diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..262bae0c1712fd17fc499abbd089e542e3dafe63 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,17 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/derain_prompt/000000_img.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/derain_prompt/000000_label.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/derain_prompt/000001_img.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/derain_prompt/000001_label.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/derain_prompt/000002_img.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/derain_prompt/000002_label.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/examples/derain_1.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/examples/derain_2.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/examples/pose_2.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/examples/seg_1.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/examples/seg_2.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/pose_prompt/000002_img.png filter=lfs diff=lfs merge=lfs -text +InternLM/tools/data/seg_prompt/000000_img.png filter=lfs diff=lfs merge=lfs -text +figs/DeLVM.PNG filter=lfs diff=lfs merge=lfs -text diff --git a/InternLM/__init__.py b/InternLM/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/InternLM/configs/kd_1b_to_300m.py b/InternLM/configs/kd_1b_to_300m.py new file mode 100644 index 0000000000000000000000000000000000000000..ca6910d809b7fdca5a5eb2aa2374af0218aa8eb7 --- /dev/null +++ b/InternLM/configs/kd_1b_to_300m.py @@ -0,0 +1,208 @@ +kd_config = dict(gt_weight=1., kd_weight=1., temperature=1) +teacher_type = "INTERNLM" + +teacher_ckpt_folder = '/path/to/teacher' + +VQGAN_FOLDER = '/path/to/vqgan' +T_SEQ_LEN = 2048 +T_HIDDEN_SIZE = 2048 +T_NUM_ATTENTION_HEAD = 16 +T_MLP_RATIO = 8 / 3 +T_NUM_LAYER = 22 +T_VOCAB_SIZE = 8192 + +teacher = dict( + checkpoint=False, # The proportion of layers for activation aheckpointing, the optional value are True/False/[0-1] + num_attention_heads=T_NUM_ATTENTION_HEAD, + embed_split_hidden=True, + vocab_size=T_VOCAB_SIZE, + embed_grad_scale=1, + parallel_output=True, + hidden_size=T_HIDDEN_SIZE, + num_layers=T_NUM_LAYER, + mlp_ratio=T_MLP_RATIO, + apply_post_layer_norm=False, + dtype="torch.float16", # Support: "torch.float16", "torch.half", "torch.bfloat16", "torch.float32", "torch.tf32" + norm_type="rmsnorm", + layer_norm_epsilon=1e-5, + use_flash_attn=True, + num_chunks=1, # if num_chunks > 1, interleaved pipeline scheduler is used. + lvm_config=dict( + enable=True, + embedding_cfg=dict( + vq_model_path=VQGAN_FOLDER, + embedding_dim=T_HIDDEN_SIZE, + freeze_vq_model=True, + ), + ) +) + +######################################################## +JOB_NAME = "lvm_llama_kd" +DO_ALERT = False +model_type = "INTERNLM" + +SEQ_LEN = 2048 +HIDDEN_SIZE = 1024 +NUM_ATTENTION_HEAD = 8 +MLP_RATIO = 8 / 3 +NUM_LAYER = 22 +VOCAB_SIZE = 8192 + +MODEL_ONLY_FOLDER = "local:llm_ckpts/xxxx" +SAVE_CKPT_FOLDER = "local:/path_to_save/" +LOAD_CKPT_FOLDER = "local:/path_to_load/" + +CHECKPOINT_EVERY = 10000 +ckpt = dict( + enable_save_ckpt=True, # set True to enable ckpt save. + save_ckpt_folder=SAVE_CKPT_FOLDER, # Path to save training ckpt. + # load_ckpt_folder= dict(path=MODEL_ONLY_FOLDER, content=["all"], ckpt_type="normal"), + # load_ckpt_folder="local:llm_ckpts/", + # 'load_ckpt_info' setting guide: + # 1. the 'path' indicate ckpt path, + # 2. the 'content‘ means what states will be loaded, support: "model", "sampler", "optimizer", "scheduler", "all" + # 3. the ’ckpt_type‘ means the type of checkpoint to be loaded, now only 'normal' type is supported. + # load_ckpt_info=dict(path=MODEL_ONLY_FOLDER, content=("model",), ckpt_type="internlm"), + checkpoint_every=CHECKPOINT_EVERY, + async_upload=True, # async ckpt upload. (only work for boto3 ckpt) + async_upload_tmp_folder="/dev/shm/internlm_tmp_ckpt/", # path for temporarily files during asynchronous upload. + # oss_snapshot_freq=int(CHECKPOINT_EVERY / 2), # snapshot ckpt save frequency. + oss_snapshot_freq=0, +) + +TRAIN_FOLDER = "/path/to/dataset" +VALID_FOLDER = "/path/to/dataset" +data = dict( + seq_len=SEQ_LEN, + # micro_num means the number of micro_batch contained in one gradient update + micro_num=1, + # packed_length = micro_bsz * SEQ_LEN + micro_bsz=16, + # defaults to the value of micro_num + valid_micro_num=1, + # defaults to 0, means disable evaluate + valid_every=0, + pack_sample_into_one=False, + train_one_epoch=False, + total_steps=40000, + skip_batches="", + rampup_batch_size="", + # Datasets with less than 50 rows will be discarded + min_length=50, + train_folder=TRAIN_FOLDER, + valid_folder=None, + empty_cache_and_diag_interval=10000, + diag_outlier_ratio=1.1, +) + +grad_scaler = dict( + fp16=dict( + # the initial loss scale, defaults to 2**16 + initial_scale=2**16, + # the minimum loss scale, defaults to None + min_scale=1, + # the number of steps to increase loss scale when no overflow occurs + growth_interval=1000, + ), + # the multiplication factor for increasing loss scale, defaults to 2 + growth_factor=2, + # the multiplication factor for decreasing loss scale, defaults to 0.5 + backoff_factor=0.5, + # the maximum loss scale, defaults to None + max_scale=2**24, + # the number of overflows before decreasing loss scale, defaults to 2 + hysteresis=2, +) + +hybrid_zero_optimizer = dict( + # Enable low_level_optimzer overlap_communication + overlap_sync_grad=True, + overlap_sync_param=True, + # bucket size for nccl communication params + reduce_bucket_size=512 * 1024 * 1024, + # grad clipping + clip_grad_norm=1.0, +) + +loss = dict( + label_smoothing=0, +) + +adam = dict( + lr=1.5e-4, + adam_beta1=0.9, + adam_beta2=0.95, + adam_beta2_c=0, + adam_eps=1e-8, + weight_decay=0.1, +) + +lr_scheduler = dict( + total_steps=data["total_steps"], + init_steps=0, # optimizer_warmup_step + warmup_ratio=0.0056, + eta_min=1.5e-5, + last_epoch=-1, +) + +beta2_scheduler = dict( + init_beta2=adam["adam_beta2"], + c=adam["adam_beta2_c"], + cur_iter=-1, +) + +model = dict( + checkpoint=False, # The proportion of layers for activation aheckpointing, the optional value are True/False/[0-1] + num_attention_heads=NUM_ATTENTION_HEAD, + embed_split_hidden=True, + vocab_size=VOCAB_SIZE, + embed_grad_scale=1, + parallel_output=True, + hidden_size=HIDDEN_SIZE, + num_layers=NUM_LAYER, + mlp_ratio=MLP_RATIO, + apply_post_layer_norm=False, + dtype="torch.float16", # Support: "torch.float16", "torch.half", "torch.bfloat16", "torch.float32", "torch.tf32" + norm_type="rmsnorm", + layer_norm_epsilon=1e-5, + use_flash_attn=True, + num_chunks=1, # if num_chunks > 1, interleaved pipeline scheduler is used. + lvm_config=dict( + enable=True, + embedding_cfg=dict( + vq_model_path='/cache/ckpt/vqgan-f16-8192-laion/', + embedding_dim=HIDDEN_SIZE, + freeze_vq_model=True, + ), + ) +) +""" +zero1 parallel: + 1. if zero1 <= 0, The size of the zero process group is equal to the size of the dp process group, + so parameters will be divided within the range of dp. + 2. if zero1 == 1, zero is not used, and all dp groups retain the full amount of model parameters. + 3. zero1 > 1 and zero1 <= dp world size, the world size of zero is a subset of dp world size. + For smaller models, it is usually a better choice to split the parameters within nodes with a setting <= 8. +pipeline parallel (dict): + 1. size: int, the size of pipeline parallel. + 2. interleaved_overlap: bool, enable/disable communication overlap when using interleaved pipeline scheduler. +tensor parallel: tensor parallel size, usually the number of GPUs per node. +""" +parallel = dict( + zero1=8, + pipeline=dict(size=1, interleaved_overlap=True), + sequence_parallel=False, +) + +cudnn_deterministic = False +cudnn_benchmark = False + +monitor = dict( + # feishu alert configs + alert=dict( + enable_feishu_alert=DO_ALERT, + feishu_alert_address=None, # feishu webhook to send alert message + light_monitor_address=None, # light_monitor address to send heartbeat + ), +) diff --git a/InternLM/configs/pretrain_300m.py b/InternLM/configs/pretrain_300m.py new file mode 100644 index 0000000000000000000000000000000000000000..18e417ba6ea824873db4f0ee55a30158059a4b64 --- /dev/null +++ b/InternLM/configs/pretrain_300m.py @@ -0,0 +1,168 @@ +JOB_NAME = "lvm_llama" +DO_ALERT = False +model_type = "INTERNLM" + +SEQ_LEN = 2048 +HIDDEN_SIZE = 1024 +NUM_ATTENTION_HEAD = 8 +MLP_RATIO = 8 / 3 +NUM_LAYER = 22 +VOCAB_SIZE = 8192 + +MODEL_ONLY_FOLDER = "local:llm_ckpts/xxxx" +SAVE_CKPT_FOLDER = "local:/path_to_save/" +LOAD_CKPT_FOLDER = "local:/path_to_load/" + +CHECKPOINT_EVERY = 10000 +ckpt = dict( + enable_save_ckpt=True, # set True to enable ckpt save. + save_ckpt_folder=SAVE_CKPT_FOLDER, # Path to save training ckpt. + # load_ckpt_folder= dict(path=MODEL_ONLY_FOLDER, content=["all"], ckpt_type="normal"), + # load_ckpt_folder="local:llm_ckpts/", + # 'load_ckpt_info' setting guide: + # 1. the 'path' indicate ckpt path, + # 2. the 'content‘ means what states will be loaded, support: "model", "sampler", "optimizer", "scheduler", "all" + # 3. the ’ckpt_type‘ means the type of checkpoint to be loaded, now only 'normal' type is supported. + # load_ckpt_info=dict(path=MODEL_ONLY_FOLDER, content=("model",), ckpt_type="internlm"), + checkpoint_every=CHECKPOINT_EVERY, + async_upload=True, # async ckpt upload. (only work for boto3 ckpt) + async_upload_tmp_folder="/dev/shm/internlm_tmp_ckpt/", # path for temporarily files during asynchronous upload. + # oss_snapshot_freq=int(CHECKPOINT_EVERY / 2), # snapshot ckpt save frequency. + oss_snapshot_freq=0, +) + +TRAIN_FOLDER = "/path/to/dataset" +VALID_FOLDER = "/path/to/dataset" +data = dict( + seq_len=SEQ_LEN, + # micro_num means the number of micro_batch contained in one gradient update + micro_num=1, + # packed_length = micro_bsz * SEQ_LEN + micro_bsz=16, + # defaults to the value of micro_num + valid_micro_num=1, + # defaults to 0, means disable evaluate + valid_every=0, + pack_sample_into_one=False, + train_one_epoch=False, + total_steps=40000, + skip_batches="", + rampup_batch_size="", + # Datasets with less than 50 rows will be discarded + min_length=50, + train_folder=TRAIN_FOLDER, + valid_folder=None, + empty_cache_and_diag_interval=10000, + diag_outlier_ratio=1.1, +) + +grad_scaler = dict( + fp16=dict( + # the initial loss scale, defaults to 2**16 + initial_scale=2**16, + # the minimum loss scale, defaults to None + min_scale=1, + # the number of steps to increase loss scale when no overflow occurs + growth_interval=1000, + ), + # the multiplication factor for increasing loss scale, defaults to 2 + growth_factor=2, + # the multiplication factor for decreasing loss scale, defaults to 0.5 + backoff_factor=0.5, + # the maximum loss scale, defaults to None + max_scale=2**24, + # the number of overflows before decreasing loss scale, defaults to 2 + hysteresis=2, +) + +hybrid_zero_optimizer = dict( + # Enable low_level_optimzer overlap_communication + overlap_sync_grad=True, + overlap_sync_param=True, + # bucket size for nccl communication params + reduce_bucket_size=512 * 1024 * 1024, + # grad clipping + clip_grad_norm=1.0, +) + +loss = dict( + label_smoothing=0, +) + +adam = dict( + lr=1.5e-4, + adam_beta1=0.9, + adam_beta2=0.95, + adam_beta2_c=0, + adam_eps=1e-8, + weight_decay=0.1, +) + +lr_scheduler = dict( + total_steps=data["total_steps"], + init_steps=0, # optimizer_warmup_step + warmup_ratio=0.0056, + eta_min=1.5e-5, + last_epoch=-1, +) + +beta2_scheduler = dict( + init_beta2=adam["adam_beta2"], + c=adam["adam_beta2_c"], + cur_iter=-1, +) + +model = dict( + checkpoint=False, # The proportion of layers for activation aheckpointing, the optional value are True/False/[0-1] + num_attention_heads=NUM_ATTENTION_HEAD, + embed_split_hidden=True, + vocab_size=VOCAB_SIZE, + embed_grad_scale=1, + parallel_output=True, + hidden_size=HIDDEN_SIZE, + num_layers=NUM_LAYER, + mlp_ratio=MLP_RATIO, + apply_post_layer_norm=False, + dtype="torch.float16", # Support: "torch.float16", "torch.half", "torch.bfloat16", "torch.float32", "torch.tf32" + norm_type="rmsnorm", + layer_norm_epsilon=1e-5, + use_flash_attn=True, + num_chunks=1, # if num_chunks > 1, interleaved pipeline scheduler is used. + lvm_config=dict( + enable=True, + embedding_cfg=dict( + vq_model_path='/cache/ckpt/vqgan-f16-8192-laion/', + embedding_dim=HIDDEN_SIZE, + freeze_vq_model=True, + ), + ) +) +""" +zero1 parallel: + 1. if zero1 <= 0, The size of the zero process group is equal to the size of the dp process group, + so parameters will be divided within the range of dp. + 2. if zero1 == 1, zero is not used, and all dp groups retain the full amount of model parameters. + 3. zero1 > 1 and zero1 <= dp world size, the world size of zero is a subset of dp world size. + For smaller models, it is usually a better choice to split the parameters within nodes with a setting <= 8. +pipeline parallel (dict): + 1. size: int, the size of pipeline parallel. + 2. interleaved_overlap: bool, enable/disable communication overlap when using interleaved pipeline scheduler. +tensor parallel: tensor parallel size, usually the number of GPUs per node. +""" +parallel = dict( + zero1=8, + pipeline=dict(size=1, interleaved_overlap=True), + sequence_parallel=False, +) + +cudnn_deterministic = False +cudnn_benchmark = False + +monitor = dict( + # feishu alert configs + alert=dict( + enable_feishu_alert=DO_ALERT, + feishu_alert_address=None, # feishu webhook to send alert message + light_monitor_address=None, # light_monitor address to send heartbeat + ), +) diff --git a/InternLM/internlm/__init__.py b/InternLM/internlm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..34e14e4993df023b99efca658bd108e8047cd017 --- /dev/null +++ b/InternLM/internlm/__init__.py @@ -0,0 +1,10 @@ +from .initialize.initialize_trainer import initialize_trainer, initialize_kd_trainer +from .initialize.launch import get_default_parser, launch_from_slurm, launch_from_torch + +__all__ = [ + "get_default_parser", + "initialize_kd_trainer", + "initialize_trainer", + "launch_from_slurm", + "launch_from_torch", +] diff --git a/InternLM/internlm/apis/__init__.py b/InternLM/internlm/apis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/InternLM/internlm/apis/inference.py b/InternLM/internlm/apis/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..88d6d50327f42700fc47191c65db27901933367b --- /dev/null +++ b/InternLM/internlm/apis/inference.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import torch +import torch.nn.functional as F +from torch import nn + +__all__ = ["SequenceGenerator"] + + +class InferenceParams: + """ + Intermediate cache objects for inference + """ + + def __init__( + self, + max_sequence_len, + max_batch_size, + sequence_len_offset=0, + batch_size_offset=0, + key_value_memory_dict: dict = None, + lengths_per_sample=None, + attention_mask=None, + ) -> None: + + self.max_sequence_len: int = max_sequence_len + self.max_batch_size: int = max_batch_size + self.sequence_len_offset: int = sequence_len_offset + self.batch_size_offset: int = batch_size_offset + if key_value_memory_dict is None: + key_value_memory_dict = {} + self.key_value_memory_dict: dict = key_value_memory_dict + self.fused_ft_kernel: bool = False + self.lengths_per_sample = lengths_per_sample + self.attention_mask = attention_mask + + def reorder_state(self, indices): + if self.lengths_per_sample is not None: + self.lengths_per_sample = self.lengths_per_sample.index_select(index=indices, dim=0) + for key, value in list(self.key_value_memory_dict.items()): + value = value.index_select(index=indices, dim=0) + self.key_value_memory_dict[key] = value + + +def _get_model_device(model): + """ + obtain the device of an nn.Module.model + + Args: + model: nn.Module + + Return: torch.device. if None, the parameters of this model is None. + """ + assert isinstance(model, nn.Module) + + parameters = list(model.parameters()) + if len(parameters) == 0: + return None + else: + return parameters[0].device + + +class SequenceGenerator: + """ + Sequence Generator. + """ + + def __init__(self, decoder, eos_token_id, pad_token_id, bos_token_id): + self.decoder = decoder + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + + @torch.no_grad() + def generate( + self, + tokens: "torch.LongTensor" = None, + num_return_sequences=1, + max_length: int = 20, + num_beams: int = 1, + do_sample: bool = True, + temperature: float = 1.0, + top_k: int = 50, + top_p: float = 1.0, + repetition_penalty: float = 1, + length_penalty: float = 1.0, + ): + """ + Args: + tokens: the beginning tokens whose shape is [bsz, length]. If shape is None, default ''bos_token'' will be + added to conduct generation. + num_return_sequences: number of returned sequences. + max_length: the max length of generated sequence. + num_beams: the size of beam search. + do_sample: whether using sample. + temperature: it's meaningful when do_sample is True. + top_k: sampling from top_k. + top_p: sampling from top_p tokens(nucleus sampling). + + Return: + the token sequence whose shape is [bsz, num_return_sequences, max_length]. If eos_token_id is not None, + the ending of each sequence must be eos_token_id. + """ + assert num_return_sequences <= num_beams, f"The `{num_return_sequences}` must be less than `{num_beams}`..." + if do_sample: + return sample_generate( + self.decoder, + tokens=tokens, + max_length=max_length, + num_beams=num_beams, + num_return_sequences=num_return_sequences, + temperature=temperature, + top_k=top_k, + top_p=top_p, + eos_token_id=self.eos_token_id, # the ending token id + pad_token_id=self.pad_token_id, + repetition_penalty=repetition_penalty, # the penalty degree for repetition tokens + length_penalty=length_penalty, # the penalty for length. if it > 1, then encourages long sequence. + # Otherwise, encourages short sequence. + bos_token_id=self.bos_token_id, + ) + else: + return greedy_generate( + self.decoder, + tokens=tokens, + max_length=max_length, + num_beams=num_beams, + num_return_sequences=num_return_sequences, + eos_token_id=self.eos_token_id, + pad_token_id=self.pad_token_id, + repetition_penalty=repetition_penalty, + length_penalty=length_penalty, + bos_token_id=self.bos_token_id, + ) + + +@torch.no_grad() +def greedy_generate( + decoder, + tokens=None, + max_length=20, + num_beams=1, + num_return_sequences=1, + eos_token_id=None, + pad_token_id=0, + repetition_penalty=1, + length_penalty=1.0, + bos_token_id=1, + feat_mask=None, + ffn_mask=None, + layer_mask=None, +): + """ + Search sequence greedily. + + Args: + decoder: the Decoder object. + tokens: the shape is [batch size, length]. If decoder is None, generating begins with bos_token_id. + max_length: the max length for generated sequence. + num_beams: the size of beam to decode. + eos_token_id: the ending token id. If None, the decode length is max_length. + pad_token_id: the token id of pad. + repetition_penalty: the penalty degree for repetition tokens + length_penalty: the penalty for length. + + """ + if num_beams == 1: + token_ids = _no_beam_search_generate( + decoder, + tokens=tokens, + max_length=max_length, + temperature=1, + top_k=50, + top_p=1, + eos_token_id=eos_token_id, + do_sample=False, + repetition_penalty=repetition_penalty, + length_penalty=length_penalty, + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + feat_mask=feat_mask, + ffn_mask=ffn_mask, + layer_mask=layer_mask, + ) + else: + token_ids = _beam_search_generate( + decoder, + tokens=tokens, + max_length=max_length, + num_beams=num_beams, + num_return_sequences=num_return_sequences, + temperature=1, + top_k=50, + top_p=1, + eos_token_id=eos_token_id, + do_sample=False, + repetition_penalty=repetition_penalty, + length_penalty=length_penalty, + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + feat_mask=feat_mask, + ffn_mask=ffn_mask, + layer_mask=layer_mask, + ) + + return token_ids + + +@torch.no_grad() +def sample_generate( + decoder, + tokens, + max_length=20, + num_beams=1, + num_return_sequences=1, + temperature=1.0, + top_k=50, + top_p=1.0, + eos_token_id=None, + pad_token_id=0, + repetition_penalty=1.0, + length_penalty=1.0, + bos_token_id=1, +): + """ + generate sequence in sampling way. + + Args: + decoder: the Decoder object. + tokens: the shape is [batch size, length]. If decoder is None, generating begins with bos_token_id. + max_length: the max length for generated sequence. + num_beams: the size of beam to decode. + num_return_sequences: number of returned sequence. + temperature: annealing magnitude during sampling. + top_k: sampling from top_k. (Default: 50) + top_p: sampling from top_p tokens(nucleus sampling). (Default: 1.0) + eos_token_id: the ending token id. If None, the decode length is max_length. + pad_token_id: the token id of pad. + repetition_penalty: the penalty degree for repetition tokens + length_penalty: the penalty for length. + + """ + if num_beams == 1: + token_ids = _no_beam_search_generate( + decoder, + tokens=tokens, + max_length=max_length, + temperature=temperature, + top_k=top_k, + top_p=top_p, + eos_token_id=eos_token_id, + do_sample=True, + repetition_penalty=repetition_penalty, + length_penalty=length_penalty, + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + ) + else: + token_ids = _beam_search_generate( + decoder, + tokens=tokens, + max_length=max_length, + num_beams=num_beams, + num_return_sequences=num_return_sequences, + temperature=temperature, + top_k=top_k, + top_p=top_p, + eos_token_id=eos_token_id, + do_sample=True, + repetition_penalty=repetition_penalty, + length_penalty=length_penalty, + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + ) + return token_ids + + +@torch.no_grad() +def _no_beam_search_generate( + decoder, + tokens, + inference_params=None, + max_length=20, + temperature=1.0, + top_k=50, + top_p=1.0, + eos_token_id=None, + do_sample=True, + repetition_penalty=1.0, + length_penalty=1.0, + pad_token_id=0, + bos_token_id=1, + feat_mask=None, + ffn_mask=None, + layer_mask=None, +): + # delete num_return_sequences=1 for lint check; + batch_size = tokens.size(0) + if eos_token_id is None: + _eos_token_id = -1 + else: + _eos_token_id = eos_token_id + + has_bos = torch.all(tokens[:, 0].eq(bos_token_id)) + if has_bos: + bos_pos = torch.where(tokens.eq(bos_token_id), 1, 0) + bos_sum = bos_pos.cumsum(dim=-1) + bos_pos = torch.where(bos_sum.eq(bos_sum[:, -1:]), 0, 1) + to_atten_x = bos_pos[:, :, None] + to_atten_y = bos_pos[:, None, :] + # attention_mask = torch.einsum('bno,bom->bnm', to_atten_x, to_atten_y).eq(1) + else: + bos_pos = torch.where(tokens.eq(bos_token_id), 1, 0) + to_atten_x = bos_pos[:, :, None] + to_atten_y = bos_pos[:, None, :] + # attention_mask = torch.einsum('bno,bom->bnm', to_atten_x, to_atten_y).eq(1) + attention_mask = torch.logical_or(to_atten_x, to_atten_y).eq(1) + if inference_params is None: + inference_params = InferenceParams( + max_sequence_len=max_length, + max_batch_size=tokens.size(0), + sequence_len_offset=0, + batch_size_offset=0, + key_value_memory_dict=None, + lengths_per_sample=None, + attention_mask=attention_mask, + ) + + if layer_mask is None: + if feat_mask is None and ffn_mask is None: + scores = decoder(**{"input_ids": tokens, "inference_params": inference_params}) + else: + scores = decoder( + **{ + "input_ids": tokens, + "inference_params": inference_params, + "feat_mask": feat_mask, + "ffn_mask": ffn_mask, + } + ) + else: + scores = decoder( + **{ + "input_ids": tokens, + "inference_params": inference_params, + "feat_mask": feat_mask, + "ffn_mask": ffn_mask, + "layer_mask": layer_mask, + } + ) + + if isinstance(scores, (list, tuple)): + scores = scores[0] + scores = scores[:, -1].float() + inference_params.sequence_len_offset += tokens.size(1) + if _eos_token_id != -1: + scores[:, _eos_token_id] = -1e12 + next_tokens = scores.argmax(dim=-1, keepdim=True) + token_ids = torch.cat([tokens, next_tokens], dim=1) + cur_len = token_ids.size(1) + dones = token_ids.new_zeros(batch_size).eq(1) + # tokens = tokens[:, -1:] + + real_max_length = max_length + max_lengths = tokens.new_full((tokens.size(0),), fill_value=max_length, dtype=torch.long) + + while cur_len < real_max_length: + # batch_size x vocab_size + if has_bos: + bos_pos = torch.where(token_ids.eq(bos_token_id), 1, 0) + bos_sum = bos_pos.cumsum(dim=-1) + bos_pos = torch.where(bos_sum.eq(bos_sum[:, -1:]), 0, 1) + to_atten_x = bos_pos[:, :, None] + to_atten_y = bos_pos[:, None, :] + # attention_mask = torch.einsum('bno,bom->bnm', to_atten_x, to_atten_y).eq(1) + else: + bos_pos = torch.where(token_ids.eq(bos_token_id), 1, 0) + to_atten_x = bos_pos[:, :, None] + to_atten_y = bos_pos[:, None, :] + # attention_mask = torch.einsum('bno,bom->bnm', to_atten_x, to_atten_y).eq(1) + attention_mask = torch.logical_or(to_atten_x, to_atten_y).eq(1) + inference_params.attention_mask = attention_mask + if layer_mask is None: + if feat_mask is None and ffn_mask is None: + scores = decoder(**{"input_ids": token_ids[:, -1:], "inference_params": inference_params}) + else: + scores = decoder( + **{ + "input_ids": token_ids[:, -1:], + "inference_params": inference_params, + "feat_mask": feat_mask, + "ffn_mask": ffn_mask, + } + ) + else: + scores = decoder( + **{ + "input_ids": token_ids[:, -1:], + "inference_params": inference_params, + "feat_mask": feat_mask, + "ffn_mask": ffn_mask, + "layer_mask": layer_mask, + } + ) + + if isinstance(scores, (list, tuple)): + scores = scores[0] + scores = scores[:, -1].float() + inference_params.sequence_len_offset += 1 + + if repetition_penalty != 1.0: + token_scores = scores.gather(dim=1, index=token_ids) + lt_zero_mask = token_scores.lt(0).float() + ge_zero_mask = lt_zero_mask.eq(0).float() + token_scores = ( + lt_zero_mask * repetition_penalty * token_scores + ge_zero_mask / repetition_penalty * token_scores + ) + scores.scatter_(dim=1, index=token_ids, src=token_scores) + + if eos_token_id is not None and length_penalty != 1.0: + # batch_size x vocab_size + token_scores = scores / cur_len**length_penalty + eos_mask = scores.new_ones(scores.size(1)) + eos_mask[eos_token_id] = 0 + eos_mask = eos_mask.unsqueeze(0).eq(1) + + scores = scores.masked_scatter(eos_mask, token_scores) + + if do_sample: + if temperature > 0 and temperature != 1: + scores = scores / temperature + + scores = top_k_top_p_filtering(scores, top_k, top_p, min_tokens_to_keep=2) + # add 1e-12 to avoid https://github.com/pytorch/pytorch/pull/27523 + probs = F.softmax(scores, dim=-1) + 1e-12 + + next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1) # batch_size + else: + next_tokens = torch.argmax(scores, dim=-1) # batch_size + + if _eos_token_id != -1: + next_tokens = next_tokens.masked_fill(max_lengths.eq(cur_len + 1), _eos_token_id) + next_tokens = next_tokens.masked_fill(dones, pad_token_id) + tokens = next_tokens.unsqueeze(1) + + token_ids = torch.cat([token_ids, tokens], dim=-1) # batch_size x max_len + + end_mask = next_tokens.eq(_eos_token_id) + dones = dones.__or__(end_mask) + cur_len += 1 + + if dones.min() == 1: + break + + # if eos_token_id is not None: + # # setting the eos at the maximum length position + # tokens.scatter(index=max_lengths[:, None], dim=1, value=eos_token_id) + # if cur_len == max_length: + # # If eos is not reached by the maximum length, forcibly replace the last word with eos + # token_ids[:, -1].masked_fill_(~dones, eos_token_id) + # TODO Here we are simply adding an extra dimension for interface compatibility, but in the future it will need to + # be able to return multiple real results + return token_ids[:, None] + + +@torch.no_grad() +def _beam_search_generate( + decoder, + tokens, + inference_params=None, + max_length=20, + num_beams=4, + num_return_sequences=1, + temperature=1.0, + top_k=50, + top_p=1.0, + eos_token_id=None, + do_sample=True, + repetition_penalty=1.0, + length_penalty=1.0, + pad_token_id=0, + bos_token_id=1, + feat_mask=None, + ffn_mask=None, + layer_mask=None, +) -> torch.LongTensor: + + device = _get_model_device(decoder) + batch_size = tokens.size(0) + + if eos_token_id is None: + _eos_token_id = -1 + else: + _eos_token_id = eos_token_id + + has_bos = torch.all(tokens[:, 0].eq(bos_token_id)) + + if has_bos: + bos_pos = torch.where(tokens.eq(bos_token_id), 1, 0) + bos_sum = bos_pos.cumsum(dim=-1) + bos_pos = torch.where(bos_sum.eq(bos_sum[:, -1:]), 0, 1) + to_atten_x = bos_pos[:, :, None] + to_atten_y = bos_pos[:, None, :] + # attention_mask = torch.einsum('bno,bom->bnm', to_atten_x, to_atten_y).eq(1) + else: + bos_pos = torch.where(tokens.eq(bos_token_id), 1, 0) + to_atten_x = bos_pos[:, :, None] + to_atten_y = bos_pos[:, None, :] + # attention_mask = torch.einsum('bno,bom->bnm', to_atten_x, to_atten_y).eq(1) + attention_mask = torch.logical_or(to_atten_x, to_atten_y).eq(1) + + if inference_params is None: + inference_params = InferenceParams( + max_sequence_len=max_length, + max_batch_size=tokens.size(0), + sequence_len_offset=0, + batch_size_offset=0, + key_value_memory_dict=None, + lengths_per_sample=None, + attention_mask=attention_mask, + ) + + if layer_mask is None: + if feat_mask is None and ffn_mask is None: + scores = decoder(**{"input_ids": tokens, "inference_params": inference_params}) + else: + scores = decoder( + **{ + "input_ids": tokens, + "inference_params": inference_params, + "feat_mask": feat_mask, + "ffn_mask": ffn_mask, + } + ) + else: + scores = decoder( + **{ + "input_ids": tokens, + "inference_params": inference_params, + "feat_mask": feat_mask, + "ffn_mask": ffn_mask, + "layer_mask": layer_mask, + } + ) + + if isinstance(scores, (list, tuple)): + scores = scores[0] + scores = scores[:, -1].float() + inference_params.sequence_len_offset += tokens.size(1) + if _eos_token_id != -1: + scores[:, _eos_token_id] = -1e12 + vocab_size = scores.size(1) + assert vocab_size >= num_beams, "num_beams should be smaller than " "the number of vocabulary size." + + if do_sample: + probs = F.softmax(scores, dim=-1) + 1e-12 + # (batch_size, num_beams) + next_tokens = torch.multinomial(probs, num_samples=num_beams) + logits = probs.log() + # (batch_size, num_beams) + next_scores = logits.gather(dim=1, index=next_tokens) + else: + scores = F.log_softmax(scores, dim=-1) # (batch_size, vocab_size) + # obtain (batch_size, num_beams), (batch_size, num_beams) + next_scores, next_tokens = torch.topk(scores, num_beams, dim=1, largest=True, sorted=True) + + indices = torch.arange(batch_size, dtype=torch.long).to(device) + indices = indices.repeat_interleave(num_beams) + inference_params.reorder_state(indices) + + # batch_size * num_beams x length + tokens = tokens.index_select(dim=0, index=indices) + # genrated token (batch_size', cur_len) + token_ids = torch.cat([tokens, next_tokens.view(-1, 1)], dim=-1) + dones = [False] * batch_size + + beam_scores = next_scores.view(-1) # batch_size * num_beams + + cur_len = token_ids.size(1) + + real_max_length = max_length + max_lengths = tokens.new_full((tokens.size(0),), fill_value=max_length, dtype=torch.long) + hypos = [ + BeamHypotheses(num_beams, real_max_length, length_penalty, early_stopping=False) for _ in range(batch_size) + ] + # 0, num_beams, 2*num_beams, ... + batch_inds_with_numbeams_interval = (torch.arange(batch_size) * num_beams).view(-1, 1).to(token_ids) + + while cur_len < real_max_length: + if has_bos: + bos_pos = torch.where(token_ids.eq(bos_token_id), 1, 0) + bos_sum = bos_pos.cumsum(dim=-1) + bos_pos = torch.where(bos_sum.eq(bos_sum[:, -1:]), 0, 1) + to_atten_x = bos_pos[:, :, None] + to_atten_y = bos_pos[:, None, :] + # attention_mask = torch.einsum('bno,bom->bnm', to_atten_x, to_atten_y).eq(1) + else: + bos_pos = torch.where(token_ids.eq(bos_token_id), 1, 0) + to_atten_x = bos_pos[:, :, None] + to_atten_y = bos_pos[:, None, :] + # attention_mask = torch.einsum('bno,bom->bnm', to_atten_x, to_atten_y).eq(1) + attention_mask = torch.logical_or(to_atten_x, to_atten_y).eq(1) + + inference_params.attention_mask = attention_mask + # (bsz x num_beams, vocab_size) + + if layer_mask is None: + if feat_mask is None and ffn_mask is None: + scores = decoder(**{"input_ids": token_ids[:, -1:], "inference_params": inference_params}) + else: + scores = decoder( + **{ + "input_ids": token_ids[:, -1:], + "inference_params": inference_params, + "feat_mask": feat_mask, + "ffn_mask": ffn_mask, + } + ) + else: + scores = decoder( + **{ + "input_ids": token_ids[:, -1:], + "inference_params": inference_params, + "feat_mask": feat_mask, + "ffn_mask": ffn_mask, + "layer_mask": layer_mask, + } + ) + + if isinstance(scores, (list, tuple)): + scores = scores[0] + scores = scores[:, -1].float() + inference_params.sequence_len_offset += 1 + if repetition_penalty != 1.0: + token_scores = scores.gather(dim=1, index=token_ids) + lt_zero_mask = token_scores.lt(0).float() + ge_zero_mask = lt_zero_mask.eq(0).float() + token_scores = ( + lt_zero_mask * repetition_penalty * token_scores + ge_zero_mask / repetition_penalty * token_scores + ) + scores.scatter_(dim=1, index=token_ids, src=token_scores) + + if _eos_token_id != -1: + max_len_eos_mask = max_lengths.eq(cur_len + 1) + eos_scores = scores[:, _eos_token_id] + scores[:, _eos_token_id] = torch.where(max_len_eos_mask, eos_scores + 1e32, eos_scores) + + if do_sample: + if temperature > 0 and temperature != 1: + scores = scores / temperature + + scores = top_k_top_p_filtering(scores, top_k, top_p, min_tokens_to_keep=num_beams + 1) + # add 1e-12 to avoid https://github.com/pytorch/pytorch/pull/27523 + probs = F.softmax(scores, dim=-1) + 1e-12 + + # batch_size' x (num_beams+1) + _tokens = torch.multinomial(probs, num_samples=num_beams + 1) + + logits = probs.log() + # batch_size' x (num_beams+1) + _scores = logits.gather(dim=1, index=_tokens) + # batch_size' x (num_beams+1) + _scores = _scores + beam_scores[:, None] + _scores = _scores.view(batch_size, num_beams * (num_beams + 1)) + next_scores, ids = _scores.topk(2 * num_beams, dim=1, largest=True, sorted=True) + _tokens = _tokens.view(batch_size, num_beams * (num_beams + 1)) + # (batch_size, 2*num_beams) + next_tokens = _tokens.gather(dim=1, index=ids) + # (batch_size, 2*num_beams) + from_which_beam = torch.floor(ids.float() / (num_beams + 1)).long() + else: + # (batch_size * num_beams, vocab_size) + scores = F.log_softmax(scores, dim=-1) + # (batch_size * num_beams, vocab_size) + _scores = scores + beam_scores[:, None] + # (batch_size, num_beams*vocab_size) + _scores = _scores.view(batch_size, -1) + # (bsz, 2*num_beams) + next_scores, ids = torch.topk(_scores, 2 * num_beams, dim=1, largest=True, sorted=True) + # (batch_size, 2*num_beams) + from_which_beam = torch.floor(ids.float() / vocab_size).long() + next_tokens = ids % vocab_size # (batch_size, 2*num_beams) + + # next_scores, sorted_inds = next_scores.sort(dim=-1, descending=True) + # next_tokens = next_tokens.gather(dim=1, index=sorted_inds) + # from_which_beam = from_which_beam.gather(dim=1, index=sorted_inds) + + not_eos_mask = next_tokens.ne(_eos_token_id) + keep_mask = not_eos_mask.cumsum(dim=1).le(num_beams) + keep_mask = not_eos_mask.__and__(keep_mask) + + _next_tokens = next_tokens.masked_select(keep_mask).view(-1, 1) + _from_which_beam = from_which_beam.masked_select(keep_mask).view(batch_size, num_beams) + _next_scores = next_scores.masked_select(keep_mask).view(batch_size, num_beams) + beam_scores = _next_scores.view(-1) + + flag = True + if cur_len + 1 == real_max_length: + eos_batch_idx = torch.arange(batch_size).to(next_tokens).repeat_interleave(repeats=num_beams, dim=0) + eos_beam_ind = torch.arange(num_beams).to(token_ids).repeat(batch_size) + eos_beam_idx = from_which_beam[:, :num_beams].reshape(-1) + else: + effective_eos_mask = next_tokens[:, :num_beams].eq(_eos_token_id) # batch_size x num_beams + if effective_eos_mask.sum().gt(0): + eos_batch_idx, eos_beam_ind = effective_eos_mask.nonzero(as_tuple=True) + eos_beam_idx = eos_batch_idx * num_beams * 2 + eos_beam_ind + eos_beam_idx = from_which_beam.view(-1)[eos_beam_idx] + else: + flag = False + + if flag: + _token_ids = torch.cat([token_ids, _next_tokens], dim=-1) + for batch_idx, beam_ind, beam_idx in zip( + eos_batch_idx.tolist(), eos_beam_ind.tolist(), eos_beam_idx.tolist() + ): + if not dones[batch_idx]: + score = next_scores[batch_idx, beam_ind].item() + if _eos_token_id != -1: + hypos[batch_idx].add(_token_ids[batch_idx * num_beams + beam_idx, :cur_len].clone(), score) + else: + hypos[batch_idx].add(_token_ids[batch_idx * num_beams + beam_idx].clone(), score) + + reorder_inds = (batch_inds_with_numbeams_interval + _from_which_beam).view(-1) + inference_params.reorder_state(reorder_inds) + token_ids = torch.cat([token_ids.index_select(index=reorder_inds, dim=0), _next_tokens], dim=-1) + + for batch_idx in range(batch_size): + dones[batch_idx] = ( + dones[batch_idx] + or hypos[batch_idx].is_done(next_scores[batch_idx, 0].item()) + or max_lengths[batch_idx * num_beams] == cur_len + 1 + ) + + cur_len += 1 + + if all(dones): + break + + # select the best hypotheses + tgt_len = token_ids.new_zeros(batch_size, num_return_sequences) + best = [] + + for i, hypotheses in enumerate(hypos): + # best_hyp = max(hypotheses.hyp, key=lambda x: x[0])[1] + sorted_hyp = list(sorted(hypotheses.hyp, key=lambda x: x[0], reverse=True)) + _best = [] + for j, hyp in zip(range(num_return_sequences), sorted_hyp): + hyp = hyp[1] + if _eos_token_id != -1: + hyp = torch.cat([hyp, token_ids.new_ones(1) * _eos_token_id]) + tgt_len[i, j] = len(hyp) + _best.append(hyp) + best.append(_best) + + # generate target batch + decoded = token_ids.new_zeros(batch_size, num_return_sequences, tgt_len.max().item()).fill_(pad_token_id) + for i, hypo in enumerate(best): + for j, _hypo in enumerate(hypo): + decoded[i, j, : tgt_len[i, j]] = _hypo + + return decoded + + +class BeamHypotheses(object): + """ + BeamHypotheses + """ + + def __init__(self, num_beams, max_length, length_penalty, early_stopping): + """Initialize n-best list of hypotheses.""" + self.max_length = max_length - 1 # ignoring bos_token + self.length_penalty = length_penalty + self.early_stopping = early_stopping + self.num_beams = num_beams + self.hyp = [] + self.worst_score = 1e9 + + def __len__(self): + """Number of hypotheses in the list.""" + return len(self.hyp) + + def add(self, hyp, sum_logprobs): + """Add a new hypothesis to the list.""" + score = sum_logprobs / len(hyp) ** self.length_penalty + if len(self) < self.num_beams or score > self.worst_score: + self.hyp.append((score, hyp)) + if len(self) > self.num_beams: + sorted_scores = sorted([(s, idx) for idx, (s, _) in enumerate(self.hyp)]) + del self.hyp[sorted_scores[0][1]] + self.worst_score = sorted_scores[1][0] + else: + self.worst_score = min(score, self.worst_score) + + def is_done(self, best_sum_logprobs): + """If there are enough hypotheses and that none of the hypotheses being + generated can become better than the worst one in the heap, then we are + done with this sentence.""" + if len(self) < self.num_beams: + return False + elif self.early_stopping: + return True + else: + return self.worst_score >= best_sum_logprobs / self.max_length**self.length_penalty + + +def top_k_top_p_filtering(logits, top_k=0, top_p=1.0, filter_value=-float("Inf"), min_tokens_to_keep=1): + """ + Based on the values of top_k and top_p, set the values that do not meet the criteria to the filter_value. + + Args: + logits: logit value, shape is [bsz, vocab_size]. + top_k: If it is greater than 0, only the probabilities of the top_k vocabulary are kept, and the rest of + the positions are set to filter_value. + top_p: according to http://arxiv.org/abs/1904.09751. + filter_value: filter value + min_tokens_to_keep: The probability of words in each sample‘s returned distribution will not be + lower than this value. + + """ + if top_k > 0: + # Safety check + top_k = min(max(top_k, min_tokens_to_keep), logits.size(-1)) + # Remove all tokens with a probability less than the last token of + # the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits[indices_to_remove] = filter_value + + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold + # (token with 0 are kept) + sorted_indices_to_remove = cumulative_probs > top_p + if min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep + # (set to min_tokens_to_keep-1 because we add the first one below) + sorted_indices_to_remove[..., :min_tokens_to_keep] = 0 + # Shift the indices to the right to keep also the first token + # above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) + logits[indices_to_remove] = filter_value + return logits diff --git a/InternLM/internlm/core/__init__.py b/InternLM/internlm/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d6b704899a1d5cfcd80a8d08aaf8743b9842c3a8 --- /dev/null +++ b/InternLM/internlm/core/__init__.py @@ -0,0 +1,9 @@ +from .engine import Engine +from .naive_amp import NaiveAMPModel +from .trainer import Trainer + +__all__ = [ + "NaiveAMPModel", + "Engine", + "Trainer", +] diff --git a/InternLM/internlm/core/communication/__init__.py b/InternLM/internlm/core/communication/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a42b9ea16f3630438c7578141c152009b5776f8b --- /dev/null +++ b/InternLM/internlm/core/communication/__init__.py @@ -0,0 +1,32 @@ +from .p2p import ( + AsynCommunicator, + recv_backward, + recv_forward, + send_backward, + send_backward_and_recv_next_backward_async, + send_backward_recv_backward, + send_backward_recv_forward, + send_forward, + send_forward_and_recv_next_forward_async, + send_forward_backward_recv_forward_backward, + send_forward_recv_backward, + send_forward_recv_forward, +) +from .utils import recv_obj_meta, send_obj_meta + +__all__ = [ + "send_forward", + "send_forward_recv_forward", + "send_forward_backward_recv_forward_backward", + "send_backward", + "send_backward_recv_backward", + "send_backward_recv_forward", + "send_forward_recv_backward", + "recv_backward", + "recv_forward", + "send_obj_meta", + "recv_obj_meta", + "send_backward_and_recv_next_backward_async", + "send_forward_and_recv_next_forward_async", + "AsynCommunicator", +] diff --git a/InternLM/internlm/core/communication/p2p.py b/InternLM/internlm/core/communication/p2p.py new file mode 100644 index 0000000000000000000000000000000000000000..e707661f6e890b0e801bae38855a09c7be242791 --- /dev/null +++ b/InternLM/internlm/core/communication/p2p.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/communication + +import operator +from functools import reduce +from typing import List, Tuple, Union + +import torch +import torch.distributed as dist + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.utils.common import get_current_device + +from .utils import gather_split_1d_tensor, split_tensor_into_1d_equal_chunks + +TensorShape = Union[torch.Size, List[int], Tuple[int]] + + +def _get_tensor_shape(tensor_shape: TensorShape, chunk_tensor: bool = False) -> Tuple[TensorShape, bool]: + """get the exact tensor shape when communicating and return whether the tensor is a chunk + + Args: + tensor_shape (:class:`torch.Size`): shape of tensor + chunk_tensor (bool, optional): whether to chunk tensor, defaults to False + + Returns: + Tuple[Union[:class:`torch.Size`, List[int], Tuple[int]], bool]: exact tensor shape, whether to chunk tensor + """ + if chunk_tensor: + tensor_chunk_shape = reduce(operator.mul, tensor_shape, 1) + tensor_parallel_world_size = gpc.get_world_size(ParallelMode.TENSOR) + if tensor_chunk_shape % tensor_parallel_world_size == 0: + tensor_chunk_shape = tensor_chunk_shape // tensor_parallel_world_size + else: + tensor_chunk_shape = tensor_shape + chunk_tensor = False + else: + tensor_chunk_shape = tensor_shape + return tensor_chunk_shape, chunk_tensor + + +def create_recv_buffer_with_shapes(recv_shapes, dtype, scatter_gather_tensors): + if isinstance(recv_shapes, torch.Size): + recv_chunk_shape, recv_split = _get_tensor_shape(recv_shapes, scatter_gather_tensors) + buffer_recv = torch.empty(recv_chunk_shape, requires_grad=True, device=get_current_device(), dtype=dtype) + return buffer_recv, recv_split + buffer_recv = [] + for recv_shape in recv_shapes: + recv_chunk_shape, recv_split = _get_tensor_shape(recv_shape, scatter_gather_tensors) + tensor_recv = torch.empty(recv_chunk_shape, requires_grad=True, device=get_current_device(), dtype=dtype) + buffer_recv.append(tensor_recv) + return buffer_recv, recv_split + + +def process_object_to_send(object_send, scatter_gather_tensors): + if isinstance(object_send, torch.Tensor): + send_split = _get_tensor_shape(object_send.shape, scatter_gather_tensors)[1] + if send_split: + object_send = split_tensor_into_1d_equal_chunks(object_send) + return object_send + + object_send_list = [] + for tensor_send in object_send: + send_split = _get_tensor_shape(tensor_send.shape, scatter_gather_tensors)[1] + if send_split: + object_send_list.append(split_tensor_into_1d_equal_chunks(tensor_send)) + else: + object_send_list.append(tensor_send) + object_send = tuple(object_send_list) + + return object_send + + +def filling_ops_queue(obj, comm_op, comm_rank, ops_queue): + if isinstance(obj, torch.Tensor): + op_to_add = dist.P2POp(comm_op, obj, comm_rank) + ops_queue.append(op_to_add) + else: + for tensor_to_comm in obj: + op_to_add = dist.P2POp(comm_op, tensor_to_comm, comm_rank) + ops_queue.append(op_to_add) + + +def _communicate( + object_send_next: Union[torch.Tensor, List[torch.Tensor]] = None, + object_send_prev: Union[torch.Tensor, List[torch.Tensor]] = None, + recv_prev: bool = False, + recv_next: bool = False, + recv_prev_shape: Union[torch.Size, List[torch.Size]] = None, + recv_next_shape: Union[torch.Size, List[torch.Size]] = None, + prev_rank: int = None, + next_rank: int = None, + dtype: torch.dtype = None, + scatter_gather_tensors: bool = False, +) -> Tuple[Union[torch.Tensor, List[torch.Tensor]]]: + """ + Adapted from megatron.p2p_communication. + Communicate tensors between stages. Used as helper method in other + communication methods that are used in pipeline schedule. + Takes the following arguments: + object_send_next (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): tensor to send to next rank + (no tensor sent if set to None). + object_send_prev (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): tensor to send to prev rank + (no tensor sent if set to None). + recv_prev (bool): boolean for whether tensor should be received from + previous rank. + recv_next (bool): boolean for whether tensor should be received from + next rank. + recv_prev_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): shape of the tensor to be received + from the previous stage, defualts to None. + recv_next_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): shape of the tensor to be received + from the next stage, defualts to None. + prev_rank (int): the rank of the previous pipeline stage, defualts to None, + next_rank (int): the rank of the next pipeline stage, defualts to None, + dtype (torch.dtype): data type of intermediate buffers, defaults to None + scatter_gather_tensors (bool): whether to scatter and gather tensor between pipeline stages, defaults to False + + Returns: + Tuple[Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]]: returns tensor_recv_prev, tensor_recv_next + """ + + # Create placeholder tensors for receive in forward and backward directions + # if needed. + tensor_recv_prev = None + tensor_recv_next = None + + if recv_prev: + assert recv_prev_shape is not None + tensor_recv_prev, recv_prev_split = create_recv_buffer_with_shapes( + recv_prev_shape, dtype, scatter_gather_tensors + ) + + if recv_next: + assert recv_next_shape is not None + tensor_recv_next, recv_next_split = create_recv_buffer_with_shapes( + recv_next_shape, dtype, scatter_gather_tensors + ) + + if object_send_prev is not None or recv_prev: + if prev_rank is None: + prev_rank = gpc.get_prev_global_rank(ParallelMode.PIPELINE) + + if object_send_next is not None or recv_next: + if next_rank is None: + next_rank = gpc.get_next_global_rank(ParallelMode.PIPELINE) + + if object_send_prev is not None: + object_send_prev = process_object_to_send(object_send_prev, scatter_gather_tensors) + + if object_send_next is not None: + object_send_next = process_object_to_send(object_send_next, scatter_gather_tensors) + + ops = [] + if object_send_prev is not None: + filling_ops_queue(object_send_prev, dist.isend, prev_rank, ops) + + if tensor_recv_prev is not None: + filling_ops_queue(tensor_recv_prev, dist.irecv, prev_rank, ops) + + if tensor_recv_next is not None: + filling_ops_queue(tensor_recv_next, dist.irecv, next_rank, ops) + + if object_send_next is not None: + filling_ops_queue(object_send_next, dist.isend, next_rank, ops) + + if len(ops) > 0: + reqs = dist.batch_isend_irecv(ops) + for req in reqs: + req.wait() + # To protect against race condition when using batch_isend_irecv(). + torch.cuda.synchronize() + + if recv_prev and recv_prev_split: + if isinstance(tensor_recv_prev, torch.Tensor): + tensor_recv_prev = gather_split_1d_tensor(tensor_recv_prev).view(recv_prev_shape).requires_grad_() + else: + for index in range(len(tensor_recv_prev)): + tensor_recv_prev[index] = ( + gather_split_1d_tensor(tensor_recv_prev[index]).view(recv_prev_shape[index]).requires_grad_() + ) + + if recv_next and recv_next_split: + if isinstance(tensor_recv_next, torch.Tensor): + tensor_recv_next = gather_split_1d_tensor(tensor_recv_next).view(recv_next_shape).requires_grad_() + else: + for index in range(len(tensor_recv_next)): + tensor_recv_next[index] = ( + gather_split_1d_tensor(tensor_recv_next[index]).view(recv_next_shape[index]).requires_grad_() + ) + + return tensor_recv_prev, tensor_recv_next + + +def recv_forward( + input_tensor_shape, prev_rank=None, dtype=torch.float, scatter_gather_tensors=False +) -> Union[torch.Tensor, List[torch.Tensor]]: + """Copy the forward output from the previous stage in pipeline as the input tensor of this stage. + + Args: + input_tensor_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the tensor + to be received. + prev_rank (int, optional): The rank of the source of the tensor. + + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: The input tensor or input tensor list. + """ + input_tensor, _ = _communicate( + recv_prev=True, + recv_prev_shape=input_tensor_shape, + prev_rank=prev_rank, + dtype=dtype, + scatter_gather_tensors=scatter_gather_tensors, + ) + return input_tensor + + +def recv_backward( + output_grad_shape, next_rank=None, dtype=torch.float, scatter_gather_tensors=False +) -> Union[torch.Tensor, List[torch.Tensor]]: + """Copy the gradient tensor from the next stage in pipeline as the input gradient of this stage. + + Args: + output_grad_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the tensor + to be received. + next_rank (int, optional): The rank of the source of the tensor. + + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: The input gradient tensor or gradident tensor list. + """ + _, output_tensor_grad = _communicate( + recv_next=True, + recv_next_shape=output_grad_shape, + next_rank=next_rank, + dtype=dtype, + scatter_gather_tensors=scatter_gather_tensors, + ) + return output_tensor_grad + + +def send_forward(output_tensor, next_rank=None, scatter_gather_tensors=False) -> None: + """Sends the input tensor to the next stage in pipeline. + + Args: + output_tensor (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Tensor to be sent. + next_rank (int, optional): The rank of the recipient of the tensor. + """ + _communicate(object_send_next=output_tensor, next_rank=next_rank, scatter_gather_tensors=scatter_gather_tensors) + + +def send_backward(input_tensor_grad, prev_rank=None, scatter_gather_tensors=False) -> None: + """Sends the gradient tensor to the previous stage in pipeline. + + Args: + input_tensor_grad (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Tensor to be sent + prev_rank (int, optional): The rank of the recipient of the tensor + """ + + _communicate(object_send_prev=input_tensor_grad, prev_rank=prev_rank, scatter_gather_tensors=scatter_gather_tensors) + + +def send_forward_recv_backward( + output_tensor, output_grad_shape, next_rank=None, dtype=torch.float, scatter_gather_tensors=False +) -> Union[torch.Tensor, List[torch.Tensor]]: + """Batched communication operation. Sends the input tensor to the + next stage in pipeline, while receives the gradient tensor from the + next stage in pipeline as the input gradient tensor of this stage. + + Args: + output_tensor (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Tensor to be sent. + output_grad_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the tensor + to be received. + + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: The input gradient tensor. + """ + _, output_tensor_grad = _communicate( + object_send_next=output_tensor, + recv_next=output_grad_shape is not None, + recv_next_shape=output_grad_shape, + next_rank=next_rank, + dtype=dtype, + scatter_gather_tensors=scatter_gather_tensors, + ) + + return output_tensor_grad + + +def send_backward_recv_forward( + input_tensor_grad, + input_tensor_shape, + prev_rank=None, + dtype=torch.float, + scatter_gather_tensors=False, +) -> Union[torch.Tensor, List[torch.Tensor]]: + """Batched communication operation. Sends the gradient tensor to the + previous stage in pipeline, while receives the output tensor from the + previous stage in pipeline as the input of this stage. + + Args: + input_tensor_grad (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Tensor to be sent. + input_tensor_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the tensor + to be received. + + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: The input tensor. + """ + input_tensor, _ = _communicate( + object_send_prev=input_tensor_grad, + recv_prev=input_tensor_shape is not None, + recv_prev_shape=input_tensor_shape, + prev_rank=prev_rank, + dtype=dtype, + scatter_gather_tensors=scatter_gather_tensors, + ) + + return input_tensor + + +def send_forward_recv_forward( + output_tensor, + input_tensor_shape, + prev_rank=None, + next_rank=None, + dtype=torch.float, + scatter_gather_tensors=False, +) -> Union[torch.Tensor, List[torch.Tensor]]: + """Batched communication operation. Sends the input tensor to the + next stage in pipeline, while receives the output tensor from the + previous stage in pipeline as the input of this stage. + + Args: + output_tensor (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Tensor to be sent. + input_tensor_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the tensor + to be received. + + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: The input tensor. + """ + input_tensor, _ = _communicate( + object_send_next=output_tensor, + recv_prev=input_tensor_shape is not None, + recv_prev_shape=input_tensor_shape, + prev_rank=prev_rank, + next_rank=next_rank, + dtype=dtype, + scatter_gather_tensors=scatter_gather_tensors, + ) + return input_tensor + + +def send_backward_recv_backward( + input_tensor_grad, + output_grad_shape, + prev_rank=None, + next_rank=None, + dtype=torch.float, + scatter_gather_tensors=False, +) -> Union[torch.Tensor, List[torch.Tensor]]: + """Batched communication operation. Sends the gradient tensor to the + previous stage in pipeline, while receives the gradient tensor from the + next member in pipeline as the input of this stage. + + Args: + input_tensor_grad (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Tensor to be sent. + output_grad_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the tensor + to be received. + + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: The input gradient tensor. + """ + _, output_tensor_grad = _communicate( + object_send_prev=input_tensor_grad, + recv_next=output_grad_shape is not None, + recv_next_shape=output_grad_shape, + prev_rank=prev_rank, + next_rank=next_rank, + dtype=dtype, + scatter_gather_tensors=scatter_gather_tensors, + ) + return output_tensor_grad + + +def send_forward_backward_recv_forward_backward( + output_tensor, + input_tensor_grad, + input_tensor_shape, + output_grad_shape, + prev_rank=None, + next_rank=None, + dtype=torch.float, + scatter_gather_tensors=False, +) -> Tuple[Union[torch.Tensor, List[torch.Tensor]]]: + """Batched communication operation. Sends the input tensor to the next stage in pipeline and + the gradient tensor to the previous stage, while receives the input gradient tensor from the + next stage and the input tensor from the previous stage. + + Args: + output_tensor (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Tensor sent to the next. + input_tensor_grad (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Tensor sent to the previous. + input_tensor_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the tensor received + from the previous. + output_grad_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the tensor received + from the next. + + Returns: + Tuple(Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]], Union[:class:`torch.Tensor`, + List[:class:`torch.Tensor`]]): (the input tensor, the input gradient tensor) + """ + input_tensor, output_tensor_grad = _communicate( + object_send_next=output_tensor, + object_send_prev=input_tensor_grad, + recv_prev=input_tensor_shape is not None, + recv_next=output_grad_shape is not None, + recv_prev_shape=input_tensor_shape, + recv_next_shape=output_grad_shape, + prev_rank=prev_rank, + next_rank=next_rank, + dtype=dtype, + scatter_gather_tensors=scatter_gather_tensors, + ) + return input_tensor, output_tensor_grad + + +def send_forward_and_recv_next_forward_async( + output_tensor, + recv_prev_shape: Union[torch.Size, List[torch.Size]] = None, + dtype: torch.dtype = None, + scatter_gather_tensors=False, +): + """send forward output to next rank and recv forward input from prev rank""" + + reqs = [] + tensor_recv_prev = None + + # prepare send opreations + if output_tensor is not None: + next_rank = gpc.get_next_global_rank(ParallelMode.PIPELINE) + + output_tensor = process_object_to_send(output_tensor, scatter_gather_tensors) + + if isinstance(output_tensor, torch.Tensor): + reqs.append(dist.P2POp(dist.isend, output_tensor, next_rank)) + else: + for tensor_to_comm in output_tensor: + reqs.append(dist.P2POp(dist.isend, tensor_to_comm, next_rank)) + + # prepare receive opreations + if recv_prev_shape is not None: + prev_rank = gpc.get_prev_global_rank(ParallelMode.PIPELINE) + # create receive buffer + tensor_recv_prev, recv_prev_split = create_recv_buffer_with_shapes( + recv_prev_shape, dtype, scatter_gather_tensors + ) + # generate async receive opterations + if isinstance(tensor_recv_prev, torch.Tensor): + reqs.append(dist.P2POp(dist.irecv, tensor_recv_prev, prev_rank)) + else: + for tensor_to_comm in tensor_recv_prev: + reqs.append(dist.P2POp(dist.irecv, tensor_to_comm, prev_rank)) + + if len(reqs) > 0: + reqs = dist.batch_isend_irecv(reqs) + + # return and do other things + yield + + # check communication completed + for req in reqs: + req.wait() + # To protect against race condition when using batch_isend_irecv() + torch.cuda.synchronize() + + # Process received data + if recv_prev_shape is not None and recv_prev_split: + if isinstance(tensor_recv_prev, torch.Tensor): + tensor_recv_prev = gather_split_1d_tensor(tensor_recv_prev).view(recv_prev_shape).requires_grad_() + else: + for index in range(len(tensor_recv_prev)): + tensor_recv_prev[index] = ( + gather_split_1d_tensor(tensor_recv_prev[index]).view(recv_prev_shape[index]).requires_grad_() + ) + + yield tensor_recv_prev + + +def send_backward_and_recv_next_backward_async( + input_tensor, + recv_next_shape: Union[torch.Size, List[torch.Size]] = None, + dtype: torch.dtype = None, + scatter_gather_tensors=False, +): + reqs = [] + tensor_recv_next = None + + # prepare send opreations + if input_tensor is not None: + prev_rank = gpc.get_prev_global_rank(ParallelMode.PIPELINE) + + input_tensor = process_object_to_send(input_tensor, scatter_gather_tensors) + + if isinstance(input_tensor, torch.Tensor): + reqs.append(dist.P2POp(dist.isend, input_tensor, prev_rank)) + else: + for tensor_to_comm in input_tensor: + reqs.append(dist.P2POp(dist.isend, tensor_to_comm, prev_rank)) + + # prepare receive opreations + if recv_next_shape is not None: + next_rank = gpc.get_next_global_rank(ParallelMode.PIPELINE) + # create receive buffer + tensor_recv_next, recv_next_split = create_recv_buffer_with_shapes( + recv_next_shape, dtype, scatter_gather_tensors + ) + # generate async receive opreations + if isinstance(tensor_recv_next, torch.Tensor): + reqs.append(dist.P2POp(dist.irecv, tensor_recv_next, next_rank)) + else: + for tensor_to_comm in tensor_recv_next: + reqs.append(dist.P2POp(dist.irecv, tensor_to_comm, next_rank)) + + if len(reqs) > 0: + reqs = dist.batch_isend_irecv(reqs) + + # return and do other things + yield + + # check communication completed + for req in reqs: + req.wait() + # To protect against race condition when using batch_isend_irecv() + torch.cuda.synchronize() + + # Process received data + if recv_next_shape is not None and recv_next_split: + if isinstance(tensor_recv_next, torch.Tensor): + tensor_recv_next = gather_split_1d_tensor(tensor_recv_next).view(recv_next_shape).requires_grad_() + else: + for index in range(len(tensor_recv_next)): + tensor_recv_next[index] = ( + gather_split_1d_tensor(tensor_recv_next[index]).view(recv_next_shape[index]).requires_grad_() + ) + + yield tensor_recv_next + + +class AsynCommunicator: + """AsynCommunicator for managing async communication.""" + + def __init__( + self, + tensor_to_send: Union[torch.Tensor, List[torch.Tensor]], + recv_shape: Union[torch.Size, List[torch.Size]], + dtype: torch.dtype = None, + scatter_gather_tensors=False, + forward: bool = True, + ) -> None: + self._need_receive = recv_shape is not None + + if forward: + self._coroutine = send_forward_and_recv_next_forward_async( + tensor_to_send, recv_shape, dtype, scatter_gather_tensors + ) + else: + self._coroutine = send_backward_and_recv_next_backward_async( + tensor_to_send, recv_shape, dtype, scatter_gather_tensors + ) + + @property + def need_receive(self) -> bool: + return self._need_receive + + def start(self) -> None: + next(self._coroutine) + + def wait_and_receive(self) -> Union[torch.Tensor, List[torch.Tensor]]: + received = next(self._coroutine) + self._coroutine.close() + + return received diff --git a/InternLM/internlm/core/communication/utils.py b/InternLM/internlm/core/communication/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f41328605a498118eb61d5f6ab68c28dc9eef340 --- /dev/null +++ b/InternLM/internlm/core/communication/utils.py @@ -0,0 +1,125 @@ +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/communication + +from typing import List, Tuple, Union + +import torch +import torch.distributed as dist + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.utils.common import get_current_device + +TensorShape = Union[torch.Size, List[int], Tuple[int]] + + +def send_meta_helper(obj, next_rank, tensor_kwargs): + send_shape = torch.tensor(obj.size(), **tensor_kwargs) + send_ndims = torch.tensor(len(obj.size()), **tensor_kwargs) + dist.send(send_ndims, next_rank) + dist.send(send_shape, next_rank) + + +def send_obj_meta(obj, next_rank=None): + """Sends obj meta information before sending a specific obj. + Since the recipient must know the shape of the obj in p2p communications, + meta information of the obj should be sent before communications. This function + synchronizes with :func:`recv_obj_meta`. + + Args: + obj (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): obj to be sent. + need_meta (bool, optional): If False, meta information won't be sent. + next_rank (int): The rank of the next member in pipeline parallel group. + + Returns: + bool: False + """ + if next_rank is None: + next_rank = gpc.get_next_global_rank(ParallelMode.PIPELINE) + + tensor_kwargs = {"dtype": torch.long, "device": get_current_device()} + if isinstance(obj, torch.Tensor): + send_obj_nums = torch.tensor(1, **tensor_kwargs) + dist.send(send_obj_nums, next_rank) + send_meta_helper(obj, next_rank, tensor_kwargs) + else: + send_obj_nums = torch.tensor(len(obj), **tensor_kwargs) + dist.send(send_obj_nums, next_rank) + for tensor_to_send in obj: + send_meta_helper(tensor_to_send, next_rank, tensor_kwargs) + + +def recv_meta_helper(prev_rank, tensor_kwargs): + recv_ndims = torch.empty((), **tensor_kwargs) + dist.recv(recv_ndims, prev_rank) + recv_shape = torch.empty(recv_ndims, **tensor_kwargs) + dist.recv(recv_shape, prev_rank) + return recv_shape + + +def recv_obj_meta(prev_rank=None) -> torch.Size: + """Receives obj meta information before receiving a specific obj. + Since the recipient must know the shape of the obj in p2p communications, + meta information of the obj should be received before communications. This function + synchronizes with :func:`send_obj_meta`. + + Args: + obj_shape (Union[:class:`torch.Size`, List[:class:`torch.Size`]]): The shape of the obj to be received. + prev_rank (int): The rank of the source of the obj. + + Returns: + Union[:class:`torch.Size`, List[:class:`torch.Size`]]: The shape of the obj to be received. + """ + if prev_rank is None: + prev_rank = gpc.get_prev_global_rank(ParallelMode.PIPELINE) + + tensor_kwargs = {"dtype": torch.long, "device": get_current_device()} + recv_obj_nums = torch.empty((), **tensor_kwargs) + dist.recv(recv_obj_nums, prev_rank) + if recv_obj_nums.item() == 1: + recv_shape = recv_meta_helper(prev_rank, tensor_kwargs) + obj_shape = torch.Size(recv_shape) + else: + obj_shape = [] + for _ in range(recv_obj_nums.item()): + recv_shape = recv_meta_helper(prev_rank, tensor_kwargs) + obj_shape.append(torch.Size(recv_shape)) + + return obj_shape + + +def split_tensor_into_1d_equal_chunks(tensor: torch.Tensor, new_buffer=False) -> torch.Tensor: + """Break a tensor into equal 1D chunks. + + Args: + tensor (:class:`torch.Tensor`): Tensor to be split before communication. + new_buffer (bool, optional): Whether to use a new buffer to store sliced tensor. + + Returns: + :class:`torch.Tensor`: The split tensor + """ + partition_size = torch.numel(tensor) // gpc.get_world_size(ParallelMode.TENSOR) + start_index = partition_size * gpc.get_local_rank(ParallelMode.TENSOR) + end_index = start_index + partition_size + if new_buffer: + data = torch.empty(partition_size, dtype=tensor.dtype, device=torch.cuda.current_device(), requires_grad=False) + data.copy_(tensor.view(-1)[start_index:end_index]) + else: + data = tensor.view(-1)[start_index:end_index] + return data + + +def gather_split_1d_tensor(tensor: torch.Tensor) -> torch.Tensor: + """Opposite of above function, gather values from model parallel ranks. + + Args: + tensor (:class:`torch.Tensor`): Tensor to be gathered after communication. + Returns: + :class:`torch.Tensor`: The gathered tensor. + """ + world_size = gpc.get_world_size(ParallelMode.TENSOR) + numel = torch.numel(tensor) + numel_gathered = world_size * numel + gathered = torch.empty(numel_gathered, dtype=tensor.dtype, device=torch.cuda.current_device(), requires_grad=False) + chunks = [gathered[i * numel : (i + 1) * numel] for i in range(world_size)] + dist.all_gather(chunks, tensor, group=gpc.get_group(ParallelMode.TENSOR)) + return gathered diff --git a/InternLM/internlm/core/context/__init__.py b/InternLM/internlm/core/context/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3fc7deb58e47f65002b8c46287c41a2ce717645e --- /dev/null +++ b/InternLM/internlm/core/context/__init__.py @@ -0,0 +1,49 @@ +from .parallel_context import ( + IS_TENSOR_PARALLEL, + Config, + ParallelContext, + global_context, +) +from .process_group_initializer import ( + Initializer_Data, + Initializer_Model, + Initializer_Nettest, + Initializer_Pipeline, + Initializer_Tensor, + Initializer_Zero1, + ParallelMode, + ProcessGroupInitializer, +) +from .random import ( + add_seed, + get_current_mode, + get_seeds, + get_states, + seed, + set_mode, + set_seed_states, + sync_states, +) + +__all__ = [ + "Config", + "IS_TENSOR_PARALLEL", + "global_context", + "ParallelContext", + "ParallelMode", + "Initializer_Tensor", + "Initializer_Pipeline", + "Initializer_Data", + "Initializer_Zero1", + "Initializer_Nettest", + "ProcessGroupInitializer", + "Initializer_Model", + "seed", + "set_mode", + "add_seed", + "get_seeds", + "get_states", + "get_current_mode", + "set_seed_states", + "sync_states", +] diff --git a/InternLM/internlm/core/context/parallel_context.py b/InternLM/internlm/core/context/parallel_context.py new file mode 100644 index 0000000000000000000000000000000000000000..1fc1542a174637e12a2ec0991fa9fa5c1ca6273b --- /dev/null +++ b/InternLM/internlm/core/context/parallel_context.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/context + +import inspect +import random +import socket +import sys +from collections import Counter +from importlib.machinery import SourceFileLoader +from pathlib import Path +from typing import Union + +import numpy as np +import torch +import torch.distributed as dist + +from internlm.utils.common import SingletonMeta +from internlm.utils.logger import get_logger +from internlm.utils.timeout import LLM_NCCL_TIMEOUT + +from . import process_group_initializer as pgroup_initializer +from .process_group_initializer import ParallelMode +from .random import add_seed, get_seeds, set_mode + +IS_TENSOR_PARALLEL = "is_tensor_parallel" + +logger = get_logger(__file__) + + +class Config(dict): + """This is a wrapper class for dict objects so that values of which can be + accessed as attributes. + + Args: + config (dict): The dict object to be wrapped. + """ + + def __init__(self, config: dict = None): # pylint: disable=W0231 + if config is not None: + for k, v in config.items(): + self._add_item(k, v) + + def __missing__(self, key): + raise KeyError(key) + + def __getattr__(self, key): + try: + value = super().__getitem__(key) + return value + except KeyError: + raise AttributeError(key) + + def __setattr__(self, key, value): + super().__setitem__(key, value) + + def _add_item(self, key, value): + if isinstance(value, dict): + self.__setattr__(key, Config(value)) + else: + self.__setattr__(key, value) + + def update(self, config): + assert isinstance(config, (Config, dict)), "can only update dictionary or Config objects." + for k, v in config.items(): + self._add_item(k, v) + return self + + @staticmethod + def from_file(filename: str) -> object: + """Reads a python file and constructs a corresponding :class:`Config` object. + + Args: + filename (str): Name of the file to construct the return object. + + Returns: + :class:`Config`: A :class:`Config` object constructed with information in the file. + + Raises: + AssertionError: Raises an AssertionError if the file does not exist, or the file is not .py file + """ + + # check config path + if isinstance(filename, str): + filepath = Path(filename).absolute() + elif isinstance(filename, Path): + filepath = filename.absolute() + + assert filepath.exists(), f"{filename} is not found, please check your configuration path" + + # check extension + extension = filepath.suffix + assert extension == ".py", "only .py files are supported" + + # import the config as module + remove_path = False + if filepath.parent not in sys.path: + sys.path.insert(0, (filepath)) + remove_path = True + + module_name = filepath.stem + source_file = SourceFileLoader(fullname=str(module_name), path=str(filepath)) + module = source_file.load_module() # pylint: disable=W4902,E1120,W1505 + + # load into config + config = Config() + + for k, v in module.__dict__.items(): + if k.startswith("__") or inspect.ismodule(v) or inspect.isclass(v): + continue + else: + config._add_item(k, v) + + # remove module + del sys.modules[module_name] + if remove_path: + sys.path.pop(0) + + return config + + +class ParallelContext(metaclass=SingletonMeta): + """This class provides interface functions for users to get the parallel context, + such as the global rank, the local rank, the world size, etc. of each device. + + """ + + def __init__(self): + # distributed settings + self._global_ranks = dict() + self._local_ranks = dict() + self._world_sizes = dict() + self._groups = dict() + self._cpu_groups = dict() + self._ranks_in_group = dict() + + # load config from file + self._config = None + + # default parallel args, will be overwritten during process group intialization + self.world_size = 1 + self.data_parallel_size = 1 + self.pipeline_parallel_size = 1 + self.tensor_parallel_size = 1 + self.zero1_parallel_size = -1 + self.nettest_parallel_size = 1 + self.num_processes_on_current_node = -1 + self.virtual_pipeline_parallel_size = None + self.virtual_pipeline_parallel_rank = None + + @property + def config(self): + return self._config + + def load_config(self, config: Union[dict, str]): + """Loads the configuration from either a dict or a file. + + Args: + config (dict or str): Either a dict containing the configuration information or the filename + of a file containing the configuration information. + + Raises: + TypeError: Raises a TypeError if `config` is neither a dict nor a str. + """ + if isinstance(config, str): + self._config = Config.from_file(config) + elif isinstance(config, dict): + self._config = Config(config) + else: + raise TypeError("Invalid type for config, only dictionary or string is supported") + + def detect_num_processes_on_current_node(self): + hostname = socket.gethostname() + hostname_list = [None for _ in range(self.get_world_size(ParallelMode.GLOBAL))] + dist.all_gather_object(hostname_list, hostname, group=self.get_group(ParallelMode.GLOBAL)) + counter = Counter(hostname_list) + self.num_processes_on_current_node = counter[hostname] + + @staticmethod + def _check_parallel_mode(parallel_mode: ParallelMode): + assert isinstance( + parallel_mode, ParallelMode + ), f"expected the argument parallel_mode to be of enum ParallelMode, but got {type(parallel_mode)}" + + def get_global_rank(self): + """Returns the global rank of the current device. + + Returns: + int: The global rank of the current device + """ + return self._global_ranks[ParallelMode.GLOBAL] + + def get_local_rank(self, parallel_mode: ParallelMode): + """Returns the local rank of the current device. + + Args: + parallel_mode: The parallel mode for the rank. + + Returns: + int: The local rank of the current device for `parallel_mode`. + """ + self._check_parallel_mode(parallel_mode) + return self._local_ranks.get(parallel_mode, 0) + + def get_next_global_rank(self, parallel_mode: ParallelMode): + """Returns the global rank of the next device. + + Args: + parallel_mode: The parallel mode for the rank. + + Returns: + int: The global rank of the next device for `parallel_mode`. + """ + self._check_parallel_mode(parallel_mode) + + # get rank and world size + local_rank = self.get_local_rank(parallel_mode) + world_size = self.get_world_size(parallel_mode) + ranks_in_group = self.get_ranks_in_group(parallel_mode) + + return ranks_in_group[(local_rank + 1) % world_size] + + def get_prev_global_rank(self, parallel_mode: ParallelMode): + """Returns the global rank of the previous device. + + Args: + parallel_mode: The chosen parallel mode. + + Returns: + int: The global rank of the previous device for `parallel_mode`. + """ + self._check_parallel_mode(parallel_mode) + + # get rank and world size + local_rank = self.get_local_rank(parallel_mode) + world_size = self.get_world_size(parallel_mode) + ranks_in_group = self.get_ranks_in_group(parallel_mode) + + return ranks_in_group[(local_rank - 1) % world_size] + + def is_using_dp(self): + """Returns a boolean value indicating whether the current device is initilized with + ParallelMode.DATA and its world_size is greater than 1. + """ + return self.is_initialized(ParallelMode.DATA) and self.get_world_size(ParallelMode.DATA) > 1 + + def is_using_tp(self): + """Returns a boolean value indicating whether the current device is initilized with + ParallelMode.TENSOR and its world_size is greater than 1. + """ + return self.is_initialized(ParallelMode.TENSOR) and self.get_world_size(ParallelMode.TENSOR) > 1 + + def is_using_pp(self): + """Returns a boolean value indicating whether the current device is initilized with + ParallelMode.PIPELINE and its world_size is greater than 1. + """ + return self.is_initialized(ParallelMode.PIPELINE) and self.get_world_size(ParallelMode.PIPELINE) > 1 + + def is_using_sequence(self): + """Returns a boolean value indicating whether the current device is initilized with + ParallelMode.SEQUENCE and its world_size is greater than 1. + """ + return False + # return gpc.is_initialized(ParallelMode.SEQUENCE) and gpc.get_world_size(ParallelMode.SEQUENCE) > 1 + + def is_first_rank(self, parallel_mode: ParallelMode): + """Returns a boolean value indicating whether the current device is the first one + among its group for `parallel_mode`. + + Args: + parallel_mode: The chosen parallel mode. + + Returns: + bool: a boolean value indicating whether the current device is the first one + among its group for `parallel_mode`. + """ + rank = 0 + if self.is_initialized(parallel_mode): + rank = self.get_local_rank(parallel_mode) + return rank == 0 + + def is_rank_for_log(self): + """Returns a boolean value indicating whether the current device should print log.""" + is_log_rank = ( + self.is_first_rank(ParallelMode.DATA) + and self.is_first_rank(ParallelMode.TENSOR) + and self.is_last_rank(ParallelMode.PIPELINE) + ) + return is_log_rank + + def is_last_rank(self, parallel_mode: ParallelMode): + """Returns a boolean value indicating whether the current device is the last one + among its group for `parallel_mode`. + + Args: + parallel_mode: The chosen parallel mode. + + Returns: + bool: a boolean value indicating whether the current device is the first one + among its group for `parallel_mode`. + """ + rank = 0 + world_size = 1 + if self.is_initialized(parallel_mode): + rank = self.get_local_rank(parallel_mode) + world_size = self.get_world_size(parallel_mode) + return rank == world_size - 1 + + def is_pipeline_first_stage(self, ignore_virtual=False): + if not ignore_virtual: + if self.virtual_pipeline_parallel_size is not None and self.virtual_pipeline_parallel_rank != 0: + return False + return self.is_first_rank(ParallelMode.PIPELINE) + + def is_pipeline_last_stage(self, ignore_virtual=False): + if not ignore_virtual: + if ( + self.virtual_pipeline_parallel_size is not None + and self.virtual_pipeline_parallel_rank != self.virtual_pipeline_parallel_size - 1 + ): + return False + return self.is_last_rank(ParallelMode.PIPELINE) + + def get_world_size(self, parallel_mode: ParallelMode): + """Returns the world size for `parallel_mode`. + + Args: + parallel_mode: The chosen parallel mode. + + Returns: + int: The world size for `parallel_mode`. + """ + self._check_parallel_mode(parallel_mode) + return self._world_sizes.get(parallel_mode, 1) + + def get_group(self, parallel_mode: ParallelMode): + """Returns the group of the current device for `parallel_mode`. + + Args: + parallel_mode: The chosen parallel mode. + + Returns: + torch.distributed.ProcessGroup: The group of the current device for `parallel_mode`. + """ + self._check_parallel_mode(parallel_mode) + return self._groups[parallel_mode] + + def get_ranks_in_group(self, parallel_mode: ParallelMode): + """Returns the rank of the current device for `parallel_mode` in the group. + + Args: + parallel_mode: The chosen parallel mode. + + Returns: + int: The rank of the current device for `parallel_mode` in the group. + """ + self._check_parallel_mode(parallel_mode) + return self._ranks_in_group[parallel_mode] + + def get_cpu_group(self, parallel_mode: ParallelMode): + self._check_parallel_mode(parallel_mode) + return self._cpu_groups[parallel_mode] + + def init_global_dist(self, rank: int, world_size: int, backend: str, host: str, port: int, use_cpu: bool = False): + """Initializes the global distributed environment + + Args: + rank (int): rank for the default process group. + world_size (int): world size of the default process group. + backend (str): backend for ``torch.distributed`` + host (str): the master address for distributed training. + port (str): the master port for distributed training. + use_cpu (bool): whether to set up cpu process group. + """ + # initialize the default process group + init_method = f"tcp://[{host}]:{port}" + dist.init_process_group( + rank=rank, + world_size=world_size, + backend=backend, + init_method=init_method, + timeout=LLM_NCCL_TIMEOUT, + ) + + # None will give the default global process group for pytorch dist operations + ranks = list(range(world_size)) + if use_cpu: + cpu_group = ( + dist.new_group(ranks, backend="gloo", timeout=LLM_NCCL_TIMEOUT) + if dist.get_backend() != "gloo" + else None + ) + else: + cpu_group = None + self._register_dist(rank, world_size, dist.GroupMember.WORLD, cpu_group, ranks, ParallelMode.GLOBAL) + self._global_ranks[ParallelMode.GLOBAL] = rank + + def _register_dist(self, local_rank, world_size, process_group, cpu_group, ranks_in_group, mode): + self._check_parallel_mode(mode) + self._local_ranks[mode] = local_rank + self._world_sizes[mode] = world_size + self._groups[mode] = process_group + self._cpu_groups[mode] = cpu_group + self._ranks_in_group[mode] = ranks_in_group + + def check_sanity(self): + """Checks sanity of the parallel context. + + Raises: + AssertionError: Raises an AssertionError if the world size does not equal to the product + of data parallel size, pipeline parallel size and tensor parallel size. + """ + dps = self.data_parallel_size + pps = self.pipeline_parallel_size + tps = self.tensor_parallel_size + ws = self.world_size + assert ws == dps * pps * tps, ( + f"Expected the world size {ws} to be equal to data" + f" parallel size ({dps}) * pipeline parallel size " + f"({pps}) * tensor parallel size ({tps})" + ) + assert self.zero1_parallel_size > 0 + assert self.data_parallel_size % self.zero1_parallel_size == 0 + + def _set_parallel_size_from_config(self, config: dict, key: str, attr_name: str): + if key in config: + ele = config[key] + if isinstance(ele, int): + setattr(self, attr_name, ele) + elif isinstance(ele, dict): + setattr(self, attr_name, ele["size"]) + else: + raise NotImplementedError( + f'{"Parallel configuration does not support this kind of argument, please use int or dict"}' + ) + + def init_parallel_groups(self): + """Initializes the parallel groups.""" + + # get rank and world size + rank = self.get_global_rank() + world_size = self.get_world_size(ParallelMode.GLOBAL) + self.world_size = world_size + + # set parallel size as attributes for global context + parallel_config = self.config.get("parallel", None) + if parallel_config is not None: + self._set_parallel_size_from_config(parallel_config, "pipeline", "pipeline_parallel_size") + self._set_parallel_size_from_config(parallel_config, "tensor", "tensor_parallel_size") + self._set_parallel_size_from_config(parallel_config, "zero1", "zero1_parallel_size") + + # the user should not set the data parallel size manually + # instead, it should be calculated based on other parallel config + self.data_parallel_size = self.world_size // (self.pipeline_parallel_size * self.tensor_parallel_size) + + # the recommended nettest_parallel_size is 32 GPUs + self.nettest_parallel_size = 32 + + if self.zero1_parallel_size <= 0: + self.zero1_parallel_size = self.data_parallel_size + + self.check_sanity() + + initializer_args = [ + rank, + world_size, + self.data_parallel_size, + self.pipeline_parallel_size, + self.tensor_parallel_size, + self.zero1_parallel_size, + self.nettest_parallel_size, + ] + + # run initialization of different process groups + initializers = [] + initializers.append(pgroup_initializer.Initializer_Data(*initializer_args)) + initializers.append(pgroup_initializer.Initializer_Model(*initializer_args)) + initializers.append(pgroup_initializer.Initializer_Tensor(*initializer_args)) + initializers.append(pgroup_initializer.Initializer_Zero1(*initializer_args)) + initializers.append(pgroup_initializer.Initializer_Nettest(*initializer_args)) + if self.pipeline_parallel_size > 1: + initializers.append(pgroup_initializer.Initializer_Pipeline(*initializer_args)) + for initializer in initializers: + parallel_setting = initializer.init_dist_group() + if isinstance(parallel_setting, list): + for args in parallel_setting: + self._register_dist(*args) + else: + self._register_dist(*parallel_setting) + + def is_initialized(self, parallel_mode: ParallelMode): + """Returns a boolean value indicating whether `parallel_mode` is initialized + in the current system. + """ + return parallel_mode in self._groups + + def destroy(self): + """Destroys the current distributed parallel environment.""" + for mode, group in self._groups.items(): + if mode is not ParallelMode.GLOBAL: + dist.destroy_process_group(group) + # destroy global process group + dist.destroy_process_group() + self._groups.clear() + + def set_device(self, device_ordinal: int = None): + """Sets distributed processes to be bound to devices. + + Args: + device_ordinal (int, optional): the device id to be bound to + """ + global_rank = self.get_global_rank() + if device_ordinal is None: + devices_per_node = torch.cuda.device_count() + device_ordinal = global_rank % devices_per_node + + torch.cuda.set_device(device_ordinal) + logger.info(f"process rank {global_rank} is bound to host:{socket.gethostname()} device: {device_ordinal}") + + def set_seed(self, seed: int, dpseed_with_tpoffset: bool = False): + """Sets seeds for all random libraries. + + Args: + seed (int): seed for random states + """ + pipeline_offset = self._local_ranks.get(ParallelMode.PIPELINE, 0) + global_rank = self.get_global_rank() + + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + assert torch.cuda.is_available() + + # data parallel seed are kept the same in the same pipeline stage + dp_seed = seed + if dpseed_with_tpoffset: + dp_seed = seed + pipeline_offset * 1024 + add_seed(ParallelMode.DATA, dp_seed) + add_seed(ParallelMode.DUMMY, dp_seed) + + # model parallel seeds are different across ranks + if self.is_initialized(ParallelMode.TENSOR): + tp_rank = self.get_local_rank(ParallelMode.TENSOR) + tp_seed = seed + tp_rank + pipeline_offset * 1024 + add_seed(ParallelMode.TENSOR, tp_seed) + + # we do not set the random state mode to ParallelMode.DATA until model is built (instead, we use a dummy mode + # during model construction), this is because the random state will be different in different tensor parallel + # device of the same data parallel group. The underlying reason is that the device of tp_rank = 0 will perform + # additional random operations during the RowParallelLinear module building process. + set_mode(ParallelMode.DUMMY) + + seeds = get_seeds() + seed_str = ", ".join([f"{k}: {v}" for k, v in seeds.items()]) + logger.info( + f"initialized seed on rank {global_rank}, " + f"numpy: {seed}, python random: {seed}, {seed_str}," + f"the default parallel seed is {ParallelMode.DATA}." + ) + + def set_virtual_pipeline_parallel_size(self, size): + self.virtual_pipeline_parallel_size = size + + def set_virtual_pipeline_parallel_rank(self, rank): + self.virtual_pipeline_parallel_rank = rank + + +global_context = ParallelContext() diff --git a/InternLM/internlm/core/context/process_group_initializer.py b/InternLM/internlm/core/context/process_group_initializer.py new file mode 100644 index 0000000000000000000000000000000000000000..97e9ef017bc761e92500103b75fa69ef45c3d315 --- /dev/null +++ b/InternLM/internlm/core/context/process_group_initializer.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/context + +import math +from abc import ABC, abstractmethod +from enum import Enum + +import torch.distributed as dist + +from internlm.utils.timeout import LLM_NCCL_TIMEOUT + + +# parallel modes +class ParallelMode(Enum): + """This is an enumeration class containing all possible parallel modes.""" + + GLOBAL = "global" + + # common parallel + DATA = "data" + + # model parallel - containing tensor and pipeline parallel groups + # this is added to facilitate amp and grad clipping in hybrid parallel + MODEL = "model" + + # pipeline parallel + PIPELINE = "pipe" + + # containing all ranks in tensor parallel + TENSOR = "tensor" + + # zero1 parallel + ZERO1 = "zero1" + + # runntime network test + NETTEST = "nettest" + + # dummy mode, only used during mode construction + DUMMY = "dummy" + + +class ProcessGroupInitializer(ABC): + """An object, knowing the parallelism configuration, that initializes parallel groups. + + Args: + rank (int): The rank of current process. + world_size (int): Size of whole communication world. + data_parallel_size (int): Size of data parallel. + pipeline_parallel_size (int): Size of pipeline parallel. + tensor_parallel_size (int): Size of tensor parallel. + zero1_parallel_size (int): Size of zero1 parallel. + """ + + def __init__( + self, + rank: int, + world_size: int, + data_parallel_size: int, + pipeline_parallel_size: int, + tensor_parallel_size: int, + zero1_parallel_size: int, + nettest_parallel_size: int, + ): + self.rank = rank + self.world_size = world_size + self.data_parallel_size = data_parallel_size + self.pipeline_parallel_size = pipeline_parallel_size + self.tensor_parallel_size = tensor_parallel_size + self.zero1_parallel_size = zero1_parallel_size + self.nettest_parallel_size = nettest_parallel_size + super().__init__() + + @abstractmethod + def init_dist_group(self, use_cpu: bool = False): + pass + + +class Initializer_Data(ProcessGroupInitializer): + """A ProcessGroupInitializer for data parallelism. + + Args: + rank (int): The rank of current process. + world_size (int): Size of whole communication world. + data_parallel_size (int): Size of data parallel. + pipeline_parallel_size (int): Size of pipeline parallel. + tensor_parallel_size (int): Size of tensor parallel. + zero1_parallel_size (int): Size of zero1 parallel. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rank_num_per_dp_group = self.world_size // self.data_parallel_size + + assert self.world_size % self.data_parallel_size == 0 + + def init_dist_group(self, use_cpu: bool = False): + """Initialize data parallel groups, and assign local_ranks and groups to each gpu. + + Returns: + Tuple (local_rank, group_world_size, process_group, ranks_in_group, mode): + A Data parallelism's information tuple. + """ + local_rank = None + ranks_in_group = None + process_group = None + cpu_group = None + group_world_size = None + mode = ParallelMode.DATA + + for i in range(self.rank_num_per_dp_group): + ranks = [i + j * self.rank_num_per_dp_group for j in range(self.data_parallel_size)] + group = dist.new_group(ranks, timeout=LLM_NCCL_TIMEOUT) + if use_cpu: + group_cpu = ( + dist.new_group(ranks, backend="gloo", timeout=LLM_NCCL_TIMEOUT) + if dist.get_backend() != "gloo" + else group + ) + else: + group_cpu = None + + if self.rank in ranks: + local_rank = ranks.index(self.rank) + group_world_size = len(ranks) + process_group = group + cpu_group = group_cpu + ranks_in_group = ranks + + return local_rank, group_world_size, process_group, cpu_group, ranks_in_group, mode + + +class Initializer_Model(ProcessGroupInitializer): + """A ProcessGroupInitializer for model parallelism (model parallel group contains pipeline and tensor parallel + groups). + + Args: + rank (int): The rank of current process. + world_size (int): Size of whole communication world. + data_parallel_size (int): Size of data parallel. + pipeline_parallel_size (int): Size of pipeline parallel. + tensor_parallel_size (int): Size of tensor parallel. + zero1_parallel_size (int): Size of zero1 parallel. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rank_num_per_group = self.tensor_parallel_size * self.pipeline_parallel_size + self.num_group = self.world_size // self.rank_num_per_group + + assert self.world_size % self.rank_num_per_group == 0 + + def init_dist_group(self, use_cpu: bool = False): + """Initialize model parallel groups, and assign local_ranks and groups to each gpu. + + Returns: + Tuple (local_rank, group_world_size, process_group, ranks_in_group, mode): + A Model parallelism's information tuple. + """ + local_rank = None + ranks_in_group = None + process_group = None + cpu_group = None + group_world_size = None + mode = ParallelMode.MODEL + + for i in range(self.num_group): + ranks = [i * self.rank_num_per_group + j for j in range(self.rank_num_per_group)] + group = dist.new_group(ranks, timeout=LLM_NCCL_TIMEOUT) + if use_cpu: + group_cpu = ( + dist.new_group(ranks, backend="gloo", timeout=LLM_NCCL_TIMEOUT) + if dist.get_backend() != "gloo" + else group + ) + else: + group_cpu = None + + if self.rank in ranks: + local_rank = ranks.index(self.rank) + group_world_size = len(ranks) + process_group = group + cpu_group = group_cpu + ranks_in_group = ranks + + return local_rank, group_world_size, process_group, cpu_group, ranks_in_group, mode + + +class Initializer_Pipeline(ProcessGroupInitializer): + """A ProcessGroupInitializer for pipeline parallelism. + + Args: + rank (int): The rank of current process + world_size (int): Size of whole communication world + data_parallel_size (int): Size of data parallel + pipeline_parallel_size (int): Size of pipeline parallel + tensor_parallel_size (int): Size of tensor parallel + zero1_parallel_size (int): Size of zero1 parallel. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rank_num_per_dp_group = self.world_size // self.data_parallel_size + self.pipeline_stage_size = self.rank_num_per_dp_group // self.pipeline_parallel_size + + assert self.world_size % self.data_parallel_size == 0 + assert self.rank_num_per_dp_group % self.pipeline_parallel_size == 0 + + def init_dist_group(self, use_cpu: bool = False): + """Initialize pipeline parallel groups, and assign local_ranks and groups to each gpu. + + Returns: + List[Tuple (local_rank, group_world_size, process_group, ranks_in_group, mode)]: + A Pipeline parallelism's information in list of tuples. + """ + local_rank = None + ranks_in_group = None + process_group = None + cpu_group = None + group_world_size = None + mode = ParallelMode.PIPELINE + + for i in range(self.data_parallel_size): + for j in range(self.pipeline_stage_size): + ranks = list( + range( + i * self.rank_num_per_dp_group + j, + (i + 1) * self.rank_num_per_dp_group, + self.pipeline_stage_size, + ) + ) + pipe_group_size = len(ranks) + pipe_group = dist.new_group(ranks, timeout=LLM_NCCL_TIMEOUT) + if use_cpu: + group_cpu = ( + dist.new_group(ranks, backend="gloo", timeout=LLM_NCCL_TIMEOUT) + if dist.get_backend() != "gloo" + else pipe_group + ) + else: + group_cpu = None + + if self.rank in ranks: + local_rank = ranks.index(self.rank) + group_world_size = pipe_group_size + process_group = pipe_group + cpu_group = group_cpu + ranks_in_group = ranks + + return local_rank, group_world_size, process_group, cpu_group, ranks_in_group, mode + + +class Initializer_Tensor(ProcessGroupInitializer): + """A ProcessGroupInitializer for tensor parallelism. + + Args: + rank (int): The rank of current process. + world_size (int): Size of whole communication world. + data_parallel_size (int): Size of data parallel. + pipeline_parallel_size (int): Size of pipeline parallel. + tensor_parallel_size (int): Size of tensor parallel. + zero1_parallel_size (int): Size of zero1 parallel. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.num_tensor_parallel_group = self.world_size // self.tensor_parallel_size + + assert self.world_size % self.tensor_parallel_size == 0 + + def init_dist_group(self, use_cpu: bool = False): + """Initialize tensor parallel groups, and assign local_ranks and groups to each gpu. + + Returns: + Tuple (local_rank, group_world_size, process_group, ranks_in_group, mode): + A Tensor parallelism's information tuple. + """ + local_rank = None + ranks_in_group = None + process_group = None + cpu_group = None + group_world_size = None + mode = ParallelMode.TENSOR + + for i in range(self.num_tensor_parallel_group): + ranks = [i * self.tensor_parallel_size + j for j in range(self.tensor_parallel_size)] + group = dist.new_group(ranks, timeout=LLM_NCCL_TIMEOUT) + if use_cpu: + group_cpu = ( + dist.new_group(ranks, backend="gloo", timeout=LLM_NCCL_TIMEOUT) + if dist.get_backend() != "gloo" + else group + ) + else: + group_cpu = None + + if self.rank in ranks: + local_rank = ranks.index(self.rank) + group_world_size = len(ranks) + process_group = group + cpu_group = group_cpu + ranks_in_group = ranks + + return local_rank, group_world_size, process_group, cpu_group, ranks_in_group, mode + + +class Initializer_Zero1(ProcessGroupInitializer): + """A ProcessGroupInitializer for zero-1 parallelism. + + Args: + rank (int): The rank of current process. + world_size (int): Size of whole communication world. + data_parallel_size (int): Size of data parallel. + pipeline_parallel_size (int): Size of pipeline parallel. + tensor_parallel_size (int): Size of tensor parallel. + zero1_parallel_size (int): Size of zero-1 parallel. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rank_num_per_dp_group = self.world_size // self.data_parallel_size + self.num_zero1_parallel_group = self.data_parallel_size // self.zero1_parallel_size + + assert self.world_size % self.data_parallel_size == 0 + assert self.world_size % self.zero1_parallel_size == 0 + + def init_dist_group(self, use_cpu: bool = False): + """Initialize zero1 parallel groups, and assign local_ranks and groups to each gpu. + + Returns: + Tuple (local_rank, group_world_size, process_group, ranks_in_group, mode): + A zero1 parallelism's information tuple. + """ + local_rank = None + ranks_in_group = None + process_group = None + cpu_group = None + group_world_size = None + mode = ParallelMode.ZERO1 + + for i in range(self.rank_num_per_dp_group): + for j in range(self.num_zero1_parallel_group): + ranks = [ + i + (j * self.zero1_parallel_size + k) * self.rank_num_per_dp_group + for k in range(self.zero1_parallel_size) + ] + group = dist.new_group(ranks, timeout=LLM_NCCL_TIMEOUT) + if use_cpu: + group_cpu = ( + dist.new_group(ranks, backend="gloo", timeout=LLM_NCCL_TIMEOUT) + if dist.get_backend() != "gloo" + else group + ) + else: + group_cpu = None + + if self.rank in ranks: + local_rank = ranks.index(self.rank) + group_world_size = len(ranks) + process_group = group + cpu_group = group_cpu + ranks_in_group = ranks + + return local_rank, group_world_size, process_group, cpu_group, ranks_in_group, mode + + +class Initializer_Nettest(ProcessGroupInitializer): + """A ProcessGroupInitializer for network test, especailly for NCCL. + + Args: + rank (int): The rank of current process. + world_size (int): Size of whole communication world. + nettest_parallel_size (int): Size of a network test group. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.num_nettest_group = math.ceil(self.world_size / self.nettest_parallel_size) + + def init_dist_group(self, use_cpu: bool = False): + """Initialize tensor parallel groups, and assign local_ranks and groups to each gpu. + + Returns: + Tuple (local_rank, group_world_size, process_group, ranks_in_group, mode): + A Tensor parallelism's information tuple. + """ + local_rank = None + ranks_in_group = None + process_group = None + cpu_group = None + group_world_size = None + mode = ParallelMode.NETTEST + + for i in range(self.num_nettest_group): + ranks = [] + for j in range(self.nettest_parallel_size): + rank = i * self.nettest_parallel_size + j + if rank < self.world_size: + ranks.append(rank) + group = dist.new_group(ranks, timeout=LLM_NCCL_TIMEOUT) + if use_cpu: + group_cpu = ( + dist.new_group(ranks, backend="gloo", timeout=LLM_NCCL_TIMEOUT) + if dist.get_backend() != "gloo" + else group + ) + else: + group_cpu = None + + if self.rank in ranks: + local_rank = ranks.index(self.rank) + group_world_size = len(ranks) + process_group = group + cpu_group = group_cpu + ranks_in_group = ranks + + return local_rank, group_world_size, process_group, cpu_group, ranks_in_group, mode diff --git a/InternLM/internlm/core/context/random.py b/InternLM/internlm/core/context/random.py new file mode 100644 index 0000000000000000000000000000000000000000..b2c0a1d29fca30aafab989da7e84c525012dce21 --- /dev/null +++ b/InternLM/internlm/core/context/random.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/context + +from contextlib import contextmanager + +import torch +import torch.cuda +from torch import Tensor + +from .process_group_initializer import ParallelMode + + +class SeedManager: + """This class is a manager of all random seeds involved in the system.""" + + def __init__(self): + self._current_mode = None + self._seeds = {} + self._seed_states = {} + + @property + def current_mode(self): + return self._current_mode + + @property + def seeds(self): + return self._seeds + + @property + def seed_states(self): + return self._seed_states + + def set_state(self, parallel_mode: ParallelMode, state: Tensor): + """Sets the state of the seed manager for `parallel_mode`.""" + assert parallel_mode in self._seed_states, f"{parallel_mode} not found in seed manager" + self._seed_states[parallel_mode] = state + + def set_mode(self, parallel_mode: ParallelMode): + """Sets the current mode of the seed manager.""" + if self.current_mode: + # save state for current mode + self._seed_states[self._current_mode] = torch.cuda.get_rng_state() + + # set new state for new mode + self._current_mode = parallel_mode + torch.cuda.set_rng_state(self._seed_states[parallel_mode]) + + def add_seed(self, parallel_mode: ParallelMode, seed: int, overwrite: bool = False): + """Adds a seed to the seed manager for `parallel_mode`.""" + assert isinstance(parallel_mode, ParallelMode), "Invalid ParallelMode" + if not overwrite: + assert parallel_mode not in self._seed_states, f"Seed for {parallel_mode} exists" + elif parallel_mode in self._seed_states: + print(f"Warning: {parallel_mode} seed overwritten.", flush=True) + + current_state = torch.cuda.get_rng_state() + torch.cuda.manual_seed(seed) + self._seed_states[parallel_mode] = torch.cuda.get_rng_state() + self._seeds[parallel_mode] = seed + torch.cuda.set_rng_state(current_state) + + def reset(self): + self._current_mode = None + self._seeds = {} + self._seed_states = {} + + +_SEED_MANAGER = SeedManager() + + +def get_seeds(): + """Returns the seeds of the seed manager. + Returns: + dict: The seeds of the seed manager. + """ + return _SEED_MANAGER.seeds + + +def get_states(copy=False): + """Returns the seed states of the seed manager. + Returns: + dict: The seed states of the seed manager. + """ + states = _SEED_MANAGER.seed_states + if copy: + new_states = dict() + for parallel_mode, state in states.items(): + new_states[parallel_mode] = state.clone() + return new_states + else: + return _SEED_MANAGER.seed_states + + +def get_current_mode(): + """Returns the current mode of the seed manager. + Returns: + :class:`torch.ByteTensor`: The current mode of the seed manager. + """ + return _SEED_MANAGER.current_mode + + +def add_seed(parallel_mode: ParallelMode, seed: int, overwrite: bool = False): + """Adds a seed to the seed manager for `parallel_mode`.""" + _SEED_MANAGER.add_seed(parallel_mode, seed, overwrite) + + +def set_mode(parallel_mode: ParallelMode): + """Sets the current mode of the seed manager.""" + _SEED_MANAGER.set_mode(parallel_mode) + + +def set_seed_states(parallel_mode: ParallelMode, state: Tensor): + """Sets the state of the seed manager for `parallel_mode`.""" + _SEED_MANAGER.set_state(parallel_mode, state) + + +def sync_states(): + current_mode = get_current_mode() + current_states = torch.cuda.get_rng_state() + set_seed_states(current_mode, current_states) + + +@contextmanager +def seed(parallel_mode: ParallelMode): + """A context for seed switch""" + current_mode = _SEED_MANAGER.current_mode + try: + yield _SEED_MANAGER.set_mode(parallel_mode) + finally: + _SEED_MANAGER.set_mode(current_mode) diff --git a/InternLM/internlm/core/engine.py b/InternLM/internlm/core/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..3af5f26e41ff6733a9762979d8f9fac8ff8b3014 --- /dev/null +++ b/InternLM/internlm/core/engine.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/engine + +from typing import List, Optional + +import torch +from torch.nn import Module +from torch.nn.modules.loss import _Loss +from torch.optim.lr_scheduler import _LRScheduler + +from internlm.core.gradient_handler import BaseGradientHandler +from internlm.solver.beta2_scheduler import Beta2Scheduler +from internlm.solver.optimizer.hybrid_zero_optim import BaseOptimizer +from internlm.utils.common import get_batch_size, move_to_device + + +class Engine: + """ + The Engine class is responsible for managing the training and evaluation process of a neural network model. + It handles the forward and backward passes, parameter updates, gradient handling, and mode switching between + training and evaluation. + + Args: + model (torch.nn.Module): The neural network model to be trained or evaluated. + optimizer (BaseOptimizer): The optimizer used for updating the parameters of the model. + lr_scheduler (torch.optim.lr_scheduler._LRScheduler, optional): The learning rate scheduler for the optimizer. + Default is None. + beta2_scheduler (internlm.solver.beta2_scheduler.Beta2Scheduler, optional): The beta2 scheduler for the + optimizer. Default is None. + criterion (torch.nn.modules.loss._Loss, optional): The loss function used for calculating the loss during + training. Default is None. + gradient_handlers (List[BaseGradientHandler], optional): A list of gradient handlers used in the backward pass. + Default is None. + clip_grad_norm (float, optional): The norm value for gradient clipping. Default is 0.0. + + Examples: + >>> # define model, criterion, optimizer, lr_scheduler, train_dataloader for your training + >>> model = ... + >>> criterion = ... + >>> optimizer = ... + >>> train_dataloader = ... + >>> engine, _, _, _ = internlm.initialize_engine(model, optimizer, criterion) + >>> engine.train() + >>> for inputs, labels in train_dataloader + >>> # set gradients to zero + >>> engine.zero_grad() + >>> # run forward pass + >>> outputs = engine(inputs) + >>> # compute loss value and run backward pass + >>> loss = engine.criterion(outputs, labels) + >>> engine.backward(loss) + >>> # update parameters + >>> engine.step() + """ + + def __init__( + self, + model: Module, + optimizer: BaseOptimizer, + lr_scheduler: Optional[_LRScheduler] = None, + beta2_scheduler: Optional[Beta2Scheduler] = None, + criterion: Optional[_Loss] = None, + gradient_handlers: Optional[List[BaseGradientHandler]] = None, + clip_grad_norm: float = 0.0, + ): + self._model = model + self._optimizer = optimizer + self._lr_scheduler = lr_scheduler + self._beta2_scheduler = beta2_scheduler + self._criterion = criterion + self._clip_grad_norm = clip_grad_norm + + # state + self.training = True # default + + # build gradient handler + self._gradient_handlers = gradient_handlers if gradient_handlers else [] + + @property + def model(self): + """Returns the model attached to the engine.""" + return self._model + + @property + def optimizer(self): + """Returns the optimizer attached to the engine.""" + return self._optimizer + + @property + def criterion(self): + """Returns the criterion (loss function) attached to the engine.""" + return self._criterion + + def _all_reduce_gradients(self): + """Handles all-reduce operations of gradients across different parallel groups.""" + for handler in self._gradient_handlers: + handler.handle_gradient() + + def zero_grad(self): + """Sets the gradient of all parameters in the model to zero.""" + self.optimizer.zero_grad() + + def step(self): + """ + Executes the parameter update step. This includes all-reduce operations of gradients, gradient clipping, + and parameter update. If successful, it also steps the learning rate scheduler and beta2 scheduler + if they exist. + + Returns: + success (bool): Whether the parameter update was successful. + grad_norm (float): The norm of the gradient after clipping. + """ + self._all_reduce_gradients() + self.optimizer.clip_grad_norm(self.model, self._clip_grad_norm) + + success, grad_norm = self.optimizer.step() + + if success and self._lr_scheduler is not None: + self._lr_scheduler.step() + + if success and self._beta2_scheduler is not None: + self._beta2_scheduler.step() + + return success, grad_norm + + def train(self): + """Sets the model to training mode.""" + self.training = True + self._model.train() + + def eval(self): + """Sets the model to evaluation mode.""" + self.training = False + self._model.eval() + + def backward(self, loss: torch.Tensor): + """ + Starts the backward propagation given the loss value computed by a loss function. + + Args: + loss (torch.Tensor): The loss value computed by a loss function. + """ + return self.optimizer.backward(loss) + + def backward_by_grad(self, tensor, grad): + """ + Starts the backward propagation given the gradient of the output tensor. + + Args: + tensor (torch.Tensor): The output tensor. + grad (torch.Tensor): The gradient passed back to the output tensor. + """ + return self.optimizer.backward_by_grad(tensor, grad) + + def __call__(self, *args, **kwargs): + """ + Runs the forward step for the model. + + Returns: + torch.Tensor: The output of the model. + """ + return self.model(*args, **kwargs) + + def load_batch(self, data_iter, to_gpu=True): + """ + Loads a batch from the data iterator. It returns the data and labels which are + already in the same GPU as where the model is. + + Args: + data_iter (Iterable): The data iterator from which to get a batch of data, obtained by calling + iter(dataloader). + to_gpu (bool, optional): Whether the data should be moved to the GPU. Default is True. + + Returns: + Tuple (torch.Tensor, torch.Tensor): A tuple of (data, label). + """ + if data_iter is None: + raise RuntimeError("Dataloader is not defined.") + try: + batch_data = next(data_iter) + except TypeError: + batch_data = data_iter + + if to_gpu: + batch_data = move_to_device(batch_data) + batch_size = get_batch_size(batch_data) + + return batch_data, batch_size + + +class KDEngine(Engine): + def __init__( + self, + model: Module, + teacher: Module, + optimizer: BaseOptimizer, + lr_scheduler: Optional[_LRScheduler] = None, + beta2_scheduler: Optional[Beta2Scheduler] = None, + criterion: Optional[_Loss] = None, + kd_criterion: Optional[_Loss] = None, + gradient_handlers: Optional[List[BaseGradientHandler]] = None, + clip_grad_norm: float = 0.0, + ): + self._teacher = teacher + self._kd_criterion = kd_criterion + + super().__init__( + model=model, + optimizer=optimizer, + lr_scheduler=lr_scheduler, + beta2_scheduler=beta2_scheduler, + criterion=criterion, + gradient_handlers=gradient_handlers, + clip_grad_norm=clip_grad_norm, + ) + + @property + def teacher(self): + """Returns the model attached to the engine.""" + return self._teacher + + @property + def kd_criterion(self): + """Returns the model attached to the engine.""" + return self._kd_criterion diff --git a/InternLM/internlm/core/gradient_handler.py b/InternLM/internlm/core/gradient_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..f2aaa1d50b26ab1cb551157a138883fe0af41207 --- /dev/null +++ b/InternLM/internlm/core/gradient_handler.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from abc import ABC, abstractmethod +from collections import defaultdict + +import torch +import torch.distributed as dist +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +from internlm.core.context import global_context as gpc + + +class BaseGradientHandler(ABC): + """A basic helper class to handle all-reduce operations of gradients across different parallel groups + before optimization. + + Args: + model (Module): Model where the gradients accumulate. + optimizer (Optimizer): Optimizer for updating the parameters. + """ + + def __init__(self, model, optimizer): + self._model = model + self._optimizer = optimizer + + @abstractmethod + def handle_gradient(self): + """A method to accumulate gradients across different parallel groups. Users should + write their own functions or just use the functions in pre-defined subclasses. + """ + pass + + +class PipelineSharedModuleGradientHandler(BaseGradientHandler): + """A helper class to handle all-reduce operations in sub parallel groups. + A all-reduce collective communication will be operated in + :func:`handle_gradient` among all sub pipeline parallel groups. + For better performance, it bucketizes the gradients of all parameters that are + the same type to improve the efficiency of communication. + + Args: + model (Module): Model where the gradients accumulate. + optimizer (Optimizer): Optimizer for updating the parameters. + """ + + def handle_gradient(self): + """A method running a all-reduce operation in sub pipeline parallel groups.""" + if gpc.pipeline_parallel_size > 1: + # bucketize and all-reduce + buckets = defaultdict(lambda: defaultdict(list)) + # Pack the buckets. + for param in self._model.parameters(): + group = getattr(param, "pipeline_shared_module_pg", None) + if ( + param.requires_grad + and group is not None + and ( + (hasattr(param, "colo_attr") and not param.colo_attr.saved_grad.is_null()) + or param.grad is not None + ) + ): + tp = param.data.type() + buckets[group][tp].append(param) + + # For each bucket, all-reduce and copy all-reduced grads. + for group, group_buckets in buckets.items(): + for tp, bucket in group_buckets.items(): + grads = [ + param.colo_attr.grad_payload if hasattr(param, "colo_attr") else param.grad.data + for param in bucket + ] + coalesced = _flatten_dense_tensors(grads).to(torch.cuda.current_device()) + dist.all_reduce(coalesced, op=dist.ReduceOp.SUM, group=group) + for buf, synced in zip(grads, _unflatten_dense_tensors(coalesced, grads)): + buf.copy_(synced) diff --git a/InternLM/internlm/core/naive_amp.py b/InternLM/internlm/core/naive_amp.py new file mode 100644 index 0000000000000000000000000000000000000000..86e4b57a2e00a3c43b0290330f2e7b0177bf9e9a --- /dev/null +++ b/InternLM/internlm/core/naive_amp.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/tree/main/colossalai/amp + +from typing import Any + +import torch +import torch.distributed as dist +from torch import Tensor, nn +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors +from torch.distributed import ReduceOp + +from internlm.core.context import ParallelMode +from internlm.core.context.parallel_context import global_context as gpc + + +class NaiveAMPModel(nn.Module): + """ + This is a wrapper class for a model that automatically casts the model, its inputs, and outputs into fp16. + It also provides options to cast the output back to fp32 and to synchronize buffers. + + Args: + model (torch.nn.Module): The model to be wrapped and cast into fp16. + output_to_fp32 (bool, optional): If True, the output of this module is cast into fp32. Defaults to True. + parallel_mode (:class:`internlm.core.context.ParallelMode`): The parallel group mode used in this module. + Defaults to ``ParallelMode.DATA``. + sync_buffer (bool, optional): If True, the buffers are synchronized. Defaults to True. + """ + + def __init__( + self, + model: nn.Module, + output_to_fp32: bool = True, + parallel_mode: ParallelMode = ParallelMode.DATA, + sync_buffer: bool = True, + dtype=torch.float16, + ): + super().__init__() + self.model = model.to(dtype) + self._output_to_fp32 = output_to_fp32 + self._sync_buf = sync_buffer + self.dtype = dtype + + if gpc.is_initialized(parallel_mode) and gpc.get_world_size(parallel_mode) > 1: + self._process_group = gpc.get_group(parallel_mode) + self._world_size = gpc.get_world_size(parallel_mode) + else: + self._process_group = None + self._world_size = 1 + self._sync_buf = False + self._first_eval_run = False + + @property + def sync_buffer(self): + """Returns the current state of the buffer synchronization.""" + return self._sync_buf + + @sync_buffer.setter + def sync_buffer(self, state: bool): + """Sets the state of the buffer synchronization.""" + self._sync_buf = state + + def _convert_to_fp16(self, input_: Any): + """Converts the input to fp16 if it is a Tensor of dtype float32.""" + if isinstance(input_, Tensor) and input_.dtype == torch.float32: + input_ = input_.to(self.dtype) + return input_ + + def _convert_to_fp32(self, input_: Any): + """Converts the input to fp32 if it is a Tensor of dtype float16.""" + if isinstance(input_, Tensor) and input_.dtype == torch.float16: + input_ = input_.float() + return input_ + + def convert_to_fp32(self, out): + """Converts the output to fp32""" + if isinstance(out, Tensor): + out = self._convert_to_fp32(out) + elif isinstance(out, (tuple, list)): + out = [self._convert_to_fp32(val) for val in out] + elif isinstance(out, dict): + out = {key: self._convert_to_fp32(val) for key, val in out.items()} + + return out + + def _reduce_module_buffer(self): + """ + All-reduces the buffers (e.g., running stats of batch normalization) across + data parallel ranks so that all the ranks will produce consistent results + when given the same input. + """ + buf_list = [] + + # find valid buffers + for buf in self.model.buffers(): + if buf is not None: + buf_list.append(buf) + + # reduce buffers across data parallel ranks + if buf_list: + coalesced_buf = _flatten_dense_tensors(buf_list) + coalesced_buf.div_(self._world_size) + dist.all_reduce(coalesced_buf, op=ReduceOp.SUM, group=self._process_group) + unflattened_buf_list = _unflatten_dense_tensors(coalesced_buf, buf_list) + for old, new in zip(buf_list, unflattened_buf_list): + old.copy_(new) + + def eval(self): + """Sets the model to evaluation mode. Buffers are only synchronized in the first eval iteration.""" + self.model.eval() + self._first_eval_run = True + + def forward(self, *args, **kwargs): + """ + Performs a forward pass on the model. Buffers are synchronized before the forward pass. + The inputs are converted to fp16 and the outputs are optionally converted back to fp32. + """ + if (self.training or self._first_eval_run) and self._sync_buf: + with torch.no_grad(): + self._reduce_module_buffer() + + if self._first_eval_run: + self._first_eval_run = False + + if args: + args = [self._convert_to_fp16(arg) for arg in args] + if kwargs: + for k, v in kwargs.items(): + kwargs[k] = self._convert_to_fp16(v) + + out = self.model(*args, **kwargs) + + if self._output_to_fp32: + out = self.convert_to_fp32(out) + return out diff --git a/InternLM/internlm/core/scheduler/__init__.py b/InternLM/internlm/core/scheduler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..369f3127943ff22ecfc7c1ebd5a6b20576df883d --- /dev/null +++ b/InternLM/internlm/core/scheduler/__init__.py @@ -0,0 +1,14 @@ +from .base_scheduler import BaseScheduler, SchedulerHook, SchedulerMetricHook +from .no_pipeline_scheduler import NonPipelineScheduler, KDNonPipelineScheduler +from .pipeline_scheduler import InterleavedPipelineScheduler, PipelineScheduler, KDPipelineScheduler + +__all__ = [ + "BaseScheduler", + "NonPipelineScheduler", + "KDNonPipelineScheduler", + "InterleavedPipelineScheduler", + "PipelineScheduler", + "KDPipelineScheduler", + "SchedulerHook", + "SchedulerMetricHook", +] diff --git a/InternLM/internlm/core/scheduler/base_scheduler.py b/InternLM/internlm/core/scheduler/base_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..2c436d04b2bbc3f66e7206c1589abc6773237933 --- /dev/null +++ b/InternLM/internlm/core/scheduler/base_scheduler.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/engine + +from abc import ABC, abstractmethod +from typing import Any, Callable, Iterable, Optional + +import torch + +from internlm.core.engine import Engine +from internlm.utils.megatron_timers import megatron_timer as timer + + +class BaseScheduler(ABC): + """A basic helper class to control the process of training or evaluation. + It mainly composes of forward_backward_step for gradient backward and + optimizer_step for parameters update. + For the convenience to enable FP16, we aggregate all codes that contain the + control of FP16 in class schedule. + + Args: + data_process_func (Callable, optional): The preprocessing function which receives a batch of data and arranges + them into data and label. + """ + + def __init__(self, data_process_func: Callable = None): + self.data_process_func = data_process_func + + @abstractmethod + def pre_processing(self, engine: Engine): + """To perform actions before running the schedule. + + Args: + engine (internlm.core.Engine): InternLM engine for training and inference. + """ + pass + + def _load_micro_batch(self, data, label, offset, micro_bsz): + assert isinstance(data, dict) and isinstance(label, torch.Tensor) + micro_batch_data = {k: v[offset : offset + micro_bsz] for k, v in data.items()} + micro_batch_label = label[offset : offset + micro_bsz] + + return micro_batch_data, micro_batch_label + + @abstractmethod + def forward_backward_step( + self, + engine: Engine, + data_iter: Iterable, + forward_only: bool, + return_loss: bool = True, + return_output_label: bool = True, + ): + """The process function over a batch of dataset for training or evaluation. + + Args: + engine (internlm.core.Engine): InternLM engine for training and inference. + data_iter (Iterable): Data iterator from which get a batch of data, obtained by calling iter(dataloader). + forward_only (bool): If True, the process won't include backward. + return_loss (bool, optional): If False, the loss won't be returned. + return_output_label (bool, optional): If False, the output and label won't be returned. + """ + pass + + @staticmethod + def _call_engine(engine: Engine, inputs: Any): + """Calls the engine with the given inputs. + + Args: + engine (internlm.core.Engine): InternLM engine for training and inference. + inputs (Any): The inputs to the engine, can be of type torch.Tensor, list, tuple, or dict. + """ + if isinstance(inputs, torch.Tensor): + return engine(inputs) + elif isinstance(inputs, (list, tuple)): + return engine(*inputs) + elif isinstance(inputs, dict): + return engine(**inputs) + else: + raise TypeError( + f"Expected engine inputs to be of type torch.Tensor, list, tuple, or dict, but got {type(inputs)}" + ) + + @staticmethod + def _call_engine_criterion(criterion, outputs: Any, labels: Any): + """Calls the engine's criterion with the given outputs and labels. + + Args: + engine (internlm.core.Engine): InternLM engine for training and inference. + outputs (Any): The outputs from the model, can be of type torch.Tensor, list, tuple, or dict. + labels (Any): The labels for the outputs, can be of type torch.Tensor, list, tuple, or dict. + """ + assert isinstance( + outputs, (torch.Tensor, list, tuple, dict) + ), f"Expect output of model is (torch.Tensor, list, tuple), got {type(outputs)}" + if isinstance(outputs, torch.Tensor): + outputs = (outputs,) + if isinstance(labels, torch.Tensor): + labels = (labels,) + + if isinstance(outputs, (tuple, list)) and isinstance(labels, (tuple, list)): + return criterion(*outputs, *labels) + elif isinstance(outputs, (tuple, list)) and isinstance(labels, dict): + return criterion(*outputs, **labels) + elif isinstance(outputs, dict) and isinstance(labels, dict): + return criterion(**outputs, **labels) + elif isinstance(outputs, dict) and isinstance(labels, (list, tuple)): + raise ValueError(f"Expected labels to be a dict when the model outputs are dict, but got {type(labels)}") + else: + raise TypeError( + f"Expected model outputs and labels to be of type torch.Tensor ' \ + '(which is auto-converted to tuple), list, tuple, or dict, ' \ + 'but got {type(outputs)} (model outputs) and {type(labels)} (labels)" + ) + + +class SchedulerHook(ABC): + """ + Scheduler Hook. + """ + + @abstractmethod + def before_forward(self, scheduler, inputs) -> None: + """Actions before forward""" + + @abstractmethod + def after_forward(self, scheduler, outputs) -> None: + """Actions after forward""" + + @abstractmethod + def before_criterion(self, scheduler, outputs, label) -> None: + """Actions before criterion""" + + @abstractmethod + def after_criterion(self, scheduler, loss) -> None: + """Actions after criterion""" + + @abstractmethod + def before_backward(self, scheduler, outputs, outputs_grad) -> None: + """Actions before backward""" + + @abstractmethod + def after_backward(self, scheduler, inputs_grad) -> None: + """Actions after backward""" + + @abstractmethod + def post_helper_func(self, scheduler, outputs, label) -> None: + """A post helper function""" + + +class SchedulerMetricHook(SchedulerHook): + """ + Scheduler Metric Hook. + """ + + def __init__(self, metric: Optional[Callable] = None, skip: bool = False) -> None: + self._post_func = metric + self._skip = skip + + def before_forward(self, scheduler, inputs) -> None: + if not self._skip: + timer("fwd").start() + + def after_forward(self, scheduler, outputs) -> None: + if not self._skip: + timer("fwd").stop() + + def before_criterion(self, scheduler, outputs, label) -> None: + if not self._skip: + timer("cal_loss").start() + + def after_criterion(self, scheduler, loss) -> None: + if not self._skip: + timer("cal_loss").stop() + + def before_backward(self, scheduler, outputs, outputs_grad) -> None: + if not self._skip: + timer("bwd").start() + + def after_backward(self, scheduler, inputs_grad) -> None: + if not self._skip: + timer("bwd").stop() + + def post_helper_func(self, scheduler, outputs, label) -> None: + if self._post_func is not None: + self._post_func(outputs, label) diff --git a/InternLM/internlm/core/scheduler/no_pipeline_scheduler.py b/InternLM/internlm/core/scheduler/no_pipeline_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..afbcd0134528c3a5e597ee3d46a9013a5780ecc6 --- /dev/null +++ b/InternLM/internlm/core/scheduler/no_pipeline_scheduler.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/engine + +from typing import Any, Callable, Iterable, List, Optional + +import torch + +from internlm.core.context import global_context as gpc +from internlm.core.engine import Engine, KDEngine +from internlm.utils.common import conditional_context +from internlm.utils.timeout import llm_timeout +from collections import defaultdict +from .base_scheduler import BaseScheduler, SchedulerHook + + +class NonPipelineScheduler(BaseScheduler): + """A helper schedule class for no pipeline parallelism running environment. + During one process, it loads a batch of dataset and feeds it to the model. + After getting the output and calculating the loss, it will use :meth:`step` + to update the parameters if it is in training mode. + + Args: + data_process_func (Callable, optional): The preprocessing function which receives a batch of data + and returns a tuple in the form of (data, label), and it will be executed in load_batch. + gradient_accumulation_steps(int, optional): the steps of gradient accumulation, 1 for disable + gradient accumulation. + + Examples: + >>> # this shows an tools of customized data_process_func + >>> def data_process_func(dataloader_output): + >>> item1, item2, item3 = dataloader_output + >>> data = (item1, item2) + >>> label = item3 + >>> return data, label + """ + + def __init__( + self, + data_process_func: Callable = None, + gradient_accumulation_size: int = 1, + scheduler_hooks: Optional[List[SchedulerHook]] = None, + ): + self._grad_accum_size = gradient_accumulation_size + self._grad_accum_offset = 0 + + self._hooks = scheduler_hooks + + super().__init__(data_process_func) + + def pre_processing(self, engine: Engine): + """Performs actions before running the schedule. + + Args: + engine (internlm.core.Engine): InternLM engine for training and inference. + """ + pass + + def _call_hooks(self, func_name: str, *args, **kwargs) -> None: + for hook in self._hooks: + getattr(hook, func_name)(self, *args, **kwargs) + + def _load_accum_batch(self, data: Any, label: Any): + """Loads a batch of data and label for gradient accumulation. + + Args: + data (Any): The data to be loaded. + label (Any): The label to be loaded. + """ + + _data, _label = self._load_micro_batch( + data=data, label=label, offset=self._grad_accum_offset, micro_bsz=self._grad_accum_batch_size + ) + self._grad_accum_offset += self._grad_accum_batch_size + + if self.data_process_func: + _data["input_ids"] = self.data_process_func(_data["input_ids"], _data["cu_seqlens"]) + _label = self.data_process_func(_label, _data["cu_seqlens"]) + _data.pop("cu_seqlens") + _data.pop("indexes") + + return _data, _label + + def _train_one_batch( + self, + data: Any, + label: Any, + engine: Engine, + forward_only: bool = False, + return_loss: bool = True, + scale_loss: int = 1, + ): + """Trains one batch of data. + + Args: + data (Any): The data to be trained. + label (Any): The label for the data. + engine (internlm.core.Engine): InternLM engine for training and inference. + forward_only (bool, optional): If True, the model is run for the forward pass, else back propagation will + be executed. + return_loss (bool, optional): Loss will be returned if True. + scale_loss (int, optional): The scale factor for the loss. + """ + + # forward + with conditional_context(torch.no_grad(), enable=forward_only): + self._call_hooks("before_forward", data) + output = self._call_engine(engine, data) + self._call_hooks("after_forward", output) + + self._call_hooks("post_helper_func", output, label) + + if return_loss: + self._call_hooks("before_criterion", output, label) + loss = self._call_engine_criterion(engine.criterion, output, label) + self._call_hooks("after_criterion", loss) + loss /= scale_loss + + # backward + if not forward_only: + self._call_hooks("before_backward", None, None) + engine.backward(loss) + self._call_hooks("after_backward", None) + + if not return_loss: + loss = None + + return output, dict(loss=loss) + + @llm_timeout(func_name="nopp_forward_backward_step") + def forward_backward_step( + self, + engine: Engine, + data_iter: Iterable, + forward_only: bool = False, + return_loss: bool = True, + return_output_label: bool = True, + ): + """The process function that loads a batch of dataset and feeds it to the model. + The returned labels and loss will None if :attr:`return_loss` is False. + + Args: + engine (internlm.core.Engine): InternLM engine for training and inference. + data_iter (Iterable): Dataloader as the form of an iterator, obtained by calling iter(dataloader). + forward_only (bool, optional): + If True, the model is run for the forward pass, else back propagation will be executed. + return_loss (bool, optional): Loss will be returned if True. + return_output_label (bool, optional): Output and label will be returned if True. + + Returns: + Tuple[:class:`torch.Tensor`]: A tuple of (output, label, loss), loss and label could be None. + """ + assert ( + forward_only or return_loss + ), "The argument 'return_loss' has to be True when 'forward_only' is False, but got False." + + batch_data, batch_size = engine.load_batch(data_iter) + + assert ( + batch_size % self._grad_accum_size == 0 + ), f"batch_size:{batch_size} must be an integer multiple of gradient accumulation steps:{self._grad_accum_size}" + self._grad_accum_batch_size = batch_size // self._grad_accum_size + + data, label = batch_data + + loss = defaultdict(int) if return_loss else None + outputs = [] + labels = [] + + # reset accumulation microbatch offset + self._grad_accum_offset = 0 + + for _current_accum_step in range(self._grad_accum_size): + if _current_accum_step == self._grad_accum_size - 1: + engine.optimizer.skip_grad_reduce = False + else: + engine.optimizer.skip_grad_reduce = True + + _data, _label = self._load_accum_batch(data, label) + + _output, _loss = self._train_one_batch( + _data, _label, engine, forward_only, return_loss, self._grad_accum_size + ) + + if return_loss: + for k in _loss: + loss[k] += _loss[k] + if return_output_label: + outputs.append(_output) + labels.append(_label) + + if not return_output_label: + outputs, labels = None, None + + return outputs, labels, loss + + +class KDNonPipelineScheduler(NonPipelineScheduler): + + def __init__( + self, + data_process_func: Callable = None, + gradient_accumulation_size: int = 1, + scheduler_hooks: Optional[List[SchedulerHook]] = None, + ): + super().__init__( + data_process_func=data_process_func, + gradient_accumulation_size=gradient_accumulation_size, + scheduler_hooks=scheduler_hooks, + ) + + def _train_one_batch( + self, + data: Any, + label: Any, + engine: KDEngine, + forward_only: bool = False, + return_loss: bool = True, + scale_loss: int = 1, + ): + """Trains one batch of data. + + Args: + data (Any): The data to be trained. + label (Any): The label for the data. + engine (internlm.core.Engine): InternLM engine for training and inference. + forward_only (bool, optional): If True, the model is run for the forward pass, else back propagation will + be executed. + return_loss (bool, optional): Loss will be returned if True. + scale_loss (int, optional): The scale factor for the loss. + """ + + # forward + with conditional_context(torch.no_grad(), enable=forward_only): + self._call_hooks("before_forward", data) + output = self._call_engine(engine, data) + self._call_hooks("after_forward", output) + + self._call_hooks("post_helper_func", output, label) + + if return_loss: + self._call_hooks("before_criterion", output, label) + loss_gt = gpc.config.kd_config['gt_weight'] * self._call_engine_criterion(engine.criterion, output, label) + + with torch.no_grad(): + engine.teacher.eval() + output_t = self._call_engine(engine.teacher, data) + + loss_kd = gpc.config.kd_config['kd_weight'] * self._call_engine_criterion(engine.kd_criterion, output, (output_t, label)) + + self._call_hooks("after_criterion", loss_gt + loss_kd) + loss_gt /= scale_loss + loss_kd /= scale_loss + + # backward + if not forward_only: + self._call_hooks("before_backward", None, None) + engine.backward(loss_gt+loss_kd) + self._call_hooks("after_backward", None) + + if not return_loss: + loss_gt = None + loss_kd = None + + return output, dict(loss_gt=loss_gt, loss_kd=loss_kd) diff --git a/InternLM/internlm/core/scheduler/pipeline_scheduler.py b/InternLM/internlm/core/scheduler/pipeline_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..8c37d48adddec7193d2b135db400c8607e8aab0c --- /dev/null +++ b/InternLM/internlm/core/scheduler/pipeline_scheduler.py @@ -0,0 +1,1363 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/engine + +from contextlib import contextmanager +from typing import Callable, List, Optional, Tuple, Union + +import torch.cuda + +import internlm.core.communication as comm +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.core.engine import Engine +from internlm.core.naive_amp import NaiveAMPModel +from internlm.utils.common import get_current_device, move_to_device +from internlm.utils.logger import get_logger +from internlm.utils.timeout import llm_timeout +from collections import defaultdict +from .base_scheduler import BaseScheduler, SchedulerHook + +logger = get_logger(__file__) + + +def get_tensor_shape(): + if hasattr(gpc.config, "TENSOR_SHAPE"): + return gpc.config.TENSOR_SHAPE + + if not gpc.is_initialized(ParallelMode.PIPELINE): + return None + + if hasattr(gpc.config, "SEQ_LEN") and hasattr(gpc.config.data, "micro_bsz") and hasattr(gpc.config, "HIDDEN_SIZE"): + if gpc.config.model.use_flash_attn: + if gpc.config.parallel.sequence_parallel: + sequence_world_size = gpc.get_world_size(ParallelMode.TENSOR) + tensor_shape = ( + gpc.config.SEQ_LEN * gpc.config.data["micro_bsz"] // sequence_world_size, + gpc.config.HIDDEN_SIZE, + ) + else: + tensor_shape = ( + gpc.config.SEQ_LEN * gpc.config.data["micro_bsz"], + gpc.config.HIDDEN_SIZE, + ) + else: + tensor_shape = ( + gpc.config.data["micro_bsz"], + gpc.config.SEQ_LEN, + gpc.config.HIDDEN_SIZE, + ) + return tensor_shape + else: + return None + + +def pack_return_tensors(return_tensors): + output, label = tuple(zip(*return_tensors)) + if isinstance(output[0], torch.Tensor): + output = torch.cat(output, dim=0) + elif isinstance(output[0], (list, tuple)): + output = tuple(torch.cat(tensors, dim=0) for tensors in zip(*output)) + else: + raise TypeError("Output of model must be tensor or list/tuple of tensors") + if isinstance(label[0], torch.Tensor): + label = torch.cat(label, dim=0) + else: + merged_label = {k: [] for k in label[0].keys()} + for d in label: + for k, v in d.items(): + merged_label[k].append(v) + label = {k: torch.cat(v, dim=0) for k, v in merged_label.items()} + return output, label + + +@contextmanager +def switch_virtual_pipeline_parallel_rank(rank): + prev_rank = gpc.virtual_pipeline_parallel_rank + try: + gpc.set_virtual_pipeline_parallel_rank(rank) + yield + finally: + gpc.set_virtual_pipeline_parallel_rank(prev_rank) + + +@contextmanager +def switch_optimizer_grad_sync_skip_mode(optimizer, skip: bool = True): + prev_mode = optimizer.skip_grad_reduce + try: + optimizer.skip_grad_reduce = skip + yield + finally: + optimizer.skip_grad_reduce = prev_mode + + +class PipelineScheduler(BaseScheduler): + """ + A helper schedule class for pipeline parallelism running environment. + It uses non-interleaved 1F1B strategy. Other properties are similar as + :class:`NonPipelineSchedule`. + + Args: + num_microbatches (int): The number of microbatches. + dtype (torch.dtype): Type of data. torch.float by default. + data_process_func (Callable, optional): + The post processing function which receives a micro batch of data, and it will be executed + in `load_micro_batch`. + tensor_shape (torch.Size, optional): Specified shape in pipeline communication. + scatter_gather_tensors (bool, optional): + If set to `True`, communication will be reduced over pipeline when using 1D tensor parallelization. + scheduler_hooks (Optional[List[SchedulerHook]], optional): List of scheduler hooks. + """ + + def __init__( + self, + num_microbatches: int, + dtype: torch.dtype = torch.float, + data_process_func: Callable = None, + tensor_shape: Union[torch.Size, List[int], Tuple[int]] = None, + scatter_gather_tensors: bool = False, + scheduler_hooks: Optional[List[SchedulerHook]] = None, + ): + assert num_microbatches > 0, f"expected num_microbatches to be larger then 1, but got {num_microbatches}" + + assert not isinstance( + tensor_shape, int + ), "tensor_shape type should be one of Union[torch.Size, List[int], Tuple[int]]." + + super().__init__(data_process_func=data_process_func) + + self.num_microbatches = num_microbatches + self.dtype = dtype + self._hooks = scheduler_hooks + + self._tensor_shape = ( + tensor_shape if tensor_shape is None or isinstance(tensor_shape, torch.Size) else torch.Size(tensor_shape) + ) + + self.scatter_gather_tensors = ( + scatter_gather_tensors + and gpc.is_initialized(ParallelMode.TENSOR) + and gpc.get_world_size(ParallelMode.TENSOR) > 1 + ) + + if gpc.config.parallel.sequence_parallel: + self.scatter_gather_tensors = False + + # cache for the batch data + self.batch_data = None + + @property + def tensor_shape(self) -> torch.Size: + return self._tensor_shape + + @tensor_shape.setter + def tensor_shape(self, tensor_shape: torch.Size): + self._tensor_shape = tensor_shape + + def pre_processing(self, engine): + types = set() + + for param in engine.model.parameters(): + types.add(param.dtype) + assert len(types) == 1, f"Mixed types of parameter detected, {types}" + + self.dtype = types.pop() + + @staticmethod + def _call_engine(engine, data): # pylint: disable=W0237 + if data is None: + return None + + if isinstance(data, torch.Tensor): + return engine(data) + elif isinstance(data, (list, tuple)): + return engine(*data) + elif isinstance(data, dict): + stage_output = data.pop("stage_output", None) + + if stage_output is None: + return engine(**data) + elif isinstance(stage_output, torch.Tensor): + return engine(stage_output, **data) + elif isinstance(stage_output, (tuple, list)): + return engine(*stage_output, **data) + else: + raise TypeError( + f"Expected stage_output to be of type torch.Tensor, list, or tuple, " + f"but got {type(stage_output)}" + ) + else: + raise TypeError(f"Expected data to be of type torch.Tensor, list, tuple, or dict, but got {type(data)}") + + def load_batch(self, engine, data_iter): + # Pipeline schedule just puts data in memory + batch_data, batch_size = engine.load_batch(data_iter, to_gpu=False) + assert batch_size % self.num_microbatches == 0, "Batch size should divided by the number of microbatches" + + self.microbatch_offset = 0 + self.batch_size = batch_size + self.batch_data, self.batch_label = batch_data + self.microbatch_size = self.batch_size // self.num_microbatches + + def load_micro_batch(self): + micro_batch_data, micro_batch_label = self._load_micro_batch( + data=self.batch_data, label=self.batch_label, offset=self.microbatch_offset, micro_bsz=self.microbatch_size + ) + if self.data_process_func: + micro_batch_data["input_ids"] = self.data_process_func( + micro_batch_data["input_ids"], micro_batch_data["cu_seqlens"] + ) + micro_batch_label = self.data_process_func(micro_batch_label, micro_batch_data["cu_seqlens"]) + + micro_batch_data.pop("cu_seqlens") + micro_batch_data.pop("indexes") + + micro_batch_data["label"] = micro_batch_label + self.microbatch_offset += self.microbatch_size + + return move_to_device(micro_batch_data) + + def _get_data_label_for_current_step(self, stage_output, micro_batch_data): + if isinstance(micro_batch_data, (tuple, list)): + if gpc.is_first_rank(ParallelMode.PIPELINE): + # for the first stage, we use the data from the + # dataloader output by default + data, label = micro_batch_data + else: + # for non-first stage, we use the output passed + # by the previous as the model input + data = stage_output + _, label = micro_batch_data + elif isinstance(micro_batch_data, dict): + label = micro_batch_data.pop("label", None) + data = {"stage_output": stage_output, **micro_batch_data} + + return data, label + + def _call_hooks(self, func_name: str, *args, **kwargs) -> None: + for hook in self._hooks: + getattr(hook, func_name)(self, *args, **kwargs) + + def _get_current_microbatch_id(self, step_id: int) -> int: + """ + Get the current microbatch ID based on the step ID. + In 1f1b scheduler, the microbatch ID is the same as the step ID, + but it is important to note that the step ID is calculated separately + for forward and backward passes. + """ + return step_id + + def _forward_step(self, engine, input_obj, return_tensors, return_output_label=True, accum_loss=None): + """ + Forward step for passed-in model. If it is the first stage, the input tensor + is obtained from data_iterator, otherwise the passed-in input_obj is used. + Returns output tensor. This is a helper function and can be ignored by users. + + Args: + engine (colossalai.engine.Engine): Colossalai engine for training and inference. + input_obj (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Input tensor for this pipeline stage. + return_tensors (List[:class:`torch.Tensor`]): A list of tensors to return. + return_output_label (bool, optional): Whether returns output labels. + accum_loss (optional): Where accumulated loss stores. + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: output or the loss value of the current + pipeline stage. + """ + micro_batch_data = self.load_micro_batch() + data, label = self._get_data_label_for_current_step(input_obj, micro_batch_data) + + self._call_hooks("before_forward", data) + output_obj = self._call_engine(engine.model, data) + self._call_hooks("after_forward", output_obj) + + if gpc.is_last_rank(ParallelMode.PIPELINE): + self._call_hooks("post_helper_func", output_obj, label) + if return_output_label: + return_tensors.append((output_obj, label)) + if accum_loss is not None: + self._call_hooks("before_criterion", output_obj, label) + loss = self._call_engine_criterion(engine.criterion, output_obj, label) + self._call_hooks("after_criterion", loss) + + loss_reduced = loss / self.num_microbatches + accum_loss['loss'].add_(loss_reduced.detach()) + output_obj = loss_reduced + + return output_obj + + def _backward_step(self, engine, step_id, input_obj, output_obj, output_obj_grad): + """ + Backward step through the passed-in output tensor. If it is the last stage, the + output_obj_grad is None, otherwise it is the gradients with respect to stage's output tensor. + Returns the gradients with respect to the input tensor (None if first stage). + This is a helper function and can be ignored by users. + + Args: + engine (colossalai.engine.Engine): Colossalai engine for training and inference. + step_id (int): The ID of the current step. + input_obj (Union[torch.Tensor, List[torch.Tensor]]): Input tensor for this stage. + output_obj (Union[torch.Tensor, List[torch.Tensor]]): Output tensor for this stage. + output_obj_grad (Union[torch.Tensor, List[torch.Tensor]]): Gradient of output tensor for this stage. + + Returns: + Union[torch.Tensor, List[torch.Tensor]]: Gradient of input tensor. + """ + + # Retain the grad on the input_obj. + if input_obj is not None: + if isinstance(input_obj, torch.Tensor): + input_obj.retain_grad() + else: + for in_tensor in input_obj: + if in_tensor is not None: + in_tensor.retain_grad() + + # Backward pass. + + # Only the last microbatch does syncing grad. + skip_grad_sync = self._get_current_microbatch_id(step_id) != self.num_microbatches - 1 + + self._call_hooks("before_backward", output_obj, output_obj_grad) + with switch_optimizer_grad_sync_skip_mode(engine.optimizer, skip_grad_sync): + if output_obj_grad is None: + engine.backward(output_obj) + else: + engine.backward_by_grad(output_obj, output_obj_grad) + + # Collect the grad of the input_obj. + input_obj_grad = None + if input_obj is not None: + if isinstance(input_obj, torch.Tensor): + input_obj_grad = input_obj.grad + else: + input_obj_grad = [] + for in_tensor in input_obj: + input_obj_grad.append(in_tensor.grad) + self._call_hooks("after_backward", input_obj_grad) + + return input_obj_grad + + def _forward_only_step(self, engine, return_loss=True, return_output_label=True): + """ + This function performs forward only computation process. The scheduling of microbatches is similar to the + warmup phase, where each microbatch first receives the forward input from the previous stage, then performs + the forward computation, and finally passes the forward computation output to the next stage. There are two + special cases to note: + 1. The first stage of the pipeline does not need to receive forward input; its input comes from the dataloader. + 2. The last stage of the pipeline does not need to send forward output; its output is returned to the user code + for processing. + + Args: + engine (colossalai.engine.Engine): internlm engine for training and inference. + return_loss (bool, optional): Whether to return the accumulated loss. + return_output_label (bool, optional): Whether to return outputs and labels. + + Returns: + Tuple[Union[torch.Tensor, None], Union[torch.Tensor, None], Union[torch.Tensor, None]]: + output, label, and accumulated loss. + """ + + # Input, output tensors only need to be saved when doing backward passes + return_tensors = [] + accum_loss_init_func = lambda: torch.zeros(1, device=get_current_device()) + accum_loss = defaultdict(accum_loss_init_func) if return_loss and gpc.is_pipeline_last_stage( + ignore_virtual=True) else None + + # Used for tensor meta information communication + forward_recv_shapes = self.tensor_shape + need_forward_meta = self.tensor_shape is None + + # Run all forward passes. + for _ in range(self.num_microbatches): + # Receive input from the previous stage + if not gpc.is_first_rank(ParallelMode.PIPELINE): + if forward_recv_shapes is None: + forward_recv_shapes = comm.recv_obj_meta() + input_obj = comm.recv_forward( + forward_recv_shapes, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + else: + input_obj = None + + # Perform forward computation + output_obj = self._forward_step( + engine, + input_obj, + return_tensors, + return_output_label=return_output_label, + accum_loss=accum_loss, + ) + + if not gpc.is_last_rank(ParallelMode.PIPELINE): + if need_forward_meta: + comm.send_obj_meta(output_obj) + need_forward_meta = False # send only once. + # Send the forward computation output to the next stage + comm.send_forward(output_obj, scatter_gather_tensors=self.scatter_gather_tensors) + + output, label = pack_return_tensors(return_tensors) if len(return_tensors) > 0 else (None, None) + + return output, label, accum_loss + + def _forward_backward_step(self, engine, return_loss=True, return_output_label=True): + """ + This function schedules the forward and backward computation of microbatches in the pipeline in a 1F1B manner. + It consists of three stages: warmup, 1F1B, and cooldown. + + 1. Warmup Stage: + The warmup stage performs num_warmup forward microsteps. The calculation of num_warmup is the pipeline length + minus the rank of the current pipeline minus 1. For each microstep, it receives data as input from the previous + stage, performs the forward computation, and then sends the result to the next stage. + + 2. 1F1B Stage: + The 1F1B stage consists of pairs of forward and backward microsteps. It performs num_1f1b_micropairs iterations, + where num_1f1b_micropairs is calculated as the total number of microbatches minus the number of microbatches in + the warmup stage. In each iteration, it first performs a forward computation, sends the result to the next + stage, receives input for the backward computation, performs the backward computation, and finally sends the + result to the previous stage to receive input for the next forward computation. + + 3. Cooldown Stage: + The cooldown stage performs the same number of iterations as the warmup stage. In each iteration, it receives + input for the backward computation, performs the backward computation, and finally sends the result to the + previous stage. + + There are two special cases to consider: + 1. The first stage of the pipeline does not need to receive forward input or send backward output. The last + stage does not need to send forward output or receive backward input. + 2. Pay attention to the communication between stages and use additional communication to bridge the gap. + + Args: + engine (Engine): The engine used for computation. + return_loss (bool, optional): Whether to return the accumulated loss. + return_output_label (bool, optional): Whether to return outputs and labels. + + Returns: + Tuple[Union[torch.Tensor, None], Union[torch.Tensor, None], Union[torch.Tensor, None]]: + The output, label, and accumulated loss. + """ + + num_warmup_microsteps = ( + gpc.get_world_size(ParallelMode.PIPELINE) - gpc.get_local_rank(ParallelMode.PIPELINE) - 1 + ) + num_warmup_microsteps = min(num_warmup_microsteps, self.num_microbatches) + num_1f1b_micropairs = self.num_microbatches - num_warmup_microsteps + + # Input, output tensors only need to be saved when doing backward passes + input_objs = [] + output_objs = [] + return_tensors = [] + accum_loss_init_func = lambda: torch.zeros(1, device=get_current_device()) + accum_loss = defaultdict(accum_loss_init_func) if return_loss and gpc.is_pipeline_last_stage( + ignore_virtual=True) else None + + # Used for tensor meta information communication + forward_recv_shapes = self.tensor_shape + backward_recv_shapes = None + need_forward_meta = self.tensor_shape is None + + # Run warmup forward passes. + for i in range(num_warmup_microsteps): + # Receive the input from the previous stage + if not gpc.is_first_rank(ParallelMode.PIPELINE): + if forward_recv_shapes is None: + forward_recv_shapes = comm.recv_obj_meta() + input_obj = comm.recv_forward( + forward_recv_shapes, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + else: + input_obj = None + + # Perform forward computation + output_obj = self._forward_step( + engine, + input_obj, + return_tensors, + return_output_label=return_output_label, + accum_loss=accum_loss, + ) + + if not gpc.is_last_rank(ParallelMode.PIPELINE): + if isinstance(output_obj, torch.Tensor): + backward_recv_shapes = output_obj.shape + else: + backward_recv_shapes = [out_tensor.shape for out_tensor in output_obj] + + if need_forward_meta: + comm.send_obj_meta(output_obj) + need_forward_meta = False # send only once. + + # Send the output of forward computation of this pipeline stage to the next pipeline stage as input for + # forward computation + if not gpc.is_last_rank(ParallelMode.PIPELINE): + comm.send_forward(output_obj, scatter_gather_tensors=self.scatter_gather_tensors) + + input_objs.append(input_obj) + output_objs.append(output_obj) + + # Before running 1F1B, need to receive first forward tensor. + # If all microbatches are run in warmup / cooldown phase, then no need to + # receive this tensor here. + if num_1f1b_micropairs > 0: + if not gpc.is_first_rank(ParallelMode.PIPELINE): + if forward_recv_shapes is None: + forward_recv_shapes = comm.recv_obj_meta(forward_recv_shapes) + input_obj = comm.recv_forward( + forward_recv_shapes, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + else: + input_obj = None + + # Run 1F1B in steady state. + for i in range(num_1f1b_micropairs): + # Perform forward computation + output_obj = self._forward_step( + engine, + input_obj, + return_tensors, + return_output_label=return_output_label, + accum_loss=accum_loss, + ) + + if gpc.is_last_rank(ParallelMode.PIPELINE): + output_obj_grad = None + else: + output_obj_grad = comm.send_forward_recv_backward( + output_obj, + backward_recv_shapes, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + + # Add input_obj and output_obj to end of list. + input_objs.append(input_obj) + output_objs.append(output_obj) + + # Pop output_obj and output_obj from the start of the list for + # the backward pass. + input_obj = input_objs.pop(0) + output_obj = output_objs.pop(0) + + input_obj_grad = self._backward_step(engine, i, input_obj, output_obj, output_obj_grad) + + if i == (num_1f1b_micropairs - 1): + input_obj = None + if not gpc.is_first_rank(ParallelMode.PIPELINE): + comm.send_backward( + input_obj_grad, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + else: + if gpc.is_first_rank(ParallelMode.PIPELINE): + input_obj = None + else: + input_obj = comm.send_backward_recv_forward( + input_obj_grad, + forward_recv_shapes, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + + # Run cooldown backward passes. + for i in range(num_warmup_microsteps): + input_obj = input_objs.pop(0) + output_obj = output_objs.pop(0) + + if not gpc.is_last_rank(ParallelMode.PIPELINE): + output_obj_grad = comm.recv_backward( + backward_recv_shapes, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + else: + output_obj_grad = None + + input_obj_grad = self._backward_step( + engine, num_1f1b_micropairs + i, input_obj, output_obj, output_obj_grad + ) + + if not gpc.is_first_rank(ParallelMode.PIPELINE): + comm.send_backward(input_obj_grad, scatter_gather_tensors=self.scatter_gather_tensors) + + output, label = pack_return_tensors(return_tensors) if len(return_tensors) > 0 else (None, None) + + return output, label, accum_loss + + @llm_timeout(func_name="nointerleaved_forward_backward_step") + def forward_backward_step(self, engine, data_iter, forward_only=False, return_loss=True, return_output_label=True): + """Runs non-interleaved 1F1B schedule, with communication between pipeline stages. + Returns a tuple with losses if the last stage, an empty tuple otherwise. + + Args: + engine (colossalai.engine.Engine): Colossalai engine for training and inference. + data_iter (Iterable): Dataloader as the form of an iterator, obtained by calling iter(dataloader). + forward_only (bool, optional): + Whether run forward step only. Default is false. If true, no backward will be run. + return_loss (bool, optional): Whether returns the loss value. Default is true. + return_output_label (bool, optional): If False, the output and label won't be returned. + Returns: + Tuple[:class:`torch.Tensor`]: A tuple of (output, label, loss), loss and label could be None. + """ + + assert ( + forward_only or return_loss + ), "The argument 'return_loss' has to be True when 'forward_only' is False, but got False." + + # Load data first + self.load_batch(engine, data_iter) + + if forward_only: + return self._forward_only_step(engine, return_loss, return_output_label) + else: + return self._forward_backward_step(engine, return_loss, return_output_label) + + +class InterleavedPipelineScheduler(PipelineScheduler): + """ + Interleaved Pipeline Scheduler. + """ + + def __init__( + self, + num_microbatches: int, + num_chunks: int, + dtype: torch.dtype = torch.float, + data_process_func: Callable = None, + tensor_shape: Union[torch.Size, List[int], Tuple[int]] = None, + scatter_gather_tensors: bool = False, + scheduler_hooks: Optional[List[SchedulerHook]] = None, + communication_overlap: bool = False, + ): + """A helper schedule class for pipeline parallelism running environment. + It uses interleaved 1F1B strategy. Other properties are similar as + :class:`NonPipelineSchedule`. + + Args: + num_microbatches (int): The number of microbatches. + num_chunks (int): The number of model chunks. + dtype (torch.dtype, optional): The data type of the tensors. Default is torch.float. + data_process_func (Callable, optional): + The preprocessing function which receives a batch of data, and it will be executed in `load_batch`. + tensor_shape (torch.Size, optional): Specified shape in pipeline communication. + scatter_gather_tensors (bool, optional): + If set to `True`, communication will be reduced over pipeline when using 1D tensor parallelization. + scheduler_hooks (List[SchedulerHook], optional): List of scheduler hooks. Default is None. + communication_overlap (bool, optional): Whether to enable communication overlap. Default is False. + """ + assert ( + num_microbatches % gpc.get_world_size(ParallelMode.PIPELINE) == 0 + ), "num_microbatches must be an integer multiple of pipeline parallel world size" + + assert ( + isinstance(num_chunks, int) and num_chunks > 0 + ), f"expected num_chunks to be an integer and larger than 0, but got {num_chunks}" + + super().__init__( + num_microbatches, + dtype=dtype, + data_process_func=data_process_func, + tensor_shape=tensor_shape, + scatter_gather_tensors=scatter_gather_tensors, + scheduler_hooks=scheduler_hooks, + ) + + gpc.set_virtual_pipeline_parallel_size(num_chunks) + gpc.set_virtual_pipeline_parallel_rank(0) + + self._num_chunks = num_chunks + self._communication_overlap = communication_overlap + # switch 1f1b loop runner function according to communication overlap + self._run_1f1b_loop = ( + self._run_1f1b_loop_with_overlap if communication_overlap else self._run_1f1b_loop_without_overlap + ) + + # states + self._pp_size = gpc.get_world_size(ParallelMode.PIPELINE) + self._pp_rank = gpc.get_local_rank(ParallelMode.PIPELINE) + + self._accum_loss = None + self._return_tensors = None + self._input_objs = [[] for _ in range(num_chunks)] + self._output_objs = [[] for _ in range(num_chunks)] + self._output_obj_grads = [[] for _ in range(num_chunks)] + + self._input_obj_shapes = [self.tensor_shape for _ in range(num_chunks)] + self._output_obj_shapes = [None for _ in range(num_chunks)] + self._send_tensor_shape_flags = [self.tensor_shape is None for _ in range(num_chunks)] + + @property + def tensor_shape(self) -> torch.Size: + return self._tensor_shape + + @tensor_shape.setter + def tensor_shape(self, tensor_shape: torch.Size): + self._tensor_shape = tensor_shape + self._input_obj_shapes = [self._tensor_shape for _ in range(self._num_chunks)] + self._send_tensor_shape_flags = [self._tensor_shape is None for _ in range(self._num_chunks)] + + def _clear_state(self) -> None: + self._accum_loss = None + self._return_tensors = None + self._input_objs = [[] for _ in range(self._num_chunks)] + self._output_objs = [[] for _ in range(self._num_chunks)] + self._output_obj_grads = [[] for _ in range(self._num_chunks)] + + self._input_obj_shapes = [self.tensor_shape for _ in range(self._num_chunks)] + self._output_obj_shapes = [None for _ in range(self._num_chunks)] + self._send_tensor_shape_flags = [self.tensor_shape is None for _ in range(self._num_chunks)] + + def load_batch(self, engine, data_iter): + super().load_batch(engine, data_iter) + # overwrite microbatch_offset, since model chunks load the same microbatch, and should tract the offset + self.microbatch_offset = [0 for _ in range(self._num_chunks)] + + def load_micro_batch(self, model_chunk_id): + micro_batch_data, micro_batch_label = self._load_micro_batch( + data=self.batch_data, + label=self.batch_label, + offset=self.microbatch_offset[model_chunk_id], + micro_bsz=self.microbatch_size, + ) + micro_batch_data["label"] = micro_batch_label + self.microbatch_offset[model_chunk_id] += self.microbatch_size + return move_to_device(micro_batch_data) + + def _forward_step(self, engine, chunk_id): + """Forward step for passed-in model. If it is the first stage, the input tensor + is obtained from data_iterator, otherwise the passed-in input_obj is used. + Returns output tensor. This is a helper function and can be ignored by users. + + Args: + engine (colossalai.engine.Engine): Colossalai engine for training and inference. + chunk_id (int): The id of model chunks. + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: output or the loss value of the current + pipeline stage. + """ + gpc.set_virtual_pipeline_parallel_rank(chunk_id) + + if gpc.is_pipeline_first_stage() and len(self._input_objs[chunk_id]) == len(self._output_objs[chunk_id]): + self._input_objs[chunk_id].append(None) + input_obj = self._input_objs[chunk_id][-1] + + micro_batch_data = self.load_micro_batch(chunk_id) + data, label = self._get_data_label_for_current_step(input_obj, micro_batch_data) + + self._call_hooks("before_forward", data) + output_obj = self._call_engine(engine.model[chunk_id], data) + # Convert output_obj to fp32 when last model chunk of last stage + if gpc.is_pipeline_last_stage(ignore_virtual=False) and isinstance(engine.model[chunk_id], NaiveAMPModel): + output_obj = engine.model[chunk_id].convert_to_fp32(output_obj) + self._call_hooks("after_forward", output_obj) + + if gpc.is_pipeline_last_stage(): + self._call_hooks("post_helper_func", output_obj, label) + + if self._return_tensors is not None: + self._return_tensors.append((output_obj, label)) + if self._accum_loss is not None: + self._call_hooks("before_criterion", output_obj, label) + loss = self._call_engine_criterion(engine.criterion, output_obj, label) + self._call_hooks("after_criterion", loss) + + loss_reduced = loss / self.num_microbatches + self._accum_loss.add_(loss_reduced.detach()) + output_obj = loss_reduced + + self._output_objs[chunk_id].append(output_obj) + + return output_obj + + def _backward_step(self, engine, chunk_id, step_id): + """ + Backward step for passed-in model. If it is the last stage, the input tensor + is obtained from the previous forward step, otherwise the passed-in input_obj is used. + Returns input tensor gradient. This is a helper function and can be ignored by users. + + Args: + engine (colossalai.engine.Engine): Colossalai engine for training and inference. + chunk_id (int): The id of model chunks. + step_id (int): The current step id. + + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: input tensor gradient. + """ + gpc.set_virtual_pipeline_parallel_rank(chunk_id) + + if gpc.is_pipeline_last_stage() and len(self._output_obj_grads[chunk_id]) == 0: + self._output_obj_grads[chunk_id].append(None) + + input_obj = self._input_objs[chunk_id].pop(0) + output_obj = self._output_objs[chunk_id].pop(0) + output_obj_grad = self._output_obj_grads[chunk_id].pop(0) + + input_obj_grad = super()._backward_step(engine, step_id, input_obj, output_obj, output_obj_grad) + + return input_obj_grad + + def _get_chunk_by_microbatch(self, step_id: int, backward: bool = False) -> int: + """Helper method to get the model chunk ID given the iteration number.""" + microbatch_id_in_group = step_id % (self._pp_size * self._num_chunks) + chunk_id = microbatch_id_in_group // self._pp_size + + if backward: + chunk_id = self._num_chunks - chunk_id - 1 + + return chunk_id + + def _get_current_microbatch_id(self, step_id: int) -> int: + # format: + # microstep_id : 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 + # microbatch_id: 1 2 3 4 1 2 3 4 5 6 7 8 5 6 7 8 + num_microbatch_group = step_id // (self._pp_size * self._num_chunks) + step_id_in_group = step_id % (self._pp_size * self._num_chunks) + + microbatch_id = num_microbatch_group * self._pp_size + step_id_in_group % self._pp_size + + return microbatch_id + + def _run_warmup_loop( + self, + engine: Engine, + num_microsteps: int, + num_warmup_microsteps: int, + receive_extra_backward: bool = False, + forward_only: bool = False, + ) -> None: + """ + Run the warm-up loop and prepare data for the 1F1B stage. + + During the warm-up process, for each execution, it first performs a forward computation, + and then sends the computation result to the next stage. + It also receives data for the next forward computation. + Since the input for the first forward computation is not considered initially, + it needs to receive data once at the beginning. + + After the warm-up is completed, we need to prepare data for the 1F1B stage. + The data preparation process should be consistent with the communication method of the 1F1B stage. + + Args: + engine (Engine): The engine to run the warm-up loop. + num_microsteps (int): The total number of microsteps. + num_warmup_microsteps (int): The number of warm-up microsteps. + receive_extra_backward (bool, optional): Whether to receive extra backward input for the 1F1B stage. + Default is False. + forward_only (bool, optional): Whether to only perform forward pass. Default is False. + """ + if not gpc.is_pipeline_first_stage(): + if self._input_obj_shapes[0] is None: + self._input_obj_shapes[0] = comm.recv_obj_meta(self._input_obj_shapes[0]) + self._input_objs[0].append( + comm.recv_forward( + self._input_obj_shapes[0], + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + ) + else: + self._input_objs[0].append(None) + + for k in range(num_warmup_microsteps): + chunk_id = self._get_chunk_by_microbatch(k) + + output_obj = self._forward_step(engine, chunk_id) + + if forward_only: + # when forward-only, no need to save tensors for a backward pass + self._input_objs[chunk_id].pop() + self._output_objs[chunk_id].pop() + + if not gpc.is_pipeline_last_stage(): + if isinstance(output_obj, torch.Tensor): + self._output_obj_shapes[chunk_id] = output_obj.shape + else: + self._output_obj_shapes[chunk_id] = [out_tensor.shape for out_tensor in output_obj] + + if self._send_tensor_shape_flags[chunk_id]: + comm.send_obj_meta(output_obj) + self._send_tensor_shape_flags[chunk_id] = False # send only once for each chunk. + + # Determine if tensor should be received from previous stage. + next_forward_chunk_id = self._get_chunk_by_microbatch(k + 1) + + with switch_virtual_pipeline_parallel_rank(next_forward_chunk_id): + if not gpc.is_pipeline_first_stage() and self._input_obj_shapes[next_forward_chunk_id] is None: + self._input_obj_shapes[next_forward_chunk_id] = comm.recv_obj_meta() + if k == (num_microsteps - 1) or gpc.is_pipeline_first_stage(): + input_shape = None + else: + input_shape = self._input_obj_shapes[next_forward_chunk_id] + + # Don't send tensor downstream if on last stage. + if gpc.is_pipeline_last_stage(): + output_obj = None + + # Send and receive tensors as appropriate (send tensors computed + # in this iteration; receive tensors for next iteration). + if k != (num_warmup_microsteps - 1) or not receive_extra_backward: + # Normal warm-up communication process, or no need to prepare backward input for the 1F1B stage + input_obj = comm.send_forward_recv_forward( + output_obj, + input_shape, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + else: + # Receive output_obj_grad for next backward, if receive_extra_backward is True. + if self._communication_overlap: + # In this case, we should handle forward and backward communication separately, consistent with the + # overlap version of the 1F1B stage + input_obj = comm.send_forward_recv_forward( + output_obj, + input_shape, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + output_obj_grad = comm.send_backward_recv_backward( + None, # nothing to send + self._output_obj_shapes[self._num_chunks - 1], + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + self._output_obj_grads[self._num_chunks - 1].append(output_obj_grad) + else: + # In this case, we should handle forward and backward communication together, consistent with the + # non-overlap version of the 1F1B stage + input_obj, output_obj_grad = comm.send_forward_backward_recv_forward_backward( + output_obj, + None, # no backward grad to send + input_shape, + self._output_obj_shapes[self._num_chunks - 1], + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + self._output_obj_grads[self._num_chunks - 1].append(output_obj_grad) + + self._input_objs[next_forward_chunk_id].append(input_obj) + + def _run_1f1b_loop_with_overlap( + self, + engine: Engine, + num_warmup_microsteps: int, + num_1f1b_micropairs: int, + all_warmup_microsteps: bool = False, + ) -> None: + """ + Run the 1F1B loop with overlap. + + The 1F1B loop with overlap consists of the following steps: + 1. Perform the forward pass. + 2. Check if the backward input is ready. + 3. Send the forward output and receive the forward input for the next iteration. + 4. Perform the backward pass. + 5. Check if the forward input is ready. + 6. Send the backward output and receive the backward input for the next iteration. + + Args: + engine (Engine): The engine to run the 1F1B loop. + num_warmup_microsteps (int): The number of warm-up microsteps. + num_1f1b_micropairs (int): The number of 1F1B micropairs. + all_warmup_microsteps (bool, optional): Whether to run all warm-up microsteps. Default is False. + """ + + backward_async_communicator = None + + # Run 1F1B in steady state. + for k in range(num_1f1b_micropairs): + forward_microstep_id = k + num_warmup_microsteps + backward_microstep_id = k + forward_chunk_id = self._get_chunk_by_microbatch(forward_microstep_id) + backward_chunk_id = self._get_chunk_by_microbatch(backward_microstep_id, backward=True) + + # 1. Forward pass. + output_obj = self._forward_step(engine, forward_chunk_id) + + # 2. Check if the backward input is ready. + if backward_async_communicator is not None: + output_obj_grad = backward_async_communicator.wait_and_receive() + + if backward_async_communicator.need_receive: + self._output_obj_grads[backward_chunk_id].append(output_obj_grad) + + # 3. Send the forward outputs and receive the forward inputs from the previous rank. + + # Check if it is the last model chunk of the last pipeline stage, no need to send forward output. + gpc.set_virtual_pipeline_parallel_rank(forward_chunk_id) + if gpc.is_pipeline_last_stage(): + output_obj = None + + # Check if it needs to receive the results from the previous rank. + next_forward_chunk_id = self._get_chunk_by_microbatch(forward_microstep_id + 1) + with switch_virtual_pipeline_parallel_rank(next_forward_chunk_id): + if gpc.is_pipeline_first_stage() or k == num_1f1b_micropairs - 1: + input_obj_shape = None + else: + input_obj_shape = self._input_obj_shapes[next_forward_chunk_id] + + forward_async_communicator = comm.AsynCommunicator( + output_obj, + input_obj_shape, + self.dtype, + self.scatter_gather_tensors, + forward=True, + ) + forward_async_communicator.start() + + # 5. Backward pass. + + input_obj_grad = self._backward_step(engine, backward_chunk_id, backward_microstep_id) + + input_obj = forward_async_communicator.wait_and_receive() + if forward_async_communicator.need_receive: + self._input_objs[next_forward_chunk_id].append(input_obj) + + # 6. Send the backward output and receive the backward input for the next iteration. + gpc.set_virtual_pipeline_parallel_rank(backward_chunk_id) + if gpc.is_pipeline_first_stage(): + input_obj_grad = None + + next_backward_chunk_id = self._get_chunk_by_microbatch(backward_microstep_id + 1, backward=True) + with switch_virtual_pipeline_parallel_rank(next_backward_chunk_id): + if gpc.is_pipeline_last_stage(): + output_obj_shape = None + else: + output_obj_shape = self._output_obj_shapes[next_backward_chunk_id] + + backward_async_communicator = comm.AsynCommunicator( + input_obj_grad, + output_obj_shape, + self.dtype, + self.scatter_gather_tensors, + forward=False, + ) + backward_async_communicator.start() + + if all_warmup_microsteps: + if not gpc.is_pipeline_last_stage(): + self._output_obj_grads[self._num_chunks - 1].append( + comm.recv_backward( + self._output_obj_shapes[self._num_chunks - 1], + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + ) + else: + self._output_obj_grads[self._num_chunks - 1].append(None) + else: + output_obj_grad = backward_async_communicator.wait_and_receive() + if backward_async_communicator.need_receive: + backward_chunk_id = self._get_chunk_by_microbatch(num_1f1b_micropairs, backward=True) + self._output_obj_grads[backward_chunk_id].append(output_obj_grad) + + def _run_1f1b_loop_without_overlap( + self, + engine: Engine, + num_warmup_microsteps: int, + num_1f1b_micropairs: int, + all_warmup_microsteps: bool = False, + ) -> None: + """ + Run the 1F1B loop without overlap. + + The 1F1B loop without overlap consists of the following steps: + 1. Perform the forward pass. + 2. Perform the backward pass. + 3. Send the forward output of this iteration to the next stage, and send the backward output of this iteration + to the previous stage, and receive the forward and backward inputs for the next iteration. + + Args: + engine (Engine): The engine to use for computation. + num_warmup_microsteps (int): The number of warmup microsteps. + num_1f1b_micropairs (int): The number of 1F1B micro-pairs. + all_warmup_microsteps (bool, optional): Whether to run all warmup microsteps. Defaults to False. + """ + for k in range(num_1f1b_micropairs): + # Forward pass. + forward_microstep_id = k + num_warmup_microsteps + forward_chunk_id = self._get_chunk_by_microbatch(forward_microstep_id) + output_obj = self._forward_step(engine, forward_chunk_id) + + # Backward pass. + backward_microstep_id = k + backward_chunk_id = self._get_chunk_by_microbatch(backward_microstep_id, backward=True) + input_obj_grad = self._backward_step(engine, backward_chunk_id, backward_microstep_id) + + # Send output_obj and input_obj_grad, receive input_obj + # and output_obj_grad. + + # Determine if current stage has anything to send in either direction, + # otherwise set obj to None. + gpc.set_virtual_pipeline_parallel_rank(forward_chunk_id) + if gpc.is_pipeline_last_stage(): + output_obj = None + + gpc.set_virtual_pipeline_parallel_rank(backward_chunk_id) + if gpc.is_pipeline_first_stage(): + input_obj_grad = None + + # Determine if peers are sending, and where in data structure to put + # received tensors. + next_forward_chunk_id = self._get_chunk_by_microbatch(forward_microstep_id + 1) + with switch_virtual_pipeline_parallel_rank(next_forward_chunk_id): + if gpc.is_pipeline_first_stage() or k == num_1f1b_micropairs - 1: + recv_prev = False + else: + recv_prev = True + + next_backward_chunk_id = self._get_chunk_by_microbatch(backward_microstep_id + 1, backward=True) + with switch_virtual_pipeline_parallel_rank(next_backward_chunk_id): + if gpc.is_pipeline_last_stage(): + recv_next = False + else: + recv_next = True + + input_shape = self._input_obj_shapes[next_forward_chunk_id] if recv_prev else None + output_shape = self._output_obj_shapes[next_backward_chunk_id] if recv_next else None + + # Communicate objs. + input_obj, output_obj_grad = comm.send_forward_backward_recv_forward_backward( + output_obj, + input_obj_grad, + input_shape, + output_shape, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + + # Put input_obj and output_obj_grad in data structures in the + # right location. + if recv_prev: + self._input_objs[next_forward_chunk_id].append(input_obj) + if recv_next: + self._output_obj_grads[next_backward_chunk_id].append(output_obj_grad) + + # receive necessary data for next cooldown loop + if all_warmup_microsteps: + if not gpc.is_pipeline_last_stage(): + self._output_obj_grads[self._num_chunks - 1].append( + comm.recv_backward( + self._output_obj_shapes[self._num_chunks - 1], + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + ) + else: + self._output_obj_grads[self._num_chunks - 1].append(None) + + def _run_cooldown_loop(self, engine: Engine, num_microsteps: int, num_1f1b_micropairs: int) -> None: + """ + Run the cooldown loop. + + The cooldown loop consists of the following steps: + 1. Perform the backward step. + 2. Send the backward output to the next stage and receive inputs for next backward. + + Args: + engine (Engine): The engine to use for computation. + num_microsteps (int): The total number of microsteps. + num_1f1b_micropairs (int): The number of 1F1B micro-pairs. + """ + for k in range(num_1f1b_micropairs, num_microsteps): + chunk_id = self._get_chunk_by_microbatch(k, backward=True) + + input_obj_grad = self._backward_step(engine, chunk_id, k) + + next_backward_chunk_id = self._get_chunk_by_microbatch(k + 1, backward=True) + + if k != (num_microsteps - 1) and not ( + gpc.is_pipeline_last_stage(ignore_virtual=True) and next_backward_chunk_id == (self._num_chunks - 1) + ): + output_shape = self._output_obj_shapes[next_backward_chunk_id] + else: + output_shape = None + + self._output_obj_grads[next_backward_chunk_id].append( + comm.send_backward_recv_backward( + input_obj_grad, + output_shape, + dtype=self.dtype, + scatter_gather_tensors=self.scatter_gather_tensors, + ) + ) + + def _forward_only_step(self, engine: Engine): + num_microsteps = self.num_microbatches * self._num_chunks + num_warmup_microsteps = num_microsteps + + self._run_warmup_loop( + engine, + num_microsteps, + num_warmup_microsteps, + receive_extra_backward=False, + forward_only=True, + ) + + def _forward_backward_step(self, engine: Engine): + # Compute number of warmup and remaining microbatches. + all_warmup_microsteps = False + num_microsteps = self.num_microbatches * self._num_chunks + + # Run all forward passes and then all backward passes if number of + # microbatches is just the number of pipeline stages. + # Otherwise, perform (num_chunks-1)*pipeline_parallel_size on + # all workers, followed by more microbatches after depending on + # stage ID (more forward passes for earlier stages, later stages can + # immediately start with 1F1B). + if self.num_microbatches == self._pp_size: + num_warmup_steps = num_microsteps + all_warmup_microsteps = True + else: + num_warmup_steps = (self._pp_size - self._pp_rank - 1) * 2 + num_warmup_steps += (self._num_chunks - 1) * self._pp_size + num_warmup_steps = min(num_warmup_steps, num_microsteps) + num_1f1b_micropairs = num_microsteps - num_warmup_steps + + # We usually need to prepare an extra backward data for the 1F1B stage when the WarmUp stage ends, + # because the 1F1B stage typically performs one forward and backward pass together, + # except in the following cases: + receive_extra_backward = not ( + all_warmup_microsteps # Only warmup microsteps + or gpc.is_pipeline_last_stage(ignore_virtual=True) # The rank is the last pipeline stage + ) + + # 1. Warmup + self._run_warmup_loop( + engine, + num_microsteps, + num_warmup_steps, + receive_extra_backward=receive_extra_backward, + ) + + # 2. 1F1B + self._run_1f1b_loop( + engine, + num_warmup_steps, + num_1f1b_micropairs=num_1f1b_micropairs, + all_warmup_microsteps=all_warmup_microsteps, + ) + + # 3. Cooldown + self._run_cooldown_loop(engine, num_microsteps, num_1f1b_micropairs=num_1f1b_micropairs) + + @llm_timeout(func_name="interleaved_forward_backward_step") + def forward_backward_step(self, engine, data_iter, forward_only=False, return_loss=True, return_output_label=True): + """Run interleaved 1F1B schedule (model split into model chunks), with + communication between pipeline stages as needed. + + Args: + engine (colossalai.engine.Engine): Colossalai engine for training and inference. + data_iter (Iterable): Dataloader as the form of an iterator, obtained by calling iter(dataloader). + forward_only (bool, optional): + Whether run forward step only. Default is false. If true, no backward will be run. + return_loss (bool, optional): Whether returns the loss value. Default is true. + return_output_label (bool, optional): If False, the output and label won't be returned. + + Returns: + Tuple[:class:`torch.Tensor`]: A tuple of (output, label, loss), loss and label could be None. + The loss would be returned only in the last stage. + """ + assert ( + forward_only or return_loss + ), "The argument 'return_loss' has to be True when 'forward_only' is False, but got False." + + gpc.set_virtual_pipeline_parallel_rank(0) + + self.load_batch(engine, data_iter) + + if return_loss and gpc.is_pipeline_last_stage(ignore_virtual=True): + self._accum_loss = torch.zeros(1, device=get_current_device()) + if return_output_label: + self._return_tensors = [] + + if forward_only: + self._forward_only_step(engine) + else: + self._forward_backward_step(engine) + + if return_output_label and len(self._return_tensors) > 0: + output, label = pack_return_tensors(self._return_tensors) + else: + output, label = (None, None) + accum_loss = self._accum_loss + + self._clear_state() + + return output, label, accum_loss + + +class KDPipelineScheduler(PipelineScheduler): + + def __init__( + self, + num_microbatches: int, + dtype: torch.dtype = torch.float, + data_process_func: Callable = None, + tensor_shape: Union[torch.Size, List[int], Tuple[int]] = None, + scatter_gather_tensors: bool = False, + scheduler_hooks: Optional[List[SchedulerHook]] = None, + ): + super().__init__( + num_microbatches=num_microbatches, + dtype=dtype, + data_process_func=data_process_func, + tensor_shape=tensor_shape, + scatter_gather_tensors=scatter_gather_tensors, + scheduler_hooks=scheduler_hooks, + ) + + def _forward_step(self, engine, input_obj, return_tensors, return_output_label=True, accum_loss=None): + """ + Forward step for passed-in model. If it is the first stage, the input tensor + is obtained from data_iterator, otherwise the passed-in input_obj is used. + Returns output tensor. This is a helper function and can be ignored by users. + + Args: + engine (colossalai.engine.Engine): Colossalai engine for training and inference. + input_obj (Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]): Input tensor for this pipeline stage. + return_tensors (List[:class:`torch.Tensor`]): A list of tensors to return. + return_output_label (bool, optional): Whether returns output labels. + accum_loss (optional): Where accumulated loss stores. + Returns: + Union[:class:`torch.Tensor`, List[:class:`torch.Tensor`]]: output or the loss value of the current + pipeline stage. + """ + micro_batch_data = self.load_micro_batch() + data, label = self._get_data_label_for_current_step(input_obj, micro_batch_data) + + self._call_hooks("before_forward", data) + output_obj = self._call_engine(engine.model, data) + self._call_hooks("after_forward", output_obj) + + if gpc.is_last_rank(ParallelMode.PIPELINE): + self._call_hooks("post_helper_func", output_obj, label) + if return_output_label: + return_tensors.append((output_obj, label)) + if accum_loss is not None: + self._call_hooks("before_criterion", output_obj, label) + loss_gt = gpc.config.kd_config['gt_weight'] * self._call_engine_criterion(engine.criterion, output_obj, + label) + + with torch.no_grad(): + engine.teacher.eval() + output_obj_t = self._call_engine(engine.teacher, data) + + loss_kd = gpc.config.kd_config['kd_weight'] * self._call_engine_criterion(engine.kd_criterion, + output_obj, + (output_obj_t, label)) + # loss = (gpc.config.kd_config['gt_weight'] * loss_gt + + # gpc.config.kd_config['kd_weight'] * loss_kd) + self._call_hooks("after_criterion", loss_gt + loss_kd) + + loss_gt_reduced = loss_gt / self.num_microbatches + loss_kd_reduced = loss_kd / self.num_microbatches + accum_loss['loss_gt'].add_(loss_gt_reduced.detach()) + accum_loss['loss_kd'].add_(loss_kd_reduced.detach()) + output_obj = loss_gt_reduced + loss_kd_reduced + + return output_obj diff --git a/InternLM/internlm/core/trainer.py b/InternLM/internlm/core/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..18a8f6f256700627426bc8f9e2e9661776168607 --- /dev/null +++ b/InternLM/internlm/core/trainer.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/engine + +import json +from typing import Iterable, Optional + +from internlm.core.engine import Engine +from internlm.core.scheduler import ( + BaseScheduler, + InterleavedPipelineScheduler, + NonPipelineScheduler, + PipelineScheduler, +) + + +class TrainState: + """ + The TrainState class is used to record the current state of training. + + Args: + train_dl (DataLoader): The DataLoader object used for training. + """ + + def __init__(self, config, batch_sampler) -> None: + """ + Args: + config (Config): internlm config + batch_sampler (torch.utils.data.Sampler): Because the dataloader loading is + asynchronous and prefetched, the batch_sampler state maintained inside the + dataloader are faster then the actual training progress, so we copy the + batch_sampler as the anchor point of ckpt reload. + """ + # The number of batches produced by the data iterator + self.batch_count: int = 0 + # Used to store the number of samples consumed in the current epoch + self.num_consumed_samples_in_epoch: int = 0 + # Total number of tokens consumed + self.num_consumed_tokens: int = 0 + # Number of batches skipped due to inf or nan values + self.inf_nan_skip_batches: int = 0 + # Records the number of updates, skipped batches and inf batches are not counted + self.step_count: int = 0 + + # Total step count + self.total_steps: int = config.data.total_steps + + # resume tensorboard folder, need load from checkpoint or set manually. + self.resume_tb_folder = config.resume_tb_folder + + self.tensorboard_folder = config.tensorboard_folder + + # learning rate + self.lr = config.adam.lr + + # smapler state + if batch_sampler: + self.init_batch_sampler(batch_sampler) + + def init_batch_sampler(self, batch_sampler): + """ + Args: + batch_sampler (torch.utils.data.Sampler): sampler. + """ + # make a copy of batch_sampler. + self.batch_sampler = batch_sampler.copy() + # Iterator for the batch sampler + self.batch_sampler_iter = iter(self.batch_sampler) + + def __str__(self) -> str: + """Returns a string representation of the training state in JSON format.""" + info = { + "batch_count": self.batch_count, + "num_consumed_samples_in_epoch": self.num_consumed_samples_in_epoch, + "num_consumed_tokens": self.num_consumed_tokens, + "inf_nan_skip_batches": self.inf_nan_skip_batches, + "step_count": self.step_count, + } + + return json.dumps(info, indent=4, sort_keys=True) + + def load_state_dict(self, other_stuffs): + """ + Resumes training from a checkpoint. + + Args: + other_stuffs (dict): Other information needed to resume training. + """ + self.num_consumed_samples_in_epoch = other_stuffs["num_consumed_samples_in_epoch"] + self.num_consumed_tokens = other_stuffs["num_consumed_tokens"] + self.inf_nan_skip_batches = other_stuffs["inf_nan_skip_batches"] + + # Because the ckpt save occurs after updating 'step_count', + # there is no need to increment 'step_count' here (Does our step count start from 0 ?), + # However, 'batch_count' is updating before ckpt storage, so it need to inc 1 when resume. + self.batch_count = other_stuffs["batch_count"] + 1 # here you need to shift a batch backward + self.step_count = other_stuffs.get("step_count", self.batch_count) + + # resume tensorboard from older tensorboard_folder + self.resume_tb_folder = other_stuffs.get("tensorboard_folder", None) + + def state_dict(self): + return { + "batch_count": self.batch_count, + "num_consumed_samples_in_epoch": self.num_consumed_samples_in_epoch, + "num_consumed_tokens": self.num_consumed_tokens, + "inf_nan_skip_batches": self.inf_nan_skip_batches, + "step_count": self.step_count, + "tensorboard_folder": self.tensorboard_folder, + } + + +class Trainer: + """This is a class tending for easy deployments of users' training and evaluation instead of + writing their own scripts. + + Args: + engine (:class:`Engine`): Engine responsible for the process function. + schedule (:class:`BaseScheduler`, optional): Runtime schedule. Defaults to None. + """ + + def __init__( + self, + engine: Engine, + schedule: Optional[BaseScheduler] = None, + ): + """Initializes the Trainer class. + + Args: + engine (Engine): The engine responsible for the process function. + schedule (Optional[BaseScheduler], optional): The runtime schedule. Defaults to None. + """ + self._engine = engine + + # build schedule + if schedule is None: + self._schedule = NonPipelineScheduler() + else: + assert isinstance( + schedule, BaseScheduler + ), f"expected schedule to be of type BaseSchedule, but got {type(schedule)}" + self._schedule = schedule + + self._schedule.pre_processing(self._engine) + + @property + def engine(self): + """Returns the engine that responsible for managing the training and evaluation process.""" + return self._engine + + @property + def schedule(self): + """Returns the runtime scheduler.""" + return self._schedule + + @property + def uses_pipeline(self): + """Returns whether the pipeline parallel is used or not.""" + return isinstance(self._schedule, (PipelineScheduler, InterleavedPipelineScheduler)) + + def train(self): + """Sets the model to training mode.""" + self._engine.train() + + def eval(self): + """Sets the model to evaluation mode.""" + self._engine.eval() + + def zero_grad(self): + """Sets the gradient of all parameters in the model to zero.""" + self._engine.zero_grad() + + def step(self): + """Executes the parameter update step.""" + return self._engine.step() + + def execute_schedule(self, data_iter: Iterable, **kwargs): + """Runs the forward, loss computation, and backward for the model. + Returns a tuple of (output, label, loss). + + Args: + data_iter (Iterable): The data iterator. + **kwargs: Additional keyword arguments. + + Returns: + Tuple[:class:`torch.Tensor`]: A tuple of (output, label, loss). + """ + output, label, loss = self._schedule.forward_backward_step(self._engine, data_iter, **kwargs) + return output, label, loss diff --git a/InternLM/internlm/data/__init__.py b/InternLM/internlm/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..23eb3ab66e777c612f86a23b246066d1e5c50a13 --- /dev/null +++ b/InternLM/internlm/data/__init__.py @@ -0,0 +1,13 @@ +from .batch_sampler import get_dpsampler_dataloader +from .collaters import jsonl_ds_collate_fn, packed_collate_fn +from .dummy_dataset import RandomDataset +from .packed_dataset import PackedDataset, PackedDatasetWithoutCuSeqlen + +__all__ = [ + "jsonl_ds_collate_fn", + "packed_collate_fn", + "RandomDataset", + "PackedDataset", + "PackedDatasetWithoutCuSeqlen", + "get_dpsampler_dataloader", +] diff --git a/InternLM/internlm/data/batch_sampler.py b/InternLM/internlm/data/batch_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..ca594346cb4cc361da45814363b6cee69533f188 --- /dev/null +++ b/InternLM/internlm/data/batch_sampler.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import math +import random +from typing import Iterator, TypeVar + +import numpy as np +import torch +from torch.utils.data import DataLoader, Dataset, Sampler + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.utils.logger import get_logger + +logger = get_logger(__file__) + +T_co = TypeVar("T_co", covariant=True) + + +class DataParallelSampler(Sampler): + """A data sampler for distributed data parallelism. + + Args: + dataset (:class:`torch.utils.data.Dataset`): The Dataset for sampling. + shuffle (bool, optional): Whether to shuffle data, defaults to False. + seed (int, optional): The random seed used for sampling, defaults to 0. + drop_last (bool, optional): Set to True to drop the last incomplete batch, if the dataset size + is not divisible by the batch size. If False and the size of dataset is not divisible by + the batch size, then the last batch will be smaller, defaults to False. + """ + + def __init__( + self, + dataset: Dataset, + shuffle: bool = False, + seed: int = 0, + drop_last: bool = False, + ) -> None: + self.dataset = dataset + self.num_replicas = gpc.get_world_size(ParallelMode.DATA) + self.rank = gpc.get_local_rank(ParallelMode.DATA) + self.epoch = 0 + self.drop_last = drop_last + # If the dataset length is evenly divisible by # of replicas, then there + # is no need to drop any data, since the dataset will be split equally. + # type: ignore[arg-type] + if self.drop_last and len(self.dataset) % self.num_replicas != 0: + # Split to nearest available length that is evenly divisible. + # This is to ensure each rank receives the same amount of data when + # using this Sampler. + self.num_samples = math.ceil( + # `type:ignore` is required because Dataset cannot provide a default __len__ + # see NOTE in pytorch/torch/utils/data/sampler.py + (len(self.dataset) - self.num_replicas) + / self.num_replicas # type: ignore[arg-type] + ) + else: + self.num_samples = math.ceil(len(self.dataset) / self.num_replicas) # type: ignore[arg-type] + self.total_size = self.num_samples * self.num_replicas + self.shuffle = shuffle + self.seed = seed + + def __iter__(self) -> Iterator[T_co]: + if self.shuffle: + # deterministically shuffle based on epoch and seed + g = torch.Generator() + g.manual_seed(self.seed + self.epoch) + # type: ignore[arg-type] + indices = torch.randperm(len(self.dataset), generator=g).tolist() + + # update for next epoch so that there is no need to call + # set_epoch manually + self.epoch += 1 + else: + indices = list(range(len(self.dataset))) # type: ignore[arg-type] + + if not self.drop_last: + # add extra samples to make it evenly divisible + padding_size = self.total_size - len(indices) + if padding_size <= len(indices): + indices += indices[:padding_size] + else: + indices += (indices * math.ceil(padding_size / len(indices)))[:padding_size] + else: + # remove tail of data to make it evenly divisible. + indices = indices[: self.total_size] + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank : self.total_size : self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) + + def __len__(self) -> int: + return self.num_samples + + def set_epoch(self, epoch: int) -> None: + r"""Sets the epoch for this sampler. When :attr:`shuffle=True`, this ensures all replicas + use a different random ordering for each epoch. Otherwise, the next iteration of this + sampler will yield the same ordering. + + Args: + epoch (int): Epoch number. + """ + self.epoch = epoch + + +def get_dpsampler_dataloader( + dataset, + shuffle=False, + seed=1024, + add_sampler=True, + drop_last=False, + pin_memory=False, + num_workers=0, + **kwargs, +): + r"""Set up a deterministic dataloader (also configure seed workers, samplers and whether shuffle or not) + + Note: + When pipeline parallel is enabled, shuffle cannot be True as it will result in mismatch between input data + on the 1st stage and label on the last stage. + + Args: + dataset (:class:`torch.utils.data.Dataset`): The dataset to be loaded. + shuffle (bool, optional): Whether to shuffle the dataset. Defaults to False. + seed (int, optional): Random worker seed for sampling, defaults to 1024. + add_sampler: Whether to add ``DistributedDataParallelSampler`` to the dataset. Defaults to True. + drop_last (bool, optional): Set to True to drop the last incomplete batch, if the dataset size + is not divisible by the batch size. If False and the size of dataset is not divisible by + the batch size, then the last batch will be smaller, defaults to False. + pin_memory (bool, optional): Whether to pin memory address in CPU memory. Defaults to False. + num_workers (int, optional): Number of worker threads for this dataloader. Defaults to 0. + kwargs (dict): optional parameters for ``torch.utils.data.DataLoader``, more details could be found in + `DataLoader `_. + + Returns: + :class:`torch.utils.data.DataLoader`: A DataLoader used for training or testing. + """ + _kwargs = kwargs.copy() + + if add_sampler and gpc.is_initialized(ParallelMode.DATA) and gpc.get_world_size(ParallelMode.DATA) > 1: + sampler = DataParallelSampler(dataset, shuffle=shuffle, drop_last=drop_last) + else: + sampler = None + + # Deterministic dataloader + def seed_worker(): + worker_seed = seed + np.random.seed(worker_seed) + torch.manual_seed(worker_seed) + random.seed(worker_seed) + + if sampler is None: + return DataLoader( + dataset, + worker_init_fn=seed_worker, + shuffle=shuffle, + drop_last=drop_last, + pin_memory=pin_memory, + num_workers=num_workers, + **_kwargs, + ) + else: + return DataLoader( + dataset, + sampler=sampler, + worker_init_fn=seed_worker, + drop_last=drop_last, + pin_memory=pin_memory, + num_workers=num_workers, + **_kwargs, + ) + + +class StaticBatchSampler: + """ + A static batch sampler that generates batches with a fixed micro-batch size. + + Args: + num_samples (int): The total number of samples in the dataset. + batch_size (int): The batch size for the current rank. Defaults to 192. + rampup_batch_size (str): A string with three space-separated integers representing the + starting batch size, the increment, and the number of steps between + each increment. For tools, "192 24 8" means that the batch size + starts at 192 and increases by 24 every 8 steps. Defaults to + "6 2 8", which corresponds to a batch size of 2 for the first 6 steps. + micro_bsz (int): The micro-batch size. Defaults to 2. + seed (int): The random seed for shuffling the indices. Defaults to 0. + drop_last (bool): If True, drop the last incomplete batch. Currently only supports True. Defaults to True. + data_rank (int): The rank of the current process in the data parallel group. Defaults to 0. + data_world_size (int): The number of processes in the data parallel group. Defaults to 1. + """ + + def __init__( + self, + datasets, + batch_size=192, + rampup_batch_size="6 2 8", + micro_bsz=2, + seed=0, + drop_last=True, + data_rank=0, + data_world_size=1, + ): + assert drop_last is True, "Currently only support drop last" + if rampup_batch_size: + # In the process increase to batch_size + start_bsz, bsz_incre, incre_every = map(int, rampup_batch_size.split()) + else: + start_bsz, bsz_incre, incre_every = batch_size, batch_size, 1 + self.raw_rampup_batch_size = rampup_batch_size + self.start_bsz = start_bsz + self.bsz_incre = bsz_incre + self.incre_every = incre_every + if gpc.is_initialized(ParallelMode.PIPELINE): + assert ( + batch_size - self.start_bsz + ) % self.bsz_incre == 0, f"{batch_size} - {self.start_bsz} should be multiple of {self.bsz_incre}" + assert batch_size % micro_bsz == 0, f"batch_size({batch_size}) should be multiple of micro_bsz({micro_bsz})" + assert ( + self.start_bsz % micro_bsz == 0 + ), f"start_bsz({self.start_bsz}) should be multiple of micro_bsz({micro_bsz})" + assert ( + self.bsz_incre % micro_bsz == 0 + ), f"bsz_incre({self.bsz_incre}) should be multiple of micro_bsz({micro_bsz})" + + self.batch_size = batch_size + self.epoch = 0 + self.seed = seed + self.rng = np.random.RandomState(seed) + self.batch_count = 0 + self.micro_bsz = micro_bsz + self.data_rank = data_rank + self.data_world_size = data_world_size + self.num_consumed_samples_in_epoch = 0 + self.datasets = datasets + self.num_samples = sum([len(ds) for ds in datasets]) + + self.get_indices() # get data + + def get_indices(self, old_indices=None): + if old_indices is not None: + assert ( + len(old_indices) <= self.num_samples + ), f"The checkpoint has {len(old_indices)} samples, \ +while the new restart use less samples ({self.num_samples})" + + else: + old_indices = np.array([]) + + # indices includes len(old_indices) but not self.num_samples + indices = np.arange(len(old_indices), self.num_samples) + self.rng_state = self.rng.get_state() + self.rng.shuffle(indices) + # Need to consider drop_last + ramp_steps = (self.batch_size - self.start_bsz) // self.bsz_incre + if self.batch_count < ramp_steps * self.incre_every: + rampup_samples = 0 + for i in range(ramp_steps): + rampup_samples += (i * self.bsz_incre + self.start_bsz) * self.incre_every + assert ( + rampup_samples * self.data_world_size <= self.num_samples + ), f"Too much rampup samples: \ +{rampup_samples*self.data_world_size} Vs. self.num_samples: {self.num_samples}" + + num_samples = (self.num_samples - rampup_samples * self.data_world_size) // ( + self.batch_size * self.data_world_size + ) + num_samples = num_samples * self.batch_size * self.data_world_size + rampup_samples * self.data_world_size + else: + num_samples = self.num_samples // (self.batch_size * self.data_world_size) + num_samples = num_samples * self.batch_size * self.data_world_size + indices = np.concatenate([old_indices, indices]).astype(int) # It needs to be spliced with the previous + indices = indices[:num_samples] + self.indices = indices + assert len(self.indices) >= self.batch_size, "The number of samples should be larger than batch_size" + self.num_consumed_samples_in_epoch = 0 + + def set_epoch(self, epoch): + self.epoch = epoch + self.rng = np.random.RandomState(self.seed + self.epoch) + + def __len__(self): + ramp_steps = (self.batch_size - self.start_bsz) // self.bsz_incre + if self.batch_count < ramp_steps * self.incre_every: + rampup_samples = 0 + for i in range(ramp_steps): + rampup_samples += (i * self.bsz_incre + self.start_bsz) * self.incre_every + assert ( + rampup_samples * self.data_world_size <= self.num_samples + ), f"Too much rampup samples: {rampup_samples*self.data_world_size} \ +Vs. self.num_samples: {self.num_samples}" + + num_batches = (self.num_samples - rampup_samples * self.data_world_size) // self.batch_size + num_batches = num_batches // self.data_world_size + self.incre_every * ramp_steps + else: + num_batches = self.num_samples // self.batch_size // self.data_world_size + + return num_batches + + def __iter__(self): + indices = self.indices[self.data_rank :: self.data_world_size] + while self.num_consumed_samples_in_epoch < len(indices): + batch_rampup_idx = self.batch_count // self.incre_every + cur_batch_size = batch_rampup_idx * self.bsz_incre + self.start_bsz + cur_batch_size = min(cur_batch_size, self.batch_size) + batch = indices[self.num_consumed_samples_in_epoch : self.num_consumed_samples_in_epoch + cur_batch_size] + yield batch + self.num_consumed_samples_in_epoch += len(batch) # Consider multiple processes. + self.batch_count += 1 + self.get_indices() # get a new round + + def state_dict(self): + states = { + "batch_size": self.batch_size, + "raw_rampup_batch_size": self.raw_rampup_batch_size, + "rng_state": self.rng_state, + "epoch": self.epoch, + "seed": self.seed, + "data_world_size": self.data_world_size, + "num_consumed_samples_in_epoch": self.num_consumed_samples_in_epoch, + "batch_count": self.batch_count, # The batch_count here is due to the existence of multiple processes, + # the batch may be oversent, and it needs to be overwritten by the external batch_count + "indices": self.indices, # The sequence used to breakpoint retraining is the same as before + } + + return states + + def load_state_dict(self, states): + for name in ("data_world_size", "raw_rampup_batch_size", "seed"): # 'batch_size' + assert states[name] == getattr(self, name), (name, states[name], getattr(self, name)) # should not change + self.rng.set_state(states["rng_state"]) + self.get_indices(old_indices=None) # Regenerate indices based on random state + self.epoch = states["epoch"] + self.batch_count = states["batch_count"] + self.num_consumed_samples_in_epoch = states["num_consumed_samples_in_epoch"] + + def copy(self): + copy_sampler = StaticBatchSampler( + self.datasets, + self.batch_size, + self.raw_rampup_batch_size, + self.micro_bsz, + self.seed, + drop_last=True, + data_rank=self.data_rank, + data_world_size=self.data_world_size, + ) + + copy_sampler.load_state_dict(self.state_dict()) + return copy_sampler diff --git a/InternLM/internlm/data/collaters.py b/InternLM/internlm/data/collaters.py new file mode 100644 index 0000000000000000000000000000000000000000..b327b544ebae09e88e54a08f5a932a93b56b2ab9 --- /dev/null +++ b/InternLM/internlm/data/collaters.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import torch + + +def packed_collate_fn(batch, packed_length): + + """ + Collate function for packed input sequences. + + Args: + batch (List[Dict]): List of dictionaries representing each sample in batch. + Each dictionary contains "tokens", "labels", "type_ids", "cu_seqlens", and "indexes" keys. + packed_length (int): The length of packed sequence. + + Returns: + Tuple[Dict[str, torch.Tensor], torch.Tensor]: A tuple containing a dictionary of tensors with "input_ids", + "cu_seqlens", "indexes", and "type_ids" keys, and the tensor of padded "labels". + + Raises: + AssertionError: If the length of a sample is not equal to packed_length. + AssertionError: If the shape of the padded "input_ids" tensor does not have the correct shape. + """ + + xs, ys, cu_seqlens, indexes, ts = [], [], [], [], [] + for b in batch: + assert ( + len(b["tokens"]) == packed_length + ), f"length of a sample should be equal to packed_length, but got {len(b['tokens'])} and {packed_length})" + assert ( + len(b["labels"]) == packed_length + ), f"length of a sample should be equal to packed_length, but got {len(b['labels'])} and {packed_length})" + assert ( + len(b["type_ids"]) == packed_length + ), f"length of a sample should be equal to packed_length, but got {len(b['type_ids'])} and {packed_length})" + + tokens = [abs(w) for w in b["tokens"]] + labels = [w if w > 0 else -100 for w in b["labels"]] + + xs.append(torch.LongTensor(tokens)) + # The labels have been shifted here, so they are aligned with the output corresponding to the token + ys.append(torch.LongTensor(labels)) + ts.append(torch.LongTensor(b["type_ids"])) + cu_seqlens.append(torch.IntTensor(b["cu_seqlens"])) + indexes.append(torch.LongTensor(b["indexes"])) + + xs = torch.nn.utils.rnn.pad_sequence(xs, batch_first=True) + ys = torch.nn.utils.rnn.pad_sequence(ys, batch_first=True, padding_value=-100) + ts = torch.nn.utils.rnn.pad_sequence(ts, batch_first=True, padding_value=0) + indexes = torch.stack(indexes, dim=0) + if len(set(map(len, cu_seqlens))) == 1: # if has uniform length, then stack to save device transfer time + cu_seqlens = torch.stack(cu_seqlens, dim=0) + + assert xs.shape[1] == packed_length, (xs.shape[1], packed_length) + + return {"input_ids": xs, "cu_seqlens": cu_seqlens, "indexes": indexes, "type_ids": ts}, ys + + +def jsonl_ds_collate_fn(batch, max_length_per_sample): + """ + Collate function for json dataset. + + Args: + batch (List[Dict]): List of dictionaries representing each sample in batch. + Each dictionary contains "tokens". + max_length_per_sample (int): The length of output sequence. + + Returns: + Tuple[Dict[str, torch.Tensor], torch.Tensor]: A tuple containing a dictionary of tensors with "input_ids", + and the tensor of padded "labels". + + """ + xs, ys = [], [] + for x in batch: + x["tokens"] = x["tokens"][:max_length_per_sample] + tokens = [abs(w) for w in x["tokens"]] + labels = [w if w > 0 else -100 for w in x["tokens"]] + labels = labels[1:] + [-100] + xs.append(torch.as_tensor(tokens)) + ys.append(torch.as_tensor(labels)) # y has been shifted + xs = torch.nn.utils.rnn.pad_sequence(xs, batch_first=True) + ys = torch.nn.utils.rnn.pad_sequence(ys, batch_first=True, padding_value=-100) + + xs = torch.cat([xs, xs.new_zeros(len(xs), max_length_per_sample - len(xs[0]))], dim=-1) + ys = torch.cat([ys, ys.new_full((len(ys), max_length_per_sample - len(ys[0])), fill_value=-100)], dim=-1) + + return {"input_ids": xs}, ys diff --git a/InternLM/internlm/data/dataset.py b/InternLM/internlm/data/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..401e510630ba8e9a1c732d541438f0592ef504d5 --- /dev/null +++ b/InternLM/internlm/data/dataset.py @@ -0,0 +1,56 @@ +import os +from typing import Dict + +from torch.utils.data import ConcatDataset + +from internlm.data.single_dataset import JsonlDataset + + +def get_dataset_dict(folder, split="valid") -> Dict: + """ + Return a dictionary of Datasets from a folder containing data files for validation. + + Args: + folder (str): The path to the folder containing data files. + split (str): The split of the data files to be used, default is "valid". + + Returns: + A dictionary containing Datasets for each folder in the given path + that contains data files with the specified split. + + Raises: + AssertionError: If the given folder does not exist. + + Example: + If the given folder is as follows, + - data + - zhihu + - xxx.bin + - valid.bin + - baike + - xxx.bin + - valid.bin + + The returned dictionary will be, + { + 'zhihu': Dataset, + 'baike': Dataset + } + """ + + assert os.path.exists(folder), f"folder `{folder}` not exists" + data_dict = {} + + for root, dirs, files in os.walk(folder, followlinks=True): + dirs.sort() # The order is guaranteed, and the newly added data starting with z needs to be ranked behind + datasets = [] + for fn in sorted(files): # Need sorted to ensure that the order is consistent + if fn.endswith(".bin") and split in fn: + fp = os.path.join(root, fn) + ds = JsonlDataset(fp) + datasets.append(ds) + if datasets: + ds = ConcatDataset(datasets=datasets) + data_dict[os.path.basename(root)] = ds + + return data_dict diff --git a/InternLM/internlm/data/dummy_dataset.py b/InternLM/internlm/data/dummy_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..fb361848bea57f0cbd136d6c3c9eb8294d7f0f3e --- /dev/null +++ b/InternLM/internlm/data/dummy_dataset.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import numpy as np +from torch.utils.data import Dataset + + +class RandomDataset(Dataset): + """ + RandomDataset for generating random dataset. + + Args: + num_samples (int): The number of samples to generate. + max_len (int): The maximum length of each sample. + + """ + + def __init__(self, num_samples=10000, max_len=1024) -> None: + super().__init__() + rng = np.random.RandomState(1999) + max_num = rng.randint(1, 30, size=(num_samples,)) + rep_num = rng.randint(10, 200, size=(num_samples,)) + data = [] + lengths = [] + for n, r in zip(max_num, rep_num): + d = list(range(n)) * r + d = [n, r] + d + d = d[:max_len] + data.append(d) + lengths.append(len(d)) + self.data = data + self.max_len = max_len + self.lengths = np.array(lengths, dtype=int) + + def __getitem__(self, index): + d = self.data[index] + input_ids = np.array(d, dtype=int) + return {"tokens": list(input_ids), "type_id": 0} + + def get_dataset_name(self): + return "dummy_path/dummy_lang/dummy_ds/train.bin" + + def __len__(self): + return len(self.data) diff --git a/InternLM/internlm/data/packed_dataset.py b/InternLM/internlm/data/packed_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..98d256d026b6d57e04b74767bc10e3734a8a365c --- /dev/null +++ b/InternLM/internlm/data/packed_dataset.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import itertools as it +import operator +import os +from copy import deepcopy +from typing import Dict + +import numpy as np +import torch +from torch.utils.data import ConcatDataset +from tqdm import tqdm + +from internlm.core.context import global_context as gpc +from internlm.data.single_dataset import JsonlDataset +from internlm.data.utils import get_dataset_type_id +from internlm.utils.logger import get_logger + +DEFAULT_SEED = 1024 +logger = get_logger(__file__) + + +class PackedDataset(torch.utils.data.Dataset): + """ + The class PackedDataset takes in a dataset and aggregates samples of different + lengths together based on the packed_length. + + Args: + dataset: The original dataset to pack. + max_length_per_sample: The maximum length of each original sample. Default is 2048. + packed_length: The length of each packed sample. Default is 4096. + """ + + def __init__( + self, + dataset, + max_length_per_sample: int = 2048, + packed_length: int = 4096, + ): + assert hasattr(dataset, "lengths") + assert len(getattr(dataset, "lengths")) == len( + dataset + ), "The dataset must have lengths attribute and have the same length as the dataset" + self.dataset = dataset + self.max_length_per_sample = max_length_per_sample + self.lengths = getattr(self.dataset, "lengths") + self.packed_length = packed_length + # Force a seed to be fixed to prevent problems caused by the seed not being restored when restarting + + self.seed = DEFAULT_SEED + self.sample_indices, self.len_samples_shuffled, self.acm_len_samples = self.accu_sample_len(seed=self.seed) + self.num_tokens = sum(self.lengths) + + def get_dataset_name(self): + return self.dataset.get_dataset_name() + + def accu_sample_len(self, seed=None): + """accumulative length of samples""" + if seed is not None: + rng = np.random.RandomState(seed) + else: + rng = np.random.RandomState(self.seed - 1) + + sample_indices = np.arange(len(self.lengths)) + rng.shuffle(sample_indices) + len_samples_shuffled = list(map(self.lengths.__getitem__, sample_indices)) + acm_len_samples = list(it.accumulate(len_samples_shuffled, operator.add)) + return sample_indices, len_samples_shuffled, acm_len_samples + + def __len__(self): + # Line 405 of document_to_sequence.py in metaseq is directly spliced, + # without additional consideration of sos or eos + n_packs = self.num_tokens // self.packed_length + return n_packs + + def cal_map(self, carriage_idx: int = 0): + assert carriage_idx >= 0 + length_train = (carriage_idx + 1) * self.packed_length + post_pos = np.searchsorted(self.acm_len_samples, length_train, side="left") + return post_pos + + def mapping(self, pack_idx: int = 0): + # pack_idx is zero-based + pre_pos, pre_token_id = 0, 0 + if pack_idx > 0: + pre_pos = self.cal_map(pack_idx - 1) + pre_token_id = self.len_samples_shuffled[pre_pos] - ( + self.acm_len_samples[pre_pos] - (pack_idx) * self.packed_length + ) + if pre_token_id == self.len_samples_shuffled[pre_pos]: + pre_pos += 1 + pre_token_id = 0 + + pos = self.cal_map(pack_idx) + token_id = self.len_samples_shuffled[pos] - (self.acm_len_samples[pos] - (pack_idx + 1) * self.packed_length) + return pre_pos, pre_token_id, pos, token_id + + def build_pack(self, pre_pos: int, pre_token_id: int, pos: int, token_id: int): + pack, cu_seqlens, indexes, labels, type_ids = [], [0], [], [], [] + + while pre_pos < pos: + sample_idx = self.sample_indices[pre_pos] + sample = self.dataset[sample_idx] + chunk = sample["tokens"][pre_token_id:] + pack.extend(chunk) + _labels = deepcopy(chunk) + _labels = list(_labels[1:]) + [-100] + assert len(_labels) == len(chunk), (_labels, chunk) + labels.extend(_labels) + type_ids.extend([sample.get("type_id", 0)] * len(chunk)) + num_new_samples, tokens_left = divmod(len(chunk), self.max_length_per_sample) + for _ in range(num_new_samples): + cu_seqlens.append(cu_seqlens[-1] + self.max_length_per_sample) + indexes.extend(list(range(self.max_length_per_sample))) + if tokens_left > 0: + cu_seqlens.append(cu_seqlens[-1] + tokens_left) + indexes.extend(list(range(tokens_left))) + pre_pos = pre_pos + 1 + pre_token_id = 0 + + sample_idx = self.sample_indices[pos] + sample = self.dataset[sample_idx] + chunk = sample["tokens"][pre_token_id:token_id] # fragement of a sample + pack.extend(chunk) + _labels = deepcopy(chunk) + if token_id == len(sample["tokens"]): + _labels = list(_labels[1:]) + [-100] + else: + if token_id > len(sample["tokens"]): + print(f"token_id {token_id}, len of sample {len(sample['tokens'])}") + _labels = list(_labels[1:]) + [sample["tokens"][token_id]] + assert len(_labels) == len(chunk), (_labels, chunk) + labels.extend(_labels) + type_ids.extend([sample.get("type_id", 0)] * len(chunk)) + num_new_samples, tokens_left = divmod(len(chunk), self.max_length_per_sample) + for _ in range(num_new_samples): + cu_seqlens.append(cu_seqlens[-1] + self.max_length_per_sample) + indexes.extend(list(range(self.max_length_per_sample))) + if tokens_left > 0: + cu_seqlens.append(cu_seqlens[-1] + tokens_left) + indexes.extend(list(range(tokens_left))) + + out = {"tokens": pack, "cu_seqlens": cu_seqlens, "indexes": indexes, "labels": labels, "type_ids": type_ids} + return out + + def cal_pos_unpack(self, index): + if index == 0: + pre_pos = 0 + else: + pre_pos = index * gpc.config.data["micro_bsz"] + + pos = (index + 1) * gpc.config.data["micro_bsz"] + return pre_pos, pos + + def build_unpack(self, index): + + pre_pos, pos = self.cal_pos_unpack(index) + + pack, cu_seqlens, indexes, labels, type_ids = [], [0], [], [], [] + + while pre_pos < pos and pre_pos < len(self.dataset): + sample_idx = self.sample_indices[pre_pos] + sample = self.dataset[sample_idx] + length = min(len(sample["tokens"]), self.max_length_per_sample) + chunk = sample["tokens"][0:length] + pack.extend(chunk) + _labels = deepcopy(chunk) + _labels = list(_labels[1:]) + [-100] + assert len(_labels) == len(chunk), (_labels, chunk) + labels.extend(_labels) + type_ids.extend([sample.get("type_id", 0)] * len(chunk)) + cu_seqlens.append(cu_seqlens[-1] + len(chunk)) + indexes.extend(list(range(length))) + pre_pos = pre_pos + 1 + + if cu_seqlens[-1] != self.packed_length: + pack = pack + [0] * (self.packed_length - cu_seqlens[-1]) + labels = labels + [0] * (self.packed_length - cu_seqlens[-1]) + type_ids = type_ids + [0] * (self.packed_length - cu_seqlens[-1]) + indexes.extend(list(range(self.packed_length - cu_seqlens[-1]))) + cu_seqlens.append(self.packed_length) + + assert len(pack) == self.packed_length + + out = {"tokens": pack, "cu_seqlens": cu_seqlens, "indexes": indexes, "labels": labels, "type_ids": type_ids} + return out + + def __getitem__(self, item: int) -> Dict: + """Given the index, it returns a dict as + { + 'tokens': List[int], + 'cu_seqlens': List[int], + 'indexes': List[int], # denotes positional vector as 'tokens' + 'labels': List[int], # corresponds to 'tokens' and shifted yet, -100 means skipping prediction + } + """ + + if gpc.config.model.use_flash_attn: + pos_before, token_id_before, pos_after, token_id_after = self.mapping(item) + return self.build_pack(pos_before, token_id_before, pos_after, token_id_after) + + return self.build_unpack(item) + + +class PackedDatasetWithoutCuSeqlen(torch.utils.data.Dataset): + """ + A dataset wrapper that aggregates samples with different lengths based on packed_length. + If a sample is shorter than max_length_per_sample, it will be merged with other samples. + For tools, given a dataset with 10 samples: + [1, 2, 3, 4, 5] + [6, 7] + [8, 9, 10, 11] + [12, ..., 100] + ... + + Args: + dataset: The original dataset to be wrapped. + max_length_per_sample (int): The maximum length allowed for each sample. + packed_length (int): The desired length for each packed sample. + """ + + def __init__( + self, + dataset, + max_length_per_sample: int = 2048, + packed_length: int = 4096, + debug=False, + ): + assert packed_length % max_length_per_sample == 0 + assert hasattr(dataset, "lengths") + assert len(getattr(dataset, "lengths")) == len( + dataset + ), "The dataset must have lengths attribute and have the same length as the dataset" + self.dataset = dataset + self.max_length_per_sample = max_length_per_sample + self.lengths = getattr(self.dataset, "lengths") + self.bsz = packed_length // max_length_per_sample + self.packed_length = packed_length + self.debug = debug + # Force a seed to be fixed to prevent problems caused by the seed not being restored when restarting + + self.seed = DEFAULT_SEED + indices = np.arange(len(self.lengths)) + rng = np.random.RandomState(self.seed) + rng.shuffle(indices) + self.indices = indices + self.cum_lens = np.cumsum(self.lengths[self.indices]) + self.num_tokens = sum(self.lengths) + + def get_dataset_name(self): + return self.dataset.get_dataset_name() + + def __len__(self): + n_packs = self.num_tokens // self.packed_length + return n_packs + + def find_offset(self, offset): + idx = np.searchsorted(self.cum_lens, offset, side="right") + if idx == 0: + return idx, offset + length = offset - self.cum_lens[idx - 1] + return idx, length + + def pdebug(self, line): + if self.debug: + print(line, flush=True) + + def __getitem__(self, item: int) -> Dict: + """Given the index, it returns a dict as + { + 'tokens': List[int], + 'cu_seqlens': List[int], + 'indexes': List[int], # denotes positional vector as 'tokens' + 'labels': List[int], # corresponds to 'tokens' and shifted yet, -100 means skipping prediction + } + """ + + start_idx, start_length = self.find_offset(item * self.packed_length) + end_idx, end_length = self.find_offset((item + 1) * self.packed_length) + pack_tokens = [] + pack_labels = [] + type_ids = [] + + self.pdebug(f"item : {item}, start_idx:{start_idx}, start_length:{start_length} ") + self.pdebug(f"item : {item}, end_idx:{end_idx}, end_length:{end_length} ") + + if start_idx == end_idx: + idx = self.indices[start_idx] + sample = self.dataset[idx] + self.pdebug(f"item : {item}, idx: {idx}, len : {len(sample['tokens'])}") + tokens = sample["tokens"][start_length:end_length] + pack_tokens.extend(tokens) + pack_labels.extend(tokens[1:] + [-100]) + type_ids.extend([sample["type_id"]] * len(tokens)) + return { + "tokens": pack_tokens, + "cu_seqlens": [i * self.max_length_per_sample for i in range(self.bsz + 1)], + "indexes": list(range(self.max_length_per_sample)) * self.bsz, + "labels": pack_labels, + "type_ids": type_ids, + } + + idx = self.indices[start_idx] + sample = self.dataset[idx] + self.pdebug(f"item : {item}, idx: {idx}, len : {len(sample['tokens'])}") + tokens = sample["tokens"][start_length:] + pack_tokens.extend(tokens) + pack_labels.extend(tokens[1:] + [-100]) + type_ids.extend([sample["type_id"]] * len(tokens)) + + for i in range(start_idx + 1, end_idx): + idx = self.indices[i] + sample = self.dataset[idx] + self.pdebug(f"item : {item}, idx: {idx}, len : {len(sample['tokens'])}") + tokens = sample["tokens"] + pack_tokens.extend(tokens) + pack_labels.extend(tokens[1:] + [-100]) + type_ids.extend([sample.get("type_id")] * len(tokens)) + + # corner case, the last sample is useless + if end_length == 0: + pass + else: + idx = self.indices[end_idx] + sample = self.dataset[idx] + self.pdebug(f"item : {item}, idx: {idx}, len : {len(sample['tokens'])}") + tokens = sample["tokens"][:end_length] + pack_tokens.extend(tokens) + pack_labels.extend(tokens[1:] + [-100]) + type_ids.extend([sample.get("type_id")] * len(tokens)) + + return { + "tokens": pack_tokens, + "cu_seqlens": [i * self.max_length_per_sample for i in range(self.bsz + 1)], + "indexes": list(range(self.max_length_per_sample)) * self.bsz, + "labels": pack_labels, + "type_ids": type_ids, + } + + +def get_packed_dataset_without_short_length( + folder, + max_length_per_sample=2048, + packed_length=4096, + show_progress=False, + min_length=50, + min_length_dict=None, + pack_into_one_sample=False, +): + """ + Given a folder, combine all the .bin files into a single large dataset. + And filter out short samples with length less than 'min_length'. + + Each .bin file is treated as a separate dataset. + + Args: + folder (str): Path to the folder containing the .bin files. + max_length_per_sample (int): Maximum length of each sample. + packed_length (int): Length to pack samples to. + show_progress (bool): Whether to show the progress bar. + min_length (int): The minimum length of the sample. + min_length_dict (dict): The minimum length of the sample for each dataset. + The format is something like {'pile-arxiv': 50} + dataset_backend (Optional[str]): Dataset storage location. Optional parameters are local, local-shm, kv + + Returns: + A packed dataset containing all the data from the .bin files. + """ + + assert os.path.exists(folder), f"{folder} does not exist." + datasets = [] + delete_samples = 0 + + for root, dirs, files in os.walk(folder, followlinks=True): + dirs.sort() # Let the folder need to be returned in a fixed order + if gpc.is_rank_for_log(): + logger.info(f"Reading {root}...") + num_token_in_folder = 0 + + for fn in tqdm(sorted(files), total=len(files), leave=False, disable=not show_progress): + if fn.endswith(".bin"): + fp = os.path.join(root, fn) + catch_ml_keys = [] + min_length_num = min_length + if min_length_dict is not None: + for k, v in min_length_dict.items(): + if k in fp: + min_length_num = v + catch_ml_keys.append(k) + assert ( + len(catch_ml_keys) < 2 + ), f"The file name `{fp}` matched the following resample keys:{catch_ml_keys}" + + ds_type_id = get_dataset_type_id(path=fp) + ds = JsonlDataset(fp, ds_type_id, min_length=min_length_num) + + if hasattr(ds, "old_length"): + delete_samples += ds.old_length - len(ds) + if len(ds) == 0: + if gpc.is_rank_for_log(): + logger.info(f"None of the data in `{fp}` is longer than {min_length}") + continue + + if pack_into_one_sample: + ds = PackedDatasetWithoutCuSeqlen(ds, max_length_per_sample, packed_length) + else: + ds = PackedDataset(ds, max_length_per_sample, packed_length) + + num_token_in_folder += len(ds) * packed_length + datasets.append(ds) + + dataset = ConcatDataset(datasets=datasets) + if gpc.is_rank_for_log(): + logger.info( + f"Find `{len(datasets)}` datasets, \ + {len(dataset)} samples, \ + delete `{delete_samples}` because of short length", + ) + + return dataset diff --git a/InternLM/internlm/data/single_dataset.py b/InternLM/internlm/data/single_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..5477d34ce2686d82ff828aa7ee9f1e5da633ab19 --- /dev/null +++ b/InternLM/internlm/data/single_dataset.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +""" +A .bin file corresponds to a Dataset instance here. +""" + +import json +import mmap +import os +import threading +from pathlib import Path + +import numpy as np +import torch + + +class JsonlDataset(torch.utils.data.Dataset): + """ + + JSONL format is expected to roughly follow that of The Pile. + One-line-per-document of the form: + ``` + { + "tokens": List[int], + } + ``` + + Note that only the "tokens" key is used. + """ + + def __init__(self, path: str, dataset_type_id: int = 0, min_length=50): + self.path = path + self.threadlocal = threading.local() + resolved_path = Path(path).resolve() + self.resolved_path = resolved_path + self.meta = Path(f"{resolved_path}.meta") + self.type_id = dataset_type_id + + # only build the cache in on the primary worker to prevent overloading nfs + assert os.path.exists(self.meta), f"The cache file:{self.meta} is not found for file:{self.path}" + try: + with open(self.meta, "rb") as f: + meta = np.load(f) + except Exception as e: + print(f"Cannot load file {self.meta}...") + raise e + self.offsets = meta[:, 0] + self.lengths = meta[:, -1] + + if min_length > 0: + mask = self.lengths >= min_length + self.old_lengths = self.lengths.copy() + self.old_length = len(self.offsets) + self.offsets = self.offsets[mask] + self.lengths = self.lengths[mask] + + def __getitem__(self, idx): + f = self._get_mmap() + position = self.offsets[idx] + f.seek(position) + item = f.readline().decode("utf-8") + try: + item = json.loads(item) + item["length"] = len(item["tokens"]) # add a length info + item["type_id"] = self.type_id + except Exception as err: + raise json.decoder.JSONDecodeError( + doc=self.path, + pos=position, + msg=( + f"Error while loading JSONL line in file {self.path} at byte " + f"{position}. Contents of line:\n{item}\n{err}" + ), + ) + return item + + def get_dataset_name(self): + return str(self.resolved_path) + + def _get_mmap(self): + if not hasattr(self.threadlocal, "handles"): + with open(self.path, "rb") as f: + mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + self.threadlocal.handles = [f, mm] + if self.path.endswith(".gz") or self.path.endswith(".bz") or self.path.endswith(".bz2"): + raise NotImplementedError( + "Compressed files are not supported because .seek() would require " + "rereading the entire file, making performance too slow." + ) + return self.threadlocal.handles[-1] + + def __setstate__(self, state): + self.__dict__ = state + self.threadlocal = threading.local() + + def __getstate__(self): + d = {} + for i, v in self.__dict__.items(): + if i != "threadlocal": + d[i] = v + return d + + def __del__(self): + if hasattr(self.threadlocal, "handles"): + # cleanup files we opened on initialization + while self.threadlocal.handles: + self.threadlocal.handles.pop().close() + + @staticmethod + def exists(path): + return os.path.exists(path) + + def __len__(self): + # Virtual length of the dataset depends on the epoch number if the number of documents + # is not perfectly divisible by the data_subshard_count + return len(self.offsets) diff --git a/InternLM/internlm/data/utils.py b/InternLM/internlm/data/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..00d8fd9ff92459175cc6b9597e3ff103a9e8a138 --- /dev/null +++ b/InternLM/internlm/data/utils.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import torch + +from internlm.core.context import global_context as gpc + +DATASET_TYPE_IDS_MAP = {"vision": 0} + + +def get_dataset_type_id(path): + import re + + match_idxes = [] + for key, idx in DATASET_TYPE_IDS_MAP.items(): + if re.search(rf"/[z_]*{key}/", path): + match_idxes.append(idx) + assert len(match_idxes) == 1, f"{path}, match_idxes should be 1, but got {match_idxes} from {DATASET_TYPE_IDS_MAP}" + return match_idxes[0] + + +def unpack_data(input_ids, cu_seqlens): + """ + input_ids: (n, packed_length) + Return: + output: (batch_size, max_length) + """ + + bsz = input_ids.shape[0] + + num_sequence = gpc.config.data["micro_bsz"] + + outputs = torch.zeros(bsz, num_sequence, gpc.config.data.seq_len, device=input_ids.device, dtype=input_ids.dtype) + + for i in range(bsz): + output = torch.zeros(num_sequence, gpc.config.data.seq_len, device=input_ids.device, dtype=input_ids.dtype) + cu_seqlens_slice = cu_seqlens[i] + for j in range(num_sequence): + seq_length = cu_seqlens_slice[j + 1] - cu_seqlens_slice[j] + output[j, 0:seq_length] = input_ids[0, cu_seqlens_slice[j] : cu_seqlens_slice[j + 1]] + outputs[i] = output + + if bsz == 1: + outputs = outputs.squeeze(0) + + return outputs diff --git a/InternLM/internlm/initialize/__init__.py b/InternLM/internlm/initialize/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..395457845872159f460a2db4d36683cd95ab56da --- /dev/null +++ b/InternLM/internlm/initialize/__init__.py @@ -0,0 +1,16 @@ +from .initialize_trainer import initialize_trainer, initialize_kd_trainer +from .launch import ( + get_default_parser, + initialize_distributed_env, + launch_from_slurm, + launch_from_torch, +) + +__all__ = [ + "get_default_parser", + "initialize_trainer", + "initialize_kd_trainer", + "launch_from_slurm", + "launch_from_torch", + "initialize_distributed_env", +] diff --git a/InternLM/internlm/initialize/initialize_tensor.py b/InternLM/internlm/initialize/initialize_tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..b317f2697765ac1c99331e42e310088ae692235f --- /dev/null +++ b/InternLM/internlm/initialize/initialize_tensor.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import math + +from torch import Tensor, nn + + +def scaled_init_method_normal(sigma: float = 1.0, num_layers: int = 1): + """Init method based on N(0, sigma/sqrt(2*num_layers).""" + std = sigma / math.sqrt(2.0 * num_layers) + + def init_(tensor): + return nn.init.normal_(tensor, mean=0.0, std=std) + + return init_ + + +def normal_(mean: float = 0.0, std: float = 1.0): + r"""Return the initializer filling the input Tensor with values drawn from the normal distribution + + .. math:: + \mathcal{N}(\text{mean}, \text{std}^2) + + Args: + mean (float): the mean of the normal distribution. Defaults 0.0. + std (float): the standard deviation of the normal distribution. Defaults 1.0. + """ + + def initializer(tensor: Tensor): + return nn.init.normal_(tensor, mean, std) + + return initializer + + +def scaled_init_method_uniform(sigma: float = 1.0, num_layers: int = 1): + """Init method based on p(x)=Uniform(-a, a) where std(x)=sigma/sqrt(2*num_layers).""" + std = sigma / math.sqrt(2.0 * num_layers) + a = math.sqrt(3.0 * std) + + def init_(tensor): + return nn.init.uniform_(tensor, -a, a) + + return init_ + + +def uniform_(mean: float = 0.0, std: float = 1.0): + r"""Return the initializer filling the input Tensor with values drawn from the uniform distribution + + .. math:: + \mathcal{U}(mean-a, mean+a), where a satisfies \mathcal{U}_{std}=std. + + Args: + mean (float): the mean of the uniform distribution. Defaults 0.0. + std (float): the standard deviation of the uniform distribution. Defaults 1.0. + """ + + a = math.sqrt(3.0 * std) + + def initializer(tensor: Tensor): + return nn.init.uniform_(tensor, mean - a, mean + a) + + return initializer diff --git a/InternLM/internlm/initialize/initialize_trainer.py b/InternLM/internlm/initialize/initialize_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..64b633045d32b7e8c8f3536f22c0d5d2c14580d9 --- /dev/null +++ b/InternLM/internlm/initialize/initialize_trainer.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# adopted from https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/initialize + +from typing import Callable, Iterable, List, Optional, Tuple + +from torch import nn +from torch.nn.modules.loss import _Loss +from torch.optim.lr_scheduler import _LRScheduler +from torch.optim.optimizer import Optimizer +from torch.utils.data import DataLoader + +from internlm.core.context import global_context as gpc +from internlm.core.context import ParallelMode +from internlm.core.engine import Engine, KDEngine +from internlm.core.gradient_handler import PipelineSharedModuleGradientHandler +from internlm.core.scheduler import (InterleavedPipelineScheduler, KDNonPipelineScheduler, KDPipelineScheduler, + NonPipelineScheduler, PipelineScheduler, SchedulerHook) +from internlm.core.scheduler.pipeline_scheduler import get_tensor_shape +from internlm.core.trainer import Trainer +from internlm.data.utils import unpack_data +from internlm.solver.beta2_scheduler import Beta2Scheduler +from internlm.solver.optimizer.hybrid_zero_optim import BaseOptimizer +from internlm.utils.common import get_current_device + + +def initialize_kd_trainer( + model: nn.Module, + teacher: nn.Module, + optimizer: Optimizer, + criterion: Optional[_Loss] = None, + kd_criterion: Optional[_Loss] = None, + train_dataloader: Optional[Iterable] = None, + test_dataloader: Optional[Iterable] = None, + lr_scheduler: Optional[_LRScheduler] = None, + beta2_scheduler: Optional[Beta2Scheduler] = None, + scheduler_hooks: Optional[List[SchedulerHook]] = None, +) -> Tuple[Trainer, DataLoader, DataLoader, _LRScheduler]: + """Core function to wrap the essential training components with our functionality based on the config which is + loaded into gpc.config. + + Args: + model (:class:`torch.nn.Module` or `Callable`): Your model instance or a function to build the model. + optimizer (:class:`BaseOptimizer`): Your optimizer for training. + criterion (:class:`torch.nn.modules.loss._Loss`, optional): Your criterion instance. + train_dataloader (:class:`torch.utils.data.DataLoader`, optional): Dataloader for training. + test_dataloader (:class:`torch.utils.data.DataLoader`, optional): Dataloader for testing. + lr_scheduler (:class:`torch.nn.lr_scheduler._LRScheduler`, optional): Your lr scheduler instance, optional. + + Returns: + Tuple (trainer, train_dataloader, test_dataloader, lr_scheduler): + A tuple of ``(trainer, train_dataloader, test_dataloader, lr_scheduler)`` + where only ``trainer`` could not be None. + """ + + if isinstance(model, nn.Module): + # first sync model across dp ranks + model.to(get_current_device()) + elif isinstance(model, Callable): + model = model().to(get_current_device()) + + # clip grad norm + clip_grad_norm = gpc.config.hybrid_zero_optimizer.get("clip_grad_norm", 0.0) + + assert isinstance(optimizer, BaseOptimizer), "optimizer must be instance of BaseOptimizer" + + # gradient handler, only support PipelineSharedModuleGradientHandler now + if gpc.is_using_pp(): + gpc.config.gradient_handler = [dict(type="PipelineSharedModuleGradientHandler")] + gradient_handler_cfg = gpc.config.get("gradient_handler", []) + gradient_handlers = [] + assert isinstance(gradient_handler_cfg, list), f"gradient_handler must be list but got {type(gradient_handler_cfg)}" + for config in gradient_handler_cfg: + if isinstance(config, dict) and config.get("type") == "PipelineSharedModuleGradientHandler": + handler = PipelineSharedModuleGradientHandler(model=model, optimizer=optimizer) + gradient_handlers.append(handler) + + # initialize scheduler for trainer + scheduler = None + if gpc.config.model.use_flash_attn: + data_fn = None + else: + data_fn = unpack_data + if gpc.is_using_pp(): + gpc.config.NUM_MICRO_BATCHES = gpc.config.data.micro_num + tensor_shape = get_tensor_shape() + use_interleaved = ( + hasattr(gpc.config, "model") and hasattr(gpc.config.model, + "num_chunks") and gpc.config.model.num_chunks > 1 + ) + scatter_gather = gpc.is_initialized(ParallelMode.TENSOR) + if use_interleaved: + raise NotImplementedError('InterleavedPipelineScheduler for KD is not implemented') + + else: + scheduler = KDPipelineScheduler( + data_process_func=data_fn, + num_microbatches=gpc.config.NUM_MICRO_BATCHES, + dtype=gpc.config.model["dtype"], + tensor_shape=tensor_shape, + scatter_gather_tensors=scatter_gather, + scheduler_hooks=scheduler_hooks, + ) + else: + scheduler = KDNonPipelineScheduler( + data_process_func=data_fn, + gradient_accumulation_size=gpc.config.data.gradient_accumulation, + scheduler_hooks=scheduler_hooks, + ) + + # initialize engine for trainer + engine = KDEngine( + model=model, + teacher=teacher, + optimizer=optimizer, + lr_scheduler=lr_scheduler, + beta2_scheduler=beta2_scheduler, + criterion=criterion, + kd_criterion=kd_criterion, + gradient_handlers=gradient_handlers, + clip_grad_norm=clip_grad_norm, + ) + + trainer = Trainer(engine, scheduler) + + return trainer, train_dataloader, test_dataloader, lr_scheduler + + +def initialize_trainer( + model: nn.Module, + optimizer: Optimizer, + criterion: Optional[_Loss] = None, + train_dataloader: Optional[Iterable] = None, + test_dataloader: Optional[Iterable] = None, + lr_scheduler: Optional[_LRScheduler] = None, + beta2_scheduler: Optional[Beta2Scheduler] = None, + scheduler_hooks: Optional[List[SchedulerHook]] = None, +) -> Tuple[Trainer, DataLoader, DataLoader, _LRScheduler]: + """Core function to wrap the essential training components with our functionality based on the config which is + loaded into gpc.config. + + Args: + model (:class:`torch.nn.Module` or `Callable`): Your model instance or a function to build the model. + optimizer (:class:`BaseOptimizer`): Your optimizer for training. + criterion (:class:`torch.nn.modules.loss._Loss`, optional): Your criterion instance. + train_dataloader (:class:`torch.utils.data.DataLoader`, optional): Dataloader for training. + test_dataloader (:class:`torch.utils.data.DataLoader`, optional): Dataloader for testing. + lr_scheduler (:class:`torch.nn.lr_scheduler._LRScheduler`, optional): Your lr scheduler instance, optional. + + Returns: + Tuple (trainer, train_dataloader, test_dataloader, lr_scheduler): + A tuple of ``(trainer, train_dataloader, test_dataloader, lr_scheduler)`` + where only ``trainer`` could not be None. + """ + + if isinstance(model, nn.Module): + # first sync model across dp ranks + model.to(get_current_device()) + elif isinstance(model, Callable): + model = model().to(get_current_device()) + + # clip grad norm + clip_grad_norm = gpc.config.hybrid_zero_optimizer.get("clip_grad_norm", 0.0) + + assert isinstance(optimizer, BaseOptimizer), "optimizer must be instance of BaseOptimizer" + + # gradient handler, only support PipelineSharedModuleGradientHandler now + if gpc.is_using_pp(): + gpc.config.gradient_handler = [dict(type="PipelineSharedModuleGradientHandler")] + gradient_handler_cfg = gpc.config.get("gradient_handler", []) + gradient_handlers = [] + assert isinstance(gradient_handler_cfg, list), f"gradient_handler must be list but got {type(gradient_handler_cfg)}" + for config in gradient_handler_cfg: + if isinstance(config, dict) and config.get("type") == "PipelineSharedModuleGradientHandler": + handler = PipelineSharedModuleGradientHandler(model=model, optimizer=optimizer) + gradient_handlers.append(handler) + + # initialize scheduler for trainer + scheduler = None + if gpc.config.model.use_flash_attn: + data_fn = None + else: + data_fn = unpack_data + if gpc.is_using_pp(): + gpc.config.NUM_MICRO_BATCHES = gpc.config.data.micro_num + tensor_shape = get_tensor_shape() + use_interleaved = ( + hasattr(gpc.config, "model") and hasattr(gpc.config.model, "num_chunks") and gpc.config.model.num_chunks > 1 + ) + scatter_gather = gpc.is_initialized(ParallelMode.TENSOR) + if use_interleaved: + if isinstance(model, nn.Sequential): + model = nn.ModuleList([model]) + + communication_overlap = gpc.config.parallel["pipeline"].get("interleaved_overlap", False) + scheduler = InterleavedPipelineScheduler( + num_microbatches=gpc.config.NUM_MICRO_BATCHES, + num_chunks=gpc.config.model.num_chunks, + dtype=gpc.config.model["dtype"], + tensor_shape=tensor_shape, + scatter_gather_tensors=scatter_gather, + scheduler_hooks=scheduler_hooks, + communication_overlap=communication_overlap, + ) + else: + scheduler = PipelineScheduler( + data_process_func=data_fn, + num_microbatches=gpc.config.NUM_MICRO_BATCHES, + dtype=gpc.config.model["dtype"], + tensor_shape=tensor_shape, + scatter_gather_tensors=scatter_gather, + scheduler_hooks=scheduler_hooks, + ) + else: + scheduler = NonPipelineScheduler( + data_process_func=data_fn, + gradient_accumulation_size=gpc.config.data.gradient_accumulation, + scheduler_hooks=scheduler_hooks, + ) + + # initialize engine for trainer + engine = Engine( + model=model, + optimizer=optimizer, + lr_scheduler=lr_scheduler, + beta2_scheduler=beta2_scheduler, + criterion=criterion, + gradient_handlers=gradient_handlers, + clip_grad_norm=clip_grad_norm, + ) + + trainer = Trainer(engine, scheduler) + + return trainer, train_dataloader, test_dataloader, lr_scheduler diff --git a/InternLM/internlm/initialize/launch.py b/InternLM/internlm/initialize/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..8f38204bf5dbbc451a987067f9fe6a9cee5519d0 --- /dev/null +++ b/InternLM/internlm/initialize/launch.py @@ -0,0 +1,511 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import argparse +import os +from pathlib import Path +from typing import Dict, Union + +import torch + +from internlm.core.context import Config +from internlm.core.context import global_context as gpc +from internlm.monitor import initialize_light_monitor +from internlm.utils.common import get_master_node +from internlm.utils.logger import get_logger +from internlm.utils.timeout import llm_timeout + +logger = get_logger(__file__) + + +def get_default_parser(): + """Reads user command line and uses an argument parser to parse the input arguments. + Input arguments include configuration, host, port, world size, local rank, backend for torch.distributed. + + Returns: + Parser: Returns the parser with the default arguments, the user may add customized arguments into this parser. + """ + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=str, help="path to the config file") + parser.add_argument( + "--launcher", + type=str, + default="slurm", + choices=["slurm", "torch"], + help="launcher for launching distributed environment", + ) + parser.add_argument("--host", type=str, help="the master address for distributed training") + parser.add_argument("--port", type=int, default=8888, help="the master port for distributed training") + parser.add_argument("--world_size", type=int, help="world size for distributed training") + parser.add_argument("--rank", type=int, help="rank for the default process group") + parser.add_argument("--local_rank", type=int, help="local rank on the node") + parser.add_argument("--backend", type=str, default="nccl", help="backend for distributed communication") + parser.add_argument("--seed", type=int, default=1024) + parser.add_argument("--profiling", default=False, action="store_true", help="enable/disable profiling.") + return parser + + +def args_sanity_check(): + assert gpc.config is not None, "config is not load!" + + # the default model type is INTERNLM + if "model_type" not in gpc.config: + gpc.config._add_item("model_type", "INTERNLM") + + # procssing the parallel config in gpc + if "zero1" not in gpc.config.parallel: + gpc.config.parallel._add_item("zero1", -1) + + if "pipeline" not in gpc.config.parallel: + gpc.config.parallel._add_item("pipeline", 1) + + if "tensor" not in gpc.config.parallel: + gpc.config.parallel._add_item("tensor", 1) + + # processing the data config in gpc + data = gpc.config.data + + assert data.seq_len is not None, "'seq_len' must be given a value" + assert data.micro_bsz is not None, "'micro_bsz' must be given a value" + + if "packed_length" in data and gpc.is_rank_for_log(): + logger.warning("packed_length would be ignored and will be setted as seq_len * micro_bsz.") + + data._add_item("packed_length", data.seq_len * data.micro_bsz) + + if "micro_num" not in data: + data._add_item("micro_num", 1) + + data._add_item("gradient_accumulation", data.micro_num) + if gpc.is_rank_for_log(): + logger.info(f"gradient_accumulation size will be setted to {data.micro_num}.") + + # batch_size should be equal with micro_num, should not use it directly + data._add_item("batch_size", data.micro_num) + + if "min_length" not in data: + data._add_item("min_length", 0) + + if "train_folder" not in data: + data._add_item("train_folder", None) + + if "valid_folder" not in data: + data._add_item("valid_folder", None) + + if "valid_micro_num" not in data: + data._add_item("valid_micro_num", data.micro_num) + + if "valid_every" not in data: + data._add_item("valid_every", 0) + + if "empty_cache_and_diag_interval" not in data: + data._add_item("empty_cache_and_diag_interval", 50) + + if "diag_outlier_ratio" not in data: + data._add_item("diag_outlier_ratio", 1.1) + data.diag_outlier_ratio = max(1, data.diag_outlier_ratio) + + if gpc.is_rank_for_log(): + logger.info("+" * 15 + " Data Info " + "+" * 15) # pylint: disable=W1201 + logger.info(f"seq_len: {data.seq_len}") + logger.info(f"micro_num: {data.micro_num}") + logger.info(f"micro_bsz: {data.micro_bsz}") + logger.info(f"packed_length: {data.packed_length}") + logger.info(f"pack_sample_into_one: {data.pack_sample_into_one}") + logger.info(f"min_length: {data.min_length}") + logger.info(f"valid_micro_num: {data.valid_micro_num}") + logger.info(f"valid_every: {data.valid_every}") + + # processing the checkpoint config + ckpt = gpc.config.ckpt + if "enable_save_ckpt" not in ckpt: + ckpt._add_item("enable_save_ckpt", True) + + # Saving checkpoint args. + if ckpt.enable_save_ckpt: + assert "checkpoint_every" in ckpt, "If enable save checkpoint, must give checkpoint_every in config.data!" + assert ckpt.checkpoint_every > 0 + assert "save_ckpt_folder" in ckpt, "If enable save checkpoint, must give save_ckpt_folder in config.data!" + + if "async_upload" not in ckpt: + ckpt._add_item("async_upload", False) # async defalut is False. + else: + if ckpt.async_upload: + assert "save_ckpt_folder" in ckpt + if "boto3:" not in ckpt.save_ckpt_folder: + if gpc.is_rank_for_log(): + logger.warning( + "Storing ckpt on file system does not support asynchronous storage, will use sync save!" + ) + ckpt.async_upload = False + else: + if "async_upload_tmp_folder" not in ckpt: + ckpt._add_item("async_upload_tmp_folder", "/dev/shm/internlm_tmp_ckpt/") + + if not ckpt.async_upload: + ckpt._add_item("async_upload_tmp_folder", None) + + if "oss_snapshot_freq" not in ckpt: + ckpt._add_item("oss_snapshot_freq", float("inf")) # if oss_snapshot_freq not given, we disable. + else: + ckpt._add_item("checkpoint_every", float("inf")) + ckpt._add_item("oss_snapshot_freq", float("inf")) + ckpt._add_item("save_ckpt_folder", None) + ckpt._add_item("async_upload", False) + ckpt._add_item("async_upload_tmp_folder", None) + ckpt._add_item("snapshot_ckpt_folder", None) + + if "load_ckpt_folder" not in ckpt: + ckpt._add_item("load_ckpt_folder", None) + + if "stop_file_path" not in ckpt: + ckpt._add_item("stop_file_path", None) + + if "auto_resume" not in ckpt: + # If 'auto_resume' is not given, we set it to True, so internlm can have opportunity + # to auto-load latest checkpoint. + ckpt._add_item("auto_resume", True) + + if gpc.is_rank_for_log(): + logger.info("+" * 15 + " Ckpt Info " + "+" * 15) # pylint: disable=W1201 + logger.info(f"is enable save ckpt: {ckpt.enable_save_ckpt}") + logger.info(f"save_ckpt_folder: {ckpt.save_ckpt_folder}") + logger.info(f"checkpoint_every: {ckpt.checkpoint_every}") + + # tensorboard writer config + if "enable_tb" not in gpc.config: + gpc.config._add_item("enable_tb", True) + if "tensorboard_folder" not in gpc.config: + gpc.config._add_item( + "tensorboard_folder", os.environ["tensorboard_folder"] if "tensorboard_folder" in os.environ else None + ) + if "resume_tb_folder" not in gpc.config: + gpc.config._add_item( + "resume_tb_folder", os.environ["resume_tb_folder"] if "resume_tb_folder" in os.environ else None + ) + + if gpc.is_rank_for_log(): + logger.info(f"tensorboard_folder: {gpc.config.tensorboard_folder}") + logger.info(f"resume_tb_folder: {gpc.config.resume_tb_folder}") + + # cudnn + torch.backends.cudnn.benchmark = gpc.config.get("cudnn_benchmark", False) + torch.backends.cudnn.deterministic = gpc.config.get("cudnn_deterministic", False) + clip_grad_norm = gpc.config.hybrid_zero_optimizer.get("clip_grad_norm", 0.0) + + if gpc.is_rank_for_log(): + logger.info("+" * 15 + " Other Info " + "+" * 15) # pylint: disable=W1201 + logger.info(f"cudnn.benchmark: {torch.backends.cudnn.benchmark }") + logger.info(f"cudnn.deterministic: {torch.backends.cudnn.deterministic }") + logger.info(f"clip_grad_norm: {clip_grad_norm}") + + model = gpc.config.model + if "dtype" not in model: + logger.warning("dtype is not set, use torch.float16 by defalut!") + model._add_item("dtype", torch.float16) + else: + if gpc.config.model.dtype == "torch.bfloat16": + gpc.config.model.dtype = torch.bfloat16 + elif gpc.config.model.dtype in ("torch.float16", "torch.half"): + gpc.config.model.dtype = torch.float16 + elif gpc.config.model.dtype == "torch.float32": + gpc.config.model.dtype = torch.float32 + elif gpc.config.model.dtype == "torch.tf32": + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + gpc.config.model.dtype = torch.float32 + else: + assert gpc.config.model.dtype in [ + "torch.float16", + "torch.half", + "torch.bfloat16", + "torch.float32", + "torch.tf32", + ] + + if "checkpoint" in model: + if model.checkpoint is True: + model.checkpoint = 1 + elif model.checkpoint is False: + model.checkpoint = 0 + else: + assert ( + model.checkpoint >= 0 and model.checkpoint <= 1 + ), f'model.checkpoint: "{model.checkpoint}" should >=0 and <=1' + + if "teacher" in gpc.config: + teacher = gpc.config.teacher + if "dtype" not in teacher: + logger.warning("dtype is not set, use torch.float16 by defalut!") + teacher._add_item("dtype", torch.float16) + else: + if gpc.config.teacher.dtype == "torch.bfloat16": + gpc.config.teacher.dtype = torch.bfloat16 + elif gpc.config.teacher.dtype in ("torch.float16", "torch.half"): + gpc.config.teacher.dtype = torch.float16 + elif gpc.config.teacher.dtype == "torch.float32": + gpc.config.teacher.dtype = torch.float32 + elif gpc.config.teacher.dtype == "torch.tf32": + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + gpc.config.teacher.dtype = torch.float32 + else: + assert gpc.config.teacher.dtype in [ + "torch.float16", + "torch.half", + "torch.bfloat16", + "torch.float32", + "torch.tf32", + ] + + if "checkpoint" in teacher: + if teacher.checkpoint is True: + teacher.checkpoint = 1 + elif teacher.checkpoint is False: + teacher.checkpoint = 0 + else: + assert ( + teacher.checkpoint >= 0 and teacher.checkpoint <= 1 + ), f'teacher.checkpoint: "{teacher.checkpoint}" should >=0 and <=1' + + if gpc.is_rank_for_log(): + logger.info("+" * 15 + " Model Info " + "+" * 15) # pylint: disable=W1201 + logger.info(f"Model: {gpc.config.model}") + + logger.info("+" * 15 + " grad_scaler Info " + "+" * 15) # pylint: disable=W1201 + logger.info(f"grad_scaler: {gpc.config.grad_scaler}") + + logger.info("+" * 15 + " hybrid_zero_optimizer Info " + "+" * 15) # pylint: disable=W1201 + logger.info(f"hybrid_zero_optimizer: {gpc.config.hybrid_zero_optimizer}") + + logger.info("+" * 15 + " adam Info " + "+" * 15) # pylint: disable=W1201 + logger.info(f"adam: {gpc.config.adam}") + + logger.info("+" * 15 + " beta2_scheduler Info " + "+" * 15) # pylint: disable=W1201 + logger.info(f"beta2_scheduler: {gpc.config.beta2_scheduler}") + + # process the model config + if "use_flash_attn" not in gpc.config.model: + gpc.config.model._add_item("use_flash_attn", True) + + # process the parallel config + if "sequence_parallel" not in gpc.config.parallel: + gpc.config.parallel._add_item("sequence_parallel", False) + else: + assert not ( + gpc.config.parallel.sequence_parallel is True and gpc.config.model.use_flash_attn is False + ), "sequence parallel does not support use_flash_attn=False" + + # monitoring default config + monitor_default_config = { + "alert_address": None, # compatible with old alert config + "monitor": { # new monitoring config + "alert": {"enable_feishu_alert": False, "feishu_alert_address": None, "light_monitor_address": None} + }, + } + + for key, value in monitor_default_config.items(): + if key not in gpc.config: + gpc.config._add_item(key, value) + + alert = gpc.config.monitor.alert + + if alert.enable_feishu_alert and not alert.feishu_alert_address and gpc.is_rank_for_log(): + logger.warning("alert is enable but alert_address is not set") + + optim_ckpt = gpc.config.hybrid_zero_optimizer + if "zero_overlap_communication" in optim_ckpt: + # Compatible with the old interfaces. + optim_ckpt._add_item("overlap_sync_grad", optim_ckpt.zero_overlap_communication) + if "overlap_sync_grad" not in optim_ckpt: + optim_ckpt._add_item("overlap_sync_grad", False) + if "overlap_sync_param" not in optim_ckpt: + optim_ckpt._add_item("overlap_sync_param", False) + if gpc.is_rank_for_log(): + logger.info( + f"overlap_sync_grad:{optim_ckpt.overlap_sync_grad}, overlap_sync_param:{optim_ckpt.overlap_sync_param}" + ) + + +def launch( + config: Union[str, Path, Config, Dict], + rank: int, + world_size: int, + host: str, + port: int, + backend: str = "nccl", + local_rank: int = None, + seed: int = 1024, +): + """This function first parses the configuration arguments, using :func:`parse_args()` in case one of the input + arguments are not given. Then initialize and set distributed environment by calling global_context's functions. + + Args: + config (Union[str, dict, Config]): Config file or config file path are both acceptable + rank (int): Rank for the default process group + world_size (int): World size of the default process group + host (str): The master address for distributed training + port (str): The master port for distributed training + backend (str, optional): Backend for ``torch.distributed``, defaults to ``nccl`` + local_rank (int, optional): + Rank for the process on the node and is used to set the default CUDA device, + defaults to None. If local_rank = None, the default device ordinal will be calculated automatically. + seed (int, optional): Specified random seed for every process. Defaults to 1024. + + Raises: + Exception: Raise exception when config type is wrong + """ + + # set config + assert isinstance( + config, (Config, str, Path, dict) + ), f"expected argument config to be Config, str or Path, but got {type(config)}" + if not isinstance(config, Config) and isinstance(config, dict): + config = Config(config) + if isinstance(config, (str, Path)): + config = Config.from_file(config) + gpc.load_config(config) + + # init default process group + gpc.init_global_dist(rank, world_size, backend, host, port) + + # init process groups for different parallel modes from config + gpc.init_parallel_groups() + + # set cuda device + if torch.cuda.is_available(): + # if local rank is not given, calculate automatically + gpc.set_device(local_rank) + + # set the number of processes running on the same node + gpc.detect_num_processes_on_current_node() + + gpc.set_seed(seed) + + if gpc.is_rank_for_log(): + logger.info( + f"Distributed environment is initialized, " + f"data parallel size: {gpc.data_parallel_size}, pipeline parallel size: {gpc.pipeline_parallel_size}, " + f"tensor parallel size: {gpc.tensor_parallel_size}", + ) + + +def launch_from_slurm( + config: Union[str, Path, Config, Dict], + host: str, + port: int, + backend: str = "nccl", + seed: int = 1024, +): + """A wrapper for internlm.launch for SLURM launcher by reading rank and world size from the environment variables + set by SLURM + + Args: + config (Union[str, dict, Config]): Config file or config file path are both acceptable + host (str): The master address for distributed training + port (str): The master port for distributed training + backend (str, optional): Backend for ``torch.distributed``, defaults to ``nccl`` + seed (int, optional): Specified random seed for every process. Defaults to 1024. + """ + try: + rank = int(os.environ["SLURM_PROCID"]) + world_size = int(os.environ["SLURM_NPROCS"]) + except KeyError as e: + raise RuntimeError(f"Could not find {e} in the SLURM environment") + + launch( + config=config, + rank=rank, + world_size=world_size, + host=host, + port=port, + backend=backend, + seed=seed, + ) + + +def launch_from_torch( + config: Union[str, Path, Config, Dict], + backend: str = "nccl", + seed: int = 1024, +): + """A wrapper for internlm.launch for torchrun or torch.distributed.launch by reading rank and world size + from the environment variables set by PyTorch + + Args: + config (Union[str, dict, Config]): Config file or config file path are both acceptable + backend (str, optional): Backend for ``torch.distributed``, defaults to ``nccl`` + seed (int, optional): Specified random seed for every process. Defaults to 1024. + """ + try: + rank = int(os.environ["RANK"]) + local_rank = int(os.environ["LOCAL_RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + host = os.environ["MASTER_ADDR"] + port = int(os.environ["MASTER_PORT"]) + except KeyError as e: + raise RuntimeError(f"Could not find {e} in the torch environment") + + launch( + config=config, + local_rank=local_rank, + rank=rank, + world_size=world_size, + host=host, + port=port, + backend=backend, + seed=seed, + ) + + +@llm_timeout(func_name="initialize_distributed_env") +def initialize_distributed_env( + config: str, + launcher: str = "slurm", + master_port: int = 8888, + seed: int = 1024, + args_check=True, +): + """ + Initialize distributed environment for distributed training. + + Args: + config (str): Config file path. + launcher (str): Launcher for launching distributed environment, can be slurm or torch. "slurm" by default. + master_port (str): The master port for distributed training. 8888 by default. + seed (int, optional): Specified random seed for every process. 1024 by default. + """ + + torch.cuda.empty_cache() + + if launcher == "torch": + launch_from_torch(config=config, seed=seed) + elif launcher == "slurm": + launch_from_slurm( + config=config, + host=get_master_node(), + port=master_port, + seed=seed, + ) + else: + assert launcher in ["slurm", "torch"], "launcher only support slurm or torch" + + if args_check: + args_sanity_check() + + # init light monitor client + alert_config = gpc.config.monitor.alert + if alert_config.enable_feishu_alert and gpc.is_rank_for_log(): + light_monitor_address = alert_config.light_monitor_address + if light_monitor_address: + initialize_light_monitor(light_monitor_address) + else: + logger.warning("monitor address is none, monitor could not be used!") + + +def get_config_value(config, key, defalut): + try: + value = config[key] + except KeyError: + value = defalut + return value diff --git a/InternLM/internlm/initialize/legacy/__init__.py b/InternLM/internlm/initialize/legacy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/InternLM/internlm/initialize/legacy/launch.py b/InternLM/internlm/initialize/legacy/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..8313654daefd2767c8a8356e59713498108fab9d --- /dev/null +++ b/InternLM/internlm/initialize/legacy/launch.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from internlm.initialize.launch import get_config_value +from internlm.utils.logger import get_logger + +logger = get_logger(__file__) + + +def auto_resume_sanity_check(ckpt_config): + load_given_ckpt = get_config_value(ckpt_config, "load_given_ckpt", None) + if load_given_ckpt is None: + return True # default value is True + else: + return not load_given_ckpt + + +def ckpt_info_sanity_check(ckpt_config): + load_ckpt_folder = get_config_value(ckpt_config, "load_ckpt_folder", None) + + load_model_only_folder = get_config_value(ckpt_config, "load_model_only_folder", None) + + if load_model_only_folder is not None: + assert ( + load_ckpt_folder is None + ), "Detect 'load_ckpt_folder' and 'load_model_only_folder' set at the same time, \ +# and 'load_given_ckpt' is True, so internlm will load from 'load_ckpt_folder'" + return dict(path=load_model_only_folder, content=("model",), ckpt_type="internlm") + else: + load_optimizer = get_config_value(ckpt_config, "load_optimizer", True) + + if isinstance(load_ckpt_folder, str): + if load_optimizer: + return dict(path=load_ckpt_folder, content=("model", "sampler", "optimizer"), ckpt_type="internlm") + else: + return dict(path=load_ckpt_folder, content=("model", "sampler"), ckpt_type="internlm") + elif load_ckpt_folder is None: + return None + else: + assert f"Unsupport data type:'{type(load_ckpt_folder)}' for config.ckpt arg: 'load_ckpt_folder'" diff --git a/InternLM/internlm/model/__init__.py b/InternLM/internlm/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8df9091250a73529d33fa2b16693b5eef854005d --- /dev/null +++ b/InternLM/internlm/model/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from .embedding import Embedding1D, RotaryEmbedding +from .linear import FeedForward, RewardModelLinear, ScaleColumnParallelLinear +from .metrics import AccPerplex +from .modeling_internlm import build_model_with_cfg +from .modeling_vit import build_vit_model_with_cfg +from .multi_head_attention import MHA +from .utils import gather_forward_split_backward + +__all__ = [ + "Embedding1D", + "FeedForward", + "RotaryEmbedding", + "RewardModelLinear", + "ScaleColumnParallelLinear", + "AccPerplex", + "MHA", + "gather_forward_split_backward", + "build_model_with_cfg", + "build_vit_model_with_cfg" +] diff --git a/InternLM/internlm/model/embedding.py b/InternLM/internlm/model/embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..6b93d6fa9b3473107d88b4f7f9089e91539ed0fe --- /dev/null +++ b/InternLM/internlm/model/embedding.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from typing import Tuple + +import rotary_emb +import torch +import torch.nn.functional as F +from einops import rearrange +from flash_attn.layers.rotary import ApplyRotaryEmb as LegacyApplyRotaryEmb +from flash_attn.layers.rotary import ApplyRotaryEmbQKV_ as LegacyApplyRotaryEmbQKV_ +from torch import Tensor, nn + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc + +from .utils import gather_forward_split_backward, split_forward_gather_backward + + +from .muse import VQGANModel + +class Embedding1DLVM(nn.Module): + def __init__( + self, + vq_model_path: str, + embedding_dim: int = None, + freeze_vq_model: bool = True + ): + super().__init__() + + self.vq_model = VQGANModel.from_pretrained(vq_model_path) + if freeze_vq_model: + self.vq_model.requires_grad_(False) + self.vq_model.eval() + + self.num_embeddings, vq_embed_dim = self.vq_model.quantize.embedding.weight.shape + + if embedding_dim is not None: + self.embed_proj = nn.Linear(vq_embed_dim, embedding_dim, bias=False) + self.embedding_dim = embedding_dim + else: + self.embed_proj = None + self.embedding_dim = vq_embed_dim + + def forward(self, input_: Tensor) -> Tensor: + + # input: N x seq + output_parallel = self.vq_model.quantize.get_codebook_entry_for_lvm(input_) # N x vq_embed_dim x sqrt(seq) x sqrt(seq) + + if self.embed_proj is not None: + output_parallel = self.embed_proj(output_parallel) + + output = gather_forward_split_backward(output_parallel, ParallelMode.TENSOR, dim=-1) + + if gpc.config.parallel.sequence_parallel: + output = split_forward_gather_backward(output, ParallelMode.TENSOR, dim=1) + + return output + + +class Embedding1D(nn.Module): + """ + 1D Embedding. + + Args: + num_embeddings (int): The size of vocab. + embedding_dim (int): The dimention of model. + padding_idx (int): If specified, the entries at :attr:`padding_idx` do not contribute to the gradient; + therefore, the embedding vector at :attr:`padding_idx` is not updated during training, + i.e. it remains as a fixed "pad". None by default. + dtype (Optional[torch.dtype]): Data type None by default. + + """ + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + *args, + padding_idx: int = None, + dtype: torch.dtype = None, + **kwargs, + ): + super().__init__() + + self.num_embeddings = num_embeddings + self.embed_dim = embedding_dim + embed_dim_per_partition = embedding_dim // gpc.tensor_parallel_size + + self.padding_idx = padding_idx + self.embed_args = args + self.embed_kwargs = kwargs + + self.weight = nn.Parameter(torch.empty((num_embeddings, embed_dim_per_partition), dtype=dtype)) + + def forward(self, input_: Tensor) -> Tensor: + output_parallel = F.embedding(input_, self.weight, self.padding_idx, *self.embed_args, **self.embed_kwargs) + + output = gather_forward_split_backward(output_parallel, ParallelMode.TENSOR, dim=-1) + + if gpc.config.parallel.sequence_parallel: + output = split_forward_gather_backward(output, ParallelMode.TENSOR, dim=1) + + return output + + +class ApplyRotaryEmbQKV_(torch.autograd.Function): + """ + ApplyRotaryEmbQKV_ + """ + + @staticmethod + def forward(ctx, qkv, cos, sin, cos_k=None, sin_k=None): + """ + qkv: (total, 3, nheads, headdim) + cos, sin: (seqlen, rotary_dim / 2) + cos_k, sin_k: (seqlen, rotary_dim / 2), optional + rotary_dim must be <= headdim + Apply rotary embedding *inplace* to the first rotary_dim of q and k. + """ + _, three, _, headdim = qkv.shape + assert three == 3 + rotary_seqlen, rotary_dim = cos.shape + rotary_dim *= 2 + assert rotary_dim <= headdim + cos_k = cos if cos_k is None else cos_k + sin_k = sin if sin_k is None else sin_k + assert sin.shape == cos_k.shape == sin_k.shape == (rotary_seqlen, rotary_dim // 2) + q1, q2 = qkv[:, 0, :, :rotary_dim].chunk(2, dim=-1) + rotary_emb.apply_rotary(q1, q2, rearrange(cos, "s d -> s 1 d"), rearrange(sin, "s d -> s 1 d"), q1, q2, False) + k1, k2 = qkv[:, 1, :, :rotary_dim].chunk(2, dim=-1) + rotary_emb.apply_rotary( + k1, k2, rearrange(cos_k, "s d -> s 1 d"), rearrange(sin_k, "s d -> s 1 d"), k1, k2, False + ) + ctx.save_for_backward(cos, sin, cos_k, sin_k) + return qkv + + @staticmethod + def backward(ctx, dqkv): + cos, sin, cos_k, sin_k = ctx.saved_tensors + rotary_dim = cos.shape[-1] + rotary_dim *= 2 + dq1, dq2 = dqkv[:, 0, :, :rotary_dim].chunk(2, dim=-1) + rotary_emb.apply_rotary( + dq1, dq2, rearrange(cos, "s d -> s 1 d"), rearrange(sin, "s d -> s 1 d"), dq1, dq2, True + ) + dk1, dk2 = dqkv[:, 1, :, :rotary_dim].chunk(2, dim=-1) + rotary_emb.apply_rotary( + dk1, dk2, rearrange(cos_k, "s d -> s 1 d"), rearrange(sin_k, "s d -> s 1 d"), dk1, dk2, True + ) + return dqkv, None, None, None, None + + +apply_rotary_emb_qkv_ = ApplyRotaryEmbQKV_.apply +legacy_apply_rotary_embed_qkv = LegacyApplyRotaryEmbQKV_.apply +legacy_apply_rotary_embed = LegacyApplyRotaryEmb.apply + + +class RotaryEmbedding(torch.nn.Module): + """ + The rotary position embeddings from RoFormer_ (Su et. al). + A crucial insight from the method is that the query and keys are + transformed by rotation matrices which depend on the relative positions. + + Other implementations are available in the Rotary Transformer repo_ and in + GPT-NeoX_, GPT-NeoX was an inspiration + + .. _RoFormer: https://arxiv.org/abs/2104.09864 + .. _repo: https://github.com/ZhuiyiTechnology/roformer + .. _GPT-NeoX: https://github.com/EleutherAI/gpt-neox + + If scale_base > 0, this implements XPos (Sun et al., https://arxiv.org/abs/2212.10554). + A recommended value for scale_base is 512: https://github.com/HazyResearch/flash-attention/issues/96 + Reference: https://github.com/sunyt32/torchscale/blob/main/torchscale/component/xpos_relative_position.py + """ + + def __init__(self, dim: int, base=10000, scale_base=0, device=None): + """ """ + super().__init__() + # Generate and save the inverse frequency buffer (non trainable) + self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim)) + self.scale_base = scale_base + self.scale = ( + (torch.arange(0, dim, 2, device=device, dtype=torch.float32) + 0.4 * dim) / (1.4 * dim) + if scale_base > 0 + else None + ) + + self._seq_len_cached = 0 + self._cos_cached = None + self._sin_cached = None + self._cos_k_cached = None + self._sin_k_cached = None + + def _update_cos_sin_cache(self, x, indexes): + """x: (batch, seqlen, nheads, headdim) or (batch, seqlen, 3, nheads, headdim)""" + if not isinstance(indexes, int): + seqlen = indexes.max().item() + 1 + else: + seqlen = indexes + 1 # eval_forward + # Reset the tables if the sequence length has changed, + # or if we're on a new device (possibly due to tracing for instance) + if seqlen > self._seq_len_cached or self._cos_cached.device != x.device or self._cos_cached.dtype != x.dtype: + self._seq_len_cached = seqlen + t = torch.arange(seqlen, device=x.device, dtype=self.inv_freq.dtype) + # Don't do einsum, it converts fp32 to fp16 + # freqs = torch.einsum("i,j->ij", t, self.inv_freq) + freqs = torch.outer(t, self.inv_freq.to(device=t.device)) + if self.scale is None: + self._cos_cached = torch.cos(freqs).to(x.dtype) + self._sin_cached = torch.sin(freqs).to(x.dtype) + else: + power = ( + torch.arange(seqlen, dtype=self.scale.dtype, device=self.scale.device) - seqlen // 2 + ) / self.scale_base + scale = self.scale.to(device=power.device) ** rearrange(power, "s -> s 1") + # We want the multiplication by scale to happen in fp32 + self._cos_cached = (torch.cos(freqs) * scale).to(x.dtype) + self._sin_cached = (torch.sin(freqs) * scale).to(x.dtype) + self._cos_k_cached = (torch.cos(freqs) / scale).to(x.dtype) + self._sin_k_cached = (torch.sin(freqs) / scale).to(x.dtype) + + def forward(self, qkv: torch.Tensor, **kwargs): + if kwargs.get("indexes", None) is not None: + return self._forward(qkv, kwargs.pop("indexes")) + if kwargs.get("inference_params", None) is not None: + return self._eval_forward(qkv, seqlen_offset=kwargs.get("inference_params", None).sequence_len_offset) + else: + return self._eval_forward(qkv) + + def _forward(self, qkv: torch.Tensor, indexes=0) -> Tuple[torch.Tensor, torch.Tensor]: + self._update_cos_sin_cache(qkv, indexes) + if self.scale is None: + return apply_rotary_emb_qkv_(qkv, self._cos_cached[indexes], self._sin_cached[indexes]) + else: + return apply_rotary_emb_qkv_( + qkv, + self._cos_cached[indexes], + self._sin_cached[indexes], + self._cos_k_cached[indexes], + self._sin_k_cached[indexes], + ) + + def _eval_forward(self, qkv, seqlen_offset=0): + """ + seqlen_offset: can be used in generation where the qkv being passed in is only the last + token in the batch. + """ + self._update_cos_sin_cache(qkv, seqlen_offset + qkv.shape[1]) + if self.scale is None: + return legacy_apply_rotary_embed_qkv( + qkv, self._cos_cached[seqlen_offset:], self._sin_cached[seqlen_offset:] + ) + else: + return legacy_apply_rotary_embed_qkv( + qkv, + self._cos_cached[seqlen_offset:], + self._sin_cached[seqlen_offset:], + self._cos_k_cached[seqlen_offset:], + self._sin_k_cached[seqlen_offset:], + ) + + def _single_forward(self, x, indexes=0): + assert self.scale is None + self._update_cos_sin_cache(x, indexes) + x = x[None, ...] + ret = legacy_apply_rotary_embed(x, self._cos_cached[indexes], self._sin_cached[indexes]).squeeze(0) + return ret + + def _single_eval_forward(self, x, seqlen_offset=0): + assert self.scale is None + self._update_cos_sin_cache(x, seqlen_offset + x.shape[1]) + return legacy_apply_rotary_embed(x, self._cos_cached[seqlen_offset:], self._sin_cached[seqlen_offset:]) diff --git a/InternLM/internlm/model/linear.py b/InternLM/internlm/model/linear.py new file mode 100644 index 0000000000000000000000000000000000000000..5a3a4ebd7e5b09b76c8181ba03200dc3715fc419 --- /dev/null +++ b/InternLM/internlm/model/linear.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from typing import Optional + +import torch +import torch.nn.functional as F +from flash_attn.ops.fused_dense import ColumnParallelLinear, RowParallelLinear +from flash_attn.utils.distributed import all_reduce, reduce_scatter +from torch import nn + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.model.utils import fused_dense_func_torch + + +class ScaleColumnParallelLinear(nn.Linear): + """ + ScaleColumnParallelLinear. + + Args: + in_features (int): size of each input sample + out_features (int): size of each output sample + process_group (Optional[torch.distributed.ProcessGroup]): The group of the current device for `parallel_mode`. + bias (bool): Whether the bias is needed for linears. True by default. But it is typically set to False + in the config. + sequence_parallel (bool): If sequence_parallel is True, we're doing Tensor Parallel with sequence parallelism: + we do an all_gather of x before doing the matmul. + If not, then the input is already gathered. + device (Optional[Union[str, torch.device]]): The device will be used. + dtype (Optional[torch.dtype]): The type of data. + weight_scale (int): For training stability. 1 by default. + """ + + def __init__( + self, + in_features: int, + out_features: int, + process_group: Optional[torch.distributed.ProcessGroup], + bias: bool = True, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + weight_scale: int = 1, + ) -> None: + world_size = torch.distributed.get_world_size(process_group) + if out_features % world_size != 0: + raise ValueError(f"out_features ({out_features}) must be divisible by " f"world_size ({world_size})") + super().__init__(in_features, out_features // world_size, bias=bias, device=device, dtype=dtype) + self.process_group = process_group + self.weight_scale = weight_scale + + def forward(self, input): # pylint: disable=W0622 + # If self.sequence_parallel is True, we're doing Tensor Parallel with sequence parallelism: + # we do an all_gather of x before doing the matmul. + # If not, then the input is already gathered. + if self.weight_scale != 1: + weight = self.weight * self.weight_scale + (1 - self.weight_scale) * self.weight.detach() + else: + weight = self.weight + return fused_dense_func_torch( + input, + weight, + self.bias, + process_group=self.process_group, + sequence_parallel=gpc.config.parallel.sequence_parallel, + ) + + +class RewardModelLinear(ScaleColumnParallelLinear): + """ + RewardModelLinear. + Args: + in_features (int): size of each input sample + out_features (int): size of each output sample + process_group (Optional[torch.distributed.ProcessGroup]): The group of the current device for `parallel_mode`. + bias (bool): Whether the bias is needed for linears. True by default. But it is typically set to False + in the config. + sequence_parallel (bool): If sequence_parallel is True, we're doing Tensor Parallel with sequence parallelism: + we do an all_gather of x before doing the matmul. + If not, then the input is already gathered. + device (Optional[Union[str, torch.device]]): The device will be used. + dtype (Optional[torch.dtype]): The type of data. + weight_scale (int): For training stability. 1 by default. + """ + + def __init__( + self, + in_features: int, + out_features: int, + process_group: Optional[torch.distributed.ProcessGroup], + bias: bool = True, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + weight_scale: int = 1, + ) -> None: + super().__init__(in_features, out_features, process_group, bias, device, dtype, weight_scale) + torch.distributed.broadcast(self.weight, gpc.get_ranks_in_group(ParallelMode.TENSOR)[0], process_group) + if bias: + torch.distributed.broadcast(self.bias, gpc.get_ranks_in_group(ParallelMode.TENSOR)[0], process_group) + + def forward(self, input): # pylint: disable=W0622 + # If self.sequence_parallel is True, we're doing Tensor Parallel with sequence parallelism: + # we do an all_gather of x before doing the matmul. + # If not, then the input is already gathered. + if self.weight_scale != 1: + weight = self.weight * self.weight_scale + (1 - self.weight_scale) * self.weight.detach() + else: + weight = self.weight + return fused_dense_func_torch( + input, + weight, + self.bias, + process_group=self.process_group, + sequence_parallel=gpc.config.parallel.sequence_parallel, + ) + + +class ColumnParallelLinearTorch(ColumnParallelLinear): + def forward(self, x): + # If self.sequence_parallel is True, we're doing Tensor Parallel with sequence parallelism: + # we do an all_gather of x before doing the matmul. + # If not, then the input is already gathered. + + return fused_dense_func_torch( + x, self.weight, self.bias, process_group=self.process_group, sequence_parallel=self.sequence_parallel + ) + + +class RowParallelLinearTorch(RowParallelLinear): + def forward(self, x): + """ + We're doing Tensor Parallel with sequence parallelism: we do the matmul and then + a reduce_scatter of the result. + """ + out = fused_dense_func_torch(x, self.weight, self.bias) + reduce_fn = reduce_scatter if self.sequence_parallel else all_reduce + return reduce_fn(out, self.process_group) + + +class FeedForward(nn.Module): + """ + FeedForward. + + Args: + in_features (int): size of each input sample + hidden_features (int): size of hidden state of FFN + out_features (int): size of each output sample + process_group (Optional[torch.distributed.ProcessGroup]): The group of the current device for `parallel_mode`. + bias (bool): Whether the bias is needed for linears. True by default. But it is typically set to False + in the config. + device (Optional[Union[str, torch.device]]): The device will be used. + dtype (Optional[torch.dtype]): The type of data. + multiple_of (int): For efficient training. Reset the size of hidden feature. 256 by default. + """ + + def __init__( + self, + in_features: int, + hidden_features: int, + out_features: int = None, + process_group: Optional[torch.distributed.ProcessGroup] = None, + bias: bool = True, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + multiple_of: int = 256, + ): + super().__init__() + + hidden_features = multiple_of * ((hidden_features + multiple_of - 1) // multiple_of) + + self.w1 = ColumnParallelLinearTorch( + in_features, + hidden_features, + process_group, + bias, + sequence_parallel=gpc.config.parallel.sequence_parallel, + device=device, + dtype=dtype, + ) + self.w2 = ColumnParallelLinearTorch( + in_features, + hidden_features, + process_group, + bias, + sequence_parallel=gpc.config.parallel.sequence_parallel, + device=device, + dtype=dtype, + ) + self.w3 = RowParallelLinearTorch( + hidden_features, + out_features, + process_group, + bias=bias, + sequence_parallel=gpc.config.parallel.sequence_parallel, + device=device, + dtype=dtype, + ) + + def forward(self, x): + out = self.w3(F.silu(self.w1(x)) * self.w2(x)) + return out diff --git a/InternLM/internlm/model/loss.py b/InternLM/internlm/model/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..a15729a4ec24fbc73341f36dea20c8845cec5cec --- /dev/null +++ b/InternLM/internlm/model/loss.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import torch.nn.functional as F +from flash_attn.losses.cross_entropy import CrossEntropyLoss as FlashCrossEntropyLoss +from torch import nn + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc + + +class FlashGPTLMLoss(nn.Module): + """ + Loss function for flash GPT Language Model. + """ + + def __init__(self, parallel_output=True, label_smoothing=0): + super().__init__() + + if label_smoothing is not None: + if label_smoothing != 0: + if gpc.is_rank_for_log(): + print(f"use label_smoothing: {label_smoothing}") + else: + label_smoothing = 0 + self.label_smoothing = label_smoothing + + if parallel_output: + self.loss_fn = FlashCrossEntropyLoss( + reduction="mean", + inplace_backward=True, + process_group=gpc.get_group(ParallelMode.TENSOR), + label_smoothing=label_smoothing, + ) # The loss in this place is bound to the gather_output initialized by VocabParallelClassifier1D + else: + # Here, the output will gather output is set in the model, so use ordinary loss + self.loss_fn = nn.CrossEntropyLoss(reduction="mean", label_smoothing=label_smoothing) + + def forward(self, *args): + if len(args) == 3: + # residual is to match prenorm + logits, _, labels = args + elif len(args) == 2: + # When using postnorm + logits, labels = args + else: + raise RuntimeError(f"The number of criterion inputs are:{len(args)}") + shift_logits = logits.contiguous().view(-1, logits.size(-1)) + shift_labels = labels.contiguous().view(-1) + loss = self.loss_fn( + shift_logits, shift_labels + ) # There is no need to consider the ignore_index problem here, because the loss calculation will be + # calculated through the calculation range, and -100 must be outside this range, so there is no problem + + return loss + + +class KLDivLoss(nn.Module): + def __init__(self): + super().__init__() + self.temperature = gpc.config.kd_config.get('temperature', 1) + self.inverse = gpc.config.kd_config.get('inverse', False) + + def forward(self, *args): + if len(args) == 3: + if self.inverse: + logits_teacher, logits_student, _ = args + else: + logits_student, logits_teacher, _ = args + else: + raise RuntimeError(f"The number of criterion inputs are:{len(args)}") + + logits_teacher = logits_teacher.contiguous().view(-1, logits_teacher.size(-1)) + logits_student = logits_student.contiguous().view(-1, logits_student.size(-1)) + + log_pred_student = F.log_softmax(logits_student / self.temperature, dim=1) + pred_teacher = F.softmax(logits_teacher / self.temperature, dim=1) + loss_kd = F.kl_div(log_pred_student, pred_teacher, reduction='batchmean') + loss_kd *= self.temperature ** 2 + + return loss_kd diff --git a/InternLM/internlm/model/metrics.py b/InternLM/internlm/model/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..b5cee5fed3042e09fc1763132da9a28c5b590bae --- /dev/null +++ b/InternLM/internlm/model/metrics.py @@ -0,0 +1,263 @@ +from typing import List + +import torch +from flash_attn.losses.cross_entropy import CrossEntropyLoss as FlashCrossEntropyLoss +from torch_scatter import scatter + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.utils.parallel import is_no_pp_or_last_stage + + +class AccPerplex: + """ + AccPerplex module for calculating model's accuracy and perplexity metrics. + + Args: + device: The GPU device. + tp_pg: The tensor parallel process group. + dp_pg: The data parallel process group. + tokenizer: For calculating BPB. + dataset_types (List[str]): Various data types that will be used in the current training process, + such as ['en', 'cn', 'code']. The order of the List should be consistent with the type_id specified + in the dataset. Changed parameters need to be used in conjunction with set_current_type_ids(). + """ + + def __init__(self, device, tp_pg, dp_pg, tokenizer=None, dataset_types: List[str] = None): + self.device = device + self.right = torch.Tensor([0]).to(device=device) + self.total = torch.Tensor([0]).to(device=device) + self.total_log_probs = torch.Tensor([0]).to(device=device) + self.tp_pg = tp_pg + self.dp_pg = dp_pg + self.tp_local_rank = torch.distributed.get_rank(self.tp_pg) + self.tokenizer = tokenizer + self.total_bytes = torch.Tensor([0]).to(device=device).view(1) + self.batch_shift = 0 + self.type_ids = None + if dataset_types is not None: + self.dataset_types = dataset_types + self.total_type_count = len(dataset_types) + self.ds_right = torch.zeros(self.total_type_count, dtype=torch.long, device=device) + self.ds_tokens = torch.zeros(self.total_type_count, dtype=torch.long, device=device) + + self.loss_with_type_id = LossWithTypeId(device, dp_pg, dataset_types) + + def set_current_type_ids(self, type_ids: torch.Tensor): + self.batch_shift = 0 + self.type_ids = type_ids.cuda() + + def __call__(self, logits, labels): + return self.update(logits, labels, type_ids=self.type_ids) + + def update(self, logits, labels, type_ids=None): + if gpc.config.model.use_flash_attn: + micro_bsz = labels.size(0) + else: + micro_bsz = 1 + if type_ids is not None: + type_ids = type_ids[self.batch_shift * micro_bsz : (self.batch_shift + 1) * micro_bsz].view(-1) + self.batch_shift += 1 + self.loss_with_type_id.update(logits, labels, type_ids) + + with torch.no_grad(): + if isinstance(logits, (list, tuple)): + logits = logits[0] + + logits = logits.detach().clone() + labels = labels.detach().clone() + + if self.tokenizer: # need to calculate bits per bytes + sequences = self.tokenizer.decode_ids(labels.tolist()) + self.total_bytes += sum(map(lambda x: len(x.encode("utf-8")), sequences)) + + shift_logits = logits.view(-1, logits.size(-1)) + shift_labels = labels.view(-1) + # There is a shift according to the current rank, because the logits are split + pred_shift = self.tp_local_rank * logits.shape[-1] + + logits_max = torch.max(shift_logits, dim=-1)[0] + torch.distributed.all_reduce(logits_max, op=torch.distributed.ReduceOp.MAX, group=self.tp_pg) + # Determine whether the maximum value of the current local tensor is the global maximum value + logits_global = logits_max == torch.max(shift_logits, dim=-1)[0] + + corrects = torch.logical_and( + (shift_labels == (shift_logits.argmax(dim=-1) + pred_shift)), logits_global + ).long() + mask = shift_labels.ne(-100).long() + if hasattr(self, "total_type_count"): + ds_acc = scatter(corrects, type_ids, dim=0, reduce="sum") + token_num_type = scatter(mask, type_ids, dim=0, reduce="sum") + if len(ds_acc) < self.total_type_count: + ds_acc = torch.cat([ds_acc, ds_acc.new_zeros(self.total_type_count - len(ds_acc))]) + token_num_type = torch.cat( + [token_num_type, token_num_type.new_zeros(self.total_type_count - len(token_num_type))] + ) + self.ds_tokens += token_num_type + sync_tensor = ds_acc + torch.distributed.all_reduce(sync_tensor, op=torch.distributed.ReduceOp.SUM, group=self.tp_pg) + self.ds_right += sync_tensor.view(-1) + + acc = corrects.sum() + torch.distributed.all_reduce(acc, op=torch.distributed.ReduceOp.SUM, group=self.tp_pg) + self.right += acc # Masked_fill is not needed here because -100 is not available anyway + self.total += mask.sum() + + # Subtract the maximum value. + shift_logits = shift_logits.sub(logits_max.unsqueeze(dim=-1)) + + # Get the partition's vocab indecies + partition_vocab_size = shift_logits.size()[-1] + vocab_start_index = partition_vocab_size * self.tp_local_rank + vocab_end_index = vocab_start_index + partition_vocab_size + + # Create a mask of valid vocab ids (1 means it needs to be masked). + target_mask = (shift_labels < vocab_start_index) | (shift_labels >= vocab_end_index) + masked_target = shift_labels - vocab_start_index + masked_target[target_mask] = 0 + + # Get predicted-logits = logits[target]. + # For Simplicity, we model_hf logits to a 2-D tensor with size + # [*, partition-vocab-size] and target to a 1-D tensor of size [*]. + logits_2d = shift_logits.view(-1, partition_vocab_size) + masked_target_1d = masked_target.view(-1) + arange_1d = torch.arange(start=0, end=logits_2d.size()[0], device=logits_2d.device) + predicted_logits_1d = logits_2d[arange_1d, masked_target_1d] + predicted_logits_1d = predicted_logits_1d.clone().contiguous() + predicted_logits = predicted_logits_1d.view_as(shift_labels) # bsz x max_len + predicted_logits[target_mask] = 0.0 + # All reduce is needed to get the chunks from other GPUs. + torch.distributed.all_reduce(predicted_logits, op=torch.distributed.ReduceOp.SUM, group=self.tp_pg) + + pred_exp_logits = torch.exp(predicted_logits) + # Sum of exponential of logits along vocab dimension across all GPUs. + sum_exp_logits = torch.exp(shift_logits).sum(dim=-1) + torch.distributed.all_reduce(sum_exp_logits, op=torch.distributed.ReduceOp.SUM, group=self.tp_pg) + + total_log_probs = -(pred_exp_logits / sum_exp_logits).log().masked_fill(shift_labels.eq(-100), 0).sum() + self.total_log_probs += total_log_probs + + def get_metric(self, reset=True): + if is_no_pp_or_last_stage() and self.dp_pg is not None: + torch.distributed.all_reduce(self.right, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + torch.distributed.all_reduce(self.total, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + torch.distributed.all_reduce(self.total_log_probs, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + if hasattr(self, "total_type_count"): + torch.distributed.all_reduce(self.ds_right, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + torch.distributed.all_reduce(self.ds_tokens, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + if self.tokenizer: + torch.distributed.all_reduce(self.total_bytes, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + + acc = round((self.right / self.total).item(), 4) + perplexity = round(torch.exp(self.total_log_probs / self.total).item(), 4) + bits_per_bytes = round((self.total_log_probs / self.total_bytes).item(), 4) if self.tokenizer else 0 + + if hasattr(self, "total_type_count"): + ds_acc = {} + ds_tokens = {} + for i in range(self.total_type_count): + ds_acc[f"acc/{self.dataset_types[i]}"] = round( + (self.ds_right[i].float() / (self.ds_tokens[i].float() + 1e-5)).item(), 4 + ) + ds_tokens[f"tokens/{self.dataset_types[i]}"] = self.ds_tokens[i].item() + if reset: + self.right.fill_(0) + self.total.fill_(0) + self.total_log_probs.fill_(0) + self.total_bytes.fill_(0) + if hasattr(self, "total_type_count"): + self.ds_right.fill_(0) + self.ds_tokens.fill_(0) + if self.tokenizer is not None: + res = {"acc": acc, "perplexity": perplexity, "BPB": bits_per_bytes} + else: + res = {"acc": acc, "perplexity": perplexity} + if hasattr(self, "total_type_count"): + res.update(ds_acc) + res.update(ds_tokens) + + loss_res = self.loss_with_type_id.get_metric(reset) + res.update(loss_res) + + return res + + +class LossWithTypeId: + """ + Notice the loss value computed here may be not the same with the main info loss, + cause loss here is the reduced result of the data parallel. + """ + + def __init__(self, device, dp_pg, dataset_types: List[str] = None) -> None: + self.device = device + self.dp_pg = dp_pg + + self.loss = torch.Tensor([0.0]).to(device=device) + self.token_num = torch.Tensor([0.0]).to(device=device) + + if dataset_types is not None: + self.dataset_types = dataset_types + self.total_type_count = len(dataset_types) + self.ds_loss = torch.zeros(self.total_type_count, dtype=torch.float, device=device) + self.ds_token_num = torch.zeros(self.total_type_count, dtype=torch.float, device=device) + + self.loss_fn = FlashCrossEntropyLoss( + reduction="none", inplace_backward=True, process_group=gpc.get_group(ParallelMode.TENSOR) + ) + + def update(self, logits, labels, type_ids=None): + with torch.no_grad(): + if isinstance(logits, (list, tuple)): + logits = logits[0] + logits = logits.contiguous().view(-1, logits.size(-1)) + labels = labels.contiguous().view(-1) + loss_list = self.loss_fn(logits, labels) + + cond = labels != -100 + real_loss_list = loss_list[cond] + self.loss += real_loss_list.sum() + self.token_num += real_loss_list.numel() + + if hasattr(self, "total_type_count"): + type_ids = type_ids.contiguous().view(-1).to(self.device) + real_type_ids = type_ids[cond] + + loss_list_type = scatter(real_loss_list, real_type_ids, dim=0, reduce="sum") + token_num_type = scatter(torch.ones_like(real_loss_list), real_type_ids, dim=0, reduce="sum") + + if len(loss_list_type) < self.total_type_count: + loss_list_type = torch.cat( + [loss_list_type, loss_list_type.new_zeros(self.total_type_count - len(loss_list_type))] + ) + token_num_type = torch.cat( + [token_num_type, token_num_type.new_zeros(self.total_type_count - len(token_num_type))] + ) + self.ds_loss += loss_list_type + self.ds_token_num += token_num_type + + def get_metric(self, reset=True): + if is_no_pp_or_last_stage() and self.dp_pg is not None: + torch.distributed.all_reduce(self.loss, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + torch.distributed.all_reduce(self.token_num, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + if hasattr(self, "total_type_count"): + torch.distributed.all_reduce(self.ds_loss, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + torch.distributed.all_reduce(self.ds_token_num, op=torch.distributed.ReduceOp.SUM, group=self.dp_pg) + + loss = round((self.loss / self.token_num).item(), 4) + res = { + "loss_from_metric": loss, + } + if hasattr(self, "total_type_count"): + ds_loss = {} + for i in range(self.total_type_count): + ds_loss[f"loss/{self.dataset_types[i]}"] = round((self.ds_loss[i] / self.ds_token_num[i]).item(), 4) + res.update(ds_loss) + + if reset: + self.loss.fill_(0.0) + self.token_num.fill_(0.0) + if hasattr(self, "total_type_count"): + self.ds_loss.fill_(0.0) + self.ds_token_num.fill_(0.0) + + return res diff --git a/InternLM/internlm/model/modeling_internlm.py b/InternLM/internlm/model/modeling_internlm.py new file mode 100644 index 0000000000000000000000000000000000000000..633679118ac893bfb23cd7bdb487d91446faa4ea --- /dev/null +++ b/InternLM/internlm/model/modeling_internlm.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import math +from typing import Optional + +import torch +from flash_attn.modules.embedding import ParallelGPT2Embeddings +from flash_attn.modules.mlp import ParallelFusedMLP +from torch import nn + +from internlm.core.context import IS_TENSOR_PARALLEL, ParallelMode +from internlm.core.context.parallel_context import global_context as gpc +from internlm.initialize.initialize_tensor import normal_, scaled_init_method_normal +from internlm.model.embedding import Embedding1D, Embedding1DLVM +from internlm.model.linear import ( + FeedForward, + RewardModelLinear, + ScaleColumnParallelLinear, +) +from internlm.model.multi_head_attention import MHA +from internlm.model.utils import gather_forward_split_backward, try_import_RMSNorm +from internlm.solver.pipeline_utils import partition_uniform +from internlm.utils.checkpoint import activation_checkpoint +from internlm.utils.common import filter_kwargs +from internlm.utils.logger import get_logger +from internlm.utils.registry import MODEL_INITIALIZER + +MODEL_TYPE = "INTERNLM" + +logger = get_logger(__file__) +RMSNorm = try_import_RMSNorm() + + +class PackedFlashBaseLayer1D(nn.Module): + """ + 1D Packed Flash Base Layer. + + Args: + hidden_size (int): The hidden size of model. 768 by default. + num_attention_heads (int): The number of attention heads. 12 by default. + mlp_ratio (int): The ratio of MLP layers. 4 by default. + attn_drop_rate (float): The dropout rate of attention module. 0 by default. + drop_rate (float): The dropout rate of the input hidden state. 0.0 by default. + dtype (torch.dtype): Type of data. torch.float by default. + layer_norm_epsilon (float): A value added to the denominator for numerical stability. 1e-5 by default. + checkpoint (bool): Whether to use checkpointing to save VRAM. True by default. + layer_idx (int): The index of current layer. 0 by default. + residual_in_fp32 (bool): Whether to use residual in fp32. False by default. + device (Optional[Union[str, torch.device]]): The device will be used. + norm_type (str): Use RMS norm or layernorm."rmsnorm" by default. + use_flash_attn (bool): Whether use flash-attn. True by default. + """ + + def __init__( + self, + hidden_size: int = 768, + num_attention_heads: int = 12, + mlp_ratio: int = 4, + attn_drop_rate: float = 0, + drop_rate: float = 0.0, + dtype: torch.dtype = torch.float, + layer_norm_epsilon: float = 1e-6, + checkpoint: bool = False, + layer_idx: int = 0, + residual_in_fp32: bool = False, + device: Optional[torch.device] = None, + norm_type: str = "rmsnorm", + dropout_selective_checkpoint: bool = True, + use_scaled_init: bool = True, + use_swiglu: bool = True, + use_flash_attn: bool = True, + ): + super().__init__() + self.checkpoint = checkpoint + # dropout selective checkpoint can only be enabled when checkpoint is disabled. + self.dropout_selective_checkpoint = dropout_selective_checkpoint is True and checkpoint is False + self.layer_idx = layer_idx + self.use_flash_attn = use_flash_attn + + head_dim = hidden_size // num_attention_heads + self.mixer = MHA( + embed_dim=hidden_size, + num_heads=num_attention_heads, + process_group=gpc.get_group(ParallelMode.TENSOR), + dropout=attn_drop_rate, + softmax_scale=1 / math.sqrt(head_dim), + causal=True, + layer_idx=layer_idx, + rotary_emb_dim=head_dim, + rotary_emb_scale_base=0, + use_flash_attn=use_flash_attn, + device=device, + dtype=dtype, + ) + + self.dropout1 = nn.Dropout(drop_rate) + if norm_type == "rmsnorm": + self.norm1 = RMSNorm(hidden_size, eps=layer_norm_epsilon) + self.norm2 = RMSNorm(hidden_size, eps=layer_norm_epsilon) + else: + self.norm1 = nn.LayerNorm(hidden_size, eps=layer_norm_epsilon) + self.norm2 = nn.LayerNorm(hidden_size, eps=layer_norm_epsilon) + + if use_swiglu: + self.mlp = FeedForward( + hidden_size, + int(hidden_size * mlp_ratio), + out_features=hidden_size, + process_group=gpc.get_group(ParallelMode.TENSOR), + bias=False, + device=device, + dtype=dtype, + ) + else: + self.mlp = ParallelFusedMLP( + hidden_size, + int(hidden_size * mlp_ratio), + out_features=hidden_size, + activation="gelu_approx", + process_group=gpc.get_group(ParallelMode.TENSOR), + bias1=False, + bias2=False, + sequence_parallel=gpc.config.parallel.sequence_parallel, + checkpoint_lvl=0, + heuristic="auto", + device=device, + dtype=dtype, + ) + for _, param in self.mlp.named_parameters(): + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + setattr(param, IS_TENSOR_PARALLEL, True) + self.dropout2 = nn.Dropout(drop_rate) + self.use_swiglu = use_swiglu + self.use_scaled_init = use_scaled_init + self.residual_in_fp32 = residual_in_fp32 # only make sense when using prenorm + self.return_residual = False + self.reset_parameters() + + def reset_parameters(self): + with torch.no_grad(): + for name, param in self.mixer.named_parameters(): + if param.ndim == 1: + param.data.zero_() + elif "Wqkv" in name: + normal_(std=0.006)(param.data) + elif self.use_scaled_init: + scaled_init_method_normal(sigma=0.006, num_layers=self.layer_idx + 1)(param.data) + else: + normal_(std=0.0015)(param.data) + + for name, param in self.mlp.named_parameters(): + if param.ndim == 1 and "bias" in name: + param.data.zero_() + elif self.use_swiglu: + if self.use_scaled_init and "w2" in name: + scaled_init_method_normal(sigma=0.006, num_layers=self.layer_idx + 1)(param.data) + else: + normal_(std=0.006 if "w1" in name or "w2" in name else 0.0015)(param.data) + else: + if self.use_scaled_init and "fc1" not in name: + scaled_init_method_normal(sigma=0.006, num_layers=self.layer_idx + 1)(param.data) + else: + normal_(std=0.006 if "fc1" in name else 0.0015)(param.data) + + def forward(self, hidden_states, cu_seqlens=None, indexes=None, inference_params=None, max_seqlen=None): + if self.checkpoint and self.training: + return activation_checkpoint( + self._forward, False, hidden_states, cu_seqlens, indexes, inference_params, max_seqlen + ) + else: + return self._forward(hidden_states, cu_seqlens, indexes, inference_params, max_seqlen) + + def _forward(self, hidden_states=None, cu_seqlens=None, indexes=None, inference_params=None, max_seqlen=None): + r"""Pass the input through the encoder layer. + + Args: + hidden_states: the sequence to the encoder layer (required). + residual: hidden_states = Attn/MLP(LN(residual)) + cu_seqlens: 1d LongTensor, len(cu_seqlens) = hidden_states + 1 + indexes: the length of index is same as hidden states, which stand for the current position + """ + mixer_kwargs = { + "cu_seqlens": cu_seqlens, + "max_seqlen": max_seqlen, + "indexes": indexes, + "inference_params": inference_params, + } + + def _dropout_and_norm_attn(_hidden_states): + _dropped = self.dropout1(_hidden_states) + _residual = _dropped + _hidden_states = self.norm1(_residual.float()) + return _residual, _hidden_states + + if self.dropout_selective_checkpoint: + residual, hidden_states = activation_checkpoint(_dropout_and_norm_attn, False, hidden_states) + else: + residual, hidden_states = _dropout_and_norm_attn(hidden_states) + + if self.residual_in_fp32: + residual = residual.to(torch.float32) + + hidden_states = self.mixer(hidden_states, **mixer_kwargs) + + def _dropout_and_norm_ffn(_residual, _hidden_states): + _dropped = self.dropout2(_hidden_states) + _residual = (_dropped + _residual) if _residual is not None else _dropped + _hidden_states = self.norm2(_residual.float()) + return _residual, _hidden_states + + if self.dropout_selective_checkpoint: + residual, hidden_states = activation_checkpoint(_dropout_and_norm_ffn, False, residual, hidden_states) + else: + residual, hidden_states = _dropout_and_norm_ffn(residual, hidden_states) + + if self.residual_in_fp32: + residual = residual.to(torch.float32) + + hidden_states = self.mlp(hidden_states) + + return hidden_states + residual + + +class PackedFlashInternLm1D(nn.Module): + """ + 1D Packed Flash InternLm. + + Args: + num_layers (int): The number of layer. 12 by default. + hidden_size (int): The size of hidden state. 768 by default. + num_attention_heads (int): The number of attention head. 12 by default. + vocab_size (int): The size of vocabulary. 50304 by default. + mlp_ratio (int): The ratio of MLP layers. 4 by default. + attn_drop_rate (float): The dropout rate of attention module. 0.0 by default. + drop_rate (float): The dropout rate of input hidden state. 0.0 by default. + dtype (torch.dtype): The type of data. torch.float by default. + checkpoint (float): The proportion of layers that need to be checkpointed compared to the total number + of layers. 0.0 by default. + layer_norm_epsilon (float): A value added to the denominator for numerical stability. 1e-6 by default. + first (bool): Whether input embedding layer or not. False by default. + last (bool): Whether output embedding layer or not. False by default. + embed_split_hidden (bool): Split the embedding layer in the hidden state dimention or vocabulary dimention. + True by default. + embed_grad_scale (float): Refer to GLM-130B, for training stability. 0.1 by default. + parallel_output (bool): If it is necessary to collect the output of parallel computing. True by default. + start_layer_idx (int): The index of start layer in the pipeline. 0 by default. + device (Optional[Union[str, torch.device]]): The device will be used. None by default. + residual_in_fp32 (bool): Whether to use residual in fp32. False by default. + norm_type (str): Normalization type. Use RMSNorm or LayerNorm. "rmsnorm" by default. + use_flash_attn (bool): Whether to use flash-attn. True by default. + + """ + + def __init__( + self, + num_layers: int = 12, + hidden_size: int = 768, + num_attention_heads: int = 12, + vocab_size: int = 50304, + mlp_ratio: int = 4.0, + attn_drop_rate: float = 0.0, + drop_rate: float = 0.0, + dtype: torch.dtype = torch.float, + checkpoint: float = 0.0, + layer_norm_epsilon: float = 1e-5, + first: bool = False, + last: bool = False, + embed_split_hidden: bool = False, + embed_grad_scale: float = 0.1, + parallel_output: bool = True, + start_layer_idx: int = 0, + device: Optional[torch.device] = None, + residual_in_fp32: bool = False, + norm_type: str = "rmsnorm", + is_reward: bool = False, + dropout_selective_checkpoint: bool = True, + use_scaled_init: bool = True, + use_swiglu: bool = True, + use_flash_attn: bool = True, + lvm_config: dict = None, + ): + super().__init__() + self.lvm_config = lvm_config + + checkpoint_layer_num = int(num_layers * checkpoint) + + if is_reward: + head_cls = RewardModelLinear + else: + head_cls = ScaleColumnParallelLinear + if first: + if self.lvm_config.get('enable', False): + self.embedding = Embedding1DLVM(**self.lvm_config.get('embedding_cfg')) + if self.embedding.embed_proj is not None: + for _, param in self.embedding.embed_proj.named_parameters(): + normal_(std=0.0052)(param) + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + setattr(param, IS_TENSOR_PARALLEL, True) + else: + if embed_split_hidden: + self.embedding = Embedding1D(num_embeddings=vocab_size, embedding_dim=hidden_size) + else: + self.embedding = ParallelGPT2Embeddings( + embed_dim=hidden_size, + vocab_size=vocab_size, + max_position_embeddings=-1, + process_group=gpc.get_group(ParallelMode.TENSOR), + padding_idx=None, + sequence_parallel=gpc.config.parallel.sequence_parallel, + device=device, + dtype=dtype, + ) + for _, param in self.embedding.named_parameters(): + normal_(std=0.0052)(param) + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + setattr(param, IS_TENSOR_PARALLEL, True) + self.embed_grad_scale = embed_grad_scale + self.blocks = nn.ModuleList( + [ + PackedFlashBaseLayer1D( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + attn_drop_rate=attn_drop_rate, + drop_rate=drop_rate, + dtype=dtype, + layer_norm_epsilon=layer_norm_epsilon, + checkpoint=lid < checkpoint_layer_num, + layer_idx=lid + start_layer_idx, # This parameter is used for caching during generation + residual_in_fp32=residual_in_fp32, + device=device, + norm_type=norm_type, + dropout_selective_checkpoint=dropout_selective_checkpoint, + use_scaled_init=use_scaled_init, + use_swiglu=use_swiglu, + use_flash_attn=use_flash_attn, + ) + for lid in range(num_layers) + ] + ) + if last: + if norm_type == "rmsnorm": + self.norm = RMSNorm(hidden_size, eps=layer_norm_epsilon) + else: + self.norm = nn.LayerNorm(hidden_size, eps=layer_norm_epsilon) + self.head = head_cls( + in_features=hidden_size, + out_features=gpc.get_world_size(ParallelMode.TENSOR) if is_reward else vocab_size, + process_group=gpc.get_group(ParallelMode.TENSOR), + bias=False, + device=device, + dtype=dtype, + weight_scale=embed_grad_scale, + ) + for _, param in self.head.named_parameters(): + normal_(std=0.0052)(param) + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + setattr(param, IS_TENSOR_PARALLEL, True) + self.parallel_output = parallel_output + + def forward(self, hidden_states=None, cu_seqlens=None, input_ids=None, indexes=None, inference_params=None): + # attention_mask: compute attention on the places where the value is 1 + if hasattr(self, "embedding"): + hidden_states = self.embedding(input_ids) + if self.embed_grad_scale != 1: + hidden_states = ( + self.embed_grad_scale * hidden_states + (1 - self.embed_grad_scale) * hidden_states.detach() + ) + if isinstance(cu_seqlens, list): + assert len(cu_seqlens) == 1 + cu_seqlens = cu_seqlens[0].to(hidden_states.device) + + if cu_seqlens is not None: + cu_seqlens = cu_seqlens.squeeze(0) + hidden_states = hidden_states.squeeze(0) # If cu_seqlens is passed in，it indicated a packed state， + # the batch dimension with a size of 1 should be directly squeezed off. + + if indexes is not None: + assert len(indexes) == 1 + # The indexes are used to indicate the actual position IDs of each token in the packed input. + indexes = indexes[0] + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() if cu_seqlens is not None else None + + for _, block in enumerate(self.blocks): + hidden_states = block( + hidden_states, + cu_seqlens=cu_seqlens, + indexes=indexes, + inference_params=inference_params, + max_seqlen=max_seqlen, + ) + + if hasattr(self, "norm"): + hidden_states = self.norm(hidden_states.float()) + if hasattr(self, "head"): + hidden_states = self.head(hidden_states) + + if not self.parallel_output: + hidden_states = gather_forward_split_backward(hidden_states, ParallelMode.TENSOR, dim=-1) + return hidden_states + + +def _build_generic_model_1d(num_layers, num_chunks, device=torch.device("cuda"), **kwargs): + """ + build generic model 1d + + Args: + num_layers (int): The number of layer. + num_chunks (int): The number of partitions in pipeline parallel. + device (Optional[Union[str, torch.device]]): The device will be used. torch.device("cuda") by default. + + """ + pipeline_size = gpc.get_world_size(ParallelMode.PIPELINE) + pipeline_rank = gpc.get_local_rank(ParallelMode.PIPELINE) + + all_parts = partition_uniform(num_layers, pipeline_size, num_chunks) + parts = all_parts[pipeline_rank] + if gpc.is_rank_for_log(): + logger.info(f"The layer sharding is {all_parts}.") + + models = [] + + for start, end in parts: + kwargs["num_layers"] = end - start + kwargs["first"] = start == 0 + # If there is no content in the final layer, assign the last layer. + kwargs["last"] = end == num_layers and len(all_parts[-1]) != 0 + kwargs["device"] = device + kwargs["start_layer_idx"] = start + chunk = PackedFlashInternLm1D(**filter_kwargs(PackedFlashInternLm1D.__init__, kwargs)).to(device) + + models.append(chunk) + torch.distributed.barrier() + if len(models) == 1: + model = models[0] + else: + model = nn.ModuleList(models) + + return model + + +@MODEL_INITIALIZER.register_module(module_name=MODEL_TYPE) +def build_model_with_cfg( + num_chunks=1, + checkpoint=0.0, + dtype=torch.float, + embed_split_hidden=False, + num_layers=48, + hidden_size=2048, + vocab_size=50304, + embed_grad_scale=1, + parallel_output=True, + num_attention_heads=32, + mlp_ratio=4.0, + residual_in_fp32=False, + norm_type="rmsnorm", + drop_rate=0, + attn_drop_rate=0, + apply_post_layer_norm=False, # pylint: disable=W0613 + layer_norm_epsilon=1e-5, + is_reward=False, + dropout_selective_checkpoint=True, + use_scaled_init: bool = True, + use_swiglu: bool = True, + use_flash_attn: bool = True, + lvm_config=None, +): + """ + Build model with config. + + Args: + num_chunks (int): The number of partitions in pipeline parallel. 1 by default. + checkpoint (bool): Whether to use checkpointing to save VRAM. False by default. + dtype (torch.dtype): The type of data. torch.float by default. + embed_split_hidden (bool): Split the embedding layer in the hidden state dimention or vocabulary dimention. + False by default. + num_layers (int): The number of layer. 48 by default. + hidden_size (int): The size of hidden state. 2048 by default. + vocab_size (int): The size of vocabulary. 50304 by default. + embed_grad_scale (float): Refer to GLM-130B, for training stability. 0.1 by default. + parallel_output (bool): If it is necessary to collect the output of parallel computing. True by default. + num_attention_heads (int): The number of attention head. 32 by default. + mlp_ratio (int): The ratio of MLP layers. 4.0 by default. + residual_in_fp32 (bool): Whether to use residual in fp32. False by default. It cannot be used temporarily + because this parameter requires inconsistent data types to be passed between pipelines, + which requires significant modifications to internlm. + norm_type (str): Normalization type. Use RMSNorm or LayerNorm. "rmsnorm" by default. + drop_rate (float): The dropout rate of input hidden state. 0 by default. + attn_drop_rate (float): The dropout rate of attention module. 0 by default. + apply_post_layer_norm (bool): Whether to apply post layer norm. False by default. + layer_norm_epsilon (float): A value added to the denominator for numerical stability. 1e-5 by default. + is_reward (bool): Whether to use reward model. False by default. + dropout_selective_checkpoint (bool): It can only be enabled when checkpoint is disabled. True by default. + use_scaled_init (bool): Whether to use scaled init. True by default. + use_swiglu (bool): Whether to use swiglu. True by default. + use_flash_attn (bool): Whether to use flash-attn. True by default. + + """ + + cfg = dict( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + checkpoint=checkpoint, + dtype=dtype, + embed_split_hidden=embed_split_hidden, + vocab_size=vocab_size, + embed_grad_scale=embed_grad_scale, + parallel_output=parallel_output, + mlp_ratio=mlp_ratio, + residual_in_fp32=residual_in_fp32, + norm_type=norm_type, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + layer_norm_epsilon=layer_norm_epsilon, + is_reward=is_reward, + dropout_selective_checkpoint=dropout_selective_checkpoint, + use_scaled_init=use_scaled_init, + use_swiglu=use_swiglu, + use_flash_attn=use_flash_attn, + lvm_config=lvm_config, + ) + + return _build_generic_model_1d(num_layers=num_layers, num_chunks=num_chunks, **cfg) diff --git a/InternLM/internlm/model/modeling_vit.py b/InternLM/internlm/model/modeling_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..2580801a4a02b5be1aa36c170da58fc6a7b15c37 --- /dev/null +++ b/InternLM/internlm/model/modeling_vit.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import math +from typing import Optional + +import torch +from flash_attn.modules.embedding import ParallelGPT2Embeddings +from flash_attn.modules.mlp import ParallelFusedMLP +from torch import nn + +from internlm.core.context import IS_TENSOR_PARALLEL, ParallelMode +from internlm.core.context.parallel_context import global_context as gpc +from internlm.initialize.initialize_tensor import normal_, scaled_init_method_normal +from internlm.model.embedding import Embedding1D, Embedding1DLVM +from internlm.model.linear import ( + FeedForward, + RewardModelLinear, + ScaleColumnParallelLinear, +) +from internlm.model.multi_head_attention import MHA +from internlm.model.utils import gather_forward_split_backward, try_import_RMSNorm, try_import_LayerNorm +from internlm.solver.pipeline_utils import partition_uniform +from internlm.utils.checkpoint import activation_checkpoint +from internlm.utils.common import filter_kwargs +from internlm.utils.logger import get_logger +from internlm.utils.registry import MODEL_INITIALIZER + +MODEL_TYPE = "ViT" + +logger = get_logger(__file__) +RMSNorm = try_import_RMSNorm() +LayerNorm = try_import_LayerNorm() + +def drop_path(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + def __init__(self, drop_prob=None, scale_by_keep=True): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) + + +class PackedFlashBaseLayer1D(nn.Module): + """ + 1D Packed Flash Base Layer. + + Args: + hidden_size (int): The hidden size of model. 768 by default. + num_attention_heads (int): The number of attention heads. 12 by default. + mlp_ratio (int): The ratio of MLP layers. 4 by default. + attn_drop_rate (float): The dropout rate of attention module. 0 by default. + drop_path_rate (float): The drop path rate of the input hidden state. 0.0 by default. + dtype (torch.dtype): Type of data. torch.float by default. + layer_norm_epsilon (float): A value added to the denominator for numerical stability. 1e-5 by default. + checkpoint (bool): Whether to use checkpointing to save VRAM. True by default. + layer_idx (int): The index of current layer. 0 by default. + residual_in_fp32 (bool): Whether to use residual in fp32. False by default. + device (Optional[Union[str, torch.device]]): The device will be used. + norm_type (str): Use RMS norm or layernorm."rmsnorm" by default. + use_flash_attn (bool): Whether use flash-attn. True by default. + """ + + def __init__( + self, + hidden_size: int = 768, + num_attention_heads: int = 12, + mlp_ratio: int = 4, + mlp_bias: bool = False, + attn_drop_rate: float = 0, + drop_path_rate: float = 0.0, + dtype: torch.dtype = torch.float, + layer_norm_epsilon: float = 1e-6, + checkpoint: bool = False, + layer_idx: int = 0, + residual_in_fp32: bool = False, + device: Optional[torch.device] = None, + norm_type: str = "rmsnorm", + dropout_selective_checkpoint: bool = True, + use_scaled_init: bool = True, + use_swiglu: bool = True, + use_flash_attn: bool = True, + ): + super().__init__() + self.checkpoint = checkpoint + # dropout selective checkpoint can only be enabled when checkpoint is disabled. + self.dropout_selective_checkpoint = dropout_selective_checkpoint is True and checkpoint is False + self.layer_idx = layer_idx + self.use_flash_attn = use_flash_attn + + head_dim = hidden_size // num_attention_heads + self.mixer = MHA( + embed_dim=hidden_size, + num_heads=num_attention_heads, + process_group=gpc.get_group(ParallelMode.TENSOR), + dropout=attn_drop_rate, + softmax_scale=1 / math.sqrt(head_dim), + causal=True, + layer_idx=layer_idx, + rotary_emb_dim=head_dim, + rotary_emb_scale_base=0, + use_flash_attn=use_flash_attn, + device=device, + dtype=dtype, + ) + + self.dropout1 = DropPath(drop_path_rate) + if norm_type == "rmsnorm": + self.norm1 = RMSNorm(hidden_size, eps=layer_norm_epsilon) + self.norm2 = RMSNorm(hidden_size, eps=layer_norm_epsilon) + else: + self.norm1 = LayerNorm(hidden_size, eps=layer_norm_epsilon) + self.norm2 = LayerNorm(hidden_size, eps=layer_norm_epsilon) + + self.mlp = ParallelFusedMLP( + hidden_size, + int(hidden_size * mlp_ratio), + out_features=hidden_size, + activation="gelu_approx", + process_group=gpc.get_group(ParallelMode.TENSOR), + bias1=mlp_bias, + bias2=mlp_bias, + sequence_parallel=gpc.config.parallel.sequence_parallel, + checkpoint_lvl=0, + heuristic="auto", + device=device, + dtype=dtype, + ) + for _, param in self.mlp.named_parameters(): + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + setattr(param, IS_TENSOR_PARALLEL, True) + self.dropout2 = DropPath(drop_path_rate) + self.use_swiglu = use_swiglu + self.use_scaled_init = use_scaled_init + self.residual_in_fp32 = residual_in_fp32 # only make sense when using prenorm + self.return_residual = False + self.reset_parameters() + + def reset_parameters(self): + with torch.no_grad(): + for name, param in self.mixer.named_parameters(): + if param.ndim == 1: + param.data.zero_() + elif "Wqkv" in name: + normal_(std=0.006)(param.data) + elif self.use_scaled_init: + scaled_init_method_normal(sigma=0.006, num_layers=self.layer_idx + 1)(param.data) + else: + normal_(std=0.0015)(param.data) + + for name, param in self.mlp.named_parameters(): + if param.ndim == 1 and "bias" in name: + param.data.zero_() + elif self.use_swiglu: + if self.use_scaled_init and "w2" in name: + scaled_init_method_normal(sigma=0.006, num_layers=self.layer_idx + 1)(param.data) + else: + normal_(std=0.006 if "w1" in name or "w2" in name else 0.0015)(param.data) + else: + if self.use_scaled_init and "fc1" not in name: + scaled_init_method_normal(sigma=0.006, num_layers=self.layer_idx + 1)(param.data) + else: + normal_(std=0.006 if "fc1" in name else 0.0015)(param.data) + + def forward(self, hidden_states, cu_seqlens=None, indexes=None, inference_params=None, max_seqlen=None): + if self.checkpoint and self.training: + return activation_checkpoint( + self._forward, False, hidden_states, cu_seqlens, indexes, inference_params, max_seqlen + ) + else: + return self._forward(hidden_states, cu_seqlens, indexes, inference_params, max_seqlen) + + def _forward(self, hidden_states=None, cu_seqlens=None, indexes=None, inference_params=None, max_seqlen=None): + r"""Pass the input through the encoder layer. + + Args: + hidden_states: the sequence to the encoder layer (required). + residual: hidden_states = Attn/MLP(LN(residual)) + cu_seqlens: 1d LongTensor, len(cu_seqlens) = hidden_states + 1 + indexes: the length of index is same as hidden states, which stand for the current position + """ + mixer_kwargs = { + "cu_seqlens": cu_seqlens, + "max_seqlen": max_seqlen, + "indexes": indexes, + "inference_params": inference_params, + } + + residual = hidden_states + + hidden_states = self.norm1(residual.float()) + hidden_states = self.mixer(hidden_states, **mixer_kwargs) + hidden_states = self.dropout1(hidden_states) + + residual = residual + hidden_states + + hidden_states = self.norm2(residual.float()) + hidden_states = self.mlp(hidden_states) + hidden_states = self.dropout2(hidden_states) + + return hidden_states + residual + + +class PackedFlashInternLm1D(nn.Module): + """ + 1D Packed Flash InternLm. + + Args: + num_layers (int): The number of layer. 12 by default. + hidden_size (int): The size of hidden state. 768 by default. + num_attention_heads (int): The number of attention head. 12 by default. + vocab_size (int): The size of vocabulary. 50304 by default. + mlp_ratio (int): The ratio of MLP layers. 4 by default. + attn_drop_rate (float): The dropout rate of attention module. 0.0 by default. + drop_path_rate (float): The drop path rate of input hidden state. 0.0 by default. + dtype (torch.dtype): The type of data. torch.float by default. + checkpoint (float): The proportion of layers that need to be checkpointed compared to the total number + of layers. 0.0 by default. + layer_norm_epsilon (float): A value added to the denominator for numerical stability. 1e-6 by default. + first (bool): Whether input embedding layer or not. False by default. + last (bool): Whether output embedding layer or not. False by default. + embed_split_hidden (bool): Split the embedding layer in the hidden state dimention or vocabulary dimention. + True by default. + embed_grad_scale (float): Refer to GLM-130B, for training stability. 0.1 by default. + parallel_output (bool): If it is necessary to collect the output of parallel computing. True by default. + start_layer_idx (int): The index of start layer in the pipeline. 0 by default. + device (Optional[Union[str, torch.device]]): The device will be used. None by default. + residual_in_fp32 (bool): Whether to use residual in fp32. False by default. + norm_type (str): Normalization type. Use RMSNorm or LayerNorm. "rmsnorm" by default. + use_flash_attn (bool): Whether to use flash-attn. True by default. + + """ + + def __init__( + self, + num_layers: int = 12, + hidden_size: int = 768, + num_attention_heads: int = 12, + vocab_size: int = 50304, + mlp_ratio: int = 4.0, + mlp_bias: bool = False, + attn_drop_rate: float = 0.0, + drop_path_rate: float = 0.0, + dtype: torch.dtype = torch.float, + checkpoint: float = 0.0, + layer_norm_epsilon: float = 1e-5, + first: bool = False, + last: bool = False, + embed_split_hidden: bool = False, + embed_grad_scale: float = 0.1, + parallel_output: bool = True, + start_layer_idx: int = 0, + device: Optional[torch.device] = None, + residual_in_fp32: bool = False, + norm_type: str = "rmsnorm", + is_reward: bool = False, + dropout_selective_checkpoint: bool = True, + use_scaled_init: bool = True, + use_swiglu: bool = True, + use_flash_attn: bool = True, + lvm_config: dict = None, + ): + super().__init__() + self.lvm_config = lvm_config + + checkpoint_layer_num = int(num_layers * checkpoint) + + head_cls = ScaleColumnParallelLinear + if first: + if self.lvm_config.get('enable', False): + self.embedding = Embedding1DLVM(**self.lvm_config.get('embedding_cfg')) + if self.embedding.embed_proj is not None: + for _, param in self.embedding.embed_proj.named_parameters(): + normal_(std=0.0052)(param) + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + setattr(param, IS_TENSOR_PARALLEL, True) + else: + if embed_split_hidden: + self.embedding = Embedding1D(num_embeddings=vocab_size, embedding_dim=hidden_size) + else: + self.embedding = ParallelGPT2Embeddings( + embed_dim=hidden_size, + vocab_size=vocab_size, + max_position_embeddings=-1, + process_group=gpc.get_group(ParallelMode.TENSOR), + padding_idx=None, + sequence_parallel=gpc.config.parallel.sequence_parallel, + device=device, + dtype=dtype, + ) + for _, param in self.embedding.named_parameters(): + normal_(std=0.0052)(param) + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + setattr(param, IS_TENSOR_PARALLEL, True) + self.embed_grad_scale = embed_grad_scale + self.blocks = nn.ModuleList( + [ + PackedFlashBaseLayer1D( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + mlp_bias=mlp_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + dtype=dtype, + layer_norm_epsilon=layer_norm_epsilon, + checkpoint=lid < checkpoint_layer_num, + layer_idx=lid + start_layer_idx, # This parameter is used for caching during generation + residual_in_fp32=residual_in_fp32, + device=device, + norm_type=norm_type, + dropout_selective_checkpoint=dropout_selective_checkpoint, + use_scaled_init=use_scaled_init, + use_swiglu=use_swiglu, + use_flash_attn=use_flash_attn, + ) + for lid in range(num_layers) + ] + ) + if last: + if norm_type == "rmsnorm": + self.norm = RMSNorm(hidden_size, eps=layer_norm_epsilon) + else: + self.norm = LayerNorm(hidden_size, eps=layer_norm_epsilon) + self.head = head_cls( + in_features=hidden_size, + out_features=gpc.get_world_size(ParallelMode.TENSOR) if is_reward else vocab_size, + process_group=gpc.get_group(ParallelMode.TENSOR), + bias=False, + device=device, + dtype=dtype, + weight_scale=embed_grad_scale, + ) + for _, param in self.head.named_parameters(): + normal_(std=0.0052)(param) + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + setattr(param, IS_TENSOR_PARALLEL, True) + self.parallel_output = parallel_output + + def forward(self, hidden_states=None, cu_seqlens=None, input_ids=None, indexes=None, inference_params=None): + # attention_mask: compute attention on the places where the value is 1 + if hasattr(self, "embedding"): + hidden_states = self.embedding(input_ids) + if self.embed_grad_scale != 1: + hidden_states = ( + self.embed_grad_scale * hidden_states + (1 - self.embed_grad_scale) * hidden_states.detach() + ) + if isinstance(cu_seqlens, list): + assert len(cu_seqlens) == 1 + cu_seqlens = cu_seqlens[0].to(hidden_states.device) + + if cu_seqlens is not None: + cu_seqlens = cu_seqlens.squeeze(0) + hidden_states = hidden_states.squeeze(0) # If cu_seqlens is passed in，it indicated a packed state， + # the batch dimension with a size of 1 should be directly squeezed off. + + if indexes is not None: + assert len(indexes) == 1 + # The indexes are used to indicate the actual position IDs of each token in the packed input. + indexes = indexes[0] + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() if cu_seqlens is not None else None + + for _, block in enumerate(self.blocks): + hidden_states = block( + hidden_states, + cu_seqlens=cu_seqlens, + indexes=indexes, + inference_params=inference_params, + max_seqlen=max_seqlen, + ) + + if hasattr(self, "norm"): + hidden_states = self.norm(hidden_states.float()) + if hasattr(self, "head"): + hidden_states = self.head(hidden_states) + + if not self.parallel_output: + hidden_states = gather_forward_split_backward(hidden_states, ParallelMode.TENSOR, dim=-1) + return hidden_states + + +def _build_generic_model_1d(num_layers, num_chunks, device=torch.device("cuda"), **kwargs): + """ + build generic model 1d + + Args: + num_layers (int): The number of layer. + num_chunks (int): The number of partitions in pipeline parallel. + device (Optional[Union[str, torch.device]]): The device will be used. torch.device("cuda") by default. + + """ + pipeline_size = gpc.get_world_size(ParallelMode.PIPELINE) + pipeline_rank = gpc.get_local_rank(ParallelMode.PIPELINE) + + all_parts = partition_uniform(num_layers, pipeline_size, num_chunks) + parts = all_parts[pipeline_rank] + if gpc.is_rank_for_log(): + logger.info(f"The layer sharding is {all_parts}.") + + models = [] + + for start, end in parts: + kwargs["num_layers"] = end - start + kwargs["first"] = start == 0 + # If there is no content in the final layer, assign the last layer. + kwargs["last"] = end == num_layers and len(all_parts[-1]) != 0 + kwargs["device"] = device + kwargs["start_layer_idx"] = start + chunk = PackedFlashInternLm1D(**filter_kwargs(PackedFlashInternLm1D.__init__, kwargs)).to(device) + + models.append(chunk) + torch.distributed.barrier() + if len(models) == 1: + model = models[0] + else: + model = nn.ModuleList(models) + + return model + + +@MODEL_INITIALIZER.register_module(module_name=MODEL_TYPE) +def build_vit_model_with_cfg( + num_chunks=1, + checkpoint=0.0, + dtype=torch.float, + embed_split_hidden=False, + num_layers=48, + hidden_size=2048, + vocab_size=50304, + embed_grad_scale=1, + parallel_output=True, + num_attention_heads=32, + mlp_ratio=4.0, + mlp_bias: bool = False, + residual_in_fp32=False, + norm_type="rmsnorm", + drop_path_rate=0, + attn_drop_rate=0, + apply_post_layer_norm=False, # pylint: disable=W0613 + layer_norm_epsilon=1e-5, + is_reward=False, + dropout_selective_checkpoint=True, + use_scaled_init: bool = True, + use_swiglu: bool = True, + use_flash_attn: bool = True, + lvm_config=None, +): + """ + Build model with config. + + Args: + num_chunks (int): The number of partitions in pipeline parallel. 1 by default. + checkpoint (bool): Whether to use checkpointing to save VRAM. False by default. + dtype (torch.dtype): The type of data. torch.float by default. + embed_split_hidden (bool): Split the embedding layer in the hidden state dimention or vocabulary dimention. + False by default. + num_layers (int): The number of layer. 48 by default. + hidden_size (int): The size of hidden state. 2048 by default. + vocab_size (int): The size of vocabulary. 50304 by default. + embed_grad_scale (float): Refer to GLM-130B, for training stability. 0.1 by default. + parallel_output (bool): If it is necessary to collect the output of parallel computing. True by default. + num_attention_heads (int): The number of attention head. 32 by default. + mlp_ratio (int): The ratio of MLP layers. 4.0 by default. + residual_in_fp32 (bool): Whether to use residual in fp32. False by default. It cannot be used temporarily + because this parameter requires inconsistent data types to be passed between pipelines, + which requires significant modifications to internlm. + norm_type (str): Normalization type. Use RMSNorm or LayerNorm. "rmsnorm" by default. + drop_path_rate (float): The drop path rate rate of input hidden state. 0 by default. + attn_drop_rate (float): The dropout rate of attention module. 0 by default. + apply_post_layer_norm (bool): Whether to apply post layer norm. False by default. + layer_norm_epsilon (float): A value added to the denominator for numerical stability. 1e-5 by default. + is_reward (bool): Whether to use reward model. False by default. + dropout_selective_checkpoint (bool): It can only be enabled when checkpoint is disabled. True by default. + use_scaled_init (bool): Whether to use scaled init. True by default. + use_swiglu (bool): Whether to use swiglu. True by default. + use_flash_attn (bool): Whether to use flash-attn. True by default. + + """ + + cfg = dict( + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + checkpoint=checkpoint, + dtype=dtype, + embed_split_hidden=embed_split_hidden, + vocab_size=vocab_size, + embed_grad_scale=embed_grad_scale, + parallel_output=parallel_output, + mlp_ratio=mlp_ratio, + mlp_bias=mlp_bias, + residual_in_fp32=residual_in_fp32, + norm_type=norm_type, + drop_path_rate=drop_path_rate, + attn_drop_rate=attn_drop_rate, + layer_norm_epsilon=layer_norm_epsilon, + is_reward=is_reward, + dropout_selective_checkpoint=dropout_selective_checkpoint, + use_scaled_init=use_scaled_init, + use_swiglu=use_swiglu, + use_flash_attn=use_flash_attn, + lvm_config=lvm_config, + ) + + return _build_generic_model_1d(num_layers=num_layers, num_chunks=num_chunks, **cfg) diff --git a/InternLM/internlm/model/multi_head_attention.py b/InternLM/internlm/model/multi_head_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..d63460503bc8d6c49da5dcb70c871b62a8e95373 --- /dev/null +++ b/InternLM/internlm/model/multi_head_attention.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from typing import Optional + +import torch +from einops import rearrange +from flash_attn.modules.mha import ( + CrossAttention, + FlashCrossAttention, + FlashSelfAttention, + SelfAttention, + _update_kv_cache, +) +from torch import nn + +from internlm.core.context import IS_TENSOR_PARALLEL, ParallelMode +from internlm.core.context import global_context as gpc +from internlm.model.embedding import RotaryEmbedding +from internlm.model.linear import ColumnParallelLinearTorch, RowParallelLinearTorch + + +class MHA(nn.Module): + """ + Multi-head self-attention and cross-attention. + + Args: + embed_dim (int): The dimention of hidden state. + num_heads (int): The number of attention heads. + process_group (torch.distributed.ProcessGroup): The group of the current device for `parallel_mode`. + bias (boolean): Whether the bias is needed for linears. Will be used when initializing QKV matrix and + output projection. True by default. + dropout (float): The dropout rate for cross attention and self attention. 0.0 by default. + softmax_scale (float): The temperature to use for the softmax attention. + causal (boolean): Whether to apply causal attention mask. False by default. + layer_idx (int): The index of current layer. None by default. + rotary_emb_dim (int): The dimention of Rotary Embedding. 0 by default. + rotary_emb_scale_base (int): The scaling factor of Rotary Embedding. If scale_base > 0, this implements + XPos(Sun et al., https://arxiv.org/abs/2212.10554). 0 by default. + use_flash_attn (boolean): Whether to use flash attention or not.If False, vanilla attention module will be used. + False by default. + sequence_parallel (boolean): If True, we're doing Tensor Parallel with sequence parallelism. An all_gather_raw + of x will be done before doing the matmul. + device (Optional[Union[str, torch.device]]): The device will be used. + dtype (Optional[torch.dtype]): The type of data. + use_flash_attn (bool): Whether to use flash-attn. True by default. + + """ + + def __init__( + self, + embed_dim: int, + num_heads: int, + process_group: Optional[torch.distributed.ProcessGroup], + dropout: float = 0.0, + softmax_scale: float = None, + causal: bool = False, + layer_idx: int = None, + rotary_emb_dim: int = 0, + rotary_emb_scale_base: int = 0, + use_flash_attn: bool = True, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.embed_dim = embed_dim + self.causal = causal + self.layer_idx = layer_idx + self.rotary_emb_dim = rotary_emb_dim + self.use_flash_attn = use_flash_attn + self.num_heads = num_heads + assert self.embed_dim % num_heads == 0, "self.kdim must be divisible by num_heads" + self.head_dim = self.embed_dim // num_heads + + if self.rotary_emb_dim > 0: + self.rotary_emb = RotaryEmbedding(self.rotary_emb_dim, scale_base=rotary_emb_scale_base, device=device) + + # notice here should change bias=True + self.Wqkv = ColumnParallelLinearTorch( + embed_dim, + 3 * embed_dim, + process_group, + bias=True, + sequence_parallel=gpc.config.parallel.sequence_parallel, + **factory_kwargs, + ) # according to https://spaces.ac.cn/archives/9577 + + inner_attn_cls = FlashSelfAttention if use_flash_attn else SelfAttention + inner_cross_attn_cls = FlashCrossAttention if use_flash_attn else CrossAttention + self.inner_attn = inner_attn_cls(causal=causal, softmax_scale=softmax_scale, attention_dropout=dropout) + self.inner_cross_attn = inner_cross_attn_cls( + causal=causal, softmax_scale=softmax_scale, attention_dropout=dropout + ) + + # output projection always have the bias (for now) + self.out_proj = RowParallelLinearTorch( + embed_dim, + embed_dim, + process_group, + sequence_parallel=gpc.config.parallel.sequence_parallel, + **factory_kwargs, + ) + # need to assign tp attribute so that internlm know it is tensor parallel module + if gpc.get_world_size(ParallelMode.TENSOR) > 1: + for name in ["out_proj", "Wqkv"]: + for param in getattr(self, name).parameters(): + setattr(param, IS_TENSOR_PARALLEL, True) + + def forward(self, x, seqlen=None, inference_params=None, **kwargs): + if kwargs.get("indexes", None) is not None: + return self._packed_forward(x=x, inference_params=inference_params, **kwargs) + else: + return self._forward(x=x, seqlen=seqlen, inference_params=inference_params, **kwargs) + + def _forward(self, x, seqlen=None, inference_params=None, **kwargs): + """ + Arguments: + x: (batch, seqlen, hidden_dim) (where hidden_dim = num heads * head dim) if seqlen=None. + If seqlen is not None, x is (batch * seqlen, hidden_dim). This is so that when we + split x during sequence parallel, we split the batch * seqlen dimension + (in case batch is small). + """ + qkv = self.Wqkv(x) + if seqlen is None: + qkv = rearrange(qkv, "b s (three h d) -> b s three h d", three=3, d=self.head_dim) + else: + qkv = rearrange(qkv, "(b s) (three h d) -> b s three h d", s=seqlen, three=3, d=self.head_dim) + + if self.rotary_emb_dim > 0: + kwargs["inference_params"] = inference_params + qkv = self.rotary_emb(qkv, **kwargs) + + if inference_params is None: + if gpc.config.model.dtype is torch.float32 and gpc.config.model.use_flash_attn: + with torch.cuda.amp.autocast(dtype=torch.bfloat16): + if qkv.dtype not in [torch.float16, torch.bfloat16]: + qkv = qkv.to(torch.bfloat16) + context = self.inner_attn(qkv).to(x.dtype) + else: + context = self.inner_attn(qkv) + else: + q = qkv[:, :, 0] + assert self.layer_idx is not None, "Generation requires layer_idx in the constructor" + kv = _update_kv_cache(qkv[:, :, 1:], inference_params, self.layer_idx) + # If we're processing the prompt, causal=None (use self.causal). + # If we're decoding, then causal=False. + causal = None if inference_params.sequence_len_offset == 0 else False + context = self.inner_cross_attn(q, kv, causal=causal) + + if seqlen is None: + context = rearrange(context, "b s h d -> b s (h d)") + else: + context = rearrange(context, "b s h d -> (b s) (h d)") + + out = self.out_proj(context) + return out + + def _packed_forward(self, x, inference_params=None, **kwargs): + """ + Arguments: + x: (batch, seqlen, hidden_dim) (where hidden_dim = num heads * head dim) if seqlen=None. + If seqlen is not None, x is (batch * seqlen, hidden_dim). This is so that when we + split x during sequence parallel, we split the batch * seqlen dimension + (in case batch is small). + """ + qkv = self.Wqkv(x) # total x hsz' + qkv = rearrange(qkv, "t (three h d) -> t three h d", three=3, d=self.head_dim) # total x 3 x n_head x d + qkv = self.rotary_emb(qkv, **kwargs) + kwargs.pop("indexes") + + if inference_params is None: + if gpc.config.model.dtype is torch.float32 and gpc.config.model.use_flash_attn: + with torch.cuda.amp.autocast(dtype=torch.bfloat16): + if qkv.dtype not in [torch.float16, torch.bfloat16]: + qkv = qkv.to(torch.bfloat16) + context = self.inner_attn(qkv, **kwargs).to(x.dtype) + else: + context = self.inner_attn(qkv, **kwargs) + + else: + raise RuntimeError("Not support this right now") + + context = rearrange(context, "b h d -> b (h d)") # recover the shape + out = self.out_proj(context) + return out diff --git a/InternLM/internlm/model/muse/__init__.py b/InternLM/internlm/model/muse/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ceaa3939a2455e319af2aec89f15b586e12eda4d --- /dev/null +++ b/InternLM/internlm/model/muse/__init__.py @@ -0,0 +1,18 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.0.1" + +from .modeling_taming_vqgan import VQGANModel diff --git a/InternLM/internlm/model/muse/modeling_taming_vqgan.py b/InternLM/internlm/model/muse/modeling_taming_vqgan.py new file mode 100644 index 0000000000000000000000000000000000000000..ab0003ea7b5da279859d5636660f320ef2cde651 --- /dev/null +++ b/InternLM/internlm/model/muse/modeling_taming_vqgan.py @@ -0,0 +1,591 @@ +# coding=utf-8 +# Copyright 2023 The Taming Transformers Authors and The HuggingFace Inc. team. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from functools import partial +from typing import Tuple + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn + +from .modeling_utils import ConfigMixin, ModelMixin, register_to_config + + +class Upsample(nn.Module): + def __init__(self, in_channels: int, with_conv: bool): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = nn.Conv2d( + in_channels, + in_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, hidden_states): + hidden_states = torch.nn.functional.interpolate(hidden_states, scale_factor=2.0, mode="nearest") + if self.with_conv: + hidden_states = self.conv(hidden_states) + return hidden_states + + +class Downsample(nn.Module): + def __init__(self, in_channels: int, with_conv: bool): + super().__init__() + + self.with_conv = with_conv + if self.with_conv: + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, hidden_states): + if self.with_conv: + pad = (0, 1, 0, 1) # pad height and width dim + hidden_states = torch.nn.functional.pad(hidden_states, pad, mode="constant", value=0) + hidden_states = self.conv(hidden_states) + else: + hidden_states = torch.nn.functional.avg_pool2d(hidden_states, kernel_size=2, stride=2) + return hidden_states + + +class ResnetBlock(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int = None, + use_conv_shortcut: bool = False, + dropout_prob: float = 0.0, + ): + super().__init__() + + self.in_channels = in_channels + self.out_channels = out_channels + self.out_channels_ = self.in_channels if self.out_channels is None else self.out_channels + self.use_conv_shortcut = use_conv_shortcut + + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=3, + stride=1, + padding=1, + ) + + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=self.out_channels_, eps=1e-6, affine=True) + self.dropout = nn.Dropout(dropout_prob) + self.conv2 = nn.Conv2d( + self.out_channels_, + self.out_channels_, + kernel_size=3, + stride=(1, 1), + padding=1, + ) + + if self.in_channels != self.out_channels_: + if use_conv_shortcut: + self.conv_shortcut = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=3, + stride=1, + padding=1, + ) + else: + self.nin_shortcut = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=1, + stride=1, + padding=0, + ) + + def forward(self, hidden_states): + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.in_channels != self.out_channels_: + if self.use_conv_shortcut: + residual = self.conv_shortcut(residual) + else: + residual = self.nin_shortcut(residual) + + return hidden_states + residual + + +class AttnBlock(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + + self.in_channels = in_channels + conv = partial(nn.Conv2d, self.in_channels, self.in_channels, kernel_size=1, stride=1, padding=0) + + self.norm = nn.GroupNorm(num_groups=32, num_channels=self.in_channels, eps=1e-6, affine=True) + self.q, self.k, self.v = conv(), conv(), conv() + self.proj_out = conv() + + def forward(self, hidden_states): + residual = hidden_states + hidden_states = self.norm(hidden_states) + + query = self.q(hidden_states) + key = self.k(hidden_states) + value = self.v(hidden_states) + + # compute attentions + batch, channels, height, width = query.shape + query = query.reshape((batch, channels, height * width)) + query = query.permute(0, 2, 1) # (b, hw, c) + key = key.reshape((batch, channels, height * width)) + + attn_weights = torch.bmm(query, key) # b,hw,hw + attn_weights = attn_weights * (int(channels) ** -0.5) + attn_weights = nn.functional.softmax(attn_weights, dim=2) + + # attend to values + value = value.reshape((batch, channels, height * width)) + attn_weights = attn_weights.permute(0, 2, 1) + hidden_states = torch.bmm(value, attn_weights) + hidden_states = hidden_states.reshape((batch, channels, height, width)) + + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states + residual + return hidden_states + + +class UpsamplingBlock(nn.Module): + def __init__(self, config, curr_res: int, block_idx: int): + super().__init__() + + self.config = config + self.block_idx = block_idx + self.curr_res = curr_res + + if self.block_idx == self.config.num_resolutions - 1: + block_in = self.config.hidden_channels * self.config.channel_mult[-1] + else: + block_in = self.config.hidden_channels * self.config.channel_mult[self.block_idx + 1] + + block_out = self.config.hidden_channels * self.config.channel_mult[self.block_idx] + + res_blocks = [] + attn_blocks = [] + for _ in range(self.config.num_res_blocks + 1): + res_blocks.append(ResnetBlock(block_in, block_out, dropout_prob=self.config.dropout)) + block_in = block_out + if self.curr_res in self.config.attn_resolutions: + attn_blocks.append(AttnBlock(block_in)) + + self.block = nn.ModuleList(res_blocks) + self.attn = nn.ModuleList(attn_blocks) + + self.upsample = None + if self.block_idx != 0: + self.upsample = Upsample(block_in, self.config.resample_with_conv) + + def forward(self, hidden_states): + for i, res_block in enumerate(self.block): + hidden_states = res_block(hidden_states) + if len(self.attn) > 1: + hidden_states = self.attn[i](hidden_states) + + if self.upsample is not None: + hidden_states = self.upsample(hidden_states) + + return hidden_states + + +class DownsamplingBlock(nn.Module): + def __init__(self, config, curr_res: int, block_idx: int): + super().__init__() + + self.config = config + self.curr_res = curr_res + self.block_idx = block_idx + + in_channel_mult = (1,) + tuple(self.config.channel_mult) + block_in = self.config.hidden_channels * in_channel_mult[self.block_idx] + block_out = self.config.hidden_channels * self.config.channel_mult[self.block_idx] + + res_blocks = nn.ModuleList() + attn_blocks = nn.ModuleList() + for _ in range(self.config.num_res_blocks): + res_blocks.append(ResnetBlock(block_in, block_out, dropout_prob=self.config.dropout)) + block_in = block_out + if self.curr_res in self.config.attn_resolutions: + attn_blocks.append(AttnBlock(block_in)) + + self.block = res_blocks + self.attn = attn_blocks + + self.downsample = None + if self.block_idx != self.config.num_resolutions - 1: + self.downsample = Downsample(block_in, self.config.resample_with_conv) + + def forward(self, hidden_states): + for i, res_block in enumerate(self.block): + hidden_states = res_block(hidden_states) + if len(self.attn) > 1: + hidden_states = self.attn[i](hidden_states) + + if self.downsample is not None: + hidden_states = self.downsample(hidden_states) + + return hidden_states + + +class MidBlock(nn.Module): + def __init__(self, config, in_channels: int, no_attn: False, dropout: float): + super().__init__() + + self.config = config + self.in_channels = in_channels + self.no_attn = no_attn + self.dropout = dropout + + self.block_1 = ResnetBlock( + self.in_channels, + self.in_channels, + dropout_prob=self.dropout, + ) + if not no_attn: + self.attn_1 = AttnBlock(self.in_channels) + self.block_2 = ResnetBlock( + self.in_channels, + self.in_channels, + dropout_prob=self.dropout, + ) + + def forward(self, hidden_states): + hidden_states = self.block_1(hidden_states) + if not self.no_attn: + hidden_states = self.attn_1(hidden_states) + hidden_states = self.block_2(hidden_states) + return hidden_states + + +class Encoder(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + + # downsampling + self.conv_in = nn.Conv2d( + self.config.num_channels, + self.config.hidden_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + curr_res = self.config.resolution + downsample_blocks = [] + for i_level in range(self.config.num_resolutions): + downsample_blocks.append(DownsamplingBlock(self.config, curr_res, block_idx=i_level)) + + if i_level != self.config.num_resolutions - 1: + curr_res = curr_res // 2 + self.down = nn.ModuleList(downsample_blocks) + + # middle + mid_channels = self.config.hidden_channels * self.config.channel_mult[-1] + self.mid = MidBlock(config, mid_channels, self.config.no_attn_mid_block, self.config.dropout) + + # end + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=mid_channels, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d( + mid_channels, + self.config.z_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, pixel_values): + # downsampling + hidden_states = self.conv_in(pixel_values) + for block in self.down: + hidden_states = block(hidden_states) + + # middle + hidden_states = self.mid(hidden_states) + + # end + hidden_states = self.norm_out(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class Decoder(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + + # compute in_channel_mult, block_in and curr_res at lowest res + block_in = self.config.hidden_channels * self.config.channel_mult[self.config.num_resolutions - 1] + curr_res = self.config.resolution // 2 ** (self.config.num_resolutions - 1) + self.z_shape = (1, self.config.z_channels, curr_res, curr_res) + + # z to block_in + self.conv_in = nn.Conv2d( + self.config.z_channels, + block_in, + kernel_size=3, + stride=1, + padding=1, + ) + + # middle + self.mid = MidBlock(config, block_in, self.config.no_attn_mid_block, self.config.dropout) + + # upsampling + upsample_blocks = [] + for i_level in reversed(range(self.config.num_resolutions)): + upsample_blocks.append(UpsamplingBlock(self.config, curr_res, block_idx=i_level)) + if i_level != 0: + curr_res = curr_res * 2 + self.up = nn.ModuleList(list(reversed(upsample_blocks))) # reverse to get consistent order + + # end + block_out = self.config.hidden_channels * self.config.channel_mult[0] + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_out, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d( + block_out, + self.config.num_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, hidden_states): + # z to block_in + hidden_states = self.conv_in(hidden_states) + + # middle + hidden_states = self.mid(hidden_states) + + # upsampling + for block in reversed(self.up): + hidden_states = block(hidden_states) + + # end + hidden_states = self.norm_out(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class VectorQuantizer(nn.Module): + """ + see https://github.com/MishaLaskin/vqvae/blob/d761a999e2267766400dc646d82d3ac3657771d4/models/quantizer.py + Discretization bottleneck part of the VQ-VAE. + """ + + def __init__(self, num_embeddings, embedding_dim, commitment_cost): + r""" + Args: + num_embeddings: number of vectors in the quantized space. + embedding_dim: dimensionality of the tensors in the quantized space. + Inputs to the modules must be in this format as well. + commitment_cost: scalar which controls the weighting of the loss terms + (see equation 4 in the paper https://arxiv.org/abs/1711.00937 - this variable is Beta). + """ + super().__init__() + + self.num_embeddings = num_embeddings + self.embedding_dim = embedding_dim + self.commitment_cost = commitment_cost + + self.embedding = nn.Embedding(num_embeddings, embedding_dim) + self.embedding.weight.data.uniform_(-1.0 / num_embeddings, 1.0 / num_embeddings) + + def forward(self, hidden_states, return_loss=False): + """ + Inputs the output of the encoder network z and maps it to a discrete one-hot vector that is the index of the + closest embedding vector e_j z (continuous) -> z_q (discrete) z.shape = (batch, channel, height, width) + quantization pipeline: + 1. get encoder input (B,C,H,W) + 2. flatten input to (B*H*W,C) + """ + # reshape z -> (batch, height, width, channel) and flatten + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() + + distances = self.compute_distances(hidden_states) + min_encoding_indices = torch.argmin(distances, axis=1).unsqueeze(1) + min_encodings = torch.zeros(min_encoding_indices.shape[0], self.num_embeddings).to(hidden_states) + min_encodings.scatter_(1, min_encoding_indices, 1) + + # get quantized latent vectors + z_q = torch.matmul(min_encodings, self.embedding.weight).view(hidden_states.shape) + + # reshape to (batch, num_tokens) + min_encoding_indices = min_encoding_indices.reshape(hidden_states.shape[0], -1) + + # compute loss for embedding + loss = None + if return_loss: + loss = torch.mean((z_q.detach() - hidden_states) ** 2) + self.commitment_cost * torch.mean( + (z_q - hidden_states.detach()) ** 2 + ) + # preserve gradients + z_q = hidden_states + (z_q - hidden_states).detach() + + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + return z_q, min_encoding_indices, loss + + def compute_distances(self, hidden_states): + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z + hidden_states_flattended = hidden_states.reshape((-1, self.embedding_dim)) + emb_weights = self.embedding.weight.t() + + inputs_norm_sq = hidden_states_flattended.pow(2.0).sum(dim=1, keepdim=True) + codebook_t_norm_sq = emb_weights.pow(2.0).sum(dim=0, keepdim=True) + distances = torch.addmm( + inputs_norm_sq + codebook_t_norm_sq, + hidden_states_flattended, + emb_weights, + alpha=-2.0, + ) + return distances + + def get_codebook_entry(self, indices): + # indices are expected to be of shape (batch, num_tokens) + # get quantized latent vectors + batch, num_tokens = indices.shape + z_q = self.embedding(indices) + z_q = z_q.reshape(batch, int(math.sqrt(num_tokens)), int(math.sqrt(num_tokens)), -1).permute(0, 3, 1, 2) + return z_q + + def get_codebook_entry_for_lvm(self, indices): + batch, num_tokens = indices.shape + z_q = self.embedding(indices) + z_q = z_q.reshape(batch, num_tokens, -1) + return z_q + + # adapted from https://github.com/kakaobrain/rq-vae-transformer/blob/main/rqvae/models/rqvae/quantizations.py#L372 + def get_soft_code(self, hidden_states, temp=1.0, stochastic=False): + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() # (batch, height, width, channel) + distances = self.compute_distances(hidden_states) # (batch * height * width, num_embeddings) + + soft_code = F.softmax(-distances / temp, dim=-1) # (batch * height * width, num_embeddings) + if stochastic: + code = torch.multinomial(soft_code, 1) # (batch * height * width, 1) + else: + code = distances.argmin(dim=-1) # (batch * height * width) + + code = code.reshape(hidden_states.shape[0], -1) # (batch, height * width) + batch, num_tokens = code.shape + soft_code = soft_code.reshape(batch, num_tokens, -1) # (batch, height * width, num_embeddings) + return soft_code, code + + def get_code(self, hidden_states): + # reshape z -> (batch, height, width, channel) + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() + distances = self.compute_distances(hidden_states) + indices = torch.argmin(distances, axis=1).unsqueeze(1) + indices = indices.reshape(hidden_states.shape[0], -1) + return indices + + +class VQGANModel(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + resolution: int = 256, + num_channels: int = 3, + hidden_channels: int = 128, + channel_mult: Tuple = (1, 1, 2, 2, 4), + num_res_blocks: int = 2, + attn_resolutions: int = (16,), + no_attn_mid_block: bool = False, + z_channels: int = 256, + num_embeddings: int = 1024, + quantized_embed_dim: int = 256, + dropout: float = 0.0, + resample_with_conv: bool = True, + commitment_cost: float = 0.25, + ): + super().__init__() + + self.config.num_resolutions = len(channel_mult) + self.config.reduction_factor = 2 ** (self.config.num_resolutions - 1) + self.config.latent_size = resolution // self.config.reduction_factor + + self.encoder = Encoder(self.config) + self.decoder = Decoder(self.config) + self.quantize = VectorQuantizer( + self.config.num_embeddings, self.config.quantized_embed_dim, self.config.commitment_cost + ) + self.quant_conv = nn.Conv2d( + self.config.z_channels, + self.config.quantized_embed_dim, + kernel_size=1, + ) + self.post_quant_conv = nn.Conv2d( + self.config.quantized_embed_dim, + self.config.z_channels, + kernel_size=1, + ) + + def encode(self, pixel_values, return_loss=False): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quantized_states, codebook_indices, codebook_loss = self.quantize(hidden_states, return_loss) + output = (quantized_states, codebook_indices) + if return_loss: + output = output + (codebook_loss,) + return output + + def decode(self, quantized_states): + hidden_states = self.post_quant_conv(quantized_states) + reconstructed_pixel_values = self.decoder(hidden_states) + return reconstructed_pixel_values + + def decode_code(self, codebook_indices): + quantized_states = self.quantize.get_codebook_entry(codebook_indices) + reconstructed_pixel_values = self.decode(quantized_states) + return reconstructed_pixel_values + + def get_code(self, pixel_values): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + codebook_indices = self.quantize.get_code(hidden_states) + return codebook_indices + + def forward(self, pixel_values, return_loss=False): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quantized_states, codebook_indices, codebook_loss = self.quantize(hidden_states, return_loss) + reconstructed_pixel_values = self.decode(quantized_states) + outputs = (reconstructed_pixel_values, quantized_states, codebook_indices) + if return_loss: + outputs = outputs + (codebook_loss,) + return outputs diff --git a/InternLM/internlm/model/muse/modeling_utils.py b/InternLM/internlm/model/muse/modeling_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b1f9988fdfd802fc85ce290dc98d5d35d228d045 --- /dev/null +++ b/InternLM/internlm/model/muse/modeling_utils.py @@ -0,0 +1,1171 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import inspect +import json +import os +from collections import OrderedDict +from functools import partial +from pathlib import PosixPath +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import accelerate +import numpy as np +import torch +from accelerate.utils import set_module_tensor_to_device +from huggingface_hub import hf_hub_download +from huggingface_hub.utils import ( + EntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from requests import HTTPError +from torch import Tensor, device + +from . import __version__ +from internlm.utils.logger import get_logger + +logger = get_logger(__file__) + + +hf_cache_home = os.path.expanduser( + os.getenv("HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface")) +) +default_cache_path = os.path.join(hf_cache_home, "muse") + + +CONFIG_NAME = "config.json" +WEIGHTS_NAME = "pytorch_model.bin" +SAFETENSORS_WEIGHTS_NAME = "pytorch_model.safetensors" +HUGGINGFACE_CO_RESOLVE_ENDPOINT = "https://huggingface.co" +MUSE_CACHE = default_cache_path +MUSE_DYNAMIC_MODULE_NAME = "myse_modules" +HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(hf_cache_home, "modules")) + + +_LOW_CPU_MEM_USAGE_DEFAULT = True + + +def get_parameter_device(parameter: torch.nn.Module): + try: + return next(parameter.parameters()).device + except StopIteration: + # For torch.nn.DataParallel compatibility in PyTorch 1.5 + + def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] + return tuples + + gen = parameter._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].device + + +def get_parameter_dtype(parameter: torch.nn.Module): + try: + return next(parameter.parameters()).dtype + except StopIteration: + # For torch.nn.DataParallel compatibility in PyTorch 1.5 + + def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] + return tuples + + gen = parameter._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].dtype + + +def load_state_dict(checkpoint_file: Union[str, os.PathLike]): + """ + Reads a checkpoint file, returning properly formatted errors if they arise. + """ + try: + if os.path.basename(checkpoint_file) == WEIGHTS_NAME: + return torch.load(checkpoint_file, map_location="cpu") + except Exception as e: + try: + with open(checkpoint_file) as f: + if f.read().startswith("version"): + raise OSError( + "You seem to have cloned a repository without having git-lfs installed. Please install " + "git-lfs and run `git lfs install` followed by `git lfs pull` in the folder " + "you cloned." + ) + else: + raise ValueError( + f"Unable to locate the file {checkpoint_file} which is necessary to load this pretrained " + "model. Make sure you have saved the model properly." + ) from e + except (UnicodeDecodeError, ValueError): + raise OSError( + f"Unable to load weights from checkpoint file for '{checkpoint_file}' " + f"at '{checkpoint_file}'. " + "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True." + ) + + +def _load_state_dict_into_model(model_to_load, state_dict): + # Convert old format to new format if needed from a PyTorch state_dict + # copy state_dict so _load_from_state_dict can modify it + state_dict = state_dict.copy() + error_msgs = [] + + # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants + # so we need to apply the function recursively. + def load(module: torch.nn.Module, prefix=""): + args = (state_dict, prefix, {}, True, [], [], error_msgs) + module._load_from_state_dict(*args) + + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + ".") + + load(model_to_load) + + return error_msgs + + +def _get_model_file( + pretrained_model_name_or_path, + *, + weights_name, + subfolder, + cache_dir, + force_download, + proxies, + resume_download, + local_files_only, + use_auth_token, + user_agent, + revision, +): + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + if os.path.isfile(pretrained_model_name_or_path): + return pretrained_model_name_or_path + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, weights_name)): + # Load from a PyTorch checkpoint + model_file = os.path.join(pretrained_model_name_or_path, weights_name) + return model_file + elif subfolder is not None and os.path.isfile( + os.path.join(pretrained_model_name_or_path, subfolder, weights_name) + ): + model_file = os.path.join(pretrained_model_name_or_path, subfolder, weights_name) + return model_file + else: + raise EnvironmentError( + f"Error no file named {weights_name} found in directory {pretrained_model_name_or_path}." + ) + else: + try: + # Load from URL or cache if already cached + model_file = hf_hub_download( + pretrained_model_name_or_path, + filename=weights_name, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision, + ) + return model_file + + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier " + "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a " + "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli " + "login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for " + "this model name. Check the model page at " + f"'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {weights_name}." + ) + except HTTPError as err: + raise EnvironmentError( + f"There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a file named {weights_name} or" + " \nCheckout your internet connection or see how to run the library in" + " offline mode at 'https://huggingface.co/docs/diffusers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a file named {weights_name}" + ) + + +class ModelMixin(torch.nn.Module): + r""" + Base class for all models. + + [`ModelMixin`] takes care of storing the configuration of the models and handles methods for loading, downloading + and saving models. + + - **config_name** ([`str`]) -- A filename under which the model should be stored when calling + [`~models.ModelMixin.save_pretrained`]. + """ + config_name = CONFIG_NAME + _automatically_saved_args = ["_version", "_class_name", "_name_or_path"] + _supports_gradient_checkpointing = False + + def __init__(self): + super().__init__() + + @property + def is_gradient_checkpointing(self) -> bool: + """ + Whether gradient checkpointing is activated for this model or not. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules()) + + def enable_gradient_checkpointing(self): + """ + Activates gradient checkpointing for the current model. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + if not self._supports_gradient_checkpointing: + raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.") + self.apply(partial(self._set_gradient_checkpointing, value=True)) + + def disable_gradient_checkpointing(self): + """ + Deactivates gradient checkpointing for the current model. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + if self._supports_gradient_checkpointing: + self.apply(partial(self._set_gradient_checkpointing, value=False)) + + def set_use_memory_efficient_attention_xformers( + self, valid: bool, attention_op: Optional[Callable] = None + ) -> None: + # Recursively walk through all the children. + # Any children which exposes the set_use_memory_efficient_attention_xformers method + # gets the message + def fn_recursive_set_mem_eff(module: torch.nn.Module): + if hasattr(module, "set_use_memory_efficient_attention_xformers"): + module.set_use_memory_efficient_attention_xformers(valid, attention_op) + + for child in module.children(): + fn_recursive_set_mem_eff(child) + + for module in self.children(): + if isinstance(module, torch.nn.Module): + fn_recursive_set_mem_eff(module) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + r""" + Enable memory efficient attention as implemented in xformers. + + When this option is enabled, you should observe lower GPU memory usage and a potential speed up at inference + time. Speed up at training time is not guaranteed. + + Warning: When Memory Efficient Attention and Sliced attention are both enabled, the Memory Efficient Attention + is used. + + Parameters: + attention_op (`Callable`, *optional*): + Override the default `None` operator for use as `op` argument to the + [`memory_efficient_attention()`](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.memory_efficient_attention) + function of xFormers. + + Examples: + + ```py + >>> import torch + >>> from diffusers import UNet2DConditionModel + >>> from xformers.ops import MemoryEfficientAttentionFlashAttentionOp + + >>> model = UNet2DConditionModel.from_pretrained( + ... "stabilityai/stable-diffusion-2-1", subfolder="unet", torch_dtype=torch.float16 + ... ) + >>> model = model.to("cuda") + >>> model.enable_xformers_memory_efficient_attention(attention_op=MemoryEfficientAttentionFlashAttentionOp) + ``` + """ + self.set_use_memory_efficient_attention_xformers(True, attention_op) + + def disable_xformers_memory_efficient_attention(self): + r""" + Disable memory efficient attention as implemented in xformers. + """ + self.set_use_memory_efficient_attention_xformers(False) + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + is_main_process: bool = True, + save_function: Callable = None, + state_dict: Optional[Dict[str, torch.Tensor]] = None, + ): + """ + Save a model and its configuration file to a directory, so that it can be re-loaded using the + `[`~models.ModelMixin.from_pretrained`]` class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to which to save. Will be created if it doesn't exist. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful on distributed training like TPUs when one + need to replace `torch.save` by another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + state_dict (`Dict[str, torch.Tensor]`, *optional*): + The state dictionary to save. If `None`, the model's state dictionary will be saved. + """ + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + if save_function is None: + save_function = torch.save + + os.makedirs(save_directory, exist_ok=True) + + model_to_save = self + + # Attach architecture to the config + # Save the config + if is_main_process: + model_to_save.save_config(save_directory) + + # Save the model + if state_dict is None: + state_dict = model_to_save.state_dict() + + weights_name = WEIGHTS_NAME + + # Save the model + save_function(state_dict, os.path.join(save_directory, weights_name)) + + logger.info(f"Model weights saved in {os.path.join(save_directory, weights_name)}") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): + r""" + Instantiate a pretrained pytorch model from a pre-trained model configuration. + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train + the model, you should first set it back in training mode with `model.train()`. + + The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come + pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning + task. + + The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those + weights are discarded. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + Valid model ids should have an organization name, like `google/ddpm-celebahq-256`. + - A path to a *directory* containing model weights saved using [`~ModelMixin.save_config`], e.g., + `./my_model_directory/`. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the dtype + will be automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received files. Will attempt to resume the download if such a + file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `diffusers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + from_flax (`bool`, *optional*, defaults to `False`): + Load the model weights from a Flax checkpoint save file. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo (either remote in + huggingface.co or downloaded locally), you can specify the folder name here. + + mirror (`str`, *optional*): + Mirror source to accelerate downloads in China. If you are from China and have an accessibility + problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety. + Please refer to the mirror site for more information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each + parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the + same device. + + To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading by not initializing the weights and only loading the pre-trained weights. This + also tries to not use more than 1x model size in CPU memory (including peak memory) while loading the + model. This is only supported when torch version >= 1.9.0. If you are using an older version of torch, + setting this argument to `True` will raise an error. + + + + It is required to be logged in (`huggingface-cli login`) when you want to use private or [gated + models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + + + Activate the special ["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use + this method in a firewalled environment. + + + + """ + cache_dir = kwargs.pop("cache_dir", MUSE_CACHE) + ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + output_loading_info = kwargs.pop("output_loading_info", False) + local_files_only = kwargs.pop("local_files_only", False) # TODO + use_auth_token = kwargs.pop("use_auth_token", None) + revision = kwargs.pop("revision", None) + torch_dtype = kwargs.pop("torch_dtype", None) + subfolder = kwargs.pop("subfolder", None) + device_map = kwargs.pop("device_map", None) + low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT) + + if low_cpu_mem_usage is False and device_map is not None: + raise ValueError( + f"You cannot set `low_cpu_mem_usage` to `False` while using device_map={device_map} for loading and" + " dispatching. Please make sure to set `low_cpu_mem_usage=True`." + ) + + user_agent = { + "diffusers": __version__, + "file_type": "model", + "framework": "pytorch", + } + + # Load config if we don't provide a configuration + config_path = pretrained_model_name_or_path + + # This variable will flag if we're loading a sharded checkpoint. In this case the archive file is just the + # Load model + + model_file = None + + if model_file is None: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=WEIGHTS_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + + if low_cpu_mem_usage: + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + config, unused_kwargs = cls.load_config( + config_path, + cache_dir=cache_dir, + return_unused_kwargs=True, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + device_map=device_map, + **kwargs, + ) + model = cls.from_config(config, **unused_kwargs) + + # if device_map is None, load the state dict and move the params from meta device to the cpu + if device_map is None: + param_device = "cpu" + state_dict = load_state_dict(model_file) + # move the params from meta device to cpu + missing_keys = set(model.state_dict().keys()) - set(state_dict.keys()) + if len(missing_keys) > 0: + raise ValueError( + f"Cannot load {cls} from {pretrained_model_name_or_path} because the following keys are" + f" missing: \n {', '.join(missing_keys)}. \n Please make sure to pass" + " `low_cpu_mem_usage=False` and `device_map=None` if you want to randomely initialize" + " those weights or else make sure your checkpoint file is correct." + ) + + for param_name, param in state_dict.items(): + accepts_dtype = "dtype" in set(inspect.signature(set_module_tensor_to_device).parameters.keys()) + if accepts_dtype: + set_module_tensor_to_device(model, param_name, param_device, value=param, dtype=torch_dtype) + else: + set_module_tensor_to_device(model, param_name, param_device, value=param) + else: # else let accelerate handle loading and dispatching. + # Load weights and dispatch according to the device_map + # by deafult the device_map is None and the weights are loaded on the CPU + accelerate.load_checkpoint_and_dispatch(model, model_file, device_map, dtype=torch_dtype) + + loading_info = { + "missing_keys": [], + "unexpected_keys": [], + "mismatched_keys": [], + "error_msgs": [], + } + else: + config, unused_kwargs = cls.load_config( + config_path, + cache_dir=cache_dir, + return_unused_kwargs=True, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + device_map=device_map, + **kwargs, + ) + model = cls.from_config(config, **unused_kwargs) + + state_dict = load_state_dict(model_file) + + model, missing_keys, unexpected_keys, mismatched_keys, error_msgs = cls._load_pretrained_model( + model, + state_dict, + model_file, + pretrained_model_name_or_path, + ignore_mismatched_sizes=ignore_mismatched_sizes, + ) + + loading_info = { + "missing_keys": missing_keys, + "unexpected_keys": unexpected_keys, + "mismatched_keys": mismatched_keys, + "error_msgs": error_msgs, + } + + if torch_dtype is not None and not isinstance(torch_dtype, torch.dtype): + raise ValueError( + f"{torch_dtype} needs to be of type `torch.dtype`, e.g. `torch.float16`, but is {type(torch_dtype)}." + ) + elif torch_dtype is not None: + model = model.to(torch_dtype) + + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + + # Set model in evaluation mode to deactivate DropOut modules by default + model.eval() + if output_loading_info: + return model, loading_info + + return model + + @classmethod + def _load_pretrained_model( + cls, + model, + state_dict, + resolved_archive_file, + pretrained_model_name_or_path, + ignore_mismatched_sizes=False, + ): + # Retrieve missing & unexpected_keys + model_state_dict = model.state_dict() + loaded_keys = [k for k in state_dict.keys()] + + expected_keys = list(model_state_dict.keys()) + + original_loaded_keys = loaded_keys + + missing_keys = list(set(expected_keys) - set(loaded_keys)) + unexpected_keys = list(set(loaded_keys) - set(expected_keys)) + + # Make sure we are able to load base models as well as derived models (with heads) + model_to_load = model + + def _find_mismatched_keys( + state_dict, + model_state_dict, + loaded_keys, + ignore_mismatched_sizes, + ): + mismatched_keys = [] + if ignore_mismatched_sizes: + for checkpoint_key in loaded_keys: + model_key = checkpoint_key + + if ( + model_key in model_state_dict + and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape + ): + mismatched_keys.append( + (checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape) + ) + del state_dict[checkpoint_key] + return mismatched_keys + + if state_dict is not None: + # Whole checkpoint + mismatched_keys = _find_mismatched_keys( + state_dict, + model_state_dict, + original_loaded_keys, + ignore_mismatched_sizes, + ) + error_msgs = _load_state_dict_into_model(model_to_load, state_dict) + + if len(error_msgs) > 0: + error_msg = "\n\t".join(error_msgs) + if "size mismatch" in error_msg: + error_msg += ( + "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method." + ) + raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}") + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when" + f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are" + f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task" + " or with another architecture (e.g. initializing a BertForSequenceClassification model from a" + " BertForPreTraining model).\n- This IS NOT expected if you are initializing" + f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly" + " identical (initializing a BertForSequenceClassification model from a" + " BertForSequenceClassification model)." + ) + else: + logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n") + if len(missing_keys) > 0: + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably" + " TRAIN this model on a down-stream task to be able to use it for predictions and inference." + ) + elif len(mismatched_keys) == 0: + logger.info( + f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at" + f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the" + f" checkpoint was trained on, you can already use {model.__class__.__name__} for predictions" + " without further training." + ) + if len(mismatched_keys) > 0: + mismatched_warning = "\n".join( + [ + f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated" + for key, shape1, shape2 in mismatched_keys + ] + ) + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not" + f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be" + " able to use it for predictions and inference." + ) + + return model, missing_keys, unexpected_keys, mismatched_keys, error_msgs + + @property + def device(self) -> device: + """ + `torch.device`: The device on which the module is (assuming that all the module parameters are on the same + device). + """ + return get_parameter_device(self) + + @property + def dtype(self) -> torch.dtype: + """ + `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). + """ + return get_parameter_dtype(self) + + def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int: + """ + Get number of (optionally, trainable or non-embeddings) parameters in the module. + + Args: + only_trainable (`bool`, *optional*, defaults to `False`): + Whether or not to return only the number of trainable parameters + + exclude_embeddings (`bool`, *optional*, defaults to `False`): + Whether or not to return only the number of non-embeddings parameters + + Returns: + `int`: The number of parameters. + """ + + if exclude_embeddings: + embedding_param_names = [ + f"{name}.weight" + for name, module_type in self.named_modules() + if isinstance(module_type, torch.nn.Embedding) + ] + non_embedding_parameters = [ + parameter for name, parameter in self.named_parameters() if name not in embedding_param_names + ] + return sum(p.numel() for p in non_embedding_parameters if p.requires_grad or not only_trainable) + else: + return sum(p.numel() for p in self.parameters() if p.requires_grad or not only_trainable) + + +""" ConfigMixin base class and utilities.""" + + +class FrozenDict(OrderedDict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + for key, value in self.items(): + setattr(self, key, value) + + self.__frozen = True + + def __delitem__(self, *args, **kwargs): + raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.") + + def setdefault(self, *args, **kwargs): + raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance.") + + def pop(self, *args, **kwargs): + raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance.") + + def update(self, *args, **kwargs): + raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance.") + + def __setattr__(self, name, value): + if hasattr(self, "__frozen") and self.__frozen: + raise Exception(f"You cannot use ``__setattr__`` on a {self.__class__.__name__} instance.") + super().__setattr__(name, value) + + def __setitem__(self, name, value): + if hasattr(self, "__frozen") and self.__frozen: + raise Exception(f"You cannot use ``__setattr__`` on a {self.__class__.__name__} instance.") + super().__setitem__(name, value) + + +class ConfigMixin: + r""" + Base class for all configuration classes. Stores all configuration parameters under `self.config` Also handles all + methods for loading/downloading/saving classes inheriting from [`ConfigMixin`] with + - [`~ConfigMixin.from_config`] + - [`~ConfigMixin.save_config`] + + Class attributes: + - **config_name** (`str`) -- A filename under which the config should stored when calling + [`~ConfigMixin.save_config`] (should be overridden by parent class). + - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be + overridden by subclass). + - **has_compatibles** (`bool`) -- Whether the class has compatible classes (should be overridden by subclass). + - **_deprecated_kwargs** (`List[str]`) -- Keyword arguments that are deprecated. Note that the init function + should only have a `kwargs` argument if at least one argument is deprecated (should be overridden by + subclass). + """ + config_name = None + ignore_for_config = [] + has_compatibles = False + + _deprecated_kwargs = [] + + def register_to_config(self, **kwargs): + if self.config_name is None: + raise NotImplementedError(f"Make sure that {self.__class__} has defined a class name `config_name`") + # Special case for `kwargs` used in deprecation warning added to schedulers + # TODO: remove this when we remove the deprecation warning, and the `kwargs` argument, + # or solve in a more general way. + kwargs.pop("kwargs", None) + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error(f"Can't set {key} with value {value} for {self}") + raise err + + if not hasattr(self, "_internal_dict"): + internal_dict = kwargs + else: + previous_dict = dict(self._internal_dict) + internal_dict = {**self._internal_dict, **kwargs} + logger.debug(f"Updating config from {previous_dict} to {internal_dict}") + + self._internal_dict = FrozenDict(internal_dict) + + def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + """ + Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the + [`~ConfigMixin.from_config`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the configuration JSON file will be saved (will be created if it does not exist). + """ + if os.path.isfile(save_directory): + raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file") + + os.makedirs(save_directory, exist_ok=True) + + # If we save using the predefined names, we can load using `from_config` + output_config_file = os.path.join(save_directory, self.config_name) + + self.to_json_file(output_config_file) + logger.info(f"Configuration saved in {output_config_file}") + + @classmethod + def from_config(cls, config: Union[FrozenDict, Dict[str, Any]] = None, **kwargs): + r""" + Instantiate a Python class from a config dictionary + + Parameters: + config (`Dict[str, Any]`): + A config dictionary from which the Python class will be instantiated. Make sure to only load + configuration files of compatible classes. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + Whether kwargs that are not consumed by the Python class should be returned or not. + + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to update the configuration object (after it being loaded) and initiate the Python class. + `**kwargs` will be directly passed to the underlying scheduler/model's `__init__` method and eventually + overwrite same named arguments of `config`. + + Examples: + + ```python + >>> from diffusers import DDPMScheduler, DDIMScheduler, PNDMScheduler + + >>> # Download scheduler from huggingface.co and cache. + >>> scheduler = DDPMScheduler.from_pretrained("google/ddpm-cifar10-32") + + >>> # Instantiate DDIM scheduler class with same config as DDPM + >>> scheduler = DDIMScheduler.from_config(scheduler.config) + + >>> # Instantiate PNDM scheduler class with same config as DDPM + >>> scheduler = PNDMScheduler.from_config(scheduler.config) + ``` + """ + # <===== TO BE REMOVED WITH DEPRECATION + # TODO(Patrick) - make sure to remove the following lines when config=="model_path" is deprecated + if "pretrained_model_name_or_path" in kwargs: + config = kwargs.pop("pretrained_model_name_or_path") + + if config is None: + raise ValueError("Please make sure to provide a config as the first positional argument.") + # ======> + + # Return model and optionally state and/or unused_kwargs + model = cls(**config) + return model + + @classmethod + def load_config( + cls, pretrained_model_name_or_path: Union[str, os.PathLike], return_unused_kwargs=False, **kwargs + ) -> Tuple[Dict[str, Any], Dict[str, Any]]: + r""" + Instantiate a Python class from a config dictionary + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* of a model repo on huggingface.co. Valid model ids should have an + organization name, like `google/ddpm-celebahq-256`. + - A path to a *directory* containing model weights saved using [`~ConfigMixin.save_config`], e.g., + `./my_model_directory/`. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received files. Will attempt to resume the download if such a + file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `transformers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo (either remote in + huggingface.co or downloaded locally), you can specify the folder name here. + + + + It is required to be logged in (`huggingface-cli login`) when you want to use private or [gated + models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + + + Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to + use this method in a firewalled environment. + + + """ + cache_dir = kwargs.pop("cache_dir", MUSE_CACHE) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + _ = kwargs.pop("mirror", None) + subfolder = kwargs.pop("subfolder", None) + + user_agent = {"file_type": "config"} + + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + + if cls.config_name is None: + raise ValueError( + "`self.config_name` is not defined. Note that one should not load a config from " + "`ConfigMixin`. Please make sure to define `config_name` in a class inheriting from `ConfigMixin`" + ) + + if os.path.isfile(pretrained_model_name_or_path): + config_file = pretrained_model_name_or_path + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, cls.config_name)): + # Load from a PyTorch checkpoint + config_file = os.path.join(pretrained_model_name_or_path, cls.config_name) + elif subfolder is not None and os.path.isfile( + os.path.join(pretrained_model_name_or_path, subfolder, cls.config_name) + ): + config_file = os.path.join(pretrained_model_name_or_path, subfolder, cls.config_name) + else: + raise EnvironmentError( + f"Error no file named {cls.config_name} found in directory {pretrained_model_name_or_path}." + ) + else: + try: + # Load from URL or cache if already cached + config_file = hf_hub_download( + pretrained_model_name_or_path, + filename=cls.config_name, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision, + ) + + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier" + " listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a" + " token having permission to this repo with `use_auth_token` or log in with `huggingface-cli" + " login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for" + " this model name. Check the model page at" + f" 'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {cls.config_name}." + ) + except HTTPError as err: + raise EnvironmentError( + "There was a specific connection error when trying to load" + f" {pretrained_model_name_or_path}:\n{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a {cls.config_name} file.\nCheckout your internet connection or see how to" + " run the library in offline mode at" + " 'https://huggingface.co/docs/diffusers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load config for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a {cls.config_name} file" + ) + + try: + # Load config dict + config_dict = cls._dict_from_json_file(config_file) + except (json.JSONDecodeError, UnicodeDecodeError): + raise EnvironmentError(f"It looks like the config file at '{config_file}' is not a valid JSON file.") + + if return_unused_kwargs: + return config_dict, kwargs + + return config_dict + + @staticmethod + def _get_init_keys(cls): + return set(dict(inspect.signature(cls.__init__).parameters).keys()) + + @classmethod + def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]): + with open(json_file, "r", encoding="utf-8") as reader: + text = reader.read() + return json.loads(text) + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + @property + def config(self) -> Dict[str, Any]: + """ + Returns the config of the class as a frozen dictionary + + Returns: + `Dict[str, Any]`: Config of the class. + """ + return self._internal_dict + + def to_json_string(self) -> str: + """ + Serializes this instance to a JSON string. + + Returns: + `str`: String containing all the attributes that make up this configuration instance in JSON format. + """ + config_dict = self._internal_dict if hasattr(self, "_internal_dict") else {} + config_dict["_class_name"] = self.__class__.__name__ + config_dict["_version"] = __version__ + + def to_json_saveable(value): + if isinstance(value, np.ndarray): + value = value.tolist() + elif isinstance(value, PosixPath): + value = str(value) + return value + + config_dict = {k: to_json_saveable(v) for k, v in config_dict.items()} + return json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save this instance to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file in which this configuration instance's parameters will be saved. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + writer.write(self.to_json_string()) + + +def register_to_config(init): + r""" + Decorator to apply on the init of classes inheriting from [`ConfigMixin`] so that all the arguments are + automatically sent to `self.register_for_config`. To ignore a specific argument accepted by the init but that + shouldn't be registered in the config, use the `ignore_for_config` class variable + + Warning: Once decorated, all private arguments (beginning with an underscore) are trashed and not sent to the init! + """ + + @functools.wraps(init) + def inner_init(self, *args, **kwargs): + # Ignore private kwargs in the init. + init_kwargs = {k: v for k, v in kwargs.items() if not k.startswith("_")} + config_init_kwargs = {k: v for k, v in kwargs.items() if k.startswith("_")} + if not isinstance(self, ConfigMixin): + raise RuntimeError( + f"`@register_for_config` was applied to {self.__class__.__name__} init method, but this class does " + "not inherit from `ConfigMixin`." + ) + + ignore = getattr(self, "ignore_for_config", []) + # Get positional arguments aligned with kwargs + new_kwargs = {} + signature = inspect.signature(init) + parameters = { + name: p.default for i, (name, p) in enumerate(signature.parameters.items()) if i > 0 and name not in ignore + } + for arg, name in zip(args, parameters.keys()): + new_kwargs[name] = arg + + # Then add all kwargs + new_kwargs.update( + { + k: init_kwargs.get(k, default) + for k, default in parameters.items() + if k not in ignore and k not in new_kwargs + } + ) + new_kwargs = {**config_init_kwargs, **new_kwargs} + getattr(self, "register_to_config")(**new_kwargs) + init(self, *args, **init_kwargs) + + return inner_init diff --git a/InternLM/internlm/model/norm.py b/InternLM/internlm/model/norm.py new file mode 100644 index 0000000000000000000000000000000000000000..51bcbaf39899430517f323902d6b1fecd64a22fd --- /dev/null +++ b/InternLM/internlm/model/norm.py @@ -0,0 +1,46 @@ +# adopted from https://github.com/NVIDIA/apex/blob/master/apex/normalization/fused_layer_norm + +import numbers + +import torch +from torch.nn import init +from torch.nn.parameter import Parameter + + +def manual_rms_norm(my_input, normalized_shape, weight, eps): + # layer norm should always be calculated in float32 + dims = tuple(i for i in range(-1, -len(normalized_shape) - 1, -1)) + variance = my_input.to(torch.float32).pow(2).mean(dims, keepdim=True) + my_input = my_input * torch.rsqrt(variance + eps) + + if weight is None: + return my_input + + # model_hf into half-precision if necessary + if weight.dtype in [torch.float16, torch.bfloat16]: + my_input = my_input.to(weight.dtype) + + return weight * my_input + + +class RMSNormTorch(torch.nn.Module): + """A custom PyTorch module for RMS normalization.""" + + def __init__(self, normalized_shape, eps=1e-5): + super().__init__() + + if isinstance(normalized_shape, numbers.Integral): + normalized_shape = (normalized_shape,) + self.normalized_shape = torch.Size(normalized_shape) + self.eps = eps + self.weight = Parameter(torch.empty(*normalized_shape)) + self.reset_parameters() + + def forward(self, _input: torch.Tensor): + return manual_rms_norm(_input, self.normalized_shape, self.weight, self.eps) + + def reset_parameters(self): + init.ones_(self.weight) + + def extra_repr(self): + return "{normalized_shape}, eps={eps}, ".format(**self.__dict__) diff --git a/InternLM/internlm/model/utils.py b/InternLM/internlm/model/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3d43ba8746faffc09b74e4c5e88073d5356b2234 --- /dev/null +++ b/InternLM/internlm/model/utils.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from typing import Optional + +import torch +import torch.nn.functional as F +from flash_attn.ops.fused_dense import FusedDenseFunc +from flash_attn.utils.distributed import ( + all_gather_raw, + all_reduce_raw, + reduce_scatter_raw, +) +from torch import Tensor +from torch.cuda.amp import custom_bwd +from torch.distributed import ProcessGroup + +from internlm.core.context import global_context as gpc +from internlm.utils.logger import get_logger + +logger = get_logger(__file__) + + +def _split(input_, parallel_mode, dim=-1): + # skip if only one rank involved + world_size = gpc.get_world_size(parallel_mode) + if world_size == 1: + return input_ + + # Split along last dimension. + dim_size = input_.size(dim) + assert dim_size % world_size == 0, ( + f"The dimension to split ({dim_size}) is not a multiple of world size ({world_size}), " + f"cannot split tensor evenly" + ) + + tensor_list = torch.split(input_, dim_size // world_size, dim=dim) + rank = gpc.get_local_rank(parallel_mode) + output = tensor_list[rank].contiguous() + + return output + + +def _gather(input_, parallel_mode, dim=-1): + # skip if only one rank involved + world_size = gpc.get_world_size(parallel_mode) + if world_size == 1: + return input_ + + # all gather + rank = gpc.get_local_rank(parallel_mode) + tensor_list = [torch.empty_like(input_) for _ in range(world_size)] + tensor_list[rank] = input_ + group = gpc.get_cpu_group(parallel_mode) if input_.device.type == "cpu" else gpc.get_group(parallel_mode) + torch.distributed.all_gather(tensor_list, input_, group=group) + + # concat + output = torch.cat(tensor_list, dim=dim).contiguous() + + return output + + +class _GatherForwardSplitBackward(torch.autograd.Function): + """Gather the input from model parallel region and concatenate. + + Args: + input_: input matrix. + parallel_mode: parallel mode. + dim: dimension + """ + + @staticmethod + def symbolic(input_): + return _gather(input_, parallel_mode=None) + + @staticmethod + def forward(ctx, input_, parallel_mode, dim): + ctx.mode = parallel_mode + ctx.dim = dim + return _gather(input_, parallel_mode, dim) + + @staticmethod + def backward(ctx, grad_output): + return _split(grad_output, ctx.mode, ctx.dim), None, None + + +def gather_forward_split_backward(input_, parallel_mode, dim): + return _GatherForwardSplitBackward.apply(input_, parallel_mode, dim) + + +def linear_bias_wgrad_torch(my_input, grad_output, has_d_bias): + assert my_input.dtype == grad_output.dtype + grad_weight = torch.matmul(grad_output.t(), my_input) + grad_bias = grad_output.sum(dim=0) if has_d_bias else None + return grad_weight, grad_bias + + +# adpated from https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/ops/fused_dense.py +class FusedDenseFuncTorch(FusedDenseFunc): + """A custom PyTorch module extending FusedDenseFunc.""" + + @staticmethod + @custom_bwd + def backward(ctx, grad_output, *args): + grad_output = grad_output.contiguous() + if ctx.return_residual: + (grad_input,) = args + grad_input = grad_input.contiguous() + process_group = ctx.process_group + sequence_parallel = ctx.sequence_parallel + if ctx.compute_weight_gradient: + x, weight = ctx.saved_tensors + if process_group is not None and sequence_parallel: + total_x, handle_x = all_gather_raw(x, process_group, async_op=True) + else: + total_x = x + else: + (weight,) = ctx.saved_tensors + total_x = None + batch_shape = grad_output.shape[:-1] + batch_dim = batch_shape.numel() + grad_output = grad_output.reshape(batch_dim, grad_output.shape[-1]) + if ctx.needs_input_grad[0]: + if not ctx.return_residual: + grad_input = F.linear(grad_output, weight.t()) + else: + grad_input = torch.addmm(grad_input.reshape(batch_dim, grad_input.shape[-1]), grad_output, weight) + grad_input = grad_input.reshape(*batch_shape, grad_input.shape[-1]) + if process_group is not None: + reduce_fn = reduce_scatter_raw if sequence_parallel else all_reduce_raw + grad_input, handle_grad_input = reduce_fn(grad_input, process_group, async_op=True) + else: + grad_input = None + if ctx.needs_input_grad[1]: + assert ctx.compute_weight_gradient + if process_group is not None and sequence_parallel: + handle_x.wait() + # we remove the cuda independence, which is different from flash_attn. + grad_weight, grad_bias = linear_bias_wgrad_torch( + total_x.reshape(batch_dim, total_x.shape[-1]), grad_output, ctx.needs_input_grad[2] + ) + else: + grad_weight = None + grad_bias = grad_output if ctx.needs_input_grad[2] else None + if process_group is not None and ctx.needs_input_grad[0]: + handle_grad_input.wait() + return grad_input, grad_weight, grad_bias, None, None, None + + +def fused_dense_func_torch( + x: Tensor, + weight: Tensor, + bias: Optional[Tensor] = None, + return_residual: bool = False, + process_group: Optional[ProcessGroup] = None, + sequence_parallel: bool = True, +): + dtype_eligible = x.dtype in [torch.float16, torch.bfloat16] or ( + x.dtype == torch.float32 and torch.is_autocast_enabled() + ) + if x.is_cuda and weight.is_cuda and (bias is None or bias.is_cuda) and dtype_eligible: + return FusedDenseFunc.apply(x, weight, bias, return_residual, process_group, sequence_parallel) + else: + return FusedDenseFuncTorch.apply(x, weight, bias, return_residual, process_group, sequence_parallel) + + +class _SplitForwardGatherBackward(torch.autograd.Function): + """ + Split the input and keep only the corresponding chuck to the rank. + + Args: + input_: input matrix. + parallel_mode: parallel mode. + dim: dimension + """ + + @staticmethod + def symbolic(input_): + return _split(input_, parallel_mode=None) + + @staticmethod + def forward(ctx, input_, parallel_mode, dim): + ctx.mode = parallel_mode + ctx.dim = dim + return _split(input_, parallel_mode, dim) + + @staticmethod + def backward(ctx, grad_output): + return _gather(grad_output, ctx.mode, ctx.dim), None, None + + +def split_forward_gather_backward(input_, parallel_mode, dim): + return _SplitForwardGatherBackward.apply(input_, parallel_mode, dim) + + +def try_import_RMSNorm(): + """ + Try import MixFusedRMSNorm from apex, if failed, return our RMSNorm + + """ + try: + from apex.normalization.fused_layer_norm import MixedFusedRMSNorm as RMSNorm + + return RMSNorm + except ModuleNotFoundError: + logger.warning("The torch implementation for MixFusedRMSNorm is slower than apex. Please note this!") + from internlm.model.norm import RMSNormTorch as RMSNorm + + return RMSNorm + + +def try_import_LayerNorm(): + """ + Try import MixFusedRMSNorm from apex, if failed, return our RMSNorm + + """ + try: + from apex.normalization.fused_layer_norm import MixedFusedLayerNorm as LayerNorm + + return LayerNorm + except ModuleNotFoundError: + import torch.nn as nn + + return nn.LayerNorm \ No newline at end of file diff --git a/InternLM/internlm/monitor/__init__.py b/InternLM/internlm/monitor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2501d66a97b5aabe21862dce2a2abcea02bece52 --- /dev/null +++ b/InternLM/internlm/monitor/__init__.py @@ -0,0 +1,11 @@ +from .alert import initialize_light_monitor, send_heartbeat +from .monitor import initialize_monitor_manager, send_alert_message +from .utils import set_env_var + +__all__ = [ + "send_alert_message", + "initialize_monitor_manager", + "set_env_var", + "initialize_light_monitor", + "send_heartbeat", +] diff --git a/InternLM/internlm/monitor/alert.py b/InternLM/internlm/monitor/alert.py new file mode 100644 index 0000000000000000000000000000000000000000..1772e7fc14bbbd5ea1ba309790d8a534e53c0f70 --- /dev/null +++ b/InternLM/internlm/monitor/alert.py @@ -0,0 +1,104 @@ +import json +import math +import os +import re +import time +from typing import Dict + +import requests + +from internlm.utils.logger import get_logger + +logger = get_logger(__file__) + + +def initialize_light_monitor(monitor_address: str = None): + try: + from uniscale_monitoring import init_monitor + + init_monitor(monitor_address) + except Exception as e: + logger.warning(f"init monitor meet error: {e}") + + +def send_heartbeat(msg_type: str, msg: Dict): + def nan2none(v): + if isinstance(v, float) and math.isnan(v): + return None + return v + + try: + from uniscale_monitoring import send_meta + + data = {} + for k, v in msg.items(): + if isinstance(v, Dict): + for k1, v1 in v.items(): + new_k = f"{k}_{k1}".split(" ")[0] + new_k = re.sub(r"[^a-zA-Z0-9_]", "_", new_k) + data[new_k] = nan2none(v1) + else: + new_k = k.split(" ")[0] + new_k = re.sub(r"[^a-zA-Z0-9_]", "_", new_k) + data[new_k] = nan2none(v) + + if os.getenv("CLUSTER_NAME"): + data.update({"cluster": os.getenv("CLUSTER_NAME")}) + if msg_type == "train_metrics": + data.update({"msg_type": "train_metrics"}) + elif msg_type == "init_time": + data.update({"msg_type": "init_time"}) + elif msg_type == "stage_time": + data.update({"msg_type": "stage_time"}) + send_meta(data, timeout=0.1) + except Exception as e: + logger.warning(f"send heartbeat meet error: {e}") + + +def send_feishu_msg_with_webhook(webhook: str, title: str, message: str): + """ + Use Feishu robot to send messages with the given webhook. + + Args: + webhook (str): The webhook to be used to send message. + title (str): The message title. + message (str): The message body. + + Returns: + The response from the request. Or catch the exception and return None. + + Raises: + Exception: An exception rasied by the HTTP post request. + + """ + + headers = {"Content-Type": "application/json;charset=utf-8"} + msg_body = { + "timestamp": int(time.time()), + "msg_type": "post", + "content": { + "post": { + "zh_cn": { + "title": title, + "content": [ + [ + { + "tag": "text", + "text": message, + }, + ], + ], + }, + }, + }, + } + + try: + res = requests.post(webhook, data=json.dumps(msg_body), headers=headers, timeout=30) + res = res.json() + print(f"Feishu webhook response: {res}") + except Exception as err: # pylint: disable=W0703 + print(f"HTTP Post error: {err}") + res = None + + return res diff --git a/InternLM/internlm/monitor/monitor.py b/InternLM/internlm/monitor/monitor.py new file mode 100644 index 0000000000000000000000000000000000000000..6a3b9dc48c370d1fb6d23160e1ba614a79a4042c --- /dev/null +++ b/InternLM/internlm/monitor/monitor.py @@ -0,0 +1,232 @@ +import os +import signal +import socket +import time +from contextlib import contextmanager +from threading import Thread + +from internlm.core.context import global_context as gpc +from internlm.monitor.alert import send_feishu_msg_with_webhook +from internlm.utils.common import SingletonMeta + +from .utils import get_job_key, set_env_var + + +def send_alert_message(address: str = None, title: str = None, message: str = None): + """ + Send alert messages to the given Feishu webhook address in log rank. + + Args: + address (str): The alert address to be used to send message, defaults to None. + title (str): The message title, defaults to None. + message (str): The message body, defaults to None. + """ + + if address is not None and gpc.is_rank_for_log(): + send_feishu_msg_with_webhook( + webhook=address, + title=title if title else get_job_key(), + message=message, + ) + + +class MonitorTracker(Thread): + """ + Track job status and alert to Feishu during job training. + + Args: + alert_address (str): The Feishu webhook address for sending alerting messages. + check_interval (float): The interval in seconds for monitoring checks. Defaults to 300. + loss_spike_limit (float): The threshold for detecting loss value spikes. Defaults to 1.5. + """ + + def __init__( + self, + alert_address: str, + check_interval: float = 300, + loss_spike_limit: float = 1.5, + ): + super().__init__() + self.alert_address = alert_address + self.check_interval = check_interval + self.loss_spike_limit = loss_spike_limit + self.last_active_time = -1 + self.last_loss_value = -1 + self.stopped = False + self.start() + + def run(self): + """ + start the monitor tracker. + """ + + while not self.stopped: + try: + self._check_stuck() + self._check_loss_spike() + except Exception: + continue + time.sleep(self.check_interval) + + def _check_stuck(self): + """ + Check training status for potential stuck condition. + """ + + new_active_time = -1 + if os.getenv("LAST_ACTIVE_TIMESTAMP") is not None: + new_active_time = os.getenv("LAST_ACTIVE_TIMESTAMP") + if int(new_active_time) <= int(self.last_active_time) and new_active_time != -1: + self._send_alert("Training may be in stuck status, please check it.") + self.last_active_time = new_active_time + + def _check_loss_spike(self): + """ + Check for loss value spikes. + """ + + if gpc.is_rank_for_log(): + new_loss_value = -1 + new_step_id = -1 + if os.getenv("LOSS") is not None: + new_loss_value = os.getenv("LOSS") + if os.getenv("STEP_ID") is not None: + new_step_id = os.getenv("STEP_ID") + + if (float(new_loss_value) / float(self.last_loss_value)) > self.loss_spike_limit and new_loss_value != -1: + assert int(new_step_id) >= 0 + self._send_alert( + f"Checking periodically: Loss spike may be happened in step {new_step_id}, " + f"loss value from {self.last_loss_value} to {new_loss_value}, please check it." + ) + + self.last_loss_value = new_loss_value + + def _send_alert(self, message): + """ + Send alerting message to the Feishu webhook address. + + Args: + message (str): The alerting message to be sent. + """ + + send_alert_message( + address=self.alert_address, + message=message, + ) + + def stop(self): + """ + Stop the monitor tracker. + """ + + self.stopped = True + + +class MonitorManager(metaclass=SingletonMeta): + """ + Monitor Manager for managing monitor thread and monitoring training status. + """ + + def __init__(self, loss_spike_limit: float = 1.5) -> None: + self.monitor_thread = None + self.loss_spike_limit = loss_spike_limit + self.last_step_loss = -1 + + def monitor_loss_spike(self, alert_address: str = None, step_count: int = 0, cur_step_loss: float = 0.0): + """Check loss value, if loss spike occurs, send alert message to Feishu.""" + set_env_var(key="LOSS", value=cur_step_loss) + set_env_var(key="STEP_ID", value=step_count) + + if self.last_step_loss != -1 and cur_step_loss > self.loss_spike_limit * self.last_step_loss: + send_alert_message( + address=alert_address, + message=( + f"Checking step by step: Loss spike may be happened in step {step_count}, " + f"loss value from {self.last_step_loss} to {cur_step_loss}, please check it." + ), + ) + self.last_step_loss = cur_step_loss + + def monitor_exception(self, alert_address: str = None, excp_info: str = None): + """Catch and format exception information, send alert message to Feishu.""" + filtered_trace = excp_info.split("\n")[-10:] + format_trace = "" + for line in filtered_trace: + format_trace += "\n" + line + send_alert_message( + address=alert_address, + message=f"Catch Exception from {socket.gethostname()} with rank id {gpc.get_global_rank()}:{format_trace}", + ) + + def handle_sigterm(self, alert_address: str = None): + """Catch SIGTERM signal, and send alert message to Feishu.""" + + def sigterm_handler(sys_signal, frame): + print("receive frame: ", frame) + print("receive signal: ", sys_signal) + send_alert_message( + address=alert_address, + message=f"Process received signal {signal} and exited.", + ) + + signal.signal(signal.SIGTERM, sigterm_handler) + + def start_monitor( + self, + job_name: str, + alert_address: str, + monitor_interval_seconds: int = 300, + loss_spike_limit: float = 1.5, + ): + """ + Initialize and start monitor thread for checking training job status, loss spike and so on. + + Args: + job_name (str): The training job name. + alert_address (str): The Feishu webhook address for sending alert messages. + monitor_interval_seconds (int): The time of monitor interval in seconds, defaults to 300. + loss_spike_limit (float): The limit multiple of current loss to previous loss value, which means loss spike + may be occurs, defaults to 1.5. + """ + + # initialize some variables for monitoring + set_env_var(key="JOB_NAME", value=job_name) + + # start a monitor thread, periodically check the training status + self.monitor_thread = MonitorTracker( + alert_address=alert_address, + check_interval=monitor_interval_seconds, + loss_spike_limit=loss_spike_limit, + ) + + def stop_monitor(self): + """Stop the monitor and alert thread.""" + if self.monitor_thread is not None: + self.monitor_thread.stop() + + +monitor_manager = MonitorManager() + + +@contextmanager +def initialize_monitor_manager(job_name: str = None, alert_address: str = None): + """ + Initialize monitor manager for monitoring training lifetime and alerting exception info to Feishu. + + Args: + job_name (str): The training job name. + alert_address (str): The Feishu webhook address for sending alert messages. + """ + + if alert_address is not None: + try: + monitor_manager.start_monitor(job_name=job_name, alert_address=alert_address) + monitor_manager.handle_sigterm(alert_address=alert_address) + send_alert_message(address=alert_address, message=f"Training in {socket.gethostname()} is starting.") + yield + finally: + send_alert_message(address=alert_address, message=f"Training in {socket.gethostname()} completed.") + monitor_manager.stop_monitor() + else: + yield diff --git a/InternLM/internlm/monitor/utils.py b/InternLM/internlm/monitor/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f64c7dcb552f14b43794b646987d30c576e8abe8 --- /dev/null +++ b/InternLM/internlm/monitor/utils.py @@ -0,0 +1,32 @@ +import os +from datetime import datetime + + +def now_time(): + return datetime.now().strftime("%b%d_%H-%M-%S") + + +def set_env_var(key, value): + os.environ[str(key)] = str(value) + + +def get_job_id(): + job_id = "none" + if os.getenv("SLURM_JOB_ID") is not None: + job_id = os.getenv("SLURM_JOB_ID") + elif os.getenv("K8S_WORKSPACE_ID") is not None: + job_id = os.getenv("K8S_WORKSPACE_ID") + + return job_id + + +def get_job_name(): + job_name = f"unknown-{now_time()}" + if os.getenv("JOB_NAME") is not None: + job_name = os.getenv("JOB_NAME") + + return job_name + + +def get_job_key(): + return f"{get_job_id()}_{get_job_name()}" diff --git a/InternLM/internlm/solver/__init__.py b/InternLM/internlm/solver/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..773f2dc8c8b589e666edb9f9c00147d541d06937 --- /dev/null +++ b/InternLM/internlm/solver/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from .beta2_scheduler import Beta2Scheduler +from .lr_scheduler import FineTuneCosineAnnealingWarmupLR +from .optimizer import HybridZeroOptimizer + +__all__ = ["Beta2Scheduler", "FineTuneCosineAnnealingWarmupLR", "HybridZeroOptimizer"] diff --git a/InternLM/internlm/solver/beta2_scheduler.py b/InternLM/internlm/solver/beta2_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..904f4e049be56bb629f863179a0f73be09f92f88 --- /dev/null +++ b/InternLM/internlm/solver/beta2_scheduler.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import torch + + +class Beta2Scheduler: + """ + Beta2Scheduler + """ + + def __init__(self, optimizer: torch.optim.Adam, init_beta2, c=0.8, cur_iter=-1): + self.cur_iter = 0 if cur_iter == -1 else cur_iter + self.init_beta2 = init_beta2 + self.c = c + self.optimizer = optimizer + assert isinstance( + optimizer, (torch.optim.Adam, torch.optim.AdamW) + ), "should use Adam optimzier, which has beta2" + + def step(self, cur_iter=None): + if cur_iter is None: + self.cur_iter += 1 + else: + self.cur_iter = cur_iter + + new_beta2 = self.get_beta2() + for pg in self.optimizer.param_groups: + beta1, _ = pg["betas"] + pg["betas"] = (beta1, new_beta2) + + def get_beta2(self): + if self.c <= 0: + return self.init_beta2 + scale = 1 - (1 / self.cur_iter**self.c) + return max(self.init_beta2, scale) diff --git a/InternLM/internlm/solver/lr_scheduler.py b/InternLM/internlm/solver/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..d71d3ad96b179b4caf487a2e9e476f9172cee223 --- /dev/null +++ b/InternLM/internlm/solver/lr_scheduler.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import json + +from torch.optim.lr_scheduler import CosineAnnealingLR as _CosineAnnealingLR +from torch.optim.lr_scheduler import _LRScheduler + + +class WarmupScheduler(_LRScheduler): + """Starts with a linear warmup lr schedule until it reaches N epochs then applies + the specific scheduler (For tools: ReduceLROnPlateau). + + Args: + optimizer (:class:`torch.optim.Optimizer`): Wrapped optimizer. + warmup_epochs (int): Number of epochs to linearly warmup lr until starting applying the scheduler. + after_scheduler (:class:`torch.optim.lr_scheduler`): After target_epoch, use this scheduler. + last_epoch (int, optional): The index of last epoch, defaults to -1. When last_epoch=-1, + the schedule is started from the beginning or When last_epoch=-1, sets initial lr as lr. + """ + + def __init__(self, optimizer, warmup_epochs, after_scheduler, last_epoch=-1): + self.warmup_epochs = int(warmup_epochs) + self.after_scheduler = after_scheduler + self.finished = False + super().__init__(optimizer, last_epoch) + + def state_dict(self): + state_dict = {key: value for key, value in self.__dict__.items() if key not in "optimizer"} + if isinstance(state_dict["after_scheduler"], (_LRScheduler, _CosineAnnealingLR)): + state_dict["after_scheduler_type"] = type(state_dict["after_scheduler"]).__name__ + state_dict["after_scheduler_dict"] = state_dict["after_scheduler"].state_dict() + del state_dict["after_scheduler"] + else: + raise NotImplementedError() + return state_dict + + def load_state_dict(self, state_dict): + # state_dict = {key: value for key, value in self.__dict__.items() if key not in 'optimizer'} + for key in list(self.__dict__.keys()): + if key in state_dict: + self.__dict__[key] = state_dict[key] + if isinstance(self.after_scheduler, (_LRScheduler, _CosineAnnealingLR)): + assert type(self.after_scheduler).__name__ == state_dict["after_scheduler_type"] + # state_dict['after_scheduler_dict'] = state_dict['after_scheduler'].state_dict() + self.after_scheduler.load_state_dict(state_dict["after_scheduler_dict"]) + # del state_dict['after_scheduler'] + else: + raise NotImplementedError() + return state_dict + + def get_lr(self): + if self.last_epoch >= self.warmup_epochs: + if not self.finished: + self.after_scheduler.base_lrs = self.base_lrs + self.finished = True + return self.after_scheduler.get_lr() + + return [(self.last_epoch + 1) / self.warmup_epochs * lr for lr in self.base_lrs] + + def step(self, epoch=None): + if self.finished: + if epoch is None: + self.after_scheduler.step(None) + self._last_lr = self.after_scheduler.get_last_lr() + else: + self.after_scheduler.step(epoch - self.warmup_epochs) + self._last_lr = self.after_scheduler.get_last_lr() + else: + return super().step(epoch) + + +class CosineAnnealingWarmupLR(WarmupScheduler): + """Cosine annealing learning rate scheduler with learning rate warmup. A linear warmup schedule will be applied. + + Args: + optimizer (:class:`torch.optim.Optimizer`): Wrapped optimizer. + total_steps (int): Number of total training steps. + warmup_steps (int, optional): Number of warmup steps, defaults to 0. + eta_min (int, optional): Minimum learning rate, defaults to 0. + last_epoch (int, optional): The index of last epoch, defaults to -1. When last_epoch=-1, + the schedule is started from the beginning or When last_epoch=-1, sets initial lr as lr. + """ + + def __init__(self, optimizer, total_steps: int, warmup_steps: int = 0, eta_min: float = 0.0, last_epoch: int = -1): + base_scheduler = _CosineAnnealingLR( + optimizer, total_steps - warmup_steps, eta_min=eta_min, last_epoch=last_epoch + ) + super().__init__(optimizer, warmup_steps, base_scheduler) + + +class FineTuneCosineAnnealingWarmupLR(CosineAnnealingWarmupLR): + """ + FineTune Cosine Annealing Warmup LR. + + Args: + optimizer: The optimizer object. + total_steps (int): The number of total steps. + init_steps (int): The number of init steps, default is 0. + warmup_steps (int): The number of warm up steps, default is 0. + eta_min (float): The minimum learning rate, default is 0.0. + last_epoch: Last epoch, default is -1. + + """ + + def __init__( + self, + optimizer, + total_steps: int, + init_steps: int = 0, + warmup_ratio: float = 0.0, + eta_min: float = 0.0, + last_epoch: int = -1, + ): + self._init_steps = init_steps + self._warmup_steps = int(total_steps * warmup_ratio) + # Use this value to calculate the lr of warmup, because warmup_epochs = init_steps + warmup_steps + super().__init__(optimizer, total_steps, self._warmup_steps + init_steps, eta_min, last_epoch) + + def get_lr(self): + if self.last_epoch >= self.warmup_epochs: + if not self.finished: # pylint: disable=E0203 + # This True switch is to avoid warning when the warmup reaches the preset value switch + self.after_scheduler._get_lr_called_within_step = True + self.after_scheduler.base_lrs = self.base_lrs + self.finished = True + return self.after_scheduler.get_lr() + + elif self.last_epoch >= self._init_steps: + return [(self.last_epoch + 1 - self._init_steps) / self._warmup_steps * lr for lr in self.base_lrs] + else: + return [0 for lr in self.base_lrs] + + def __str__(self): + return json.dumps(self.state_dict(), indent=4, sort_keys=True) diff --git a/InternLM/internlm/solver/optimizer/__init__.py b/InternLM/internlm/solver/optimizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..99051f457d0cd992a10dd2ba80a3add5148fbfd7 --- /dev/null +++ b/InternLM/internlm/solver/optimizer/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from .hybrid_zero_optim import HybridZeroOptimizer, reload_zero_fp32_buff + +__all__ = ["HybridZeroOptimizer", "reload_zero_fp32_buff"] diff --git a/InternLM/internlm/solver/optimizer/hybrid_zero_optim.py b/InternLM/internlm/solver/optimizer/hybrid_zero_optim.py new file mode 100644 index 0000000000000000000000000000000000000000..31af6d364d809485b46ab49d504d08e6aa723441 --- /dev/null +++ b/InternLM/internlm/solver/optimizer/hybrid_zero_optim.py @@ -0,0 +1,815 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import math +from functools import partial +from itertools import product + +import torch +import torch.distributed as dist +from torch.optim import Optimizer + +from internlm.core.context import Config, ParallelMode +from internlm.core.context import global_context as gpc +from internlm.monitor import send_alert_message +from internlm.solver.optimizer.store import ( + BucketStore, + GradientStore, + ParameterStore, + TensorBucket, +) +from internlm.solver.optimizer.utils import ( + DynamicGradScaler, + ParamBcastSyncHandler, + flatten, + get_grad_accumulate_object, + has_inf_or_nan, + reduce_tensor, + release_param_grad, + split_half_float_double, + sync_param, +) +from internlm.utils.common import get_current_device +from internlm.utils.logger import get_logger +from internlm.utils.megatron_timers import megatron_timer as timer +from internlm.utils.timeout import llm_timeout + +from .utils import compute_norm + +inf = math.inf +logger = get_logger(__file__) + + +class BaseOptimizer(Optimizer): + """ + Base Optimizer. + """ + + def __init__(self, optim: Optimizer): # pylint: disable=W0231 + self.optim = optim + + @property + def param_groups(self): + return self.optim.param_groups + + @property + def defaults(self): + return self.optim.defaults + + def add_param_group(self, *args, **kwargs): + return self.optim.add_param_group(*args, **kwargs) + + def step(self, *args, **kwargs): + return self.optim.step(*args, **kwargs) + + def zero_grad(self, *args, **kwargs): + self.optim.zero_grad(*args, **kwargs) + + def load_state_dict(self, *args, **kwargs): + self.optim.load_state_dict(*args, **kwargs) + + def state_dict(self): + return self.optim.state_dict() + + def backward(self, loss): + loss.backward() + + def backward_by_grad(self, tensor, grad): + torch.autograd.backward(tensors=tensor, grad_tensors=grad) + + def clip_grad_norm(self): + pass + + +class HybridZeroOptimizer(BaseOptimizer): + """ + Hybrid Zero Optimizer. + """ + + def __init__( + self, + optimizer: Optimizer, + cpu_offload=False, + grad_scal_cfg: Config = None, + zero_cfg: Config = None, + param_bcast_sync_handler: ParamBcastSyncHandler = None, + ): + # DynamicGradScaler related args + if gpc.config.model.dtype is torch.float32: + initial_scale = 1 + else: + initial_scale = grad_scal_cfg.fp16.initial_scale + min_scale = grad_scal_cfg.fp16.min_scale + growth_interval = grad_scal_cfg.fp16.growth_interval + growth_factor = grad_scal_cfg.growth_factor + backoff_factor = grad_scal_cfg.backoff_factor + hysteresis = grad_scal_cfg.hysteresis + max_scale = grad_scal_cfg.max_scale + + # Zero related args + reduce_bucket_size = zero_cfg.reduce_bucket_size + clip_grad_norm = zero_cfg.clip_grad_norm + self._overlap_sync_grad = zero_cfg.overlap_sync_grad + self._overlap_sync_param = zero_cfg.overlap_sync_param + + super().__init__(optim=optimizer) + + self._dtype = self.optim.param_groups[0]["params"][0].dtype + self._cpu_offload = cpu_offload + self._zero_local_rank = gpc.get_local_rank(ParallelMode.ZERO1) + self._zero_world_size = gpc.get_world_size(ParallelMode.ZERO1) + self._broadcast_parallel_mode = ParallelMode.ZERO1 + + # ParameterStore will manage the tensor buffers used for zero + # it will not manage the tensors used by mixed precision training + self._param_store = ParameterStore(ParallelMode.ZERO1) + self._grad_store = GradientStore(ParallelMode.DATA) + self._bucket_store = BucketStore(ParallelMode.DATA) + self._bucket_in_progress = [] + + # fp16 and fp32 params for mixed precision training + self._fp16_param_groups = dict() + self._fp32_flat_param_groups_of_current_rank = dict() + + # communication params + # self._overlap_communication = overlap_communication + self._reduce_bucket_size = reduce_bucket_size + + self._comm_bcast_stream = torch.cuda.Stream() + + # gradient scaler + self.grad_scaler = DynamicGradScaler( + initial_scale=initial_scale, + min_scale=min_scale, + growth_factor=growth_factor, + backoff_factor=backoff_factor, + growth_interval=growth_interval, + hysteresis=hysteresis, + max_scale=max_scale, + ) + self._found_overflow = torch.cuda.FloatTensor([0], device=get_current_device()) + + # gradient clipping + self._clip_grad_norm = clip_grad_norm + + # need to record the rank in which parameter groups are not assigned parameters. + self.param_group_has_params = [] + self.param_group_no_params_ranks = [] + self.padding_grad = torch.zeros([32], dtype=self._dtype, device=get_current_device()) + self.padding_tensor = torch.zeros([32], dtype=self._dtype, device=get_current_device()) + + self.rank_unique_id = ( + f"gpus-{gpc.get_world_size(ParallelMode.GLOBAL)}_" + + f"pp-{gpc.get_local_rank(ParallelMode.PIPELINE)}_" + + f"tp-{gpc.get_local_rank(ParallelMode.TENSOR)}_" + + f"zo-{self._zero_local_rank}.pt" + ) + self.params_per_rank_id_dict = [] + self._param_bcast_sync_handler = param_bcast_sync_handler + if self._overlap_sync_param: + assert self._param_bcast_sync_handler is not None + + # iterate over the param group in the optimizer + # partition these param groups for data parallel training + # and add buffers to parameter store for future access + for group_id, param_group in enumerate(self.optim.param_groups): + group_params = param_group["params"] + + # add the fp16 params to fp16_param_groups for bookkeeping + self._fp16_param_groups[group_id] = group_params + + # assign parameters to ranks the params in the list are sorted + params_per_rank, no_params_ranks = self._partition_param_list(group_params) + self.param_group_no_params_ranks.append(no_params_ranks) + self.param_group_has_params.append(self._zero_local_rank not in no_params_ranks) + + # store the mapping between param to rank each param should belong to only one rank + for rank, params in enumerate(params_per_rank): + # check whether any rank is not assigned params. + if len(params) != 0: + self._param_store.add_fp16_param_list_by_rank_group(rank, group_id, params) + for param in params: + setattr(param, "group_id", group_id) + self._param_store.set_param_to_rank(param, rank) + + # move to cpu to make room to create the flat tensor + for param in group_params: + param.data = param.data.cpu() + + # flatten the reordered tensors + for rank in range(self._zero_world_size): + # No flat fp16 buffer is allocated if the process has no parameters. + if rank not in self.param_group_no_params_ranks[group_id]: + tensor_list = self._param_store.get_fp16_params_by_rank_group(rank, group_id) + with torch.no_grad(): + flat_tensor = flatten(tensor_list) + flat_tensor = flat_tensor.data.cuda() + self._param_store.add_flat_fp16_param_by_rank_group(rank, group_id, flat_tensor) + sync_param(flat_tensor=flat_tensor, tensor_list=tensor_list) + + # create a copy of fp32 weights of the parameters for which this rank is responsible + # No flat fp32 buffer is allocated if the process has no parameters. + if self.param_group_has_params[group_id]: + fp16_flat_current_rank = self._param_store.get_flat_fp16_param_by_rank_group( + self._zero_local_rank, group_id + ) + fp32_flat_current_rank = fp16_flat_current_rank.float() + device = "cpu" if self._cpu_offload else get_current_device() + fp32_flat_current_rank = fp32_flat_current_rank.to(device) + fp32_flat_current_rank.requires_grad = True + self._fp32_flat_param_groups_of_current_rank[group_id] = fp32_flat_current_rank + + # need to replace the params in the `params` field in the optimizer + # so that when the optimizer calls step(), it only updates the tensors + # managed by this data parallel rank + param_group["params"] = [fp32_flat_current_rank] + + # set reduction state + for param in self._fp16_param_groups[group_id]: + self._param_store.set_param_reduction_state(param, False) + + assert len(self._fp16_param_groups) != 0 + + # If a rank is not assigned any arguments, 'has_params' is False. + self.has_params = sum(self.param_group_has_params) != 0 + # flag used to skip unnecessary gradient reduce operation when gradient accumulation is enabled. + self.skip_grad_reduce = False + + # reduction hook is only used if overlapping communication + # if it is stage 1 without overlapping, no hook will be attached + if self._overlap_sync_grad: + self._attach_reduction_hook() + + @property + def zero_local_rank(self): + return self._zero_local_rank + + @property + def zero_world_size(self): + return self._zero_world_size + + @property + def dtype(self): + return self._dtype + + @property + def loss_scale(self): + return self.grad_scaler.scale + + @property + def num_param_groups(self): + return len(self._fp16_param_groups) + + def _partition_param_list(self, param_list): + no_params_ranks = [] + params_per_rank = [[] for _ in range(self._zero_world_size)] + numel_per_rank = [0 for _ in range(self._zero_world_size)] + self.params_per_rank_id_dict.append([[] for _ in range(self._zero_world_size)]) + + sorted_params = sorted(param_list, key=lambda x: x.numel(), reverse=True) + for i, param in enumerate(sorted_params): + global_id = str(i) + for j in range(len(param.size())): + global_id = "_".join([global_id, str(param.size()[j])]) + if self._overlap_sync_param: + rank_to_go = self._param_bcast_sync_handler.get_rank_by_param(param) + else: + rank_to_go = numel_per_rank.index(min(numel_per_rank)) + params_per_rank[rank_to_go].append(param) + self.params_per_rank_id_dict[-1][rank_to_go].append(global_id) + numel_per_rank[rank_to_go] += param.numel() + + # check whether any rank is not assigned to parameters. + for rank, params in enumerate(params_per_rank): + if len(params) == 0: + no_params_ranks.append(rank) + + if gpc.is_rank_for_log(): + logger.info(f"Number of elements on ranks: {numel_per_rank}, rank:{gpc.get_global_rank()}") + + return params_per_rank, set(no_params_ranks) + + def _attach_reduction_hook(self): + # we iterate over the fp16 params + # on each param, we register a hook to its AccumulateGrad object + for group_id in range(self.num_param_groups): + param_group = self._fp16_param_groups[group_id] + for param in param_group: + if param.requires_grad: + reduce_rank = None + + def _define_and_attach(param, reduce_rank=None): + # get the AccumulateGrad object of the param itself + # If these objects are not kept, reduction hooks may not be attached successfully. + accum_grad_obj = get_grad_accumulate_object(param) + self._grad_store.add_accumulate_grad_object(accum_grad_obj) + + reduction_func = partial( + self._store_and_try_reduce_grads_by_bucket, + param=param, + reduce_rank=reduce_rank, + ) + + # define hook + # NOT IMPORTANT BUT GOOD TO KNOW: + # args here is not grad, but allow_unreacable and accumulate_grad + def reduce_grad_hook(*args): # pylint: disable=W0613 + if self.skip_grad_reduce is False: + reduction_func() + + accum_grad_obj.register_hook(reduce_grad_hook) + + _define_and_attach(param, reduce_rank) + + def _store_and_try_reduce_grads_by_bucket(self, param, reduce_rank=None): + param_size = param.numel() + + # check if the bucket is full + # if full, will reduce the grads already in the bucket + # after reduction, the bucket will be empty + if self._bucket_store.num_elements_in_bucket(reduce_rank) + param_size > self._reduce_bucket_size: + self._reduce_grads_stored_in_bucket(reduce_rank, last_bucket=False) + + # the param must not be reduced to ensure correctness + is_param_reduced = self._param_store.is_param_reduced(param) + if is_param_reduced: + msg = ( + f"Parameter of size ({param.size()}) has already been reduced, " + + "duplicate reduction will lead to arithmetic incorrectness" + ) + raise RuntimeError(msg) + + # the param must have grad for reduction + assert param.grad is not None, f"Parameter of size ({param.size()}) has None grad, cannot be reduced" + + self._bucket_store.add_num_elements_in_bucket(param_size, reduce_rank) + self._bucket_store.add_grad(param.grad, reduce_rank) + self._bucket_store.add_param(param, reduce_rank) + + def _reduce_grads_stored_in_bucket(self, reduce_rank=None, last_bucket=False): + # reduce grads + self._reduce_grads_by_rank( + reduce_rank=reduce_rank, + grads=self._bucket_store.get_grad(reduce_rank=reduce_rank), + bucket_size=self._bucket_store.num_elements_in_bucket(reduce_rank), + ) + + params_in_bucket = self._bucket_store.get_param(reduce_rank=reduce_rank) + + for param in params_in_bucket: + # the is_param_reduced flag should be False showing that + # this param is not reduced before calling self._reduce_grads_by_rank + is_param_reduced = self._param_store.is_param_reduced(param) + + if is_param_reduced: + msg = ( + f"Parameter of size ({param.size()}) has been reduced, " + + "duplicate reduction will lead to arithmetic incorrectness" + ) + raise RuntimeError(msg) + + # update the flag + self._param_store.set_param_reduction_state(param, True) + + if self._param_store.belongs_to_current_rank(param): + self._param_store.add_reduced_param_for_compute_norm(param, last_bucket) + else: + self._param_store.add_previous_reduced_param(param) + + self._bucket_store.reset_by_rank(reduce_rank) + + def _reduce_grads_by_rank(self, reduce_rank, grads, bucket_size): + grad_buckets_by_dtype = split_half_float_double(grads) + next_bucket_list = [] + # add parameters into bucket for reduction + for tensor_list in grad_buckets_by_dtype: + param_bucket = TensorBucket(size=bucket_size) + for tensor in tensor_list: + param_bucket.add_to_bucket(tensor, allow_oversize=True) + if not param_bucket.is_empty(): + self._reduce_and_copy(bucket=param_bucket, reduce_rank=reduce_rank) + next_bucket_list.append(param_bucket) + + # wait for the completion of previouce bucket list reduction, and do unflatten_and_copy() + # here we can also overlap the communication with some memcpy operation caused by bucket.flatten() + for bucket in self._bucket_in_progress: + bucket.commu_handle.wait() + bucket.unflatten_and_copy() + bucket.empty() + self._bucket_in_progress = [] + self._param_store.clear_grads_of_previous_reduced_params() + + # after the completion of bucket list reduction, add new buckets into _bucket_in_progress + self._bucket_in_progress = next_bucket_list.copy() + + def _reduce_and_copy(self, bucket: TensorBucket, reduce_rank): + # flatten the tensors and do allreduce + bucket.flatten() + bucket.commu_handle = reduce_tensor( + tensor=bucket.get_flat_tensor(), + dtype=None, + dst_rank=reduce_rank, + parallel_mode=ParallelMode.DATA, + ) + + # update the reduced tensor + if reduce_rank is None or reduce_rank == self._zero_local_rank: + bucket.set_unflatten_and_copy_flag(flag=True) + + def _has_inf_or_nan(self, tensor): + try: + tensor_mean = float(tensor.mean()) + except RuntimeError as instance: + # We want to check if inst is actually an overflow exception. + # RuntimeError could come from a different error. + # If so, we still want the exception to propagate. + if "value cannot be converted" not in instance.args[0]: + raise + return True + else: + if tensor_mean == float("inf") or tensor_mean == -float("inf"): + return True + return False + + def _sync_grad(self): + # update param already reduced flag + reduction_states = self._param_store.get_param_reduction_states() + for tensor, _ in reduction_states.items(): + reduction_states[tensor] = False + self._param_store.reset_reduced_data_for_compute_norm() + + # accumulate gradient + avg_gradients = self._grad_store._averaged_gradients + for group_id in range(self.num_param_groups): + # the following operations are performed only on the rank to which parameters are assigned. + if self._zero_local_rank not in self.param_group_no_params_ranks[group_id]: + param_group = self._param_store.get_fp16_params_by_rank_group(self._zero_local_rank, group_id) + + if group_id not in avg_gradients: + avg_gradients[group_id] = [] + + param_idx = 0 + for param in param_group: + if param.grad is not None: + if len(avg_gradients[group_id]) == param_idx: + avg_gradients[group_id].append(param.grad) + else: + avg_gradients[group_id][param_idx].add_(param.grad) + param_idx += 1 + + # the gradients needed are stored in the avg_gradients buffer + # thus, can clear this + self.zero_grad() + + def zero_grad(self, set_to_none=True): + """ + Set parameter gradients to zero. If set_to_none = True, gradient + will be set to None to save memory. + + :param set_to_none: Whether set the gradient to None. Default value is True. + :type set_to_none: bool + """ + for _, param_group in self._fp16_param_groups.items(): + for param in param_group: + if set_to_none: + param.grad = None + elif param.grad is not None: + param.grad.detach() + param.grad.zero_() + else: + pass + + def backward(self, loss, retain_graph=False): + loss = self.loss_scale * loss + loss.backward(retain_graph=retain_graph) + + # Gradients may not be fully synchronized here. + + def _compute_norm_with_stage( + self, + group_id: int = 0, + last_bucket: bool = False, + last_stage: bool = False, + previous_norm=None, + ): + # compute norm for gradients that have been reduced + params, grads = self._param_store.get_reduced_param_for_compute_norm(group_id=group_id, last_bucket=last_bucket) + if len(params) == 0: + grads = [self.padding_grad] + params = [self.padding_tensor] + + norm = 0 + if self._clip_grad_norm > 0: + # this norm is before scaling, it will be very large + norm = compute_norm( + gradients=grads, + parameters=params, + last_stage=last_stage, + previous_norm=previous_norm, + ) + + return norm + + @llm_timeout(func_name="optim_step") + def step(self, closure=None): + """Performs a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + Returns: + Union[bool, float]: Whether the gradient is success updated, and the gradient. + """ + assert closure is None, "closure is not supported by step()" + + # if not overlapping communication (no reduction hook is attached) + # we need to manually reduce these gradients + if not self._overlap_sync_grad: + for group_id in range(len(self._fp16_param_groups)): + for param in self._fp16_param_groups[group_id]: + if param.grad is not None: + self._store_and_try_reduce_grads_by_bucket(param) + + # we need to reduce the gradients left in the communication bucket + self._reduce_grads_stored_in_bucket(reduce_rank=None, last_bucket=True) + + # compute norm for gradients in the before bucket + groups_norms = [] + for group_id in range(self.num_param_groups): + groups_norms.append(self._compute_norm_with_stage(group_id=group_id)) + + # clear reduced grads + # grads in the last bucket is reduced + for bucket in self._bucket_in_progress: + bucket.commu_handle.wait() + bucket.unflatten_and_copy() + bucket.empty() + self._bucket_in_progress = [] + self._param_store.clear_grads_of_previous_reduced_params() + + # compute norm for gradients in the last bucket + total_norms = {} + for group_id in range(self.num_param_groups): + group_name = self.param_groups[group_id]["name"] if "name" in self.param_groups[group_id] else "default" + group_name = f"{group_id}_{group_name}" + total_norms[group_name] = self._compute_norm_with_stage( + group_id=group_id, + last_bucket=True, + last_stage=True, + previous_norm=groups_norms[group_id], + ) + + timer("sync_grad").start() + self._sync_grad() + timer("sync_grad").stop() + + return self._step(closure=closure, norms=total_norms) + + def _step(self, closure=None, norms=None): + assert closure is None, "closure is not supported by step()" + + # check for overflow + found_inf = False + found_nan = False + # if there is INF values in grades, compute_norm func would also returns -1 + # thus, we try to avoid call _check_overflow here + # found_inf = self._check_overflow() + # Because you may encounter inf when computing norm + + if -1 in norms.values(): + found_inf = True + + if -2 in norms.values(): + found_nan = True + + loss_scale = float(self.loss_scale.item()) # backup + if gpc.config.model.dtype is not torch.float32: + self.grad_scaler.update(found_inf) + + # update loss scale if overflow occurs + if found_inf: + if gpc.is_rank_for_log(): + logger.warning("Overflow occurs, please check it.") + send_alert_message( + address=gpc.config.monitor.alert.feishu_alert_address, + message="Overflow occurs, please check it.", + ) + self._grad_store._averaged_gradients = dict() + self.zero_grad() + return False, norms + + if found_nan: + if gpc.is_rank_for_log(): + logger.warning("Nan grad norm occurs, please check it.") + send_alert_message( + address=gpc.config.monitor.alert.feishu_alert_address, + message="Nan grad norm occurs, please check it.", + ) + self._grad_store._averaged_gradients = dict() + self.zero_grad() + return False, norms + + # copy the grad of fp16 param to fp32 param + single_grad_partition_groups = [] + for group_id in range(self.num_param_groups): + # compute norm + # The following operations are performed only on the rank to which parameters are assigned. + if not self.param_group_has_params[group_id]: + continue + + # create flat gradient for the flat fp32 params + gradients = self._grad_store.get_averaged_gradients_by_group(group_id) + with torch.no_grad(): + flat_fp16_avg_grads = flatten(gradients) + self._grad_store.reset_average_gradients_by_group(group_id) + gradients = None # release cuda memory + + dtype = self._fp32_flat_param_groups_of_current_rank[group_id].dtype + flat_fp32_avg_grads = flat_fp16_avg_grads.to(dtype) + flat_fp16_avg_grads = None # release cuda memory + + param_shape = self._fp32_flat_param_groups_of_current_rank[group_id].shape + assert ( + param_shape == flat_fp32_avg_grads.shape + ), f"fp32 param and grad have different shape {param_shape} vs {flat_fp32_avg_grads.shape}" + + single_grad_partition_groups.append(flat_fp32_avg_grads) + device = self._fp32_flat_param_groups_of_current_rank[group_id].device + self._fp32_flat_param_groups_of_current_rank[group_id].grad = flat_fp32_avg_grads.to(device) + + # unscale and clip grads + # get the global norm + global_norm_groups = {} + if self._clip_grad_norm > 0: + for group_name, norm in norms.items(): + global_norm_groups[group_name] = norm**0.5 + + # the following operations are performed only on the rank to which parameters are assigned. + if gpc.config.model.dtype is not torch.float32: + if len(single_grad_partition_groups) != 0 and self._clip_grad_norm > 0: + self._unscale_and_clip_grads( + single_grad_partition_groups, + list(global_norm_groups.values()), + loss_scale, + ) + + # update the parameters + timer("step").start() + + # For those ranks that are not assigned parameters, we just wait for other ranks + # to send them updated their own parameters. + if self.has_params: + self.optim.step() + # release the fp32 grad + release_param_grad(self._fp32_flat_param_groups_of_current_rank.values()) + # update fp16 partition updated by the current rank + for group_id in range(len(self._fp16_param_groups)): + if self.param_group_has_params[group_id]: + fp16_param = self._param_store.get_flat_fp16_param_by_rank_group( + rank=self._zero_local_rank, group_id=group_id + ) + fp32_param = self._fp32_flat_param_groups_of_current_rank[group_id] + fp16_param.data.copy_(fp32_param) + + torch.cuda.synchronize() + with torch.cuda.stream(self._comm_bcast_stream): + self.broadcast_params() + + timer("step").stop() + + # update gradients may not be needed here, because the sync_params function is used in initialization, + # so synchronization is maintained + for group_name, global_norm in global_norm_groups.items(): + global_norm_groups[group_name] = global_norm / loss_scale + return True, global_norm_groups + + def broadcast_params(self): + handles = [] + + for rank, group_id in product(range(self._zero_world_size), range(self.num_param_groups)): + # The following operations are performed only on the rank to which parameters are assigned. + if rank in self.param_group_no_params_ranks[group_id]: + continue + fp16_param = self._param_store.get_flat_fp16_param_by_rank_group(rank=rank, group_id=group_id) + # grank = gpc.get_ranks_in_group(group_type)[rank] # need to model_hf to the global rank + # assert grank == rank, f"{grank} == {rank}" + g_rank = gpc.get_ranks_in_group(self._broadcast_parallel_mode)[rank] + handle = dist.broadcast( + fp16_param, + src=g_rank, + group=gpc.get_group(ParallelMode.ZERO1), + async_op=True, + ) + + if self._overlap_sync_param: + self._param_bcast_sync_handler.add_bcast_handle(rank, handle) + else: + handles.append(handle) + + for handle in handles: + handle.wait() + + ################## + # FP16 Utilities # + ################## + + def _check_overflow(self): + # clear previous overflow record + self._found_overflow.fill_(0.0) + + # check for overflow + for group_id in range(len(self._fp16_param_groups)): + # The following operations are performed only on the rank to which parameters are assigned. + if self._zero_local_rank not in self.param_group_no_params_ranks[group_id]: + for avg_grad in self._grad_store.get_averaged_gradients_by_group(group_id): + if avg_grad is not None and has_inf_or_nan(avg_grad): + self._found_overflow.fill_(1.0) + break + dist.all_reduce( + self._found_overflow, + op=dist.ReduceOp.MAX, + group=gpc.get_group(ParallelMode.GLOBAL), + ) + + return self._found_overflow.item() > 0 + + def _unscale_and_clip_grads(self, grad_groups_flat, total_norm_groups, loss_scale): + # compute combined scale factor for this group + combined_scale_groups = [] + + if self._clip_grad_norm > 0.0: + # norm is in fact norm*scale + for group_id, total_norm in enumerate(total_norm_groups): + combined_scale_groups.append(loss_scale) + clip = ((total_norm / loss_scale) + 1e-6) / self._clip_grad_norm + if clip > 1.0: + combined_scale_groups[group_id] = clip * loss_scale + + for group_id, grad in enumerate(grad_groups_flat): + grad.data.mul_(1.0 / combined_scale_groups[group_id]) + + def clip_grad_norm(self, model, max_norm): + # will conduct in the step() + pass + + def state_dict(self): + states = {} + grad_scaler = self.grad_scaler.state_dict() + states["grad_scaler"] = grad_scaler + optim_states = self.optim.state_dict() + states["base_optim_states"] = optim_states + + flat_fp32_weights = {} + for group_id, param in self._fp32_flat_param_groups_of_current_rank.items(): + if self._zero_local_rank not in self.param_group_no_params_ranks[group_id]: + assert param.grad is None + flat_fp32_weights[group_id] = param + states["flat_fp32_weights"] = flat_fp32_weights + states["zero_devide_optim_plan"] = self.params_per_rank_id_dict + + return states + + def load_state_dict(self, states): + # TODO: Need to take into account the change in the number of DP. + assert "grad_scaler" in states, "Not found grad_scaler state!" + grad_scaler = states["grad_scaler"] + self.grad_scaler.load_state_dict(grad_scaler) + optim_states = states["base_optim_states"] + self.optim.load_state_dict(optim_states) + + # load fp32 model weight. + flat_fp32_weights = states["flat_fp32_weights"] + assert set(flat_fp32_weights.keys()) == set(self._fp32_flat_param_groups_of_current_rank) + for group_id, param in flat_fp32_weights.items(): + if self._zero_local_rank not in self.param_group_no_params_ranks[group_id]: + self_param = self._fp32_flat_param_groups_of_current_rank[group_id] + assert ( + self_param.shape == param.shape + ), f"The loaded parameter shape is inconsistent, {self_param.shape} != {param.shape}" + self_param.data.copy_(param.data) + + # Load the fp16 model weights. + for group_id in range(len(self._fp16_param_groups)): + if self._zero_local_rank not in self.param_group_no_params_ranks[group_id]: + fp16_param = self._param_store.get_flat_fp16_param_by_rank_group( + rank=self._zero_local_rank, group_id=group_id + ) + fp32_param = self._fp32_flat_param_groups_of_current_rank[group_id] + fp16_param.data.copy_(fp32_param) + + if "zero_devide_optim_plan" in states: + self.params_per_rank_id_dict = states["zero_devide_optim_plan"] + + +def reload_zero_fp32_buff(optimizer): + # If we use AMP optimizer, we need to update its fp32 buffer as newly loaded weights value. + # Or we must ensure that loading model weights must be done before zero is initialized. + if isinstance(optimizer, HybridZeroOptimizer): + for group_id, param_group in enumerate(optimizer.optim.param_groups): + if optimizer.param_group_has_params[group_id]: + # flatten fp16 params have already been updated by 'load_model_checkpoint' + fp16_flat_current_rank = optimizer._param_store.get_flat_fp16_param_by_rank_group( + optimizer._zero_local_rank, group_id + ) + # param_group["params"] is fp32 flatten optimizer states of this zero rank. + param_group["params"][0].data.copy_(fp16_flat_current_rank.float()) diff --git a/InternLM/internlm/solver/optimizer/store.py b/InternLM/internlm/solver/optimizer/store.py new file mode 100644 index 0000000000000000000000000000000000000000..adab6c91c52dd0687b5142f373842aa9712493fc --- /dev/null +++ b/InternLM/internlm/solver/optimizer/store.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from typing import List + +from torch import Tensor +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc + + +class BaseStore: + """ + Base Store + """ + + def __init__(self, dp_parallel_mode=ParallelMode.DATA): + self._world_size = gpc.get_world_size(dp_parallel_mode) + self._local_rank = gpc.get_local_rank(dp_parallel_mode) + + @property + def world_size(self): + return self._world_size + + @property + def local_rank(self): + return self._local_rank + + +class BucketStore(BaseStore): + """ + Bucket Store + """ + + def __init__(self, dp_parallel_mode): + super().__init__(dp_parallel_mode) + self._grads = dict() + self._params = dict() + self._num_elements_in_bucket = dict() + + self.reset() + + def num_elements_in_bucket(self, reduce_rank: int = None): + return self._num_elements_in_bucket[reduce_rank] + + def add_num_elements_in_bucket(self, num_elements, reduce_rank: int = None): + self._num_elements_in_bucket[reduce_rank] += num_elements + + def add_grad(self, tensor, reduce_rank: int = None): + self._grads[reduce_rank].append(tensor) + + def add_param(self, tensor, reduce_rank: int = None): + self._params[reduce_rank].append(tensor) + + def reset(self): + keys = [None] + list(range(self._world_size)) + self._grads = {rank: [] for rank in keys} + self._params = {rank: [] for rank in keys} + self._num_elements_in_bucket = {rank: 0 for rank in keys} + + def reset_by_rank(self, reduce_rank=None): + self._grads[reduce_rank] = [] + self._params[reduce_rank] = [] + self._num_elements_in_bucket[reduce_rank] = 0 + + def get_grad(self, reduce_rank: int = None): + return self._grads[reduce_rank] + + def get_param(self, reduce_rank: int = None): + return self._params[reduce_rank] + + +class GradientStore(BaseStore): + """ + Gradient Store + """ + + def __init__(self, *args): + super().__init__(*args) + # bookkeeping data structures + self._averaged_gradients = dict() + + # for backward reduction hooks + self._grad_acc_objs = [] + + def add_accumulate_grad_object(self, obj): + """ + Keep :class:`AccumulateGrad` objects. If these objects are not kept, reduction hooks may not + be attached successfully. + + :param obj: An object of :class:`AccumulateGrad` class + :type obj: :class:`AccumulateGrad` + """ + + self._grad_acc_objs.append(obj) + + def get_averaged_gradients_by_group(self, group_id: int) -> List[Tensor]: + """ + Return average gradients of a parameter group + + :param group_id: The index of parameter group + :type group_id: int + + :return: Return the list of averaged gradients of a parameter group. Each element is a gradient, + not a parameter. + :rtype: List[torch.Tensor] + """ + + return self._averaged_gradients[group_id] + + def add_average_gradient_by_group(self, group_id: int, tensor: Tensor) -> None: + """ + Append an average gradient to the list of averaged gradients of a parameter group + + :param group_id: The index of a parameter group + :param tensor: A :class:`torch.Tensor` object + :type group_id: int + :type tensor: torch.Tensor + + """ + + if group_id in self._averaged_gradients: + self._averaged_gradients[group_id].append(tensor) + else: + self._averaged_gradients[group_id] = [tensor] + + def reset_average_gradients_by_group(self, group_id: int) -> None: + """ + Reset the bookkeeping data structure for averaged gradients to an empty list + + :param group_id: The index of a parameter group + :type group_id: int + """ + + self._averaged_gradients[group_id] = [] + + +class ParameterStore(BaseStore): + """ + Parameter Store + """ + + def __init__(self, dp_paralle_mode): + super().__init__(dp_paralle_mode) + # param partitioning data structures + self._fp16_param_to_rank = dict() + self._rank_groupid_to_fp16_param_list = dict() + self._rank_group_id_to_flat_fp16_param = dict() + + # param reduction data structures + self._is_param_reduced = dict() + self._reduced_param = [] + + self._former_bucket_reduced_param = {} + self._last_bucket_reduced_param = {} + self._former_bucket_reduced_grad = {} + self._last_bucket_reduced_grad = {} + + def set_param_to_rank(self, tensor: Tensor, rank: int) -> None: + """ + Set the mapping between parameter to rank, each parameter should be owned by a rank. + + :param tensor: A :class:`torch.Tensor` object + :type tensor: torch.Tensor + :param rank: The rank of which the process is responsible for updating the parameter + :type rank: int + """ + + self._fp16_param_to_rank[tensor] = rank + + def get_param_rank(self, tensor: Tensor) -> int: + """ + Gives the rank which the parameter belongs to + + :param tensor: A :class:`torch.Tensor` object + :type tensor: torch.Tensor + """ + return self._fp16_param_to_rank[tensor] + + def belongs_to_current_rank(self, tensor) -> bool: + """ + Check whether a parameter is supposed to be updated by the process of the current rank + + :param tensor: A :class:`torch.Tensor` object + :type tensor: torch.Tensor + + :return: True if the parameter should be updated by the current rank. Otherwise false. + :rtype: bool + """ + + tensor_rank = self._fp16_param_to_rank[tensor] + return tensor_rank == self._local_rank + + def add_fp16_param_list_by_rank_group(self, rank, group_id, tensor_list) -> None: + if rank not in self._rank_groupid_to_fp16_param_list: + self._rank_groupid_to_fp16_param_list[rank] = dict() + + if group_id not in self._rank_groupid_to_fp16_param_list[rank]: + self._rank_groupid_to_fp16_param_list[rank][group_id] = [] + + self._rank_groupid_to_fp16_param_list[rank][group_id].extend(tensor_list) + + def get_fp16_params_by_rank_group(self, rank, group_id) -> List[Tensor]: + return self._rank_groupid_to_fp16_param_list[rank][group_id] + + def add_flat_fp16_param_by_rank_group(self, rank, group_id, tensor) -> None: + if rank not in self._rank_group_id_to_flat_fp16_param: + self._rank_group_id_to_flat_fp16_param[rank] = dict() + + self._rank_group_id_to_flat_fp16_param[rank][group_id] = tensor + + def get_flat_fp16_param_by_rank_group(self, rank, group_id) -> Tensor: + return self._rank_group_id_to_flat_fp16_param[rank][group_id] + + def is_param_reduced(self, tensor): + return self._is_param_reduced[tensor] + + def set_param_reduction_state(self, tensor, state): + self._is_param_reduced[tensor] = state + + def get_param_reduction_states(self): + return self._is_param_reduced + + def reset_previous_reduced_params(self): + self._reduced_param = [] + + def add_previous_reduced_param(self, tensor): + self._reduced_param.append(tensor) + + def add_reduced_param_for_compute_norm(self, param, last_bucket=False): + group_id = getattr(param, "group_id") + if last_bucket: + if group_id not in self._last_bucket_reduced_param: + self._last_bucket_reduced_param[group_id] = [] + self._last_bucket_reduced_grad[group_id] = [] + + self._last_bucket_reduced_param[group_id].append(param) + self._last_bucket_reduced_grad[group_id].append(param.grad) + else: + if group_id not in self._former_bucket_reduced_param: + self._former_bucket_reduced_param[group_id] = [] + self._former_bucket_reduced_grad[group_id] = [] + + self._former_bucket_reduced_param[group_id].append(param) + self._former_bucket_reduced_grad[group_id].append(param.grad) + + def get_reduced_param_for_compute_norm(self, group_id=0, last_bucket=False): + if not last_bucket: + if group_id not in self._former_bucket_reduced_param: + return [], [] + return ( + self._former_bucket_reduced_param[group_id], + self._former_bucket_reduced_grad[group_id], + ) + else: + if group_id not in self._last_bucket_reduced_param: + return [], [] + return ( + self._last_bucket_reduced_param[group_id], + self._last_bucket_reduced_grad[group_id], + ) + + def reset_reduced_data_for_compute_norm(self): + self._former_bucket_reduced_param = {} + self._last_bucket_reduced_param = {} + self._former_bucket_reduced_grad = {} + self._last_bucket_reduced_grad = {} + + def clear_grads_of_previous_reduced_params(self): + if len(self._reduced_param) > 0: + for param in self._reduced_param: + param.grad = None + self.reset_previous_reduced_params() + + +class TensorBucket: + """ + Tensor Bucket + """ + + def __init__(self, size): + self._max_size = size + self._current_size = 0 + self._bucket = [] + self._flat_tensor = None + self._unflatten_and_copy_flag = False + self.commu_handle = None + + @property + def max_size(self): + return self._max_size + + @property + def current_size(self): + return self._current_size + + def is_full_or_oversized(self): + return self._current_size >= self._max_size + + def is_empty(self): + return len(self._bucket) == 0 + + def set_unflatten_and_copy_flag(self, flag): + self._unflatten_and_copy_flag = flag + + def get_unflatten_and_copy_flag(self): + return self._unflatten_and_copy_flag + + def get_flat_tensor(self): + return self._flat_tensor + + def add_to_bucket(self, tensor, allow_oversize=False): + tensor_size = tensor.numel() + + if not allow_oversize and self.will_exceed_max_size(tensor_size): + msg = f"The param bucket max size {self._max_size} is exceeded" + f"by tensor (size {tensor_size})" + raise RuntimeError(msg) + + self._bucket.append(tensor) + self._current_size += tensor_size + + def will_exceed_max_size(self, tensor_size): + expected_size = self._current_size + tensor_size + return expected_size > self._max_size + + def get_bucket(self): + return self._bucket + + def empty(self): + self._bucket = [] + self._size = 0 + self._flat_tensor = None + self.commu_handle = None + + def flatten(self): + self._flat_tensor = _flatten_dense_tensors(self._bucket) + + def unflatten_and_copy(self): + if self._unflatten_and_copy_flag: + unflattened_tensor_list = _unflatten_dense_tensors(self._flat_tensor, self._bucket) + for old, new in zip(self._bucket, unflattened_tensor_list): + old.copy_(new) diff --git a/InternLM/internlm/solver/optimizer/utils.py b/InternLM/internlm/solver/optimizer/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1e2242739ee778293e8d8a5ff1fc830046547933 --- /dev/null +++ b/InternLM/internlm/solver/optimizer/utils.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import math +from abc import ABC, abstractmethod +from collections import OrderedDict +from functools import partial +from typing import Any, Dict, Optional, Union + +import torch +import torch.distributed as dist +from torch import Tensor, nn +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.core.naive_amp import NaiveAMPModel +from internlm.utils.common import get_tensor_norm, move_norm_to_cuda +from internlm.utils.logger import get_logger +from internlm.utils.parallel import is_model_parallel_parameter + +logger = get_logger(__file__) + +try: + import amp_C + from apex.multi_tensor_apply import multi_tensor_applier + + APEX_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + logger.warning("The torch implementation for cal_l2norm is slower than apex. Please note this!") + APEX_AVAILABLE = False + +inf = math.inf + + +def flatten(input_): + return _flatten_dense_tensors(input_) + + +def unflatten(flat, tensors): + return _unflatten_dense_tensors(flat, tensors) + + +def get_grad_accumulate_object(tensor): + """ + Return the AccumulateGrad of the input tensor + """ + + # grad_fn reference: + # https://discuss.pytorch.org/t/in-the-grad-fn-i-find-a-next-functions-but-i-dont-understand-the-meaning-of-the-attribute/24463 + # expand_as reference: https://pytorch.org/docs/stable/generated/torch.Tensor.expand.html#torch.Tensor.expand + # + # `next_functions` will return the backward graph where + # the first element is the AccumulateGrad of the leaf nodes. + # we want to get the AccumulateGrad of the input tensor instead of the leaf + # node in the whole computation graph. + # Therefore, we call expand_as to create a dummy graph + # where tensor_tmp and tensor indeed point to the same object. + # You can check this by print(tensor.data_ptr() == tensor_tmp.data_ptr()) + tensor_tmp = tensor.expand_as(tensor) + grad_acc_obj = tensor_tmp.grad_fn.next_functions[0][0] + return grad_acc_obj + + +def split_half_float_double(tensor_list): + dtype_buckets = { + "torch.cuda.HalfTensor": [], + "torch.cuda.FloatTensor": [], + "torch.cuda.DoubleTensor": [], + "torch.cuda.BFloat16Tensor": [], + } + + for t in tensor_list: + dtype = t.type() + if dtype in dtype_buckets: + dtype_buckets[dtype].append(t) + + buckets = [bucket for bucket in dtype_buckets.values() if bucket] + return buckets + + +def reduce_tensor(tensor, dtype=None, dst_rank=None, parallel_mode=ParallelMode.DATA): + """ + Reduce the tensor in the data parallel process group + + :param tensor: A tensor object to reduce/all-reduce + :param dtype: The data type used in communication + :param dst_rank: The source rank for reduce. If dst_rank is None, + :param parallel_mode: Communication parallel mode + all-reduce will be used instead of reduce. Default is None. + + :type tensor: torch.Tensor + :type dtype: torch.dtype, optional + :type dst_rank: int, optional + :type parallel_mode: ParallelMode, optional + """ + # use the original dtype + # if dtype is None: + assert dtype is None + dtype = tensor.dtype + + # cast the data to specified dtype for reduce/all-reduce + # if tensor.dtype != dtype: + # tensor_to_reduce = tensor.to(dtype) + # else: + # tensor_to_reduce = tensor + + # world_size = gpc.get_world_size(parallel_mode) + # tensor.div_(world_size) + group = gpc.get_group(parallel_mode) + + # if rank is None, all reduce will be used + # else, reduce is used + use_all_reduce = dst_rank is None + + if use_all_reduce: + handle = dist.all_reduce(tensor=tensor, group=group, op=torch.distributed.ReduceOp.AVG, async_op=True) + else: + ranks_in_group = gpc.get_ranks_in_group(parallel_mode) + global_rank = ranks_in_group[dst_rank] + handle = dist.reduce( + tensor=tensor, dst=global_rank, group=group, op=torch.distributed.ReduceOp.AVG, async_op=True + ) + + return handle + + +def has_inf_or_nan(tensor): + try: + # if tensor is half, the .float() incurs an additional deep copy, but it's necessary if + # Pytorch's .sum() creates a one-element tensor of the same type as tensor + # (which is true for some recent version of pytorch). + tensor_sum = float(tensor.float().sum()) + # More efficient version that can be used if .sum() returns a Python scalar + # tensor_sum = float(tensor.sum()) + except RuntimeError as instance: + # We want to check if inst is actually an overflow exception. + # RuntimeError could come from a different error. + # If so, we still want the exception to propagate. + if "value cannot be converted" not in instance.args[0]: + raise + return True + else: + if tensor_sum == float("inf") or tensor_sum == -float("inf"): + return True + return False + + +def release_param_grad(tensor_list): + for tensor in tensor_list: + tensor.grad = None + + +def sync_param(flat_tensor, tensor_list): + """ + Synchronize the flattened tensor and unflattened tensor list. When + a list of tensor are flattened with `torch._utils._unflatten_dense_tensors`, + a new tensor is created. Thus, the flat tensor and original tensor list do not + share the same memory space. This function will update the tensor list so that + they point to the same value. + + :param flat_tensor: A flat tensor obtained by calling `torch._utils._unflatten_dense_tensors` on a tensor lsit + :param tensor_list: A list of tensors corresponding to the flattened tensor + :type flat_tensor: torch.Tensor + :type tensor_list: List[torch.Tensor] + """ + updated_params = unflatten(flat_tensor, tensor_list) + + # update the tensor data + for p, q in zip(tensor_list, updated_params): + p.data = q.data + + +def multi_tensor_l2norm_torch(tensor_list, per_tensor): + # Convert tensor_list elements to torch.float32 + tensor_list = [tensor.float() for tensor in tensor_list] + norms_tensor = torch.stack([torch.norm(tensor, p=2) for tensor in tensor_list]) + l2_norm = torch.norm(norms_tensor, p=2).unsqueeze(0) + + if per_tensor: + per_tensor_norm = norms_tensor + else: + per_tensor_norm = torch.Tensor([]).to(norms_tensor.device) + + return l2_norm, per_tensor_norm + + +def calc_l2_norm(grads): + norm = 0.0 + if len(grads) > 0: + if APEX_AVAILABLE: + dummy_overflow_buf = torch.cuda.IntTensor([0]) + norm, _ = multi_tensor_applier( + amp_C.multi_tensor_l2norm, + dummy_overflow_buf, + [grads], + False, # no per-parameter norm + ) + else: + norm, _ = multi_tensor_l2norm_torch(grads, False) + return norm + + +def calc_lp(grads, norm_type): + norm = 0.0 + for grad in grads: + grad_norm = torch.norm(grad, norm_type) + norm += grad_norm**norm_type + return norm + + +def compute_norm(gradients, parameters, last_stage=False, previous_norm=None, norm_type=2): + """Get the norm + Arguments: + gradients (Iterable[Tensor]): The gradient value. + parameters (Iterable[Tensor]): The parameter each gradient corresponds to. + norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for + infinity norm. + + Returns: + Total norm of the parameters, need total_norm**(1/norm) before using. + """ + + enable_cuda_kernels = gradients[0].device.type == "cuda" + # Norm parameters. + norm_type = float(norm_type) + + # Calculate norm. + if norm_type == inf: + total_norm = max(g.data.abs().max() for g in gradients) + total_norm_cuda = torch.FloatTensor([float(total_norm)], device=gradients[0].device) + + if last_stage is False: + return total_norm_cuda + + if previous_norm is not None: + total_norm_cuda = max(total_norm_cuda, previous_norm) + + # Take max across all model-parallel GPUs. + if gpc.get_world_size(ParallelMode.MODEL) > 1: + dist.all_reduce( + total_norm_cuda, + op=dist.ReduceOp.MAX, + group=gpc.get_group(ParallelMode.MODEL), + ) + total_norm = total_norm_cuda[0].item() + else: + tensor_parallel_grads = [] + for g, p in zip(gradients, parameters): + # TODO: consider the pipeline shared parameter + if ( + gpc.is_initialized(ParallelMode.PIPELINE) + and hasattr(p, "pipeline_shared_module_pg") + and dist.get_rank(p.pipeline_shared_module_pg) == 0 + ): # if shared between different pipe, only count o + tensor_parallel_grads.append(g.data.float()) + elif ( + gpc.is_initialized(ParallelMode.PIPELINE) + and hasattr(p, "pipeline_shared_module_pg") + and dist.get_rank(p.pipeline_shared_module_pg) != 0 + ): + continue + elif ( + gpc.is_initialized(ParallelMode.TENSOR) + and not is_model_parallel_parameter(p) + and gpc.get_local_rank(ParallelMode.TENSOR) == 0 + ): # if not used in each chunk, such as layernorm + tensor_parallel_grads.append(g.data.float()) + elif is_model_parallel_parameter(p): + tensor_parallel_grads.append(g.data.float()) + elif gpc.get_local_rank(ParallelMode.TENSOR) != 0: + continue + else: + raise RuntimeError("Should not arrive here") + + if norm_type == 2.0 and enable_cuda_kernels: + tensor_parallel_norm = calc_l2_norm(tensor_parallel_grads) ** norm_type + else: + tensor_parallel_norm = calc_lp(tensor_parallel_grads, norm_type) + + # If norm is type of float, then we model_hf them into torch.Tensor. + tensor_parallel_norm = get_tensor_norm(tensor_parallel_norm, enable_cuda_kernels) + # If grads are on CPU, the norms is also on CPU. Cast them to CUDA tensors + if not enable_cuda_kernels: + tensor_parallel_norm = move_norm_to_cuda(tensor_parallel_norm) + + total_norm = tensor_parallel_norm + + if last_stage is False: + return total_norm + + if previous_norm is not None: + total_norm = total_norm + previous_norm + + # Sum across all model-parallel GPUs. + if gpc.is_initialized(ParallelMode.MODEL): + dist.all_reduce( + total_norm, + op=dist.ReduceOp.SUM, + group=gpc.get_group(ParallelMode.MODEL), + ) + + # This is because we use zero1, so we need to use this reduction. + # TODO: Check zero group to be a subset of dp group. + dist.all_reduce(total_norm, op=dist.ReduceOp.SUM, group=gpc.get_group(ParallelMode.ZERO1)) + + if torch.is_tensor(total_norm): + total_norm = total_norm.item() + + # Scale. + if total_norm == float("inf") or total_norm == -float("inf"): + total_norm = -1 + + if math.isnan(total_norm): + total_norm = -2 + + return total_norm + + +class BaseGradScaler(ABC): + """A base class for the gradient scaler. + + Args: + initial_scale (float): the initial loss scale + """ + + def __init__(self, initial_scale: float): + assert initial_scale > 0 + self._scale = torch.cuda.FloatTensor([initial_scale]) + + @property + def scale(self) -> Tensor: + """Returns the loss scale.""" + + return self._scale + + @property + def inv_scale(self) -> Tensor: + """Returns the inverse of the loss scale.""" + + return self._scale.double().reciprocal().float() + + def state_dict(self) -> Dict: + """Returns the states of the gradient scaler as a dict object.""" + + state_dict = dict() + state_dict["scale"] = self.scale + return state_dict + + def load_state_dict(self, state_dict: Dict) -> None: + """Load the states of the gradient scaler from a dict object. + + Args: + state_dict (dict): the states of the gradient scaler + """ + + self._scale = state_dict["scale"] + + @abstractmethod + def update(self, overflow: bool) -> None: + """Update the loss scale. + + Args: + overflow (bool): whether overflow occurs + """ + + pass + + +class DynamicGradScaler(BaseGradScaler): + """A gradient scaler which uses dynamic loss scale + + Args: + initial_scale (float): the initial loss scale, defaults to 2**16 + growth_factor (float): the multiplication factor for increasing loss scale, defaults to 2 + backoff_factor (float): the multiplication factor for decreasing loss scale, defaults to 0.5 + growth_interval (int): the number of steps to increase loss scale when no overflow occurs, defaults to 1000 + min_scale (float): the minimum loss scale, defaults to None + max_scale (float): the maximum loss scale, defaults to None + hysteresis (int): the number of overflows before decreasing loss scale, defaults to 2 + """ + + def __init__( + self, + initial_scale: float = 2**16, + growth_factor: float = 2, + backoff_factor: float = 0.5, + growth_interval: int = 1000, + min_scale: Optional[float] = None, + max_scale: Optional[float] = None, + hysteresis: int = 2, + ): + super().__init__(initial_scale) + if min_scale: + self._min_scale = torch.cuda.FloatTensor([min_scale]) + else: + self._min_scale = None + + if max_scale: + self._max_scale = torch.cuda.FloatTensor([max_scale]) + else: + self._max_scale = None + + self._growth_factor = growth_factor + self._backoff_factor = backoff_factor + self._growth_interval = growth_interval + self._growth_step = 0 + self._hysteresis = hysteresis + self._hysteresis_step = 0 + self._sanity_checks() + + def _sanity_checks(self) -> None: + """Check if the arguments are correct.""" + + if self._min_scale: + assert self._min_scale > 0, "The minimum gradient scale cannot be zero or negative" + if self._max_scale: + assert self._min_scale > 0, "The maximum gradient scale cannot be zero or negative" + assert self._growth_factor > 1, "The growth factor cannot be equal or smaller than 1" + assert self._backoff_factor < 1 and self._backoff_factor > 0, "The backoff factor must be between 0 and 1" + assert self._hysteresis >= 0, "The hysteresis cannot be negative" + + def update(self, overflow: bool) -> None: + """Update the loss scale. + + Args: + overflow (bool): whether overflow occurs + """ + if overflow: + self._hysteresis_step += 1 + self._growth_step = 0 + + if self._hysteresis_step >= self._hysteresis: + self._backoff_scale() + if gpc.is_rank_for_log(): + logger.warning(f"Overflow occurs, the loss scale is adjusted to {self.scale.item()}") + else: + self._growth_step += 1 + if self._growth_step == self._growth_interval: + self._growth_step = 0 + self._hysteresis_step = 0 + self._grow_scale() + if gpc.is_rank_for_log(): + logger.warning( + f"No overflow for consecutive {self._growth_interval} steps, " + f"the loss scale is adjusted to {self.scale.item()}", + ) + + def _backoff_scale(self) -> None: + """Decrease the loss scale""" + + self._scale = self._scale * self._backoff_factor + if self._min_scale: + self._scale = torch.max(self._scale, self._min_scale) + + def _grow_scale(self) -> None: + """Increase the loss scale""" + + self._scale = self._scale * self._growth_factor + if self._max_scale: + self._scale = torch.min(self._scale, self._max_scale) + + def state_dict(self): + """Returns the states of the gradient scaler as a dict object.""" + + state_dict = dict() + state_dict["_scale"] = self._scale.item() + state_dict["_growth_step"] = self._growth_step + state_dict["_hysteresis_step"] = self._hysteresis_step + + return state_dict + + def load_state_dict(self, state_dict): + """Load the states of the gradient scaler from a dict object. + + Args: + state_dict (dict): the states of the gradient scaler + """ + + self._scale = self._scale.fill_(state_dict["_scale"]) + self._growth_step = state_dict["_growth_step"] + self._hysteresis_step = state_dict["_hysteresis_step"] + + +class ParamBcastSyncHandler: + """ + Model Partition Handler for overlap broadcast with forward + """ + + def __init__(self, model: Union[nn.Module, nn.ModuleList]) -> None: + self._block_to_param = OrderedDict() # + self._param_to_rank = dict() # + self._block_to_rank = dict() # + self._bcast_handles = dict() # + + zero1_size = gpc.get_world_size(ParallelMode.ZERO1) + total_param_num = sum(p.numel() for p in model.parameters()) + avg_param_num = total_param_num * 1.0 // zero1_size + + # just want to share same for loop for ModuleList and Module + if not isinstance(model, nn.ModuleList): + model = [model] + + # record the parameters to transformer/embeding/head/norm block + for _chunk in model: + if isinstance(_chunk, NaiveAMPModel): + _chunk = _chunk.model + + for _, children in _chunk.named_children(): + # should be the transformer block definaton in modeling_xxx.py + if isinstance(children, nn.ModuleList): + # record the block that a parameter belongs to + for _, block in enumerate(children): + # self._block_to_param[f"{name}.{idx}"] = list(block.parameters()) + self._block_to_param[block] = list(block.parameters()) + else: + # record the block that a parameter belongs to + # self._block_to_param[name] = list(children.parameters()) + self._block_to_param[children] = list(children.parameters()) + + alloc_num = 0 + rank_to_go = 0 + + # process the parameters in block_to_param sequencially, + # allocate each parameter to a local rank of ParallelMode.ZERO1, + # NOTE that we do NOT consider following scenarios: + # 1) whether a parameter is trainable; + # 2) paramters maybe in different optimizer group + for block, params in self._block_to_param.items(): + # allocate a model block to a local rank of ParallelMode.ZERO1 + self._block_to_rank[block] = [rank_to_go] + for p in params: + alloc_num = alloc_num + p.numel() + # in this case, allocate the param to next rank if possible + if alloc_num > avg_param_num * 1.01 and rank_to_go < zero1_size - 1: + rank_to_go = rank_to_go + 1 + alloc_num = 0 + self._block_to_rank[block].append(rank_to_go) + # allocate a parameter to a local rank of ParallelMode.ZERO1 + self._param_to_rank[p] = rank_to_go + + # initialize an empty list for _bcast_handles of each rank + for rank in range(gpc.get_world_size(ParallelMode.ZERO1)): + self._bcast_handles[rank] = [] + + # register_forward_pre_hook for transformer/embeding/norm/xxx block + self._register_sync_parameters_hook() + + def _register_sync_parameters_hook(self) -> None: + def _pre_forward_hook(model: nn.Module, inputs: Any): # pylint: disable=W0613 + bcast_handles = [] + # gather all required broadcast hanles into a list + for rank in self._block_to_rank[model]: + bcast_handles.extend(self._bcast_handles[rank]) + # need to clear _bcast_handles since they would be processed later + self._bcast_handles[rank] = [] + # wait all required broadcast handles to be completed + for handle in bcast_handles: + handle.wait() + + # register_forward_pre_hook for transformer/embeding/norm/xxx block + for block, _ in self._block_to_rank.items(): + block.register_forward_pre_hook(partial(_pre_forward_hook)) + + def get_rank_by_param(self, param) -> int: + return self._param_to_rank[param] + + def add_bcast_handle(self, rank, handle) -> None: + self._bcast_handles[rank].append(handle) diff --git a/InternLM/internlm/solver/pipeline_utils.py b/InternLM/internlm/solver/pipeline_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c57765e4973f507b732ce9f9ad41d810ccf6075e --- /dev/null +++ b/InternLM/internlm/solver/pipeline_utils.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from internlm.utils.logger import get_logger + +logger = get_logger(__file__) + + +def partition_uniform(num_items, pipeline_parallel_size, num_chunks): + assert ( + num_items % num_chunks == 0 + ), "Layer length should be divided by the number of chunks, otherwise parameter method is recomended" + + parts = [[] for _ in range(pipeline_parallel_size)] + partition_items = num_items // num_chunks + for idx in range(num_chunks): + base_idx = idx * partition_items + chunk_size = partition_items // pipeline_parallel_size + left = pipeline_parallel_size - partition_items % pipeline_parallel_size + if chunk_size == 0: + raise ValueError("Some nodes in Pipeline have no requests") + + for p in range(pipeline_parallel_size): + st = base_idx + base_idx += chunk_size + (p >= left) + parts[p].append((st, base_idx)) + + indexes = [] + for _parts in parts: + for s, e in _parts: + indexes.extend(list(range(s, e))) + assert len(indexes) == len(set(indexes)), indexes # should have no duplicates + assert set(indexes) == set(list(range(num_items))), (indexes, num_items) # should have the same indexes as expected + return parts diff --git a/InternLM/internlm/train/__init__.py b/InternLM/internlm/train/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..98bd57a34fc4312b3790cff3b67279745aa4497b --- /dev/null +++ b/InternLM/internlm/train/__init__.py @@ -0,0 +1,23 @@ +from .training_internlm import ( + get_train_data_loader, + get_validation_data_loader, + initialize_llm_profile, + initialize_model, + initialize_teacher, + initialize_optimizer, + load_new_batch, + load_new_batch_stop, + record_current_batch_training_metrics, +) + +__all__ = [ + "get_train_data_loader", + "get_validation_data_loader", + "initialize_llm_profile", + "initialize_model", + "initialize_teacher", + "initialize_optimizer", + "load_new_batch", + "load_new_batch_stop", + "record_current_batch_training_metrics", +] diff --git a/InternLM/internlm/train/training_internlm.py b/InternLM/internlm/train/training_internlm.py new file mode 100644 index 0000000000000000000000000000000000000000..4172e28f7a7d45cc236f454cb6090e6aa67da90e --- /dev/null +++ b/InternLM/internlm/train/training_internlm.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import os +import time +from functools import partial +from typing import Callable, Union +from typing import Iterable + +import torch +import torch.distributed as dist +from torch import nn +from torch.utils.data import ConcatDataset, DataLoader + +from internlm.core.context import global_context as gpc +from internlm.core.context import ParallelMode +from internlm.core.context.random import set_mode +from internlm.core.naive_amp import NaiveAMPModel +from internlm.core.trainer import TrainState +from internlm.data.batch_sampler import get_dpsampler_dataloader, StaticBatchSampler +from internlm.data.collaters import jsonl_ds_collate_fn, packed_collate_fn +from internlm.data.dataset import get_dataset_dict +from internlm.data.dummy_dataset import RandomDataset +from internlm.data.packed_dataset import (get_packed_dataset_without_short_length, PackedDataset, + PackedDatasetWithoutCuSeqlen) +from internlm.data.utils import DATASET_TYPE_IDS_MAP, unpack_data +from internlm.monitor import send_heartbeat, set_env_var +from internlm.monitor.monitor import monitor_manager as mm +from internlm.solver.beta2_scheduler import Beta2Scheduler +from internlm.solver.lr_scheduler import FineTuneCosineAnnealingWarmupLR +from internlm.solver.optimizer import HybridZeroOptimizer +from internlm.solver.optimizer.utils import ParamBcastSyncHandler +from internlm.utils.common import DummyProfile +from internlm.utils.logger import get_logger +from internlm.utils.megatron_timers import megatron_timer as timer +from internlm.utils.model_checkpoint import get_current_device +from internlm.utils.parallel import ( + is_no_pp_or_last_stage, + sync_model_param, + sync_model_param_within_tp, +) +from internlm.utils.registry import MODEL_INITIALIZER +from internlm.utils.timeout import llm_timeout + +logger = get_logger(__file__) + + +@llm_timeout(func_name="initialize_teacher") +def initialize_teacher(): + """ + Initialize model with Automatic Mixed Precision. + + Returns: + torch.nn.Module: + The neural network model to be trained or evaluated. + """ + + model = MODEL_INITIALIZER.get_module(module_name=gpc.config.teacher_type)(**(gpc.config.teacher)) + + if isinstance(model, nn.ModuleList): + model = nn.ModuleList( + [ + NaiveAMPModel( + model=_m, + output_to_fp32=False, # manually controlled by interleaved pipleline scheduler + dtype=gpc.config.teacher.get("dtype", torch.half), + sync_buffer=False, + ) + for _m in model + ] + ) + else: + model = NaiveAMPModel( + model=model, + output_to_fp32=is_no_pp_or_last_stage(), + dtype=gpc.config.teacher.get("dtype", torch.half), + sync_buffer=False, + ) + + # This sync is very important, cause the model weights kept in optimizer are copied + # from the origin parameters in the memory, so we should make sure the dp sync + # does not influence the model weights in optimizer be different with the origin parameters. + sync_model_param(model, parallel_mode=ParallelMode.DATA) + + # This function is needed to make sure parameters that are not splitted by tensor parallelism are + # the same across tensor parallelism. + sync_model_param_within_tp(model) + + # Change random state mode to ParallelMode.DATA after model is built, guaranteeing the random + # state in the same dp group are all the same. + set_mode(ParallelMode.DATA) + + folder = gpc.config.teacher_ckpt_folder + if gpc.is_rank_for_log(): + logger.info(f"loading teacher model from: {folder}") + + tp_size = gpc.get_world_size(ParallelMode.TENSOR) + pp_size = gpc.get_world_size(ParallelMode.PIPELINE) + tp_rank = gpc.get_local_rank(ParallelMode.TENSOR) + pp_rank = gpc.get_local_rank(ParallelMode.PIPELINE) + + if not os.path.exists(folder): + raise RuntimeError(f"teacher ckpt at '{folder}' not found!") + else: + fns = os.listdir(folder) + + max_pp, max_tp = 0, 0 + for fn in fns: + if fn.startswith("model_t") and not fn.endswith(".md5") and not fn.endswith(".upload_record"): + segements = os.path.splitext(fn)[0].split("_") + max_pp = max(max_pp, int(segements[-1][2:])) + max_tp = max(max_tp, int(segements[-2][2:])) + + assert ( + pp_size == max_pp + 1 + ), f"The weights are save for {max_pp + 1} pipelines, while current has {pp_size} pipelines" + assert ( + tp_size == max_tp + 1 + ), f"The weights are save for {max_tp + 1} parallelism, while current has {tp_size} tensor parallelism" + + should_load_name = f"model_tp{tp_rank}_pp{pp_rank}.pt" + fp = os.path.join(folder, should_load_name) + + assert os.path.exists(fp), f"{fp} is not found!" + with open(fp, "rb") as f: + states = torch.load(f, map_location=get_current_device()) + + missing_k, unexpected_keys = model.load_state_dict(states, strict=False) + if len(missing_k) != 0: + logger.warning(f"Warning: missing keys {missing_k}") + if len(unexpected_keys) != 0: + logger.warning(f"Warning: unexpected keys {unexpected_keys}") + + # avoid to cuda oom, Ref: https://discuss.pytorch.org/t/load-state-dict-causes-memory-leak/36189/11 + del states + torch.cuda.empty_cache() + + if gpc.is_rank_for_log(): + logger.info(f"loading teacher model successfully") + + return model + + +@llm_timeout(func_name="initialize_model") +def initialize_model(): + """ + Initialize model with Automatic Mixed Precision. + + Returns: + torch.nn.Module: + The neural network model to be trained or evaluated. + """ + + model = MODEL_INITIALIZER.get_module(module_name=gpc.config.model_type)(**(gpc.config.model)) + if isinstance(model, nn.ModuleList): + model = nn.ModuleList( + [ + NaiveAMPModel( + model=_m, + output_to_fp32=False, # manually controlled by interleaved pipleline scheduler + dtype=gpc.config.model.get("dtype", torch.half), + sync_buffer=False, + ) + for _m in model + ] + ) + else: + model = NaiveAMPModel( + model=model, + output_to_fp32=is_no_pp_or_last_stage(), + dtype=gpc.config.model.get("dtype", torch.half), + sync_buffer=False, + ) + + # This sync is very important, cause the model weights kept in optimizer are copied + # from the origin parameters in the memory, so we should make sure the dp sync + # does not influence the model weights in optimizer be different with the origin parameters. + sync_model_param(model, parallel_mode=ParallelMode.DATA) + + # This function is needed to make sure parameters that are not splitted by tensor parallelism are + # the same across tensor parallelism. + sync_model_param_within_tp(model) + + # Change random state mode to ParallelMode.DATA after model is built, guaranteeing the random + # state in the same dp group are all the same. + set_mode(ParallelMode.DATA) + + return model + + +@llm_timeout(func_name="initialize_optimizer") +def initialize_optimizer(model: Union[nn.Module, nn.ModuleList]): + """ + Initialize optimizer. + + Args: + model (:class:`torch.nn.Module`): Your model instance to be trained or evaluated. + + Returns: + A tuple of (optimizer, beta2_scheduler, lr_scheduler). + """ + if gpc.config.hybrid_zero_optimizer.overlap_sync_param: + param_bcast_sync_handler = ParamBcastSyncHandler(model) + else: + param_bcast_sync_handler = None + + adam_cfg = gpc.config.adam + naive_optimizer = torch.optim.AdamW( + params=[{"params": model.parameters(), "weight_decay": adam_cfg.weight_decay}], + lr=adam_cfg.lr, + betas=(adam_cfg.adam_beta1, adam_cfg.adam_beta2), + eps=adam_cfg.adam_eps, + ) + + optimizer = HybridZeroOptimizer( + naive_optimizer, + grad_scal_cfg=gpc.config.grad_scaler, + zero_cfg=gpc.config.hybrid_zero_optimizer, + param_bcast_sync_handler=param_bcast_sync_handler, + ) + + beta2_scheduler = Beta2Scheduler(optimizer=naive_optimizer, **gpc.config.beta2_scheduler) + + lr_scheduler = FineTuneCosineAnnealingWarmupLR(optimizer, **gpc.config.lr_scheduler) + + return optimizer, beta2_scheduler, lr_scheduler + + +@llm_timeout(func_name="get_train_data_loader") +def get_train_data_loader( + num_worker: int = 0, dataset_generate_func: Callable = None, train_sampler=None, train_collate_fn=None +): + """ + Generate and return the training data loader. + + Args: + num_worker (:class:`int`): number of subprocesses used for dataloader. + dataset_generate_func (:class:`Callable`, optional): generate function for dataset. + train_sampler (:class:`torch.utils.data.sampler`, optional): dataset sampler for training dataloader. + train_collate_fn (:class:`Callable`, optional): collate function for training dataloader. + + Returns: + A tuple of (train_dl, dataset_types). + """ + + # Get the dataset types + dataset_types = None + dataset_types = list(DATASET_TYPE_IDS_MAP.keys()) + data_cfg = gpc.config.data + + # Get the sample weight dictionary + train_folder = data_cfg.train_folder + + if not train_folder: + train_ds = RandomDataset(num_samples=1000000, max_len=data_cfg.seq_len) + if data_cfg.pack_sample_into_one: + train_ds = PackedDatasetWithoutCuSeqlen( + train_ds, max_length_per_sample=data_cfg.seq_len, packed_length=data_cfg.packed_length + ) + else: + train_ds = PackedDataset( + train_ds, max_length_per_sample=data_cfg.seq_len, packed_length=data_cfg.packed_length + ) + else: + if dataset_generate_func is not None: + train_ds = dataset_generate_func() + else: + train_ds = get_packed_dataset_without_short_length( + folder=data_cfg.train_folder, + packed_length=data_cfg.packed_length, + max_length_per_sample=data_cfg.seq_len, + show_progress=dist.get_rank() == 0, + min_length=data_cfg.min_length, + min_length_dict=data_cfg.get("min_length_dict", {}), + pack_into_one_sample=data_cfg.pack_sample_into_one, + ) + + if dataset_generate_func is None or not train_folder: + # partition already completed + assert isinstance(train_ds, (PackedDataset, PackedDatasetWithoutCuSeqlen, ConcatDataset)) + # Create the training dataset sampler + train_sampler = StaticBatchSampler( + train_ds.datasets if isinstance(train_ds, ConcatDataset) else [train_ds], + batch_size=data_cfg.micro_num, + rampup_batch_size=data_cfg.rampup_batch_size, + micro_bsz=data_cfg.micro_bsz, + seed=1024, + drop_last=True, + data_rank=gpc.get_local_rank(ParallelMode.DATA), + data_world_size=gpc.get_world_size(ParallelMode.DATA), + ) + + if dataset_generate_func is None or not train_folder: + train_collate_fn = partial(packed_collate_fn, packed_length=data_cfg.packed_length) + + # Create the training data loader + train_dl = DataLoader( + dataset=train_ds, + batch_sampler=train_sampler, + num_workers=num_worker, + pin_memory=True, + collate_fn=train_collate_fn, + persistent_workers=num_worker > 0, + ) + + return train_dl, dataset_types + + +@llm_timeout(func_name="get_validation_data_loader") +def get_validation_data_loader( + num_worker: int = 0, dataset_generate_func: Callable = None, val_collate_fn=None, dataloader_func=None +): + """Generate and return the validation data loader.""" + + data_cfg = gpc.config.data + + if not data_cfg.valid_folder: + val_ds = RandomDataset(num_samples=gpc.get_world_size(ParallelMode.DATA) * 500, max_len=data_cfg.seq_len) + else: + if dataset_generate_func is not None: + assert val_collate_fn and dataloader_func is not None + val_ds = dataset_generate_func() + else: + val_ds = get_dataset_dict(folder=data_cfg.valid_folder, split="") + + if not isinstance(val_ds, dict): + val_ds = {"val": val_ds} + + if val_collate_fn is None or not data_cfg.valid_folder: + val_collate_fn = partial(jsonl_ds_collate_fn, max_length_per_sample=data_cfg.seq_len) + + val_dls = {} + for val_name, ds in val_ds.items(): + if dataloader_func and data_cfg.valid_folder is not None: + val_dls[val_name] = dataloader_func(dataset=ds, collate_fn=val_collate_fn) + if gpc.is_rank_for_log(): + logger.info( + f"load validation dataset {val_name} with valid batch size {str(data_cfg.valid_micro_num)} and " + f"{ds.size} Byte samples." + ) + else: + # making the batch_size of validate larger can speed up the evaluation, but it should not be too large, + # otherwise too much data may be dropped + batch_size = min( + data_cfg.valid_micro_num * data_cfg.micro_bsz, len(ds) // gpc.get_world_size(ParallelMode.DATA) + ) + batch_size = batch_size // data_cfg.micro_bsz * data_cfg.micro_bsz + + if batch_size == 0 and gpc.is_rank_for_log(): + logger.info(f"skip validate {val_name}.") + continue + + val_dls[val_name] = get_dpsampler_dataloader( + ds, + shuffle=False, + num_workers=num_worker, + batch_size=batch_size, + collate_fn=val_collate_fn, + drop_last=True, + ) # drop_last=True, otherwise it may cause problems in the last batch + + if gpc.is_rank_for_log(): + logger.info( + f"load validation dataset {val_name} with valid batch size {str(batch_size)} and " + f"samples {str(len(val_dls[val_name]))}." + ) + + return val_dls + + +@llm_timeout(func_name="load_new_batch") +def load_new_batch(train_dl: DataLoader, train_iter: Iterable, train_state: TrainState): + """ + Load and return the new batch data based on training data loader. + + Args: + train_dl (torch.utils.data.DataLoader): Dataloader for training. + train_iter (Iterable): Data iterator from which get a batch of data, obtained by calling iter(dataloader). + train_state (TrainState): Current training state. + + Returns: A batch data and the updated train_iter. + """ + + timer("batch-gen").start() + try: + batch = next(train_iter) # structure is ({'input_ids': Tensor, 'cu_seqlens': Tensor}, Tensor) + if hasattr(train_state, "batch_sampler_iter"): + next(train_state.batch_sampler_iter) + except StopIteration: + if gpc.is_rank_for_log(): + logger.warning('Reach the end of dataset! Epoch %s Length is %s', gpc.epoch, len(train_dl.dataset)) + gpc.epoch += 1 + train_iter = iter(train_dl) + batch = next(train_iter) + train_state.num_consumed_samples_in_epoch = 0 + if hasattr(train_state, "batch_sampler"): + train_state.batch_sampler_iter = iter(train_state.batch_sampler) + next(train_state.batch_sampler_iter) + timer("batch-gen").stop() + + if batch[0].get("type_ids", None) is not None: + # if use_flash_attn is False, we need to unpack type_ids + if not gpc.config.model.use_flash_attn: + batch[0]["type_ids"] = unpack_data(batch[0]["type_ids"], batch[0]["cu_seqlens"]) + + return batch, train_iter + + +@llm_timeout(func_name="load_new_batch_stop") +def load_new_batch_stop(train_dl: DataLoader, train_iter: Iterable, train_state: TrainState): + """ + Load and return the new batch data based on training data loader. Return None when StopIteration + + Args: + train_dl (torch.utils.data.DataLoader): Dataloader for training. + train_iter (Iterable): Data iterator from which get a batch of data, obtained by calling iter(dataloader). + train_state (TrainState): Current training state. + + Returns: A batch data and the updated train_iter. + """ + + timer("batch-gen").start() + try: + batch = next(train_iter) # structure is ({'input_ids': Tensor, 'cu_seqlens': Tensor}, Tensor) + if hasattr(train_state, "batch_sampler_iter"): + next(train_state.batch_sampler_iter) + except StopIteration: + batch = None + return batch, train_iter + timer("batch-gen").stop() + + if batch[0].get("type_ids", None) is not None: + # if use_flash_attn is False, we need to unpack type_ids + if not gpc.config.model.use_flash_attn: + batch[0]["type_ids"] = unpack_data(batch[0]["type_ids"], batch[0]["cu_seqlens"]) + + return batch, train_iter + + +def initialize_llm_profile(profiling: bool = False, start_time: str = None): + """Initialize and return the profiler context manager instance.""" + + if profiling and gpc.get_local_rank(ParallelMode.DATA) == 0 and gpc.get_local_rank(ParallelMode.TENSOR) == 0: + llm_profile = torch.profiler.profile + logger.info(f"Do profiling in rank {gpc.get_global_rank()}!") + else: + llm_profile = DummyProfile + + return llm_profile( + activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA], + schedule=torch.profiler.schedule(skip_first=5, wait=1, warmup=1, active=1, repeat=1), + on_trace_ready=torch.profiler.tensorboard_trace_handler( + f"{gpc.config.JOB_NAME}/{start_time}/traces/rank{gpc.get_global_rank()}_" + + f"dp{gpc.get_local_rank(ParallelMode.DATA)}_" + + f"tp{gpc.get_local_rank(ParallelMode.TENSOR)}_" + + f"pp{gpc.get_local_rank(ParallelMode.PIPELINE)}", + ), + with_stack=True, + with_modules=True, + ) + + +@llm_timeout(func_name="record_current_batch_training_metrics") +def record_current_batch_training_metrics( + get_tflops_func, + logger, + writer, + success_update, + batch_count, + batch, + train_state, + optimizer, + beta2_scheduler, + trainer, + start_time, + loss, + grad_norm, + metric, + update_panel, +): + """ + Print some training metrics of current batch. + """ + + set_env_var(key="LAST_ACTIVE_TIMESTAMP", value=int(time.time())) + + timer.store_last_timers() + if success_update in (0, True): + train_state.num_consumed_tokens += batch[1].nelement() * gpc.get_world_size(ParallelMode.DATA) + if is_no_pp_or_last_stage(): + acc_perplex = metric.get_metric() + + if success_update and gpc.is_rank_for_log(): + lr = optimizer.param_groups[0]["lr"] + if hasattr(trainer.engine.optimizer, "grad_scaler"): + scaler = trainer.engine.optimizer.grad_scaler._scale.item() + elif hasattr(trainer.engine.optimizer.optim, "grad_scaler"): + scaler = trainer.engine.optimizer.optim.grad_scaler._scale.item() + + num_tokens_in_batch = batch[1].nelement() + num_samples_in_batch = sum([len(b) - 1 for b in batch[0]["cu_seqlens"]]) + max_length_in_batch = max([(b[1:] - b[:-1]).max().item() for b in batch[0]["cu_seqlens"]]) + max_samples_in_batch = max([len(b) - 1 for b in batch[0]["cu_seqlens"]]) + min_samples_in_batch = min([len(b) - 1 for b in batch[0]["cu_seqlens"]]) + + tk_per_gpu = 0 + tk_per_gpu = round( + num_tokens_in_batch + * gpc.get_world_size(ParallelMode.DATA) + / gpc.get_world_size(ParallelMode.GLOBAL) + / (time.time() - start_time), + 2, + ) + + tflops = get_tflops_func((time.time() - start_time)) + + if isinstance(loss, dict): + infos = { + "tflops": tflops, + "step": batch_count, + } + infos.update(loss) + infos.update( + { + "tgs (tokens/gpu/second)": tk_per_gpu, + "lr": lr, + "loss_scale": scaler, + "grad_norm": grad_norm, + } + ) + else: + infos = { + "tflops": tflops, + "step": batch_count, + "loss": loss.item(), + "tgs (tokens/gpu/second)": tk_per_gpu, + "lr": lr, + "loss_scale": scaler, + "grad_norm": grad_norm, + } + + infos["micro_num"] = len(batch[1]) + infos["num_consumed_tokens"] = train_state.num_consumed_tokens + infos["inf_nan_skip_batches"] = train_state.inf_nan_skip_batches + infos["num_samples_in_batch"] = num_samples_in_batch # the number of batches which have the most samples + infos["largest_length"] = max_length_in_batch # the longest input + infos["largest_batch"] = max_samples_in_batch # the batch with the most samples + infos["smallest_batch"] = min_samples_in_batch + infos["adam_beta2"] = beta2_scheduler.get_beta2() + + fwd_bwd_time = round(timer("fwd-bwd").elapsed(), 2) + infos["fwd_bwd_time"] = fwd_bwd_time + + for key, value in acc_perplex.items(): + infos[key] = value + + line = "" + for key, value in infos.items(): + line += f"{key}={value} " + if isinstance(value, dict): + writer.add_scalars(key=key, value=value, step=train_state.step_count) + else: + writer.add_scalar(key=key, value=value, step=train_state.step_count) + + if gpc.config.monitor.alert.get("light_monitor_address", None) and batch_count % 50 == 0: + send_heartbeat("train_metrics", infos) + + if update_panel: + # metrics shown with dashboard panels + if isinstance(loss, dict): + panel_metrics = { + "step": batch_count, + "lr": lr, + "num_consumed_tokens": train_state.num_consumed_tokens, + } + panel_metrics.update(loss) + panel_metrics.update( + { + "flops": tflops, + "tgs": tk_per_gpu, + "acc": acc_perplex["acc"], + "perplexity": acc_perplex["perplexity"], + "fwd_bwd_time": fwd_bwd_time, + } + ) + else: + panel_metrics = { + "step": batch_count, + "lr": lr, + "num_consumed_tokens": train_state.num_consumed_tokens, + "loss": loss.item(), + "flops": tflops, + "tgs": tk_per_gpu, + "acc": acc_perplex["acc"], + "perplexity": acc_perplex["perplexity"], + "fwd_bwd_time": fwd_bwd_time, + } + for norm_key, norm_value in grad_norm.items(): + panel_metrics[norm_key] = norm_value + + logger.info( + "{line}", + line=line, + extra=panel_metrics, + ) + else: + logger.info(line) + + # if loss spike occurs, send alert info to feishu + if isinstance(loss, dict): + loss = sum(loss.values()) + + mm.monitor_loss_spike( + alert_address=gpc.config.monitor.alert.feishu_alert_address, + step_count=batch_count, + cur_step_loss=loss.item(), + ) diff --git a/InternLM/internlm/utils/__init__.py b/InternLM/internlm/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/InternLM/internlm/utils/checkpoint.py b/InternLM/internlm/utils/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..31a97af635a08a79a914c62b0e6205196eb93184 --- /dev/null +++ b/InternLM/internlm/utils/checkpoint.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import weakref + +import torch +from torch.utils.checkpoint import check_backward_validity, detach_variable + +from internlm.core.context.random import ( + get_current_mode, + get_states, + set_mode, + set_seed_states, + sync_states, +) + +from .common import get_current_device + + +def copy_to_device(obj, device): + if torch.is_tensor(obj): + # Notice: + # When in no_grad context, requires_gard is False after movement + ret = obj.to(device).detach() + ret.requires_grad = obj.requires_grad + return ret + elif isinstance(obj, list): + return [copy_to_device(i, device) for i in obj] + elif isinstance(obj, tuple): + return tuple([copy_to_device(v, device) for v in obj]) + elif isinstance(obj, dict): + return {k: copy_to_device(v, device) for k, v in obj.items()} + else: + return obj + + +class CheckpointFunction(torch.autograd.Function): + """ + Checkpoint Function + """ + + @staticmethod + def forward(ctx, run_function, activation_offload=False, *args): # pylint: disable=W1113 + check_backward_validity(args) + ctx.run_function = run_function + ctx.activation_offload = activation_offload + ctx.device = get_current_device() + + # preserve rng states + ctx.fwd_cpu_rng_state = torch.get_rng_state() + sync_states() + ctx.fwd_seed_states = get_states(copy=True) + ctx.fwd_current_mode = get_current_mode() + + if hasattr(torch, "is_autocast_enabled"): + ctx.had_autocast_in_fwd = torch.is_autocast_enabled() + else: + ctx.had_autocast_in_fwd = False + + if activation_offload: + inputs_cuda = copy_to_device(args, ctx.device) + else: + inputs_cuda = args + + with torch.no_grad(): + outputs = run_function(*inputs_cuda) + # Save non-tensor inputs in ctx, keep a placeholder None for tensors + # to be filled out during the backward. + ctx.inputs = [] + ctx.tensor_indices = [] + tensor_inputs = [] + for i, arg in enumerate(args): + if torch.is_tensor(arg): + if activation_offload: + tensor_inputs.append(copy_to_device(arg, "cpu")) + else: + tensor_inputs.append(arg) + ctx.tensor_indices.append(i) + ctx.inputs.append(None) + else: + ctx.inputs.append(arg) + + if activation_offload: + ctx.tensor_inputs = tensor_inputs + else: + ctx.save_for_backward(*tensor_inputs) + return outputs + + @staticmethod + def backward(ctx, *args): + if not torch.autograd._is_checkpoint_valid(): + raise RuntimeError( + "Checkpointing is not compatible with .grad() or when an `inputs` parameter is " + "passed to .backward(). Please use .backward() and do not pass its `inputs` argument." + ) + # Copy the list to avoid modifying original list. + inputs = list(ctx.inputs) + tensor_indices = ctx.tensor_indices + + if ctx.activation_offload: + tensors = ctx.tensor_inputs + else: + tensors = ctx.saved_tensors + + # store the current states + bwd_cpu_rng_state = torch.get_rng_state() + sync_states() + bwd_seed_states = get_states(copy=True) + bwd_current_mode = get_current_mode() + + # set the states to what it used to be + torch.set_rng_state(ctx.fwd_cpu_rng_state) + for parallel_mode, state in ctx.fwd_seed_states.items(): + set_seed_states(parallel_mode, state) + set_mode(ctx.fwd_current_mode) + if ctx.activation_offload: + tensors = copy_to_device(tensors, ctx.device) + + # Fill in inputs with appropriate saved tensors. + for i, idx in enumerate(tensor_indices): + inputs[idx] = tensors[i] + detached_inputs = detach_variable(tuple(inputs)) + if ctx.had_autocast_in_fwd: + with torch.enable_grad(), torch.cuda.amp.autocast(): + outputs = ctx.run_function(*detached_inputs) + else: + with torch.enable_grad(): + outputs = ctx.run_function(*detached_inputs) + + if isinstance(outputs, torch.Tensor): + outputs = (outputs,) + # recover the rng states + torch.set_rng_state(bwd_cpu_rng_state) + for parallel_mode, state in bwd_seed_states.items(): + set_seed_states(parallel_mode, state) + set_mode(bwd_current_mode) + + # run backward() with only tensor that requires grad + outputs_with_grad = [] + args_with_grad = [] + for i in range(len(outputs)): + if torch.is_tensor(outputs[i]) and outputs[i].requires_grad: + outputs_with_grad.append(outputs[i]) + args_with_grad.append(args[i]) + if len(outputs_with_grad) == 0: + raise RuntimeError("none of output has requires_grad=True," " this checkpoint() is not necessary") + torch.autograd.backward(outputs_with_grad, args_with_grad) + grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else None for inp in detached_inputs) + return (None, None) + grads + + +def activation_checkpoint(function, activation_offload, *args, use_reentrant: bool = True): + """Checkpoint the computation while preserve the rng states, modified from Pytorch torch.utils.checkpoint. + Args: + function: Describe the forward pass function. It should know how to handle the input tuples. + activation_offload: The variable to check whether we should offload activation to cpu + args (list): Tuple containing the parameters of the function + use_reentrant: Bool type to check if we need to use_reentrant, if use_reentrant=False, there + might be more flexibility for user to define there checkpoint function + Returns: + Output of running function with provided args. + """ + if use_reentrant: + return CheckpointFunction.apply(function, activation_offload, *args) + else: + return _checkpoint_without_reentrant( + function, + activation_offload, + *args, + ) + + +def _checkpoint_without_reentrant(function, activation_offload=False, *args): # pylint: disable=W1113 + # store rng_state + fwd_cpu_state = torch.get_rng_state() + sync_states() + fwd_seed_states = get_states(copy=True) + fwd_current_mode = get_current_mode() + + # check if use autocast + if hasattr(torch, "is_autocast_enabled"): + has_autocast_in_fwd = torch.is_autocast_enabled() + else: + has_autocast_in_fwd = False + + # using WeakKeyDictionary to store all the activation the first time we call unpack + storage: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary() + weak_holder_list = [] + + # class for weakref.ref + class Holder: + pass + + # return a Holder object for later unpack process + def pack(): + res = Holder() + weak_holder_list.append(weakref.ref(res)) + return res + + # unpack hook + def unpack(x): + unpack_counter = 0 + + # re-compute all the activation inside the function when we first call unpack + if len(storage) == 0: + + def inner_pack(inner): + nonlocal unpack_counter + unpack_counter += 1 + + # If the holder went out of scope, the SavedVariable is dead and so + # the value will never be read from the storage. Skip filling it. + if weak_holder_list[unpack_counter - 1]() is None: + return + + # Use detach here to ensure we don't keep the temporary autograd + # graph created during the second forward + storage[weak_holder_list[unpack_counter - 1]()] = inner.detach() + return + + def inner_unpack(packed): + raise RuntimeError("You are calling backwards on a tensor that is never exposed. Please open an issue.") + + # restore rng state + torch.set_rng_state(fwd_cpu_state) + for parallel_mode, state in fwd_seed_states.items(): + set_seed_states(parallel_mode, state) + set_mode(fwd_current_mode) + + # reload arg into device if needed + if activation_offload: + for arg in args: + if torch.is_tensor(arg): + arg = arg.to(device=device) + + # rerun forward, the inner_pack will store all the activations in storage + if has_autocast_in_fwd: + with torch.enable_grad(), torch.cuda.amp.autocast(), torch.autograd.graph.saved_tensors_hooks( + inner_pack, inner_unpack + ): + function(*args) + else: + with torch.enable_grad(), torch.autograd.graph.saved_tensors_hooks(inner_pack, inner_unpack): + function(*args) + + if x not in storage: + raise RuntimeError( + "Attempt to retrieve a tensor saved by autograd multiple times without checkpoint" + " recomputation being triggered in between, this is not currently supported. Please" + " open an issue with details on your use case so that we can prioritize adding this." + ) + + return storage[x] + + # get device if we need to offload the activation + if activation_offload: + device = get_current_device() + + # run function with pack and unpack as saved_tensors_hooks + with torch.autograd.graph.saved_tensors_hooks(pack, unpack): + output = function(*args) + + # offload activation if needed + if activation_offload: + for arg in args: + if torch.is_tensor(arg): + arg = arg.to(device="cpu") + + return output diff --git a/InternLM/internlm/utils/common.py b/InternLM/internlm/utils/common.py new file mode 100644 index 0000000000000000000000000000000000000000..f3b58c0c0eaf21f558e4c31738fa8eb0392c5faa --- /dev/null +++ b/InternLM/internlm/utils/common.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import bisect +import inspect +import os +import random +from contextlib import contextmanager +from datetime import datetime +from typing import Union + +import numpy as np +import torch + +import internlm + +CURRENT_TIME = None + + +def parse_args(): + parser = internlm.get_default_parser() + args = parser.parse_args() + + return args + + +def get_master_node(): + import subprocess + + if os.getenv("SLURM_JOB_ID") is None: + raise RuntimeError("get_master_node can only used in Slurm launch!") + result = subprocess.check_output('scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1', shell=True) + result = result.decode("utf8").strip() + return result + + +def move_norm_to_cuda(norm: Union[float, torch.Tensor]) -> Union[float, torch.Tensor]: + if torch.is_tensor(norm) and norm.device.type != "cuda": + norm = norm.to(torch.cuda.current_device()) + return norm + + +def _move_tensor(element): + if not torch.is_tensor(element): + # we expecte the data type if a list of dictionaries + for item in element: + if isinstance(item, dict): + for key, value in item.items(): + assert not value.is_cuda, "elements are already on devices." + item[key] = value.to(get_current_device()).detach() + elif isinstance(item, list): + for index, value in enumerate(item): + assert not value.is_cuda, "elements are already on devices." + item[index] = value.to(get_current_device()).detach() + elif torch.is_tensor(item): + if not item.is_cuda: + item = item.to(get_current_device()).detach() + else: + assert torch.is_tensor(element), f"element should be of type tensor, but got {type(element)}" + if not element.is_cuda: + element = element.to(get_current_device()).detach() + return element + + +def move_to_device(data): + if isinstance(data, torch.Tensor): + data = data.to(get_current_device()) + elif isinstance(data, (list, tuple)): + data_to_return = [] + for element in data: + if isinstance(element, dict): + data_to_return.append({k: _move_tensor(v) for k, v in element.items()}) + else: + data_to_return.append(_move_tensor(element)) + data = data_to_return + elif isinstance(data, dict): + data = {k: _move_tensor(v) for k, v in data.items()} + else: + raise TypeError(f"Expected batch data to be of type torch.Tensor, list, tuple, or dict, but got {type(data)}") + return data + + +def get_tensor_norm(norm: Union[float, torch.Tensor], move_to_cuda) -> torch.Tensor: + if isinstance(norm, float): + norm = torch.Tensor([norm]) + if move_to_cuda: + norm = norm.to(torch.cuda.current_device()) + return norm + + +def get_current_device() -> torch.device: + """ + Returns currently selected device (gpu/cpu). + If cuda available, return gpu, otherwise return cpu. + """ + if torch.cuda.is_available(): + return torch.device(f"cuda:{torch.cuda.current_device()}") + else: + return torch.device("cpu") + + +def get_batch_size(data): + if isinstance(data, torch.Tensor): + return data.size(0) + elif isinstance(data, (list, tuple)): + if isinstance(data[0], dict): + return data[0][list(data[0].keys())[0]].size(0) + return data[0].size(0) + elif isinstance(data, dict): + return data[list(data.keys())[0]].size(0) + + +def filter_kwargs(func, kwargs): + sig = inspect.signature(func) + return {k: v for k, v in kwargs.items() if k in sig.parameters} + + +def launch_time(): + global CURRENT_TIME + if not CURRENT_TIME: + CURRENT_TIME = datetime.now().strftime("%b%d_%H-%M-%S") + return CURRENT_TIME + + +def set_random_seed(seed): + """Set random seed for reproducability.""" + # It is recommended to use this only when inference. + if seed is not None: + assert seed > 0 + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + # if you are using multi-GPU. + torch.cuda.manual_seed_all(seed) + + +@contextmanager +def conditional_context(context_manager, enable=True): + if enable: + with context_manager: + yield + else: + yield + + +class BatchSkipper: + """ + BatchSkipper is used to determine whether to skip the current batch_idx. + """ + + def __init__(self, skip_batches): + if skip_batches == "": + pass + intervals = skip_batches.split(",") + spans = [] + if skip_batches != "": + for interval in intervals: + if "-" in interval: + start, end = map(int, interval.split("-")) + else: + start, end = int(interval), int(interval) + if spans: + assert spans[-1] <= start + spans.extend((start, end + 1)) + self.spans = spans + + def __call__(self, batch_count): + index = bisect.bisect_right(self.spans, batch_count) + return index % 2 == 1 + + +class SingletonMeta(type): + """ + Singleton Meta. + """ + + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + else: + assert ( + len(args) == 0 and len(kwargs) == 0 + ), f"{cls.__name__} is a singleton class and a instance has been created." + return cls._instances[cls] + + +def get_megatron_flops( + elapsed_time_per_iter, + checkpoint=False, + seq_len=2048, + hidden_size=12, + num_layers=32, + vocab_size=12, + global_batch_size=4, + global_world_size=1, + mlp_ratio=4, + use_swiglu=True, +): + """ + Calc flops based on the paper of Megatron https://deepakn94.github.io/assets/papers/megatron-sc21.pdf + """ + + checkpoint_activations_factor = 4 if checkpoint else 3 + + if use_swiglu: + mlp_ratio = mlp_ratio * 3 / 2 + + flops_per_iteration = ( + checkpoint_activations_factor + * ( + (8 + mlp_ratio * 4) * global_batch_size * seq_len * hidden_size**2 + + 4 * global_batch_size * seq_len**2 * hidden_size + ) + ) * num_layers + 6 * global_batch_size * seq_len * hidden_size * vocab_size + + tflops = flops_per_iteration / (elapsed_time_per_iter * global_world_size * (10**12)) + return tflops + + +class DummyProfile: + """ + Dummy Profile. + """ + + def __init__(self, *args, **kwargs) -> None: + pass + + def __enter__(self): + return self + + def __exit__(self, a, b, c): + pass + + def step(self): + pass diff --git a/InternLM/internlm/utils/evaluation.py b/InternLM/internlm/utils/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..5ae216c0c208d9e3c6aacd7d2608d276cbed0802 --- /dev/null +++ b/InternLM/internlm/utils/evaluation.py @@ -0,0 +1,170 @@ +from contextlib import contextmanager + +import torch +import torch.distributed as dist +from tqdm import tqdm + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.core.scheduler import SchedulerMetricHook +from internlm.model.metrics import AccPerplex + + +@contextmanager +def switch_evaluation_no_pipeline_scheduler(trainer, grad_accum_size, grad_accum_batch_size, metric_hook_list): + if not gpc.is_using_pp(): + prev_data_process_func = trainer.schedule.data_process_func + prev_grad_accum_size = trainer.schedule._grad_accum_size + prev_grad_accum_batch_size = trainer.schedule._grad_accum_batch_size + prev_metric_hooks = trainer.schedule._hooks + try: + trainer.schedule.data_process_func = None + trainer.schedule._grad_accum_size = grad_accum_size + trainer.schedule._grad_accum_batch_size = grad_accum_batch_size + trainer.schedule._hooks = metric_hook_list + yield + finally: + trainer.schedule.data_process_func = prev_data_process_func + trainer.schedule._grad_accum_size = prev_grad_accum_size + trainer.schedule._grad_accum_batch_size = prev_grad_accum_batch_size + trainer.schedule._hooks = prev_metric_hooks + + +@contextmanager +def switch_evaluation_pipeline_scheduler(trainer, num_microbatches, tensor_shape, metric_hook_list): + if gpc.is_using_pp(): + pre_data_process_func = trainer.schedule.data_process_func + prev_num_microbatches = trainer.schedule.num_microbatches + prev_tensor_shape = trainer.schedule.tensor_shape + prev_metric_hooks = trainer.schedule._hooks + try: + trainer.schedule.data_process_func = None + trainer.schedule.num_microbatches = num_microbatches + trainer.schedule.tensor_shape = tensor_shape + trainer.schedule._hooks = metric_hook_list + yield + finally: + trainer.schedule.data_process_func = pre_data_process_func + trainer.schedule.num_microbatches = prev_num_microbatches + trainer.schedule.tensor_shape = prev_tensor_shape + trainer.schedule._hooks = prev_metric_hooks + + +@contextmanager +def switch_sequence_parallel_mode(): + prev_mode = gpc.config.parallel.sequence_parallel + try: + gpc.config.parallel.sequence_parallel = False + yield + finally: + gpc.config.parallel.sequence_parallel = prev_mode + + +def evaluate_on_val_dls( + trainer, + val_dls, + writer, + logger, + step_count, + update_panel: bool = False, + streaming: bool = False, +): + with switch_sequence_parallel_mode(): + torch.cuda.empty_cache() + trainer.eval() + verbose = gpc.is_rank_for_log() + data_cfg = gpc.config.data + + for val_name, val_dl in val_dls.items(): + if not streaming and len(val_dl) == 0 and verbose: + logger.info(f"Validation dataset: {val_name} is empty") + continue + + val_metric = AccPerplex( + device=torch.cuda.current_device(), + tp_pg=gpc.get_group(ParallelMode.TENSOR), + dp_pg=gpc.get_group(ParallelMode.DATA), + ) + val_sche_metric_hook = SchedulerMetricHook(metric=val_metric) + + val_loss = 0 + val_idx = -1 + for val_idx, batch in tqdm( + enumerate(val_dl), + desc="Val.", + total=len(val_dl) if not streaming else None, + position=1, + disable=not verbose, + leave=False, + ): + with torch.inference_mode(): + if gpc.is_using_pp(): + total_val_bsz = len(batch[1]) + assert total_val_bsz % data_cfg.micro_bsz == 0 + num_microbatches = total_val_bsz // data_cfg.micro_bsz + tensor_shape = torch.Size( + [data_cfg.micro_bsz, batch[0]["input_ids"].shape[1], gpc.config.HIDDEN_SIZE] + ) + + with switch_evaluation_pipeline_scheduler( + trainer=trainer, + num_microbatches=num_microbatches, + tensor_shape=tensor_shape, + metric_hook_list=[val_sche_metric_hook], + ): + _, _, loss = trainer.execute_schedule( + batch, forward_only=True, return_loss=True, return_output_label=False + ) + else: + total_val_bsz = len(batch[1]) + assert total_val_bsz % data_cfg.micro_bsz == 0 + grad_accum_size = total_val_bsz // data_cfg.micro_bsz + grad_accum_batch_size = data_cfg.micro_bsz + with switch_evaluation_no_pipeline_scheduler( + trainer=trainer, + grad_accum_size=grad_accum_size, + grad_accum_batch_size=grad_accum_batch_size, + metric_hook_list=[val_sche_metric_hook], + ): + _, _, loss = trainer.execute_schedule( + batch, forward_only=True, return_loss=True, return_output_label=False + ) + if verbose: + if isinstance(loss, dict): + loss = sum(loss.values()) + val_loss += loss.item() + + assert val_idx != -1 + dist.barrier() + + val_res = val_metric.get_metric() + if verbose and (streaming or len(val_dl) != 0): + val_loss = val_loss / (val_idx + 1 + 1e-6) + infos = { + "step": step_count, + f"val/{val_name}_loss": val_loss, + f"val/{val_name}_acc": val_res["acc"], + f"val/{val_name}_plex": val_res["perplexity"], + } + + for key, value in infos.items(): + writer.add_scalar(key=key, value=value, step=step_count) + + if update_panel: + logger.info( + f"Validation on {val_name}: " + " ".join([f"{key}={value}" for key, value in infos.items()]), + extra={ + "step": step_count, + "val_loss": val_loss, + "val_acc": val_res["acc"], + "val_perplexity": val_res["perplexity"], + }, + ) + else: + logger.info( + f"Validation on {val_name}: " + " ".join([f"{key}={value}" for key, value in infos.items()]) + ) + + trainer.train() + torch.cuda.empty_cache() + dist.barrier() diff --git a/InternLM/internlm/utils/gputest.py b/InternLM/internlm/utils/gputest.py new file mode 100644 index 0000000000000000000000000000000000000000..ddb4932e222892dbabb3ab9d4c37d26f24142384 --- /dev/null +++ b/InternLM/internlm/utils/gputest.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import math +import socket + +import torch +import torch.distributed as dist +from flash_attn.modules.mha import FlashSelfAttention, SelfAttention +from torch.utils import benchmark + +from internlm.monitor import send_alert_message +from internlm.utils.logger import get_logger +from internlm.utils.megatron_timers import megatron_timer as timer + +try: + import GPUtil + import psutil +except ImportError: + GPUtil, psutil = None, None + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.utils.common import get_current_device + +logger = get_logger(__file__) + + +def empty_cache_and_diag(batch_count, interval=50): + """empty cuda cache and run diag bench or tests.""" + if interval <= 0: + interval = 50 + if batch_count % int(interval) == 0: + # there is no need to do diag on the first batch + if batch_count > 0: + if gpc.is_rank_for_log(): + logger.info("Empty Cache and Diagnosis GPU/NCCL/Timer ...") + with torch.no_grad(): + timer_diagnosis() + bench_gpu() + bench_net() + # do empty_cache after the bench + torch.cuda.empty_cache() + + +def benchmark_forward( + test_fn, + *inputs, + repeats=100, + amp=True, + amp_dtype=torch.float16, + **kwinputs, +): + """Use Pytorch Benchmark on the forward pass of an arbitrary function.""" + + def amp_wrapper(*inputs, **kwinputs): + with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): + test_fn(*inputs, **kwinputs) + + bench_timer = benchmark.Timer( + stmt="test_fn_amp(*inputs, **kwinputs)", + globals={"test_fn_amp": amp_wrapper, "inputs": inputs, "kwinputs": kwinputs}, + num_threads=torch.get_num_threads(), + ) + used_time = bench_timer.timeit(repeats) + return used_time.mean + + +def flops(batch, seqlen, headdim, nheads, time_f): + """Compute the flops value of a GPU with give flashattention function""" + + flop = 4 * batch * seqlen**2 * nheads * headdim + return (flop / time_f / 10**12) if not math.isnan(time_f) else 0.0 + + +def get_gpu_temperature(): + """Get current GPU temperature.""" + try: + gpu_id = torch.cuda.current_device() + except AssertionError: + gpu_id = -1 + + if GPUtil is not None and gpu_id >= 0: + gpus = GPUtil.getGPUs() + gpu_temperature = gpus[gpu_id].temperature + else: + gpu_temperature = -1 + + return gpu_temperature + + +def get_cpu_temperature(): + """Get current CPU temperature.""" + + if psutil is not None: + cpu_temperature = psutil.sensors_temperatures()["coretemp"][0].current + else: + cpu_temperature = -1 + + return cpu_temperature + + +def timer_diagnosis(): + """Diagnosis running time""" + + if len(timer.names) == 0 or len(timer.times) == 0: + return + + world_size = gpc.get_world_size(ParallelMode.DATA) + if world_size < 2: + return + + # if gpc.is_rank_for_log(): + # logger.info("Diagnosis running timers ...") + + # detect slow rank compared to other ranks in the same DP group + running_time = torch.Tensor(timer.times).to(device=get_current_device()) + avg_time = running_time.detach().clone() + if world_size <= 4: + dist.all_reduce(avg_time, op=torch.distributed.ReduceOp.AVG, group=gpc.get_group(ParallelMode.DATA)) + else: + running_time_max = avg_time.detach().clone() + running_time_min = avg_time.detach().clone() + dist.all_reduce(running_time_max, op=torch.distributed.ReduceOp.MAX, group=gpc.get_group(ParallelMode.DATA)) + dist.all_reduce(running_time_min, op=torch.distributed.ReduceOp.MIN, group=gpc.get_group(ParallelMode.DATA)) + dist.all_reduce(avg_time, op=torch.distributed.ReduceOp.SUM, group=gpc.get_group(ParallelMode.DATA)) + avg_time = (avg_time - running_time_max - running_time_min) / (world_size - 2) + + diag_result = running_time > avg_time * gpc.config.data.diag_outlier_ratio + diag_result = diag_result.tolist() + avg_time = avg_time.tolist() + + for slow, name, time, avg in zip(diag_result, timer.names, timer.times, avg_time): + if slow is False or avg < 0.5: + continue + msg = ( + f"Rank {gpc.get_local_rank(ParallelMode.GLOBAL)} is slower than avg on {name}, " + f"Hostname {socket.gethostname()}, " + f"its time {time:.2f}, avg {avg:.2f}, " + f"CPU temp {get_cpu_temperature()}, GPU temp { get_gpu_temperature()}" + ) + logger.warning(msg) + send_alert_message( + address=gpc.config.monitor.alert.feishu_alert_address, + message=msg, + ) + + # detect slow rank compared to historical timer data + for name, time in zip(timer.names, timer.times): + if name not in timer.hist or len(timer.hist[name]) < 5: + continue + hist_avg = sum(timer.hist[name]) / len(timer.hist[name]) + if time > hist_avg * gpc.config.data.diag_outlier_ratio and time > 0.5: + msg = ( + f"Rank {gpc.get_local_rank(ParallelMode.GLOBAL)} is slower than hist avg on {name}, " + f"Hostname {socket.gethostname()}, " + f"its time {time:.2f}, hist_avg {hist_avg:.2f}, " + f"CPU temp {get_cpu_temperature()}, GPU temp { get_gpu_temperature()}" + ) + logger.warning(msg) + send_alert_message( + address=gpc.config.monitor.alert.feishu_alert_address, + message=msg, + ) + + +def bench_net(): + """Benchmark nccl performance for slow node detection.""" + + if gpc.get_world_size(ParallelMode.GLOBAL) <= 1: + return + + # if gpc.is_rank_for_log(): + # logger.info("benchmarking network speed ...") + + repeats = 100 + input_data = torch.randn( + 8 * 1024 * 1024, + device=get_current_device(), + dtype=torch.bfloat16, + ) + + def allreduce_fn(inputs): + dist.all_reduce(inputs, op=torch.distributed.ReduceOp.AVG, group=gpc.get_group(ParallelMode.NETTEST)) + + bench_timer = benchmark.Timer( + stmt="test_fn_amp(inputs)", + globals={"test_fn_amp": allreduce_fn, "inputs": input_data}, + num_threads=torch.get_num_threads(), + ) + allreduce_time = bench_timer.timeit(repeats).mean + allreduce_time = allreduce_time * 10**3 + allreduce_time_this = allreduce_time + allreduce_time = torch.Tensor([allreduce_time]).to(device=get_current_device()) + dist.all_reduce(allreduce_time, group=gpc.get_group(ParallelMode.GLOBAL)) + allreduce_time_avg = allreduce_time / gpc.get_world_size(ParallelMode.GLOBAL) + allreduce_time_avg = float(allreduce_time_avg.item()) + + if allreduce_time_this >= allreduce_time_avg * gpc.config.data.diag_outlier_ratio: + msg = ( + f"Rank {gpc.get_local_rank(ParallelMode.GLOBAL)} NCCL test is slower than avg, " + f"Hostname {socket.gethostname()}, " + f"allreduce_time {allreduce_time_this:.2f}, avg {allreduce_time_avg:.2f}, " + f"CPU temp {get_cpu_temperature()}, GPU temp { get_gpu_temperature()}" + ) + logger.warning(msg) + send_alert_message( + address=gpc.config.monitor.alert.feishu_alert_address, + message=msg, + ) + + +def bench_gpu(use_flash_attn=True): + """Benchmark single GPU performance for slow node detection.""" + + # if gpc.is_rank_for_log(): + # logger.info("benchmarking gpu speed ...") + + headdim = 64 + dim = 2048 + batch_size, seqlen = 2, 1024 + nheads = dim // headdim + + inner_attn = FlashSelfAttention if use_flash_attn else SelfAttention + inner_attn = inner_attn(causal=True, softmax_scale=None, attention_dropout=0) + + qkv = torch.randn( + batch_size, + seqlen, + 3, + dim // headdim, + headdim, + device=get_current_device(), + dtype=torch.float16, + requires_grad=True, + ) + time_f = benchmark_forward(inner_attn, qkv) + speed = flops(batch_size, seqlen, headdim, nheads, time_f) + speed_this = speed + speed = torch.Tensor([speed]).to(device=get_current_device()) + dist.all_reduce(speed, group=gpc.get_group(ParallelMode.GLOBAL)) + speed_avg = speed / gpc.get_world_size(ParallelMode.GLOBAL) + speed_avg = float(speed_avg.item()) + + if speed_this <= speed_avg / gpc.config.data.diag_outlier_ratio: + msg = ( + f"Rank {gpc.get_local_rank(ParallelMode.GLOBAL)} GPU is slower than avg, " + f"Hostname {socket.gethostname()}, " + f"tflops {speed_this:.2f}, avg {speed_avg:.2f}, " + f"CPU temp {get_cpu_temperature()}, GPU temp { get_gpu_temperature()}" + ) + logger.warning(msg) + send_alert_message( + address=gpc.config.monitor.alert.feishu_alert_address, + message=msg, + ) diff --git a/InternLM/internlm/utils/logger.py b/InternLM/internlm/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..61115533bb4fc17e1999b4eb54e05c3085e74305 --- /dev/null +++ b/InternLM/internlm/utils/logger.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import logging +import os + +LOGGER_NAME = "internlm" +LOGGER_FORMAT = "%(asctime)s\t%(levelname)s %(filename)s:%(lineno)s in %(funcName)s -- %(message)s" +LOGGER_LEVEL = "info" +LOGGER_LEVEL_CHOICES = ["debug", "info", "warning", "error", "critical"] +LOGGER_LEVEL_HELP = ( + "The logging level threshold, choices=['debug', 'info', 'warning', 'error', 'critical'], default='info'" +) + +uniscale_logger = None + + +def get_logger(logger_name: str = LOGGER_NAME, logging_level: str = LOGGER_LEVEL) -> logging.Logger: + """Configure the logger that is used for uniscale framework. + + Args: + logger_name (str): used to create or get the correspoding logger in + getLogger call. It will be "internlm" by default. + logging_level (str, optional): Logging level in string or logging enum. + + Returns: + logger (logging.Logger): the created or modified logger. + + """ + + if uniscale_logger is not None: + return uniscale_logger + + logger = logging.getLogger(logger_name) + + if logging_level not in LOGGER_LEVEL_CHOICES: + logging_level = LOGGER_LEVEL + print(LOGGER_LEVEL_HELP) + + logging_level = logging.getLevelName(logging_level.upper()) + + handler = logging.StreamHandler() + handler.setLevel(logging_level) + logger.setLevel(logging_level) + handler.setFormatter(logging.Formatter(LOGGER_FORMAT)) + logger.addHandler(handler) + + return logger + + +def initialize_uniscale_logger( + job_name: str = None, + launch_time: str = None, + file_name: str = None, + name: str = LOGGER_NAME, + level: str = LOGGER_LEVEL, + file_path: str = None, + is_std: bool = True, +): + """ + Initialize uniscale logger. + + Args: + job_name (str): The name of training job, defaults to None. + launch_time (str): The launch time of training job, defaults to None. + file_name (str): The log file name, defaults to None. + name (str): The logger name, defaults to "internlm". + level (str): The log level, defaults to "info". + file_path (str): The log file path, defaults to None. + is_std (bool): Whether to output to console, defaults to True. + + Returns: + Uniscale logger instance. + """ + + try: + from uniscale_monitoring import get_logger as get_uniscale_logger + except ImportError: + print("Failed to import module uniscale_monitoring. Use default python logger.") + return None + + if not file_path: + assert ( + job_name and launch_time and file_name + ), "If file_path is None, job_name, launch_time and file_name must be setted." + log_file_name = file_name + log_folder = os.path.join("RUN", job_name, launch_time, "logs") + log_dir = os.path.join(log_folder, log_file_name) + file_path = log_dir + + logger = get_uniscale_logger(name=name, level=level, filename=file_path, is_std=is_std) + if isinstance(logger, (list, tuple)): + logger = list(logger)[0] + + global uniscale_logger + uniscale_logger = logger + + return logger diff --git a/InternLM/internlm/utils/megatron_timers.py b/InternLM/internlm/utils/megatron_timers.py new file mode 100644 index 0000000000000000000000000000000000000000..d5d89e5e4d9591070e42bb412254268d7f4de78a --- /dev/null +++ b/InternLM/internlm/utils/megatron_timers.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import time + +import torch + + +class _Timer: + """Timer.""" + + def __init__(self, name): + self.name_ = name + self.elapsed_ = 0.0 + self.started_ = False + self.start_time = time.time() + self.stream = torch.cuda.current_stream() + + def start(self, reset_all=True): + """Start the timer.""" + # need to reset all timers in a new batch + if self.name_ == "one-batch" and reset_all is True: + megatron_timer.reset() + + assert not self.started_, "timer has already been started" + self.stream.synchronize() + self.start_time = time.time() + self.started_ = True + + def stop(self): + """Stop the timer.""" + assert self.started_, "timer is not started" + self.stream.synchronize() + self.elapsed_ += time.time() - self.start_time + self.started_ = False + + def reset(self): + """Reset timer.""" + self.elapsed_ = 0.0 + self.started_ = False + + def elapsed(self, reset=True): + """Calculate the elapsed time.""" + started_ = self.started_ + # If the timing in progress, end it first. + if self.started_: + self.stop() + # Get the elapsed time. + elapsed_ = self.elapsed_ + # Reset the elapsed time + if reset: + self.reset() + # If timing was in progress, set it back. + if started_: + self.start(reset_all=False) + return elapsed_ + + +class Timers: + """Group of timers.""" + + def __init__(self): + self.timers = {} + self.hist = {} + self.names = [] + self.times = [] + + def __call__(self, name): + if name not in self.timers: + self.timers[name] = _Timer(name) + return self.timers[name] + + def store_last_timers(self): + """Store timers to two list""" + self.names = [] + self.times = [] + for key, value in self.timers.items(): + senconds = round(float(value.elapsed(reset=False)), 4) + self.names.append(key) + self.times.append(senconds) + if key not in self.hist: + self.hist[key] = [] + self.hist[key].append(senconds) + if len(self.hist[key]) > 10: + self.hist[key].pop(0) + + def write(self, names, writer, iteration, normalizer=1.0, reset=False): + """Write timers to a tensorboard writer""" + # currently when using add_scalars, + # torch.utils.add_scalars makes each timer its own run, which + # polutes the runs list, so we just add each as a scalar + assert normalizer > 0.0 + for name in names: + if name in self.timers: + value = self.timers[name].elapsed(reset=reset) / normalizer + writer.add_scalar(f"time/{name}-time", value, iteration) + + def log(self, names, logger, normalizer=1.0, reset=True): + """Log a group of timers.""" + assert normalizer > 0.0 + string = "" + for name in names: + if name in self.timers: + elapsed_time = self.timers[name].elapsed(reset=reset) * 1000.0 / normalizer + string += " | {}: {:.2f}".format(name, elapsed_time) + if not len(string): # pylint: disable=C1802 + return + string = "time (ms)" + string + + logger.info(string) + return string + + def debug(self, names, logger, normalizer=1.0, reset=True): + """Log a group of timers.""" + assert normalizer > 0.0 + string = "" + for name in names: + if name in self.timers: + elapsed_time = self.timers[name].elapsed(reset=reset) * 1000.0 / normalizer + string += " | {}: {:.2f}".format(name, elapsed_time) + if not len(string): # pylint: disable=C1802 + return + string = "time (ms)" + string + + logger.debug(string) + return string + + def reset(self): + for _, t in self.timers.items(): + t.reset() + + +megatron_timer = Timers() diff --git a/InternLM/internlm/utils/model_checkpoint.py b/InternLM/internlm/utils/model_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..cdd1a84dcc8a6a18daed1acb505fcdefb74e12a6 --- /dev/null +++ b/InternLM/internlm/utils/model_checkpoint.py @@ -0,0 +1,828 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import copy +import inspect +import os +import socket +import time +from enum import Enum +from typing import Callable, Dict, Union + +import torch + +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.core.trainer import TrainState +from internlm.initialize.launch import get_config_value +from internlm.initialize.legacy.launch import ( + auto_resume_sanity_check, + ckpt_info_sanity_check, +) +from internlm.monitor import send_alert_message +from internlm.solver.optimizer import HybridZeroOptimizer, reload_zero_fp32_buff +from internlm.utils.common import get_current_device +from internlm.utils.logger import get_logger +from internlm.utils.megatron_timers import megatron_timer as timer +from internlm.utils.storage_manager import ( + get_fns, + get_storage_manager, + init_storage_manager, + llm_load, + llm_save, + try_get_storage_backend, +) +from internlm.utils.timeout import llm_timeout + + +logger = get_logger(__file__) + + +class CheckpointSaveType(Enum): + NORMAL_CHECKPOINT = 1 + SNAPSHOT_CHECKPOINT = 2 + + +class CheckpointLoadType(Enum): + INTERNLM = "internlm" + + +# The load method implemented by internlm by default does not use string representation types, +# but uses enumeration types defined in advance. +LOAD_TYPE_DICT = { + "internlm": CheckpointLoadType.INTERNLM, +} + + +class CheckpointLoadContent: + MODEL = "model" + SAMPLER = "sampler" + OPIMIZER = "optimizer" + SCHEDULAER = "scheduler" + + +class CheckpointLoadMethod: + """The registration class of the checkpoint loading method, + users can define their own custom ckpt loading methods.""" + + LOAD_FUNC_SIG = None + LOAD_TYPE_FUNC = {} + + @staticmethod + def convet_load_type(load_type: str) -> Union[CheckpointLoadType, str]: + if load_type.lower() in LOAD_TYPE_DICT: + # The ckpt load method implemented by internlm by default. + return LOAD_TYPE_DICT[load_type.lower()] + else: + # If it is a user-defined field, we do not do any conversion and represent it as a string. + return load_type + + @staticmethod + def register_ckpt_load_type(load_type: Union[str, CheckpointLoadType], load_func: Callable): + if load_type in CheckpointLoadMethod.LOAD_TYPE_FUNC: + logger.warning(f"{load_type} has aleady been registed!") + return + + CheckpointLoadMethod.LOAD_TYPE_FUNC.update({load_type: load_func}) + + if load_type == CheckpointLoadType.INTERNLM: + CheckpointLoadMethod.LOAD_FUNC_SIG = inspect.signature(load_func) + else: + if inspect.signature(load_func) != CheckpointLoadMethod.LOAD_FUNC_SIG: + logger.warning( + f"registe load model ckpt signature is not same with: {CheckpointLoadMethod.LOAD_FUNC_SIG}" + ) + + @staticmethod + def get_ckpt_load_type_func(load_type: Union[str, CheckpointLoadType]): + return CheckpointLoadMethod.LOAD_TYPE_FUNC[load_type] + + +class CheckpointLoadMask: + """ + According to the content field in the incoming ckpt_info, decide which components to load. + """ + + LOAD_CONTENT_DICT = { + "model": CheckpointLoadContent.MODEL, + "sampler": CheckpointLoadContent.SAMPLER, + "optimizer": CheckpointLoadContent.OPIMIZER, + "scheduler": CheckpointLoadContent.SCHEDULAER, + } + + def __init__(self, content: tuple) -> None: + self.load_set = set(map(lambda x: x.lower(), content)) + if "all" in self.load_set: + self.load_set = set(CheckpointLoadMask.LOAD_CONTENT_DICT.values()) + else: + self.load_set = set(map(lambda x: CheckpointLoadMask.LOAD_CONTENT_DICT[x.lower()], content)) + + def need_load(self, content: CheckpointLoadContent): + return content in self.load_set + + def not_only_load(self, content: CheckpointLoadContent): + return content in self.load_set and len(self.load_set) > 1 + + def only_load(self, content: CheckpointLoadContent): + return set((content,)) == self.load_set + + def __str__(self) -> str: + return f"{self.load_set}." + + def __repr__(self) -> str: + return f"{self.load_set}." + + +def get_model_topology(model): + """ + Returns: + { + '{name}': {'dim': int} + } + where name is the name of the module, and all parameters under this module are + concatenated along the dimension 'dim'. + """ + + from flash_attn.modules.embedding import VocabParallelEmbedding + + topos = {} + for name, module in model.named_modules(): + # If it does not meet these conditions, it is shared between various tp/dp, and it is necessary to assert + # that they are consistent. + if isinstance(module, VocabParallelEmbedding): + topos[name] = {"dim": 0} + return topos + + +def try_load_internlm_ckpt(ckpt_mm, load_info, train_state: TrainState): + load_content_str = "" + load_ckpt_folder = load_info["path"] + load_content: CheckpointLoadMask = load_info["content"] + + if gpc.is_rank_for_log(): + logger.info(f"Try load_ckpt_folder: {load_ckpt_folder}") + + if load_content.need_load(CheckpointLoadContent.MODEL): + load_model_checkpoint(folder=load_ckpt_folder, model=ckpt_mm.model) + load_content_str += f"{CheckpointLoadContent.MODEL}, " + + if load_content.not_only_load(CheckpointLoadContent.MODEL): + # load training states. + load_context(load_ckpt_folder, train_state) + + # load optimzier states. + if load_content.need_load(CheckpointLoadContent.OPIMIZER): + load_optimizer_checkpoint(load_ckpt_folder, ckpt_mm.optimizer) + load_content_str += f"{CheckpointLoadContent.OPIMIZER}, " + else: + if gpc.is_rank_for_log(): + logger.warning("CheckpointManager has no 'optimizer', skip reload optim checkpoint!") + + # load lr scheduler states. + if load_content.need_load(CheckpointLoadContent.SCHEDULAER): + if ckpt_mm.lr_scheduler: + load_scheduler(load_ckpt_folder, ckpt_mm.lr_scheduler, ckpt_mm.optimizer, train_state) + load_content_str += f"{CheckpointLoadContent.SCHEDULAER}, " + else: + if gpc.is_rank_for_log(): + logger.warning("CheckpointManager has no 'lr_scheduler', skip reload lr_scheduler checkpoint!") + + # load dataloader sampler states. + if load_content.need_load(CheckpointLoadContent.SAMPLER): + if hasattr(train_state, "batch_sampler") and not isinstance( + train_state.batch_sampler, torch.utils.data.sampler.BatchSampler + ): + load_sampler(load_ckpt_folder, ckpt_mm.train_dl.batch_sampler) + # track the actual updates of sampler when using weighted sampling + train_state.init_batch_sampler(ckpt_mm.train_dl.batch_sampler) + load_content_str += f"{CheckpointLoadContent.SAMPLER}, " + else: + if gpc.is_rank_for_log(): + logger.warning("CheckpointManager skip reload 'batch_sampler'") + + # reload data state dict. + if hasattr(train_state, "data_state_dict"): + ckpt_mm.train_dl.dataset.load_state_dict( + llm_load(os.path.join(load_ckpt_folder, "sampler_0.pt")), ckpt_path=load_ckpt_folder + ) + load_content_str += f"{CheckpointLoadContent.SAMPLER}, " + else: + if gpc.is_rank_for_log(): + logger.warning( + "CheckpointManager has no 'data_state_dict', skip reload data_state_dict checkpoint!" + ) + return load_content_str + + +def save_model_checkpoint(folder, model): + """ + Save the model according to the relationship between tp and dp. The principle is that the data of each tp + will not be gathered and saved separately, which is equivalent to actual sharding. The saved weight is named + - folder + - model_tp{tp_rank}_pp{pp_rank}.pt + + If the tp is inconsistent with the saved one in the future use, the weight needs to be converted before loading. + + Args: + folder: The folder to save the model + model: The model to be saved + """ + + states = model.state_dict() + topo = get_model_topology(model) + + if folder is not None: + dp_size = gpc.get_world_size(ParallelMode.DATA) + tp_size = gpc.get_world_size(ParallelMode.TENSOR) + dp_rank = gpc.get_local_rank(ParallelMode.DATA) + tp_rank = gpc.get_local_rank(ParallelMode.TENSOR) + pp_rank = gpc.get_local_rank(ParallelMode.PIPELINE) + + # TODO In theory, we should also consider pp level, but since pp is generally a state across machines, + # even if pp is not considered, it will definitely not be written on the same machine. + should_save_rank_pair = set() # (tp_rank, dp_rank) + for i in range(tp_size): + should_save_rank_pair.add((i, i % dp_size)) + + if (tp_rank, dp_rank) in should_save_rank_pair: + fn = f"model_tp{tp_rank}_pp{pp_rank}.pt" + fp = os.path.join(folder, fn) + llm_save(fp, saved_obj=states) + topo_fn = f"topo_tp{tp_rank}_pp{pp_rank}.json" + topo_fp = os.path.join(folder, topo_fn) + llm_save(topo_fp, saved_obj=topo) + + torch.distributed.barrier() + + +def load_model_checkpoint(folder, model): + """ + There should be weights with names similar to the following under the folder. + - folder + - model_tp{tp_rank}_pp{pp_rank}.pt + + If the tp is inconsistent with the saved one in the future use, the weight needs to be converted before loading. + """ + + tp_size = gpc.get_world_size(ParallelMode.TENSOR) + pp_size = gpc.get_world_size(ParallelMode.PIPELINE) + tp_rank = gpc.get_local_rank(ParallelMode.TENSOR) + pp_rank = gpc.get_local_rank(ParallelMode.PIPELINE) + + fns = get_fns(folder) + max_pp, max_tp = 0, 0 + for fn in fns: + if fn.startswith("model_t") and not fn.endswith(".md5"): + segements = os.path.splitext(fn)[0].split("_") + max_pp = max(max_pp, int(segements[-1][2:])) + max_tp = max(max_tp, int(segements[-2][2:])) + + assert ( + pp_size == max_pp + 1 + ), f"The weights are save for {max_pp+1} pipelines, while current has {pp_size} pipelines" + assert ( + tp_size == max_tp + 1 + ), f"The weights are save for {max_tp+1} parallelism, while current has {tp_size} tensor parallelism" + + should_load_name = f"model_tp{tp_rank}_pp{pp_rank}.pt" + fp = os.path.join(folder, should_load_name) + states = llm_load(fp, map_location=get_current_device()) + + missing_k, unexpected_keys = model.load_state_dict(states, strict=False) + if len(missing_k) != 0: + logger.warning(f"Warning: missing keys {missing_k}") + if len(unexpected_keys) != 0: + logger.warning(f"Warning: unexpected keys {unexpected_keys}") + + # avoid to cuda oom, Ref: https://discuss.pytorch.org/t/load-state-dict-causes-memory-leak/36189/11 + del states + torch.cuda.empty_cache() + + +def save_optimizer_checkpoint(optim, state_path): + """Store the state of the optimizer to the local file system or remote OSS. + + Args: + optim (Optimizer) + state_path (str): The state loading path of optimizer. + """ + + # TODO sanity check for optimizer type + zero_rank = gpc.get_local_rank(ParallelMode.ZERO1) + tp_rank = gpc.get_local_rank(ParallelMode.TENSOR) + pp_rank = gpc.get_local_rank(ParallelMode.PIPELINE) + tp_size = gpc.get_world_size(ParallelMode.TENSOR) + pp_size = gpc.get_world_size(ParallelMode.PIPELINE) + fp = f"optimizer_tp{tp_rank}_pp{pp_rank}_zo{zero_rank}.pt" + + states = optim.state_dict() + if isinstance(optim, HybridZeroOptimizer): + if gpc.get_global_rank() < optim.zero_world_size * tp_size * pp_size: + llm_save(os.path.join(state_path, fp), states) + if "zero_devide_optim_plan" in states: + params_per_rank_id_dict = states.pop("zero_devide_optim_plan") + fp_meta = os.path.join(state_path, optim.rank_unique_id) + llm_save(fp_meta, params_per_rank_id_dict) + else: + llm_save(os.path.join(state_path, fp), states) + + +def load_optimizer_checkpoint(folder, optim): + """Load the optimizer state from the local file system or remote + object storage Service (OSS). + + Args: + optim (Optimizer): optimizer + folder (str): The FS/OSS path where the optimizer will be stored. + """ + + fns = get_fns(folder) + max_tp, max_pp, max_zero = 0, 0, 0 + for fn in fns: + if fn.startswith("optimizer_") and not fn.endswith(".md5"): + _, tp, pp, zero = os.path.splitext(fn)[0].split("_") + max_zero = max(max_zero, int(zero[2:])) + max_tp = max(max_tp, int(tp[2:])) + max_pp = max(max_pp, int(pp[2:])) + + zero_size = gpc.get_world_size(ParallelMode.ZERO1) + zero_rank = gpc.get_local_rank(ParallelMode.ZERO1) + tp_size = gpc.get_world_size(ParallelMode.TENSOR) + pp_size = gpc.get_world_size(ParallelMode.PIPELINE) + + assert ( + zero_size == max_zero + 1 + ), f"The weights are save for {max_zero+1} data parallel, while current has {zero_size} zero broadcast range." + assert ( + pp_size == max_pp + 1 + ), f"The weights are save for {max_pp+1} pipelines, while current has {pp_size} pipelines" + assert ( + tp_size == max_tp + 1 + ), f"The weights are save for {max_tp+1} parallelism, while current has {tp_size} tensor parallelism" + + fp = f"optimizer_tp{gpc.get_local_rank(ParallelMode.TENSOR)}_" + fp += f"pp{gpc.get_local_rank(ParallelMode.PIPELINE)}_" + fp += f"zo{zero_rank}.pt" + states = llm_load(os.path.join(folder, fp), map_location=get_current_device()) + + if isinstance(optim, HybridZeroOptimizer): + fp_meta = os.path.join(folder, optim.rank_unique_id) + try: + zero_devide_optim_plan = llm_load(fp_meta) + states.update({"zero_devide_optim_plan": zero_devide_optim_plan}) + except Exception as e: + logger.warning( + f"Read zero optimzer split file '{fp_meta}', for '{e}'" + f"Please check whether loading ckpts are saved with the HybridZeroOptimizer." + ) + + optim.load_state_dict(states) + del states + torch.cuda.empty_cache() + + +def load_sampler(ckpt_path: str, sampler): + sampler_states = llm_load(os.path.join(ckpt_path, "sampler.pt")) + sampler.load_state_dict(sampler_states) + if gpc.is_rank_for_log(): + pstate = copy.deepcopy(sampler_states) + pstate.pop("indices") + pstate.pop("rng_state") + logger.info(f"reload sampler_states:{pstate}") + torch.cuda.empty_cache() + + +def load_context(ckpt_path: str, train_state: TrainState): + context_stuffs = llm_load(os.path.join(ckpt_path, "context.pt")) + train_state.load_state_dict(context_stuffs) + if gpc.is_rank_for_log(): + logger.info(f"reload train_state:{train_state}") + torch.cuda.empty_cache() + + +def load_scheduler(ckpt_path: str, lr_scheduler, optimizer, train_state: TrainState): + learning_rate = train_state.lr + scheduler_states = llm_load(os.path.join(ckpt_path, "schedulder.pt")) + if learning_rate != scheduler_states["base_lrs"][0] and gpc.is_rank_for_log(): + logger.warning( + f"Using new learning rate {learning_rate} to replace old learn rate {scheduler_states['base_lrs'][0]}." + ) + + base_lrs = copy.deepcopy(scheduler_states["base_lrs"]) + scheduler_states["base_lrs"] = [learning_rate] * len(scheduler_states["base_lrs"]) + if "after_scheduler_dict" in scheduler_states: + scheduler_states["after_scheduler_dict"]["base_lrs"] = [learning_rate] * len( + scheduler_states["after_scheduler_dict"]["base_lrs"] + ) + + lr_scheduler.load_state_dict(scheduler_states) + lr_scheduler.last_epoch = train_state.step_count + 1 + + ratios = [learning_rate / lr for lr in base_lrs] + for idx, param_group in enumerate(optimizer.param_groups): + param_group["lr"] = param_group["lr"] * ratios[idx] + torch.cuda.empty_cache() + + if gpc.is_rank_for_log(): + logger.info(f"reload load_scheduler:{lr_scheduler}") + + +class CheckpointManager: + """StorageManagerContext""" + + def __init__( + self, + ckpt_config, + model, + train_dl=None, + optimizer=None, + lr_scheduler=None, + model_config=None, + model_config_file=None, + feishu_address=None, + ) -> None: + """ + CheckpointManager is used to decide when to store ckpt. If it is an asynchronous + upload mode, you must call wait_async_upload_finish at the end of the program to wait + for the asynchronous ckpt upload to complete. + + Args: + ckpt_config (dict): model checkpoint config. + model (nn.module): model obj. + optimizer (object): optimizer obj. + lr_scheduler (object): lr_scheduler obj. + model_config (dict): model config. + """ + self.enable_save_ckpt = get_config_value(ckpt_config, "enable_save_ckpt", False) + self.checkpoint_every = get_config_value(ckpt_config, "checkpoint_every", 100) + self.save_ckpt_folder = get_config_value(ckpt_config, "save_ckpt_folder", None) + self.oss_snapshot_freq: int = get_config_value(ckpt_config, "oss_snapshot_freq", 50) + self.stop_file_path = get_config_value(ckpt_config, "stop_file_path", None) + if self.save_ckpt_folder: + self.snapshot_ckpt_folder = get_config_value( + ckpt_config, "snapshot_ckpt_folder", os.path.join(self.save_ckpt_folder, "snapshot") + ) + self.async_upload_tmp_folder = get_config_value( + ckpt_config, "async_upload_tmp_folder", "/dev/shm/internlm_tmp_ckpt/" + ) + else: + self.snapshot_ckpt_folder = None + self.async_upload_tmp_folder = None + + self.async_upload = get_config_value(ckpt_config, "async_upload", False) + + # initialization storage manager + init_storage_manager(self.enable_save_ckpt, self.async_upload_tmp_folder, self.async_upload) + + self.feishu_address = feishu_address + self.storage_manager = get_storage_manager() + self.snapshot_counter = 0 + + self.model = model + self.optimizer = optimizer + self.lr_scheduler = lr_scheduler + self.train_dl = train_dl + self.model_config = model_config + self.model_config_file = model_config_file + + # Register defalut internlm ckpt load type. + self.defalut_load_type_func = {CheckpointLoadType.INTERNLM: try_load_internlm_ckpt} + for ckpt_load_type in CheckpointLoadType: + CheckpointLoadMethod.register_ckpt_load_type(ckpt_load_type, self.defalut_load_type_func[ckpt_load_type]) + + # Init alter file. + if self.stop_file_path and gpc.get_global_rank() == 0: + dir_path = os.path.dirname(self.stop_file_path) + if dir_path != "" and not os.path.exists(dir_path): + os.makedirs(dir_path) + with open(self.stop_file_path, "w", encoding="utf-8") as f: + f.write("0") + + self.load_ckpt_info = get_config_value(ckpt_config, "load_ckpt_info", None) + if self.load_ckpt_info is None: # (legacy): Try Compatible with old interfaces + self.load_ckpt_info = ckpt_info_sanity_check(ckpt_config) + + # Auto-reload latest checkpoint, it will overwrite the setting of 'load_ckpt_info'. + self.auto_resume = get_config_value(ckpt_config, "auto_resume", None) + if self.auto_resume is None: # (legacy): Try Compatible with old interfaces + self.auto_resume = auto_resume_sanity_check(ckpt_config) + if self.auto_resume: + self.load_ckpt_info = self.query_lastest_ckpt() + + if self.stop_file_path is None and gpc.is_rank_for_log(): + logger.warning("no set stop_file_path, quit_signal_handler is disable") + + # model_hf to internal representation + if self.load_ckpt_info: + assert ( + "path" in self.load_ckpt_info + and "content" in self.load_ckpt_info + and "ckpt_type" in self.load_ckpt_info + ), "please set content in ckpt setting, eg: ckpt = dict(path='', content=['model'], ckpt_type='internlm')" + + # replace load_ckpt + self.load_ckpt_info["content"] = CheckpointLoadMask(self.load_ckpt_info["content"]) + self.load_ckpt_info["ckpt_type"] = CheckpointLoadMethod.convet_load_type(self.load_ckpt_info["ckpt_type"]) + + # test storage setting is ok. + if self.enable_save_ckpt: + self.try_ping_storage() + + def quit_signal_handler(self, train_state) -> bool: + """ + Exit signal detection function, if we write the exit step in the 'QUIT_FILE_PATH' file, + all ranks will save ckpt and exit. + Negative integer step means save ckpt. + Positive integer step means save ckpt and quit. + + Args: + train_state (TrainState): + Returns: + bool: whether to quit. + """ + now_break, now_save_ckpt, save_type = False, False, CheckpointSaveType.NORMAL_CHECKPOINT + + if self.stop_file_path is None: + return now_break, now_save_ckpt, save_type + + with torch.no_grad(): + action_step_t = torch.zeros((1,), dtype=torch.int64).cuda() + if gpc.get_global_rank() == 0: + with open(self.stop_file_path, "r+", encoding="utf-8") as f: + f.seek(0) + msg = f.read() + action_step_t.fill_(int(msg)) + + torch.distributed.broadcast(action_step_t, src=0) + action_step = action_step_t.item() + del action_step_t + + if action_step < 0 and abs(action_step) == train_state.step_count: + now_save_ckpt = True + + if action_step > 0 and action_step == train_state.step_count: + now_break, now_save_ckpt = True, True + + if action_step != 0 and gpc.is_rank_for_log(): + msg = "Stop" if action_step > 0 else "Save" + action_step = abs(action_step) + if train_state.step_count <= action_step: + if self.feishu_address: + send_alert_message( + address=self.feishu_address, + message=f"training will {msg} at step_count {action_step}!\ +now step_count is {train_state.step_count}", + ) + + return now_break, now_save_ckpt, save_type + + def is_now_to_save_ckpt(self, train_state, data_iter_stop=False) -> (bool, CheckpointSaveType, bool): + if data_iter_stop: + return True, CheckpointSaveType.NORMAL_CHECKPOINT, True + save_ckpts, save_type, now_break = False, CheckpointSaveType.NORMAL_CHECKPOINT, False + if self.oss_snapshot_freq > 1 and train_state.step_count % self.oss_snapshot_freq == 0: + save_ckpts, save_type = True, CheckpointSaveType.SNAPSHOT_CHECKPOINT + if train_state.step_count % self.checkpoint_every == 0: + save_ckpts, save_type = True, CheckpointSaveType.NORMAL_CHECKPOINT + now_break, singal_save_ckpts, singal_save_type = self.quit_signal_handler(train_state) + if save_ckpts is False: + save_ckpts = singal_save_ckpts + save_type = singal_save_type + + return save_ckpts, save_type, now_break + + def try_save_checkpoint(self, train_state, data_iter_stop=False): + if not self.enable_save_ckpt: + return False + + save_ckpts, save_type, now_break = self.is_now_to_save_ckpt(train_state, data_iter_stop) + + if save_ckpts: + # Wait for the previous round of asynchronous upload storage to complete. + self.storage_manager.wait() + if save_type == CheckpointSaveType.SNAPSHOT_CHECKPOINT: + # Snapshot number, with only two snapshots written alternately. + self.snapshot_counter = (self.snapshot_counter + 1) % 2 + save_ckpt_folder = os.path.join(self.snapshot_ckpt_folder, f"{self.snapshot_counter}") + else: + save_ckpt_folder = os.path.join(self.save_ckpt_folder, str(train_state.step_count)) + + self.save_checkpoint( + folder=save_ckpt_folder, + model=self.model, + optimizer=self.optimizer, + scheduler=self.lr_scheduler, + train_state=train_state, + model_config=self.model_config, + model_config_file=self.model_config_file, + ) + + return now_break + + def wait_async_upload_finish(self): + """wait for all checkpoint uploads to be completed""" + self.storage_manager.wait() + torch.distributed.barrier() + + def query_latest_snapshot_step_boto3(self): + """query_latest_snapshot_step_boto3 + Returns: + Tuple(str, int): path of latest ckpt and ckpt step, if not found, None will return. + """ + ckpt_list = self.storage_manager.get_fns(self.save_ckpt_folder) + if ckpt_list is None or len(ckpt_list) == 0: + return None, None + + max_normal_step = 0 + # Return ckpt_list look like: ['pings', 'snapshot', '4'] + # Here we only try to find the ckpt folder named after step, ignoring snapshot and other folders. + ckpt_list = [int(fn.strip("/")) for fn in ckpt_list if fn.strip("/").isdigit()] + if len(ckpt_list) == 0: + logger.warning("Not found avaliable normal checkpoint!") + else: + logger.info(f"Found avaliable normal checkpoint: {ckpt_list}!") + ckpt_list.sort(reverse=True) + for ckpt in ckpt_list: + fns_list = self.storage_manager.get_fns(os.path.join(self.save_ckpt_folder, str(ckpt))) + for fn in fns_list: + if fn.endswith(".step"): + max_normal_step = ckpt + break + if max_normal_step != 0: + break + + max_normal_step = ckpt_list[0] + load_normal_ckpt_path = os.path.join(self.save_ckpt_folder, str(max_normal_step)) + + snapshot_path_0 = os.path.join(self.save_ckpt_folder, "snapshot", "0") + snapshot_path_1 = os.path.join(self.save_ckpt_folder, "snapshot", "1") + ckpt_list_0 = self.storage_manager.get_fns(snapshot_path_0) + ckpt_list_1 = self.storage_manager.get_fns(snapshot_path_1) + + def found_latest_snapshot(_ckpt_list): + _max_step_snapshot = 0 + if _ckpt_list: + for ckpt in _ckpt_list: + ckpt = ckpt.strip("/") + if ckpt.endswith(".step"): + _max_step_snapshot = max(_max_step_snapshot, int(ckpt.split(".")[0])) + return _max_step_snapshot + + max_step_0 = found_latest_snapshot(ckpt_list_0) + max_step_1 = found_latest_snapshot(ckpt_list_1) + + if sum([max_step_0, max_step_1, max_normal_step]) == 0: + return None, None + else: + snap_load_path = snapshot_path_0 if max_step_0 > max_step_1 else snapshot_path_1 + snap_step = max(max_step_0, max_step_1) + load_path = snap_load_path if snap_step > max_normal_step else load_normal_ckpt_path + return load_path, max(snap_step, max_normal_step) + + def query_latest_snapshot_step_local(self): + max_step, max_step_path = 0, None + save_ckpt_folder = self.save_ckpt_folder.split(":")[1] + for root, _, files in os.walk(save_ckpt_folder, followlinks=True): + for fn in files: + fn = fn.strip("/") + if fn.endswith(".step"): + # We assume that both internlm ckpt and snapshot ckpt will store the '.step' file + # as an integrity flag. + step = int(fn.rsplit(".", maxsplit=1)[0]) + if max_step < step: + max_step = step + max_step_path = root + + return max_step_path, max_step + + def query_lastest_ckpt(self): + latest_ckpt, step = None, -1 + # Training was automatically restarted by the process, forcing the latest snapshot to be read. + if self.save_ckpt_folder: + backend, _ = try_get_storage_backend(self.save_ckpt_folder) + if backend == "boto3": + latest_ckpt, step = self.query_latest_snapshot_step_boto3() + if latest_ckpt and not latest_ckpt.startswith("boto3:"): + latest_ckpt = ":".join(["boto3", latest_ckpt]) + elif backend == "local": + latest_ckpt, step = self.query_latest_snapshot_step_local() + if latest_ckpt and not latest_ckpt.startswith("local:"): + latest_ckpt = ":".join(["local", latest_ckpt]) + + if gpc.is_rank_for_log(): + logger.info(f"Found latest ckpt {latest_ckpt if latest_ckpt else 'None'}, step: {step}...") + + return dict(path=latest_ckpt, content=("all",), ckpt_type="internlm") + + def try_resume_training(self, train_state: TrainState, current_time=""): + if self.load_ckpt_info is None or self.load_ckpt_info["path"] is None: + if gpc.is_rank_for_log(): + logger.info( + f"===========New Run {current_time} on host:{socket.gethostname()},rank={gpc.get_global_rank()}," + f"tp={gpc.get_local_rank(ParallelMode.TENSOR)},pp={gpc.get_local_rank(ParallelMode.PIPELINE)}," + f"dp={gpc.get_local_rank(ParallelMode.DATA)}===========" + ) + else: + load_path = self.load_ckpt_info["path"] + load_content = self.load_ckpt_info["content"] + load_type = self.load_ckpt_info["ckpt_type"] + + load_func = CheckpointLoadMethod.get_ckpt_load_type_func(load_type) + load_content_str = load_func(self, self.load_ckpt_info, train_state) + + # If we only load model weight, we need rewrite zero optim's fp32 buffer. + if load_content.only_load(CheckpointLoadContent.MODEL) and isinstance(self.optimizer, HybridZeroOptimizer): + reload_zero_fp32_buff(self.optimizer) + + if gpc.is_rank_for_log(): + logger.info(f"load_ckpt_info : {self.load_ckpt_info}") + logger.info( + f"===========Resume training from `{load_path}` {current_time} on host:" + f"{socket.gethostname()}===========" + ) + if load_content_str: + logger.info(f"===========Load contents are: {load_content_str}") + + @llm_timeout(func_name="save_checkpoint") + def save_checkpoint( + self, + folder, + model, + optimizer, + scheduler, + train_state: TrainState, + model_config: Dict = None, + model_config_file: str = None, + ): + """ + Save checkpoint to the given folder path. + """ + + start = time.time() + self.set_save_folder(folder, train_state.step_count) + torch.cuda.synchronize() + torch.distributed.barrier() + if gpc.is_rank_for_log(): + logger.info(f"Saving checkpoint to `{folder}` at batch count:{train_state.step_count}...") + + timer("save-model").start() + save_model_checkpoint(folder=folder, model=model) + timer("save-model").stop() + + timer("save-optimizer").start() + save_optimizer_checkpoint(optim=optimizer, state_path=folder) + timer("save-optimizer").stop() + + if ( + hasattr(train_state, "data_state_dict") + and gpc.get_local_rank(ParallelMode.TENSOR) == 0 + and gpc.get_local_rank(ParallelMode.PIPELINE) == 0 + ): + llm_save( + os.path.join(folder, f"sampler_{gpc.get_local_rank(ParallelMode.DATA)}.pt"), + saved_obj=train_state.data_state_dict, + ) + + if gpc.is_rank_for_log(): + if scheduler: + scheduler_states = scheduler.state_dict() + llm_save(os.path.join(folder, "schedulder.pt"), saved_obj=scheduler_states) + + if hasattr(train_state, "batch_sampler") and not isinstance( + train_state.batch_sampler, torch.utils.data.sampler.BatchSampler + ): + sampler_state = train_state.batch_sampler.state_dict() + llm_save(os.path.join(folder, "sampler.pt"), saved_obj=sampler_state) + llm_save(os.path.join(folder, "context.pt"), saved_obj=train_state.state_dict()) + + if model_config is not None: + # Model configuration dictionary. + llm_save(os.path.join(folder, "model_config.pt"), saved_obj=model_config) + + if model_config_file is not None: + # The complete training config file content, stored in binary format. + llm_save(os.path.join(folder, "config_file.pt"), saved_obj=model_config_file) + + torch.distributed.barrier() + + if gpc.is_rank_for_log(): + timer.log(["save-model", "save-optimizer"], logger=logger) + logger.info(f"Step: {train_state.step_count}, rank 0 save ckpt use {time.time() - start:.3f} s") + if self.storage_manager.async_mode is False: + llm_save( + os.path.join(folder, f"{train_state.step_count}.step"), + saved_obj=dict({"step": train_state.step_count}), + ) + + def set_save_folder(self, folder, step): + self.storage_manager.latest_save_folder = folder + self.storage_manager.latest_save_step = step + + def try_ping_storage(self): + if gpc.get_global_rank() % 8 == 0: + buff = torch.ones((1, 64, 64), dtype=torch.bfloat16) + test_fn = os.path.join(self.save_ckpt_folder, f"pings/{socket.gethostname()}.ping") + self.storage_manager.save(test_fn, buff) + self.storage_manager.wait() + self.storage_manager.load(test_fn) + del buff diff --git a/InternLM/internlm/utils/parallel.py b/InternLM/internlm/utils/parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..8e8618b3fd6f15cb92ea0c15813a8a4a03ee7c41 --- /dev/null +++ b/InternLM/internlm/utils/parallel.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import torch.distributed as dist + +from internlm.core.context import IS_TENSOR_PARALLEL, ParallelMode +from internlm.core.context import global_context as gpc + + +def is_model_parallel_parameter(p): + return hasattr(p, IS_TENSOR_PARALLEL) and getattr(p, IS_TENSOR_PARALLEL) + + +def sync_model_param(model, parallel_mode): + r"""Make sure data parameters are consistent during Data Parallel Mode. + + Args: + model (:class:`torch.nn.Module`): A pyTorch model on whose parameters you check the consistency. + parallel_mode (:class:`internlm.core.context.ParallelMode`): Parallel mode to be checked. + """ + if gpc.is_initialized(parallel_mode) and gpc.get_world_size(parallel_mode) > 1: + for param in model.parameters(): + ranks = gpc.get_ranks_in_group(parallel_mode) + dist.broadcast(param, src=ranks[0], group=gpc.get_group(parallel_mode)) + + +def sync_model_param_within_tp(model): + r"""This function is changed from colossalai, which is ``sync_model_param``. + + We modified this function to make sure it only sync parameters within tensor parallelism + but they are not splitted by tensor parallelism. + This function is used to make sure parameters that are not splitted by tensor parallelism + are the same across each tensor parallelism. + For tools, parameters like RMSNorm, LayerNorm... + + Args: + model (:class:`torch.nn.Module`): A pyTorch model on whose parameters you check the consistency. + """ + parallel_mode = ParallelMode.TENSOR + if gpc.is_initialized(parallel_mode) and gpc.get_world_size(parallel_mode) > 1: + for param in model.parameters(): + if not is_model_parallel_parameter(param): + ranks = gpc.get_ranks_in_group(parallel_mode) + dist.broadcast(param, src=ranks[0], group=gpc.get_group(parallel_mode)) + + +def is_no_pp_or_last_stage(): + return not gpc.is_initialized(ParallelMode.PIPELINE) or gpc.is_last_rank(ParallelMode.PIPELINE) + + +def get_parallel_log_file_name(): + if gpc.is_rank_for_log(): + fn_prefix = "main_" # Indicates a rank with more output information + else: + fn_prefix = "" + + log_file_name = ( + f"{fn_prefix}dp={gpc.get_local_rank(ParallelMode.DATA)}_" + f"tp={gpc.get_local_rank(ParallelMode.TENSOR)}_pp={gpc.get_local_rank(ParallelMode.PIPELINE)}" + ) + return log_file_name diff --git a/InternLM/internlm/utils/registry.py b/InternLM/internlm/utils/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..7cbfcc5ec1b7d6a147f4592fccd306282fe8590b --- /dev/null +++ b/InternLM/internlm/utils/registry.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + + +class Registry: + """This is a registry class used to register classes and modules so that a universal + object builder can be enabled. + + Args: + name (str): The name of the registry. + """ + + def __init__(self, name: str): + self._name = name + self._registry = dict() + + @property + def name(self): + return self._name + + def register_module(self, module_name: str): + """Registers a module represented in `module_class`. + + Args: + module_name (str): The name of module to be registered. + Returns: + function: The module to be registered, so as to use it normally if via importing. + Raises: + AssertionError: Raises an AssertionError if the module has already been registered before. + """ + + assert module_name not in self._registry, f"{module_name} not found in {self.name}" + + def decorator_wrapper(original_func): + self._registry[module_name] = original_func + return original_func + + return decorator_wrapper + + def get_module(self, module_name: str): + """Retrieves a module with name `module_name` and returns the module if it has + already been registered before. + + Args: + module_name (str): The name of the module to be retrieved. + Returns: + :class:`object`: The retrieved module or None. + Raises: + NameError: Raises a NameError if the module to be retrieved has neither been + registered directly nor as third party modules before. + """ + if module_name in self._registry: + return self._registry[module_name] + raise NameError(f"Module {module_name} not found in the registry {self.name}") + + def has(self, module_name: str): + """Searches for a module with name `module_name` and returns a boolean value indicating + whether the module has been registered directly or as third party modules before. + + Args: + module_name (str): The name of the module to be searched for. + Returns: + bool: A boolean value indicating whether the module has been registered directly or + as third party modules before. + """ + found_flag = module_name in self._registry + + return found_flag + + +MODEL_INITIALIZER = Registry("model_initializer") diff --git a/InternLM/internlm/utils/simple_memory_profiler.py b/InternLM/internlm/utils/simple_memory_profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..9caf0a2bafe0855675e89b11b4ca1a29b104a6b3 --- /dev/null +++ b/InternLM/internlm/utils/simple_memory_profiler.py @@ -0,0 +1,672 @@ +import os +import time +from collections import OrderedDict +from functools import partial, reduce +from typing import Any, Dict, List, Tuple + +import pyecharts +import torch + +from internlm.core.naive_amp import NaiveAMPModel + +mb = 1024 * 1024 + + +class SimpleMemState: + """ + A class to represent the memory state of a model layer. + + Args: + layer_name (str): The name of the layer. + layer_mem (int): The memory usage of the layer in bytes. + """ + + def __init__(self, layer_name: str, layer_mem: int = 0) -> None: + self.layer_name = layer_name + + # Memory status of the current model layer. + self._layer_mem: int = layer_mem + # Total memory status of the model and sub-models, initialized with layer memory. + self._total_mem: int = self._layer_mem + # SimpleMemState of sub-models. + self.sub_model_stats = OrderedDict() + + @property + def layer_mem(self) -> int: + """ + Get the memory usage of the layer. + + Returns: + int: The memory usage of the layer in bytes. + """ + return self._layer_mem + + @layer_mem.setter + def layer_mem(self, new_layer_mem: int) -> None: + """ + Set the memory usage of the layer. + + Args: + new_layer_mem (int): The new memory usage of the layer in bytes. + """ + diff = new_layer_mem - self._layer_mem + self._layer_mem = new_layer_mem + self._total_mem += diff + + @property + def total_mem(self) -> int: + """ + Get the total memory usage of the model and sub-models. + + Returns: + int: The total memory usage in bytes. + """ + return self._total_mem + + def add(self, layer_name: str, layer_mem: int = 0, flush: bool = True) -> None: + """ + Add a layer to the memory state. + + Args: + layer_name (str): The name of the layer. + layer_mem (int, optional): The memory usage of the layer in bytes. Defaults to 0. + flush (bool, optional): Whether to update the total memory usage. Defaults to True. + """ + path = layer_name.split(".") + + target = self.find_layer_state(path, create=True) + target.layer_mem = layer_mem + + if flush: + self.update_total_memory() + + def delete(self, layer_name: str, flush: bool = True) -> None: + """ + Delete a layer from the memory state. + + Args: + layer_name (str): The name of the layer. + flush (bool, optional): Whether to update the total memory usage. Defaults to True. + """ + path = layer_name.split(".") + assert len(path) >= 2, f"Only support deleting non-root layers, layer_name: {layer_name}" + + parent_path = path[0:-1] + layer = path[-1] + parent = self.find_layer_state(parent_path) + + if parent is not None and layer in parent.sub_model_stats: + del parent.sub_model_stats[layer] + + if flush: + self.update_total_memory() + + def update_total_memory(self) -> None: + """ + Update the total memory usage of the model and sub-models. + """ + self._total_mem = self._layer_mem + + for stat in self.sub_model_stats.values(): + # Update sub-model status first. + stat.update_total_memory() + # Add sub-model total_mem to model total_mem. + self._total_mem += stat._total_mem + + def find_layer_state(self, path: Tuple[str], create: bool = False) -> "SimpleMemState": + """ + Find the memory state of a layer. + + Args: + path (Tuple[str]): The path to the layer. + create (bool, optional): Whether to create the layer if it doesn't exist. Defaults to False. + + Returns: + SimpleMemState: The memory state of the layer. + """ + current_node = self + + for _node in path: + if _node not in current_node.sub_model_stats: + if not create: + return None + # Create a layer node. + current_node.sub_model_stats[_node] = SimpleMemState(_node) + + current_node = current_node.sub_model_stats[_node] + + return current_node + + def dump(self, prefix: str = "") -> str: + """ + Dump the memory state of the model and sub-models. + + Args: + prefix (str, optional): The prefix to add to the layer names. Defaults to "". + + Returns: + str: The memory state information. + """ + cur_prefix = prefix + "." + self.layer_name if prefix != "" else self.layer_name + res = f"layer: {cur_prefix}, layer_mem: {self.layer_mem / mb:.2f} MB, total_mem: {self.total_mem / mb:.2f} MB\n" + + for sub_layer in self.sub_model_stats.values(): + res += sub_layer.dump(cur_prefix) + + return res + + def to_json(self, base: int = 1024 * 1024) -> dict: + """ + Convert the memory state to a JSON structure. + + Returns: + dict: The JSON structure of the memory state. + """ + children = [child.to_json() for child in self.sub_model_stats.values()] + if len(children) == 0: + return {"name": self.layer_name, "value": self.layer_mem // base} + else: + return {"name": self.layer_name, "children": children} + + +class ActivationMemState: + """ + Activation Memory State + """ + + def __init__(self, num_chunks: int) -> None: + self._num_chunks = num_chunks + + self.inited: List[bool] = [False for _ in range(num_chunks)] + self.states: List[SimpleMemState] = [SimpleMemState(f"activations_{idx}") for idx in range(num_chunks)] + + @property + def total_mem(self) -> int: + return sum(state.total_mem for state in self.states) + + def dump(self, prefix: str = "") -> str: + return reduce(lambda x, y: x + y, [state.dump(prefix) for state in self.states]) + + def to_json(self, base: int = 1024 * 1024) -> List: + return [state.to_json(base) for state in self.states] + + +def _unpack_naive_wrapper(model: torch.nn.Module) -> Tuple[torch.nn.Module, int]: + num_chunks = len(model) if isinstance(model, torch.nn.ModuleList) else 1 + + if num_chunks > 1: + model = torch.nn.ModuleList([_model.model if isinstance(_model, NaiveAMPModel) else _model for _model in model]) + else: + model = model.model if isinstance(model, NaiveAMPModel) else model + + return model, num_chunks + + +class SimpleMemoryProfiler: + """ + A memory profiler for a llm model. + + Args: + model (torch.nn.Module): The model to profile. + optimizer (torch.optim.Optimizer): The optimizer used for training the model. + log_file (str): The file to write the memory state information to. + total_steps: number of steps to trace. + """ + + def __init__( + self, + model: torch.nn.Module, + optimizer: torch.optim.Optimizer, + log_folder: str, + total_steps: int = 5, + ): + self._model, self._num_model_chunks = _unpack_naive_wrapper(model) + self._optimizer = optimizer + self._log_folder = log_folder + self._remaining_steps = total_steps + + self._stoped = False + self._record_start_time = time.time() + + # For activation memory state. + + self._activation_mem: int = 0 + self._activation_mem_max: int = 0 + self._activation_base_mems = ActivationMemState(self._num_model_chunks) + + # Check or create log folder + os.makedirs(self._log_folder, exist_ok=True) + + # Register activation memory tracking hooks + if self._num_model_chunks > 1: + for chunk_id in range(self._num_model_chunks): + self._register_activation_trace_hooks(chunk_id, self._model[chunk_id]) + else: + self._register_activation_trace_hooks(0, self._model) + + # Calculate static parameter cuda memory + self._param_mem_state = SimpleMemState("param_mem") + self._calc_tensor_memory(self._param_mem_state, self._model.named_parameters()) + # Calculate static grad cuda memory + self._grad_mem_state = SimpleMemState("grad_mem") + self._calc_tensor_memory(self._grad_mem_state, self._model.named_parameters(), True) + # Calculate static optimizer state cuda memory + self._os_params_mem_state = SimpleMemState("os_params_mem") + self._os_state_mem_state = SimpleMemState("os_state_mem") + self._calc_tensor_group_memory(self._os_params_mem_state, list(enumerate(self._optimizer.param_groups))) + + # Generate the first memory record + self.point(with_options="params,grads,os_params", create=True) + + def point(self, with_options: str = "", create: bool = False) -> None: + """ + Record the memory state. + + Args: + with_options (str, optional): The options to include in the memory state. Defaults to "". + create (bool, optional): Whether to create a new memory record file. Defaults to False. + + Returns: + None + """ + now = time.time() + file = f"{self._log_folder}/memory.log" + + if with_options == "all": + options = ["params", "grads", "os_params", "os_state", "activation_base"] + else: + options = with_options.split(",") + + total_mem = ( + self._param_mem_state.total_mem + + self._grad_mem_state.total_mem + + self._os_params_mem_state.total_mem + + self._os_state_mem_state.total_mem + + self._activation_mem + ) / mb + + # Generate summary information for memory state + summary_info = ( + f"total_memory: {total_mem:.2f} MB" + + "\n" + + f"params_memory: {self._param_mem_state.total_mem / mb:.2f} MB, " + + f"grads_memory: {self._grad_mem_state.total_mem / mb:.2f} MB, " + + f"os_params_memory: {self._os_params_mem_state.total_mem / mb:.2f} MB, " + + f"os_state_memory: {self._os_state_mem_state.total_mem / mb:.2f} MB, " + + f"activation_memory: {self._activation_mem / mb:.2f} MB" + ) + + # Generate layout information based on selected options + layout_info = "" + if "params" in options: + layout_info += "params_layout:\n" + self._param_mem_state.dump() + if "grads" in options: + layout_info += "grads_layout:\n" + self._grad_mem_state.dump() + if "os_params" in options: + layout_info += "os_params_layout:\n" + self._os_params_mem_state.dump() + if "os_state" in options: + layout_info += "os_state_layout:\n" + self._os_state_mem_state.dump() + if "activation_base" in options: + layout_info += "activation_base_layout:\n" + self._activation_base_mems.dump() + + # Write memory state information to log file + file_mode = "w" if create else "a" + with open(file, file_mode, encoding="utf-8") as writer: + writer.write( + "Memory State:\n" + f"time: {now - self._record_start_time}\n" + "---summary---\n" + summary_info + "\n" + ) + if layout_info != "": + writer.write("---Layout---\n" + layout_info) + writer.write("\n") + + def step(self) -> None: + """ + Update the memory state of the optimizer state. + + Returns: + None + """ + if self._stoped: + return + + self._remaining_steps -= 1 + if self._remaining_steps == 0: + self._stoped = True + + # Update os state memory usage + self._os_state_mem_state = SimpleMemState("os_state_mem") + self._calc_tensor_group_memory(self._os_state_mem_state, list(self._optimizer.state_dict()["state"].items())) + + if not self._stoped: + # Do we need to print os_state_layout every time? Is it always constant? + self.point(with_options="os_state") + else: + # Dump memory layout + self.point(with_options="all") + # Generate sunburst charts + self._render_sunburst_chart(self._param_mem_state.to_json()["children"], "params_memory_sunburst") + self._render_sunburst_chart(self._grad_mem_state.to_json()["children"], "grads_memory_sunburst") + self._render_sunburst_chart( + [self._os_params_mem_state.to_json(), self._os_state_mem_state.to_json()], + "os_memory_sunburst", + ) + self._render_sunburst_chart(self._activation_base_mems.to_json(), "activation_memory_sunburst") + # Generate summary sunburst chart + summary_sunburst_data = [ + {"name": "params", "value": self._param_mem_state.total_mem // mb}, + {"name": "grads", "value": self._grad_mem_state.total_mem // mb}, + {"name": "os_params", "value": self._os_params_mem_state.total_mem // mb}, + {"name": "os_state", "value": self._os_state_mem_state.total_mem // mb}, + {"name": "activation", "value": self._activation_mem_max // mb}, + ] + + self._render_sunburst_chart(summary_sunburst_data, "summary_sunburst") + + def _render_sunburst_chart(self, data: Any, name: str) -> None: + pyecharts.charts.Sunburst(init_opts=pyecharts.options.InitOpts(width="1000px", height="1000px")).add( + name, + data_pair=data, + highlight_policy="ancestor", + radius=[0, "95%"], + levels=[ + {}, + { + "r0": "10%", + "r": "35%", + "itemStyle": {"borderWidth": 3}, + "label": {"align": "left"}, + }, + {"r0": "35%", "r": "55%", "label": {"align": "left"}}, + {"r0": "55%", "r": "70%", "label": {"align": "left"}}, + {"r0": "70%", "r": "80%", "label": {"align": "left"}}, + {"r0": "80%", "r": "90%", "label": {"align": "left"}}, + { + "r0": "90%", + "r": "92%", + "label": {"position": "outside", "padding": 3, "silent": False}, + "itemStyle": {"borderWidth": 3}, + }, + ], + ).set_global_opts(title_opts=pyecharts.options.TitleOpts(title="CUDA Memory")).set_series_opts( + label_opts=pyecharts.options.LabelOpts(formatter="{b}") + ).render( + f"{self._log_folder}/{name}.html" + ) + + def _inner_activation_trace_hook( + self, + chunk_id: int, + layer_name: str, + model: Any, + inputs: Any, + output: torch.Tensor, + ) -> None: + """ + Hook function to trace the activation memory usage for a inner layer. + + Args: + layer_name (str): The name of the layer. + model (Any): The model. + inputs (Any): The inputs to the layer. + output (torch.Tensor): The output tensor. + + Returns: + None + """ + del model, inputs + assert isinstance(output, torch.Tensor), f"Invalid output type: {type(output)}" + + if self._stoped or self._activation_base_mems.inited[chunk_id]: + return + + # Delay updating the total_mem of activation_base_mem here, it will be handled in the forward ending hook. + self._activation_base_mems.states[chunk_id].add( + layer_name, output.element_size() * output.nelement(), flush=False + ) + + def _activation_trace_hook_forward(self, chunk_id: int, model: Any, inputs: Any, output: torch.Tensor) -> None: + """ + Hook function to trace the activation memory usage for a forward pass. + + Args: + model (Any): The model. + inputs (Any): The inputs to the model. + output (torch.Tensor): The output tensor. + + Returns: + None + """ + del model, inputs + assert isinstance(output, torch.Tensor), f"invalid output type: {type(output)}" + + if self._stoped: + return + + # Check if the activation memory has been initialized + if self._activation_base_mems.inited[chunk_id] is False: + self._activation_base_mems.inited[chunk_id] = True + # Update the total memory of the activation base memory state + self._activation_base_mems.states[chunk_id].update_total_memory() + # Set with_options to "activation_base" to include activation_base_layout in the memory dump + with_options = "activation_base" + else: + with_options = "" + + # Accumulate activation memory usage for each forward pass + self._activation_mem += self._activation_base_mems.states[chunk_id].total_mem + if self._activation_mem > self._activation_mem_max: + self._activation_mem_max = self._activation_mem + + # Trigger a memory record + self.point(with_options) + + def _activation_tarce_hook_backward(self, chunk_id: int, model: Any, inputs: Any, grad_outputs: Any) -> None: + """ + Hook function to trace the activation memory usage for a backward pass. + + Args: + model (Any): The model. + inputs (Any): The inputs to the model. + grad_outputs (Any): The gradients of the outputs. + + Returns: + None + """ + del model, inputs, grad_outputs + + if self._stoped: + return + + # Release activation memory usage for each backward pass + self._activation_mem -= self._activation_base_mems.states[chunk_id].total_mem + + # Trigger a memory record + self.point() + + def _register_activation_trace_hooks(self, chunk_id: int, model_chunk: torch.nn.Module) -> None: + """ + Register activation trace hooks for the model and each submodule in the model. + """ + + # Register inner activation trace hooks for each submodule in the model + for layer_name, sub_model in model_chunk.named_modules(): + # Register the hook + if len(sub_model._modules) != 0: + continue # TODO: in some special cases, we may need some additional configuration to correct + + sub_model.register_forward_hook(partial(self._inner_activation_trace_hook, chunk_id, layer_name)) + + # Register a forward hook for the main model to track activation memory usage + model_chunk.register_forward_hook(partial(self._activation_trace_hook_forward, chunk_id)) + # Register a backward hook for the main model to release activation memory usage + model_chunk.register_full_backward_hook(partial(self._activation_tarce_hook_backward, chunk_id)) + + def _calc_tensor_memory( + self, root_stat: SimpleMemState, named_tensors: Dict[str, torch.Tensor], require_grad: bool = False + ) -> None: + """ + Calculate the memory usage of tensors and update the memory state. + + Args: + root_stat (SimpleMemState): The root memory state. + named_tensors (Dict[str, torch.Tensor]): A dictionary containing the named tensors. + require_grad (bool, optional): Whether to consider tensors with gradients. Defaults to False. + + Returns: + None + """ + for name, tensor in named_tensors: + if require_grad and not tensor.requires_grad: + continue + + layer_splits = name.split(sep=".") + layer_stat = root_stat.find_layer_state(layer_splits, create=True) + layer_stat.layer_mem = tensor.element_size() * tensor.nelement() + + root_stat.update_total_memory() + + def _calc_tensor_group_memory(self, root_stat: SimpleMemState, tensor_groups: List[Tuple[int, torch.Tensor]]): + """ + Calculate the memory usage of a group of tensors. + + Args: + root_stat (SimpleMemState): The root memory state. + tensor_groups (List[Tuple[int, torch.Tensor]]): A list of tuples containing the tensor groups. + + Returns: + None + """ + + def _normalize_helper(named_tensors: Dict[str, Any]) -> List[Tuple[str, Any]]: + """ + Normalize the named tensors. + + Args: + named_tensors (Dict[str, Any]): The named tensors to normalize. + + Returns: + List[Tuple[str, Any]]: The normalized named tensors. + """ + res = {} + + for name, tensors in named_tensors.items(): + if isinstance(tensors, torch.Tensor): + res[name] = tensors + elif isinstance(tensors, (list, tuple)): + for index, tensor in enumerate(tensors): + res[f"{name}.{index}"] = tensor + elif isinstance(tensors, dict): + for subname, tensor in tensors.items(): + res[f"{name}.{subname}"] = tensor + else: + raise TypeError(f"unsupported normalize value type: {type(tensors)}") + + return list(res.items()) + + def _value_check(tensor_or_tensors): + """ + Check if the input is a tensor or a collection of tensors. + + Args: + tensor_or_tensors (Any): The input to check. + + Returns: + bool: True if the input is a tensor or a collection of tensors, False otherwise. + """ + if torch.is_tensor(tensor_or_tensors): + return True + elif isinstance(tensor_or_tensors, (list, tuple)) and all(torch.is_tensor(x) for x in tensor_or_tensors): + return True + elif isinstance(tensor_or_tensors, dict) and all(torch.is_tensor(x) for x in tensor_or_tensors.values()): + return True + else: + return False + + # Calculate the memory usage of a group of tensors. + for idx, tensors in tensor_groups: + # Normalize the named tensors + named_tensors = {f"{idx}.{k}": v for k, v in tensors.items() if _value_check(v)} + named_tensors = _normalize_helper(named_tensors) + # Calculate the memory usage of the tensors and update the memory state + self._calc_tensor_memory(root_stat, named_tensors) + + +if __name__ == "__main__": + + class SimpleModel(torch.nn.Module): + """ + A simple model with three linear layers. + + Args: + skip_layer2 (bool, optional): Whether to skip layer2. Defaults to False. + """ + + def __init__(self, skip_layer2: bool = False): + super().__init__() + self.layer1 = torch.nn.Linear(5120, 5120, True) + self.layer3 = torch.nn.Linear(5120, 5120, False) + + if skip_layer2: + self.layer2 = None + else: + self.layer2 = SimpleModel(skip_layer2=True) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + """ + Forward pass of the model. + + Args: + inputs (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor. + """ + output1 = self.layer1(inputs) + if self.layer2 is not None: + output2 = self.layer2(output1) + else: + output2 = output1 + output = self.layer3(output2) + + return output + + def _simple_schedule(_num_chunks, _model_chunks, _input) -> torch.Tensor: + if _num_chunks > 1: + _output = _input + for _model_chunk in _model_chunks: + _output = _model_chunk(_output) + else: + _output = _model_chunks(_input) + + return _output + + # num_chunks config + _num_chunks = 1 + + # init model and optimizer + if _num_chunks > 1: + _chunks = [SimpleModel(skip_layer2=idx % 2 == 0) for idx in range(_num_chunks)] + _model = torch.nn.ModuleList(_chunks).cuda() + else: + _model: torch.nn.Module = SimpleModel().cuda() + _optimizer = torch.optim.Adam(_model.parameters()) + + # init profiler + profiler = SimpleMemoryProfiler(_model, _optimizer, "./test_simple_memory_profiler", total_steps=1) + + _optimizer.zero_grad() + + # inputs + x1 = torch.randn((128, 5120)).cuda() + x2 = torch.randn((128, 5120)).cuda() + # forward + out1 = _simple_schedule(_num_chunks, _model, x1) + out2 = _simple_schedule(_num_chunks, _model, x2) + # backward + out1.mean().backward() + out2.mean().backward() + + _optimizer.step() + + # Update the optimizer state memory usage and record the memory state + profiler.step() diff --git a/InternLM/internlm/utils/storage_manager.py b/InternLM/internlm/utils/storage_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..9204b25ad0f49a46b74eee1907093753bad8b7e7 --- /dev/null +++ b/InternLM/internlm/utils/storage_manager.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import asyncio +import concurrent.futures +import hashlib +import io +import os +import pickle +import re +import socket +import stat +from asyncio import InvalidStateError +from asyncio.tasks import ALL_COMPLETED +from datetime import datetime +from typing import Any, Awaitable, Callable, Dict, List, Union + +import torch +import torch.distributed as dist + +from internlm.core.context import global_context as gpc +from internlm.utils.common import SingletonMeta +from internlm.utils.logger import get_logger + +try: + import boto3 + import botocore +except ImportError: + pass + + +logger = get_logger(__file__) + +boto3_url_re = re.compile(r"([^\.]+)\.([\d\.]+)") + +MB = 1024**2 + +storage_manager = None + + +def check_folder(fp: str): + storage_manager.assert_fp_exists(fp) + + +def get_fns(fp: str): + return storage_manager.get_fns(fp) + + +def llm_load(fp: str, **kwargs): + return storage_manager.load(fp, **kwargs) + + +def llm_save(save_path: str, saved_obj: Any, **kwargs): + storage_manager.save(save_path, to_save_obj=saved_obj, **kwargs) + + +class StorageClient: + """ + StorageClient as a client for s3 storage access. + """ + + def __init__(self, handler) -> None: + self.handler = handler + + @staticmethod + def load(*args, **kwargs): + raise NotImplementedError + + @staticmethod + def sync_upload_fileobj(*args, **kwargs): + raise NotImplementedError + + @staticmethod + def async_upload_fileobj(*args, **kwargs): + raise NotImplementedError + + @staticmethod + def assert_fp_exists(*args, **kwargs): + raise NotImplementedError + + @staticmethod + def get_fns(*args, **kwargs): + raise NotImplementedError + + +class Boto3MetaInfo: + """Boto3 meta info for save/load etc.""" + + def __init__( + self, + is_async, + handler: StorageClient, + bucket_name: str, + endpoint: str, + file_path: str, + async_upload_fn: callable, + local_nvme_path=None, + ) -> None: + # all need info. + self.client = handler + self.bucket_name = bucket_name + self.file_path = file_path + # only save need info. + self.local_nvme_path = local_nvme_path + self.is_async = is_async + self.endpoint = endpoint + self.async_upload_fn = async_upload_fn + + def __str__(self) -> str: + return f"is_async: {self.is_async}, bucket_name:{self.bucket_name}, endpoint:{self.endpoint}, \ +local_nvme_path: {self.local_nvme_path}" + + @staticmethod + def unpack_boto3_save_meta(meta): + if meta.is_async: + return meta.client, meta.bucket_name, meta.file_path, meta.local_nvme_path + else: + return meta.client, meta.bucket_name, meta.file_path + + @staticmethod + def unpack_boto3_nosave_meta(meta): + return meta.client, meta.bucket_name, meta.file_path + + +class LocalMetaInfo: + """Local meta info for save/load etc.""" + + def __init__(self, file_path: str) -> None: + self.file_path = file_path + self.async_upload_fn = None + self.is_async = False + + @staticmethod + def unpack_local_save_meta(meta): + return (meta.file_path,) + + @staticmethod + def unpack_local_nosave_meta(meta): + return (meta.file_path,) + + +def unpack_save_meta(meta: Union[Boto3MetaInfo, LocalMetaInfo]): + if isinstance(meta, Boto3MetaInfo): + return Boto3MetaInfo.unpack_boto3_save_meta(meta) + elif isinstance(meta, LocalMetaInfo): + return LocalMetaInfo.unpack_local_save_meta(meta) + else: + raise ValueError(f"unkonwn meta info: {type(meta)}") + + +def unpack_nosave_meta(meta: Union[Boto3MetaInfo, LocalMetaInfo]): + if isinstance(meta, Boto3MetaInfo): + return Boto3MetaInfo.unpack_boto3_nosave_meta(meta) + elif isinstance(meta, LocalMetaInfo): + return LocalMetaInfo.unpack_local_nosave_meta(meta) + else: + raise ValueError(f"unkonwn meta info: {type(meta)}") + + +def compute_file_md5_by_chunk(file_name: str): + hash_md5 = hashlib.md5() + with open(file_name, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def try_get_storage_backend(path: str): + sre = path.split(":", maxsplit=1) + if len(sre) == 1: + if path.startswith("s3:"): + backend = "boto3" + if gpc.is_rank_for_log(): + logger.warning(f"path: '{path}' not start with backend prefix, guess it is the backend of boto3.") + else: + backend = "local" + if gpc.is_rank_for_log(): + logger.warning(f"path: '{path}' not start with backend prefix, guess it is the backend of local.") + return backend, sre + else: + return sre[0], sre[1] # (backend_prefix, splited_path) + + +class Boto3Client(StorageClient): + """ + Boto3Client + """ + + def __init__( + self, + s3_endpoint_url: str, + use_threads: int = True, + multipart_chunksize=8 * MB, + max_concurrency: int = 10, + multipart_threshold=100 * MB, + ) -> None: + """S3 object/file storage management class + + Args: + s3_access_keys_id (str): S3 access key ID. + s3_secret_access_key (str): S3 secret access key. + use_threads (bool, optional): Whether to enable multipart. Defaults to True. + multipart_chunksize (_type_, optional): Defaults to 8*MB. + max_concurrency (int, optional): Defaults to 10. + + Raises: + RuntimeError: Connection failures caused by misconfiguration or network problems. + """ + super().__init__(boto3) + self.botocore = botocore + try: + s3_access_key_id = os.environ["S3_ACCESS_KEY_ID"] + s3_secret_access_key = os.environ["S3_SECRET_ACCESS_KEY_ID"] + except KeyError as exc: + raise RuntimeError( + "Please set boto3 bucket 'S3_ACCESS_KEY_ID' and 'S3_SECRET_ACCESS_KEY_ID' using environment variable!" + ) from exc + + self.client = self.handler.client( + "s3", + "", + use_ssl=False, + verify=False, + endpoint_url=s3_endpoint_url, + aws_access_key_id=s3_access_key_id, + aws_secret_access_key=s3_secret_access_key, + ) + + self.config = self.handler.s3.transfer.TransferConfig( + multipart_threshold=multipart_threshold, + max_concurrency=max_concurrency, + multipart_chunksize=multipart_chunksize, + use_threads=use_threads, + ) + + @staticmethod + def sync_upload_fileobj(handler, bucket_name: str, fp: str, saved_obj=None, **kwargs): + assert saved_obj is not None, "saved_obj is None!" + try: + with io.BytesIO() as f: + torch.save(saved_obj, f, **kwargs) + f.seek(0) + handler.client.upload_fileobj(f, bucket_name, fp, Config=handler.config) + except handler.botocore.exceptions.EndpointConnectionError as exc: + raise RuntimeError( + f"Boto3 Network Error: Please Check your Internet Connection in {socket.gethostname()}" + ) from exc + + @staticmethod + def load(handler, bucket_name: str, fp: str, **kwargs) -> Dict: + """ + Args: + fp (str): Path to save, eg. s3://opennlplab/model_weights/xxx/ddd.pt + """ + try: + with io.BytesIO() as f: + handler.client.download_fileobj(bucket_name, fp, f, Config=handler.config) + f.seek(0) + states = torch.load(f, **kwargs) + except handler.botocore.exceptions.EndpointConnectionError as exc: + raise RuntimeError( + f"Boto3 Network Error: Please Check your Internet Connection in {socket.gethostname()}" + ) from exc + return states + + @staticmethod + def assert_fp_exists(handler, bucket_name: str, fp: str): # pylint: disable=W0613 + assert len(list(handler.client.list_objects(Bucket=bucket_name, Prefix=fp)["Contents"])) > 0, fp + + @staticmethod + def is_fp_exists(handler, bucket_name: str, fp: str): # pylint: disable=W0613 + re = handler.client.list_objects(Bucket=bucket_name, Prefix=fp) + if "Contents" in re: + return len(list(re["Contents"])) > 0 + else: + return False + + @staticmethod + def get_fns(handler, bucket_name: str, fp: str): + """ + Ref: https://stackoverflow.com/questions/54314563/ + how-to-get-more-than-1000-objects-from-s3-by-using-list-objects-v2 + """ + if Boto3Client.is_fp_exists(handler, bucket_name, fp): + paginator = handler.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=fp) + folder_name_list = [] + for page in pages: + if "Contents" in page: + for obj in page["Contents"]: + pth: str = obj["Key"] + folder_name_list.append(pth.split(fp, maxsplit=1)[1].strip("/").split("/", maxsplit=1)[0]) + return list(set(folder_name_list)) + else: + if gpc.is_rank_for_log(): + logger.warning(f"'{fp}' not found!") + return None + + @staticmethod + def async_upload_fileobj(handler, bucket_name: str, fp: str, local_nvme_path: str): + try: + with open(local_nvme_path, "rb") as f: + handler.client.upload_fileobj(f, bucket_name, fp, Config=handler.config) + except handler.botocore.exceptions.EndpointConnectionError as exc: + raise RuntimeError( + f"Boto3 Network Error: Please Check your Internet Connection in {socket.gethostname()}" + ) from exc + except Exception as e: + raise e + + @staticmethod + def delete_obj(handler, fp: str): + raise NotImplementedError("boto3 not support delete_obj") + + +class LocalClient(StorageClient): + """ + Storage Client for local NFS. + """ + + def __init__(self, *args, **kwargs) -> None: # pylint: disable=W0613 + super().__init__(None) + + @staticmethod + def sync_upload_fileobj(fp: str, saved_obj=None, **kwargs): + assert saved_obj is not None + fp_dirname = os.path.dirname(fp) + if not os.path.exists(fp_dirname): + os.makedirs(fp_dirname, exist_ok=True) + torch.save(saved_obj, fp, **kwargs) + + @staticmethod + def load(load_path: str, **kwargs): + assert os.path.exists(load_path), f"{load_path} is not found!" + with open(load_path, "rb") as f: + states = torch.load(f, **kwargs) + return states + + @staticmethod + def assert_fp_exists(folder): + assert os.path.exists(folder), folder + + @staticmethod + def get_fns(folder): + if not os.path.exists(folder): + if gpc.is_rank_for_log(): + logger.warning(f"'{folder}' not found!") + return None + else: + return os.listdir(folder) + + @staticmethod + def delete_obj(fp: str): + if not os.path.isdir(fp): + os.remove(fp) + + +def get_tmp_file_name(tmp_local_folder: str, fp: str): + """ + It should be noted that all our temporary files will be stored in the same folder, + so the file name passed upstream must be unique. + """ + base_path = os.path.join(tmp_local_folder, fp.split("/")[-1]) + current_time = datetime.now().strftime("%b%d_%H-%M-%S") + pid = os.getpid() + # step = self.step_counter + return "-".join([base_path, current_time, str(pid)]) + ".tmpfile" # , str(step) + + +def get_boto3_meta(fp: str, tmp_local_folder: str, is_async: bool) -> Boto3MetaInfo: + assert fp.startswith("s3://"), f"Path '{fp}' is not a boto3 url" + parts = fp.lstrip("s3://").split(os.path.sep) + match = boto3_url_re.match(parts[0]) + assert match is not None, f"url '{fp}' is not a valid boto3 url" + bucket_name, endpoint = match.group(1), match.group(2) + endpoint = "http://" + endpoint + ":80" + if is_async: + tmp_step_file = get_tmp_file_name(tmp_local_folder, fp) + else: + tmp_step_file = None + return Boto3MetaInfo( + is_async=is_async, + handler=None, + bucket_name=bucket_name, + endpoint=endpoint, + file_path=os.path.sep.join(parts[1:]), + async_upload_fn=Boto3Client.async_upload_fileobj, + local_nvme_path=tmp_step_file, + ) + + +def get_local_meta(fp: str) -> LocalMetaInfo: + assert not fp.startswith("s3://"), f"Path '{fp}' is not a local path" + return LocalMetaInfo(fp) + + +def get_mount_point_free_size(path: str): + """ + Returns the remaining space of the temporary storage mount point as a percentage. + Args: + path (str): temporary storage folder path. + + Raises: + FileNotFoundError: If the temporary storage folder does not exist, + an error will be reported。 + """ + if os.path.exists(path): + st = os.statvfs(path) + # f_bavail: Number of free blocks for unprivileged users. + # f_bsize: Filesystem block size. + # return unit is TB. + return st.f_bavail * st.f_bsize / (1024**3) + + +def check_tmp_folder_accessibility(tmp_local_folder: str): + """ + Check access permissions for temporary storage. + """ + ret = True + if os.path.exists(tmp_local_folder): + ret &= os.access(tmp_local_folder, os.W_OK) + ret &= os.access(tmp_local_folder, os.R_OK) + if ret is False: + error_str = f'{socket.gethostname()} dose not have read and write permissions on {tmp_local_folder}"' + raise RuntimeError(error_str) + + +class StorageManager(metaclass=SingletonMeta): + """ + Storage Manager for saving or loading checkpoint. + TODO: add a thread to poll the asynchronous storage state. + """ + + BACKEND_TYPE = {"boto3", "local"} + BACKEND_INIT_METHOD = { + "boto3": Boto3Client, + "local": LocalClient, + } + CLI_DICT = {} + + def __init__(self, enable_save, tmp_local_folder="/dev/shm/test/", async_mode=True, n_async_workers=8) -> None: + self._exception_list = [] + self._to_be_del_files = [] + self._async_stack = [] + self.upload_count = 0 + self.tmp_local_folder = tmp_local_folder + self.async_mode = async_mode + self.has_warning = False + self._async_loop = None + self._thread_pool = None + self.latest_save_folder = None + self.latest_save_step = 0 + self.async_task_peeding = False + + if enable_save and self.async_mode: + self._async_loop = asyncio.new_event_loop() + self._thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=n_async_workers) + + check_tmp_folder_accessibility(os.path.dirname(self.tmp_local_folder)) + + # Try to create tmp folder + try: + os.makedirs(self.tmp_local_folder, exist_ok=True) + os.chmod(self.tmp_local_folder, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + except FileExistsError: + pass + + # In case it is a directory created by other users, we check the permissions again. + check_tmp_folder_accessibility(self.tmp_local_folder) + + # Try to clean tmp folder's empty folder. + self.try_delete_tmpfile(self.tmp_local_folder) + + # Avaliable storeage space check. + free_size = get_mount_point_free_size(self.tmp_local_folder) + if free_size < 0.1: + logger.error(f'tmp_local_folder only have "{free_size}" GB free space, less then 100 GB!') + raise RuntimeError(f"Insufficient temporary storage space on {socket.gethostname()}") + + def _get_client(self, path: str, async_mode: bool = False) -> Union[Boto3MetaInfo, LocalMetaInfo]: + """ + tools: + local:/path/to/checkpoint + boto3:s3://model_weights/0331/120bi + + Args: + path (str): _description_ + """ + backend, path = try_get_storage_backend(path) + + init_args = (None,) + if backend == "local": + meta_info = get_local_meta(path) + backend_key = backend + elif backend == "boto3": + meta_info = get_boto3_meta(path, self.tmp_local_folder, async_mode) + backend_key = backend + ":" + meta_info.endpoint + init_args = (meta_info.endpoint,) + if ( + "http_proxy" in os.environ + or "https_proxy" in os.environ + or "HTTP_PROXY" in os.environ + or "HTTPS_PROXY" in os.environ + ): + if not self.has_warning: + logger.warning( + "HTTP/HTTPS proxy is detected when using boto3, incorrectly setting \ + the proxy may make boto3 unavailable or affect performance." + ) + self.has_warning = True + + assert backend in StorageManager.BACKEND_TYPE, f"Unkown backend: {backend}" + + # boto3 backend need special treatment. + if backend_key not in StorageManager.CLI_DICT: + StorageManager.CLI_DICT.update({backend_key: StorageManager.BACKEND_INIT_METHOD[backend](*init_args)}) + + meta_info.client = StorageManager.CLI_DICT[backend_key] + + return meta_info + + def assert_fp_exists(self, folder) -> None: + meta = self._get_client(path=folder) + meta.client.assert_fp_exists(*unpack_nosave_meta(meta)) + + def get_fns(self, folder) -> List[str]: + meta = self._get_client(path=folder) + return meta.client.get_fns(*unpack_nosave_meta(meta)) + + def save(self, save_path: str, to_save_obj: Any, async_upload=None, **kwargs): + + if async_upload is None: + async_upload = self.async_mode + + if not save_path.startswith("boto3:"): + async_upload = False + + meta = self._get_client(save_path, async_upload) + + if async_upload: + assert ( + self.tmp_local_folder + ), "StorageManager is not setted tmp_local_folder, so async save cannot be performed." + tmp_step_file = meta.local_nvme_path + self._to_be_del_files.append(tmp_step_file) + with open(tmp_step_file, "wb") as f: + torch.save(to_save_obj, f, pickle_protocol=pickle.HIGHEST_PROTOCOL) + self.async_executor(meta.async_upload_fn, *unpack_save_meta(meta)) + os.chmod(tmp_step_file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + self.async_task_peeding = True + else: + meta.client.sync_upload_fileobj(*unpack_save_meta(meta), saved_obj=to_save_obj, **kwargs) + self.upload_count += 1 + + def load(self, load_path: str, **kwargs) -> Any: + self.wait() + meta = self._get_client(path=load_path) + return meta.client.load(*unpack_nosave_meta(meta), **kwargs) + + def delete_obj(self, fp: str): + meta = self._get_client(path=fp) + meta.client.delete_obj(*unpack_nosave_meta(meta)) + + def _del_tmp_folder(self): + for fp in self._to_be_del_files: + try: + os.remove(fp) + except FileNotFoundError: + pass + except SystemError as e: + logger.error(f'delete file: {fp}, failed for reason:"{e}"') + else: + pass + + def try_delete_tmpfile(self, tmp_dir: str): + """Delete temporary files in tmp_dir.""" + + for filename in os.listdir(tmp_dir): + if filename.endswith(".tmpfile"): + file_path = os.path.join(tmp_dir, filename) + try: + os.remove(file_path) + logger.info(f"Delete tmpfile: {file_path}") + except OSError: + # Ignore deletion errors + pass + + async def _sync_tasks(self) -> Awaitable[None]: + if self._async_stack: + await asyncio.wait(self._async_stack, return_when=ALL_COMPLETED) + count = 0 + while self._async_stack: + t = self._async_stack[0] + try: + e = t.exception() + if e: + self._exception_list.append((e, count)) + logger.error(f"File:{self._to_be_del_files[count]}, upload failed for {e}") + # raise e + count += 1 + self._async_stack.pop(0) + except InvalidStateError: + # Not finished. https://docs.python.org/3/library/asyncio-task.html#asyncio.Task.exception + pass + + def async_executor(self, fn: Callable, *args, **kwargs) -> None: + """ + Overview: + Execute task in background, then apppend the future instance in _async_stack. + Arguments: + - fn (:obj:`Callable`): Synchronization fuction. + """ + if not self._async_loop: + raise RuntimeError("Event loop was not initialized, please call this function in async or parallel mode") + t = self._async_loop.run_in_executor(self._thread_pool, fn, *args, **kwargs) + self._async_stack.append(t) + + def wait(self) -> bool: + """Wait for async operations to complete.""" + + if not self.async_mode: + return + + if not self.async_task_peeding: + return + + if self._async_loop: + self._async_loop.run_until_complete(self._sync_tasks()) + + if self._exception_list: + for error_msg, file_id in self._exception_list: + logger.error( + f"Node:{socket.gethostname()}, Error: Checkpoint {self._to_be_del_files[file_id]} " + f"failed on step {self.upload_count}: {error_msg}" + ) + + # TODO: Re-upload in sync mode + raise RuntimeError( + f"Failed to upload {self._to_be_del_files[file_id]} " f"on step {self.upload_count}: {error_msg}" + ) + + self._del_tmp_folder() + self._exception_list.clear() + self._to_be_del_files.clear() + self.async_task_peeding = False + + if gpc.is_rank_for_log(): + self.upload_count += 1 + if self.async_mode and self.latest_save_folder: + self.save( + os.path.join(self.latest_save_folder, f"{self.latest_save_step}.step"), + to_save_obj=dict({"step": self.latest_save_step}), + async_upload=False, + ) + self.latest_save_folder = None + + +storage_manager: StorageManager = None + + +def init_storage_manager(enable_save_ckpt, async_upload_tmp_folder, async_upload): + global storage_manager + storage_manager = StorageManager( + enable_save_ckpt, + tmp_local_folder=async_upload_tmp_folder, + async_mode=async_upload, + ) + + +def get_storage_manager(): + assert storage_manager is not None, "storage_manager has not been init!" + return storage_manager + + +def wait_async_upload_finish(): + dist.barrier() + storage_manager.wait() diff --git a/InternLM/internlm/utils/timeout.py b/InternLM/internlm/utils/timeout.py new file mode 100644 index 0000000000000000000000000000000000000000..7a96841fe8727dca4bb198f8c74d9f35dac96a0b --- /dev/null +++ b/InternLM/internlm/utils/timeout.py @@ -0,0 +1,113 @@ +import datetime +import os +import signal +import socket +import traceback +from functools import wraps + +from internlm.utils.logger import get_logger + +logger = get_logger(__file__) + + +class Timeout: + """Timer to execute code + + Adapted from https://github.com/reasoning-machines/pal + + Args: + seconds (float): The maximum seconds to execute code + error_message (str) + """ + + def __init__(self, seconds=1, error_message="Timeout"): + self.seconds = seconds + self.error_message = error_message + + def timeout_handler(self, signum, frame): + raise TimeoutError(self.error_message) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.timeout_handler) + signal.alarm(self.seconds) + + def __exit__(self, error_type, value, traceback): + signal.alarm(0) + + +ENABLE_TIMEOUT = os.getenv("INTERNLM_ENABLE_TIMEOUT", None) + + +timeout_threshold_dict = { + "initialize_distributed_env": 120, + "nopp_forward_backward_step": 360, + "initialize_model": 10, + "initialize_optimizer": 20, + "optim_step": 30, + "get_train_data_loader": 600, + "get_validation_data_loader": 60, + "load_new_batch": 10, + "record_current_batch_training_metrics": 10, + "save_checkpoint": 1200, + "interleaved_forward_backward_step": 600, + "nointerleaved_forward_backward_step": 600, +} + +if ENABLE_TIMEOUT is not None: + os.environ["NCCL_ASYNC_ERROR_HANDLING"] = "1" + LLM_NCCL_TIMEOUT = datetime.timedelta(seconds=int(os.getenv("NCCL_TIMEOUT", str(60)))) +else: + timeout_threshold_dict = dict.fromkeys(timeout_threshold_dict.keys(), 0) + LLM_NCCL_TIMEOUT = datetime.timedelta(seconds=1800) + + +def try_get_gpc_rank(): + try: + from internlm.core.context import global_context as gpc + + rank = gpc.get_global_rank() + except: # noqa # pylint: disable=bare-except + rank = "unknown" + + return f"host-{socket.gethostname()}-rank-{rank}" + + +def llm_timeout(seconds=0, func_name=None): + """timeout decorator, Note that this decorator cannot be reentrant, + otherwise the signal will be reset. + + Args: + seconds (int, optional): timeout threshold. Defaults to 300. + func_name (str, optional): the func who is been waited to timeout. + """ + + def decorator(func): + nonlocal func_name + if func_name is None: + func_name = func.__name__ + + @wraps(func) + def wrapper(*args, **kwargs): + def _handle_timeout(signum, frame): + raise TimeoutError + + nonlocal seconds + seconds = timeout_threshold_dict.get(func_name, seconds) + + if seconds > 0: + signal.signal(signal.SIGALRM, _handle_timeout) + signal.alarm(seconds) + + try: + result = func(*args, **kwargs) + except TimeoutError as e: + logger.error(f"TimeoutError at {try_get_gpc_rank()}: {func_name}\\n {traceback.format_exc()}") + raise e + finally: + signal.alarm(0) + + return result + + return wrapper + + return decorator diff --git a/InternLM/internlm/utils/writer.py b/InternLM/internlm/utils/writer.py new file mode 100644 index 0000000000000000000000000000000000000000..b519b954ee77278edbbaa322be2da1abd7464906 --- /dev/null +++ b/InternLM/internlm/utils/writer.py @@ -0,0 +1,150 @@ +import logging +import os +import socket +import sys +import traceback +from functools import partial + +import torch +from torch.utils.tensorboard import SummaryWriter + +from internlm.core.context import global_context as gpc + + +def tb_save_run_info(writer, config_lines, global_step=0): + writer.add_text(tag="cmd", text_string=" ".join(sys.argv[:]), global_step=global_step) + lines = [] + for line in config_lines: + if line.strip().startswith("#"): + continue + lines.append(line) + writer.add_text(tag="config", text_string="\n".join(lines), global_step=global_step) + + +def init_tb_writer( + job_name: str, + launch_time: str, + file_name: str, + tensorboard_folder: str, + resume_tb_folder: str, + step_count: int, + config: str, + logger: logging.Logger, +): + tb_log_file_name = file_name + if not tensorboard_folder: + tb_folder = os.path.join(job_name, launch_time, "tensorboards") + else: + tb_folder = tensorboard_folder + + if gpc.get_global_rank() == 0: + # If we don't load ckpt, 'resume_tb_folder' is set as the tensorboard + # dir of the last task by 'make_launch_script.sh'. + # If we load ckpt, 'resume_tb_folder' will be overwritten as the + # reloaded 'train_state.resume_tb_folder'.s + if resume_tb_folder is not None: + assert len(resume_tb_folder) > 0 and resume_tb_folder != "/" + if not os.path.exists(resume_tb_folder): + logger.error( + f"Can't found resume_tb_folder{resume_tb_folder}, \ +please make sure this folder is located at local file system." + ) + else: + logger.info(f"Try mv tensorboard logs: {resume_tb_folder} to {tb_folder}... ") + os.system(f"cp -r {resume_tb_folder}/* {tb_folder}/") + os.system(f"chmod -R +w {tb_folder}/") + else: + logger.info(f"Login tensorboard logs to: {tb_folder}") + + tb_logdir = os.path.join(tb_folder, tb_log_file_name) + writer = SummaryWriter(log_dir=tb_logdir, max_queue=5, purge_step=step_count, flush_secs=3) + writer.add_text(tag="job_name", text_string=job_name, global_step=step_count) + writer.add_text(tag="tensorboard_folder", text_string=tb_logdir, global_step=step_count) + + torch.distributed.broadcast_object_list([tb_folder], src=0) + else: + objects = [None] + torch.distributed.broadcast_object_list(objects, src=0) + tb_folder = objects[0] + tb_logdir = os.path.join(tb_folder, tb_log_file_name) + writer = SummaryWriter(log_dir=tb_logdir, max_queue=5, purge_step=step_count, flush_secs=3) + + if gpc.is_rank_for_log(): + tb_save_run_info( + writer=writer, + config_lines=config, + global_step=step_count, + ) + + writer.add_text( + tag=f"mapping_{tb_log_file_name}", + text_string=f"file_path={tb_logdir} hostname={socket.gethostname()} device={torch.cuda.current_device()}", + global_step=step_count, + ) + writer.add_scaler = partial(writer.add_scalar, new_style=True) + + return writer, tb_logdir + + +class Writer: + """ + Customed writer based on tensorboard for recording training metrics. + + Args: + job_name (str): The name of training job, defaults to None. + launch_time (str): A string representing the launch time of the training. + file_name (str): The log file name, defaults to None. + tensorboard_folder (str): A string representing the folder for saving tensorboard logs. + resume_tb_folder (str): A string representing the folder for resuming tensorboard logs. + step_count (int): An integer representing the step count of the training. + config (str): A string representing the configuration of the training. + logger (logging.Logger): A logging.Logger object for logging information during training. + enable_tb (bool): A boolean indicating whether to enable the tensorboard writer. + + """ + + def __init__( + self, + job_name: str = None, + launch_time: str = None, + file_name: str = None, + tensorboard_folder: str = None, + resume_tb_folder: str = None, + step_count: int = 0, + config: str = None, + logger: logging.Logger = None, + enable_tb: bool = True, + ) -> None: + self.enable_tb = enable_tb + self.tb_writer, self.tb_logdir = init_tb_writer( + job_name=job_name, + launch_time=launch_time, + file_name=file_name, + tensorboard_folder=tensorboard_folder, + resume_tb_folder=resume_tb_folder, + step_count=step_count, + config=config, + logger=logger, + ) + + def add_scalar(self, key, value, step): + try: + if self.enable_tb and self.tb_writer is not None: + self.tb_writer.add_scalar(tag=key, scalar_value=value, global_step=step) + except Exception: + traceback.print_exc() + + def add_scalars(self, key, value, step): + try: + assert isinstance(value, dict) + if self.enable_tb and self.tb_writer is not None: + self.tb_writer.add_scalars(main_tag=key, tag_scalar_dict=value, global_step=step) + except Exception: + traceback.print_exc() + + def add_text(self, key, value, step): + try: + if self.enable_tb and self.tb_writer is not None: + self.tb_writer.add_text(tag=key, text_string=value, global_step=step) + except Exception: + traceback.print_exc() diff --git a/InternLM/requirements/runtime.txt b/InternLM/requirements/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..f46d7ad91a5ee1b4aaaeb6082b0e89dfa974ded7 --- /dev/null +++ b/InternLM/requirements/runtime.txt @@ -0,0 +1,16 @@ +transformers<4.30.0 +sentencepiece +numpy +tqdm +psutil +packaging +pre-commit +ninja +gputil +pytest +packaging +boto3 +botocore +torch-scatter +pyecharts +-f https://data.pyg.org/whl/torch-1.13.1+cu117.html \ No newline at end of file diff --git a/InternLM/requirements/torch.txt b/InternLM/requirements/torch.txt new file mode 100644 index 0000000000000000000000000000000000000000..3b428b0d045cbe9b8adb9d10d0346be142e565d6 --- /dev/null +++ b/InternLM/requirements/torch.txt @@ -0,0 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/cu117 +torch==1.13.1+cu117 +torchvision==0.14.1+cu117 +torchaudio==0.13.1 \ No newline at end of file diff --git a/InternLM/tools/convert2hf.py b/InternLM/tools/convert2hf.py new file mode 100644 index 0000000000000000000000000000000000000000..11bb8e99794b4017c3d9f72353695fa86327929a --- /dev/null +++ b/InternLM/tools/convert2hf.py @@ -0,0 +1,163 @@ +import argparse +import json +import math +import os +import re +import tempfile +import sys + +import torch + +from model_hf.modeling_internlm import InternLMConfig, InternLMForCausalLM +sys.path.append('../') + +def convert2hf(model_config, states_tp_pps): + with tempfile.TemporaryDirectory() as folder: + states = merge_pp(states_tp_pps)[0] + + dims_per_head = model_config["hidden_size"] // model_config["num_attention_heads"] + base = 10000.0 + inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head)) + + current_states = {} + + vq_model_embed_weight = states.pop('embedding.vq_model.quantize.embedding.weight') + embed_proj_weight = states.pop('embedding.embed_proj.weight') + current_states["model.embed_tokens.weight"] = vq_model_embed_weight.mm(embed_proj_weight.T) + current_states["model.norm.weight"] = states.pop("norm.weight") + current_states["lm_head.weight"] = states.pop("head.weight") + + for i in range(model_config["num_layers"]): + states.pop(f"blocks.{i}.mixer.rotary_emb.inv_freq", None) + + wqkv = states.pop(f"blocks.{i}.mixer.Wqkv.weight").reshape( + 3, model_config["num_attention_heads"], -1, model_config["hidden_size"] + ) + bqkv = states.pop(f"blocks.{i}.mixer.Wqkv.bias").reshape(3, model_config["num_attention_heads"], -1) + + current_states[f"model.layers.{i}.self_attn.q_proj.weight"] = wqkv[0].reshape( + -1, model_config["hidden_size"] + ) + current_states[f"model.layers.{i}.self_attn.q_proj.bias"] = bqkv[0].reshape(-1) + current_states[f"model.layers.{i}.self_attn.k_proj.weight"] = wqkv[1].reshape( + -1, model_config["hidden_size"] + ) + current_states[f"model.layers.{i}.self_attn.k_proj.bias"] = bqkv[1].reshape(-1) + current_states[f"model.layers.{i}.self_attn.v_proj.weight"] = wqkv[2].reshape( + -1, model_config["hidden_size"] + ) + current_states[f"model.layers.{i}.self_attn.v_proj.bias"] = bqkv[2].reshape(-1) + + current_states[f"model.layers.{i}.self_attn.o_proj.weight"] = states.pop( + f"blocks.{i}.mixer.out_proj.weight" + ) + current_states[f"model.layers.{i}.self_attn.o_proj.bias"] = states.pop(f"blocks.{i}.mixer.out_proj.bias") + + current_states[f"model.layers.{i}.mlp.gate_proj.weight"] = states.pop(f"blocks.{i}.mlp.w1.weight") + current_states[f"model.layers.{i}.mlp.down_proj.weight"] = states.pop(f"blocks.{i}.mlp.w3.weight") + current_states[f"model.layers.{i}.mlp.up_proj.weight"] = states.pop(f"blocks.{i}.mlp.w2.weight") + + current_states[f"model.layers.{i}.input_layernorm.weight"] = states.pop(f"blocks.{i}.norm1.weight") + current_states[f"model.layers.{i}.post_attention_layernorm.weight"] = states.pop(f"blocks.{i}.norm2.weight") + current_states[f"model.layers.{i}.self_attn.rotary_emb.inv_freq"] = inv_freq + + config = InternLMConfig( + hidden_size=model_config["hidden_size"], + intermediate_size=compute_intermediate_size(model_config["hidden_size"]), + num_attention_heads=model_config["num_attention_heads"], + num_hidden_layers=model_config["num_layers"], + rms_norm_eps=1e-06, + bias=True, + ) + + if model_config["vocab_size"] != -1: + config.vocab_size = model_config["vocab_size"] + + config.save_pretrained(folder) + torch.save(current_states, os.path.join(folder, "pytorch_model.bin")) + + model = InternLMForCausalLM.from_pretrained(folder, torch_dtype=torch.float16) + del model.config._name_or_path + + return config, model + + +def compute_intermediate_size(n): + return int(math.ceil(n * 8 / 3) + 255) // 256 * 256 + + +def merge_pp(states_tp_pp): + max_tp = len(states_tp_pp) + max_pp = len(states_tp_pp[0]) + + full_states = [] + for tp in range(max_tp): + layer_shift = 0 + + tp_states = {} + for pp in range(max_pp): + _layer_shift = 0 + states = states_tp_pp[tp][pp] + keys = list(states.keys()) + for key in keys: + match = re.search("\.\d+\.", key) + if match is not None: + s, e = match.span() + layer_idx = int(key[s + 1: e - 1]) + layer_shift + _layer_shift = max(_layer_shift, int(key[s + 1: e - 1])) + name = key[:s] + f".{layer_idx}." + key[e:] + tp_states[name] = states[key] + else: + tp_states[key] = states[key] + layer_shift += _layer_shift + 1 + full_states.append({(key[6:] if key.startswith("model.") else key): value for key, value in tp_states.items()}) + return full_states + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--src_folder', type=str, default='/path/to/intermlm_model/') # internlm model folder + parser.add_argument('--tgt_folder', type=str, default='/path/to/hf_model/') # hf model folder + args = parser.parse_args() + + + def load(fp): + with open(fp, "rb") as f: + pt_data = torch.load(f, map_location="cpu") + return pt_data + + + folder = args.src_folder + target_folder = args.tgt_folder + model_config = load(os.path.join(folder, "model_config.pt")) + + fns = list(os.listdir(folder)) + + model_fns = [] + for fn in fns: + if fn.startswith("model_t") and not fn.endswith("md5"): + model_fns.append(fn) + + max_tp, max_pp = -1, -1 + for fn in model_fns: + _, tp, pp = os.path.splitext(fn)[0].split("_") + max_pp = max(max_pp, int(pp[2:]) + 1) + max_tp = max(max_tp, int(tp[2:]) + 1) + + states_tp_pps = [[]] + + for pp in range(max_pp): + model_name = f"model_tp0_pp{pp}.pt" + states = load(os.path.join(folder, model_name)) + states_tp_pps[0].append(states) + + config, model = convert2hf(model_config, states_tp_pps) + + os.makedirs(target_folder, exist_ok=True) + model.save_pretrained(target_folder, max_shard_size="20GB") + # TODO There should be a better way to add this. + with open(os.path.join(target_folder, "config.json")) as fp: + config_dict = json.load(fp) + config_dict["auto_map"]["AutoModel"] = "modeling_internlm.InternLMForCausalLM" + with open(os.path.join(target_folder, "config.json"), "w") as fp: + json.dump(config_dict, fp, indent=2) diff --git a/InternLM/tools/convert2hf_vit.py b/InternLM/tools/convert2hf_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..b4572a830b0fb99979d2311ed543dd8398b3ffa6 --- /dev/null +++ b/InternLM/tools/convert2hf_vit.py @@ -0,0 +1,168 @@ +import argparse +import json +import os +import re +import tempfile +import sys + +import torch + +from model_hf.modeling_vit import InternLMConfig, InternLMForCausalLM + +sys.path.append('../') + +def convert2hf(model_config, states_tp_pps): + with tempfile.TemporaryDirectory() as folder: + states = merge_pp(states_tp_pps)[0] + + dims_per_head = model_config["hidden_size"] // model_config["num_attention_heads"] + base = 10000.0 + inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head)) + + current_states = {} + + vq_model_embed_weight = states.pop('embedding.vq_model.quantize.embedding.weight') + embed_proj_weight = states.pop('embedding.embed_proj.weight') + current_states["model.embed_tokens.weight"] = vq_model_embed_weight.mm(embed_proj_weight.T) + current_states["model.norm.weight"] = states.pop("norm.weight") + current_states["model.norm.bias"] = states.pop("norm.bias") + current_states["lm_head.weight"] = states.pop("head.weight") + + mlp_bias = False + for i in range(model_config["num_layers"]): + states.pop(f"blocks.{i}.mixer.rotary_emb.inv_freq", None) + + wqkv = states.pop(f"blocks.{i}.mixer.Wqkv.weight").reshape( + 3, model_config["num_attention_heads"], -1, model_config["hidden_size"] + ) + bqkv = states.pop(f"blocks.{i}.mixer.Wqkv.bias").reshape(3, model_config["num_attention_heads"], -1) + + current_states[f"model.layers.{i}.self_attn.q_proj.weight"] = wqkv[0].reshape( + -1, model_config["hidden_size"] + ) + current_states[f"model.layers.{i}.self_attn.q_proj.bias"] = bqkv[0].reshape(-1) + current_states[f"model.layers.{i}.self_attn.k_proj.weight"] = wqkv[1].reshape( + -1, model_config["hidden_size"] + ) + current_states[f"model.layers.{i}.self_attn.k_proj.bias"] = bqkv[1].reshape(-1) + current_states[f"model.layers.{i}.self_attn.v_proj.weight"] = wqkv[2].reshape( + -1, model_config["hidden_size"] + ) + current_states[f"model.layers.{i}.self_attn.v_proj.bias"] = bqkv[2].reshape(-1) + + current_states[f"model.layers.{i}.self_attn.o_proj.weight"] = states.pop( + f"blocks.{i}.mixer.out_proj.weight" + ) + current_states[f"model.layers.{i}.self_attn.o_proj.bias"] = states.pop(f"blocks.{i}.mixer.out_proj.bias") + + current_states[f"model.layers.{i}.mlp.fc1.weight"] = states.pop(f"blocks.{i}.mlp.fc1.weight") + current_states[f"model.layers.{i}.mlp.fc2.weight"] = states.pop(f"blocks.{i}.mlp.fc2.weight") + + if f'blocks.{i}.mlp.fc1.bias' in states: + mlp_bias = True + current_states[f"model.layers.{i}.mlp.fc1.bias"] = states.pop(f"blocks.{i}.mlp.fc1.bias") + current_states[f"model.layers.{i}.mlp.fc2.bias"] = states.pop(f"blocks.{i}.mlp.fc2.bias") + + current_states[f"model.layers.{i}.input_layernorm.weight"] = states.pop(f"blocks.{i}.norm1.weight") + current_states[f"model.layers.{i}.input_layernorm.bias"] = states.pop(f"blocks.{i}.norm1.bias") + current_states[f"model.layers.{i}.post_attention_layernorm.weight"] = states.pop(f"blocks.{i}.norm2.weight") + current_states[f"model.layers.{i}.post_attention_layernorm.bias"] = states.pop(f"blocks.{i}.norm2.bias") + current_states[f"model.layers.{i}.self_attn.rotary_emb.inv_freq"] = inv_freq + + config = InternLMConfig( + hidden_size=model_config["hidden_size"], + intermediate_size=int(model_config["hidden_size"] * model_config["mlp_ratio"]), + num_attention_heads=model_config["num_attention_heads"], + num_hidden_layers=model_config["num_layers"], + norm_eps=1e-06, + bias=True, + mlp_bias=mlp_bias, + ) + + if model_config["vocab_size"] != -1: + config.vocab_size = model_config["vocab_size"] + + config.save_pretrained(folder) + torch.save(current_states, os.path.join(folder, "pytorch_model.bin")) + + model = InternLMForCausalLM.from_pretrained(folder, torch_dtype=torch.float16) + del model.config._name_or_path + + return config, model + + +def merge_pp(states_tp_pp): + max_tp = len(states_tp_pp) + max_pp = len(states_tp_pp[0]) + + full_states = [] + for tp in range(max_tp): + layer_shift = 0 + + tp_states = {} + for pp in range(max_pp): + _layer_shift = 0 + states = states_tp_pp[tp][pp] + keys = list(states.keys()) + for key in keys: + match = re.search("\.\d+\.", key) + if match is not None: + s, e = match.span() + layer_idx = int(key[s + 1: e - 1]) + layer_shift + _layer_shift = max(_layer_shift, int(key[s + 1: e - 1])) + name = key[:s] + f".{layer_idx}." + key[e:] + tp_states[name] = states[key] + else: + tp_states[key] = states[key] + layer_shift += _layer_shift + 1 + full_states.append({(key[6:] if key.startswith("model.") else key): value for key, value in tp_states.items()}) + return full_states + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--src_folder', type=str, default='/path/to/intermlm_model/') # internlm model folder + parser.add_argument('--tgt_folder', type=str, default='/path/to/hf_model/') # hf model folder + args = parser.parse_args() + + + def load(fp): + with open(fp, "rb") as f: + pt_data = torch.load(f, map_location="cpu") + return pt_data + + + folder = args.src_folder + target_folder = args.tgt_folder + model_config = load(os.path.join(folder, "model_config.pt")) + + fns = list(os.listdir(folder)) + + model_fns = [] + for fn in fns: + if fn.startswith("model_t") and not fn.endswith("md5"): + model_fns.append(fn) + + max_tp, max_pp = -1, -1 + for fn in model_fns: + _, tp, pp = os.path.splitext(fn)[0].split("_") + max_pp = max(max_pp, int(pp[2:]) + 1) + max_tp = max(max_tp, int(tp[2:]) + 1) + + states_tp_pps = [[]] + + for pp in range(max_pp): + model_name = f"model_tp0_pp{pp}.pt" + states = load(os.path.join(folder, model_name)) + states_tp_pps[0].append(states) + + config, model = convert2hf(model_config, states_tp_pps) + + os.makedirs(target_folder, exist_ok=True) + model.save_pretrained(target_folder, max_shard_size="20GB") + # TODO There should be a better way to add this. + with open(os.path.join(target_folder, "config.json")) as fp: + config_dict = json.load(fp) + config_dict["auto_map"]["AutoModel"] = "modeling_vit.InternLMForCausalLM" + with open(os.path.join(target_folder, "config.json"), "w") as fp: + json.dump(config_dict, fp, indent=2) diff --git a/InternLM/tools/data/derain_prompt/000000_img.png b/InternLM/tools/data/derain_prompt/000000_img.png new file mode 100644 index 0000000000000000000000000000000000000000..ab47c6d4e9d1ba905908a0cf00d10606e02eb368 --- /dev/null +++ b/InternLM/tools/data/derain_prompt/000000_img.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a093e28cc58f2c6c548eb8987b572230fd5060754960ded58e17f6db23d7a7b +size 125199 diff --git a/InternLM/tools/data/derain_prompt/000000_label.png b/InternLM/tools/data/derain_prompt/000000_label.png new file mode 100644 index 0000000000000000000000000000000000000000..7790600401e0588c5d4c6a44569014377e4ae95a --- /dev/null +++ b/InternLM/tools/data/derain_prompt/000000_label.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ebab2858e59c13adebb8d236a17bef8655a18710cb1a84cdbca7100a6f2a53 +size 124163 diff --git a/InternLM/tools/data/derain_prompt/000001_img.png b/InternLM/tools/data/derain_prompt/000001_img.png new file mode 100644 index 0000000000000000000000000000000000000000..31d1e59342550c60061232d3093eeeec286d9017 --- /dev/null +++ b/InternLM/tools/data/derain_prompt/000001_img.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6845215b57ef4b08fe92d0cbac064ddc6e8f6ef99da646493cb4b16f1bf684cc +size 119004 diff --git a/InternLM/tools/data/derain_prompt/000001_label.png b/InternLM/tools/data/derain_prompt/000001_label.png new file mode 100644 index 0000000000000000000000000000000000000000..b3044f8a58153a3522f94c28ce9deca70c4fb640 --- /dev/null +++ b/InternLM/tools/data/derain_prompt/000001_label.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d8b880f095d4470c5821677a94825fc723b094bee67bd1175970b2bdb6d42d +size 112877 diff --git a/InternLM/tools/data/derain_prompt/000002_img.png b/InternLM/tools/data/derain_prompt/000002_img.png new file mode 100644 index 0000000000000000000000000000000000000000..87ab5ad6b20288292f9eb850a39ecadb7faf1f78 --- /dev/null +++ b/InternLM/tools/data/derain_prompt/000002_img.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61b8f4dd1fa056bf4049a59e439a2a6b02af6947b248f4c5f7897eb0bb229755 +size 133866 diff --git a/InternLM/tools/data/derain_prompt/000002_label.png b/InternLM/tools/data/derain_prompt/000002_label.png new file mode 100644 index 0000000000000000000000000000000000000000..d1f84925f4493d15428a12fa9d4670fd923f6552 --- /dev/null +++ b/InternLM/tools/data/derain_prompt/000002_label.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a46b849110f6cd22ea85357a034b25f6c330c916e8740f2cfa5e28d5fee761f +size 129395 diff --git a/InternLM/tools/data/examples/derain_1.png b/InternLM/tools/data/examples/derain_1.png new file mode 100644 index 0000000000000000000000000000000000000000..043acc64f23ccaab877db64a1ffdfeaf2717a490 --- /dev/null +++ b/InternLM/tools/data/examples/derain_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1397f93d68b09cdcc9760ed07dc0924cd9aa38937283fd647677e7b0f158975b +size 112605 diff --git a/InternLM/tools/data/examples/derain_2.png b/InternLM/tools/data/examples/derain_2.png new file mode 100644 index 0000000000000000000000000000000000000000..904e20edca45869d3bbaef3751a48e4cb59ab937 --- /dev/null +++ b/InternLM/tools/data/examples/derain_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:339649329df66d4d0a7fdf54dc04b109dd1c4f7a8cd39395efef5546c504e254 +size 130336 diff --git a/InternLM/tools/data/examples/pose_1.png b/InternLM/tools/data/examples/pose_1.png new file mode 100644 index 0000000000000000000000000000000000000000..13d08f01a7d24f5002e13c16a19cf6ded07f1e38 Binary files /dev/null and b/InternLM/tools/data/examples/pose_1.png differ diff --git a/InternLM/tools/data/examples/pose_2.png b/InternLM/tools/data/examples/pose_2.png new file mode 100644 index 0000000000000000000000000000000000000000..91050314ef8efd0bb742d86041abdc88db82989f --- /dev/null +++ b/InternLM/tools/data/examples/pose_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d0bf066409a16af1f91ec9dcd081bd28b9e9c90607fc4938c41d4c2eb4161ee +size 116785 diff --git a/InternLM/tools/data/examples/seg_1.png b/InternLM/tools/data/examples/seg_1.png new file mode 100644 index 0000000000000000000000000000000000000000..3bc9a29d9436762fa722821d090d6d8d880c5b8b --- /dev/null +++ b/InternLM/tools/data/examples/seg_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f601ee7205066e8a0a817cf65f73faafd83d8976919df3d51a68b71eca88c0aa +size 117262 diff --git a/InternLM/tools/data/examples/seg_2.png b/InternLM/tools/data/examples/seg_2.png new file mode 100644 index 0000000000000000000000000000000000000000..7a412ca78d29d97db0dbc207c3bd5c72889992ec --- /dev/null +++ b/InternLM/tools/data/examples/seg_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cd3ca8b81b86fdca535eeaacb202d87206fe7572355ec56bbe8309c0410ef1c +size 105428 diff --git a/InternLM/tools/data/pose_prompt/000000_img.png b/InternLM/tools/data/pose_prompt/000000_img.png new file mode 100644 index 0000000000000000000000000000000000000000..b7f470b755819587d0a176c79c69a0c07be251b8 Binary files /dev/null and b/InternLM/tools/data/pose_prompt/000000_img.png differ diff --git a/InternLM/tools/data/pose_prompt/000000_label.png b/InternLM/tools/data/pose_prompt/000000_label.png new file mode 100644 index 0000000000000000000000000000000000000000..3a515dba771c89471ca61dbe05bace1c90bf19ad Binary files /dev/null and b/InternLM/tools/data/pose_prompt/000000_label.png differ diff --git a/InternLM/tools/data/pose_prompt/000001_img.png b/InternLM/tools/data/pose_prompt/000001_img.png new file mode 100644 index 0000000000000000000000000000000000000000..3c5481e2668193ea222d4125fa393200d4565847 Binary files /dev/null and b/InternLM/tools/data/pose_prompt/000001_img.png differ diff --git a/InternLM/tools/data/pose_prompt/000001_label.png b/InternLM/tools/data/pose_prompt/000001_label.png new file mode 100644 index 0000000000000000000000000000000000000000..1be7247af1a0b93877bb78690c4f307fc81e7db3 Binary files /dev/null and b/InternLM/tools/data/pose_prompt/000001_label.png differ diff --git a/InternLM/tools/data/pose_prompt/000002_img.png b/InternLM/tools/data/pose_prompt/000002_img.png new file mode 100644 index 0000000000000000000000000000000000000000..0c22c1b3f148a8e8db3068f44cfc327ee47ffca7 --- /dev/null +++ b/InternLM/tools/data/pose_prompt/000002_img.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8172c437cd808746c5c184512771c42497b203fa6c49bdd4bd508df0a8860d29 +size 114715 diff --git a/InternLM/tools/data/pose_prompt/000002_label.png b/InternLM/tools/data/pose_prompt/000002_label.png new file mode 100644 index 0000000000000000000000000000000000000000..e60863408747bc3b0d2b1f3bd8fa3a012c930b29 Binary files /dev/null and b/InternLM/tools/data/pose_prompt/000002_label.png differ diff --git a/InternLM/tools/data/seg_prompt/000000_img.png b/InternLM/tools/data/seg_prompt/000000_img.png new file mode 100644 index 0000000000000000000000000000000000000000..e9f2cb7721f4a4ffa8fe88a507c510a754c69fa3 --- /dev/null +++ b/InternLM/tools/data/seg_prompt/000000_img.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93bd2b077a7f24e57cac839dc857749b07fc0b32de48c3ff65ff337e90fe387c +size 117972 diff --git a/InternLM/tools/data/seg_prompt/000000_label.png b/InternLM/tools/data/seg_prompt/000000_label.png new file mode 100644 index 0000000000000000000000000000000000000000..8546fd7de4d3bab3af90ccbedcf7abc133e20004 Binary files /dev/null and b/InternLM/tools/data/seg_prompt/000000_label.png differ diff --git a/InternLM/tools/data/seg_prompt/000001_img.png b/InternLM/tools/data/seg_prompt/000001_img.png new file mode 100644 index 0000000000000000000000000000000000000000..4fda2c701a3db9831582b4cca59dd1f2099c2f98 Binary files /dev/null and b/InternLM/tools/data/seg_prompt/000001_img.png differ diff --git a/InternLM/tools/data/seg_prompt/000001_label.png b/InternLM/tools/data/seg_prompt/000001_label.png new file mode 100644 index 0000000000000000000000000000000000000000..ea70ebdcec62efe1bd26ca5c7a39ca7ecb64dc46 Binary files /dev/null and b/InternLM/tools/data/seg_prompt/000001_label.png differ diff --git a/InternLM/tools/data/seg_prompt/000002_img.png b/InternLM/tools/data/seg_prompt/000002_img.png new file mode 100644 index 0000000000000000000000000000000000000000..08b97902799b09ff9aa7254ba2d8f5ea9c662236 Binary files /dev/null and b/InternLM/tools/data/seg_prompt/000002_img.png differ diff --git a/InternLM/tools/data/seg_prompt/000002_label.png b/InternLM/tools/data/seg_prompt/000002_label.png new file mode 100644 index 0000000000000000000000000000000000000000..347b633cf0a106e97535441fe08438f3927d6649 Binary files /dev/null and b/InternLM/tools/data/seg_prompt/000002_label.png differ diff --git a/InternLM/tools/demo.ipynb b/InternLM/tools/demo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..359bde47460d4f731c8a1bff20307b25ec5525d9 --- /dev/null +++ b/InternLM/tools/demo.ipynb @@ -0,0 +1,275 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bc90fb20-685c-4649-bae1-614ec2d45926", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Step 1: convert imternlm model to hf model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "aecea625-1a52-4615-94ce-2f068b06546b", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/ma-user/anaconda3/envs/PyTorch-1.8/lib/python3.7/site-packages/requests/__init__.py:104: RequestsDependencyWarning: urllib3 (1.26.12) or chardet (5.0.0)/charset_normalizer (2.0.12) doesn't match a supported version!\n", + " RequestsDependencyWarning)\n", + "2024-02-05 18:08:41,399\tWARNING utils.py:30 in -- The torch implementation for cal_l2norm is slower than apex. Please note this!\n" + ] + } + ], + "source": [ + "!python convert2hf.py --src_folder /path/to/intermlm_model/ --tgt_folder /path/to/save/hf_model/" + ] + }, + { + "cell_type": "markdown", + "id": "0674bb47-59ac-4418-baa5-d9a99c1eeb8d", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Step2: Prompted inference" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "3e56d9fe-e74d-46e9-adf0-b50957b6ae18", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# prepare parameters\n", + "prompt_path='./data/seg_prompt/' # path to prompt\n", + "input_img='./data/examples/seg_1.png' # path to input image\n", + "lvm_path='/path/to/lvm_model/' # path to converted hf model\n", + "vqgan_path='/path/to/vqgan_model/' # path to vqgan model" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "84bd8204-146b-4383-baf7-b9e4434bc8d5", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# import packages\n", + "import os\n", + "\n", + "import torch\n", + "from PIL import Image\n", + "from transformers import AutoModel, GenerationConfig\n", + "\n", + "from model_hf.muse import VQGANModel\n", + "from utils import convert_decode_to_pil, encode_transform, patchify, unpatchify\n", + "from torchvision import transforms\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "4060b444-061d-41b0-b919-fe0105c55f36", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# prepare models and config\n", + "model = AutoModel.from_pretrained(lvm_path, trust_remote_code=True).cuda().eval()\n", + "vq_model = VQGANModel.from_pretrained(vqgan_path).cuda().eval()\n", + "\n", + "generation_config = GenerationConfig(\n", + " temperature=0.1,\n", + " top_p=0.75,\n", + " num_beams=1,\n", + " early_stopping=True,\n", + " max_new_tokens=256,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "cefbaecf-31ae-4711-a714-d75b064798f0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "prompt: 000000_img.png\n", + "prompt: 000000_label.png\n", + "prompt: 000001_img.png\n", + "prompt: 000001_label.png\n", + "prompt: 000002_img.png\n", + "prompt: 000002_label.png\n", + " torch.Size([1, 1536])\n" + ] + } + ], + "source": [ + "# prepare prompt\n", + "img_names = os.listdir(prompt_path)\n", + "img_names = sorted(img_names)\n", + "\n", + "seq_prompt, names = [], []\n", + "for i, img_name in enumerate(img_names):\n", + " print('prompt: ', img_name)\n", + " img_path = os.path.join(prompt_path, img_name)\n", + "\n", + " image = Image.open(img_path)\n", + " image = encode_transform(image)\n", + " image = image[0:3,:,:].unsqueeze(0)\n", + " seq_prompt.append(image)\n", + "\n", + "seq_ids = []\n", + "for images in seq_prompt:\n", + " images = images.cuda()\n", + "\n", + " # tokenize\n", + " quantized_states, indices = vq_model.encode(images)\n", + " prompt_ids = indices.reshape(1, -1)\n", + " seq_ids.append(prompt_ids)\n", + "\n", + "seq_ids = torch.cat(seq_ids, dim=1)\n", + "\n", + "print(type(seq_ids), seq_ids.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "db51d90c-7655-4598-97ad-3deefd2c696b", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " torch.Size([1, 1792])\n" + ] + } + ], + "source": [ + "# prepare input\n", + "input_img = Image.open(input_img)\n", + "img = encode_transform(input_img)[0:3,:,:].unsqueeze(0).cuda()\n", + "quantized_states, indices = vq_model.encode(img)\n", + "input_ids = indices.reshape(1, -1)\n", + "input_ids = torch.cat([seq_ids, input_ids], dim=1)\n", + "\n", + "print(type(input_ids), input_ids.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "1df0f68b-a9ee-423b-bedc-ef61d789cc2a", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# generate\n", + "with torch.no_grad():\n", + " outputs = model.generate(input_ids=input_ids,\n", + " generation_config=generation_config,\n", + " max_new_tokens=256,\n", + " return_dict_in_generate=True,\n", + " output_scores=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "0fbf738f-7381-4cd8-9214-a4438db038ca", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn8AAAEtCAYAAAB54AaaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOz9bZYcObKuiz1mANw9Ij/Iqu7ee597dY50tZZmoX+ahgag2WkKmoKGIC1J957Tu7uqyMyIcAdgph+GiEwmk2SSxaquD1otFoMR/gGHA47XXzN7Tdzd+Wbf7Jt9s2/2zb7ZN/tmfwrTf3UDvtk3+2bf7Jt9s2/2zb7Zr2ffwN83+2bf7Jt9s2/2zb7Zn8i+gb9v9s2+2Tf7Zt/sm32zP5F9A3/f7Jt9s2/2zb7ZN/tmfyL7Bv6+2Tf7Zt/sm32zb/bN/kT2Dfx9s2/2zb7ZN/tm3+yb/YnsG/j7Zt/sm32zb/bNvtk3+xPZN/D3zb7ZN/tm3+ybfbNv9iey/NIN5f/2/0ANpDsiCRXBATv/LoLjuAN+/u7y8eG7dw767m+Pv36J8rS88znO/459SL9a5J3fHp9Px08+vld9OK4A7k4SSAL7SZhUKEm5WQq7nFgm4d+/X/j364kJePP2nupCzpmpJK5m4WZOWDP++eORv79Zebs5P9413tZKRTlW537duK+NoyXM44RX+4kMvL1fsQqUxG4/8d0uc5OchPN6N/HmeASUeZowSby53zg1xySwvrqzS7Ak599vCvud8re/3PBqX7De4jyqHA4rP/10ZN06+wn+crtQBOYEuyWRVMgpAYKqkFKidXBz3IxtaxiwWwpT0Rgf5rjBaTX+/s873hwqN397RWPix/uNN8fOD8fGfXVah22rvD2caMB//csOVUgKVhuzOv/2emE/F5aS2BWhYOwm5fvrieSONcgCUxGmEvdza862GZIyXYTdLiECp7VzXDcMZe0xkOckzDlRVMA721apm5M0MRclibOflCknene2rbM1YXOnWsyRhnLqxtphbY21Qu2CueB05jmxNfjp7ZF17ZQS48W6cTiduN86ac78t1eZ6wJ1Xem9MU+ZV1cL17uJ/ZyZS7RT3EhJmYuCgrnz5mCsDUgT1Zy1Gc1ioB+PlaTOPsNfXhWud4nk4P1hHrk7KQmguETbD6eKqrKuje5ON8PMcTeaGW/vDpxqI6syT5ndfmEuCVwAGeNHqL2Ts4J0/q//l//DC2b+79fk//6/wL9vkC2evgrgIB7PJX30AIUnD8JPPBXPP7/kAfol0v7y6U1evu/nHOyX2vZL7Rc8x7ML5BfY1yjd8Gt05c+yL7zIl+72UiDyoW0/BHae61d5BiDJow/vfBYwAUvgj/i7miHt4Sfw//P/85NNfjH4o8ZCJhIXYONZdbnmM1r6CvYlt/Q94AcfuXnPoM3xtT0+lsgAt+cDKUYDdDyzcwDF7rTNMFGECe+ZNwcheaVaiUO32KeJcAIwQZKy7GZWcXau1EOjrp2tbhxPR9bayWVhnnfklEhAb4Yfj9AqkyyUU6VV4U4C/Pm2kHImpULtQkdYTVjN49qsg1V6zqhkjl25LVfM0w1Tnmg0ahO6CN1AtFImSAl6d+YiaE6IJEQU0YSqgiiG4N4xj/Fh3ke/JcyUrJmUhd6No68gzjInkgjNoTXj7rhx3JzVhbUZ96cTb+7vQJ1/vIV5Sry6ueb61RWzdFyFU2u4dTKZac60Lry570hvSO9MkrBZ8DmRS6EbrM1pWyOVTCrgonQc0USvHetx1w3HpAcAd8MdzAxFsCaUFPffcMycrTbuT5W1G6ksuGbWatzXzoawVjgcN9baERU0Q5UZk4xppnsjudB6x62TEtzuEnPJIEpzJ5WJOSeWohTJiAtuICSSKt4aYgHYplQGyDuOeyYkVUqKydua4d5xAxUHyWzVsG3De0NQVAKoWVdyLqAJITFPMSZUFTBUld4atVV6W9Fk7HNhmmaWaWK3lJhq5pj7eHl0coGiCbM/QbGhfxfwCgiYxZumPHq49/jpvWfpR4HdkwXF5d19vpZ97vHeadPXbMjv2D61Rr4I7P9KqOxffs9+qQH8QlT3otN/ZK49Pt3TF7qPze+nX8ozB3fi+XH+XRxKAgxuX9LuzwF/IuCKn7k+AT+3VB5BrycXJe98+BAT98F/fNQeH+0BhD48COXRD+/17aMO9TOyPh/kURN8fOHI8JELabCcp1qRokxTwnBOWyWlxGGbaL3hrZJVKCUxZ6VMibIkNAm9Vcoy8WpR5NixpDSMk1VkbQEm3MjqTEVIoxU5C1fLBC4sRVgmYcmJKStzKeyWhZwLLsqxdn68O3J/ONFQ0jSRskLPdDc2N+5a5y9p5oeTc+qV5EZvFayjYqgm5gRTHtexFKas8QKSc4AmF8ygm2OD7UooWmYQo6OxvolgDs2dnhJ52XG82/jpbmW1ylqdw7bxdjB/Wzdq67gq5o3DWpnngmpGkuLeObXKJIYmOFVHBaasiGaWUtCSMWDLimtCUQzoIlTv9NbRVZGs1NpYTxsmkFMKlnGMCxdBJAPCPGeSCorjolhKbKJs1rm3xEGEXgTTTO3CyWBzODajVlhdWd3ozVGEhCCiuCxUadA7agYW9yErJBfECrkkppzIpkxqJHXmkpjnjKrTeyMlmKZCKQnVgoijxfHqmAST3LqzbZXWOkkTfT3SBHordAQ0o6qoQFIll0RJGdWEo5gksAy1ka3TTYJMT4p7Gn86qSRub3bsdvsAzPbkNU0ct/aUjP/j2k7iDV0NRMdz5wnbx/v//OLv/5X29AH97ocX2G+eevp8+y3ep69hv4Vb9dx4+9r9/SHc+LHrf44R/GC7Hjf8MRAZL4mX789/dLgqbRx3g/yyi/4M8JcQBURwd+QxFen6qGHni/vcHvq5Ns4vz5zjaXuEh7djeNhn9KuMdj5eplRlvGAH8NPR71qEMmVUhNYqh7pSDsJSlCwBg7LFfjIYsi7gqpRSmEshT0Ltb2neqa6svbFWxc3JdBbpLJNScsJd2JcrajuylMTt9RU31zv2y8ScJ4Rgc46nSjWj9Y1aT1iaEXdUwm/au3O/NTY3prdHdscjsxhzEuiVrLCbC9dLIU+FMimShA5Ug946snm4+x3cY1yoEiAERVMCYG3GqXbENhzHEMxTgCPZeHM88Y+3J364q/z97Yn7DpYnJE94zghC3wY72JXT6YR3obCh3skYSxJqTmxlZVkKen1N1hRsHsK2OWwNcwumrkdLVJXaANmobQV3UsljXZZoK0J3QVwhxzX13uker0J1c4xObc7JMicRTq1zv3XuTpVTc7buHLZOd6NbMGYAKSkpN1SdjLDWytpWEo2iwpTCnT4VZb9MXC2ZKcf8LsnJSclTQrOiKmSFrIqoYq54h7V1nEw3o26NtTa2rVFbwxxSCjczNLZqlJRJOca0jvnSHcQFNaH3RutbML84KUHSBDiWBStCM2FqSkozV8vEPBdaM4b/4MFr4AZJSUmR0bd/aCtr+NMvz6Cnr6VfsrJ8refq0zb8K1b03yo4/Jrn+i0gpd+D/Zx++tQ8+hWwyGfvw6Pp/OgYFyZf3r+Ec5ydjB0dEOMl9hngj/Hg9gGAHp3/4qaQd54bHyJZf6luF//QkQOkvLvxe3s/sJMOPsDieb8zoej4xX2GOPgENoE2ugDmbOZMUiAFGDV3WmtsayeRgzlKmSlPiE6kSdjPK7VXBEFlprBwX2AuO15/d8P1siMn2JpxOG3844e3TGK83if+9pcbrnc7ssC6bpy2Ci7UrkDHvYHlgK0SLJO5sLWV+8OG/+NHXu2URZyrqXA1JXIOAJCniVQiNq1142SGOPRaA+ylse1ghSQJItFHqWR632i1Yq1jFkFkohlX6Dgm0Hrnp7c/8N//sfJmE3qZmaZMKhPdwbph7qzrxpYrNRFMqHRUOipOswAc3ROGUl242xqYBVPrnW4dq41eG1YNESjzxDwbmuLupiS4G9ahu5IEmjhJEioJR2g13MzhA3Zcwm3sJDZX7qpxWDs/njbeHFdO1Vlb57A1WjccIUmijL7T3shiTOK0umHbEWVD54k079jtZq6vFvb7mSkrRZzkTs4+sGin9Y5qQnMJIOaC+xh71aiWqM25P66cTivbVjGHXDKlTEw5kzTRXNi6MynIYHjdOq011lNDXBFRVJWSBRRyVnR8p4MVN+9sraII85QpSZhzxi3mVrh4R/ywxlzSPwH2iwfzozd4f0J5ij+zbnwpMDzbh/xMH9r3Kz6hP8nA/BbA3tcGz7+APT70e4upvP/9L3n+5+wXO/cXHviDWODRYR/chY8+fwInvJh5/8hxPsUEnrd9DOjOxNZTUu3SZo2/TcE6L7GXg78HJ+8Tcu2M/F5yk74SHHzuxdQf//Dhd9d3++5dgDdQ30N/+gB7g+2Lv88drQPUCat3qjaaVXJKTNXZTwlXp2O0bhGD1SJmr5syLzlYVHOsdRxnLrFI7jJcZed0mihp5vX3N7y6vgFxTmvjHz/8xOlHY05wneG2JG6WgmBoh6yZMk2kyfjPt2+D4euCiiPiAQrEaQhba/j9PUlmbICqJSc0FXKZSakAHetGc8MlAE+PzA7ocX2lOKU4SIIUsWGiCet1uIUFR0dikNCasbVKt0hOEA+ANZcCy440z5ASvbUAjeaYN9waU96xnzNTNpROUaMkYUqJqSTKNNE1UWultRaxktbpvdK3Sl8btnVEnHncj2lOpASoo90HQDFa0JrRt6qYEYyZRZycm0X8qyZcYXPjzdY4bZWf1pW7dSR4tMbaOq1HAIHnYILFFBMBjCQxFlNKJCKJZDcX9ruZ3TwxZSWdx+iIF3OE3iPOMieJOMKUom0dukEzaD0A8lqN+/sTdd3QlNCUgYSKXhK0ajXwFmNTAO/0ttFbxTuUMjEvEUIgEnGySRMpZZLqYM6dqW24W8QM4kwljftPgL/x1hr/7sFK/9HNO+/E5J0fNvKpZ+PPARbP7fu5x/vC838xifhbAIVf+xw/Y937V7v7f21H3lc5iT/78f3v5OX9+9zLzEvfvx5v+DEweMEyT2i+x8DkPWAjjzyYL+uzl2f7vtOU8+PdL79xAUjvXME7+/OB3z49gj/WUy+xhyfQGTw//vthmyfHftKHPt6yDEUJ4LZaw7qDVryvZE3giaITNkdcWZJOAjJOTo25OvsKZTJEK7017o9HkkBJQp4LRXdsJWEGu+Ts50SeJnalczoc+cvNnpKURRPJjEwwJ3NWppK5KXvyofP93T0N4c2x0hzMjeoecXdEzGhkfoKp0LqxtQB03UZM+sXdfwa/jkuwS9aN1ipb7ZScKVNhnudgcQS2GsdzA0RRj+zTtVVO20brxjQXrvbXvO7GQqHniZYSW7eRvNBJEu7MkpXdPHG9X5gnyGIkseHuFHIK13p1OGyd0+kU4M873jte4w89XJBeO5IqBiyLDsZ8ZCW70M1pzehWY3y4YnSaRf9473SL/jBprAbH1llb481aWVun94iBEzknWkiARRJmMbOSgCch50LOQtbCbp5YloV5KpQUiTGYDSDbyElJnuIxcY64kEgkEoHuTmtO7U7rcY/ryEiuW6cUxU1wU3oHtXi9aWJ4M7o2BAM33BoQiS7h3h+uWjHUhTSysJNGxrwgKBkwsIidTAgpyQUARvZ3p9YBZv8MylP26KH+1IXz3LPN3/vwzD4fefn+5OPypYvreYC9cPPPbsdjeymZ8LnbfsYhf9FzfOKe/V7sd9P8r9jQj4JIPjI/ntzvD4HKpwd+DPgu235gZz/HEH918CfxgjpQk49zOA8Ps8cM9NOLe9G1vrwxH/7u8TPqGeb1IbnOH/pIuLh5x2W+t6OMg0eWtWEo7k51oxOLo3inNfB2Qt1p+0LG41xuJPFI4Dit7E6V3W6ilIS3zv3hHhVnN0+kFPF+uNN753S65+3dxNW1YLWTNfPdd9/jZqgWenesGzkpSykwFcoys7aN17fXmEC6O3I4GYctgElrRg00iI0gfU2RDHE4rQFxxcmyJyUNSZdzwo/7BQiaC6013BtmEc+HJowNHXIxvY0MaQlJGPNO7RWzTkpCWSZuX7+i68reMyeHYzd6bWjvJIRSCjuVuL5xm1USpWSyGokAHeYhMbO2yg8/veW0HikiSIpEnexnkKORqZsSzRxpnUVCvsa80syoHbbmHNbK8VSBylR2OJ1tSJu02thqpxo00ciSdqW2jWPtdIuEiZQiKUckBQvXDWud7uGuzUUhKyllck5MeUeZMrlk0jn5QmPed2t4rzgFkYxKIudgDCN5KeIzvTe2XgP4NdhqZCM35xLH2DqXJJ3WDesNErg4zTtYxy2AcsrKXKIt6dweInYyZFtsAL+wnAQdbt3z66L4CAsg5pwTkO+cNPSHt8fSDOeH+/nB9HkHeh4YfvaD9WPu3w9s/th+FfbnV7anbKWPPvrZ1+rPfP4jduCf2D44nT4xcfzJeHgc/vE0LOTp4S7nPH/5tWP+ntgZiz7E2fkTtvFdNCYfHeSfMxHeZQEfYOeH3Cbv7vPcJziD2vdbdLkXZ3DtABbxfQNEGg33jpqj4+HeWmPdjNWDLUkig71ymjmtH+lu7PcztM7pNALBzSlTQRRMMqJKq/DDj2/5+z9/4nB3ZCoz8xyyHoiCK+KJKU9ocTrBpEzJud0vIEIuE2/vKvL2xFprMDkuwUQ4KAkhMl4Pxzt++qFy/9NM/f6W/v0e3y/MqRAuUAJgAE64PVWVaZ7IudDN2A5HrDvH4xFwVKN9OSckaSQq5IU0F9KccJnIeWWtxrF37taO1U5TyDqxn2aucmZXhHo6caDS10LfZfZziixmd2rduLs/8MPbA2/u7zCc62mmMEVXEe7WospSEssyU0ac4rQL7b7TqdGas3bnWDt3p5W390e6bSwzeHKq+cgOrpy2ztoM04RME6oZaw3vhmihaGYqhVwKhnDahgu6d3rviI4Y0TkhKgwaEykKGqzcujUQR63ReyVJR1MmZ8gpR9wcivWQZEmaaD3cz7VHlvHhVLk7nDieVnpt5OZoWdjtOgyWrtYG5ixTvAyEJ1ZJouQk5MEsl5JQMdwsGNccrJ6OeWIOaYwRGYHK7pEoI6aR6OGGewBhlcjG/sPbxaXDU9fDx3Z4Yh/Z54tIjpfs9GuClM8BpL8WiPpMkPzNfkP2JXPsZ9gHh8rH2vGMm/E5KHNOEruAw0c+2fOz5YUvkp+V8HH+8Bjk+aP/n9+Q3p2OnwPoXj6R3z/2Y1fKS4Dmy5r0oSMZNjIqwU0QSXQVMgkkYw5rk4iFQkZg+xDdVagOx7UzzU5Baa703imu7KaFZZ4RglU7HO843h34+z/+yd//t7/z/fd/5fpmx9X+il3vJAn9vXkSlqlQinJ3OMDpyI1CnmDRieJCPa0c6KCwAussqHfcKr0Kvp04Ht7QtiO2FiapJF3x9Yrr/Y6bZWYWpaXI8O5Wg70rhVIKbk6rla0Jx2Pl/s09qDHPBXYRj5YI2ZA8ZdIU177fJW52e+63yv3WuDp1ppRZ9IAjXO9mrqaJRKduB/rxjuO9sb4R7suIjVsmJCl9q7TjkVmEq+sr9tMcfd97uMg1MU+ZZSks80RRaGZc7ZZgabWDCi7BzNXW2dZOs8zmIaVSTehb47QZ26mybg0TpeycMju+VZIkbvbX3N7sWeYZU2HthtgR2xrikaBjUmg5Y5JpCrhjzem9sqlRMGgb7iem5FzvMjfXM54ndFri2iySKFoD9w50jqcaLmhz3q6Vu7Vy2DYO60avjZKcNB24ahkXR1LHt0rroKlwvd8zFeX8QAm9Pw8WMylOZP1OZcdUYnyLBCMu7pgNNrg/YtbHFIwkHsOIRCARBcnA9cvm5+/W3nmL5ON+pM9YLD54rI/t9xUWvc/Epr8r+znRSd/sz20vibv7OZj0HeD3yIX5GfZ5On+PWvSUGYMngPPZrLVndvqkE/39fb50jgmP7sknHloPWPfxl2em8Sx1E2BXtIQLlE7VTBrbdmAuM3kqw1UWkjEyVP1N4d6EJSV02eNtYyVzMiWTmLKQM1yla8pyxe333/Mf//FfqK2SJeLDrBtvfnrDdnrL6XjD97e3LLuZXlfq25+oa0XISBfs/oQf37CrGzudaVPmfmu8Pb1hPTT+9re/knd7Xt8uJHd2JXO7n7hZEvtJmLOR3FhIrJKo3kgjw7fkTMkJs8a2hoYdCSobh+NGOQnHQ2W3zJQ5DdmbiInMZeL69pr9zYLkhV1u3E7Gd1PicJU5bRVNhZJLZBWbcjw67XTHejjQTsZ6zKT9nt3tLddXE2Z7cofr/cLV1TXNHasryVrEVZaz4PTGT4dKmRa2mujeWC3R3Knd2JpTXeg5Yz2NUAdhc8G0IHNhSYll6iBCmQvTXGAJ0eZ5KUxLChczod1YkpMl4t1yd+q60XOmXhmuIW7treFthbaCb5hv9HrHDuP1beY/2n9hmoApM2smSyfRMAvGsZtyf1i5O1Yqzj3OirOZUT2i91QiK9et4mQ0J65f37Lbzcwls5TMrIA1uvWQpZEYxxAAbsqFnNKjuT9miFm8oIowVGDCzesh8BzPB79sB+P7P7o9fXj5+TnCI8bg6dv1I1fQw5cfOcEnV40vsC/Z7+egoc9l9H6N9n3oHI+Yly9yu7/UvqHLr2e/5LPmhffpcTzfs6TVcx5MfxdYXty9cnmmvtR+ttv3U9993H4DD/sXzye/4NmH6xxxgxIZjoKAKg0HGaxgAy0JE6G6MRlMWkAj47O6YdXpnqi1Y8eNuq0cD4WlKMtcuFoWrm52oMqrqyvMGgDWO4c3b9gOFW2Cbyt3dwfuD0fSNJghGfInW2feF/5t+gvf/y3TW2LD+Mebf/DT/+c/SfOe2+tXXO0TRSGLsJTEvghqjdpP9LVRRTi6sW4bQg0R6GUhp4IQEibTNAL9aw1A0u44VqFJo7fG3hdSLnSLKheNjbsffuD4VtnvrlAX+tZJ5ny3FPKrK3JZKLKn64qworantRvW02nIhxCZpSmxu5q52hWyJ/a7a3bznkpF+onkFRGnu1O3jm+NkhRrjf/xnz8xX03B3IqQsrK/KpTlmqtmWBc6kTBxm0LeJqWE0sneKSkzjQQNFePuWGlro7Yj/RTp98lhJ44scHTFutK2RqtH1lOittgXHqo/uIN1Ics1MiXKfIXKzN3bIw3najezK4mkNmRqjHUz7k/bJbGnGqzmbAhNMiTHS8KnCXZ7lutrrqfCnDNLnimlU2gUayBCE/DeyWW4dt2YSiYlDSav+UgEsYfnkctlnoyk6QEAY+KJnJlCQD4eGPKHsstb6DPxOx+038Cz8l9ivyN36++kmd/sl7YvCVv4nG2fC/jzd58pL7DPAn+fckL40wf413jxe6nJM/98RD6+yxv6y9r2jOtX3kHoT5zPorhAD+chvRtbb2x2j4jRe8ObMWtiP2eWRcnJqevK4f4ee/sTkwivXt8ir64oaUYk49LZ6krKGe8bWAvA6Y2pKGU3MamwaELN6JJHAsdETnMUo5PKbhEsJXpXtpNxdzwhLNh/fMdfrv6dv92+QtOGtRWV0G+blgnrSq3OunXuW8XbkfV4TxbneroilTkyXnuIPE85fNvdnTRn0qYkV6ac2V0t3Ly6ZdktEaO3bdRWMa/YtnLYGmbG2/sKknn9+pplSaR+ROWAWKPkcD96Vq6u9qiEgPG6nTDr3GiUgcta2F/NlF3BRJDWseZRvqwbuTieJo6+cf/DWxDB9JaUHNEQYJaUyCOLuluidYNszPNEUcWs0cxRd+Y5s7vZsVtm8I2snTe2od7oEjVvC8oiypUXtqvM4XbPm5tKl0ZZMtWcu9OR9bTCSBKacuJ6ueb1fuZmP3O7y+RJ6Gnj1DqsG9VG3CNOb8apNqoHy3fqnWN1Dt04IayiaIY8zbQy09NC9ZnmQnan20qqG04DDRHnPOmjmt7B2J113VVDusW7XYBcgDsuoSBIZAj7yAzuZ7kgIhlHVaKG4J/CBMT+eHjujw5+/ujX981+f/ZY+PlSKehrx/yd7VFihJzLuj17rpfE8X1kNj3Nankc4PiJSfgoPPHJpv5OlvRzrusPtu5JYHaAy0d0K4xoR43ut87WB2tSBRELWZBqnPrGtgo3LbHMynY8ctg6STNzLkzzwu3tDX/97pbr/R43Yz1Eksbp9Ja6nkJEWYySM9P1MqoyLKCZdauwVWqvIR8iQu2NMs0spWAJFofchDklrtMVN2Xm+6nSfKWmhubMbpeYloz1uKa+nSK2rRtNUujTlhIxjFbJREUKVSUj7JNwe3VNmQpqAcbKPLPsJvb7mZSU4z2kTYGM9YxqjpJiudLNydLopzfU1kjilCkhWdEUtYndhdpX1nWl1WNkVO9n5jkjolylhvc7ulfcKuLGpCFK3JJSN8OTIruJrBJl77YtxKXPzJXbOA9gzi4nbN3YrA8AY2hRJIP2E7MkHKckeH2zJ6cUFaFHmbTQ2xO2Krw5wXK9clgP0Z+nld4q27ribizTxDQt7K937JaF3a6QltBb3IagZ/dw52pNMS6ts9XOaVu5uzuxdqFrVNho44UEiwz1XjcmhYlGS42rWfCikCFNgk8pQJnKCFmQ0KwcJTqiri8jlGHk/pqFu1cDLJo7KgmzSBDp3UI+pze6G+5KLiGf9Ie3Dz0rn6vfOX76zYHE3+1t+gpxWL+q/aYa83Xs6Xj+3HD/rzYXPnSizzjBR1MLLiDpeRfp01O+RxZ+LC7tKfP33sk/0rAH+wzw9+jEF7/OB1zVnzj/Zw1p+cDnD34Vo+u5qEJB3on5+1mxg3ABhI9viQxAaKI0gah7rGQyOYcrt+DMGa6XzJw9siJLR/rC9Txz9fqK/fUt19evuN4t1NMJp7NuG+vhnvu7O1IqXF/vmeaZeQpXZdeZ7sr9eodvBw7biiOkHFpwywI7S5ScKNLZlU5h4tXyml3eMyXn1EO2Q3IOQJkySZyuRk6V3TSTdaH7hrqxTAmkYb2NOvWxz5wje1hDmYVEGiXVnJSUoiH6q/sdU9pwDLMJEYlM3Fw5nU60UXJtGnImy7xEVQzCZXg4HTkeDqynDTEjLxNCYp6mcDOKYa1hVkflkSg7l0ngkHpjKhOyi/nazNl6pzUfOocObphFnWFByPuJFsWMSSKUlCkphdRMa1hrTCXxej/jBBu3mTBNocO0tYcptGSn5o40G2Oq0SaYpCCi7KaJ3ZzJ3jkd36I9ZHl6EqxWBBsl1TI555BXIa65rZXteOJUHfIORt82UbbeOKwnjr2R7chNccpOKJqpKNNgj1EdYtAPJdkkCSEHPuaUR9vdA9xFJZdw6yZVHAnQ14LV7d0iEcRsgD9Hh/j4n8Oec9uc7QV9cMn4+7qt+qT90W7PYyDyR7u235q9FGs9nQ6/tRefx/ZB7+4L39i++No+dPzPey58ZrbvU8h+Vvl7bhZ9eFZ9DNO+/80HHbd8eJTIOx8fN/2BFfQP7fHOEd+pqHLOZn7qDn787/NnGbVgIUCQDLdnzuyyspuU1/sJr3dIy0xaEO/sSsiQKBHT12oL8JMzb+42Docjx+OBeb4ipJ0TtROabNpAI/j+eDiw1kr3cEU7cH/Y2B2OXO1m5qIRw5WiMsOUCq2t8ccc2lige6WkBNbYLwWfElOZcGsojayO9xXxFsznqCKSklM0UdKOnBSVqNZR6xYxi+2IeadIYppS6ANaCjbJIWdBslMdSk5cXV1RJuVqd42mwrpWmnU4hhZdVmGaFq72O5ZpIqmyHqMqhZmFGzen6B9JgOJmuAmYRIhiqyGAbI6dBa7NMevxW6tMecIssVsSZUlkkdDhE0jiQ8sRJgUdWoHdGnRh68EqnjZjbXDanOPa2U5HVJ2SCs039tpZSlz3PCVKBqyyHe85rB2ryn7ekTTiMxEQDXdqGfp7poJlZS2RRd6toihJM2kq3NfK3bbR1yMHray318juChGn9Y4N4JdSIuWEppjnquB9DHQf7l7zh2QO69gQ7HMxeg9R6N7aqD9+jv8LNzhYtP2FwqS/e3uv7uZL3pyFTz7wv5Z91dvwqSfsr20f6qffQtues5etlN/sa9rn9O9HxvdnPc8+wyv6LKwaIOUCQH4xnT9/5tMzv1+eST93AH+UWw2Oz98FXs815+FwD+7kp6I0T08r7/32yBUt8lD/d2zj58NfsoATiiAOKkIWKAKzOjeTcpM9XJvLBKpkjOSdiQ7biXoQjr1TVDBvvL2/5+3hjlY3pvk1QqF3oY5qGeaQ0ozgmApIiEXXGuXjWjtxvLvntJvZLTPzPHNzsyAaGo1ra1HdwwzHWWtjXUPXDRdu9jekoVcnnkl0ijjWFOsbiqBGoAOPbOYEkaErgiQQE5pFVZC2VVRD1DinqBgB4R5HGvuieJmZpsLtq1u0CPvpCpEC/Q39FOzjlBUtE8u0sNstlJwH01QDyMI4drBWogIiWHesBiCra2XbKq2N+sMi6Lm022CzJoGrObNfCjdXC3NJ6OD2IJJ/5iJMaijQthNbDbdnNeG0bZxOp5COMdi2xmmt9NaZ9wtX80KRiSUTtXtLoZSJPES/T1LpbWXWcKnnFNp7ZYgvz1OmlABs8yTsp5DLOW6N2iTurQtNDe0F2xLVFKFD71FrOClK1MI+1+tNo89EwgMc3SL0Ud/YRqk5GwAQuAA8GwkoZ0B4Dt2QUWNaIEBmHpqVfxr7IG3w/mZPP5+frS88xGc363yOf7V97Hn+pezQ02P+Fq7zj2yfgWtevP/Z/pWs4AVwfKARH5tHH/JwfGifD17n+aXwy+zF4O85dvZr2qfc4i/f81EPnu/N43v0XsPlciPlXNf3PRR/3m4sTk/UuP0M/87/c+NcC1g0gVkcu3cMwyU6PvXKbkn85fYKkpPNoJ5CAoRG6huph4jz4fSW+9MbtnYi5Yn97oplXiJ2rVW29cS6buCFq9s9024GLch2ipZKRCNuW+P+/khtxiudSGVGxCBBH8LSnT5issIdG4BAES8su4nmNsp1WbgDUc6l3mywwaZC80pScJ8oJUft1tiU7oaYIW4gwT4WCU0Qk07KoNNMzqEfeL2/wlMnyXDXYkhrzFmY8kLSQskTOZ916RjnVqLCCgEqHSR7SM14w/tG345Y9SjBVzKi51i/wWhZp/eooPHq9oqrqz3LbmZKOcqbMRgs4nqkr9SutNZxV8yilvHp1FnXztYrW2us68a2buSycLPf8W/fv8J747RtJG8DGBemnBERtm1HXQ+ULAE4PVizeZpY5pkypQDSedTrNeHV9TVb65jB/Wnj/rhRa2M3KftJ6G0HvjLPOeRd5sJEJ+VzJQ8ZJcPHeH54K4r+sTPgO1fMGeXbnBHjFyAwwJ6g52QQRuIIBnKOJ/wTgD//wD8GKP7sY3yUXPuYX/MTT/AXv7P/EuhJPtyGs714AXrSvpfu98nL+qVQ4x8MjT7b37/mNf5CCPFy2E9cy5ec/kVZu4/Oe3FRvrBNwz6D+XuX0XvX1Rs/CO+6Ut+j477CfRB593gyaD9/dAIZaYjnGL8LJhug4H2gefHVPjT8iWs3wMAAe483k/OC+HAMPy+XIpFlSsRFtdY5dZhUad3ROXNzs+ffXr+C1PB1hRWs1lExo5CyRgWJ01vUV/bLzM3+e/72l9cs+x1mFu7TPNG3EPZtW+X6Zs8yJbZN8A4pJQw4njZOpxXNiZQF6zAvhXOySkfYetTk7bXjbuQUrM26/pObqz1lSqh0CsaaFcWQXkk4c8kkDcHfJMKcU8TGZZhKwpOynX2qI94xqaJZSOIIkdCB6APzlTO7qdBxrJ2CLU2deVbm3RWqCXEdCRpnt7OQdzORj9EjycA7bh0sMlezdOa0MelG0sSy7JjmeWgQwjnWz3rH3Jinidurhev9QpoyJZWoyTuEimtdqeuRtq2cupLLnilNHNaO1xO9O+vaOG0ra9tY10qvlXnZ8ddXt/zPf3lN61tU87Bo0zxNLFNBJbFuK73uceus65G2ntCs7PczyzJiIVOA1JwysxQ6M7UHQD+sK3eHE6etstVK67eIOof7N0x5Yr9MXO8WEp3McMVeNFrG2O7hrj27gK1bsLrvuDnCtWsqdOziMhaVYIclDebcqL1hw/2b0x9s4XvOXqTD9QHA8kFX0lOwJO//9tGufaHr+GsI1/7q9oWu528e1z+I/RL08Ze24Wuc42PHesx2fdq+WOfvgSP7ZWeEfPCBxyNUF8zDeVE6f2/D9/1Oacb3MetD1vLlIervHP9Sv/hRv18O8ZhZFIkH5JB8ceGiY9Zcol4vneaZ5ol5d8O03zNNmX0RjA1RJe9mRDNoovcUZeCK8LfvvsMtcXV1zfX1Aip0c5Ylk/SG3bwwz0fmAnN2dvsd3WfElZIzqhNrN07rkd4qp3XleHxLSnvEKnU9sG0brXasxxXmnEgp0drGm7dvERGu2GH9RLLOgY5bw62RFeZp4np/xTIVbueZecrs5sx+npnKhI1sW5szW1txc+ZSopQYAsT5UAk2MBemnJgSdIFqKwJMV8putwuxYk1AsGttJE6ICmpKb1HXttWODOZVJCHqqDqaFVKiHhPLbub6+lWIdrud87ajAoV3lpKYk7LMiVQyU4k2mztbM44muIZ0ScQVjjJnKWPd2dbK8Vip3Ull4dWrG1JqLLvEMgtbPUFzboqwTDuWKcr45Rxj5/6U6XXCzagL0COWtJSoBx0i4o5qJ2dhyooZTEP8M0/KTibqkiK5hngBOK4LIs5+mSgpXL2zCllsvLQYqoLqqG3djZx1xFHGi46MMAj3wRAOhs9LuIPFH7DLA/jraFd6b6h08p+B+YN/rbvqm325/ZZc4t/sZ9hv/QZ+Fcr7k/ZZ4O+Mcy5d9x5wFuRJ495nAp/74ePnfP8QgjwGdli4j87beyz+jLijdw50ZinkEQqURzymP8QRno8cuC+YscsPAwAGsxiMhpyBqDtaBgBUhTQ0vRzcjG098tOh8881sd8yx7uNZRFSXVGrkT2aFS0FdOLmemH/Cu5PnXYSeoc5KZoVo7Cfp2ibwZu3B97++N+5//Etf3l1y/X1LXWLpAdE2MmM7Qq1bvznPyp3hwNHjF5X1tOBbh0VZdHI2s2lkNLEm4Oz1jsO90f280RycOuYd3qrdKtYSmjqWG9AYjcnrpfENOfQ5iuG0knZEU/s2ONC1D7WM4MagEbGvUg0ijfEQHyjaLB64WYNh2uACo8kEc8E8BfEOr0Zx+MKzUFBSkamAjkkTK5S5tXtX7HVES0su5kkid5H2TF3kmSyGBmhyACNWlFvg203sBbAskzIbs/90fjx7crdesepGvf3J376xx2n2liuJr67veZ2vwc2SqpM/cDxp3te73Z8Vxb2BTRtZInKKaiF2DYNVcc83PDaI9FEpKNqaBpCy6JkKQGY3eg+QgkSTACqF0/BssRYnVK8nFgf++eYz9aN3kOuJaU0hJ1jzuQSWdzyaG7pyKg2iyeGyHAdn5lygT6qkKQcupMqnZTLyx4KfxQbzwTgZeTAcw/Tl65jz4W7vLP/V1oQP8Nb9bu1z11/P+ua/wgI82PhBr9V+1g7fw7geoaF/9n2dQDgi8GfvPfpKZ3+hbElH7CPHU3OLBtn1m4APWxUDYiFSQczFjs+tEMk4ovOTIS7X1zCgd/OVQj8wv6dj3mGgeY9HMsOIuO8xOIZjYl6vg6QBfeo8rFunXU9cNfeYKkzZ8Gyk17NvJojk7bkxDJlpikhRUkIpV7zl1uhrsr92w0wJDuaMykXkmZaN9Qrs0zUrZFOB1KBJcHmja0eI3bNBW+dXFe8npAizNNgh5qQUiHnHTkXJCWaKUsupA6+NSZJXO0yU96zWyKzNlhXR2hkKrMIi1SusjJnp+RO0g5ioEZSJecJhJAGiVRQwAfDpAMsdJRgiXprCFA0kgVatzEYesQ0Sjj7nRFzJi2yWxF6LnSfISc8xb1QjQzsRYQ0A6I0TvHCoOchI6OmrZBGgof4uMdW6WaoOEsRUlImV7TMuBiHU8PXxt39Pa02rm4Lr/OOm5sd37/a891uz9WUmYvQW0XVuZoTc4F0AXFb6OpZ53oCphFTaSmYOxSxcyxG6P8FgA63bFJhygmToaVoxKuKJ6wLtXfchNbbJTHDvdFFsJSHrMujuXd+MRrzIYkiapfs3ZhLwXarKpoUvYRHxFHcQgzcAHowg0nSu/V///D2ZFH4M136n81e7on7A9kf6YK/xD38Jdf/kvN87C3x8wD3F2b7nvm/f8ET6z0qMBgjFWBot4koUwkWIXkfLI6PwvMPmYYOAVrOmbvDVYUGGDy7bhFCzuOMCKzResOtk1Qpw2EZC2AnwEiwcd0S3oXmsb+q0kmcVmG5O/K//uMNbUfIg+SF/W5hP0/MWUPzxCpeG3scmSZ8husMjlJ7o1sLV5sHSPpuX/hufs0PP52YZGNmQly5zhktmdac+9OJu+3AHof9wvXNNde3u3CfumKiCBlB6d65P63MWeBwz04yO6AA37+65fY6Ek+KRsWRrd1zuvuJ7MZ+UcqslCkYs6xO0ujLPAq+do9qGmaG6oNoMFgICasMUOeggg22RICUBO8dt+GWTIqL0o0Lq2UIMi1IKRhRcq/j9PF/NUPVyZJCs88r3RtCCEmHL1WQIiGV505fO94b3vrQp4OUMyIZdcVV2O0Lr/2GY4e39xuOsJsKV8vE7dXMX64W/rKfuZpDJqY1UO1MxRGpODbq74Z+oNCHeDKoFpIOdyt+KbfmA5Resv4fzdEsGmx4AiWhFDDYmrPWEKvuvUWtXQxMsD5kbIYrOESbz2AOQGPeuD28LBEZvuJ9yN+Eizc486gC0mrUCg5s7ShDGPrP4vZ9x/5Ii+Q3+2bf7F83pz8Pk32Gzt+7zNfDyd7/+I69EyT8qGFfih0v+40Pg7CLxUkuAX6OBROTZrpZxNuJ0Hunti1ci5IuDZexsIEj7ghGtxauPxQ0gXUmdbQYRRw3KHRUgtEq8zycv0pPnUPrJDrVhG4VdEKmTM57/JRwMqfjiR+acqVHXl3NvNaCpUwTEGtI36AdSAK5OynPMCeaERpybfzpjbZt+FoR6by+/Z6l3LBfFnoPceKIP4NpKlztdtzdn1gbLPuZq/1VJAxowlVwF6w5tTautHOdCldyTbvfcO7YaWZmZpKJXZlC1NmERTI3r1+N+D9hLoWiEkkEYgNLBTOFWQAFjUSRiMVLFwDY7SwADLjyEMX5aDBoRiSYurNvMYuEpp3A1sFNBiwZ8WdWcVupbFgvUbNXM3m0K2nEw4lqZDMn0NRRGuIC1Ag0UFATfNSoFQdvnVOtnLrSe8JOjXZYsbXTp0bH8angXegINur4OhWGLApSAwgTSTBRUo3Li0t6RBTlKfqnVb/o752ZbHGj1YqK4qoDXAkuDR+gtfeKWUPUWaZMdxvj5TzHHku+pOHCDWbWBujzc3ba47nuERvYpSMiAfJFoNs48jkr2y/MoMqfoLzb+dl1dvn+3rxjP9d+1ev9JU7ySzb8tzgIPteT91u8hq9pz13fv2ASP3W8nr8U56XA6uXg75JJ+YDaXnLJ74gkP93ynX++H5v37hfnf+rD14MREpHIGFUZDFJimWdyCQmR1hqtNlpruEcGq6EjoB+iTm64HN07KkbCSERliKQZqU7tnYaxWzLLPCMGfdtwF26ub/kv//Ff6f2O7XTP0VZWh8PaeGMnTgO1JCnkuSBph4mHwO62wn2nHBrLqpATsxjZnR2w5EzRTi6KXWrECnhIyKiAirHahm13GJ2r62uu9xPznNnWNsCsoxaZv6+uF75/tWOrnUlDyNe8072H/5TIVvbkbFmjDFy+Jn0PJRPZviUxT5WZA8VLALoJNBWShLuxpMKsABvileAsLcaTGCURyRkSLkpNAaRwyBIgHne8K0okv7z73uD4IwmeiP1M5BQgCe+DRQuI5WYk6WSpFD/RvKM2oRYZxzkPWZvh7j3r/ElvaI1sV+2gkrEkmBLM4Khb6wLNhNSM5BvYkeQrjpM9gPAyFZaRye3mtB5l5/QSUqpD8DqHgMwol2YmD1J4bqgq1gUXp/cB/hxaN8wbQkccqrULcxeZzBKncMOsMS4BBKY0BRj0SPaZSiblNCqjjKQcDze7d7tMYn/yzAmmPdreR1k3vWS++xn3YeMP8B60/2PaV3bxvucJ+Rn7/5rr9gf74JlGfHY41gd2+FDE0ovsZ3bOR+/5MyTKc4vrF4+bF7b9Y9Fbn3usP5U90ydfgge/Bob8jDHy+VIvlzg4f/bXj3mkP27y7gHeAZgPlURk/H35bsRuiaRggMabdcqJXAqaMu5OrcHWMOLB3A3NaWSKRjaoake8omIUJRiWJBRNmCfuN2PKE6+uXnN7c424UdcjAnz36jv+y9/+gtnC8X7i1CtlmjkcTvyv//mf/HRsrDXi29QK5EJtG/eW6Vuj2QZ/vwcpHF7teb1LXOeQPckpk9RwzREo32uA8AH8cla6wtwc30Wc4e3VMgSAo9KGwIWpSZooWUmiOIkikVm8tRA57maxIJ8x1ay82u8wZhKR9enWEJELQCg6ypt5SK0EeDJK6qQR5H8GCHK5g3KJ7dMzgB//Dmw7Yi/F8TSYoT6SEB4NQ5fBdAGBjoScEkInqWAEK6eAayQ35DSTPQegMg1XatYRC/rQQgaDRo+xo664JFzyZfyhfokRlOSkrAH+xFgW4fZ2obuSU2F/dcV+v6NM87heGzF6BSTEvAMYx6vJ4yzZ84SwwaK5QW82MnA73kes46idC07WhGgJYGyjv1KwodHJcb0hgh2hETlNETMo8aKgo0/PL1yh6dyHft9w5454P8Z9OMdLmj8SgLahiPnOy2C8ZOAWMjx/dvvQg/SlC8I5KufnnPux/RbW+s9eTD7Qac8d5zfBvD5qxNM2fpUXhRe+Ifgzn38L9/9r2x/l2p5t/8sv6jPLu32ZncnCjw3BZ6HkRYLlkbDMhdn0YMF0fB51RbvZCEAPfbYzU5lzuBN771G+KiLfL/FIcxKQWKiShhZfxrEGRZxUgjUpeeb66obr3RXqDaZMScLNbuKKjWlyqhaa7JjyxL0KfliY7cC9d071RO2d3hLWKrWF3tmPd0e8r3ivvHmz8P2S+Mu+8NfbhXY9czNPaO14r7S6BaBTCeCV82B7CvNccEI+pbdwU045smdlZGgG4HKm4qgGS4YKU0u05gMAjtixIcDrCN2FSHmIfoqM0Ie6to9LfKk4SZ0kjMQND7fheMAFWItz6ACHkYUdTns/jxs5uwgBdXSUmAi2yC/HeofuHtqBcd+Vbo49dkmK42SSRxk3MQfSJRMV8UdDcIxAUcwDBJkrzSKZKM5tFxa6dag9Kre4wO5qgSldtPaWeUeeCgi0ISdzEU/2KDknA1RFfJ9f4hmjZKBEzowKfbBnjgXrZwHqrCvWosyfFmXK00Bj5yobwVD6CGh9eLEaiVLj23ALx8vA2aUbLvmIo/WRQX7ptyez2i4I/dEL28gARs73N/rF/lTJHs/Ypy7/xaBuvGX97ha2312Dv9nvyt71F/0m7ZNT4OvOkS/W+XvamU+b9R6we2fzM6T7GMU/AN8AgO8otlxIyAAB7jbW3ygOb+6YOa0b3bfhChbKiFlK6eLwjQXIjIxFvVqF3TJzvUwkN+oKyYV5LmQTumd6qxzu7ykYS3YmFWS9Y7MD85Vzu5/IywyW0KPwl2VGa2OxlQOdQ19prqx9QzWjJXGsxvFU+eePnXo48EaNH4rw5rtr1n//K+27G6YS4KubUl3Jmih5hpJRNRZRhMiCtdZotdI3p8+FqGYb5dbwqMyhQlTvKAEMkgeLlJLQLZglGUxfN0daVLGAiHHUBCU5KhFTeUkYEHAZSRij1q+oXFikyBEwzjUelIeQAsGHwHksYEP05cLGiZ+H0mAFz5nalz9nVBGJETkrtQfrdWYIwS+M1CVe0Ad4vYyyACU+koMQx0YWa+ud2jr9DEJlMHEitA5rM9bWcVXK1R66o9VDEiVlDGdrK9adJn5JGglaraMjczq09aK8WnIBDab6Evd3ybB9uPwou5ZCRgXAEyKFQIb9wspyBo7nzJjznLOo0nIG4mcA5+cC0USfmUXt57i3aYRd6Ljnep6e4OdMeHl4Dox75OfRZH0k8/wZAMALfSMfYql+7S76UDNf5CJ8qX3ti/qC4z17nfLsx69vv/RN/RnH/xRe+lXG44fmzBcOwg9qB/8C4PDxIT/VxOcu5UOM2Vfo988Af49KmJ1ZEXnUkvf67TM78jmGXny4aoEzRzLYiscB42ZgMmLfNDJ+WzPExiL/aCE1i8XK2qjDkRTrjdYqbo15mbjZ3/DX19dMqdNOE9I6JRf8UHl7f+Jw1zkhLCowK2WGnIzjqTFJZr+85qoEENgtmb+9umaXEnflxP26cewdRLk7OalMUBLrTqhbJYvRrfN2PXF8u3I6HUELXZXvbmaWkqNqiCiFQmKmk1CFMs/0WrHe2HrDEGpzPEFWoQw9xKRRQSNiH/twIfYAheIBDFVw0yGrc46zC1bPvQ5/pGF9sGLd6C2CznIa3JE9ALc0kgZUhtsdHUBkrGjnhIDLeBhgb7B/OoDtcIbyUEP2wd143u8s1ZNLGYLCgwkeQPFSZxbHNfazS/DZQziBX9jIcfU9spNbq2x1ZIRfQE1U+d2MqMiyNbRkTHPETopgBGDcesV6JAPJuLbzG05gUQ+GT5WUjWSCdolM3RxJKFnGtY+5IZc4utA4dAng7SLBbsoDszcwGUZsH5jOufiFhUcAfLB1Hn3Ue4tu1gcwrgEXQRlagAGSzYb+5iUbWC7zPOL9nO5GbZ2o2vdniPl7aj8X0X3ChfdLLc5fbZ38Wg38vbw4/Jrt/L30yYfsY5TSx7576fE+9PvTyfM1BvsToPpiUPgMMBI+/th4YZd8QczfBzrkPVb1YzSrvPfP97Y8fydn1m+ANQ8WRHBMhH6hMALInTMjT+sWrNJw/3nQWCPjUeke8h4dEIWK09yYUqJMO3a7a67nTrqeya3jvvGPu3us3pPSK5ZpYRLAGt47eQoAKOLUreG1oZJ5dbvw/fWO4/WOH3+84+54Ik2ZeT9zv27gQh9uw7U7XleO90cOB6WusHnnp/u32N+N43HP7fUSLmhxSk6s1pmLMCdlTsK6grtC3pGKc2rGREESyIWxEzyBKSHI3Mat8sbZ8ZcItusCpEKHhead1iOxol/iuPSMGS4xmKoP7kRcLsLNF0Cn4zfxcbvHDX/kQjwfT86sEsqFmxtjwvwBCJ5HpfeQj8kpBJ+TDIfnmWFTGUAy2DPzALIuo5qHwMhpfQBKgxns3Wg13J7dbTBYISHTTNiqc9w6vYO1xjqArtk0jhXsWpfh8tXocXMimE6iLB4I0kfGrysiBjWYaU0wFce9IaMSXbRCR4xgJK+YO20IZCcJUBldFQLMAc7OTGscR0d1mjMry0iQ6hbMZW8NEHLO5JQe6v9qCDunFFnBPpJvrA8J9h53JzQYjW4h9Fx7o7WOJI2qNt/sA/Y7cFu9yH4PgORrtPFfcZ2/4jm/+ql+S+PiJaDzubn4Kab4K13jWCc/vsGn7YsqfIQ9/jSitN6jK5828BnA+NznseXF0/QIb2YXyohdc3H6WPqyKFmDF7JzzIuMDEeJBIZu9hCPlkKPrfbKlBI6F3KLxfKnux9IbLSrwvdXBZOoDTsX5fvrhaVckdLMlB36PWoHTGC5uWW32zGVBZLi1tnlkAORWdgmQ2vn+mbPf/xP37FcFax27g+nYGlSZlbn7lD54c2BH9/8xNuffsRb5f7tPW/fHMhZ8ZKwtrGUiddXN7y+vWa/Kw86eqlgkigorTYWOuQJLQkTo8oDx5IwxGVosxlIGm66s2v8rBsX8X3dnBApGfSQPbA5IpGxK4PByiMWMJI9BgAQewB3cgaAI+7v/OXlt7GdyiUu8IzzLsfwMxt8bmskh8jIFI5rG8kJCXBFLOIGHRsMlYesTW8jzi5i0C5sYSC/GFtmI6wgpHWaGSZnfUFh64p5yOUcj5W32xZxlUtGJdE9Sv2lMwLuA1QOBOYS0oJnYSV1R/38gtMRbyTplGwIDdVIlMl6TgxKqDiZsxRLQ8d08OGCPbObnSHKPMqkpAH84vfQq3SPSieRfOOXe6kqpJLJKfQDk4Tr95xNLBJ97xZHq9YD8JmNuNtR5aOf9QsT6U+J/eTpg/UD5u/89aJ15AnZ8M1+hn2O++6PZB8iw77ZI3thpzxdvD56mEcMyAfH3vnB8Xgjf7LDh+3lFT6eAWrvXsL7bN6DxssLAgceBSk/AL53R54LdI3ESxdhUyEKiaUQ5JVgozpG0ULJcXndLArS48E4qVAQnBbabeZ0i6zUSSMpobbG8WT80FaW1Fly4f/0t3/jx/VE3wp1axTpaOkkzez2M1dXr9iXHYd1pf50z80us1mnbxug7HeZXdmzy8KuHfiflh3TlDjmkMRoGEsp5OuZw+uF/7xb+PHumjeHE/84GIdT58cf7/nf/seP/ON//Cf7Uvjf/cff+K//U+f1qwWRqIJh1qm1saQM3Xn93Q1whV7NeBKCjmkkb0yq6L5EPKSGsLNoaO65ebiFh96eCExzJuk5Zu7R2LiAMb/c9zO4E4gYPxUuWiXu4/Ngbc9xevLwWS5ob7hj/XGU6FBU1Aem0Nw4V/fwIXrsg13sZogFILThcjQCfKzrSqsBaERD29AG4HuoGBPtVckUNUwnPDvex7ZAngopX2Eyc3d/4njo/HhfkdS5Lok8TaNdFvv1RrNOrw0j2NxuIaHjNiqfCJc4P5Vw9yqVhEWpO3VKEooKOTWSJnJSikIWmJJActqI7XMbcX9yrqXMAIQS4t5DBuZ879QDXMoZrBJjdZ4mcimX5448QutmRkgORjZ5V6hCsMbNIqmo+0gmiTjRuQj9z+T2femj8Wuc4zdhvxfU8IJ2/lxP/e/N/Mnnby8V/Osv/OlN+Xx7MfiLzM/H33wtN8RZNPqR+04uTqdH53r3j/pQ2NAHlmdjlNvShKYIJt9aVOOQ4X6L2KTMskzk1JFq2PFAWjeSJF7vr5jyRG/Gm1459pU9J/7t+1dcXe+QvGNLDhLxba0bKRVurv7CPN9gJA7bSusbglGyXWILc85Mi1LE8F7Z7hP7GXZi5Bl0CsZpnoxrz+z2r3j1+oYfD53d241//vCG9dSp9Z66GQcOHG3laJW0OvV0xLdKrSvWG1Kj9irSo+Zum5kTiHWwTpbO7X7PUguThPQLCKJp3AOAFCBtZE5nFXKaLq5QfFQ1OcfeycNdAuOcCSsSoDvq4J7ZwNCJu4wrf3AhP2WRHwPMy5fnrwZbJToYuiGWHFIyEiXK+shi9rhnrYXuY611gKxE1gLdRlxjAKQok3ZmtQpKxiiQjczMZC1ADILnGctXbL2gW0cV+naiOeT5SFGltU6rJ6yttG2lWwNalLqDSBjZCutpY6sb3ZWUZ/I0o5IHK9jJwJLDnTtlZ8pKUadIJ6dOSTBnpWaltk6WRNCMFrB5VPxIZ+YuFxKZLkJCSNLjDxYsnvjF/W4W40qS0JsNIXQnwGGHoYzT3Kk9ogpOm9FH0ot5VF5xjbElyWEac/OPbh99ZL70mfqYAv/QcV50wp9nl+b+q+7b1zjvFx7jPfDza/bBr3iu54bPS3DHBxnDX6ntT0HqY/uXPWbk2Y+fs9v73335XP8Ct+/FyfuV7LnXqOcozHNwOZfFq0iUrcqSKKpsZTAUHnFTDdBSKFlHjFMkM1wtO/52+5rJ7jjdOWtx0IlZlJtlZr6a6eJspwPSQ16ln+45rDtub2/I84xNmTfbHfcVylzYzTPb1vnp/sRWPQST31amAiVnUnFydiZ1qjt96xyPP/KfU+dmmVAVbq+UPGXuV+O0Ne6OnUMXzDOvr/aoKn/921/5b//H/4W7wx2Hwxtudgs31wtuG6c7QaxzOCSOa4Nk3N7egGbu70/09cSkIaw8TwvztLDLe9QLtTWqhxM9edQMbu7UFmAtiZI10myyRBJJuOUjw9rPem6jDq1w1ogLl27czo+/Lmp6YI9UH7mBzy7h82iRB7foGexd0i/SgCDDTYnnka1rtNZZ68ZWG9tW2dqGdyepkrLTHM4O8XMpwJSiVOA0zeRUUIRmSuqdrRqtG67BfFlSVnHcNybt7Arc7hLHrTPLhq/G8c0d93d3UXkDYb+byJOSveHnWNbthK8bba00F3ra6L2R04IBrTVUlLpbmOaJ1YTcU4REiFN6xJ/O3SkbLBrxhUkN0ejnlBMZJXXo3cnmkU3u0DCUeDlIEklAZ6m/KL4imIRLurZRPQcY2UFAJJxUgdqMuzXqE0sSclkQzQ8MrdlgIyu9/wnA3x/JflPM4jf7TdlvgR39w4/Ppx0sH3YpP2OfDf545JpxGc/795zSz78a+OPg/g8c/fzp8q/HgV5nlnCgQFehn92F3iMmSjMk2GxjrbGgpHP5ApzSjbRu9PuKSedVMa5urtkve26WQlPHBMwap6ScMHJvweD1DekrqNO00vspEj4sczwdWWvjzY9v2aqxbUdS6uSUyDqRUiGnREmJJcGVGlcTJG+8+q6Ql8JPBteLkqVzOBz555s3HLeOlD1ld4M3J03waoartMf2hat94Wo/k7OQ1OhuvL3b6A1Oh5VeN073d/S6kucd1zev2M0TS5m5mQtFM0kjtgvCFWfnerrn+L+zWJ9yqULhI17sLNhxrqKRVCPxRqPfn95rPbuHL9p+Zzevnx3Ew806bv246zL+8Q6rOEZL1KMdAOXidg5WUTPUrbHWE1tv1F4D6LZGr8FSSQo5m6gzGwBUEYqGhmJJiQTBqJVMcUeak6dMN8Vax3qltpXWjWKJXYF2vUc1SvEdto3D3VvUDmQ9kopzNe0i/vOvf2NeEjolujvrduLu7sjbw4F1M3pLtC603unbynaqHE6N0/GG+dUNmhM5RQJQSUpRpbhyNKPQSe4kMVKKKiJJhVyIFxMc6x2lMk9R5k0wkhoJJ0loNapGMkeAP2fxSDypdQg4y9mVLiMxprM1w3rl7nhCzMlpIuVMylOwy4Bbo9UTgnE6GvC3Z54NfyR76u/91MP6Q2/28sGvPnraX2JBfrrI/qsX/V/TnsoT/Zmu/Vl71AHPwYCv3T9fAvAubXlCSvxa9+6L58sZW31gB3nu4B+2l7t9n55FHj4+vrEXTtAfFu8R1vWBY8U3Lo/YxMsKf0EKl882kjwUp+M0Gs2d5k5QNw3VEDpWzeHWEomaonVD+wnvTtVGyjBdTfzbPvOX64VpyRytsW4btW0kKiVDXq64vf2OV1fXpCy4Vaqd6ITL8HTaMDlADhfW6X7l/m7FxcilMJeCagePtNqkzi7B1WzkrFynyn4tzHnlaq5cTUJdN346bjR39rPgrfHjT3e8OR04HSvucHtzTUrfMU8TJSd2JdMddrcT26Hx49b5+4//nTf/+B9MwF7/Rn71mv3+mv1UUIJF6q3R60qZJiQVZCQm+GDyhJBvEYnqIS4O3gOEW9y/lDSqfFwqZIy4OxvOPL1EmI2M4Meg7+mT4czayeXr85A3Y+zrj8IEnoxFP8f9dVoLZktEyElxTyMJYbg+XSmlRBmzFMBFZWgSpkTOiVKCGSQ1yA31zm4Suofmom1K3RykReZ2D7fozX5mu71m7cbbw4HT1YToK0QjBGA/LyzTxOurmXmCMstg9oytO1tvHI6NN28ab+4rtXbu7t7y9x/vMdvQdCLrRDNl60qteWTcJkqKOZLEyQK4UVJDzUhaKblG2UCJ+MMszmmKxKMAfxIl8hJkDVCvqTGVmZRLyABhbGb0ZvRulwxis8iGPrvWt7qCC1k7ORdSjtAMhKhoYo0kEszvn8FeFO/3UV/PF9hTP9zPPc6vbV/zvF96rF/72n8r8+G30o6n9gR8/KzjPLbHx/y16cNPzdPn2vRl7fyMCh+jruiTN53HIR/vnP5dwu7ph2eO/5ghfEJfOgxtEFyhSzAMpjzEBo7UYHchG6SUyTnAiljo2UEE9ecpU+aZKQtzKRxr5c3xyI4JkU5uG307IeuJWZS5XHF79T1XVzs0R+bi7ImuCdOV09o5HiupJDqKpJk0Rfvm/VVIX1in9UZtnVo7P6yNdH8gJeX2XrjdGVf7zNUszAkSHWcKWZcyYa7crZ0ffjry008r3Z23B2OticPmXO8nihrdO9khmdH7if2U2f3139lPhdevb/ju5pqb/YKKUY8rh3VFPDIuaxuVIqpH8oiF8PWUFKaJMi+UnC6ZnBHm9VCdQ0d5tgCARLUJ8WAAz0kc438XyZdxgy/fP+D8d25/gEkuQPACEC9sMuP+v/sCYr2HlMgAF1kymieKCb11FGUqiWnKlzeVuJ502UcTpOK4WMSn9XOtXdDQRWHKmZQyixTMUtTaRaklxvPrXYlKGcKo2BEs8FyEkhTVAGmigactQzeh5sz3i3OqUVHk7nDFfz01frw78Pd/3FFTZjVhrUazSuuV2qHnjPfIHM6ENmHyleyNKSd28wSl0LVDq3hKICVAPoaQ8PSQ7BLZ2p3NT3gLJlw03Oh1ayEsbuNP9wsAxENrUkWRHILpPiqjiApJ8yU8YP5TpPv+VhfSr2S/+cv7zTfwN2JfuZ9+rW5/yTvOM97ST3zxFUweYZwPgc0P7Pep437W9g/2cubvMbvy3O+X/z1mVR+xgJd9H0q1Pd3fH5DBw7nG4nqu6nvGHJe/CR2xAJsegf3aHw0CRyzi9vAW2ZKaWeY986Qsu4T2IyQjF2FKCUmZ7Ip3x5qwaGaX84h3y3SJSg15WkhNkbbhFnp0k07YLspiCcZ313tMjHXrUB1TpbbCejIObzesVtpNJvmMJkI8uFeupsJ3r65YlgXNOYhLlKYT921j26JU3NbfcOpwe70gtlG3A7p1rqfM7U3h5mrPzbRnNxV2u4n9MjGVkDfRDG4FSYWtFU7d6dWij0ZSivUaFUQgMjxTCd51xHDpSBx4uM/yKDtWhgYcg2U7M3WPnMH+eOCfjyPvjbMLyzdYZL9k4z4aUXouNRYCx12MqFfhl+MFQE0BAufQAEyD2fIhAi2PXkQiMzZKrjk9WEMbcjXm0OSSeDRLRlPBXDETzJzDtpGycjspZhpxfaMiiIiTxaJah3XoUYu49w13UAvgNmflpsxoKvSbTDV4e7rm/7WbOTTnvhn3p43DceO0VtbaYI03pVQSao4dD/R6pPtG3i1oFqZZWSYlTTNLKUzLjqlMQBuxjsEgeg8Gr3cDiwQQ3LDa2E4bda2XMnDB/jGqhCglT6Q8M5WotpJTvmgbnl8odczkUgrf7Je2D7mRv5L9IrFe8mhR+RnH+DX2e65LP3mIfwUg/a2D4Jeyep+4jg8d5sX36bkvPzVvHjFf8uT7T+334m2f2/7z5vMX6fydo7MeOJjx+zvnf5Sv+4D8Hh3pUb3e57Z9/Pcjf/ADsPfhfoxj2RkKukQGpUQsUhmLbFStyGSFPM/srq/YzZk5N0qrzEtmWTK7IiQS82ykbNS1czVnZjV63Vi7cN9W1trpJKQUiicwyHOCpGjubOsdvp7ItpEmIU9OQdDm1O7U1jmeVqRXAHbLxDxPvLk/sK0NpHDDDLLQzDluG9UFz8HYuEdCy6E28uEUFU68sR3u8FOF/cLN1St2ux1Xu12AWoXaGlKjzi55YushsHtfI7tUzJlVmFQAHeXCnNaiskOS0PGzUa4t5YQO8HeuBHG5q26PavD6GDfnzw/k7jv4z+WdIfx46DxSfbkkmKRL4GBsqBri1OIM0WQQHef2oQNoPcr9TSXabm3sPmIYdRz/HFZo53QSxzVGm3gkPMhwTyeUpI5i2Hg5cBUwUO2oDElEHxdyBpjmZC0hS2ORGa40zCPWzjFcIo4yZUFUaA5zSvi/3XBocLd13h5OvL1T3tw7h0OnVSOVxDwlUu/MXVnNEXP2U+JmN3Gzn7naz8wlsyuFaV4oJWOjP8ooAdhbZ62NWlv0ywDZ3Z1eG94bmKHGEJmGJIk8zez2V0zTRCkj23gwq49GCW6hJXhWAfpmX9NesnC+dNH4hRiRs70f/fG+fWrR/hxv06fa89XstwC0fgttGPbM+/7z7fsNtflsZ1fnOyzec+jyc9r+r7nOl+v8XT7IhX0ReVioLwvzBcYNNu5C1Ty49C76bX5x4IHYOMtzHSiXgH8gKjFIpBi4GOfg/ktDxqHPosM5JZLm0ELTEeO1u2I3C317SxqyJdWM7IKWxJyvSWXCurFPM0UTb7fK2+PK3fHIBuR5j+uEpVGjVhKaM1Iq5EQ/dbZt5WY3s1sm+uTkY+O4HenthErl5tXMd9/fcPv6ipQTdtyw5Jxc+Of9yrE6kpxjPXGqIUlyc3PF0gyhkwbzZjgpCZaVlp2aha4FpplWElHzt+LdWJjZ7Wc2d344RdbrDz++gb6x5MychKUklpIouUACHwkAXNzn+sjdO+6PPnC6PlzwVvv4/HjRP5dyG5PonDnk8JAx+sge3jpwF2DUlcXoHoBPRqyhi4x7HzGKUcdZ6T00C3tvtBZxbzk7ZoK44TrKzenDGAyGEeRhRI+5H1nPIiGULE7EyvUezJ4mREIIe8kS49R71FceZfF8lEQTQM5M4sikLSWNYXzOYjaQUVLPUySjiPPX68RqiVfVuZuUN0W5mRKHpdCas1ztKCWjrbGddpzWHeYb1/srbq+v2S8T+3miJGXJSs4BYM3kXNJk3DeBMsIozOkeFUxcRqUPOc/cUdZPBE2ZPM9c3VyTy6go0teoSvJoTotr1Dhuo+zgn94+/w3+65zzK9qzIO7xgvnknE8v93Mv/53t5d3vXnxpX3sB/sj1/eJr/W8VNMGjReDnHeep/Qxy8L1jv2j7LwV6X8t+/gV/htv30UyWd5m/8wkHn3Ph8S7F3P2B7YlsSr0soHEUewQk5WGAiDxggfG3n8ugjnO8u895Y0FIiKYowdUZjFCGUYlhPTRybby9+4nFN0wSJ3OWSdjPhd00sUxXLCkjngDltN3x03rkzaHT3FncsWTcnTZOp5Wkwn63sPXOsUGZb5iub9jd7thPihpMh8qKclpXbvItf/23v/I//9t/kNLEoW7kqyssb9wdj/zjzX9i3clZWObMNEViwtXrHb1Vtm0Dh2UKd66qsfWNilDnHYc085YJl8KsSjWhm3FqmbsDnKzxz/uNH378if/f//v/y5LgZjch3ri9XviPv33H315fMyUlecd6p/dKTuP+jzgwuQCwh/qx4/Y9AH15d7zIOxsN9nYweUHkBRi8QCQHLIDIuZavmw+GdxzaBe/g6BhyaSSaGHjHulG70VrFkpL7BkDSjHmIWQsa0jAerGcoi8c4vKSspMhxDnd2jnHpoV03sDGio7SZdUzskRvA3nErK4pYVCW5gEHkUv7Oz6lN0i4uaFyYspM7zJrZSeJalVd55n5faP0aScput0Oz0Lchrmwh21JyZhrVORKOdUOtoVpJQElRJ7t36B7Z70kkajh75E41cUgzyzxHhQ50SPqM2M+SQTPTVcKpnI4VoUcVGT/XX44KINYckUQuf4aYv+fs6cP6F15MPhsYfa2TPnPCXxrnfvJaf0HQ96vbbxD0PbVf+73m6Xk/t4u+tL2/g1vxGU/bM9x78vfZrXfe5NJZI74JuXTEpY7qEPkN9khxb5dg/qfne0gZHue7vNidwaI+Igj1YZF1QV3pvcfi2RpNa8hVuFOPK9eTUVhZMWquXKnSpbDWjTesTDmkPrYO++WKN8fKD8fG3RYL4MEq1StvjifujoeIi9vt6K0x54nvb65Ju2uaTlSErCAls+ycv/1lJvW/Ml0V5v0tc1lY738C23jz04kffvyJ4/FE7U4pwr9/94rX04yWKSpurCd0PZEksU+Ja3O2ZrAqubxif/sa2V9zsMR6CI23Vo1WK24r7p1q0fb745Ef1sY+Kae+kqwyLzumZcfuak9qDuuJunV8KRdA7z5cqRYxgpqUlMYtOws+yxgrj327wgOjdLZ33MMP4wU//x1ZuIgOBlkvIMoYWaZoJAQ59CYjYxnM4jXD1PHUse4hNaIzIili1FofsXzBwulwJ59xpVi0tZiMrOeMSYuKKUL4dMc4dRd6dVqt4Dbq8UZJuTzc73AmQnSw135hqyNZJA6po0ThWTZHOog1pJ+wrmAF8RLZ3LPweil08ni/UlSMLuCTID6Rk4+EqtBrVIE+C73LpYJK1gBy1iUSOfpDRQ4ddYLnKZFLCSmbUFl6TP+jWZEUoovNnDk5kjPmsX1twfb1tbGtHc1KKv+qleGPYi9gDX/NLv7Yub7d6m/2JfYOm/yC7b7k2L91k0/N86/M/ME73rcHN90F+PnDBo8X+0dMDQAaFRNCxPcBqD0c2R+BufOCJLg8YoDGKfThqOgoFdbH+VWVIhM9KW4biCMR2Y+58abes3XnapfYRPjp7R3p/sBSlCkpWUIqozXDXMllYrPG2jrNBSTR1xOnZpxqpfcoX9bvTojD7SycfvyJH7cj18vE9ZQpSenmbF2wIUb95m7j4G+4mlZ+ePtPttao1fA040XYXSVeXc28vr2i5Mz//L/7C33daKeJ3DeKA905nk7cnRplWUAK9/eNbgdOcwGM2hundaWuJ/q20uvKRuOuO8fNECm4JqrA65srltffU3a3aF4oKeisUoSUFbxHHNrZzcoA9gO0qIB1J6Tc5N016ZFXRiUYXBn3TBhacaNaiIxx4fioPXxmxhRPkZPaDeIFoICkcNsaVOC4NVo70lvH3C51hqUUUilImdhW8G5gia113C207ZSoBSwMt2uUV6tmGAW1xFRylLk7uzABMaPVRquVXlcwZyqZeSnIkJCRBOcklDaGvHcfdYijdzoRX6gqZFeUDETMXhZFyFQBTUbqW8QfAiYJ83wBq3jDzm5wEt5DKulhqsoog1dBNnKaSf5Q5i2N6ahpAOnxAqc4OYc7G4k62W48AMAWZex6h04j6msLW+usW2erTq2dbduwJhRX2p0D1y96Fv1x7OeuOB9iDV+Arj5Axn09e3LwXwTw/ZwL+NJ9n3mg/UvsM+71v8Se6ZsXvJ/83FP8rO1+qf1/1omfhkv4+5tcPry8oZ/nZzm77h4TNo8CH/0dQeZ0+T6NYu9nAu9BzmPEZyHEyvG47WcgOC7WH9zK7zidRR+mwFjkIqNSsRTtcAu33uYOHjV8mwv365HahWWaMFN6WxH3sZAGK5IkYyiNA2iwMU0UUqaZsW4tqmCoopoxIHvnWCtSD2SVS1zV1TxzvbuilAlPynba+OnugP/jCCqs9UCrnWWemJeZ3evvuN3v2Rdh3i/krKwkyn7H9e3C1WL42rl/syLHxtX3BVkm/vOfR/7x41ve/m9/pwDVGw3HZMSQuVNrjexeKZhBcfgfb99ws5u4/t/fossNlAVSRrNSpqhO4gmsgvcegEh1gDi4OGkHgInYt8vtexhGMsSeR2KGyoMUDDLKq9k2tOOCJpMcdXW7WZQMqxZgfmgSrnVla4ab0mrU6+22sZ0Og/01cpmYlx1lWkgOx1pZ7w9B42KoKimdpWv6GFshWxLl+kBzxtMEObKwzUcts27QG7QG3kbSwwDCzZCmlCyYS4DaS5+FSPVjBn1E+T2AQjPEbQhNK1kmaolsdO1OQN1g9ERC0Pl8D0wi1EJEUQK8uUX95z4qoxgd1FDpCIUkEeYQjeqkxEXd2883cMT31Y0RhzlA/rkUCJFRXnulese8c9wqtbfB+p1rbhugmCt96+8/c/6Q9nuhGD7Hvvaq/kvbH+ke/Jb6/tyWj/TvCzb5xe1xGz7Wlt9S135l+wydv7N8w6PeeCTMfIGAA9mJx6J9BgcQ+0esz7lPQw6D4c57x338CMV++N48uNpiOZLB0CRmUdQ7lhrmndrr0GnLWJqQKdNcMBWazSRmOo61I7DhajRVXIYfUxJdRjWJM5Oigk8SiQ4umGaYFqTBev8Tqc+YO9JnDg73Dv9cT/R+iOxkD9DTqGx9AzXmaYos3m2jmHPfGrMo/00zSZz//o+/czVPXO8LeMObkyUNN+01WxeaGP+0O37aGm3b+On+hCchzQnJBTQSYJp5CDz3xiIVq5XbVzN5Tog667by9u0GizDPwdSeTpW2bhemS0e9V01pgPyEjKCAbn4p7fZA5p5dxY8pXKU7FybRSFRL9J5wUrgQJbJcmwlrc7Ye2d5Giwoa68r9/T2H45HD/YpZY7/MTFPh+voWGfV5S45KKwCn44H7uzu8NVKCkifySGxAPJJEeqeNTOjeG0kVm3ZYPoFEIgNm0Cve6pBrYYhDF+YyGN9+oGRl2lJUY8lCLjnmSwyv0PdzuzCpeBDq2h1r4Ci5ZHpKGAUbgNM86ue2UGIBj0xk3MIbnfTi4hUZLzB0oiJxIqXBSmomSwaLF7Uo1Tem2LjPaUx/k0RvPmpnO5ekq4u7PJKyau+8PUUG+2mrtO4YwUIGiIyMcReiJOEf3r7GiveSFenXZIV+Z0DqnTfRz9lRPvD5c8//dQ7zrv2rkdTH/v3EHoOGn0O+vuQcH9rnKXD51DT5Re7Zh+wzEOfPaMtnM3/n2qfvwL4LWNNHG57vrFwe9EpopXUzWg9ph5BrHnF7T0vlPD7xI4T+rlLgqDfrUNxpg3FSj5qh3YMFiazYIbA7pF+2MgfjoDPOMtiliW4rlUpTgXwuFjsi+VWDhdT80B8aQezjgmgAZY9LR9SAyNB0PwMYAkwg5HlBk7PZCe+gnnBVrDWOpyP3nEhubFsjiXNY77HWKCWuY84T+2nHvByRcsdqnW3d+OntkQZI2VHnQncfiQyCVdBuF02/KQm31ztuv7vm++sZofP2zVtKXamT0/cKu4lM1Ho9x3EmQgpEFLQpORcmAlx1i+xWHwkcAXSBUfXjDKIRpatEti0Btk7d2CxhEvF1DM3FbsJhNY6b0S3qJzsRj7auG4fDxvFQ2baIL7uabtjf3pDzxOFwYNvumRLMU6GUzJQzV1c7ejd6byMmFZrFGDdzWt1Y12MwiT1c1FYjdrAatFH72azhLZJiVIwpZ3bzzG5JzKUwJ+V6v7vIziAZSUIqiiO4EnWCz4xbIEDUfIArv8y6c2GVCIVUXCYaTvczYDPEw7lr7qgJPVnE4BG+7KQS2ccpqrK4xbw5Y/LebegeBrMXXWKXZ1JUIqnU3i9u5JiOMSu7G7VVqjvbtrHWTm013MA25HhSvgBxcFp/KB35zc72MUriCw/3B2Uy/lz2rwJ7vzOg/7uyT/Xt1+37z6rwcU69fZ/vI34brEX8lICH+K3Yevw+Njq7ui6u43eu7d0LPbuMz2xIuJkZACza5gYmQrUai1M/oUmYUmESEAxpIecSLrlMHW3QNJgkicWpEwxZBH4N4DfKs6Epsj8vfmx9AKVmiAmaCs0TmuGEXdRMZNQ/JRdUEl0SVTqeNFy/omwuqCnJQkeum/OfdwdUhbUbrYW7t6iSpaLSEL1DSqaUggpUjcW8W6JnaBZF8dRD2kSssc/CblZud4Xvdom9OsUrvjlSJ/Iuk9TBnK1VJKWIDRzSJC5CGsczMdxbAOM83J1+rsUG8OCKxDxKvznx9wBD6uGOP1XYHFwyzpACMedYjcPaOaydZg2zYHW9G33d2O43tkOlmZPmCfeMmXJ3f+THH98gdHQ/s98Vdrsd05TDm2nAKEe2bY1mwUDNWdnNhaurhXWrbFulu9EtsUoKt3TvbNWoW6O2irfOpI5lQ8xRCr12dLegaaKkTMrBlLqmAH4ygFZ7mFduhtmIhjizokC3ite4duSBSY8KGzEeVQYoH38/ZtAhXgLSKMWXNdHN6AQoNzvH9Q1m3s/za0wBDzd4bUZrDRc/q/eNZwO4Gd161CIWQTQzzRlNmW5R+UQl/q3juZLVMTkDwW/2y9l4EH0VAPghxuc3iC4vLJN8Adv3la/nZ63hv1Pw9ZVI069y/pds489897Mb4M98fum+H7Bn2dOXNfozmL/nABpPruEB1Z1Dys9Lgsk5kum8w1CEjZX3UaPHn7PGy5nqOLt3B5Px4HJ+kBe5kINj0VumiaslkyWYwOzCIkrRgqjypjdOrVOlYbbRo9otiRoB9wym5wJaz67pYDZj3XoQnRGCkkk+ZD+S0IShfycP1xN+tNhX+ri0AjL2c8FJiObYXBvH1jFalLSTjJvQ+iNpFXXEO5OPGDyxWMStB4tjRtJgoZYysczKrEaSzm4WrufCTU5k4Ltd4eZqxzxlkkZhvLUHKJhUoAspCVlD4w8/M3kDYA5XbigWn+VZGELGfQg0nwkuGQ7DcPEea+V+a1RXum+0UXZurZ1jNY5bZR0uWLdOx1EHaR3fwj0KiUyG7tT1xHo6kBV2yxW73cw0LUxlIhchpx6YXkYb3NCeEYGcUmhEprj209po1tmsky3B1uia2IZOXceHC9xJuTCVmWVeKEVY9jsk5wHyFVcdIXVxj1SD8XMEG6XS8DP4C4AoZ3eqG9YbrdcoGXdh2EPvkJQeaicH3T1A1pgvl888eoECM4vQxTP48wEEzQZYt/Gd0XrU83UJgBez/HyweGHpHsdyh5QSOU/jRUkvJ3VrdOtkTVztlk8+hf7Y9nSB+FrH/Nh5vtYxP+f3n2O/FsD8V6OVP5j9qm7T587/gjH/G3xn+bB9rBNfdiGfFfP32Nn6bhueAYZyWQpgBKJfWD8ZrmL3sRAYH7yYR+7ei8P3wmZczvCQbekBJyYX5lS4nSeyJtw7s07cpEzxzGpDqJaGWAsG0JTkDfWKYvReI3NSwu11eSx7lACTs4/MDaQDgnpUaTAMRAerqcNdfNb4OLvOg1BlLPDhVmuE/IfSNV1AUSdcZgGqdIjwytCXc1QdVNgGI+kews/eo4KCAEWUSTJXKbMvCtKZ8sTrV1f829XCgoM1bveF/W4iF0D6YFGdVjubCMmdpSiSJbJiL4t/AIjWLIShSeHuPeu52QASFokG4hHj1wlg24CftsbhuLFuznGrHNc1kmqqc5Jws/bW8N6D+UPYlYk5RcJNmpSUlGVXBulo7KZC2hd2y0JOSkmh/adOyOZ4B4u6z6VkSvIL25ZUo/auKFOaadbY6JReSKWiWiNGzoSNzCTO1T5zc7Xnel5Cf1GMnAVXD1Fo5OL2NkZmtAY7fJaqcWNkAA/wJOky3HtvtN7Yags3syopBYupMmpwSzDjqvI4B2O8NMWEMs5VTIRI+4hM4D7cveYMdtMiacb6ZbxFvO5o/2U6+mVcx3FkvO55iHJLJ+XQRwRotbGux8iOTplp92fV+Xtsv7Bf9sE//6kNnrFfe9V+jop57rcvOd6/0F6M7//V7f1Xn/9r2nOs28fG1y9tnzr3L9/3n+f2Bd7lGeXdf1/aG8yGnOPgHjEOZ6nnh/30AoYeneyZf8b2rsEoPO4vJbhDPQPJIXsh3WmboUXJJTFPhbnMeIN6OgVA6hGvl9BYKE3oKGIgblTriMbi72edORcu2cnOA30y2BfOxMoZ0NEHO/fA/D3I3HgI3+KYt7gQySAZx0NM1+WitSGiAarU41wWS61LMJH97GL3NAD3YCUlYi+VaL+4UjGupyteX/+NmyuFFuXmpiWTpoSL0d1oBqJKq4ZbpQwAqICl9KCJNwjaPrQcw2U8+sGhW2jHnf99Tg5v5tRkHK1xV437U+f+7sSbt0feHo6ctgCGrZQYht3BOnhDUmIuSplmisaALiWxzIk5CTnBNE3kUphLCj1Cj+SWHurFFzFqJeozgwerNdjODrhGg3MqqBZymigaiUV7hGtJbHMjJ2F3NXF9fcV+WZiTRDygbWiK0miGX4CUQJwnRa3kM0j2AewvLxhYjO8RNtAtwlDdBvebdEgoKRcRpXO85QCC7z1khoDzORrXPIZfF4YL2C71ersZ1vp455Ax18a9PB86QkovgNDPLwVjrljvUVNae4DlWjmtG7UGa7ww8ee039Ii+1tqy+NF+ue067d0TS+132ObP2GfuqSv7mp9cu4PvT/84rjvl76XX+Yx+Czm78FX9OjzY6HesyCznMWdxwIRyIcAIfqEQRwAkKcA8LH5w3HOChQXGnE4jUf7OiMGCrDaaOYsi3OTFiTN9DyxWWcTqKa0LjiZqexCI65Xau1Yi7gyB9wD/NmoscrIgrwsfOIwip+Z5AARGMnBvQKGjv8uVN/IeHXrqIy4R+s4gugQGyZiDH2cZ8CT0c0eqEEZgsrnOK1zP56zlIe+oQdoXrsFq2lGk84uO6ej8YNtKI2bJUEukKKyhHWjuaMkNmu0teEirISOX7ZIPDnfSyEhcMmC9ZEkgQzwZwI2WFoNgLM1Y9PGXYPD2jluxt2p8ua48tNh47A1WhLcompLssjsVpRlKuRlYdov7IpEabqkzFlQ7/S20rYN60fEJko6J/BEKEJKhZyDNXMLPUA3aLXTu6OitGKjpm6P+tClkBNcZ2H2xJ7Cq6JxzWi8bEwzu6kwpVCBcTdE+qh/bBe3agyFUSLv7HK1AF9x13W4UyNswse8egisCKF01YSm9CCZMxKuZCQ6PZ5Jj+dsyCP5xd3bzIccjNGtRYzecP9GQknUGjaNF4JLeb5L1Y4zM/jA0rsM0WgXWh3H7TZcw0pzpbVGva8feQb8Ue25h/WXLhafwRr+AbHFJ+1F6+O/smP+rOf+NUw+8Pn3Zk8Q6+VZ/vkeg8/O9n2o2CGXdjwUb3hw9T400R9cvc8c73nm8COzVP3Cnl38+Gc2hLNIbyIcgj5CBxVNecRcJbQ4MxNNnNQiqWBeZpIK3gXRSlOQ1ocbWQZQ8YcWnbXpfPTJWO4usY6iZAFcQ67DRvkzHSW7PILiVf3RfYuFXYnjnsWFbQDnNKpJnMHfORbxzFw9xFQK51rJ8VsAVnPj1FZqd+48Ege2w8rh+JabxXm9X/hv//4XjggTIcSdR1ZobY3ana0GIE2iaM6koc6RBFTT6IdRz3aUWtPMEGz2aEf42xFRmjnrsXKyxn13jh225nRNtJJZc+bY4eBAHaCxd7JVZhF2SzBxKRWmObMUZSeQqEg1Wts43t3h3LFN1+yWa5Z5epBzscZWa/SwE9U7HGqNiigg5CmTS6K2jdO2kcvEd7evWYqiauQcotBYQFJJiiRIYmQV5kkwT9QaSSXhQhXcFUacnhGhEQa4Rhk+MSf5cKd7v8yH3hmZzoZIIuVoX0oy5qOPqiKCJkU07n2MVb1MOx/z0hD6SKqJ5JEeZel6VEM5M+SRpR9Z1pJBCdZXB0D1czm4eG2gi8W1eYDZ1pxaPUS3u4fgeykknFaN4/HPIPXyJfaUovjY4vX0t1/bnfU17Gstzp97nF8ZFPySTNcvYb8GgfWzdn401uWZg33o+I+/f++ePEcPPrfDp07y0gZ8pj12xH6mfVZ5tzPme8zzvFO2iwdcFv9/FMs32Knn7XwFT/y5j879rpuY/z97f7YkObKsa2KfmhkG95gyK6vWWns6wz69hS08PN3SJC94yyfgq/IVeM9LCskWCtl9TvfeZ69aq6oyMyLcAZiZKi8U8Ck8MiOycqrBSrLCHQ6YGcwMwI9fVX89nlyT3QNNRDziFaNrk4OXJlIxhjJhYgSEsVamkrEQZ6BVsRCRKM6gRIMpEqoRJM2+aoaWugNbzP5bsDyM/TTjjuH0CFfP3lVhZpUMd4Bv2kSIMjOlkVp0ZhQDxYxi1QWrlzGQvTlvJ6x7NEZxN1aCzFG1tsuSUoFihrjeBjYVxlzZTAM3F5GpVpq+odjAtm940SUu20AjUPPIOG6pZQJzsBNLQ1PM/dGcOkNiIFic14l5VGqdT3oJDFFnPbPBOFW2dyNjETZVGSWxKZW3U+H1ZLyejNsRNipUsoPzcWLdGN+sV3NQjAMMVaFoYBClTYvnpZHaFrgkpX72jUsOiHGJF11kfMzBeQiJGB0EenSqr3WzwDhNIEJVz3IS0mz2VptN8OIvKNF1JYsZnie4zBG8OsuxyAy+nDULzGbS4K4RIbKLmjX1oJs5fQ2lmG9fpFjmNeGhveqXRxRn6xbHvPk6k/mLAOiiszmvE1XXfpxlbxaTry7Dg89pxaAWRNLMmjaIhjkgpO6uaJf/q+RqWK6UycjFfTZVhZgiKSY//5BowuF6/rWWcw+Tw9+e8zD5kHaX8rWCwi+1Bn4La+8zlc8Oaj9GQ++r4xMAt49dngkEn2f2FYBw5P63v4Uc3kzsIat3tM8p0/dYg4cfD+jNo+OWh5cDo9gEdAaZIbp5bsqZXDLbcSBFN5MN40Axl3sBYzNMxOS5aQu4ALREx1MWZxjh7YQ5uGTBvbZjUZwBWQIJKupYIAZSiqQUPbfw/JBrQgSLVPMsB9kq2ZwVCtTZZ2+JNF7S5DE/9G0npbibj8VUKLD3SZyjRkXdb3A2B0dJFIu7NF55Kry9zUj9ibs3wo+rxMu+46ZrWKdAMBcwDrNZr0mBWgOqkVo8s0kVSAlCAzE4UI8BbFK0zEBiNm1baCgqbKbK7XbW9kuRe1V+ut/y01B4PRTe3k/c3hcmnMGSCmuDIMa6TSQBzSPbu0y9VxIefLNOxkWTWLUNF5fdrO/oARyet9aDGJjBisxMJDNLZShBIrGJpNQQYqQxYdVdEFNgKhkdK7OdcxcJG8QQUddrrD4mwzwXsjPXhpn9nc34JlA8KMKqYWVm9YBg7jrhl54LVadmdh9YMqSEQEwuIeNp1DxYyaqLOcsclb17jZij501xH0yc9ctqO3/DnZlXlpSIRq1QxNm9kBpSctHwgsyMn+7Sv1UchlYr5FLQyd00ESG2DU1wxlJSRM3Iludr8TdQBB7XNH3fgR/pmC/COn3Kxp5R99G5f+6H95ds7yNM+nPeTT7ZGvuZQOxd/fqKsNzzy0My7l3lebl9bTE57po6/7L6McvZQJOzO2LBZSQikVy3vL334IQUPepxKrM/U50hVYwuVIzQirDNE5XCpNWFmgNIdLmVMjN7S9TykoMVwGSOyDRPDbcu7laWgCQRF1Bp0TmKN8RAZTHjyZzVQcm4tAmqHol5KBFzcP6LXxWwi+zdE6HzBC2ZWHb9nNm3IC7269EalALRKpVARfnpzT23bybe9A3DzRV2cwmrxGXX0rWRjsBFE7nqhTYFZ8iQWXTZCFUoGG3jkcDTWLBanP2snv+3SqAG2JTCNiuvDXIIDDHydhJeq2dDmQRoA1ehYdUmXqzXtDHQz8Etl23iomtpo5BEiXO0dM0Tt1Pl4sULLi6vWXXJde9wdivnTMnFvTDDHESCZ7gwjFxd+8UFkJPrG4ZEagKpFkqd2Gw36Ma8HvMI45gCKQVSEprgskTuxaaziXQxqc5TYzq7DugsYl2po3okuTio8/USSVFIwVmyRRwpzD6TaorU6ibY2Zzt0d4zC8g+Z/ASFa9aD5hmD7oxnYWW1RirXwOV4C8+s0lcUuuMKIqavynVMqKlOlhOCZGEVsWKMowDuRQX34mzdmZwrT+ddTRHK0zILFvzGyhPvl/+op9Ev5ffy+/lKy7PftW2A2S5pKA6Dvo43Hn3v8MNB/udsngLsjzYvgMxsv+8q2D/sDBx02awRLAEVihkZxVkjn9cgiJ0MZYpsVZiFSbKHIRaqUGpsg+dWPykYpiZvQOgtXQ3hOBPZDWmOQg3YugswoxNnhM3OxPYJM+qYCbkMvmDWoQgbqrG3ESrGEh4OMQ74ewDgHhIRy7jtERbHIFHpZhCch27NhrrAFDZ5IJYYNVckC5eEtYrSjJKCnRBWHUtFzEQpJLVH/BiRi6FWgoxBjprSVlJEmiAtgmktqGq7XwHRy3cFsjSUqKyLcbtpvDTmHk7TAxlomA0q8A6NrzsWl6sV1yuLmhNaKMwjhvyOLnUTRSCmWsPrjpiCHSrntX6gos+IS6qh6F0Jc2mR88j7HmGBa1GntysH0OCOcetFmZWqyIWsApDHsnFTa+BRJMaWtzcGqVB8ZR1HuntAUMoVEueci3MungVz7dGJBfzQJslSnuOqm0QGjUo4wzYlrzKhlpAxGiiA1BZroX5e9gtnHnua6XUWSR6Yat1H9WspkzZgZ+kDouNG4vNwWkp7vfn7w/uM1lzIUWjTRFJfl8IpiiVOlVWsXHWU6AgqLmPYy7KsM3cbUdqhay/g519+dRjMdd/9Cb/tZqCHys/d4w+NRP5SxvP95Rzp/Moi3YCBJ7CAj6ZKfyEtPUXYcSfWc4ZVI++P63zHyD1clDC4Q3k5PdFx4Izv53bvA/hPd7h0La6ox2XiFc53Ztq1TX8UNeQE0PDbCZV3YkvB2EHCJJWTJ2dsOAPLWJx5k06zKLnnLWwC7ZYJKYPey0hzKygUcQfynu9wzn4xCAcBGNU1TkZSnD/w5B2/mBmEOZAiYDspLDnBGEzQDiZmkU/Zf67i/4keS8UdrBWKjFEGhHPbBICowrJGrTpKCHxtlQ2paBxRb/qGVrIQybqhNZCFKVaZZhGxrIlReWiv+JmfUkfPJPERWxorMEC5BTddFgT1gqaI1uDH+9u+fP9HW+mkVGqz1GItE3Dxapj1QZqNLZSyK0wFmUsWzabe/ogXK86rvqWtm/pu451t2cF27SwZDJrO0baZs6uYWXO0jFSy0RsGpJEjLTPsIHPa6mVaSoelGEOFhF2zKDMgFElMrnRFiUiVJeBVM+KUQ6jaNUI0iAWqXnyaypGNAoWPGpcaoWSsTxBLYQATdN6kEfwQI+pVGz0KPLQRJq2cf/B4CZmCGgtsPjy2WxWDg5YF03uWhPSNtQMkyWyCaW6H6CWiZoVBc9SIgEr1YNK2hbpvE1qYSqVrJODcoRcCqMYUyiotJ6Sbypsc2YzgUhL/q0wf19t+ZSA5WM+TT+0Lvl5hz+njU/f0HvKV2TT/Bow8NfQhyeXU5LrOYj06XP8AeBvNiQtuO8I5B0i/Ccuvh0pdQAk3xeps2T/OGgWzHXpUCyKp+gSB2QeqavYTv5EZtGMgIRIIdBIBFr3xjKPHRYrLEEWJocw75jBdF5xrjcGlzjBmQ7PvGGzkprXoWbUogda1zO7Z27uszjnrlX3B0tmJNeMpiz4NyyM6J7729/cTm5yJru2928FnmGl2sS9hjmLh6LWUFV5u6lEuWfVCus+ktorXtqKUUcSiTEPaCnUkslaGHNhnFzP7vqiIaQrtG9pLCK1JZEoqKenU5fX2Wrg7ZD56T7zl7uB72+33JUJawNd39LFhmZOiTaMmZxHlC2CEvOEjgOSC/Q9l01HDDMDl1pSSqx6B4FdAyLVc+fWiooH5BySz8kCIXSoenyKg/QIIWIWHSzFgMXoZuwQSF3rGT1SSwjJczJLINuc/9ncV87MiKboVBjGkSFPDgKz554OoUFIaPaAJEJEUnI/PTGsVixP1DwgqnRNQ7suXIYViFByRWvxHNYx0ph5/twCAXdbmHIhT54OL0bXK4zJXzhslmwxBNGEBMg1cztmNlNmzBktE1Kz75MiKSWS+Pn3TU+tkalEdyu1RGgDF6s19a4y3GfCzAhuMbaTMoyVzVDYjJlt9nmoZ+8Zv4XytQCG0/K1sVdfw9h8DWzTZxyHR5s6Q/Ys23fPnTP7nvNJ+xqmdSlfU1925blA/mMzf/ONYMkxupNwgfcAvsPf7XhBPHZv2aEZOd3woMqDru0IyIrtjreZLdzlJz0AsYb4AyckEkZdNOrEwAKiDhL38ZFLho59xxdB3UN9NZt/3wG7A+PscuyCcR23+gfFMMvOYM7txlk5N84O+Ee+fcgBsydH47UDpEdDNB/DLkYZVaExcVauGm1qwZRtAbsbSaGyXjWovaVMI5dtpTOwOQuKZ6KIlBrIChTYbAtDNmKCHIRcAhRjLIVhcr/GKhN31XizmXizvePHceDH0T0hkyZsFLQoq2jEGHem2TxlxnGibu4Yt/d0Flk1rYOxOfJ0Ee329HM6pzozyJ4ZxNnPGYjPUbMxRGKafd+Ka+15AEjCLGDqEbtBxP3WaiYEITbzy4V4cEdVg1KoxX0otXp8rOTsqeamiVIzOc//qmFMhJhm+Z+CEDzv7sxQY4rlTK0DwQS1grQBrT0a3EcwpbgLaiFAteIB5nXLkJXt4NHqyCJd0xAsIMWtzjJfI1aNqRg/3g/8dLfhbhgZp4zm6lHsKdKselZdpE2RNnZoCUgtSM6eQSUlYmowFcZibGsg18pWA7em3E6F++3A/XZiMxbGDFkr+puI9v3Y5dyYnWMNPrS+0yf5L3GOfm6fvzYG7zOWw+Xzzq6ckjHv2+9gp9P9j5avnPnh/VU+qXzIe83pGDz5snhKJz/vXH+Azx97YLZj7eYfH+u7PPhw8vXMCMrJjk+ZqF2+X9gBNZYH/fLlsKI58jTJ7J81Z9pY+uRW212U5tFbzUHfjru6SKvMv9mZq2enEXjUsznd1pIQy5kjm4Fh2IX2Hl4ThxfcAulOeUBZzgbZHR/m0QnoLBIsxpxIbjZ9S2BQQ7MxambKb3lzK7xcN/RL5owATRDPnSwBkvvIDZa5L0IdFbVMnxqKFe6Hge12YMwTg1XuMe4r3A2Z+1y5UwjFKLXSROh7YR2EV6uWm4uISeTubsvr17ds6pa6rcTgfUnRwV+bEl2K9CkQNVNypgaPsPacubb7Z3POWnA/TBOIyV8DgsoM+AK1elDOIr68yLp4zmrBwpIJ2rPAVINJ50Qkai7zMk1M40AteedrWIuRqydWc9mf5IEcpojOKo47YOZsZYqRpm1Z9T19v6LrEzGZR1fbnIvXZk2+Uim1UmpgFltEwfMmu/IQNgt+L35/uRTuh8wPb+94u9mynQq5uFxMSpFGGtraoDUyidCHQKggKGLV9R+jYeI5mKe7jJmRtTLUxJtauZ1G7sfMdqqMWRmLMJY5yOr38mXKex/yvyTQ90hfP+gUPvF5P5VZ+9zlLFb5mH36kLp+SWvwtPzcl7Cn7Pu8Np4n9XLGJY9TEHKmz7uu2Xt3fcgmL/87IbgOuLWdqfh4nR50eNHKO8WQ5pk0CIFs4p8PtdMQlkCTQ17tsT4/HCI7/Hnm4/aahHvi1D+pmT+R56wJYp4rOGogqXr0J3sxaZvB7k5A+R39O4Soe1DofmU65wEWiQ6MxEWqFShiaFWG+5Hbe+N+qPSt0IjRB2PdJPomkWIiioEnHeEv9xPNUBlKpZdMrpXNMDKME0MeuauZIQbG2DJWKEXYTIJsJpowsVr3fHPV8zcXK/7uquNvvl0hqeH1m44fxPiJkb5siCRu1h0XXcMqRVZtYt0IXYKgGS1GmdOeqc7rxJRaPPBhmZVgeArm5H5whIDVQCk25yv2YAethlUHQ6pGrdUZvzlzTTVnD0e1A7++iuVCHka0uN+dzCA84qkHL1Yr2rb1dII1I3P89TJrQVpEjK5tuOh7Li/XXK6u6FYBif7C4ZG6rgtopVCK943o6e2qznl6MabqEeWlKrkYpQq1CmOu3G4H3t6PbCdlqoLSINH9F0uNDNvCphhdU+iikUIkSnUASEHN2JbMZhiRosTkUkDbbLwulfuqDDUwmTBKYMQYTJnq12Ri/Fzll/xA+5TluePyhP0fGpC+fHnAe3yhDj5K3LynPx/lkj0ZhEM94HcS24+RSY/s/9zylKn44uvpwzvw7Awfzy8H7NYJRnkyAXry5fjrYaXHv+1N03uW8nhvF4xW4VhMGWak6qLJchCMssOgRyLTx+cnLOZeOe3tXLXt4jJ2o7NDgt6WGERTkgkJl2MJFnftHAf0eFs2fz6EebugEGR377MdQyoutiwFFXfkX7I81JnxdKE+odbMkD0wYtUFGsusgjE0ia5xUWRRxXSibeA2v0WAIRfEEnUqnuECZaByp5UxJTRFJDZIqQybSnM/crEWvltf8O9fXfIfXl3xh4uWV22kbQLX1x3X9YrLMHIpSrTE9dUVL67XXK4aVo3QBvUQDWGnqaiqznDO415r9X+zmbVBiERqnrNPR2Ym2ShaySVTioNFT0vm2ngVoyBkM4oquZZ5O/sUbgZWZp/DogTDGbwm0XYt/brn+uqSvm0opljNmGa0ZjA3ezcpkJpE0zS0MdE0HW3T7uRjcqkeHW6BakJFqMEB/DAqKp5STedI72LqUddZGbMxFWGqQpkq2yl7lhUipOC+jxI8Ovd+oNaJrous+pYkIyHNun1RQHRm+QrVhBQjRkBr5L5E3ioMRLJAFmWgMFIZ1djk32LAx4cgkve87J2t73Tb1wy0P+ET9clmzMMDPuUT/oujhw8rX2L5fO42v0pPhw9j+c6Vjwj+7JgFfFLf3nFhyREUO6jvYcVHAOtgbHYJ7k8OO3rJ2mEvPdomJzvYwbfjW+9BP3cP+z0jeXSGsgekesbxVeao0j1F6jpvWYQYIyWEOfBkKeGwsztz836PfevLWexAH26qQwqV4oLUYUEq4hIooSVFNxHXoJjsBacR5uwakVyFMRemMqE6EpJh20IphhVcD65CLpDFyI2Sk5Bx82ffNsTRZXdu2pZ/82rNf/y33/Kf/u0r/vbFBW3IpDxx0Sg364Yr1vTxG276S8SEVd+y6hu6FGc/RDcgxphIKSBRXGZlSVc259N1f0WjWkEkEmKDzlk1ggUQD7wptTCWiVx1B62rCCUKxYxJjbF6EMOUJwwjpdb1tPGo5dAEYjRC29KI0KRIv2q4vFizuryi71a0QV3kuxZ0lmOJIh7E0s4AW5wcrlXYThUdq0uwaNn5iS6ZPUpWN+Peb5mKg3sWyRWDrMaYhc0EQzHGYmgxxuLA2ADXmPZGtRS2my0SlKvLFQBDKYgGSpNoQgKBHI3aRCrC5n6gTMo4Be6myD3mIupVmbLOUdtGyfD2N2n3/aqeLF4+20Pv+FX9wxr9GZ18Z5OfGfB9aaDxwKr3GTvyrqY+JuB7Km76ePjqmeXzLoLnRfs+Z1COEc+Z7TOzdFThyc3g7HGyI/J2dT+aNu7cMJ7b1x7s7PhtZtSWz2cviD04PDwNMTkAWo/1b89I7oCzLm0KJQgFgSAo0UWPZc8mnSMfz7W0wL9d93aAd5mbyG4phBld4CnbJICp0kQh9ZF1k7jqGjrpuF6taCSQqzLmAqVh0ob7PHikqxqrrqdMDjwygQnXULQoxEZIIRBLoa3Ki6sV//Dimv/dP/6J//QfXvDvv+u4DoZtIJRKZ0abDGkiXK657npqdb89mVPphbgEcbg2HeKAybNeuM+fhEBMkaoFK4B5DmAJkZCEUutMfUIpMEzZ9QktYEGIqcVoWPzlSjFyrUya2eaCEblqWvq29cwgIRBFCVJpQqAJwf0UU2S16uhWKw9YCROWHfxNtSCqxBRpo9CkQJTgJuhqTFMhl0quxVlIs52O4pQnpjx6OkITNuNELjq7KnpAksQE0jBW2A7GfakMxfMuj2poLtQy+wMaBAJdk7i6uuD65pL1xYrb2zuGzYaqEyEaqUlIGygpsDXjbiyM95lpGBlGGLWhRMh4juMpZ8ailCogLvj8e3lqeQRAPNj82IPkfezhY/X9nPKuNj8j6Pukdf3Mdj47CHykodNn78cGQh9yfh9rTB4jwb/C97BPWZ7B/M1QRuQQ7hyUcADCFoSy//pYncdrzN59b1gQls2A7ID1AvbBxPJYsw9hmMzgTI43Hv2VRc7mzEVwft0c9svFm3fi2AeRwKfteBth7ufCwnld+aDxY63Bx8ZsPwi7MTkShj4srgPnQoICc9Sz1EKZMqUWGoyuCTQiXK17rtc9ry56gsF2mk3CU+HN/VvuRui7Fc068TeXr5gK3A0bhknZTCOTjlhQLCgpQWOFy1b57rrj7/9wxd/98YJXly3rKHQobSzUfIvlwnZKRLlkHRvqMGFVPeI2xZn3qj6j87DUogQRTJUwM56L2T2GSGZOt6aejYLsUbsWGsD94DKgTUugYSyFKY9kUyZzWSGNEWmE1LesopuP2yDE6qLXjVSiGGm2ogsVLZVcK5q3RIwUDI2FiFJyJtQ5dVwWVCqhjQRmE2/eB6qYwZQrw7TI7WSXdanFA2DahtgkRpvIuRBTpO9bmq5xseVtwfJEmYSyy+2rlFoo40QtRowN3aqlX/VcXHa8fHmDRHhzl1ASw1QwzVjIFDG2otxp5W6q5EnRsVC1oWJkqYw2m8CXRNcudPnOl7jfTHn0QfQ5nsCfcvw/1ZP7ucc+wkR88vIlkcXPbPuTX5an/TtjyTsqz+zQo6e/e8A/r76fXR67ln/uGnnePeJnJNNc6LeHEycwS6y8uyOnv567LA/Zqr3RkvNvpfLI55O+7Wk6Odj6jsk4MEGbnG8a9tbevdDySVVHTbzrjXzple2Oe7do9wnqXbadbeNk28K+Lv0Ong0jqHqKutCw6ldctR2XfUOQwLrvubxqCV1whmicGIeJu82Wu80AU2B1fcn1zZo/XLxkRcPb7RvebDa83lTu8wCp0Kwa1pctYTNxaYGblfDiMrBKSt1ueTtmrK28WgVS37AdJk+U1gXElJB0ljbxiOUgDmIliYsLB89dXKuzgXXOmGLz9BQ1TAJmzopVy6gp2QqZlhB6Ii2Is4hjKWQrFFOGaSKjSGzcvNy0tKsOyUbeTthmi9WMThtKI4S+x0hordSaUZ0QK7QhUNoGIlgw2uCZZLrLDgzKlOf1GSi29wios2l3LKMD0qpM1c8xxoamW83i05EuROKKmSWtmCm5KJtx4K8/veWvP9xzNwqklbOkqGcAwQjBiLKkzRsZN5nbu5YUW4ZhwIJQQ/Ro3VGZVNmY8lYq92Zsi2saigpCoc45n2ebOK43Oa/JZ+Sl/L38UsunALG/l9/LL7V8uevhedG+cAi/9vbHEyC4YwY/4nnZccuP7PPErbuUZ0cbHx5nh+d2WNM7eOIDhk1O9z9l304B2076Zf//HQhejttvOOjCuX6cbFum6oiVlJM+GLOTHlZnOZIQCJbow5pV6mliwMx4+3bgPlW0VkquZEBWa27WlwT1nLmNRn64e41qoeYtDcYfLyN9+5L1RYI+MeSJt+Nr7u//R6T+iX/zp2tCo2RTCgENga0INQD9CmjZaHRduC5Rx8IwjeS8BaBNgS5BtEibBKk2Z1RRT6sXW0QC1YyUK3kcqVo8WKMUhuoRwlUD42TkPDBNmaHANqvL1ORM1Yo2idCtSa3RxEQiEBWkgoaGBiFFQ1OLhQ5Comohm0c/B4PYRIYyQFWiRPou0a3WSJvc/y7MrwEVB6lVGbOSFcZs3N0O3A1bigVi4ybkFFssBEJsKBpQEsEMCcZUJjbjhrvNhjdv3/LTT294+8MbhnrJxYuerk2E1KCWKNH1A7u+oekaFyxHmWphKJntNDJlJVukBMgxkklMamzJ3GmZI6zBREF0f+2puazRksrm1I/ht1a+Rjz0s6bj9PX9o1X8AeW9rMBnbP/j7fqZK/u45X3r/dzykQcfPlJ5pL69f9T5Xd9rLn5XPz/hmnxidc/y+dvNxy7DxikqOjMaTwaAxzvZwraZnfmVZ4zXuwd5R9ad++UQl2EHDOHRDw/692A4YLHVvqeryxiHuQfe1s7Ed1Tn3IclQPJ06M9ShYegXQ62HdQVBCOSUiDMDNBmypTpR/56H7heNazahElBrDiAMSFIpEokqwEVscpPtxve3r3lzf1PdO2Gq9Rz3V5xs1pzM6y5ulpz0fekyxf86fq/p2962pj4/vu3bBS+veipr1YMVcjW0jYNQRM/3U0MpVC1sL295+3rH9ncvyaK8fL6mm9erFmt1pRUaYPnwG3bSGoSAdciFNU5qMIDQEqp7hNYDdVILpFxUDbbkdvNPffTxP2kTFMlF8/SElcXtGulaZXACKYEAklcviV0KyRFKsY4VqIVoGBkINC2Ddp1jGoElBIC1IhlQZZglOJrod5Pu3SA95uRbMI0VraDMk2GBQjR05OEOUtIrjBOypi3DLWwHSeGPM0RyRUNK1ZXDSFeYZq4uXlB2ySqQmwSKTWktkFSQMUjo0spHiE8FUwal8eJiVATMQixCtRKKVvUFKx1djk2D9fs4UuY1d26/738jPLQr+UZ5eRm/Zjx4MkdOfeS/XPKVwxmHi2/xD7/VsvHfAM7BT6H3z/hmnhG1c9m/g5Np8dg5KSc4osPHdczBJacDuKC095DyO07dXxXe5cBdl/lA/rvpFeHP83AVdibgNkD2mVnWXT65jMKchwcIrteL0D4MfbxxMT83hVwDhCLNyCz6LM4SHKBXnXwq4aOxu20RcQzxoo5oWMIKsHzEqshVtHq2m2pu6RrL2ialr7vubm54I+vbnj14op1l8iDMWzfsNlu+csPb8nThl4ib26ueGvC5UVH3/UwCMkKf319z+39PTmPDPe3vPnxB4bNGy7aRIwN/aqh6YUuRKoZpuZ6fTiAgcBUPNVaKYWcR8acmcpIVmWybpYegdvRuBuFbYkMGqnSUJNQTRitYcquxejT4JlF2hToQqLmQNRArQXGglCQaIQoNCGi5mbZESPOun6bEdKm7qdDfB1EQKKQmgYNEVNBRT37SUjkUtjmiXEYWHVrVEbeDpXNWKlV2ORp/pcxg1XfcXGxYtUHxnYiSuTlxRVFK9OU6VctqW1IbQ8xMNXJfTZrZZhcG9A1AAPFhGIeQVzU9QZrnRerLObdBIQDxlrYuxws+/7+oHxQju5pX2B8HgDAL2Gm+tDzfuJxn+GZ/N7yNS79d5G3P6e+x+p8B1/xRcpCoDzmE/jOvr1r0H7WW9Xj5ZlVPtvnb3fpi3lE667BMydrcBQEclTB8Y77eo/2fuyA4zbtqIaj3ZesFvtevGcln6/m0R2d0AsH/V1Sq82m3x0APHP4HlLuwelJ+8u78zI2AnPgscwE3tLeceXHo/bICe00Euf+SZg7olQBMUGX+ZizkuSqTGRMnakJEolEF4WedeXQiqoSSTTdJes2sO4arvqeP171/OnFmpc3F6wuEipKLQOTwVAq45iZpso9So4j93JPezvRxpE8TaSqbLd35Dwy5S3D9p7727eEXOjjihh6UrOiSR2padHRQ2XUDC2VWj0F25AncimUapQKE4kJYUK5zcJGlfts3BXlrhqDRbIEaopoDWQzRhNsqlhRDzNRIaG0MdI1RpM8CEqK0lRn/mIDbRtZJWGsla2OztqFSioVRRGdwJQouDm5aYkpkZpEN2cBqVUpVqmiqBhFM6VOBCKiganCXYYsiW69ptMWyyNl2FKLZxTp+5a+b1mverrUsAo9wzjRBOi66HEYsVJEmWpmKJmxVjwsxRnpccq8zYWhRooKkxqbamQt881T5r9x/ryYeI+X5pJz+/dyrvyMcfkkwObB5H3Myj9CnV/DOvrUfXgfMvsaxuCkfDWuDV/h2HyB8vz0brtPcgRGjkHaKRPFk8ZbDnc8edt8F8Z8rJHzTcr5Hezk76NNHNe6yKXsgete4uV4v3fbYx6Ftwcp4OzEIV4OzGZnX5p2mO90TM84MSw7qwdEuKKfYSgEKESiJOpMPyrFgy9oEHHwVxFUjBBc3LoJ3ewjWJlq4n4UfiRjesftkGlaIWshbwbKdsM0ZXIuqEFKgY1V/rK5pxYXCTSb6CUiTHQJsMo0VpSGi/UFly++4cWrb7m+viSmBERSNMQKWN2lPXNtOc+rOxVFRciWGCwyWuXelJ+myt2k3OfKbVG2Clk96KJUI5sxYRSBYkbR4JIo5iLTKZZdyrgWYRVwIekKa4tcWWQVjVUQQtPRtNCFCZ0yVgqmQhRIFmgk0sSGloaSjWE7kaeJWjNWC6UWqlWIQtMmrI1YEdrU0sU1MTVEg1Rb2ETyNNGIEIMD0dA2dKmlsYQEoxSwoOSaGXVitOCizxpAWooqb7cT95uRYZp4WyqDBcrMAk6z4PXev08OFt/yInTuevj9prwr77AwfLlx+lzt/r4Onl7OgcDfx+/d5Yko9IOigE/H/hPMy6PY6qkgxsvzdP72BkhvYglsOGSQdmU20C5WnsMuHdXF8XF28v3BsQcs2eH2k/N9l5Vk19PDe+kHWjMEdjqAe8bvKZN8eiN/5DiZa7XTLr6jjXNjcfTTXJMs3w7mcwcYjSXXMRJQnJkJIc2MYDNr6bnJD/OsFgY0EmhjQyeRSGDMmaFMbHTi1jLfx0rfR7q2BZyRS1bc906NKpEmJeJglDIwbgfKeE+XlOvVmnUXWEvksu+JoeXySnh1fcEf//gNr/7mJRfrlmmTMZSmSVhVinh07FBdzHisgalUtqNSRcgK91Nmq5lBhduh8HpTuBsrm1zZVmFbK7ka1YLLlgTxY4vNengBV2Scxy5ETFwepw1KKy7pcpkD2z6yjkqXA21IXKYGN7knVJKn2zMjVqGESC0thUgpW+7u7pkm9zEUgCjE4ELQoe2Q2JEkYNaCdGzyRDGddQ/drF9U2U6ZODU0UYCARagRxqqM48h2ytwbTJIQWkSFMQtvNpm/vt1yv5nIWtkGGINQsDlV4OFb4WLqPb3A5OTvV0ML/LLLJxnOx+41v4OMDyrnSIcPOvgp2z+g+nfxFD9nXf0ml8snPmk5/XKGeHukPIP5O0l/drDl1OR49E3k+Bc7t98jJuOH1b2re0fV7Mz1D6p9BxJ/cMw5MDZ3bgapO8i2k4M5BbLMgHUPhI9Mwo/4PhyNjsnsH7ifg/09/sSk+2RMePyU2E/LYga2PSiez1lRF50WYc4uTFWXRvHswAHE89VKcDkWEd/mONJz0I6lMA1K0kDbJFJsICSUwFSNatHbWcSaG6OiWGs0Fxes1j1NNC6u1jQBblYNf7i54A/fXrG+acAUDd7v2AZqdZNjKZ7SLAejWGVQYzPBWDPF4Hbccj9uyCZssnF7X3g7GJNFCo0fa+rmbQkYgqprCIpCMEW0YlZQVSxE1+Erlfs80kRoUiD3PbltSHi0a3Mx8mK95jJWOsDUwVgISghGGgtNMEKAUjdMgws4x5CIIZJCQx9akEQpkVo9j/HdODHWkTfD4EFaIbiMy6whmIqyZSKJkUKhbRMFGKfCNE5stpmNQgm+3utY2U4TP20m7sZKyUIxYWxgQlCb9SKP3shkv6DPXvs8YNN/O+Wp521Pvx9+Egz9qRnHj41Yn9nfT3JaH2PMvtB18a5mn0qSnE7pb/US/6jlzCCe5YyeNtgfpPP3YF4PaTb7OZew7HzAH2vzlAI84g/ksG07Zm0fmGvP1H3w2yms3XEXtkA+B2WPVnJu24MLRx583o/t8v3haB4BWDvZYnDoy3cMuM+VA9rzSJtxQcIORNwK7Dpyvu8izOvsoMnMJApk3NRaEKQq1cRlowVE3EysImxVaDQQzYjmfnkpNfTNii5EVinQJ/eHMyt00XjVtlz3HdEyTQPrtmHdRLq2Aw3c3WbqNNAK1L6hSiR0QhQlZuYoXw+CUKuemm4Ai7MwtFWGMmIEQvCsGVMxmr5jlRoSUKSlmgsukzOxVqJVkimBgulEMAEVqoFpRWQimJIkkXTCtoHtVBiyUV+/4U17QZsAC4Qm0XQdbdd4vlwUNBOBKEqSnhQToQYaC7RTYqNCnGTOAOL5hTdTZsjKWPPsvyeEGAgiJIk0RLZbA1NCdYBYU6RLa7TA61x5M2bup8Iw3DMOGa2+FKbQYObuAVkEXTJ06Mz0mXm2GHz5HK39Q/eFxS/w/Qv1V1Y+09PwYzTz2R7gXwghfEmA8rWAonNczM/p2+mlfAhSPoEl9NOWdyHew98+1sm8D32faf8Dmn5WtO9xxKvM8QFz1oQZ9S1Z4E6Nujt+6Wwn5czmRwb8PSf5TtbuxGj6LgOtnfmynOe5486dweHBdjBJCyg9xMyPsiKPrDnHfI8tyL1x9+jD0u6DH0BYMoscom9BbAZ/MAd51PkElF04ajioS4yKMVCJOENYzZAQaSWSNCDqkiVTLTCNUMHKRLRAFyOX64EXFx3dqsWiG1Hb2LBuA31K9ASCwHaaqFOmNBEtI1NJdAFCnli1kc1Y2bSJECrVPNfwMHoWjKKKlorpxDQp0kRWF1dcvrzh25QYNfJqNF7eTry+GzENjFm5n0beDgPDMLHZbGhEeXXZcL2K9AlWnYtit01LJ4FsEdRoopJSJfYdRmBzP/D6pzt+vB35aauITIzjxFADWhLjdouFFglhN3Gr1NA3nndZrZKSkERwEUDdBeUs422K++kt2U8sIMUIWolmBBNEjYDRmSDRiE3PqlW2eeDNOPEmK/dZGEehZKGWSghCSbpngK3u7gkIc4pAfI0YEJeXioOrbvm4e4H6RTwFvlx56vB8kmH8FJV+5DrPgZfD377Ie4Uc/Xl++YQo6Utcbr/od7uPjY4/UjnL+n105m9JgzYzUjOCsPmmfgoodvGuRxfeIXg8/u1hQMQJgHl6N/cf30WH7vwV9wzeaeK6w1PaAT47AXrv7aA8+P8REF7a3tGUB+d9MiTL+cjywJxxge1OZzn2ENS9r6/L3NlRHX7SsmMRXWtwmaw676Pz8TIDAUDc88/ige9XVUwEDRFV0KwUzVSpNCHSNg1d6BCEaEKKiazwZrvlzf2E1UKbEi/WPVOX+ItWdNyiltFppI1wuWq5uVxxtWppKaxS4vrygqv1GiFTpg3T5Dlvc5mcmEo93776I3/3dzdI2xE61x6+3WTe3k80OrK6TPyh6xjGiTf3A6MIL8z46+2WsVbaPvNPf7zhH//uO169vODiouHiMhJTgw1uFg5J6IIisXju2xC43WR+/MuWv74e+Ol+4m4ofP/nv/DTbeWHaaSUxKSVah5BLSJsk7HqI1OGqarnIsZN7iEAQagYFoRmznBSKuhQ5hcXB+9WJqzmnd5ea4kVQJuooaBsyeWOgpKlpaizqi7qN6+RMl+8jRwuTDA9WBvLy97BPsvbz/LiEg7Y5t+S+ffJD8Lf0Jg8p3zMYfnUQ3wOjH6p8pS2f19yJ+V9IPxLD9g5SvX95Vlm3yPsa/Bh0TCHlfy8w39e0wuoef/EHcKyPTw8hHNPKAdA15+HsmNN97vIHGV7si8Lw/oIV/kYa/iUbu3oxz0IPGQUd/lXwaN6EZA072+7/WV5wAvOCAu785uiEBSSedCDxIyFSoqBFAJJhBQiEpJbCEMgJvcbxIAKYzF+2GwZSqRFCdVomxaJgWHcerq1bGzuN0jZ0gfjolvz4vKKVy97LteJdX+JcoGaECSRUkeSjtfbe2y6ZftDYTvesx0nxpzZjpXNdqIUpVt3/MOLC9btFRIS9b/5ExqFbTD+/sU1f/tyxfU60ETAil8aSWmDsFpFmiYCEQ2CiTDFhjex4+6l8npwTcEf/vZb/uWnkf/y0xv++Yd7vv9pw2YYZiu7+1COg2Chd9N5KSBGahIWhWzmpmjzlzKJPm/ZPLd0tdn+qgIkYgg0oSEGXIga2GJoLUiIaEhYaMEilACpgaJQ5rzDTcJBXsTR527RcKTht7gOLO4IJvu3G9X5ZUKB9tnr9xdfviIS4fOV39TJ/l5+L+8oX4yWfh74O4Ue+0jexzt/zqizz4HLI+d9jGTP7fL47eN9N5Z3ofj3I+hT8+7+3m1HBMb+8wELt3v2nTlxm8Hdme7L0afFZHYwG3Li3/jIA+W01X34iAdz2Dwx+/4bEoQYov+dc+eGBSwuRnBTf4jvCEmHseByIY36tsaUPiirBE1sQJIHR1hGdQRrIAZMIsUEK4ZOI3XcEkOiW3UIECWQogOHNjWkGOmjcL1KdI0xbQvDNGHDBtVMam7o2hdcri9cJFnNAzKqkje3TD/+1XXyimJBuGhbbrqWocvctc5uvrq+5OrqgrbpCCFhIRCbllXT0kUh2kTYZhqUUEc6mc3jFiAnNIGESGpczplJSUOmy8pLjVzHxB9frfnTRct/928veT1Wfnib+fHNwNvNwDAVihbup8LdVnkzZAqGSKIaVMTFuWNw7b8AIXhauCD+exBnc0MMRDoaiUQBtYqKMkmgqIC0EKKDMp0DdtIctJMEYoIqEA5MuIfsXphzugXxtbGATlnAInu62jsHlg8Y5d9Q+TXjoLO30w884Q8jN75g+TkT+wlO9te8zj5r+VhzImc+f2jdD7HEU8qTwd8MOw6Knvx60BE52SrLLwurJRw7vh2WPb+2fDs+N3/g2Ml47UHWKbh7ZEB3dlU52OU8MDzCaidsp5383ZtPvQ4zOwKAe+C2jMcCoezwp72MznxY2Gn67cdNAFuA3wHg3I//w0UgZxaaAUECITg7IzMrGSQSQyTGCOIyJqUUdhIjhycVPBuIYG47nddBm1oED0xYS+SmEa67QNMmSmowMZoEFoycjbGMbMaBbR7QUtC8pUxvPOak+ZYX19/yh+tL1ikRUKiVZJVVDFyvOm4uO0p5Rc4TaCGJ8uKi5/Jixbpt6JuGEKBqdbDXRy76P1Cq583VRmi7FmJkqpVcKklg3SRSSLQx7q2cElm1QtXKOE6UcSDXQhMqYdWiMVBKJU+ZUAS1DAhVzVPJBUgWMYwkSimVSypBI9+0gX+46diuEttpzVQ9QjlPhbf3yv/7hw1vpomCcD/BUGAymIARo4RKbBrHXRWCeRq7ZQkJhkZFg1G0MNXi0brSQvCIaerMygkOBgVn+BSCqLPUqUJU3B68MIBz0Mfhm9/OUiCzu8J83dkSHWwQPij+7JddnsX8PWHnT8EkfmhdD269H6FTP7eKk0fVpytnrDRP7vu5jr3P9PgFy1O684sB7R+5POl6/LmD82Hs4c/I8HFotDzvs2Onxzz45QPKqV/QY+QdM7h6tKmDvi/+iw9+PwVtj3RpOeK0L/awu3DMDC6V7siTGYwesm/vOoPDbCI70HfS7d3+8wNYZihvB30Jc2Tm8tdPVx3HqZIkomaIKaaLr5+DUk8H530JuFUxAtmUIIGIpyfDYCzKJigXKdCGjtQ2rJKRmsqkytvNhOZCkIkQM6HNqFQkKC965UWnXDfGqjHaJKxTz7qJtEHoU+CybwmhdWCIEilIrQiFvL1DB+9zCJ6/+OJijfYtZgEzpVBdnDnIjtUUrUQ1tOK5LcyZTTVBx8o2jwx5RLUgVIoYpuoMY6lMk1JzYRgHSoGm6+hXHVggq/J2u2UYR3KuFFNC0yApkUJHoGWdEl2TyJawpuOyhRJXfL/ZMijcFmUzVbalMmhm1Eq2SpFCahOoIarkKp7pRA0TNz9bMEyUBmdCDUHEczorlSA2B+3uX/Yk+LU1hYKHUFewBDT+dwF2AqJ+DRoBYvS0gBIJIbCsDEu+3urX9lD7pOUMOHh0P3hUu+qTla9gLj60C+eOe+q2j1k+Wf1PQRSf0J/gQ6r80szth7Z7+tw/hwPkdIdP2aEnlCe64z3/VXthnE47/wD42OOA6R3jczaH7ungfrK3t/cgaDnd55hbE7fp7rt2AsaO8N4e7bGHWxwcYO5TZ0sgxX6/B9p+j5bjB8yhzz0s8dhh5/e3I0MXLtKgUqA4o+SAzwFemI27y3/+KA9E8aCDKEK1TK2zXLQqtSrZKsNk3E+VVa70bWJrmRAKiKI6cBOF/uqSPghJlBhe0abEN+sbLvqeXgJBKinBKiqrVuibQB+Nix7aGNwiqe7TWbNLoMhsZQzi0cJNgC4GZI4oFgLERAiBKgW1CmpoUTRnLMTZ5L1I33h6uJq3WJ0wqw5Ugay4BmBVTI1aM7UUzJxJjU3r45szeRoYN1t0VGgEacOcyi1AFQatTEUZ1NhW434o3Gtlwk2/BGiSYlSCKr0BGtgihGDEYKyITBaZVMlaPQcv5un1RCBENIDOkd0aAmaNM38zkysCEt18XKqDf4tCExJx9umT+WXBzNO/BTEPWAlQG8iqBMHnFUWoVKtEoMb4vgX9Gy528vlTIYtzT+nTbU9s+7OxbD+nfEKQ9MnLad/PDPKXPq3TLn4YSfXh7X7sep5b72cd/+e9ID5T6uW0ndOb0IEp8cCsa8cHne56CGdOGnvChXnUr4d9OWv+3lFeD6p62NYZNu/x/sgRUBU77NxeEnsfM31yi92By33HTRykeXcPergM/YF57Xyv9mN4fjRtxzYaeICH7EGsg0CbTbrO8pm4l+AiXB0k+H6iVHPtN0UolilmRIRkzqJpLdRaUZ2oZKZB6DTThEwTlYsu8uJizXdXL7hZr+i6QJOMi67lm64FgbLdkvMWrDLle8YxQxNo1j2xvSKFCLUQFEothOrp3VKMs6h0IgQhRKGNSpSIakWCEVKkadzEWoqzX5rcf88ZPwENqAYmNUwrnSgpKmVOHxdNiFpJFkEEa4Izk03CSKS2J6WGaubAsG9ozNCkSAqEviF2LUED21yYpsxmUu6y8eNU+MvdHW+HyqTBg09iJImwju5TqaZEi7xar2nFCNNEo8JWegZVxloY1RhNyVTUIFgkti0htUhsaASwjNYJLcWNuiESYiRr5W4zkoOxXq1YN4E8FcZpQsuExAChgxAIS8aPmKhNYCgZUxBVwhwYo8EIBA8o/k2VZwCPpz5EvvTDHt7Nsr0LU77r+Gc1/pUgzfd147PP1VcCdJ/b/PuYty9ddv15z4l99GHfW1gew1bvKk/3+ZsHXWAXrGfvPJvH3xYfJwRP63v3aD0Eow83yDktPDn5+4Q23jm/R00cnN3Z+o8MtfM5H3yXY3OvHNT31HW/r/GQTVxM4If9O2ApF7A8p3iT+XdZ2EnxPQxFd2tO0Hkt1JkRhFkWRkFnKZgUAgkhBgUpSM1ENciFlBJrgT5U1l3iu6s1f7i+5rvrG15cruk6IcTKZRO5aYVSjSE0jGMlTwPf392zHe7JMZDqJb2BNgmxyqrpCeaMVQhCkyJd29CmSIyCRGhaJVlB6wL+3JycVMnmGYwJ5uLIJIQEmigqSMmIVRpJGFCsOsOIIEEI0hAkzP5yzpUa0f+J779qoG+vKJdGmTwziAajAFOeKGOhTJk8VcZRuR9HXm9fM24jbXfJqol0fUcfhaCVXCYm9azLL19cs4qZfAv324lsRh8jKUUaNRLKpM5eRmvo2zVd2xGayFUTiTJScyRopQuRNjZYTGxy4X61JaTM9WVLmwI/vL7l7nbEQkFo0BSgSQRxgFoQRpQhVEqps+C1M8SCEMwlcX4vTymfYZweuXc9/5ivvXxqBvU3Xp4zDIdQ4bEH3dcG/J5THrzknKCL92JzOfN5BxgOfv4UZt8l6OAAOxzLleylU5b5OxUzOapqt/U4Z/CzunNQvR38/2GbT4dN549/rNZn2rWPmnmIQvdA/rAve0pkxwDaXm76VJ/wSKplAeoHNue919/yvz37t+/bnPFi9/vMQsoBmypLoIrNQNEPV6uoCVGEGKCJhSSQ1BCpiGVCKbTS8IfLjptWCFL59sUN//DdN/ztzSVXXUeXhCQVs0pjlcaEiNKtA7lbs90Im3FFjEIiIBrZ3A/ULtKEwMUq0abo0jNBiTGSghBnMBiTy9UIlRg9WjkGB04mQgx+sqbLubmvICGQ1OOZg0TUOtQaOplZ0GoYuguikRCREGlCgjnzb9XCVJWiga5dUVQoxaglM+SRsVQqlS4KF11DTIHQFHIHU9vRTRfcXL/iarVm3SZCrQzDltutsS0gJG5iBBnYtpXv7wZ+rAPEnhRaLAjBIIbg6elMiBiBESvZl0Ko9Fboo3DZtXTR/fP0omX9zYomZmpTyCa0oVBWlVgyWgObmijirgExBO5z5sfhLWOeSBK46jterHr6tqEJHjPSpebd181vsXwxDPEpGraDao8eALufP0obDz4/TkR81vLJWKt3VXp4zu+iX7/G8pUwlY+Wn9u/n7MYDsmdk3l9YneeqfM3PwLl3PYZVOwQxSI/crzzh5zuo+dyBPw+7SvB+bN4rGen9uRjaOu5es8cc4zIjgDbkUl43lcWkHbS5vEnedCjo6oONy7M7hKNOXN9h8fv8OHc10XgeQFH7mfnvnXrmDCpyOzK38ZIDA0GtKnh1arn33xzyVRGMONytebF5Zpvrzuu20BLJWrB1IWMkwRyKNAkRBNNs6Jd/y1TnsjDRB5GtBQXjW4SqUk0KREC+3+C5xzGIXWYzdGLGRv1PMJV/UxjbDCZ8+GKYYuEi+Cp3IIPWCQSgkdHmxouW+KZNkIQQkizVM6SExlCDeSiRDWiGG1M1Gh0bWKqSl2DScRCogpsaua+brktlVTWXDeXaMkM2w23m3sCGyROtFI9MKd8j4UCTcWaEauFcXRJnRAaJLhZupZKySNdMIp6nt9xGCEUmiiklBDLkAodxs31FTd9ZLMdyJq5vLngH7/7Bi03TNvM/WZiOxhDruTtiOjEj9MdI/+FMbbE9gWvrq/5h1cv+eOLSy5bY7sZCPpLfq1/Ztm/HT/++9dePnYf33c7/eD2TtfV5wA/p5aXR3b5lF0429j7tsuZ305R64fQee8rcr7ac8/JBVec/vbFbh/vmsgzv507J+AMKDiz38dbLM+P9j1gGY9RhBzFZdgenezna/lyDko9srbe/7L0jl/P/PTc6+2x2uX0y67iB1Tk4y3L4eEzoFsWtRnM0bbLmB9CyKMIX3uszYNNtoDzvSl3PxMLkvPPsntDl+O5MdsxiXsga/vAFlUaHBA2avRVyRh909DFyHXT0kUPtlhF+ONqxcvrHo1K01a+7Tr6VlBx06VoIZjSNoGmDYgIORueT7jQNi1N01O0p/SVMha0FmJQUgi0ceUBHrNvmSikGAhL1GlwE7XDvoQZHshQwUw9ijk4e2foDGwDIs7qpRB3QTmYB0KEKG7m5ZiPNVxDEIyqFUPd9ByS+weWglkli2KSaCW56HVsXLsPQyVRZUXTCHdvlVUQ7qfKXTSuu4b68tqDUlJD0zZsFUjKXzYbphSZ/vIT3/90x5SFkFbEbkUMkRoKxSrDVikIf3p1xUW34nqduGwDfRJWQViHwIuupe9hzFsuL4XU9vSdkKJgTUtue6YVlOzp+4Zyj2b4B038YxBGvSQ2l1z2F1z1PZdt4jJUchspZXy4eH8vTy8f47nwIc/1Jx/zS0C0H6Gcw0+/6PI5TuIEALxrqfySltHZvn4C1H+IOz4283fIH8lBQw8/PaSW7exvBz18H0j6KOWYfXv8Teb0yn3uG8/5ffZn7vUpNgOGZbvLbMTgkapqS0q1mSk8Ol7m384xiMfFW6vLzGHmEZ2ie9YOcBMnMgd3eP2+JRBEUFOqd2Z/QsHzyJqpm3VjYGWBYorUylgKF6sV//Ttv+Gby8RlX0AqU3bpE0V5o8rf//GGy1Z5IUaMlSIuuSI6UEphq4FGEkGE7ThBNawKE4KEjpga2iaxvmwINGAFQWgCpGDud4YRg9CEMI95xUx20a1LKrud+dug1opRkSDOkhlzfmMB1Nk181jnGDxbiQAqs2TKIfDD2cMlTV4MgkTxsTIlJQGt9J2CVmox1NRN5BUsCGqChQTbQmOG1EKb4MV1RwhrQghEiTSxIQvc4Wwhavw338E3q4Z/7m95c5shtFibuJsyQSsk4+7NX1mlFU1zw9V6xWWf6KIiNlLLiEbldmu8va80MfLN5SWpwDTdM5kRmxaJLUlBrGA60cZKSIHUXSLNDaQOIc1jJrSh0pSJUreU+Eu6q/+Kytc+7F97/84WOfpzuvlzNf9xKnkK+PvEAHFX/Yee2EfEFbuqnlDXuWH54C6c4pTl37vY28fLM33+eNDQ+5k5zgTW7hDPfodPsXjeV+0hc/mk5t9Buz5rQt1PzuZkbg45qsuCWIQQicG5I4cRgRDiHEywgJe48688jMbd98M/zJDSySnbw3AzZ7ea1LATa5EwYzuHLWrq9YbgshxiFJQoc/4OU7ejEhACSEJjRPNECg0iGWki/9P9f+V/2oyIFBqBVUg0AuOwYSVQ49/zj3+6RsjIdsuYI68uW67antAJhMBARacBEYgR+hRpghAjpNZoGgUqNXue25QSUTyaVjVjpphEVBIi7mVqCyN9YNPWA5raxMGwVhc8llnTcJkxJPrvjgoRszndshzfGMyjoUOI/lk9FMbUZVACcQ42CTQJpCRKVmp187FYdVkYiZ5pIxhJivvmiYsvhyikICQKWCHXSsRoLPCiVbpvLvn3317zH777E69vRzZDYZOVH7YDP95P3A8jP9U162bNVSts714jg6GpEuuWId8xSuayW7Fa9bTtNW3f0SCUEUoe0ToQUiZGoY2FdTJSbEhAjIakgCabAbUD3i4KnQm1QAz9cy6i38vv5QPLJ3re/GrLpxqvJ9b7q5yqUyD3c0/yOTjGy7OkXvzvHskt8Z+HbzmHvnd7rH0Qx3uovSKnex4es9T7/rNxy+cj+50FZUc9enS/8zWehbH7Xw/Z1yNy8yA2WsyjUHFw4vzabAIM0SNLQ0CknQWUHXQM00geMwUlBdejWxo8ZPFsZg5VFQmCZ+cA0zJPlxBmbkoMF/RVD4Ko1VyeQ122REIgmrN/LvasdDiA0mCeGQJ1bTyMrRqrvufv09/xKq55a3/mL/VH7vM9tU5QKn2IXLYtWgr3wwj/v/+Z8e6KdVBajJtVz/3La17drLjqG/o20bQN3bqnjdXlSzJomcAqWkeqBJoYSW1AKIRouBCzEWJZ+FJMjELYS9Ts1nLYrV0xXPQZF362JbhmBxaNMPv9ue+j6+UVcS1BkVnAWIIDdDGStER8jIoWSslUCkYmhkgTDQnJwf+89DVkF44OEGMiSiKrUqQQWyO1DabBM3fMOoYSBQuFWDNtcZ9D6eGyRmJMfNMF7i86bofK3VT5e26YEMZs/PWnO358vUFsYJtvSRpozYNkQtPTho6uXbNe9TTNms220kogVEHE/TsBwnJfMAWddis/l4rFBg+Arpg6mzxpIEWIv+V4j18ku/WlyrsYqafaXD/mgP8SJu90zJ7b509xju+zqn3t4/phjNsBmDqo53Tbh/bHngKZgGeBv71O3W7Tgy2+dd+RUxFk9gDxERr3vNzLu0VlvHuP7fFuWtR/PRz8fd/OpUd7X3nQjYOmj3CvCdEEtYqJ0DWJkBIq7ju2pFULKWLqTvngPlFR4pzuzZnDIwbzoB9xTknmFt3g+Wu1EgzE/G/ASBGIiaYJs/+aR7664VIIaiQgmJLrSK+VKg4uW8tUFNOAiqIhk6XyWv+Cdj0SsosupyvnMWvG8kRWZdU2qBaG7Zaf3hiyXtH2KyBxP2SEyubOiFpZd5Gbqwuu1w0hJUJRyjiBZZoEDQ0hRmIQtM7nIbpzURAJeJ4ROUDotpswlTJnHVQs+PGqnopNa6WqA0mbJ1LECI3nPBaEKEoMDgpFPIuGg+xISGG/1tTN5M4XVubEthgRrXiksComAY2958kVwyRg0niUSp5ZY0+xQpzXgoihZExHYjDaviMQCCouBq2ZNiTWnfAiBYpGlIQlKBXur3r+9foN//rDX9lEIalw0bf0fUebIqvUoNWIIdCENL/A6A4Iz9E0nkHEPIWd1YpUJSUFCTSdIaKIFGqoFAE00bUtq/SrfMU/X35Dp/rxyrvu8R9y3EcsXxtOOTJS/RKYzneA0qeO7TtP8V2VPBMQP7rbO8CsPLbfUyp+N2Hmb/0LaPHMVk8pHyGZ5rKwZP728AROrcWP6vk9SqOdnPwT5up4t3dP6tHvR8LM86Z3fDv3RrVoCzowO3+0CSiCWdwzRcG18gR22nTBPF0aAilGQpeoNTtGNZ1Tth2egXh61ZkJjHM2iyUNG8EZvR2zJ+bgbDZArvqei94DM3Iu3G0Gap6QGVwkhFryHIiitCgSQcWYtFCrEKiMwbizhlUjBJv8bxBS0xBTRKzQxMSEEnMgppaUOmo2bqctw/1b3siI1C1JMzcXPfnbP8CLa2q/pguBLrW0MTnoiu4TJxW0AhYgzMMvASy6f98CvuZxXRhAUqDKzJQqoK5hWAWqzBHA5kEfSvDI4WpoddY2CTTRo3Z3a0qAUKGCMaEi1Fqpc/ROkIBYIs4Uogf9uJkfIkEi/uLjQN0wVAPRZBacxsGXOfjzc/MsIxoMJJIIBAqRiTpL8Ij5uvCJa9DiIi99rIQ1dDUy5iuCJFarFU3Xk2JDHxpqMaZxRKlEnfYvgCa7F4A6i3nnUtFcsOK+nik19IBEJQbFolHnXME6KUwTcPH4hfqrLF8BavhQMuiTlY8FVr6aE/qNlXPM7GNs7btY3CeWJ7tsHbb3KcvHauM8EQbLHznevGj5PrH5ZwZ8HJJMh3DjOCr07DB/ypePMybfQ67wfeTyeyp/5vbTcry4HQgvbOYSfevpwspUqDMZlZplFitqSqnOEKWUUPMsGaa2Y+iO8LPMx845VitKMY8yLaazqTJAdIBpUShaqVOhlsI0eK5czCgloyXPciuBi76j7XtSigQ8IEFFyaKM6tHJoY5ElCb1rGMkihKimwGDKkmElFoHFN9+S8R4cdnSmFK3WyxXN1GLEkOgb3rW/TUX3ZpVs6JPrg3XJAhSITiQ0+qkZbBEXRIML6vT3DXA1NOt+WzMkb4CtRqZ4vNiOCBRRbXOPpU6gzPZzeSSJ1dc8G9OjGKzpMzMzhre5mxyrlp9/ENA5rzHMtP1i2l6WS+yZFWxGaWKA7y4EG3z/mI2m6oNU5lFpQ0sEKVxs20sZIxqdc6pG4i41E01Y6oZlZEXXWYVV2RtQRKp6YhNg4RAIlCyMYVKLjKTljaPk1LN12RVo9RKLQUtFZRZS1ApqXhG3xB8Dc5XgtbKOE5PvKZ+JWV30/yFgpRndfs5O7/PnPtBHfiE5SubQ3nwgY/bv/c9+879/r7n6Ncwfj8PKbzzmKPTfE/d78JKsxuRP9/nnVXcQrQ09JgL3El5Nvh7EL1x+H3HerCzcR7p7x2e81P691ggyGdcJ8+FeP5F9tuOKlg8/5bgjAWUuNhv2enL4Q/QovMxS4o1IcbZo8r2Es+P9t0Tbcwmy/nhjBFiJMVEStCkFhFlmgZqqYzjlnFO59U1iVWb6Nvo0iYSWKWGy7blsu+Jalid2JSBjRVnyDQzbTc06ZI/3Ky5aSNRCjlPiCkRo287LtfXpKbj8mpFxLXl7t68pVTlou+47i+56CN9I3QpcdWteLFec9klumiIZLBK1hlcAKZCI9A1rQerLFBKFauOnTyAes43bM6+VlUyhWyT7+Ah0ajqLPnik+n+lwGbpV4W6jDM9XgOWweTYc6Bu1weC4BUZrkccSkZz4e78zPYQ79Z6mf+AuYrx+NrnA1cltfiu4m6D6NYSzBDLSKSiEmc1YxLbuJItECjEN1QjFVn51qtrKyhmJvfZWakmcWwSzIaMXINVG2oVdH5nxTBsrOhUh20BnGRbcTXnLOkrocYBBfZnn1gay5PvNp+heVL4Iev4Xn7ay9fBNv8Eif2icDrcDyfRSZ9jjF54mTvdnuE9bTzm3dV+wNl/jITAztQ+PRB+bDcvjuWSXbgc5c9bLfPsuH4rvbonD06Xk+Y5TN5es8Bzb1EjexSqB1HyPJk1LzIr+wCOY6c/Q76fNR9OdrXABNnhXTed+aI0OrsCcIc2LEwUcwJP+ZBl8NztblvPuZq867mDEsUQ4IRgzlzFoQ+dZgUKJkys1NzfDBdEC6alou+o2sC6zYybkdWAa5aobWIVk/blhRIQlYYiFx3l/zbV9+wbpwdzMNEDJFV13B9seLm+gqbgYGUkXE7UaQQVoFvL6/5w4trXlx0tI1niWgQOgkkqYhNmGVnMmtB68x4GWj0wAb/alR16RTUCDYb1WfAXc1QFSY1smZynTDN7tunHKxf2wkiB4luToY5a0f0+dlNgkc+xxko+rVZd1HBtizVXYaWZS6XxSLuy7dfIUe3lBA9WtZED3S4o5uW1bAaZtDlfUEiNRgqATfau5k4FSP6q4HrMkalEcEqTKZMOKMp5nTqsqZiEDRUYisUhVxBC2iNxOwm8GDuH1rmsQoIITSklEgNpCYQouyzvYgRI+QnXXm/wrLcIj83APyswOQrY8c+tJyewtnk8T+n8g81jx3045PN66cy3X0t5eeM/2F5ZALk9MtpWwfH2cn3s/sdfDwi4D4y83eunLJ6J4neDtzn7ECk+GF5AJ7s8OHnezx+Oqcg68zAnG10BqfynpvSmTnaZecV3CS3P4uDY+Tk43FFi9iLhYCEg2y/akf3SRF25t0QXPxFLOzSvM2GQU6BwsIbKUITWxAlSCUwITYR6oipM0sSA40pQSDGliCJYJVGIJQCoyAWSa1nmhiHDfdSqY1HsLZNoA0tKUVUeuR6zTera172HTVvPH/v1QVdSvRd4mod6Vt4u91wN9xieSSq8odVJDZXvLy45NVlz1UfScxsnVaSZkwzmQm1PIM3Z0mt6g7UhTxhMaG6aOVVgpiLMM9iLDuCz3x8a3HTuunMvho7xlUEB1cSiLJ/iZBo0ERSiJ7KTszTxgVxAekDMO4BGsacE9Hbx1xOJiwIXubApQPmePfCwJxeT5FoaPJ6qvoYSFCkzABR/TzNgrObFUw8z260ipRCLAVR9wEkBOKsNwjm50VxBtMU00JRRSWSQuPmbqlUrcTgLCeSMEk0IVGahpwTReucNg5Camm7xiVfRFFz30exgGglSKAJ7ePX4a+1nIKHL8IUPbN8zX37mOUMgfCwvGfCnjVWPwd4PEYT/RLLE18UHiHOHtb1vgpO933OuB0+rN/X9o6Fek+/5jqPEkbIAV4BdwJfCLaDgRB1P/MnlGdF+9psejrHS/gD6gAOzp08JP8enK7t6z1o6EhE2vc79OA7HLiD3uzQpbBkrzVs1nR7ZKB3uG152LMnYXanLWcnNRxSnrtWDyqWfb7ic/eQZf+l/mUMJLoX2ELqwfJ82OdNXqRH3GfwcC0ZFhxgVFOQSCCQDBClEaV1XoqshVJGKoHUtFyvWvq2Zd301DxSxnvKtKWULdsBxo2i9ZJX1y+polQdqKL0Tc+qbRwANonLdc+qN6Ik7t5u2GzuaNZrYidENcr9wJv7zGsb+Ncfvufu/n8mhiv+8e//yB+++SMpBfpG6ZgIRcCMVlywGVWqVDIBk+Sm8fklg2B7di27vVupqDroFamkAA2eBzhImNdrQFQI0pCCUmf6XFHcVrx/cQnixzk+MyzgvpSYazPGGfSJUBd4v1yjEmGO0DVm/cHqQIjoWn0SBBWf1xDcL49ljnGAR6qeCDcVignKXjMwCiRc/kdUoEIt/oJhJBoRYi0ELVjNqGYKAZ1NsKYVswLR5hzHExIKVSpEmW9AlZIzKoblcfbliwgtYgmLnlJvtY4swSvBBDOhiQGsUKt6XuPqvqhqHpWeUnf+Ov01lV87efJrKaf3/HcyFz+nkU+4IM5Zw56Dhb7Gcq6PHzyMP3fsD4inp3bk7C62fwmUg21LpOduk+xxzmlWr8VNKFQ+TbSvLC35l7PqKjvQZ3uUeo6MO2G39uhwCR45ZD8Odz8cuWPmcf/Rv7g5bmHn7PiIU5B/iilP2j4/qXby6dhE9/D9RQ72X1ghDs77+HxsHpPF5Hb4m0j0jwvNO2vN2XJccP+viJJqodpI0YKYkebxDhaRCjeXK66veq6uVlxdrlzupN6gJVPGgpmDgapKFxvyVPnx9g3j9paSW5KsuVytWfdCDBs2dyOltpRifPvyW767uWSYMnkaud/ecr95y5S3/Hj7mqb5lpcXV5AumEhcrTuuVpE+CFI8kMCiS5KEpoFq1GiMc57anDO11Flg2YhRKJYJCFWLAyGrCAWLYU4JlwjBmLRSixClJ0VD6pqk2WfHClVHl6Zxas3NvCkRUiREwWLwCGYCIhGbmVk1RYvPqC7zJ2DBBaNVnWEMuryciPsKmnqE9+FSFGGxM9vsiygSsJp8jam4OLgaauIR5BiCB+io2RyBbSiJZBXTimqlVKOa6zWqREQiYg2mEaHx6zfgJm8V0IRoxIrrQqLNHHwS/d+cCk+CeIa7YHNUeERLpeaJcdwwTRNDzoylUixSaRGJfPPyl/D0+cRlT+Lvvz8oT2RGPlX5YHby+CX/9/Kp3wROSZT3jPl719zy42cArKcE0NL2A7esT7GOHmNRT9evHOx+ShKd0JI7jPMIftld06eTYMfzZoJHMs7b63xMXLpkT56eZ+r82QmPdfj7ue+PncyZAw4A2iFdeAivzqn9vWvqD484AnCyN5XK/P2otXOVnkHrj4+x7UbKDhjAx+vybQ+btZ1v4nGRk797oLjbbAG3LVZEK2kHJt3e2UTBEiQysWzQTUZDYSgjQYS2DVytGi5erEkhYaXydsxs7ifyqAx3mWFzyxgjDIVxKIAy5glTuLr+luv1Dfda+fH1X7nfDAzTQJ62TNOGQSrNxQ3/9Ld/y9/98YaLriE0LZvkLxUuZT0y5YxuR+frmkQIUEullJFpuMfGAupJ2oIIEoymaVj1awctGCKBFFvaFCEUKhVCoI2RFIUpF1oAScTYgyiVBrXWxw88O0dIu6VrZlRzP0yq+xZm3Ztt1YoHeCxRw7W4CTqKM4Qic0pgTx0XiRgeeWviQShhx/yFmeX1iBWrznZWoEpACS4dNOcOFoGoYbZpe9aRaEojE0EqKlAkMRGpeCSvzJG33nc8h3EUQgqYOxQiMqccrBFqAG0IIjQhEVMkNuZR3QIxVtwRVDAC1IaSq7OU0mMRNGemYSKXgCSl6eOjV9SvppzeEk8v7S/BDH4wmPu9HJcvDMo/VnmwHj7TonwXv/Iph/VJbTx1DB6hjA+jdB+tz47/zvfzfXDHnujxh8ccdLiQcrMmLPo0WPcs5s/hjO6+Lf+3gz32nT+l1A4HRY4/nQODR0zXAhjnet63EA6a21VtJzscdGv303Neat4LEN9/I1jMuNjpeJ20Ywedlf0Pe/+wBTTLTvePIB4EMfuwtTEQZ+Zr3TV0Fy1dF2lMuUqJi67jsmmYtHA/bSjbic0kTHcBnZS77ch2GJkGyNOGMm1ZFWMdWzo6Ouvpuob+puNi1XNXjNe3t/yX138mF2edjOQMnTSEtiW1F/x0m8nTD7y4WnN10dNFIVKwMqLTiE4DWgvJBBWh71usZLDJc9tKoEmJGBJJHFgZgRoqqYukxn0RPUFJZcyFaaykJDRtxOZo1iQJkQLi4CmJM14htITQEEP0a01hsolxGBmHgVyUoBDUM6PIrKuoVlAtFHV5GZ39K0MIpBhJKZBSICZnFIUwX7w+nwZUqyw8YJjlgILO5mrRJSiZKu7bKYA74XkQhtQ53tmgMY8qNhGKRrIaWYwswlQUpRLEx4HozGaTvCdaM2bFAWJMaKhQG1Z9z6rfvYeiVKpNlFrJs6uGM48VKwktiVIhRNeRjG2iW0HRLY0Iiaf5qvy6yq8EMPzmy+GD+9c0n1/ibeRM+eLD+okZz6WJw7Kcs8DORCiuyuC/qbvjBBzJpRZogKepJjyL+XPuaJ8SawdJdkDrgBm0U+bOUVZ4xwzujj4lCQ+3PcqCvXvTY9seTOcDDGqPLrzdnCyfH+wz842znMehNWf5/fj7Mnon4FhgF9F8dvgCh+nwbAGJAUp1MV5DaGLDTX/Bq+trXr5ac/2iIRRYi2u4TVPGxpF1G7A6eoBIhJwmugSTDdysO65e3dCkF1y2DVf9iqv1Bau2A4zNmHm7veen13fcV6GmC6RtiKGZB1wRGwlkxk3mp/uJbWPcvv6RvmvpUiJibpacBqbtPXm7QdRoQsv6suNq1bLuE11MtOsV/WrFetXTNZ5ATYuDuKZJdKveTbRqkMHqhMukzAxXErQatQyejSP61RYkEGdpkmDOLvqbWEVKJqL0TXJzeK5zvt45Nd7MCJbq+XnrHKUtEmialhBa4mzmFXWfw70moeySrCz5l73oDO4NKBjFBaiRmT0UJEbUIM96i0kEncUPRzOqGKUYpTpAthApMZJrZcwjVo1ARUogFSMNlUghSiUmmyN0A/0qES8CbbwDmzOfmJuzixY36daCJCHEFtNK3ia0tgRpkeB+mEkKsXWtxTqM3N6+Bq7OLfDfy1H50uBiYSQOvp7d5zdaHmN2f2lD8ijWeRd79QnbtqcM5hMH/LGun9suDz48tsPB98OHdZhJu8fMvMrRmOq873Lvn/3TZzFXiDabeeu8XZ0FrJdgDUxPE8p/Mvjzvp9Dv+cQFA4SF7OqLfB1AZDzYXZ6zPHfHVP6obaRB2tUzra9A6+n57f8cMaOvsjcHPo97nc7swAPqz5cG0eAznYIcg+oD6s6vtu6K9khUNzH/YpEiMHTjAX3+FJRhpp5vb1j+nHLD28rkittdLAzTZmc3VwZTP2hH/FAjr7n6rs/0Sfjom+56BI3XUPXNKTG09INY6FsR7bTyP00MEoH7QoLzbzGCzZtYbhDULrVFTdtS9sLiUzOE7fjBi0FVc+Bm/PIVCZsHLlqe/oQWV2/5Obqiut1w3XfsZrTj0UMqYVAcqDSJCz664aY+TUYmPX28LRoUVxwGcMqlOpec8os2oxr5AEH6d2UNiWI5r5s1Sl3z7oREXReS4v/XXQhZomk1NG2PSkBZNd4rC63IsasEyi7oG9sCRqZXQDMPPAD86jbxQFYo0c+40CsMSgSsRTn4CQjC5QUUIuoRKoZwzixGbaYKikEahBEIrUoWTPRlDYJXXC9wLZxP9KYJqJkpsHIQ6ZUD2ZREVQmttMtllskZvrGRcGn4nMsIq6DGDO1bhjLxDQVRk3AP/CrLqf3/89eDh9UJx149MH31Lfpj1k+A9PyUcuHjMfnOsefM1e/NMT6WHnXedjRnwfHfBCAP3y4Hwgww54l2qViM+a3f/8sy985eCMuZM5MEoQD9xhLUBq4b+EvLfxlCz9N8D+8v4cfFO378CSXEzuFb6eI5/DvuXE83ld2A3GmzSWF2lFqs8f6zgFFtwdXp0SaPCL7cqgFeLoOjrCpLFzn6c57aZjDCgz2gO7xYTqhP+3BrkeR1Zg73ksgznl9rVaqGpM5OLJRGUoBmdBxomtamqanbzpi07rZMwhtEFJQUhT3LQsOGO5GyFmZxkrfCjEqFWMslWEMZFmjjTrDpEIe7hGtRBtpbeKihVfrC759+Q3XFz3ri46UhO/f/MDt/R11zDQxkppIiLgOnCrfXKz47uUlN9eXdE0iis1RruLaheJANUmCIGgKLr5cPSbWY2QiFnzZqyQq0YMklihcq1jNnsYtCarsQJepQoDYRKIEtColZ2op1FJJEmkbF1L2qFkBmXUH1QF233e0XUJi9aCJWqkI1L0Xqkv/+IQuUeBmy/VnUN1Mq6aemcUUIRCjp85DjNHAxHMURxoMI5uHgShCFc/qMmQ36ds00sRIbBuX+7FANNc2zMWBbK1GTUKSSkz+glczVBVMAsXcxaAaqEWQ4EE5paDTBi1CUKHrGqLAOL7lxzd/5i8/vqVIZHW5Av7Tg+vv113ec/d6J7t2Zt9nPatP347PMTpf6uH/pYDfu1itDxiLw+oe8Caf6xx/7SD6Xcc/pb7Tfd4xXkeXyhMvyh3Ym3HLDmcIzClHHYrpvNmYxVLdFSgGCAv7588Uxha2AV6v4L+ukf+lpf3+Jd+wIurlE/r1TJFnwuxTZvvMBQ9O9OiQPfvBzqhru2171uwYiR1dfrbfcjTUsrAq+7YeA+7HdcsOOy4BLLaj7B5hNo++naU5HwC0p5czAFmOP9rRvsyU6MI87rKreniAzAsFSGYO/vCoy2ou4EtVUioe9EFizEZMgZSWSFgHUzE48Cpa2U6FN5sJKRmJkS4l1inSzIK9hlDUI0urCffb4uyaKTLe00WlS8qqiby6uOAfvn3J37/6louLRNsmRIS+rWyvVtRpoo2Jvm3pmkCbAq0IV13iatXQdtFTypXivnTiPnpJXMpl5vpm7lkpUqkUzKqLJEsiq1A1sJ2EqUKqAkVws7QHZLQmNGpYMU+rhhGC0aaAFqWUQp0KZVSoQmgSTWwIohgFoXrwRYIQkmdVaQIxKubhGhh11vwL800ioFZdc9DcHODBGPtwK5kdfkv1iOUp3yOS6Ls1LRENMNZKQcAiokZFKKpk9UARRclamIaJ7WbCxkwKmW6l9G0gSZqDM8Tlg7RgQdEMWSoSjCAJs0SQ6CkEtVBtnhOrjMOGt3d33A4jiZ42NVx0a2JM5HHD29vv+de//gv/8ucfqbHjm7+5fsZ180stB1f0V0NufcgD83OX56DgDyk/s95z8/jcuf3Up/jVl89w4h/rejM4tsS9jx0/c7EfYgapIC6ztfshyuzHFzyIQ4ABuBP4c0f812te/PVv+O7N3/PN/be8nK55mXrq+DTJrGcEfNguD+2SNuswJ+nh+RxiujPVnPjGHQ/QsRlVjn4+IsPsdPuJv+CuuRlkHqJvjlm+HSBcTuCwR6ck3iPh8nKocbhr6+FKW8A/R6f2iIj1g/QvMzhYPjoScCaY+UUBnA2LEK2ATZ6X1ypY3AUctLElRXF9PK2YCWNx4eQQBEFBnDGraoxjZRwmqJXQJLrG6FMhhv1L0KJPaKps7m9dVzAYK8l8c9Fxc3XB9eUFry6v+dP1JS9XDX0HKVaPeL3uKBctVgpJovczBNoIfYj0SegCqGa2OhHUpVFE3OtRDNTmQBfEgWeAEJSQPJtHJTIZbCtkEhuL3OcK2Rm1gJJCpFsl+qYhSQAUihKtkkQ9//Lk42IaiCJIgCRLfK4SRAnBQIK3HyshCFBcdNkKqnke+7CbcjOYciXXPDN9gsx5cGVeqwKg4tIuVahF5ihoZZJCFhhxlq+qoQUmdf/PKRemUplKcTPslJnGDVE9bVtfjKt1Q59cH7KglFAooRDFaGJAInMUe0At0zQJiS4OLVTysGW7/ZG//vRnfvjxLfdD4frmG2IQxm7FWyplfMt2+yOv73/k9f0PbIug/Yuz19avqjwV8L2LgHrfMb+Xr7cczv+zmdqP2fgz2K7Tcq6Kpx5+uv7fz7f8vPJBYG/p1Lsm6LTj70LuByBCYC/OfFBimf8BS+pQAmgP2wg/NnAL8kPH5U8rXvz0R755+7f8zfhv+Bv+jpfxGy76RG8w6NMG8Ok+f2qevH5H+c30pR3wcjs2cOaiRA/GT/a/yTFTJwfpP44MyzIfdRjtOn+yRynzU/bx9IM9XG/LhOz22//qre6Fo4/akN0QnLR62rdTRMnBOc2bdl99jPeJHvbn7ME2goqn5XI9Z4fiwfBcu3NE5yqBUNA6EmbR3hQiqxRZNy0xKForakaIiVoLd9uRQJq5M0OpZCrFAlMWasWzf4RANCPk4nl5MaLYzLxVohb6WOikcNNFXl2s+Ntvb/juxRUvLi+46jtWEoiaiZPRtkZKLe1FoCpYEcRcHFiAhNFHpYuRRoRRXTNO0fkiceZczI8T8+he1HMhhz4SU8SqBzxMpgwVNgXuq3E3KZvtljJNRIPVqucqJdZtR5ciUQxCJWghURl0QkolmHl2D3f/cwBmniIua0apDtyC+Fiqg8vgaVvcr0884tgILh+jxlAquVYwI0ic0/IpO/pWQTTN7Fug764cBEcHa5MJWRomCwzZGLMxVKXkyjhltsPEdpgYx4lpypSyoQ9Cn6IH/zTQthEPMMmo4P6CKqgkYmxd3LoqYx5oa2W1SlysA1oLd5uf+OuP/8J//fN/5ocffiKERNMNbMeBH4NgeSTUgZhGRt2wtR95s50Ib97wezlXPgZF+IFP1M8OUL5EeYRh+MAqnjVVH8IYnu3ju8DcKQP1zPK0R+2H1fUpme9PVveZMTyCBgdfDi2KBxhkH0jqbjoesZtm9ia6hMN9gDcN/HOk+c9XvPjLNVd3f8vf1W/5h/QP/LH7G27aG7qwwiR6fnVV4hO18p8M/oLpnKlAd8LCAp70nX0+UpEwp83yLLWehkT2EyHzPhiLKrXuIN/hRSgHARS62wbHrB+7njxtpg9JwOM6lr8PwaPO2w6Tpz14MZfjczys5niz7Hbf03dznTsKTXZspC8QcVMkniVCLSOWaVFE65yGi504cAyBPlTaMAP2GGhTy7pbs256okGetoxz4IM7+geKE1MskjEmLhJczMha9tsKTDVTNZMEVklYt/7CIpqJofJ3f/qGF6vIH67X/PF6zR+vL1inSCtGMqWhkhoHuiFBEz1tWTXICFIhLBp1wUjBXMolCa30RCseXVoqOosci4mLDot4HIckAgnNYEmptVIVCIlqxg+v73g9VDbZ+PH2DW/evCXUxMuX19xsjZtruFy1rJK334ZAjEsErZ9Hb0YfEnFe01kLpWay1h3rHIPMqeFciDoFIcTkwtASsOr+dKVOFFW2WsmlUmshxUQfG5oQ/VxLhWpEgxQh9ZGm79AANSzSLsJUAm+2hTf3ldtt4X4qlFwpRcm5ksfiwK9WRBVrhNh0SLMi9ResLtesBJCCRGcxy+im5xoaxlLYTPcU3A9UtBDyxP3mB/7r9/8r3//5X7i/vWMafyClyk8//MT9uIEgJFFWSWlqZjPdsx1vUTHup/G91++vrjwlveTuw3vuce8iH37x5SOd1AOQJsdDe3YMn9n2u3b/6IDkXczUc+t5HyX3OcohYOILruUnNny62yGRdPrmZAc7BYPFL9wEpAFLMAb35fvB4H8Vmv/Piov/7yXf/fh3/Kfmv+ffX/07vr26YNU0mCS0BqoWSphxQGC2ML2/PJ35m53Kq2aSFWIwQowgCZ1TXqnabP40MHV5DGQHEJd/u1wWDyizw8GyQ9Lwwdg9LPNi3VUsZ/ezg//v76vnJuikrcdurCcvVQ+YxhOkuYd7crDFjn457YvMeWATSqKSbCLqSKdKCAGd2aUQAmLQxMB11/Ky7ximwqRGv+q5ubzkar0iBqj1AkMwC4xTdoDZBihGnirbYaAUj0bN05bBRteyU0U1zsBeiQYXKfLqZsWLmzW5bhnY8u/+8IL/+Pff8d3FirVAL2C1UPOEaCaRCcn51BACVKFORlUQa4nBzalBxLOfSaFQyFX8LUcrJRdyBa2eq1fmaNc0v0UFEw/IqMYkSlYXOB5L5O3biT9//xNvszHS8v1t5l++31AG48Vt5vr1wNWq57pruGqFdSdcrBuuX1zTX/c+OeYsWyAh0rrwsm7Z1sJYquvnWUGiQEqkJrJqGhpJiCRi00BIBINChdBSJUMfsZCxnCmqDMX97kQCqUukICTpaZIHxEwKw+Qm+yLG2ynzdlJ+3FR+vC3cbQr3JVOmgsPp4P+ljraFi/aCl5cd64sLvru+4aJfIbGwzQM5+8uPmVFH9SjuYmzevoVQ+Pf/7jsSmX/983/h/v5fub/7nu+//1/485//hZQKNy8Uq1vub79nzBWLhmrmTclonthOW+5rJa0vmKbMb7p8igfdg7RRH7l8kno/5RP/qXV/JtTxLsbwSV145MH0Qdjt9EH7c+r6uWVu8H1ZSX5m9cDDZ/zp9nMb34kLTt8m5ue7MYsyq5t3U4NLUDSgLWw6+An4z0b8/wRW/+U7/ub+f+D/dPXf8m//6QUv4zWNzsGEC+Ems29+EDcKVYgPsqCcL8/w+SsIidRGF8A1jzZULU5dBsCcWfHMCoEYImrOYsnBgPgj33YMl82s2WGaN9vnPXtQFhPpIZDbYT9ZPBJ3e++PgRNUvq9sF4hzggMf6cF7xuo9v+8yjBzsb4fraP9d5i4GhGSzlmOIBG1pmf3MUkPTNaz6DrPAatXyzdUFlymwHQZiiFyuWvouElIAUVSNiPB2628Mk46kGgkYQUewkZACq6ZFVgKXiVDVF1YNUJRvb6749uU1/aqhX0de3HQ0nRGt0BO4icJlLXTJsFyp2U24S3YLVV8v0zjnzDX3Y/Nw44jE2WRq6jp6ppSiVBsYxoEpl1ljLriZmOgspkTEIlMujNOGrJVRM0MujMW4G5R/fb3ln//8hrdFkNVLbqfAlktyVKwkpiGwUeN+GHkjmVWoXF+u2YxCs12BKH2MrEKgEaEPSh8T0LIpHW+2hfvR2JbRo/XbRNt1XK8bUgKjIlGR6ALWqKJlQkuG6iLNpmClIFppk9AG5fLqihgjISaPGg6Zaahsh8z9UNhk5Ydh4r7CfYW7Ubg3YVOig+DqQK4NwqptWPcdF6s1/eUFq+ueHAN3tfLj3S3D5p7tdmQzupk4T4XRMnVQ2N5zfQmiE/e3/yt3t/+F7d2f2dx/z+3999xvf0AYSM0aLXf0bcfmbsPt/cZFv6uRc2WbMyPQV+Gb7mlRar/aco4EXF5mn/UM/Ix0yS/GLHymk+f6/anP5XOO1c8CbWdMV5+8HJM/D7d9jHKOEXrkBE/Zp9PfDtmend/2bOmU5fNc/2LaFQ+4BCBeQO5gaOBfC/yPG+T/1XDxv/yRf6r/kf/jq/89/+5v/pZGAikEDKgioAkjYCpz1iVDq7haoIk/n59Qngz+vr1qKAXP9Wmz+UddjBfUJWnEncypLqMbTEiA760I7rLkz/dADIHJ9hxfmE1nO/+6GQcYBqoHy8AR7ymQW0yqR6Bwrms/v6c24xPEd6YInLroPdznfPjzYwTkae0s6fN2Dc02b8PTlgWgWEZx0CaSmLSgYnRA37Z88+KGrltxuVpx0UV0GrnsOnQqUCb33wrGJm95c3vL7f09b28nuhAIjbNrUjKhZNZdw6sXL/jjdc+3Fy+57BLrvqFvAil1FFVajMs2seqCSw+ZksJEqIVYhLZAyopIJsaIhOQBJFY8MIOGkg2d4y3AM2B45o1AUSWouw+UUijq+YXVjM1QmSbPgBFkBsSpJaaGEOIcjFGZsgO/bZ643W55u5346X7ir3cTm1q5n2Az3TLSId0loQ1I0zBF9z+8n1PjdQTCOHFx/5bw/VuQSh/hKiWu1td8s3rJy1VDTIE3OXJfekYmSC2leJRs1MhGlRhGaq1udlUljyM6DIzjwDjcQnGB5raJrLuWm8s1L19cQX9Bs37BlAtRGrIAkihRGRB+GJUf7jOvJyWnyGTCFmGMgdI05KDkqh4lbIFqiTIlRAIThftppE9CsZHNNLj2XlamnNASMR1JMWFp4s3mJ4ZxQ5A3vHn9n7m//Wfy8FfG4S9M+S9ovOP+7i+Ev6wRU759+Ypt2TLpxGjBM4yULdO4oYgg0+zc/Jsrvwjk9Astj1E6X3nZRQV+7Ho5Zqoea+LQGPXFhu5jNv5z1sFj1+eh5W5hTA/qV3C0N283hajQzP46FmHq4J9XxP97ov7f3vBv/vIf+d9+83/gn/7wH/jD1Uv61BMEqhVyUU/zKZ5HPZGoVUGLtyLJYwHMg3KfUp4M/v7xZY+R2NZKzUquE7UUtChWKlp08UVn0uwP31nDxhPbByS69peaeA7TkBzohTm8QOvMdAkhLnGTy8CJJ1hgiTSefQJtEXxZJkCOpPOCsAOSC8icXRa97X1kxcww7yd798kWAHhiLuZDbt3HRl/Hnvu+L+ywvzDo3Hx0RjMKq6anFSOIm4GvVi2X655V62LHiBHyhmnM/PWHnyi5erBOzDSNa9RZFEYypYF+Heio9KuGIA2tBl60F3xzccE311d8d33Fd6uOVTSiVIRMbIy2bQmA5oFQlKQGVjAbECvEsELMxShFoGpBKS5ErC6irBZnHT1ngjDFQiGGMkeUKnVmmHPNs19cJaWGru9o2gbVOShiTsEmQahaMc0UnVAxbI6WNYkULYxlomqg7Xou2obtXWbcZogtEj1rRlVnJnOt5FIQc//GdlsJQak6EaLSmNCFW1bhL6xjIiUoQZHkqeVCiIQYCGrEUDGdyOPAOGwZx4FSJlIdkNKQLM65iTu6JlDN3SnCtlBlSyHRryo5K3/Nt4RY2Q5bfrrf8vp24IfbiZ8G2ErHFCJVIkUazJRaoVTzoGaEYMYdlYaR1/EWQYlUSAEVIRcjTwM6ZaRUklW6ONHETEcF7hmmP/PDW2PSH1n9MXNFYrxrGbY9FpXhnyPf327puxXT25HNtlBE6W9arppAGUbevh6wElhdTCibZ19Nv/zyi6HOfmHlCXfpx1x5Pmk5S+0e4xI7s+3o8FMU95TmFlLh5LdzbSys1u7LwY4/C0+fOejUrPuztQ/fNR6PEDSHx+yG9cTsLOfG+2AwZAcS2JFTVAhlj7JMgATSeyDH/2OA/+sbvvl//rf8n//D/4V/+t/8E1erC9o2EpJQqJQZH0ic9QBVQQOVirrtGIBqEZVZpeOJQ/hk8Pff/cNLxmHkvkS0ArOGmaoxjRkxo2pFgzCVwnZyKYnNMDBl97eqFKoKRcGi2649Qb3LWajMMjKmmHoKriCB6ikO5nFdwJpruM3idjPiFbAlgdx+BHagjxmEn64POfqzO8r/yJ6MO9h/gXA7zvF99/AHlui5ht36kQcL3wHwDA6jEOkIqaeJAcxTjCGN+8jVSN1M3G7v+GGYEC3kXBl1zloRlNAAUakSKGZEhU6NSOZ6/YJ//NvvuGkbUh2JJXMR4VvZ8lIHuqhEG4GM6IpYegyXMYkzQC9amHTDFN8S+Rsub66gNgybgTp6JgyzgIgHQBjJ/4oioWBqKB4wobPZ31AMJZdKroqpYKKECKlJB8DZ91eK+6hRseAschMTsWtI6zXd5Q3rF8rNqNxNxu0ENBvGcsdk2TObIOgsqaPmYLVUg5DoJBGBYg2Wp/8/e3/aJEmWpeeBzzn3XlU1M19iy8jKrMrqQlU1GuhuoJsAQQxACCkcfuDHkRmZ+WvzF2Y+jMgIfwBGBEMRooEhiSYWAo3u2rIyK7dYPNzdFtW7nPlw1NwtPNwjPLOygK4Gr0hmmJvpcvXq9t73nPO+SAOtja5lEhWh0UKgaUY1E2dtzk4gSAXJlDZSppFWMjFEjrojTlYnnC574qB0fc9imVikQO9TP0qF0hLPznacby5YX67ZTTumUjjbbjnf7FhvC5sWyeGERodpoM5AutV2dRwmgmogRaVF13FsrVB2hTEXSoNajDJlwrilrzuWXSYcZxarAq3w6sUviHKGSqNbrsnjhrFsyLsL935uhd3Uc3YxsmqJqSY2uy3oREoRLBKKsBhWDOo5nuuXL99yA/0Vb+8iGH6dbfym2l9qzPqXEVR/W/35pgDpAMTdCfpu/n3IcNk33/Wd/fm229vozK/b3jaBuBHSs4PvWwTKPmzpEi4GlAi7BXya4J+cI//0Mf/t6v/GH/83f8ij7iF9SEgQrDVKM0BR0xnbzJFNM7Dq20evCCs37PSO3CxavavdG/z94otL+lZZpJ5eEjEmQgKJghz3bgofBI2Kqsefc6uMuVKqsSuNzVjYjBOvLrdcbKcZIFbYv1zFKyBF9qBK3S91Dv+WmUp1eZOKSPP8STxnzAuL2zzwRrVrcVzkMBa/Z/8OL+zX21VO3mvJgK+Hma/X3IvdzczgbfeXHH7cM3yvXz2NfcWvXW3O++nhTbHCVLbk0qg1E1vmYgtna2ERI91sUybFNdgk9BTHDgRmlZBZwHgZjT4UlmSeLo74/tMnvH98wkqVrnUkKyykcCSZjolOjBACMc5yLGQQJSUlCJRsHsoUpcoppJ4NoLUyTY2ya35agBCFEHWmv2ewvz9H4hIzNlccNzzMO1VPZm1m1JwJ8+Ueos7jeS0lZPN14cLK5pON1gjm46RLoV8IxzWwyhC6gULkbJ3Bonve0uaUhIbQ5iKSQEgdGkBMqNZBaYg5u9oMJASq6KyvV0EqwWCU6tW+SUE6CJEUYEiRVR856XuiKlPJXE4b5FLoQ2DoArTGdrsjRYFWKW1imzO70Z1bxmbkEpgKVIukENEQUJSCT5wkqmseihd8SAg+dsHv1VInqjZSnpimibRMDKGyRFjRGHQkpg19f864PmfbfYrampZ3tDxyub5ge7llGndMeWTKxvkZtNwRlse0NjBNG6w2Rsn0C+hTz3A8ICExTpOnjPyn0r7NqNa+fZt45z7b+db292vRSV9vF39Z22uvggO26Q3S4J7t5rn5j8Jy3mz7ifpbFnlbgcd/iL7LGx9u2e9doLDNwExA8/zSbZAbtB5eDPBvQf7JxNGf/pD/9qP/E3/04Q85TgssJsqsnQoRmyOYCmCuUSsSqHuMYO2aZMRZxqtHyj3Z03uDv3/3+Y5ljATdIm3tPqrBRXQ7FVKAPipdHxj6SIqREAMxJroY6ZKwXECulUcPjx0UNtiNhbEZu6kyZSOXxjh5cn6tjdqgimNa3eNbbWhzlq+hM915fTLabJOyzx70b4XZ7oKrcovb8dz1CZWbIPBgkbfRxzPFuOcnrwpbzFd0qCN+AmEGl1yz0jf6pSaEOtfUaKXgxQ9mHirP2RhLplOvKFVR8pxnkDTSxQStUfIEpdIH4biH0wFOUuK9ReS9LvBQGwuFKC5o3EtmYCIyQW0u6SGRSoNa5otPqQR39pCIBc/3bAZ1ajBWdmPGJpCiLposAQ0BaS6b4u4bE1h14NbkilWus2WYz4Tk6ry2XGm1Es1z1gQPkaIO9p1BNnL1fMGcM7VCk0AMiZPYswwdyxhY6sAiDDw737DeFNbjxLq0GYw6m1f39m61ehg3BILNExFtiNaZnY5o8NtKzUF6o7qlWjA0QNBESkqnyrKPPOgjJ0mRJkh1RnwsE7vq10kMAiFyOWVKqYAxNhepbq1honS9shygDx3LbgUicwpGJVsB3HvXq/H9vo2doKlHtKPUwhQLU9kRT4/pYyXZOX2ZiOMFdfcFefslaXmOTq9Yds9odWLa7RjHifOLSzbriXGa2Ox2XK4bZUycHj0g6DGtNnbbTKAiGdIicDz0HJ0ukNix3u0olt79IPrf29vbNwZkvwk0+h+z/WVEezfH964+2rsXeW3Zb0IP39KXPeNw52Vw32tkv9xdiPMbItE7w+C3/vHudd/VbgOCd+3n6qs6s33mOX4VyBGeB/jnheEfJ77/ye/zN9/7h/zR9/+AZRfdnADBWpg95WczDTOaNXQOPe7TwPb7uypuFa7CzfI1Dvbe4O/5BBdNkFCRmolzoYeJkQySGikIIR74wsZAioEwS5GIePFCikoIgT5E+hiQEKkdXO4KuVR2UVmPwpgLU3GAVA1ybVc2yIrnjGFK0zk3EJ3DiEaTuh8h/3cOF+8JvGZ2FUF+fcj2wPCA5pbXsd+eRDxsr02yZpB3xTLuWamZ7WtzHB/mflxtxK77cE1SuhhwFZoIplDVxXatgcyuEk2ENgMri+rFAA0WJpgKTB5K76kso3LaJ56sIg8G5TjAUo2BkV6USMakoFKJClp1DtnaLNk4B7uDuF8ruN4QOJPUGm10F9lajKlUB3RmJIAKUgSr7j7SpNKsOCwWQ8StzVpzcFtbo7HP2xM0+LEUfGxUgwMaMUKIfn6bUefq89aCB2RFEPVQcynGWEdyC1gxltoYwsS2XhBqoceTa1UBVUpTWhPX2lMI0QWWJahL1czHbSiIy9eoBIIFWstufXYVkhYCSpNAE2XXIE4FLZXLaaSUTKsTISqFRugTXfTcwdoirVaSFUwS0SClQJciyz5yMgws4pJtbox5opi7ftRi5MZcaOHamjEqoYsE7cACOxG2Y+HhgwHjnO35C6bNr9hdfkHdfEYtX5Lqli6s6XnJOmc2F5nLPHG53TFNlVob0wTT1kXES4VahFwm6lTohkjqOvquYzEkVouEELHWs5sOHm7/e7vRviE4+yYY6D7r/GXEVre1t/XzLgLnP1j7FsK3d/38ru9uYwV/nS7dp71129/wBHzT/h6GaW/N59svI7cvcwgCDkGiyGzR1rwAwjrX7Pv/Zfp/suS7P/8R/9mT/wO/99f+gNVy6UWYBazuo312YHrhuKFxaJGLT+L3v77WZXu9q+9o9wZ/kwayuTZhECXqXEWLR59HM6TalRxIsoppnZkvD92quDRJJ0YXnB1M0fXOFGXMZS4UgSRCiIkhAKaI2WwcH5jmsF4xZdzzawKYV9GYKFlHLzTZg78rytmZOJMDHcErRokZ7MnV93t96muXj/2cZv/t1bAf3JI2R5gPEBzX/8rVia0HkWfXSLyefc2AVL3L1TyPrZpQMdf1CwmRNDOiXAkyFwyCkiJkc3cHqUa0vV2g0i97lquB415JbSKXzDRmNCoWGqghquTo8iW5FWgNyX5xaVSYQYM019qbmgsvT2ZMdYIqlCbUOU9vH2IEcYuz6hW5jYqpM1NNZ6Dd2iwF4/+1GcibzTZieyJ3z2YZQCUEI2gkl8pUJzREVDqqCE2FYsZ2M3G+WXO52TIZZBNyabzabJimkSQ9XVxQusS2CCEa4wRj9vzD1hpSC2jA5mIlpF3lGIo05hgrgeCAtDZKxe3cWqOKkE0YEc6nQFNjsMo0jViu9FrpUXpR+iBeLR1cQklackFtSaQgLLrIcug5XvYcDwNiPa+2I9OUqTR2UsjZ2FVjVwu7XKmluDC4NGIqKI3UGgsx3j+Cbc2MZy958eJjLr76OWH6nNPVFj0ylB3jdMbFeuRs3XhxviNXI4REFzqO+o54VNjtMq1k1heXXFxsKDSOj3r6YaDrI6qVVtZ0smShgWna3vdx9Fvc7PBBwf1fel/jLfdNQ4VfZ/lvM8T8m0Acvy3g9Lb2tfp+y8L3OTdvczq4c9t3naeb4OnG33bzt1tQ9zcOS38bF/htfXj9nX37evtjOWSGdBZwbp77txng31yi/2Pgycff528++S/43b/+e5ycrqjWnBiyuUpXGtr2CMJ1PV7HJdd9vKaJXudY4fZTe1u7N/grs9+cwpysP4Mu9hptjlijNYY5V28yzzkSMVeeniVL1rUSiyGjoZoxrQ5fzPO7mgZC6OhjRycRQ4jNWCG0EthQGGtDkJnBaz5wBMQiMLETyGJz7thcIWxX0OwKZJm9HgJ+3cv3+uQ22TN6+9/fvIBtb3cnHnb00K5XFKtdryPW5r8ro7ZZBNtQmeP4+xDmfA2JzqBhhqB+NLOj72wP5hHJQq2V0twbt++Vbn+RdC6jgkCJys4CuxoZC+QC07glb41ln1j0SugDnUbMIglhrBUxdxQJgFoH9CARaUYtmakYtWV2ZuysUrO55l6MRFU0Ck0CZuqgsDozWPdh+iC0eG0haLPun3vYVijmupLqnr4pOtBXrsP7tTVUG9tc2eUJjUbSyDZXSqusp4mzi3NenL3g/PIVuQrdYkmIA2LGg8VAl1ZIWDISWReIk7GZGmx2BNvnITZadb1Eo86sJXM5vrN+fr94iBUKgQJSCdKcKRdzmzztiLHjqO9ZNNAqnA6Jo0Hoe6/0antD6CZ0GtEUqBUqlS4Yi2isFFJt5JKJ1ohB2eaCtormRrBGaIXOjwBpDaUS2w6CsOwECRPRRjSfQ7lkffmSFy+/YNBnnJ54MvLFtOPZZuRsu2M9wXpTsJZYDT19t2JIA0eDcXl5zmZX2V5e8PyzDUePhRA7+r6nS1DyORevdpwshZiOsFrv+zj67W3fkMD739s72k3c8VvRvq2L4S5gctvfN/f5NqRw2zo3199Dj7cdyzc4Obctesiw/dqU383391vQ511DdHVYh31RkDIzNhF+2eB/rJz+7K/x45O/z9/83T/k9PEJpTjxEVVnEbx5O6/ta+991g4w6KEx7iEE3P99fzB8f4cPEULwKkHZy9fMOzZ3IyY06FpDm6cfNk2gShX3iN1Lq2iUuaqyokFmFsPLO5p5RbBIZKqgVGrzVPCuBVQzI0ZuzhQ5Qi6IVJBGah0wUswFgBuBNrN5OgeN3VPV2ciZmNwTfrxe2LEfzjlHL8zf1esL3vYi0fOqV1xg8xOpzOF/mB1PjNAasYFZoaZMC+6m63XKOhOR16BUBAJGj0u8NKlM5oU0ZopKJKCEVrGaaRQWUbwyVX1mIZJQEVoztmPljB01j1wkIdSRWEcexshqEI5XicVS6KuyjUJssN0y12MogYaMzcO1+0Gcma1aYWywA3JriCoxGSkGokSSKh1KQrAgTBWvNi3moeUmqBgyCztjzUF7BZsqtAmhoMEoNWBm9N1ASj0SOxrKdiqcjxO7sZDbiDCy3o2MeWLKE9txw3r7imKXqByxSEsePXyPB6sjOumYWmNdKue5MrZC0swQoXaVsVRaU4oBZlh1+RuRhgS31ovq4y6mUAp1N9HKmlUyFn3v/Y1CUkjB6DRxPCx5cJSIsbDsBp4er3iyVDqp5HFiu5moxYWyFSEL7ErmcspstjvWl8YLAqoR1YHULxFpvDxfM04ZK4VKY5rD5UEjMQaUCmPBUsfQJS63L/jXn/171s8umabnSB55dLJk6I7ReMknXz5nXSvbnFhnA+1ZLSN1Z4QasKmjakCD0Q8dmip5FzlZRZYrOFr0LLqEMrHb7tjlCyQsYRLOt/8JSL3cfO/+RsDKwUvk6zp83JeB+UvFrMmb4/gbwVSHX3xbJ+4W4HHnpm87r3eciEP65zAOePWiu4UevFlg8tpKt7S9Ztp+mTfY7LdcTO8iEa93cseCv4EL8Jts8hD4KXhOkxexclHgfz4n/ssn/Hjx9/jDH//nPHxwghUnBrBGq4ZowErDVOYQ70Ei2mu42g5G9HW+77ZP72r3Bn9dEIaY3JmhVndW2OeymfuqRvOuF3WdtFqaOzSoQ7RKxWolaiQFxczlOySYG9NTwRq1GtSJIDIzHpWmjbF5En42sKZEEkG9prGKCx4XiofgcEbomiYVggQHTwh5Rn06V4E2oM0J/nCTOp05t9mubqbtHPLJgajiPmQr7rG3P1kuXWOei2izfa4ppsqUMk0byTqwhBDmPjvoiQhJlRNNnDYj5xFrmUndX7c0PH8yJlIUrFRK3ZB2o6uHx84BshUsGzU3dkAdM3VQtsEI1uhoTNpYlsJZaSy2mcVQWfU9HbCbCnNpBlQHIa3A7L3G/kJtkpmkMoX9NWvIWN05RBspBHoNJI3EkCjVWcWSQXxkEAy1hrQM8zWRzOjUfXGlOaA2M5TC0AvD0BO7jqlU1rvC5XrN+fmabc6U4vl/01gIqaMflqxWJ3TdwGqxxCwSJTFlY5cLL3cbXmzWXI6ZXXGrtlavK8fNootRa0WtgE0EbSQRUgh0EYI0ggQIlXF6xWY6I7Utx+k9Hi3e4/jImcZWJsDzDTtpHA2R06Mlj1ZLHvaBnkxWZVRFG1hRtnni1WZkzAXbrcnjS7a7Rp4Elciif8jJg4RR0VYZJCDR79OqiqiQuo7Ud1TFQaFUAsaLZ5/yj/+n/zvbjx+xionHR5Whf8XavuCL8orzywuKHnNy/JShP6GWRNlmmDKp7xniEsHYrM8Zx5EPP3zK8fEDwu/uaHXLyWlAGbl89YKzs5dkMyS8QnTLVy+f3/vB9dvb7vuG+ctEZf1l6svN9k3e2LeBuLcBu98w0r23hdktIPfm77duf+/6vv/7cFk7+EevwcytQ3Rj57cBzFlh4/Vtvy10erO9ZUb0dS6/m7u6OoaveS6vFr8Jlg/A7mF1dsig1e3afnkGf3LJB/wdfvz9H/P00elVhFRfx25XNQV70Hf4+WrbcAA2D8/TN7te7w3+VmIEy4g2mHPJsjWaqeMd8Zf7ZJ5XlgT6kmm50SwzS1675AZC6iIpVBp5Piadq1k8UVKaV0uGoEgAUaNoppmQcDu51nbsU8S9+tVzD4NExAQ197qVuSrGnTFmKDfTp6oOPJsZZS8dM3vmXVnMzahbZlcRORhgX6JxZdlmfuraLPznpK1Tfy5doogFpCQG6ynS0RhBChGZRakNrILVmZ1sTAV2ZtTdmt32nDgcs+oXDIuOxaKj6yMtGLvdmnF014yuC2hSWimUYrSgSHBfWZu1A+MiMgShpxEFRrwyfZsbqU3ETaaVic12x1YKO3V2Uk0JTebq9kbL5mHcUKipUqMRpIfm1bWRiFpAzQhixOBMsos4z2kBraHWSMHlY4JAUqFPidAprROqNUKbEBW6TlgMC7TrGWtjs96y3W253Gw4e/GMy82IxkQXIhIjp8fHHJ+sOD495uR4xenRQIo9n3yZ+bc/+4Sz9Zrd6AUqU62MZa4Wnv2DkyhD17PsOoYUCFLQ5ux0p4VFFzgaBpZDRxfBGmzzmnYUCPGE1dGHnBy/x+nRQ4aug5Kp04RJI9FAC7U1ujpyVAJ9iiR1jcaYoI+JQMeXLwsxBWQCk2nOhS2ICrGLDEtFZaS2wuOTnhgUq3WWohHCXPWeC2yLuh9kMo6XPQ9Xj3mv/GesfrDi9ETpwgvG9Y7nZ5XLHezyKSo9bDuCPmBcb/nsz17SrUYe/u5TPvzghK5LnG+NxfCUH37/e6xOIimc8/LLr2i24cVXZ2xfnXkByLJnNyqvLr7kcvzLCC7+Y7VvaSzuSma/a3+3LvYf+rzc7PNdx/DrgLIbx/QG63WfcOg3ab9On+9Y94AZevc+7ur71wFot613UKx1CBbvcr66s92Xwv2m/bzru3tu7zUmjutx34O5JlAiQwns/skl/DLxw49+lw8efUiniVDqbHPWrjbk6iThSqdvppU4xBi3d/OW+2TGLvdp9wZ/ywgxKkTF2kTvafquGzfn3ZkVvwRKJeWJVadElN6EZJ4baNLoFJYJOoWmkSpGjJEmDuK0NYKbeniRQHRWrjJwORbG6hpqVl2Ad6zFw1mlUg2KCBoCJi4oXcWrgHUWg664eLSGmZ6akywj0HRmZNnLYsBeQFjY5wjudf0OzoQx51EdgD/lqtLXJRg9/1DNPW6PG0TKvHkvJsAmxIqDP7ywoRN4slzwdEisZAHlCWl5xGq1ZHm0JC0TLYnnOAaX9djudkxThqgEU1pR1uvK5UVmu3Ums4uR0ClB/diKGM2U9XbHZnOJTZkkgSaNLZULyWwx92+uQipCaIoWHzQDSizkviIJgk1EAp1UYk2oeXGKWJ29oQuNitKIQeiDkkJwa7MYWXSBZR/R2LGrhmUIYixCdMcXFaT2tCmSpbDdrTm7OGM7jjx6dMpf+8ExEgN9FyEKp8cDXUi0migWmYryaq1MGHEYiKUipUCLvp8u0CFuz1YrgebVyrZDKxz3HX0KLOLAsldOVgMnq2P6FMF21JrRtGB11DPEwKqLhNCRYo9i1FyoGRIKrVBadpkdXZN0JBRD1OgRiEKKC4TG6XEgrjqGPnN89IjH0xE5O7gLMdF1K1QjopUY/TpN0tGlhKgxjRMXFxvybkSmSGzCsAz0aeL0tOP/+n/5P/Pqq88Z2+e0mqm7p7x3mXj+8oJnX12wWQei9Cz7UwKJj/5gzaOHj/j+9x/x4Fgp45YHJ0d88PQjnr14xs9/8Qvee2/JMkE0WAw9pw+PCRvlcpM5e3lOtsCH7z267+Por077RqHfmyzEX9H2mwyL/za1e43DPa6JQwHaq23eSAt42yZuZdMOAIjLTtxgo74JAHzL/t+2nTui2d/6jq/GT3itoLTAvhJx9++ewf9gPFr9Q773wR9yPKygzoxfNURnUgg8Qsm87k1293DXr3XjcKHDMb7/QN8b/LVpCzWhxRCZgMnDmRWkCWjDotD6REqJLsIqwKLredQvedQtOe4ii+i2ZL0UkkKR5qGfeRCTuOl8py4grCFQxRitkQtUa+xKoewmylQp2dfftcrFbst2nNi2TAYucmGXG2UGfJXZuovg+oDmgsWiHezDug0vwNDrtMq9BdyeMcSuJ4rXvxgqnr/YHAFfra8SPfHfHXqR5vmPkygm2XMCq5BScEZJ61U1cgiBVep5b7Vg6AKPFkd8cLx0Bq1k12tLzvhUoEhwK7TFkl2YC2Oan5tFN1HaBefn57QRahooSWlaPUwdoLQAqhjuUDG2zE5gI7AjsK6VnEGrESqk5gBQTDwXsUEuHt7upZLEiCZeBV4K0vBZTy2YubMEZnTB6KLShUiXAl00htIYcmMxGkMXWCZYdImi0AXPL8wx01UP9TeM1WrF8fEJD5crnp6cMqx6limxnjyEux0LX70657OXGy6nxs4EV5+LpGHBAqOr6sLQonPlcaa1jLZMJxWKkHMlqDIsOx4e9zw47lj0bpGXREgiLELPYhE5OuqwKqgVWi2E1khBoDMsKHmcaNJI/UAIEORaHqiKUXFHlKrJczuHwGKqdKsFD5YL11hs8/QkRogJFKoAWtCg9FFZxcy4y1xKIbaeVT8wkWb/45FxOkdY8+Ch8KuffcbPf/K/MO0+5fhYOD5esNTASd+h24A1IciOo6PGB999n5PjxqPTwhDO2K0Luw38q3/zz/nlZ1+wuVjTfu8j3n/6EEHYTsaUJ5TM44cD733nAdov+eg7H977wfVb3w79W7/Ry+q2h7y8ZVvvYIW+Th9+07hz/5K7F+DZr3Dj4x3D87ZVX1/u10Ard43P1cv5xgK3LX+zf3ce1wFYuDO/015fTg6uO4Obee5X/Xzt91v6IFzlrl0BoDf6+y0Bv8P9Hvb9al/7Ph6Or72+/GvtvkyfXAO91zpyuHOvN0AClApjhX92DufK3/vbf8T3Hr3v0acmIAlTnSt950JXuB5jO9jdbdfo1WLXC/jlalfDf9/I9r3B3/d6o9aMUOmjEN0rDGvJU+C0Mkll7I1qQtAIYkyaeVk37KbKoiZWIXDUKcdRWMwXpYrOUjCKzs4dNleLBossY+AkCIVCbS7nEYdEkp6gXvE5mrEpmV01znNmnRtn48S2Gtsps5ky26mwLYWpeY5fNXMW0TKlmjtJmF2xsnvP4NZA41w4UudQt+hVGLjNkjIqSpRAc+hElb0QdplZRw/nhtbm4g1hUC9kWXaRk+UxfRqwWGnqgKPUiTLt+OTZOZtW+ZsP3+eIzMMhIXVDrI3eIqkGcjPKbsvlq5dsdi/ZMEBMWIGaGxONQZQPHiV2W69RtS4hXUfqB6oq611lrI0pByYg54lta6wbSOrmEHagUV1xvHpRjyCUEMlBGDVSyTRVsinBBG3i/85Av2nCcEu/MmaSgZrRNWeHOoSEEXOhj5VhihwvEkNrDAF6bayCsYyNZVSGTln0C04XieMu0ir88vma3VfnDHEBYmymzCZPXO5GLnaZbTG21pDYebKtGag6+yyCzBW6ojtq2WAlI63Sh8CT445Fb/S6Y1l3dKMQWySEnr7ribHSK1Aq4xrPUY2ANQ/Di4t0izWm0fX/WmmYKqL7EipAFI0JTRF2DcosYVSyjx+z0XeMhKhUFXatMe2mWRLICEFpwVMlNtsKObCKiUVQzksklxFrW46PFqxOP+Cf/cn/l1/85Ke0aeLB6RH9kLk8v+DlV2dsLifIK4IsGC+ek+2Co+URoQkXzyZe7HaM08R6U/jqqzXbTeHR0w9ZHB0z1ca43SEIH370HU6OArk0Xl4KYzVWpyf3fRz99rb9DH1fvf3ai+pboLruE/l7rT+/3u5+s+3m2/5rrnLYbq5+G5D5tto7u33LTu/q99X3tyCAq8+3XUvzv8YNJGFzzrq8zsgdrn7oa6t2PUm5qka1623vGT/MZ5t7HLj/7mr/d7RD4Pb6Ad+18N3fvcFkvm2/96EIrwHZnWhcZovZ/fEW4HIHf2p89P4f88MPf0gXE6VVrIUrAsnmQsZDoLd397oegWtw9+ao3PjmlvnPu9q9wd9ff5jcpQQlmYfmVDvEOsCotmPNxGUorgeY3AmiVAeN61LYGrxsjT5AH4Q+GkGDv74MYhCWKXDcJU77xEkfPQyo4BWehRA8DOrXp7svRHFR6VVo5ArvDYmsiUuDsQljcbZwVwu7WpmqYc3YjZmdRdcLHBvjOLHJmV2ZKNVD2GaGFTCKV9dWr8o1T0p0u7H5/rDWCKo0wUGk2HxxeH5hs4q1SgKOYuSo6wkSWIWeR93A+0fHHK2WyApqnMg1M44T065QtjsixqJlplfPwJb0PfRJGRKINPKUKXWk65W+f8RxtyT0A6Eqlj2PrQaQmKg1UZrQQqRGIwu83Bi7smYcC+djZswVLNAsYs2AiEqkBUU1oBaR5obT1YwaGiQlaKM0JRvk0jzkj+tD6lwdbECTiGAUOnLNBG2M5qxiLEYwEBphNLpYWE6VO3uUqgABAABJREFUfoAhCoNUhlmw+igpx52y6gMvt41FqHSdsL6ceHm5pvEKjRHTfRG2ELqBRRdQGttdJedKNtdZKlNzTUMymYK1iZYn+i5w8mDFMiWOOpfaSVJIoWJtYtwaMSxmj9/EhCJzWkJrMHQJsTaD+uKFTWbU0qhW5+fkbOsz++/G0BE7kFApecSyu614CZUimpDYiMlhYLXGZpzYbbfQDA2KaSTGxDlGyQXMi58QY2rNhcOpfPXsKy7On/Hlp59gecPJSeTkUY+1wuX5yG63JqWe00fHRI64LJes+iUhNC7XG7aXl0y7idIq56+2iPQ8fO8xT58+4Oj0iDJOXG4mUsgEFbquY5wS02bCJPHVJ1/d93H0V6zdB/D9usDwjlfCNwE/3wCTfa32tQ/zHh161zYPQdbN8JncBBhw37yqd4K6d47fNwC9NhuKz7no144Bs0XCPmx1hX/2bkY39qn4982uwdJexuQK/O0XV3/XFcG1LW6wbu8Et/vtfEsU9Nca34OFX2Pj7cYycgBkDwH0PBaiQIJffAqfwl/74d/jZPgOKSygROyqoMYJrqs0U+OW62lm8vabfgMfH0q+3HJI92j3Bn/fP0lMRdhNIzY1r2ZE3CNUXavtyJSl4KLCVCwoBRfWzaVQql98Vis5BEpuLk2BEEwJQdhNhc2ucB5GFtHDVUNQgmQ3kY+BPrhwrleOGp14FWhQaBSkGQkjaUJSwLpAaYGxRsbq5vVWGjYESCtMIyUb22nkcnLf1NL8xmnNkDm8PGWjNWhNYGZZttbYFf+9Vmc9TWCqlWINohAW8SqkmqtXhR7FjgfDkkE6OhNOxOjbjjhlYoA4KKmLsFhCA50WjOsdF2dnlGnHdl3JIzNQCEjqkK6jXz1kYXC+XRPSgIYOpLn+XmtcTiPb9ZZSvNI2464hpVbW2VgX91s2CjEKgYQ0paOBKZMJWRSTWXBYBFMHtk1dj0glzA4srk2o84Vs8/MBk1kr0q9qSR1Ez0drtVDxim6qZ0pqM0JtrDPELEQNDFqJLbNQYRWV4wSrYPTBq6MXg9KnxEV28KWdF5j4c06IKqABTZHFamAg0qQhpbG92DJu15RW5vzPHkk9cdkznB6x7BJqE5eXW4YgBFGqBe9vhUJDc6NLIKqYGkJDxuIh5OqFHWUv39TMhaPb3u5PEA3EIKRQCMEfzqVmrGYEt8Cr5hXyEgQN7t27f6YXN3LGpNLmF4BSKKV5DuPklfcWQDrh2dnH/Ouf/b/ZfL7E7IKQtsAl42ZLbZ6/GEJgMQysjgd6XbCwDkkj2+krXly+YrvZUHOj1EqdGg8fLXn83oK+m4uGdpX1JtNYU2zL87NMyQOXF43TB8dcnF/e93H0W9zuYi9+TcYPuOVN+leo3Yep2S/HPZd92/rf9Pd9uwcwuXWdr3EdvAZQbuzX9iBsz5Q0txvTmZC4eiDPlYgyV+na/D3GXsTCu+Xvsiu0IjNgNHHAp3P40/D1Da706fYs901i8rC/bxz7bQvfGNMrsHjbmN3GJt5F/97Gqr6Gpt/8vek8Tvi4ioIF2AX48wJ1wXcf/oguLAmzDi+1zZJ2hnvz6mtb3ROXr3F/b31cXB/7NW63e19B9wZ/7y8SUxY21ihmqGWCGikZXQqgiakZWwlUU3IdEVGseXVrbUYxwSQwTjuaqWv6Ic6cMXvy1sY4VSYrrGF+ASpBd0RpdKljCJEudgxdZJUCqxjok2CqzmJUQ6vLuMRY/UWPM3FZoATX51MNEJ01q2KUqIxDT7GeNheJgCCzj2zNXtSRZ4Y7i7LD2DWj1IbNlaHFjKlkaqlYBIaIRaHMk4QgylITR6ljRUecKiFv6GqFPEKr2GhMUSC5fVmq4on5IVGSMjUhj42asxcjdEp/vCSmY8x2XG4ru8tzmkFrmVwLoxU2tbCZKlNRGonczOV4qhfSVM3QoDNQCwQzaJGGYiZ0BCbnQanzzKdRaeKgxExRiy7Vghc2+VVsnuNnMwATL4oRdRu1oIq0CiaozjeACDoDTCyTZ1AuEinBkJrZiTFGZRthoUaHuX7eGh4/PCETaCEQNF3NclUUYk9IHaLmY9AqGrxq1tQgRn+oCYhGYoiEGKktMWbXJ9xkgeZejNrc6s4sohmQimoDmSva1TApUMu1OPSc2yfzzFFbnSeRs9q7FIJORBU0QLNKrdN8yweQ4GO2DxeYM89dn9AYXJbJjCbORtOMko08QR3NkyaDoWacne/49NNLLj59yWq15snjiZDWtLZl2o2Mmy1lzBSt7MYtliIPH3yX8+2XbHfKmBc0SUxtYrfbsewjXVoSaYyXr8i7zNR6GsJuEtZfTdAK2kZoCQk7Vsty38fRb3l718vqHb/tH/xvY5++Lgb6prjxDuzxjfvxrbVvccdfB5e/azxufvfGtg9Yplu3d4OF3APAKyCob/7upuf+h4hLkeghk7UvfBSoew06c6KwMQOdBnkGOAho8e3YDILmQkMCzAnIXDFpr7kj3ARwM4DcM4lf63qV15f/Nq6xm/fUFb46kLDZH0PDQe++UKMovKjwr2C1/Bs8PXlKkgG1gFuTXm/0OrHHp+t241BuB8oHP98gLL/JBPLe4G8QI6oRopONyYwUGkMyFskgzDZZElGc4fJLLpFCh0oETTSNrHcju1wpNTO1xmRejFGskGsll0rNboNls5NCboXcjF3OjNLo1BhTocTAlAIxCi0YMSlGIGAk9ZBjF+wqJK8ofZxHWueLs2RCMzqERfQqYcS9V0WE0AUPoZnQcGZqMiMDk8HUvKiECliYPWcHWp7FrZNSo8wVqkqQQC9KJ4GlKXFs1EmJ84t/QphaYWxuuZbNaC3QxZ6j1YAuO3IxrGayZkquiJ8Apk3GZKSWxuX5ObsxU8lIDEgKBBX6ELy4RRopCjJXe/TV/PjmG16bS7qYRGLoqEHYjbCuE9UaVV0zcaIyWsYN69xsrlpFZtY3ztpGbm02Ax3EAZIHhF2Wpxa3oMMLXWIKxKDzuRNacTmToIpYhahoc7fc7DF6CpAwNEOYCq05o5xwL+qWC4pbzg2aUPDK6O2EhuoV29JD17t/L87eRUekbDeZCZekqRaYBCQ7QM5FmZrOBT94QRTNw+tUUNc73Guj7p9fiqtSSnOvErO9a8gsEh68hsMn4O0qzKBiaPMngcz2gEGNrkLs8JBvcMs8E/MCmQKlKkby0H0yUjD65UMeH/9tXtX/me32nNgpyI6c1+Ts+afWKrvdSJM1fQ9Pnhwzji8xHrJYHjPlkV0+p+ma1PeIdFj2e2s3TozVZr/kJZfrDZuLDZSJoV9QNfPd7yy+1sPrt7K98wX1tqf+bczFfVDbbxJ53YX2ftP7+U21d7x1b13+PojvPr8dLHPF7NxoN0/3IQN0CATnyR7zuwwLkHGA0gRq9v92o1coVoMsMM3/VbveV/WohiBOElqcixUaFgKcdLSnAZYCsXjRA8prIdI3hmkPpA4RzG2A/QAMv3a8t2zrW22H4PtGf+zg+z24bgq7Cj97BT8JfP+9v8vJ8gQlQXMLhz1ie/1q2RuefvMuHnTUv77nxu5f7WuNEKEzI2r1nCs1Bp0YRDEr5PmlFQDU7ddCcPYuRpdfaVLJUbkcC8WEMgPGNqPhgleLTqWSs1ebjs2YrCfnRm1CrIIUD51tpsy6OTjc1oluEdHUsYiJRepJ3UAXZ3cPc5YrRYXoNboRgWJzibq6KwMNzG3ngsxhy2Zu3WWZIJUBiDRCbYRaPUxcma26IhIDLVTGWuYXurNYgUCwSgTUhM4aqoYtGlEDISSyRAeXrTG1TJ4yrUEIwhAivXTUarQSyDlRSiNXzxsLdokF4XS1ouZM1EyWRup7YhcRFXIzpsklZtwzNqLRnxW5ZLDmlUmmDipDx6JLrNvI+TpzvvPQd0MoBluUjUXGZjPT6NWsDbdlC8Vndy6BM2uYC1QpBIVikU4j0ioRB1pJDLWCFJsrbitMIxKVGAJ1crBZJnfdGFtDMUIQwqwTeL6+oE6uK5hSoNZCmTJWGjEmVkcnnD44YrVakKeMyYTEQNcLUeLVM9TTOjw0u915MVIQCEmx6jm+rcGuCLs5HIsZV4XNzajV/YuDugtIkODjazLPKZUws9PgNoe1eQWzmv8n87RGMaQKWipqHs6Oyf8LQC6FYI0YAiT3BC4YORemLNTspcCqSg/E0Dg5fcD3vvsDXnzxE6JuGIYtm4stl+dnmBkp9SyPe3KO7HaVUr2oYyod6BH9QiibCyRkhlUg9QpS6YfAyXLJ+SRsn63BlKgd0ibK2JBmMBTW240Xo/xVb4fMh938cM+n9q0UwduWeRuQka+16zdXfwdIemO77wjj3bm5r0MLvWN/b92M3XI67LWfb9/+fbb9Nfr/1rHYP5kO+zAzbHtKaD/DbDMpsTNYG+wUXWe680w/ToR1RnNBMuikyCjo5PJdiiFRUSuEMj95TAlVic093Sc1zt9rfPm3A+VHCzgewfJB12475kMx6cNjuW3Zm8f6lkXvbHctvL+mboLTm/05nBDojNfmMdZ5pp+Blzv406+Qi0f8/t/6mwzdgMx+7h7IOjhuE+YcqAPm7paeH5Cit/f/m7f7O3wseoiGNiGW7CAQr9wUf+MTyCgTCfcvFYKLXteKMoFFqvhLLmhzeRhVatC5dKPRUKoEWgvUFplq9EINCy71UhUpHh72KHFjLIUyjdS8Y70FmRKT9GxsB2HrTsPV7d6sC0gSrAsQogtCTwWxBBqdDZHmIe0YGIIyVA/rChC10WQkBHcu8de2uiBvHzASKQUsQCtC19RDs3g1a2zmYUdrzgJGQfpAC8IuV1oZXbOtGcxahH3wnMEuwUIaWnc02xGiIl2kWSC34KADZSewDAtOjxdkaw6ka6W2Sm0uhq2Dz+RUAhbEQ5LNaHVmQ3E630RoYhhb2rQldJUjBSwwVmNsMGrHGAJja4xToeWCdYFqjaYZncCqUc1ZOvaFMKFBqOTsbKz00UV4RNhcrPnqi694+ewZZfSQgl2eQz/MbG2BOUxMafNs01xdPMQ9TebLmXi+yv5mr758XHR854On/IN/9F/w4NETjEwphdrcCi2Ik8NhjoDU1qi5sGvNfZwzpABhDmdnEhOJ0pRWHeja/ABu1twKrnmeaxQlirj4tQhR1XNWg4d8WzNyLeRWXQi7eL5gMGNQIVlHZ8xMqFGkEsxIKgxdRFSp6MycV0ptTKWRS6BVoRWDltllr3Tv4oQp/Oj3f4Tkjjz9nCkLF5cTVitHy8iwWLI4OuL5s8pmnfnVl7+gFFiPZyyOOkwbq2VHDBHaJY0LhgcPOHmwIm5hysbldmK7LVgZiVZZrjpOHx4haYfqfwrg7/APOXgJ/jobO1j5627nm+z3WyNabgC6b4M0vK1vV8P0NUH2fbf/2k7u1ZE3F7sP0XvXcvuF9ZD9a/7c2xm8HEmfFJYfK/2zwHIbeXre84GcsEwLBpQokSCJRKInMtDTRSOmSIpCp0f0dITWEYKSNEKrPLt8yf/0v/6Ef/zycz4fOvhRcoFeKz7e7cZh28FB3HV8d1rWvW0cbg6k3A7e39auNnEQlkGuI4QEH1Pbv8PKDOACbAx+fgn/Co4Wf4vvPfwdOu3dCrZ51Alxf1gv4ohzr+T6sA9ZTg5cevfn9LCj9uvfLPcGfz8522FSGfolJwS2OIuUNLEKwpASaKHTgWCFSHV9O1OiKGHOfVIDVWGZFDP3c21W3AOY5myHGYi7U7AQaisu8tsi0ON+u8ps+UoRyHbK2W7DxW7nzMQOLDdKLoyjV+8WgXFsTGKzvl3ETGlZUOswDbNwdfMcMIxoSiiFTjzEKMFo7Ii9EmLy4gZRuhhYLQaWy0iYGkghioPi6qJxRIGkStBAU0U6JaaIdoEqlZKre8GOmToVNET61TEalKTKsoOFGaqFwg4LhoQe1Z6ldGCRUpXBKq0WVKCaV/iW2txD1zwJN4oQ8OKUSqGYO6C0qnj+pVJao7SJXCdq2RFChSEwlUBtyq4auwYjQlEoGslRydPsMRvds1BKoFV8bMkY1WO7qbgod3YgtFgOaC1ETfzkL37O7pc/Jf/FL0ASLDsPU0iCnCG50DNBYSoez6T6TEyjx0oNrkQ4W/WHiib/frehfHHJl5fn1H/4d1n2HWMThIaUHa0UNDYWKbioNDDVxmQFk8zUKlNVzBJWM9UqRToyg2sl0tz32Fw3sbaGSZxHVubwtGKa6DRiYbb9o2HmntVFAlmEMj8MTJo/VJs/fLs+YFGQUP2/2AhRiCnSmnnqhFVyLlQUtCOkHtGISsOa0KXIYhHoug3f+533OHv5ik9/+gsuzkdKU7rFQACWq2NC6ohpwcV64uJyw+df/IoQE89fPqdbBB4+PuL0dAE18+zZ5xytMrtxSanGydFA/E7Pl1+d8cXlM9p2TaLy+EHP+x88IvYFrdtf+4H2W9Heyfh9Gyjo6/ZlbneCm7dSUX952tu6eF/y8DcRRbxXe8cYHwKS1/6eW5vz0uL8WxW4aMSfVn74Lwb+u598n99rP+DJ9z7k0ZOe1BnaB5wy8MdjUJcs8+JeT9Xx39SL+MDVK8R1ctePn/Lw0TGX/+b/w3//9Jfwnfdhpf5cpl7LylyFSg8O9RBsfWsn5sZk4opVu20fd0w8XrOjE65yEqt62LzO8i4ItOBh8WcT/IuX6GdL/v4P/2seDQ9cGaQd7MAcB9hb9Rjv5j9f7+dds4b736f3Bn8/PxtZdpFuu+VFrQyxkIaebkgcp8QqNJIai05Ydc4YBQGpzS25rGBlwpq/ynrmatHgL1w1CDP31+awoIjRqhHF6EN0t4+aqWZeLRp0PjcKkni4OKXWE6biSe27XNjsYDeNbPPERc6c5Yk8exNP2cO1uXVIaxQaI178IWZOTDV3HIkIMue40gKtFVSMoEoMQpeUGDOaJkQqcYDFskdVoVYsN0ITeol0XUdIEekbccpo78n6jAW2DpbUEp0krPauedgqRYUsQhKl6RFVK00dKIhNwAyOphHZjB5GNSOGyDIKGgOSFA1+8wbUq30BpGMZFwgKGn18zTUQs20wa5w1lxcpFsjWMRZjnArTmJn2+ZsCZupFN5IoDbQlxIRiroCIzvYtobngdlOiwvHJEdTMYhjYvnzGJ46W4fghHB17eN4EpjybJ88G2tpgMc/0mkFITtdRr9fZbP0hk3HguVhA20LbsVgk3n//IdO4QUaQanSx43gVeHI08HjoSRG2uWPcwE/GC75YT5xvK3mqjJtLcq6oeDi4iId5m0SIAaWjpUS1eRZo9SqcHELwgg6Hfc6+Fi/iaXOhR1JBCEQTOkusROi1Y4gD0nfO4vYV7SspNbqGTx6mhqd2Ghq6mU1P0IJrLtIQmcjTBV989hUny8y//dNf8tmnv2TcPGdz/gWw4dHDE4aUGKdGZWKXG2cvR548GXj65AlVLtlOa16ebVivYdEpIQmpW5HHSGs9MfUsF/D4ZMH2pXDZGzUZQz8h7RVHiwWL/ui+j6Pf3navGfstLMb1Bm5Z7q71v0675UX4xmbk9d/3f9y13L3prPv291tAZYeA6Q1GZf/lLf0+9G9947vbdvSWvr7xAn/XcvdY/lC4uXkxGDXAp5c8+Ynyx5ff5x+8//f56NEjBhoDlUL156E6yDGT+fK0K1tUEKQKrc1qAmpUaT5Jtkpvxof2gL81/JD//l/8BfyDp7C83qbnwc9jeqVt8rbjuHngN679N8LI72C/D5nGO3ctd4z1DZ9iyzPr1/z9I+a5kZcVfrlD/iRyvPhj/vCjP2CQRNeEyRKFuYBvplVe78RdYO1tz4DDtfZ84cH1ec+kv3uDv8+3imzNBZZboZdMiJkYRo40sJTCEGHZu2Dxoose1gq4W4fO2mLNvUWTBNz8TUniY6l4XpRZc/M4a7QZeXcz0pYArVVazXhlqCKiVJtt0pqQ8By2LiVWS6g1sC0D22o8NeMyV8ZaWU8TzZSLUdmMsC1uq7aThsxhuYCHrVs1yB62TdaBOaulQAqBFAOi4pWcwPm6cDZWYt/TxUiviV4FJWHW0cZK3W7JrdBiISahm0WiVRJRA0UCVoXQDBWXntkpBFx7sBWdQ7LOGYmCUhgoxB5kzjH0ogWAchUut+CXYSQhwCIK0RpNErXNeXMhzJX/HvK3aWI9GlYyLWfSbHnWq4tRT1YYgyLaYdGFoGttWFGsGEIhSCUFSKmjRZhkIFcn8rrQKDXTl8iyVjqdGUIzdJoQ7agSoA/zcdhszReu7xMxIEGbrh80cQUnR2i3wKYJYwdtDZtXgDGOWx70gsRASomjGPnuowVPTntOu8TSOprN2HwJXDxkt3tOli0iEzEYtaVZBifQooJ64Y5JRIJXCwPug1yz6wiaEOsOq4HZrhozQ9sEaqQoXtQShEBErSJNSSgx9GhIs32fYLmRc2Ws2Scb5pqUIkopgBRqHqnZqEVIGlkuEsul+rVWDdEeaT3bXeP5l+dM2y0PH3WgkZfnlzTrefLeE378137AsPxzjlcrfvC9HzH+2Qt+8osvWW8uSQEenPZ89OFDPnj6mCcPTihj43x7zvHxMU8ePECrcbJasN68Yr1d8/yrX/Lw5Ac8/e5/KvZuNx7o/9GYpne0+xIywG8NM/hrt2/rON8G3G8DObetdwhMZhC+1+ATcVH4Lza89+I9/nr313h0+pgYPP4yVSdoMM8fFgLXAuQHRJ01lLla1fBCu/lzoBGlchThvdUjhj+H3U9GeO8JhM3Vs/s6v+1QAfqQChTeBDv3HL+bYPLq9BxOSm5hA+3GOq/9cRNwVp+0twyxOQMUor9UpwJfruGfnpO+XPBf/fF/yenTJdtWiXSukrHPv3SvJuyNffzHafcGf5MmdqURrTBYI7cKUwMqlyYkyyTzF3un5lp14mHOPnhYNMaAhkBKHYuuZ9F3hBhcmy0I1oxabHbQwAWgY0I1klRoVmYfYdCkxJRcR82uL/5SMobLvuTSmKaMxEhIgQVCrJBKowKntZEn4TQ1tn3lshS+LBOleYh47zMrraC1sTBhIcJClaiGVnfr6ESIGikom2rsxuo2dBjTNJFbZpJAkQ40uOxHaFQRqhnrzY6xjPQh0MWOLnR0ITCEQJkiUY0YjN3YXPZF5apABWxOUXALtxgDFiNxrt6P4j7CSkPqBLUiVrBSEGsEEkmCs3ItIIwECXT0iBhZG8RIWiWWnRBCpY4N0UqpFWsgwwBywiiBtbj2I9LccaJUpq0nAPed0Uc83FyNXW2MNVHLXOiQKsWgS0pXG1JcxiTNIXVInF2OxG6gqNEfPyJ0HdM4UqYJTwKt0ITlyQOGqGw2G4Imdq1yEh8yTi/YbF9AvYA2YutGPj9Hd2u2588Z85rcBajHXI5LVouOXjPNErvR+PJyw8fnG84vJ1ouxDanIEgiCdTZd7jhAuOl+bmR6ow3exUEFaTJrHTggp+qoMEnWJWJggIRMfWKbIlAoIr7EU91jey8oMUsM9WRqUx4oU6HSE8rLl4u6rqHFENawWImT5ec5YkgExp2PHs+YgS+8/QDnjyEnI8RNog1NCaWx09ZrB7N+o2Z9faMxoZhcNceRkjROE7C6bDkweKE4+OOF59/gQVIXeLx6ZIHqyMuTyMX50vWGy9Q+uC9pwy1+zafbX95mxhXGmHA1Qvvykpqv9zB5/3L7E7mav8yvI2Je2eH3ujK11r3qgs3d3xbR+7LfN6XYft1212szx37M24c550n7I7fv05fbix/G2P4hjTJzEhFczmWbUNzIvYdopVWDAl2FYkVCUjVWXLKr8lmrjig6kWJZuLPealzRKygktAakQo9He8dPeRDHvDTf/oZ/PFTWMzyL8Z8/vaAT+4GXnLzu9vOzeECdvDvHlDdHOs7T6L/J4ff6cGm9x/mfD2dx7TLUCawBGWAz0f45+eEf7biO0f/iN//3t8lhkdUS5Q2ebGMNWYDWAw7AIP7/e55u3tc4AeH+OveDvcGf1vxAonRhKAJib33wtznc2rFmQmrpOqzgoCgc2Wks8DqQy4jnYyorAkCncJChSDV9QDnAdJgsy+ukmYwGURp1eVJUheRoBQaUymousabSMDMreFqnYscAI2J0C8QApspc7HbsRkdfOVW2TVjHZWaAl2X6Gav3T52HEfhVCLHGujDgpiUhQrSMrW63lFusAXG3cSz3SUvL7dspx0iBdWImTHWSq0jVioNl7ZpVJrCtjayVbZ5JIrSCfRBSUHRKKQUCWL0mpBcndlRn4SELqIhYjOFFAKEoFiIxKAkMjJGym5kyltKKdAgKpSZ+fHQuyHiOotWGhOZQoHLwNQqpWTKNqPV6NNAPwx0w0CKS6p1hN3IxfaS2hqSQEw4HXqenPYMqZBoxFkzsBTh+XklmDDpyC4pZUiIBAbJ6PYSzl/RdIEcHdP1AycSMA2ERU8Ix4x5SwrC8mgJ0rhcX3C0esCH3/0e6/MvEBrjboutX3HGOax3cHkObKFfIAtjESohr2H7nFpGzi+V8xdbYpdIg09QpuL5qZuS2bYAlkjVaE0IeO6mqZKD0GJAglGaMpbAbqpMu4lmbum3dzpR9fxXCZ43apoxClGdkU1zvuK+IE4lErpjprJz9pBMq5mxVmfTu4FhOMZMaBYI1mOtslpO5DIxpIEuKEOfSKFS6wVTy9RiPFks+flPPme3/ZLjZY+1Jev1wDROdH1ieXRMacaXv/oZ01j48qvPWG/OWOmG1TLxez/+iGmzJqrx8NERjx6fcny6ZLkQwnsPEdsgbc2r52vaCO+dPOBHTx/x7OUZvzw7QyO8enXxaz7OfguayDXIe+O3/b83X2rtxgJ7gPe2jbzru/v09Zut9vXbTYrxbQDxPmHvd7RbK5Tvu63Dt+/Ndd4Vpn8XCLQb/95Y9orZO/ziBsgxYBbkp4JUY7DASjt6IpLzjDcaah3RHJS4mL0cbLHNxRpyJY2FGsGMYMWL2cTfNSrKauj5IB3x00/OYJvhwRwyVbvGfeBjvw//HuDBN7DaXcynHLCch6HkvUj1YWh5v8xhDt9d983+vtyzhraX1CrAPs1IoXW+TInwfIJ/tYN/DkebY/7R3//veP/Rd9gUsNYoxUi4jZubGuydT9rru37jw42hOPzyWyTX7w3+qkAXIUoHTdyOal8Bo4GmhlghtIKrmlXEIFmdNR/1ygZNpHiIl4aaEawxiAOPao0inlvZAogVpEEUQ5oh6tp7CpiYLztLY6i51RtzSbXOeYXV5nwqVYg7r4S0xtQqY4VW3aFkBMbis6M0NUoU1sEIWnlhjSWRIyKLuGTZ96yGRNcpEh3UqgrbOiG9EkxZto7QBTQ0oibP09LEoIJoo4kXX2yobGtlmip+q6nfM82QVikYRefCmGpoq4TJ0CwEnGUNnUIfCEmQaIg0Ukr0IbCIgUQjNeaqZkX6hecrdsmBq1ViC7RWKblQmajg+opWKDS2NtGAThJJF2QZWFelrXdYW9NKYztNbLcj01iJUkldx2LRM60TKTSSQh8TQ0wstOdxSizwsfikbDBRUpc4GaDrwizJU6h1x4PTRzztTvjJL35Bbkq/HBl3a+p2Q9OARKXt1mxtx7hJfPHZX9Cev6ANCx/rzQjbWY09rqAvYFvq7pKHq8jD+Ijpcs3FujGOXghUqeTQaDbnrhJYxEjQDhX1wuGpUEultUan14VIEpRaqzPHAoJ4KoQKqi7REtRzLN0WsNHIJBEXlVaZ9REbtUEIEetP6NsR0txSUchAwyxj+DUvEqE1Wm6Ien5OLSPb3ZbaRWLXMfSB42WgG5bkKuTLyv/2px8zTl8xXp5TxucEHd1iznq0dEybrefwlh0vnr1gmgoX61/ywYe/w8OHp6idIGbEqNSambYvqDt3+qAo2/WWzcWOzfmG3bhj8+Ah63FHpfHVixf88pe//PaebH9Zm+xFIPf/E64VW28DCje+e+1NcUgBvC1Ge1/m6X6LvPFWeo09ue9GbtnmawTawR9XTMltYOs2tu2gD/c+nj1IOATWh3043Odtn285jjv3d2OBu/DsG/mht00a7Pq/PcjZV2/UhlQjWiBZIrToKSIFlIa2HlV/dwax1wm5WXxezCXRRDzy0ObijILSxCewSGPoIt8dHsCvPoGXGd6fWbQ5Xem1/tredu7GceyXe43hPJj87Mf2CsvtJ1I3BvANBeQ92yhcW9PpAXC8OZbt+vNelWJfPV2BvCRdQPuzM+o/37H87Ak//tH/kR9/9LewtmQQz8NX90DDpM2sn8zw+nB/77pn73lJ3Ry6e7R7g7+oQpqZCssOsBTBLFCrXt2gXhDjByVWmSyguD9steAzDmbpDpk1c1pBG4jufUaNSVxAOEggIUgtc3jXcwajuA7aZI2KOyW0kudTLUTxbDfBQ5rFZieEudjAFIq5PEkDmqg7f8yTianBWBxwQvE8q9ZYtUYnO4aNh7hDbHO+ohAVpjK5lp5lanWtu6Ces7clE6UQmW+2+YIfpVLVQaqqeLhblBj9JgxBCL0RUvXk/VKpF8K0cdFiiuckWKlIFCR42UwXKp0qnQhdcHHnODNOKSmDBkwjrRSvJo490mBTtmxzdp1Ba0w0t+rTAVNldDdmQmlInYs4WoFaybVQo7orSDNKMXYbyOOOEApJoZvZzJ7IMg2spRGkcLnbMtYL+uF91mevKNvttaQLRt8rMRSGNDG2yoPjU1aLjvWrS6QZi8VAXQ7UkunCmpgyZQWro57haMXmxSs2Z1vqNAuQTiPoSJTCk+OEpSNqExZqlEWiiZAjlIDfwNIwSUhaYOaAv7ZCyeKKMhimkYxQgpAJHJE4KUqtjZKbs7UGe63DMBdzYEZuFREjBSHgYXma55VqjARtrK3RpCMEGEIkqkvSiGZy2TGNE7UU8rRFsksWhZBZSKXG6pZztiOPhS2Z9XbD2dlzvvj4Y/78L/4lx4vG0crol0d06ZhcRkot5HGiFXj04AGLUdh+MHCySvzge4/5ne+dslyE+f4zjMarF2us+b0hLbK58Cp8CR1NMs/PLrncjpgIaXFMnaZbmJS/gu2KmdnLRsxfXlluHbITbwM33Hhv/KbG7sb+hds/37sft7Eur+/iegwOQc09j+9eix2g10Mg+5sYwlvw2+vgxt6JX6+3NXfSDn68yWTtPzbxFJ/qhYapqb/6zLezdx+yef83pxp+WV6Pu4ldgRfBXbucC6sgRtdFTrsjuAA+2cKPTqDfg66bx3Uw2G/8fBOswT69abYpul6mHQC8q48z6FS7Bs9X99gB+LvyMj6cdO23P0u4yBzuDXlet3NpkY1Qf3GO/fMN/HnPo+WP+OMf/ZeshhNqho5G1+p8avRqdw7/9kNitx/+jdN49283r+Gbk8F3t3uDvyRuJKx4SNYf8riLwSxMa1LZqbkDjHJAnSrNwJqgFrxyVsTZmuYesJjRtIA6S5dxn9xYA4O4o0M1c+oZIwo0jGLuKhFEKergSOrsjGB+qTp73eZJgQtVGkYRpUilaqVK9QpUbJ4QuOvIvoCgEsHcyUOrEfOIWnNNQDxkquL5d1WMkNzNYDZgQKzOW/Qx1PnClNYo2qhhrtoVRWe7sxTc3zUEhdKQZEgTkkHNQmu4Y8bhQ7L5mXHnruKiyebjlebcyhSFPglDJwxjJohx3A+e00DkAmVtxlQr2RyIZzNMlRCjy94Erz7FKlabC1k2oxIRjYSUiJJI5q4dTXaYNWqrjFYo2ZjMHUySQB8am82G7fYcOzpmvNxSdjuoGbXG8fGC3/u9jxi3O2o752KX+cEP3qeWiWfP3Inj/ffeY7Xs+PLzMywkVn/jhy5VID3Hp0d88cnnfPJnH3Ox3YF2Lo3QzD2OLVHm/lsSuq6niQPpGECkEYIS04KY+jkvZtaOtOQi+dUoFpgMsgpbE6YMXSdYE8YMNbtYdC3urdsMkib3kGak1YmplDlP0ycD+7zWmjeM5YzRFqj4uewE99mWiVLWtLwjAK157uaq60ldQ8qWl+cXaEmM7YLt9gum8SVT3TKOG55/8Rnj7jknq47lMvF4sSTGyOVG2WzW5FJZDMrjxwPb7ZqFPuDJw4FHD49ZxEqwTKsjrfmsOWih66GMhd16y/nZhpwbp48esDrpuDy/5Oxyg6bAMi0oRTg5eXrfx9Fvb2sR1webX177F/ke+F6xGgd/2MGD/XCBuwDLTUDwtvYaEOGObb5tW7fRVvL6bzfJjsPFjdsBwhXw2IOjW7b5Rjjvbe2OgXoDdN2Fbu/5crUb5+eN39/GzdyCAu9cfH+irgbxgP6Rq/+kqjN/ooQ5Oubzj4Zpnd/degN7+7b3c5PrX80l75oDrH3UDjE6CTxcPGAYYffTl/APTn2Ccwiwr7q9H8e3IF45HKvDfw8A3SGQ29vVXf3dXtvc9eeDd6W/mP3vNq8vbb4/M1C48kK23p1PLit8vKH96QX8BbwnP+APvvcP+f7x7xJrcIm4NhFaxdzK4eCy2ef2vZ2fewPcHR7GG5fIN5+x3Bv8hbkfZuaMBM2xBuL5fGZApalSVGnBZg6g0VrBmucuRZNZyiX4y1V1rohxANbUpV5sHztrxmQgIZKxOYw1aw+Z0OacBYi0ABEhNsBc3NdQVJWm1T1bxcGRzQ9IjQGRCiHP9Ky3CrMejzOcYgqq7NRvqtCMaEIkzNqafgFpDExWSeo3imsmy34EAQ/J7mdxQaBJoZiHwa1WhOxuIKKEvf+rNp/4AL1EpIjbr2FX1l6er2vzPaNQjdDmPI3WCOLFBCkIfYJu5zmFixQpo3IejCaBTcnsSqFU9+/NYoxWaIRZ8FPpOoiiyOw+QfMZIZoIEgmyIGpkCJVohcw8E5s17DDQJhCULgqLJPR1wOoJi7SgkzA7q3jYchg6PvjgMevzC56/6CDB6apjtyscDcpi6PjwOw9YLXt2l1vOtxu+/9FHmCklCw8eHmO58Oxnv+KiFQg9mGI1cDkqn11AHd1aGYFIpWHU4tqBUYwUEqlVkmWievXyHABBxChWmaaJXc1MKFsLXK4rYzWwnVewt0gzZwJrdkDZQu/sX/AiijzuWKSeRRfpukBFmKZGLVta31Nbo5gyTbAtmVa2REYCW1IsrE4GVoOQt37Ox8svefH5r/jy5SseP36IccHzFz/l5YtPmKYtIkIpGx49CC69Ils0+eNBGBl64eRoAK0cr0ZC23L6ZMDySN1VLp6t6ReRMlszqrrWoCbj8tWOqSjrzUStjYchsFwNbHeFcbul1omL6TmtRR4dP/ymz7HfntZ6n5mqvzSvCz8OwlDAa2Dv5gP/jQKIG8DoMKfwEDPaayvd0uaFbgLCGz/fDvj2L8+7tnuz3XwB3rXMTRB1AFLvbO84xtu6/9b+3QS1dyx2nx/fORR79uldgHt/rm4Z8/3zvymhBVILRAtXOKaKK2Q4MKkzzrabG3hjPjAHgGlitNmWIYi/i2MLPB5OOSnC7mcXsFU47eaQ6fWz/7V8vdsO+zWwOC/T5gWugOO88P6A1CDMkT1TZ9u0vqHUcoWVsVkTES9GkD34O7zn5nEV8xy0HOGVwJ+t4X9dw7/reNR+h9//6B/x+x/8AwZ7SCwRzMX4X/Msnncsb+Ty3mxvgn+5otnubq+tdZ950NzuDf5EvKrRL4AZwIjLi4Sr89kQcS26Ku4w0PYnU3291FwuBRw4NhpNjEIDzZhkzOrshqH0NqBmVEvO2lHcjcKaV0FWIRhEMsGgLz1D8zJsVQ83izhoz1r9RXcF/swZyDkBv8gc7bfrOYa7l0DFGbk6awvKfO2YXSv3mAktKhMZSwWkYqK05smenjfhFdL+DBWSBkTKnCOZ2T98zdxCDVOHnzKL/5pr46nNwBK/6N1BAoyGziA7zDeJiqBB3T3D/EJ3b2CDAC0XNmxpjIzivsVtniEaOqc6CBZgzMUzKTeFIEIfYYheaBOCukxNSPShJ5hhoVLUMBFqdeHlaIGIskRYdR1Hy8Cqz2yHJYvlEb2esOgHokSouKf4NPHy5SvqtOH58y+pYrx6/pDL9Ss263MGOWF7vub5l8/51a8+oUjh8dMnjFMDi1CXpBAJiwEWvZ/cqWAIl6Pw6VlGa3VZFBq0aa5qrfNkoxEkgk7EJAxJ6SPz71BrYzONrHde5DPWxCSR9SYzlYzxihQLQVaE0MEskppMyeOl+0AGY715wcvnzzhZHPP49BgxpdaJ7WaklZ7jDx6T1GeQpRR22wum3TmLWOjClmATYksW/cDZly95tr7gq89/yleff0pNwmr1ESHuaPU5Vl+w215wuV6j0jg56ehjIE9n7CZDilLHwnLR8+TJgMYdeXpFlXNS1/j5r37Jo2lBnU4wjqgGY86IekX8eDGy2RXQ4kUpKdLUJzWiiWqJi82O8/UrKokUlvd9HP32trGHVCBWrkIU+8z3m2G9N5iRq7fXwb83XlrvbG9hW26nquZFbmGlbm7j5kv7PkDzrf26pR9vANIDILTv553smlyvcwWEb273rnXfhhhvO0/3OS9vGW9uHNdr273x79Vu9vucWa8mhKqkGojVvdodTPhzWJqAysz07bs5s1MHOYA+XP4et32tgvp3Fa6UC07DipOmfPlxhZfAkwQ64fL18w7eOOUHCPPqpfvaH/hLimswFWwW87d5IjV/F5p30BJXAHq/XpPrzTWd/7a9xtx1DYaJvy/MaxkoGTYKLw1+McKfXNL/+QnH8mP+9of/FX/w9B9yGr9PK50/06rnMBdVtDrm2V+Tvnu7BeLtR+cm8Hvz037N26+ovebfu+4lb/fP+cPQ4LPTQHA2bc5b0waqns8XTek9nZQ6e/6K+AkUjSTzSuDa3P6q4tZUU3BGT8zZlGiNzhJRGjKXh4g0nENySjaZkJqgzRz8sSPVU9CEqSIWiQRwKWG38GV2trB2FQJ2JfN9Yuc8iOagsms+qUACLQRUgxeYSCVbw5qhew9cUe9ZcHZzf6eYOAizGTB7uNDBYqku/OjgztlCEa+g0pn5s6YuC1J1zpVVRI0qhSyNKu2KQAjNyeZg5sUevTJIY4XOOoKem7i/D6NGFGFbPcz7qmZG298n0bUYRebxD1dhPcyINPKUmbR6LqEk99HVkWEGScSMzbO/iNGJMuCgMklgZRFtwk6FGsGipwKoBdSiX6Kz37K0yrBISOxImgjSE+MRi8HouxXWlMv1jt1UOD09wZoQJFJyYdpmpAV0OIZ+hPXkt0mMXObCi/VIL3Kdj2+45JDhFobmwuDGiGj1Ip7oIflmilWhVGEqA7UtKOKOMak3GhPKI4I0kgopuqSLWWE7nrNev+R02RNFSLLjFx//G5ZxyfT+exwvlO3mnFcv1wgP+MHwkKyn9N3KRfTzmoVWFqmw3XzF57/6BZ9+3Hh48pg///OfUdqWMr1EZWJYRF6ev2B7+ZLNxZdM4zmX6y2ffPo5SmBYRX743ccMjBxpYtktEW0MMbPqJyRO7GyHrIT15Tlx2bM6ecDx6QNiH2l5AjNKazx7ccZ6s+b0wVM260xJgbRYspuMVicqHd3wEBkDu7Fgouzy/R5av9XtJXDSwyL7g8VDGFxdeHXORwWuQ5832SC57X1w+3d3/f3Gj++ipOyW7w7afemGWze/Bx8HoOwNIMn1y/+NfR0uc9exHC571zZurnvbcof7ewuos/1yhyD+DrZn//thmPMNhHAAKPeFDje/P9yHiTN/c8g3zaYL+315rt5N4Hfd/zd2PbemBtVTqho6l0Z4hG2hAydNkM/BPt/B7xxBB1dFH20+rkOZvz0buD/vZrx2PHbjO2kQp5k5V5iLOsnmgv4jMJqzPVuBCXflyG7S4Ab2gl2JTzdn/9q8bZ2RYJug7Vwd4quKfBKIP++Izx/w/Yf/NX/rg/+GHz35fY7jI6xEmgiTVUyE3KJDPI1Xxc23nPU72rsmIXf9fmMCdI92f2/f4IKOjUpUIyhYVEznuPacExead6Kag7YwD4DPNgKhGuRGqUYxT0gXcTHnZgGVHqUjKA42rUeDYqqkeZaXUeq83yoBTN3I3paoRqZgmPqLXFuFOTermoKI5wKazRVNwcFQcVW1OoMoaUKswqJAbEaVmRlUQUIim7l1msx+vfMFWlCaRYxpnsTMOZI6h8nNZmDhXrFTq1hz8Kfikh1XBR8aMBF35GhCNAe6LRstGjkYWeZKYFWiKQuJnAblo8eP+eGTB6TlyEIzfcsM4jm4SZSFdizTgl0VCpVfnm35dy9ecZZH9wIuhtVApKNX1yyKqhRxeRgxrz4uFkEbWZVKwFqgawK5YNOcxxkENaMTYRBYBWXq3JqsTkYrhcv1xMuyhu3Ew164KNlDxarEmHjw4BGP3jvm7OUZjx89IXQnyHCE4BI4ejJgJwPJKv3Fjiff/Qi05+g48MWvnvHy7Jz1ppC3I1xuvKKni5iN7LY7SqmYOGsKRhTFNMwFQI2ZVvVbL1Rya6w3O8ockhAJLkskioRI0IEhdbSYKWHCakG1IFrpoqCaOb94zqe/+DeoNB4MgWEIRNtQ8zO2u8Aze8G5VqZxSy7CYlG5PP8U68744leNk6MjHixWvHr5Kb/4yb/l3//0f+DjT/5XToaH/O73/4jnZ684fbxg0WesXSAYLU+0uqaWV0zjBdNuopRMa5nuOLHLa6ZpR9uNvHfyhKPVgiEkYqu0XHn/6QNaO+H87JQPP1pSpor2kc125Ox8S2nQLxaMtXCxqyxFOd+MXFzuOFrB5VmhFiHGJSEuSOmUYeHWOdN/CuDvX/wS/vi70BawEohlZjHadUK7ulrBay/5K0Zkz/bcAICv/XEbt/CWF8Jr5NIdbNMVLrpjO1f9+zrttn6+bRtvAXZ3uke8i9W8+ftdSPpd4PgmCLu5zB643mAbbwLgN7axB/937Zfrcd9nELQZzFTzXPG6T9PaRzIEpFylBl5vXm7b+lXfr76fmbmwX9oqMcJqmegIcFngRfFQadNrpu4wrCs2X+92nZ/X5s6YzgL+M0OnDWRyt6dgkAwHaNWtPrcCzyp8keDTinwyIs8znAl60dAMWoJXO1dxB6rmbqG1zmlSLeOyLnN/UkOSgCbQxxzpd/no9G/yOz/+Q75/+kc8Pn6fhUa0NoxCE2VshWiVaF7IKlcC17eP6pvt9nv49nnAu0Diu9u9wV+vFbVKbZkg4mCq+iZUE20GO35uvcBABUT9Qqkmc+hQiEGgOCvXTGizFjZUl4PBiz5EwYJBwyteJeGh50K2hqigEjGJGEYm0wJkye5he2XRJoRqSNtXE/s1ZM3z7GhevCAWIMicp+egzaLTtBEDqVRAQ6SJUi3SSqPgAEFnMUdoiBWaFKoEPxY7qJRSRQnuy6GBeRpCk8o+r8JzgHxWeJWJKCDmeorWoEhm1JFRMhUl1iUtO5C+/OIlf/H8FTEZKRp91wihELQRDGouhP39h7BYLBmJjBIoxDnNQmgSPCzf/LwY4gxqNQ+DayAnFxs0FKvOy1rMZKuUJrTi+X2xCQOwpPFKK5/GRheZbeQKY5vIm5d8+OCIV2JkDVAa0/qCjz/+KX/yzyralD/5p3+CHJ8S+wV5s6WViX7Zs1gtably+eIVX371nC8/+xyJik0jqT+hFMF2PiNj8ORcxkqpBZlfwIYzfHkuxgBx5lZwr5RaaZOhUUnLB0hpbMfCbmxQC4MKQYxJBJWRIRpluyHnHZvxku10zqOTnge9sP3yZ4xf/pzSRv63j/8VD5+cMhwnTu0VZy8u+OLzLatoHB0tOD19wOnJCYN+TkiPyL3w8Og7/PGj7/M/P/+XfHz+F7TtSx4vj/nwvROSfsr7TzNNf8Wrl89p4xlaFpw+XnA8CMeDEeIDyhT46z82Li7W7KbK+Rdf8atP4BGQfhRYfs9mBrij6waiDnzx5Qt2o/HegxUvLl/RpkauSqvRx3IUchY2m8anH7/k8mLrloxtS53OKdWIYSDEBbnCOK4pNVO30zd+kP3WtP/nGfxsgv/iu/DRMZxWZwHj5KK8sjeBB7czMubyteuXyD4sdpP82X++7YVwayh03sBrzNRNJugmm3AHiDoEhlebuwlibnx32KcrACOv68Ltt7fHXzex0GHX5WCbV0DwEEzdBE23gc+bx3RzR3csexNAv4FBDyV+Dr/fry9cCXoervzadvbbkINTNS8v9Xrj+wIIUUzcR74FJy6UfY78DX7vVrbolnM+b9tJPH/nqbmPuMZEkw5shK8u0fIejQSMnoMX8FwlDjc7P4/3KRBXVSbGlcZeZ15lS4DYe9HUmOFVgV/u4N9t4N8I8rOnPHzxXR52DzjRI1Z6xCC9F8RpJEiHJJfZQgvSeqwGzCI0w5rvQyUSQkJTIvYDqV+wPH3M0fGKxXDEol8RwwBmiLi5QLOMWMaTwDxMvQ/wyhvX0h3tznv4bdfpjWvlfqQf8DXA33vJEI2U4HdmtcrU3Oljb8NW6gQ4yyPNqGQaXvRRxZk9QWcj+wXYgIrikK4RzYHSPpy89/xthVlTsBBaJgKLKtA8vFq1zMUchWkmo6VlzFyA10V1PVevijFJZhKg6pXsRjQAJbbgcjZqmBYupRAEegM1j5c6qwmdelK7RfOXmxV0yiQDa4Uas9uloXNZPLiwRyBoTwyeg+EFIG5rt893VJtzK8EvMgwJioXsxxojKXSgStIODUpKAxoTmv0q2FUI1dhhrMlzdpo4q6pptgsriBq6fgXSI2HFKi5IIWDamGpGzFimiAG5ef+yNXKDdcXL/oPOBd4V8PNhs+aiBPG+N6U2YcIwqXOOZ6OakCTSNMCwJKfZIi0GCIEQex48eMzT995nzJnj94+IwwlpteTsy4pY4MnTxzx+/Jg2Fl4uVjz84CFx0fP8+Wew6FkuHnC5qWSrkJI/Ry82UDIhKY+fHhMD1HFLLtlnhdmBYZnD94IQO5caAqh1pBOQZAzRUHENRvLEKjVWw4KjZaTUnmodz77a8tUXF5x/8TEv18+ZLj7nvQfKy7Nfsj7/nC2nsFnw/pHwZDhid5Hpw8Rqmam8oF5Wvvz5BcePPuR77/8dvneS2NY/43LzZwzdK370O6eIDSyHwjCMnF2+4rNffsb6xYbYB+pqRdsFEGe9A4kYBpbLwMmw5OT0mOP/7A/I0yVl3GFj42gYWPSJcXJT5GdfbDg7m5jqxPZnX/D8+Uu6/ohilfPLV5xfnrPdZqaxQlKCXLLZZN57+pRmcLG9ZNF3rI4WDItjNmPhxcsXfPHFhiePvsaT67e1fQbx/7Gh/etPiX/vPcofndA+inBqsDBXHKjzv+BgqMBrIc8r0DcDG+OOl/eNdit5dfjy2COs+7IUhy+124AjN7Z14/OV3IZcg7/XwpB6vfgbfb9tXze7d4NFewPs2cG4yQHBd7jteZ0roHXH/t85bHJ9vHaw7iH4mwmt14HrXQd3ox9v/H09SRDzMG2YMWGbd3yosWy3bOnmnry118bCa5eEhHK6OKGEHsoFPIc4DhQyTedc9+vkeF4LW7cZ+FqbK+Bn4kPbHFScz1NJcNHDC4WfBfiXjfjnT3my/SF/K/4NvvvhD/nO7z7iaNGT0hyJiUqIvn2b+10aV9e6MtvWoTOYC0D0ItR9xYgpuYBJI4SGkp04kAayA9shYnPhK0CkSnrnmfv229djA+8N/n6wNOLJwDYYu11mnIxtNqZqYMHDtpKcbcZflFU6mrh6+FwSi82erCJQ6oS1QKmNVnEPU91P/JyFCSKECK01pKl/ZwEVcXZ7ZmpaKVSZCOrWbmrisf6ZcTNRGjJXxXoyq2pAraLmIVM1BdP5WBo1NFry+L21Oem0uQRH1Di7YcwFIyIICY2ZrhnZlECPxEiS6FpsZi7p0ZSkkX7fH3F7nWpeHOKzqYrY/jZ1ocguCI3kRSgqpBRIIdHUJUESkZg6QlGieFFDUEHF5pzCSrVKaYUmrj0oFVpxHSMVoAVsLGQKxlzpGj1BOLfGtF2jNHKByYxJFEJHDJUUghetaKOqEEKkk+h2bRqIpsTmuX+qAQsKakQVt8ApoEnRfgGpw0ICSZ6X2YTf+f5HhChc7C5J/QmxH1h/cEnLWz747kM+/N4HrM8mvvzsOafvnTKacf7ykk4Dfdfz8cfP+Om/+gmvvnzprEqIkDp2uw0a4fTkCGkRyyNdBnKl1EyWufK3Qp0yuTa6qMRhRa1GCxGJoNKwKdM2WxRjMWS6vjKqUIvw7PNzLl99zLj7Ciln1PYlj7qBVXpJ0wsunp3zfFfpHp6wPDoipMJYtpy/+Ippm1n1j0mLM2z9FS8/+XN+lnqWRz1Vtjx8ALkY6/U55+vn1JIYLyfaq0a3fsii74h2RLAFKQVKGVmf7yjlAqWxWEYeff8Bf/jjFZ/+svHlZ5dcjmu2ZULtCO17dtbYTZXVycCSjhfPzlgsVvT9klcXl+QJLl4Jz76qiAqnj3pqDdjUkzihsqZUFyTvlrNuZQXplP5IOH303td50v1WtriGE2D80x3jv/8U+3dfwT9Ywu8P8EGERfWXXfBIA6154nmL80tyzwQyszDgD6YbdNlN7CA3vtz7uF79foMde+0tJTc+H7xk2o0XzU1BXw5ZsMP9N97cl3BdDXoISvfVdQebvQnGbubV3WRRDoWi30A6/oJ/EwaJj/8VA3sIwIUrNu+1w7PrvlztY96mHiynro93tbsWoQUP+bc5PLU/1aZcyQO9PmDXn+UQyNrV324YoHNeuvNQnvU+FyFex5XuAUyul9xHsQyozdU5JBoxVJ+sfPEKLib/nDzv/oqR1DrnVylXcnBaZ0BYQfwd4vZTPWwA6+AXBn/yHP6k0v/0O/xu+jv80Qd/zPfe+4jTBw9IKYK6tDJiaHAyycRxwF52zcPMLvfW5rGz+fjkyjJ1nvCbK310IVBwhQhpI2Y7LEyY7iBmsibEBmL1QtRrwza5Memw68vs3cP85nneb/mNbRx+cT+Ief9q380X/N7vHMFJ5csXhbNXW85bZqyK6BIzQZK4fEtKhLigoogkxCBbZlu3THWcpUuUba5Yq3TF5VRaLS4L4zXAIA3VSAzClCsiiVa4yo+eZZKv5iAZZ5qmakTmXDlxpF/NKHN1chQ/7UkqJoXaGsESg3VEorOYVimtUnOjWKE0vGBF9/xd8UrUg2dVa5WFOnizmtF6SbCeoVuiOAjT6EUEta6xZkQx2jzRqW1/PEIAZ+TmHEC1SBJ3vNCW3f5OCoaR3UKFGBrKRAwyF8BEzBpqfpvKvB8vfJqZ1STksXC5nShmxNCTJJJzBmksVh29NrabHT2BSGXRKZqEYo3RKqONmJo7UOB5bUUiqrDsIl1k1kKs7lpijb0MUJW9DlIjxICUhErnI1DnvA4xSs58+smXXK5fsH51xpPHx7x68Yqzs5f0Ecp0TNll8pi5uFxzOW7olivKmImxZzttKWPxwp69wrxVWs589tkXfPnVV/SdEuqWYJkliS4IEiKkgCQgJmptPnGZdqTuCGtK1y84Oo7EaJyfjXz+1XNquaAPl+ymzJPvfQil5+WznkdPBzbbyObllvOvfsUvfraj1S3LtOA76YSPX3zG8/IVz18952i1QFpmu74gCDw4Mnq5ZP3yOaMFhsUxEh9w9KBDh5H19oJ2eUbJa0aWdGHJyUnEUuTo+MTFnzeRyEAeG2cvz6i1cXo8sIgLQhU+/fQlv/rsM549+4peViyOFnRHQnckpE6RjTGWHZv1Fk3GkyfvM22NzW6kZmN9Ubg4h1KNs1c7jhbw+L0P2G4LIRU0CLkULi62hCBMk2BmDKsFMvT3fRz91jap0KL682Yr7P6XifL5Dv6sg79zBD/q4b0lHDeI+3BZ9upgs3kC6pPUefbL68LQN9v8Qtjjw31r1YHG/sVkMxjbPyT28hz7Usg9rfEaoJGDZeC60lau/97vG67Zrauw5NypdsgCHgBaZQYGzIyQ53dfJartxwL1Yr3XjvcQFLXrvuwrPPdSIXv275CNOzyGfTGA7cGKXR/DfpzazFqF/fczhReM1178e2Z179O9RyP7XDc1sDzTaZ4jT9kDYpdlceC5p9Dm32zenxyO/fycq+a57ximc1FjdQWD17G/vElu3tJsBvN2wKp6PacQEkiY7+EvRuzCnK2zMF9XeyZv3kmZx6fV+fq0ueiyuC9w8FqCuF1Q/kWF/9eXdP/2u/xe/5/zN77/x3z/yQ94dPyAoeuQENEYKK3SqiEqM7j1bh4SqkKjzTJurlk8I+2D8fNQ7awoMgMuP60NbCKwf4d4Cliw5AB+vo9cHeVwewfX5hvA77ZBP5icyR2n5XDS9A0oxXuDv89/8a/57mkjnnXIaDww5aRL5JQ89w0lBXGJkBCIcWCslSk3WjWyNBZBKDF4TpVVsB25FnLFBWlj9Hi8OOXamH15zdAAKQWsU1QTVv2mrK2Sa6bO4UgNAVUPO++rwWOAJo78mzR0r/5tlaKFqVY6E5YZulI9IVSbs4rSqGquzap73cLKbOjmYEKgmFFqZpE6uthhKMucQSKLLmAY1QpCRWdJHGuCWCPPIM+iYBL8nM8ASeCKYVQrICOtrmmWidqhmojzDC9JIwQhWiWwQyy6MHbzC55qHlaWgtsyOnwekjF06tVJNdCHgMySLstB6aPSrVy3Dx1o2hjHxuW4ZbLkOnWxQ4KSyw5aY2phnhxPsBvxYMM+9DD7Sc75PUXMg9IijOOGXRoBQ0W9z6VizVitBlodWJ9fEMJLRpTNess6T0iD9avCtJv46U9/zrbuCLGj7NYshoEHD94j9SekRe+M38yeksLspzyxGScs7zDLrKkkohffNIEp+7NIFI2K0LM5uyTngiB0nXsot1oZbUezS16+/IrnX73kpx//DDV4+eoTUnqFbp7z6tWnXO7OEBrb8ZJBK09XKxanj8myYVNHnr88Q1rmaNnx+PEpw1LIect6+pImC1adUqpR20DUhrQtebyktZEUTlh1x/9/9v7r2ZYkS+/Efi5DbHXkVSmqslQDVd0NNAdoGAASJIczZhwz8Inif+Mz+T7Gp6EwQgwbGAiyqxuoLp2ZlZlXH7VVKJd8iL3POVfmvVXVaFTNuNkRO3aEh7uH8M+/tda3aNMFMQWIns1Fy9VXG+q5RRhI0TJblBwdHHJ0OEEKwa8+/4Kf/Owp3g88OLXoskd6IASCULiY6IeBs4uHzOd3WK2W9I2nHwZ8DKSU0Lt5amgzSwfCrHh+ecnJSUE5meD9wGrokDKTsyYOGWM0MXTv/wb7HSsBiWdMUamTZLq19J97+ssBvgrwtxT88QT+4AjmNRSAGcZJUe9AYEw3wG8/SezZqf1i6noyuAVqbvsKqj3Lst++2/ca1HAzIcMtUAMviOvuGbG8i77cg7FrJLEHKYz7JLkDlml3gp1/441fDDeSB2o0gcu9CsMO0MAIAANj+h24qWBvHr9elY++5Dcs5J592oEz2I2ZuOnjteTObixChmx2DJ0cGbrr8+zGTaWdLMRubAVj+/cM636I5e5ceyCL3CGTHSDN8aZtOYzHBbnbX93Ss+Pm7/UA7gHDboyEQki1Y/5uEh5wnfDgNg7Jr1b7AsC/dU/tyYO8IxAYh8hlRS7syNRtMmnlyb2FqRnBXN7dOzKPwC7sz65AJ5D7CPgM6LHfF474z57B/w2+sfon/OHdP+W7d7/HyeIetqxu3I3IpLibV5HktGfzxI7Fy7c6lnasZbruUN6N3Y0XxC02cIcVRotmQIoxlebIxFsQGb0L1hLk6wjqcT32wsP41iJeuApv22v33+7ee8EF9D2A4DuDv6uv/orP7ZaQoOsDs8khd+9/k+PTj9CmGB0kcyaGnhAC2TUj25Y1QwKXPVEGgkr4HNnGQEwbpOjxwhBQFHpBpctR+FmOQpIxjhfNx1GaJeXRVJp2q8GUMkaKUW5DFaQ8psoaxYcFMYxBGEKNYCRmrnPkVnWBFwqfIspDlT01Ah13GRVSGoVlVCIrT5BjdGsWuywgQrLXO8pIkBarxnRqLkZC70hxwAiJLi1ZSlIaffu0AClHPcKgIMvxZZd3eYnHd+VoIpfJkxIYKRDCk2QgR89EaaRMYwCJTOSUMEoCkZwdCTsCNoAUSDmSckQSRwFmW6CUpK7tqOWXJaHzDJsNsVshoiM1kjY4gpYM/UDMnkFE1q3jfNOh6gOOTh9wcucOs2oCscQNIEQ5mod7tyMnxtWQQo6sLJKsFC4HAhGXw5gDOjpMGP0mlRy1lgQjy3F5vuIXv/glTx9+wcXWk3XNsN6SvMe7hPeSHCNt05OtIA6eMDhijDgnkHag6R3Ycpy0hgHSOG4b17Hue+LQ0fuBECUCvROsBi0ihTAYqUcdRStxPuIGj+s9ofcolZkuJNUMtqunPPriFyzPNmyWS7QaMLahqh2rq6dcnD3CDQ5jLG0vCCpyclBwdDJHpS39+VP6boWWEa0rykqjyzxKLRiPURpbeJTpIEWyy2TXkfxADjtzYYZpVaAnBVJIfBon58m8xhYFMWWMVdRFjVaKi6srvnrynLOrAa0yXfDUcUC5RNx2pBgwsqSezqjqKdIYNpuG6DO2KDg9PUWImqJoabaey8sGHzJDF3B5oHMlx3eO6Nolm3ZLii0CQ4oKIRSr1eZdX0e/w8WOpjKRSFJiksY4Rb4MhM4TnwzwWYQfePjjA/hgAvNq9Acs3DiJXoObHarYW3wlr59s9tsyNwzMHqBdM2K7w65DQDNkPW7cm5tfCGTYAbhrsLQHcbcQwu2/ewAR087EtzP37UFLEjtgyM70CUQ9/si8A6u7Y9XO30pmMNzM2OlWu/ZjIfbn2/d134cdCBW32MTbQ7cHddfjuwfEOzCU9sB3B2BHvbMRoO2DGoQeAU6XoI/Qedh6aPwoQTLsxkuXUE9gXo4/dYLSg+x3gDLfyFDsr/v+Wu3B/i7CdGxv3AXO7DJQifF9K9MoYXYrNORF0LD7dAv2vHgnvYQF90MkskAKBcKgVDl+uQX5LJIbQz6wY0DTPj9uELtrm26u+d4sLHbAumNME/cvNuR/0fPJ9p/yD775X/HNe9/gsJ5jdQlSknIi7aKZRzAqdxJujBamPEK7seni+vf14mV3r7/uscnXfvceECNYEhEhAjerld3YXTN9L+ZIvjVSLxXxwvcvPi1fBwBfgIAv7P32o14s7wz+1udf8sw6Np1nvWk5PnyANSWT2QFGFdQKCrVb6ShQPuJ9BC0ZyAxxlDVpnaOJw0hB50AnIkkqnNBjzlcpr8W2rda79Fqa3kd8jPgwshh7u7oWjL5wQmHMuAoUu4tGAucFyNH8ixhv/BgzEsnRwYQ2eCKR2AxUOXFkLXU0EDw9mZZIlxMhOmKWxB1bJUZDJkLK8QGUCqLi4uyMp2dXDMlz/uQZqY8cnBzx0bc+ZHE8AyFJJGRKyB3hTBY7Aef9GmRcaeQdfSxJ+OiZmmJHJ4OIoHMDUaAxWF2Q5fgiCCnjU0KJXYTg/uUs2bV5JycjDYVRFEazbragFORAsz5HxQbXtXz+xVOWzy+QStA2LaLQpEKw7QPrISDNlOPTZ3z4wQM++uABH9y/R6kLZFSjW4suMUaT5Zi9ZSTyJSprkky4DIGEy5Ihju/JmUjYOAb7INXIxIZEjpmyKqnmc7ItUEU9mjVy4OBozsm9IwprmM2nYMbX2LDdklKkGyLLbSKkPF4rdkyFlOA9z58/hxwxMGY4SYKQNQmFRGIR1DlSC8OkMEjhUUaRkyYHhcSghCdnhxs6zp4/4eGXX3H25CladlTlgOkb1lcdm+2arh8QwhIpkDmRPMQoqKdzUlI85zlE0EqNgD6PWorFXOOHeidG3ZFj2KU0FIg0StSErMaE7ho+uHOKLQraYcvsjmJwPeCxZtTddM6Tck/Mhn5oCSmiDGNeaZOQeoy0TznSDT1YgyksVTnFh8zy8gqBYjpXGGPQRpEIpJwwhcJYhVElMSWaxkMq0XqCVI7BOaJ3CCylKcYI/d/zMmYLGtkILzKJgEZQRIHeKkIDcRkJzzbwNMG3W/hYwp0CDhXUEowYwdDobb/34h/Lnq0CbiaRHcjKcO1nJdTOfMy4It6blAPXEiFjWH/e+4ns3u1y9MWSuzbslb5U3jnm52v/7usJVuzAEbfYRiTEHZuVBHgBXYRmgOUAGwHNjmWrFRyWMDcwk1BqMOlGQkQx1it3sCbtx2H/I29YtixGIBUTNGM+cnwcyUEU4w0vRz81xWiilLt6FCDHvPSjVEK+CWJAj2Y/L2AdYNXCOsMSuEhwFcfo1HUYpVC6PGrTCcB0UPVwVMADA9+y8B0LxxbKYTzvTmpqvMZ7wLcHrnsQnW/aKv0YIK527j4ZVJLktE/btgd7N2uH28Dh+v+Xo8RzvvX9LnI4wz7rlt77TnaS9MWWvJrAPcU1uNxf61CM95dOu9y5u/54xijCz3r4Zw3yzxTfE/8b/v63/imf3PsG9aRCCD2eLQsy+iZxiNhFG+yyYsVX5FD2QGkvp/wSBM4vPjXyeq2Trrfv76d0G3S9APbE9Z+95e7V8urWG3D6SjXvBejeZ/93Bn9Dt2ToarabnofPrgjJMD97AoVlUi9ZFFPuH51wdFAhUqAuFG3rISUCmSASzge2yWGjo9aWwda0WNZZE2TButnyfHNG1zukVCzmNaURTI9O0GoUUE4IZBb43SpWiTEYIeU4ig5HN0q4pITICZ0EUipCzPgESimUVlglWRgxJr42kmEQlIXkUGqmSYEL9ElQIWizwu8YxSR3tn45mmiFNkhtQRW4PvBXX3zBT/7yx6Sq4urpGax7ZicLCp05nH6X2WxCJIwANcdryeqUAymMq5abSF+BFAIlM0EEjmxJN/QMzTki9CyvLkgxMZnMMYenTBcHJBQ+CUyWiCzHvMlCIKQYwd1OIBoB/dBy/vSMEBznlxfYsoCQWD57xGJmyCHw819+yvOHzzFW0XcOO51CIenDGO0b/Tnnj55y+cVXDN/5Nh8u5hzODxAxIKLHqoJS6VFiRwkyEZkSInsCPUENJCRewKAEPiumSmBjRPgwvvTEyPBOqprvfPcbmLkBOUXZmqvlFVomTk8OOD463AFbQe/asc+xx+gaPUQ2w3Z8S8Q41qskaEUeHOurNXVpmdgKFzxRjpHGY15dhfeCzZDphkAsNVUhKGtJjFCZktnUomTHxj/lbPmML3/1JZv1hvPzh9y5I7BFIAwr2rajG0ZXhxQz0iW0qMA5tusGP4lImSmMZFJZqkJSWI3IAaMEp6cTrDhgudwQU0cOHrIiRo0Qkkk1o1RQqpLSKBaTmpgdLjumC8vT5y1XVxvqumJxMB/vfw2QmUwsJ8cLBKANHB/XLBYTrNWE4BlkJiTJZtuPPny6wPUD2hQUhcFHQT90bLZb2iYjpaYqSqQsaTcdm03P1WFLUYOUBZBwfoAUKEpFUdj3eM39bhZBJieFyGokQfI+XWXGMPr2xkaSv4ikq4b8iwY+BO6XcLeAIwVzCVMFpRx/VNqhdXGL8dqbQBlng7ADCCHtAI8DL3cMVBqByLVAboI+7bZzTW6BGH1ojIRCQSGhElALqIGZGkV9Sw1WghUjINRpnGkEQBhZaQT0EtoMywDnAZ45eNrCo24Uw253bZ9oOKnhgxI+kHDfjMBoqmBidkCQcUW8W+TfMHfcApcZNh7WCVZxBGJNHPvqGQGx1mAVzDRMBRwXMJFjqrJSjH1St7Kz7OYfvIDnHTwL8NkAX27g2S7TxRJYg2hBODG6UXh50z4CiBWphHwqyd8t4R8s4B8fwT0But/5fO6Ysj3o2/t77vu4B9lq96PFqImnrj3eRmbsFgh6ASpcBz7cYPfbZU+O5f3pGElOSSLnACLsVDOApImfnsN6AZidn+Pu4EEiBjNWX3Zg/ShAG8UI/H/WwL8c0P+84mP/d/lHf/f/xPfufRdhJEJoUtajZm9WiKwRWY1qGyJcP2M3nNwuG9lL8GovyXbTqGuk91LZQTixJwpvch7nPWQWL9Tywv8vltdDwTftIl77xS1A+TrQ+R5Q8d1z+yqPlB5dQhc71v2SR08/42pzTlUsqHRN+N7f4v697zO0G4a4GbOj5pE5sFJSK8FEJyZJIesJgwAvMq0q6ZD895//kp/87Odcnq+x0nLvzgmH84I7f/L3WMyOSEqTYsZqzXYI1w+595EUHDk6XNeBACMyxu5eVjKy2m7ZbluS0FRVTTaKi+UFXXDcOTmiUIZSaUoRsT5AcuicqaWh3wWBZOWIO0HKLBRZjfULaxG6Zh02PP3iMy5+/jO4+wGECN2WzfOW9fP76G9/k5PjisBoblRkIoleRkJ2xLj3JGQnUaPQYgwySYXmUCseXSz56uc/RPqBR7/6kpgFh3ceED/5DvN5TVlNEdIgpSU4ScChxWg+jWI0tSopyGlgffGYf/Mv/zsuny3pk8caS78dcMOK0/uHzGYHrMNAmChsMeYk9TkxtAODG+VtcAOBDbQNp1VBnRxHCqRyCNOQo0P5MSNKkqPwT06RlDzkFm0GBHOUsWgEWWpqbagk6OxHJc4xyTMuOFIYkD6iqzFmTYRAFpGh7TnzF6xWay7PLwmxRSrF1eqM06P7oCe0247g3LjqZ8eQpIRwidKW3L97f3xntiuENtTVjMKWpCjpu0C7CTTNwHbVcvzRgtIomjBgZKa2o6uA22x49uQhj776HCUiqIGm21JPAsG3RCmJAprWsb1q0NJyfHSAVZlNs2brrhDWUU8lVT3HGklhQeuItYLDWUHqHUPf0juPkBCjw3lPDJm6WlBNq7FfHlbbS1bbCwIDczXj0cNzzs8bTu/OmM01ZW2ZHkhEikwmlrI44viwxFpDUVYYO2ZI8a4jJ9g0Pev1Jd477pzOOTw6YnEw5869u1wuG8qzJdNpRYoeH9KOXJXEHtwanj09Z3qkMHpkdWJM5BRo2w4hq3d+cf0ul5THhd3eyT7uft9MUJIiCPxlJF4J+Axy0ZNnPRwy/hwxgsCFHXXQJtXIxJnd1LP3LRvV5KELI3AYIrQOVg56RvNax83/AyMQ8oBjzG+/n8nSDUGTLVDtfubAAXBiYaphUcPcwsyMQtYVMFE7QMIIPoOASw9PevishU8HeJQQFwLRSoSDm4ksAhvS0Zr8EfDdEr4zhQ+KERTPgFqPwG2fpckzBhL4OAK8qwRPB/i8gS8dPAVWQMN4rp1LGorRlDwDjoFvFnCvgm8ewKmChRyZSCN3jKsAl+F8gL88hx8O8FMQZyBa8UJAsGL3Xmd0KcnXac92xGuXCF8m4tOW9NhBOYU/1eO4FjvAmXbg+TroJ4wXR4TRJ9QEUG4E4VYQLQQViXLM4ZvlCE8kowvVyxk+XoUQLwHEPJ42CBAioXa510c8L0lK3uz4cDneZ6lkTBY+jIuBnJEEosyjObgI47UbIny+hn/eYP7siAf5v+B/9Xf+93z8wSdIY4Ax29W16+cuElcKNfr4iX3mrp1+7z4g5zrS+1Y3bm17WYfvFQZ0v4barynIO3J9rON1kE68MJpvL+ItNXz99hdh5u01z9eVdwZ/WSUa13L3o48IQLPxnJ89JD2TaFEhkyQPK779zVPIjkdffkFdLjiYzDmaz5gVJTJItl1PGByV8dfsWbAGJzU/So7h2SWXvzpDR0l6tqSfTeDj73Pv+COWTcuzsysuQ8/z8zN88GQh6F3Ch8xsOuHwzgEf3D3k/p0Zk5lmCNAPkZ//8ku++PIRZ5crQJHdwOrsjOmk4h//w3/ARx9/k3lpqXymwI1pz7Ih5YJBjjkMkxhT0ckkkMKCVKSdCn9Kgm3fE64uYdvBNEBhx9XMtkf1jqlQHCtLF3vwfszqIRO9TESlEdKO2kSIUZ5EKkqlqU3B4LfYLFn2W5a/+BHSJbqH5yRb0oiC9uguyUExLSgLw7yej4DFg0IjZYEXkqR2TOLgGZbnfP6XP6R9vMLcu0NlS7pNT9SR7aRD2xpTV2jnCWh8cKTg6foWkkQouVvF5zEARUUOCsWUgZw7VN4gRIkQEi8yPgtiEiD0GLUsINspRpVkacl5FAqvrGVWjmwSMpF8T9tv2fRrLp8/5S/+3Z9h7zzgzp1vsLxc0rstB0cHnJ7cI4VE9DCdHVLWmigctqyx5YJtb3DrRMw91+bwnMFI5tOaDz86xW9bjBiQUTHThkprgkg0KnKw0LQqEtYtB1WmrGAxMaTQguwxNnBqND7UfPGLgUcPvyCLNd5taTYO1wwECnwsCQ66qw6VBxaLCbNZgesbktogbWRRGupJjRSZ0HeInRD4xUXLetmipKYsLD5Fhr5nGBIpa+rCIJRACUsA1s2Stu+IDLizgc8/dTQDTBYBH3uMSUjdkwYARVkKirJG69EXz/cS7xTBKVwnuLroIENVa7bLNQfzKZ/8wT1Wy46r5TnKCD765n22m47lqiEOozj73Qd3uCi29L5FNoLJpEJJjVaGkEemPqfff7PvjoZjL5YRGDPjhGsuJqJ26SgLFDkbsst45wmbTH6c96vDEejYfgQrdhjf5uqlU6XrU44s3yjDiRi4ATy3yu2pZUycNQYiCF4EBoG9pNftAxxYB7olF4xs4DUwBEpgsjt4DTwGno1AiVYgs8QwZhUal9txRzoKIOIvI/4S4k96OOhHRvTbwIcFfHwMM7sLiPEjiN32cNbCYwePgK8EnGVY30zNo/OOAsZgvj0kyE+BX0D+NwNMBjhd7s5XwocTODAj07bp4GkHnwb4xdgn4UfsNaYxkNcc023zYWYHUK5HdIxNNQiSg+0vAvxffwHDMfxXd0CYMdeoDlwHgtyOppYRjNuxn7s6tSKYhNORKPMoeJ9GGZM9A7gP+7t9/fMrHwR7rVkpdhm2REblgMpuDHaQJVFY2tsi3ucgvxrgQpDqemT4bAsTT6y6cWGixibndYCfXsL/vUf960MepL/Pf/NH/0e++cEHREpCtFgJ16k5kLt++PFJEuM1HM3Qo7UoZ/GS/93tft7u9UtPwQuC5nnf/Vvf3z7mHQytLxOsb9/7tYe/vbwGAL5DeWfwl7Kg7ToeKMO0ntJ3S1yI9INjc7VChMThvOT/8f/8f3Fy55Rf/vLnaGn4o7/9R/zt737CdDbBlAWlEfgmoUKPziClopgcUc7m/OknP+DRT1e4xwHnW3QI5CGQNpFvzBYUXvHF88/4d//xz/jJT37MdjugqjmimNF4QFnu3Jnz8f0j/uRvf4dJrTm9c0DKms9/9jl/9R9+xNnVGqMLFJn1xTkiwt2DBzw4+QaTcsJUeZRwSJHQWqJ1SS88bcpIUaCVHqM/o8T5nugapPbEwqIGx6S0yPunpJ1sSlxMwUvK0jC3koXMGOcImxUKhzABW02IskQLQZYCLSU2i/GdLmASx1RzQlwxMy2+e8KTn37F6d1TJvfnqEXGxTVX6+csDmfMCssHh5peSS63A8EFkndIkUlJYYxlSIrceTQVgkBYZ8rTA+Z3LUE5pBXEMOZHVtS43tGc7ewwpUEVlok1pDTg4oCyCl0ovBsQqaBWmamuMBREIQlSAzUZTRKBQEObNnjpibSYNBBDIg6CGDt8PxBTBmtQdcX84ADXObbbFnU0ZXowA6WxVUE1L3nw4QO+8fEnWFVwdbGmNAqXW6bTOW0fiEmhpEIYMyp25zQGfIQEXcJ3AYNES4gETAhM2oyNA0lLzESjbE1nBU3uWF8+ZKsSfdfQuzWTScUHHx5zuJBciYBbPaZb/RUffvOYYXNJbDPdMrJZd2S7IMsKVdeIFDDaIBSAZ9OcMy0MdVVQF2Me3ME3dNstw0bSVBVFZVkcLHAhsG17lm1LioKyKKjqmrqeU+spsRnTGh7fPcZayePnTygmLaKEajLeB0IolucX4ypaTQk+MFnUdK5nvepo156hiwxDJEXN4cEDKmtxccXZ+WOK8gO+/NVDlssNF+dbfLSUZc2kKilKw/JsS/A9h6cHqELQ9ZH5wYSDxQyy4HKpuTy7oqwMi/nsXV9Hv8PlRgEs7+DVOMXso1L3UlcjRhI7cKJQXGdS3fmaZQ+p34OIF72YXp4A5M4Qpl7gJPaz0gvGPrgGQSN6fN1ehht3vz2MIELqxr3i5sZi/ErR4xdCKFRU1wFgI8CMwMCwi5xMO7cYAVgsNQrvAv65Jz4HfggwwOTxCDTr3c7DSz9BQrCYnJEY2MHtff/2f9Pur97ZZTIQG/AN8Cvgz3oo+pHN3Ov4NozMaVJoEqNQ1W5RzE4jdHdVd6qzt8bztvl3BKMKQRk0/a8C/F8uRpr1H92BQwFF2kXG5p2/ZwQTR/Pu3uYailFiJRlyagkJHHkcUzGaMeMtE++e/xplB/cm0NtsX75uosgZyZgiVSeBDhkVdnZPJxn2KRqVBRFI/+EJ+sMaqY4JR3kct7LfrT4yWEm1hf7Pn5H+W4/4yYxvTv4+/8vv/h948OGDkQwJgpg8YWfJVmJ3f+5B2nUAk9wFhXK9/Zqhu81sittw6TX3/zUreAssvqJfefvTbe7u3c2ur5SX2Nd3Puh90eSuvDP4m8zvkX3Lk4dPWDejCchqzZADITSUJqFUz+df/ZhPf6UYBkdZVDx5fsq3PvmYPsVR2iKNq7iiqEedvBhxrqO/0sx0zXdPP8J/1HHVPcdnz8mDOxzffYCx0DYOhszjXz7k6vESFw1zqTg4WnBY1KPYs4/87D9+zmd/+VdU0vHhhw+oJ3OeL7eEduCwniJVSXIDcjHlyZfP2VyumGtNIQJl4SmrkUWPbkvIW4bsMIVFYlEUWNWjjKdWjtz1CAxBSsoswDsWkwJRHUAqCDgmxnBa1sxipsoOYRO5slhhCSkjpCHq8XWTk0SmUTQ6p0CIHatuzRdf/ZA+PeEPvjfnk0/usvVfcfJgxsmdU5Zt5tOf/pCf/OJH/C/+4X/JR3ePaA7uclDfQwpBlTPJbzBKk2QJMRKCo8Dw4f1PcHcqfJYc1adoHQls8bInKY0UFfUscuGfIAtBjpqyKCmswmqJMhN8LEY9xQBtl6k+nFEzpcqGKtbYOL5IXQbPQJAeRyZGTwpfoNQzXG+J7QOGzkKpca0necZ0ar7HDR337p7yj//LP+GHP/kxws7QsmZ5ccn66pz5rELqzKNHX/LTn/wCGRJNc0m/fgS5ADMHPQNfQNy9tbMYZyctmVrL4aREyC0TVzANAgZHygPGluhFyVZEovecd0v+4of/Px49/IonDz/nD/7wE/7kT37ARw8mlLnCLdcczSz5zoS8fYrcZnRdYEWByJkwqHElLhTGapbLLc4H1leXtO6Mj9UJNs+YTiDnAT+M0j62ntIFT4oaMQxjXuQskNoipSSkyLK5xMWeFC65eHgFZI5O5hwuRk3CxUTwxVfw7GHgg7uB+UnF0DqcD7hhTK9mjaZ3A8OwpWkHQj/qdZaVZXFQsZiXXF21/Mmffp+6qmianuM7Rxyc9PzyF8/54osv8CFy794pQnXMqinTw1G6KeVE9A19J1GmwBrN4mCBtRWh//2XetnDPIgIApa96/0+qeU4fbgdvTcKP/nd/rtoxmt6b4ye3xuf9nPAm71/bpRR98Au3Z7gXnPcDWv14hxz2yB7G8bsvajGPAk30+v43xi5r8KNCO7Y60DcsXzsAgd6SnY2YqAfTYsEFB4LGMQI3/Ju1LYQt+P5x/ACNVo80Oz9u0ZY1zOKaMVb4PbFAICxD2nHi43gTF73QpAHyEPeXSExBnuxY8SIO5I1kHZHyTHj+ytjN4LqxBhmJhHXPGfGAMYpNl9G+D8/gX/1BP5xDd+ajT6IKo+m0roEq0ezd20gD6MZOpTwsEEvBzQRYRPKCEg7EymZvU36dsA316NxbeNE3M7GQUTmgM4SkRWOEkHE5IY6ZmY70KtSwmTw/2FNqH4FsYXvz+CjcvTR1Hk0U69WtD/cwH+b4VeSH9z7B/zDD/53fHTwDUzWpFQi0AgGUgp4UZCUJeV9oAkvmHVfTaW2+/xyOK949el48ers+/t1ZbyKN7W/5in5DfDg7XO8057vAQTfGfwdHS1ol4mch1HOxYGSmoOpYmrmiOQhbpDKMgwJPyRcv+HRo8948uw+bbfm6ZPndE3DtNZ86+Pv8cG9b1MYhSlmtC5y//gOP/jeAGnDwycdy2ZJ3F7xVz/6H5gVBa27YDaVHM0t7azAFhXH9w9ZnM6JWRG8InSZbjJgDBwdHnN6dEqWkn6zZNlc4TdQTA8orUYcVNRqxvFiYFY5bBHJNqJKmGqNSoq279lebmi2CSkMMhlk2iDVI0oxY1HfZ7GY4Yyi/2LNdrkk9wEhLb7fwNBQHpXcOZlx9/SYk/mCqBOiEuig0NkwKE9Lj/R+F8RXoI1G1pYuJ3756Gf8+x/+C9b9Y/7qpxU5JT7++PvYsiLkRA4O3Ia+j2yXF/zk7AnmDyzH3/iAqqooSBSpROiSaDVd3vLF+ozLZ79iefkQa+9weHSXrvmKYVgj1YC0giQLcrLkIJCu5XheIWU5shLZMfi4kzUIVHVBOa+YHE1Hdi17VqmnMQI5OJRviHg8jpgckcQgAsYeIpKlmtXkg0NcbymrU44+PcMWgBjT2pEjV8sl/+Jffs5f/OxHfPTh90nZ8OzxE9rVJScnJ3zjkx2gKiqO7sz4w7vfpW3X5GzxydAHxdPPn7B6fD76Y6Zh59QEzeoKt7rE9peUoUW5yHL5jLPtJcnWTGcfMUjDpnM8fv6UP/sffkqKG773vW/y4MO7+LDl6cWvqOf3efDREcvNt9B2xbOnW6oJlPUcO60oDwu6oOj6QFsMhGHg6vMnrE4l0yphbUFhDCl5um2HUZnZvNoJqUuarmPbr3j8qEPZEmsBoWi3PatVS4xwuJhCLvn8R0+pa2j7Bd1wwGa94vGXnu1jKGLP82cr5vWExcEJqhhT13VNg4sBqRWz+YKyGM29zaaj7Xo+/exTEC21sRTlfYy0FLZmsx7Yrh1N63l25tgsYbV8yp1jybxeoMKAlYnQDaz7wNXFFmstZVVjTUXfbHBavfkF9HtS9q9xcYtVGwHd+LO31MrdHhJ/baDag6PM6/i0G0hxMx3tz3jDNeXrPXjh+5vp5SYWci+Nwa6GF+eVm8ko3vp7my/ZQ7yb3xGIoxj/rkejcTeTRic1BBG7E6gHszPL7tuTducaWdNxa0Bcs5oKdQ1HxRh1vvOr2/NuI1QcGVeuW/Jiz3YQZ/dbXG+BvTDNTc9G4Sx24FUyjNozjAKNcgci3WjlYW+Fvz2R7wMw5K5Hicg+n7xigmW4GAj/nwR/3kLVjqbzfQDNXnGkAE53fyOQzuAMNks4Ox5YlZmhsuiQrhMIjKbMm7vxhRHY6yS+QA/uTL8AIhCFQBpFkuBRLNuB2I4LSLVTq8gb4N9cwZdX8Ecl/JM78IGCAzeab3+4hv8O+Dn8yYf/NX/3G/8Nd2afoJTC6GJ0OdmpcwjsOPZp3+KXr9vLAGkPBG934tekyX5nyrsDxXcGf6tnD2mGBqTeyZJE+n6NEplKG9LgGdYB70EVFfODOUOIPL/8kv/3f7/F6pKuCQxNQKnMH/zBc/63//Uxs2rCsDlH6wIjD6ltx0El6KYlQxf4+adf8PAnDzmYDdz96D5FOfDdb864f/ghdVlycnxCPVuw7QPbRhI6i5WKgyPDBx/c4w+/+T8j58y/+ot/zb/+857nFxvSMDCrJfW8xHUCLRyb7XN6H0i+oS4002pkaZyLLNdb/uxf/ivOLz7nf/2P/yl3Tw+YzxeUVY20klRGktrg/DnSenI30G6fEcNottv2JSk3GBvQZSTHAY2jCJK2XSFQzKzGGoVOOw5AJ3p6zpdf8u//v/+cTz/7KZfNYz58cJf54oj19pKzqwum9Zz55JBCRIwINKsL1ldr/vSP/xF1NaEowbXL3TPQILLFSIXOiYsnX9CulgzKo2TP0DSgIikP0IKxFWU1QdqSoY/gM4GeNiaSGMW3g+9JwpH0lCQ9VQFCO7b9hiH2aGkQKWGFRwmJ0Aql61HMO2WsOqCiYVIYVmLCaikp7JTClCg95qFNYaDrtqzXS676Dc4HHj35itoeUChLdXyXoizZbrdURUldFRTVaHwprMV5gRGK9XaD980o1yAT5NHTO6/X/PJHv+Cr79xnzhIjHEOz5ac/+gu+fPQrDu5/zA/+aMrZZce//Td/yexwxsQmpqXlO58ccnAi8OGMX/78l/zipwPRdTTbc4QuEeUxUTg2QWA02EqRQ6bpGtbrJeG8R3i4+2DGN++fcDAtsVWm225YLzsWBxWzxQzvHU8frliutqy2HdttJtOiNZQlnF9mrp6NpsD6eMW904bqyNJtHU8eryhLRYwObTOy2MUB9BKjC6rJhCFs2Wwdzx4/4+l5w6TWY0Tw/ITg4MnjC54929I1macX8OBDR+RTrs43pCxJaIYgEDkznyhUipweWz744JSzxxvOnp1jiwnTeo6UA84HtLKUdrTTpZAR6n8M4O8mEnFv6L1tCt5PYPp6/xtAuDfAilfA38tm2xvgtj/bbRPX60Hdbd5vX7984ZvXsSI3Z36VQXy5fbdBT7retgdiGsEYBjEGRvQIHJJAAhyjf+Rtjm7kQPdjsgdoiTGUbmciv+61IlPs/jcohuve3ey1C0i7huS32TDJnjkV13B3X/ZXdA9cE/u8sGJ3lUeG8zpXxAtckcbtZL/EjiPMOyCdkPRUCFwS+A2kbYYzruURX7gVdpdViRtZQJlg1bc8Ptnyd+4ZihAQLiCT3kXs3rDNaWcK3uvl7a/U7WuaGdO1ZpXH8Y6BLAVeSDZ4robHwGjhuXZB3UL6HDjr4c+/gimjH2jLGA19CR+f/D1+8NH/nPuH30TnmixKCBIdAzKGUcZMCMj7iF7J9V2Xby9UrunKm3vvBUo8X//7Tnp8r8VRN+f5daDkq0/ry59fB+DezOe/0q53pP/eGfw9fvwYVRliFBSmIMeAb7f0wRNtQVVWaGuwhYDCEIXHe4dB06zP6JUlBMV63eG9J+WBu3dOuXf3lNVySdf1BK+4uNxycb7BDZ6iDCxmkbNnD/n8s3/H82cTSNA2S+bTmtCd8/hXjyjqmoOjYz46/ohZfcyPf/IFX/7yGYvFH3G1qokhYuSSTz6ecvdeOaZDQ1LUkll5wP0jw/MnP+Ps8il9u6EqDNNJCTHTdwFjLb/6/Mf88tOf8re+/QOWVxZjI3fv3+XjD75FcXCIo2XbPEXEBk2LLgtcSETXUZiEG85Zbp6wmCascMxnBTMS2+EMmWaYVDF0mT5lTFEhbYULgefnl3z26UPaLjCZ3aGcHCBVzWJesWkghIRUisVhyebxJT/7q//I6nLJH3zyHb5x/yOKqqQoAjkMFHWNLgQ+wsnJnG999xMePnyM95FSDtQTRbtdMjsoOTg5RqqSbTvgfM9EB5yItG2PaBPFBEyp2fqWIXWIlCAsicMzyqlm3Te4YUMbPKWtkLtE22Knz6RRlIUgR4/VAiFgcAEXFDk5OtcTo4PkEVmgC8384JD/4g//Lt/66rv85Ge/4mh+hzAEZnXJbFYyW1QsphU//+WXNO2Wxw/PePz8r/BNADUjekHsBAg9Sgv0A6QGtKMoHMKvOD//GZXaBQ8dGjbdhMW85PRoztQ+4NG9z4hpzT/8uwdsrp6yevgf6TeA8Pihww0dbmippmaMSDYTnp2vcIPiZF5RZRjaln67BnpEmTn40PCtjz7gb3/nA7brSz779DPWyy1KwmJRMVtUaGNATDg8WHB6z3JxuSaEnsViihSCojxDyQ2uh/nMcHp6l+XzKzbecXIsOT455OTkiG9/K7BcroDAwcGEwhqW52c8fP4Vl6tEWQbW28ymjZA9VdEwr2vqmWXaKM6eB2QFH30As+kx265ns+6YzY4pJwecnk4wcsLl2ZLTD+bcub/g2eMlw+BROlKUFVLbcWrOgq7taJuGoigYIxJ+v8vtKepFFhBu4NEekO3hycgKjvBjzxa+zGbI6xpvwNwN0LgBly9OLTfG15tz3/y+QRY3+93+vP/vZkK6AVS369kXdX2MuN5/36aIJDAK6uZrdZnEqKkWdrxYFvtvwihPeGsM97WnXbjIbXi6z0S7Z1MlDnYg7YbTvPGd3B+ZiS9Mu7dxhGDPQI4s4Qgx0w783YTD3IC5zIsjkq/7ur+yeygWBfQ57YBwZjTw7+QFSaO+6XVL5Gja3dWoUTu+M45qL8mTUk/KgSjiGKic9gD6tseo2GWzuM2siRfo3FHnW6DSeKxKctQZNZb50ZwkDTq5Wy2TFIzyNnGV6LeZqBkZyp0L+UFxjx/c/yd8ePR96uKQ7DQySmIYg0oMCQ+QFDdm3RHkvEDsva6Im3a/vP1tEOnrubOXFz9v2uc3tfm+CSK+Bhy+0ZT9+vLu0b4oYhJcnJ2TYto5XoLRmigsQYGQFiHAOY8TEd97+vUlUsJkMceWc6o6o1ymbS749NO/ZLM55OpiRdcPtF1g0/Q4lzCmwBrLpAqk48DV2a9oVwXWlnjnWfdLcvD4bmBoFUb00Pe46Rnt+iHL82c8flTgh0u6dmC1WdKHHiXHhyl6sN4wSY5+2fFw84izy0uGYaAsLXVdEkNiddkwqQpSaDk+Lmm7C56fdfR9y7PLCy5WDUdPnjGEyLOHP2NWhXF9VwZQhr5VzMuM0R6lE7bS1BRMasHEDthhgnSGg8mEpRnIWSONJRtN6MD7gFGWoYmYZNEfTJjNTqjUjBw0m+0lpa2xlUXIS/q+4fh0gVCZ+aFG2Z715ZrnTx8xXcyZHx7TD5Hn50ukHJhMNL7rqEvBvCgZ6gpTCqwacNGhk6MuDSdWsZY9vkjkUzB1Aday8Yq1M1STktO5QaYtKaxp+0vWmxXO9/TdQIobFtWUu4f3mNVT2nb04dhsl1gVWSzuMTBFMWfwa7p+QwgD5DCmqHRbtqvnrM4qVk+fcvH4M7qrK/wgKHXBdFozm5QUVrJcrTk8nlPMJ7j+PmImWK1booHOdbiu2Ynk7vQs+o6zR5/Tth8yLVtE6Lk8e8LFs58jQ8+9xcd8cqqY3r2H6v+YP/8P/5xme4mMl7TrFUOXqCaaotRoK4idJ3uBFwCSGCQ4AQG67Ybl1Zq+6zhcCE6+MefB/bvcPz3EmIH1esnQOWbTBfW0QukdY6Asy1XD0GXu3D1BCEU9mXJ0dEgI8IGxzBcdzbqHmNAoXNfx4YeW7/3BR3zjWw/48MMjtDacPb1kvdpgrEZZyePHl3SDY7XJJAEhgBsgxcQgE8VxZjG35Bg5PoC/892a733nLsoUxIuGwlu2baTttkhpkEJxfDzn+9//Jif3Cj7/xRlSWh48+ICimnB5uebZ8+csr7Y020jfR6o6UE5+300y8CJr8PLkcAOZ0u0Zlx3Lwn5Svu2sv2cLX/Yp26u53a75Nv93s+fL/70Otr2pJ6/f70013GaR9kBV3oJeN32L1wDyJiJWAmPU6Uu1CshijP4kcwt43Ziw9y0W1/XH67PlF/bbf37xyBdH+HaJ13uIHbenriH2zbE34PrVclN32sUdcx1jEcSoua12o3PDQd6M/20mUtxq0zUwiYnsA8GnnUZe3GWgiy+Ao+vrcivaNe++G//ZcaiZMZUqeaxHC4QSSAJ98te+n2NtN1p6MkGZRvLOD+ATFAi+cffv8I3jP2KiTlHRklJmn+QgCkVE74z7crcU2LXzNvLaheMKceveE3DNpr+M3N+lvBOW+vXA3bs24X3g43XCkncs7wz+Nm3HsOnYthuCd4issKZgNp1RFTWqKhjIGD2GqhspmEwmuDyu0qSIBNegtWFSGlKIrK6ekPyS9WpMA9UMnrYfkFKj9JSULXWVMLrgan1J6A2xnBC8x21bJlVJVZZAollf4rYd0W0oS4/RjvXyCUr3NJuBEBxKJ0SORO9G+YQkiTFyvvR0AdwuifgwwHo9jvrQeHwrMbrn7umczeY5V6uGfnD0ybHcrkdn9Zh4fnZOaVvKQiPKMWn0YDUqBZJb0bXP2WwMje9pm0RVRM62a3RQDLGldYmqPGJaHxCyYL3dcHZ2weAGFkdzqsmEFASb9ZaVW7ParAnRE3zAFob5bEJRJU4OT9Aq8K//9T9DW7i6vOTq6px6OqGeTuiGQNs51psVswNBriRaeeLQIESmbwIx9ihbMK0k87milorDQiGCpJOOpDN9DpgomVCjjSa0V/z8p3/B1eVnbLuOzbYly8Rm3RBcw+FsxubumoPZHN872q7h4bOfcDx/wMffAGMPOTk5phsEKQ/E2EMeECkicXjfcnH+lOfPv6BfPUHlhEgF680GLU44ns/xfU/fbNmawND3bK96DuYzFlWBkHDlGsLQjNkkdB7lEXpH6C85e/w5wa5QqadZrWibc4TIhM1jzr/6S9S84VuHJerud/m37Z+xFS1llTEFlDqhRSCqyGxqEErSuhY/dBxVE3RhKApJFIo7JxOOs6GqEndPZjy4O0MKx9B57t45oCpLVlctGUE3dDRXLQhB0wz0TWK17Ik5MpsXdO1A9KC0xFjNfDanthXzST36m9aRu3cWHB/UDN2WHnC+oXcNncuUk5J2GGjajPNwdQk+wHxSMa0rCqM5nE84Oqg5OqwILnB4rJgfzXn6dMWzpys2m8BymUhRYJSiKAz37h0wdPD5zy8ATVFZQgCTErbQKCXoe8d2k8fgsTKiRPHub67f2XIbBtwAmdvbXo7DfNPrP791n9flF9ilurx1vt+Ul7jdlq+fqG5m4OtMRtcg7GYPeHEURgbpJifF7V5lIOUxRjbncF3vyz9jPXtm7TYAfhXS3YCq259v5ca9tV2wN8Pf/L0NJV+M7P36sudw96boMfHKDZy6PQr7LTfs4c012PtfyswIpnYZk0ZJFMgp7lxvBFLkW2nQXuSU9/6WMrPLonH72wwiE6XEicCvLp6wxb8GNN+0WjHWMwJamKojvnn/TzisP0Clape1LxDzqOObxcgsZuSo4/eCMf7lscu3/n95VG/z0S+Xt/N3X1feyj6+c5XipTrEq5veqfJ378M7g79ls2a7bZA6MZlOsbZGaYMua8SkIhWKLrlx3ZMTJYJFpYlmQiLgU2ZwgZwCRmrqacng1qyvNoQgUNpSFGOaNiEkQox6cjlFqqLCKIHRAqMTYRjIsRnz0k5qhBC4YUArSWU8d+4f4YZLvLvCuzH7hzICawU5OnIcCHiaxuNSxMXEEDNJjQMXwvgQGK3QBfRNw6Q2zKdThv6KEAe0LXC+4/lZS0qji64PDiU9ShmE0CA1thTkkNiun/DpL3/IxdmnxL6H5CENBB+Z1xMklm7IzOenfOPj77NYnI6JqkkgPNODAmMUIQScW9N1DSkHtIaYPVBw5/SYbTegZWS1fMovf/ojkIKYA847rDVIrQghgVQYK5CqxUxGUHzVLrG6Gl8eIiJ2zuYuBpRUDHRkkWhCR8bSRwG6oCwKpHSsNk/5Dz8+R/0so4ygHyJCSwaXyCHwvKp4/OyMQo15HXMKfPHwx3zvOwpRTinMmsX8Q5yT9P2SGDdI4xAWlHaUlWBxVHK3O6Bvjjm9eweZCi4vGg4Ppjy4d4e+65lOC0wZOX/yDH0w4WhR4foABFIbCN0wxtiVmmQUwhmOFpn15SOGvMTgcc2GGALGSM7PvuKH3Yb57Occzz/mZH5KXSvEdvRxHO+rgW7b47Mf87knjUoR+o7788NRJFtLZKFxvme7uSLHgYnVxKHharUhC8mduw+Yz6dcXmxYLhs224626UlZoJTBdXB1sQI1ipsPfUCg0EoymdZMDisODxdMS4s7nVNVcOd0gSbx7Mlz2n7L4CP9MIowm8bS9x6tKo6P7CjvI+BgVjKfTVgsKqpCYRWcTmq0FOQcaLqO82cXPHu6ZbvJrFZAHqUtp9MCdzjnyaMVF+eXhKCIWfDlw2cYo5AK+mFAKkFRZqSUTKblKP/ye19emjyB2xzgy6/ud7Bo3fqU3/jti7Dqt2+Mun2Gt08/r7KNr8LWF+u5Xd9tc/PeNCmvZ8kbU+/N39tg78UzvWnM31RuDOt7KLhn+/bbXoXU7wP6Xj7Xbdj5crkGh7eg/O2+7D8nRv/evGfuREbIREoRtcu3DqNocWI0+97cH/ka7F2nFYXr9M8R8GTaOPBodcm//PFf0L90f93OofFyfw2Ce4ff5sO738PK6ahtJCOJRBSZJAUijzHTIkuESNcjfJs9fu0AXn9++Vrkt1z090Rw73Jx3+kBfn/Q+fqF1vuZmd8Z/AmtsXaCtpmjk3tU1QShFVlJYha00ZO6jhAkRQKLoW17jFZkyQjUCoF3DpE9k6LAu4GcdpkslMGUFlNYnIt0Xcfl5QXe9Xzz40+oqxIpEkZnRCWwymBsAtFjtMYoqK2ktJH7pwvOzgvOV2u836CM2uX6y+TsSXhC9rRDi08RW1cYMk3X4p1D64K6mlCVihwTQ4LjxQRTWq7WG0KI1JOahMD7HucHpJQUVkJ2tO0KqRVlOaUoSoQU9P0Fn3+2RAiIIeI6T7O+orSW+aSmKCasm8hkNuf56oIP732bnCDEBlOCGwY2TYdWipgSKQdKaxAy0fcdUkkODo8oQia4lsuLx5xfPKd3ifnhlJShdy1593DbotjpJA2jT0r0ZJ2R1ZguJydBG3qGbmAzgLaS7aohxojPDmNrkpDYukJFh4gZFz19M+D6nqrSDCERURhbQxZs+p4vHz+hWa4QKXL35IjVquPO3ZZnZ8+Q6ZyDyQOaLrC8+grEhmo62grq2lNWHqF6PvzgBJEC8/kE10VigrqOWJuZFAs+nBzjwiWpW1OqisNZwdOn5/ihZa47culARmZzPSZVT3BQKcgbNpsLrAwQA8ZKlFKsmy1Pz8+R5gtms8/52/e+TxIDMbaQPEpofN/Tdlui8CgtKasph/MjsvNMCoudWpISdD6xumxZX26pqszQepYXKzarNZPZDMElISuabc9q2bLdOrwTCKmxtiCFfJ3IPHoF2SClIqdE9IkYAt4PdMKhVObo6ICDxZTl5ZLNZosLLVnKUVcwQrMdAEU9qdGmpiwmxBgJ3jG4gYxBKvDBY3xCFYYsBcGDUgWHRxqZPYJIijuAnRKXVxu6YaBrHFLVRCJPnl7gQ6IqBMZItBFM5hIpNWVVUBblO7+4flfL7fCC29tuGLnX6/2/rrzLy/+NAr5/jeV9zvE+0+0ezNzsM+4ld95yN9P76wHmm0Dm29p2e8xG3ulm643Z+8bf7rcxvreXBHsm8KZ36Xp7fs299HIbMuBDovcelzx6p4WjyCQZR5OuGIMpMlzPD3nXzSx2MdZpBE0pRwIZnyJ9dGxcy+PNkh89/AV/9rP/+MY49D0IvD1OBstH97/P0fweWhjIibwTs99n7xrB387P8VbAz+vv9Zvywv0jXvz3fa/Rq1HE71PJ7XvlNQ1645bbR79r2QPbdzvincHfydFd9Ikm5oApKrwfc+eSYRgGQhjo1iumhWBWGFxQbFcrFrMpVhsKWzKpaorCjPFdMiNVprSWzXpg6LsxLzgafKRbt1w9v8SHju988m2SUWw3a1K01JOCaCyuH9hsBiZlxbSeoHKi3WxYn11RGoMViuQCdqLofU/btATvxzHKEp8SMSWQkEKiaUYftfnsgFJPKE0GDXc+vMe8mnK+XdF0HYODwiakEWg9PiBaQV1ZRO65uFiN+YPtmJvXGI0Qkc4NBB+QSpOyoHcNSgQefvmEBx98SNfDQI/7zPP5l78k+kjXbUlywMWebdcjciYl0FpALojBMwwDq+WWYQgYW2J0puu3ZOmQY9phZGLH9oTRvyNlLIaitDSbDW3bYktLEoKub+kGhw+RnDM6SmI3+nDGPHrLWJHQVpBiT2gbSImJEigdMUUC4cgqMkSJ0iU5j67FW9dxtjoj9gOTqWZ+WOP8GPma2swv81+w3HSsrx5RFQMHi4QoPNO5p3WP+ff/9sfcv/MRBHj6+BGbVYsfAqG/olSR4/ldVCp59PSXPH/yFYtSUcYD3PYZrt9SZo+xjoxjUQqKqcKnDKnDFhbXOjAJayRGaUQaHY9zjKQYOdt+RbfaYKsKP2zYtCtmkylKSgojUYXFGMt0vuDe6YMxQjo6tFRsmpaf/vgRT77YsJgbvveDGf2QePZshVaCqSp4/nw1+uVEhdY1IguiD2Q0bkhEl0EUKBNJUbJZD8QhICUUVc/Q9Qzdmjsnc8pCU09qYgxcLpfEBAeHd0ALeufpuoiQniEE/KajabfIo9G0u16uGPotrp/y4P4hs2mJ84GuHdDGMFss+M7fOuRB63n2cEXTDaxWG87P1izXnv7ZJXWtULLCOY+uDf2Y2AajQGlBShADZJlpmw71chL538Nym3R4VXLlZqLJL229gTS3uZ13PdPLZ3n90V830fx6RrFXy/uwbW8+9rap81Vz3texib/O+W7YtRev0bv0521j+7IZe3+1b8Ddi6bYl3nM/TGvqwdg23c8vbzkyeUFE2GYFRadM1kmUkwopclkktiJUafR5zSLSMyZQMSHQPCeIXqa5NkOHZt2w8VmyWeXT/j50y950o9CNy+35eWxGuEyVMx4cPd7FHaGlGP+cpHkmGAh78B2ZtQZlLdr4ZZJNF/TkS/2/2UI/PZr8Ebg+Fsqe7b4nYu4Oe4NX/H6u/qvgfkzUmGNJibLMESavqftejrfk3LEGsn2akk7haZQaJHpmxbvE/P5jKwLRIQiC3SGwUdiSqzXS6QcJ5yuX2NsjZEltS6pKaG0lMYgiQxa4WKP9pKYEtuuJbpAcAkrKwor8cHx+Refo4uCup7g03iewXk225ZhGDDGYosaH8ebJMYMIeH7Ade3NDEjvaMuNbPFAllUfP7kKy4ulyhVcnxwTFVNaIaGGHpy2oeiG7TVTGYVrh/ouxYhMtGUaCUIyeFzQAuLrQpO70+ZCsNZf8VERHJhWQ09Z8Nj8i5xtxSZYmKQKjGZGUQWVGVB2/dkAiGOsiUpOy6XZ4QQQQQKq5nNKqSQ9KFDG0lWkeAGgvM03Zqut0ynU1bLNc1mSz2ZMJ+VbDYt58tLkHBwdEQxLdlstohCoaUmdB4pxgVj13X0fkAJsIUhMrKEUYK2CuUSXb+ldYFClphC8uDDO8jowQzowrLZXpJjxK8HuquGvouEpmduR0mZJmzZbge6uOTRlxcsnz0aswNogcKgpaZve7766oovHWNiejFGD262GdE9IaUt3WZJrRVzIyA7lPfEZszUMgRHUR6gpCDliPOe0HdYYSl1gahqvIPQtDjVUNUGayG4MV3ebGKZzedMFzVSjiINYbhk21wxWRxg6hqdAn6QZC2Y3q05uXMHSWa+aLHKojB0mzVJRBAWkTTJCbq1x+8WLXthL11D9Jm+G8jN7gVYwuygQegpH3x0zIP797h3d0HrWparDdrqkQlwgc26pWk9CUM/RIYhsFptePb0CmMsZTE+72VVMJnVHB5PECoRmkjXZ3wYkLLER4+pJFNTkYj03hNVh0GgZUHOJc+fnFOmCikFMoNWBVJkutazWSekSSgdGJr+nV9cv+vldewfvP7V/bqJ7FVu63VlH7jweo7w1b2/vrxtuvlNJ823AdNXR+DNn1/Xprd5R30d4H25rszbjn3/kX0ZPL7par+t5syNWfp2WzKZy2HDn3/1U7JP3C2nLKoSnRQowdA7chZEmfAy4HMipH2GmcCQIy6nnatIy9b1rKJjO7QMQ8vgO7o40F6fezQp7NvzchvFre1azDg8+hZSleRbCz+R2UlzC8jqmjd7sW8vjem1iPPb7o6Xv3vLHfsuKPGt5euA2dc8Le+zQso7pnRX72uZyteUdwZ/XdvRycDTZxfoqgAlWW22bNerXXLphDaCPnm2RjArLZOqBjFqgPUhsV0tCb1jojTiZM7RQc0wKGbzY549W+JdIBPJOSBzQsWELATRtxgrKAtF7z3ojFGGgorQD2gjsYVmWk1Ybh1ZeApbMp8e8vDsOVfnSzZdS9t1xJSYGoOInt71QKISJcgx52HvetpuzXqbmc1qhNIMg+P8cs2m6ZnUR8wmguD8qHEXB1LwI5iNA0pr5rMJvrBU9RQhE955nM8473DeI7yjKhwTOarWz44OIWZ0TOA9PjmyBKMNuihIKaGywEoJQpJjYFZVZBJtinQuEv3o2OtiImRHloGCAlON2lbSCrSMCB8Z+o5+6LnaBNSZJoaEbx2rtaMsjlHCEFzPpmsIQlJNKqrFDO+H3f0lCcHTtR29d2N6XK0REpJU9FKD1RSFpRCZFDNGg1EFhVIYFVFizFGZXWB1tUZ7jQyZzfIc+jF90KyUiMoQm8iquyJmz907NdE7SLCoZ8wmFW3bst4u2fSRMAgmdsK0LlhUGkti6M9xfsP508+ZV1Ps4THWQgoRhEFkhbGWfnAoPDJHRIyILCi0ZlFOMdMCVdQs6kO+uniMEhJTfURMESsFWia0yRglGbUJPV989ZDlZo2XiqtmoA8DyTg+/taETz45xBaa2HXonJE541pPDqOZfLVqWa8G+jYiRIFSihgjDB5SIEhDRpFXuwfUjoyaFpl25Xjy5ZJZNQEMq82ap+dbJrOSy9WamDOrdc/V1Zac1XhvukTbZ9otTGaO4GBaWXIE5xzLq4hUYHRBypLPP3+EHwxPHl9RTyfM5nNOTg65e+8YHxKbZc9P/upz+qbDNRBVR1lZjAApRsZB6MRkIZnPa6RSNJvmXV9Hv8NlP52/bFB8dZ/f5vn+usrLU+tfB2vy65TfJkv5bud52xm/Hma+CRa8z9Xb77s3E+895DyBnzZP+OWnT6iAArOTuJa38pyMMtXx1hlHAe7bLXrRqHub4bOMwTThljTOy0EoL6pHgpCCnMZvldylLcyjCV+KDDmTxC3vx5yvMc7XsWgvj/ibzMCvPe4tg/7rLnR+3fvxTXD29f3/a/L5673HlJrJYUVC03QdfthCaCGDLCSn906wRnG0mFIWChE9cQgkNZp2SBBSoA+Op09aXDdntqjxuWd6WBBWnmFoETJRHUiOPp7QbLdcXV0gjWS7aQgi4Jslk+kULRWFKSlMidUGqxVGy51cReDyySXbrmW1XbHpOyJgrB1lBDy7PIAB5xymUBwcHaCLTI6BQpfUVYWQgsvlmscPnzKZHaCkRCpJEo6MR6mEVJLBR5q2YVJV2NJS1QXeR/p+oN02SKWRWmMrSxaJbb/F48iTCVpaLrYbDo6OmNclsY10Q0/vB3q/oTIlZWlwvaMbBobgOZgvqAqL0YJYaqRUaFuCkqTsiXEL0SOzGbXns6YwglRJhiHTDgM+BZTUaKnBSHIWpBCoipJiUrGKW3zuR3ZRSJqmIafIpCyZ1RVZAlKN+XGdR8c8mu0xCDKuC+ChKMb0bzllnO9IOVJajVUGLSyzOwUFEhUjMibsRJFVwWXyrNqeoD3Otaw2Ld+6/wl1vWCzXJHjBluMjsCrzZbN9gorK8pKs5hbTPK0mys220u6/ow+9sxVjbIZYSNNXAMTqtkBQgQePv8K+o66tBxMZhxPFpRmisDig2DdbRj6zNCOZlapNdOJ4XBR4YeB9XrDsmlp+571tiELQVHP+OyzL3j6BJKE6RzuHM4pVILYInNgVldIZXAetpuIFBJjLQdHFVWd6ZpIxoJUo0xMswElqWxJPFUYowl+QIqI1ZYYNF9+cYWLYK2C5JkdzemGlr5vkVLiQ8L7zDAEmi1U1nD/7imX5oKr1cD8zpTFdEJha7abgSeP1rRNR2E1Ryd3GXrB8mpDPySycnTunLIwTGczbFGODH2pCZuBgzsWU9XEFFi1LevVuGAoaji9a7lz5xQXAkK+Xkzj97O8iZF4M5f0OoDwLq/6d5143mW/l/d5n7p/HVPzm87z24S0r2vD23yw3m3P2+XNzN3Xla8z07/83d6k+rpjAoINmQ0epNyhiD0EUJDTyCJde+jta9lxinuNxWuWbm/EHTNS75ON3G7Ly+29DQyVzkg5IKQnpzzGo+zONZqd39Tz2zW+yvG+mfl68xX47S9abi8L3g2ovt8e+Y1fv+uz8c7gD61ZN1uc75HWYktBPa9Ialwv1IViNplweFwyX1hkDGzONjSb7ShFYksm5YJaVRiR0SJhpWWz7Vg3HqkKXHA0fYNAUpoSU0nKZNAFtEPPdhjQRUlRzQgukKKj1CUzWyIjrLYrlusrYoKmD3gyeqLH9GByzJerjQWpST4hbeZgPkMqmE0rFtOSYZgRQiC4iB8CQx8gZ+7cO8XoCmKg2S7JOdC5DTFFpNYYWzCt5gxDRyaxWXWsN2ukVmhtMXZM++P7QIiZoc/0ceD84oLj8piD+ogmtHgSB4uS02JOcIHNekPfD3SrFUpbJhoUAZNbVE5MjIAhst2ucUJSlBZdKmTyzGyN8A4fPEUe87+WxjA5PGCiLc4ndCioqhqjC6QEoTTbZgthwJAoiYTNFTmDDpGqMMy0weRMPakpqpLgIv22wQ/9uHbsPd1yzRB7jiZ3ES4RvGDT9oTssGWmKgy1sYiYWUwEhZFE0eGVIwpDNZmjfCS2gSQShdEsRCIPHUkoaiMIsSc5i4gGQiYOA6oU1DYgZUvf9qyvLqgnma7rmU01B0c1utQ0fcfWdyxmE6qFZBgMylSUuuLOyQEH5YSCMbp202yJydMPA7ZYI5Xi6OgAJzqabkkMG7Q0FMWEws5pu3OQiemspF0vufvhA773x1OqqiSHiIye2kApYLltCTGgsmHoI199ec56nZEGDo+PkLLApzHvbmaM9paFJstE27Yk1+OMJpNIIdJue5QBv4bPfrmmmMPpMfzR5NucnB7hCst6vUZEx7wuoS65d1Rj1Ix6cohGczDrMcqSguDxVxv6sCFkR1EK1s3AavMV84MFRVVSRsPycokLo/N4Xa+pqpKu8UzrKRwX+C5higIfBV0nMBZms5LFYspisWC6mPPFl19wcHrwzq+j39/yfqv3/6n8T+V15eYOum0MFjvzYLpxIckvHiCuAd1reNx8k53mTaD0dSby17HDe8Hy4AYQbnQjuj6deE0N72wD/c+Kgf7Pubwz+IvBj7p1fmT8rDXMZjV37x9SVQVlZamMYXlxwdOvzpjP7A7keLarnrqUTE4EZVlRK8Fqfc52KDicHtCHRN8EOu/JCFL2+ByZz+bMp4dIK9k875BGIaTEqIKh9WyXG7xqUUfQD/2YmF4IZkcHzOc1l23L1m8JOaOsJWfJMCS6psUaiY+OSaXxzlMXApFHjSEtMrY0qLomkWiHlm7wuAGaVUdKGa0VUkmqSYkylhhBSUXUFi2hjwmrC8qqoKhKhNAMbcRvPd3asb1qcLlhHddwuoDBocsWTOLORPHg6ACjSi7risurKy7alhrJfDqj7x1D6Miiw5qCbC2hc6Qc0SmjXMTakmlRkXPAi4jKieQ8UmqssuQiMQhHlpIHJyfMqkO0Tjy9OiN4wZ3DA+azEi0kOQRKXaKk5Wg2w9iCnAI2CsqYQCbKAvoMzg2EFGiWS9r1isnRFDvVTIo59eJg9CNxLd16wzYuKXSm1AqNQhejAuiQIsGBF5LFwlAvjvFS0A+B7DJSjgaJrUg4HxFIqkpxcjxhYg1Ktrh+i/eBepY5OJkgZ3fJMZKlYNVvSDlweHTCYj5Fi4QoJR9/dA+bwGqFTILoAiJLCqtxEZTMmEnJanOFW/WoItJ1V2glqaua6VSgVYmwCqssLnsu1xdMp4dolVEqE11kfXHBsm+4c3DEx3c/xFaWsi55dr7hyy+est1m+g6ePrlCKkWKiuggx/GlbetRMy9tehCZ2PtrYS9RC6bHBff+1jHTqqSelXz88YIf/OEDDo4iX376iC9/BSpLcqmoikOKasLQZp6ePefy+XMOjg7JMbPe9PShp08JoSTH04r5XLNdLqlnhpAET84u8DlSz8T4AmfUCfRZsFmu8X0mB01i9A2WMnP/wR2+9Z1PsMZydvacrm1YzOZMjw7+ut5z/1mV3wzavWgU/HVNhH8T5V3ZtL8J+Pv+rOffLLx4CbO9cdsLptqXMRUCrgWx0zXwu8kJsxfgjuzh2j7DzP7ee9drtd9H3focgoec0VKR8h4S7ng7cQMbXxWy+Q3LdVVvpszE7Q1vJtlePVa8ayvFDYP6hgPEra9vxJtfM9pvcxp9S3ln8LdtttSTKcYU9N0GFzIxFzvbfBj19EJgvW3wXUtpLFZXnNwtqKcNSmjKaQmFYMiRQXrWQ4dSNdOiRhUlIQZySiit0CrTdg3T6RRkImcPORCHgJeZ7eqKfrtCTWp8nqKCRGnFN+99m8v+CisVk7Lk0eVjvrp4gilrFgfHmMLSR4dzmZwV2+2A0pHLi82OM88opZlOLXVV0ruevvdstw19C77PxCSZLCqUVmQliCrjUsA1DX7w5JCpCsvpfIEw4ELAO4/GcmcxhVrSLjpWQ8dBGshtotsGyhiwU0m/3nLmI4UsSBHuVDPsoaIwisVkij2QPN9ekkhMpjOUMAxHI5UfRaJzHUYZtJGE1BNdS84BXUhsqTDWYCaSvrekAPVc4MMSnxJqIjheHDHzM4ahJzhP7D0HszkyRrTMGGvIUYOIYyqI6KhE5HBWocQobXM6rblcbSjVASCwNlJNLC4aLgdN22S890jrceUWnRXTeU2IimGIxBxQWjEvJaYuiChcEGw3mZg0QzNKi/RdS1XVnBxNOT20KBJ1ZSBH+q5nUs0xU8tMLoi9J7lADOO9ZI1ERo/0A1IYcBkfQBhD3w6kLjCvDjg+uEfnAs36jMYNPHt+STts+MYnd0fR1DxK8Vy6c4TQ2KKk1oblqiV1nmV3SbfdUFYlh5MZs3pGMZtwOKlRUrLZbujcwGY7UJUlZd2RdSYkiDER9+/vFCAIpJ1icmZY9+M9OzGYcsxpfHBc8tG3TvjTv/d3+OPvfZfTuxNivuLHP/4Lnnx+Tm0rPv7wIxbzjq73+F4QU6BtOn7+4y+5ugrY6jnzA4MtC6SWuF7igmc28Zx8fJdPvnHK0UnBMEQUkudPl2ybjsFFYgg437JaRppmNOXUxagrKST0Dtqup202eGtYri+ILvONjz7Bhf/xBHy8f3k5mnHcdvtN//6hBv9pyl8HTHpbna+ax19nHrzZ8z3c8F+717ua7V492/ud59Xynlf3Gk/dIMDXtexFQLkXH0q3K3ihheKlz28q++/2XKSLPc61ZMKoi3t9mV48z2vrzPCipMnr/3+jT+bLF/21QyneMsS376eb/8UrgO7N3oevNwe/wQHhdeDy13VA3JV3z/CxWuFT2iFQhVGGqqgpbEn0jmazQkmN7xJKVOSo0VazOKxws4qh9UQ3sOq2kNNI8eqMyx1BGnzc3QxhQAXwIhPjwOA6Do7nSJkIQ0PfOciewW3xogdrUWVCqDBGE5YDgsCT88cMArp+S9+3DDEitaGsJghl0EqihECagJYSUh7ZLaUxUjNRFZWyDDi8S0QHeEEOGRcj5SSRZaLvBnASoRS6UkghsGgqXYzpvLqermuIKVGWJfVUMpnMKBcV3eUluZe4tsf3fpTA8JrUC5qhow0thTEc3zvg4HDK+fqcZr1GVDXSZ6LvyUKjLFR7914NE1Oy3XS02wZdQKUUSQac7+hiQwwGqSyz2lBPKoqyZ73uSLvrkuPI4thSYq0haoFUEa3BdS39EJFZUVZ29CPMAXAoFZgUmsWk4LCqWZQQPFxeXJBFQdATUtLkpietW8p5ZDotsDqQkiMMgIQYB2LI+A6SlBRDRZKSkAtcFzDFHFtJbC+JrkNkMEKTUwdxoNBT6rpAHEwA6PyAcJB3MjdKJoIfCENAl5qyrFBJ0DctKo85iIOIBBXxNuDKwCAd58/OWa9afNeiRETGyHRWMZlYYghsN1sG30EKDN4TupZCKGw5YTqfUNUFk7JkUmoWM8NiWtJvM+vzDe3g2bSemDJCjm6UKQpyFBDyuCGAsAIRElIoxgsSoA0kGciTgkRmebXipz/9KcJ33L06pNk+54tf/RySpyqqMceyMNSmoI8BF8bUT+1ZoDQQugiLAjJslp4QIocnNUeLOdlnDo6mPP7yOcurFilLqmoK0lCEjHMQ1yP7a8x4/YUWhBQ4mB8QUiREz/PnZwgpWG+3aGm5WjdE6X+9t9jvUNkv0t8fcPx6MO5NU89fFyj8m+PD3jjNv3Xr2795eY9Xc6a8TxGvreH1IOLXqfn2J26d6yaj88vnfREav7594qU93n4H3d6aXrPXba9ex8C2WZFSgDwGioyL6RF0jaLO+0rEdXNeaKF4w539dTTzb/wAvAR5xb4p70z93ZSXmLu3+wjuAenrOvB+nXpn8KekZnOxQtQWaRVKW5S0CGEIwbPZOsgjMzGrRvYnpIEYJUpCYTRDCLRDP0bzakkZDZQBR49LER97EImUgZQx1oJIGGuo65LV1QofB/pBYQtFUU0pp+Xos6oSWUSerc5o00CbOlyCLEFIwaZpaYfEbOE4PDjGKonRmpQczgdqO+YkVYwTY4iCbedpuoGcBdPJnEGBVI6ERBpFSIFt1+JioqxLZpMJGoXRFqsMnff0XSJ4kFqCjHgavBIU5ZR6kLg+o8SYFssIUEmiQkbkRIqRLASubalKg86jtIquSu7Opqy2CZkSuAHvBgY3oIzg8GBOFzzCOZTWCCUIafSJG0KH63bBCtMp5aEmZU8WPdJoZIhsNz0hgzQlShmkhLbtMFIRw0AYHEoatBEoLUmMosDOJ9pe4VLCKjVKeISe0go8DuciWVimM40xMwa1xZSGpAONO4N1zWRSk3ZCwd5nQhLkmJBaIo1AhYjRgRQcMnjiMJByJmWDEGMGmb7dYmSiKIuRWQ+RftsRQ8TYsT9ajKZjbS37tWgYApmMkx4fI1mCS4Flt6IdepbtiuVmzUJrDqdTVIoYYakKi1eCrpUw5DGKOGdOT0+YzeZoo5FyjAImeOIQSbUACcqO/qjD4MiMYspS59EdZ//2jAl8HqV/pKEwBSEktLEEF0d/x87RG4GQjraJnD274MnDr/ijP/omxjjWmy0KQQ6GuiqZTGfMpguKsmLoMkP7K779vYGQPU07oAqIKeI7jzGao4M5hbWsV0ucm3B53tB2kbIe9SP7ISCkoSgL5rIkBst22xI9HBzOUVJS1pbBaXxwrDcNQoCPGWkFz86eI817vbt+b8qrUr1fX973iNdN6e9TfhsQ5X3KuzJ7v0497z/SX9/X11ndXr1GL0/rL876b4eWr7vit+t7VSHypuWvgxMvGm3f1kvxwu999O1oJn4XLHV7wbM/6ygl07NulsQUURJuOf3tTKJvvtOyuNXuF4DTW1oh3rLjm8ym7/GgvS6v7tuWJK/v3W/2VL3Pe+GdwV9ZTdlsnpODGM1qUhJzxvlI0w6sNh0EsNoym0BWgd43LFeO0lgKW2KlQTlNPzi861A6c3BUk21AEjERjC0QGfwQ0VrhwwBCMD+Y0XYDwhiG3lEWJVVlqCuLMhKpBCEGzlcXRJnAjJkDpmrCiT5BrTtaF3BpoHUNHsFiVhG9RwTPbFaRhaDfSWlsXU+fHJtuzeAik3pOlBmMRGuJ0pCSJCMJPtG1Ad9vUDHTFzWhqgkhMXgHAmxp0JUgm0gXN7h+IGaP1JG61pRZYk1AygEjFNYoyBYlBNE7knVMjCX5DoXndDaj0BOkkKSQueg71u0WKTO5LFDRMyvGrCleBLwLpDCa1b1PJDdK5HRDSdc5XEqobOhcZNOsyUJhUkIKjRKSvu8gBKbTimIiyCkxxJ44CBABHzOuj4Sh4+nFwGxeUGiNzYJqNgXv6EOPVInFfI4xJU/OGgY/EHNi3TbEACgLYpc0SYoxulokrBBUOo3XNg6ktiN1PdKN6c2k1phCgTL0/UCKmaJ3WGsRUjMmhM/XrLM2Cq0Vyhi6zpNQpCwJIdG5LW4YkEIQomAIiXboxpzS3jNEENNAih7fS5o1IDPWaNRsilaaROLwaIGUks1mTd8P9F0/sshWo/QYJW51ga0LSu+pq4qzqy06gFSBmHYZTlNGCIE2BqMtREHyCS0kqipBjBILKQX6PpLCQOihadbcu1fzwceHTKYzkktMpgfU5ZSqnHJ4eMLJyV0uz7YcHWz5wR8XPH/+hPV6Q+saslQoUWOMpSoMPjic37Jed/iQ0KWl94GmHcZnU0bqSo+5vo9H7S4pJJNZjZSjP+jgAiEEcs4Yq7HWgpQsr1YI9TfHG/2nLLe9mcR7va5frONvovz1nfe3c+3fBEZe3+7fjCN805UTr/x+v/O/ac/b3N6Ln18P4G5//yKe+fVYUHHr7z693euOe9k4/CauMxHZNEtiSmglb1g/EoibwI18reH3cr9ervU14O7XWal83bFvfQje1POXtosXv3nT4uGN7XvtDb7jbt+xr+8e8IGA0qKtpixrjDFkAYN3bLuOru+RQYERuDDQO4/rWwYUqa5RpcWUBRUT3MrRLa8wDjCZrAKqgkprisIg0bhunCQ6n9hsWmYHcybTKaasuLy8Gpkya1FakUiElOkHR98PUIzAL8uEnWpOqkPqxYJ127FpWjbdegyQEDNsBitBl4qcoWt6go/4HFl3DVebFd5LitKDzxilqCaaghJbaia5Bjyd81xdXpFdYD6ZMcwGCmvpQocQCY3AakNUmd51dM2SrvcYNaWcTCjIKOFJsQVj0bZCZ0UhCg7qCUaDsIq2zzTNkqZIxBiwtsKUhjApaZ0lJY/3Y4qv6bzETgw6wbofBaGl1kij6IeeACw3HaurBmE0SEfTR7ZdjylKRErkMGYV6V2Hazcsju4zmRmGLtJtB4KXSK2IwtAFWF91+KFn2ifuHB0ylQJjLSmPhL/UAV24UaLG9TRLh6pKvJvQU9G0BmU0iYDUGWslViSshFIlVB7oBwchUoqMtJrCakorsYUABaEJtL2j7QcmdcX0YD6CYOcI2RN9QitFYUsyks1mA0ojhCZEx3qzpm07tJBUZUCbnnboSH0AB+vWU4kNZSmwHmgTRWmo6hJjRo37wQ9411LWJf3Q4ENiGHpSUoioaFpPTFsmdUQYSTkpkbOKajZhiAO270bB7p0PidSSwhZIqembnugjgogpNKpQkAeSHAMqhNbUtaWoFG5wOBc4PD7BtY7pZEaOmqFPtI2nKQbOzi9AwMnRMe3mgujBx0A1m3B6UpPyaIrv2hapJJvtQJYKHzPbTYcPmYwmuESbBqSwFMWEuq5RWuD8gMh5915w5BRHgW6tMIVl6D1tF/7GAM1/ypJfmjTz9ZZ3B4G/zXH6TczBL09a73O+m3OK92rDi3zV2+v/urp+/SJ2db8Mv26+e/3Z3z5a7wsUX+bt3mwKfJGvvdnrZUmXG2D4KgC5yVv84h5vBi+vKy8Dw9X2cgwmNaNQjBjRHjmLm7zCLxx/q2XvcPO9k3vFG8HUu5WXgfnLV+W1B7xDGdOvvrkhL1+5mwPfrf53Bn+r8yVUirqesJgdIMjknBl8ZPAJKQzGjKzdatnQDwMhBCqrSUKBGZgqSzEpmaQZy+0V2mp8zviux1hNSpGm6Ue2SSnQCm0sFxdrVtueGCOT2ZR6MiWHiHeRFAJCZFIYRZQTGakl3g+4mEAplC3IqsBYgx4MTTPQbBtScizqknld4uMoVREIRJlIJHwONG3Hdp1JokVFwWxakYWhnoCtamyh6PqID5FmGBA+MKlrhuhQQjPkgeAdok9kY5Eu493AZtkw+MCkkmgVSDJjZEDIno6RpatFyel0zoPFKVd+SdN39K5hs11yubmidx31ZMr9k7tM65p5nNO5jj5HNkOHyBKSASSD8wxDpNQ1RTkjpI7BJ/r1wHLVoYwlIWldIESQRhJjJnpPGDzOdXjXAxEpRvNvSgmlDFkYQlaECIMvGJzE+ILeFcQUyX0kJo+SUBvJ4AKbdcB1kPoCqUp0LhDJMPQGnSQpOZTeRSmrRBJyxzI3uOCobc3scIJAkQUInZAmI7VAB0PTjhI3KSuK2QiIfNvihoacI4UpKYspKUj6NiDkmHkip9EE6wZHQBF9R8qOvncUKTBNhj4lmibi3YA+nLE4nFMUhiwyMSVCCqw3WzabhrIqca7DliW6qsY0dynTtI7ttmNtW6pZNQojP1sSYxoZsUrjXSL3mSwFWkiEyLhuNK/v/WRC8uisyDKSZACruXt/zscf3edwUSFkx7OH55wen5A1rNYt7SaQguTifMun4jFXyxUZyeF8jhJj/tLCKEptODiYE3zk2dmGPjgWJzO220QWhsvzFd4LtCmpJwV+CLjes922TGtNYS2D67g4v4KcOb67YDLR9P3IegoxsubNpsW7d39p/W6X27wfvJse/7tCrN+lAXx9n/7z7MGvA43ftbwrNfWbnPtt3NL7o578CoTbl3cDK7fL1eYZndtSlndH3V0yCHZJK8SuulfrfJ2Z+s2m69+yRWGPO6+btWfbbhu3Xzp3fjM4E7s+v1Leurr6ze/JdwZ/4WkHdy3y0ICQDMNAs93S9gNuSJBKcpSkdqDJLfbQkCIs6VmWkdU2M58FqtJgC5gfnlBVE3onSSEgvd/lJe2JIYJUaK1R0tBsWzZn52gtOc5QmAqjDDEmhn4g5QiZ0U8sj/l6pVG03tO2HoTClJPxx1qUKmj7OIoXRzCq5Go5oJTAe48yCmU0ZiiwuqQoIiFItMzUk4LJtAIp2Gw62j7RuUjA74QuBaawTOZTRmI7jeYsBTlngg+4IRBCpCxrrDUMbsAHRyHHVGGRRLKZqjQIZemc5vHqis3mCqSg+/+T96drkiTZmSb2yqabbW6+RWRkZi0ACo2NzWZPz5DNX7wO3gEf/pp748Nb4Mw8ZDcxPdPNRgOFQmVlZGy+2KKbrPwhau7mHu6RXgVg0AAkn/QwUxMVFRVd5JPvnPMdG9kOHc4OWCSx2FIqh+8Dzo/YYAmTKbofOxLgnUHQEHyNGwus9exut9g04jzIMSKUyX6SomCwBcELgpU5AESWNHUFqabrEvttwLmIKTVxtLQd7PeJPhr0fMliuUKYAjuO2HbAh4RWoIRGJo0fI0pXVMsSzIyxdSSn6MeIKSRaNegyIY0H6Rh8YHCBru3RGmpVUAqDi4Jd2zL4HlVJiqZkv7Ncf9pl4K9qhjYHKlkbc0BFSnjr6fYjSpZoNUcKhXBZiqA0M0QhUSiaas58tqRSBr/vSbbHy5YgeqqyzmmJZMQGy26zp2s7ZCnZ7Do2mz1CKIQILNYrVicrBIJ227Hb7RgHR4yJ1dmKFDTbznH1scVbQd87ZMpsZkgRQmK0Ftt5DimUIBI92DGQo15AK83Z6Qm/+MVPefXqlG684T/9h7/kN9994vrjhu2mQ0znVaiKsXN03UAzKwhDz+3NJ4QYeXV5zmy25M03r9n3HR+uPiDQEAs+fr9lcTpnHCPeKYhQlgopc5CW9yOV8Qih2O1butZDhDffVqi1Qmx2pNz7DBitRwoI/3XO/H/nJT34fDD9wks9dn4rK9Rv2daPtf23m3Kez0vwsr1f8uvv4kH5XPtfHp1j4P40tHrZmb10jJ/a/rSR9/mjTIbBL9Z8qvWHbXx5yXIY/5ews9v+E/t+w2qekEpBCvcKKI9Jy0PrTw7r0+f02AR619xxu8/h2KkP6bNK0853gO+56/zIDP0Z6Hui3hO/PXv8Z/Z9aXm5yPPlHErDbtvjnESQGHuL67KvH7ohakWOvmiw3dQ5F9jZlM3DQ6SuFYWRLJqGUUn2uz1KZnmXdtPRty3D4LA2oYRkNitJRG62I2UBg33PxekFTV3g/IB1HQJBYSoCsN+3dG6kms9BJnyC4BM2WIyXyKJEak1ZVRA9MSpSKujbgBAC5yNKR7RRGFlwfvaKxTywue1QgBIGIbKpcxhdzrjhMmtYlZokMoUhlSTFwHzWoLWmKA3GCIQIKFFQqAZdlMQIojAoUaEFaFUghQQpaJPgb3Zbru33vP1whbV7Zss5o8kSLcEYOl/ithHBQOgDQ79FFI66bEi9pRst2ujMznmFiwafJKPV3N46BjdS1QtUqRGmBAH93tLvA0UpMUUWydYJhNT0XUlMkZtbhXeeauZIQtL1ga4LOF9Qz+boskHVEi8Su7YjOEcRPWqIxBQJDqzzbG4H9FzS7nMyb4SEpkQana+BiiRp6YYWax3DIDBKE4uGAcV213K73WHjSLkomYsSHyRt7whppGoWjBZQCedDluJxFhEH+p2jqhZIVWTtQx8w0jCvZgwJCmM4Xa5Zz04po0bOIrv9RwYPUs+43l7T9Xu2uw0+Rvb7nn7woCAI8AFIAVMICu9xKWCUYnAD27bFOwgB3NUtpZlRNktM6UgJwt6TfEIhKZWClBh6BwhkqYkuQYwIrdBFIoRAJNF2A//lv/wNV59uSMHTuT0pJMpSI4XE9jlHcFXU1OWMqiipihlNVZOCIyVHVddcnn3NMDrevrtl1+0ZRse+H3AkPnxo+aYsWJys+fTDnnZiI1N0pBio6xpdSrpu5PY6s3pVCVJIqrqi7wastQzdwL51BAulgW73O7/H/lEWMf0HX+Zifryd+/IUMPi7AIf/W+Hy5wiQ362Fpw3KTxs6Hx/pd59Uf7s9f9zr8/HvL63/4/34XWHyc/167Fv48N+nyo39xG13w2viPRgRCTHFBYvp+33k8gEZiqN/nmDagMfX+0GNl5624KGizO9Qnj7/5w3lDxjMdFjMfGEUEw9R7t+12bc4OcXFQLA97e5m0obUYBZQaJAaWSqiLGDvYPSIhYGipq4UdV0iFbihI7rIem5wwbMbBnzIZiDbWRQFVdNQVQKBJvnEp4+f2O1gfQmVlvghYpMjEVBCTYnoa7quxY4DLkToHM2ioqlqUBofwHsYe4ezgbppUBoapdCVxpQFgmzy64eBsjQslgtWq4bb6y3bzSZnDtl3KFMgVYlUktWyQBWGqCXzbcXNx08Mfsd2n6N4EQYtcnJqlyCFRAygi4JxDOzbHiUFZWFQQqAKiQRUyllTzNKwl46dHzF1jV/WhCjp95mpU2VDUArvHEGOeFMSo0criXOBt7+5yaxds0IojSglSWqEnFE2Z5Qye37sxo5u1yOAWVNRLgxRJcaQM7EIbbBD4GaXSC6QQk1R14zR0XYt3gbqesm6XKF1QbvfY6Oi3fds93tkjESVcH2HJjKrKlKSWL/H9pLoJNFLSIZyvmBRr5DCsd9uGdzA6DzbXUvcQyE0uwCL+ZzgPCLNqYsllSoJe7BjTwoV0Xv21w5JO/n6dRgFMmqSc6At/bAnDEDMAtqz2YwkC1AenSLajSyS4yenJ1ycrvj+uuAvvgus5jNQJftxhxSKod+xbz1jEGzaxEF3uTJwcmJo3+65uRlZLEq61rLZQUqCRVNxfn7J0EEKAjdIhj6gZY3QgmQjbvBgPYLs34ipsCjSuCeFSPQCWQjwibiBfRew3Q0h5XseC2ffauqmYAiR9toTug5BR9lIztdLTk5+RvCOs8uvePXqnHm14q9/9T1/9au/5uPNDf0YAcnZpWC2mNENiWZZk6Tn+maD/9ShFJysSi5eVSxPS7b7PaOH1TJHAUcb0ZVBCUVwEet8li4cYezIi8h/FuXxtHgMR+4B4fP7Pfd2fxrIvJwd+rw8xy+8HOA8dzaPz+V50PYU//Q0YHvc+o8xdy/b63Ht56DG4ftzjB1P/Pa4zo/Dzx9r4cvA78Azf84sHZ99enIkxN1v9wLPXxqLp3p4YAQPKeg8t3y8/Rt+/9seIxeEkNtXSKS4zxMc4BGNJ7JZ+Kjxp0DyU314rqTjSp8N8xfupaP6z3N4n/fsYUaTLz+RT7b7zIrxpc/mi8Gf3W6hMLBrYTNmaqOsodZQStCSKAGjYXsFQZKamqZZYlTA2xzcXZWG87OG1bLBCceHT1fsuy6bTj8lkiXnZl3NWCwKYgjsPgEj7BWsyorazEg2+9LNFyVfn7+iKWv+4ru/RsSYo4X7nkErbAwkoQlJEJC4lNhuO4RP1EVFWUG3sxSixJgcZTqOgX6wjKOjrlvGdmC1LNlv9wjhMdIwdD1j8AhlqKuGalZxumho1Ctk8ow2sOt6SJp+9JhCI1XWcNNC0VvP7rbj6uaWqjCsT9aYwuD6HmKk0pIYQemBskqUp3OiD7SdZwhgETgbGUJASZEDpDxsb3tSaGlmc66uWzbXkrqREDyohAyCWJBNst6QyKnEUjQMLvvlVaQc1TpkRqcUBis9yUouzl8hRCDGnlIbgs++mKMbUVGiFbhgaW+3mMZgvcd6x7opmFUF0SZU9MyXc8rSoBvJGBU6lYTB0+g5J7MzgorsXM92GNi3ln47IpLChJJmNUeJOW8Wf8zpouVj+wPt2OYIdCLeR6I1VEXNrF5ih8i2H5A4Tk9qFvMSwoAbW2y/pW8DhWxogbHvkVpQFhKTBLHr2YYr3hPZ+g0pwi++/hkxREYhqFkw+BG0QVY1m33Pbtwy7KcHTEn2rWDfR16/EixUiSkkQjr2bcB6x+mFpCgK+sFNrHeEqCBKnPWM/XD/tsRDMvfL0RByxLVL2RosIDkYe1AlsAdqUKrk9mqg3wTCeP/usENkv+/44YcfsO0N3/78K/7ln/0BrtP0bcfY9agIM20IHlJfcttuGD/1zOee25uOaANSgVECiaDbeMZ+wzg4zk4Lzk7OcaOn6wZ2u5YYAiSBtwk7QBqhXIH6ZyT1kifoxEE6I/3Iy/+3L78N//TbHPtLoO136ctzU/RLOcdjAP0Y9hx//zGu7CXHePz9MWPzUibtd+V2X8bnPTzGU4D3y3D66c8PwYuYMn48taB4DoAdrurh98Mr7d2Hv6K3OxpzgZKalAIpjXf1VIIgHl7nLy2Pnlr2PF3zYW+fu1Neeif+3ZXHo/bjz8B9rZezuS83+2LzP/MKdAFBg6iy0KwEgoPbkEVnhw6QMJSEQmPdSLAjRgu0NnRDoBw8Qfg80dYFIQxZ2qIDFRW6kMTgkCIxQ9ISEbcSayxp7tFaZoFZUVOHGXWcczY7R5xK+hC47XtGATZGvB8hSZLS+BiJ/QBW4wYIQk+O5wEfPT7GDMljwntLcIJZU7FczdnPC+wwUi1LPAG3a7FjvlEBlicli7omDh69MDjR46cUYSEIUIpCG6TQ2E3P2Ef6nYUq4UpLignrA9E7olakmBht4GR9Qikr7LCj6/vs8O8hjIpEIGDwAWKwRGFo6iUEhds7FnXBYjkHIeidw48J3xcoJK5zuGRZLObMZI0XHlEFCgOm0JgAgx+I1kICNwo0eV/rBpScU1Kj7YjoLEF6OteCDuy7K1In0VohggARUVGx0DPWyyWXJytav2MwW6LtmUlJKiIrnZhJyY6Ei5ExRpLWBJ0zaVSLgtX5BT89/ymr4g3atXRhxIvA4C1tP9AOeyBR6oqmbOhsl+VipMYITSEkSShiSCxmc3B7RAjIOIKyCBQ4BU4QPOy6lr4d0DNFXVYUusF5wd4PqFrQDwNBgFQFPg54B/SAgmJlIEiWc5Mjf/uADIrVco3Unv225/3bDbNmjnOQvKIxM/rOE8ZAGNMR8CM/2G56vpBAhJgOwlmgQFSKoqrRBbRqz2w+o20DfecmczEgJbo0NFVJUxpsF+i2ke4m8O7tBjtEfvkfPmC94OLygpQEt9uOaAX9VlDM5oixILR7pIL5vGI+rzBa0m0HuqEHJTGlRgCmkNzcdKSUqKqautKQBEqOFGvDfLFgc7t9+evoH21JDybJOKXXSk/WeKo8/3L/7aa64+P9eJ3Pa3/+/Tk4mJ745cc5zPtfH3vxPd/fz73ZDubD+++Cz1uID74/xYQdb314Be9bjVPqs8cmyueFnR8XwUPT6WND6mHv9Oj7l8qPMavHvXsKVCWO7zlxdO5P9+6+PL5TnwKHAG8//SW77h0ns28Q5AQRUuXxlIfkBSK3KKb+HFi/h1fg8zPNa+TH+x+23S8y0lFmjs/GQHze7pfP+qn6T+393Oh9DvKP74mnzvxeNOrlC4QXg7/FWUNRlAzO06ee5D1GR6rGYErD0FrazQ76DqoGKsPyrKFssr+Zc5rge5xvaTtJ00iKUnKxXrBYnYCThJ9DbBOlKqnLmhhyQvvTkxP6dmAxWyBFpC4l0VvKYsmqWmLkDEHBrD6hqZZUVcN1GrmlZ/CB9rbD+4RPgtv9nq4YWM9PuDy9wHvHarVAysjN5hMJOD07Y7mYUdeGSmter84phcyilPs9Yq7Yjx3XN1v27UgIWTtu6B3BB4ILrOZzzpczXJsnXOsszlnckHX/RIRFOYM1aAlhtNxub4kxUGpQVYkjkkKglXuEaRh3Hd3NnmbWUOkKCo0QBaaoUcrg/Mg4QhhbTB95c3LC+eo1RXnCfuh5v/nATbcljJGqFhRpZL/vWFQFi1mNUTO6uKUu4auvT5DAD9+9xfeOmsSoFMJuGHZ73O6WsOgoq5qVCSgTCHZH6gVFFVkVe05PXyNFydhbxGgR0TNbrvi2+Smv1Yz/z+7Pufl0hbUOPdNIYdjaDic+oZsVq9kCLx2ftht0I1g0c86WS9bVCUpoIiOdvaHtNoyuwwZPu9/y7vt3mKLiJ5c/4fX6DVf7H7gZrlkslizqGhkdbuwRSXC2OmFeVhS6IqWADR6ZDLUsqFJBETWlKlmslrz+5gKk5NffvcOUDa+M5j/9zZ9zdf0BURmS1vR7y/4TsM3P4E5YTAP/6s/+gOVJxdW7D+xuWlIyWW7GGIb9OMm3QBwjRVWRSIzREWLOSpKO3xM6orQgBOBRUgylFfWioSorQrTo04bZYsWHT++JuwhKo0qN1gpjcg7i4CLD3vHhnefnPy/4P//bP+Xq08j1+w2uDyyaBbfbHW33HlEUFGpgXq4mey3IEgqlqUyBIOEChJBIMdLuR4L9wKs35ywXc5x1OdIXiZnPYblACkHbD/iXClT9Iy5PswpfNsc+nuqf495eyjk9x4c9nOaf7sOXeL+nJvqHddKDY3yZozgek6drPQ2Yn659/Kt44vNxrQNEeAhaP2/1KT7uqfF8eg94ODKP9z4+l3T0TTxR+ykY/HlvH573EfB5NA73Xmby7htHIPoeZBzv87AcZ/c4hs9PXZvb7h3X+7dcnraUoiYKnfsrIiKFqX8KREAcomYPWE0cnf+jm+4O+D0GuXf73I/IY9/BdB/p8cTVf/SeSk9f/8d1vrTtc1WXoxE+OlzWqv38PfmyZ+pheTH4e7OasV7VbAfPh2QZ2haBo9GRxXyFMxC6kbgQlLMKVQh+7w9PmM0NQ9+y3was9RSmZLVecnExZ7ky+C6iqKj0jPP5JaUs0aKkkBX4SEo5QKC/ccyrCm8dm+17unZLkom6KpFKc7Xdsd3t0MagVIMqC4x3gMJh0VLgksBgstyJKSgrhW+zHp82gn6o8CFSmIp5s2RWl3T7PVc3W5K3IBJRJupCMqtmzL6a8+H9De9++EAIlqKoEAFCb/G7FuOrLHMSIkWI4CLeexAJk0AKwWo2w0gYhx1t8hRE1mXNfDbDmCqnogswrw3VbMVCNTRlxcXynIvmjJ+//n0uVmfshx2//PBLvv/h1yAS56sTVFnyL775Y5Rc8PbTJ/79X/3P/PW7XxJEpBYCvVwgElRKMA5biIKTumZWGV7PFjRVTTE43r97y/Bhi9sIkg+UWiMLRyl3nNSS9bxgV1XsNntShNlMEdWcf/mzbzmZv+I//NVfcbvbQ0p00fJ9955Wzrhpe0RqkDERY40sKro44tyO2oKqNMtGkaJiGAK18ZSixw4/8MP2Pe7shutux667xkeLUpJZZVguK/ASTaKQEpUE3fuOpGaoUqGFQMgapSI6FZiyQCvNYHsW9ZxX55ecLdYUQePagLewWJ7w5tXXLNY1F5c/Yb/rSVLw7/7T/8jNTUe9mmMalaUKDMgTmC8q9uNABOZNzaKu+RQSm9sBa8ecG9pLZIQYPNFnl4W+u81BGoVgMauRSjOMns2mJQmBrgtS8OBSjpCdiAshQYiI2/ekwVPPCmQCu9uiQwKlSFKjpEImiKNnHCNjN7IXidFn3c56vqRud5yeZ5O5FoawvcHanuW84ny9ZAyOrt+TRKQqCyptICZcCDgPJEMkEGJEKVitGzrj2G484zgSQySFRIiJEKAfPMM+Pvv++adUjl/Uhy0PX95fBoOft3Zc+2Hrx/DheGL4Eufw+PenIPlzoO/4+4/xXfe9FI+2PQ+b7ts51Lln2R5zIZ/v/xDcSZ4qD0fgsZTxU726b/sh4/f8yKUHvz7F1TwFth/v+fAKP1/7cU/EE0ee+LRH4DVy4CKf5KGOQckxkDn+cITLDpglxvvFrJAQouWH93/FNxf/e4r5SVZIQKIRCALqgMOSRKbHd810/kddEimRmCx4aUKBKXEIHxbx0JeEEBNMPdQ7PsN0f5zju+LJq5rgsYYniKNLcDyGExg9Zi/jVEOkz+qn9Pj6fV5+lxj3F4O/eaOpC4k2NZVc03ZZFsUYQV0FutRTLx1nZ2cURU1ZCX7xc01TSz7+4JDdgEuJ2byiKATD/ppGV4ROcHv7nhgM4k1gvVpTlRUx9DkyM0Ezm9PHln63YTmbYxoNFoZ+QGmoC8kYLLt9R4iJT7s99eWSzvUkJNEGEhB8IFpHU5QURuJsi5YBrSNSCGRKJOvp3I4r69gbze3NBoVEqZgvlpDMTkrW5yesT+aYmAhtR0qeWdUQY8R2PV5G3A0IFMYYKlNSaz0FfeS2OucolKASEhEDpwo0ghNtmKsCbQqULpAo5krTe49OkllRcnm25Hwx4/x1ydmqpB4cwaxZLxR1UfH7r37O6vRrmlJjo+RyfYksIvOVZtNucDGwHyTDvmO3ueVq2FMYwe+vXnG5KvnZ+ZxXJ19xoWr+p5sbPoaRMgVU2LM+OUPqOXVZsprNkVrQqEDhPEYVzBclqixZ1ycsmjnn6xPqsski3OPIbtiBiczKBbOixrVZ7NcDVVUgpaCQCRMjpZKURcUYBE1ToJRk2G3oe8e2sHgPsxKUahBISqGwzQn7TU+/bdk2G3aftthrizpXLIoZdaVxo2FzY3FdIISAlpG2HzlZz1kvz3l1eomOmr0ZuL1p2beOXRf55udf4WLNu/f/BZdGfEg57W4QFBiqsmS59pRFyXK5otzuSUTGvSOUNXaAdh+wY8JUAT8KZBIs5xV1qXI2lbYjKj+JTENZaYSqefPmjLKuEIXCjQN9N9Lusyg5CKQQBOcZB0fTNJyfntANW1xIVLpgtxtxMSIFSLLppFCaeiERBJLy9H3HL//yHSkkZosFVQMxJopdhdSCEAJNUzBud9hxQAkolEGi8DbQj5bRBpKUCJnQJlDUWZrGe4sPFh89dgyMQ07hJwTYAeLtb/Xu+kdZBA/mxLttz5lq7oHQc4DmqWnhsXHu8OlzpunxlkOb91P+U7DgYS8k9+09V+fQ6n15DFAenn+8q/1YFPtxOfBYzxn+HkLCh315HuIej8tDY+xjoPrU/s9dy8OIP80APmT5noJdD4HpQwh53E767LeHkOkxxH4S2j04t3T06e6+OBoAMWWvPJzjvUdgegD87hB3yHVlyl4tHz78DdvdFevm52hZIqNHk7WE79LIiUTO0CQAmfs0dUUgjnFT7kHKaTtTzBlDDmDrICEjxPGZyul8Eg/NLJ+Pxf2IHN2Z4nB88dke98/Q0y1+tmxIR8+qeAzrnri3DoM7gd0MaH+8vBj8dcOA85a6aZjNSpRuGNxIigLvB1IcOFlofvHTM2ptuPxKUTcSTcRqTy8DXkkqIRnanuvr99hVSbSSdz/saFvYbywXF2vquoYI/b4nhsR8ueLq/S1jb3nz1QWzqmS729DudiztnLVc48LA6IcsyjxYXlcJ6z1106CJjKMlOk8tJWenawoZuf30EVVKwrhHa422jjBY7GDZ3tyipGJ326OkZL4q8SGQArixRISEGh1h21H6hFGGtS6x3hKSRPWBftNRNjPmJxWVLEArnAzY0aOUQo8jZSWZGZ0ZGXNCDIHKlJRVjTYFctI6JEq6q4TrIlaMWNfyaTswxC2zqybfEjFRzXJmjt1wS+1P2LsdUSjGBM0MztczpBrZdnuG3lOlQDcOuGGPAow6oywCUgwUynG6nPH69IzCFuyv99QLwen5nOWyYNHMWNVnIGDfzulOTyl0TT2rUKZhuVhhyopf1CVuCOzbPW3XkmKiqWqqpkTIhGtVzpUcHM28zJk4ekf0gZg8vnLYwnFyucD5kduwZSw9WsFp09A0c0pT413idtOiB8PO5SCMIpUU1NRUzHXDTDXMi5IkC2QfKEzJ9W6DE5AGTXO55PXZG06Xa0RQGDUSU8Hmdk9IWcbo48e/5le//jWd26BVwWp1gqwqtC5Z1PnxrKqK+WLF6fqE7aZldzWgk8GOEoFBK9BKMYw9SkDT1Cy0QQuJTpFxzLkz1cQ0z1dz5ienlKXJ99GQU6p13UhRGEpTkQJsN3tur7Ys53PWqyVXt56yqPEu8j5eY71Fm4rS1FRFQVUpCg2d6/h0EzCV4W9+/QPzxZyimeV0i0oyisTb958Ye58DhIInxUShFUZpiDAMjrYdcSGia5kFoxVIDbvdju22p+8tyhikzosxN4IpIY7cuRX/Uy6Hue8pJkvwcOp/WO63PQRaPxYq8sjP6e5IT01Pz7N2x0fkUZ30xG/Pl2Mg9xh8iqN+PPz0eQv3Rz8GYZ/D1ONpW9z9fTwGj4Hc4cj37T2n6fcUHP3S+D2EUfftPobPBynlz8/++bF+qqXjEX54Hzz89BDuP9XiU/fHAfAd/57S/T2REkQxsVcCUnzYhk7ZcHF185ZPn77ncvEnzKs5SUiS8rl+Ime7EAIxPUCRaRvT1ZqYu5TStF0g0kGRMEPRnDUkt5PuzNiHgCtxF32b7gBgugN1aTpZcQCdEzCF3Obj5ycD0MOITvfT3aU4vq8zSynElOcnPbp2aTo17rHpfSvivrmpLgLSU0mGnygvBn/XtxuESCz9QF0YrBtx3uF9ynlGZWI1q5gpxdLA5QxuboccudhHaqnxUiB9JDrHsPG0PuJtor2GtoXb6pY49Dl1XIShG3Lka12z33YEnyDsOV2vGPqBoeuzNhmR0Y2AIzDi3IDb7RECmnmFIBDsSPSRxWzBV69fYZTn3333N6SgCJsb6uUKWWhCUoxRkmLEGMNqUSGVYHHS0PdDXmEUCj0E0rZngWF29pqmMJytV+zbgdfFkiAineqolwvmJzNkkSftmCTBJcqywrnAoimYFQrjoawLRjcghEApTULgiSAk7b7DN5IhBIrCMbqedu+42XyiMNl3zOiCsm749P6W/e6Gy69eMzupWc6WOJfohpEQLXUFwmiELCiqFWtXMO80RSFYrSuSsnzYvsNHR2XmfPOTc85PTvh0fUPVlMxmNacnJbOyZFnNEEKyjGXOmSsLiqLEJcWiWSCE4FytsYNj6HtSyhlCtDCYuiAEhx8VTT1DqMCimbNr97guEJ0gRIghYMfI6as1fdfRXfZ4ItH31LMZJ8sVddngveD2uuWvTn9Dv434Apr5kg/rt4Rrz7o6QXTgRkdTN3x9+lNm8znreksScHO942L5hrPlq2zy1iXnpwUXF5bttuX07CuqomHfbri++sAPV98hCs3ZyRlRa5wPhORodIWWChE8y+UKu4tsb24ZW0eIgnk9J075LCktWgkKJRFEgpsCOFLWijSlppqVLE5mLBY13TggfH6DlqamWs84PTuhqWrsGPigr3Cdo5AarQSzquCry1Pa/cA47HFBUpZL5vWKuiowxhPxMEReFxecf3NK60bcXpKSYl2WrNdLkpasf/UDH7+/IoZEGD16klmSh/zWraPbOSIQSBRCoCuJUIp239H1A87Dal5jdMJ2ATeE/EL7ZwD8AIppMjBCo6XO94AU3LFXdxPa9Dflaerw0r+fXBNkJ4P770eMRUzpwbZcLxFTPPLeSvfMzLOs0edMxvHk/XDSe2rSOZ4SH8Kix3zI/ZHlUXuPWarDd/HZXoe/n5uNPwc74rMjHJ9lOvr9sTTKYds9FH8IJJ+DnsffjmHv5+N6//kphvAR4/TguM/Bzs/L8XimI+726Z4ft5seHP0wNvIO7NzfUZPxlXh0+R8vFDSZ+dsOH/jN27/g1cmfUX11ilYCrxMxpOOjkFI+ehDqwajIzH5koAlTXvGEPHyf7qkkBFlM7f5KM5l+M4aSd8+NmDosxDTiYgKT6Y5ufLiQuMtHPIG7I1Pz8bimJCesloCQGchjhpNjn9Nj3vkI7D66mLnv4j5w5QXl5dG+weGBod/RtQlnPTElhJKoJEkiEYPh+9+85wc78td/mWUuJIqU7u3qPgYKIVnpgtILiih4VUesEtQJ0t7ihEMKgYkRrQRVAl0ZRBAUKVKRKJuaRpkMi0OgKQrEakXTlNRFRx0BIoUb0TJhCkXQkrrRvJ4bLlav+XT+S0iJudRcliXN+ozK1ASfTVFNU1OYMpt9C8E4WpAS6y1D31EVhlldURcFTVUwa2bsuo6iKhiDm1Ya+SZ1KZGERKkCrSpKU4IWNKXBEImDRWtDwKGkJEUYnWfEEWJgsD3L1zN8CKgC7Njh+oBIkbJQGJE1Cjfthu/fvuX6/fdY2XHOGS52ICRKV6yXc+rZJbKEvd3T02KDZRy/QgpNO7bs7Q1j3LB3nqIOzM8airli/dOvmDcrrq4/kPTIxm25vf1ETKALRV1VyCDQUWFjZOsMMTmaakaMAlNI6rKm0BUxCvbW0/cDmoJZeYY0kkEMiDKxXi1ZzM4wZk6KCaEK6mJJ9CCiyVHURLSUrOcnzOs5AkXXO359/ZbtxuGjQOuaH979JY2vKEpHu7lm3HUUpuLrN6+4fPOKy9cXaFPy53/+v9D1IzdXHUMVuDirOVutOTstCK8Ds/kJaMuri0tKrfn04ROy0Lz56TdoXXKz39Bt9mgjEVKyu/rE7mqLGyRj27P51LM4WTEvauw4grecLitEEviuYwBElAx9j9aKosj+iEVpaOqK1XxG5was92hlqMs6LxJiot13DL1lt91xe3XDNgiaxdcsVzNOXy2xwVPWimgTSQZ2/YbbnSUxEkJAlprXX73m8s0lg3N8fP+R3W7g/OyE35eJsiiZNTWb0tD3Ae8ChSqRKNzgCckz9I6YVVzwfURXmrouKOqaYbdDaokR2QdQKUXTaLwLpEPQyj8DqZevzBwhNE0xY1bMmFU1pZmE3VNWQyCFDIBSZh0iMftIpjQBuERKkUTMU3HKv4XoAUGM2U86xjjVS4Tpu/MTOE+BkDxhiqZMU2spJaJIE9sCz3M/3G1/CIIe8kN3E1U62naYpASIJPNxjqItSeKRT/ujCNv0EI5lpmbaJx1N5Edw5P48BI/DET43kh5Dg8wMiSQmyCAnkC6zFh0Sgcp1p3RkT2WVuOv2Z3Dz6LQOaOJuZOWDHooneni/9QDhxYNf7885f/ochh/A32Hr432O+/JwqXD8f2bFctahQADC9F86xOvmNKyAJ+HJag5FTKgYGWLPb374//HV+X/h4uJbinJFFIqkIoGUAz1iVpjLi4MM5kRKE7uYPRSTyPNs9vnL92SSYrrnc/+FyM/NYUElpntRTvelIN0BMqYzF9OlyTnqcw7ifB+Lo1v7iCUkIYS+A2OCY9ZuApBiOtZhgTaxfofFnBCT80O6v6KHJ+pz8M+DY72kvBj8lQbKeoUpEs5HZhUoLRFSk4JA+MBqMaffbhGDxjrP+XqJdAkVoTAKqRQpwXy2wBdfEcOIDAoaQVNXFMuGGAVKappZRVkWSAMnJ0ustcTOoRrJN99+w3y+YBwsbTcQhaBqGtpu4HZ3yy//5tc0qsDHDqkDs6pCiAw8pVEsV2su6tfI/8N/QwqBWhuqQjFfnXCyWGG0BhJSCJSUNNWMIfREHyirgn3b4a1FIIh4AlmyRimDriqSkhQuEXNWLgZrkS7gQ2QU0AmIIjAmSzPL+n9DP9IUM5QSENOksWcRWiOEQheKW35gJKKTIaYeHx2EiE4yZ71IiVAI1j8zzL+5oDkp0fNIKLLp2kWLEJFZVTBb1Lx//wO3wzW60lgx4Fzk3ce3fNi2mCLy6qzBqw0CTRgDdT2ndSUfr94xm1Volei6ntE6iqpkHhrGsafd77A+slzOkcJTGINQGlPkfK9KaJz1fPy04cP7G0ot+ebya+aLs3zNZUGpG05mr5iVp8QgWczPKfQpp4sL5tWcQpdoY0ghEFVB68GHAc/I6XrJyVIRkkLIkq9ez/mzP/oJKQWG3Q439hSVYbGcsTxZ8tOffYsUM07XZ3z/9j3WBWKKXH/a8t2vfmDf7rjdXPPtz3/CYtGQYuTy8pLzs3OUlFyuzsEoorWIYFFKsT5fM9ie9++uKY1GLbJtUxMZ93s2tzukgLOLktFaiqKhKBtKU1IWNUVhsN5ih8BuZ1GmY99GPl1fYYzhZLnGdjuc8wigLutsXwmeZl7S1A0XZxfs91e0naUfBwIB60ZsOzK0DtuPmFIiFTQnS/ox8v7DFoqCm63j9mrH5mqL3Y1cnp8RrUeKLN2iVU2hNNZZrM2ZZFyIJAVCC4QSGFNjbWJ36yEIjKkIeMa9pTCaulIEq9iPAdOQdUL/iZd/89WfUsxWlFphVIFBokWGEAfP7pSyZAjxMD1n21eKiYMP0HE5YIoDoDqUNE0kTGAOEj6EPMkR8CEQD+YtMqiMMd6DMXkAT7lv2byVOGYXDtsgA6Q8IcZDr+7gS2YiM4CSUhLjZOpKR30nMzMikY+d0n0UpIQUD8790ww7/Xjw1RdCEKdMNwJx93tMMRMVU39jjESRHkyk9xD3oXSGmIBmmtCwJPvWHkCsVgqJyuciDkzRMbOU7pkZeXeIo/OejpvE3XW8u6jpADA4ike4N08edrgzTd+B6Htm6lDjri+HHe+A/QFcHO6Z41vomGnKqPb4fMTUzmFchRBIKUHmBYuPkSgCgUgQgSgmQCYCNo20/Z79buSm7fjU7vkw/pq/fvvveXP+M+a/90cZVEtNSg4hwn0fUgQiCiCmu4AIIQCV86DHCFoEUgrEGCB58pyeEEIShEQIlc85RsQUCBdjvl/SZJ++u7/l5BNIQsiYr7HIz2Scng2mMQg+og7V0cSYHgysICFiRAuyUojUhCimMxLIFBDY+1tB3N+R99A+Hl1ZOGQCQWSLzUvKi8Hff/uv/htElIxhjx0jPoYsblxVqCRYzVZ8/fo1H75/D70j3XZcnJ3grzvUKFmcLFleLJnNZ5ydX6BMQYyJalaghUYbzWw1p2xqqqKiLmuqoiQKjy40fnqAXRipizlKa4SIjLZjsCMSw37Y8un2ivXpCtsP3Gzf0tlrZrXgZLGirit0oSmqhkIY1OwNu20Lg2PsW7ZXP9DvPiIVBO+wQ4d3lma+yEArCcqmYbgd6bcdzlvQkXJRsFgvqWcNwzDmFU8ICCUJKfDp4we628y+UWk8EcvAPnWcnC04vVgjTOJ2SAgfSWMk+kiUkqgkPgikCvRhgzQFWhVQZgCJD0QloCzRWtNoTeEbXCywbqALHftuQ/AWhEAZzftREd4lbjYbUuGpQokgEXzk7M0J6zdzQuiR0hN0R/CRbb9hxoqZWqDmASda0IpUWIT0iFLiVc923PDhustSck1iuaoI0iFNpI8do8uyIEmCanrWl4LkI1fdL3FsWMkFgoK3bwduP/57XCcxRcPl+c+YN5f86z/9N7x5JZjXJ0hdIIzBJ8+23bG7vSGMPdWspNAzoioBy3xWs1p+kzUHU0CKhFKCFDy9HXn/oSX4DU2z4o//5BxlDEoAKV/Htt3z6eM7Ts7PWJ7MCTawXi9ZLJbcbjZIndh1GxaFpj5dUlU15+eviVgaVVMvZlxffaSdC0pT8fbtjg9dZNZoThc1aT4nJU3wiVoLLi5fobXhenPLdrtl2DtauWfjt7gQKeqC3cdbCiNJSeKcQ46RumpYz+fMy5KmaYjWst9sGIeRzXZHdIlC5Uh8lYqsOSmz8LgUBV3vkW1E+ETfejbXG/Y3G/Yfd4Q/sBRTwI0dR4wxKARSarSO0Gf6rqpKyqYmAVqUDH2WW4rBM5s1yGQQ0SKiQAmBjJLkA7OKzHr9Ey+vFqucQQaBCipP6Gky4B5e4OkA1o5Kugcij3mZ+22P+Z6HDJoQAk2W0SBFChJCHzFoE9g6SAvdTTSPiSHx8PsRDrv7/QGBdTfxiQf7CfFUnS+0q45wygFETSAmHXzcNZ+H8T4m2RJ3vmOP69xh4cO2xEMfqrvfDyzL5DM22TbvJEMeXb+7sXwA3ODeaU48GJejQxzRawda6eEJHfDe4V66A7Li4Xg/vD3uB/bALh2a/oxtOnTongy779L0Rd5tEwh5AKQJkieKkGMvJCAFSgmiToS5xTeCzdDxm9uP/Jfrjww//K/8Wi44LQIXb34GRY1UEkLI6ghSIZK4V1RMiRQiKXg8Fmd73NDS327o2mt27S2b7Uf6cUcQEaRAKI0qKqp6RlnM0LrEFA11OacwFUpppJRoqSnLGikyyQUSoxRCKqTMgD8pgZpYZ6kkMSbKUiKmBZwLMafhlBKlJVJCCB4lFda56TxASkGICqJAJ4FGgJgWYjJxcPG4v6AHV5F8UQ6s8o9pLx6XF4O///7/9n/n+uqKwXUobRhHh8BQFzWl0Syqcy7r11x/+iUmeeZGUFKQ3IhtLUloiqamqCuMKRC6yBdVJpTWpBRRqgCVQbYSihizT6CzQ75ZZcSNA127Z7PbMNoe5we6Yc92u2FvdwSR+P7Tez69veJvfv2fuBluWZ5qfvL1Jacnp9kkFS3IgqY4oVRrVuslRa2QKmGMIEcmepSLjD5iXYtWkqpqqEqNWYAaE/2Y+1SYIt8QQnG6XvHq7BVFCbf2ilbd0PxkRm8Lksg3iXUBOxiGYHBYaG6RpaBwAoMi2IB3kTEE+m6ksz2jc5gK6qahmi/BJ4yWiJQIzjHaHTs74joLUbBcn7G7+UTf9TmdnNY0sxnFvEEVknEYGXe3FIXi9NWc5cWKH379CT/2LOYVuloQoyWGgJSGqEoKI1icatZmgR8tCTBjQd97BCOmkpzWDfO1ph+3SNETGWiqBUWZsvNHiJACziZU6lg1FaacY4xEBoGMEaVUZr9MRzUv+dlP/4A/+MN/zc9+/xeczF6hUkUhNMF5XAgorZmvSqpqTfBz6maB1hVRKEAjZT50P46IlFBSoCNoU1A0BikSXben60ak8wgx1VM5t3HdzPnJz/4AgOgC0Qf+9I//JX/6J/+SGD2IwDDs6W1LVAFEJPjE7c0GnyLVasanH96zv7lh6Hv+w//857xa/AXnpxf86Z/9CVU159PHW95/+EiKifXinNXJOUHkVfRiUbNcNtjRc3OzIXnYbrcEF7JYcyGp6hKBZOxHhr7He8d33/06B2FsLaP1WBeQShET+DEw9A5nLSkGdONJUZJCxMfE0HconxBOsr/tef/2E1VTc/Npw2bTEgM0p0uKQmUzcK3RpuHkbM1iNWccHbtuZL/fYQlopUk6Z9Exukal7PsrvKeULvv8yX8G1B8GkST4IzmKoyKO/gL3E/fRZH38+xEH9wA7HUDJU5GLeb6YaKjAEZMn7jBISo982cTnzTxq8smt6QGIeNifh+EFD/t/2OsYiz0AjUcA8fjYAu6kjx6M1iM8lsLRAe57O30QRzvej484buRwXpKjc5woupfOvoeDPB7bp/Y/Av6P97+zUIoHWx+Mg3jw4b7Pd9c2TcDtwSDdd/Gu3XS8aWL+7o4zAdkwsaxJAAaJJOGzeZ5EkgmhEhqJFIIlNf/i5Od8VX/D1aajvf4V//H/9f/g/Ouf8+rbnzCbzThE44oEPilcElMWqp6h39Purxn7ll17w7C/oWtb9uMtDstAj8XhySbkAyMn0GTzsUCgUJRINAqNwlBQUJgaY/TkriUxuqIsS8qqoCwLtM7pKiUCrQRalzTzGd6NjENPv2vxMSJkZoylkTTzhsXylMs3P2HeLFB6lYlJN0wpLsUk35rdmkiH2Pf74KbjeF45XZT41L30hfJi8Hey1mw2LX68gVAw2o6+dbxtR9r9llW94o/f/CH20ydse8tivuTk5IxZXRNFwFnHEAVNqjHaICRY6wnRUVYlzg703cAwdpiyoFksSDHS9yMiBazzODfgnacPIzfbLf3QIU1OdD+4gcEHRuewaWDx2vBH578A6YjSIWXCpXy8qANCJXrTMgTP2O6IridGi5IRJSCJgBcjY3RYD0UqKEZN6CXBg1QGKyzOW4SDQpc0VUNsE7+yf836bM6QNtzuPzH4bQZ+UeJ9xHsIQTCMPT6OlF2NUImqrNAobG9xPt+o23bkr7/rMDNYLOHNzNC1e7r9nhAGVss5i/mMWaWoUsFooakqzi5nlJWj3ZUoJSmMYbYsMaWg63qsbzk9V5yuT5nNDW7XUSjY7Dfst7fMFjXNvKYsK+pZg6BCyOzY723AeZuBQ0p5pSTI34Vkfbqi7jT7dkOKgb7b0e0ThdYoowge7DilkivnoFdIUTIva0QwzOozXq9XlD+vWdQrLs/esD5/w/r0NftNz/b2PdfvPhAAa/Oqez6fc3p6SlXXeLfDj7egBDFAiAEpFcoYhFTEJBhdpO8CUuUXUFkqzGmV0wBGiD5LomiZKXzvA10/5EWBygsTYwpCyCtsTcOiLAgE+q5n7EYWxQXNoqIwFd/+wU8Y+h3v3r1nrs/5sz/479BS0lQFfT/QvKoxUfHDu3cYBL//zTd8/c03zBcLEDB2PcF6utc5wj6bkKYVpRS4EOj6kbZtadsdQz9SSIMdR/rBUs1qtpsdbduy2+35tLum23eURmCKHH07kwljR/bXG9rdgNSSpdEUpQYPYUxINDLkiWjZLKnqkqurq5zFBYGwjtj3GKlJgyfsPTjL6tU5hddENxKNJKaIHR3eBSQGrcnBLv/USxIwOXxnU5/MHw/+fLnSkwDg2H/uM8DDQyD4YPdphk+PEQJH/krHaOp3LvftPz7S4/6Jo89/m6M9PNJToPgJFu1RpWdP+YtjMZ3ngzrPL16eul7Pnftj8jFvPDbtft7OHbg7YoCPScLDKD1eCzwYG45H8/Mxvfv6CBxzHO0q8t0XySzynadgyqb/CIgAd54BQmQrgJQsTUVzarDBM7gt7ff/b/7jr/4HQgRQGfTEiI/Zfy9ETwjZrCtSzIEcQpCwiJiocFTacN68wlQ1UhukVBRlDSK7oIWUiEngXSD4RAjZHcKHiHcOZzv63pOSwOMnJy/PIe1SEnfesnfjlv8NOeNLeuQCICJqeuYX8pQ3X73hF3/0b3h1+Q2LZkVRzoGCFDMw9UKgyAGoxAMIlHfBJSmv3zK+QJHwU/DLj5cXg7//6f/7/2Szfc++3eFs4t37K65ubvn0sef6w8jvfd0Q/9X/kSYYZpVhJgJSVdn9U3i87CF6hl7SjREtJAlNFBbbBz5tN7z9zXt222vqVcnq4hQCuDHQDS3WO+xoMwCRiV2bdQBVqTHLElUklCkYRsfW7lAmUdaJKDyDHRjtQEoOKQUohXOCMO7wnefry28pqsR+d4MGCq3xydOOe2x0hCBIg8BHS4yCqq7RSdG7kbbv8DbTuKWuKKqSeqZ5P2pmC4mPI5vdFus9kQlYxPwvKWVxaT+Ch9Fn05cxkigiLniaU8MfzE7wKWR9PBUZfQ8m4nzEBUeIFpBIJSkLgzGK/XZPUUrWry45PV/y619+z2Z7TWkUIUaGfo8QCinADgO//NVbhj5xu8ugJyZNWTbIqiRFQVGUdG2bV3HO463DO4fWiqapKIzBhzyhD5sRJWvOlkuESDjnSD4ipSYl8N4Tx4C3gY+/GRijoAgFr09KLk9O0OIcXInzirFR7IOjvX7Hp2KDcwOr+ZxlM88k90zhnc+v3bFDFIGoFV3X0Q89KWYn3exXV6K0hkTWpGsHlNKMw4DS+V+kxI4BISRFqSiMwDvLzc0G53yOcHfZ1CCkwrqcscV5S0oBYxTBe7zzCK0ptMFZh9IG7x1d1+GCzw9xiIz9nrff/UDAI1D88pd/zbJe4FrL9tMNb77+msVykc0ICUohERrKuplyUetsbgiBsPCEeIqfssz86R/+IVIpXEwUVYUdBra3W/qux44W5xwBz+bmhtt+i7cjNgxsVyf0Y8SNCRs9pjCUVc0YAieLS1bzlv1+T4wSo2c0lSeO2RfJ6IIYJHZw9N2IdAJNibKgNAzTdY/RT2qvhm7XoQ1UTfXS19E/2nJnJjzMzEncUVl3kO2OaTqYdz5nex6Wp6DWc3XvoeJDHulh3O3vBsk+75cQx62SJ/sv7HEgjx62ekz53e94L8Fx3JZ4sM9nZ/Klobrr5OPjP9nT58vDbj5Jvj7J7j57iCfq/ujBn+JRnyvP1PnCLZW+8Huagi8y+LtvO6Vs1jz47JGyFp+SgpBAC4FRhlJpGgOrKtHbkJNYJp91eicx/DymhhizrzsBbAi0ydPHiqZqWL/6ltNXX7M6OaMoy/yECZEtDHdRvxxFy2eoGsnBUfl/QUqeGLKrW/SREC2RDDq9Dxkk+pHgMxninSelEe8tSprsPqs0UikgYu3Ah4+fuLq54t+//Xf8+fv/hVfygq/Ov+Hrb3+Py8ufsVy+pqoWKFOBUERyLISctBSYzMpp0h4GiY8pm4/ly1ZwLwZ///5v/kcUHSlFYpTEpqfCc1YKFuuC5UzyYfgNcpSswoydveXd5j1aG5qiRoqEiz1SRipVAFmYZdPtcNGybXuub26wocfsFdp9TxwTOmUgFohEnyCCjYFu36N0idYl9BtMcpyUK7xRWNkSQ4+VCVVK4lJk6Q0hEZMJ0I+wv97g7cAQTggi4XRLvawwRuP6SG8dnc1pwGLKPnb1rME0YrIbWLrWsemzQ6mMjq8uLSLUvP1uR13D2VmN9ynr+CmDUUV27K8rZo0hJIuzkttNS991KCk4e3VJWRS03R4XI26UDHZkebLCKEGpA2M/QBnuIoOtd2z3G6QQVGWVg0aCZ9517Pd7bm9vGdqeVsjscI3EOc+7D+/QGra7wIcr6HpYlLBugD4wxhbbwumrU4yc0yxmYD3dfmTosnSMRJGCBhdILrG9uUGkkn3bI5TAj5FlNcOYkpAEIhkW5QXr1RnLr89BzGnKhiLWvDr/inm1oBIzvI1YZ1FGI7RmOVvQttdZa3FwjKPFu5F+GBjHntEO9GNHNInRRqwbsN7S9x3bbUtVliil7nyaxsHixnDnq+GsI4SI1oegBoHSgpgCbdvi/eT0HiXBhymdnyfevTACKQSkytcgO7QLRtvTte3k5C4xRlOWJUYZxrFHOLA2558+Oz1hVpa07Q2fPr5ltZqxPllSVjVCCJzLjsgpRpybos+EgJhXokopdFkgKomafDyjVCipETHw6vw8m2OmZWNKidEO+Gjp+g6fRmJIKFFiraAbRpz3tP3IDx8/4VKgGyy/+e57rm5vIAnWyxPO16cslysWJ6fousQnz812x/ff/Yau26KEQhnNMOwZbc849nkFHwOlMsiUcDf/9PVeRBQTAEoPfbTEQ/g11T76y2cT/0vZsykY9uEen7FHhy+5srizrR5RR8+Wx4baw27i/vg8wm5fQg/PncejfqQntt33YALQT2Ggo2P/GBZ8sO9nnXmq7udQ8WHVLxzx0Xk8PuBTmU++VO70757qy93AiWfa/AJEP1B8ifsbiWOhksM9kyOJD9p1dwI5Ik45rclqGGnyO54YMZkyY6hUQpcms+JREXXKwRlH2YFiSvgUsMFhRyDVnL3+PU5/8lPm6zeYusmR9IBIEiEF6U4EOdvL1cRS5ldiRB/nehbcRQDn6lN0vLiPvs/+pgkxRZHnuKwIIiLipPYn5RTRHwnB0Q8du/2Ot+8+8u7Dr2i3N/zy3f/Kr6//M3NzzuXqp3z15uesz98wm61o6prFbDkNfsz9itxJ36QUSVoT0qNU8F8oLwZ/3336FUbGKegl27fFTFJVhmKusNbywX5EB4VkZHQSad9RSsN5eUahDHu3YzNsc/CGiHjpSSJSNJp98gxNz2A7nHeIoKlUQ12WmBIUAek0bTey6zoG0VPVC3zhGX2LHCxqZlFmRtBbhmFHnzLdnLxCaoVSE12aJikV3+N9pBtvCd1ATAMmzEkmYqUn6oAsdLb560wTV1VJVWncaCkRLBoxtQXBJUyRSMLjk+NmB0WZqCpDVdUgBc4FhqHFWUeymrIqUEhqXaHrgFLguj0qViyaClkYrj517PeOoizQCIq6pFAVpS2wY8t+10ISuAHAo0XCmILoA0PnMMqxqE8o1QJnA1JIZo0iJEghr3C+eR1ZNAPOSiqtWcxKKmNQUmEKQyHnjO2GzW4kOk/0AjtKhkluRiSILqKSwoQTmvqUpgnMmpyHeFEvISmEMpRFTWNm1MWc2pzQdZFCl/ge3NZwfd2SfMvQj+z2e7qup9u3mFKz77bZpy0E3OCxweKDz0E2yTLYnqgi1iWEcMQU8cExDGN+bEJApERZFpiioO+yzp6b/OGCD5RVSfCe4DOFLlUO7XcuIEReRJRFRfAB7yOjcySyycCOOc1cXdUURhN9oB86+q7PGTWMIZYlSih0qRFRUBQ5oq3rRqILEBwkz9npSc6gUxdIXSClIgmfrxkxm1JizA7JIpuvpTjILmV2UQpIKgNTEUPW45LZ2JimqElVFSAMdaWINFO04sTgucgwOnbdQFFV3Oy26F2HEIqhG/Gjo2lydLExFeuTU05O1yQZeOMdFydr+n6HiClncXED1g05gvp2w3azY+gtSnhkfLny1D/WIiOgRDbdH7NwR5jpM6yRntr6tOjwUyU9/HM44mf1xJFZ+I6ZfAQan279c6Bw7Dt2F/v7CLg97v0xTntc/4mvT/biWXPvk408xaDeb39sqn1wpj9iEn74czr65+i6f/nwT7b78q2Pm3vUp2MXgMfaNJ+1+vz1F0eg71AzRfnwaEke3UZp0tpTJAJ3EUbTDXMI5D5mv7O0TnYtChMICykSoseGkS55nFQUZ19zdvFzZudvMIsVQi9IUmYrQ8ix5jKCFPHO//AA2PIQZDYtm2oPfZ2EcA5Ky+T+JuTUBqDuAe/9o5rnjBQPki1MUeKJqCNlMWc2O2M9/4afffMTrq4+8OH6Bz7dfGS7v2bbbfhu85fU5ZJ1ccqrywvO1pecnJ6zWq8pqiYTXwiYrIlRxBzU8gX3g+Py4rdtlD0OSB6EgiQUIPPE6xUxRIQxzGaCsRjxbkQrOJmdsCw1SpZ0Y0c7WnZujzAJVERJSSxKum6AOhKSx44eow2yMcRCEktIPqsH9Th2oSMVDqlGZMqRswJLFw0KGEJH50ZiDoZFFgod880To4ckkQf6WcLtbkOMDqUjZhjxSeICICVlXVIWGlMKpFLZ80AEYnSk5GlKhZGKMURIWesupcTJXBGSRE83upZZPiZEl6niFOhcRMYlMFKKgrKQKJXw45ZubEl+TtFUGOEoTUKmgEqK6ANaaoSuCdbh/YjRhtVyRgwuXxeX0LKEIAkjVE2DKTUjAeuyGFtdZE3DYCNKWfTc4kdB9NDdjlx1O0JIzGaGkxNBGAKlqAhWUuiaRWmYJY9IEqMKlBbIBEaVKJX1EeuipDIVgknTLUCyia7v2LmOyC373YjziW5rkVLj7IgPHmc91lpGOzAMAz55nA8M44iQkuQhxDhR3wmfRkJyCCMIMaFVIsaQo6pCRGtJihnQla5ES03XDajpWZFK4axHtiKbGGK685fUWuWoWqkxhUErRYj3UhnZoVchQ5ax6PueFAu0UBSqxCmf6X/0lLVSZ42zCKMfCSGSYiCGQDErWK/PeHX5ipOTUwpTEWIkhoSWmkgG8AKJgUkd/vByOaiY5VUyKSG8J5HHQgiBiDKP2aQDlxnxdJcTMyUIwRKjyi+VmM00ZWkwg0YrnVlI63AkEAnnQs5sY8qclUYlpEyU0lAtzyiNpmlqSPkcu75js9uy2ey4/nSNtS3LxelLX0f/aItMB5FY+Mx36kH5MdPNl35/EkK+oPx2TNzz+x+zQS89jwPgeg4BfQ5+H3J8L2Txvnh88dnWx99fdozH5/C3HdeXli/BwKdB+sM6x9u/cN0+Y06PFxUHyZHDhkR+I00M4IGGFoBQE4n4GKZyxxIetCcFEpEEKQXs9P+IZ9CSaBbUq3Pmpz9jcfotpm6IQhMjEDwiZU1YScpiKikvnHM34t3QiLtxOhqudARt74BpTo+ZBA+iww/SN+lIhzLXv38v5+2ZQFBCsVjOaRYN8/kZ52dfc7O94WZ3ze32hv3+mt3N93ThLe+3mrJuOJ1dcHn6ivXpJavVBc3shKZeUBQzlNJ4AVL/HUu9qIxd0EVmQkJMOOexY8KN2easRGBWC0YxgNuipcQWJTu9JcSBK/Zs1YA1HaaUKC2wPjG0ls1+RGmwhyhSFRkZGCyIKBAp4kdNG0ZaPHUtkVXul5EGpMALxTBaunHEAaYs88Sm5RTlG0hJoZVGKUE9N1BEkgdlKrROpGTwLrN5Mmmk0KgEBkFZaNzosL1laHuc9RS6oKpLZlJQGI3WhuQTKgq6vif6LHdJGanKkrLWOBewgyf6HnxkGAeIA1p5FrOKDBIDfhgm/wFFoSRjb3FjIgQwusyPhy9QIafBnlVLVCVw1tP3I7OiQkqJkQqGEu8FdvSMdroxK83QOcZ9ZBwdTdHgB4g20m48u1tBdBCXiuaioNQzlrNLkk+YYJibCqkkySdkkhid8yhrStrdDjeM2axdOcbBMY45G8vgBj5dXedUYAisi7gA7T7L6XhvSSLe0fUhBVAwOos2JdZ7JsWtDOSlBkHOGRt6os0PdFUXOd+tT0Sf1d5DyPqNhZEkIZFCY20PZLYUwFl/rzGGQMiDnEMkJodzAYXCe8c4+sxsSMVBAd5og0CglaYpq5zRROnsY+cCY+8IrsOVjuAtMiVOT9d8dTFnvmg4Ozvn1atL3nz9htPVRQbxyRFSRMqJ4VOHjBCZzYYjgdFJhywEN/nLZCCLEEgpJrAoQEqmzSCyyTgdzBZxUtOPWfvNh3AX3KO1oW4aClNkll5Ium5g346MNtAP45TzuyQGQTNfUCmNFgVGG0ylmRWWWbViNWtZVkusHWma5UtfR/9oy2MZjc/Kb4nZPmvlc/z14vIUWyWO/n6p9v22aQJ8tPnxKYvHuz1q8UtD9OTxH5NrP9LTp39JR5ueYEaf+/LSa/Zj1+ILv/8u8PHxPk9exc/O4/BCeKLug3vimKE7sGKThLZ4uA0ikkOgw0GCKDFpv2S1G5nlW6TIzNtBGs+nwBg81jqcj9gUGUVEmgIzO6Gql9SLNeXqHDM/JemGUUiIChk9Ekv2mMt9y2/yIzB6dO7HYuCHwbkDso8AXn7fHi+ypiflaPf74+W2EsdDKyDlBMciKep6TlPPOVlfcjkM7Hcb9psP7Nr37PZXXHfXXN9suP3wgbdv/5LZcsVqecZ6ccF6ecnq5IJydUq1PGVWLp68fo/Li8FfSPlEihISEj9EgkuElAUdhYj4NBCTxKYR71tSAL/1fCduGCy0LuCiwzBSIlGjpu8ju8Ex7HOKFinBVOClww87UkpIrRGqIPqJ9Wg01bKkrgqUVHh0jvyJht5afEzUjaaaz+l9Vrf3PpAiVIWhbup8GwaJioCIh+QviJBIPvsXxhjwySMDGFEQJIztyDBYxj4gkkbqCq1qlFEUJvsUaGMQ0bDddIx9DgKodcx1qhpHwrctQz/gjMD3gm7bomWkOKvQZUVpCrQsCBEUhohHyID1AeUVJpYZBEiBUBkgF2NBYxqiDmgxol2JVNmMO7SeYTPQtRZrA0ppVCXxvcLvwA8GNVsiHKgkKf2KtYpIoSlEwZo5wnnmdoUi6yBGn7O82N5hnScpcGPeZ99dsb35xDhYZqs51uXITqUVnsCn21t2mxaPmK5viXc5+tG5bDrVKgdnWO9AJXxKSJNIUmSTLCqvzITKfnqqYHQd3dASIxitqBYzSAobM/u37zIbVmiRw+yF5Xa0+OCoioK6KbPQbQzZpColxuSsC+Dw1mJ7j5pNq1bvgOxHIoSiMAWzqkEXGqUUhdLEkGjqBqctgxgZe0f0EV0blidzlrM5f/iLP+Tr199ycXmW8+qWRRaQlVmXTSk1mXeBNH1m0nc6EstlAoT5k4KUI84k4k6lPoO/+3yXOatDpNAGo7PgaPASZwU2hUlgFXxImfaXitlsyXyxpG1bqqoieOh2LR8/fcIFjzGS1XKBEhrrIn602NHTlDV1mdlxbxPJw2K+xGgzRfX90y+f8XJP2Sy/xHx9Pud8Xvfvm2z6bdr/rUDPE5VfAq7+brD071D+t2L1njvWS1D+S/r4FFR8evQe8q0H0HSfIzfrzmULQ36kc54PMZlXBdlak/3nMgAMZIyRUo7oHbyn944heLrR0jtPQCNMgWwaFstzTpZfcbq4pJ4t8EoSpWZIOY5MxIhOFoXNws5oSIr4QJT8AH4PwU8H4eTjMxSfPXoPRumJPLqf88fyfjgnqjCbvgVi8ic8MIlGaxazJYtmSbq8wIWfsNtcsbm+YnN7zc3mmq3bcLV9x9XNO8pY0RRzFssVy/M3fP3Vz3lz+Qb47568dsflxeBPStACvM0ihSlnckEEMBKqQjAvEwUDKXQEnwd23w6kNDKExBDS3VgGGyAFugFuB5BBUMjM5PkA/T4TyFWVJ62YJD5ECq2ZNzWzRqEIhOBwIRGiIIWItT4HU8wKfEy0bctgLaNNlEZRqoJx0r4rVElZlag6R7yKlOi7jrG1JD9dKylRpsAjGPYWO474KCiaOVqVSCQ+ZPOjH3Pkz6yeUxdzFs0J2o9Z19Bqhk3E9xFSgXAzUh9IpuKkOmWxCLh+QOwNjBJd1zTzOUVtqKoZ1gnqZk7wEWE1IhZE7/DOTqrzimHoYEj0fQe3gu2tJXjLYlUhIuhBoG8Fw8YjlWB2UfN6tqI8m+N8j4o6q6GLfO1STAQf6DvLcLPHhT3jZotAMoaeEH121B89/ejoiPgokUmjy0jre65ubiiH7u6xqKoapRWjlIwaSDKn3/EOITRaK5AmM1wpB1ZAVu4XShKDR4oM6IUih7+Le9FNrXUOgLH5YdPK4GKaFNs1vXeE4DhxoGRB5x0xKYzWSFFSqJpIwPoRESJKKERQ2WRtRc5HGyXNumFezRCraW03pQYKyRNjIEQHImGdp9v3GFVSlIbTyxOaZs5yccLFxSWvLi9Yn6y4uHhFWTQkEkVhSClMPiNTSq4UMgiF/OBNOmyHbAp3yvt3xJKYzLr5ZSOnIIN0yBRxWIFPn9JRIBCJHCUdIy7kqHPrLN3Q0VuLcx6lCmaLJT5EyipHpNng8SEw2gHnBOM4MA6Our6irioKU2SNRZXN6ImYI7/JQH0+m730dfSPthzmgONcvb91+Vtgu8dT+vPtiC9+fTnYeI5Fenlr9/TL4wZ+fPSehzBPHVE8PUDPHkY839TLuvdkbz6Hc39bYPf8Ue4yjWQKavrlnu9Nz+7/aPvB1Dk57WUvP4FKwLRoPTBhIaXpfa0IQWBJtNYyxEDvHe0wsOt6WuuJWiNVw2L5ivOzr1icnDFfL7Iwsygw0hDQSBQyJsoU8CqgZEKlPD+Rcrq3hCSmu5wzd0AVcYB9h5F5fObPX8gHOtrPVvu8LREnNo10Z+HKmUYyCZYT7BQUxRnz+ow3ryJ2bLm9+sBmt2Gzu2K/v6VtW3rb8/56x7tPb/nuP/45r2dfA//XZ/t8KC/3sHbQ2xxUGCPZdi6gVFAZQak1c53zjI5RkUxAiYKUSlJwSGkpZUKpvA8RRgvrteBUFRijsinNRbzzuXNKIpMCWRKFZm874uCzvxvgU8L6gIuHdCuB0mhA0W88QUs2O0dvE4WAujLMTU0IITNqCRgDbswAQ2vFvJpRSokbs+9cURasTmaYsma0CaMc2lQoUTH2I307ElyikJrCFDSm4qQ5ZVHOOVNfYdeeumyoTZWVyJE0swWlqYgh0e/3nJwsmZUN+9stV2+vGG3EqIpCGqL3DFcuBzfEkW7fMvYDthOM3cjY7hGAKQwhZD1F6zqSg6poaGYlyfYE6xBRcqI0F+cnCC2Isad/19LbAtlIhq4nxoApS6TJWVVG7wgehrbDhpa6qnE+4JOlbhSl0dhgudl3XPeWzkfwgvV6SVBgVYnvJTZ4lBSsFqes12v81Xdsbm3OEdsUCDnFOQSPMQVGGWKIk49mIhCRKXuBQCJFS0QSYkRqgdQGISXay7t3kAREEIjkSdajdcFZWRITFCohQg6CSCEwDiMq5pdV9J5xzFGvs9mcxWJBsTIQE/vbHbpU/PwnPyF5z27fMQwj4zBgrcVHRwhZ5mWxLDg5WfH1xVecrs549dUlF5evWK/PmM0WGFNSV3msrXcE5xm9JTiP1golBMiE1BJFBrjZJy9Mi8gDuDt4k0y6bSIdUpxOfyZB4ZTrCJGFTUPK+oVCZ/X6IQi8FdmR2kacDXTDwL5raYcOGyzWO6IEqTVVVbHX2fe3qkrms4aQIrP5jHEY6fuOrh3o9j0hZqHnA2OrjUZNzpbCZOC5mNcvfh394y33TMKdZOudeezvapJ/vrwMjzxVS3zht3+I8iWA+dvYYX+k7hGz82NX6On2/67KY7r3JYzfcUn39xuT+fLwy5HV4GH9+893vz2yqwtxny1ZTO+XmCYpZZEz+eZsyFPqMqmRSAI5//vOOT71PR+3t1zZlh6PEgWLcs3lN9/y5s1POD07p1nMERGCEDjCXbq7mLLsiU6gYiDnzwKinAzOJi9op14KnjIv3DN992bep+m+Y6+A+zy+99vSUcX7q3S/SJiSBk4mmEO6wvzn4I4jEHemcznpGwbAVJKz1zVnrxPOD9ixp287dts9+82Ofrtjf33N2377xDl+Xl7u85czo9CPEHL+cMoSZrVi0VQYBclZ9i5yuw00FZwsl5AK+m5DSpGylDR1gVQF+9sdlkQtziiqiqKSuHEkRY8QOQefSoCH5BMxWRZ1hXOBFHJka4wSKFEI+q7Hp0hTF1l8N3pCn9DeMnSWUSQWSiJNSaUNxCz4qIKhUJpSlZkpiY6AIxWghEZJSdlJlNVczNa4wlLVc+qqwbwuIUmizaZikxR1tcpyLfM5MgqEKBm7HoVABIUdA+04YneeplyQ9jOuPt3wrt/Qbvf88P17Nps94zhiY08fdlzdbNl1MNfQNIL1xSlazIkuEa1HCc2smbNazjmdnZLEmCXU+pGmKFgtZsQEbvS0Xctut8X7AVloCl1SzxSUgX37kbbr0WNNUdYIpZFSU1cV+yGyHUdubh3jYPHRslgqmplhGDzXveN2EIw+IlBUSRFcYrBkBi9mB1zrNc4bQjCEZEgUBKfIkamS6BxRgNIKIQWBxOAHkghoWYBQUw7NQEhxgjYB77PWYYoCESH57Kcp4sDQbvCD4+TyHK0Mu+3IuNsTypLTVcXNTcduA6UesENmOxfzBedn57x+9RVff/U1pydnJBH4q//8K95dfU9KsNnvub66ZRwtZVWyPjvjZL2imTWsViecnZ1xcXbB6fqEpp5RVVXWOpxeQTFFApEUHBBRGpqiBBJKpMlUK0HmaF6SynpXE1g4pJQ6et/A/evj6E12EPPNgqYxHtLdC6JUuJBIIU06lIlhzD42o7WMfU8/jrgpAEak/B6IYcwLNimy8JAivmkAAO5mSURBVPXkS1hVFeuTNeMwYIqCpgx4F+j7FiEVY+oAQYgB532OepYBUxTcbm5e+jr6J1Sy7enY/PRc+dtAid95379nq+bvDKaOwcpn+O2IkUvwOPb2UH6M0Lur9YRp72XlGC4+xXXeP7HPHv5BWz/2+anyFDAW06fHAPIJzvHBQuW+2wf9xsObBCGnAAgBWhF8yjmrpZkCICRRCPok8Qluh4Ffv3vLL90NPQVzccbFxe/zL779Ca9en7OYZ8m1gyRQEJBEIqQMKoUIdynUQBBSRIUEqLtRTcgpk80xQ/sUgL73735q2J5jqZN4WEk8qvT8lUn3lOHRVcjizFnIWR1u7bu8hYfxlwilKY3B1AuaReTsMhJ9JjDcruX9++HZIx+XF4O/AlBCUjbZXi9FdmqPLrC/6SBkh3qhBTUCMSgCI2XpOJ2VxKQZR0cYCsrlDCNL5oCwM7a7AQm43pF8QmuJMRLnIyJpcBJVGprZEl1ICtOAEFlsd9KK9HOPMopZUyNKxerynHGI+B5CgFlVsigLSi3RIuvtkUTO1hEltS6oZwV+tESXtehIOXMCMTD2YIoKMQ+gS7SpQFl2uxtuP90wbHt870n+Ch9GrOvYXm/otpYQHHYYcKPHuWyatsOITZH9vmf0HdIHqsJQ1WXOQUtAiBEKx6qAmYBZUTCrK87qNfPyFC1qIF8LLRQuDqSBLFBclLhCcX214XbjMbqhoECkE0oahGgJI7y7/kSjAuWi4tc/XLEdHEL0KGVymHqEWbUgJclmb2k3EVEFTNWhvKRSM6KKBCWQpkYIjwigZE2IFh968AnvHVJ4XBpAR+SU5xAppkTvEHxiv+8oa4OpDUIJgkgM7mCKTtSTeVLrHGxwRG4BKftgRGhKmM9K6lpzzbR/HBiso+9HQpB0/ZZqNgX0aPjjX/yCV69fs1otubi8YN4sICm22z3/6S//go8f3nF7fcMwDozrNfP5nJ//7Peo6zkXF+d88+0bXr15Rd3MKIoSKRQxZPeGFLMJOCZ7F6Gcs3NkqdEQAoiE1pNo87SqzUBOAlOy8HhH6XGf1+nRi4V0/9Y+miAnaVUQk25lyitK5yI2BFwIDC5m/0zvsqi187goiBikFBSFAe8pS0lhFErLCfzpHO0tISZLInB+usLoBu88w2BRUtD3A3Z0WOdwNms1OjcCI/afi9Pf30t5AYv1uzT5D17+Hs7rv4ryD3tOPx6F/ZBZFXf7pCnALD+r+c0UScmTxYYTYmL2BTnlaRQCGxK33cB3t5/4z/sfuMVzwZJ/8fW/5Y/+7F+zPF2gjc4Aj+w6E6PjLhp3AuByWthmrBbvmMiIwgoDFHfnloP20gMJo3/o8jRfO43uAYAeXuGHRf7RAkYiIGRlCjHJzSSV1SaMKtFlxdfnLzvXF4O/WZpRKc3ebompoqhXaFVgh47t9ho3JgqVma3T01NEUIy3PZae1cmS9ekryvki59QtG+RrzbJaUxrDbrdDRIUSBpkERiuMzpIm3kqEEyihmNU1q+WSSteMyTP6SIh5AgveE2KkLg2x1ChT0oo9loAQmlnToDUM3Z7udmToIylGvI8km7h1PVq1uLFn3I+MvceGbGoNStJtPPtdTx93DMLn66U9znXYtieOgeCzactZSxA56wchh5cnElobCl0ilaY+qSk1rL8+yaa9kIh9oBCKWhfM6pL5QlEWEe8CN9uO3o0oKamLOVoXDN7S9l0OthCSKBNJZUVyELSMOBwplsgum8xxiRig7/cIoXF+x7Ju+OZsRU9iaz0+RFIaICSMUAgqjGwYW0mgJPqBEBQmOMbksTExuoizkhDSRMfHTE17i0yBGC3oiDAOVXqkjCA8qLwijNOKNAqXtZlUyjJAwOgzMJK6JCWDUgIhHN57vI/UxZyyrHMmj3Fkcrdj7AN2HBn7hB1gu2kxOrFalKxP1ry6/IrTk1P+5A//JOe8tZBEpG33/PDn72j3PePosDYQQ2TWzPjpT3/GV69e85Nvv2G5mLNcLqmaLGCdBUQFQiiI6SgVt5i09cCIHMBxYP8PK8bsiBxQk+8iiaN0XBnMhZijkO+Swcujt8nhpXGwKoh7806a+hIBoSbN/ZgmtfrJdzBGfIg4ZxnGgWF0WDtlAQmBENIkKC0hCJQQGC2IwdHu9lnGRWTmfOg67DiipWLeHJjIHLyjVY1aLhBCMlo3pRrM6Rd99C99Hf3TKAdW5Ufe1S97lT+czP92TN9D89/Tvfnbg5fPRa2fr/k7//zgfB6N0Z0J8yk26MhkJ36kB88NxZeG8Uvlx/Duj7X7ov2nSk/UOxBkhzV1mlKnSZFTtGUrBNm3OSWEUpMVIgcgenLK1ZBgPwy8vb7iu+0VN96i1Qlv3vyf+L/80b/g6/NTStUQkEQCo7dZE2/qn8AcejD16ZDGLk3vzfwSzK9ESUqGhMygkDBZiB5Tcuno36MX8LOD+czQPa7/wPnvcUmPqj2+56ZzeYxRUzr+h3tFxUkybUpnd/hFCkUqMsH1kvJi8Pd763/LopkRUo+UBWXVUBQFyQV8b9GypCwU3nqiFUihs7CiiGglmC8WzBenSJWlL5Qsaco1lZrxvvrI2HUIJIVWKJklNeyY2bGQPH3bs9mOtLd7hOpzkpOQMvjzkbEbsG7MbEKIWO/Yb68ILhCiJMaA8yPWD/jBMu4dwecJTxuJHQIyRUQMRBeISZC0JBUgjSHGxPa6JQRPz0jXB4pZoihBJ0mVvbIomorlvMKUhpQiBkGhFEJqhFT5Jg0JHxNeW0wdUFGgywKKmlJWFFKjJRAcoXfgJSnUdD7hI2x2e1xs2Xc57Z00evIbs4xjT9Folqcr9n7gptsyuARWoj1UukRLQ9cOSFES4oCpK5zJOaV9zIrpd2KbKIKMFCqSdMQLj02WKsZs/g8qP5TJZSmUnL+OGLKMifcDWpZEESZfugEfLEKBD4EQQ/ZtS5LgPCkmlDJZwS4JSGry30iIpNBSYYRmiB0yCVTK41tKyWg9YXDYPu+qpUIpzfmpoqlqTk/XLGYzFvMF65M1s6bmZnNL33sKNN99/x3b3Y66KZFKUFczztavWJ+c8fXXb3j11SuasmAxW03R4SLnBtY6m6hjDo5IKWJUZiiJZF9ElV0ISEzK7JPJJEUQ2eygRHFvmYhTRe5X6UlMyu53lgxxpzMlIOtgwZTn8bBazJp+kUAUKUcQAxBBRoL3jDbSj5Z917PvBvphxFrP6DwheGKMOYcmCaMNiCycjYxok7OgSDMlSJ8miOATwWUXgBg8MeSOxpTwbpz8OSOmFNTLee6f/6fI8DxTnpx0XwisxBe//u7lyYb+tq0/RiL/kAzMc8cWT35+UU+/eKkemhEf+4h9Efx+yVr8O43ho33S8bYDcDk+6CHsYQJ70/IxiSywLMlZgzKxoUBKvIBUKHo38Bd//Zf8MG7YxYSWa15f/An/u5//AZeXX9HM50QBMgW8jJN1JEf9pilzSAZ+WRImpoQ8WHWmF2gOXjuMYPblkwxAtmqk+8i3J8bhGAA+HuMJ6d8jrs9HOz26ckdNPn1lBMdm3ufq5MMe9+kADB/3e9pDHNqbtmcF/xerJrwY/P3pz/9bKtFQSpklWXSeuGzvGZIneAE+IZWhDx1EQXAWqQQpOLpNYh93ONchlMEUFbrwaHb8+ofv+PCbdzjvMKVAmUhwA0PX4mwWRu67jtEPeEHWBHKZ9RNS3ZnXQhqzxl9ICA0+dYx9ZoKCz/5V6EQhBTpJRu9ACoRIxJAohKbWKgeaiDy9uzFQqRJd1Zx9syAJQTfs2W1vWJwWLNdzSlWgnaISBY0omTc187nBCEn0nr63hJQwushRuWPO5rDxW+LY4X3gauPxbkldLChVgdEKoQOj3zMMI4OLxLpisNBvBlCSwQ7YaNGVoRAlMQZ616MGgfY1LsFubwlIhIukJLPpNUaciBgZsckhaoFXgSAESEEK6Y52VhpkAWOyWOXwIjC6ERUjBAOhQiWZU/NET4rZAbfvW6wf8uoQS4ohCwnL/CAkBCFEIKf3idETYiAkckBJcFNAQLgLbDAqUhgByWVTqg+IGJHeghsRwVMVglfnNVXV8O3XP+XrN6+oqoqhs+x2LSkEwpjYb1v6wfLd2++4vW2pTQ0STs8veP3qkm+/ecPF+QVNPafQFXVVY8oCJQRFUaDImT1CjCRvc3SyFJRaEWMWLxAiIbTgPtotg6NAXtC4mNlqKTKzfbcqnMRM7x/7DH7T4aJMCdJz3TSxhOSX8yTUHCfLTDpkuUkJHxLj4KeAp0CIkdF6RhcZncfazKT6kKWRgs/Ruzl6Ocv6xBQ4wNGyyrJJm92OYRyyIHppkCLnmdZaoQ5BOjHrBIbg8CGbdbKMU+5bEmCK4qWvo3+05Xh6fZiw69lp4wttfenXv28g/Rz99AXa6Xdlwv6uy2997Ptzuluc/dYA7AUM4nOH/tIxf6TB9Pib4CHVfESA3bNpB0Zs+jGpnJ41hMncyJ0On1SKKBVjDLy93fB+e8V16lGp4fVP/zV/fHnBannKvF5RFw2FLqaF7pQ7N0w532N8YN4UKSGS5iAGw/RJ3HU69+/Y6yVbXY5Aori/boeMNUdCL18cN3F0pJeVA1T+sRae9kA91H366TlsPX5fPESbkskcn5gSD/x4eXlu3//hP8IYKeUsp2FSkUD2n+o2I24yjUmtGPYdIiiit3nS8CMpBpJQJKkQpgCh8anAmJLr7ZbudocPFm3A6JRlTIaBlFRmGpLDR5/1BpUk+JzSRClNYQxSJhIOGxM+QYgOUXmGMUu4OJ+1zEqtIBVopTFaoUpF13eUZUFT1JQJpM/JpJXON3bdFMiqQBYFoFmuZlyezXHpmqIM1IVGOMlMV9RxhnOB/d5TKsG+7Ri9x8WAUH12kLdZL24/XHPd59Rgn/YR6zyremBZzljMFugouN3v2Ld7IoLlrCbJgn3XIUtBPya8FFRSobTCWoefaHMpE2FMhCjJmih5hRYOLhQokog5LZqMJHXQjDvcUOKwkCAIiDiCDriYAUJIgpRKhC/RIWeakAi0SCQCzo7EFLMOIfnBzL59ghiyBIuUBmJeQabgQaS8HXHnKxeizy+GCN73OKshBFJyDEOiVFBXBeuTee6sOKUsa6qyQcuCoqhZNHNE7Gn3PTfbW4ZhQKmcvSWKxOnpGRfrV/z8D37G+fkZdVGzXMwpi8zExTCB1inAxNkBoRVGi8k8ch9NK0Vm+iT3543IK9qQ8qo1TaYIqbIupCDntYyIu1xY4u5FJw60XhZGFYlDAvLj6D24fxGmJOAQ1DG9UBPToilkAfFxtBN4TVl+VUq0KSnRIDRKWUwIOYNPCPjo8T5AgtFahiESYoRkGPpI23YsT0CeVighiUkjKBGpIMVuEvo/ANNAnKK2fRTgHD55mtU/k2jfpyaex+zDsxX+IZmzl5THE8+PfYeHgPG/tvN7xI79bcuTJOjDtp8i/R7FFvxWh/nsh3RcQ+asM5N8FGRptSQiggBC/f/Z+7MnSZIsvRf7HVW1xbdYMyJyray9ume6e4DBcEAS4MXlFcEDyQf+veQLSN4rAC4EAhkIgBn0YHqrNddYfTEz3figauYekRGZkVlZvY6KVKWHu5mampqp6qffOec7mekrKHVJiJJcR0Rog+N8NefF/Iyn9pxlp9nZOuCTu/+Uo4M71Fs7lKMRpkgRvtKnxAieQeAZQUU1zGb9TnatYtC387Kv8ybI6v8bZsOM0Ae8vDlNXuqINzGv17yT1xGokP0gX1c2rnep7v7XVwH5cMYmGyj9rH+5/YkwTfJdtzP6vgX4+9///X8F59Nk7pOjoZjE3qyWHbZ1CGknb7sOvELweGchJsdN0QpVFjg0ISpaZxBT0CwdtS7Q4jDWYVRAxUD0PuUvzSK0kYDRyZzqQ0CiRsWIwqecwyLUxhBQLBtHOUpRxODwzqJEM6rGGBlhQhLhLUpNV48YVSVVUaK9w3iPiWkRX2U/JOsCLov+VlWJVpGL85c4Z4mjhq4JNAYqH/BW2JvNOLUXvDg+wUnE64gqADx+1VFE0BouGo8FlgLnWEJYEH1ER0MlBU2wtMGmSEy/IkTB64DogqgNSmmUKVK2BpV2YlpS+3ERtYHoJKacohITQxxCSCa6nGNxCAgYXjQBND6mKKtowFlHUBGvVIrW9TUqCBoPOJTkkP4oGGWoyzr7EKZ32HYea0My7YomRIPRJd4GlEoad4UxKdVYSH6ZwaWdpmsTs1gWmq2dfba3FYXA3vYW08kEyWxTWY4hCF1nuTg/5/z0jIuLC9rO0jqLLgqmsym7Ozvce3CX/Tt3GJUzDg6PmEzH4EPeSSW/FmIYgFeUNFGFGDE5WXfKM9kzlCAi60iwDQGpPHZT/0gCXGlsp+eTJRaTv16O9iWLkg4m383J7Aq9L9KbgWPu77iRfqg3KyQTbspd7HEx+wPm/JApF2USlU5BKRptIspnkWsXkh+gU7QdWJ/+XVw4tPEUdYrE89YxriOBAusj1nmE5Fbge78hgRAi3ik6FxjV1W2noz/g0nN+Ny/Qr4cZ8bLZ6ftgkp4g4TIZ9KZTrmnRrY77bZehDZdQwHVHvNuTeNdy21pvOu7ynbzuTbrmzBsYRBnWiawFmuc4UZIIAEAbg4+RedfyYnXBy2bO0juQkurOPQ73HrC/fcTe9n2msyrNR6xTtJE/9xvjIZI4p1C7JDkjrCNd1wdeAnjr5CJDUrmNrrhpQ7EWcb5Nd73tG/DGsdv/c7lJZBYh/x2vPNKNgXq5puH3ZPnJ6/wbgWgqtwZ/T57NKaoC75P+WQgRbUzKJ+fBxmQGVjal4oqKZDotkkkrxuQwHrXDuUiIhkYswUHbBkJRMB0pTKmoTEyRjTFFQUYx+A6UKpmOa6J3+M6iokYrjVKCroQohsoUiNLMR556p6YcgQ8rolvktF4TiGPaZaQQw6gskGmFqED0DlUGxoWiUhrbRZqLyPzCsgqOYALaCFUHvltxeuEpx4rWWi4uWlRwxPaM2tRsT/d5MV/yfHGOk4gZG0yhCN7SdQ0TKdgbj4AG72BUK859wOLpaGhZAR4nPmXwIEJsEAQpPaiALhSiBa1SeHtK+6XQEpAQkZDAXyBngIgJAKoMxEJc08QS80OLKg3GHJ4PGu+T+novZIxAEPAIIRj0oNKepTQl6TklX7CK5WqFJ70fzqeAE5XzOhIjRjRNiMTok8SJSgBCa6EqFWWVrj8eVWxtbbE1G7Ozs4OpKnwbcG0ySRaiMaZCK82L4xO6tqWsy5QayFr29vZ58Oghu7u77O7sMZ3OODy8w2x7im8jMfsdKpKIdD/mLueeFFBkMJ4As+/NsFGy/60MwRiSQWPP8EnvoCuRIa9aPx1JQHLWluFyrOfrfpEepjUVBwC4mTYsTayBGNZsZX78aVIP/X8p4MOH5IPaWVLKRudSKkQEpU0CqiGpuvsspO69JLBIAtu69NgozJs2mZStpaob9qOmsY6maRHRWGfx0WF0GrshCkRF8Aaj69tOR3/YZSP68Pql+yaQcgOBce2hcs1Br17thkxeN1z9pt/fQyU/OFx83YJ4y2vfFmO97kq36fDLZ1zTgOs3D9e9CvHSjxuoI24ekxaEtOnsN5CaSP5PCyvrODs/44KW81XD3Hp8VTHevcOdnbvcvfeI7f0jRJLgsvM+K9SlhTzDEjY5u741g58bl824feuu50Wv4qd45cvXvZVveAZvsbO61dMcHsQNm414pZ545eleexFZH7vR3rd5RW8N/oIeQVkhwVEghMzmIIbSaJQOdLZDicOUGpQhLZEKLS3eBTrnCQjapFRtPgacFCkTx4VjVEI9Fna3FHWdmL/OQ12NWZwESjVmOi2JzhJsh5ZEJdvg8Qoiip3xiLIQTlYNZqrY3h8TvMJ2nqRBIrSNZ37RAJqdraTLd3ZxxvnFkqoO6FFBMa7wK0VztuL4vCPoKc50KB0oXEe7nNNaD3VBcAXnK8ty3mKXLVv1ikcPOk5XS85tg4uByahGFSVOBVoclS5w2oAoDEJdjrhYrSi1UFQFuhSi8kSJaANaoDbp4WoVcxh8An4C2U8vBdioPsopAH3Ghss23fRuBRKKCwodCnQokOCyflIauCFofNe/jwIhJvAYYvJ3iwk0BRSOgCNFNweVQL82GiU2+8AlRpAgyZ9PIjF2RF8mljdYIh6RQFkKk+mY3d2KrS1F10Z2t2bcPTxgOhujTUVUgg2OdmlTDmRAmZKyUrTO0XaWYlQn+ZatLR48eMSD+/fZ2dmhKCpCiFhnsZ1HSNGmOkhKjK0SCxZDPzL76SvpR0HSq0rgKgG5lGdX5Wwa68V92OUOOS/j2h1loPEF3ScLIQy73TRnrMGgGlBgX3v/k6z/TU41QMpRPRiIVRIyL0qTckfH5OuqlCJ6D86nKO2uxXqHaIORHMThPa6zBN+bbIWqGjNDc+9ega5e0oU2pfiLjpXtWNoWh2LVtVwslngUbbvEW0dRFJRVhVEaowKByNlZe9vp6A+2DH7o8FZg4v0yEN//+N9Gxd+7Te8BrP3uy/e5ietetnX+jtD7FYuQUrAphN4ikVJNLJxnvlpx3M35+vlL5iOoqgkHdz/g/v2PuXN4RF3VmKLAheS+kjFkDsiQDdWpjbSSAH3YXGbx3rwJuvrdm7clb1Xk8sd36/mr17yuDbdp19UWXD3n+talbGi3u+9bgz9T1Dgn+GCojEYVAWcD3oMkL30KH5nOJliVcoHqHDShcajCEQtNpMS5ipcri185nKrQ0RBdRCvLbFvx+KOSw0PB2RJrS7bHe3z7K49tYGdnQrtcMF846lHFqB6xaFq+e/ES6yzltEJL5Pz5U1QsqWYHGNXRugVReapSaLqOr79+Tm00d/amKKM4Pn3JN08tuwcwOtBs7c6otsa4E8uFX7G9d8hyucTZJYUqWSwXhOgZ6RozmhJVZGEbkMjKCwu7SGrkWStwZoS6VnROWBVCLA1eF5RVSVXVVKMZk8ZT1xVb0ymTcU0Mjqqq6LzD6IDRJbHLIfY+pbxTWqUFPjM4ZAbM94EUMZKGeM6Nkc3o9IEBOapUoTIzqHImjR48RDpr0SblklU+ED0oAsE1BBqCEgIpG0fIefGCBFRUGEJKw5fD70NYEuMCU1YYZemixbYerQPGpOczGSv292uOjvbZ3dkD0ZweL+i6lN3F6IL5ckXbtuDzPUngbLHg+PyC2XiLnb1dDvcP2N3b5YtPP2d3d5sYJLOSOkXhxoiKEaMNSgkm2cLT+25SIFHMkRMiOWGRKEQnOZYQknZ8b5olqsSSkRi/2IuQ9mzgwMHJsMPtAzQkPYHhiATY4wZjlwe3StqWQjKLp4pUnsPTM0sAMYFQ+uctEHX622hNUZTU45S+zflAZz3atCiJBN8SmoBrG5q2JUZF13na1RK0wigotmpG4x0klty1wt7pt5wtj2lCy8Xigme2yV4DHct2xdw2dJ1jMZ+zWq4oiiLLLxm0Ao9j1d5OnPQPulwzZ29O8zczgTdM6HLtx5sPuvao94SSfmjy7q3KD9CYG4ibtynXPcm3XuKvanjmA68niC7zbZLJAEjWF+cDRitEpyhRH5PLisXThMB3F6d8dfyERak43P+QTz77c+4eHTAdT9CqSGsOaU7SuU6yq1TMfnqSc4/HnGrtpnt9hc+8Dke9Rd8Ph972uW10629z3/DKc5ONf/p14saD89eyhtS3ffVvL/LcBcRUrFYeCjAFlKKT31xUBJOiCqeq4syeEXxgezYmKo2zjkI8s3FFVY14eQZPzxfoqGAU0guqI6oUilqY7gs7R5r5ywLvHU0zRwrN8++eokYHtLHh5eoZO8U2ZVliu8DL8xNMralGilVjefa8Q7Udo+kUpSwnpw4fOnZ2BWe3OD2Hve3AzvY+s52ar779DlOcYeqAVIIvBBcDq+BoYqRyK9rOslqtKNqW84sVgcDMBkqf0sxFXeBcYNXYlBs1RrrOozXUWlGLwjrorOArRedSBKbRGi0Gg8FEQ6lKFIbWeYLXEAuUCkn4MkuJJF24nprvHWYTeNNKJ3Ym+6oF0RA0Juq074tk/zSfAwJy0m0SKxej2gAPEdt5dE7jJTEN8kJBlIZQLAgSCNIMbUs7wIhET/QOZ1c46zAafJgT1QnleEZgSdfAwX2FqQrKccm9g31GdcVsNmJUjFAILgqF1rw8OWa17Lh3/wFbW9swVbx49pLjkxOc88y2Zjz84AEfffARX/zoc/b29lPUeWdZLVqCtwigdXpvE24SoneISj6QUZKcSQiRQNrEKOnNuSrJ9UjygzEq83qiCIOTsuSAj5jTCm1saa8VZM5AW/qf14xfzKf0+XkBCHEgAoeQO0K6Zm5Oeg/WoqE6m6FDFndWYtBFBJOyoERUYqdnYLuW5XLG2ekZy2VD23Q0bQeuIXYrls2KFxcLZnvbjMYTCpMm/q1pDWYKK8/5haNtV/jY4OMCHywudkn3UCuqUc1oVFPXNRDpbMvZxQnLxR8/87dZroK+VN5+2fmdYq53vfhNi9h7W3VvW9E73MC700K3bIKsp41bdciGNWf4kP/LaFORtFeTZSUx/UoVFEVB8Dn4ohAab3lxccbXp8/52p4jesxnD37KF5/+U+49PmTuPSFvOoNPqduSYLxCoYmiSDItWa8vs1DJLeZ6mZPL2OyWgkeX0NDbPYz3gOFfU67U2l9MbttOefXP150mG4QB1ywxrym3Bn+Hu2CMYHZH1HqC8yu6boWzqX1mUlIQOTrY4+l8xfnZkqO9MdVOjVI12FUS5NYlxbTku5Mznp2BF4+qwBRC41ecnluefFfiOkW7OGc5d/gw5dkT+ParU9TIMJrWnC7goj0DM6JQI6yPnJy0fPPyjMPJNkWlWDSB83PDbFKyNeqYFBW7O0ecu4r9gxdsjUcIkW4FRiaE7oLn3wWicrhmjoSOi4sWHyOnJw2Igmhwnce2EAzMF0tCCDSdRemKqtJsjRXjMhJiOwxeFTpwSRYkZKbO5Ry5RIsNPkVBhjD4ZCX/OgEjBJUk9VBJ0ds5EhhBITHpMPmYAZdEILE44FKEFkkF3UeSP1hMwskJCDpQDsjA0kuWJlGJ/deawhRY58hBuPgk40gTPEE5HI4+C0WAwXdP6UjnAs6BD9C0Lc5dMJts8dnnR9jO88nDz5htTUB3jArFt9894fTkmHPRBKdYzB0iI+59eJ9la3n+5Jxf/fo7tNHsbO/w4eOP+eDRIz777DMePLyLRI1zQuhSCEtRaowZ4XyZctkCRWEGk7lk1iwFIfS7J0m5mDe26mt3nRxFq/PoIzGKZPCd4HTu27zzTedvmmmzAaafEKW/TuytwWwG/g/N2pgckhxRH77dg8Z+Cu3rlQT0gwzt7n9XEhHRiNYU0RCM4ESolGF3ukXXtZxfnHN+fs7FwrCoQV0EQhkJdFycfk3TrtCqIERhtVzw8vlzvv3mOxbn5zR1xfz8mGgtRUzv9ag0SFWnhcKBMpq6Shl77J8A9hseMzcabt4OjryGFLx9eYfl8Ack1t5L+R4r+5sMbm93icvUrGx+t0HUvLItfF1nxOtaFYcJSgbLQgZeWWMT0clXX9Lmvg2gSoXUmtOLOb/+7hlPFqcErdnffshff/KvuP/Bx4zKCiUFiw5ENCpuzpuktST2DKDbnCiB+Ab8+r6R9GbN1wCxS8Riame85rf31ICN8mrlsvnhBsz4ump7rmATZG/wf28stwZ/e7uCcxeYoDkcpajPY9dyYhucFurpmL0J3H/Q8fKrBav2nHLvgK0dxfHLc46PX4JTTCfbRDOlqsCUis4HREpMbUA6iJHFRcA2yeRUlRPGkwMumu+wGibjKVvbuzw7Pme5mmOdMJntUI53CfElxc4WZjymnk6wnTCud9neqmjnnuXFKdYtkElNVY5ZLRpenJ1xeDBGmRpTFUjlmI1hVhuaZoS3DW3jqUaKoigpxdA2C4oKxIAEi7cKpQXbNkTX4EJEQpf89gSsg6ZzjEKS63Au0LaW8XhMmUWXi1hgokZiEkmWmIPhJZlYoyRWKqokzts/5JB3XhJABSG4ZEKEzPRIMufGPqdrTMAzxoBIyO+MBVJeZectziuUMqAEg0IbQVTEFEmGxfkNLjoGRAWU8RA7rI10DuZ1y2xaMt4uKecRM4bRCB4+mPHFn93jw8d3efjwHkbXvPjugq+efEezvEAZhW0b7MozGW2xvb3H+KAGrTk5XTI/XxItfPbJ5/zsJz/l8UeP2dqeokXjXWC1sHi/AlHUZUUKUtAopSkrQ12ngJaUEcbnwSTZzJtYvIF7kzU4U9IrK2+Aw3z/MV7K5ZGA5PAXPSf7yrAcgKVsLviydvHLrF/P/F3OsxkRlXIZh8FHkwzm+11/+rdPARyiDBOdluSWoZRKAVoxRd7qUlMZjQ0erSLG7LCzPUtam+2KxaLh6ekxz54+xxiDnxQ4Gzk7v0B8R61gq6rQ4zHiAs+/fsJifoFzFufAeQHlqUd10k00SaXemIKV/Amgv38sf1Dld8qqvtKC+MovAhuZhOImnMlm15RdCDRKFWghpSwtFKto+bvf/He+bo5po2ZaP+KjT/8lH3/0EVtb20hZYrNPsjIp/abERDiQs3xEQspo1M83b9VhP1Tv/u6f2g9SLqHFm9Dq7VHsrcFfPRlxcjynbVvqO9sUpmYVO47bSIOnkBVdESjvbBO+WzHv5ljlqSYF3dxzJil37d52jY8VqkzprEQJJvtJpfQxmi4DiOVSOLxTUJiSmCHNpKpSKjUbKIyhqmuKosQYQ10VmLLAG08XliAVmIBXDau4ookdExXQRKx1BO9TOjQRoopEleRYpNLIuKQyBaNxSVQNpjIoVRLo0mKrIuNKMSoUWivqUmUpEUdUglfgel8rBWIKEIPzKdpRK2FkDI0yFBi0773yUqi2IqAlogWiMhBcCiItFNpExHkkRIgp5ZeEgISAlsR2RQkElcFLDtQI3idhbDyotFtLSSgsKWFgRCmdzAI9dySKGDUuKHzwKDSlcRQKygKKQojiaZ1NmUQiKAVFBdv7mvtHB/zkp59SFAqJMQVyjEvmFy3LRUvnXvLVt8c8/fYlZVlRlRX3Htzl6OAewSuefPeCJ19/g+8Uu3t7/Kt//n/hs08/ZTadUZWjTL8lU2hRGurSpKAFmzQGI4oiR1JInw0HlbQFg8H7JLTsfUhSKypFTEsGRlm55hKfPpAtGzv3zaFHz9hmmKayT96wO4sbMDBv3+I1jM8li0E/rV/i9SVrOqqNRvSR2mS5mbX5JwWVyMDOJpbZIjGD4ADBeQjpvdMFaCUELyAVk1HNzhbsHOzz0YePcW3D+dk5FxdzxlXJeFIDgePnL1mdn7O6OOfXbsVMxownU6wNFHXS8ou2SwLjIphC06zmfxpm3ysk2w/He7xLuRlorH9/P629xID9MJd4q3LT2Lv9OZcZvlfuQa4cs3nchu0/zS3XMLGX2D4ZLLowTDUkS1DEWYtoybqtapDfsgScRF52c7756ilfh1PG5S4/+sn/zNG9h2zPDiiLMYLKri1CKclcHLxD65StJ7ENamNzrPI6sRHIdqVbru/fd3jQ174fm1+++uTi1UOvPeqmL99/ee21L+HnzcZu3kUcvr6yGrDeALy53B787dS4c4vzC6YPR9T1iFPdobo5oQk0ElhYS6sNuhzRuRO6ECnKUTILAbFUlLMC5xUoDdJnMdB4H7BecLFEigJjNLFZQdRUWjEdT3j+7IKFc4zLOmUGCOQgA0tRKEoHOkfBKqVSIIo0iMpC0d7StktqYxGliaJprUseb0rjVXqhbUhghujRZYGoJgGihKFANKXWlEVi/NCgUehCo61GabAh4mPyo8CCD4pIgTGRwrSJmwsGjQEcHkcfmUtMkclCQKuUUWLpHUShlIJCLJ0kXzhItPtgJs5amClIQCHBoHxidWKW9tASE6PYg0scUdkksYJP/ZVD9K3tcDZQmCllkcBm16VmOg9bs5bxBMYzYS8I9ahmOhvz8IMt7t/fQWLBbDrCNpbnL17w8uU5PrQE7yj1GKiIUfPTn/0Vd4/u8M1Xz3h5fM7XX/0DtnWMqhEff/gjfvrnP+P+0QO2pjOQgIpZczB0g5i0VjpPgILRBh/dwGj5EFFRoZRfm0ljAk7GmCFSVyQ9U5GU77bPRBFlc0jKWkV+Y9hdWtLWIlRXjlnvy/uy9vVL51xO8bMm8q8uNgNDmCV4eo2/kHfh6yZolM4yPiST9KDqGCI+dNjgB/2t1NbEIBeapG0YBBc83npUiGyVI7yp2Z7McM7StC2rruV8ueIvf/oTlss5T759yqLtuDhfUI1qjl+cp1zNrcVHTyTifYqddiGgbfemaeiPtLwZ8VyFA9dalN5m8XrjsTcccEupkndeR39g4Hdzu96xxTedtrnZgyvPSDa+3vwhzwxDH1x+L3oMkKw269RmfXHBUqikV+sjOBIJ0cbIolvx5PwFv754Qhs1h1sP+auH/4IHHxwhozGqKBOg6+cOFDH2YvUmzSshtSKqmMHnxpzYu6+84fnd2Mvy6jt946i4NBiu7qI25tXXDKvfyv5iYAlucRy89uDrfrkGY79VuTX4qypFxNK6ji4uqVAo7VE6LTbOOxyWoiAtniYvMqIolGBUZq1GFbE1BCEFFuSABZ/XIkfSTYs+4H1EogMsIg4PBKWyxlp+7XUgKpsARbCIWLQusxOqw0tHlIIYU17cEENi1rL5NIYsOimk/LMhYgEnDgkt3reISouTxBTdTNQoVeVA9cS6KCUpy4gh3XtI3RvyMotWRK3wgA+C9yo5zQVPFIXt/fzIC7TvF8YkpdPFgHeaaJNjbcqSlt7upNOX70fy5BE0EnqKh8QMEhPzozPA2ZAC1zpQlIqiSiLZIeR8jtGDWIrSEVSHEyjLxGaaAkYTw97+jHsPJ9STMaPxlP29beqiYz4/ZjFf8vz5S85P58wXC0xRYkxF6AxFtc3BnXucXjTMlw3/6T/+N06PF0xnO3z60afcv3uPuwdH3D26h1EjjC4pyxJnG7z3KKWTILRWaKVQSifztrcp3dqQ0SSZub2LRC3J3JnEE9Nzy3l4hZRxI33up9WYzeRryDYIuWTdjsvaXZsT/nqHrgBRGwEgPbDbCObI0O9SVTGuzSlrkYbNy/Xai4ISiFl1sQe365RwG2xf7DOSyCDorEz24+nvLsbsxxhz1DgUUVEYRRd88lkloHSK/h9VJbM4ZWfH092xeNvx6O49ynGF1oZm1XB8ekHbWJrW0dmOpm1YLpfpHflWMf9T0fm7obweAvbP/nVI73fLIf6WiJNblvVYeRds/Mbj5c1HXbdle+X3V8C0XPmXK481olAb7iBpkypGo7XCuoguhKAUK9fx9cvnfLN4ySJ0VHrK4YMfsX/vI/a295mODzBVzjkverA+iOQkHDEQ1UBHDvNZj1/XwDW+Onfle7h6d2/EQ2/zCt/UdVe/u7HOd30z3rJsotDXtOf1rbiuJ68/7L0zf9XYUNaGZROJyqNMoKjAjDTaJ8CnCkdRldR1TVEWqEKoRgWjUY0pDVErgqSIWG8j0WuCGLzovIAm02JrHZGGlQ84vyKGLgUy0Kdwjlgf8eIJKmcaCR4XYgJwxJSIPipcsPgIPgSsBxegMIkZakKHc0mrrgdCWbUGZyN0FttZkEjXWcrS5F1XjmwigUUVsv5ajnByQWiDQnRJRKcoXQS0GhbZGFRmlgKioYsOGwPBxqRPpxXRp0HoRBLaitB5n9qqIqiAT+rNKTlYWGfSIIB4hQpJCmYQ2lQRZRRaa7yLKaBDgdIB0QEXbMrAIQZtwIsjikeqOcYEiik8OizYnm0hynF0tMN4NEEXFaPZGK0MpoS2azibX3D28piuc6wWnsUFTMYjtg+P2N7fph6POD2Z8923L1guO0ZqzF/87J/y8P5jHjy4z87WNnVRYYxmuWxZNHOEMaUx6KpI9yRkU63GKEOKXI05naCgVWIwo/TKhYIStQHYEuRJAKvPjBHoXwhBUv5nejDFEO2bJAwus3uw9u8bInVjHEzJg6BqXCvpX07b0U+i/VQaNza0PZLcBIFpNu6TwanshX2ZW5D+9tILHrOAc88Y4nO1MgDCgVmMOb0dIYukK0xU+JDzB4siOJ/Gy0a0s1PC9mREFCiLkiIKo4M65XMWTQgB61qsbVgsliw/vIv7E7D6XtqtS/+2MCwK160NVxnfV8otgMgrtd6Wlbim/CBL5bURDrdBAlfOu+aUtyNE38fd5fGZq3rTXax9eS+P6zTkM4QcXEVinpuSZp4iBwOGFJy3oOXpi2e8dOechQZrDdVon/t3jrizd5f9gyN0vUVZVGhVZrkoBVEPm990jbyI9BvI2LcwuwNtup9cus8rGTRuC+auOe77bmVuwlm3hFHf43rXbNLfgPquC4p502bwEgF6y7M2y+2lXuoSMYlJccGD9pg6UoxAtaALjakqdFGidao2RI8SRaErRApckMT4kVNmuRKUIRqdmShFiB3WdXjp6BTYaHNO1JilLDLoCTFHyAa01hl0pfdYaY3POUy99wQUUSW9u5yABBGDd9C2KXl9j5o2H5gxiQlDuiSImQdACsLIpwRS1ow+n2uM+AhtVGhdIug8lvqsFy5PCpEYHd47lKnpQkcXHAbobIMvBI1GkASWXUQrhfUdffqv2Lv6Skq/Q4jJbyuSoju9RgVNiMl0mdjCXq+uBzakDCKVJyqP9R7vA9qEzOB6zAgmu57tnTE7dxWff3jA7myH5WpJjJHlckWzWOBiizYlF/NTmtWcpl1yftYSg6FQOxzubjGebFGZEat5ZH6xZLFo2N0+5IvP77JT7fLZx19wZ++AuqrSa+wDwXm0EqqyAFJmCm109k/MRFpIeYoT6gUkiY0qSb4vRQ/mMli8pCnV6+nFkPLfhh74ZGCVwVD6M52j1KaY86vbrX7CHIJr8qQqPYjrn8UmA3hpzEpu15p9e5VlTMdJD9SGZ6o2oCMDQJWQvS1yLs2++L6NkcuTem5XcidImwyR5Hurtc7MqML79K4UaIJovBGsFsZFwaptCb6j0sK4rAgx+ZUmH0tFjCV2UiGHOySJiD+hskk8XFlL3z+H9xbA8C3Pfrdymxrf7aq3PUuu+fRu5d2e2FVwIFfSeqVNQZ4H8kY3ZGAmpPy6nhTwN287jk9PeRrPObloCLMx9e4eR1t32d++z9bWHcbjCVVd46OQHJVk2PcmUgN6Hb63A8zccP/rULfXHhl7yLiGTrfvzTdBpFd/3dxmr///6jFvvu7tyMdXfpTrv77u+Ffafw2BuAk6hcsuSq8rtwZ/ShdEBOvAuoiPHqUDpog52lNhyipRyEqI0RODJ1iQUBCDwYZI1CE5kleSdzCalBImJL8kiSABbcBEiGr96iS/fY9on1zSfIQAKmYgABhtqIo6/Z2ZiUyNECQFYUTvIeqUU7RzSd/Np4W5X0RRQlWVTLcKlHSpngzaRNbgIcasaq4MSjLoleRvIWgk6CScnAMPrEl6bEp7onTJT0PAdRbvHKqIySwcVJLIyeuu98nnwicRvQx6NhTaZU0OhRhyYu7EKkUCMfjEPPasIR4RT5Rkvq3GEVWElB+4ilQjGE0U9bRi+47h4Kjk4GCXwijubB9SFyMuLkqePHvO2fkpXRcoWwdiaJqO5UXDdDJjOr7LbLzHqNyGYGg7x/xixWrZopXmwf3HPHr4gM8+/xTVaipTU5mS4FLmDK2SiVYKUuBPDLRdBxJRpkjyLDGbdX1+NkqlCDTJkyYpL2Ufh6Hy4EhahzE/xzyEenIt9sE3G260gzROMgWHQRnmum1bnvYyu9fP6zFnBBEy85ZB6ebuOg6BISmwO9CbdnNO3FxirrSfXiU9bmQAcOnaMcSsQ7hu6iA/gyKihlR//XUHcJyG48Aeo3oTj+qHCaIVhS6TuVz1DuKpb5u2pe0aEEHpJLXjQsroEbKPayUFRWkg6lvORn/o5brJefO7S95Ubz71ba58DRt1VQZ481d55cNbXW2jrvcPH2+6ZD8W3lTWIOD7tPGGFX3jGm9sxY2LdU9IJA9crfSwHoTsm77sGs66FcftirOzC9xWzf6DT9g6fMidgztsb+9SFCMIyRUpeEGLAfQAygJ+o8OubkKv3lm89p2Qjd/ftgfSUXHj83suV7t3uMU1ZHr7um7bSrnhZbxhXG8ceil4+rVN7O0Hb9dztwd/KkW1RgRrBdsJMWRTVqaflCoIQVGYApG8kwgKhckRo54oHlVEilqQUkiBhglxKdGowlDPNHVdoC86+tiDJCxJlueIaB0xUaMwEA3GFEmHUBUJQNQ10WoKM8qMnEqpp3QkhCRwGaLG+zSolE6gVYYUt4IyQlEmk7Q2Ok8skkFiTzmlHZlWGq0LUGmghiBEl1ZjhUrZLgyUKqBVRLTDi8XHgA6R4APeptRt3ncEb/BB0TYpZVmMCp/zsUYh5SLOrvtaGZRSKQevgIsBUS6n6bHE6PHB5ejVFF0Zg8dHC4AuYTwumG4btvaFQpVsb9Xs7NfsHZbsH5aUhTAa1bQLx/nZBQvV0XWe8/mKVRuJsSC0YFtHs3QEVzKZ3eXhvYfs7xzQtZHf/OZrnj9/SVnWfHD/I+7sHfLJhx9BgJpxilD2DmvTszZFgcmgzbs0VRtjsNbldyGsB0jGwCrRmxv5eDNL1/vbQTb7kv1Ms2dfD3jihjxLX3E/xHrfvxjwISSG7ZU1Uwbg1wdiAENe3T5rR68tSH6G101AyQqbTMspSXpI2n4DYCVPZGmRCENTZAB3kdRHPo+f4Yjh+gncSQ7w6beb0vv3xD7/iAxCrZGU2k9EQEVKU2QWIf2mWG/aVCVUhUmLFoI1HucCFLlngyf4jkIbXHi7yesPvvQbi8u4L//W//1bBk/vrbweFP2+lLfv3dsd+VouarjoGmil0bI29fZTCdGjpEhZhVSBirCyLXO34qJb8XJ1wbNmSTs2HB7c5ccf/pTHH32C0wWi1UAI0M8xEejlWtIub7BEpVZsgAhhWNulb+8rfXHpjN/B036HOeMWOOl19/S+7vWtzr+pvdcAy9vWe2vwJ0pjdEqL1TZC22iiM6igcC4g1uJcgUtSfdgGXOtpXcQ6oWuFFY6mWVBLiahEKcTo8d4hKkWhFqOK3fsVWzsW+dUFXeNpCVwsWmwH3qc8r94DyiCmTFG2pUEp6FzKGFKWJdYVEEYphRmaSFrwoxjKSmHKi+SwXo8pRxVSJPBnKjBVwEdH10WUipmR3Fh0e9/9Pj2aBER5tErfCQrrHCEkU6Qojy47CrGIAXI+U0QTfIG3DsnZG2yItM7jg2V+0VHVIeX5zU8rhj6MxEJIBL4xCjy0MafrFQ+Fx4nDe0uM6XqiDUWZImUb65lME2D0PvLhJzvs39OMRweUpqYeC9t7QlG0nB+fcn664svfPMW3NUbNUKpk1YBtK6LXmKJiWm3zaH+X2dYW25Ndnj15zi9+/jf4TjEejfnowad8+umnfPLph3SrjugVp89O8Ftj6sJgCpP9EX2O1PUopairMqc1g7qsiOTMMNn5OcRenCaC+AFQaZ2AXgw++fJFQKUsKEYpYgaFIayZvk1OYECWvUuA9PvlBIhE5XM2d+9x43xgCB3ZiMAlT7gDONyUktk0vW4wdT3b9ureuq8j/S8MjOYweJNO4ZWJYmiG5OldXZ02hF4+omcU+8jqSMrdrERhVGa8Q3rfeyY9SPITrAqDiCbESEEBWUqCnLEmhBIJ0DrHH3vZ3CNcN6H3e8pLJ3yf62xWcy1T0/99M3fzh1Be6aZbEyGv6+A3d/5VjvTVy173kNMCst76ZQiV5wfJYswiyT/ZR0WHZ+U9XQw8nZ/w3fyYs9Ciyy3uPvon3Pv0U452tinVBJ/Cy4g+t0lyBg8iUXyyykWXrSapjf2GdNhsXMHuV2e0zfu9erfX9torL+P13XMNAfa9y3Xtfl0bXnc28co3r9zsdTf6bnd1q6E/PLi3L7cGfwEFuqDzwqoLtDaZIUNIwRDORZpRQbO0BCs0p3B+fM6L43OW5y2r88Bpt+Lpd0+4s7VHcDpH5AouZt+9qLKPnkeKSBfhdAE7M4+ezIhViyVde34BUSWQ1OGw3tHYyKpzzFcrzk/nLN0W+iyJJttlwC7SHYftkAIILlpWey1d8ChdIGIIUfAu4loPvqFrhOAhBktRV1l5uTcdpsXQeovzK5xtE2DxniAh+SSGiHfpmKCTWcv7iFORQIrejYWA9/QYwgewwSE+KaaHYBJrFLJ4c0gMi+0i1gciNvkHxgQzJCQTeas6vEvRytFB10QKY5lOQYxi+QJ2J8J4JOztb3Pv4R2arsA5w8X5BcvFOe6ppywD3jmCMpydwep4xeJihYo1o3rK1myfvZ07HOwfsbOzg1Y1f/fz/8b/57/9WxSGP/+zH/PpT3/Mh48eszXdojQFpS5SwJAZEdolwVqC0WijMdln1HuPtSnriM7+fuLIPp7J9t2bQ7VIMmlLAlqD2LIwMFHBJ7kbiInJzj6DAwNDP/etGbp+QiaPsZRV43LatjicuA7O6OUYoBdZlSEYp/cVXJvz/aV54bKWHz0Zl+qQtSJgYgN7ljEfyHouiLmuGNYAcd3mCD4M7V1PTz207YNaVP5WDaAl6Jy10ydNQRcDKkp2CxRU0j5CpwTK677KABuTzdBEopLk9qGhMn8KZt/8rr02hcObJ/PXLSfXcm5vsT4Mo+FdK7hteccV/tZM09UDf4eoNrUzb/YkG9qDT+OJkJlzyXJTJgmAhZRgcqU8X50+4dcnT3gaFmjZ4uG9z/knH/0ZR7v3GW9NaLJGbFQmiTGjSYRHWhQUOn/ngTaruvTQsy+v0M7X9q8Mxw7bx9c+hxsB4Q3H/jDlbd/fdw39uc1Z72csbW7XIq+bEa4vtwZ/KI0yBbaFrrE4a6nqgvFkQlm0SXNOBGUq6nqEAkxhqMqaznQImkI021tjTGnougZvC4gaXdRoCYTo6TpPszA0o0DrLUunsD6mxV+Djx5dGspa0XlFpEb0GLLMh0hNVW4hBWnRkzLJu+SgkNZFVs5hnU/ZJLpIZztEFFoUtu+VMhBXXaLak4d72pVln0a0T+Y4FMFDaF0CFy5xQkF5rPNEFCFIjmD2mELAZ9BbaKIvicEQo0qsYY5ZCNlPS5vssxgE16UoyeAFbYTgfBIz1hCKiDHCaJTEeYNYHIFilF4SY0AVcOeg5uGjA0a14eDut9x7cJe7d7Y42r/D05fHfPv1c1zwnF6ccHF+zqSesH93n4vTNqVza6bEULI9m7A73ef+4X32D/Ypq4qT4zP+1//fv+f5k5fcf3CPv/jJX/IXP/kJH3/0mMKMEi7zAY2nRKFrw6geY9sRznvKQqElZUqBmNgqLUl2Jtt3dVFssFUkZkokS7bkfLwZbyTXxpiigSVlOemDh9ICnNm+Ycj0C/M1wG5jhzUAxXzAJbmWAf1tTKVRkh5kXGstxmyKjr2pM1e6ZgLz19LLL6h1kA7Qm3N70Lc2467TxUXyGFAM2V0gg8FNB8DNsskE0McdqyQfI+u2aYGoBaIgXgaM199K+r+ijyQOwefo4Ij3NqcZ3NxEXTvr/NGWS1jkbVbH62v4AcoVCvJt27exYbqpmW9q/e3g8Rsads3Prz/jTTd6GSyt0zBuHLFxz3EDICXHhzxmJLliKE1KMiCJmwshYoMlCJy5lqcvn/Pzxdd0wOMPf8rnDz5h784jRuOttIGmQAiMAeuSMoPRVQrKjL2FyEGMKArSluwqX3nz7b+C/195aLeC4T/I3uH25S3aFje+eBOifT9Xfw/lWm7zjeX24A/LeFpRjAVTwWQ6Yn9rTFVOeH4WuVgtMZWiqg3T2YjtPWE0KajrmjAJzKY1rl1QFQVts0CUQ0wkWoeIRyRm3zRwHdhVpE8TWNQakY62haZtmE0mmCriWkeMHYpIqQsIkWjbBChVgVKOGFq88yTKCGzr6eyKLkcKKxMJ0aYglEw8iErpbFSlMIVOmR9iJJAEoQmOqAKBHFRB8tmTmCRvYkx+eFoXaF3QBqHzjmg8RZXFqaPHFBpjNKaMiBaUF4KNUEBhoFDQdh7nOoQkvRJCoFl0jLZHjOoE6rQJBNfhnSVAis7dntK6JeOZY2d3l6O79zi8v8PBvQm7uxN0VLx8ehe7WnFxcsx/+y9zpBBOXr7AeuHlScv8PLCqO5qzhuU8MhlvYU7H3H3wMZ9/8Tnb0zGnx8f84he/4KuvvsR2HUdHd/ln//d/zo8++4KDg31C64htR7QrjE4SM9ropDuodAZgiq5taAtHXY/ROgfSqBSIYKODIClStX8dNxcX+iAclY0bG+nOEESSjIESPzBnyaQc8nVyxXEtmBo3JoGN+ImhAQlABTZngQH4sT43zf0RH3NgUR+1l0FliiNSlysfwOMGJB3AXfq5F5lOzbyMHlKAycbfPXjNgTHro8igbhNs5jsIG+xnPjbFl+eUgPk/JSopCypBSRhAnMQeqGp8zvmMJCZCRY8ZAHi+3fCnBwD/sfy+lbdnh179a+0znAAXICHPSWlsueAwOklYAaCS7q0lsnAN35w+5ecvf8Uplm11hz/74l/zycePqbe2UGWNFUXImZoSs5e0cpWkPI3R92qfvZRU+pzCAz2Xcoa/Y0/9Y/kdlBt3jJufb78pvDX4i66lLjVVKSmSr3+dRagLmC8dUTSSFzrRJG0zTa8NCXhsWOFoCTGAlIj2hOhIdJjDO7Cd4FohOjBeMKoEtUgRwLpM2ncuLRZ9NGOMJHpbBUJo8L5LtxfBdp7SqCwxAc75ZE6NFqV6SRfBmIiJSQpGa6EsFaIC1kFVK5SukybfMpDckyIqhwhrJQQcTdsxGtVISraLNv1mQhNQdL4HNmnHp0sYj0rOFprQwXwFTmC6I5ipEJqAtR0jbVBGCK2jmXdUM0OpEwjxwRGiIwRoLVR3FD/9yU84OVvwi1/+iq3dHb740Ud8+PEhZeU4ef6Cb776lr//+X/h9ALuH4yYHT4AU9J2BS+eX3B62uGsQbkxE9nn4/uP+fiTz9nZ3uMffvEb/u2/+fecnR5TloqDwz3+yV/8jEcfPODu/l0m022qok5mQR8olaE0kaKqKKsKBFarFrtqkg9ioVBK8NbhfEhgLKbctUpAclCGNhqtNHGQ3emjFPNEJimCLfRadQMTFUm0Y2IP+8DtkHUWQ/CDPBFsAKE8eW4Co0uD79pN76u7xkQwxktBQgnPpfuWjYoH4daegcts4KXNaW8/XtMLuZ1h4+/s9xcl+zNugsl0M+uoYHJ7NpnNyxNIzyYqYpZr6qVhBqNVj/gyUboWwl7Xm4NA6P0PU/tSJLDKHOEfeXkdy3dpQ7NZbjrhBtPU5nt64+r+ln3dt/sNp71yudcc/27A4+3Ouv7o23GKN5XLXbvm4tebRclBVAyi+zGmDDm6MBhd4Hyy6ujS0PqOXz35ml+efcNJWFKaGXfv/pifffLnHB7eZVTuYXQ220bBkBnFmJMW9GMxz28xuI0N4+aoD7zqbZDbvfH9q2LNt+2fq1Txxub1TdVdAjVvus7VCt7jvCF9G94/k3m1lbeqQl69w35efaWWt2zTrcGf82nH0Fq4mM85OTvD+Zbzecf84oLVYkmpA4tlSwywXMDiYsmqaWnalqZ1zJeOszOYzUaU4yTXgoso3UtyJKanD0wM+Rnk/VPOOBYpilGKHnYOaxusa1K0rAfnm8wcJXDoo8F7Czr5U8SB8Ug6f94mIFFooSgEE6AoEtgrMRR1oJ4KZWXwCNanxQ8MKhokpFyHHk/EYx1o2zFvlrRNQ9e5bH8U7FKhdEj3pzUehRQV2oyw7gwbC9Soo5xGRttCOQF/Ap1EVFhSS0GZH3BnV0kuI4AuFLt7I/Z2t7C+YW+3Zns25s7RFlu7wouXz3n5/Oeo+ISDwx3KUmHKwKINnC7g8E5kfuw4P285+yawejliq77LvUcPefjgA/b2DtnfecSvfvNz/rd/8285OT1Hi+LhoyM+/eQjHjx6wN7BPtuzGThoO4uNK0ajGUYpwrJN0c8xg5qsKxcloIDZZCv1U0gTT3Jy7rWthMKYJBZcFigRfEoPMwCjkNlDyaHaKjOBibkKaTLOo6iXNNEb4yfGiHM2m1ZlACxDdG7s65P1mBtG43rEXVq0e/Pt8H8ZjlmnX+9NxiqnkUug6uoMHXN9vf5q6Nu0OWnHxEQOQixDIAroHu1uMpPZBH3JnzFv5ogQ9Rpap7uMecMHazd11sDzlRkjw/Isik4+rwe+6ce+H1I71SsBJ3/k5aZ1Ll75KNf5Xr1m0Ys3fP4T6951ef2N34oHu7xPyt/lDczwPpNZ/RRcljL6SCYHsk+5U0QjdNry7cun/MPFr/guHCN+zP7sQ35873MePfiQw7t3aILGqAqlTJ4zdWIGhMHfd7jwYD3IAPBqu6/c2lW/zlu9Gte+bte/mW9dXjMGXn/MD7VhfFsQ+rqarrbxNeDyTVj22tPidTvGN5a3CPiIYJKcSNO1NLah6IS2s7jse+asx7oIVNgOnA1Er/Au0tqAi4rRpKAajQlcJAZMJy2XmFNGeRdx1uO8WwPA6NAaihKsb/GxQ5uIcuSo4cTsaBOJ2hFocD4QYh4gEhETMSUUhaIeVYxGhuBOcDZiXUdgLYQcYvbOMBDEYWoQnUxp4t2w2MUo+ADiPN4kfbkIOB9o2xVRkh5i1EJE41YFSvU8SYGLOa2dLokSUYVmZAqmM890G8aTyHgGiwV0LuJlST0tmDRkczXoMewdbPPZ5495/PgIowORhqKwNPNTfHuBtwu8DSyXkWZVoW3FfGHxTqOD5+nTDu0cdbHNxx98zPizPcpygtEVtnP8/d/+nKb5b5xdnKOV58eff87DB/c5ONznzv4+VVWiBIxPgKQe1ygNdV2hY6CRCDFgvSO0AVMYirJKUcui0EVBNRphW4sxZfbPs4MIijaaHjhEAkbpBPhiTP2f2bTB+plTEokihT5Lv5mIGfRA1HEIyFibbzcBWh5U/RfXlEETsj90M1SzD8aIOSRD+kjdNWMXY0w+iKT+2aysB7Y9uAKIspGVI5tktUrZN30OmopZjDkEyUEtat1+RWbaWINTn0zkAgNYlH4Q9JuuOLzx6+N6IJ9uNt/DNbNWTH5MvZ8lxAHb9uRfkIBoRdjIU/rHWzYpNLlhMv89LJtEQ7z+p99eI96hvJGxvMp33YSaN0FW3iQOLHzOIa4yydDPL4CPSRpq6SzffveU/+G+YWEd08kRf3b/n3J49Jit2QFVuUVd1qhCUwdFDH1+3aQ+QVxH/Q/t7DVKX/OMrv9BXrm76499Gxbu0kx6y/NereNWp94GMH7v8v5AYF/eWNNbTwvv1rZbgz+tktkNzSDUXFSGyiUxZGNWiBa0NqhKUY8EZSJFpSjrElNoKinY2d2mKEYofYEuFLaNBJ+8ERSKGByL+Zyo5ixaKIqIKWPK4UtaCEXHJMcSIihPFIfSEVUCRUenFintWwRRDoqWaiewFxX1RHFwMGOxKjHlU4q6YDQucbSoIhJdygFMIUz2CqZ3NOqXc1zXELwkUyYeEUNWTE6prrKoLZIibl1wFKVJeWODJqIhFPncJOJsg8eRch77sMJHR6kjxiTBYueFsoYuphgth+X+/Ts8/Ow+rVvx5NvnOGnZP1Ls7CmKYoWPDV07xwWFGKFZzYk+UlY1q7nny+ULoje8fLng6TeB5gLuPDrk3p3PubN3RFVuY7vI2fGcJ0+fcH46p3OW6WyLe0f3+ODhfe7fu8/2zoRCwWhUUxYaIwolJoESCSilKApBRYWuDFoE51xi8ooKpRTWdkAC3ONRzdxnx2Sd8lSSteSS30wKjvDRY8zlTXfvCxdC2KDtyfNiAlYDmMrjxPuQ8zEnTcb1gIsJZ0oGLJJAVJrr1/p5akMW79JGbk2IDV8oWAdk9FhvCBTpl4f0aahO8oLUCznnv0UpdIyDxuEaQ2QkOgg/k8FxGls9YBNhIwBX6DOhpMvnAI/sgD7gubgm6noWXmR96/3v65Jh7hX6IfRfJMHADAITSA4xrH//Yy7D+/maGV6u/fiO13rXcvVZyDW//baR603vx83w5bWnXfr5KhsjG8pIG6bTwWe2999NYyoEnzRnydHuyqQ0owoa13GyPOfp4oTnfk63ckx3PuBHf/4J09kh0+0dqvEUbcrEkoeka6qiQrKKQz+PDE66uV1r397rI3Ovfc9u9X69ES3/cKTbu5brbua9tfFdKnrd+LgNtfkO17qNunkubyHynHLiqqxRp8sUiFEHYTQy6AtyzlhFUJGiBF0KqoiYUjBFDjbXBqVMHjTZT0mSc38f2dg0DauwoAmwV4NSkaLsRWaFbHfNsisRVEh+XCoS1ApvVkgRUx5TOqRoGO8F6rHQnAjTsWE8qynGJdqA6IBSObdtl6RWRAv1tGC6ryjMEr8KiLeoKCjls39ZWnRVVClYxWdtughEjy4KlAGJMeXUpcBIgSjBhoCJlqhAjKWzFic+pV6lT6wtTKaa2Y6nBA6Pxjx8vMMHnzzm/Owc21zw5Kxl2S05u3iCSMVidQbBc3T3AYf373By3NA1HctFy2q5ZHkeCJ2mNDUfHP2I4mjM47tHjIsZ8zPPiX3BctHQrFps4ygKzZ07Bzx89JCd7W0O9vcpTUFVgOCptKE0JmlAopM8gbcJePsUxKO0oI1JQS1KUegiYSyfs1uESFkUKCVYZ7Mj9Jqh8hnUhRCTxlyMeB+HSNkoCZL0AuAJJMnAmvV+dJsALGX3UIMAdMwIZm167cdUBm99Srh+sk9U4mWGMJdhodj8Z/h3Y8++weClQzNIHWq5LMacxkkyPxN6H8Y1kJANpqEHcj0T2IPDQZJwg6i7hNHyva3N89nvsQdz0h+/ThPYZw8ZwOzmepNR4xAyEjcumINfkMwO/kmUDNJv+ukPoWw8XHnViez3qFzTtpuaex1pzYY/Xz9WYj+eUvF5ThqwoxKCpDSmJ6sFx+05p3bBRdfRBIO6c8T9yR6P9v+M+w8fJR8jSZa1GJOg+zpAq7cG9Buufme5wVLetNi/YWPxql/fdYhQXu0XefXzAD9/H4fwbV7P99Huq2D4Cglx26a8j3LTNuBqeQvwl6REtBGiuKTFp31iAEvSi5p1vNLinBbREB0Bh4+BpnVcnK/Y2qoxpkgRfsFTlP2CnNi64CNNF1hE2NaAjxSFziyDEHySV4kxAT5UgJyqzNFiOU+BJi4Qug5bdXhxoAM+wMV8RQgFSmlsZ2maBV4nU7GP5MhFRXAKYyqqssZKCVISJBBUQEufEk5jUIg4gu9SFFZIEhZKRSI2MZNSoUOFoPARrLWYykIMBFkmUU+ddfoUjMcVB3emmMKwf+QwEQ72dzBK2NkzTOpdvroz48nZgrOLFduLQD2acbE8R2Mglig1IsYRy4tTTo4XuM5QyBajYou92Q6fffEFKhZE73n27ClPvnuGD4FRPWJ3d4ejR4fsbO+wu73L/v4e08mUyhhc22EUFKbCFJmlE8HnCGoycAg5sCbpY6dJMpkhY04oTk4H5il1hUhiA6uyZJh6pQ8wSjttbcocuOM29uPpuN7/T0sylKyBVrzkyxc2TJBKqQ0Q9upM10cG9/RbYvF69LfB4GV2LrUjH5+vsTmPD4tJBqXB+zXnsDmBbIC5SwK8YS0Lu9HM9fX7K2ywhZsl5kWGS8zjxjjv081tgL5XeL0+T3I+UwiDz+JwrNq49tCujYbG9fPtkbG8xa71D7+8OkHL8P/X9cMbJva3WGGu4/Jud/zviva5+ebkyqf+rV0PifVccDsmMAGkuPHdkC879ptASeMxpL8dwjJ0XLSnnIUVLy7mXLiOWJWMtw95tPuY/YcPOdjZoS63sg90CrzqR3oCXCHjroDEtKG9nK/11TF5XX/c/ogr/Rpv+kNe+fMSFpRrTvl9Le+7jTd05Q1f8brRd93xPwRwfAupl8SuKZ1Mmta3ON/lSD+P9xbXlQQfEaVxHXRNRwjpt6Z1XCwc8/mK7a096nqCxFUy7ZHTRWVTmlaasjAsnUs6SKFPa0Xye1IFmQRPZjHl8Hi8pOwYF83LnLpLcF3ALR0vX7Qcn8Dpk8BOccp8HglOcD6yahpc7LBWkv6YUrhOOH/uWLwsiFLi0ShdE12LFo9WSb9PRYWWkkQhNsTgs36ZSsEJzmWWShE6hfOwXAZsGyjLjug7nL+griOjScVoYtjfDzx4uM/Hn95nNNN416JiQQjC3//9r/jmO83dnYfcvbvPdyfnLP2cyaSiLMeIrGhb4etvLnjyXUuzioyKfYrZIZNqh8OD++xs7+NsoFt2vHjxnKZb0HUrQgjs7Gxx/8ED7h4ccWd3l93dHSaTCcSA+EhphHE9Irrk1ByiHyJzXJYyiRGMZNFplycqlV7wEEPWMQz5mafnqE0SXe7aFAneM14p33PKSQkhvQuqSOxg1JdNLzGCX++QU6q3Xh9v/b6koKJkvkn5dXu/HWFtT84sWkwyPoEc2a0VEmSQbbmaoaOfxAcwmdnHwGVw1oOvNegUiGth5cHMu2FfjSFiQ3IbSJlF+gXj8ozc98elNS9dlCFKcKPNMbcnzeYBJWmjtdk2JdlvcXADTJX0z3NYjyLDOeleesZUXW7Mxm44qnzOnwz7l8paR/H6X29Twyulf4wbf95c2+1B5qu4/LfN+N1wr1cRyOYNr+mxKz9e8328vAFLI1YBfhgbgXVqSPJ4UCicD1jvOQsNT5sTvjt9wUnZUpY7HB58zKPDjzk6eMhka4bXKTtOyFWTN5cRMosqCCa31F2aklKTr39mm+7Gvc/Gm9+r1z3DWzzfV5iuN5QbEembns8bTv9By+3f8xuPHG7vptH42x5LqbwF+FuLx1pn6WyLCx0Rk/Tvos9O5x6FQhcJKOpCoav0sos27OxNqEc1Iiu0mJTNIYKSlLVBac1ktkU5NaizpxRFurLWNdok+RitTUpn1qWo3cJUBCKtA2vBq0hnwVQFpirxRmFVkkE5nkfKkeXsokWpgnpc4b0hUIKucV3H6UtPGRuOpeH4xQkvXkAIE5QSnO0wIaCKMjF/StBFCaVC2iXKrLIenyc4RbuKBA3WtZwvX2BbWJ6TIpOnK+pK4/0Fh4fwyRf3uHt/n6ruOLhbc/hgyvnFS77+8hc8ePiA6DXL7gX2Rcfu5A7T6R6728/Rq4CKO5yfKJ5/5wnO8MItmdSBR/c/4p/91U+5t/cBi4XlyXfP+Pabb/nqy684PXlJVZfsHW5zdHjA7vaMo8Mj7t49ZHu2TVVUGMni00TEeESBNhGfJyGlTNaXAhPSYu+cpyirJODrQ5YjkIwBZTDd9mnPjNGE4ClLQ9usCN6jjU4mWVGZRUoA2lpLWZYIyTewB3FCYhW1TtqBQv9O5cCLHuAlDJnT8kWizgCoDzHPNlElvRBrZinz5BpDIA7BG5syD2u2jQ0geGnyTjM8axYsXe+SJl9gYDkHIEdm7DI4EgEVddos9QAzksDjADi5Mqf0PnghW4zX0+jAwPVtDIl9GFhcIiH3xwBmQ0xBWT4MLEjftt5Psu/ztcm7r25jwd747dX0cn+sRV775/tcDG5b0/u64tstlddFib+5xqss37Ulrv+5fPya+VuDiY0xGNlgoFO+eec6lM4bohx05WOkcx2OwIXrOD0/4+/nT1ialrv7n/DPP/oxhw8/YVxP0VGhSOkNNSkgMIZIoU1yFcKv55Hezy/lxxlu5tI9yPr7a/vlum65FWp6E7x6tTffugyXeDd+62a+7C3KrfviLcrrDv+hEes7Dt63yPCR/HJMGTFa0IWiqjVaCmZbBaMZmDJgKsEExdYu1BNFMfFMjDC9o5lrRVVp8C3Rd9SVYtVFQrBoVRB8xFmHEsV0pGjaHLgZwdoyEQPaUBbTlE/UQ2gK6MZIULQdRAcH08c03a+odYs2E6IWijqtadXMsLs1o+0MFyfnIBVlOaLxQlQaH4TQaUJX0aE5m1+w6KCuPIGGGJO0i6YkaoVGUxY1qiqZuDOWzSqBHBUoyqwfWMFoFLFhgRjhsx/PIEZM0fHRx3c5uDfmq9/8kvH2gsnuDKVaVm7J8dk5y9Wc82bJdnNMe6Jx3rOzt00xHTF/fsbZmeP584758YJZvUO8uMunn/85nzz8C+4/PIJo+fbrJ/zvf/83nJ2dsVqtWDUrXOjYP9jncP8OH378kAcP73Hv8A6TcpRYS5Wlh2NAYkArA1phO0tnPS54TGEwRYlEhfchma4VaNFoldkppZGQI6GDX++YtUaFzPApoW0appMJ88Ucazt0MUr+oz0IESHEgMtsYQqwScxJSD4GaK0xxuBjTIwrPsvGhGyG79PrOZxLOYO9XzN4fe7fCBit0b3qN1ye9+IwJLKO10as4CsLy1UMdpltUKyDKzZniR48kRebYbHMpu2eI1jXu2m2XoNRrtS9lqtJbV6bxuUSU7k+Ig5YVXLkiiBoSQwsRV/3xgzXd0VMZvWrs1+8JPmy7o9Nf8g/+pKfy5qjiTe8MLer6lK5jv36Abr27aq8frG/XR2bW6zrqtz0gUt/ycZwXLNqV07L47bXx0ygby3GbEOL6IqiMMQoeFFYktB54zueXLzk75//mpeqZSJb/OiDv+ZHX3zBZHcHipomgutj0CSz7kGhxRAjeCeAYUMWGqRvQbg0Pq5/jLdh7uLlP19b4rWfL9d0ixfqKoa8lujqH9DVH1898aYrXQdVr9z1W5SbzrxFTa9F3W/fgu9bblvPrcGfxMi0nlBpRdslmZGQ8prhXINzLcoYQrCs2iXLVaRqLRfzU6L34C2u7Tg5PUZPSlzTEbxGK5PNhw7vLK71xKCZzgrGs5LFicO6wOnLY+wKbNOyWM5pOkcIGkKFbwu6NhIaKNqS3fBTdsdf0awMqwV4A6qrmJTQKKFkxERpbLOiWc1YNhZLi22X2KWjiYpVrZjMasYzR3y5QmmHbTvKoqIQk/2eetOAShOIcxiVIqPHowIRx2wS0SUE3/H02469nQP+b/+Pv+aLz/6MX/7mP1COLJ//9AH/7//Xt/ynv3nByfyU8dQzGZfc9XvsHc24f/c+5+ctz74+4/QESgn81+Pf8OTJBd4Z7h18wuH+I+4fPWY8GrFYRo7PTvlP/+E/04U5bbfEtRYlQlkVTMZj7h8dcHS4z8O7Rzx6+IDJeIQWoRBB6xwAQQAlGF0NfnitaFbtikILpSkHqZHUB71/CmuTpIq4rkOZajAj5nSxKcNKBpZtbAfxZescdR5z3ockRyKC9yllUQyRGDwuT9rBbwZ1hBzMkc5P5lqVxZzdEOBRmGzSzRHFfYaQkBfhPnJYJLOJw+57g6XLxyZGS1K7+uwiPaNFYsJQa3ZsjcrWQOuys/TaJEzMNUpOYYdCwtXj0zmbfnyXvs65oNMitCG63Cd2l/77PnBj3SepLyMxJN/LzcWkB9ObF7vsn9g/71cn1iGKOD9DlTcLfxrlMoPyfaiB348eu+5evm/L3sT4veY6rwC9XreTS/65/fhV6GFTBcmPTxcFpTF0NoIGKRQtkV89/YZfnn7Nib/AmAn3jz7mzz79c46O7jMd3clOUAqJmpJ+c5hY+zT+s68zki0Ur26OLgEvufrN25abwNXtYJJc+Xzj0W8iDK9rCnCtGTte/vBK+rxbXPJNHOb15R3e3c39900t3dyJDB9eHTOvrjDv0rrb3/ntRZ5bh6LEOzg7hqf1Gdp1VLpkfhY4O45U1RK7shhRtAsIHWAhp6Gga+D8GA4nUGhNcC2hU5i6Tv5EQTASmU40dx9MMUXgGS+YVCWzacdUoIoW7QNhFYmNwsSa6Xifozv3aVdL7ow/QT17xOODn3LRCMFP6fyMqY3QzZmvXuDKkhgDug6MtjRVPUPFEvEG5aFWBdulYVZYYuUpInQrx1dfwo8/coxqg+8sq2ZJDBe4uGKkpuxtTZhsL3E+4v0Zs8mEv/7rzzh6uM033/6KZ0+ec2cbdvfg4G5H4woumgv+6q9m/M1/3KXYWRAKx8l5JATDj3e2efz4iF/+53/gF//wnK9/DXvbE8rZI2aTO0wONVufHHJ09xFFMeLL33zNf/mvP+c3v/yG2W7BYtEQushoVLGzt8NsOmFrNuPu0V0+/ugx9w4PmI3GRO/omoZ2uSJqzXQ6oSxKvPM0tsUGi4ikIB1RSFaeMpLy7Ia43j2n9zqBQaUMIbSDLIlsAA+VTYMqs2uRgPNJA9B2HmcdIDjrCd5TVGVqU5aDcd6TCcE0tQvkyBL6qyTCK5vmdZLksdYCfaBHZqe0WoPHfG5vTtb6ylDLqMUPAQ6Xd8Sy8ak3c25KxFwtawHky8xZzIxE71uolMIYkwMyUqR8X2UyHfXgN4PKsGk0WrdQrnzaoOmubd+lRsEl38YQ/MZ9pDqGCWpwOkw+k8g6mGR9PCRx7stkwJ9E+VO6199CeT0wSRsjGY7bHLueqDQSVU4OaYhKsE6IBax0x5cvn/CLky/5Nr7EsM29vU/4y4df8Ojhxxzc22PZeSIFESGE5IoUXdbjUz3wjJA3x1dH4qutvwoO/rH8qZfbTxe3P/L2Is8+EFxSHC9MSwguyTNojfcpijZCDldN63BlSvZmeyyWZ0iEUQl3dhXTWU3gHCGb4nzEeYsQKIxhVI8YT8YEabHO8OW337E89XzxZ3t8cPc+1nZs7xtmO/vs7G6RJDICMZY8e9KgFv8DG5YECSztkq7ruDgdw6rAHtcUh9tUU0cxEmzX0SxXKFNQj8bMwopaLP7UYrtAqeBoHzoH7j5s7VS0p5G2bSA4qrpgPCnY25/yk599ztZd+NVX/8Dxd8+YzCbs7Rr+6q8f8dHzGf/j737Fy2fH/Of//O95/uRXjGaBqvI8f3KBijVbSlieJP+Ts9by7/+35/zNf5jz8smC8egj/s9//QEHuw8oZYopZjgLT5+84O///t+xXC1ZLpa0vmOyY1gtzxlXFaOdKXuzXe7dO+LhgyOODg/Y29lmVI0wSqOzeKhWkel0RIzp786RJswcmeuswzmL9yntmjEFgZQhxYcwgIKEeAI+Bowk82nTAybv8X6d47brunzeGCI4Z9FKs+iWaK2YFJqi0DjJPngkUBNCMoz4kHxlhgk2E1MhM4whZxLR2Z+UmKVSNjmEbFLt3QskHZQWgxyM0vvb9dM25NRwAwm2nrwH7Euk19Xr5WN6MNqzDDEfpnSWcQ3xFZmUwR+PlJaQ6JLZvPdlJCWI39wXXgKaub6B5UTQWqcAmd5fMgPzEMMguZLuPdcb+/NTD/S+gIMfZWroxryT/kg+mz3LmMW0N6OhhwVYsm7inwIiWu/uX0sEvGHtfyM0uBV2eDf28Z1hyW1PvJHi6HstXtvcyyxVOrY/LMplkimEpCCRpLUUiMILBAIuRM66hi+/+oZfhmf4WHL33o/4nz/4gHsPPmQ83kbFAqLQdlBQEjJ0VAI+dKk1fW5Tes3A63wcX/3mSss37u32PX8zGL5ax+uZoquv4m8Fkr7pIt93mriWDL0Kvt+yjmvLld67Hdn6vcrbMJ5vEfBhibFDqYjSEPAUtWJ7e0KUjv0zMMZQ1QW1MoxHKUsDwWIUbG8LsSw42NtmMjFszUq0mSMSIIYs6huJ0bFqF5yfR6K0dF1AqRKMpwuWVrUsuhaHQ4xi5Y95sWh5efaSk5Mz1NaE5dYZp8uXBKVYdQHvHItmQWg6Gt8S5YwuzJkvIqPiJcfPNHpU4f05ZZGEloOBl0t4egqNg7KEP/9Rxf/0v/xP/Md/87f89//8LasmokzEhRYXVoynisOjCTbuMao8L49f0HTCat6ys7XDg4cPaBrLd998h/MNP/3ZjyE6/t3/+g0vnq4IbeS//11gPFJ8/OEWe3c+4ujwHp/eHTEe7zOutzl+cc4v/v43/PrnfweFp6iSqKiKitqMGNUjUJbtccHWaJtPPv6UR48ecWd/j/GoojQKo0GCJzpP5xucjYmdVTqDq4gXv/Y7yxG3kZTmT1QK0ujBWJRkXg2ZPYsh0nUJs7hoccESY5V22SFcymfrvaVtW5zrsNYmGRdncU4jTNBaY22H9+k3JPnspWwsuVFRcvaOACbmCHIGE08CKgmghCwwnACsyhbY9EPYnGhlYxhFBvPnpnyMlt6U1PsMDWfDBgDtQeTmCFU9CJLhK8zAQK6Hr8jlcwdTcA6YiSKXFwZZt3+tu9fn991gBTfEnePwW7x0nbxsre9NUnuMMv2JWdPyyvmxP1/W0ZFE+lDh1N1rMJlAbMqW88de+vje10/QN6O/97f4vs8VaLOuG1r4WlvWFWR24yIZX/NnqiP2wWX0gWB91H2kd01RYsBoBIMSsMHTRsdFt+TZxTFfL15yGjqmfsqPP/kXfPajP0PqKUVdQc5IJZIAXQwgsUCTXSBU717RP8PLN35bEDcw4d/jgd8eCLwN2pIbvrlFNW9dXq3wdZd4E6y9sT/eY7tf3be87gn8FjZcrym31/kTjymgroTpDHZ2RuzubrGzPcX7ObOZQeuColBEt/bLWrUXaBWYjRTeCzoIoYtURUEMFiUBpZPEig8R5wLNynLy4pymfc6zp44727sEveLF2Rl7B57WC62NeNtyVl0kB9ymwbuAKSOhaOjsgno6QasVqBYX5iy6yLIT5sunrNoFMUJZBoxx1JMCdeFwXWR8OOLB/TFoz9bxKTZqPvviR+zOdviLv/yAf/ibX+GD0LVQ10kCRMTTtQ3eGQpxEDqUBGbTLc5PWpQ4nj095dmTUxaLgNGaF89anG2R71q6ecWPP/7n/OyLfQShKgx1PUZLRYiKX/79c7578recvJzjukDTRmpdUJoaYwzROrxtMaK5u7fH559/yN3Du+ztHFCPRhSFTqbCGJL+nndorSiMweiksSNR4ZxlAzMBianywRMCecEO2GBRqJS+Lm4s4iJ45/ExUFZZr08JSqdMGpfX9yQLZLLpN5mWDbooEujJpmGArusoygqtU2SwUgplUk7cPtftoN4XQWtDjIllLEwSUnXOkrKdJW1C1ed6jhBFCC6lFNR6HXCRtCj77BMJdPWgM+S+GcBWf0/93WXw5X1gUzOs39cPBFgmzhIDmL/cAF3DpHVJGiYdp/IBA/G62b0bfyiV/B77dssGe5eAfQ8U123sg00S8wnaZLmX4QYF1TMncQ0Q++atpWgY+hl6ZnbdP73E059EuWlBf19kw3Ur4Hss37+6m2r4vst60stLbHs6fq0E0PswZ9koSeoTKx9og+W70+c8bY459w1B1UwOPuLHDx9xNN1jZ3qX8XRGuJQfu9f/TKPY41Ex5aaPIW1k1kZmNtrab9Aub5Suv6/XvSTv96HeBIpehXn/WG5T3vh03oaee+uLXtlIvabcGvxFH5jV45TAJpDlMhJr4L2jbQPeWS7mF0zLMeOJZjIdIdoTgsVaT7PyNBcrVKgJ1ifhXwmIBLxN11EoohPsyrNYNLx8rjjc17joWCw83gp4jW0dXduyWmqM9ji7Agm4eErrPT602C4Qo8WUPkXcCigjoOaY0vLx58LHHx6xVU+Z7Y3w5pTTU8vh/REPHu9Rj4QP4w5RKj7+4sesTiMSTRY4Xv+ntaT0dzoyGpUo0RS6ZHtywPlJw8/nv6Guxzx/ds7xixYtY7ZGD8FucbS7w/beDhenHZEZZTVlubCcnJzx5a+e8/zbc8qpolk5Vo1FYvJRnExNyrYiHghMZyP2d+5ysLfPw3t3eXTviOl0AvTabEkoOQVM+KGvk29eMjsmAiu9PH3QQv8aDX5bWghZvFtS+hJExWGhFyWECKKEolD4kPpGDUEV6TdjDM5ptMpRtTkLRFWWjOoa7z226wZwpJTOkjAp0rcwBUVRYF0HPmaTbzJRa60oy2Jg65SkdkW1loYhJkDic8Rxb4aNIeBiRCmddQI3Up31YyGzhZIZrl655KrmXm/yHRjBPiCjP176dGrps95gCDfzhQ5wMh932Ty6ZlF7KjHSM4ubbUnALbEgYcibPOgkXq5xiMJOD3+tWcgl83W+/+He1+erQRNwoBeH+5LhGn1/begS/tGXDVBwzVqeekTg8iO+fd+8ER+8r1XnNi3aeG+vvXJ/Yze16eo14nB8T6zFsHF+9Ek3NA2AvEkTRJLMSgCiFpa25cnxc17aBfNgsc4w2trn7v4+s9k+s50j6p09xmVJKRXeAzG5gqTrK0CjohDEE3EgYWCw4/AU4zXPQ4Z/Zbif/H28pieuOf9txsltj33bsfe+2b7hvd+cP6/ulOKr59ym3pv+/iFg7ffpjnd9Bu96H7cGf/PzOQd3CkIXWV3AsrIsFy21aVktHYtF8guz1uMLQQxENNY6VHSsVoGzs0Cz11JpD06jMWufoOxfFGNIAsLOgxdWLSAtMaxoV+CaBaENuCaZCV23JDiPVg1VCZE5iGdrJjjXMBkpxrOK7d2aejRCq5K6Vrw4O2XvcMIXX3yGXTi0iRwvFKoSJttgqkA9Nmzv7FFWO9RGYWPg5Mk5y4sLCJ5CpxfUO4+1jlXTsFpaFnNH0whGJpzPz/Hec3Cwy+7WI6oPjxjVM3ZmuwQb2JrexXWe+fKMFy9e0qxesFp5lquG89MLzp4vGG1pJuMxs8k2RVkQXESLwhQwHo3Y29ni6OiQe4dHHOzvsT2ZMioMRgttl4Wncx7bqCD6zPQJBOdzJGpKSt77fMkwz8nA6mQ5P0JYA96Y2cAkAJ6CEVQQTGFQOscBqxzcoYSIQuu1DhxkQJCBRlkUVGXJYrGga9thqjPGoFUSjnbZPqi0oIMmet+niyWExKqqnKWFYAneo6IMWn6RmANKEguldC+MLIMAdAgRY3TSGxSFiz7v6tmkx4Z72AhlITdl+P/mKB1kVvogEOnBX/aPy4gxxMAgs7fJBCpJm68hunmj+qEtDAvRZkmX7oWe49DuNQJNm8b1ef3evxeijRvn9cBvc3VavytRYnYZ7P38NszO+RTVs6XSw9w/gTLY2i/D7T4nc/7r8m+Rjd38dctX/9xf7cGbsODb9PX3Xt9vYjuvOw4YIqUggTd6yaH0lgxVbSSgTt0a0gZHhhcMouBioPWO02bJ3C05dgvOzi9gdwc92+PO+IiD/Qfs7x4wqaeoosQrGawKSnR+Z/NmKZuWBYOKhiA5reTgxxE3p4jXlFch8Y3Qrh+jv/Xyhmu+JRn56mFyw3v/vstVqHRdT69Frm5XNoHqm657+firv70z+N6sWtiYj99cbg3+Tl8es7zX4R24DmwH7SrSlGCtgqhSTlwpcV5oG8/8rOX8eEVtPMt54Pw80iwCuxOhoKJQVdJfiikwwNkG71u6tsM2ERUFU0SMXqGVxXlw3TGFCIVJ6uejUWBnJszGW8y3PaMJ7OzXqPsFZycXhBDZvbPF4d07bO9uU5QlPkaevHjKwaMdHj46ZHna4n3L+OuCzgtIh7UrlJpQVZqd3QnPvz0h2gnPnnzLV1+ecLF0SICuDYSzjhCXfPWb55SjAudKop9iqil7e7vU5YjdnQPqcobEAkFzenzKi5fPWSye89XX3zJvVswXHW0DWlXU9YR6tM2DD3dRAqXJCwQexDMqxxzc2ePh/bs8fHCPO3f2GI9qyqJABw/OgQgq+iRxIir7XwlRknki5Py4Mfrkn+d9jixN4sh9FolAYve0lswYRbTRGFEE77NcikIrlXJdIhRFkTI2hIjJotwhaJCYmcb+JU7bd6MNUZJ5Umfhb5+23Bit0CZLIkjOpZmDfJQWxDNknhCJEFNghBaSL59zRJX83pTSKBFc9Ano5oVGKQ0KQgZXm/BniB7Ok3tP0AXWPoADkNtYiIeA20uTWw/4rnJlw08DkxaFjawX8RJT2F/z+p1tNrHmgJPNwAwZ6t6UnmHN8glZE21tpt1c+Ab/xgF4XgaKw3LWy+30Nu24bu86SCTLyZD8RuNln4A/ztKjXdZsh8AVMHgdWroGyF97zFUwcbVsnnl15X7dwvH9luZXzfqX3/9LLelx8MZGJGmOps3GGgsnVt8DRqk+0RBRBBs8nbM0tmPRdZzaJceLhi4G4mTCzr2POXz8Gdt3DhiNt9G6xKDRcS1Y70KEKCitk3sLkURShDRnkky9V5fwS7D+bbrtTY/hLYHWb7Xc+Opc/6beXH7om3td/e967Zse3JX64vXfv9NVrwz110HMm8qtwd/F+ZzFYo6IoSyFQpcEV9C1BpGa0hQsF5Zm2aFDiW1hERrOXmpa45ifNyzP4fw4sF230JVUpsoZAkLWDPMQO5z1tKuIKiN72zCpYTYuODhU7O5p9qY1k21HFzr2d0Z8cPce21v7XCwuKCrP9u4Yo8d8+eU3HJ9ecPdwn6N7dynrEY2ztLbl4GibyaxkuZhTjqbUk5rd3W0WF3OIkaIsmU52qIoxk8mMJ+0K5eDrX3/Nr79sebGE7UowwRBag1mUnD2zLI6Ejz75nJ1phzE1Skom4ymrheP42YrlYsVivuLZ8+94/vwli+WS5bylnIwZj6eM6wnjesJoNKYuy6RH5y22W9B0Cybjkjv3jvjg3mM+/OBDDvb3qasS8Ig4RDxEi+saiAqNJIHlGHGdpRdYHhz+FQOr0JuDRekB+EFMKdwkIEpDyOCPHriRJ+mUns17hxLJItwRCWS/Qo1XgRDSZKlUYg8Lo4ZMHlGlaxmdfAQjASRSlEUy24aAKRSmLhLIw6csMiYOgEopiN4RQy/qnNwKVDYHaU0SmFaCd9nUKwlYqexzGDJoJcYMQJPQtN4IiFlL0vW0aA8U+4GZwdUG+Lk8KNNKtfaX3PQXYs2wydWFc4N9vDRr9IAtP8uBDdnYWWenvPTPesJKpugMxi5dq9dLXLMuyaE+gujLDE0Gi2lhDsPzGGCG9BqIanB5kv5FkD5y+fdxVXv/pQd9N/92zfc/yML/eoD5pnOutvU6buWNXMewMdr0lU2buLWLRC8KHulD+oeRImnTGLUAhkW3og2Wc7vitJlztlpy4TqaIlJN9rm385gPH/2Mo/u7WISgNZ4E8lLQmEeiSpvZpGCe5pKoM/uYrBtByO9zv2NRa8h+bTfe1Lfv8YFe9zB+oPLKc3wT8PmdlreHnm8CUJtj+OYRsVleHcBvatX1tb2pha9rw+Vya/C3mFu61iFSIkEhwbBaBIJd4gO0C4drwLYBi2dxBnVlsfMOKTy1DcyisJgHvv72lJEaYcoKU7a0tgMtlCWMKs240sxGmvHMc2fqOdjaZ3fi+Bf/py32trbY3ppC9JwvniEi3Nl5SFVO+ObrDhvOObgzhVDwt4vnfPvkjKpSjEcl48mUWAjON1i/4Pj5MYd3H2AqzeKkY3ke8U2BKab4rubF04Z2oQhdx4snK6qiYNUI9+5NaKRjf1Ly4GCP3f0DDg/u8fGnj9GVsL07xYc5i1XDz//rLzg9WXB6cs6q6SCmdGhFUaCLEaaq2SpB65LReMSoKNESid0S6xd48QiecVVysLvLx598yI9//Gfsbe9TFWMEwfmOrmsJvkMbTaUUo/GYQmsMkdZ1tM4Nkh4yOOgLSEqZlsycZgBcMYsp98EUwZPAmYfoI8Gl9T9vhEkRO+mzBIWKgrUdzjpMYYguZCYwEHzKbCE5kASSr551lpWShCtUxHYWay1VXUNMen9KIsqkNsToiJ4UcKP7TBiKLji61lFqGbKB9KZH7xMo0lrhPYQcOZxMRwnAiop464ZgDyQFkGilsRsCyImN8DniOZuZB0zW/525CpGNKGW1jvbt2Yps4/WhZxc3JokNlq4Xf16Dwst7vl6W5vJ8kJ65UnG4Xm9mTJDM9y3Om4LIps+h5DYQeo+/Hgz2DE3qv94HMLGUJFeDQZdQhrYo0UO9ifhPN6P/BESe+6n50r+bRF//TN+pKy5RzDde/93KzWD1ytUvZ9N4IyWx3hmtX03J85PO8kAK55NAOxsZYyIpMZoTT+Mdq9jw1elznp8c85IlURfsbB3x4N6PObr/EXv7DyikRmKBJQeiDexiDmzK4FI2Bmf0fmOvlTdlVxbZSxuXeOW+37bcfv0ernftdz/AcHq1yt9nOhJ+G+1ad/VtHtq6v37XPXf7DB9oFqdLYhfRAsFFzl8uUNIwmkyILcxfRC5enjG9s0WtYG82YX9rl9mkYHcyp/2oZTLbxphdzk9q/u5Xf4frIrocY72D2DEeV/zo88f85T97zGTmia7FoJifnXN4tI8qBNu2iDJMij18cDjrifGCeXNM050R431KrYgSsD4mCQkFo62aYlJw9pszjk/OQWB7JHTP56zmkfZkC9OVFN2U1bEwDx2zTw5ZHhdsje5y9/4HPLr3F/zrf13yP/72K9oWIim7iHfCd9+u+Ptf/HdevHjKsrWMpjVd41heeEQMo3rCeDKjLscUZUVZGZQ2abH2lq6Z49sF4BATUFqzPZty584hH3/4CR9/+AFbWzsIycwBFu8tXbPEWosp0qLa+WQajQEskc4Hgo+EIOCTVIr0JmB89vXL0bvBD2yhDwFvfdaFizgHBEFTImicj3Qu4G0PaAwKSdIuOX+uMSVVke7RW5dMtekKBAfBWnzliQQ624AE6vEoBebYZNYpjMa5HHEbU8YQ5yxlWSSxZp8YOhEwUiIh+f4praiKAmstGsHGBChjjFRVYlVTBLPHu4AuDKVJSdVb71M0b8gsWvZplCDrqNwQ8T4zYiGACoPJHMkMo6wzicSYAi4uM3B9BO+6xBgIXhJQyv6S/QQRekDuPT2o6lnNQT8ws3+J+cxshjIoMTm4JZm8ez+qEP2asWMtxpxAZ9+yCIPJC1Imlb69MW8aLs8YibPJEckkcD0IgW6W/FyTM+OfSrl56n/btf+6cl0d32+RiRsA9XU1yeV/4+Yyt1ld/x70AVhxvZGQdKKIIWTJoLQxTayxjwFHwEvkwi54Mn/Gb559w9fyEiVj7k0+5Kcf/AsePPqQ2fYOUlSIKvEh581NgwC8DAy99JsQ+rd2fdtsENyXGdjXAOLNPrr+47orNn99Xdj77xhf/b7Cu5vL99vqvPokro7ZN43S7zuKf9hya/BX1xXd8oLgVmzNxsQYWJwvKVTN1qxgb7ZFuzhjf2ePQjSmEx7efcD/8i//FavVKV9/+SVt21FNphg1pb2wQAXSEZ0lhqQjGINiMqq5f3SfyVSzOj/FtStkalFRETqPQjGejonA6ckZy8UZMXoWiyWjyYxvvz5lNfc8f7og2MhybnnxcsHKGgKKL39zwqpTbO0ccP5kiokVW9ND/g9/8TPapcd5D8oRWRFsR+sFo7b49qs5Z6fPWDaOb799welZw7KxzM8bTl8sOL9YYWkhghLNtIP97V12Hk4xpkCLwUiJEYMRhbUr2osW2zWEsESwlAZ29ra5d/eIDx7e58PHH7K3f4eyHFFoTVUY2maB7xxFWVKXhtGsxgVNwCOqwFtoO4cNgkZRFGOUcnS2Q4CiTGr01nbJz4+INmWadxyoQg0TotKCMQnoxcwWFjnvrfd+SFskKjlFW5tYyKlMhkAYpaAokunWiKEwZTKxakVEUZmCQKB1LVprqqKgrkZ4m+ouTJHMQiGbYWNEEdAiVMbQ+Q7nXfL5UYIWRcymYy2KpnUYUnYMLXoQjBZJALUHSd55rNjElElqc4gR23W4zuHEZUCZMo10zvbGp5TtIgpapVydSutL/oCFLoiAs45IRKksoeM9fQ7fBJcYUkINMinDwpiAWRRPhn7ZXE1mNiH69eLbr089yHc+DKnunO9wwSXTlZYhshlyuryNsd/rMiqlICiCd4ScgTRFUqd8pdKbgft8pGoNGGKmuORS2wau5xK7+cddepkfWf+1cd8xA59Xz9rklq41DLMJuNYHx+HXdyqbq+Ab1rLLwCiZUjdvZxMvrfmzPhhoDQW1ksSAZya4I6JIOXXbYHkxv+DJyUuerV7ylFM0hsd3/5z/60f/T+4+/JhRMaJUFZKdUyCN8yLx30P/hBiQGDeieAHU+j299n3cuKdrfv7+zOproP9t2bw3HnMzBJU3Hvf7Wt5/Wy/XeLMB9q0rfDNBf+mUy1d+zSBMO/hbN+f2zJ+KKOPZ2jYghqqsUfsFe/v7PP7oEc+evOBv/+5vefTBIZ89+il7e2O++epXuOWCu/t7fPToPmU1oelgcSFodYr7//4HVs2KajzBiIagCTaBlBChMlMYRaxodvf36bqWp8+f0nVL5ivH+cWc5WKVIrx8ILQTvnt+wjfxnCdfn/F3v3B8+LjkaO8R9+99QKBisfB88eFHbM0OmUy2KMsx0Wt8Z1IO4ZNnfPPVU+YXZ0RtOT19xsXZBcvlivNzS7Nq0XWFVgXWgvcKpUqqUc3haEpV1ShToLVGaU0hoJUGH1Ie5NAQQsQSQXlU6JiVmvF4yuMPHnFwtM/R0V0O7txhazqhLEuUAts1dHaFuCIFOCB0TYN3UI0KyrKgtRHXObxzGF2AKSCqJMIsCm3qZNaUSNO22JjThukUmWu7Dl2XFGWFDx5rE0NUKI1SKcOE1gplNF3wBAGMTuZAlZg1HyPKKMSotKdX/WdHxA+BFcSI1nrIz6u1YjSuk7kZoSwqVtJiO4d3yVSMSAapgeihaz1aCkRV2VcvpGwzMbFUFkBpdFGhTAE+Ym1D13ggpUsrdImPflgIYsg5h5WiNMWwADjrk9ah91jlQCJ1VeWsJSnTiFKS9AXJoC77WgaXoBoCxhSJfcvp4Yzu/Sslm2tliBeOkZylZJ2Oqr+GVuk6MQPw3takh3R5MUU9qgwmQ4QY0nUJRJXAm9ZmYDJ7gBmzb1XPxHgvoDKDmfs0CUMnM5zW62wo0Juze/Ykcqly2ZC4kE3IovgTsPreorwrW/A2yOCW17i2ukvobeMYubq25U/5v8H3dR105ELyQRY2Nh9Rpehagc55vIazZsnTk2d8t3jGE/+SJYGJ3ufx7uf85Sc/4+jhAcV4CmZElwPO4iXgmdweknyVyu1L5mRUfC3Z9jbl7V7f98Hv/mP57ZXfh+f1fttwa/B3uLvH8XdfsgiBw8N7fPDoMcEpnHPYlUURUeKpq5rzizO+/eobtmYj/uKvPmVUVTz57jlffvUrnr6YszV9jNYFLkBZjSh1SWsbQvCIlJydLvjb//YrtiYjmuacbnXBbHdGs+w4Oz2ncwFjCozRFGaC0SUEqM0dJgcf8ujxY54enfAv/0XB0eEh23u7oAzegxwZtK44ebnk5ZMF3zz573z75CXz0wUX5+ecnp/StqtknhQQ8TSrDusDdbXFdG+HsqiIolEYlBQolUL+o4foPUaZFJ0aAl23oFu2CIGy1Izqmul4zHQ8Yms2YWd7xIePP+TevUNG4xlVXUIE7wLeWrrlHF0khqWuK+oiSb00bQNaKIxBq4LgUqo113kKXbAz3WLhhfmyYWFdSuGmFYVJJsyoFQqNRqNMCqAolEoiygjWe3xMrI4qCryLiIGiqiEkU6EuCyR4cDnqVkuSRtEaY0wyW5YKXepk5XERowqM0ljnsDnVWyDJzMQMe1wOxAgh0LWO1aqlrgrCIH2isDb5qTmd3gWtNRGP+ATMvEvPTRtDL/KaTLcJuEQApTCVBp/8OkUpTGFo2pbgQmZETdasC5SmoGsd1iUGtTRl8pcbgJnCGE3XWbz3aA1FUbLqUhCM0lCYBM482Vcx65GFmPo0bAR6iEgykEtKx9YLOvfi1M55nMvALge7qHQgADbagb31MTEnMSTQnbKbrE3EwzKdA1oiYa2hJhBDWrIT+6sQnd8dnaQ1khGvZ/EABHwg4geQlwBtb/5lA6+sTW9/EkUGHPT25RYY783szVWYJlcYg+sWGbn8WTbB1WWJDNlwY1hHnaskm0Kk8x5t0qZn0/gbJOIk0lnLyrb88ruv+dY/55gVozDjcPchf3nv/8jBvYdMZ3co9JjCjJFeNioKRc84xhyCEdetI7sniOiNbpSN9m/e+e3Dj97tuCv9GzePy20SXjGxy2AW3/xS3gMm+EMefL+ttr/S8fnfmzr/8u9vA92ujLbBUjCUGyu6/VVuDf4+//xHxI8/47w9YzzZ5f7dD1kuLKcvTyhVyWyyzaScJbNW7djZ3qXrXvLv/u1/4PjlE7755gnLVWS2c4+f/OQjzlfndKHDYRiJpjAVi6bl9KThy18/Y1SX3D28w8mL59RlSYwFzVLhuoJxNeHO/gGz2ZTRaMRoNE4BFKqkrmtG05IiPmG5aDk/bvj2yTOePHnG8+cvaZqGtms5OV7R+cjx2SnWRSpTE2OSFBmNtygKg3MeEUU10kkLL1agyhwJpjFZNoSYF+AQCD6wXJ1huw4F1CNhOiqYjCbMZmN2drc5uHPIg3t32d/bpa4N48mY8XiMc3nRjWnRVBqMLlGSQEtRGFCaIA6lDRDRpkDpIkkRhLTjdQgr51hZh/Mej0ObzNDolGw8uLTwxwyICJ6AAtHZlJhYOkEQrVHoZFJUKXOGUhpTlGgXCOIRY1IkcVGgNakunc71koBGHxsSwjqwAQSXGUObtfdEkjSNVoYYLc451GgEkpi34AOgUUonIW8i2qgk8KpVMhsR6UJAO4+zllJFqqpCA53tsDGQolYSE6CMyQBRY0xBZzt8ZzEhoHXKCFIWyXDU2ZbgA9a0KdVdYRCXJ+3AOjgj59ItyyKLS6d3pDcrS9Y+1JJkKmwIkCOuoZeUybyY9Kb4bI5HEUip8kLPrGRUISKEXmePFPUcVIq8zo84gTZSBHXwHk8YCJr0DFQ6NtcnklLrJde8LH6TEHR677OjfDJBJ7oxhkCMAr35Ny9kfoCaqb0xy/+44d7/uMsm8Bs+S//35Vww62+v1nBz3Zeuc8NxiRDbAA1xDYVSUcOR6+/6GuPlRseQBU/8APz6iPIeuyit8T7Vpo3BRxAlOKCNlovlBc8vXvJ8dcxpXNHEQOl2uHv45/zs4UfcuXNAUY0x9RhVVIgYJOZUa1EgKPAKo4TQ6/9JsgBc9TO9LJKeF+ZNHPyGXcjbQo3XRmpfg+VeOe/q1QcWfV3H1To3D7/03R8yxnul/OHcTL9R+v4tHpyBLtX9Ltj/1uCvKqdUWyXzJw2nJ3Oq4pxCj9C6Tlp3KJwVTk8uGJXbXCwu+PrLX/LkyS+zdEfJncM7HN37lP2DI/7ul88Qregay7JZ4pzDWYcvFKPRDof7jzm6c8jB3gdMRjWzrT201ChdUOoxo1Gd9OBEcN4zny9ZLTvmi+d8+91XfPvtU05ennJ+ek4bW5quoWlWONsRQkSrimo8pa6mjKc1tRlhdJ2YHq0J3hNDm0U9FUZ08mtCEUNHDB7rO7y1+djkq1KYglEFu1szxnXFnb1t7uzvsL+/x87ONrPZhMl0yvbWLKUdCy0BsK7D2QRkTE671vt04dOiGoIkM2MIA4jqbMDF5EcW0ZhKE4PQOE/TNDgX0EZRFCabEcMgrhsi5Oy02OyLZowZfMxEK4zWaKNwvp/2A9Y5qqqkT1oeRbIcXXL6D0AXAjbnARafzI0+eEw0BMA5hw8hmxUd1lqscxhjkulYK1ShQSl8BJTKQcV92wVTFHjnEhMZwtA3pqhSfl+lUiCRTrp+hQK0EGxiSUMiBlAq+Rla72htl0Cs0tkcnJfBHGWrtUBHSmEXPEYptAghR/MmX0YNGLx3hODQOfgmhIALbvCf06qXdQloFEGlfMkhi1jHvoGSgjQGrkKSK4GQruWRgTHsIxajSEJ5mVZRIohJ0ZMpmtdn3z8/sJwJ6KnMpKyX/TUrpwb2cVC46NnDgVyJINkfUJECYobsHj1Y6JFOTBuW5Hw5SA390Zdhppbr/nkz0HsdKXfDKa9rx6uHr2NZe5B/2eOwZ/g2GxOSX14MA6MXSRscJAVpBAVd9he9WDWczy946c5ZxBbrIzFqynqfD47uMt7aYWvrHtNqm9lkl6Is80Yhvy8oetWCxJx7YvSoYNYtHfqpZ5rffvW9mf9bj83NSl9b/Xu1Gr4Ptu/WV3rr8n6x5vsDeT9Ml71ps/B2CPym8XjrcsubvL3Z9/CIeqz59ZdfcXa6RMspe7sFoDg5PWPZrjg+Pk9A0AdOz85YLDrGoy3GkxnjyYytnUN2tg6Yz1tePD/DdWkAOWdps2+XdwXj0S4PH3zMJx99yPbujOAE20GzCixXDSdnC3756285Ozvl/OKC+WLFYrHk/HzBxXzO6fkZq6ZJjvptRJeKejKiKCYUeoQoRaVLpChzvtgaHQ0xs1vep6iwSkuKnvSe2HlsaME7nFsSQu/DBlVRMZtO2d3bZm//iDv7e+zv7lFXJeNRzXg0ZjIZU1Vl9o8StNIohBAVQkAilGUJJBApJB8wn5kuEQg5WjZkfzmdI1M76xLzIpFCFbjg6bqOyP+/vT9bkiTL9j6h3x51MDMfIjKrsqrOqfN9zdcItIAI0x3DDdzyHiDC88AL8AZctsAj9B0g3c2HCNLQ51RVZka4u5kOe+RibTX3iPCI8BgyKyvT15FT6eFupqampqZ76X/9hxZ15jzOO9lWyk3520jWSt5jrk0MUIs0UqWchQi5VknVUIpSJBINpYkpE1NEBAyCJOVaqamiUyDViqmQi4wbz24wtXkXNwuHEBLaGIx3oA2pVpxS8m+jOC0zQ+gpShYPqCijMc6TqWTqefulcOYaWi+JNMooUYXnbcwpiGRV+qx8znU71gnnu+YFKDwhVbdmNaERM2vlrGA0pVJV2WzyqLVglRMOZS2CkmIaD0lsVbTS1Noao7Z4WKMxVRY14USW5psn35GyIS0FjNWSqtIC5mU6u8XT1XNvVRroJgksDa1rKOymOjbNjqacuVjb8P1eAbntg+I+Cq6229hthCvRgaqN1moblZtmHt2Qojeanu0cFINwawxVuadejv5x6xG05l1k7fFL/cM7/I8uBh9o7EBGolW9iS692ZM2bmaR6YekWjwcoXIe0yst6L3Z8rDbNuTTLaRcmMPC3XLi1XTLLSemlKlLBTcw7L/h+vAN+8ML9heXjJcv8eMO24+oorHbedGAPDmTxYtPTs0NL90Q6Afv/Ixkb2/y4RHZvrRPPajvHsf3LrTvbOsDTeTTNvBlpT74z69eH9/+3we1ew/4+rO+3td47P0z6juXjKdu58nNnzUdMazc3UxMx0DvF64uKlpbjseZu+mEsQPDcIFRjovLa3rf8U9/+AP73Y6+3zHsLrHugjUPXBx+R9ftSSlilSbpFds7drsR53piqkxzJNUjr76/5S9/ueH7v97w6vUrbl6/4ofXP3Jze8NpmghroirIWXQVrnOgNd3+wLDzWGvohh7rnJD5q8IqBGGqhhwLJSeoERqqYoyGmslxJadILlHsNWrC6MwwWvb7kcuLPddX1/zu22/59nff8s233/HixTcc9heUZnmilWnWKiCEZ1lojbYyslPicWacF9StCIlfUrza6ERJNBsYQbWMQRsrKGBpVh1KU6qYJeecRBjQ+Hd6Ex/ocjYT3mxJaqnSfEkIb4sVq+fRUK3IQm4ad84YlDWCeiLImbai9t2WCBRtNN0Ud1qjnQVjqUr85FBV0kaUpu97FJnU7GWqMWjnKApuj3f0Q89uvwMbMZIdKPtRjBwztfF+OCOFnFEsQbxSbl6DDa2TXTVgtLxulQUx162Z0+fjk3ORMW6teOcEtaWSYpTRuRKiOlX4mqVZWeSUhT5glKidt2PeVshaiozA2DwV5ftWN3uU7WxpKEuzQBOEpf1e+IL3Zs1VKxmFKUHiclUt9L5dK2rLeW6XCmMMqmwqx3tvvrohO5t6t/2fqi27V4HWrm1re46s0FZZtDYkve3w5hDYEMqyocSCFCmtaCf4b6DU+V7+/QvP42iSevsXj9UHVrM3cjW2x9XtL7KK1O2VzqPd9sk15KzZrzf+Z7sBUJAatWPNiVAyU1g5xZlTmTnGhXkNZBR55/CXL7joLrkcv+Ow/4bd4QXDeMB2FqUtRYv5skZudExr9OT/78/Hcz60El7h+ebuvQfh3d+rR3/78Xr6Yv2hR35ZE/TBG4K/T3/1Vr3VbL/zu79/vb03n9sMfm7D+/c6Gk9u/v76txtOx1tOdwGNw/uBYTyg0XR+R8Xz8j/7A3/+5/+Ene/5n/1PL7FGc31xhbU9zvegPMdTZgmWP//7hYvLl8TXt1htickwDI79YeDuOPNf/Bf/D/7L//L/zek08er71/zwwx03ryeWdRa7D6NkvdEKYxzWe3bdiPM9feepRtN3Hbr6FkwvPLOcCqqAIYvpbcmQk3wAtaBVxWqxzyg1kdMMgO8U++GCw2Hk+nLP5dWel1cXvLi65OryiovLS3a7HV0/ipWJNsyzcMKM1g014Tzu25qFbNqduAKU8LhKVWc0TTktH5PiTFQWtKqhMhURNZzjuQwGfW6GjDWt4ZCm0BpL1TJikYZRBAfedueVpRKFpqVEnWrdffRarQVtLcY5VC5Y55r4xjaRQ2tsbYcxUV5DW7QCq8BajyrSDNZSZVvWY11PUYkUItUoqnZi5qw107pwWlfGiwPaObzvCetKLFkaJyVIHlWQuFiKnBtNRFJpiGNM5zGR1q2ptlp8+pt62RhHrqIa1lrjjFjUlIZqlCzNVW0/C3pXG1onyOiSI6WmFjJfie0Gx2grNjStH1KNb1iaCndb2OQ/G4K32U7I77aGSWx2HlimNANneUxpz5Xzo5ybOESZvambURgtSl1phmtrS6Th3BTCcmaptglpNDfVskafo6+2Uq0RlHsA1RJTGnq5+RrWLUdaeKrlQa70r7vuza3va7sR2ZqzB0fzjW7v047P44vKA5jgjebvviXdzheFUCdKy2cWdHhD9+pZpFRTYiUxzRNTWTmllTlGpjkwp0TsFLr3jNff8PLyWy5efMfl9e/Yjwe67gDVyU3u+e3p5ida0RihPNS32uW63ck8PHYPD2w9H9cPtl4PkMEva03e+owe+6ge2Zf68LHqjf988it//Jc/RX2odX57Jx5BS39ZfSDweTcDX/dtPG0PHr0qvH9w8E49ufkL2ZPywO9//58wHi747rs/8c3L35FjpeaRQuVPf/oDf/zjd5hi2F0MVDLTcWVdIBXNNAX+7S93fP/qln/961/IBXLUGGehOKiOeSr81//Vf8v/8//+H/Gd8KRsNSjt8N2Ow9ijtJERaUMMjLUNzdFo7WQttApTRZghql0l2V5WoatG5ZVUIiEvEhmmIeVAKREA5y1d57m+7uj6nsPVBX/67t/xz//0J16+uGboPM4anNFyd9rGXSEEwhJwxhJTECsPp0TZiYJSzsrW0JqVXArO2XuuP1uqQvu5jWOVKuSUhL5cYcu6VPreamMbvwoyJo2ht7JthVi+SJ6vPK9ScNbReU9ovL/cuHfWWpyTEWbOWbz62lhHK9O4bdJgGq3luHu5SMvIVBAB5zoUVTwDjYVSMcmxeZcpqxF7OiMNua6gDdZr/NBTrSZTBJ1DEL+iKikGGbNaK4rTWskIQmcbUrkhaqlkUkqYluKxiWjQihBkpKuUwnhPSZmkClYpqpXmWFVDKRmtZV9TTqSU8J14AeaS5BgYoTGIDiJhjaa/6CXBJGdiah+y3lS8DUHbros1g6pkLfypzRpDay1qXmsopZJyAjYT6LaP7YZgG+MLXcC0vwsxXvQX6gFCos6cQBFqtAW1IlzWum27iTharqkYYGeqUWKBs0mx21g4Iv6OKIgxtkZSmmQFZzGKaqN1KvdZyL/6ulfHvtsI8kiTcO7O3vjnh+ox5EudUdyGvm4j0q0DadvfdEabrydKREUSaSafYyriubfkwF2YOcYTr25e84qJ5EAZy9BfcDn+mRcv/8Q33/6Jy5dXWOfBeFAWqiI3C5at8ZR0D4UqTRWuJVptO2YKzcNe6a1df/MgPqHOa2Xdrrdfcxl/Ahb0U93vvNFQPoa+vbEX73niW499847kked/xpv5ZYKBH0QDf2G7+tn15Obvf/A/+l9hihDefdfT9zvAcJoWDi//RdSKYeXHm8Tx+Irb//o133//PX/5b7/nb3+943haiTETYuL2eOJuWjieZi5eXELR6M7jnMFpWWi1NuwOjsv9AV0McS1o36GMlgZAy9jSGFGxKi0oRimZZZolPUJV0JEznqFpHDeIYWaaJpaw4LzhsBvZ73qG4YLDYeDli2t+/+23vHj5gmHccXF5Rd/vcNZSkqxUNSfWGMkpEWMAZBH3XUe1mePpxHI6NQFBWwnbxdNaITBvQhfXeXKqLdZLUD/ayNEYQy4JqiLGeDbW1VpyejdD4VorrvPUUgkhUqo0bMIl3JIm7hdqYw0pJYZhxHsvi3TjmmmtKSWRUjyr31yV5tY5xxLEzNoaQY1qUzrr5geXU2x2LxpnzNm0eLt/l9QQGc32wyBIolO4UOhLEoSyFvaHPS9iYBhGdrvduVmlFnLKZz9FbWTks66RWjPOerrBt8ZOxpLVNQVzEzRoI1FRoQZUkfQLYzxKF8imoYPCYzTOyY0OBmUtxrb0lIqghNKhSf4v0kct80ynm2pZ1WYx00x+qwgzVG2jYONRtZDLKs1ta+JqFd8y1TpEZRpCorUIntrY1DrbDJyrCFViYOPqGWuxRqOLIicoRbwWBUXOcl7UjReoW1MgCy9VRvG1pHbegTVWRuMKcqMc0Mbosi2gSrPrvRcTaN1a/bYY6ebvVlFtYiyil99eqUeWWfXmj/W+QXn4u/tf1XefWe8fvN1E3nMK5IFay7VQNd/NUuXmpoAIK+R2UQRaJRFLYk2R07pwu0zcTCemMPPaTFSleNF9x++/+U+5/v0/c315zf5wRdfv0cqhMO1GosX9tSzdzfpnQ/W25Bhhsm5Z4g8RsvMt49tH6t1/P0DS3l68H21TPgE1+XipR3560sO/1gN/pu08bP0+twX/svo55gU/xfv42DY/BVN9ynPeric3fy+++T173xHWhb/88Jp/++GvvL654//z3/w3/Ff/r//INAdSXvnbv/5/+eGH71lXyUZ1rhPz5qKx1tEPYxtrWl6++D3D7iDQPpDyitUKowspL6Q0k4twkq6uLzGuJ6Xc0Ks2+srC4XPeoqmsoXDoR9ASul1KRjVELi4L1ILRikLB2MKffvd7vv3dFf/y53/Hv/zLn7m+umQ39FilJUtWO3KSSLA4TyQjhtJxicR1YTktLNNCzhnrJRvXKAu6sBxXULDbD0KKTpJja72j70VYcbo74bwX65Cc7jl4WkjwuRSMNRJRpo3w1troteu6toCXBzwyQVrWEEgxs9uNYuURMzElckvkMEaO4RoicZ/IObFMgVILRjc0FlG1osRPUGlFzoV+6DgdZ0SZbLHtsSlJdm3JFd85whpJpUjTVirWGow1lJSJIZxTQUxniCljvCCUKQvy6LzldDzxr/+/f6XvHWM/ENYFpRVhjeRc6DpP33XnNI5lXoGKHq0IiLI0J8ZYaCNcpeRGwXeOkgtWezmGLUrNOEdOGaVqQ73k9/O8nHlw2hisEmWhqYKqKK1AGbwbKbXQqV4UrKZDa7DanxvrWiFG4QMWjDSVzc9vXUPzcBywTm4CSsmknMhFYZ1nGEeJvCtZRC+xNZelNGWwFcqBscRcSQWUkuY/N/GO0Q1RzZlQCk5lMctuYzaJpKskwflwGHRRVCOqZKPFWHsTCW2xdaVKTnMIgT0GZRRWu/PNl3qryasUGbl3T74c/ePWGR1VbyyW7wedHsEg3uzhzj8D7SbrvtFTitZAbcki7VkN1dvQNKUNumwj3pYAVCO3pxPHNPPqdMOr+ZbbfOSoZiIFr/b8Yfwn/sf/8r/km2++ZffiJf1hT1ISxSaj3AeNmrjNS8oGLZmmoZHy44aHIrzgh8fnLVPEpy3G76ok1fkYPNzOg0d9cQP4icrMn6CreGOTb0+h3+MJeEag37D82Vrytzb22Os88u9P29HPr/dt5v1N0Mde+APo5xfULw0xfPLV9v/0f/w/s85Hbl6/5tXNK25uj8zrwpqiLC4Rht1I3xlsf8HF1Y6u6/CuI+famg6J80IprPUYxLhXVYcxilIcMa2EsAritUVlaSOJE/Od2GCUglGF0TdrEyDGiWlZSCkzl0yMq6ASFaw2DEPHNy+u+OMfvuNP//QHvnnxgqvLl3zzu2/59vcvKCUTQ2Jd1ibskMzb5XTHjz/8yLyeUBqGbmToB7xzHMaOb672GKPJtRBjopSMMaJOc1bUvTnns9JUotAUIE3kt998g/eC1m1iC7iP1Np4+qalMOQsTdxuNzZ0DuGWoYhJuIu1FuZ1xTrbLEG457C135VaWNdVFLFFEKiUsvjlIQ1KzmL42w89yzyjlGK325Fy4vbmDhCFsu88uRROpxNUMTb2vefm9WuoinG3I8Uo6uRSiDEyDh3WWXzneX17hzKwu9jjrGdZFsIaMcY0Q+wdP/74mu//+iO+94R1keSPNor2nUdrTQiBNaxobfDdnYxjkwgL5ikAkstrjMb5xlOMiRgi2mhClP9umcEbH9IYi7WG2tC80rKTu76XEWsqZ36l1poYEiFEOf87T14l8k5tI2e9xW6AUg7tnby2tFRo21HJrGtiBawzKCeKZKrEzTnnRUjSzhVtDFW3NJEsCJsxjffpjYz6S1NJm3skRWuL0xXbWgOtDU6LkrI2b8JKpWRBWiulCXYUWUnCB9WAkgawUkAbnLdNve0oVNZaheuJ3LxsPcx2adXW4s1voPkDQN0vwh9aEdQb/2k/bcrWB4tSfbhMtRjGc4MtQptURI3bZEIUVUWcZSC25j3WwpoTp/WGH29vuZlu+Uv5kRMBy8BF9zt+//v/wH/2u+94+fL37K9eYnWHsWN7cU1RpjVy8v4Mrenj/lyqZO59Ih+2S09Byz68hJ59EtWbj3y4jG+PeRw7/LL6ey/wn4IIbb97KgL15jH8CvVVkdbH6/Pat6/b9G1b/JxHfv09ua8nX23/8//r/42L/Q5rNQmoytH1HaNzdF2PsQaUmCTXkqSJ0ZaqDEpltCrkmgltpEdJqFqoRZFTICuNt5bedXTWkXNHKQldC3oTZ9SM66CkQI6B43El59gWMkXVGtDs9wdeXH/H1YuX/Mu//zPffPOSi/0FY9fT955x7Ck5o5RhDYF/+7e/YJQW3lctlJQorZFyxvLd716Q8wUVaaC0shhj6byXRIwqRrnebTyzxqVzTvh6IUIJbURtyVkWcOsc4zASY2RZFpRSTUBhRe1bixgZx0hsBHlrZeUW5EsixIwxoGQkHGOi7+X9pdpGmg25ctrdx5EpQ4yJGMUPcBi6Nh4Un78YEjFJAyZ5uBrfdXjnOR1PeO8Zx11rQAvUxNCP50aMWtmNe3xD5aRRUeQU8c5LbJ2Rsa87zewPl1x+8wKFZTqeWO2Cs5rOO64PV/zbv/0bv3/5LX/+l3/ixx9/IMSE80100FI7YhKRhfOONSYZgWZFjCsgSFZpxn1aa2qBdVkaulnPY3BjNLmUs9VLSplcEDQrF6YQ8NqAsShdMU4891IRQ+kcEjFl5jXge0ffeVLOpCCNZo4FVTXGOJzfmk0gwXpaUboIkm0VKUdSEUVxzhJL13WiNI5rQaHvDZb1/ThV/NaEoG/0hrSodqNA+8xqow7I+VxqxhlF5zwgYhbxBVQoY2U83WyKlBaEetuOURutoI2qlQLtQYlnpqLZEzXUUDwnxbQbJei+1eYrXtp+mSVGOpuqW73hlby1JfdzXnjI3duaqtrEDpqGiKnNtomzPdBGWdgUufpMKpU2LKnKnBamvHA3HXl1vOGH+TU3HJlJOHZcqt/z7//lf8Ef//gn9hcv6IYR4zuU8yhtyQ3Jy+3cohqI8i62FJrtXcF2xyFWTed3dH57j7Qm9a1/P/rz/ZE7//wYjLodUvXu49+pRzqcr9bsPNzYY6v6Ow3/e3fpk2sDWh95Ofn5A13Gu+35V6q35/FfsT62zx9uqr7iznxG9/ZBY/DtD+0a8r5z5mP15Obvn/78Z2qUhkYuSZpSZPHWZlOAqbNqcbPdWJbQkg4qKDEwFnW+mOQ65UBnLAZdclOqZkoNlCgCjBhXptOJECKqFIyp9J0sqtcXF7y4vubF9Ut+/8ffc339DS9ffsM4jri+w9omLFBKUixyZDkdCXHFu45lWYCCtRprNEPnUM6SYm0WAy0vtVpqzaS0jcwUMQXWIFdXbQyuJUOs6yqLaZEFLoTQFjhLSsIRdE72P6bAqx9fYYyl7zu0UmJcjFzEYkyEZcX3HcPoyamQUqaWLGhpzoQkIhVlDJ131JKZjxO+9/RDD0gub2hImXNeGu4UCWvAeS8X71JZ5hmtFOsaCCHivWOxIpDIKXA6Fe5ub8+omHWi8l2mmbgmyn7AOk+OiXWeqTlTfLOwqZDPvMKM0YIEpzUAheUoudFxDtQUQHtqFnRuPc0cj7fc3dyxLgsxJqbjSRpko1tclIxT022SbN5uQCuLM2I7crw7kmumHwfGcaQk8RPs+oF1ndGm8f7ae+o6jwKWaRV0uQS6YWgja4lyK7ngOy+K6Jax653j7vZITpVUIrvDReNFKkrKLV9ZohFNU1APw0BcpXFMMYgxd2dRRlHIZJVBQ4qS2xzmBec987SwrkvjK4oAJ8VITEEaWi03OBv6XHKSnOIkuc2lihCnFhk3uxbLR0v8OGsst/FR45HC5uvWLGLaeLwCtW3bWIXrO6i07726zyY27ZxqDbO1Wri0v/bSD0bedbMquUft3uDlAfefgDo3MNLkCXSqlaitlaJxoZu6XdFi9wq5ZHItrDkyrQs/HO+YlhM/5jvu6olcNb254PLyz3z3u+847F9y/e13DMMFrttjlNwUggiGmp2jcFG1acKi7ZwRTE2xUVFqe6ymKvHM3LrUdzGOt396u94dND7+2E9dBt9jDfNTNSZnxPP+fz+21v+9EUWpX8ZefHp9+Cx5WhP4+demT0X9noZIvwmbvmsk/fF6cvNXqkObIl/sdgWoWfzkxG7FSGSXkubEGoN1ht71zdoEasmShdrQGl3FiCOrTEkL07owHSfWZWlKw4LRFWsU+3HPn/47/8Lh4sCf/vgd11dXXB4O7Pqezju89Shr8F0naJSSC+GGLqQtszQXVM54q2AN9Na0bFh5HasVKSVKTtQMxShyTKDE1mTsJUtyCSspJbE5ac3NsiyEIHmyruuopbKu0jh4J6PEHJNwoJwl5szp7og2mn7oZPz7YCHNKRHCiuscw9CTYiSl1pRoRViDJFpYi/OenDNhDcK1uti3phzCGrDWipeeUqyr7Gff9S35Q1Ac31nQlbAE8V+0mt1uwBjDNJ0k/URnSsn0Q49vo9M1Ny6jMzhr6TvPXBPGSJPcOTk3ci6ixq0Zox3GanISVNc5Q+c8CkO1CV0FaTIarFY4Y6EWdFONWmPJIQqKVMRYue8HUs6cjgFVNMpXUphJIWONwyqDqpWyJpYyU7IYeMc6sy4LNVdcbzFRFLV5Xam5sK6hpYFkVNVQKikHliyCkxoSaRvvx9hQwIRShpAieRFRhTUSB1iL8CILtESVxHS8k/FzbXpGrVCzRltp/uKWDGIdVhdSDGIzg6bvR+Er2iZSyYIEGqtx3lOreCeKJyRn1InNGxFJFRGLvtq+J2J+XUohxkCMiVwym61ajFFeQxlSFrqDTKVlqJdDJIVEbp6DSy3n4yPf7QfUhkazON0c+Z8/+dL1j1mC1Il6dat7CYNc0Gs7d+5/V99oEZSqTd0OSlssVgRhCJc3UZhjYIorp3Xidr7jbj1xl48c68xaKkMZub76J/549Q37i2/YHa7p9xe4YY91PSiHtu48VSkPrvuC3Cq22D/TUD0FVLU1gA+RJGlC1TnhpR2Lx4/Q/f9+cP37xEbkPQ9/8la+0rzzl93c/Vrrw138w9/Udx77ZTekb5zHH93Ug4buoyeE+uKT5snNX86SFJBVkdSAqshtIdn0arm08ZlWWKNQtUAWFMYqRa6BsE6EGGVUlSsxBsIaqTlhjHCQLncj19dXfPPymm+/ecHvfv8N15eX7Pe786jUWiuoXOMQ1QLLslBUJpWM6zpQlhQEeTTa4J0RlCNnjAZVxXNNA3Feic3ywhiNsx40xJCoJdMPHc53lCxNVwX6rpfs2iY+UVoxjqOMdktmXVcZl/ZNkJAFTXFaY52TUa/Wwn8zVviCWpSrMsKNOC/vt1KJMaKawln4hQVrbFNUKtZV0jKc9+z3I2sIrPOKQjdfNjEdjlFGsNY5SFByJaXEZpha4WzzsqGVKWV83xHWVUQWfS9eiG3hV00Jui1WJcmY07ZIsZJS441JE5HbKLA2/hy1UlJEaxkhlxKJIVOSJseAUoV1XljjynQ6opr5q1JQcyVpTW7jclUqtSZMs7PJJaCNwhtFKgVVKq7ZnmRVUTnSayQCThlKDFgjUErJFafkJkcbhUmiojVoUotlM7VAEr+zGiM5i/jDOI1DUda1iWGFC5pjEgRZKwyGSiWs0iBqI6PPkgpkSElMp2PzQTEuE1CtqY/EkFof15q4ljGttKLrnGSqJmkWtTXnZBWtRCVtrIxiS04oZaGKSMv6ph6ula52gDR2xhoxti615RLL57sN+PS5mZSbBGVp42GxuJEbmHQeD9cqj80psF5ff9HF7B+hZIxuODP3NvIl6mxdp7AyyW0In24myueYPVXa+VTl2NXCkldCydzdzNwtR16lO27LxFoyJSu82rF78SdeXF4zHq4Zu0t240uG7oB3I9p5lJHscDZFbqMUSOBMMz6/39u2mDX+3sM1a5s/PzASb+/+wZF4/zDu/SPZT1/tni6/+HKE5+FWnuuXWp9CNPx5JxEfOusf/ea894bkae/vyc2f1pJ8IaQRSTpoznJtkpHRKlGqXAxSjKQonDytKppCiCvLfEJrTTd0XF28YDe+4OLikt3Ysxt7hn5g7Ef2uz1XF3t2u5GLiz2dd7JtJYpTakXV9noNSVE1UVIVrlMBasRT0BaMrRgjxrmFLN7OOVJoC3Au6KZM3GKDShZ6tO8HjLWEkIipoIzCO+G2nSPOasV5jzUyEsspo7TBNQ5fTs1/zjqMMaSGjHjvca1p0Uo4g1ukm7HS2JVaCKsgSs5ZQdGKoE7Ce5NmMaeM85ZxHKRRTy06y1mcddTSmjzAeY/RhlhkJKhbwxviImM6KyKHnJKM5RpqVXLFO49RpgkARAygGn/MGLFnKDmjlSB2mw1MrVuEWPMCVFpQUGtRVUyYM4mSREldSj3z+pwzrMvEus5yfHM6iydqKWjpIlAVnFaSVZxkjJl0xSCvb1TFKLBIhF2tknU89J55Tqiam59fU/JqGf1nEMRRSbyed4ZiHClHapLz0mtB2tZlJaZA5y3WGea4ism1toJ2bXMzVbBatQziQiXTW+FvhiA8UhpC5rRCG4fSwtUsbb/FrqOcBR2lJZFoowhnQllD6FMU8+8Ho0fV7IJykuav1oI2Dqs377/mxdeSOlRQrEHORW91uya0MaA2WNNERprmwakF9W/70nXlvompLY6vVnIJb45Ef6VlzJZN3Tq985RXfhDf+sYL5F6BXqnNnqeSqM12JXA83nG73nFT7ziVwBIyJRq063H733PYHRjGS3b9Jbura3aHa3y3x9kepbz4QGKg6jNaJ57Och2UMW0T6TyohwuQ9K/3K9Fjw6i3n/juAPfzl9qvM/p94qt/Su/wOfvxCPXx117ntvsroatfUm+igF++jb9PPW3Pn9z81RRRZvuKFsl6bBSVXColJ3KO5CzjIBkLJxkFOsfQ93h/QOtv2e/3XFwd+O73f+LF9TXXV9fsRhkjemvbvhdcsyMxRhFXMWU2zW5EK1mYtWr+ZBW6vmVA6qZsLAlnLVq8fCk5NgUrEihfACOLUec8RlswkKtwoqjNoNholjW3yDSDdQ6tDSFEQpRIOEHWtIgDcgI0fUNeYohtjGqbN5vk8W6WLttJbxtiGINw/rbGbplFlDAMPcZYQhCOn2nbSymxroFa6/kxYktSREzQ0KQYo4wWtW4j5kIMAarCtWY250wKsq+Ke86OsYa4isegVpoUgohb2ng6p4R2YrBdkljHCI/SoERwi1FakjIQLpptvEirrfiIhSQIYJF9ANCdxPP1vefVj69Z5onOedYSxGTbOmKKDf1tCKTsBHFZUJ1DKxlj5oY2KWuhFFKQkbz1jt45wrqN0jNeC5e1NGqCMpq4RkoIWO8wqqJUJVWx0UGp1sg7qhPLnxSCePBJQAslBbSzeGewWsl5khMKTefMOdVAmlRZgLekj83PEjQYQ8pyTgsnUKNqFbUvgmgqJSPGs6chophXWkkDUlvz2/KMa5Yc4lIzmZWiBFVtDunSBCLnx5oSGkW0LTmm+bUZc88NK1k+dOutWOYYGfErFNaKj+fGKdTONPuhX7/gQ2nVkk5Ky6nd+gk5xveHW244qqokRCi3hpUlzBzjxG2emNLCPAViKJSdg26Hv7xktJccdt9wOLxkf7hiGPfYrkMbj7KSYa6raRxVaeqpCn0ex26NqcSrNadJzmPnWh9ZXt5mKj19CVRv/ffDj3rzd3+PhfZnfc3H1vFP2IEvH1x+nXrKLj/lMZ/3Xh4ehY+9ysfO4K91NB9rM9+/b4/+5eca+8blRFYi3Ki1jVprS6uoNC5LxRmN95bduGccJfv28nDB9dU111dXHC72jLsd/dAzDjuM0ZTmx0aVcN6cIqUkilHoWmSxi80DT4EzwgXTzTeMZhKqzX36QcoVg0JXS8liZVCSIA1GiwDA94aqC1oZjBL0LZVMyqKStM6hjWNZF1ISBWY/9FRgbWNZEGsTYy0xRtlPrei7AWsNIcYmADGSRFIhxEApIlSgylhMaUlKSClReegFFxuKqEBrUkyCqjWEKKUkY/RSpdFV0mxK1Jq+bzaDKIpzSvhNyLCshDUyDAO27X9YQ/MUvDemVkpGtiEEhn4QDmWS8bJqKuMUBYnLObVjnZtqWXKIz+bH971EW/zlRqGkQlQR3xTSGhFx9L5Doeg6T4yJZV44HA7kJpaQZBSxVkkofC+Cl5orqQRKzs1iRppSAJoiu7TjuqGU1hjWeaXkCJ0XRLg2TqExrGVlnk5cdVfEuMrYuQqSl3Mmsgra6Qza9tzd3QGZYdxDrYRlQVGwXY91lqQqy7JSCwzDiNGOsCxi5G0tzhkKVdI6aqYkwak759sNUSAnGeeapr6tpbRGrqDtvcF2yUk4tqp9p8gtyUFTszR6JUvzp40VU2Z1H+NFaXhtSvRVRr4WSTHJzYhXq0ItkZILKclokmRIpWCtCE9KLtCEIkYJUmycRmvJCf+1V2nXz/rGDFVQNq1lfFpKIZXIWlaWtDLlwCnN3K0njqcTp7gQXEV3jn5/xbX7lqtv/5n94Rt2l9/g+xHjvQiYqpXRPHJNK1ki0xRaaDnQduKeV7iZg3NuAuVR96SQD687nzpofWo9aD8/6/mf/mpvV7sh+uBjfppX/rm29wsA3z7IDng/WeAfrz7nRum99Rkf3NM5f2Em1dT894TIb61mHB1D17PrBw6HA9cvr3n5zRXffvsdl5dX+IaqiBVAC5FviEZcF+7WtdmqKKyRC01JwvmyWuGNITU1rVWCqDgl0b5UyM1qoFaIJYt9SZaEAmcMNYkiWcZnGtcZrG32F0oRS6IoWNd4NineeHRozRwCtYogQxkIKYtasqGAvuswRlDAtXH4dn3zq1tnlnnBWEM39BitWdeVeV5k1Dh0aNNQqMYVM1ZGuVSa8KRgvUNpzboGYog4b7HGEWNiXSVpo+s7aZCCxI51fSfehW3Mu67ScBprBU1cVo7HE9bKCL7vek6nmXVZRbnceTEBzpIIEkLAWkFx59h4nNYQ0z2CKUkj7QRULUu5FKz3QD6HwafUsnOVWJeIHYWodV3nUSHgncM2QUkpGec9aBl373Y7YvOCLA3NqlWO4W43EI1qalJNiOFsZ2KVo2SxmtBGGkrhUgZCWpp3n6fMkmzinBVroyQZp9Y5QhCfvxBWShWfw37ozhY6wg0UUUjfO2rVUAMFcL1FUYlpkcZaK1zvmk1NAOvEzy8ViYZr0Wwo4YrljT9bKg5FqhHSKlF7yoqaUldq+65pDVaVxsmraKvPx6nm3AQhndxI1MoaI7pUtM4NBbo3uN5Q7VIzro2vtYKqdUMJpaMX+kLFWvlO6ioIdaViqqDtcuMDxmoMlTRPhBDQ6teP/BmnqEnoE2zTf7mlJuRIJDLFiZvpyOvjLbfTiaNemW2kGk3nRq52/8y/u/z3fPPtH7m4+h3jxZ6ixcQc5eRzQAQhtUqKx5aecn+EG+qnNtxRvZElXds44mwK/dby+qXL1bvP/3sPy56yFH+dffzirXzy6Pnp9ff+FD63fj6E80NH6EN78OYe/vw3Lu/Wk5u/9XjH5fWel9dXfPvyG15cv+Dlyxe8fHnN1eUl+2FkGDp836ONWLzkWEkpUMJCiEFsIowl5SzpA9YyaEM1RcapVhbsWgwpG3LOeOvwzpFrpWSJEVsXEZaIRbw5Ww+kXChV4rSsd43XJu71qo0v0TJKKUoxLyupjW0lWFRygrU18npRuHzWWZrIk3URTpz3nq7rKBRO89TEDZ6u70BVlnUR6wxv6boeVGWaJ2mumkHwZpuzrCulFBnFIk2f2EBoQemUOicm6OZlmJoPX84iEqHCuq7EkKTZ9I7jKbKGVcQZWp2zU5VSjdemzukhMnKOrcEwbZwvo8EQAikmLi4OEuReC13XY50TDqEWNNV7j9UepZLY21TxsENBTIlUEvMykVOhG4Y2KlTUmrFO4TtzNmE2ToQtaFmIfOfw3ogSsl38Sk6EWs7NxbIu7MoejUbrwjgOhJsgY09V0U4TleTTUgvOCjJ6Ot7x+vVr+r5nGDrWsLAsM7UWictDxCjWKawXz76u65mmmVIyne3BOU7zzDzNZ8PpYS+I6mk6UnJh3O2x1hHDyhoWfBPOrOvK7e1rOitCJ20Nr1+/Yl0ru92OcRxIJXN7e0dYF/K6st/vGDvDUi0xBuZ1wTsvIz4tv1tD5Hh3RwqRcT+iUTLqTbVxUCtG5eZvWJpKvrYkGrF16jqPNSIUUS0TLsWI1k7GggpsG/+WApkso3JtxDDcSFxYCBFvLd4YQb2QbOC8iGJ7sJ51Q2Z/xVWtl9jEKNOO3JS5r453/PXmR35I33PDDVVrrN5x7b7jd5f/fb793T9z9fI7docrfH9A2xFQ7Sa2osjnGEOFQVGFB0ulVkmr2f6Peo+dqdbkbyPUN5h7j6AJT7GU+ODS87H18SPb+ZRF8ydZYJ/AyftZGqjfIDfw0XpwHNT7jsGD8/jnRwgf3FScv2Cf82G9/Zz66PfzU9rgJzd//4f//f+O65dXXF0OOONkdLC9m5olbi0FprvAaTmJiEAZOmewSuG9xRqPBnJWON8Jeb2pDFUjjkskV6VUje0kFzRk8ahaQ8QqTWcdxnnhIFbk8akIUuMc1hiqMmgjCAUlU5XEUMW1cROTmABbY/C+w3abT1zjnKXUzJBVGyEL+rZx6LquJ6yBeZ7P4gvrLc47Qlg5TXdY5xn3I0Zb5mkmBvG4s1ZEHqWUJtLwZ2PhdV6k2WnNJcBpmsk5yUJsLaXCPM+EltDhOoktSyG1MacmlSqjdFXPJsDWSqKGUYbb2xvWsGKdoD7zPDOdJhlZDx3WanJO57xf5yXG7e7mjqobglEzKUdKaxS0VWSkyatUnDdgBCVbY2gj4cbR1KAaQmecGGUb58QGpilQK5BzoioRsnjXUalMy4mUo+QpK+jtIAiaKqzhJMhpqvjes6s7lnkhpcThMKKNlUY6CdfxMAy4viOFwDxP6GYXVJFmP8bcKAmVw8UFKVWOxyOHw45hGFnDyuvj6fzZ9H0v6uhmu9MPPbvxwDIvhDlg9lYayJIFFVaOru/byD3y6uaGYTew2+9JMRNz5u400XnHbjdyOk3kGImpnoUwJRvxcUwJE1bMIAimUGcLx/YZadNG6lpTij2nduQ047uOfhgIMYjCvQo3dgkJSOcM4L7zKN3JOF9pwpooJQgy33iiIspKpLSde5w9GNudFiACMj/0glQCqvwjDm8+rf719b9xsyxMp5VXd6/4W/6BV9wRAM/AQV3yT/p/wh//+T/w+z/8mf3Vt+huj/M92lrQSO5zszzacroVGlO1IM3n2lo1feZRn+u9h/p9i8ebo9b7fIyv2IWcEa0v4/L99Pjc+xiPT9nmr/8c/6r1odPrg4fy7U/kaVSBL/t0ns4n/Jr1uWfVk5u//95/+GdA+Gm6mfSGGEgpCI/J2sa/qgy2Clm5ZlwFbwVBAMW6BJzrmlo0YdCCOlE5nY5MtzMVhdsNVKWY4kxaMn70jMNA5zqUonHzZFE22tLtB4w11NRGxzmzxvVsRVKbdYKMCCs1Z4a+l3SSJuIIIbCsK6HZw2yLue+8iDtCkPFIqdwtNyKE0Jq+E48+YzWn08x0OrHb76V5K/D69oZ1XiSurP0+pSQCglLYdV74e0toIhMniBmVeVqYp4lxHARBRMbiIUhcWefltTfTXq01xjqWeeJ4vBPvuyA8LN93aKNZw8o0LTjncVYa7JQE4XPOtlgwDQg6sa4rXdc1PqC8bjSWQmWaZqgVqw1BraBktH2aJg4HOR9iEcuXlEJr7uS9lZJZ88K6Ts3mZGtoOta4EEPAWCOoXy/5zT/+8CN9N7Dbi0dhzFG2SyXXzLzOYCrKVWJdJRt3rVAyuSZR2yIJJgpY1hlnpYGyzghv1RlSEoFCVWBaLnOKAaM0Xd81XqcnJTkmUYkK2lmxEzoYTVgXUoj45tN4ujty++q1jOiHjmItd7c3+LVjt99zfen42/c/8PrVj3R+oB/6RikI3N4chYdpFFnB6x9ft5GzZ2goshiMzzKiNxbvHcPYY5zm9vUNd69e0/UD+91I7x05F0HoU2o3PEIJsEYTYyE0L0uFkqQW70ixjbUp5FSb0KaSciTH1OL2HLUioiOBDNG63KvoCy0Vp43/nUU7+5tAL/4v//E/5xaD58AFV7zo/1P++epPvHj5By6uv2W/v2DYXaK0x+lO6BMUUi1Nza7P8guAFtL24OL/UJm7kWt5gPbJ7x81NebBY858xPu/vU3F+twW6NdZz43dp9XDE+xriih+wpn4r6yeLvhYT9RUic3YFraYIWhXKOYwi7gBxW4/tngzQwgrr15L1qpznhJbI6U0fdcTTxPrKs3ReLHDWeGt5JIwztLvD5JvqiylyHiv1IKzgrTJ2E8Lb2+VEeXDaC6tRdHpfYfWipQiqoeu61EolmXh7ngiZ1nonJW8WO89IIvYuizknLHWMwxiliwiFElW0GhOpxPzPGOtZWiN2mmaKSnhvWtZx9KoLZMghsO4QyvN7e0dtRb6YcBvC/Ms5P9xHNtoTfZ1DYv41nkvY9sQJC3CmRabZri5vQElWba1mfNqBTEElmnGWMX+MOK8pVaEy7aNUGtto97QjKztWRBSiyBqxlnCPElKi5PPoFQZxYpgwZyft1UpYpNiEERP4sdMs8qBQYtYyBgLSnh5fS/vcTMRjilivaBzt+sta5jRxUgj6x2piPJZoVjMincDxlnmaSHdCULprMN6w3w6EW/v6Loe34mHYYgnhmHA+561rqzrAlXQUN91VAfLzcK6Foz1ON+BkkZvWRZ044t671mmmeM8sasDwzjQ9T15mkTAU0QUlFNlOk2UXLl6ecXF5RWn45H5tHB3e6IbOvb7HbvDgfk447TB9JbT3Y8YCyVDqhVtLM4plrwKui2ALmGRTOrd4cAw7gnryjStGBNFUDN0lCLN2nSaUEajlcZZy7jfNWPm1NDu7iwKKjUDSjKS2/gfVPvMnUS1KdXoCYEqNn8yyjcKSjO7rpUSs4wofwPX7P9u979l//J3/OGfXnK4usL6AWU7iQo0QntAGyqK1K5HSmlJPqoijpK1Ure1ziK22ggVZlNVnZNC3hzTvmnE8r56MB87c//u+8g3l9gv+9DeeXZV7/nDJ2zj/Nv3NRWft8/vHLl3NvOp231z0P7pVd/88R/m+6Pe89+H9Z7P7tFfqwd/+8BBeOLI/mu2oh98/Q89c/v+qke39N76lH1/cvMn6leD84h1wxbthNg8UApGK7wVy5CUE+vdRG7+bhXVGq8CWrhM3nuWZUVpxeXVVbOykJGCQjzChLjv0I2btswLtdKaPuGmrWuUEVnK6KbSrSVDLfS9b2NRUaKGmPC+o28L2Wk6iidc4SxocM6KRUsSQUVYV6iacTfS+a6NAaEfemlmS+HudMe6rIz7HeOwQynF8XQ8Z/YOvZhEV2SUWErFe0Hz1jWQcm6JG77xGlfWNWCdpdti6oogbSkl9hd7vJfM3BCEm9j1A33fE5tIQ2slViNGPP2UEe5gVVWQxL4ThXRO5Cy8S+9lhFySqGStc8yLRL4pxLC66zrxLmwqWe/cA75ioVIE7RrlBmBr5FGKrhMvR2tta+rq2e+PCroZDdfmg1GVomrhJO0PI3/5678SU2w3Ek5yb6kYo/G+QwUaOlkx2WFMPFu2KCX8J2M81npCCHJ+UtmNO1ISsUwuGe9Ne7zkKxeKcCOdYxgGjsc7YnjFMA5n25y8LGwQSy0F3/dUtUrDGhzWWfaHA8uycHt7JxzD3cDQD/zww498/5fv2V9cyHfDejn3YuDu9ig3Dr0YhZdUefHtS+bTzLpGnK+YKrSFcRyJLjFPEzFUdoc9uSRuXt2iFIzjIDdliEJ5nhZSFkukw+WeWpDIuBAw2bTPXXh+Cwrfe0a7Y54myR2OSfbNO0F5UyLG1DwojSTIIL6Pcm2WiMdatSB/pVCoIvp6L2nn11P/w//1/wZnu6bYNnIcELGF+HSLT+YWjSbcSPFVFKqk/O7eWLn5ksIDxE9+/2a9PUr9GEHqzcc9PkT7lPr8RuyxBfmnOlM+abuf1bd9+Eh+6vuq7/vHz/5VehtN/rwd+LkkEW+92Bv1if3W169PaRLfOglF6vW0nX9y8+ecw2otKR4tnPzepT+To1z0lRLRhKIpe7VCjKB1MwGWRT/XwjSt0Pht3ntQoi7NKbZmxGKNvNZ0PJJyETWs9eI1l7a0gEqtSgjlXdeMbsXcWIyYY/Nc09KMaJiXhRTF1sRafx53+ibCWENgnhdJ3TCacdxhrWOepqZ8dXRdTy1VELoQ6YeBvuuoNbMsUVCjxrXTWothcvPAc97jOy9k/Vl4aqLQLaxhJTY7F6XFniFVQdTEP0/81HIqws8CbPPpK6WKV13OOCsol1Li9ZZSlEYWya+tpYBRlCJeh8Mw0HkZNVVdz/YTG0LjjEZr1+xjRM3pvafzHdY4Sq0kApL2phtfUhrd2kbIzjYvvI2wWsFo8eqLa0RVpCk5LTIuNBqdm2LRyutO0yxj6CB2K61LlGSWrgelWJeFZV4oudL3HeO4J4TAGhO1rnQdoA0lyVjbd9IIx2WmLu29G0s/DCJ+yYnj6UjX9XRdxzAMzPNKTAnXxEWiHE6sy4Ied2Kh4wzHuzvu7u4E/WsUgVIK8zyjjWbXfC9f/fiau9vXXF5d0w0dxln0rJlOE7c3t4y7gcPlhfBiU2K33zf1+ApNhCMIo6DBYVlQRjMMAxeXF6zLSojxjNZaZ+m0Jh4TawyYYBh6aR6XeZYLROclGWRdmVf5TJx30qwnzrQA085JlCLGQA4RreX9KiMCmNJiAEuR5kUb8VLMtTT7ol9/82cvrzBZbmZKi0vT7fxFgaq6AXcVVQ1KlYb4bdCOeoDmtYbuvFjodvF/s953VN9YOh6CJw+3/WB7G7D40Q2/99U+od56E5+GyGzI3N/3fPrpX/1xbmRt/3PGfD9jR74UG/3c+tgWfrbB+te82/hJd/rzd+zpxlotm7MW8apKOTWemYgJhPdlMU42mWNuBrce3ZIYQOw/4rKQS8U6MXW2VvJB12WW0WrjK4lqMoq60GiGrmtoXyUFWWCgNksOR0VRUmyNp8S4xZjIzSnf2oboxSiqTKDvBW0TgUElRHm9EKUBdday2+/RWnE6imqz6zxd30ujtq7knBiGgX4YMEYxTQvHu6OMLbuu2atItFrOuZksd2ilOU4TpRQ6K4hXCIGwrvKYrhPVpDYsiyhQO9/JPhvLcTpRsqiRrRHFb4yR6XSCAt55Sd/QpUV4ZUly0AatRCRQqCzzRIqRcRjEJzCG5hNYSUVQn80vUGxORHUcQ8S6rRFsKRNVPBVN25/SBDQ5Z9ZlBQ+pb4ICJJGi5EzNRZrcKDxOrU0zlDYtcs+eT/MQ1mZBk8/WMjkLD1I1c2at9fmqV6uMG2ut1CIq5tiQJuscOUoDA0hmcog4F/Gdx9qOlDN5zayriHHGoacfBknaaMIO08bcqnEew7JCXzFa03Udy7KKGbYRMcswDG2EukUAei4vLzgeT9y+vqFrHDvfRE9Lox2EZcFaERXpNia3Rsbipd2IWGfY7Ub6rmOeZpZpYhgGzG5gnhdiFC6fd6Lk3e9HSpHxcwypobZDMywv54xgQmCaJuxqJW96HAWFzbV9ZqrlMQuqKJGBYJ2j876dV5JAotrNoNIajaEYTfn5Lu1/t7LKNIQP8YhsPMgzorcJZNrodkP/gLagv0XCe6PBeQxFukcG3m2g3sUI3m0n3vzNG0Pk2pCGn6jL+TS08ZG/ft7k7NPrwUH94knwV6gvR2l/ufX2e/m7NYNfXA/fyQP07u3XeR+6/N79efpt0pObP0nFkJD3Ao2DJaaxRsuC5oylIko/13zaJDWitIUkN6RGGhPXOHUxiFFxbQ2L7zqJHkviawfQdR2dF1Xk2ixarDZ452SsVyRlQe6EFSnm1vhobCd8mpwz67RSsoxEO+/pu57U0AuFWJLkZjnRdx2d71AV1mWlUgQda8+Zl5Pk7zZLDIUixMQ0TeSc6YcdXdedzZpjjI0j2AG1jR3zWaG8htAeo8QGpOsb/y4yzxOlVPa7vvnriVjEWoNrzWUphXWZzwIN08bvuvGFxJRaxrS2JY+AGA2XLOhfac2b/FwJ6yqfs95G8pXc/BBTTsIZpJ7HpSGsYgXje1H1tpMx57QBfbKNnJv2sJ695CT5QdBGn2WUqlASG+bEhNv7XrwcYxRhSjOiLqUQlgXfe5y35GhbgoFYiqAEva5VvAd1Q7+c6zjFE/Oy0HfC4ct5FkuUIskeWktSSUBJdF1DvJ2z5yYHELqAc4LqTTOFwjD0LQdZizI8l7OISGv53TSdSDkxjj2gON0dOZ2OuOAZxoFh6PHOcpombm/v2I07EYI0Ac7QD3g/kBtNIa4JMw7CYwyB2Jr1ru8Yhh7dDKJDCBTrhJLQOdY1NENyje97JDYwkGK5532GSGhNv/ES9xhJ5yzrbZxvlCUuK2URdFRuBup5tCuN+L0SGP3QdPjXW6bSkL57JE+ph/ga93yfty7ijy7qj8z9ZCT8lJnk243kJ2Nr26s9ePXH6v1/ed9znzYM/ng9vfF5bDF+82/v39av/6blub603h3Rvvm3xx7/mfXEu7FPaP5qs2ChRTFJw2G0xmrbsj8FWTHGtmgy03hU+YwaGm3EvNcYiVqrksbgXSeLv9kaFeHnmUagN8ayro1s3pS4ztrmX1ekqWwNSkn5LF4wVsxvSymC4OSMNY6uaxm7MbGGQFX35gXe+3s0scI8TdRSGfc7vOsb924ipSxNad+3uLfAtAgnr++FVyjopTQrpRRpYpq1SsnSQDjnyW2bWmmslxGa1oaUJJkjhMAwjM3qJTNNJ0Bhuu6MapVaWIMohrvOC3k8ZWqprEF8B70T4Ylu6QqVQk6SYKGUiGFyzmeENeZ8Vh9rpVDWElNmXYVXJ0kkhZgKuWSWRThkvnNn41+jJI7MOUPXuWZzIzY61hiCDpRUZGzrvHAOizTHMUWKkvNOa83+sON0OpFSZBgOTdAiTWqKAecM1jick+a21Iz0ZgrrDLlYQezaTYixYvadYiRZc0Z0FTJCL1WUsNoYduOONTSTbiOm1lsubq1yfmldzw2Q8OQQEU9TjG+8QutENLVF6uWGUluj6Qbx/ktZqBTe+XPzVWohpcS437XUFeHnlSooo+/E/HueJqEW9N05haWUjGtjZ610s3SBZVlw3tP3Hq0RFf2WSmPu852NlTH4liATlrWlixhKLo2WkPH4lvGrRYyUk6D83uKNWBylpk5XW0Zx+xx/7aVrG58239HzknC+cd1oDNszHr+Qfx5T7iP12KLxCQ3hl+zJ10anvub2Pmu6/Us4lX9NkN976u+GBP4MdX9teN9fv6yenvCRRa7X9SNG6XZRqG3iW1sjJ6Nf5x21FuZ5IiVRCjrrzqkLNNPikiWCatwNdN5LU7TMzPNCisKl2u12QGVeV9YlYJvqcGsgtrGTte68qEjOb4fthIc2h1X4gk5GuMY4aGkg6zKjlWnCDTE43hbbUktbhDN9N+C9WLScThMhRvq+ZzeOaC2o3bKu59HsOI7oNkKMIUqz5S1KSbzbugRZcBsaFYJYbhjfoYCUMpWVnMRLzxhD3/fQhAdhXQVdqzTOo6RfoES4YoxtPoeFnFu0Wy7CwWqNjXOOJayAou87jNKsSZprEuSG7vpe0klAkNmYW/6td61B3hBDGZvnlAWNzRm0bh6QEnsn/E5HSlESLpyn5DtiDGfxyDY2zjmzhgXXSbPSdR195/jhb4KCdr6T7NtcySpRqqLmIqkSWrHkJIi1ErFBN/TiKdjQu3W996erjdZgraHzncTolQJKbkSstRwOB+bvV+7ujlxcXGCcQQFaiTIztbg/6zy73Y6721um0wRKju8wDufs4C3yTyl1tkcJIVJLwXl/tm4pJbOEFe9EtLMbd4JIVjGxdl48JE/Liu8c4064qbevb8h5YX/Y43c7puOJaT4Rm0BjUz2nlJhOE+sa2e1H9vs9x+PEMi9nhbpSSkb2JPpxh3OOaToxx4VuEAW7XIykwZSoPy0ehEWOrdFys+GtEzPiyhkxpVaMUuebt19z6caSUVU4s/f07G20+9boR8nv1IN/3l/3P4AY1LceW996xlPXjg2g/Nhj3n7YW8/5en3IZ2CCT5zFqo/8/RNe8TFY9uet30Dj91i93Xv/Mnpx9dZ/P/LQe5bHWz+8ZxNvfUef+tE/Xe2rBeUQW5MtPiifuUYyVvNYbUkptjix3IQMPVZb4Vo10YLSimEUbz5dRWAR07aAKvpR7DFqhuPpSEyBzkuEWS3SDCpoOaSqeQ7mc2qCdYbTNMuoU6szf8pYy7oE1llyVruhl0UwJlQbj26NnMSWKcZxlEi0eWWaTpRSuLy4ap55Mj4TVCc2QYAX64xpJsTU0D0HyFi5lErXdzjX8mqXBWsMh4sDWtuzXcq67YNSHA57Ou/FTmaa6DrZ71rEAyy1kV+thWEchXxfxDcvZRmJd12HNbapYUWssc4zpiWO1CpEfOcdMUTmaZHR69ALn84YtDWEICknu8MB77Zos9YswdnsN8VIUZLKsixBLEj6xu1LGWUqKSahEqTmA4ciROGahXVlniYRCTUz6hgS02ltMW2QY6LmilEa5ZxkG6SCKpUcozR/WgsaZ1TLG0bGsmGVBqf3KA1xbeNvYF0XnLP044A2hnVdWrKH5/V0pHLAW8M8i6hHKVrDW2TMrBXWiTVRTpGcxCRbG826LsQI4zDS9x3zPLPMM/0wno2XTUs6CTkR1hVvHeMwohWcjidOdyd2hz3OWmrnWRFaRmoI8fX1FdM8Mx2P7PcH+S4hCOnaRvlbE9h5187jpfERe7TmnrdnNNpq4TY2VN9aS6qRnCJJa6x1GNPGxCmyBkG1+96f/ThzTKxyKmK0AmuIMQkinxJ+8E+9HP3DlqLZZFEeTGYfKinur/5v9g/3Aox3t/nuT4//+2P12BL5vm18ZDl9RxlSP3F3ft7u5Yte7b1P/mW0Hs/1G6r6dIHPk5s/bWRcuSyLkLtbOoP4zxkUhZCSiA2ojOOervMYpVjCynGZUVWUgn03npWg0xSaD52Qn7u+Y+h7rLUcTxOvXr9GK8XFxQVaKeZpRinJZVVauEvrKnm0u8MeYzUhRuaGpOVS6buebuwptXC8O4m601kOu0tyzhzvjlQqve/OBrUxRFJJGOPOXDGxWcns9ruWoCBI5TTNVCpDP3BxcUEpmRAC87LivW+KXbEM6bsehiq2GCFwPE7UWthdjc0ypBJjuc/i1aJaHseRkivLLPFhnesbclIxzqJ0EYRKuzPKmpOIZ4zRdF5UtkZLRuuGrN3e3gnh/pDJpaKtjPDX9Sij5nGgZOHsjfueSpVGLiQOh8P5c0OJr6LbEEdlyZQzX7C0RiLEwBJW8Y6rllBCy3utaLQ0g+sqCKZxZzGOaQjd4bBn3I0cbyeWy7XxyzTGamqRm4DpdqLbiwI5IzYwVO4NihtxXhvbvikVow1Jmxa/J6KRCg0VdszzzM3rV+wPB/p+R2g2PDEJIm6MlmxarYWysEaG/Q7vM6fjHfM844sIeIZhd06QkRzmgePtkdPtkf3FAWc90yypLkppFLAsc4vjc2jjON3dknNit9vT9yPOdyzzwrpGYOZw2HMwjr99/z1/++v3XF1eMA4DRluWeWKdJVKw6zp2hz0+JY53d9y8vuFwcUHfD8zTzDzPGKPphwFnHafjCZc8425EDSPH4x3zNNN15ey5mZJwHnPnuTj4M8dTBCCSxCOqcgVqlTFySuT068/2Fe8C3rpCv9XwvVEPBBeK9zzm/b9936Meb0c+pVn51KZQPQ1F/MR67Hi95w8feM5X3YFPfNBXagx/gl75Q2fDU17uMYbbz9UGf+ltEHyMcfvUd/L5H8wbdIknUnc/pZ7O+WtqTlE1+hbsrig5S0Zuy6P1XqLBSsnc3t2C0s2Drcf7DmNMG49OgqT4jq4foWn9MorjHKhloZbC/rBvTedKrdD1A76Zyq5hpdSKb6MoQMbDYWVdVg6HC/bjSG5NX04J6yzXV1dopbi5uSHFxLgbGYYBrdSZIA9IIobWguCFlWVduH7xknGQmK1pmclJ0KDdft9G18KhijHSD9LEphRRxkoz3PwKYwqENk70nUVpzskiOSWMEV9FrTRd77HW8cPrH9BGc3V1KV6KUYGSZnc6nlDA9YsLGU9H8cLLJZNjbgkhnTSdMaK04Xg6cTxO/OmP30lW7emE1dLMeS+o0LgbqaWKOKRWbm7umE4z4zjKvsZIRTVrFHme5BYbqB7VvACNcXSdqHdzzFQq1hr6buDu9o51Xs4NwpZUUlIhLoloIzVkvPXSEBrL7c0t9Z8KqoqKWbdvSskZbcB7QbCEmygNbw4F5QpWmxbDlri7u2O/3+ObMnZaVvaHPcM4iuq7iSWGvmeeJhQw7nfc3dyITctuR4pJ0jCaJ6Agp4GSxWzcOMt0OqG1YbfvqLaQUmCZV2rNHC4vefnNC3784Uem05Fxv2d/kNdfw0pYA8scmaeJl9+85LvffcvfjObm5kdSSlxcXHE47Ol9x83tLXe3t6zzzNX1C15eX3OaThxPd+zY0/eiIl7m+YFFUc84DBwOB453R25ev+L6+prDYYfz9nw+d52Mrud5YpkVu8PIbj9w8/q2mbSLUMnoHuc10zxze3fDuNtzeXng7m4ixoUQRaXtrMM7OfdTUmdR0a+55F6jcu+t8qYa9/0jxLf/8QSS+OP95cO9eXQf33Bse2OTX9IUPvX5H9rqx47NU7bxBRv4YH3ue/vC13/y0z+0fx+O0/vcPXzCGQr8cnHRD7/vLz1vnn6z9q550/tvFz9Fef8JnD/hWFGLkNtTU/7mjDEbN08amHldoULfC9Hdmi2ZYz1boxjruLi4RCtYl3DOx3VOTIZjLcQcIYM1lnG3B6VY55nbWdA/5z1dL3mXIUTCEtqYSnH94oUYFIflnPjhvBO/tuMdAL7rzpzCdVkaF0k83iTiDFKWnFqlNNcvXjAOPdMyc3c8im+dc83s2RJzaoKULFmpfUdsJsxFQ8yJJSyiri2VcRjbfkiqxe3tDesS6XtP733jpmW86zhNR1CK/f6A1gbQ9L0YFS/zHWsI7Pd7rHPiT2c8p+XIq9evWWfJAO57g7WtAauZEFe63jHu9njXYS4kLm2eF4y1TWEtozhrMsoqtIJx6MWUW2sqipgSIYVzBu7Ffg+A0chYXyn61EGVJtZoMQBW2gpfLolVSt95ci0Yb6lVMe5HXCeok4iMNEZ3HC4O/O0vf0UBnffkc25wIaZMTQVzzCijMaoZSYPkUTfKglZN/Zs3lbgIJrKGXBKDH4khcjoeSTFycXmJUkaydYvwMENYm6clYkZdxPLGGIcZhuZDKOKk3W4vfM9m47P5/aWcWJaZvh84HPa8fv2aVz/8yDAO7MYdne9JsaCN2BXN84w1jmHoCOsgXMMcW/MlcYQKON7d8m//7b/y7e++5fLiQIxBmkIvfMTdOGKNWAgBZ1R7fzhQyo5j8yX0nadk3zKFE7uLHRf73/H69S2vvn/FuN/x4sVLjscjU6NZ9H3fqBeO0+nI6XTCKMUwdI26cBRer034znN5OHCcjqT8S10GfqL61EkoP/Mw9DfKG3uu5/ot1JObv/04NMVt4DQdCWHFGnNGEkKIxCS2J323k3g05ym1EnO8T8oAxt2e/X7PukZevb4BJSPTfhiByjxNzPOMUoZxJ7yoUiRrNOYELa5NxrEQFln8tdGM/b4JSxTTaRLbEO/bKG4FpZqgQ6LQqFXsMZq3nTWWrhchQ06ZFAKpCEJptOXm5lbUns06wzVT6GVexX+uZFEnO0kIyTGLmKKpJte1JT5Yi+sMOQoHbl1OpJjwTixPagttV20Ueby7pdRK3ztSjmh0s9wpIrxwBxFA5ELMC/0wtoZGMe56vLWUZjCdi6CsRhsuD5eM/cC8nISTmSOvXr1uAhdPyvGsJC5RuIXOe8Z+YFln4Wkm4S2uS8BaQXZzEjTMd52gjyk19LbS914i65Q0WgC+k7xfVQvOOGl8jcHpkf2wZy3if3hxccV+t+N74HQ84qwjhpWUxIuwa4hdWjPWaWoSLmqOiyCwbYxutKjNAcK6oBFKAlpG0MZouk6aM200ziliLPSD53g8iao5iUXRxcUFnXcNuRS7GxFxeFJo3ERSU5xDZzqss+SUiKmwLokUIsMwSiJGLeScmkWKZRh7cpIxfkyR73/4G/vdnpcvXnKaJ7khCived4K8a4lnizVyOt1i7TXXl5ccrSXGlXmZ5SbLCCoZw0rX9ez2O5bphLYG33f3yLrzXFzuCWuQKLoB+l0PUyUsK945+Y6evRYVLYOCoRuYl4XpNGFDYOzFf3CaJqZpateNjDaWGJevdV375dYDcUR966eHI8sP9l1vPeXJPdoDEuGjz3mDNP4FjfjbYodHHS4eed+f/Bqf/KdH6j3DvV9y4/sVFNhv1lvb+9L3/ohg6UOlzo978/XrI7v266kP0CPeAfDvf1E37sSDh90rgjdjtacdtKcjfzGyLjOn40LVEg/mvSWnzOl4QmvNOAw47zFaRpF3QcaitYLS6ox2bKPU03QUwnnfoY1iXibiGkHBbr8TFaGCEBfWWZqzrt/SQJQsvs0+wxhDP/QS25Yix7tTOzCKcOYJdmKai5jQLssivm3NeNk514x6xRpDlMqavi3kORdOp0kUwd7T9WJgLSpjaZKsk9i5kgrrKk3i4IemZF6YZ1Fldl1HivncME1zGykOe7pevAFrU5sKLxCxlFGGFCQXNwQZS4IISHwTkKAVJkWmeSGl0hBBOWYah9KVZZ1ZloX9OIiVTAz0w0jMmZQK2WVyjqQkJ5J3nuk4cfPqlsvLPSklaqmNk2dJKZNMYr/bobXmtC7nk1MrxRQkqkxse9y9QjmLynbLLc45o62MBdewtOQHyfpNzVTcey83IaeJF9ffYo1Y3BTEkqVUEXv4fqDGgPWeum4q5kRtFjOV5tmXYUXsTuR8LRyPdxhrWzNWmSaxylFKEjlS2ux2CtQWpacspVbWZSWGRD+IKbkCtB2YjhOxKbe99xjfEVPidLxlv5NmfbcTb8i745HXr36k6zoOF3vG8cDxOHG8u2t8V7Ga8d6znGaWeSU31bvrPFfdC+Zp4u7uhhwSL755KchqEuuaWgrWOvrOcLcG1mYe3Q3CP8wlyTg2BqzWQmEwltvbW374618Zx5Fxv4NaOR3v6PuB3X7PPE/Mp5maK/v9jm4cRH18e0NMiZpFkHRxcSAG4euu64w27iczC/6l1dtBWJLu0dwT7n95fvSDJz74+Z5Ad/+sh6bNDxeRT1lBP/DYrzn/e/S98qAT+Iov9Qmb/Jiw8lPqHcXpI3TEpy7Un7wvf8/v0juEuye+y6fOiX+yeh/L8SvvxAcPxmOv/fbxe+y7/+n7+OTmL6TMPE1oK1YrpRYROrR4tmEYWt6rLMShIT+1Vpz1jQtoyUVMkNc1Mo49u3FHQbHMp6ZChM5Jli1KxsjrujRz3K6ZJrckjCCZuMKPkxHi5jNnrHgMrrOIQcahPwsqQmrJDCGIoKIt8sZYUktReNhQaq1ZZmmQjBXxhPjY6YZwiZeZMUaEB82OpSIcyY23lUumG0SMIY2doEsxiQJaK01Vwq+kmRFLExpa0+ugtszbWgkpkHLEey/oorasYUVhuDvesSwLw9gxjoMkauTaPPQKy7LgvWV32IulSIVCIaeIc2LTsSm6tZKYvpgiaPEqzDWixJuXUgq5RHJJGKspVY6/b8KTUjQ5Sfi8d+LztzXdpUi2rlCh7t9bLpUQV0qODHUUfmmRUbXvJBpvWWfWKLw5sRnSqNb8lWZNlKsYVnvvmRfhbpaaqV44okqrJgwC38QhqZSWTyv7Os8zr5eFFASxvX4pHDtqJeXIzc0NwzA03qjwDnPO5JIljcXaZnWiySExTzOxRQQ65zhcHLhriPI4jrhmjh6b1+E8LxLbWsUCJ4XIKR8JZsFYzTD00Lwjw7pyuLjgcHGg7zpOd+p8w3K42DP0fUPhAzll+lGi3453R47HO7EfGjtSFI/JnDJzmc+I9sXFvvlMzhhr8X7z8xNqQdd3UMXU/Pbull3eoVB0nWWeVk6nU0NG/TnGTGtNLUma/F95KbZED/m3UErkLw8RnfvG4U2VxPlfG7p2f9v/Gfy8tx7/1da4T3ztd173C8n0P1XD8KTt/kxQ1W/kRunnrfd9dl/jM30AbX7sBueNO5C3Gr/H/F/eQAG3YIWP15OvtnGNEkmmaTYb92bIktWKZNLGJE0Cknhgmz9ZLpnTtFBrxhjLfj+im/hj40YZa8+RU7kk1nBvH9L1HuOsCDJiIBfJ/h2bmXEtYs+RS2kZtxIb5lvuqnUte7YZ0aaU0VrGjbplCscYSTmhUHgvCJ5qCKM0lZFxGPGdw2jVDITjGfXTzqEREYwkQDR7lxAkjquloCil0ErSKUrNnE5HAIahY2iNdYhRjmUbo4kBskMBShumeWZdlsZPFPuIVHLj4VXmeSKugd1uxHlLyYVUczNkDqxhZbcTdDXUiKpiuyIjbfFl1EqL0EeVpmBehbPXrHFKKbKvDcHJrRFPMWK0wlgt2ygRhRwPSRfRMnquEvWVc8J2nShxS5EGuKVoQCUXQSCV4YycGW+ZloVlXXDO3dsQtbSQqjJVVcmOzQnvLWtQlFRa3Jyok8Ug+z4hRWmF03K8ai3UqtrnBd5bJM2w0nWe0Hcsr6d2LspNkbUSTRiCmBcDZxPn7SYl53T2zdRavA2dc2KbVBMlKzrn0MiNQIqBuaXVDL0nhWZYrgoaLTc1SCOVYiTGQFxXnLH0/dBev91gtCa0ZFHW5tRyfq1hmWag4nc7TO8xQTJ5U0rM89yMtXt2uxGjDDlH5imKQr5USkpYL0rgdV6Z54mpTlhjsN7Sd+IdueV8C7qd2/Wk8PfOYv156s351v2YcesG29/Vg/++rb54mN7xHmDwybvyzj9+4ublk3by4bF6/wa+ECx8d3tftC31yE/bvz9k0/PUP3zl+ple5+2X+UnOssdBs8d34IvrfRt85AWfJNl9wqbh/uSsZ8ik/Xe7Xjz92D65+XNGoYzB2PZCShITZCwGy7qIL9u6opRit9udkcClJQIo3eLb2uj3NE1nXt4wjPS+l8SOxnna7EFQkhpSchYVa5FECtNSD7TSpCLNSC2FeVlFeeo8nRdLmFyyRFPFgEIJ589JzFqKsXkMiv9g33d4L3zCnBKnk3j79V3HMAwtUaQQYpAxK+KZt8WsaaWl+dFG0JgQKKVinZbkDqXRWiLattg38XwbzkhTWAOhCRl2O+Ex9r6T/W/IltJiELzZtlBlP6ZllgSN9ndJV7nP3c0546xlHAdRG9Z7RFDei4y/2SK4lGoKXDHAdlaa8FzE2HtLzDDtva1hRSnISYNBDJ2bPY919iy4EJWwpDtYZ1FGQ+MX1lrRxlB1yweuGYxmWk7s91d0Q8exoZu+6/Bdd0aSlFISLacq1klM4ND3eG+p5IYuCp/SWovWtTV6sl9KG0ETSwTEzsg6g1KwTE3J3Ru6xjvNy3IeYZpGf9BKtXM4QUN6jTF4Z0labGeEp6jx3rE/7ESI1PJcxUKpnlE6pWIzuR6gJFIK8qVvDbIzhtp3BAUxBuH17S+4vLwExHrleHtso2gLXlToKQaoBWtErS3N49o8AA21WmrNzUdSxvHjMOIOjmk6Ct/WyM2WVhptLF3n0ENPaWIW1Xf0pkN1zSMSQVxt+56kLOrvXy2950EJBUo9APm2z1B+fjMsbbuSK+7/9+FPT73Hf0+9M+P8CbxYPvbaH3259zdTj24PPtq9ffQlHz0R3x5Nf079nDc3T/w2Pfawn6EjfcqWHgO5Puke5T3MgifV2/cdb87qP/LEp2z8U3blQx+SevN3n7DpJzd/XeeoKLTVGGPP7vQpJmLOElzfjJKHXvzE4hoJzbR5U/I6KzzBNS4s60wtBd9LwseGwIUoHn2qNQfGiEpTAuRb1qt1bVwEqS1OObdRYi445+k6SabYkKl4tqNx+JbZezpNbApQpRTOumYEXFlDPDeGnfcMw4ixljAJChJCOHsTdl0PbCNLBAnJMj6sVUQHktJgmzeeEnRvDeexudGWECLLsp6P2/Y8MXs2rEH2R6w3ejHKNqYhVRWnNcuykmJmGHoRO6Qk0WyohiYm+q5nN+wku7dklLZnxXXfi9hjy/utKPEG1Ib9uBejaCJK6TaOpOXkeox11HURI2eVQenWHIqQQz7jJDcpLeHeNPNopTS1bp6PSJOCFhMgrVBasy4Tl84yjAOvXr8mREmQkbhAQWCVVmf0V9JXJKXF2ob+Zskmrm3Mq5RkVz+cxyklNwwbIu2cQ2vDMq0s69osjyz7/V6SSNZArUeM0lxdXwGKZZ6IOT1oRuV9WdW+P0mQ5pRkdO+8Z10CJVd87xtK2NTibV8k61ifs3JzLigkUcN7oUrkObMuC7NxXFweOBwuyKmwtFi6faNglJoJa5DM5M7RDx3LsrLM8vm5hn47hCebSpHvu43i19n3lFxb7yIZ3ussxuTeyTldW/JLikniFo3Y4MQcYePYGieCot9Atu8G6FXVLun1YcP1/o7ofi162tX9ay3TPxlw8hn1lPX7k5qK5/oqn+s/6vH8pez3xz6DR1u/DzxJPdEx6+kkG23wxpKrZMFuY8YQV9aGAnXOsx93WG+5u71hmVeGYWDcjU0UIHy/uRlFd0PH1eU13jnWJTCHmUpDRaIoh4dxxDmJAyttkQGoRVGgqSLbKDdmoLA/XErEXCksi+xfDBGloOt6+l5QouPpxLwsjOMOZwTFNFayR5dVMlxLLuwOe/rOo5RhXcN5vI1SDH1H14sVSQwy7p6aGthVaWB663BOLEFKU9rCdt3XHMYdve+JraFcmiq670XAApBKZF5XpmUmrImcIvv9QZrOWok5tRFsYl0jMWRevtxhtCVXWXgrEmm3rKtYcRjPWldAUypt0Rdls7OeVCPVelItzPMsvMthpNaCsx5DJSbhNnovXofOiQimVjFO1sqQkqCutQBFk6JwGrVVLRN3y9pVrbmRZtZoQ1Wt8Wy5v9pYlNbNgqRSqjTFSkm0G80Spmb5PCSBRM4D78X4uRTIVfwPq3ENrTVtjJ1RtWCNI5bUzrvaULuuIbaBdQ0M3cAw7lhDZF1vmCaxYRE7Hi0CkhVyLnhjN9gHrZSox41lWSemaYaqGIcd03GWY6oqwzjS9QMozbos1NKaPa3pu4EQpdHKyCjVWEtvDForbl/f8Pr1K0rJXFxecv3iBa9evZLvSc50rsO7jhTKBv6KTYw23NzcEOOJoQ5tlC2fqauSujLNEzZaoQ10npgyVpuzACvGSBmkoby4vOTm9S2naabv+5adrFiXhZRWvK8oJyNoa3/9Js91u2pv49x3buAfjn0/Vu9/zBsY3pO5gOrxSSvwaQrTJ+z7p3QdX9qhfJXO9UOr7cee+xXajC+b6b+1H1+plf+UWe77zqvPeZ33/e7Df3irPkQn2Lh579nWV74TevNTUW/97RE0Xqlt4POenXraTfTTOX8J0jqhjMZaQbm2MWzOib7vubq8whrPjz/+jZLy2QIDpUhRbE6WRXh/Qz+y3+05HPaUXLh5dcNpPrWFrWd/cRA0UFtSjszzTEyZoe8Zhl58A5dFvMVSwhqD8x19L757JSfWGJlPE0q1UaazZ9HJuoiZ8/X1Fbkk7NZQFFHpTvMEKIbd0JTNjhQzr378kaIKxkqO6m4cocJ0mgRF0zAvM4f9QcQSWtEPkqcblpWUxW4kRckkPlwc8KYjxsASZ1LMzRvOiCWJsYR1xnvH7d0tyzzjux5rNMZJY5VTotaCNkYSUKjs9yOdH0TMkDNQZUydAtoIgrvmQExi+Jxi4DSd6Fwvo9wUyTm2Tz+zrrO0Zlp8GWutqJbfuzXq1hpKLa0RMeJVWMXbLsaVVCKuNt6i0RitOU0zc1OabtFq2/h2o6+WXMSKRTs65ykxYo0GCst8JMSZDi+jVatbI1mpOaO9QiG8RGfltY1WUO95nqb5DW7Xp5wTRnu8c+iWQ70ugaET26G6CBfTGUvne/a7nYgwTifWsPLq1SuGfsQ6jbeWNSwS2WYfjLAUQl3QYlJ9PB65urrmxYsrjqdblpZvvduPjLsBZw3T8cTpeIfvRERCrWjdUMqc5P1qjbeeoR84Ho+8evVKfAovLtjtdtz8+IrXP74SY+dxZL/bMc0nUgzyPTGGzonwKYZAzQXr5LujaIhvLYR1wbbPOaeVojS+63DeEpbIukwsS+Xi8oKrF1cc7+6YTkcRLw07rN4R19Di+YQWoNWvP94to9BKum39Dj9HPxi/tlIy7XhT6PG+et/qqj6heXtT+PHBl/vQaOxnqMeb2y/Z0Mfq/s0+6SmfsF8PG4APfIr3jztv+3M6qo91aZ9WT/4cNrraJ7/Cxzb6OVt8mzLx2DZ+RhrE+SXv9+P9VMH2HX3IB95GCgaeapf6dOTPQW8synqUsZyOR6b5iLaaqxdXeNuJgpBE53u6XjJeJcLsjmUWcUI/7jhcXIuYosLf/vpX1nmhVvC+F2uVYWA3SILE8XjHusz4YaDrRoaxY56npsgNGKtwXd/EHz3GaKbjHUprMRN2MpqUsZojhcDd3R0aLaH3nWeek6BMpZ7zZFNKDONO0CxjOJ1OTRASGS92zf9NINacMqVmnLdM80TXWXZjT1hnEWSoZiysFaZqai2sy4Rz4slmDSyTWLaEKLm5nR9E/JFz46Up5vlEyQljFZ3rGXtpgkuSZielwI8/fo9W8PLlH/HOiGimJowSQYFCvAKHoUMjzZZ3mpAkQ7jrLc42M2S1jUSFFznuRkl22U62lqm8LjPOapyVRkZ4a5lacxN2JLw3AkerirUidkErjKl4C9YorNEoKyjkJgpJORLDjLZiFu6MJ4SMcwM5FX589QOHix3XF5cYZRj6QZCn1pAarVFKU1pUmncOncWKKKfMdDxxcXGB1RqsY13kRkK1RbiWQkkR7aCWTNc5bm5WKopxGAAxPz8cDpRSmKcjx7vMfjdK9m8IDJ0n5CyfpTGt0dEoY3G2Y1rv8J1HGwVYcq4oRCxTc6GmjLOeYai8fv0jJSf6iys65wlhBQNGN45mFSsc4frBNB25u30NVA77C65fvOTm5jVhjRLJ1nf0fse8HAnzwsXlNbtxxw8/fC+8WWtxRrdjYqhdZV4m2beSscZzcXHJPE3N+seBgznIaH2dlpZSs4NSWcMCVdTHxmo6O8oYvlRq+aUMYn66ylZBaNw+Vak03gT1QYPW/ivcGuAMGj8o9cZCIb962Jw8ZAq17TxsAD/YtH2AY/R2E/mRtfFtPOWxpfqNxzyyvfe9xJPOli8B7B59Qn36a39Gfcju6P0Utq+1N09opH7qXuiRl3//Hj3c369xDB45M98+39938n7Wyz9yvN86vu98zurBCz7oTSsVtKYoTdH5Sa/+dJ8/EkGBqaByQVvDbn/AdhZlDaEpXmMKGG+hed+Vkuh9R296QKO8bm9AGiKnHLazhBQxzmKcZMOG1OLkjKPzFaUllSKWhKKQ88rx+FeMMVxc/gFrDTFlYioo47FWA8LzEp9BQYQqirGXBAqlNDFknOkpKGpMUKpElyECl1IKa5VsY1sqfT9gtUFrS6mKZQ1QCtoAFIxSWOfJKaKVpaJJzUy4UMXSC8U47lDGUoE1RaquOGUY2+jNdh0xF1HOOrE5sZazmlQpQ6qyeGRVG1+qSP5uAQyscRalsLMUXUEVvLM434OGXBPFKNaSQGUuLw9yvJrIIqlCLTJKPxz2dLuBQiZVJMmi1hY9d4HrOmLOhGWlVgXaEmqBmrFW49R4bnwySlA6Vai6YpRqivAoxy5DKlCrxtqB/a4TzpyzsIrp9Tj2HHY7GRUWWNcojWQToaAMRUFW0gzlKCO3pMRGphaF9wPOD6IIpmKURhvx+kPJGNJ5DYgYJlUxAR/HPSUVMJZUCyWJXc0weHQdWE4zIQas0ZSsMM5hxMOHWltTh6KWjDGKYTxQciHEhLOW/e6SGOMZQa1VFOfaOi4vr0UhXRD7GC2WQ6oRPdS2bSU3PFpdNLqE4nRa0EYx7g/nq0qKGescXR2Ei9sSSK6vXxCCoNQoQ0W+P9ponPEo22G9oWThYmrEBzOp0kQfA3HNaNcRkoidjBsYu7GJXQpVi2rbFsM6JUJcn3o5+oetrO6odY/JUI3wae/ZfPeznIfylzcJ3xti8e5q8zaS8eF1+lNXq79/Y/60vuPL9vPvz228/+R/Tj7e+86pn+O1P16f0v5/+qu+/xlffi69+1k+Ml7+SL0phWsdn9o2JdeQqhVGaVRnUXZ+2v7V+vbt43M913M913M913M913P9WuvXn6T+XM/1XM/1XM/1XM/1XOd6bv6e67me67me67me67l+Q/Xc/D3Xcz3Xcz3Xcz3Xc/2G6rn5e67neq7neq7neq7n+g3Vc/P3XM/1XM/1XM/1XM/1G6rn5u+5nuu5nuu5nuu5nus3VM/N33M913M913M913M912+onpu/53qu53qu53qu53qu31A9N3/P9VzP9VzP9VzP9Vy/ofr/A7CQkvvPQ2s7AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# visulization\n", + "generated_tokens = vq_model.quantize.get_codebook_entry_for_lvm(outputs.sequences[:, -256:])\n", + "generated_img = vq_model.decode(generated_tokens.view(1, generated_tokens.shape[1] // 16, 16, -1).permute(0, 3, 1, 2))\n", + "generated_img_rec = convert_decode_to_pil(generated_img)[0]\n", + "\n", + "fig, axes = plt.subplots(1, 2, figsize=(8, 4))\n", + "axes[0].imshow(input_img)\n", + "axes[1].imshow(generated_img_rec)\n", + "for ax in axes:\n", + " ax.axis('off')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "PyTorch-1.10.2", + "language": "python", + "name": "pytorch-1.10.2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/InternLM/tools/model_hf/__init__.py b/InternLM/tools/model_hf/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/InternLM/tools/model_hf/modeling_internlm.py b/InternLM/tools/model_hf/modeling_internlm.py new file mode 100644 index 0000000000000000000000000000000000000000..07956acc7f49a7d884c9e3924132387fcbd34594 --- /dev/null +++ b/InternLM/tools/model_hf/modeling_internlm.py @@ -0,0 +1,1062 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch InternLM model.""" +import math +import queue +import threading +from typing import List, Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss +from transformers.activations import ACT2FN +from transformers.configuration_utils import PretrainedConfig +from transformers.generation.streamers import BaseStreamer +from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, \ + SequenceClassifierOutputWithPast +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, \ + replace_return_docstrings + +logger = logging.get_logger(__name__) + +INTERNLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {} +_CONFIG_FOR_DOC = "InternLMConfig" + + +class InternLMConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate an InternLM + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the InternLM-7B. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the InternLM model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`InternLMModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings(`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + """ + model_type = "internlm" + _auto_class = "AutoConfig" + + def __init__( + self, + vocab_size=103168, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=None, + eos_token_id=None, + tie_word_embeddings=False, + bias=True, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.bias = bias + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +# Copied from transformers.models.bart.modeling_bart._expand_mask +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +class InternLMRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + InternLMRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + + # model_hf into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states + + +class InternLMRotaryEmbedding(torch.nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim)) + self.register_buffer("inv_freq", inv_freq) + + # Build here to make `torch.jit.trace` work. + self.max_seq_len_cached = max_position_embeddings + t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False) + self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case. + if seq_len > self.max_seq_len_cached: + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1).to(x.device) + self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False) + self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False) + return ( + self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype), + self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype), + ) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2:] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids): + # The first two dimensions of cos and sin are always 1, so we can `squeeze` them. + cos = cos.squeeze(1).squeeze(0) # [seq_len, dim] + sin = sin.squeeze(1).squeeze(0) # [seq_len, dim] + cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class InternLMMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + ): + super().__init__() + self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False) + self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.act_fn = ACT2FN[hidden_act] + + def forward(self, x): + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +class InternLMAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: InternLMConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.max_position_embeddings = config.max_position_embeddings + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias) + self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias) + self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias) + self.rotary_emb = InternLMRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + # [bsz, nh, t, hd] + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)) + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class InternLMDecoderLayer(nn.Module): + def __init__(self, config: InternLMConfig): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = InternLMAttention(config=config) + self.mlp = InternLMMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + ) + self.input_layernorm = InternLMRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = InternLMRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +INTERNLM_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`InternLMConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare InternLM Model outputting raw hidden-states without any specific head on top.", + INTERNLM_START_DOCSTRING, +) +class InternLMPreTrainedModel(PreTrainedModel): + config_class = InternLMConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["InternLMDecoderLayer"] + _keys_to_ignore_on_load_unexpected = [r"decoder\.version"] + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, InternLMModel): + module.gradient_checkpointing = value + + +INTERNLM_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to model_hf `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +@add_start_docstrings( + "The bare InternLM Model outputting raw hidden-states without any specific head on top.", + INTERNLM_START_DOCSTRING, +) +class InternLMModel(InternLMPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLMDecoderLayer`] + + Args: + config: InternLMConfig + """ + _auto_class = "AutoModel" + + def __init__(self, config: InternLMConfig): + super().__init__(config) + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) + self.layers = nn.ModuleList([InternLMDecoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.norm = InternLMRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + @add_start_docstrings_to_model_forward(INTERNLM_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + seq_length_with_past = seq_length + past_key_values_length = 0 + + if past_key_values is not None: + past_key_values_length = past_key_values[0][0].shape[2] + seq_length_with_past = seq_length_with_past + past_key_values_length + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + # embed positions + if attention_mask is None: + attention_mask = torch.ones( + (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device + ) + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length + ) + + hidden_states = inputs_embeds + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if use_cache else None + + for idx, decoder_layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + # None for past_key_value + return module(*inputs, output_attentions, None) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(decoder_layer), + hidden_states, + attention_mask, + position_ids, + None, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class InternLMForCausalLM(InternLMPreTrainedModel): + _auto_class = "AutoModelForCausalLM" + + def __init__(self, config): + super().__init__(config) + self.model = InternLMModel(config) + + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(INTERNLM_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + """ + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + if past_key_values: + input_ids = input_ids[:, -1:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -1].unsqueeze(-1) + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "position_ids": position_ids, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past + + def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = []): + prompt = "" + for record in history: + prompt += f"""<|User|>:{record[0]}\n<|Bot|>:{record[1]}\n""" + if len(prompt) == 0: + prompt += "" + prompt += f"""<|User|>:{query}\n<|Bot|>:""" + return tokenizer([prompt], return_tensors="pt") + + @torch.no_grad() + def chat(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = [], + streamer: Optional[BaseStreamer] = None, + max_new_tokens: int = 1024, + do_sample: bool = True, + temperature: float = 0.8, + top_p: float = 0.8, + **kwargs): + inputs = self.build_inputs(tokenizer, query, history) + inputs = {k: v.to(self.device) for k, v in inputs.items() if torch.is_tensor(v)} + outputs = self.generate(**inputs, + streamer=streamer, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + temperature=temperature, + top_p=top_p, + **kwargs) + outputs = outputs[0].cpu().tolist()[len(inputs["input_ids"][0]):] + response = tokenizer.decode(outputs, skip_special_tokens=True) + response = response.split("")[0] + history = history + [(query, response)] + return response, history + + @torch.no_grad() + def stream_chat(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = [], + max_new_tokens: int = 1024, + do_sample: bool = True, + temperature: float = 0.8, + top_p: float = 0.8, + **kwargs): + """ + Return a generator in format: (response, history) + Eg. + ('你好，有什么可以帮助您的吗', [('你好', '你好，有什么可以帮助您的吗')]) + ('你好，有什么可以帮助您的吗？', [('你好', '你好，有什么可以帮助您的吗？')]) + """ + + response_queue = queue.Queue(maxsize=20) + + class ChatStreamer(BaseStreamer): + def __init__(self, tokenizer) -> None: + super().__init__() + self.tokenizer = tokenizer + self.queue = response_queue + self.query = query + self.history = history + self.response = "" + self.received_inputs = False + self.queue.put((self.response, history + [(self.query, self.response)])) + + def put(self, value): + if len(value.shape) > 1 and value.shape[0] > 1: + raise ValueError("ChatStreamer only supports batch size 1") + elif len(value.shape) > 1: + value = value[0] + + if not self.received_inputs: + # The first received value is input_ids, ignore here + self.received_inputs = True + return + + token = self.tokenizer.decode([value[-1]], skip_special_tokens=True) + if token.strip() != "": + self.response = self.response + token + history = self.history + [(self.query, self.response)] + self.queue.put((self.response, history)) + + def end(self): + self.queue.put(None) + + def stream_producer(): + return self.chat( + tokenizer=tokenizer, + query=query, + streamer=ChatStreamer(tokenizer=tokenizer), + history=history, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + temperature=temperature, + top_p=top_p, + **kwargs + ) + + def consumer(): + producer = threading.Thread(target=stream_producer) + producer.start() + while True: + res = response_queue.get() + if res is not None: + return + yield res + + return consumer() + + +@add_start_docstrings( + """ + The InternLM Model transformer with a sequence classification head on top (linear layer). + + [`InternLMForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + INTERNLM_START_DOCSTRING, +) +class InternLMForSequenceClassification(InternLMPreTrainedModel): + _keys_to_ignore_on_load_missing = [r"lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = InternLMModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(INTERNLM_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + sequence_lengths = (torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1).to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + labels = labels.to(logits.device) + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) diff --git a/InternLM/tools/model_hf/modeling_vit.py b/InternLM/tools/model_hf/modeling_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..05a90cf4da8c2cd7e03070e96fdbc92c32693d36 --- /dev/null +++ b/InternLM/tools/model_hf/modeling_vit.py @@ -0,0 +1,1048 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch InternLM model.""" +import math +import queue +import threading +from typing import List, Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss +from transformers.activations import ACT2FN +from transformers.configuration_utils import PretrainedConfig +from transformers.generation.streamers import BaseStreamer +from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, \ + SequenceClassifierOutputWithPast +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, \ + replace_return_docstrings + +logger = logging.get_logger(__name__) + +INTERNLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {} +_CONFIG_FOR_DOC = "InternLMConfig" + + +class InternLMConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate an InternLM + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the InternLM-7B. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the InternLM model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`InternLMModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings(`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + """ + model_type = "vit" + _auto_class = "AutoConfig" + + def __init__( + self, + vocab_size=103168, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + hidden_act="gelu", + max_position_embeddings=2048, + initializer_range=0.02, + norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=None, + eos_token_id=None, + tie_word_embeddings=False, + bias=True, + mlp_bias=False, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.norm_eps = norm_eps + self.use_cache = use_cache + self.bias = bias + self.mlp_bias = mlp_bias + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +# Copied from transformers.models.bart.modeling_bart._expand_mask +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +class InternLMRotaryEmbedding(torch.nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim)) + self.register_buffer("inv_freq", inv_freq) + + # Build here to make `torch.jit.trace` work. + self.max_seq_len_cached = max_position_embeddings + t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False) + self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case. + if seq_len > self.max_seq_len_cached: + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1).to(x.device) + self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False) + self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False) + return ( + self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype), + self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype), + ) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2:] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids): + # The first two dimensions of cos and sin are always 1, so we can `squeeze` them. + cos = cos.squeeze(1).squeeze(0) # [seq_len, dim] + sin = sin.squeeze(1).squeeze(0) # [seq_len, dim] + cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class InternLMMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + mlp_bias: bool = False + ): + super().__init__() + self.fc1 = nn.Linear(hidden_size, intermediate_size, bias=mlp_bias) + self.act_fn = ACT2FN[hidden_act] + self.fc2 = nn.Linear(intermediate_size, hidden_size, bias=mlp_bias) + + def forward(self, x): + x = self.fc1(x) + x = self.act_fn(x) + x = self.fc2(x) + return x + + +class InternLMAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: InternLMConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.max_position_embeddings = config.max_position_embeddings + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias) + self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias) + self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias) + self.rotary_emb = InternLMRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + # [bsz, nh, t, hd] + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)) + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class InternLMDecoderLayer(nn.Module): + def __init__(self, config: InternLMConfig): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = InternLMAttention(config=config) + self.mlp = InternLMMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + mlp_bias=config.mlp_bias + ) + self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +INTERNLM_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`InternLMConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare InternLM Model outputting raw hidden-states without any specific head on top.", + INTERNLM_START_DOCSTRING, +) +class InternLMPreTrainedModel(PreTrainedModel): + config_class = InternLMConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["InternLMDecoderLayer"] + _keys_to_ignore_on_load_unexpected = [r"decoder\.version"] + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, InternLMModel): + module.gradient_checkpointing = value + + +INTERNLM_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to model_hf `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +@add_start_docstrings( + "The bare InternLM Model outputting raw hidden-states without any specific head on top.", + INTERNLM_START_DOCSTRING, +) +class InternLMModel(InternLMPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLMDecoderLayer`] + + Args: + config: InternLMConfig + """ + _auto_class = "AutoModel" + + def __init__(self, config: InternLMConfig): + super().__init__(config) + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) + self.layers = nn.ModuleList([InternLMDecoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + @add_start_docstrings_to_model_forward(INTERNLM_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + seq_length_with_past = seq_length + past_key_values_length = 0 + + if past_key_values is not None: + past_key_values_length = past_key_values[0][0].shape[2] + seq_length_with_past = seq_length_with_past + past_key_values_length + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + # embed positions + if attention_mask is None: + attention_mask = torch.ones( + (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device + ) + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length + ) + + hidden_states = inputs_embeds + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if use_cache else None + + for idx, decoder_layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + # None for past_key_value + return module(*inputs, output_attentions, None) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(decoder_layer), + hidden_states, + attention_mask, + position_ids, + None, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class InternLMForCausalLM(InternLMPreTrainedModel): + _auto_class = "AutoModelForCausalLM" + + def __init__(self, config): + super().__init__(config) + self.model = InternLMModel(config) + + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(INTERNLM_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + """ + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + if past_key_values: + input_ids = input_ids[:, -1:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -1].unsqueeze(-1) + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "position_ids": position_ids, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past + + def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = []): + prompt = "" + for record in history: + prompt += f"""<|User|>:{record[0]}\n<|Bot|>:{record[1]}\n""" + if len(prompt) == 0: + prompt += "" + prompt += f"""<|User|>:{query}\n<|Bot|>:""" + return tokenizer([prompt], return_tensors="pt") + + @torch.no_grad() + def chat(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = [], + streamer: Optional[BaseStreamer] = None, + max_new_tokens: int = 1024, + do_sample: bool = True, + temperature: float = 0.8, + top_p: float = 0.8, + **kwargs): + inputs = self.build_inputs(tokenizer, query, history) + inputs = {k: v.to(self.device) for k, v in inputs.items() if torch.is_tensor(v)} + outputs = self.generate(**inputs, + streamer=streamer, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + temperature=temperature, + top_p=top_p, + **kwargs) + outputs = outputs[0].cpu().tolist()[len(inputs["input_ids"][0]):] + response = tokenizer.decode(outputs, skip_special_tokens=True) + response = response.split("")[0] + history = history + [(query, response)] + return response, history + + @torch.no_grad() + def stream_chat(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = [], + max_new_tokens: int = 1024, + do_sample: bool = True, + temperature: float = 0.8, + top_p: float = 0.8, + **kwargs): + """ + Return a generator in format: (response, history) + Eg. + ('你好，有什么可以帮助您的吗', [('你好', '你好，有什么可以帮助您的吗')]) + ('你好，有什么可以帮助您的吗？', [('你好', '你好，有什么可以帮助您的吗？')]) + """ + + response_queue = queue.Queue(maxsize=20) + + class ChatStreamer(BaseStreamer): + def __init__(self, tokenizer) -> None: + super().__init__() + self.tokenizer = tokenizer + self.queue = response_queue + self.query = query + self.history = history + self.response = "" + self.received_inputs = False + self.queue.put((self.response, history + [(self.query, self.response)])) + + def put(self, value): + if len(value.shape) > 1 and value.shape[0] > 1: + raise ValueError("ChatStreamer only supports batch size 1") + elif len(value.shape) > 1: + value = value[0] + + if not self.received_inputs: + # The first received value is input_ids, ignore here + self.received_inputs = True + return + + token = self.tokenizer.decode([value[-1]], skip_special_tokens=True) + if token.strip() != "": + self.response = self.response + token + history = self.history + [(self.query, self.response)] + self.queue.put((self.response, history)) + + def end(self): + self.queue.put(None) + + def stream_producer(): + return self.chat( + tokenizer=tokenizer, + query=query, + streamer=ChatStreamer(tokenizer=tokenizer), + history=history, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + temperature=temperature, + top_p=top_p, + **kwargs + ) + + def consumer(): + producer = threading.Thread(target=stream_producer) + producer.start() + while True: + res = response_queue.get() + if res is not None: + return + yield res + + return consumer() + + +@add_start_docstrings( + """ + The InternLM Model transformer with a sequence classification head on top (linear layer). + + [`InternLMForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + INTERNLM_START_DOCSTRING, +) +class InternLMForSequenceClassification(InternLMPreTrainedModel): + _keys_to_ignore_on_load_missing = [r"lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = InternLMModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(INTERNLM_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + sequence_lengths = (torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1).to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + labels = labels.to(logits.device) + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) diff --git a/InternLM/tools/model_hf/muse/__init__.py b/InternLM/tools/model_hf/muse/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ceaa3939a2455e319af2aec89f15b586e12eda4d --- /dev/null +++ b/InternLM/tools/model_hf/muse/__init__.py @@ -0,0 +1,18 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.0.1" + +from .modeling_taming_vqgan import VQGANModel diff --git a/InternLM/tools/model_hf/muse/logging.py b/InternLM/tools/model_hf/muse/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..65814a82380e47e54434c4be97026141772f7298 --- /dev/null +++ b/InternLM/tools/model_hf/muse/logging.py @@ -0,0 +1,338 @@ +# coding=utf-8 +# Copyright 2023 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Logging utilities.""" + +import logging +import os +import sys +import threading +from logging import CRITICAL # NOQA +from logging import DEBUG # NOQA +from logging import ERROR # NOQA +from logging import FATAL # NOQA +from logging import INFO # NOQA +from logging import NOTSET # NOQA +from logging import WARN # NOQA +from logging import WARNING # NOQA +from typing import Optional + +from tqdm import auto as tqdm_lib + +_lock = threading.Lock() +_default_handler: Optional[logging.Handler] = None + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + +_tqdm_active = True + + +def _get_default_logging_level(): + """ + If muse_VERBOSITY env var is set to one of the valid choices return that as the new default level. If it is + not - fall back to `_default_log_level` + """ + env_level_str = os.getenv("muse_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option muse_VERBOSITY={env_level_str}, has to be one of: { ', '.join(log_levels.keys()) }" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + global _default_handler + + with _lock: + if _default_handler: + # This library has already configured the library root logger. + return + _default_handler = logging.StreamHandler() # Set sys.stderr as stream. + _default_handler.flush = sys.stderr.flush + + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.addHandler(_default_handler) + library_root_logger.setLevel(_get_default_logging_level()) + library_root_logger.propagate = False + + +def _reset_library_root_logger() -> None: + global _default_handler + + with _lock: + if not _default_handler: + return + + library_root_logger = _get_library_root_logger() + library_root_logger.removeHandler(_default_handler) + library_root_logger.setLevel(logging.NOTSET) + _default_handler = None + + +def get_log_levels_dict(): + return log_levels + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """ + Return a logger with the specified name. + + This function is not supposed to be directly accessed unless you are writing a custom muse module. + """ + + if name is None: + name = _get_library_name() + + _configure_library_root_logger() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """ + Return the current level for the 🤗 muse' root logger as an int. + + Returns: + `int`: The logging level. + + + + 🤗 muse has following logging levels: + + - 50: `muse.logging.CRITICAL` or `muse.logging.FATAL` + - 40: `muse.logging.ERROR` + - 30: `muse.logging.WARNING` or `muse.logging.WARN` + - 20: `muse.logging.INFO` + - 10: `muse.logging.DEBUG` + + """ + + _configure_library_root_logger() + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """ + Set the verbosity level for the 🤗 muse' root logger. + + Args: + verbosity (`int`): + Logging level, e.g., one of: + + - `muse.logging.CRITICAL` or `muse.logging.FATAL` + - `muse.logging.ERROR` + - `muse.logging.WARNING` or `muse.logging.WARN` + - `muse.logging.INFO` + - `muse.logging.DEBUG` + """ + + _configure_library_root_logger() + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """Set the verbosity to the `INFO` level.""" + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """Set the verbosity to the `WARNING` level.""" + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """Set the verbosity to the `DEBUG` level.""" + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """Set the verbosity to the `ERROR` level.""" + return set_verbosity(ERROR) + + +def disable_default_handler() -> None: + """Disable the default handler of the HuggingFace muse' root logger.""" + + _configure_library_root_logger() + + assert _default_handler is not None + _get_library_root_logger().removeHandler(_default_handler) + + +def enable_default_handler() -> None: + """Enable the default handler of the HuggingFace muse' root logger.""" + + _configure_library_root_logger() + + assert _default_handler is not None + _get_library_root_logger().addHandler(_default_handler) + + +def add_handler(handler: logging.Handler) -> None: + """adds a handler to the HuggingFace muse' root logger.""" + + _configure_library_root_logger() + + assert handler is not None + _get_library_root_logger().addHandler(handler) + + +def remove_handler(handler: logging.Handler) -> None: + """removes given handler from the HuggingFace muse' root logger.""" + + _configure_library_root_logger() + + assert handler is not None and handler not in _get_library_root_logger().handlers + _get_library_root_logger().removeHandler(handler) + + +def disable_propagation() -> None: + """ + Disable propagation of the library log outputs. Note that log propagation is disabled by default. + """ + + _configure_library_root_logger() + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """ + Enable propagation of the library log outputs. Please disable the HuggingFace muse' default handler to prevent + double logging if the root logger has been configured. + """ + + _configure_library_root_logger() + _get_library_root_logger().propagate = True + + +def enable_explicit_format() -> None: + """ + Enable explicit formatting for every HuggingFace muse' logger. The explicit formatter is as follows: + ``` + [LEVELNAME|FILENAME|LINE NUMBER] TIME >> MESSAGE + ``` + All handlers currently bound to the root logger are affected by this method. + """ + handlers = _get_library_root_logger().handlers + + for handler in handlers: + formatter = logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s") + handler.setFormatter(formatter) + + +def reset_format() -> None: + """ + Resets the formatting for HuggingFace muse' loggers. + + All handlers currently bound to the root logger are affected by this method. + """ + handlers = _get_library_root_logger().handlers + + for handler in handlers: + handler.setFormatter(None) + + +def warning_advice(self, *args, **kwargs): + """ + This method is identical to `logger.warning()`, but if env var muse_NO_ADVISORY_WARNINGS=1 is set, this + warning will not be printed + """ + no_advisory_warnings = os.getenv("muse_NO_ADVISORY_WARNINGS", False) + if no_advisory_warnings: + return + self.warning(*args, **kwargs) + + +logging.Logger.warning_advice = warning_advice + + +class EmptyTqdm: + """Dummy tqdm which doesn't do anything.""" + + def __init__(self, *args, **kwargs): # pylint: disable=unused-argument + self._iterator = args[0] if args else None + + def __iter__(self): + return iter(self._iterator) + + def __getattr__(self, _): + """Return empty function.""" + + def empty_fn(*args, **kwargs): # pylint: disable=unused-argument + return + + return empty_fn + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + return + + +class _tqdm_cls: + def __call__(self, *args, **kwargs): + if _tqdm_active: + return tqdm_lib.tqdm(*args, **kwargs) + else: + return EmptyTqdm(*args, **kwargs) + + def set_lock(self, *args, **kwargs): + self._lock = None + if _tqdm_active: + return tqdm_lib.tqdm.set_lock(*args, **kwargs) + + def get_lock(self): + if _tqdm_active: + return tqdm_lib.tqdm.get_lock() + + +tqdm = _tqdm_cls() + + +def is_progress_bar_enabled() -> bool: + """Return a boolean indicating whether tqdm progress bars are enabled.""" + global _tqdm_active + return bool(_tqdm_active) + + +def enable_progress_bar(): + """Enable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = True + + +def disable_progress_bar(): + """Disable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = False diff --git a/InternLM/tools/model_hf/muse/modeling_taming_vqgan.py b/InternLM/tools/model_hf/muse/modeling_taming_vqgan.py new file mode 100644 index 0000000000000000000000000000000000000000..ab0003ea7b5da279859d5636660f320ef2cde651 --- /dev/null +++ b/InternLM/tools/model_hf/muse/modeling_taming_vqgan.py @@ -0,0 +1,591 @@ +# coding=utf-8 +# Copyright 2023 The Taming Transformers Authors and The HuggingFace Inc. team. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from functools import partial +from typing import Tuple + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn + +from .modeling_utils import ConfigMixin, ModelMixin, register_to_config + + +class Upsample(nn.Module): + def __init__(self, in_channels: int, with_conv: bool): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = nn.Conv2d( + in_channels, + in_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, hidden_states): + hidden_states = torch.nn.functional.interpolate(hidden_states, scale_factor=2.0, mode="nearest") + if self.with_conv: + hidden_states = self.conv(hidden_states) + return hidden_states + + +class Downsample(nn.Module): + def __init__(self, in_channels: int, with_conv: bool): + super().__init__() + + self.with_conv = with_conv + if self.with_conv: + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, hidden_states): + if self.with_conv: + pad = (0, 1, 0, 1) # pad height and width dim + hidden_states = torch.nn.functional.pad(hidden_states, pad, mode="constant", value=0) + hidden_states = self.conv(hidden_states) + else: + hidden_states = torch.nn.functional.avg_pool2d(hidden_states, kernel_size=2, stride=2) + return hidden_states + + +class ResnetBlock(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int = None, + use_conv_shortcut: bool = False, + dropout_prob: float = 0.0, + ): + super().__init__() + + self.in_channels = in_channels + self.out_channels = out_channels + self.out_channels_ = self.in_channels if self.out_channels is None else self.out_channels + self.use_conv_shortcut = use_conv_shortcut + + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=3, + stride=1, + padding=1, + ) + + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=self.out_channels_, eps=1e-6, affine=True) + self.dropout = nn.Dropout(dropout_prob) + self.conv2 = nn.Conv2d( + self.out_channels_, + self.out_channels_, + kernel_size=3, + stride=(1, 1), + padding=1, + ) + + if self.in_channels != self.out_channels_: + if use_conv_shortcut: + self.conv_shortcut = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=3, + stride=1, + padding=1, + ) + else: + self.nin_shortcut = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=1, + stride=1, + padding=0, + ) + + def forward(self, hidden_states): + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.in_channels != self.out_channels_: + if self.use_conv_shortcut: + residual = self.conv_shortcut(residual) + else: + residual = self.nin_shortcut(residual) + + return hidden_states + residual + + +class AttnBlock(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + + self.in_channels = in_channels + conv = partial(nn.Conv2d, self.in_channels, self.in_channels, kernel_size=1, stride=1, padding=0) + + self.norm = nn.GroupNorm(num_groups=32, num_channels=self.in_channels, eps=1e-6, affine=True) + self.q, self.k, self.v = conv(), conv(), conv() + self.proj_out = conv() + + def forward(self, hidden_states): + residual = hidden_states + hidden_states = self.norm(hidden_states) + + query = self.q(hidden_states) + key = self.k(hidden_states) + value = self.v(hidden_states) + + # compute attentions + batch, channels, height, width = query.shape + query = query.reshape((batch, channels, height * width)) + query = query.permute(0, 2, 1) # (b, hw, c) + key = key.reshape((batch, channels, height * width)) + + attn_weights = torch.bmm(query, key) # b,hw,hw + attn_weights = attn_weights * (int(channels) ** -0.5) + attn_weights = nn.functional.softmax(attn_weights, dim=2) + + # attend to values + value = value.reshape((batch, channels, height * width)) + attn_weights = attn_weights.permute(0, 2, 1) + hidden_states = torch.bmm(value, attn_weights) + hidden_states = hidden_states.reshape((batch, channels, height, width)) + + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states + residual + return hidden_states + + +class UpsamplingBlock(nn.Module): + def __init__(self, config, curr_res: int, block_idx: int): + super().__init__() + + self.config = config + self.block_idx = block_idx + self.curr_res = curr_res + + if self.block_idx == self.config.num_resolutions - 1: + block_in = self.config.hidden_channels * self.config.channel_mult[-1] + else: + block_in = self.config.hidden_channels * self.config.channel_mult[self.block_idx + 1] + + block_out = self.config.hidden_channels * self.config.channel_mult[self.block_idx] + + res_blocks = [] + attn_blocks = [] + for _ in range(self.config.num_res_blocks + 1): + res_blocks.append(ResnetBlock(block_in, block_out, dropout_prob=self.config.dropout)) + block_in = block_out + if self.curr_res in self.config.attn_resolutions: + attn_blocks.append(AttnBlock(block_in)) + + self.block = nn.ModuleList(res_blocks) + self.attn = nn.ModuleList(attn_blocks) + + self.upsample = None + if self.block_idx != 0: + self.upsample = Upsample(block_in, self.config.resample_with_conv) + + def forward(self, hidden_states): + for i, res_block in enumerate(self.block): + hidden_states = res_block(hidden_states) + if len(self.attn) > 1: + hidden_states = self.attn[i](hidden_states) + + if self.upsample is not None: + hidden_states = self.upsample(hidden_states) + + return hidden_states + + +class DownsamplingBlock(nn.Module): + def __init__(self, config, curr_res: int, block_idx: int): + super().__init__() + + self.config = config + self.curr_res = curr_res + self.block_idx = block_idx + + in_channel_mult = (1,) + tuple(self.config.channel_mult) + block_in = self.config.hidden_channels * in_channel_mult[self.block_idx] + block_out = self.config.hidden_channels * self.config.channel_mult[self.block_idx] + + res_blocks = nn.ModuleList() + attn_blocks = nn.ModuleList() + for _ in range(self.config.num_res_blocks): + res_blocks.append(ResnetBlock(block_in, block_out, dropout_prob=self.config.dropout)) + block_in = block_out + if self.curr_res in self.config.attn_resolutions: + attn_blocks.append(AttnBlock(block_in)) + + self.block = res_blocks + self.attn = attn_blocks + + self.downsample = None + if self.block_idx != self.config.num_resolutions - 1: + self.downsample = Downsample(block_in, self.config.resample_with_conv) + + def forward(self, hidden_states): + for i, res_block in enumerate(self.block): + hidden_states = res_block(hidden_states) + if len(self.attn) > 1: + hidden_states = self.attn[i](hidden_states) + + if self.downsample is not None: + hidden_states = self.downsample(hidden_states) + + return hidden_states + + +class MidBlock(nn.Module): + def __init__(self, config, in_channels: int, no_attn: False, dropout: float): + super().__init__() + + self.config = config + self.in_channels = in_channels + self.no_attn = no_attn + self.dropout = dropout + + self.block_1 = ResnetBlock( + self.in_channels, + self.in_channels, + dropout_prob=self.dropout, + ) + if not no_attn: + self.attn_1 = AttnBlock(self.in_channels) + self.block_2 = ResnetBlock( + self.in_channels, + self.in_channels, + dropout_prob=self.dropout, + ) + + def forward(self, hidden_states): + hidden_states = self.block_1(hidden_states) + if not self.no_attn: + hidden_states = self.attn_1(hidden_states) + hidden_states = self.block_2(hidden_states) + return hidden_states + + +class Encoder(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + + # downsampling + self.conv_in = nn.Conv2d( + self.config.num_channels, + self.config.hidden_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + curr_res = self.config.resolution + downsample_blocks = [] + for i_level in range(self.config.num_resolutions): + downsample_blocks.append(DownsamplingBlock(self.config, curr_res, block_idx=i_level)) + + if i_level != self.config.num_resolutions - 1: + curr_res = curr_res // 2 + self.down = nn.ModuleList(downsample_blocks) + + # middle + mid_channels = self.config.hidden_channels * self.config.channel_mult[-1] + self.mid = MidBlock(config, mid_channels, self.config.no_attn_mid_block, self.config.dropout) + + # end + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=mid_channels, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d( + mid_channels, + self.config.z_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, pixel_values): + # downsampling + hidden_states = self.conv_in(pixel_values) + for block in self.down: + hidden_states = block(hidden_states) + + # middle + hidden_states = self.mid(hidden_states) + + # end + hidden_states = self.norm_out(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class Decoder(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + + # compute in_channel_mult, block_in and curr_res at lowest res + block_in = self.config.hidden_channels * self.config.channel_mult[self.config.num_resolutions - 1] + curr_res = self.config.resolution // 2 ** (self.config.num_resolutions - 1) + self.z_shape = (1, self.config.z_channels, curr_res, curr_res) + + # z to block_in + self.conv_in = nn.Conv2d( + self.config.z_channels, + block_in, + kernel_size=3, + stride=1, + padding=1, + ) + + # middle + self.mid = MidBlock(config, block_in, self.config.no_attn_mid_block, self.config.dropout) + + # upsampling + upsample_blocks = [] + for i_level in reversed(range(self.config.num_resolutions)): + upsample_blocks.append(UpsamplingBlock(self.config, curr_res, block_idx=i_level)) + if i_level != 0: + curr_res = curr_res * 2 + self.up = nn.ModuleList(list(reversed(upsample_blocks))) # reverse to get consistent order + + # end + block_out = self.config.hidden_channels * self.config.channel_mult[0] + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_out, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d( + block_out, + self.config.num_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, hidden_states): + # z to block_in + hidden_states = self.conv_in(hidden_states) + + # middle + hidden_states = self.mid(hidden_states) + + # upsampling + for block in reversed(self.up): + hidden_states = block(hidden_states) + + # end + hidden_states = self.norm_out(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class VectorQuantizer(nn.Module): + """ + see https://github.com/MishaLaskin/vqvae/blob/d761a999e2267766400dc646d82d3ac3657771d4/models/quantizer.py + Discretization bottleneck part of the VQ-VAE. + """ + + def __init__(self, num_embeddings, embedding_dim, commitment_cost): + r""" + Args: + num_embeddings: number of vectors in the quantized space. + embedding_dim: dimensionality of the tensors in the quantized space. + Inputs to the modules must be in this format as well. + commitment_cost: scalar which controls the weighting of the loss terms + (see equation 4 in the paper https://arxiv.org/abs/1711.00937 - this variable is Beta). + """ + super().__init__() + + self.num_embeddings = num_embeddings + self.embedding_dim = embedding_dim + self.commitment_cost = commitment_cost + + self.embedding = nn.Embedding(num_embeddings, embedding_dim) + self.embedding.weight.data.uniform_(-1.0 / num_embeddings, 1.0 / num_embeddings) + + def forward(self, hidden_states, return_loss=False): + """ + Inputs the output of the encoder network z and maps it to a discrete one-hot vector that is the index of the + closest embedding vector e_j z (continuous) -> z_q (discrete) z.shape = (batch, channel, height, width) + quantization pipeline: + 1. get encoder input (B,C,H,W) + 2. flatten input to (B*H*W,C) + """ + # reshape z -> (batch, height, width, channel) and flatten + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() + + distances = self.compute_distances(hidden_states) + min_encoding_indices = torch.argmin(distances, axis=1).unsqueeze(1) + min_encodings = torch.zeros(min_encoding_indices.shape[0], self.num_embeddings).to(hidden_states) + min_encodings.scatter_(1, min_encoding_indices, 1) + + # get quantized latent vectors + z_q = torch.matmul(min_encodings, self.embedding.weight).view(hidden_states.shape) + + # reshape to (batch, num_tokens) + min_encoding_indices = min_encoding_indices.reshape(hidden_states.shape[0], -1) + + # compute loss for embedding + loss = None + if return_loss: + loss = torch.mean((z_q.detach() - hidden_states) ** 2) + self.commitment_cost * torch.mean( + (z_q - hidden_states.detach()) ** 2 + ) + # preserve gradients + z_q = hidden_states + (z_q - hidden_states).detach() + + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + return z_q, min_encoding_indices, loss + + def compute_distances(self, hidden_states): + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z + hidden_states_flattended = hidden_states.reshape((-1, self.embedding_dim)) + emb_weights = self.embedding.weight.t() + + inputs_norm_sq = hidden_states_flattended.pow(2.0).sum(dim=1, keepdim=True) + codebook_t_norm_sq = emb_weights.pow(2.0).sum(dim=0, keepdim=True) + distances = torch.addmm( + inputs_norm_sq + codebook_t_norm_sq, + hidden_states_flattended, + emb_weights, + alpha=-2.0, + ) + return distances + + def get_codebook_entry(self, indices): + # indices are expected to be of shape (batch, num_tokens) + # get quantized latent vectors + batch, num_tokens = indices.shape + z_q = self.embedding(indices) + z_q = z_q.reshape(batch, int(math.sqrt(num_tokens)), int(math.sqrt(num_tokens)), -1).permute(0, 3, 1, 2) + return z_q + + def get_codebook_entry_for_lvm(self, indices): + batch, num_tokens = indices.shape + z_q = self.embedding(indices) + z_q = z_q.reshape(batch, num_tokens, -1) + return z_q + + # adapted from https://github.com/kakaobrain/rq-vae-transformer/blob/main/rqvae/models/rqvae/quantizations.py#L372 + def get_soft_code(self, hidden_states, temp=1.0, stochastic=False): + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() # (batch, height, width, channel) + distances = self.compute_distances(hidden_states) # (batch * height * width, num_embeddings) + + soft_code = F.softmax(-distances / temp, dim=-1) # (batch * height * width, num_embeddings) + if stochastic: + code = torch.multinomial(soft_code, 1) # (batch * height * width, 1) + else: + code = distances.argmin(dim=-1) # (batch * height * width) + + code = code.reshape(hidden_states.shape[0], -1) # (batch, height * width) + batch, num_tokens = code.shape + soft_code = soft_code.reshape(batch, num_tokens, -1) # (batch, height * width, num_embeddings) + return soft_code, code + + def get_code(self, hidden_states): + # reshape z -> (batch, height, width, channel) + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() + distances = self.compute_distances(hidden_states) + indices = torch.argmin(distances, axis=1).unsqueeze(1) + indices = indices.reshape(hidden_states.shape[0], -1) + return indices + + +class VQGANModel(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + resolution: int = 256, + num_channels: int = 3, + hidden_channels: int = 128, + channel_mult: Tuple = (1, 1, 2, 2, 4), + num_res_blocks: int = 2, + attn_resolutions: int = (16,), + no_attn_mid_block: bool = False, + z_channels: int = 256, + num_embeddings: int = 1024, + quantized_embed_dim: int = 256, + dropout: float = 0.0, + resample_with_conv: bool = True, + commitment_cost: float = 0.25, + ): + super().__init__() + + self.config.num_resolutions = len(channel_mult) + self.config.reduction_factor = 2 ** (self.config.num_resolutions - 1) + self.config.latent_size = resolution // self.config.reduction_factor + + self.encoder = Encoder(self.config) + self.decoder = Decoder(self.config) + self.quantize = VectorQuantizer( + self.config.num_embeddings, self.config.quantized_embed_dim, self.config.commitment_cost + ) + self.quant_conv = nn.Conv2d( + self.config.z_channels, + self.config.quantized_embed_dim, + kernel_size=1, + ) + self.post_quant_conv = nn.Conv2d( + self.config.quantized_embed_dim, + self.config.z_channels, + kernel_size=1, + ) + + def encode(self, pixel_values, return_loss=False): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quantized_states, codebook_indices, codebook_loss = self.quantize(hidden_states, return_loss) + output = (quantized_states, codebook_indices) + if return_loss: + output = output + (codebook_loss,) + return output + + def decode(self, quantized_states): + hidden_states = self.post_quant_conv(quantized_states) + reconstructed_pixel_values = self.decoder(hidden_states) + return reconstructed_pixel_values + + def decode_code(self, codebook_indices): + quantized_states = self.quantize.get_codebook_entry(codebook_indices) + reconstructed_pixel_values = self.decode(quantized_states) + return reconstructed_pixel_values + + def get_code(self, pixel_values): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + codebook_indices = self.quantize.get_code(hidden_states) + return codebook_indices + + def forward(self, pixel_values, return_loss=False): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quantized_states, codebook_indices, codebook_loss = self.quantize(hidden_states, return_loss) + reconstructed_pixel_values = self.decode(quantized_states) + outputs = (reconstructed_pixel_values, quantized_states, codebook_indices) + if return_loss: + outputs = outputs + (codebook_loss,) + return outputs diff --git a/InternLM/tools/model_hf/muse/modeling_utils.py b/InternLM/tools/model_hf/muse/modeling_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f31c337e9f4715768928dbb7df60cda46a8cd3d3 --- /dev/null +++ b/InternLM/tools/model_hf/muse/modeling_utils.py @@ -0,0 +1,1170 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import inspect +import json +import os +from collections import OrderedDict +from functools import partial +from pathlib import PosixPath +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import accelerate +import numpy as np +import torch +from accelerate.utils import set_module_tensor_to_device +from huggingface_hub import hf_hub_download +from huggingface_hub.utils import ( + EntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from requests import HTTPError +from torch import Tensor, device + +from . import __version__, logging + +logger = logging.get_logger(__name__) + + +hf_cache_home = os.path.expanduser( + os.getenv("HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface")) +) +default_cache_path = os.path.join(hf_cache_home, "muse") + + +CONFIG_NAME = "config.json" +WEIGHTS_NAME = "pytorch_model.bin" +SAFETENSORS_WEIGHTS_NAME = "pytorch_model.safetensors" +HUGGINGFACE_CO_RESOLVE_ENDPOINT = "https://huggingface.co" +MUSE_CACHE = default_cache_path +MUSE_DYNAMIC_MODULE_NAME = "myse_modules" +HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(hf_cache_home, "modules")) + + +_LOW_CPU_MEM_USAGE_DEFAULT = True + + +def get_parameter_device(parameter: torch.nn.Module): + try: + return next(parameter.parameters()).device + except StopIteration: + # For torch.nn.DataParallel compatibility in PyTorch 1.5 + + def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] + return tuples + + gen = parameter._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].device + + +def get_parameter_dtype(parameter: torch.nn.Module): + try: + return next(parameter.parameters()).dtype + except StopIteration: + # For torch.nn.DataParallel compatibility in PyTorch 1.5 + + def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] + return tuples + + gen = parameter._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].dtype + + +def load_state_dict(checkpoint_file: Union[str, os.PathLike]): + """ + Reads a checkpoint file, returning properly formatted errors if they arise. + """ + try: + if os.path.basename(checkpoint_file) == WEIGHTS_NAME: + return torch.load(checkpoint_file, map_location="cpu") + except Exception as e: + try: + with open(checkpoint_file) as f: + if f.read().startswith("version"): + raise OSError( + "You seem to have cloned a repository without having git-lfs installed. Please install " + "git-lfs and run `git lfs install` followed by `git lfs pull` in the folder " + "you cloned." + ) + else: + raise ValueError( + f"Unable to locate the file {checkpoint_file} which is necessary to load this pretrained " + "model. Make sure you have saved the model properly." + ) from e + except (UnicodeDecodeError, ValueError): + raise OSError( + f"Unable to load weights from checkpoint file for '{checkpoint_file}' " + f"at '{checkpoint_file}'. " + "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True." + ) + + +def _load_state_dict_into_model(model_to_load, state_dict): + # Convert old format to new format if needed from a PyTorch state_dict + # copy state_dict so _load_from_state_dict can modify it + state_dict = state_dict.copy() + error_msgs = [] + + # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants + # so we need to apply the function recursively. + def load(module: torch.nn.Module, prefix=""): + args = (state_dict, prefix, {}, True, [], [], error_msgs) + module._load_from_state_dict(*args) + + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + ".") + + load(model_to_load) + + return error_msgs + + +def _get_model_file( + pretrained_model_name_or_path, + *, + weights_name, + subfolder, + cache_dir, + force_download, + proxies, + resume_download, + local_files_only, + use_auth_token, + user_agent, + revision, +): + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + if os.path.isfile(pretrained_model_name_or_path): + return pretrained_model_name_or_path + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, weights_name)): + # Load from a PyTorch checkpoint + model_file = os.path.join(pretrained_model_name_or_path, weights_name) + return model_file + elif subfolder is not None and os.path.isfile( + os.path.join(pretrained_model_name_or_path, subfolder, weights_name) + ): + model_file = os.path.join(pretrained_model_name_or_path, subfolder, weights_name) + return model_file + else: + raise EnvironmentError( + f"Error no file named {weights_name} found in directory {pretrained_model_name_or_path}." + ) + else: + try: + # Load from URL or cache if already cached + model_file = hf_hub_download( + pretrained_model_name_or_path, + filename=weights_name, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision, + ) + return model_file + + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier " + "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a " + "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli " + "login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for " + "this model name. Check the model page at " + f"'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {weights_name}." + ) + except HTTPError as err: + raise EnvironmentError( + f"There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a file named {weights_name} or" + " \nCheckout your internet connection or see how to run the library in" + " offline mode at 'https://huggingface.co/docs/diffusers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a file named {weights_name}" + ) + + +class ModelMixin(torch.nn.Module): + r""" + Base class for all models. + + [`ModelMixin`] takes care of storing the configuration of the models and handles methods for loading, downloading + and saving models. + + - **config_name** ([`str`]) -- A filename under which the model should be stored when calling + [`~models.ModelMixin.save_pretrained`]. + """ + config_name = CONFIG_NAME + _automatically_saved_args = ["_version", "_class_name", "_name_or_path"] + _supports_gradient_checkpointing = False + + def __init__(self): + super().__init__() + + @property + def is_gradient_checkpointing(self) -> bool: + """ + Whether gradient checkpointing is activated for this model or not. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules()) + + def enable_gradient_checkpointing(self): + """ + Activates gradient checkpointing for the current model. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + if not self._supports_gradient_checkpointing: + raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.") + self.apply(partial(self._set_gradient_checkpointing, value=True)) + + def disable_gradient_checkpointing(self): + """ + Deactivates gradient checkpointing for the current model. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + if self._supports_gradient_checkpointing: + self.apply(partial(self._set_gradient_checkpointing, value=False)) + + def set_use_memory_efficient_attention_xformers( + self, valid: bool, attention_op: Optional[Callable] = None + ) -> None: + # Recursively walk through all the children. + # Any children which exposes the set_use_memory_efficient_attention_xformers method + # gets the message + def fn_recursive_set_mem_eff(module: torch.nn.Module): + if hasattr(module, "set_use_memory_efficient_attention_xformers"): + module.set_use_memory_efficient_attention_xformers(valid, attention_op) + + for child in module.children(): + fn_recursive_set_mem_eff(child) + + for module in self.children(): + if isinstance(module, torch.nn.Module): + fn_recursive_set_mem_eff(module) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + r""" + Enable memory efficient attention as implemented in xformers. + + When this option is enabled, you should observe lower GPU memory usage and a potential speed up at inference + time. Speed up at training time is not guaranteed. + + Warning: When Memory Efficient Attention and Sliced attention are both enabled, the Memory Efficient Attention + is used. + + Parameters: + attention_op (`Callable`, *optional*): + Override the default `None` operator for use as `op` argument to the + [`memory_efficient_attention()`](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.memory_efficient_attention) + function of xFormers. + + Examples: + + ```py + >>> import torch + >>> from diffusers import UNet2DConditionModel + >>> from xformers.ops import MemoryEfficientAttentionFlashAttentionOp + + >>> model = UNet2DConditionModel.from_pretrained( + ... "stabilityai/stable-diffusion-2-1", subfolder="unet", torch_dtype=torch.float16 + ... ) + >>> model = model.to("cuda") + >>> model.enable_xformers_memory_efficient_attention(attention_op=MemoryEfficientAttentionFlashAttentionOp) + ``` + """ + self.set_use_memory_efficient_attention_xformers(True, attention_op) + + def disable_xformers_memory_efficient_attention(self): + r""" + Disable memory efficient attention as implemented in xformers. + """ + self.set_use_memory_efficient_attention_xformers(False) + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + is_main_process: bool = True, + save_function: Callable = None, + state_dict: Optional[Dict[str, torch.Tensor]] = None, + ): + """ + Save a model and its configuration file to a directory, so that it can be re-loaded using the + `[`~models.ModelMixin.from_pretrained`]` class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to which to save. Will be created if it doesn't exist. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful on distributed training like TPUs when one + need to replace `torch.save` by another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + state_dict (`Dict[str, torch.Tensor]`, *optional*): + The state dictionary to save. If `None`, the model's state dictionary will be saved. + """ + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + if save_function is None: + save_function = torch.save + + os.makedirs(save_directory, exist_ok=True) + + model_to_save = self + + # Attach architecture to the config + # Save the config + if is_main_process: + model_to_save.save_config(save_directory) + + # Save the model + if state_dict is None: + state_dict = model_to_save.state_dict() + + weights_name = WEIGHTS_NAME + + # Save the model + save_function(state_dict, os.path.join(save_directory, weights_name)) + + logger.info(f"Model weights saved in {os.path.join(save_directory, weights_name)}") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): + r""" + Instantiate a pretrained pytorch model from a pre-trained model configuration. + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train + the model, you should first set it back in training mode with `model.train()`. + + The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come + pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning + task. + + The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those + weights are discarded. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + Valid model ids should have an organization name, like `google/ddpm-celebahq-256`. + - A path to a *directory* containing model weights saved using [`~ModelMixin.save_config`], e.g., + `./my_model_directory/`. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the dtype + will be automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received files. Will attempt to resume the download if such a + file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `diffusers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + from_flax (`bool`, *optional*, defaults to `False`): + Load the model weights from a Flax checkpoint save file. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo (either remote in + huggingface.co or downloaded locally), you can specify the folder name here. + + mirror (`str`, *optional*): + Mirror source to accelerate downloads in China. If you are from China and have an accessibility + problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety. + Please refer to the mirror site for more information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each + parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the + same device. + + To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading by not initializing the weights and only loading the pre-trained weights. This + also tries to not use more than 1x model size in CPU memory (including peak memory) while loading the + model. This is only supported when torch version >= 1.9.0. If you are using an older version of torch, + setting this argument to `True` will raise an error. + + + + It is required to be logged in (`huggingface-cli login`) when you want to use private or [gated + models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + + + Activate the special ["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use + this method in a firewalled environment. + + + + """ + cache_dir = kwargs.pop("cache_dir", MUSE_CACHE) + ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + output_loading_info = kwargs.pop("output_loading_info", False) + local_files_only = kwargs.pop("local_files_only", False) # TODO + use_auth_token = kwargs.pop("use_auth_token", None) + revision = kwargs.pop("revision", None) + torch_dtype = kwargs.pop("torch_dtype", None) + subfolder = kwargs.pop("subfolder", None) + device_map = kwargs.pop("device_map", None) + low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT) + + if low_cpu_mem_usage is False and device_map is not None: + raise ValueError( + f"You cannot set `low_cpu_mem_usage` to `False` while using device_map={device_map} for loading and" + " dispatching. Please make sure to set `low_cpu_mem_usage=True`." + ) + + user_agent = { + "diffusers": __version__, + "file_type": "model", + "framework": "pytorch", + } + + # Load config if we don't provide a configuration + config_path = pretrained_model_name_or_path + + # This variable will flag if we're loading a sharded checkpoint. In this case the archive file is just the + # Load model + + model_file = None + + if model_file is None: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=WEIGHTS_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + + if low_cpu_mem_usage: + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + config, unused_kwargs = cls.load_config( + config_path, + cache_dir=cache_dir, + return_unused_kwargs=True, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + device_map=device_map, + **kwargs, + ) + model = cls.from_config(config, **unused_kwargs) + + # if device_map is None, load the state dict and move the params from meta device to the cpu + if device_map is None: + param_device = "cpu" + state_dict = load_state_dict(model_file) + # move the params from meta device to cpu + missing_keys = set(model.state_dict().keys()) - set(state_dict.keys()) + if len(missing_keys) > 0: + raise ValueError( + f"Cannot load {cls} from {pretrained_model_name_or_path} because the following keys are" + f" missing: \n {', '.join(missing_keys)}. \n Please make sure to pass" + " `low_cpu_mem_usage=False` and `device_map=None` if you want to randomely initialize" + " those weights or else make sure your checkpoint file is correct." + ) + + for param_name, param in state_dict.items(): + accepts_dtype = "dtype" in set(inspect.signature(set_module_tensor_to_device).parameters.keys()) + if accepts_dtype: + set_module_tensor_to_device(model, param_name, param_device, value=param, dtype=torch_dtype) + else: + set_module_tensor_to_device(model, param_name, param_device, value=param) + else: # else let accelerate handle loading and dispatching. + # Load weights and dispatch according to the device_map + # by deafult the device_map is None and the weights are loaded on the CPU + accelerate.load_checkpoint_and_dispatch(model, model_file, device_map, dtype=torch_dtype) + + loading_info = { + "missing_keys": [], + "unexpected_keys": [], + "mismatched_keys": [], + "error_msgs": [], + } + else: + config, unused_kwargs = cls.load_config( + config_path, + cache_dir=cache_dir, + return_unused_kwargs=True, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + device_map=device_map, + **kwargs, + ) + model = cls.from_config(config, **unused_kwargs) + + state_dict = load_state_dict(model_file) + + model, missing_keys, unexpected_keys, mismatched_keys, error_msgs = cls._load_pretrained_model( + model, + state_dict, + model_file, + pretrained_model_name_or_path, + ignore_mismatched_sizes=ignore_mismatched_sizes, + ) + + loading_info = { + "missing_keys": missing_keys, + "unexpected_keys": unexpected_keys, + "mismatched_keys": mismatched_keys, + "error_msgs": error_msgs, + } + + if torch_dtype is not None and not isinstance(torch_dtype, torch.dtype): + raise ValueError( + f"{torch_dtype} needs to be of type `torch.dtype`, e.g. `torch.float16`, but is {type(torch_dtype)}." + ) + elif torch_dtype is not None: + model = model.to(torch_dtype) + + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + + # Set model in evaluation mode to deactivate DropOut modules by default + model.eval() + if output_loading_info: + return model, loading_info + + return model + + @classmethod + def _load_pretrained_model( + cls, + model, + state_dict, + resolved_archive_file, + pretrained_model_name_or_path, + ignore_mismatched_sizes=False, + ): + # Retrieve missing & unexpected_keys + model_state_dict = model.state_dict() + loaded_keys = [k for k in state_dict.keys()] + + expected_keys = list(model_state_dict.keys()) + + original_loaded_keys = loaded_keys + + missing_keys = list(set(expected_keys) - set(loaded_keys)) + unexpected_keys = list(set(loaded_keys) - set(expected_keys)) + + # Make sure we are able to load base models as well as derived models (with heads) + model_to_load = model + + def _find_mismatched_keys( + state_dict, + model_state_dict, + loaded_keys, + ignore_mismatched_sizes, + ): + mismatched_keys = [] + if ignore_mismatched_sizes: + for checkpoint_key in loaded_keys: + model_key = checkpoint_key + + if ( + model_key in model_state_dict + and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape + ): + mismatched_keys.append( + (checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape) + ) + del state_dict[checkpoint_key] + return mismatched_keys + + if state_dict is not None: + # Whole checkpoint + mismatched_keys = _find_mismatched_keys( + state_dict, + model_state_dict, + original_loaded_keys, + ignore_mismatched_sizes, + ) + error_msgs = _load_state_dict_into_model(model_to_load, state_dict) + + if len(error_msgs) > 0: + error_msg = "\n\t".join(error_msgs) + if "size mismatch" in error_msg: + error_msg += ( + "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method." + ) + raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}") + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when" + f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are" + f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task" + " or with another architecture (e.g. initializing a BertForSequenceClassification model from a" + " BertForPreTraining model).\n- This IS NOT expected if you are initializing" + f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly" + " identical (initializing a BertForSequenceClassification model from a" + " BertForSequenceClassification model)." + ) + else: + logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n") + if len(missing_keys) > 0: + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably" + " TRAIN this model on a down-stream task to be able to use it for predictions and inference." + ) + elif len(mismatched_keys) == 0: + logger.info( + f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at" + f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the" + f" checkpoint was trained on, you can already use {model.__class__.__name__} for predictions" + " without further training." + ) + if len(mismatched_keys) > 0: + mismatched_warning = "\n".join( + [ + f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated" + for key, shape1, shape2 in mismatched_keys + ] + ) + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not" + f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be" + " able to use it for predictions and inference." + ) + + return model, missing_keys, unexpected_keys, mismatched_keys, error_msgs + + @property + def device(self) -> device: + """ + `torch.device`: The device on which the module is (assuming that all the module parameters are on the same + device). + """ + return get_parameter_device(self) + + @property + def dtype(self) -> torch.dtype: + """ + `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). + """ + return get_parameter_dtype(self) + + def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int: + """ + Get number of (optionally, trainable or non-embeddings) parameters in the module. + + Args: + only_trainable (`bool`, *optional*, defaults to `False`): + Whether or not to return only the number of trainable parameters + + exclude_embeddings (`bool`, *optional*, defaults to `False`): + Whether or not to return only the number of non-embeddings parameters + + Returns: + `int`: The number of parameters. + """ + + if exclude_embeddings: + embedding_param_names = [ + f"{name}.weight" + for name, module_type in self.named_modules() + if isinstance(module_type, torch.nn.Embedding) + ] + non_embedding_parameters = [ + parameter for name, parameter in self.named_parameters() if name not in embedding_param_names + ] + return sum(p.numel() for p in non_embedding_parameters if p.requires_grad or not only_trainable) + else: + return sum(p.numel() for p in self.parameters() if p.requires_grad or not only_trainable) + + +""" ConfigMixin base class and utilities.""" + + +class FrozenDict(OrderedDict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + for key, value in self.items(): + setattr(self, key, value) + + self.__frozen = True + + def __delitem__(self, *args, **kwargs): + raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.") + + def setdefault(self, *args, **kwargs): + raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance.") + + def pop(self, *args, **kwargs): + raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance.") + + def update(self, *args, **kwargs): + raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance.") + + def __setattr__(self, name, value): + if hasattr(self, "__frozen") and self.__frozen: + raise Exception(f"You cannot use ``__setattr__`` on a {self.__class__.__name__} instance.") + super().__setattr__(name, value) + + def __setitem__(self, name, value): + if hasattr(self, "__frozen") and self.__frozen: + raise Exception(f"You cannot use ``__setattr__`` on a {self.__class__.__name__} instance.") + super().__setitem__(name, value) + + +class ConfigMixin: + r""" + Base class for all configuration classes. Stores all configuration parameters under `self.config` Also handles all + methods for loading/downloading/saving classes inheriting from [`ConfigMixin`] with + - [`~ConfigMixin.from_config`] + - [`~ConfigMixin.save_config`] + + Class attributes: + - **config_name** (`str`) -- A filename under which the config should stored when calling + [`~ConfigMixin.save_config`] (should be overridden by parent class). + - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be + overridden by subclass). + - **has_compatibles** (`bool`) -- Whether the class has compatible classes (should be overridden by subclass). + - **_deprecated_kwargs** (`List[str]`) -- Keyword arguments that are deprecated. Note that the init function + should only have a `kwargs` argument if at least one argument is deprecated (should be overridden by + subclass). + """ + config_name = None + ignore_for_config = [] + has_compatibles = False + + _deprecated_kwargs = [] + + def register_to_config(self, **kwargs): + if self.config_name is None: + raise NotImplementedError(f"Make sure that {self.__class__} has defined a class name `config_name`") + # Special case for `kwargs` used in deprecation warning added to schedulers + # TODO: remove this when we remove the deprecation warning, and the `kwargs` argument, + # or solve in a more general way. + kwargs.pop("kwargs", None) + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error(f"Can't set {key} with value {value} for {self}") + raise err + + if not hasattr(self, "_internal_dict"): + internal_dict = kwargs + else: + previous_dict = dict(self._internal_dict) + internal_dict = {**self._internal_dict, **kwargs} + logger.debug(f"Updating config from {previous_dict} to {internal_dict}") + + self._internal_dict = FrozenDict(internal_dict) + + def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + """ + Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the + [`~ConfigMixin.from_config`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the configuration JSON file will be saved (will be created if it does not exist). + """ + if os.path.isfile(save_directory): + raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file") + + os.makedirs(save_directory, exist_ok=True) + + # If we save using the predefined names, we can load using `from_config` + output_config_file = os.path.join(save_directory, self.config_name) + + self.to_json_file(output_config_file) + logger.info(f"Configuration saved in {output_config_file}") + + @classmethod + def from_config(cls, config: Union[FrozenDict, Dict[str, Any]] = None, **kwargs): + r""" + Instantiate a Python class from a config dictionary + + Parameters: + config (`Dict[str, Any]`): + A config dictionary from which the Python class will be instantiated. Make sure to only load + configuration files of compatible classes. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + Whether kwargs that are not consumed by the Python class should be returned or not. + + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to update the configuration object (after it being loaded) and initiate the Python class. + `**kwargs` will be directly passed to the underlying scheduler/model's `__init__` method and eventually + overwrite same named arguments of `config`. + + Examples: + + ```python + >>> from diffusers import DDPMScheduler, DDIMScheduler, PNDMScheduler + + >>> # Download scheduler from huggingface.co and cache. + >>> scheduler = DDPMScheduler.from_pretrained("google/ddpm-cifar10-32") + + >>> # Instantiate DDIM scheduler class with same config as DDPM + >>> scheduler = DDIMScheduler.from_config(scheduler.config) + + >>> # Instantiate PNDM scheduler class with same config as DDPM + >>> scheduler = PNDMScheduler.from_config(scheduler.config) + ``` + """ + # <===== TO BE REMOVED WITH DEPRECATION + # TODO(Patrick) - make sure to remove the following lines when config=="model_path" is deprecated + if "pretrained_model_name_or_path" in kwargs: + config = kwargs.pop("pretrained_model_name_or_path") + + if config is None: + raise ValueError("Please make sure to provide a config as the first positional argument.") + # ======> + + # Return model and optionally state and/or unused_kwargs + model = cls(**config) + return model + + @classmethod + def load_config( + cls, pretrained_model_name_or_path: Union[str, os.PathLike], return_unused_kwargs=False, **kwargs + ) -> Tuple[Dict[str, Any], Dict[str, Any]]: + r""" + Instantiate a Python class from a config dictionary + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* of a model repo on huggingface.co. Valid model ids should have an + organization name, like `google/ddpm-celebahq-256`. + - A path to a *directory* containing model weights saved using [`~ConfigMixin.save_config`], e.g., + `./my_model_directory/`. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received files. Will attempt to resume the download if such a + file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `transformers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo (either remote in + huggingface.co or downloaded locally), you can specify the folder name here. + + + + It is required to be logged in (`huggingface-cli login`) when you want to use private or [gated + models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + + + Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to + use this method in a firewalled environment. + + + """ + cache_dir = kwargs.pop("cache_dir", MUSE_CACHE) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + _ = kwargs.pop("mirror", None) + subfolder = kwargs.pop("subfolder", None) + + user_agent = {"file_type": "config"} + + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + + if cls.config_name is None: + raise ValueError( + "`self.config_name` is not defined. Note that one should not load a config from " + "`ConfigMixin`. Please make sure to define `config_name` in a class inheriting from `ConfigMixin`" + ) + + if os.path.isfile(pretrained_model_name_or_path): + config_file = pretrained_model_name_or_path + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, cls.config_name)): + # Load from a PyTorch checkpoint + config_file = os.path.join(pretrained_model_name_or_path, cls.config_name) + elif subfolder is not None and os.path.isfile( + os.path.join(pretrained_model_name_or_path, subfolder, cls.config_name) + ): + config_file = os.path.join(pretrained_model_name_or_path, subfolder, cls.config_name) + else: + raise EnvironmentError( + f"Error no file named {cls.config_name} found in directory {pretrained_model_name_or_path}." + ) + else: + try: + # Load from URL or cache if already cached + config_file = hf_hub_download( + pretrained_model_name_or_path, + filename=cls.config_name, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision, + ) + + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier" + " listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a" + " token having permission to this repo with `use_auth_token` or log in with `huggingface-cli" + " login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for" + " this model name. Check the model page at" + f" 'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {cls.config_name}." + ) + except HTTPError as err: + raise EnvironmentError( + "There was a specific connection error when trying to load" + f" {pretrained_model_name_or_path}:\n{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a {cls.config_name} file.\nCheckout your internet connection or see how to" + " run the library in offline mode at" + " 'https://huggingface.co/docs/diffusers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load config for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a {cls.config_name} file" + ) + + try: + # Load config dict + config_dict = cls._dict_from_json_file(config_file) + except (json.JSONDecodeError, UnicodeDecodeError): + raise EnvironmentError(f"It looks like the config file at '{config_file}' is not a valid JSON file.") + + if return_unused_kwargs: + return config_dict, kwargs + + return config_dict + + @staticmethod + def _get_init_keys(cls): + return set(dict(inspect.signature(cls.__init__).parameters).keys()) + + @classmethod + def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]): + with open(json_file, "r", encoding="utf-8") as reader: + text = reader.read() + return json.loads(text) + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + @property + def config(self) -> Dict[str, Any]: + """ + Returns the config of the class as a frozen dictionary + + Returns: + `Dict[str, Any]`: Config of the class. + """ + return self._internal_dict + + def to_json_string(self) -> str: + """ + Serializes this instance to a JSON string. + + Returns: + `str`: String containing all the attributes that make up this configuration instance in JSON format. + """ + config_dict = self._internal_dict if hasattr(self, "_internal_dict") else {} + config_dict["_class_name"] = self.__class__.__name__ + config_dict["_version"] = __version__ + + def to_json_saveable(value): + if isinstance(value, np.ndarray): + value = value.tolist() + elif isinstance(value, PosixPath): + value = str(value) + return value + + config_dict = {k: to_json_saveable(v) for k, v in config_dict.items()} + return json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save this instance to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file in which this configuration instance's parameters will be saved. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + writer.write(self.to_json_string()) + + +def register_to_config(init): + r""" + Decorator to apply on the init of classes inheriting from [`ConfigMixin`] so that all the arguments are + automatically sent to `self.register_for_config`. To ignore a specific argument accepted by the init but that + shouldn't be registered in the config, use the `ignore_for_config` class variable + + Warning: Once decorated, all private arguments (beginning with an underscore) are trashed and not sent to the init! + """ + + @functools.wraps(init) + def inner_init(self, *args, **kwargs): + # Ignore private kwargs in the init. + init_kwargs = {k: v for k, v in kwargs.items() if not k.startswith("_")} + config_init_kwargs = {k: v for k, v in kwargs.items() if k.startswith("_")} + if not isinstance(self, ConfigMixin): + raise RuntimeError( + f"`@register_for_config` was applied to {self.__class__.__name__} init method, but this class does " + "not inherit from `ConfigMixin`." + ) + + ignore = getattr(self, "ignore_for_config", []) + # Get positional arguments aligned with kwargs + new_kwargs = {} + signature = inspect.signature(init) + parameters = { + name: p.default for i, (name, p) in enumerate(signature.parameters.items()) if i > 0 and name not in ignore + } + for arg, name in zip(args, parameters.keys()): + new_kwargs[name] = arg + + # Then add all kwargs + new_kwargs.update( + { + k: init_kwargs.get(k, default) + for k, default in parameters.items() + if k not in ignore and k not in new_kwargs + } + ) + new_kwargs = {**config_init_kwargs, **new_kwargs} + getattr(self, "register_to_config")(**new_kwargs) + init(self, *args, **init_kwargs) + + return inner_init diff --git a/InternLM/tools/utils.py b/InternLM/tools/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..424cd1cb0bc072c78656c4449f5f3efe83b2e208 --- /dev/null +++ b/InternLM/tools/utils.py @@ -0,0 +1,52 @@ +import numpy as np +import torch +from PIL import Image +from torchvision import transforms + +encode_transform = transforms.Compose( + [ + transforms.Resize(256, interpolation=transforms.InterpolationMode.BILINEAR), + transforms.CenterCrop(256), + transforms.ToTensor(), + ] +) + + +def convert_decode_to_pil(rec_image): + rec_image = 2.0 * rec_image - 1.0 + rec_image = torch.clamp(rec_image, -1.0, 1.0) + rec_image = (rec_image + 1.0) / 2.0 + rec_image *= 255.0 + rec_image = rec_image.permute(0, 2, 3, 1).detach().cpu().numpy().astype(np.uint8) + pil_images = [Image.fromarray(image) for image in rec_image] + return pil_images + + +def patchify(imgs, p): + """ + imgs: (N, C, H, W) + x: (N, L, patch_size**2 * C) + """ + assert imgs.shape[2] == imgs.shape[3] and imgs.shape[2] % p == 0 + + in_chans = imgs.shape[1] + h = w = imgs.shape[2] // p + x = imgs.reshape(shape=(imgs.shape[0], in_chans, h, p, w, p)) + x = torch.einsum('nchpwq->nhwpqc', x) + x = x.reshape(shape=(imgs.shape[0], h * w, p ** 2 * in_chans)) + return x + + +def unpatchify(x, p): + """ + x: (N, L, patch_size**2 * C) + imgs: (N, C, H, W) + """ + # p = self.patch_embed.patch_size[0] + h = w = int(x.shape[1] ** .5) + assert h * w == x.shape[1] + + x = x.reshape(shape=(x.shape[0], h, w, p, p, -1)) + x = torch.einsum('nhwpqc->nchpwq', x) + imgs = x.reshape(shape=(x.shape[0], -1, h * p, h * p)) + return imgs \ No newline at end of file diff --git a/InternLM/train.py b/InternLM/train.py new file mode 100644 index 0000000000000000000000000000000000000000..c240acdaa5d5cf40c78615cea809c2da20d27ebb --- /dev/null +++ b/InternLM/train.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import socket +import time +import traceback +from functools import partial +import os + +import torch +import torch.distributed as dist + +import internlm +from internlm.core.context import ParallelMode +from internlm.core.context import global_context as gpc +from internlm.core.scheduler import SchedulerMetricHook +from internlm.core.trainer import TrainState +from internlm.initialize import initialize_distributed_env +from internlm.model.loss import FlashGPTLMLoss, KLDivLoss +from internlm.model.metrics import AccPerplex +from internlm.monitor import initialize_monitor_manager, send_alert_message +from internlm.monitor.monitor import monitor_manager as mm +from internlm.train import ( + get_train_data_loader, + get_validation_data_loader, + initialize_llm_profile, + initialize_model, + initialize_teacher, + initialize_optimizer, + load_new_batch, + load_new_batch_stop, + record_current_batch_training_metrics, +) +from internlm.utils.common import ( + BatchSkipper, + get_megatron_flops, + launch_time, + parse_args, +) +from internlm.utils.evaluation import evaluate_on_val_dls +from internlm.utils.gputest import empty_cache_and_diag +from internlm.utils.logger import get_logger, initialize_uniscale_logger +from internlm.utils.megatron_timers import megatron_timer as timer +from internlm.utils.model_checkpoint import CheckpointManager, load_model_checkpoint +from internlm.utils.parallel import get_parallel_log_file_name +from internlm.utils.simple_memory_profiler import SimpleMemoryProfiler +from internlm.utils.writer import Writer + +# global llm logger +logger = get_logger(__file__) + + +def initialize_llm_logger(start_time: str): + """ + Initialize customed uniscale logger. + + Args: + start_time (str): The launch time of current training job. + + Returns: The instance of uniscale logger. + """ + + uniscale_logger = initialize_uniscale_logger( + job_name=gpc.config.JOB_NAME, launch_time=start_time, file_name=get_parallel_log_file_name() + ) + if uniscale_logger is not None: + global logger + logger = uniscale_logger + + return uniscale_logger + + +def main(args): + # init setting + skip_batches = gpc.config.data.skip_batches + total_steps = gpc.config.data.total_steps + valid_every = gpc.config.data.valid_every + label_smoothing = gpc.config.loss.label_smoothing + + get_tflops_func = partial( + get_megatron_flops, + checkpoint=gpc.config.model.checkpoint, + seq_len=gpc.config.SEQ_LEN, + hidden_size=gpc.config.model.hidden_size, + num_layers=gpc.config.model.num_layers, + vocab_size=gpc.config.model.vocab_size, + global_batch_size=gpc.config.data.micro_bsz * gpc.config.data.micro_num * gpc.get_world_size(ParallelMode.DATA), + global_world_size=gpc.get_world_size(ParallelMode.GLOBAL), + mlp_ratio=gpc.config.MLP_RATIO, + ) + + # get and broadcast current time + current_time = launch_time() + objs = [current_time] + dist.broadcast_object_list(objs, src=0) + current_time = objs[0] + + # initialize customed llm logger + uniscale_logger = initialize_llm_logger(start_time=current_time) + + # initialize model + model = initialize_model() + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + logger.warning(f'Model parameters: {n_parameters / 1e6} M.') + + with open(args.config, "r") as f: + config_lines = f.readlines() + + # initialize loss function + criterion = FlashGPTLMLoss(parallel_output=True, label_smoothing=label_smoothing) + + # initialize the train and validation data loader + train_dl, dataset_types = get_train_data_loader(num_worker=4) + val_dls = get_validation_data_loader() + + # initialize and resume train state + train_state = TrainState(gpc.config, train_dl.batch_sampler) + + optimizer, beta2_scheduler, lr_scheduler = initialize_optimizer(model=model) + + ckpt_manager = CheckpointManager( + ckpt_config=gpc.config.ckpt, + model=model, + optimizer=optimizer, + lr_scheduler=lr_scheduler, + train_dl=train_dl, + model_config=gpc.config.model, + model_config_file="".join(config_lines), + feishu_address=gpc.config.monitor.alert.feishu_alert_address, + ) + + # Loading other persistent training states. + ckpt_manager.try_resume_training(train_state, current_time) + + # initialize customed llm writer + writer = Writer( + job_name=gpc.config.JOB_NAME, + launch_time=current_time, + file_name=get_parallel_log_file_name(), + tensorboard_folder=gpc.config.tensorboard_folder, + resume_tb_folder=train_state.resume_tb_folder, # resume from ckpt. + step_count=train_state.step_count, # resume from ckpt. + config=config_lines, + logger=logger, + enable_tb=gpc.config.enable_tb, + ) + + # initialize metric for calculating accuracy and perplexity + metric = AccPerplex( + device=torch.cuda.current_device(), + tp_pg=gpc.get_group(ParallelMode.TENSOR), + dp_pg=gpc.get_group(ParallelMode.DATA), + dataset_types=dataset_types, + ) + + # initialize trainer + scheduler_hooks = [ + SchedulerMetricHook( + metric=metric, + skip=( + gpc.is_using_pp() + and hasattr(gpc.config.model, "num_chunks") + and gpc.config.model.num_chunks > 1 + and gpc.config.parallel["pipeline"].get("interleaved_overlap", False) + ), + ), + ] + + if gpc.config.get('kd_config', None) is None: + trainer, train_dl, _, _ = internlm.initialize_trainer( + model=model, + optimizer=optimizer, + criterion=criterion, + train_dataloader=train_dl, + lr_scheduler=lr_scheduler, + beta2_scheduler=beta2_scheduler, + scheduler_hooks=scheduler_hooks, + ) + else: + # initialize teacher + teacher = initialize_teacher() + n_parameters = sum(p.numel() for p in teacher.parameters()) + logger.warning(f'Teacher parameters: {n_parameters / 1e6} M.') + + teacher.requires_grad_(False) + teacher.eval() + + kd_criterion_type = gpc.config.kd_config.get('type', 'kl_div') + if kd_criterion_type == 'kl_div': + kd_criterion = KLDivLoss() + else: + raise NotImplementedError + + trainer, train_dl, _, _ = internlm.initialize_kd_trainer( + model=model, + teacher=teacher, + optimizer=optimizer, + criterion=criterion, + kd_criterion=kd_criterion, + train_dataloader=train_dl, + lr_scheduler=lr_scheduler, + beta2_scheduler=beta2_scheduler, + scheduler_hooks=scheduler_hooks, + ) + + # initialize simple memory profiler + if args.profiling: + memory_profiler = SimpleMemoryProfiler( + model, + optimizer.optim, + log_folder=f"memory_trace/rank{gpc.get_global_rank()}_" + + f"dp{gpc.get_local_rank(ParallelMode.DATA)}_" + + f"tp{gpc.get_local_rank(ParallelMode.TENSOR)}", + ) + else: + memory_profiler = None + + # initialize the batch skipper + batch_skipper = BatchSkipper(skip_batches) + + trainer.train() + + # transfer the train data loader into train data iterator + train_iter = iter(train_dl) + + with initialize_llm_profile(profiling=args.profiling, start_time=current_time) as prof: + # start iterating the train data and begin training + for batch_count in range(train_state.batch_count, total_steps): + empty_cache_and_diag(batch_count, interval=gpc.config.data.empty_cache_and_diag_interval) + start_time = time.time() + timer("one-batch").start() + + # load batch data + # batch, train_iter = load_new_batch_stop(train_dl=train_dl, train_iter=train_iter, train_state=train_state) + if gpc.config.data.train_one_epoch: + batch, train_iter = load_new_batch_stop(train_dl=train_dl, train_iter=train_iter, + train_state=train_state) + if batch is None: + now_break = ckpt_manager.try_save_checkpoint(train_state, data_iter_stop=True) + break + else: + batch, train_iter = load_new_batch(train_dl=train_dl, train_iter=train_iter, train_state=train_state) + + # record the consumed samples in training + train_state.batch_count = batch_count + train_state.num_consumed_samples_in_epoch += len(batch[1]) + if batch_skipper(batch_count): # skip this batch + if gpc.is_rank_for_log(): + logger.info(f"Skip batch count:`{batch_count}`...") + timer("one-batch").stop() + continue + + # zero the grads of parameters + trainer.zero_grad() + # process data + if batch[0].get("type_ids", None) is not None: + metric.set_current_type_ids(type_ids=batch[0].pop("type_ids", None)) + + # do forward and backward + timer("fwd-bwd").start() + + _, _, loss = trainer.execute_schedule( + batch, forward_only=False, return_loss=True, return_output_label=False + ) + timer("fwd-bwd").stop() + + # update parameters, and returns (success_update, grad_norm) + trainer_result = trainer.step() + assert trainer_result is not None + + success_update, grad_norm_groups = trainer_result + if success_update: # update parameters successfully + train_state.step_count += 1 + else: + train_state.inf_nan_skip_batches += 1 # record the amount of updating parameters unsuccessfully. + if -1 in grad_norm_groups.values() and gpc.is_rank_for_log(): # -1 encodes a specific failure case + logger.warning(f"Warning: skip parameter update at step {batch_count}.") + send_alert_message( + address=gpc.config.monitor.alert.feishu_alert_address, + message=f"Warning: skip parameter update at step {batch_count}.", + ) + + # calculate and record the training metrics, eg. loss, accuracy and so on. + record_current_batch_training_metrics( + get_tflops_func=get_tflops_func, + logger=logger, + writer=writer, + success_update=success_update, + batch_count=batch_count, + batch=batch, + train_state=train_state, + optimizer=optimizer, + beta2_scheduler=beta2_scheduler, + trainer=trainer, + start_time=start_time, + loss=loss, + grad_norm=grad_norm_groups, + metric=metric, + update_panel=uniscale_logger is not None, + ) + + timer("one-batch").stop() + + # evaluate on validation data loaders + if valid_every > 0 and train_state.step_count % valid_every == 0: + evaluate_on_val_dls( + trainer=trainer, + val_dls=val_dls, + writer=writer, + logger=logger, + step_count=train_state.step_count, + update_panel=uniscale_logger is not None, + ) + + # checkpoint the training states in specific steps, which is determined by the args "checkpoint_every" + # # save batch sampler that tracks the true consumed samples + now_break = ckpt_manager.try_save_checkpoint(train_state) + if now_break: + break + + if memory_profiler is not None: + memory_profiler.step() + + if batch_count % 2 == 0: + prof.step() + + ckpt_manager.wait_async_upload_finish() + + +if __name__ == "__main__": + args = parse_args() + hostname = socket.gethostname() + + # initialize distributed environment + initialize_distributed_env(config=args.config, launcher=args.launcher, master_port=args.port, seed=args.seed) + assert hasattr(gpc, "config") and gpc.config is not None + + # initialize monitor manager context + with initialize_monitor_manager( + job_name=gpc.config.JOB_NAME, alert_address=gpc.config.monitor.alert.feishu_alert_address + ): + try: + main(args) + except Exception: + logger.error( + f"Raise exception from {hostname} with rank id: {gpc.get_global_rank()}\n{traceback.format_exc()}", + ) + mm.monitor_exception( + alert_address=gpc.config.monitor.alert.feishu_alert_address, excp_info=traceback.format_exc() + ) diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a319b450a41f18cf8b6f7d712dd6914a8e1afc22 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +## Implementation of "[Data-efficient Large Vision Models through Sequential Autoregression](https://arxiv.org/pdf/2402.04841.pdf)". + + +

+ +

+

+

+Training general-purpose vision models on purely sequential visual data, eschewing linguistic inputs, has heralded a new frontier in visual understanding. These models are intended to not only comprehend but also seamlessly transit to out-of-domain tasks. +However, current endeavors are hamstrung by an over-reliance on colossal models, exemplified by models with upwards of 3B parameters, and the necessity for an extensive corpus of visual data, often comprising a staggering 400B tokens. +In this paper, we delve into the development of an efficient, autoregression-based vision model, innovatively architected to operate on a limited dataset. We meticulously demonstrate how this model achieves proficiency in a spectrum of visual tasks spanning both high-level and low-level semantic understanding during the testing phase. Our empirical evaluations underscore the model's agility in adapting to various tasks, heralding a significant reduction in the parameter footprint, and a marked decrease in training data requirements, thereby paving the way for more sustainable and accessible advancements in the field of generalist vision models. + +#### TODO List +- [X] Code about training models. +- [X] Code about inferencing models. +- [X] Huggingface & InternLM ckpts. +- [X] Code about data generation. + + +#### Set up +``` +based on InternLM-v0.2.1dev20231121 +``` + + +Install: `https://github.com/InternLM/InternLM/blob/v0.2.1dev20231121/doc/en/install.md` + +Put your training data to `/path/to/data/vision`. + +Training command: +`torchrun --nproc_per_node 8 train.py --config ./configs/pretrain_300m.py --launcher torch` + +Training via KD command: +`torchrun --nproc_per_node 8 train.py --config ./configs/kd_1b_to_300m.py --launcher torch` + +Convert model and inference example: `./tools` + +The corresponding huggingface ckpt can be downloaded at [LLaMA-1b-hf Onedrive](https://unisyd-my.sharepoint.com/:u:/g/personal/han_wu_sydney_edu_au/EQx8q3DvqP1CqOddm0aYN4wBBywVAOSvyB1P12ItzuNDmw?e=uOkUnP) / [LLaMA-1b-hf Baidu Disk](https://pan.baidu.com/s/12oI_TOVHtbhriM1Bu1TXmw?pwd=1234) and [LLaMA-300m-hf](https://github.com/ggjy/DeLVM/releases/download/hf-ckpt/llama_300m_hf.zip). + +#### Data generation +Please refer to [data_generation/README.md](data_generation/README.md). + + +### Citation + +If you find this project useful in your research, please consider cite: + +```bibtex +@article{guo2024dataefficient, + title={Data-efficient Large Vision Models through Sequential Autoregression}, + author={Guo, Jianyuan and Hao, Zhiwei and Wang, Chengcheng and Tang, Yehui and Wu, Han and Hu, Han and Han, Kai and Xu, Chang}, + journal={arXiv preprint arXiv:2402.04841}, + year={2024} +} +``` + +### Acknowledgement + +We maily follow the directon of project [LVM](https://github.com/ytongbai/LVM). And this repo is based on [InternLM](https://github.com/InternLM/InternLM), [huggingface.co/transformers](https://github.com/huggingface/transformers), and [huggingface.co/openMUSE](https://github.com/huggingface/open-muse). + +### License + +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) diff --git a/__pycache__/dist_train_utils.cpython-37.pyc b/__pycache__/dist_train_utils.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9284f8470e34bd376f724271aa65564982b9517f Binary files /dev/null and b/__pycache__/dist_train_utils.cpython-37.pyc differ diff --git a/__pycache__/dist_train_utils.cpython-39.pyc b/__pycache__/dist_train_utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..15e54a082e7356331dcb6027a35b279382f56d02 Binary files /dev/null and b/__pycache__/dist_train_utils.cpython-39.pyc differ diff --git a/__pycache__/lr_utils.cpython-37.pyc b/__pycache__/lr_utils.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5dd1f08eafe2bf804b92528ce84b351745d7a058 Binary files /dev/null and b/__pycache__/lr_utils.cpython-37.pyc differ diff --git a/__pycache__/lr_utils.cpython-39.pyc b/__pycache__/lr_utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3ff2f448428ae89ea2e6ce1d26c216a060e9da39 Binary files /dev/null and b/__pycache__/lr_utils.cpython-39.pyc differ diff --git a/data_generation/README.md b/data_generation/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b39121b688021a3346bc406f25142e365f0d3dec --- /dev/null +++ b/data_generation/README.md @@ -0,0 +1,88 @@ +# Data generation + +## Preliminary + +1. `pip install -r data_generation/requirements.txt` +2. Download the vqgan checkpoint from [CowTransfer](https://cowtransfer.com/s/d771c6d3d8344d) or [Google Drive](https://drive.google.com/drive/folders/1CyucT_QOArUH_Au8dfzRSwseyiCGserF?usp=share_link), and move it to `./weight/vqgan-f16-8192-laion`. + +## Human keypoint + +1. You can generate the keypoint image refer to [mmpose](https://mmpose.readthedocs.io/en/dev-1.x/demos.html#d-human-pose-estimation-with-inferencer) , and + change the inference cmd like this + + ```shell + python inferencer_demo.py data/path \ + coco/train2017/images \ + --pose2d configs/body_2d_keypoint/rtmo/coco/rtmo-l_16xb16-600e_coco-640x640.py \ + --pose2d-weights ./pth/rtmo-l_16xb16-600e_coco-640x640-516a421f_20231211.pth \ + --det-model demo/mmdetection_cfg/rtmdet_m_640-8xb32_coco-person.py \ + --black-background \ + --vis-out-dir coco/train2017/keypoints \ + --skeleton-style openpose \ + --disable-rebase-keypoint \ + --radius 8 \ + --thickness 4 \ + ``` + +2. Generate vq codebook by VQ-GAN + + ```shell + python generate/generate_coco-keypoint.py \ + --input_data coco/train2017/images \ + --target_data coco/train2017/keypoints \ + --output_path vq_token/coco-keypoints/train2017 + ``` + +## Deblur + +```shell +python generate/generate_GoPro.py \ +--input_data GoPro_train/input \ +--target_data GoPro_train/target \ +--output_path vq_token/GoPro_train +``` + +## Derain + +Here we use Rain13K data in lmdb fromat. + +```shell +python generate/generate_Rain13K.py \ +--input_data Rain13K_lmdb/input.lmdb \ +--target_data Rain13K_lmdb/target.lmdb \ +--output_path vq_token/Rain13K +``` + +## Video dataset + +Here we use the HD-VILA-100M dataset. + +1. You should download the dataset refer [hd-vila-100m](https://github.com/microsoft/XPretrain/tree/main/hd-vila-100m), + and use [src/cut_videos.py](https://github.com/microsoft/XPretrain/blob/main/hd-vila-100m/src/cut_videos.py) to cut + the videos to clips. + +2. Generate vq codebook by VQ-GAN + + ```shell + python generate/generate_hdvila_100m.py \ + --video_info_json hdvila_100m/cut_video_results/cut_part0.jsonl \ + --data_root hdvila_100m/video_clips_imgs \ + --output_root vq_token/hdvila_100m + ``` + +## Segment mask + +Here we use the SA-1B dataset. + +1. Download the SA-1B dataset. + +2. Generate vq codebook by VQ-GAN. + + ```shell + python generate/generate_SA-1B.py \ + --tar_root SA-1B/tar \ + --img_json_root SA-1B/tmp/img_json \ + --mask_root SA-1B/tmp/mask \ + --output_path vq_token/SA-1B/token \ + --dp_mode + ``` \ No newline at end of file diff --git a/data_generation/generate/generate_GoPro.py b/data_generation/generate/generate_GoPro.py new file mode 100644 index 0000000000000000000000000000000000000000..d20813b130841e47a456581848e8c0482cc4f0ed --- /dev/null +++ b/data_generation/generate/generate_GoPro.py @@ -0,0 +1,107 @@ +import os +import pathlib +import sys + +parent_path = pathlib.Path(__file__).absolute().parent.parent +parent_path = os.path.abspath(parent_path) +sys.path.append(parent_path) +os.chdir(parent_path) +print(f'>-------------> parent path {parent_path}') +print(f'>-------------> current work dir {os.getcwd()}') + +import argparse +import glob +import multiprocessing +from PIL import Image +from os.path import join + +import torch +from torch.utils.data import DataLoader +from torchvision.datasets import VisionDataset + +from vqgan.load import encode_transform +from generate.img_to_token import img_to_token + +CPU_COUNT = multiprocessing.cpu_count() + + +class GoProDataset(VisionDataset): + def __init__( + self, + root: str, + target_root, + transform=None, + target_transform=None, + transforms=None, + transform_name=None + ) -> None: + super().__init__(root, transforms, transform, target_transform) + self.target_root = target_root + + file_list = glob.glob(join(root, '*.png')) + ids = [os.path.basename(i).split('.')[0] for i in file_list] + self.ids = list(sorted(ids)) + + self.transform_name = transform_name + + def _load_image(self, id: int): + path = join(self.root, f'{id}.png') + return Image.open(path).convert("RGB") + + def _load_target(self, id: int): + path = join(self.target_root, f'{id}.png') + return Image.open(path).convert("RGB") + + def __getitem__(self, index: int): + id = self.ids[index] + image = self._load_image(id) + target_img = self._load_target(id) + + images = self.transform(image) + target_imgs = self.transform(target_img) + + data_list = [] + if self.transform_name == 'six_crop_encode_transform': + for _img, _target_img in zip(images, target_imgs): + _data = torch.stack([_img, _target_img], dim=0) + data_list.append(_data) + else: + _data = torch.stack([images, target_imgs], dim=0) + data_list.append(_data) + + data = torch.cat(data_list, dim=0) + + return data + + def __len__(self) -> int: + return len(self.ids) + + +def convert_img_to_token(args, device=None): + dataset = GoProDataset(args.input_data, args.target_data, transform=encode_transform, + transform_name='encode_transform') + data_loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_work) + img_to_token(args, data_loader, args.output_path, device=device) + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--input_data", type=str, default="Rain13K_lmdb/input.lmdb") + parser.add_argument("--target_data", type=str, default="Rain13K_lmdb/target.lmdb") + parser.add_argument("--output_path", type=str, default="vq_token/Rain13K") + + parser.add_argument("--num_work", type=int, default=64) + parser.add_argument("--batch_size", type=int, default=16) + parser.add_argument("--dp_mode", action='store_true', default=False) + parser.add_argument("--model_name_or_path", type=str, default="weight/vqgan-f16-8192-laion") + args = parser.parse_args() + + return args + + +if __name__ == '__main__': + args = get_args() + + device = f'cuda:{0}' + convert_img_to_token(args, device=device) diff --git a/data_generation/generate/generate_Rain13K.py b/data_generation/generate/generate_Rain13K.py new file mode 100644 index 0000000000000000000000000000000000000000..a854e1a311be2b2cf20c9144b3c49482735ab60b --- /dev/null +++ b/data_generation/generate/generate_Rain13K.py @@ -0,0 +1,127 @@ +import argparse +import multiprocessing +import os +import pathlib +import sys + +import cv2 +import lmdb +from torch.utils.data import DataLoader + +from generate.img_to_token import img_to_token + +parent_path = pathlib.Path(__file__).absolute().parent.parent +parent_path = os.path.abspath(parent_path) +sys.path.append(parent_path) +os.chdir(parent_path) +print(f'>-------------> parent path {parent_path}') +print(f'>-------------> current work dir {os.getcwd()}') + +import numpy as np + +from PIL import Image +from os.path import join + +import torch +from torchvision.datasets import VisionDataset + +from vqgan.load import encode_transform + +CPU_COUNT = multiprocessing.cpu_count() + + +class LMDBDataset(VisionDataset): + def __init__( + self, + root: str, + target_root, + transform=None, + target_transform=None, + transforms=None, + transform_name=None + ) -> None: + super().__init__(root, transforms, transform, target_transform) + self.target_root = target_root + + self.img_db = lmdb.open(root).begin() + self.target_db = lmdb.open(target_root).begin() + + with open(join(root, 'meta_info.txt'), 'rb') as f: + file_list = f.readlines() + + ids = [i.decode().split(' ')[0].split('.')[0] for i in file_list] + self.ids = list(sorted(ids)) + + self.transform_name = transform_name + + def _load_image(self, id): + img_byte = self.img_db.get(id.encode()) + image_buf = np.frombuffer(img_byte, dtype=np.uint8) + img = cv2.imdecode(image_buf, cv2.IMREAD_COLOR) + image = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) + return image.convert("RGB") + + def _load_target(self, id): + img_byte = self.target_db.get(id.encode()) + image_buf = np.frombuffer(img_byte, dtype=np.uint8) + img = cv2.imdecode(image_buf, cv2.IMREAD_COLOR) + image = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) + return image.convert("RGB") + + def __getitem__(self, index: int): + id = self.ids[index] + image = self._load_image(id) + target_img = self._load_target(id) + + images = self.transform(image) + target_imgs = self.transform(target_img) + + data_list = [] + if self.transform_name == 'six_crop_encode_transform': + for _img, _target_img in zip(images, target_imgs): + _data = torch.stack([_img, _target_img], dim=0) + data_list.append(_data) + else: + _data = torch.stack([images, target_imgs], dim=0) + data_list.append(_data) + + data = torch.cat(data_list, dim=0) + + return data + + def __len__(self) -> int: + return len(self.ids) + + +def convert_img_to_token(args, device=None): + dataset = LMDBDataset(args.input_data, args.target_data, transform=encode_transform, + transform_name='encode_transform') + data_loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_work) + img_to_token(args, data_loader, args.output_path, device=device) + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--input_data", type=str, default="Rain13K_lmdb/input.lmdb") + parser.add_argument("--target_data", type=str, default="Rain13K_lmdb/target.lmdb") + parser.add_argument("--output_path", type=str, default="vq_token/Rain13K") + + parser.add_argument("--num_work", type=int, default=64) + parser.add_argument("--batch_size", type=int, default=16) + parser.add_argument("--dp_mode", action='store_true', default=False) + parser.add_argument("--model_name_or_path", type=str, default="weight/vqgan-f16-8192-laion") + args = parser.parse_args() + + return args + + +if __name__ == '__main__': + args = get_args() + + # input_root = '/home/ma-user/work/data/tmp_data/Rain13K_lmdb/input.lmdb' + # target_root = '/home/ma-user/work/data/tmp_data/Rain13K_lmdb/target.lmdb' + # out_root = '/home/ma-user/work/data/vq_token/Rain13K' + + device = f'cuda:{0}' + convert_img_to_token(args, device=device) diff --git a/data_generation/generate/generate_SA-1B.py b/data_generation/generate/generate_SA-1B.py new file mode 100644 index 0000000000000000000000000000000000000000..1ac4d7f6c1b88bcb785620da52ff95de61a4db89 --- /dev/null +++ b/data_generation/generate/generate_SA-1B.py @@ -0,0 +1,216 @@ +import os +import pathlib +import sys + +from mmcv import DataLoader + +parent_path = pathlib.Path(__file__).absolute().parent.parent +parent_path = os.path.abspath(parent_path) +sys.path.append(parent_path) +os.chdir(parent_path) +print(f'>-------------> parent path {parent_path}') +print(f'>-------------> current work dir {os.getcwd()}') + +import glob +import json +import argparse +import subprocess +import multiprocessing +import numpy as np + +from tqdm import tqdm +from PIL import Image +from os.path import join +from joblib import delayed, Parallel +from pycocotools import mask as mask_utils + +import torch +from torchvision.datasets import VisionDataset + +from generate.img_to_token import img_to_token +from vqgan.load import six_crop_encode_transform + +CPU_COUNT = multiprocessing.cpu_count() + + +def convert_anns_to_mask(sam_label): + # device = f'cuda:{0}' + device = f'cpu' + + image_info = sam_label['image'] + anns = sam_label['annotations'] + width, height, file_name = image_info['width'], image_info['height'], image_info['file_name'] + + if len(anns) == 0: + return + + sorted_anns = sorted(anns, key=(lambda x: x['area']), reverse=True) + mask_img = torch.zeros((height, width, 3), device=device) + one_img = torch.ones((height, width, 3), device=device) + + for ann in sorted_anns: + mask = mask_utils.decode(ann['segmentation']) + mask = torch.tensor(mask, device=device) + mask = torch.repeat_interleave(mask.unsqueeze(dim=2), repeats=3, dim=2) + + color_mask = torch.rand(3, device=device) * one_img + mask_img += color_mask * mask + + del mask, color_mask + torch.cuda.empty_cache() + + mask_img_npy = mask_img.cpu().numpy() + mask_img_npy = (255 * mask_img_npy).astype(np.uint8) + + del mask_img, one_img + torch.cuda.empty_cache() + + return mask_img_npy + + +def convert_sam_label(json_dir, out_dir, tar_name): + print(f'>----------------------: convert sam label: {tar_name} ...\n') + + os.makedirs(out_dir, exist_ok=True) + + def _convert(_json_path, _out_dir): + data_name = os.path.basename(_json_path) + out_path = join(_out_dir, data_name.replace('json', 'png')) + with open(_json_path) as f: + sam_label = json.load(f) + + mask_img = convert_anns_to_mask(sam_label) + mask_img = Image.fromarray(mask_img) + mask_img.save(out_path) + + json_list = glob.glob(join(json_dir, '*.json')) + + Parallel(n_jobs=CPU_COUNT)( + delayed(_convert)(index, json_path, out_dir) for json_path in tqdm(json_list)) + + +class SamDataset(VisionDataset): + def __init__( + self, + root: str, + mask_root, + transform=None, + target_transform=None, + transforms=None + ) -> None: + super().__init__(root, transforms, transform, target_transform) + self.mask_root = mask_root + + file_list = glob.glob(join(root, '*.jpg')) + ids = [os.path.basename(i).split('.')[0] for i in file_list] + self.ids = list(sorted(ids)) + + def _load_image(self, id: int): + path = join(self.root, f'{id}.jpg') + return Image.open(path).convert("RGB") + + def _load_mask(self, id: int): + path = join(self.mask_root, f'{id}.png') + return Image.open(path).convert("RGB") + + def __getitem__(self, index: int): + id = self.ids[index] + image = self._load_image(id) + mask_img = self._load_mask(id) + + images = self.transform(image) + mask_imgs = self.transform(mask_img) + + data_list = [] + for _img, _mask_img in zip(images, mask_imgs): + _data = torch.stack([_img, _mask_img], dim=0) + data_list.append(_data) + + data = torch.cat(data_list, dim=0) + + return data + + def __len__(self) -> int: + return len(self.ids) + + +def convert_img_to_token(args, img_data_dir, mask_data_dir, out_dir, tar_name, device=None): + print(f'>----------------------: Convert img to token: {tar_name} ...') + + dataset = SamDataset(img_data_dir, mask_data_dir, transform=six_crop_encode_transform([800, 800])) + data_loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_work) + img_to_token(args, data_loader, args.output_path, device=device) + + +def unzip_data(tar_root, img_json_dir, tar_name): + print(f'>----------------------: Unzip data: {tar_name} ...') + + local_tar_path = join(tar_root, f'{tar_name}.tar') + os.makedirs(img_json_dir, exist_ok=True) + + cmd = f'tar -xf {local_tar_path} -C {img_json_dir}' + subprocess.check_call(args=cmd, shell=True) + + +def remove_tmpfile(img_json_dir, mask_dir, tar_name): + print(f'>----------------------: Remove tmpfile: {tar_name} ...') + + tmp_files = [ + img_json_dir, + mask_dir + ] + + for tmp_file in tmp_files: + cmd = f'rm -rf {tmp_file}' + subprocess.check_call(args=cmd, shell=True) + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--tar_root", type=str, default="data/SA-1B/tar") + parser.add_argument("--img_json_root", type=str, default="data/SA-1B/tmp/img_json") + parser.add_argument("--mask_root", type=str, default="data/SA-1B/tmp/mask") + parser.add_argument("--output_path", type=str, default="vq_token/SA-1B") + + parser.add_argument("--num_work", type=int, default=64) + parser.add_argument("--batch_size", type=int, default=32) + parser.add_argument("--dp_mode", action='store_true', default=False) + parser.add_argument("--model_name_or_path", type=str, default="weight/vqgan-f16-8192-laion") + args = parser.parse_args() + + return args + + +if __name__ == '__main__': + args = get_args() + + exclusion_data = [] + + tar_name_list = os.listdir(args.tar_root) + tar_name_list = [i.split('.')[0] for i in tar_name_list if i[-3:] == 'tar'] + + if os.path.exists(args.output_path): + exist_token_name_list = os.listdir(args.output_path) + else: + exist_token_name_list = [] + + tar_name_list = list(set(tar_name_list) - set(exist_token_name_list)) + tar_name_list = sorted(tar_name_list) + + for index, tar_name in enumerate(tar_name_list): + + if tar_name in exclusion_data: + continue + + print(f'\n\nProcessing sam data: {tar_name} {index + 1}/{len(tar_name_list)} ...') + img_json_dir = join(args.img_json_root, tar_name) + mask_dir = join(args.mask_root, tar_name) + out_dir = join(args.output_path, tar_name) + + unzip_data(args.tar_root, img_json_dir, tar_name) + convert_sam_label(img_json_dir, mask_dir, tar_name) + + device = f'cuda:{0}' + convert_img_to_token(args, img_json_dir, mask_dir, out_dir, tar_name, device=device) + + remove_tmpfile(img_json_dir, mask_dir, tar_name) diff --git a/data_generation/generate/generate_coco-keypoint.py b/data_generation/generate/generate_coco-keypoint.py new file mode 100644 index 0000000000000000000000000000000000000000..43207ad16e0b4abc3f0db9b452b298a46df12cff --- /dev/null +++ b/data_generation/generate/generate_coco-keypoint.py @@ -0,0 +1,106 @@ +import os +import pathlib +import sys + +parent_path = pathlib.Path(__file__).absolute().parent.parent +parent_path = os.path.abspath(parent_path) +sys.path.append(parent_path) +os.chdir(parent_path) +print(f'>-------------> parent path {parent_path}') +print(f'>-------------> current work dir {os.getcwd()}') + +import argparse +import glob +import multiprocessing + +from PIL import Image +from os.path import join + +import torch +from torch.utils.data import DataLoader +from torchvision.datasets import VisionDataset + +from vqgan.load import encode_transform +from generate.img_to_token import img_to_token + +CPU_COUNT = multiprocessing.cpu_count() + + +class KeyPointDataset(VisionDataset): + def __init__( + self, + root: str, + target_root, + transform=None, + target_transform=None, + transforms=None, + transform_name='encode_transform' + ) -> None: + super().__init__(root, transforms, transform, target_transform) + self.target_root = target_root + + file_list = glob.glob(join(root, '*.jpg')) + ids = [os.path.basename(i).split('.')[0] for i in file_list] + self.ids = list(sorted(ids)) + self.transform_name = transform_name + + def _load_image(self, id: int): + path = join(self.root, f'{id}.jpg') + return Image.open(path).convert("RGB") + + def _load_target(self, id: int): + path = join(self.target_root, f'{id}.jpg') + return Image.open(path).convert("RGB") + + def __getitem__(self, index: int): + id = self.ids[index] + image = self._load_image(id) + target_img = self._load_target(id) + + images = self.transform(image) + target_imgs = self.transform(target_img) + + data_list = [] + if self.transform_name == 'six_crop_encode_transform': + for _img, _target_img in zip(images, target_imgs): + _data = torch.stack([_img, _target_img], dim=0) + data_list.append(_data) + else: + _data = torch.stack([images, target_imgs], dim=0) + data_list.append(_data) + + data = torch.cat(data_list, dim=0) + + return data + + def __len__(self) -> int: + return len(self.ids) + + +def convert_img_to_token(args, device=None): + dataset = KeyPointDataset(args.input_data, args.target_data, transform=encode_transform) + data_loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_work) + img_to_token(args, data_loader, args.output_path, device=device) + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--input_data", type=str, default="coco-pose/GT/val2017/visual-crop/images") + parser.add_argument("--target_data", type=str, default="coco-pose/GT/val2017/visual-crop/keypoints") + parser.add_argument("--output_path", type=str, default="vq_token/coco-crop/val2017") + + parser.add_argument("--num_work", type=int, default=64) + parser.add_argument("--batch_size", type=int, default=16) + parser.add_argument("--dp_mode", action='store_true', default=False) + parser.add_argument("--model_name_or_path", type=str, default="weight/vqgan-f16-8192-laion") + args = parser.parse_args() + + return args + + +if __name__ == '__main__': + args = get_args() + + device = f'cuda:{0}' + convert_img_to_token(args, device=device) diff --git a/data_generation/generate/generate_hdvila_100m.py b/data_generation/generate/generate_hdvila_100m.py new file mode 100644 index 0000000000000000000000000000000000000000..2269a366baaf58d6cf14815035e5c03ee75bf4ec --- /dev/null +++ b/data_generation/generate/generate_hdvila_100m.py @@ -0,0 +1,118 @@ +import os +import pathlib +import sys + +parent_path = pathlib.Path(__file__).absolute().parent.parent +parent_path = os.path.abspath(parent_path) +sys.path.append(parent_path) +os.chdir(parent_path) +# print(f'>-------------> parent path {parent_path}') +# print(f'>-------------> current work dir {os.getcwd()}') + +import argparse +import glob +import jsonlines + +from os.path import join +from tqdm import tqdm +from PIL import Image +# from joblib import Parallel, delayed + +import torch +from timm.data import ImageDataset +from torch.utils.data import DataLoader + +from vqgan.load import encode_transform +from vqgan.utils import init_vqgan_encoder, get_multiprocess +from generate.img_to_token import data_loader_to_token, save_bin_and_meta_file + + +class ImageDatasetNoLabel(ImageDataset): + def __getitem__(self, index): + img, target = self.parser[index] + img = img.read() if self.load_bytes else Image.open(img).convert('RGB') + self._consecutive_errors = 0 + if self.transform is not None: + img = self.transform(img) + return img + + +def convert_img_to_token(args, data_dir, out_dir, encoder, device=None): + all_data_bin_list, all_cu_seq_len_list = [], [] + + for sub_dir_name in os.listdir(data_dir): + input_dir = os.path.join(data_dir, sub_dir_name) + + if not os.path.exists(input_dir) or len(glob.glob(join(input_dir, '*'))) == 0: + # print('Path not exist: ', input_dir) + continue + + dataset = ImageDatasetNoLabel(input_dir, transform=encode_transform) + new_multiprocess_ctx = get_multiprocess() + data_loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False, num_workers=8, + multiprocessing_context=new_multiprocess_ctx) + + data_bin_list, cu_seq_len_list = data_loader_to_token(encoder, data_loader, device) + all_data_bin_list.extend(data_bin_list) + all_cu_seq_len_list.extend(cu_seq_len_list) + + save_bin_and_meta_file(out_dir, all_data_bin_list, all_cu_seq_len_list) + + +def convert_single_gpu(video_name_list, num_work, index): + work_len = len(video_name_list) // num_work + start, end = index * work_len, (index + 1) * work_len + if index == num_work - 1: + work_video_name_list = video_name_list[start:] + else: + work_video_name_list = video_name_list[start: end] + + device = f'cuda:{index}' + encoder = init_vqgan_encoder(args.model_name_or_path, device) + + for video_name in tqdm(work_video_name_list): + data_dir = join(args.data_root, video_name) + out_dir = join(args.output_root, video_name) + + convert_img_to_token(args, data_dir, out_dir, encoder, device) + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--video_info_json", type=str, + default="hdvila_100m/cut_video_results/cut_part0.jsonl") + parser.add_argument("--data_root", type=str, default='hdvila_100m/video_clips_imgs') + parser.add_argument("--output_root", type=str, default="vq_token/hdvila_100m_2") + + parser.add_argument("--num_gpu", type=int, default=-1) + parser.add_argument("--batch_size", type=str, default=128) + parser.add_argument("--model_name_or_path", type=str, default="weight/vqgan-f16-8192-laion") + args = parser.parse_args() + + if args.num_gpu == -1: + args.num_gpu = torch.cuda.device_count() + + return args + + +if __name__ == '__main__': + args = get_args() + num_gpu = args.num_gpu + + print('Convert video info json: ', args.video_info_json) + + with jsonlines.open(args.video_info_json, 'r') as f: + video_name_list = [l.split('.')[0] for l in f] + + if os.path.exists(args.output_root): + exist_token_name = set(os.listdir(args.output_root)) + else: + exist_token_name = [] + + video_name_list = list(set(video_name_list) - set(exist_token_name)) + + for i in range(num_gpu): + convert_single_gpu(video_name_list, num_gpu, i) + + # Parallel(n_jobs=num_gpu)(delayed(convert_single_gpu)(video_name_list, num_gpu, i) for i in range(num_gpu)) diff --git a/data_generation/generate/generate_laion.py b/data_generation/generate/generate_laion.py new file mode 100644 index 0000000000000000000000000000000000000000..cd2de54a288c93290883812b4f0814d51af3c185 --- /dev/null +++ b/data_generation/generate/generate_laion.py @@ -0,0 +1,95 @@ +import os +import pathlib +import sys + +parent_path = pathlib.Path(__file__).absolute().parent.parent +parent_path = os.path.abspath(parent_path) +sys.path.append(parent_path) +os.chdir(parent_path) +print(f'>-------------> parent path {parent_path}') +print(f'>-------------> current work dir {os.getcwd()}') + + +import argparse +import subprocess +import tarfile +import time +from multiprocessing import Pool + +from timm.data import ImageDataset +from torch.utils.data import DataLoader +from tqdm import tqdm + +from generate.img_to_token import img_to_token +from vqgan.load import encode_transform + + +def extract_tar(tar_info): + tar_file, folder_path, extract_path, prefix = tar_info + tar_file_path = os.path.join(folder_path, tar_file) + target_folder = os.path.join(extract_path, f"{prefix}_{tar_file[:-4]}") + + with tarfile.open(tar_file_path, 'r') as tar: + tar.extractall(target_folder) + + print(f"Extracted {tar_file} to {target_folder}") + + cmd_txt = 'yes | rm -r ' + target_folder + '/*.txt' + subprocess.run(cmd_txt, shell=True) + cmd_json = 'yes | rm -r ' + target_folder + '/*.json' + subprocess.run(cmd_json, shell=True) + cmd_tar_file = 'yes | rm -r ' + tar_file_path + subprocess.run(cmd_tar_file, shell=True) + + +def extract_all_tarfiles_parallel(folder_path, extract_path, prefix, num_processes=4): + file_list = [file for file in os.listdir(folder_path) if file.endswith('.tar')] + tar_info_list = [(tar_file, folder_path, extract_path, prefix) for tar_file in file_list] + + with Pool(num_processes) as pool: + pool.map(extract_tar, tar_info_list) + + +def list_subdir(folder_path): + subdir = [f.name for f in os.scandir(folder_path) if f.is_dir()] + return subdir + + +def get_args(): + parser = argparse.ArgumentParser() + + # unzip + parser.add_argument("--folder_path", type=str, default='/cache/data/laion400m-images/part0/') + parser.add_argument("--extract_path", type=str, default='/home/ma-user/work/laion400m-images/part0_jpg/') + parser.add_argument("--prefix", type=str, default='laion_part0') + parser.add_argument("--num_processes", type=int, default=10) + # vqgan convert + parser.add_argument("--data", type=str, default='/cache/laion_jpg/part0/') # folder of unziped imgs + parser.add_argument("--batch_size", type=str, default=256) # folder of imgs + parser.add_argument("--output", type=str, default="/cache/laion_train_convert/part0/") + parser.add_argument("--model_name_or_path", type=str, default="weight/vqgan-f16-8192-laion") + args = parser.parse_args() + args.data = args.extract_path + + return args + + +if __name__ == '__main__': + args = get_args() + + unzip_start_time = time.time() + extract_all_tarfiles_parallel(args.folder_path, args.extract_path, args.prefix, args.num_processes) + print('########### unzip time: ', time.time() - unzip_start_time) + + dir_names = list_subdir(args.data) + + device = 'cuda:0' + + for idx, sub_dir_name in enumerate(tqdm(dir_names)): + output_dir = os.path.join(args.output, sub_dir_name) + + dataset = ImageDataset(args.input_data, args.target_data, transform=encode_transform) + data_loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_work) + img_to_token(args, data_loader, args.output_path, device=device) + + print('Finish convert...') diff --git a/data_generation/generate/img_to_token.py b/data_generation/generate/img_to_token.py new file mode 100644 index 0000000000000000000000000000000000000000..3ac3e6b88811a1889b4b61ae38e21be4d3a03757 --- /dev/null +++ b/data_generation/generate/img_to_token.py @@ -0,0 +1,63 @@ +import json +import os +from itertools import chain +from os.path import join + +import numpy as np +import torch +from torch import nn +from tqdm import tqdm + +from vqgan.utils import init_vqgan_encoder + + +def data_loader_to_token(encoder, data_loader, device): + cu = 0 + data_bin_list = [] + cu_seq_len_list = [] + for _data in tqdm(data_loader): + _data = _data.to(device) + + if _data.dim() == 5: + data_list = list(torch.split(_data, 1, dim=0)) + data_list = [i.squeeze(dim=0) for i in data_list] + data = torch.cat(data_list, dim=0) + else: + data = _data + + _, out_tokens = encoder(data) + + indices_list = list(torch.split(out_tokens, 2, dim=0)) + + for indices in indices_list: + tokens = list(chain(*indices.tolist())) + seq_len = len(tokens) + saved_bin = str.encode(json.dumps(dict(tokens=tokens)) + "\n") + + data_bin_list.append(saved_bin) + cu_seq_len_list.append((cu, seq_len)) + cu += len(saved_bin) + return data_bin_list, cu_seq_len_list + + +def save_bin_and_meta_file(out_dir, data_bin_list, cu_seq_len_list): + os.makedirs(out_dir, exist_ok=True) + out_bin = join(out_dir, "train.bin") + out_meta = join(out_dir, "train.bin.meta") + + with open(out_bin, "wb+") as bin_file: + bin_file.writelines(data_bin_list) + + cu_seq_len_list = np.array(cu_seq_len_list, dtype=np.int64) + with open(out_meta, "wb+") as meta_file: + np.save(meta_file, cu_seq_len_list) + + +def img_to_token(args, data_loader, out_dir, device=None): + encoder = init_vqgan_encoder(args.model_name_or_path, device) + + if args.dp_mode: + encoder = nn.DataParallel(encoder) + + data_bin_list, cu_seq_len_list = data_loader_to_token(encoder, data_loader, device) + save_bin_and_meta_file(out_dir, data_bin_list, cu_seq_len_list) diff --git a/data_generation/generate/token_concat.py b/data_generation/generate/token_concat.py new file mode 100644 index 0000000000000000000000000000000000000000..344a8afca6599e18a256e0f72d6f71a00cf80ad3 --- /dev/null +++ b/data_generation/generate/token_concat.py @@ -0,0 +1,69 @@ +import json +import os +from itertools import chain +from os.path import join + +import numpy as np +from joblib import Parallel, delayed +from tqdm import tqdm + + +def img_concat(data_root, output_root, data_name, token_num_per_sentence): + data_dir = join(data_root, data_name) + out_dir = join(output_root, data_name) + + os.makedirs(out_dir, exist_ok=True) + + data_bin_path = os.path.join(data_dir, "train.bin") + out_data_bin_path = os.path.join(out_dir, "train.bin") + out_data_meta_path = os.path.join(out_dir, "train.bin.meta") + + with open(data_bin_path, "r") as bin_file: + data_bin = bin_file.readlines() + + cu = 0 + new_data_bin = [] + cu_seq_len_list = [] + + sentence = [] + for index, data in enumerate(data_bin): + data = json.loads(data)['tokens'] + if index > 0 and index % token_num_per_sentence == 0: + tokens = list(chain(*sentence)) + seq_len = len(tokens) + saved_bin = str.encode(json.dumps(dict(tokens=tokens)) + "\n") + + new_data_bin.append(saved_bin) + cu_seq_len_list.append((cu, seq_len)) + cu += len(saved_bin) + sentence = [] + + sentence.append(data) + + tokens = list(chain(*sentence)) + seq_len = len(tokens) + saved_bin = str.encode(json.dumps(dict(tokens=tokens)) + "\n") + + new_data_bin.append(saved_bin) + cu_seq_len_list.append((cu, seq_len)) + cu += len(saved_bin) + + with open(out_data_bin_path, "wb+") as out_bin_file: + out_bin_file.writelines(new_data_bin) + np.save(out_data_meta_path, cu_seq_len_list) + os.rename(f'{out_data_meta_path}.npy', out_data_meta_path) + + +if __name__ == '__main__': + token_num_per_sentence = 6 + file_name = 'Rain13K' + data_root = '/home/ma-user/work/data/vq_token' + + data_dir = join(data_root, file_name) + output_dir = join(data_root, f'{file_name}-sentence_{token_num_per_sentence}') + + # for data_name in tqdm(os.listdir(data_dir)): + # img_concat(data_dir, output_dir, data_name, token_num_per_sentence) + + Parallel(n_jobs=64)(delayed(img_concat)(data_dir, output_dir, data_name, token_num_per_sentence) + for data_name in tqdm(os.listdir(data_dir))) diff --git a/data_generation/requirements.txt b/data_generation/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..801b9b8fb3dcdafd48dec6f323863ab45dae9343 --- /dev/null +++ b/data_generation/requirements.txt @@ -0,0 +1,5 @@ +timm==0.6.12 +accelerate +jsonlines +nvitop +multiprocess \ No newline at end of file diff --git a/data_generation/vqgan/__init__.py b/data_generation/vqgan/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data_generation/vqgan/__pycache__/__init__.cpython-37.pyc b/data_generation/vqgan/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef607f5ebfa0bd9fe5138425af80542d33489608 Binary files /dev/null and b/data_generation/vqgan/__pycache__/__init__.cpython-37.pyc differ diff --git a/data_generation/vqgan/__pycache__/__init__.cpython-39.pyc b/data_generation/vqgan/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bff07fadd6f19dceb8750fea8c5eda0acc063fda Binary files /dev/null and b/data_generation/vqgan/__pycache__/__init__.cpython-39.pyc differ diff --git a/data_generation/vqgan/__pycache__/calvin_img.cpython-39.pyc b/data_generation/vqgan/__pycache__/calvin_img.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9b611edc2242b73b33894333c8f0eccc5994ce2 Binary files /dev/null and b/data_generation/vqgan/__pycache__/calvin_img.cpython-39.pyc differ diff --git a/data_generation/vqgan/__pycache__/load.cpython-37.pyc b/data_generation/vqgan/__pycache__/load.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a4d995f6a902098a3315bf3c3feb874332e18e3a Binary files /dev/null and b/data_generation/vqgan/__pycache__/load.cpython-37.pyc differ diff --git a/data_generation/vqgan/__pycache__/load.cpython-39.pyc b/data_generation/vqgan/__pycache__/load.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..883de5140702771e77b602f224372f33af08c82e Binary files /dev/null and b/data_generation/vqgan/__pycache__/load.cpython-39.pyc differ diff --git a/data_generation/vqgan/laion_convert.py b/data_generation/vqgan/laion_convert.py new file mode 100644 index 0000000000000000000000000000000000000000..270fde1cd46ef74c13819ce8e73e705907c97660 --- /dev/null +++ b/data_generation/vqgan/laion_convert.py @@ -0,0 +1,157 @@ +import argparse +import json +import os +import pathlib +import random +import subprocess +import tarfile +import time +from multiprocessing import Pool + +import numpy as np +import torch +import torch.nn as nn +from timm.data import ImageDataset +from torch.utils.data import DataLoader + +torch.set_grad_enabled(False) +random.seed(42) + +from load import encode_transform, load_model + +# set seed +random_seed = 1 +torch.manual_seed(random_seed) +torch.cuda.manual_seed(random_seed) +torch.backends.cudnn.deterministic = True +torch.backends.cudnn.benchmark = False +np.random.seed(random_seed) + +# args +parser = argparse.ArgumentParser() +# unzip +parser.add_argument("--folder_path", type=str, default='/cache/data/laion400m-images/part0/') +parser.add_argument("--extract_path", type=str, default='/home/ma-user/work/laion400m-images/part0_jpg/') +parser.add_argument("--prefix", type=str, default='laion_part0') +parser.add_argument("--num_processes", type=int, default=10) +# vqgan convert +parser.add_argument("--data", type=str, default='/cache/laion_jpg/part0/') # folder of unziped imgs +parser.add_argument("--batch_size", type=str, default=256) # folder of imgs +parser.add_argument("--type", type=str, default="internlm") +parser.add_argument("--output", type=str, default="/cache/laion_train_convert/part0/") +parser.add_argument("--model_name_or_path", type=str, default="/cache/ckpt/vqgan-f16-8192-laion") +args = parser.parse_args() +args.data = args.extract_path + +# unzip part +unzip_start_time = time.time() + + +def extract_tar(tar_info): + tar_file, folder_path, extract_path, prefix = tar_info + tar_file_path = os.path.join(folder_path, tar_file) + target_folder = os.path.join(extract_path, f"{prefix}_{tar_file[:-4]}") + + with tarfile.open(tar_file_path, 'r') as tar: + tar.extractall(target_folder) + + print(f"Extracted {tar_file} to {target_folder}") + + cmd_txt = 'yes | rm -r ' + target_folder + '/*.txt' + subprocess.run(cmd_txt, shell=True) + cmd_json = 'yes | rm -r ' + target_folder + '/*.json' + subprocess.run(cmd_json, shell=True) + cmd_tar_file = 'yes | rm -r ' + tar_file_path + subprocess.run(cmd_tar_file, shell=True) + + +def extract_all_tarfiles_parallel(folder_path, extract_path, prefix, num_processes=4): + file_list = [file for file in os.listdir(folder_path) if file.endswith('.tar')] + tar_info_list = [(tar_file, folder_path, extract_path, prefix) for tar_file in file_list] + + with Pool(num_processes) as pool: + pool.map(extract_tar, tar_info_list) + + +extract_all_tarfiles_parallel(args.folder_path, args.extract_path, args.prefix, args.num_processes) +print('########### unzip time: ', time.time() - unzip_start_time) + +convert_start_time = time.time() + + +# convert part +def list_subdir(folder_path): + subdir = [f.name for f in os.scandir(folder_path) if f.is_dir()] + return subdir + + +dir_names = list_subdir(args.data) + +print('Strating convert via vqgan...') +print(args) +print(len(dir_names)) + +vq_model = load_model(args.model_name_or_path) +vq_model = vq_model.cuda().eval() + + +class ParallelWrapper(nn.Module): + def __init__(self, vq_model, func='encode'): + super(ParallelWrapper, self).__init__() + self.vq_model = vq_model + self.func = func + + def forward(self, x): + return getattr(self.vq_model, self.func)(x) + + +encoder = ParallelWrapper(vq_model) +encoder = nn.DataParallel(encoder) + + +def dumps(data): + seqlen = len(data) + saved_bin = str.encode(json.dumps(dict(tokens=data)) + "\n") + return {"bin": saved_bin, "length": seqlen} + + +for idx, sub_dir_name in enumerate(dir_names): + if idx % 10 == 0: + print(idx) + + output_dir = os.path.join(args.output, sub_dir_name) + + pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) + + out_bin = os.path.join(output_dir, "train.bin") + out_meta = os.path.join(output_dir, "train.bin.meta") + + pathlib.Path(out_bin).touch(exist_ok=True) + pathlib.Path(out_meta).touch(exist_ok=True) + + dataset = ImageDataset(os.path.join(args.data, sub_dir_name), transform=encode_transform) + loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=8) + + from tqdm import tqdm + + cu = 0 + cu_seqlens = [] + with open(out_bin, "wb+") as bin_file: + for i, (imgs, _) in enumerate(tqdm(loader)): + imgs = imgs.cuda() + quantized_states, indices = encoder(imgs) + + for indices_i in indices.tolist(): + token = dumps(indices_i) + seqlen = token["length"] # 256 + token_data = token["bin"] + bin_file.write(token_data) + # print((cu, seqlen)) + cu_seqlens.append((cu, seqlen)) + cu += len(token_data) + cu_seqlens = np.array(cu_seqlens, dtype=np.int64) + with open(out_meta, "wb+") as meta_file: + np.save(meta_file, cu_seqlens) + +print('########### unzip time: ', time.time() - convert_start_time) +print('Finish convert...') diff --git a/data_generation/vqgan/load.py b/data_generation/vqgan/load.py new file mode 100644 index 0000000000000000000000000000000000000000..a4b89c8534188ec0df63d7788246b72c5ad61b50 --- /dev/null +++ b/data_generation/vqgan/load.py @@ -0,0 +1,557 @@ +from typing import List, Tuple +import sys +import os +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +# ROOT_DIR = os.path.dirname(BASE_DIR) +# sys.path.append(ROOT_DIR) +sys.path.append(BASE_DIR) +import io +import torch +import torch.nn as nn +import torchvision.transforms.functional as F +from PIL import Image +from torch import Tensor +from torchvision import transforms +from torchvision.transforms import Lambda + +from muse import VQGANModel +from losses.vgperceptual import VQLPIPSWithDiscriminator + + +class ForwardWrapper(nn.Module): + def __init__(self, vq_model, func='encode'): + super(ForwardWrapper, self).__init__() + self.vq_model = vq_model + self.func = func + + def forward(self, x): + return getattr(self.vq_model, self.func)(x) + +class Train_VQGAN(VQGANModel): + def __init__( + self, num_embeddings, + ): + + # super().__init__( + # resolution=256, + # num_channels=3, + # hidden_channels=128, + # channel_mult=(1, 1, 2, 2, 4), + # num_res_blocks=2, + # attn_resolutions=[16], + # no_attn_mid_block=True, + # z_channels=256, + # num_embeddings=8192, + # quantized_embed_dim=256, + # dropout=0.0, + # resample_with_conv=True, + # commitment_cost=0.25, + # ) + + super().__init__( + resolution=256, + num_channels=3, + hidden_channels=128, + channel_mult=(1, 2, 2, 4, 6), + num_res_blocks=2, + attn_resolutions=[], + no_attn_mid_block=True, + z_channels=64, + num_embeddings=num_embeddings, + quantized_embed_dim=64, + dropout=0.0, + resample_with_conv=True, + commitment_cost=0.25, + ) + + # super().__init__( + # resolution=256, + # num_channels=3, + # hidden_channels=128, + # channel_mult=(1, 1, 2, 2, 4), + # num_res_blocks=2, + # attn_resolutions=[], + # no_attn_mid_block=True, + # z_channels=256, + # num_embeddings=1024, + # quantized_embed_dim=256, + # dropout=0.0, + # resample_with_conv=True, + # commitment_cost=0.25, + # ) + + self.loss = VQLPIPSWithDiscriminator( + disc_start=0, + disc_in_channels=3, + disc_conditional=False, + disc_weight=0.5, + disc_num_layers=2, + codebook_weight=1.0, + perceptual_weight=1.0, + disc_loss='hinge') + + def encode(self, pixel_values, return_loss=False): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quantized_states, codebook_indices, codebook_loss = self.quantize(hidden_states, return_loss) + + return quantized_states, codebook_loss, codebook_indices + + def decode(self, quantized_states): + hidden_states = self.post_quant_conv(quantized_states) + reconstructed_pixel_values = self.decoder(hidden_states) + return reconstructed_pixel_values + + def decode_code(self, codebook_indices): + quantized_states = self.quantize.get_codebook_entry(codebook_indices) + reconstructed_pixel_values = self.decode(quantized_states) + return reconstructed_pixel_values + + def get_last_layer(self): + return self.decoder.conv_out.weight + + def get_code(self, pixel_values): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + codebook_indices = self.quantize.get_code(hidden_states) + return codebook_indices + + def forward(self, pixel_values, return_loss=False): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quantized_states, codebook_indices, codebook_loss = self.quantize(hidden_states, return_loss) + reconstructed_pixel_values = self.decode(quantized_states) + + return reconstructed_pixel_values, codebook_loss + + def training_step(self, x, optimizer_idx, device, epoch_cnt): + + xrec, qloss = self(x, return_loss=True) + + if optimizer_idx == 0: + # autoencode + aeloss, log_dict_ae = self.loss(qloss, x, xrec, optimizer_idx, epoch_cnt, + last_layer=self.get_last_layer(), split="train") + + # self.log("train/aeloss", aeloss, prog_bar=True, logger=True, on_step=True, on_epoch=True) + # self.log_dict(log_dict_ae, prog_bar=False, logger=True, on_step=True, on_epoch=True) + return aeloss, log_dict_ae + + if optimizer_idx == 1: + # discriminator + discloss, log_dict_disc = self.loss(qloss, x, xrec, optimizer_idx, epoch_cnt, + last_layer=self.get_last_layer(), split="train") + # self.log("train/discloss", discloss, prog_bar=True, logger=True, on_step=True, on_epoch=True) + # self.log_dict(log_dict_disc, prog_bar=False, logger=True, on_step=True, on_epoch=True) + return discloss, log_dict_disc + + def configure_optimizers(self, lr): + # lr = self.learning_rate + opt_ae = torch.optim.AdamW(list(self.encoder.parameters())+ + list(self.decoder.parameters())+ + list(self.quantize.parameters())+ + list(self.quant_conv.parameters())+ + list(self.post_quant_conv.parameters()), + lr=lr) + opt_disc = torch.optim.AdamW(self.loss.discriminator.parameters(), + lr=lr) + return [opt_ae, opt_disc], [] + +def load_model(path = None, num_embeddings=None): + # Load the pre-trained vq model from the hub + # vq_model = VQGANModel( + # resolution=256, + # num_channels=3, + # hidden_channels=128, + # channel_mult=(1, 2, 2, 4, 6), + # num_res_blocks=2, + # attn_resolutions=(), + # no_attn_mid_block=True, + # z_channels=64, + # num_embeddings=8192, + # quantized_embed_dim=64, + # dropout=0.0, + # resample_with_conv=True, + # commitment_cost=0.25, + # ) + vq_model = Train_VQGAN(num_embeddings) + if path is not None: + ckpt = torch.load(path, map_location='cpu') + vq_model.load_state_dict(ckpt, strict=False) + # vq_model = VQGANModel.from_pretrained(path) + return vq_model + +def load_model_no_disc(path='./vqgan_ckpt'): + vq_model = VQGANModel.from_pretrained(path) + return vq_model + +def load_encoder(path): + vq_model = load_model(path) + encoder = ForwardWrapper(vq_model) + return encoder + + +def load_decoder(path): + vq_model = load_model(path) + decoder = ForwardWrapper(vq_model, func='decode') + return decoder + + +def load_decoder_code(path): + vq_model = load_model(path) + decoder = ForwardWrapper(vq_model, func='decode_code') + return decoder + + +def convert_decode_to_pil(rec_image): + rec_image = 2.0 * rec_image - 1.0 + rec_image = torch.clamp(rec_image, -1.0, 1.0) + rec_image = (rec_image + 1.0) / 2.0 + rec_image *= 255.0 + rec_image = rec_image.permute(0, 2, 3, 1).detach().cpu().numpy().astype(np.uint8) + pil_images = [Image.fromarray(image) for image in rec_image] + return pil_images + + +class SixCrop(torch.nn.Module): + def __init__(self, crop_size): + super().__init__() + self.crop_size = crop_size + + # def get_dimensions(self, img): + # """Returns the dimensions of an image as [channels, height, width]. + # + # Args: + # img (PIL Image or Tensor): The image to be checked. + # + # Returns: + # List[int]: The image dimensions. + # """ + # if isinstance(img, torch.Tensor): + # return F_t.get_dimensions(img) + # + # return F_pil.get_dimensions(img) + + def get_dimensions(self, img) -> List[int]: + if hasattr(img, "getbands"): + channels = len(img.getbands()) + else: + channels = img.channels + width, height = img.size + return [channels, height, width] + + def six_crop(self, img: Tensor) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + """Crop the given image into four corners and the central crop. + If the image is torch Tensor, it is expected + to have [..., H, W] shape, where ... means an arbitrary number of leading dimensions + + .. Note:: + This transform returns a tuple of images and there may be a + mismatch in the number of inputs and targets your ``Dataset`` returns. + + Args: + img (PIL Image or Tensor): Image to be cropped. + size (sequence or int): Desired output size of the crop. If size is an + int instead of sequence like (h, w), a square crop (size, size) is + made. If provided a sequence of length 1, it will be interpreted as (size[0], size[0]). + + Returns: + tuple: tuple (tl, tr, bl, br, center) + Corresponding top left, top right, bottom left, bottom right and center crop. + """ + # if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + # _log_api_usage_once(five_crop) + + crop_height, crop_width = self.crop_size + _, image_height, image_width = self.get_dimensions(img) + + # if crop_width > image_width or crop_height > image_height: + # msg = "Requested crop size {} is bigger than input size {}" + # raise ValueError(msg.format(self.crop_size, (image_height, image_width))) + + if crop_width > image_width: + crop_width = image_width + crop_height = image_width + + if crop_height > image_height: + crop_width = image_height + crop_height = image_height + + tl = F.crop(img, 0, 0, crop_height, crop_width) + tr = F.crop(img, 0, image_width - crop_width, crop_height, crop_width) + bl = F.crop(img, image_height - crop_height, 0, crop_height, crop_width) + br = F.crop(img, image_height - crop_height, image_width - crop_width, crop_height, crop_width) + + if image_height > image_width: + center_top = int(round((image_height - crop_height) / 2.0)) + cl = F.crop(img, center_top, 0, crop_height, crop_width) + cr = F.crop(img, center_top, image_width - crop_width, crop_height, crop_width) + return tl, tr, cl, cr, bl, br + else: + center_left = int(round((image_width - crop_width) / 2.0)) + ct = F.crop(img, 0, center_left, crop_height, crop_width) + cb = F.crop(img, image_height - crop_height, center_left, crop_height, crop_width) + return tl, tr, ct, bl, br, cb + + # center = center_crop(img, [crop_height, crop_width]) + + def forward(self, img): + """ + Args: + img (PIL Image or Tensor): Image to be scaled. + + Returns: + PIL Image or Tensor: Rescaled image. + """ + return self.six_crop(img) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(size={self.crop_size})" + + +def six_crop_encode_transform(crop_size): + t = transforms.Compose( + [ + SixCrop(crop_size), + # transforms.Resize(256, interpolation=transforms.InterpolationMode.BILINEAR), + Lambda(lambda crops: + [transforms.Resize(256, interpolation=transforms.InterpolationMode.BILINEAR)(crop) for crop + in crops]), + Lambda(lambda crops: [transforms.ToTensor()(crop) for crop in crops]), + ] + ) + return t + + +encode_transform = transforms.Compose( + [ + transforms.Resize(256, interpolation=transforms.InterpolationMode.BILINEAR), + transforms.CenterCrop(256), + transforms.ToTensor(), + # transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ] +) + +encode_transform_no_crop = transforms.Compose( + [ + transforms.Resize([256, 256], interpolation=transforms.InterpolationMode.BILINEAR), + transforms.ToTensor(), + ] +) + +encode_transform_2 = transforms.Compose( + [ + transforms.RandomHorizontalFlip(), + transforms.RandomVerticalFlip(), + transforms.RandomRotation(180), + transforms.RandomResizedCrop(256, interpolation=transforms.InterpolationMode.BILINEAR), + transforms.ToTensor(), + ] +) + +encode_transform_rain_random = transforms.Compose( + [ + transforms.RandomHorizontalFlip(), + transforms.RandomVerticalFlip(), + transforms.RandomResizedCrop(256, interpolation=transforms.InterpolationMode.BILINEAR), + transforms.ToTensor(), + ] +) + +encode_transform_rain_random_2 = transforms.Compose( + [ + transforms.RandomHorizontalFlip(), + transforms.RandomVerticalFlip(), + transforms.RandomCrop(400), + transforms.Resize(256, interpolation=transforms.InterpolationMode.BILINEAR), + transforms.ToTensor(), + ] + +) + +import random + +from io import BytesIO +from PIL import Image + +# def decode_image(data): + +# # assert 'rgb.jpg' in data.keys() +# # print (data.keys()) +# rgb = data['video_hand.pickle'] +# rgb = random.sample(rgb, 1)[0] +# # rgb = data['rgb.jpg'] + +# rgb = Image.open(io.BytesIO(rgb)) +# rgb = encode_transform(rgb) +# # rgb = (rgb * 2) - 1 + +# return rgb + +def decode_image(data): + # assert 'rgb.jpg' in data.keys() + rgb = data['rgb.jpg'] + + if 'crop.pickle' in data.keys(): + crop = data['crop.pickle'] + else: + crop = None + + rgb = Image.open(BytesIO(rgb)).convert("RGB") + if crop is not None: + rgb = np.array(rgb) + rgb = rgb[crop[0][0]:crop[1][0], crop[0][1]:crop[1][1], :] + rgb = Image.fromarray(rgb).convert("RGB") + + rgb = encode_transform(rgb) + # rgb = (rgb * 2) - 1 + # rgb = (rgb + 1) / 2 + return rgb + +def code_usage_stat(model, ds, save_dir): + + import json + model.eval().cuda() + + stat = {} + + cnt = 0 + for i in range(8192): + stat[i] = 0 + + for d in ds: + + rgb = d.unsqueeze(0).cuda() + + with torch.no_grad(): + idxs = model.get_code(rgb)[0] + + idxs = idxs.cpu().numpy().tolist() + + for idx in idxs: + stat[idx] = stat[idx] + 1 + + print (cnt) + cnt += 1 + + with open(save_dir, 'w') as f: + json.dump(stat, f) + +def plot_code_stat(stat, out): + import json + import matplotlib.pyplot as plt + + with open(stat, 'r') as f: + stat = json.load(f) + + x = [] + y = [] + + for k, v in stat.items(): + x.append(int(k)) + y.append(int(v)) + + plt.bar(x, y) + plt.savefig(out) + + + +if __name__ == '__main__': + import numpy as np + from calvin_img import CalvinImgDataset + import matplotlib.pyplot as plt + import webdataset as wds + from tqdm import tqdm + from copy import deepcopy + + def generator_to_webdataset(ds, save_path: str): + sink = wds.TarWriter(save_path) + for data_dict in tqdm(ds): + sink.write(data_dict) + sink.close() + + + # plot_code_stat('./calvin_vqgan_stat.json', 'calvin.jpg') +# + vq_model = load_model('/mnt/bn/roboicl-jirong/codebase/RoboICL/src/vqgan_muse_pretrained/vqgan_ckpt/ckpt.pth', 8192) + + + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/git/DeLVM/data_generation/vqgan/vqgan_ckpt/ckpt.pth', 8192) + # # # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_muse_cfg_vl_robot_1e-4_192_codebook_0.1/checkpoint_vq_epoch_70655.tar') + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_muse_finetune_calvin_datacomp_1e-5_192_codebook_0.1_const_lr/checkpoint_vq_epoch_64511.tar') + + # # ds = "/mnt/bn/roboicllq-data1/calvin_img/calvin_img_{00000000..00000239}.tar" + # # ds = wds.WebDataset(ds).decode().map(decode_image) + + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_resume_aug/checkpoint_vq_epoch_97499.tar', 2048) + + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_aug_disc_0.5_resume/checkpoint_vq_epoch_43999.tar', 2048) + # # code_usage_stat(vq_model, ds, './cofinetune_vqgan_stat_24999.json') + + # # plot_code_stat('./cofinetune_vqgan_stat_24999.json', 'calvin.jpg') + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_1024_fintune/checkpoint_vq_epoch_9999.tar') + # # vq_model2 = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_cofinetune_2500/checkpoint_vq_epoch_1999.tar') + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_muse_ego4d_llava_calvin_3e-6_disc_0/checkpoint_vq_epoch_66999.tar', 8192) + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_256_ego4d_calvin_real_aug_disc_0.5_3e-5/checkpoint_vq_epoch_57499.tar', 2048) + # model_state_dict = vq_model.state_dict() + + # keys = list(model_state_dict.keys()) + # for key in keys: + # if 'loss' in key: + # del model_state_dict[key] + + # torch.save(model_state_dict, '/mnt/bn/roboicl-jirong/codebase/RoboICL/src/vqgan_muse_pretrained/ego4d_llava_real_calvin_2048_finetune/pytorch_model.bin') + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_256_datacomp_calvin_real_aug_disc_0.5/checkpoint_vq_epoch_8999.tar', 2048) + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_aug_disc_0.5_resume/checkpoint_vq_epoch_72999.tar', 2048) + # vq_model.eval() + # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_256_ego4d_calvin_real_aug_disc_0.5_3e-5/checkpoint_vq_epoch_57499.tar', 2048) + # # vq_model = load_model_no_disc('/mnt/bn/roboicl-jirong/codebase/RoboICL/src/vqgan_muse_pretrained/muse_cofinetune_8192') + # # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_muse_ego4d_llava_calvin_3e-6_disc_0/checkpoint_vq_epoch_66999.tar', 8192) + # # vq_model = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_muse_ego4d_llava_calvin_1e-5_disc_0_resume/checkpoint_vq_epoch_4999.tar', 8192) + vq_model.eval() + # # # # # trainset = CalvinImgDataset('/mnt/bn/robotics/manipulation_data/calvin_data', use_hand_observation=True) + + ds = '/mnt/bn/roboicl-jirong/codebase/real_data_img_0122.tar' + # ds = '/mnt/bn/roboicllq/img_wds/ego4d/ego4d_0000_mLNrAABME3.tar' + # ds = '/mnt/bn/roboicllq-data1/video_img_subset/ego4d/ego4d_0003_6IzswDQ9HomKtv5.tar' + # # ds = '/mnt/bn/roboicllq/img_wds/ego4d/ego4d_0000_IdrgFUyeJM.tar' + # # ds ='/mnt/bn/roboicllq-data1/processed_real/hand_imgs/real_data_hand_img_0130.tar' + # # ds = '/mnt/bn/roboicllq-data1/video_img_subset/ssv2/ssv2_0000_2yW4Acq9GFz6Y1t.tar' + # ds = '/mnt/bn/roboicl-jirong/codebase/calvin_img_00000046.tar' + + + # # ds = '/mnt/bn/roboicllq-data1/llava_pretrain/LLaVA-Pretrain/wds/llava_pretrain_0060.tar' + # # # ds = '/mnt/bn/roboicllq-data1/aligned_robot_ds/calvin/view_0/calvin_00000011.tar' + # ds = '/mnt/bn/roboicllq-data1/datacomp10m/datacomp_0000_6s2NkdV2Ijp0JTTNG5EUpt6AJ95uV9.tar' + ds = wds.WebDataset(ds).decode().map(decode_image) + + cnt = 0 + for d in ds: + + rgb = d.unsqueeze(0) + print (rgb.shape) + + with torch.no_grad(): + recon, _ = vq_model(rgb) + # recon2, _ = vq_model2(rgb) + + recon = convert_decode_to_pil(recon)[0] + # recon2 = convert_decode_to_pil(recon2)[0] + rgb = transforms.ToPILImage()(rgb[0]) + + fig, ax = plt.subplots(1,2) + + ax[0].imshow(np.array(rgb)) + ax[0].set_title('rgb') + + ax[1].imshow(np.array(recon)) + ax[1].set_title('rgb_1') + + # ax[2].imshow(np.array(recon2)) + # ax[2].set_title('rgb_2') + + fig.savefig('/mnt/bn/robotics-data-hl/jirong/git/DeLVM/data_generation/vqgan/recon/{}.jpg'.format(str(cnt).zfill(4))) + cnt += 1 + print (cnt) + if cnt > 100: + break diff --git a/data_generation/vqgan/losses/__pycache__/discriminator.cpython-37.pyc b/data_generation/vqgan/losses/__pycache__/discriminator.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0824d8bac391dafee4c7f532bcb560fed69c51b1 Binary files /dev/null and b/data_generation/vqgan/losses/__pycache__/discriminator.cpython-37.pyc differ diff --git a/data_generation/vqgan/losses/__pycache__/discriminator.cpython-39.pyc b/data_generation/vqgan/losses/__pycache__/discriminator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee891125d28d8f10abfca825f3ead2d67716d78f Binary files /dev/null and b/data_generation/vqgan/losses/__pycache__/discriminator.cpython-39.pyc differ diff --git a/data_generation/vqgan/losses/__pycache__/lpips.cpython-37.pyc b/data_generation/vqgan/losses/__pycache__/lpips.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d2a39c733c719e9a3f1ac133ee45e20fddcc5887 Binary files /dev/null and b/data_generation/vqgan/losses/__pycache__/lpips.cpython-37.pyc differ diff --git a/data_generation/vqgan/losses/__pycache__/lpips.cpython-39.pyc b/data_generation/vqgan/losses/__pycache__/lpips.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c561cc278827b622baf758a689e041a0a6d777e8 Binary files /dev/null and b/data_generation/vqgan/losses/__pycache__/lpips.cpython-39.pyc differ diff --git a/data_generation/vqgan/losses/__pycache__/vgperceptual.cpython-37.pyc b/data_generation/vqgan/losses/__pycache__/vgperceptual.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0d0f39ba8de512cb69a7c040558945ae5feadb34 Binary files /dev/null and b/data_generation/vqgan/losses/__pycache__/vgperceptual.cpython-37.pyc differ diff --git a/data_generation/vqgan/losses/__pycache__/vgperceptual.cpython-39.pyc b/data_generation/vqgan/losses/__pycache__/vgperceptual.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ef60d326f50b035a8484cbfdeff545faed0bc3f Binary files /dev/null and b/data_generation/vqgan/losses/__pycache__/vgperceptual.cpython-39.pyc differ diff --git a/data_generation/vqgan/losses/discriminator.py b/data_generation/vqgan/losses/discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..f198553e5e1c4274827e4cdc8adc41932f5c3be5 --- /dev/null +++ b/data_generation/vqgan/losses/discriminator.py @@ -0,0 +1,149 @@ +import functools +import torch.nn as nn + + +class ActNorm(nn.Module): + def __init__(self, num_features, logdet=False, affine=True, + allow_reverse_init=False): + assert affine + super().__init__() + self.logdet = logdet + self.loc = nn.Parameter(torch.zeros(1, num_features, 1, 1)) + self.scale = nn.Parameter(torch.ones(1, num_features, 1, 1)) + self.allow_reverse_init = allow_reverse_init + + self.register_buffer('initialized', torch.tensor(0, dtype=torch.uint8)) + + def initialize(self, input): + with torch.no_grad(): + flatten = input.permute(1, 0, 2, 3).contiguous().view(input.shape[1], -1) + mean = ( + flatten.mean(1) + .unsqueeze(1) + .unsqueeze(2) + .unsqueeze(3) + .permute(1, 0, 2, 3) + ) + std = ( + flatten.std(1) + .unsqueeze(1) + .unsqueeze(2) + .unsqueeze(3) + .permute(1, 0, 2, 3) + ) + + self.loc.data.copy_(-mean) + self.scale.data.copy_(1 / (std + 1e-6)) + + def forward(self, input, reverse=False): + if reverse: + return self.reverse(input) + if len(input.shape) == 2: + input = input[:,:,None,None] + squeeze = True + else: + squeeze = False + + _, _, height, width = input.shape + + if self.training and self.initialized.item() == 0: + self.initialize(input) + self.initialized.fill_(1) + + h = self.scale * (input + self.loc) + + if squeeze: + h = h.squeeze(-1).squeeze(-1) + + if self.logdet: + log_abs = torch.log(torch.abs(self.scale)) + logdet = height*width*torch.sum(log_abs) + logdet = logdet * torch.ones(input.shape[0]).to(input) + return h, logdet + + return h + + def reverse(self, output): + if self.training and self.initialized.item() == 0: + if not self.allow_reverse_init: + raise RuntimeError( + "Initializing ActNorm in reverse direction is " + "disabled by default. Use allow_reverse_init=True to enable." + ) + else: + self.initialize(output) + self.initialized.fill_(1) + + if len(output.shape) == 2: + output = output[:,:,None,None] + squeeze = True + else: + squeeze = False + + h = output / self.scale - self.loc + + if squeeze: + h = h.squeeze(-1).squeeze(-1) + return h + + +def weights_init(m): + classname = m.__class__.__name__ + if classname.find('Conv') != -1: + nn.init.normal_(m.weight.data, 0.0, 0.02) + elif classname.find('BatchNorm') != -1: + nn.init.normal_(m.weight.data, 1.0, 0.02) + nn.init.constant_(m.bias.data, 0) + + +class NLayerDiscriminator(nn.Module): + """Defines a PatchGAN discriminator as in Pix2Pix + --> see https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/master/models/networks.py + """ + def __init__(self, input_nc=3, ndf=64, n_layers=3, use_actnorm=False): + """Construct a PatchGAN discriminator + Parameters: + input_nc (int) -- the number of channels in input images + ndf (int) -- the number of filters in the last conv layer + n_layers (int) -- the number of conv layers in the discriminator + norm_layer -- normalization layer + """ + super(NLayerDiscriminator, self).__init__() + if not use_actnorm: + norm_layer = nn.BatchNorm2d + else: + norm_layer = ActNorm + if type(norm_layer) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters + use_bias = norm_layer.func != nn.BatchNorm2d + else: + use_bias = norm_layer != nn.BatchNorm2d + + kw = 4 + padw = 1 + sequence = [nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)] + nf_mult = 1 + nf_mult_prev = 1 + for n in range(1, n_layers): # gradually increase the number of filters + nf_mult_prev = nf_mult + nf_mult = min(2 ** n, 8) + sequence += [ + nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias), + norm_layer(ndf * nf_mult), + nn.LeakyReLU(0.2, True) + ] + + nf_mult_prev = nf_mult + nf_mult = min(2 ** n_layers, 8) + sequence += [ + nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias), + norm_layer(ndf * nf_mult), + nn.LeakyReLU(0.2, True) + ] + + sequence += [ + nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)] # output 1 channel prediction map + self.main = nn.Sequential(*sequence) + + def forward(self, input): + """Standard forward.""" + return self.main(input) \ No newline at end of file diff --git a/data_generation/vqgan/losses/lpips.py b/data_generation/vqgan/losses/lpips.py new file mode 100644 index 0000000000000000000000000000000000000000..ae06dc802574470e0ea2c479cee25bc3177b2396 --- /dev/null +++ b/data_generation/vqgan/losses/lpips.py @@ -0,0 +1,167 @@ +"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models""" + +import torch +import torch.nn as nn +from torchvision import models +from collections import namedtuple + +import os, hashlib +import requests +from tqdm import tqdm + +URL_MAP = { + "vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1" +} + +CKPT_MAP = { + "vgg_lpips": "vgg.pth" +} + +MD5_MAP = { + "vgg_lpips": "d507d7349b931f0638a25a48a722f98a" +} + + +def download(url, local_path, chunk_size=1024): + os.makedirs(os.path.split(local_path)[0], exist_ok=True) + with requests.get(url, stream=True) as r: + total_size = int(r.headers.get("content-length", 0)) + with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: + with open(local_path, "wb") as f: + for data in r.iter_content(chunk_size=chunk_size): + if data: + f.write(data) + pbar.update(chunk_size) + + +def md5_hash(path): + with open(path, "rb") as f: + content = f.read() + return hashlib.md5(content).hexdigest() + + +# def get_ckpt_path(name, root, check=False): +# assert name in URL_MAP +# path = os.path.join(root, CKPT_MAP[name]) +# if not os.path.exists(path) or (check and not md5_hash(path) == MD5_MAP[name]): +# print("Downloading {} model from {} to {}".format(name, URL_MAP[name], path)) +# download(URL_MAP[name], path) +# md5 = md5_hash(path) +# assert md5 == MD5_MAP[name], md5 +# return path + +class LPIPS(nn.Module): + # Learned perceptual metric + def __init__(self, use_dropout=True): + super().__init__() + self.scaling_layer = ScalingLayer() + self.chns = [64, 128, 256, 512, 512] # vg16 features + self.net = vgg16(pretrained=True, requires_grad=False) + self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout) + self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout) + self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout) + self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout) + self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout) + # self.load_from_pretrained() + + ckpt = torch.load('/mnt/bn/roboicl-jirong/codebase/DeLVM/data_generation/vqgan/vgg.pth', map_location='cpu') + self.load_state_dict(ckpt, strict=False) + for param in self.parameters(): + param.requires_grad = False + + # def load_from_pretrained(self, name="vgg_lpips"): + # ckpt = get_ckpt_path(name, "taming/modules/autoencoder/lpips") + # self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + # print("loaded pretrained LPIPS loss from {}".format(ckpt)) + + # @classmethod + # def from_pretrained(cls, name="vgg_lpips"): + # if name != "vgg_lpips": + # raise NotImplementedError + # model = cls() + # ckpt = get_ckpt_path(name) + # model.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + # return model + + def forward(self, input, target): + in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target)) + outs0, outs1 = self.net(in0_input), self.net(in1_input) + feats0, feats1, diffs = {}, {}, {} + lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4] + for kk in range(len(self.chns)): + feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk]) + diffs[kk] = (feats0[kk] - feats1[kk]) ** 2 + + res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))] + val = res[0] + for l in range(1, len(self.chns)): + val += res[l] + return val + + +class ScalingLayer(nn.Module): + def __init__(self): + super(ScalingLayer, self).__init__() + self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None]) + self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None]) + + def forward(self, inp): + return (inp - self.shift) / self.scale + + +class NetLinLayer(nn.Module): + """ A single linear layer which does a 1x1 conv """ + def __init__(self, chn_in, chn_out=1, use_dropout=False): + super(NetLinLayer, self).__init__() + layers = [nn.Dropout(), ] if (use_dropout) else [] + layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ] + self.model = nn.Sequential(*layers) + + +class vgg16(torch.nn.Module): + def __init__(self, requires_grad=False, pretrained=True): + super(vgg16, self).__init__() + vgg_pretrained_features = models.vgg16(pretrained=pretrained).features + self.slice1 = torch.nn.Sequential() + self.slice2 = torch.nn.Sequential() + self.slice3 = torch.nn.Sequential() + self.slice4 = torch.nn.Sequential() + self.slice5 = torch.nn.Sequential() + self.N_slices = 5 + for x in range(4): + self.slice1.add_module(str(x), vgg_pretrained_features[x]) + for x in range(4, 9): + self.slice2.add_module(str(x), vgg_pretrained_features[x]) + for x in range(9, 16): + self.slice3.add_module(str(x), vgg_pretrained_features[x]) + for x in range(16, 23): + self.slice4.add_module(str(x), vgg_pretrained_features[x]) + for x in range(23, 30): + self.slice5.add_module(str(x), vgg_pretrained_features[x]) + if not requires_grad: + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X): + h = self.slice1(X) + h_relu1_2 = h + h = self.slice2(h) + h_relu2_2 = h + h = self.slice3(h) + h_relu3_3 = h + h = self.slice4(h) + h_relu4_3 = h + h = self.slice5(h) + h_relu5_3 = h + vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3']) + out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3) + return out + + +def normalize_tensor(x,eps=1e-10): + norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True)) + return x/(norm_factor+eps) + + +def spatial_average(x, keepdim=True): + return x.mean([2,3],keepdim=keepdim) \ No newline at end of file diff --git a/data_generation/vqgan/losses/vgperceptual.py b/data_generation/vqgan/losses/vgperceptual.py new file mode 100644 index 0000000000000000000000000000000000000000..e7c94f16188a74f853f7186f620a41caf3126bfa --- /dev/null +++ b/data_generation/vqgan/losses/vgperceptual.py @@ -0,0 +1,136 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .lpips import LPIPS +from .discriminator import NLayerDiscriminator, weights_init + + +class DummyLoss(nn.Module): + def __init__(self): + super().__init__() + + +def adopt_weight(weight, global_step, threshold=0, value=0.): + if global_step < threshold: + weight = value + return weight + + +def hinge_d_loss(logits_real, logits_fake): + loss_real = torch.mean(F.relu(1. - logits_real)) + loss_fake = torch.mean(F.relu(1. + logits_fake)) + d_loss = 0.5 * (loss_real + loss_fake) + return d_loss + + +def vanilla_d_loss(logits_real, logits_fake): + d_loss = 0.5 * ( + torch.mean(torch.nn.functional.softplus(-logits_real)) + + torch.mean(torch.nn.functional.softplus(logits_fake))) + return d_loss + + +class VQLPIPSWithDiscriminator(nn.Module): + def __init__(self, disc_start, codebook_weight=1.0, pixelloss_weight=1.0, + disc_num_layers=3, disc_in_channels=3, disc_factor=1.0, disc_weight=1.0, + perceptual_weight=1.0, use_actnorm=False, disc_conditional=False, + disc_ndf=64, disc_loss="hinge"): + super().__init__() + assert disc_loss in ["hinge", "vanilla"] + self.codebook_weight = codebook_weight + self.pixel_weight = pixelloss_weight + self.perceptual_loss = LPIPS().eval() + self.perceptual_weight = perceptual_weight + + self.discriminator = NLayerDiscriminator(input_nc=disc_in_channels, + n_layers=disc_num_layers, + use_actnorm=use_actnorm, + ndf=disc_ndf + ).apply(weights_init) + self.discriminator_iter_start = disc_start + if disc_loss == "hinge": + self.disc_loss = hinge_d_loss + elif disc_loss == "vanilla": + self.disc_loss = vanilla_d_loss + else: + raise ValueError(f"Unknown GAN loss '{disc_loss}'.") + print(f"VQLPIPSWithDiscriminator running with {disc_loss} loss.") + self.disc_factor = disc_factor + self.discriminator_weight = disc_weight + self.disc_conditional = disc_conditional + + def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None): + if last_layer is not None: + nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] + g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0] + else: + nll_grads = torch.autograd.grad(nll_loss, self.last_layer[0], retain_graph=True)[0] + g_grads = torch.autograd.grad(g_loss, self.last_layer[0], retain_graph=True)[0] + + d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4) + d_weight = torch.clamp(d_weight, 0.0, 1e4).detach() + d_weight = d_weight * self.discriminator_weight + return d_weight + + def forward(self, codebook_loss, inputs, reconstructions, optimizer_idx, + global_step, last_layer=None, cond=None, split="train"): + rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous()) + if self.perceptual_weight > 0: + p_loss = self.perceptual_loss(inputs.contiguous(), reconstructions.contiguous()) + nll_loss = rec_loss + self.perceptual_weight * p_loss + else: + p_loss = torch.tensor([0.0]) + + # nll_loss = rec_loss + #nll_loss = torch.sum(nll_loss) / nll_loss.shape[0] + nll_loss = torch.mean(nll_loss) + + # now the GAN part + if optimizer_idx == 0: + # generator update + if cond is None: + assert not self.disc_conditional + logits_fake = self.discriminator(reconstructions.contiguous()) + else: + assert self.disc_conditional + logits_fake = self.discriminator(torch.cat((reconstructions.contiguous(), cond), dim=1)) + g_loss = -torch.mean(logits_fake) + + try: + d_weight = self.calculate_adaptive_weight(nll_loss, g_loss, last_layer=last_layer) + except RuntimeError: + assert not self.training + d_weight = torch.tensor(0.0) + + disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start) + loss = nll_loss + d_weight * disc_factor * g_loss + self.codebook_weight * codebook_loss.mean() + + log = {"{}/total_loss".format(split): loss.clone().detach().mean(), + "{}/quant_loss".format(split): codebook_loss.detach().mean(), + "{}/nll_loss".format(split): nll_loss.detach().mean(), + "{}/rec_loss".format(split): rec_loss.detach().mean(), + "{}/p_loss".format(split): p_loss.detach().mean(), + "{}/d_weight".format(split): d_weight.detach(), + "{}/disc_factor".format(split): torch.tensor(disc_factor), + "{}/g_loss".format(split): g_loss.detach().mean(), + } + return loss, log + + if optimizer_idx == 1: + # second pass for discriminator update + if cond is None: + logits_real = self.discriminator(inputs.contiguous().detach()) + logits_fake = self.discriminator(reconstructions.contiguous().detach()) + else: + logits_real = self.discriminator(torch.cat((inputs.contiguous().detach(), cond), dim=1)) + logits_fake = self.discriminator(torch.cat((reconstructions.contiguous().detach(), cond), dim=1)) + + disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start) + d_loss = disc_factor * self.disc_loss(logits_real, logits_fake) + + log = {"{}/disc_loss".format(split): d_loss.clone().detach().mean(), + "{}/logits_real".format(split): logits_real.detach().mean(), + "{}/logits_fake".format(split): logits_fake.detach().mean() + } + return d_loss, log \ No newline at end of file diff --git a/data_generation/vqgan/muse.jpg b/data_generation/vqgan/muse.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d82972c586ddf00a6d591684d8fe5527b73d8b36 Binary files /dev/null and b/data_generation/vqgan/muse.jpg differ diff --git a/data_generation/vqgan/muse/__init__.py b/data_generation/vqgan/muse/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ceaa3939a2455e319af2aec89f15b586e12eda4d --- /dev/null +++ b/data_generation/vqgan/muse/__init__.py @@ -0,0 +1,18 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.0.1" + +from .modeling_taming_vqgan import VQGANModel diff --git a/data_generation/vqgan/muse/__pycache__/__init__.cpython-37.pyc b/data_generation/vqgan/muse/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3fc505cf9dca534bbea93d7e5dac7c2edb968175 Binary files /dev/null and b/data_generation/vqgan/muse/__pycache__/__init__.cpython-37.pyc differ diff --git a/data_generation/vqgan/muse/__pycache__/__init__.cpython-39.pyc b/data_generation/vqgan/muse/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5818a935539bd9c03f5b1f21e167a9140415dba3 Binary files /dev/null and b/data_generation/vqgan/muse/__pycache__/__init__.cpython-39.pyc differ diff --git a/data_generation/vqgan/muse/__pycache__/logging.cpython-37.pyc b/data_generation/vqgan/muse/__pycache__/logging.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f5e81a73495205fd142452a5c3cec09b68fd229 Binary files /dev/null and b/data_generation/vqgan/muse/__pycache__/logging.cpython-37.pyc differ diff --git a/data_generation/vqgan/muse/__pycache__/logging.cpython-39.pyc b/data_generation/vqgan/muse/__pycache__/logging.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..09c613a6d4abaefa5275918e10bdc09335ce6b4d Binary files /dev/null and b/data_generation/vqgan/muse/__pycache__/logging.cpython-39.pyc differ diff --git a/data_generation/vqgan/muse/__pycache__/modeling_taming_vqgan.cpython-37.pyc b/data_generation/vqgan/muse/__pycache__/modeling_taming_vqgan.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b483217fa18bcdcda78704679981c2f5590dcbf Binary files /dev/null and b/data_generation/vqgan/muse/__pycache__/modeling_taming_vqgan.cpython-37.pyc differ diff --git a/data_generation/vqgan/muse/__pycache__/modeling_taming_vqgan.cpython-39.pyc b/data_generation/vqgan/muse/__pycache__/modeling_taming_vqgan.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..887597340065b793c8f7609b50960404c36af776 Binary files /dev/null and b/data_generation/vqgan/muse/__pycache__/modeling_taming_vqgan.cpython-39.pyc differ diff --git a/data_generation/vqgan/muse/__pycache__/modeling_utils.cpython-37.pyc b/data_generation/vqgan/muse/__pycache__/modeling_utils.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..99ca3ddbf8f0edcbb3a225c54f6420d82d2aff22 Binary files /dev/null and b/data_generation/vqgan/muse/__pycache__/modeling_utils.cpython-37.pyc differ diff --git a/data_generation/vqgan/muse/__pycache__/modeling_utils.cpython-39.pyc b/data_generation/vqgan/muse/__pycache__/modeling_utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5541a3e0042a1fa7e959502dc58726323bb611f8 Binary files /dev/null and b/data_generation/vqgan/muse/__pycache__/modeling_utils.cpython-39.pyc differ diff --git a/data_generation/vqgan/muse/logging.py b/data_generation/vqgan/muse/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..65814a82380e47e54434c4be97026141772f7298 --- /dev/null +++ b/data_generation/vqgan/muse/logging.py @@ -0,0 +1,338 @@ +# coding=utf-8 +# Copyright 2023 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Logging utilities.""" + +import logging +import os +import sys +import threading +from logging import CRITICAL # NOQA +from logging import DEBUG # NOQA +from logging import ERROR # NOQA +from logging import FATAL # NOQA +from logging import INFO # NOQA +from logging import NOTSET # NOQA +from logging import WARN # NOQA +from logging import WARNING # NOQA +from typing import Optional + +from tqdm import auto as tqdm_lib + +_lock = threading.Lock() +_default_handler: Optional[logging.Handler] = None + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + +_tqdm_active = True + + +def _get_default_logging_level(): + """ + If muse_VERBOSITY env var is set to one of the valid choices return that as the new default level. If it is + not - fall back to `_default_log_level` + """ + env_level_str = os.getenv("muse_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option muse_VERBOSITY={env_level_str}, has to be one of: { ', '.join(log_levels.keys()) }" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + global _default_handler + + with _lock: + if _default_handler: + # This library has already configured the library root logger. + return + _default_handler = logging.StreamHandler() # Set sys.stderr as stream. + _default_handler.flush = sys.stderr.flush + + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.addHandler(_default_handler) + library_root_logger.setLevel(_get_default_logging_level()) + library_root_logger.propagate = False + + +def _reset_library_root_logger() -> None: + global _default_handler + + with _lock: + if not _default_handler: + return + + library_root_logger = _get_library_root_logger() + library_root_logger.removeHandler(_default_handler) + library_root_logger.setLevel(logging.NOTSET) + _default_handler = None + + +def get_log_levels_dict(): + return log_levels + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """ + Return a logger with the specified name. + + This function is not supposed to be directly accessed unless you are writing a custom muse module. + """ + + if name is None: + name = _get_library_name() + + _configure_library_root_logger() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """ + Return the current level for the 🤗 muse' root logger as an int. + + Returns: + `int`: The logging level. + + + + 🤗 muse has following logging levels: + + - 50: `muse.logging.CRITICAL` or `muse.logging.FATAL` + - 40: `muse.logging.ERROR` + - 30: `muse.logging.WARNING` or `muse.logging.WARN` + - 20: `muse.logging.INFO` + - 10: `muse.logging.DEBUG` + + """ + + _configure_library_root_logger() + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """ + Set the verbosity level for the 🤗 muse' root logger. + + Args: + verbosity (`int`): + Logging level, e.g., one of: + + - `muse.logging.CRITICAL` or `muse.logging.FATAL` + - `muse.logging.ERROR` + - `muse.logging.WARNING` or `muse.logging.WARN` + - `muse.logging.INFO` + - `muse.logging.DEBUG` + """ + + _configure_library_root_logger() + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """Set the verbosity to the `INFO` level.""" + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """Set the verbosity to the `WARNING` level.""" + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """Set the verbosity to the `DEBUG` level.""" + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """Set the verbosity to the `ERROR` level.""" + return set_verbosity(ERROR) + + +def disable_default_handler() -> None: + """Disable the default handler of the HuggingFace muse' root logger.""" + + _configure_library_root_logger() + + assert _default_handler is not None + _get_library_root_logger().removeHandler(_default_handler) + + +def enable_default_handler() -> None: + """Enable the default handler of the HuggingFace muse' root logger.""" + + _configure_library_root_logger() + + assert _default_handler is not None + _get_library_root_logger().addHandler(_default_handler) + + +def add_handler(handler: logging.Handler) -> None: + """adds a handler to the HuggingFace muse' root logger.""" + + _configure_library_root_logger() + + assert handler is not None + _get_library_root_logger().addHandler(handler) + + +def remove_handler(handler: logging.Handler) -> None: + """removes given handler from the HuggingFace muse' root logger.""" + + _configure_library_root_logger() + + assert handler is not None and handler not in _get_library_root_logger().handlers + _get_library_root_logger().removeHandler(handler) + + +def disable_propagation() -> None: + """ + Disable propagation of the library log outputs. Note that log propagation is disabled by default. + """ + + _configure_library_root_logger() + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """ + Enable propagation of the library log outputs. Please disable the HuggingFace muse' default handler to prevent + double logging if the root logger has been configured. + """ + + _configure_library_root_logger() + _get_library_root_logger().propagate = True + + +def enable_explicit_format() -> None: + """ + Enable explicit formatting for every HuggingFace muse' logger. The explicit formatter is as follows: + ``` + [LEVELNAME|FILENAME|LINE NUMBER] TIME >> MESSAGE + ``` + All handlers currently bound to the root logger are affected by this method. + """ + handlers = _get_library_root_logger().handlers + + for handler in handlers: + formatter = logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s") + handler.setFormatter(formatter) + + +def reset_format() -> None: + """ + Resets the formatting for HuggingFace muse' loggers. + + All handlers currently bound to the root logger are affected by this method. + """ + handlers = _get_library_root_logger().handlers + + for handler in handlers: + handler.setFormatter(None) + + +def warning_advice(self, *args, **kwargs): + """ + This method is identical to `logger.warning()`, but if env var muse_NO_ADVISORY_WARNINGS=1 is set, this + warning will not be printed + """ + no_advisory_warnings = os.getenv("muse_NO_ADVISORY_WARNINGS", False) + if no_advisory_warnings: + return + self.warning(*args, **kwargs) + + +logging.Logger.warning_advice = warning_advice + + +class EmptyTqdm: + """Dummy tqdm which doesn't do anything.""" + + def __init__(self, *args, **kwargs): # pylint: disable=unused-argument + self._iterator = args[0] if args else None + + def __iter__(self): + return iter(self._iterator) + + def __getattr__(self, _): + """Return empty function.""" + + def empty_fn(*args, **kwargs): # pylint: disable=unused-argument + return + + return empty_fn + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + return + + +class _tqdm_cls: + def __call__(self, *args, **kwargs): + if _tqdm_active: + return tqdm_lib.tqdm(*args, **kwargs) + else: + return EmptyTqdm(*args, **kwargs) + + def set_lock(self, *args, **kwargs): + self._lock = None + if _tqdm_active: + return tqdm_lib.tqdm.set_lock(*args, **kwargs) + + def get_lock(self): + if _tqdm_active: + return tqdm_lib.tqdm.get_lock() + + +tqdm = _tqdm_cls() + + +def is_progress_bar_enabled() -> bool: + """Return a boolean indicating whether tqdm progress bars are enabled.""" + global _tqdm_active + return bool(_tqdm_active) + + +def enable_progress_bar(): + """Enable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = True + + +def disable_progress_bar(): + """Disable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = False diff --git a/data_generation/vqgan/muse/modeling_taming_vqgan.py b/data_generation/vqgan/muse/modeling_taming_vqgan.py new file mode 100644 index 0000000000000000000000000000000000000000..af46bf01431e2664002ef0513477f6061b5b79c7 --- /dev/null +++ b/data_generation/vqgan/muse/modeling_taming_vqgan.py @@ -0,0 +1,586 @@ +# coding=utf-8 +# Copyright 2023 The Taming Transformers Authors and The HuggingFace Inc. team. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from functools import partial +from typing import Tuple + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from .modeling_utils import ConfigMixin, ModelMixin, register_to_config + + +class Upsample(nn.Module): + def __init__(self, in_channels: int, with_conv: bool): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = nn.Conv2d( + in_channels, + in_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, hidden_states): + hidden_states = torch.nn.functional.interpolate(hidden_states, scale_factor=2.0, mode="nearest") + if self.with_conv: + hidden_states = self.conv(hidden_states) + return hidden_states + + +class Downsample(nn.Module): + def __init__(self, in_channels: int, with_conv: bool): + super().__init__() + + self.with_conv = with_conv + if self.with_conv: + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, hidden_states): + if self.with_conv: + pad = (0, 1, 0, 1) # pad height and width dim + hidden_states = torch.nn.functional.pad(hidden_states, pad, mode="constant", value=0) + hidden_states = self.conv(hidden_states) + else: + hidden_states = torch.nn.functional.avg_pool2d(hidden_states, kernel_size=2, stride=2) + return hidden_states + + +class ResnetBlock(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int = None, + use_conv_shortcut: bool = False, + dropout_prob: float = 0.0, + ): + super().__init__() + + self.in_channels = in_channels + self.out_channels = out_channels + self.out_channels_ = self.in_channels if self.out_channels is None else self.out_channels + self.use_conv_shortcut = use_conv_shortcut + + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=3, + stride=1, + padding=1, + ) + + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=self.out_channels_, eps=1e-6, affine=True) + self.dropout = nn.Dropout(dropout_prob) + self.conv2 = nn.Conv2d( + self.out_channels_, + self.out_channels_, + kernel_size=3, + stride=(1, 1), + padding=1, + ) + + if self.in_channels != self.out_channels_: + if use_conv_shortcut: + self.conv_shortcut = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=3, + stride=1, + padding=1, + ) + else: + self.nin_shortcut = nn.Conv2d( + self.in_channels, + self.out_channels_, + kernel_size=1, + stride=1, + padding=0, + ) + + def forward(self, hidden_states): + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.in_channels != self.out_channels_: + if self.use_conv_shortcut: + residual = self.conv_shortcut(residual) + else: + residual = self.nin_shortcut(residual) + + return hidden_states + residual + + +class AttnBlock(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + + self.in_channels = in_channels + conv = partial(nn.Conv2d, self.in_channels, self.in_channels, kernel_size=1, stride=1, padding=0) + + self.norm = nn.GroupNorm(num_groups=32, num_channels=self.in_channels, eps=1e-6, affine=True) + self.q, self.k, self.v = conv(), conv(), conv() + self.proj_out = conv() + + def forward(self, hidden_states): + residual = hidden_states + hidden_states = self.norm(hidden_states) + + query = self.q(hidden_states) + key = self.k(hidden_states) + value = self.v(hidden_states) + + # compute attentions + batch, channels, height, width = query.shape + query = query.reshape((batch, channels, height * width)) + query = query.permute(0, 2, 1) # (b, hw, c) + key = key.reshape((batch, channels, height * width)) + + attn_weights = torch.bmm(query, key) # b,hw,hw + attn_weights = attn_weights * (int(channels) ** -0.5) + attn_weights = nn.functional.softmax(attn_weights, dim=2) + + # attend to values + value = value.reshape((batch, channels, height * width)) + attn_weights = attn_weights.permute(0, 2, 1) + hidden_states = torch.bmm(value, attn_weights) + hidden_states = hidden_states.reshape((batch, channels, height, width)) + + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states + residual + return hidden_states + + +class UpsamplingBlock(nn.Module): + def __init__(self, config, curr_res: int, block_idx: int): + super().__init__() + + self.config = config + self.block_idx = block_idx + self.curr_res = curr_res + + if self.block_idx == self.config.num_resolutions - 1: + block_in = self.config.hidden_channels * self.config.channel_mult[-1] + else: + block_in = self.config.hidden_channels * self.config.channel_mult[self.block_idx + 1] + + block_out = self.config.hidden_channels * self.config.channel_mult[self.block_idx] + + res_blocks = [] + attn_blocks = [] + for _ in range(self.config.num_res_blocks + 1): + res_blocks.append(ResnetBlock(block_in, block_out, dropout_prob=self.config.dropout)) + block_in = block_out + if self.curr_res in self.config.attn_resolutions: + attn_blocks.append(AttnBlock(block_in)) + + self.block = nn.ModuleList(res_blocks) + self.attn = nn.ModuleList(attn_blocks) + + self.upsample = None + if self.block_idx != 0: + self.upsample = Upsample(block_in, self.config.resample_with_conv) + + def forward(self, hidden_states): + for i, res_block in enumerate(self.block): + hidden_states = res_block(hidden_states) + if len(self.attn) > 1: + hidden_states = self.attn[i](hidden_states) + + if self.upsample is not None: + hidden_states = self.upsample(hidden_states) + + return hidden_states + + +class DownsamplingBlock(nn.Module): + def __init__(self, config, curr_res: int, block_idx: int): + super().__init__() + + self.config = config + self.curr_res = curr_res + self.block_idx = block_idx + + in_channel_mult = (1,) + tuple(self.config.channel_mult) + block_in = self.config.hidden_channels * in_channel_mult[self.block_idx] + block_out = self.config.hidden_channels * self.config.channel_mult[self.block_idx] + + res_blocks = nn.ModuleList() + attn_blocks = nn.ModuleList() + for _ in range(self.config.num_res_blocks): + res_blocks.append(ResnetBlock(block_in, block_out, dropout_prob=self.config.dropout)) + block_in = block_out + if self.curr_res in self.config.attn_resolutions: + attn_blocks.append(AttnBlock(block_in)) + + self.block = res_blocks + self.attn = attn_blocks + + self.downsample = None + if self.block_idx != self.config.num_resolutions - 1: + self.downsample = Downsample(block_in, self.config.resample_with_conv) + + def forward(self, hidden_states): + for i, res_block in enumerate(self.block): + hidden_states = res_block(hidden_states) + if len(self.attn) > 1: + hidden_states = self.attn[i](hidden_states) + + if self.downsample is not None: + hidden_states = self.downsample(hidden_states) + + return hidden_states + + +class MidBlock(nn.Module): + def __init__(self, config, in_channels: int, no_attn: False, dropout: float): + super().__init__() + + self.config = config + self.in_channels = in_channels + self.no_attn = no_attn + self.dropout = dropout + + self.block_1 = ResnetBlock( + self.in_channels, + self.in_channels, + dropout_prob=self.dropout, + ) + if not no_attn: + self.attn_1 = AttnBlock(self.in_channels) + self.block_2 = ResnetBlock( + self.in_channels, + self.in_channels, + dropout_prob=self.dropout, + ) + + def forward(self, hidden_states): + hidden_states = self.block_1(hidden_states) + if not self.no_attn: + hidden_states = self.attn_1(hidden_states) + hidden_states = self.block_2(hidden_states) + return hidden_states + + +class Encoder(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + + # downsampling + self.conv_in = nn.Conv2d( + self.config.num_channels, + self.config.hidden_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + curr_res = self.config.resolution + downsample_blocks = [] + for i_level in range(self.config.num_resolutions): + downsample_blocks.append(DownsamplingBlock(self.config, curr_res, block_idx=i_level)) + + if i_level != self.config.num_resolutions - 1: + curr_res = curr_res // 2 + self.down = nn.ModuleList(downsample_blocks) + + # middle + mid_channels = self.config.hidden_channels * self.config.channel_mult[-1] + self.mid = MidBlock(config, mid_channels, self.config.no_attn_mid_block, self.config.dropout) + + # end + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=mid_channels, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d( + mid_channels, + self.config.z_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, pixel_values): + # downsampling + hidden_states = self.conv_in(pixel_values) + for block in self.down: + hidden_states = block(hidden_states) + + # middle + hidden_states = self.mid(hidden_states) + + # end + hidden_states = self.norm_out(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class Decoder(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + + # compute in_channel_mult, block_in and curr_res at lowest res + block_in = self.config.hidden_channels * self.config.channel_mult[self.config.num_resolutions - 1] + curr_res = self.config.resolution // 2 ** (self.config.num_resolutions - 1) + self.z_shape = (1, self.config.z_channels, curr_res, curr_res) + + # z to block_in + self.conv_in = nn.Conv2d( + self.config.z_channels, + block_in, + kernel_size=3, + stride=1, + padding=1, + ) + + # middle + self.mid = MidBlock(config, block_in, self.config.no_attn_mid_block, self.config.dropout) + + # upsampling + upsample_blocks = [] + for i_level in reversed(range(self.config.num_resolutions)): + upsample_blocks.append(UpsamplingBlock(self.config, curr_res, block_idx=i_level)) + if i_level != 0: + curr_res = curr_res * 2 + self.up = nn.ModuleList(list(reversed(upsample_blocks))) # reverse to get consistent order + + # end + block_out = self.config.hidden_channels * self.config.channel_mult[0] + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_out, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d( + block_out, + self.config.num_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, hidden_states): + # z to block_in + hidden_states = self.conv_in(hidden_states) + + # middle + hidden_states = self.mid(hidden_states) + + # upsampling + for block in reversed(self.up): + hidden_states = block(hidden_states) + + # end + hidden_states = self.norm_out(hidden_states) + hidden_states = F.silu(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class VectorQuantizer(nn.Module): + """ + see https://github.com/MishaLaskin/vqvae/blob/d761a999e2267766400dc646d82d3ac3657771d4/models/quantizer.py + Discretization bottleneck part of the VQ-VAE. + """ + + def __init__(self, num_embeddings, embedding_dim, commitment_cost): + r""" + Args: + num_embeddings: number of vectors in the quantized space. + embedding_dim: dimensionality of the tensors in the quantized space. + Inputs to the modules must be in this format as well. + commitment_cost: scalar which controls the weighting of the loss terms + (see equation 4 in the paper https://arxiv.org/abs/1711.00937 - this variable is Beta). + """ + super().__init__() + + self.num_embeddings = num_embeddings + self.embedding_dim = embedding_dim + self.commitment_cost = commitment_cost + + self.embedding = nn.Embedding(num_embeddings, embedding_dim) + self.embedding.weight.data.uniform_(-1.0 / num_embeddings, 1.0 / num_embeddings) + + def forward(self, hidden_states, return_loss=False): + """ + Inputs the output of the encoder network z and maps it to a discrete one-hot vector that is the index of the + closest embedding vector e_j z (continuous) -> z_q (discrete) z.shape = (batch, channel, height, width) + quantization pipeline: + 1. get encoder input (B,C,H,W) + 2. flatten input to (B*H*W,C) + """ + # reshape z -> (batch, height, width, channel) and flatten + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() + + distances = self.compute_distances(hidden_states) + min_encoding_indices = torch.argmin(distances, axis=1).unsqueeze(1) + min_encodings = torch.zeros(min_encoding_indices.shape[0], self.num_embeddings).to(hidden_states) + min_encodings.scatter_(1, min_encoding_indices, 1) + + # get quantized latent vectors + z_q = torch.matmul(min_encodings, self.embedding.weight).view(hidden_states.shape) + + # reshape to (batch, num_tokens) + min_encoding_indices = min_encoding_indices.reshape(hidden_states.shape[0], -1) + + # compute loss for embedding + loss = None + if return_loss: + loss = torch.mean((z_q.detach() - hidden_states) ** 2) + self.commitment_cost * torch.mean( + (z_q - hidden_states.detach()) ** 2 + ) + # preserve gradients + z_q = hidden_states + (z_q - hidden_states).detach() + + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + return z_q, min_encoding_indices, loss + + def compute_distances(self, hidden_states): + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z + hidden_states_flattended = hidden_states.reshape((-1, self.embedding_dim)) + emb_weights = self.embedding.weight.t() + + inputs_norm_sq = hidden_states_flattended.pow(2.0).sum(dim=1, keepdim=True) + codebook_t_norm_sq = emb_weights.pow(2.0).sum(dim=0, keepdim=True) + distances = torch.addmm( + inputs_norm_sq + codebook_t_norm_sq, + hidden_states_flattended, + emb_weights, + alpha=-2.0, + ) + return distances + + def get_codebook_entry(self, indices): + # indices are expected to be of shape (batch, num_tokens) + # get quantized latent vectors + batch, num_tokens = indices.shape + z_q = self.embedding(indices) + z_q = z_q.reshape(batch, int(math.sqrt(num_tokens)), int(math.sqrt(num_tokens)), -1).permute(0, 3, 1, 2) + return z_q + + # adapted from https://github.com/kakaobrain/rq-vae-transformer/blob/main/rqvae/models/rqvae/quantizations.py#L372 + def get_soft_code(self, hidden_states, temp=1.0, stochastic=False): + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() # (batch, height, width, channel) + distances = self.compute_distances(hidden_states) # (batch * height * width, num_embeddings) + + soft_code = F.softmax(-distances / temp, dim=-1) # (batch * height * width, num_embeddings) + if stochastic: + code = torch.multinomial(soft_code, 1) # (batch * height * width, 1) + else: + code = distances.argmin(dim=-1) # (batch * height * width) + + code = code.reshape(hidden_states.shape[0], -1) # (batch, height * width) + batch, num_tokens = code.shape + soft_code = soft_code.reshape(batch, num_tokens, -1) # (batch, height * width, num_embeddings) + return soft_code, code + + def get_code(self, hidden_states): + # reshape z -> (batch, height, width, channel) + hidden_states = hidden_states.permute(0, 2, 3, 1).contiguous() + distances = self.compute_distances(hidden_states) + indices = torch.argmin(distances, axis=1).unsqueeze(1) + indices = indices.reshape(hidden_states.shape[0], -1) + return indices + + +class VQGANModel(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + resolution, + num_channels, + hidden_channels, + channel_mult, + num_res_blocks, + attn_resolutions, + no_attn_mid_block, + z_channels, + num_embeddings, + quantized_embed_dim, + dropout, + resample_with_conv, + commitment_cost, + ): + super().__init__() + + self.config.num_resolutions = len(channel_mult) + self.config.reduction_factor = 2 ** (self.config.num_resolutions - 1) + self.config.latent_size = resolution // self.config.reduction_factor + + self.encoder = Encoder(self.config) + self.decoder = Decoder(self.config) + self.quantize = VectorQuantizer( + self.config.num_embeddings, self.config.quantized_embed_dim, self.config.commitment_cost + ) + self.quant_conv = nn.Conv2d( + self.config.z_channels, + self.config.quantized_embed_dim, + kernel_size=1, + ) + self.post_quant_conv = nn.Conv2d( + self.config.quantized_embed_dim, + self.config.z_channels, + kernel_size=1, + ) + + def encode(self, pixel_values, return_loss=False): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quantized_states, codebook_indices, codebook_loss = self.quantize(hidden_states, return_loss) + output = (quantized_states, codebook_indices) + if return_loss: + output = output + (codebook_loss,) + return output + + def decode(self, quantized_states): + hidden_states = self.post_quant_conv(quantized_states) + reconstructed_pixel_values = self.decoder(hidden_states) + return reconstructed_pixel_values + + def decode_code(self, codebook_indices): + quantized_states = self.quantize.get_codebook_entry(codebook_indices) + reconstructed_pixel_values = self.decode(quantized_states) + return reconstructed_pixel_values + + def get_code(self, pixel_values): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + codebook_indices = self.quantize.get_code(hidden_states) + return codebook_indices + + def forward(self, pixel_values, return_loss=False): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quantized_states, codebook_indices, codebook_loss = self.quantize(hidden_states, return_loss) + reconstructed_pixel_values = self.decode(quantized_states) + # outputs = (reconstructed_pixel_values, quantized_states, codebook_indices) + # if return_loss: + # outputs = outputs + (codebook_loss,) + # return outputs + + return reconstructed_pixel_values, codebook_loss \ No newline at end of file diff --git a/data_generation/vqgan/muse/modeling_utils.py b/data_generation/vqgan/muse/modeling_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3194d3daacf07642933dc88ed1fa602a586cb4a9 --- /dev/null +++ b/data_generation/vqgan/muse/modeling_utils.py @@ -0,0 +1,1170 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import inspect +import json +import os +from collections import OrderedDict +from functools import partial +from pathlib import PosixPath +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import accelerate +import numpy as np +import torch +from accelerate.utils import set_module_tensor_to_device +from huggingface_hub import hf_hub_download +from huggingface_hub.utils import ( + EntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from requests import HTTPError +from torch import Tensor, device + +from . import __version__, logging + +logger = logging.get_logger(__name__) + + +hf_cache_home = os.path.expanduser( + os.getenv("HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface")) +) +default_cache_path = os.path.join(hf_cache_home, "muse") + + +CONFIG_NAME = "config.json" +WEIGHTS_NAME = "pytorch_model.bin" +SAFETENSORS_WEIGHTS_NAME = "pytorch_model.safetensors" +HUGGINGFACE_CO_RESOLVE_ENDPOINT = "https://huggingface.co" +MUSE_CACHE = default_cache_path +MUSE_DYNAMIC_MODULE_NAME = "myse_modules" +HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(hf_cache_home, "modules")) + + +_LOW_CPU_MEM_USAGE_DEFAULT = True + + +def get_parameter_device(parameter: torch.nn.Module): + try: + return next(parameter.parameters()).device + except StopIteration: + # For torch.nn.DataParallel compatibility in Pytorch 1.5 + + def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] + return tuples + + gen = parameter._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].device + + +def get_parameter_dtype(parameter: torch.nn.Module): + try: + return next(parameter.parameters()).dtype + except StopIteration: + # For torch.nn.DataParallel compatibility in Pytorch 1.5 + + def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] + return tuples + + gen = parameter._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].dtype + + +def load_state_dict(checkpoint_file: Union[str, os.PathLike]): + """ + Reads a checkpoint file, returning properly formatted errors if they arise. + """ + try: + if os.path.basename(checkpoint_file) == WEIGHTS_NAME: + return torch.load(checkpoint_file, map_location="cpu") + except Exception as e: + try: + with open(checkpoint_file) as f: + if f.read().startswith("version"): + raise OSError( + "You seem to have cloned a repository without having git-lfs installed. Please install " + "git-lfs and run `git lfs install` followed by `git lfs pull` in the folder " + "you cloned." + ) + else: + raise ValueError( + f"Unable to locate the file {checkpoint_file} which is necessary to load this pretrained " + "model. Make sure you have saved the model properly." + ) from e + except (UnicodeDecodeError, ValueError): + raise OSError( + f"Unable to load weights from checkpoint file for '{checkpoint_file}' " + f"at '{checkpoint_file}'. " + "If you tried to load a Pytorch model from a TF 2.0 checkpoint, please set from_tf=True." + ) + + +def _load_state_dict_into_model(model_to_load, state_dict): + # Convert old format to new format if needed from a Pytorch state_dict + # copy state_dict so _load_from_state_dict can modify it + state_dict = state_dict.copy() + error_msgs = [] + + # Pytorch's `_load_from_state_dict` does not copy parameters in a module's descendants + # so we need to apply the function recursively. + def load(module: torch.nn.Module, prefix=""): + args = (state_dict, prefix, {}, True, [], [], error_msgs) + module._load_from_state_dict(*args) + + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + ".") + + load(model_to_load) + + return error_msgs + + +def _get_model_file( + pretrained_model_name_or_path, + *, + weights_name, + subfolder, + cache_dir, + force_download, + proxies, + resume_download, + local_files_only, + use_auth_token, + user_agent, + revision, +): + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + if os.path.isfile(pretrained_model_name_or_path): + return pretrained_model_name_or_path + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, weights_name)): + # Load from a Pytorch checkpoint + model_file = os.path.join(pretrained_model_name_or_path, weights_name) + return model_file + elif subfolder is not None and os.path.isfile( + os.path.join(pretrained_model_name_or_path, subfolder, weights_name) + ): + model_file = os.path.join(pretrained_model_name_or_path, subfolder, weights_name) + return model_file + else: + raise EnvironmentError( + f"Error no file named {weights_name} found in directory {pretrained_model_name_or_path}." + ) + else: + try: + # Load from URL or cache if already cached + model_file = hf_hub_download( + pretrained_model_name_or_path, + filename=weights_name, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision, + ) + return model_file + + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier " + "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a " + "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli " + "login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for " + "this model name. Check the model page at " + f"'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {weights_name}." + ) + except HTTPError as err: + raise EnvironmentError( + f"There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a file named {weights_name} or" + " \nCheckout your internet connection or see how to run the library in" + " offline mode at 'https://huggingface.co/docs/diffusers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a file named {weights_name}" + ) + + +class ModelMixin(torch.nn.Module): + r""" + Base class for all models. + + [`ModelMixin`] takes care of storing the configuration of the models and handles methods for loading, downloading + and saving models. + + - **config_name** ([`str`]) -- A filename under which the model should be stored when calling + [`~models.ModelMixin.save_pretrained`]. + """ + config_name = CONFIG_NAME + _automatically_saved_args = ["_version", "_class_name", "_name_or_path"] + _supports_gradient_checkpointing = False + + def __init__(self): + super().__init__() + + @property + def is_gradient_checkpointing(self) -> bool: + """ + Whether gradient checkpointing is activated for this model or not. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules()) + + def enable_gradient_checkpointing(self): + """ + Activates gradient checkpointing for the current model. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + if not self._supports_gradient_checkpointing: + raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.") + self.apply(partial(self._set_gradient_checkpointing, value=True)) + + def disable_gradient_checkpointing(self): + """ + Deactivates gradient checkpointing for the current model. + + Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint + activations". + """ + if self._supports_gradient_checkpointing: + self.apply(partial(self._set_gradient_checkpointing, value=False)) + + def set_use_memory_efficient_attention_xformers( + self, valid: bool, attention_op: Optional[Callable] = None + ) -> None: + # Recursively walk through all the children. + # Any children which exposes the set_use_memory_efficient_attention_xformers method + # gets the message + def fn_recursive_set_mem_eff(module: torch.nn.Module): + if hasattr(module, "set_use_memory_efficient_attention_xformers"): + module.set_use_memory_efficient_attention_xformers(valid, attention_op) + + for child in module.children(): + fn_recursive_set_mem_eff(child) + + for module in self.children(): + if isinstance(module, torch.nn.Module): + fn_recursive_set_mem_eff(module) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + r""" + Enable memory efficient attention as implemented in xformers. + + When this option is enabled, you should observe lower GPU memory usage and a potential speed up at inference + time. Speed up at training time is not guaranteed. + + Warning: When Memory Efficient Attention and Sliced attention are both enabled, the Memory Efficient Attention + is used. + + Parameters: + attention_op (`Callable`, *optional*): + Override the default `None` operator for use as `op` argument to the + [`memory_efficient_attention()`](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.memory_efficient_attention) + function of xFormers. + + Examples: + + ```py + >>> import torch + >>> from diffusers import UNet2DConditionModel + >>> from xformers.ops import MemoryEfficientAttentionFlashAttentionOp + + >>> model = UNet2DConditionModel.from_pretrained( + ... "stabilityai/stable-diffusion-2-1", subfolder="unet", torch_dtype=torch.float16 + ... ) + >>> model = model.to("cuda") + >>> model.enable_xformers_memory_efficient_attention(attention_op=MemoryEfficientAttentionFlashAttentionOp) + ``` + """ + self.set_use_memory_efficient_attention_xformers(True, attention_op) + + def disable_xformers_memory_efficient_attention(self): + r""" + Disable memory efficient attention as implemented in xformers. + """ + self.set_use_memory_efficient_attention_xformers(False) + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + is_main_process: bool = True, + save_function: Callable = None, + state_dict: Optional[Dict[str, torch.Tensor]] = None, + ): + """ + Save a model and its configuration file to a directory, so that it can be re-loaded using the + `[`~models.ModelMixin.from_pretrained`]` class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to which to save. Will be created if it doesn't exist. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful on distributed training like TPUs when one + need to replace `torch.save` by another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + state_dict (`Dict[str, torch.Tensor]`, *optional*): + The state dictionary to save. If `None`, the model's state dictionary will be saved. + """ + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + if save_function is None: + save_function = torch.save + + os.makedirs(save_directory, exist_ok=True) + + model_to_save = self + + # Attach architecture to the config + # Save the config + if is_main_process: + model_to_save.save_config(save_directory) + + # Save the model + if state_dict is None: + state_dict = model_to_save.state_dict() + + weights_name = WEIGHTS_NAME + + # Save the model + save_function(state_dict, os.path.join(save_directory, weights_name)) + + logger.info(f"Model weights saved in {os.path.join(save_directory, weights_name)}") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): + r""" + Instantiate a pretrained pytorch model from a pre-trained model configuration. + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train + the model, you should first set it back in training mode with `model.train()`. + + The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come + pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning + task. + + The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those + weights are discarded. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + Valid model ids should have an organization name, like `google/ddpm-celebahq-256`. + - A path to a *directory* containing model weights saved using [`~ModelMixin.save_config`], e.g., + `./my_model_directory/`. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the dtype + will be automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received files. Will attempt to resume the download if such a + file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `diffusers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + from_flax (`bool`, *optional*, defaults to `False`): + Load the model weights from a Flax checkpoint save file. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo (either remote in + huggingface.co or downloaded locally), you can specify the folder name here. + + mirror (`str`, *optional*): + Mirror source to accelerate downloads in China. If you are from China and have an accessibility + problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety. + Please refer to the mirror site for more information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each + parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the + same device. + + To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading by not initializing the weights and only loading the pre-trained weights. This + also tries to not use more than 1x model size in CPU memory (including peak memory) while loading the + model. This is only supported when torch version >= 1.9.0. If you are using an older version of torch, + setting this argument to `True` will raise an error. + + + + It is required to be logged in (`huggingface-cli login`) when you want to use private or [gated + models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + + + Activate the special ["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use + this method in a firewalled environment. + + + + """ + cache_dir = kwargs.pop("cache_dir", MUSE_CACHE) + ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + output_loading_info = kwargs.pop("output_loading_info", False) + local_files_only = kwargs.pop("local_files_only", False) # TODO + use_auth_token = kwargs.pop("use_auth_token", None) + revision = kwargs.pop("revision", None) + torch_dtype = kwargs.pop("torch_dtype", None) + subfolder = kwargs.pop("subfolder", None) + device_map = kwargs.pop("device_map", None) + low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT) + + if low_cpu_mem_usage is False and device_map is not None: + raise ValueError( + f"You cannot set `low_cpu_mem_usage` to `False` while using device_map={device_map} for loading and" + " dispatching. Please make sure to set `low_cpu_mem_usage=True`." + ) + + user_agent = { + "diffusers": __version__, + "file_type": "model", + "framework": "pytorch", + } + + # Load config if we don't provide a configuration + config_path = pretrained_model_name_or_path + + # This variable will flag if we're loading a sharded checkpoint. In this case the archive file is just the + # Load model + + model_file = None + + if model_file is None: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=WEIGHTS_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + + if low_cpu_mem_usage: + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + config, unused_kwargs = cls.load_config( + config_path, + cache_dir=cache_dir, + return_unused_kwargs=True, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + device_map=device_map, + **kwargs, + ) + model = cls.from_config(config, **unused_kwargs) + + # if device_map is None, load the state dict and move the params from meta device to the cpu + if device_map is None: + param_device = "cpu" + state_dict = load_state_dict(model_file) + # move the params from meta device to cpu + missing_keys = set(model.state_dict().keys()) - set(state_dict.keys()) + if len(missing_keys) > 0: + raise ValueError( + f"Cannot load {cls} from {pretrained_model_name_or_path} because the following keys are" + f" missing: \n {', '.join(missing_keys)}. \n Please make sure to pass" + " `low_cpu_mem_usage=False` and `device_map=None` if you want to randomely initialize" + " those weights or else make sure your checkpoint file is correct." + ) + + for param_name, param in state_dict.items(): + accepts_dtype = "dtype" in set(inspect.signature(set_module_tensor_to_device).parameters.keys()) + if accepts_dtype: + set_module_tensor_to_device(model, param_name, param_device, value=param, dtype=torch_dtype) + else: + set_module_tensor_to_device(model, param_name, param_device, value=param) + else: # else let accelerate handle loading and dispatching. + # Load weights and dispatch according to the device_map + # by deafult the device_map is None and the weights are loaded on the CPU + accelerate.load_checkpoint_and_dispatch(model, model_file, device_map, dtype=torch_dtype) + + loading_info = { + "missing_keys": [], + "unexpected_keys": [], + "mismatched_keys": [], + "error_msgs": [], + } + else: + config, unused_kwargs = cls.load_config( + config_path, + cache_dir=cache_dir, + return_unused_kwargs=True, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + device_map=device_map, + **kwargs, + ) + model = cls.from_config(config, **unused_kwargs) + + state_dict = load_state_dict(model_file) + + model, missing_keys, unexpected_keys, mismatched_keys, error_msgs = cls._load_pretrained_model( + model, + state_dict, + model_file, + pretrained_model_name_or_path, + ignore_mismatched_sizes=ignore_mismatched_sizes, + ) + + loading_info = { + "missing_keys": missing_keys, + "unexpected_keys": unexpected_keys, + "mismatched_keys": mismatched_keys, + "error_msgs": error_msgs, + } + + if torch_dtype is not None and not isinstance(torch_dtype, torch.dtype): + raise ValueError( + f"{torch_dtype} needs to be of type `torch.dtype`, e.g. `torch.float16`, but is {type(torch_dtype)}." + ) + elif torch_dtype is not None: + model = model.to(torch_dtype) + + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + + # Set model in evaluation mode to deactivate DropOut modules by default + model.eval() + if output_loading_info: + return model, loading_info + + return model + + @classmethod + def _load_pretrained_model( + cls, + model, + state_dict, + resolved_archive_file, + pretrained_model_name_or_path, + ignore_mismatched_sizes=False, + ): + # Retrieve missing & unexpected_keys + model_state_dict = model.state_dict() + loaded_keys = [k for k in state_dict.keys()] + + expected_keys = list(model_state_dict.keys()) + + original_loaded_keys = loaded_keys + + missing_keys = list(set(expected_keys) - set(loaded_keys)) + unexpected_keys = list(set(loaded_keys) - set(expected_keys)) + + # Make sure we are able to load base models as well as derived models (with heads) + model_to_load = model + + def _find_mismatched_keys( + state_dict, + model_state_dict, + loaded_keys, + ignore_mismatched_sizes, + ): + mismatched_keys = [] + if ignore_mismatched_sizes: + for checkpoint_key in loaded_keys: + model_key = checkpoint_key + + if ( + model_key in model_state_dict + and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape + ): + mismatched_keys.append( + (checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape) + ) + del state_dict[checkpoint_key] + return mismatched_keys + + if state_dict is not None: + # Whole checkpoint + mismatched_keys = _find_mismatched_keys( + state_dict, + model_state_dict, + original_loaded_keys, + ignore_mismatched_sizes, + ) + error_msgs = _load_state_dict_into_model(model_to_load, state_dict) + + if len(error_msgs) > 0: + error_msg = "\n\t".join(error_msgs) + if "size mismatch" in error_msg: + error_msg += ( + "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method." + ) + raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}") + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when" + f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are" + f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task" + " or with another architecture (e.g. initializing a BertForSequenceClassification model from a" + " BertForPreTraining model).\n- This IS NOT expected if you are initializing" + f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly" + " identical (initializing a BertForSequenceClassification model from a" + " BertForSequenceClassification model)." + ) + else: + logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n") + if len(missing_keys) > 0: + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably" + " TRAIN this model on a down-stream task to be able to use it for predictions and inference." + ) + elif len(mismatched_keys) == 0: + logger.info( + f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at" + f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the" + f" checkpoint was trained on, you can already use {model.__class__.__name__} for predictions" + " without further training." + ) + if len(mismatched_keys) > 0: + mismatched_warning = "\n".join( + [ + f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated" + for key, shape1, shape2 in mismatched_keys + ] + ) + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not" + f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be" + " able to use it for predictions and inference." + ) + + return model, missing_keys, unexpected_keys, mismatched_keys, error_msgs + + @property + def device(self) -> device: + """ + `torch.device`: The device on which the module is (assuming that all the module parameters are on the same + device). + """ + return get_parameter_device(self) + + @property + def dtype(self) -> torch.dtype: + """ + `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). + """ + return get_parameter_dtype(self) + + def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int: + """ + Get number of (optionally, trainable or non-embeddings) parameters in the module. + + Args: + only_trainable (`bool`, *optional*, defaults to `False`): + Whether or not to return only the number of trainable parameters + + exclude_embeddings (`bool`, *optional*, defaults to `False`): + Whether or not to return only the number of non-embeddings parameters + + Returns: + `int`: The number of parameters. + """ + + if exclude_embeddings: + embedding_param_names = [ + f"{name}.weight" + for name, module_type in self.named_modules() + if isinstance(module_type, torch.nn.Embedding) + ] + non_embedding_parameters = [ + parameter for name, parameter in self.named_parameters() if name not in embedding_param_names + ] + return sum(p.numel() for p in non_embedding_parameters if p.requires_grad or not only_trainable) + else: + return sum(p.numel() for p in self.parameters() if p.requires_grad or not only_trainable) + + +""" ConfigMixin base class and utilities.""" + + +class FrozenDict(OrderedDict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + for key, value in self.items(): + setattr(self, key, value) + + self.__frozen = True + + def __delitem__(self, *args, **kwargs): + raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.") + + def setdefault(self, *args, **kwargs): + raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance.") + + def pop(self, *args, **kwargs): + raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance.") + + def update(self, *args, **kwargs): + raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance.") + + def __setattr__(self, name, value): + if hasattr(self, "__frozen") and self.__frozen: + raise Exception(f"You cannot use ``__setattr__`` on a {self.__class__.__name__} instance.") + super().__setattr__(name, value) + + def __setitem__(self, name, value): + if hasattr(self, "__frozen") and self.__frozen: + raise Exception(f"You cannot use ``__setattr__`` on a {self.__class__.__name__} instance.") + super().__setitem__(name, value) + + +class ConfigMixin: + r""" + Base class for all configuration classes. Stores all configuration parameters under `self.config` Also handles all + methods for loading/downloading/saving classes inheriting from [`ConfigMixin`] with + - [`~ConfigMixin.from_config`] + - [`~ConfigMixin.save_config`] + + Class attributes: + - **config_name** (`str`) -- A filename under which the config should stored when calling + [`~ConfigMixin.save_config`] (should be overridden by parent class). + - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be + overridden by subclass). + - **has_compatibles** (`bool`) -- Whether the class has compatible classes (should be overridden by subclass). + - **_deprecated_kwargs** (`List[str]`) -- Keyword arguments that are deprecated. Note that the init function + should only have a `kwargs` argument if at least one argument is deprecated (should be overridden by + subclass). + """ + config_name = None + ignore_for_config = [] + has_compatibles = False + + _deprecated_kwargs = [] + + def register_to_config(self, **kwargs): + if self.config_name is None: + raise NotImplementedError(f"Make sure that {self.__class__} has defined a class name `config_name`") + # Special case for `kwargs` used in deprecation warning added to schedulers + # TODO: remove this when we remove the deprecation warning, and the `kwargs` argument, + # or solve in a more general way. + kwargs.pop("kwargs", None) + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error(f"Can't set {key} with value {value} for {self}") + raise err + + if not hasattr(self, "_internal_dict"): + internal_dict = kwargs + else: + previous_dict = dict(self._internal_dict) + internal_dict = {**self._internal_dict, **kwargs} + logger.debug(f"Updating config from {previous_dict} to {internal_dict}") + + self._internal_dict = FrozenDict(internal_dict) + + def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + """ + Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the + [`~ConfigMixin.from_config`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the configuration JSON file will be saved (will be created if it does not exist). + """ + if os.path.isfile(save_directory): + raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file") + + os.makedirs(save_directory, exist_ok=True) + + # If we save using the predefined names, we can load using `from_config` + output_config_file = os.path.join(save_directory, self.config_name) + + self.to_json_file(output_config_file) + logger.info(f"Configuration saved in {output_config_file}") + + @classmethod + def from_config(cls, config: Union[FrozenDict, Dict[str, Any]] = None, **kwargs): + r""" + Instantiate a Python class from a config dictionary + + Parameters: + config (`Dict[str, Any]`): + A config dictionary from which the Python class will be instantiated. Make sure to only load + configuration files of compatible classes. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + Whether kwargs that are not consumed by the Python class should be returned or not. + + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to update the configuration object (after it being loaded) and initiate the Python class. + `**kwargs` will be directly passed to the underlying scheduler/model's `__init__` method and eventually + overwrite same named arguments of `config`. + + Examples: + + ```python + >>> from diffusers import DDPMScheduler, DDIMScheduler, PNDMScheduler + + >>> # Download scheduler from huggingface.co and cache. + >>> scheduler = DDPMScheduler.from_pretrained("google/ddpm-cifar10-32") + + >>> # Instantiate DDIM scheduler class with same config as DDPM + >>> scheduler = DDIMScheduler.from_config(scheduler.config) + + >>> # Instantiate PNDM scheduler class with same config as DDPM + >>> scheduler = PNDMScheduler.from_config(scheduler.config) + ``` + """ + # <===== TO BE REMOVED WITH DEPRECATION + # TODO(Patrick) - make sure to remove the following lines when config=="model_path" is deprecated + if "pretrained_model_name_or_path" in kwargs: + config = kwargs.pop("pretrained_model_name_or_path") + + if config is None: + raise ValueError("Please make sure to provide a config as the first positional argument.") + # ======> + + # Return model and optionally state and/or unused_kwargs + model = cls(**config) + return model + + @classmethod + def load_config( + cls, pretrained_model_name_or_path: Union[str, os.PathLike], return_unused_kwargs=False, **kwargs + ) -> Tuple[Dict[str, Any], Dict[str, Any]]: + r""" + Instantiate a Python class from a config dictionary + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* of a model repo on huggingface.co. Valid model ids should have an + organization name, like `google/ddpm-celebahq-256`. + - A path to a *directory* containing model weights saved using [`~ConfigMixin.save_config`], e.g., + `./my_model_directory/`. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received files. Will attempt to resume the download if such a + file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `transformers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + subfolder (`str`, *optional*, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo (either remote in + huggingface.co or downloaded locally), you can specify the folder name here. + + + + It is required to be logged in (`huggingface-cli login`) when you want to use private or [gated + models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + + + Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to + use this method in a firewalled environment. + + + """ + cache_dir = kwargs.pop("cache_dir", MUSE_CACHE) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + _ = kwargs.pop("mirror", None) + subfolder = kwargs.pop("subfolder", None) + + user_agent = {"file_type": "config"} + + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + + if cls.config_name is None: + raise ValueError( + "`self.config_name` is not defined. Note that one should not load a config from " + "`ConfigMixin`. Please make sure to define `config_name` in a class inheriting from `ConfigMixin`" + ) + + if os.path.isfile(pretrained_model_name_or_path): + config_file = pretrained_model_name_or_path + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, cls.config_name)): + # Load from a Pytorch checkpoint + config_file = os.path.join(pretrained_model_name_or_path, cls.config_name) + elif subfolder is not None and os.path.isfile( + os.path.join(pretrained_model_name_or_path, subfolder, cls.config_name) + ): + config_file = os.path.join(pretrained_model_name_or_path, subfolder, cls.config_name) + else: + raise EnvironmentError( + f"Error no file named {cls.config_name} found in directory {pretrained_model_name_or_path}." + ) + else: + try: + # Load from URL or cache if already cached + config_file = hf_hub_download( + pretrained_model_name_or_path, + filename=cls.config_name, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision, + ) + + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier" + " listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a" + " token having permission to this repo with `use_auth_token` or log in with `huggingface-cli" + " login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for" + " this model name. Check the model page at" + f" 'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {cls.config_name}." + ) + except HTTPError as err: + raise EnvironmentError( + "There was a specific connection error when trying to load" + f" {pretrained_model_name_or_path}:\n{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a {cls.config_name} file.\nCheckout your internet connection or see how to" + " run the library in offline mode at" + " 'https://huggingface.co/docs/diffusers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load config for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a {cls.config_name} file" + ) + + try: + # Load config dict + config_dict = cls._dict_from_json_file(config_file) + except (json.JSONDecodeError, UnicodeDecodeError): + raise EnvironmentError(f"It looks like the config file at '{config_file}' is not a valid JSON file.") + + if return_unused_kwargs: + return config_dict, kwargs + + return config_dict + + @staticmethod + def _get_init_keys(cls): + return set(dict(inspect.signature(cls.__init__).parameters).keys()) + + @classmethod + def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]): + with open(json_file, "r", encoding="utf-8") as reader: + text = reader.read() + return json.loads(text) + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + @property + def config(self) -> Dict[str, Any]: + """ + Returns the config of the class as a frozen dictionary + + Returns: + `Dict[str, Any]`: Config of the class. + """ + return self._internal_dict + + def to_json_string(self) -> str: + """ + Serializes this instance to a JSON string. + + Returns: + `str`: String containing all the attributes that make up this configuration instance in JSON format. + """ + config_dict = self._internal_dict if hasattr(self, "_internal_dict") else {} + config_dict["_class_name"] = self.__class__.__name__ + config_dict["_version"] = __version__ + + def to_json_saveable(value): + if isinstance(value, np.ndarray): + value = value.tolist() + elif isinstance(value, PosixPath): + value = str(value) + return value + + config_dict = {k: to_json_saveable(v) for k, v in config_dict.items()} + return json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save this instance to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file in which this configuration instance's parameters will be saved. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + writer.write(self.to_json_string()) + + +def register_to_config(init): + r""" + Decorator to apply on the init of classes inheriting from [`ConfigMixin`] so that all the arguments are + automatically sent to `self.register_for_config`. To ignore a specific argument accepted by the init but that + shouldn't be registered in the config, use the `ignore_for_config` class variable + + Warning: Once decorated, all private arguments (beginning with an underscore) are trashed and not sent to the init! + """ + + @functools.wraps(init) + def inner_init(self, *args, **kwargs): + # Ignore private kwargs in the init. + init_kwargs = {k: v for k, v in kwargs.items() if not k.startswith("_")} + config_init_kwargs = {k: v for k, v in kwargs.items() if k.startswith("_")} + if not isinstance(self, ConfigMixin): + raise RuntimeError( + f"`@register_for_config` was applied to {self.__class__.__name__} init method, but this class does " + "not inherit from `ConfigMixin`." + ) + + ignore = getattr(self, "ignore_for_config", []) + # Get positional arguments aligned with kwargs + new_kwargs = {} + signature = inspect.signature(init) + parameters = { + name: p.default for i, (name, p) in enumerate(signature.parameters.items()) if i > 0 and name not in ignore + } + for arg, name in zip(args, parameters.keys()): + new_kwargs[name] = arg + + # Then add all kwargs + new_kwargs.update( + { + k: init_kwargs.get(k, default) + for k, default in parameters.items() + if k not in ignore and k not in new_kwargs + } + ) + new_kwargs = {**config_init_kwargs, **new_kwargs} + getattr(self, "register_to_config")(**new_kwargs) + init(self, *args, **init_kwargs) + + return inner_init diff --git a/data_generation/vqgan/muse_vqgan_stat.json b/data_generation/vqgan/muse_vqgan_stat.json new file mode 100644 index 0000000000000000000000000000000000000000..a8f746180582e0c72dd7288ce2d296fa0bc55fdf --- /dev/null +++ b/data_generation/vqgan/muse_vqgan_stat.json @@ -0,0 +1 @@ +{"0": 690, "1": 1844, "2": 1962, "3": 352, "4": 38756, "5": 10645, "6": 65328, "7": 2349, "8": 10827, "9": 13644, "10": 12988, "11": 1, "12": 136777, "13": 136569, "14": 2393, "15": 46723, "16": 32766, "17": 15673, "18": 55700, "19": 23255, "20": 127193, "21": 24008, "22": 4515, "23": 44445, "24": 25996, "25": 294981, "26": 256, "27": 78756, "28": 5, "29": 27471, "30": 90333, "31": 12795, "32": 789, "33": 364, "34": 80875, "35": 202204, "36": 43025, "37": 221915, "38": 5636, "39": 6741, "40": 19807, "41": 493, "42": 195, "43": 1819, "44": 0, "45": 50620, "46": 23403, "47": 87, "48": 3002, "49": 542, "50": 6878, "51": 30698, "52": 0, "53": 38090, "54": 109125, "55": 2840, "56": 105840, "57": 5027, "58": 0, "59": 32034, "60": 4, "61": 0, "62": 3958, "63": 46714, "64": 0, "65": 14497, "66": 36527, "67": 34889, "68": 6990, "69": 15667, "70": 44636, "71": 11408, "72": 11215, "73": 10566, "74": 43400, "75": 264405, "76": 43400, "77": 83632, "78": 96832, "79": 48136, "80": 27746, "81": 35554, "82": 0, "83": 70926, "84": 53106, "85": 2844, "86": 50964, "87": 7947, "88": 360334, "89": 74123, "90": 66513, "91": 45977, "92": 6142, "93": 16502, "94": 18393, "95": 31842, "96": 188068, "97": 18259, "98": 94768, "99": 24333, "100": 33279, "101": 95543, "102": 2, "103": 23742, "104": 91937, "105": 11110, "106": 115075, "107": 33054, "108": 11081, "109": 41539, "110": 7759, "111": 42804, "112": 11643, "113": 120022, "114": 38942, "115": 16211, "116": 208194, "117": 33862, "118": 378, "119": 50979, "120": 6640, "121": 1826, "122": 7351, "123": 284934, "124": 100106, "125": 174055, "126": 125829, "127": 9211, "128": 287141, "129": 45808, "130": 4345, "131": 1047, "132": 24751, "133": 125, "134": 7039, "135": 83, "136": 48223, "137": 73231, "138": 34857, "139": 6980, "140": 179578, "141": 116558, "142": 208339, "143": 6334, "144": 442, "145": 14718, "146": 419, "147": 57297, "148": 0, "149": 158554, "150": 6, "151": 20511, "152": 61954, "153": 2815, "154": 908, "155": 1958, "156": 84781, "157": 43216, "158": 97754, "159": 7826, "160": 0, "161": 23743, "162": 37270, "163": 171279, "164": 1858, "165": 149387, "166": 52740, "167": 0, "168": 44802, "169": 21, "170": 7285, "171": 19038, "172": 151988, "173": 60023, "174": 162160, "175": 7247, "176": 79828, "177": 23713, "178": 2505, "179": 109218, "180": 23050, "181": 4531, "182": 146395, "183": 1087, "184": 105236, "185": 10600, "186": 1588, "187": 49159, "188": 4481, "189": 26971, "190": 459, "191": 206, "192": 31016, "193": 172242, "194": 0, "195": 47310, "196": 119, "197": 0, "198": 25, "199": 2118, "200": 13290, "201": 40793, "202": 113013, "203": 297646, "204": 0, "205": 39366, "206": 20236, "207": 2960, "208": 27851, "209": 10561, "210": 54730, "211": 324, "212": 21654, "213": 9534, "214": 2925, "215": 26273, "216": 47053, "217": 279369, "218": 86657, "219": 88724, "220": 43, "221": 10702, "222": 26655, "223": 0, "224": 516288, "225": 106943, "226": 22144, "227": 2953, "228": 52, "229": 736, "230": 62226, "231": 54059, "232": 66820, "233": 7535, "234": 61275, "235": 94989, "236": 4858, "237": 112102, "238": 12646, "239": 115013, "240": 62362, "241": 32227, "242": 78319, "243": 3, "244": 10308, "245": 1, "246": 825, "247": 114424, "248": 34101, "249": 2679, "250": 260989, "251": 57208, "252": 39133, "253": 58174, "254": 55191, "255": 336960, "256": 1841, "257": 167673, "258": 27228, "259": 67657, "260": 79915, "261": 44966, "262": 72811, "263": 59771, "264": 22717, "265": 81077, "266": 27155, "267": 25, "268": 184055, "269": 0, "270": 58595, "271": 3178, "272": 249560, "273": 1325, "274": 13950, "275": 0, "276": 50859, "277": 64112, "278": 256141, "279": 238, "280": 73, "281": 2657, "282": 88932, "283": 52610, "284": 9185, "285": 544, "286": 54854, "287": 58714, "288": 5921, "289": 898, "290": 56005, "291": 112538, "292": 3523, "293": 2117, "294": 8929, "295": 188871, "296": 111364, "297": 6964, "298": 0, "299": 39938, "300": 11966, "301": 346, "302": 32164, "303": 2775, "304": 105908, "305": 257, "306": 48480, "307": 5170, "308": 0, "309": 19649, "310": 6678, "311": 11151, "312": 7885, "313": 27, "314": 146900, "315": 4044, "316": 234, "317": 10128, "318": 32863, "319": 52844, "320": 1075, "321": 80709, "322": 13414, "323": 2897, "324": 0, "325": 388, "326": 99, "327": 84767, "328": 54132, "329": 78042, "330": 48819, "331": 44397, "332": 55570, "333": 50645, "334": 8776, "335": 31332, "336": 14863, "337": 52078, "338": 1, "339": 64630, "340": 86354, "341": 330, "342": 3330, "343": 28040, "344": 106951, "345": 322, "346": 99236, "347": 675, "348": 66450, "349": 52119, "350": 23312, "351": 4109, "352": 75980, "353": 195, "354": 3018, "355": 19442, "356": 73, "357": 2127, "358": 80775, "359": 7223, "360": 20267, "361": 43786, "362": 97529, "363": 0, "364": 1224, "365": 1, "366": 157553, "367": 1186, "368": 51370, "369": 16555, "370": 62419, "371": 50690, "372": 13313, "373": 3091, "374": 20311, "375": 648, "376": 0, "377": 0, "378": 69058, "379": 26337, "380": 44126, "381": 19428, "382": 4467, "383": 29293, "384": 121668, "385": 20161, "386": 21290, "387": 92378, "388": 157353, "389": 65177, "390": 11074, "391": 1881, "392": 48996, "393": 962, "394": 3204, "395": 2307, "396": 0, "397": 5387, "398": 23, "399": 53256, "400": 0, "401": 234, "402": 39864, "403": 41566, "404": 7408, "405": 162020, "406": 0, "407": 141676, "408": 15212, "409": 1, "410": 137906, "411": 57389, "412": 64339, "413": 179365, "414": 13471, "415": 32367, "416": 4353, "417": 158564, "418": 24368, "419": 116, "420": 36330, "421": 4059, "422": 172245, "423": 2183, "424": 8493, "425": 9741, "426": 9718, "427": 76708, "428": 116257, "429": 256173, "430": 19544, "431": 0, "432": 3153, "433": 28188, "434": 365, "435": 430258, "436": 48154, "437": 73851, "438": 344, "439": 24480, "440": 353843, "441": 6014, "442": 10993, "443": 17467, "444": 37972, "445": 426, "446": 135627, "447": 47, "448": 4015, "449": 11, "450": 71600, "451": 4839, "452": 21733, "453": 657, "454": 159516, "455": 93115, "456": 16082, "457": 107590, "458": 15833, "459": 144396, "460": 4519, "461": 68458, "462": 22752, "463": 13597, "464": 96735, "465": 60186, "466": 111913, "467": 7764, "468": 54461, "469": 3038, "470": 53206, "471": 2082, "472": 64346, "473": 8554, "474": 94883, "475": 159, "476": 2984, "477": 38539, "478": 3626, "479": 22890, "480": 199, "481": 14, "482": 43, "483": 29547, "484": 2543, "485": 1, "486": 413099, "487": 47899, "488": 66647, "489": 117223, "490": 395215, "491": 240220, "492": 0, "493": 94425, "494": 351, "495": 6154, "496": 3533, "497": 2966, "498": 13401, "499": 11880, "500": 53219, "501": 770, "502": 36864, "503": 101495, "504": 2027, "505": 53645, "506": 41128, "507": 32709, "508": 34771, "509": 50642, "510": 36587, "511": 55225, "512": 90443, "513": 12533, "514": 79803, "515": 78779, "516": 2372, "517": 43496, "518": 0, "519": 2, "520": 27306, "521": 158745, "522": 62354, "523": 1534, "524": 54474, "525": 7201, "526": 381, "527": 48693, "528": 65807, "529": 105419, "530": 16203, "531": 22689, "532": 41138, "533": 53911, "534": 255411, "535": 7, "536": 0, "537": 2903, "538": 119115, "539": 10821, "540": 248461, "541": 90, "542": 44655, "543": 20453, "544": 1876, "545": 16762, "546": 23439, "547": 4634, "548": 14605, "549": 20, "550": 59694, "551": 70582, "552": 4, "553": 20360, "554": 17777, "555": 45238, "556": 631, "557": 129881, "558": 96730, "559": 87604, "560": 1880, "561": 22827, "562": 1350, "563": 7, "564": 28, "565": 66782, "566": 47522, "567": 4992, "568": 40083, "569": 55868, "570": 446, "571": 2026, "572": 822, "573": 76755, "574": 68533, "575": 863, "576": 3153, "577": 155919, "578": 2628, "579": 19339, "580": 31076, "581": 44, "582": 99244, "583": 14284, "584": 4153, "585": 53556, "586": 55353, "587": 7429, "588": 42, "589": 11731, "590": 43965, "591": 62767, "592": 5353, "593": 244, "594": 99608, "595": 13433, "596": 5310, "597": 12454, "598": 7448, "599": 1489, "600": 98601, "601": 10912, "602": 53024, "603": 9458, "604": 23474, "605": 114372, "606": 48913, "607": 48691, "608": 25262, "609": 89514, "610": 35561, "611": 106190, "612": 12805, "613": 21080, "614": 9368, "615": 222356, "616": 30990, "617": 56330, "618": 36902, "619": 39848, "620": 51692, "621": 66815, "622": 4931, "623": 13124, "624": 54970, "625": 26205, "626": 7, "627": 588, "628": 43916, "629": 190799, "630": 108, "631": 82597, "632": 9675, "633": 0, "634": 68100, "635": 4329, "636": 143890, "637": 84286, "638": 1339, "639": 74240, "640": 232, "641": 6016, "642": 9922, "643": 31646, "644": 124252, "645": 31265, "646": 13601, "647": 69552, "648": 51403, "649": 28008, "650": 32774, "651": 4360, "652": 8982, "653": 61343, "654": 2869, "655": 41340, "656": 7523, "657": 1854, "658": 93080, "659": 132032, "660": 186826, "661": 17335, "662": 20989, "663": 109041, "664": 0, "665": 284337, "666": 39420, "667": 8504, "668": 44525, "669": 4576, "670": 351, "671": 63088, "672": 23840, "673": 277729, "674": 268232, "675": 13299, "676": 6885, "677": 10991, "678": 37050, "679": 29610, "680": 238, "681": 238, "682": 59892, "683": 1248, "684": 18616, "685": 3570, "686": 135416, "687": 162681, "688": 0, "689": 53143, "690": 32309, "691": 66590, "692": 62516, "693": 15918, "694": 8612, "695": 28141, "696": 62532, "697": 7937, "698": 218, "699": 5044, "700": 51135, "701": 8289, "702": 3533, "703": 29584, "704": 54192, "705": 88457, "706": 70590, "707": 15492, "708": 3069, "709": 59658, "710": 73684, "711": 50857, "712": 6718, "713": 54783, "714": 41655, "715": 9206, "716": 22520, "717": 21590, "718": 108698, "719": 82177, "720": 76338, "721": 8603, "722": 2002, "723": 54964, "724": 46432, "725": 2553, "726": 66231, "727": 166, "728": 125249, "729": 17228, "730": 13689, "731": 3, "732": 7333, "733": 92060, "734": 113903, "735": 6120, "736": 1866, "737": 3958, "738": 0, "739": 48, "740": 6258, "741": 1206, "742": 40066, "743": 1935, "744": 296930, "745": 55604, "746": 278, "747": 2167, "748": 244526, "749": 7121, "750": 4, "751": 87555, "752": 8285, "753": 95, "754": 218572, "755": 18094, "756": 1, "757": 1890, "758": 23339, "759": 62626, "760": 105, "761": 39315, "762": 17268, "763": 8872, "764": 17254, "765": 343, "766": 11941, "767": 75512, "768": 39, "769": 2916, "770": 20371, "771": 6761, "772": 20349, "773": 114817, "774": 116173, "775": 1826, "776": 7789, "777": 10527, "778": 86327, "779": 1918, "780": 5811, "781": 0, "782": 430836, "783": 1769, "784": 5240, "785": 591, "786": 30376, "787": 213836, "788": 16591, "789": 75, "790": 76869, "791": 118851, "792": 37865, "793": 586, "794": 103662, "795": 18993, "796": 61666, "797": 92096, "798": 14884, "799": 160580, "800": 48651, "801": 205, "802": 0, "803": 22654, "804": 314658, "805": 13145, "806": 34638, "807": 9064, "808": 13870, "809": 275, "810": 8715, "811": 18323, "812": 142203, "813": 74, "814": 3971, "815": 159, "816": 14857, "817": 51512, "818": 14943, "819": 58101, "820": 16003, "821": 7266, "822": 947, "823": 10009, "824": 10191, "825": 310821, "826": 9371, "827": 9510, "828": 189804, "829": 37679, "830": 5188, "831": 69952, "832": 978, "833": 1710, "834": 44159, "835": 681, "836": 144820, "837": 84216, "838": 9741, "839": 7474, "840": 48022, "841": 25013, "842": 2963, "843": 4779, "844": 104363, "845": 70543, "846": 23336, "847": 17888, "848": 5262, "849": 4402, "850": 38665, "851": 51581, "852": 17073, "853": 4734, "854": 3644, "855": 63095, "856": 39650, "857": 1004, "858": 56975, "859": 118804, "860": 4159, "861": 20090, "862": 16426, "863": 27776, "864": 29457, "865": 169862, "866": 10901, "867": 665750, "868": 36457, "869": 40189, "870": 108615, "871": 59355, "872": 26239, "873": 0, "874": 140, "875": 0, "876": 51, "877": 2942, "878": 12449, "879": 8896, "880": 3487, "881": 48026, "882": 24268, "883": 28307, "884": 67108, "885": 3141, "886": 585, "887": 195066, "888": 37239, "889": 1720, "890": 3620, "891": 23244, "892": 10552, "893": 5342, "894": 4, "895": 19956, "896": 3297, "897": 44008, "898": 59597, "899": 443, "900": 40772, "901": 30809, "902": 5723, "903": 13788, "904": 62146, "905": 68698, "906": 126033, "907": 1521, "908": 86871, "909": 65375, "910": 13038, "911": 73771, "912": 22652, "913": 137049, "914": 674, "915": 26712, "916": 90705, "917": 9586, "918": 113313, "919": 3365, "920": 87797, "921": 12845, "922": 3206, "923": 95005, "924": 17837, "925": 44089, "926": 56018, "927": 44704, "928": 167341, "929": 12446, "930": 32822, "931": 51247, "932": 15420, "933": 16259, "934": 5786, "935": 6003, "936": 0, "937": 6513, "938": 170514, "939": 86044, "940": 19744, "941": 733, "942": 22852, "943": 76297, "944": 4166, "945": 7646, "946": 6406, "947": 68, "948": 8686, "949": 62237, "950": 38716, "951": 9393, "952": 3605, "953": 7096, "954": 23056, "955": 98779, "956": 164896, "957": 52448, "958": 591, "959": 21590, "960": 408, "961": 10159, "962": 24979, "963": 13792, "964": 164575, "965": 60296, "966": 96765, "967": 154843, "968": 78820, "969": 6390, "970": 42989, "971": 237, "972": 13, "973": 15046, "974": 16425, "975": 4714, "976": 1859, "977": 7556, "978": 7615, "979": 16879, "980": 63093, "981": 73510, "982": 90135, "983": 71, "984": 85180, "985": 926, "986": 25063, "987": 42309, "988": 12, "989": 57617, "990": 8859, "991": 1395, "992": 140992, "993": 3706, "994": 1678, "995": 598, "996": 1392, "997": 14065, "998": 551, "999": 2260, "1000": 129495, "1001": 33584, "1002": 65696, "1003": 34519, "1004": 63028, "1005": 9101, "1006": 300035, "1007": 118, "1008": 113561, "1009": 87383, "1010": 77424, "1011": 76351, "1012": 8690, "1013": 21692, "1014": 9289, "1015": 45655, "1016": 57300, "1017": 89272, "1018": 0, "1019": 1033, "1020": 109777, "1021": 4686, "1022": 5826, "1023": 40934, "1024": 46135, "1025": 59108, "1026": 8062, "1027": 6474, "1028": 100790, "1029": 25589, "1030": 1496, "1031": 139551, "1032": 10929, "1033": 18247, "1034": 64021, "1035": 2107, "1036": 2399, "1037": 96259, "1038": 4056, "1039": 18543, "1040": 21831, "1041": 1613, "1042": 31407, "1043": 5504, "1044": 86024, "1045": 23117, "1046": 27373, "1047": 5, "1048": 1620, "1049": 245, "1050": 1098, "1051": 3714, "1052": 16, "1053": 75599, "1054": 128876, "1055": 72426, "1056": 147, "1057": 10189, "1058": 24091, "1059": 420, "1060": 120, "1061": 45237, "1062": 47311, "1063": 31424, "1064": 82237, "1065": 2595, "1066": 100567, "1067": 4122, "1068": 21360, "1069": 7507, "1070": 23980, "1071": 83772, "1072": 48353, "1073": 20390, "1074": 5366, "1075": 2487, "1076": 12594, "1077": 217, "1078": 3469, "1079": 49838, "1080": 21191, "1081": 51982, "1082": 77645, "1083": 54424, "1084": 8274, "1085": 3869, "1086": 1960, "1087": 1698, "1088": 10, "1089": 15675, "1090": 116483, "1091": 99980, "1092": 25103, "1093": 16904, "1094": 7939, "1095": 70015, "1096": 123506, "1097": 92825, "1098": 12243, "1099": 49499, "1100": 67856, "1101": 118261, "1102": 49748, "1103": 21314, "1104": 10641, "1105": 4985, "1106": 2507, "1107": 85634, "1108": 23185, "1109": 71874, "1110": 85069, "1111": 12653, "1112": 3190, "1113": 17918, "1114": 28601, "1115": 12531, "1116": 4938, "1117": 2104, "1118": 37089, "1119": 14424, "1120": 17020, "1121": 46651, "1122": 15636, "1123": 23263, "1124": 2, "1125": 339608, "1126": 131, "1127": 82132, "1128": 210821, "1129": 2476, "1130": 134950, "1131": 44497, "1132": 17937, "1133": 206, "1134": 20219, "1135": 22, "1136": 23794, "1137": 16732, "1138": 46241, "1139": 9405, "1140": 74092, "1141": 5662, "1142": 588, "1143": 2075, "1144": 81608, "1145": 2478, "1146": 25311, "1147": 20371, "1148": 4282, "1149": 48960, "1150": 370863, "1151": 10730, "1152": 10837, "1153": 299, "1154": 96283, "1155": 16768, "1156": 15289, "1157": 4735, "1158": 10660, "1159": 73276, "1160": 5835, "1161": 95760, "1162": 2235, "1163": 12461, "1164": 231680, "1165": 26462, "1166": 10620, "1167": 2513, "1168": 17352, "1169": 40141, "1170": 1230, "1171": 75153, "1172": 0, "1173": 6252, "1174": 74817, "1175": 19585, "1176": 32642, "1177": 108357, "1178": 7, "1179": 31034, "1180": 81100, "1181": 349, "1182": 7342, "1183": 1024, "1184": 35057, "1185": 42564, "1186": 6767, "1187": 23865, "1188": 660, "1189": 32113, "1190": 44255, "1191": 12, "1192": 79230, "1193": 135980, "1194": 14127, "1195": 32592, "1196": 11488, "1197": 60516, "1198": 2671, "1199": 17207, "1200": 1848, "1201": 316304, "1202": 27894, "1203": 46473, "1204": 47819, "1205": 169338, "1206": 88885, "1207": 6487, "1208": 4200, "1209": 8453, "1210": 181936, "1211": 9492, "1212": 78908, "1213": 14263, "1214": 23966, "1215": 16983, "1216": 532, "1217": 176108, "1218": 33040, "1219": 89998, "1220": 59155, "1221": 136980, "1222": 887, "1223": 9620, "1224": 66908, "1225": 69313, "1226": 76929, "1227": 805, "1228": 167070, "1229": 21232, "1230": 33963, "1231": 30728, "1232": 56136, "1233": 0, "1234": 27329, "1235": 51367, "1236": 51093, "1237": 113702, "1238": 191269, "1239": 24985, "1240": 156222, "1241": 80428, "1242": 33320, "1243": 39149, "1244": 1357, "1245": 4412, "1246": 1843, "1247": 92868, "1248": 38840, "1249": 30005, "1250": 128378, "1251": 1, "1252": 64369, "1253": 15, "1254": 1346, "1255": 84155, "1256": 76810, "1257": 58, "1258": 11836, "1259": 23930, "1260": 10124, "1261": 120246, "1262": 37418, "1263": 20931, "1264": 47541, "1265": 40018, "1266": 114, "1267": 31, "1268": 7319, "1269": 174950, "1270": 5077, "1271": 17391, "1272": 1138, "1273": 29321, "1274": 8756, "1275": 456, "1276": 76585, "1277": 42576, "1278": 125712, "1279": 72213, "1280": 10886, "1281": 17461, "1282": 4063, "1283": 14622, "1284": 48271, "1285": 19371, "1286": 29202, "1287": 109487, "1288": 51086, "1289": 244784, "1290": 106908, "1291": 121988, "1292": 19968, "1293": 974, "1294": 9335, "1295": 137593, "1296": 35280, "1297": 96426, "1298": 30624, "1299": 4701, "1300": 2553, "1301": 1072, "1302": 418, "1303": 13195, "1304": 24971, "1305": 32039, "1306": 111924, "1307": 73277, "1308": 86958, "1309": 28897, "1310": 20, "1311": 30069, "1312": 479, "1313": 521, "1314": 32404, "1315": 89439, "1316": 21104, "1317": 14526, "1318": 7286, "1319": 752, "1320": 99032, "1321": 26815, "1322": 103219, "1323": 259566, "1324": 135628, "1325": 20352, "1326": 4158, "1327": 481, "1328": 79222, "1329": 0, "1330": 43545, "1331": 28358, "1332": 2948, "1333": 106553, "1334": 112628, "1335": 59159, "1336": 141247, "1337": 48407, "1338": 168400, "1339": 4967, "1340": 56693, "1341": 7149, "1342": 95027, "1343": 27509, "1344": 9, "1345": 16178, "1346": 3087, "1347": 31045, "1348": 11500, "1349": 12683, "1350": 419558, "1351": 241, "1352": 0, "1353": 96349, "1354": 298079, "1355": 1351, "1356": 94836, "1357": 1655, "1358": 6404, "1359": 761, "1360": 117865, "1361": 90737, "1362": 19231, "1363": 42931, "1364": 23522, "1365": 10485, "1366": 10, "1367": 4562, "1368": 2874, "1369": 4754, "1370": 0, "1371": 125978, "1372": 3571, "1373": 125, "1374": 4082, "1375": 4357, "1376": 26546, "1377": 32235, "1378": 57795, "1379": 5059, "1380": 229, "1381": 0, "1382": 17071, "1383": 20281, "1384": 88404, "1385": 73625, "1386": 46725, "1387": 27, "1388": 36807, "1389": 9908, "1390": 15154, "1391": 0, "1392": 71001, "1393": 18300, "1394": 0, "1395": 7355, "1396": 145136, "1397": 9131, "1398": 6065, "1399": 27959, "1400": 1776, "1401": 21575, "1402": 1719, "1403": 75559, "1404": 84094, "1405": 2191, "1406": 0, "1407": 7621, "1408": 44, "1409": 10327, "1410": 11189, "1411": 3984, "1412": 14483, "1413": 270, "1414": 885, "1415": 17826, "1416": 24534, "1417": 36966, "1418": 1883, "1419": 0, "1420": 29876, "1421": 88144, "1422": 52186, "1423": 5792, "1424": 2330, "1425": 44430, "1426": 1885, "1427": 43083, "1428": 67147, "1429": 77403, "1430": 9134, "1431": 44733, "1432": 0, "1433": 44, "1434": 131204, "1435": 1007, "1436": 75917, "1437": 1924, "1438": 349, "1439": 22189, "1440": 4686, "1441": 38145, "1442": 29061, "1443": 2566, "1444": 12622, "1445": 0, "1446": 147345, "1447": 56698, "1448": 12863, "1449": 133446, "1450": 971, "1451": 133830, "1452": 152909, "1453": 308, "1454": 138, "1455": 27496, "1456": 2321, "1457": 344, "1458": 81240, "1459": 50361, "1460": 7083, "1461": 3755, "1462": 7946, "1463": 3688, "1464": 23088, "1465": 111479, "1466": 12787, "1467": 14917, "1468": 393, "1469": 31192, "1470": 897, "1471": 163002, "1472": 0, "1473": 132, "1474": 12307, "1475": 14948, "1476": 0, "1477": 203065, "1478": 91503, "1479": 779, "1480": 44603, "1481": 4028, "1482": 17198, "1483": 61704, "1484": 48383, "1485": 10286, "1486": 30902, "1487": 124974, "1488": 1384, "1489": 114068, "1490": 131442, "1491": 225068, "1492": 645, "1493": 8032, "1494": 18383, "1495": 9425, "1496": 173224, "1497": 39140, "1498": 24356, "1499": 52421, "1500": 353, "1501": 27549, "1502": 35340, "1503": 44907, "1504": 75860, "1505": 5378, "1506": 58909, "1507": 6726, "1508": 409503, "1509": 0, "1510": 58649, "1511": 52004, "1512": 13988, "1513": 24952, "1514": 91642, "1515": 116971, "1516": 8430, "1517": 84976, "1518": 5689, "1519": 34547, "1520": 78475, "1521": 25461, "1522": 756, "1523": 50401, "1524": 41009, "1525": 63039, "1526": 8948, "1527": 122801, "1528": 8063, "1529": 57690, "1530": 10394, "1531": 16989, "1532": 14364, "1533": 127912, "1534": 56503, "1535": 59774, "1536": 66608, "1537": 243538, "1538": 50580, "1539": 32753, "1540": 152995, "1541": 29136, "1542": 2508, "1543": 15678, "1544": 621, "1545": 55329, "1546": 221677, "1547": 0, "1548": 2251, "1549": 45853, "1550": 168490, "1551": 16794, "1552": 76316, "1553": 31270, "1554": 261, "1555": 45891, "1556": 8, "1557": 74725, "1558": 257, "1559": 39819, "1560": 77496, "1561": 5259, "1562": 116330, "1563": 135819, "1564": 83, "1565": 35150, "1566": 10312, "1567": 4243, "1568": 13427, "1569": 96476, "1570": 18147, "1571": 79067, "1572": 26258, "1573": 41190, "1574": 206247, "1575": 70, "1576": 29035, "1577": 4554, "1578": 63771, "1579": 31303, "1580": 7962, "1581": 8985, "1582": 20585, "1583": 8630, "1584": 55866, "1585": 37155, "1586": 381, "1587": 44131, "1588": 5418, "1589": 6588, "1590": 30972, "1591": 1720, "1592": 127666, "1593": 108344, "1594": 42961, "1595": 2453, "1596": 3185, "1597": 1096, "1598": 3047, "1599": 6200, "1600": 124221, "1601": 13828, "1602": 123437, "1603": 0, "1604": 806, "1605": 229, "1606": 294, "1607": 4566, "1608": 170907, "1609": 17552, "1610": 20155, "1611": 104340, "1612": 98933, "1613": 2765, "1614": 0, "1615": 14411, "1616": 5689, "1617": 47898, "1618": 249, "1619": 42759, "1620": 7919, "1621": 7307, "1622": 83600, "1623": 15070, "1624": 31024, "1625": 26541, "1626": 82326, "1627": 38011, "1628": 4772, "1629": 22115, "1630": 469, "1631": 1682, "1632": 7166, "1633": 40912, "1634": 50678, "1635": 26, "1636": 88976, "1637": 11609, "1638": 17394, "1639": 3843, "1640": 43144, "1641": 24741, "1642": 14816, "1643": 35778, "1644": 23955, "1645": 25794, "1646": 42479, "1647": 77657, "1648": 224, "1649": 395, "1650": 44529, "1651": 26723, "1652": 519, "1653": 19874, "1654": 103527, "1655": 2168, "1656": 30021, "1657": 215, "1658": 3131, "1659": 970, "1660": 734, "1661": 1, "1662": 72107, "1663": 5818, "1664": 182325, "1665": 2199, "1666": 47371, "1667": 4498, "1668": 2703, "1669": 230411, "1670": 601, "1671": 1025, "1672": 46472, "1673": 85522, "1674": 25921, "1675": 35464, "1676": 8883, "1677": 39740, "1678": 4918, "1679": 35665, "1680": 48, "1681": 563, "1682": 4382, "1683": 2210, "1684": 57727, "1685": 118767, "1686": 36469, "1687": 563, "1688": 47577, "1689": 66, "1690": 236, "1691": 67438, "1692": 281010, "1693": 25494, "1694": 42289, "1695": 367, "1696": 78505, "1697": 20995, "1698": 453, "1699": 36753, "1700": 38928, "1701": 5456, "1702": 40668, "1703": 0, "1704": 143029, "1705": 2994, "1706": 10373, "1707": 62545, "1708": 486266, "1709": 37178, "1710": 2459, "1711": 48632, "1712": 3424, "1713": 3783, "1714": 8757, "1715": 23191, "1716": 62059, "1717": 8249, "1718": 145397, "1719": 8325, "1720": 107, "1721": 5858, "1722": 78239, "1723": 2247, "1724": 39191, "1725": 219171, "1726": 78351, "1727": 219683, "1728": 43073, "1729": 11624, "1730": 47474, "1731": 4682, "1732": 4313, "1733": 5112, "1734": 115, "1735": 193, "1736": 10455, "1737": 68919, "1738": 15977, "1739": 11311, "1740": 46862, "1741": 1, "1742": 27724, "1743": 3125, "1744": 58210, "1745": 61454, "1746": 1494, "1747": 1585, "1748": 12326, "1749": 4313, "1750": 9237, "1751": 201, "1752": 56493, "1753": 79502, "1754": 117401, "1755": 0, "1756": 1508, "1757": 5, "1758": 112025, "1759": 46250, "1760": 50438, "1761": 50538, "1762": 2043, "1763": 2158, "1764": 120125, "1765": 10809, "1766": 26971, "1767": 26071, "1768": 31644, "1769": 180099, "1770": 9444, "1771": 220797, "1772": 24070, "1773": 20978, "1774": 12090, "1775": 11078, "1776": 49310, "1777": 16959, "1778": 53400, "1779": 755, "1780": 16293, "1781": 23013, "1782": 95, "1783": 53, "1784": 1, "1785": 6003, "1786": 153, "1787": 14679, "1788": 7068, "1789": 13189, "1790": 14758, "1791": 2176, "1792": 9036, "1793": 6259, "1794": 12664, "1795": 24, "1796": 8746, "1797": 717, "1798": 3018, "1799": 14600, "1800": 28128, "1801": 68053, "1802": 3131, "1803": 4622, "1804": 28819, "1805": 8063, "1806": 7492, "1807": 68112, "1808": 3975, "1809": 51831, "1810": 71848, "1811": 33438, "1812": 28980, "1813": 309, "1814": 219516, "1815": 993964, "1816": 1653, "1817": 4057, "1818": 47124, "1819": 60931, "1820": 360834, "1821": 16470, "1822": 521, "1823": 10444, "1824": 113116, "1825": 81752, "1826": 36928, "1827": 3521, "1828": 0, "1829": 409, "1830": 148420, "1831": 6, "1832": 0, "1833": 86752, "1834": 58201, "1835": 6188, "1836": 4364, "1837": 85809, "1838": 1775, "1839": 0, "1840": 62137, "1841": 67755, "1842": 28111, "1843": 3, "1844": 69272, "1845": 44839, "1846": 841, "1847": 16286, "1848": 254969, "1849": 258, "1850": 20754, "1851": 16064, "1852": 22872, "1853": 23523, "1854": 74264, "1855": 7100, "1856": 10725, "1857": 120172, "1858": 118477, "1859": 13190, "1860": 26, "1861": 32045, "1862": 3602, "1863": 21900, "1864": 222, "1865": 62253, "1866": 357267, "1867": 220, "1868": 19964, "1869": 5472, "1870": 276, "1871": 72397, "1872": 8430, "1873": 0, "1874": 44327, "1875": 43664, "1876": 38, "1877": 94, "1878": 3763, "1879": 20, "1880": 19605, "1881": 23379, "1882": 1850, "1883": 9266, "1884": 79094, "1885": 636, "1886": 9001, "1887": 0, "1888": 9, "1889": 10147, "1890": 45422, "1891": 488623, "1892": 142398, "1893": 64307, "1894": 56864, "1895": 4170, "1896": 11088, "1897": 19600, "1898": 17091, "1899": 80678, "1900": 22746, "1901": 292731, "1902": 52300, "1903": 146062, "1904": 22547, "1905": 16099, "1906": 1148, "1907": 28917, "1908": 26328, "1909": 62692, "1910": 0, "1911": 8364, "1912": 116891, "1913": 0, "1914": 187696, "1915": 10690, "1916": 46927, "1917": 0, "1918": 61057, "1919": 27051, "1920": 21761, "1921": 48604, "1922": 9299, "1923": 78729, "1924": 2991, "1925": 98796, "1926": 6103, "1927": 10990, "1928": 94386, "1929": 205, "1930": 29309, "1931": 1302, "1932": 10434, "1933": 4234, "1934": 15322, "1935": 255872, "1936": 43090, "1937": 41822, "1938": 0, "1939": 64884, "1940": 19466, "1941": 127290, "1942": 44924, "1943": 23178, "1944": 1310, "1945": 1722, "1946": 20675, "1947": 4327, "1948": 19179, "1949": 91443, "1950": 39203, "1951": 4475, "1952": 7996, "1953": 32532, "1954": 51490, "1955": 29410, "1956": 150106, "1957": 1966, "1958": 17877, "1959": 32293, "1960": 20340, "1961": 132482, "1962": 4665, "1963": 6327, "1964": 58451, "1965": 8646, "1966": 27690, "1967": 2126, "1968": 34008, "1969": 5323, "1970": 27447, "1971": 17295, "1972": 188, "1973": 204, "1974": 446, "1975": 2545, "1976": 56584, "1977": 17004, "1978": 5604, "1979": 251, "1980": 88583, "1981": 75886, "1982": 7813, "1983": 41618, "1984": 8010, "1985": 80763, "1986": 15846, "1987": 6580, "1988": 56613, "1989": 133087, "1990": 49712, "1991": 5, "1992": 74705, "1993": 39883, "1994": 456, "1995": 116894, "1996": 65219, "1997": 19628, "1998": 1083, "1999": 16018, "2000": 6894, "2001": 77159, "2002": 45522, "2003": 92529, "2004": 26980, "2005": 63003, "2006": 53728, "2007": 33255, "2008": 7377, "2009": 31603, "2010": 6162, "2011": 2946, "2012": 26632, "2013": 16533, "2014": 20279, "2015": 1749, "2016": 17214, "2017": 145955, "2018": 37110, "2019": 368887, "2020": 16670, "2021": 33579, "2022": 60016, "2023": 25814, "2024": 45692, "2025": 22491, "2026": 215356, "2027": 21565, "2028": 67374, "2029": 69, "2030": 514291, "2031": 7428, "2032": 9610, "2033": 5027, "2034": 6, "2035": 112, "2036": 46095, "2037": 50185, "2038": 22730, "2039": 52084, "2040": 191105, "2041": 17338, "2042": 747, "2043": 44450, "2044": 217129, "2045": 45884, "2046": 11106, "2047": 21946, "2048": 23674, "2049": 112769, "2050": 8195, "2051": 33083, "2052": 1827, "2053": 183614, "2054": 50866, "2055": 8939, "2056": 615, "2057": 40991, "2058": 6693, "2059": 3040, "2060": 1539, "2061": 0, "2062": 775, "2063": 1731, "2064": 839, "2065": 6158, "2066": 34629, "2067": 2336, "2068": 196872, "2069": 27757, "2070": 140092, "2071": 18025, "2072": 6260, "2073": 246909, "2074": 71615, "2075": 2747, "2076": 240559, "2077": 3349, "2078": 2, "2079": 10709, "2080": 46091, "2081": 3653, "2082": 1082, "2083": 171929, "2084": 15124, "2085": 31097, "2086": 7327, "2087": 0, "2088": 7928, "2089": 13923, "2090": 374672, "2091": 726, "2092": 717, "2093": 48936, "2094": 19862, "2095": 88534, "2096": 6475, "2097": 12547, "2098": 6326, "2099": 41082, "2100": 85300, "2101": 0, "2102": 16502, "2103": 27231, "2104": 33264, "2105": 243885, "2106": 1372, "2107": 1487, "2108": 153649, "2109": 2204, "2110": 116274, "2111": 422258, "2112": 5850, "2113": 17032, "2114": 1585, "2115": 37653, "2116": 26419, "2117": 76171, "2118": 9262, "2119": 113201, "2120": 19805, "2121": 233359, "2122": 113, "2123": 306, "2124": 7562, "2125": 202, "2126": 38057, "2127": 29319, "2128": 0, "2129": 44138, "2130": 42072, "2131": 23212, "2132": 26, "2133": 108584, "2134": 148691, "2135": 17691, "2136": 68468, "2137": 259117, "2138": 66756, "2139": 52557, "2140": 21127, "2141": 3804, "2142": 30336, "2143": 74917, "2144": 110333, "2145": 0, "2146": 0, "2147": 7408, "2148": 7, "2149": 4884, "2150": 368, "2151": 45583, "2152": 5905, "2153": 5, "2154": 87137, "2155": 29117, "2156": 31155, "2157": 10537, "2158": 3583, "2159": 47195, "2160": 3507, "2161": 57449, "2162": 2464, "2163": 5164, "2164": 10318, "2165": 6822, "2166": 61284, "2167": 48133, "2168": 22887, "2169": 83255, "2170": 63761, "2171": 0, "2172": 48580, "2173": 129256, "2174": 206192, "2175": 124121, "2176": 10219, "2177": 15135, "2178": 27676, "2179": 200, "2180": 27983, "2181": 63, "2182": 90935, "2183": 18783, "2184": 25318, "2185": 4458, "2186": 46595, "2187": 9442, "2188": 163725, "2189": 53188, "2190": 85378, "2191": 43720, "2192": 2813, "2193": 11313, "2194": 21625, "2195": 30845, "2196": 9349, "2197": 149, "2198": 4244, "2199": 10237, "2200": 13525, "2201": 100104, "2202": 75, "2203": 77467, "2204": 145842, "2205": 5417, "2206": 78034, "2207": 50303, "2208": 31092, "2209": 6953, "2210": 208401, "2211": 3390, "2212": 91867, "2213": 13053, "2214": 14846, "2215": 4885, "2216": 46, "2217": 1183, "2218": 123141, "2219": 27, "2220": 27706, "2221": 34415, "2222": 11525, "2223": 6858, "2224": 9987, "2225": 8394, "2226": 34522, "2227": 27262, "2228": 7218, "2229": 7998, "2230": 15543, "2231": 49464, "2232": 4968, "2233": 303, "2234": 17321, "2235": 111795, "2236": 29332, "2237": 33642, "2238": 15056, "2239": 28990, "2240": 17730, "2241": 6075, "2242": 431, "2243": 13436, "2244": 11793, "2245": 2494, "2246": 26797, "2247": 54916, "2248": 9010, "2249": 15362, "2250": 19657, "2251": 49822, "2252": 419228, "2253": 43857, "2254": 13040, "2255": 59966, "2256": 17323, "2257": 23762, "2258": 26045, "2259": 216061, "2260": 11850, "2261": 17093, "2262": 1802, "2263": 13, "2264": 6899, "2265": 3577, "2266": 62633, "2267": 22, "2268": 218513, "2269": 63659, "2270": 93891, "2271": 77656, "2272": 16027, "2273": 6307, "2274": 90991, "2275": 3530, "2276": 6781, "2277": 12873, "2278": 9609, "2279": 120067, "2280": 90845, "2281": 40640, "2282": 11830, "2283": 10828, "2284": 80919, "2285": 3099, "2286": 8, "2287": 120777, "2288": 105363, "2289": 29133, "2290": 10, "2291": 13648, "2292": 9551, "2293": 7981, "2294": 19385, "2295": 60853, "2296": 17831, "2297": 1, "2298": 50242, "2299": 1134, "2300": 65740, "2301": 149571, "2302": 114305, "2303": 10, "2304": 3272, "2305": 630, "2306": 7343, "2307": 17025, "2308": 86010, "2309": 3950, "2310": 97798, "2311": 31055, "2312": 19096, "2313": 141421, "2314": 4406, "2315": 107106, "2316": 63482, "2317": 1272, "2318": 559, "2319": 47200, "2320": 113, "2321": 38806, "2322": 196, "2323": 10105, "2324": 1861, "2325": 176098, "2326": 12597, "2327": 328836, "2328": 14249, "2329": 46, "2330": 63606, "2331": 72155, "2332": 124475, "2333": 1442, "2334": 76847, "2335": 1584, "2336": 2028, "2337": 5328, "2338": 16541, "2339": 109772, "2340": 3782, "2341": 209990, "2342": 35931, "2343": 66620, "2344": 61097, "2345": 4232, "2346": 56009, "2347": 44260, "2348": 0, "2349": 39750, "2350": 262, "2351": 57567, "2352": 98679, "2353": 17429, "2354": 4137, "2355": 43334, "2356": 28023, "2357": 17972, "2358": 32585, "2359": 36072, "2360": 92932, "2361": 66808, "2362": 36999, "2363": 37982, "2364": 351, "2365": 26620, "2366": 12164, "2367": 131602, "2368": 182265, "2369": 23278, "2370": 1048, "2371": 32817, "2372": 0, "2373": 7578, "2374": 18919, "2375": 49190, "2376": 229625, "2377": 103, "2378": 134, "2379": 1399, "2380": 47260, "2381": 3335, "2382": 29744, "2383": 46713, "2384": 55661, "2385": 85697, "2386": 18448, "2387": 14753, "2388": 57640, "2389": 26795, "2390": 19821, "2391": 8357, "2392": 662, "2393": 0, "2394": 77701, "2395": 29142, "2396": 859, "2397": 5635, "2398": 8403, "2399": 60472, "2400": 41, "2401": 82054, "2402": 14774, "2403": 61023, "2404": 69642, "2405": 43812, "2406": 55671, "2407": 621, "2408": 977, "2409": 5257, "2410": 353, "2411": 8626, "2412": 846, "2413": 25810, "2414": 7036, "2415": 1282, "2416": 29736, "2417": 0, "2418": 85044, "2419": 99365, "2420": 7126, "2421": 3713, "2422": 761, "2423": 3059, "2424": 22623, "2425": 41461, "2426": 84501, "2427": 977, "2428": 48313, "2429": 14345, "2430": 13103, "2431": 6249, "2432": 356, "2433": 60247, "2434": 25381, "2435": 19358, "2436": 31602, "2437": 95671, "2438": 15891, "2439": 106613, "2440": 38574, "2441": 12251, "2442": 3964, "2443": 19328, "2444": 9129, "2445": 16949, "2446": 21111, "2447": 0, "2448": 116285, "2449": 31168, "2450": 94147, "2451": 102188, "2452": 22469, "2453": 9393, "2454": 58166, "2455": 1143, "2456": 49434, "2457": 60261, "2458": 11273, "2459": 18460, "2460": 3020, "2461": 771, "2462": 33693, "2463": 8094, "2464": 10580, "2465": 50916, "2466": 47459, "2467": 60812, "2468": 69090, "2469": 2210, "2470": 69884, "2471": 3256, "2472": 8855, "2473": 15148, "2474": 297, "2475": 65228, "2476": 11590, "2477": 29245, "2478": 61699, "2479": 91578, "2480": 221, "2481": 18022, "2482": 14493, "2483": 1264, "2484": 49482, "2485": 386299, "2486": 14384, "2487": 22734, "2488": 74073, "2489": 596, "2490": 871, "2491": 7380, "2492": 622, "2493": 413, "2494": 2200, "2495": 28609, "2496": 15, "2497": 220260, "2498": 165109, "2499": 13339, "2500": 23487, "2501": 6837, "2502": 2635, "2503": 39823, "2504": 57112, "2505": 83375, "2506": 80490, "2507": 8261, "2508": 21065, "2509": 58034, "2510": 8191, "2511": 26038, "2512": 42225, "2513": 49083, "2514": 4676, "2515": 4447, "2516": 712, "2517": 13, "2518": 103596, "2519": 1163, "2520": 44471, "2521": 40954, "2522": 6, "2523": 32238, "2524": 25117, "2525": 176487, "2526": 4027, "2527": 13465, "2528": 1053, "2529": 193934, "2530": 218180, "2531": 34159, "2532": 26278, "2533": 1353, "2534": 13149, "2535": 27455, "2536": 106342, "2537": 1526, "2538": 26276, "2539": 99, "2540": 28, "2541": 20234, "2542": 1067, "2543": 38573, "2544": 17849, "2545": 65911, "2546": 81720, "2547": 63438, "2548": 3609, "2549": 71442, "2550": 30221, "2551": 6674, "2552": 15030, "2553": 1592, "2554": 27, "2555": 42813, "2556": 30051, "2557": 32207, "2558": 11, "2559": 23882, "2560": 4551, "2561": 11800, "2562": 28565, "2563": 17872, "2564": 1747, "2565": 119774, "2566": 67847, "2567": 193068, "2568": 98125, "2569": 18, "2570": 4282, "2571": 30705, "2572": 126788, "2573": 25806, "2574": 54302, "2575": 25112, "2576": 142473, "2577": 12851, "2578": 84, "2579": 384, "2580": 16240, "2581": 2728, "2582": 32826, "2583": 17805, "2584": 53029, "2585": 30867, "2586": 25437, "2587": 57503, "2588": 43789, "2589": 2385, "2590": 28014, "2591": 13519, "2592": 25434, "2593": 5423, "2594": 13418, "2595": 154272, "2596": 35780, "2597": 16303, "2598": 50870, "2599": 9542, "2600": 1749, "2601": 136928, "2602": 53962, "2603": 80299, "2604": 78963, "2605": 443, "2606": 37643, "2607": 72348, "2608": 205, "2609": 106749, "2610": 6324, "2611": 69946, "2612": 51006, "2613": 207290, "2614": 198316, "2615": 57025, "2616": 13087, "2617": 103245, "2618": 62108, "2619": 25846, "2620": 12222, "2621": 32250, "2622": 145607, "2623": 31592, "2624": 337, "2625": 127928, "2626": 5915, "2627": 292, "2628": 32931, "2629": 293, "2630": 0, "2631": 422680, "2632": 132122, "2633": 3394, "2634": 12390, "2635": 445, "2636": 37492, "2637": 27919, "2638": 29532, "2639": 33198, "2640": 1076, "2641": 15621, "2642": 555, "2643": 24345, "2644": 49708, "2645": 98, "2646": 1670, "2647": 0, "2648": 13578, "2649": 2909, "2650": 30871, "2651": 43528, "2652": 1442, "2653": 29432, "2654": 953, "2655": 52786, "2656": 2548, "2657": 178337, "2658": 34981, "2659": 4405, "2660": 176764, "2661": 4178, "2662": 68629, "2663": 160773, "2664": 6407, "2665": 14575, "2666": 45795, "2667": 7517, "2668": 524, "2669": 223277, "2670": 4744, "2671": 26506, "2672": 29880, "2673": 0, "2674": 41344, "2675": 101395, "2676": 435, "2677": 19, "2678": 197032, "2679": 2404, "2680": 25974, "2681": 42822, "2682": 169337, "2683": 32581, "2684": 15882, "2685": 7248, "2686": 1042, "2687": 45462, "2688": 14122, "2689": 18902, "2690": 15033, "2691": 2002, "2692": 25103, "2693": 15552, "2694": 4847, "2695": 0, "2696": 6900, "2697": 0, "2698": 71005, "2699": 20358, "2700": 11154, "2701": 12209, "2702": 44593, "2703": 1509, "2704": 204, "2705": 32657, "2706": 8755, "2707": 38285, "2708": 17791, "2709": 3431, "2710": 416, "2711": 75923, "2712": 30214, "2713": 17, "2714": 6299, "2715": 31024, "2716": 7990, "2717": 7532, "2718": 110881, "2719": 68727, "2720": 20448, "2721": 125576, "2722": 103, "2723": 34567, "2724": 16169, "2725": 580, "2726": 9922, "2727": 17392, "2728": 2996, "2729": 125490, "2730": 17152, "2731": 42, "2732": 34061, "2733": 13396, "2734": 120501, "2735": 205846, "2736": 41, "2737": 106, "2738": 41651, "2739": 6432, "2740": 7832, "2741": 2357, "2742": 2238, "2743": 28809, "2744": 17955, "2745": 2245, "2746": 20697, "2747": 2595, "2748": 1484, "2749": 0, "2750": 1192, "2751": 15, "2752": 52542, "2753": 75541, "2754": 10347, "2755": 1320, "2756": 31162, "2757": 5262, "2758": 62990, "2759": 31823, "2760": 0, "2761": 33089, "2762": 1305, "2763": 18175, "2764": 4932, "2765": 17374, "2766": 22, "2767": 417, "2768": 110779, "2769": 20632, "2770": 111113, "2771": 7268, "2772": 81961, "2773": 14715, "2774": 130569, "2775": 84851, "2776": 144286, "2777": 143968, "2778": 146008, "2779": 21475, "2780": 63, "2781": 22710, "2782": 182191, "2783": 33092, "2784": 116141, "2785": 55414, "2786": 2246, "2787": 54814, "2788": 41932, "2789": 60074, "2790": 133682, "2791": 4421, "2792": 0, "2793": 77491, "2794": 10787, "2795": 5739, "2796": 118550, "2797": 122, "2798": 3, "2799": 22962, "2800": 33390, "2801": 12032, "2802": 58576, "2803": 378, "2804": 54830, "2805": 261, "2806": 2044, "2807": 213954, "2808": 62091, "2809": 3356, "2810": 76, "2811": 25719, "2812": 43605, "2813": 6237, "2814": 483, "2815": 35425, "2816": 720, "2817": 55470, "2818": 4303, "2819": 209177, "2820": 4795, "2821": 66256, "2822": 427403, "2823": 15041, "2824": 45514, "2825": 3147, "2826": 2168, "2827": 17713, "2828": 1988, "2829": 40190, "2830": 3191, "2831": 62894, "2832": 64409, "2833": 6343, "2834": 88, "2835": 15351, "2836": 64376, "2837": 5189, "2838": 7578, "2839": 32652, "2840": 16742, "2841": 75781, "2842": 33767, "2843": 0, "2844": 80334, "2845": 62, "2846": 1384, "2847": 88358, "2848": 2649, "2849": 25255, "2850": 164128, "2851": 75070, "2852": 0, "2853": 17075, "2854": 111252, "2855": 12861, "2856": 87691, "2857": 1370, "2858": 889, "2859": 32496, "2860": 243, "2861": 12590, "2862": 285825, "2863": 575, "2864": 5575, "2865": 25755, "2866": 19450, "2867": 390, "2868": 711, "2869": 85, "2870": 47889, "2871": 40796, "2872": 12192, "2873": 136653, "2874": 17436, "2875": 2833, "2876": 516, "2877": 99639, "2878": 1433, "2879": 113130, "2880": 2785, "2881": 12283, "2882": 40955, "2883": 38012, "2884": 22387, "2885": 22518, "2886": 19, "2887": 3820, "2888": 36138, "2889": 1330, "2890": 715, "2891": 2155, "2892": 86380, "2893": 156431, "2894": 3306, "2895": 5105, "2896": 85537, "2897": 10025, "2898": 26649, "2899": 2435, "2900": 35059, "2901": 15918, "2902": 64161, "2903": 435, "2904": 0, "2905": 15699, "2906": 8, "2907": 94987, "2908": 140, "2909": 9111, "2910": 98893, "2911": 61577, "2912": 51267, "2913": 212683, "2914": 55244, "2915": 117380, "2916": 59664, "2917": 247, "2918": 17, "2919": 36461, "2920": 7599, "2921": 6828, "2922": 29313, "2923": 79108, "2924": 60, "2925": 20793, "2926": 73479, "2927": 5709, "2928": 448, "2929": 22020, "2930": 2113, "2931": 803, "2932": 20079, "2933": 4034, "2934": 455, "2935": 6176, "2936": 27097, "2937": 18410, "2938": 194, "2939": 10837, "2940": 68860, "2941": 1327, "2942": 18796, "2943": 616, "2944": 1825, "2945": 2044, "2946": 387, "2947": 116641, "2948": 116549, "2949": 62716, "2950": 29219, "2951": 1449, "2952": 168021, "2953": 46969, "2954": 0, "2955": 39054, "2956": 108720, "2957": 38501, "2958": 7502, "2959": 149060, "2960": 3350, "2961": 423, "2962": 8574, "2963": 24391, "2964": 35011, "2965": 2, "2966": 24515, "2967": 41020, "2968": 121810, "2969": 25717, "2970": 24281, "2971": 64030, "2972": 11937, "2973": 1339, "2974": 5549, "2975": 16562, "2976": 171791, "2977": 157, "2978": 69960, "2979": 1091, "2980": 74294, "2981": 5316, "2982": 156, "2983": 7788, "2984": 179510, "2985": 53, "2986": 6687, "2987": 51976, "2988": 12560, "2989": 11059, "2990": 113569, "2991": 38049, "2992": 8022, "2993": 54874, "2994": 134246, "2995": 20601, "2996": 130189, "2997": 106132, "2998": 30755, "2999": 99728, "3000": 60654, "3001": 914, "3002": 226, "3003": 24427, "3004": 80368, "3005": 10350, "3006": 627, "3007": 34570, "3008": 74942, "3009": 131470, "3010": 90307, "3011": 4803, "3012": 609, "3013": 182065, "3014": 1808, "3015": 78135, "3016": 85798, "3017": 354, "3018": 55346, "3019": 207, "3020": 133144, "3021": 2931, "3022": 954, "3023": 0, "3024": 33059, "3025": 19196, "3026": 127590, "3027": 16622, "3028": 88196, "3029": 9214, "3030": 0, "3031": 220395, "3032": 112030, "3033": 45732, "3034": 9275, "3035": 61133, "3036": 52707, "3037": 3095, "3038": 43306, "3039": 117255, "3040": 63117, "3041": 67513, "3042": 47663, "3043": 0, "3044": 570, "3045": 27486, "3046": 13116, "3047": 9553, "3048": 139586, "3049": 129368, "3050": 1640, "3051": 23579, "3052": 33242, "3053": 28097, "3054": 57352, "3055": 1539, "3056": 14686, "3057": 32509, "3058": 156052, "3059": 21, "3060": 7, "3061": 111375, "3062": 4406, "3063": 39922, "3064": 103, "3065": 159379, "3066": 59952, "3067": 147152, "3068": 4476, "3069": 2400, "3070": 18835, "3071": 31903, "3072": 384883, "3073": 30441, "3074": 216171, "3075": 794, "3076": 2, "3077": 51094, "3078": 57523, "3079": 1, "3080": 13642, "3081": 262278, "3082": 1315, "3083": 12037, "3084": 134, "3085": 3420, "3086": 51720, "3087": 16557, "3088": 41932, "3089": 323565, "3090": 50198, "3091": 1706, "3092": 1244, "3093": 53861, "3094": 473, "3095": 11159, "3096": 5986, "3097": 6861, "3098": 7523, "3099": 189273, "3100": 19131, "3101": 6498, "3102": 1306, "3103": 3005, "3104": 9687, "3105": 1943, "3106": 49926, "3107": 64, "3108": 86352, "3109": 257405, "3110": 147782, "3111": 78756, "3112": 22226, "3113": 82862, "3114": 77560, "3115": 4372, "3116": 1405, "3117": 8725, "3118": 3353, "3119": 28330, "3120": 82313, "3121": 111421, "3122": 38489, "3123": 109804, "3124": 14912, "3125": 12557, "3126": 90990, "3127": 147473, "3128": 77, "3129": 335, "3130": 13, "3131": 0, "3132": 4878, "3133": 29315, "3134": 179583, "3135": 8788, "3136": 35016, "3137": 80648, "3138": 19982, "3139": 57605, "3140": 68780, "3141": 107081, "3142": 57640, "3143": 6886, "3144": 165099, "3145": 2576, "3146": 7, "3147": 0, "3148": 218516, "3149": 25861, "3150": 71577, "3151": 225696, "3152": 15, "3153": 3641, "3154": 790, "3155": 0, "3156": 6525, "3157": 570, "3158": 306788, "3159": 175, "3160": 133306, "3161": 17039, "3162": 6481, "3163": 0, "3164": 79770, "3165": 1707, "3166": 232263, "3167": 11744, "3168": 155, "3169": 21601, "3170": 125801, "3171": 205581, "3172": 26045, "3173": 90436, "3174": 2585, "3175": 8322, "3176": 75690, "3177": 10972, "3178": 301, "3179": 1617, "3180": 51131, "3181": 90, "3182": 34040, "3183": 16031, "3184": 3156, "3185": 30679, "3186": 146072, "3187": 655, "3188": 14, "3189": 1200, "3190": 941, "3191": 7430, "3192": 27, "3193": 22478, "3194": 204603, "3195": 9988, "3196": 77309, "3197": 65387, "3198": 45471, "3199": 152, "3200": 7780, "3201": 88414, "3202": 33447, "3203": 17893, "3204": 4942, "3205": 78, "3206": 36349, "3207": 1457, "3208": 21747, "3209": 2411, "3210": 2773, "3211": 893, "3212": 11, "3213": 8191, "3214": 187754, "3215": 33711, "3216": 15767, "3217": 3743, "3218": 3344, "3219": 609, "3220": 2788, "3221": 15103, "3222": 18238, "3223": 286260, "3224": 184, "3225": 416, "3226": 9783, "3227": 972, "3228": 1447, "3229": 122343, "3230": 33219, "3231": 13588, "3232": 1128, "3233": 50730, "3234": 8848, "3235": 13546, "3236": 284, "3237": 6582, "3238": 60541, "3239": 29984, "3240": 106464, "3241": 7673, "3242": 52017, "3243": 0, "3244": 21525, "3245": 1301, "3246": 37777, "3247": 28587, "3248": 23611, "3249": 33007, "3250": 153653, "3251": 5248, "3252": 111850, "3253": 40872, "3254": 101381, "3255": 300, "3256": 155998, "3257": 8635, "3258": 84070, "3259": 21366, "3260": 28227, "3261": 51802, "3262": 6195, "3263": 182451, "3264": 10, "3265": 95, "3266": 52047, "3267": 4507, "3268": 833, "3269": 15455, "3270": 1362, "3271": 675, "3272": 14933, "3273": 26714, "3274": 23958, "3275": 45263, "3276": 3502, "3277": 44760, "3278": 43859, "3279": 88245, "3280": 1796, "3281": 42390, "3282": 62478, "3283": 33430, "3284": 121, "3285": 12776, "3286": 207788, "3287": 3, "3288": 9944, "3289": 298, "3290": 16960, "3291": 5118, "3292": 19896, "3293": 113544, "3294": 68719, "3295": 152820, "3296": 14998, "3297": 10295, "3298": 12128, "3299": 6136, "3300": 23735, "3301": 17, "3302": 72, "3303": 1923, "3304": 30407, "3305": 140425, "3306": 2962, "3307": 20396, "3308": 20455, "3309": 43113, "3310": 51915, "3311": 3268, "3312": 5239, "3313": 63252, "3314": 981, "3315": 104947, "3316": 27314, "3317": 3348, "3318": 1394, "3319": 0, "3320": 46052, "3321": 28066, "3322": 13197, "3323": 5222, "3324": 2033, "3325": 341, "3326": 4163, "3327": 93, "3328": 13385, "3329": 14019, "3330": 0, "3331": 27234, "3332": 17100, "3333": 116642, "3334": 47022, "3335": 1364, "3336": 175777, "3337": 5, "3338": 3, "3339": 15, "3340": 39, "3341": 738, "3342": 78935, "3343": 11920, "3344": 653, "3345": 25725, "3346": 118293, "3347": 113779, "3348": 55345, "3349": 11853, "3350": 8362, "3351": 472, "3352": 24093, "3353": 746, "3354": 110253, "3355": 8052, "3356": 39268, "3357": 22563, "3358": 2468, "3359": 46111, "3360": 81022, "3361": 29563, "3362": 12692, "3363": 279851, "3364": 38001, "3365": 60, "3366": 125010, "3367": 26563, "3368": 36503, "3369": 1042, "3370": 13897, "3371": 11394, "3372": 42596, "3373": 18701, "3374": 28532, "3375": 22124, "3376": 69246, "3377": 123493, "3378": 1505, "3379": 3361, "3380": 68122, "3381": 6008, "3382": 61871, "3383": 24304, "3384": 45641, "3385": 39161, "3386": 6764, "3387": 6074, "3388": 96644, "3389": 48450, "3390": 5034, "3391": 177, "3392": 574, "3393": 2852, "3394": 56884, "3395": 55896, "3396": 4842, "3397": 286528, "3398": 381, "3399": 153020, "3400": 28089, "3401": 8928, "3402": 244408, "3403": 1417, "3404": 43319, "3405": 88101, "3406": 61700, "3407": 2015, "3408": 292994, "3409": 10392, "3410": 171375, "3411": 883, "3412": 53386, "3413": 2909, "3414": 50635, "3415": 71137, "3416": 4776, "3417": 343, "3418": 5, "3419": 209828, "3420": 17902, "3421": 45184, "3422": 27572, "3423": 44031, "3424": 10378, "3425": 89233, "3426": 6101, "3427": 32746, "3428": 0, "3429": 15467, "3430": 42880, "3431": 3557, "3432": 6596, "3433": 25872, "3434": 97478, "3435": 267, "3436": 727, "3437": 121435, "3438": 28619, "3439": 71677, "3440": 884, "3441": 79744, "3442": 32525, "3443": 500, "3444": 0, "3445": 64355, "3446": 10903, "3447": 8597, "3448": 129242, "3449": 361048, "3450": 86146, "3451": 2386, "3452": 71396, "3453": 58266, "3454": 62464, "3455": 0, "3456": 82780, "3457": 478171, "3458": 52853, "3459": 821, "3460": 2964, "3461": 23206, "3462": 1018, "3463": 124460, "3464": 12, "3465": 52874, "3466": 31249, "3467": 2178, "3468": 0, "3469": 9569, "3470": 7860, "3471": 0, "3472": 50699, "3473": 58991, "3474": 11368, "3475": 27431, "3476": 127105, "3477": 85385, "3478": 23541, "3479": 75722, "3480": 9025, "3481": 26002, "3482": 37984, "3483": 0, "3484": 91614, "3485": 255, "3486": 56817, "3487": 86474, "3488": 341485, "3489": 693, "3490": 4346, "3491": 5308, "3492": 9277, "3493": 4966, "3494": 11654, "3495": 7656, "3496": 45581, "3497": 50204, "3498": 632, "3499": 4822, "3500": 0, "3501": 531, "3502": 54128, "3503": 36444, "3504": 12005, "3505": 2725, "3506": 184526, "3507": 14664, "3508": 66766, "3509": 25433, "3510": 19046, "3511": 1108, "3512": 40763, "3513": 695, "3514": 42911, "3515": 19026, "3516": 41134, "3517": 11082, "3518": 16844, "3519": 43199, "3520": 29694, "3521": 79399, "3522": 55770, "3523": 33, "3524": 111795, "3525": 0, "3526": 6508, "3527": 74721, "3528": 14258, "3529": 47075, "3530": 122263, "3531": 123673, "3532": 40786, "3533": 81292, "3534": 36450, "3535": 64883, "3536": 70423, "3537": 643, "3538": 85443, "3539": 7294, "3540": 84862, "3541": 4, "3542": 37104, "3543": 3944, "3544": 24237, "3545": 14663, "3546": 508, "3547": 10416, "3548": 2188, "3549": 99201, "3550": 2688, "3551": 565, "3552": 4183, "3553": 9294, "3554": 75199, "3555": 13345, "3556": 32804, "3557": 12, "3558": 38730, "3559": 76, "3560": 1341, "3561": 119475, "3562": 10211, "3563": 2031, "3564": 9620, "3565": 122776, "3566": 19425, "3567": 21554, "3568": 4306, "3569": 190, "3570": 17018, "3571": 10141, "3572": 73538, "3573": 12158, "3574": 959, "3575": 76, "3576": 28687, "3577": 58, "3578": 56169, "3579": 21370, "3580": 23377, "3581": 13869, "3582": 2661, "3583": 585, "3584": 57239, "3585": 25344, "3586": 52851, "3587": 62388, "3588": 15, "3589": 79154, "3590": 396445, "3591": 10052, "3592": 69948, "3593": 2451, "3594": 0, "3595": 81944, "3596": 52103, "3597": 44803, "3598": 134456, "3599": 63436, "3600": 121353, "3601": 44644, "3602": 20356, "3603": 76865, "3604": 56646, "3605": 187270, "3606": 42854, "3607": 84613, "3608": 3730, "3609": 216353, "3610": 12466, "3611": 1494, "3612": 74446, "3613": 92961, "3614": 813, "3615": 27558, "3616": 73175, "3617": 43714, "3618": 1329, "3619": 432, "3620": 42930, "3621": 106518, "3622": 613, "3623": 7140, "3624": 33760, "3625": 40877, "3626": 137538, "3627": 3253, "3628": 42761, "3629": 32299, "3630": 4836, "3631": 25401, "3632": 23249, "3633": 11905, "3634": 285708, "3635": 73432, "3636": 27607, "3637": 187, "3638": 56, "3639": 20362, "3640": 8663, "3641": 1170, "3642": 118413, "3643": 7375, "3644": 21607, "3645": 85480, "3646": 36654, "3647": 0, "3648": 33082, "3649": 14734, "3650": 45024, "3651": 8002, "3652": 218916, "3653": 81588, "3654": 116516, "3655": 3149, "3656": 42232, "3657": 91465, "3658": 68419, "3659": 25717, "3660": 58007, "3661": 2, "3662": 1024, "3663": 127, "3664": 258158, "3665": 1, "3666": 15, "3667": 94363, "3668": 56621, "3669": 818, "3670": 56268, "3671": 93012, "3672": 49264, "3673": 104175, "3674": 40303, "3675": 146674, "3676": 9878, "3677": 180815, "3678": 50361, "3679": 85484, "3680": 30923, "3681": 12874, "3682": 1248, "3683": 337250, "3684": 6, "3685": 7540, "3686": 0, "3687": 22850, "3688": 48307, "3689": 2, "3690": 16966, "3691": 10461, "3692": 24900, "3693": 10930, "3694": 77718, "3695": 10339, "3696": 7404, "3697": 129324, "3698": 2753, "3699": 1074, "3700": 50571, "3701": 27, "3702": 14, "3703": 0, "3704": 3412, "3705": 134815, "3706": 4096, "3707": 22956, "3708": 46994, "3709": 50540, "3710": 27778, "3711": 336274, "3712": 57797, "3713": 2156, "3714": 482, "3715": 24, "3716": 0, "3717": 87872, "3718": 28118, "3719": 5548, "3720": 132, "3721": 0, "3722": 38914, "3723": 19849, "3724": 8494, "3725": 351176, "3726": 98, "3727": 1214, "3728": 199090, "3729": 2636, "3730": 63232, "3731": 47054, "3732": 103141, "3733": 210498, "3734": 7239, "3735": 13234, "3736": 82673, "3737": 55224, "3738": 41279, "3739": 20057, "3740": 0, "3741": 10812, "3742": 39538, "3743": 78357, "3744": 188205, "3745": 7797, "3746": 6133, "3747": 37084, "3748": 2626, "3749": 167391, "3750": 150, "3751": 12658, "3752": 5876, "3753": 26943, "3754": 123802, "3755": 90490, "3756": 267, "3757": 4854, "3758": 104, "3759": 20850, "3760": 71721, "3761": 11444, "3762": 14421, "3763": 29582, "3764": 73007, "3765": 634, "3766": 21051, "3767": 4438, "3768": 61591, "3769": 18266, "3770": 0, "3771": 50620, "3772": 1675, "3773": 7949, "3774": 13, "3775": 53, "3776": 8315, "3777": 3, "3778": 3892, "3779": 43211, "3780": 4520, "3781": 77151, "3782": 9624, "3783": 261437, "3784": 19, "3785": 56438, "3786": 2559, "3787": 1738, "3788": 16941, "3789": 13063, "3790": 26687, "3791": 142020, "3792": 80821, "3793": 56010, "3794": 0, "3795": 4104, "3796": 15235, "3797": 44500, "3798": 13315, "3799": 2260, "3800": 571, "3801": 73731, "3802": 6728, "3803": 2321, "3804": 474669, "3805": 1023, "3806": 0, "3807": 47285, "3808": 16950, "3809": 13303, "3810": 0, "3811": 13010, "3812": 12867, "3813": 1166, "3814": 2507, "3815": 17903, "3816": 74471, "3817": 25622, "3818": 9424, "3819": 15699, "3820": 22312, "3821": 1271, "3822": 4118, "3823": 12515, "3824": 0, "3825": 132359, "3826": 76908, "3827": 15591, "3828": 22826, "3829": 6133, "3830": 164643, "3831": 113, "3832": 186, "3833": 139586, "3834": 0, "3835": 11427, "3836": 32482, "3837": 63158, "3838": 120, "3839": 1157, "3840": 6534, "3841": 27594, "3842": 24708, "3843": 3385, "3844": 48291, "3845": 3710, "3846": 2722, "3847": 68841, "3848": 1090, "3849": 19962, "3850": 13919, "3851": 24886, "3852": 5133, "3853": 26414, "3854": 53785, "3855": 8081, "3856": 15, "3857": 86170, "3858": 31663, "3859": 21354, "3860": 7691, "3861": 34435, "3862": 6842, "3863": 314229, "3864": 4950, "3865": 9905, "3866": 69166, "3867": 0, "3868": 365, "3869": 268231, "3870": 1322, "3871": 26450, "3872": 8605, "3873": 29533, "3874": 28626, "3875": 87, "3876": 64624, "3877": 38276, "3878": 1670, "3879": 74192, "3880": 84903, "3881": 16800, "3882": 15353, "3883": 16685, "3884": 421, "3885": 234, "3886": 109982, "3887": 26915, "3888": 11116, "3889": 110363, "3890": 35571, "3891": 44191, "3892": 4903, "3893": 15066, "3894": 21877, "3895": 979, "3896": 50845, "3897": 47564, "3898": 112748, "3899": 29009, "3900": 22632, "3901": 65857, "3902": 13180, "3903": 32665, "3904": 130381, "3905": 36108, "3906": 23140, "3907": 39994, "3908": 291546, "3909": 14894, "3910": 12588, "3911": 5590, "3912": 16986, "3913": 7717, "3914": 47469, "3915": 54, "3916": 61250, "3917": 128, "3918": 42845, "3919": 15354, "3920": 54444, "3921": 0, "3922": 21836, "3923": 247, "3924": 8218, "3925": 2638, "3926": 42810, "3927": 30626, "3928": 139417, "3929": 1, "3930": 2124, "3931": 0, "3932": 2048, "3933": 19900, "3934": 84605, "3935": 37348, "3936": 59557, "3937": 9190, "3938": 356251, "3939": 33375, "3940": 13840, "3941": 52733, "3942": 43813, "3943": 102875, "3944": 12980, "3945": 4310, "3946": 46689, "3947": 7125, "3948": 2272, "3949": 7479, "3950": 127155, "3951": 149, "3952": 6535, "3953": 29, "3954": 23473, "3955": 68257, "3956": 3663, "3957": 423, "3958": 105598, "3959": 264155, "3960": 18434, "3961": 1, "3962": 160020, "3963": 6039, "3964": 2890, "3965": 36591, "3966": 6814, "3967": 27649, "3968": 93246, "3969": 127906, "3970": 68734, "3971": 56146, "3972": 138, "3973": 11, "3974": 253, "3975": 7324, "3976": 63732, "3977": 122783, "3978": 3876, "3979": 15306, "3980": 0, "3981": 301, "3982": 9994, "3983": 80416, "3984": 26055, "3985": 44781, "3986": 478, "3987": 41584, "3988": 28371, "3989": 22167, "3990": 18863, "3991": 24250, "3992": 4472, "3993": 1074, "3994": 39994, "3995": 145725, "3996": 43902, "3997": 14, "3998": 95531, "3999": 973, "4000": 3942, "4001": 1082, "4002": 62293, "4003": 6268, "4004": 108928, "4005": 49006, "4006": 29771, "4007": 21991, "4008": 56396, "4009": 312248, "4010": 70701, "4011": 6892, "4012": 18498, "4013": 46744, "4014": 33598, "4015": 45687, "4016": 2200, "4017": 2393, "4018": 50047, "4019": 48063, "4020": 190, "4021": 3528, "4022": 5412, "4023": 3075, "4024": 48028, "4025": 41634, "4026": 75988, "4027": 41275, "4028": 601, "4029": 87000, "4030": 37001, "4031": 6671, "4032": 102758, "4033": 75472, "4034": 5688, "4035": 126228, "4036": 18191, "4037": 23514, "4038": 1644, "4039": 9882, "4040": 128513, "4041": 12526, "4042": 47000, "4043": 64159, "4044": 8792, "4045": 17118, "4046": 18348, "4047": 8148, "4048": 32349, "4049": 48564, "4050": 1977, "4051": 124230, "4052": 15823, "4053": 110, "4054": 25193, "4055": 45732, "4056": 917, "4057": 73291, "4058": 16073, "4059": 98090, "4060": 64579, "4061": 13801, "4062": 84728, "4063": 6694, "4064": 101578, "4065": 214931, "4066": 5, "4067": 0, "4068": 4945, "4069": 109035, "4070": 516, "4071": 59058, "4072": 108, "4073": 7427, "4074": 165407, "4075": 1904, "4076": 37816, "4077": 3530, "4078": 126947, "4079": 145714, "4080": 84, "4081": 23822, "4082": 8349, "4083": 0, "4084": 4920, "4085": 6074, "4086": 10612, "4087": 0, "4088": 2019, "4089": 542, "4090": 10369, "4091": 15849, "4092": 119397, "4093": 3508, "4094": 85693, "4095": 5683, "4096": 59698, "4097": 8831, "4098": 10535, "4099": 40, "4100": 30266, "4101": 3526, "4102": 36147, "4103": 11009, "4104": 3638, "4105": 16821, "4106": 25949, "4107": 21133, "4108": 16246, "4109": 18653, "4110": 0, "4111": 213, "4112": 112484, "4113": 37669, "4114": 20600, "4115": 211014, "4116": 505, "4117": 14542, "4118": 15921, "4119": 14706, "4120": 2604, "4121": 103258, "4122": 15494, "4123": 1469, "4124": 9887, "4125": 62264, "4126": 20102, "4127": 8890, "4128": 939, "4129": 20654, "4130": 38850, "4131": 13874, "4132": 0, "4133": 43415, "4134": 69882, "4135": 42960, "4136": 26788, "4137": 43, "4138": 18394, "4139": 8102, "4140": 40137, "4141": 387, "4142": 0, "4143": 50440, "4144": 14984, "4145": 69690, "4146": 18562, "4147": 8397, "4148": 30767, "4149": 40778, "4150": 20815, "4151": 74555, "4152": 24897, "4153": 2186, "4154": 64357, "4155": 31845, "4156": 3272, "4157": 2725, "4158": 166222, "4159": 12204, "4160": 17, "4161": 15738, "4162": 9899, "4163": 43207, "4164": 5, "4165": 655, "4166": 146168, "4167": 140164, "4168": 5139, "4169": 365150, "4170": 21934, "4171": 26491, "4172": 1781, "4173": 102789, "4174": 300438, "4175": 8547, "4176": 8388, "4177": 16, "4178": 3323, "4179": 3051, "4180": 65280, "4181": 5210, "4182": 451, "4183": 28092, "4184": 118126, "4185": 108771, "4186": 1042, "4187": 6303, "4188": 0, "4189": 33, "4190": 27123, "4191": 265523, "4192": 60, "4193": 13367, "4194": 20484, "4195": 2519, "4196": 37, "4197": 67590, "4198": 0, "4199": 546, "4200": 65100, "4201": 80385, "4202": 125629, "4203": 40554, "4204": 1106, "4205": 21006, "4206": 4241, "4207": 53600, "4208": 27490, "4209": 72740, "4210": 19716, "4211": 27038, "4212": 16858, "4213": 598, "4214": 51049, "4215": 196242, "4216": 18660, "4217": 4343, "4218": 105486, "4219": 4481, "4220": 0, "4221": 3783, "4222": 11835, "4223": 359393, "4224": 11051, "4225": 38793, "4226": 4535, "4227": 8700, "4228": 40176, "4229": 303506, "4230": 1711, "4231": 74867, "4232": 36043, "4233": 14272, "4234": 30830, "4235": 13349, "4236": 26328, "4237": 60153, "4238": 65587, "4239": 626, "4240": 438899, "4241": 61073, "4242": 50073, "4243": 35877, "4244": 30885, "4245": 128826, "4246": 1137, "4247": 2171, "4248": 6605, "4249": 4046, "4250": 5122, "4251": 11800, "4252": 48005, "4253": 9342, "4254": 68, "4255": 2, "4256": 7948, "4257": 129977, "4258": 2968, "4259": 72530, "4260": 82538, "4261": 68075, "4262": 52134, "4263": 122421, "4264": 73108, "4265": 127503, "4266": 23508, "4267": 20677, "4268": 34661, "4269": 13764, "4270": 50761, "4271": 30619, "4272": 30417, "4273": 103470, "4274": 0, "4275": 80763, "4276": 36839, "4277": 47998, "4278": 575, "4279": 32983, "4280": 15545, "4281": 60913, "4282": 26705, "4283": 61456, "4284": 19543, "4285": 66684, "4286": 9335, "4287": 237, "4288": 25974, "4289": 44011, "4290": 44226, "4291": 40495, "4292": 7492, "4293": 5, "4294": 47578, "4295": 31721, "4296": 13162, "4297": 4461, "4298": 68358, "4299": 132746, "4300": 111343, "4301": 2117, "4302": 174, "4303": 14706, "4304": 312, "4305": 3998, "4306": 38202, "4307": 1247, "4308": 0, "4309": 59333, "4310": 0, "4311": 82791, "4312": 270, "4313": 19278, "4314": 33199, "4315": 46989, "4316": 21728, "4317": 84, "4318": 11485, "4319": 51138, "4320": 2086, "4321": 119413, "4322": 3878, "4323": 63742, "4324": 1, "4325": 2416, "4326": 39004, "4327": 31321, "4328": 0, "4329": 713, "4330": 23876, "4331": 194629, "4332": 12262, "4333": 83, "4334": 0, "4335": 12314, "4336": 68550, "4337": 321445, "4338": 1062, "4339": 35650, "4340": 41717, "4341": 57617, "4342": 30991, "4343": 163464, "4344": 105439, "4345": 9484, "4346": 80768, "4347": 10450, "4348": 28988, "4349": 34689, "4350": 25981, "4351": 16944, "4352": 8, "4353": 83527, "4354": 3421, "4355": 64555, "4356": 25932, "4357": 77716, "4358": 67365, "4359": 24703, "4360": 1774, "4361": 11474, "4362": 445641, "4363": 1562, "4364": 65437, "4365": 3800, "4366": 33233, "4367": 22647, "4368": 28547, "4369": 80031, "4370": 0, "4371": 10511, "4372": 36036, "4373": 12178, "4374": 2225, "4375": 91942, "4376": 44699, "4377": 289242, "4378": 96418, "4379": 5753, "4380": 59169, "4381": 10, "4382": 120561, "4383": 84244, "4384": 43, "4385": 6471, "4386": 56821, "4387": 56162, "4388": 7375, "4389": 946, "4390": 14559, "4391": 16414, "4392": 15340, "4393": 7554, "4394": 254417, "4395": 77107, "4396": 104, "4397": 172259, "4398": 40094, "4399": 986, "4400": 166807, "4401": 24942, "4402": 18241, "4403": 1120, "4404": 110241, "4405": 46136, "4406": 34825, "4407": 14117, "4408": 20561, "4409": 92308, "4410": 16092, "4411": 48574, "4412": 79408, "4413": 5192, "4414": 172960, "4415": 37069, "4416": 36838, "4417": 52417, "4418": 40348, "4419": 493, "4420": 44834, "4421": 873, "4422": 8981, "4423": 26356, "4424": 109, "4425": 41, "4426": 2742, "4427": 25417, "4428": 68433, "4429": 6085, "4430": 829, "4431": 60564, "4432": 108550, "4433": 467678, "4434": 68515, "4435": 19800, "4436": 0, "4437": 40893, "4438": 207001, "4439": 21205, "4440": 4611, "4441": 48441, "4442": 15355, "4443": 25829, "4444": 213, "4445": 4972, "4446": 124919, "4447": 78, "4448": 152909, "4449": 28757, "4450": 21697, "4451": 80727, "4452": 40568, "4453": 56249, "4454": 250, "4455": 23784, "4456": 124625, "4457": 67671, "4458": 36037, "4459": 90, "4460": 16467, "4461": 25557, "4462": 58459, "4463": 99813, "4464": 12285, "4465": 19248, "4466": 57460, "4467": 88881, "4468": 5718, "4469": 81094, "4470": 44919, "4471": 17634, "4472": 80291, "4473": 68751, "4474": 33598, "4475": 174081, "4476": 1516, "4477": 81734, "4478": 121103, "4479": 3259, "4480": 47380, "4481": 55044, "4482": 50501, "4483": 55544, "4484": 3335, "4485": 672, "4486": 284, "4487": 13107, "4488": 26370, "4489": 8272, "4490": 48818, "4491": 120629, "4492": 7261, "4493": 57977, "4494": 16675, "4495": 78352, "4496": 23435, "4497": 79701, "4498": 106, "4499": 92971, "4500": 40357, "4501": 29692, "4502": 459456, "4503": 52686, "4504": 34074, "4505": 17233, "4506": 63, "4507": 18206, "4508": 5914, "4509": 1, "4510": 48442, "4511": 221502, "4512": 64821, "4513": 0, "4514": 27, "4515": 467, "4516": 16582, "4517": 28234, "4518": 782, "4519": 16244, "4520": 17636, "4521": 10007, "4522": 3472, "4523": 33045, "4524": 15771, "4525": 73515, "4526": 484, "4527": 8868, "4528": 0, "4529": 2, "4530": 39640, "4531": 565, "4532": 89279, "4533": 65234, "4534": 207, "4535": 3309, "4536": 80920, "4537": 47056, "4538": 19583, "4539": 437, "4540": 25199, "4541": 1666, "4542": 196497, "4543": 5420, "4544": 31853, "4545": 5444, "4546": 27140, "4547": 118592, "4548": 33700, "4549": 39094, "4550": 56090, "4551": 30377, "4552": 36051, "4553": 9508, "4554": 28646, "4555": 1148, "4556": 22960, "4557": 5851, "4558": 83877, "4559": 13799, "4560": 0, "4561": 4560, "4562": 4262, "4563": 4278, "4564": 3120, "4565": 7567, "4566": 35106, "4567": 57, "4568": 103183, "4569": 24483, "4570": 49789, "4571": 26676, "4572": 12782, "4573": 104045, "4574": 23338, "4575": 103107, "4576": 0, "4577": 10102, "4578": 76907, "4579": 889, "4580": 2331, "4581": 25834, "4582": 52, "4583": 112288, "4584": 0, "4585": 5082, "4586": 130977, "4587": 0, "4588": 703, "4589": 12256, "4590": 37209, "4591": 160660, "4592": 25774, "4593": 220791, "4594": 30951, "4595": 5385, "4596": 220, "4597": 85488, "4598": 14194, "4599": 9287, "4600": 30776, "4601": 27915, "4602": 33251, "4603": 22055, "4604": 56439, "4605": 5217, "4606": 514033, "4607": 117142, "4608": 3195, "4609": 143, "4610": 8417, "4611": 510, "4612": 18053, "4613": 992, "4614": 7219, "4615": 3776, "4616": 40586, "4617": 26741, "4618": 83085, "4619": 25312, "4620": 30522, "4621": 8777, "4622": 40969, "4623": 28763, "4624": 86798, "4625": 537, "4626": 22394, "4627": 274114, "4628": 35962, "4629": 3853, "4630": 3518, "4631": 9429, "4632": 54520, "4633": 451, "4634": 1393, "4635": 41523, "4636": 4560, "4637": 62599, "4638": 1, "4639": 189650, "4640": 19185, "4641": 91705, "4642": 120840, "4643": 937, "4644": 28131, "4645": 58637, "4646": 35897, "4647": 11695, "4648": 8167, "4649": 83243, "4650": 6688, "4651": 50040, "4652": 14227, "4653": 65680, "4654": 28901, "4655": 4548, "4656": 50911, "4657": 117335, "4658": 573, "4659": 379, "4660": 16885, "4661": 152408, "4662": 87537, "4663": 85631, "4664": 14610, "4665": 63469, "4666": 17868, "4667": 70, "4668": 13201, "4669": 44194, "4670": 9731, "4671": 78594, "4672": 82842, "4673": 9801, "4674": 105571, "4675": 24519, "4676": 1504, "4677": 5670, "4678": 46997, "4679": 1047, "4680": 8275, "4681": 40308, "4682": 812, "4683": 144861, "4684": 2221, "4685": 68420, "4686": 3971, "4687": 112879, "4688": 35913, "4689": 8695, "4690": 43283, "4691": 24635, "4692": 16545, "4693": 31325, "4694": 1454, "4695": 38003, "4696": 88762, "4697": 29613, "4698": 27434, "4699": 37717, "4700": 5175, "4701": 48402, "4702": 1782, "4703": 49553, "4704": 7459, "4705": 112164, "4706": 103714, "4707": 9487, "4708": 2318, "4709": 19192, "4710": 46050, "4711": 78045, "4712": 277406, "4713": 2404, "4714": 3397, "4715": 17927, "4716": 781, "4717": 137782, "4718": 1634, "4719": 3016, "4720": 54026, "4721": 64300, "4722": 15852, "4723": 5736, "4724": 99934, "4725": 61685, "4726": 17298, "4727": 3435, "4728": 91255, "4729": 146901, "4730": 12023, "4731": 8765, "4732": 5306, "4733": 447340, "4734": 330022, "4735": 3260, "4736": 12155, "4737": 52097, "4738": 108814, "4739": 61634, "4740": 8294, "4741": 19716, "4742": 87427, "4743": 7504, "4744": 0, "4745": 17725, "4746": 113013, "4747": 11987, "4748": 12175, "4749": 1, "4750": 89190, "4751": 525, "4752": 53191, "4753": 816, "4754": 88693, "4755": 7314, "4756": 4313, "4757": 16958, "4758": 4029, "4759": 58038, "4760": 48456, "4761": 237038, "4762": 21819, "4763": 348, "4764": 6811, "4765": 180342, "4766": 26322, "4767": 2175, "4768": 85415, "4769": 30452, "4770": 18764, "4771": 194259, "4772": 57614, "4773": 259719, "4774": 29037, "4775": 613, "4776": 7444, "4777": 19671, "4778": 15491, "4779": 14898, "4780": 53488, "4781": 573, "4782": 26488, "4783": 23714, "4784": 88252, "4785": 9, "4786": 4854, "4787": 102298, "4788": 4399, "4789": 2067, "4790": 2696, "4791": 17492, "4792": 88355, "4793": 11924, "4794": 30738, "4795": 6767, "4796": 61808, "4797": 2637, "4798": 1074, "4799": 17254, "4800": 105579, "4801": 1985, "4802": 65668, "4803": 33722, "4804": 48, "4805": 17189, "4806": 3039, "4807": 9317, "4808": 8747, "4809": 1720, "4810": 131826, "4811": 104466, "4812": 33751, "4813": 1002, "4814": 0, "4815": 1583, "4816": 18570, "4817": 117053, "4818": 2826, "4819": 3497, "4820": 7971, "4821": 34563, "4822": 4219, "4823": 42130, "4824": 33696, "4825": 103294, "4826": 1788, "4827": 43836, "4828": 14234, "4829": 12883, "4830": 38338, "4831": 140813, "4832": 5083, "4833": 23336, "4834": 43572, "4835": 11212, "4836": 16042, "4837": 43247, "4838": 0, "4839": 319794, "4840": 16937, "4841": 43819, "4842": 4678, "4843": 84414, "4844": 50980, "4845": 108701, "4846": 29201, "4847": 32092, "4848": 228031, "4849": 11, "4850": 4488, "4851": 7434, "4852": 154037, "4853": 12355, "4854": 13688, "4855": 381, "4856": 62, "4857": 22956, "4858": 61168, "4859": 141367, "4860": 4114, "4861": 2980, "4862": 3109, "4863": 20991, "4864": 1377, "4865": 141522, "4866": 106665, "4867": 489, "4868": 26890, "4869": 84472, "4870": 69910, "4871": 96591, "4872": 49, "4873": 26591, "4874": 0, "4875": 16879, "4876": 32189, "4877": 56774, "4878": 27064, "4879": 173637, "4880": 9412, "4881": 92580, "4882": 11787, "4883": 34082, "4884": 62032, "4885": 10195, "4886": 3704, "4887": 168, "4888": 75022, "4889": 23276, "4890": 160199, "4891": 0, "4892": 396, "4893": 495, "4894": 49100, "4895": 580, "4896": 0, "4897": 176941, "4898": 37703, "4899": 10948, "4900": 48017, "4901": 18, "4902": 8639, "4903": 26050, "4904": 60373, "4905": 15072, "4906": 142241, "4907": 0, "4908": 130130, "4909": 22, "4910": 0, "4911": 6, "4912": 23894, "4913": 13452, "4914": 972, "4915": 4042, "4916": 37073, "4917": 8, "4918": 64109, "4919": 51621, "4920": 887, "4921": 523, "4922": 5146, "4923": 76323, "4924": 9059, "4925": 107165, "4926": 10126, "4927": 33397, "4928": 5083, "4929": 14931, "4930": 56785, "4931": 0, "4932": 254, "4933": 56664, "4934": 27764, "4935": 46446, "4936": 5002, "4937": 81737, "4938": 21402, "4939": 190, "4940": 99685, "4941": 4572, "4942": 1766, "4943": 2292, "4944": 39490, "4945": 854, "4946": 24003, "4947": 84809, "4948": 64293, "4949": 885, "4950": 364, "4951": 17919, "4952": 31618, "4953": 74, "4954": 98760, "4955": 3052, "4956": 12441, "4957": 37, "4958": 94725, "4959": 76346, "4960": 190068, "4961": 16080, "4962": 6508, "4963": 28016, "4964": 478, "4965": 71596, "4966": 102962, "4967": 1, "4968": 80845, "4969": 234, "4970": 103392, "4971": 5888, "4972": 62613, "4973": 3416, "4974": 65736, "4975": 87120, "4976": 42511, "4977": 10669, "4978": 0, "4979": 110541, "4980": 738, "4981": 30, "4982": 1003, "4983": 46344, "4984": 12178, "4985": 29, "4986": 86048, "4987": 3037, "4988": 3465, "4989": 0, "4990": 3227, "4991": 9859, "4992": 7730, "4993": 242122, "4994": 8516, "4995": 5165, "4996": 86446, "4997": 8408, "4998": 117508, "4999": 11874, "5000": 201487, "5001": 56628, "5002": 67640, "5003": 4706, "5004": 1026, "5005": 9614, "5006": 29633, "5007": 6169, "5008": 26446, "5009": 31960, "5010": 11875, "5011": 90676, "5012": 33691, "5013": 2899, "5014": 19774, "5015": 188501, "5016": 1154, "5017": 19736, "5018": 0, "5019": 0, "5020": 1019, "5021": 71752, "5022": 21296, "5023": 34858, "5024": 52981, "5025": 4597, "5026": 101630, "5027": 19506, "5028": 77899, "5029": 12109, "5030": 85826, "5031": 107416, "5032": 18410, "5033": 15663, "5034": 322383, "5035": 130593, "5036": 32150, "5037": 4620, "5038": 3818, "5039": 217, "5040": 16754, "5041": 32675, "5042": 49936, "5043": 40616, "5044": 67051, "5045": 50284, "5046": 1438, "5047": 34837, "5048": 24052, "5049": 21968, "5050": 7350, "5051": 40836, "5052": 10805, "5053": 2020, "5054": 2286, "5055": 37582, "5056": 42125, "5057": 483, "5058": 9463, "5059": 1124, "5060": 59787, "5061": 2675, "5062": 49, "5063": 7266, "5064": 9043, "5065": 325, "5066": 8583, "5067": 45625, "5068": 61063, "5069": 222326, "5070": 3296, "5071": 67070, "5072": 19220, "5073": 12921, "5074": 4862, "5075": 119720, "5076": 4224, "5077": 24518, "5078": 6101, "5079": 673, "5080": 9074, "5081": 0, "5082": 10, "5083": 7351, "5084": 92795, "5085": 16033, "5086": 7715, "5087": 17304, "5088": 5217, "5089": 32098, "5090": 1360, "5091": 28076, "5092": 77381, "5093": 3680, "5094": 8868, "5095": 17499, "5096": 510276, "5097": 24701, "5098": 56117, "5099": 5699, "5100": 4577, "5101": 270, "5102": 80098, "5103": 12040, "5104": 10710, "5105": 77182, "5106": 6086, "5107": 47297, "5108": 9534, "5109": 111191, "5110": 499, "5111": 5, "5112": 16871, "5113": 26271, "5114": 101090, "5115": 2722, "5116": 16226, "5117": 10658, "5118": 24743, "5119": 70776, "5120": 150, "5121": 30656, "5122": 50327, "5123": 26294, "5124": 61121, "5125": 439, "5126": 70473, "5127": 57967, "5128": 81445, "5129": 5952, "5130": 16010, "5131": 59873, "5132": 3521, "5133": 86205, "5134": 2963, "5135": 11021, "5136": 15396, "5137": 59816, "5138": 4121, "5139": 2466, "5140": 35728, "5141": 23315, "5142": 36706, "5143": 231108, "5144": 79733, "5145": 10497, "5146": 157, "5147": 712, "5148": 566, "5149": 634, "5150": 60036, "5151": 25307, "5152": 366, "5153": 102669, "5154": 31145, "5155": 515, "5156": 328, "5157": 26574, "5158": 5, "5159": 397, "5160": 78978, "5161": 43240, "5162": 48479, "5163": 34726, "5164": 55523, "5165": 856, "5166": 340098, "5167": 37743, "5168": 1480, "5169": 11797, "5170": 23844, "5171": 18841, "5172": 191203, "5173": 155, "5174": 21220, "5175": 32390, "5176": 37286, "5177": 46597, "5178": 12112, "5179": 43435, "5180": 9267, "5181": 135, "5182": 3358, "5183": 10978, "5184": 29981, "5185": 9705, "5186": 104672, "5187": 49429, "5188": 2875, "5189": 92685, "5190": 2713, "5191": 48573, "5192": 185, "5193": 57969, "5194": 12246, "5195": 83313, "5196": 39454, "5197": 82, "5198": 37477, "5199": 10369, "5200": 280446, "5201": 39, "5202": 6, "5203": 2, "5204": 629, "5205": 290460, "5206": 7059, "5207": 65387, "5208": 15325, "5209": 26849, "5210": 4539, "5211": 601, "5212": 12136, "5213": 72152, "5214": 3590, "5215": 97608, "5216": 3631, "5217": 58145, "5218": 141636, "5219": 7217, "5220": 28313, "5221": 12865, "5222": 27, "5223": 57097, "5224": 18301, "5225": 107964, "5226": 68436, "5227": 195641, "5228": 63025, "5229": 360032, "5230": 37721, "5231": 5938, "5232": 103327, "5233": 1870, "5234": 5041, "5235": 34344, "5236": 50416, "5237": 136976, "5238": 369, "5239": 66586, "5240": 22766, "5241": 82072, "5242": 621, "5243": 6527, "5244": 18507, "5245": 30309, "5246": 55230, "5247": 2631, "5248": 4182, "5249": 8567, "5250": 79252, "5251": 22744, "5252": 9812, "5253": 5375, "5254": 65614, "5255": 1631, "5256": 18365, "5257": 9493, "5258": 75286, "5259": 74674, "5260": 49648, "5261": 2420, "5262": 105519, "5263": 22588, "5264": 118020, "5265": 28446, "5266": 2906, "5267": 9368, "5268": 16822, "5269": 248, "5270": 36474, "5271": 7970, "5272": 28398, "5273": 106471, "5274": 293, "5275": 4, "5276": 2, "5277": 1401, "5278": 16571, "5279": 35488, "5280": 0, "5281": 106507, "5282": 1099, "5283": 172820, "5284": 1571, "5285": 80, "5286": 78315, "5287": 96277, "5288": 4433, "5289": 24232, "5290": 49177, "5291": 0, "5292": 291, "5293": 31975, "5294": 29949, "5295": 495, "5296": 88746, "5297": 44286, "5298": 35338, "5299": 4234, "5300": 102696, "5301": 12279, "5302": 29, "5303": 358, "5304": 3147, "5305": 33, "5306": 69839, "5307": 3606, "5308": 12872, "5309": 46618, "5310": 41002, "5311": 8440, "5312": 8333, "5313": 105753, "5314": 32978, "5315": 49854, "5316": 64232, "5317": 41769, "5318": 66415, "5319": 25595, "5320": 355, "5321": 314426, "5322": 22800, "5323": 63519, "5324": 6298, "5325": 157535, "5326": 24216, "5327": 168833, "5328": 136337, "5329": 74822, "5330": 598, "5331": 1028, "5332": 14746, "5333": 14548, "5334": 204487, "5335": 81341, "5336": 97532, "5337": 121641, "5338": 25, "5339": 254149, "5340": 5, "5341": 74546, "5342": 10914, "5343": 0, "5344": 3838, "5345": 1, "5346": 71558, "5347": 16134, "5348": 5454, "5349": 0, "5350": 46971, "5351": 390, "5352": 51603, "5353": 173558, "5354": 20973, "5355": 3517, "5356": 2452, "5357": 33182, "5358": 2507, "5359": 6807, "5360": 13815, "5361": 3219, "5362": 9531, "5363": 10869, "5364": 2752, "5365": 55859, "5366": 18597, "5367": 23771, "5368": 19648, "5369": 24508, "5370": 25125, "5371": 56795, "5372": 224415, "5373": 15522, "5374": 90, "5375": 159764, "5376": 39376, "5377": 24582, "5378": 131944, "5379": 51801, "5380": 19130, "5381": 16756, "5382": 206995, "5383": 56615, "5384": 45, "5385": 16932, "5386": 9469, "5387": 124670, "5388": 17222, "5389": 19016, "5390": 129476, "5391": 58380, "5392": 33174, "5393": 1916, "5394": 28889, "5395": 49555, "5396": 18974, "5397": 37377, "5398": 2532, "5399": 67910, "5400": 30, "5401": 93998, "5402": 24040, "5403": 101780, "5404": 103, "5405": 30594, "5406": 95158, "5407": 116314, "5408": 163212, "5409": 100006, "5410": 81130, "5411": 22645, "5412": 17, "5413": 19237, "5414": 3522, "5415": 0, "5416": 2558, "5417": 1340, "5418": 42993, "5419": 103602, "5420": 208188, "5421": 559, "5422": 21246, "5423": 33564, "5424": 202, "5425": 63346, "5426": 9570, "5427": 86622, "5428": 19139, "5429": 52862, "5430": 60946, "5431": 1629, "5432": 28963, "5433": 9257, "5434": 487, "5435": 11599, "5436": 15942, "5437": 3521, "5438": 525, "5439": 82053, "5440": 17199, "5441": 4809, "5442": 94565, "5443": 86837, "5444": 36127, "5445": 2463, "5446": 11479, "5447": 13, "5448": 43854, "5449": 66350, "5450": 569, "5451": 36031, "5452": 121508, "5453": 14732, "5454": 16, "5455": 173705, "5456": 10621, "5457": 9856, "5458": 20453, "5459": 18542, "5460": 44, "5461": 1559, "5462": 40358, "5463": 93954, "5464": 40060, "5465": 74460, "5466": 16477, "5467": 287, "5468": 1492, "5469": 0, "5470": 2881, "5471": 37052, "5472": 1142, "5473": 17380, "5474": 872, "5475": 59953, "5476": 36275, "5477": 313469, "5478": 164239, "5479": 91071, "5480": 11667, "5481": 27927, "5482": 14035, "5483": 18078, "5484": 35, "5485": 355, "5486": 8991, "5487": 26696, "5488": 12365, "5489": 168229, "5490": 40985, "5491": 7, "5492": 162191, "5493": 7197, "5494": 9308, "5495": 14581, "5496": 79, "5497": 6810, "5498": 0, "5499": 53800, "5500": 33687, "5501": 4195, "5502": 19839, "5503": 13164, "5504": 66913, "5505": 6164, "5506": 22670, "5507": 55, "5508": 56714, "5509": 22746, "5510": 22423, "5511": 109556, "5512": 106887, "5513": 620, "5514": 10572, "5515": 5798, "5516": 12054, "5517": 284, "5518": 42459, "5519": 1158, "5520": 38675, "5521": 0, "5522": 622, "5523": 8090, "5524": 16208, "5525": 29885, "5526": 66147, "5527": 12662, "5528": 7342, "5529": 41572, "5530": 45, "5531": 51509, "5532": 8596, "5533": 9826, "5534": 3732, "5535": 30500, "5536": 118334, "5537": 11390, "5538": 19376, "5539": 12317, "5540": 9598, "5541": 7667, "5542": 21025, "5543": 184512, "5544": 118708, "5545": 8231, "5546": 9520, "5547": 89487, "5548": 15018, "5549": 13371, "5550": 2057, "5551": 22005, "5552": 73120, "5553": 5730, "5554": 9599, "5555": 6715, "5556": 25668, "5557": 6406, "5558": 252412, "5559": 1916, "5560": 14996, "5561": 89494, "5562": 33952, "5563": 74785, "5564": 236113, "5565": 253, "5566": 13082, "5567": 3718, "5568": 249, "5569": 944, "5570": 13298, "5571": 8114, "5572": 438, "5573": 59325, "5574": 37262, "5575": 44328, "5576": 4963, "5577": 198, "5578": 1014, "5579": 36005, "5580": 15151, "5581": 14628, "5582": 65087, "5583": 1684, "5584": 22765, "5585": 116, "5586": 19657, "5587": 421, "5588": 103197, "5589": 53, "5590": 9468, "5591": 602, "5592": 30349, "5593": 167787, "5594": 43833, "5595": 93788, "5596": 18795, "5597": 9721, "5598": 31, "5599": 12701, "5600": 2266, "5601": 0, "5602": 70429, "5603": 104472, "5604": 1, "5605": 37705, "5606": 56603, "5607": 46711, "5608": 14145, "5609": 748, "5610": 52746, "5611": 16702, "5612": 55751, "5613": 12690, "5614": 1992, "5615": 3211, "5616": 111270, "5617": 792, "5618": 24135, "5619": 195759, "5620": 0, "5621": 100498, "5622": 62311, "5623": 54725, "5624": 12966, "5625": 2932, "5626": 34, "5627": 273, "5628": 1426, "5629": 8091, "5630": 1268, "5631": 21258, "5632": 54950, "5633": 72149, "5634": 9959, "5635": 50958, "5636": 38215, "5637": 46796, "5638": 3805, "5639": 221060, "5640": 159979, "5641": 2469, "5642": 19098, "5643": 11221, "5644": 22119, "5645": 58606, "5646": 11303, "5647": 0, "5648": 2131, "5649": 11482, "5650": 806, "5651": 17888, "5652": 108664, "5653": 7563, "5654": 4371, "5655": 7449, "5656": 95080, "5657": 12431, "5658": 190054, "5659": 19190, "5660": 41141, "5661": 96, "5662": 10784, "5663": 29493, "5664": 55930, "5665": 22673, "5666": 73581, "5667": 39376, "5668": 20378, "5669": 2944, "5670": 450, "5671": 5665, "5672": 45580, "5673": 68416, "5674": 61187, "5675": 9808, "5676": 24225, "5677": 18220, "5678": 11322, "5679": 36182, "5680": 788, "5681": 85, "5682": 105856, "5683": 10570, "5684": 31946, "5685": 5560, "5686": 301, "5687": 12735, "5688": 29731, "5689": 99960, "5690": 52664, "5691": 23390, "5692": 45354, "5693": 138854, "5694": 11277, "5695": 8801, "5696": 3002, "5697": 58451, "5698": 8456, "5699": 192286, "5700": 129996, "5701": 34394, "5702": 383, "5703": 49068, "5704": 41481, "5705": 76237, "5706": 3862, "5707": 0, "5708": 649, "5709": 1, "5710": 456, "5711": 129511, "5712": 89631, "5713": 10378, "5714": 11477, "5715": 2560, "5716": 98483, "5717": 96721, "5718": 442226, "5719": 70013, "5720": 18304, "5721": 22933, "5722": 167, "5723": 7648, "5724": 20105, "5725": 402368, "5726": 16225, "5727": 30158, "5728": 48983, "5729": 11886, "5730": 222768, "5731": 9959, "5732": 4025, "5733": 44130, "5734": 1284, "5735": 70346, "5736": 5285, "5737": 116, "5738": 46457, "5739": 2691, "5740": 49026, "5741": 2061, "5742": 0, "5743": 15957, "5744": 21456, "5745": 26466, "5746": 1904, "5747": 172048, "5748": 3118, "5749": 1803, "5750": 1437, "5751": 13388, "5752": 92, "5753": 211851, "5754": 11525, "5755": 47866, "5756": 16132, "5757": 5124, "5758": 73468, "5759": 11906, "5760": 7704, "5761": 1523, "5762": 1, "5763": 51306, "5764": 18905, "5765": 44557, "5766": 63, "5767": 15994, "5768": 19664, "5769": 327062, "5770": 60274, "5771": 32691, "5772": 2497, "5773": 2960, "5774": 4819, "5775": 27259, "5776": 11711, "5777": 84132, "5778": 1984, "5779": 516, "5780": 3688, "5781": 142951, "5782": 4262, "5783": 5646, "5784": 1075, "5785": 21707, "5786": 108824, "5787": 17013, "5788": 104912, "5789": 6483, "5790": 47268, "5791": 17102, "5792": 547, "5793": 42041, "5794": 71427, "5795": 41964, "5796": 124057, "5797": 103, "5798": 69057, "5799": 27357, "5800": 10398, "5801": 10739, "5802": 9796, "5803": 36889, "5804": 439, "5805": 22612, "5806": 33427, "5807": 2908, "5808": 332670, "5809": 16952, "5810": 9401, "5811": 53020, "5812": 8314, "5813": 13, "5814": 34615, "5815": 204291, "5816": 23168, "5817": 5601, "5818": 171, "5819": 13431, "5820": 86070, "5821": 6621, "5822": 10926, "5823": 3873, "5824": 141598, "5825": 17220, "5826": 5710, "5827": 21433, "5828": 49434, "5829": 2377, "5830": 12219, "5831": 2, "5832": 2630, "5833": 11840, "5834": 4610, "5835": 382, "5836": 9552, "5837": 36565, "5838": 6424, "5839": 3542, "5840": 58972, "5841": 25199, "5842": 303259, "5843": 65466, "5844": 41261, "5845": 2043, "5846": 7139, "5847": 27377, "5848": 2849, "5849": 4130, "5850": 0, "5851": 1960, "5852": 16976, "5853": 19382, "5854": 38, "5855": 45826, "5856": 92634, "5857": 232, "5858": 545, "5859": 47786, "5860": 94515, "5861": 163263, "5862": 25705, "5863": 2, "5864": 5, "5865": 0, "5866": 559, "5867": 18091, "5868": 1, "5869": 66202, "5870": 60812, "5871": 8595, "5872": 30259, "5873": 380, "5874": 2508, "5875": 6051, "5876": 29790, "5877": 148, "5878": 35845, "5879": 15099, "5880": 13887, "5881": 3938, "5882": 5747, "5883": 56205, "5884": 32269, "5885": 29014, "5886": 2899, "5887": 1048, "5888": 28776, "5889": 25, "5890": 7234, "5891": 30746, "5892": 5816, "5893": 43817, "5894": 536, "5895": 17530, "5896": 4772, "5897": 19133, "5898": 43260, "5899": 28360, "5900": 165743, "5901": 0, "5902": 1153, "5903": 117515, "5904": 251, "5905": 11844, "5906": 108863, "5907": 1091, "5908": 881, "5909": 13798, "5910": 34040, "5911": 3653, "5912": 26375, "5913": 30923, "5914": 3337, "5915": 13847, "5916": 310238, "5917": 33115, "5918": 143897, "5919": 252, "5920": 8815, "5921": 37395, "5922": 154755, "5923": 103963, "5924": 15308, "5925": 1390, "5926": 776, "5927": 397, "5928": 15748, "5929": 789, "5930": 12004, "5931": 1133, "5932": 15415, "5933": 191, "5934": 435, "5935": 22982, "5936": 13131, "5937": 47902, "5938": 10388, "5939": 3012, "5940": 4914, "5941": 20309, "5942": 1486, "5943": 20976, "5944": 715, "5945": 12896, "5946": 4711, "5947": 32636, "5948": 207639, "5949": 1676, "5950": 484, "5951": 171512, "5952": 60064, "5953": 84227, "5954": 3822, "5955": 328, "5956": 14114, "5957": 1128, "5958": 43072, "5959": 197579, "5960": 9724, "5961": 67984, "5962": 3910, "5963": 47910, "5964": 46656, "5965": 77907, "5966": 39331, "5967": 16947, "5968": 91424, "5969": 652, "5970": 24281, "5971": 35898, "5972": 39104, "5973": 16172, "5974": 71415, "5975": 2, "5976": 3810, "5977": 76778, "5978": 136760, "5979": 344, "5980": 3237, "5981": 4681, "5982": 5965, "5983": 8, "5984": 89034, "5985": 4238, "5986": 4561, "5987": 277, "5988": 38138, "5989": 117788, "5990": 58482, "5991": 10480, "5992": 126616, "5993": 33389, "5994": 32413, "5995": 5564, "5996": 42539, "5997": 48710, "5998": 62114, "5999": 94, "6000": 5191, "6001": 25685, "6002": 99538, "6003": 672, "6004": 109885, "6005": 5773, "6006": 45015, "6007": 2058, "6008": 82643, "6009": 52472, "6010": 6018, "6011": 5070, "6012": 69860, "6013": 52, "6014": 45568, "6015": 1, "6016": 50272, "6017": 5703, "6018": 172374, "6019": 88, "6020": 515, "6021": 794, "6022": 20729, "6023": 21085, "6024": 92905, "6025": 121007, "6026": 110809, "6027": 7, "6028": 114596, "6029": 1136, "6030": 22720, "6031": 2066, "6032": 2525, "6033": 765, "6034": 1386, "6035": 78220, "6036": 3702, "6037": 4607, "6038": 8062, "6039": 7617, "6040": 6559, "6041": 21, "6042": 13095, "6043": 79497, "6044": 24378, "6045": 91658, "6046": 51, "6047": 32039, "6048": 28624, "6049": 0, "6050": 64227, "6051": 6000, "6052": 55558, "6053": 11702, "6054": 1098, "6055": 23775, "6056": 31144, "6057": 25960, "6058": 139689, "6059": 148, "6060": 50448, "6061": 112740, "6062": 38631, "6063": 35726, "6064": 17624, "6065": 71387, "6066": 4048, "6067": 27148, "6068": 138390, "6069": 6, "6070": 7297, "6071": 185489, "6072": 276103, "6073": 26889, "6074": 37824, "6075": 15943, "6076": 49907, "6077": 100882, "6078": 31889, "6079": 1116, "6080": 2311, "6081": 15071, "6082": 3467, "6083": 198551, "6084": 151020, "6085": 88343, "6086": 75245, "6087": 37162, "6088": 41648, "6089": 189948, "6090": 76236, "6091": 19088, "6092": 40270, "6093": 47921, "6094": 28665, "6095": 3976, "6096": 10240, "6097": 12570, "6098": 54049, "6099": 18, "6100": 10687, "6101": 37957, "6102": 4052, "6103": 47753, "6104": 8634, "6105": 9115, "6106": 68316, "6107": 49657, "6108": 17192, "6109": 45806, "6110": 110129, "6111": 2700, "6112": 62196, "6113": 32863, "6114": 570, "6115": 48154, "6116": 11552, "6117": 33280, "6118": 2547, "6119": 11772, "6120": 106088, "6121": 13718, "6122": 326, "6123": 12187, "6124": 39389, "6125": 31391, "6126": 12282, "6127": 33660, "6128": 2951, "6129": 42380, "6130": 152282, "6131": 10704, "6132": 60564, "6133": 15, "6134": 144182, "6135": 57, "6136": 486243, "6137": 1645, "6138": 70164, "6139": 68925, "6140": 32996, "6141": 10595, "6142": 33834, "6143": 591, "6144": 3339, "6145": 244832, "6146": 2115, "6147": 1878, "6148": 23995, "6149": 9583, "6150": 2256, "6151": 71832, "6152": 127785, "6153": 304, "6154": 3548, "6155": 3286, "6156": 11445, "6157": 13703, "6158": 11122, "6159": 1928, "6160": 17195, "6161": 22136, "6162": 15574, "6163": 15969, "6164": 27215, "6165": 29448, "6166": 17292, "6167": 267498, "6168": 170, "6169": 82275, "6170": 1180, "6171": 69431, "6172": 37158, "6173": 78090, "6174": 33548, "6175": 83831, "6176": 3700, "6177": 3868, "6178": 8610, "6179": 3426, "6180": 19, "6181": 4, "6182": 59565, "6183": 30191, "6184": 49889, "6185": 3005, "6186": 14612, "6187": 19918, "6188": 121091, "6189": 0, "6190": 58170, "6191": 27283, "6192": 3112, "6193": 26119, "6194": 20231, "6195": 83569, "6196": 53101, "6197": 3308, "6198": 26358, "6199": 144572, "6200": 2973, "6201": 28, "6202": 16314, "6203": 12007, "6204": 11731, "6205": 11591, "6206": 11316, "6207": 10705, "6208": 28169, "6209": 33589, "6210": 88290, "6211": 397, "6212": 104364, "6213": 19526, "6214": 101313, "6215": 40365, "6216": 5862, "6217": 45633, "6218": 10707, "6219": 84411, "6220": 93687, "6221": 72443, "6222": 1934, "6223": 0, "6224": 11970, "6225": 3951, "6226": 631, "6227": 50234, "6228": 85129, "6229": 44843, "6230": 45420, "6231": 14345, "6232": 2014, "6233": 38051, "6234": 102866, "6235": 85822, "6236": 35, "6237": 14118, "6238": 24218, "6239": 112031, "6240": 94641, "6241": 25950, "6242": 11271, "6243": 3704, "6244": 224329, "6245": 62098, "6246": 12341, "6247": 111876, "6248": 153711, "6249": 26971, "6250": 0, "6251": 22026, "6252": 291, "6253": 51426, "6254": 18169, "6255": 2, "6256": 14706, "6257": 72860, "6258": 162, "6259": 0, "6260": 19203, "6261": 31115, "6262": 76668, "6263": 22689, "6264": 27387, "6265": 6214, "6266": 56188, "6267": 120927, "6268": 88705, "6269": 94753, "6270": 1646, "6271": 42153, "6272": 8401, "6273": 11164, "6274": 14056, "6275": 23630, "6276": 14326, "6277": 96871, "6278": 60, "6279": 7404, "6280": 9, "6281": 43976, "6282": 46919, "6283": 181, "6284": 59329, "6285": 1419, "6286": 8064, "6287": 141851, "6288": 18978, "6289": 65213, "6290": 213509, "6291": 27183, "6292": 1616, "6293": 57759, "6294": 208714, "6295": 15563, "6296": 30183, "6297": 6102, "6298": 112, "6299": 2017, "6300": 53197, "6301": 581, "6302": 12214, "6303": 17603, "6304": 26708, "6305": 40250, "6306": 8643, "6307": 35206, "6308": 550, "6309": 60744, "6310": 156864, "6311": 180, "6312": 23066, "6313": 2075, "6314": 69, "6315": 29778, "6316": 71680, "6317": 100829, "6318": 281477, "6319": 6093, "6320": 27221, "6321": 99643, "6322": 18568, "6323": 20179, "6324": 27909, "6325": 64199, "6326": 15459, "6327": 7, "6328": 26690, "6329": 307158, "6330": 20554, "6331": 17971, "6332": 68804, "6333": 35902, "6334": 99479, "6335": 4994, "6336": 2313, "6337": 530, "6338": 70714, "6339": 14484, "6340": 12302, "6341": 390, "6342": 97984, "6343": 8615, "6344": 81316, "6345": 137112, "6346": 207214, "6347": 6467, "6348": 23641, "6349": 18755, "6350": 37170, "6351": 68718, "6352": 11577, "6353": 14391, "6354": 22813, "6355": 896, "6356": 10791, "6357": 0, "6358": 128351, "6359": 631, "6360": 165770, "6361": 0, "6362": 1830, "6363": 19712, "6364": 35899, "6365": 12831, "6366": 1283, "6367": 6303, "6368": 64474, "6369": 1211, "6370": 156042, "6371": 229938, "6372": 937, "6373": 28443, "6374": 111355, "6375": 95248, "6376": 12801, "6377": 43587, "6378": 99813, "6379": 40755, "6380": 11751, "6381": 420, "6382": 1812, "6383": 10956, "6384": 147822, "6385": 1024, "6386": 8863, "6387": 3056, "6388": 5072, "6389": 11054, "6390": 355, "6391": 123961, "6392": 32530, "6393": 108562, "6394": 31780, "6395": 4493, "6396": 39968, "6397": 1638, "6398": 172, "6399": 34795, "6400": 18068, "6401": 15524, "6402": 66077, "6403": 72193, "6404": 16625, "6405": 1366, "6406": 1040, "6407": 307, "6408": 162872, "6409": 166494, "6410": 3261, "6411": 102975, "6412": 3732, "6413": 87774, "6414": 12710, "6415": 12719, "6416": 26294, "6417": 8219, "6418": 89988, "6419": 41342, "6420": 438, "6421": 611, "6422": 658, "6423": 11233, "6424": 50359, "6425": 57472, "6426": 11553, "6427": 1468, "6428": 160251, "6429": 37816, "6430": 56399, "6431": 15469, "6432": 418, "6433": 781, "6434": 18501, "6435": 31325, "6436": 1216, "6437": 775, "6438": 45370, "6439": 4459, "6440": 22038, "6441": 76509, "6442": 10, "6443": 28786, "6444": 21621, "6445": 56520, "6446": 653, "6447": 4505, "6448": 110039, "6449": 16165, "6450": 14508, "6451": 89390, "6452": 982, "6453": 65403, "6454": 6455, "6455": 22852, "6456": 4081, "6457": 4051, "6458": 421, "6459": 2638, "6460": 33471, "6461": 42972, "6462": 25258, "6463": 843, "6464": 56055, "6465": 25958, "6466": 2811, "6467": 28625, "6468": 105282, "6469": 60182, "6470": 22113, "6471": 9285, "6472": 3728, "6473": 68626, "6474": 97381, "6475": 3616, "6476": 2270, "6477": 4688, "6478": 6969, "6479": 41323, "6480": 0, "6481": 71283, "6482": 5, "6483": 10698, "6484": 58308, "6485": 6959, "6486": 58290, "6487": 22417, "6488": 168818, "6489": 57366, "6490": 11702, "6491": 113, "6492": 15319, "6493": 1183, "6494": 543, "6495": 73895, "6496": 39953, "6497": 2878, "6498": 2, "6499": 5612, "6500": 95702, "6501": 14652, "6502": 74013, "6503": 18242, "6504": 0, "6505": 2018, "6506": 232938, "6507": 180199, "6508": 92165, "6509": 116935, "6510": 59428, "6511": 0, "6512": 65604, "6513": 2, "6514": 51999, "6515": 11869, "6516": 32268, "6517": 2478, "6518": 90535, "6519": 433, "6520": 2036, "6521": 4087, "6522": 6427, "6523": 66029, "6524": 48640, "6525": 4208, "6526": 13020, "6527": 18570, "6528": 31944, "6529": 34823, "6530": 3, "6531": 6711, "6532": 1612, "6533": 285123, "6534": 50819, "6535": 36235, "6536": 1178, "6537": 41964, "6538": 69782, "6539": 37813, "6540": 58239, "6541": 94170, "6542": 30446, "6543": 29174, "6544": 15895, "6545": 78754, "6546": 12369, "6547": 29234, "6548": 47, "6549": 0, "6550": 47301, "6551": 13436, "6552": 18018, "6553": 2808, "6554": 27147, "6555": 87637, "6556": 657, "6557": 3, "6558": 272375, "6559": 3472, "6560": 21794, "6561": 5734, "6562": 131, "6563": 31, "6564": 97, "6565": 20468, "6566": 31849, "6567": 9747, "6568": 7795, "6569": 6284, "6570": 49317, "6571": 14651, "6572": 72761, "6573": 23205, "6574": 11146, "6575": 19348, "6576": 78944, "6577": 25587, "6578": 73997, "6579": 48207, "6580": 5536, "6581": 11097, "6582": 150017, "6583": 48396, "6584": 79038, "6585": 331, "6586": 1352, "6587": 3582, "6588": 1603, "6589": 61573, "6590": 36900, "6591": 17, "6592": 2327, "6593": 128613, "6594": 53379, "6595": 10257, "6596": 17347, "6597": 120353, "6598": 17677, "6599": 32080, "6600": 68270, "6601": 123738, "6602": 55, "6603": 624, "6604": 1486, "6605": 671, "6606": 55994, "6607": 114386, "6608": 4215, "6609": 8253, "6610": 0, "6611": 11044, "6612": 14246, "6613": 26700, "6614": 139364, "6615": 56275, "6616": 126813, "6617": 35896, "6618": 2033, "6619": 16252, "6620": 4951, "6621": 2041, "6622": 25952, "6623": 24072, "6624": 75480, "6625": 110624, "6626": 80883, "6627": 790, "6628": 1585, "6629": 1095, "6630": 1871, "6631": 16203, "6632": 13143, "6633": 2461, "6634": 51, "6635": 4766, "6636": 40981, "6637": 315, "6638": 10237, "6639": 91, "6640": 286, "6641": 47522, "6642": 8913, "6643": 221890, "6644": 7002, "6645": 276269, "6646": 82988, "6647": 3726, "6648": 5343, "6649": 8344, "6650": 16980, "6651": 4736, "6652": 73318, "6653": 1186, "6654": 1625, "6655": 141768, "6656": 46094, "6657": 54149, "6658": 31074, "6659": 154202, "6660": 47973, "6661": 9035, "6662": 80246, "6663": 979, "6664": 207609, "6665": 26633, "6666": 54479, "6667": 3590, "6668": 14626, "6669": 18916, "6670": 1791, "6671": 116, "6672": 101169, "6673": 121, "6674": 4914, "6675": 5259, "6676": 135605, "6677": 27935, "6678": 49690, "6679": 189137, "6680": 2105, "6681": 155683, "6682": 1158, "6683": 126215, "6684": 24007, "6685": 26644, "6686": 42405, "6687": 28007, "6688": 30143, "6689": 105366, "6690": 2557, "6691": 100530, "6692": 1008, "6693": 2, "6694": 1, "6695": 39180, "6696": 68237, "6697": 12, "6698": 54580, "6699": 5584, "6700": 39, "6701": 0, "6702": 62973, "6703": 48581, "6704": 19453, "6705": 140696, "6706": 5466, "6707": 36746, "6708": 5343, "6709": 43903, "6710": 56890, "6711": 2712, "6712": 11717, "6713": 268, "6714": 49234, "6715": 106367, "6716": 4334, "6717": 210, "6718": 108, "6719": 154725, "6720": 0, "6721": 2060, "6722": 0, "6723": 66536, "6724": 41577, "6725": 1252, "6726": 42, "6727": 154042, "6728": 27323, "6729": 48463, "6730": 680, "6731": 0, "6732": 5, "6733": 32457, "6734": 21042, "6735": 34187, "6736": 3184, "6737": 4076, "6738": 400, "6739": 58722, "6740": 1163, "6741": 27911, "6742": 122480, "6743": 34812, "6744": 15078, "6745": 54011, "6746": 1852, "6747": 1888, "6748": 113640, "6749": 32575, "6750": 10448, "6751": 90642, "6752": 1454, "6753": 3534, "6754": 53148, "6755": 4993, "6756": 104280, "6757": 34174, "6758": 15825, "6759": 37143, "6760": 23397, "6761": 2303, "6762": 24279, "6763": 14730, "6764": 33795, "6765": 14648, "6766": 27368, "6767": 194584, "6768": 2212, "6769": 13064, "6770": 10186, "6771": 54392, "6772": 39311, "6773": 55542, "6774": 604, "6775": 74595, "6776": 61214, "6777": 4331, "6778": 42865, "6779": 10180, "6780": 150109, "6781": 10, "6782": 29349, "6783": 47604, "6784": 53285, "6785": 143667, "6786": 138, "6787": 3415, "6788": 1954, "6789": 10659, "6790": 21453, "6791": 5849, "6792": 5464, "6793": 19806, "6794": 1465, "6795": 32049, "6796": 4123, "6797": 99305, "6798": 14073, "6799": 91543, "6800": 97522, "6801": 60, "6802": 341118, "6803": 13274, "6804": 7441, "6805": 85869, "6806": 14107, "6807": 99136, "6808": 104678, "6809": 113789, "6810": 46326, "6811": 16801, "6812": 3, "6813": 10758, "6814": 174178, "6815": 2955, "6816": 104639, "6817": 196, "6818": 5879, "6819": 27457, "6820": 5813, "6821": 12429, "6822": 11739, "6823": 621, "6824": 860, "6825": 2504, "6826": 118073, "6827": 28434, "6828": 101208, "6829": 43678, "6830": 79456, "6831": 149974, "6832": 125079, "6833": 96, "6834": 121418, "6835": 0, "6836": 0, "6837": 11248, "6838": 61136, "6839": 65629, "6840": 40032, "6841": 29669, "6842": 1, "6843": 752, "6844": 28825, "6845": 95016, "6846": 2186, "6847": 19502, "6848": 23421, "6849": 32901, "6850": 122710, "6851": 15207, "6852": 5174, "6853": 69191, "6854": 33170, "6855": 140, "6856": 74, "6857": 40248, "6858": 0, "6859": 549, "6860": 71671, "6861": 74055, "6862": 22434, "6863": 33668, "6864": 63478, "6865": 45137, "6866": 4769, "6867": 19725, "6868": 45270, "6869": 14, "6870": 59930, "6871": 855, "6872": 19946, "6873": 53255, "6874": 74015, "6875": 6037, "6876": 838, "6877": 98587, "6878": 6054, "6879": 9472, "6880": 108, "6881": 104056, "6882": 36524, "6883": 24077, "6884": 32735, "6885": 4563, "6886": 24329, "6887": 20982, "6888": 2096, "6889": 24061, "6890": 12447, "6891": 15827, "6892": 3171, "6893": 52968, "6894": 7153, "6895": 37801, "6896": 137388, "6897": 2569, "6898": 2443, "6899": 18809, "6900": 7968, "6901": 12531, "6902": 36831, "6903": 29875, "6904": 14574, "6905": 25801, "6906": 2774, "6907": 11309, "6908": 21732, "6909": 77183, "6910": 5564, "6911": 8438, "6912": 203534, "6913": 232311, "6914": 5351, "6915": 3648, "6916": 175, "6917": 14895, "6918": 2275, "6919": 76551, "6920": 77287, "6921": 33361, "6922": 4198, "6923": 9649, "6924": 356, "6925": 21849, "6926": 72207, "6927": 190520, "6928": 150, "6929": 12877, "6930": 138426, "6931": 561, "6932": 243, "6933": 35790, "6934": 8909, "6935": 37287, "6936": 78843, "6937": 57777, "6938": 34156, "6939": 2477, "6940": 38450, "6941": 5073, "6942": 136988, "6943": 47219, "6944": 11531, "6945": 27385, "6946": 18351, "6947": 2013, "6948": 8130, "6949": 5701, "6950": 85352, "6951": 47941, "6952": 94279, "6953": 0, "6954": 2558, "6955": 28079, "6956": 142296, "6957": 23092, "6958": 115253, "6959": 23, "6960": 7836, "6961": 17504, "6962": 86626, "6963": 10374, "6964": 105380, "6965": 4943, "6966": 5668, "6967": 1231, "6968": 196547, "6969": 36, "6970": 23283, "6971": 1285, "6972": 25412, "6973": 68996, "6974": 12760, "6975": 4756, "6976": 134579, "6977": 7048, "6978": 1335, "6979": 65744, "6980": 5313, "6981": 4752, "6982": 9894, "6983": 55996, "6984": 12489, "6985": 11668, "6986": 804995, "6987": 88052, "6988": 57491, "6989": 18084, "6990": 93789, "6991": 11165, "6992": 68625, "6993": 9408, "6994": 15391, "6995": 167017, "6996": 1075, "6997": 37812, "6998": 140392, "6999": 21082, "7000": 23240, "7001": 26100, "7002": 32475, "7003": 24622, "7004": 29622, "7005": 17687, "7006": 23198, "7007": 106300, "7008": 399999, "7009": 4856, "7010": 21, "7011": 16146, "7012": 16176, "7013": 148255, "7014": 39593, "7015": 25668, "7016": 0, "7017": 2927, "7018": 41526, "7019": 834, "7020": 59746, "7021": 45974, "7022": 7257, "7023": 61001, "7024": 76993, "7025": 2762, "7026": 61875, "7027": 3379, "7028": 38207, "7029": 22888, "7030": 77053, "7031": 13500, "7032": 43163, "7033": 26955, "7034": 0, "7035": 72412, "7036": 0, "7037": 1169, "7038": 79773, "7039": 52009, "7040": 12607, "7041": 44694, "7042": 23759, "7043": 11768, "7044": 50731, "7045": 76432, "7046": 83383, "7047": 69621, "7048": 47492, "7049": 3, "7050": 280, "7051": 57130, "7052": 49993, "7053": 11922, "7054": 12933, "7055": 37729, "7056": 20918, "7057": 20164, "7058": 724, "7059": 903, "7060": 2, "7061": 31579, "7062": 3621, "7063": 4389, "7064": 3550, "7065": 28964, "7066": 55685, "7067": 29462, "7068": 729, "7069": 109, "7070": 82618, "7071": 64399, "7072": 11253, "7073": 10795, "7074": 33002, "7075": 0, "7076": 60838, "7077": 14351, "7078": 52870, "7079": 108590, "7080": 20648, "7081": 97, "7082": 4277, "7083": 7, "7084": 8714, "7085": 12076, "7086": 50115, "7087": 133255, "7088": 47746, "7089": 72975, "7090": 12557, "7091": 59735, "7092": 5419, "7093": 0, "7094": 17364, "7095": 16136, "7096": 1563, "7097": 210948, "7098": 35489, "7099": 2761, "7100": 3979, "7101": 32761, "7102": 98350, "7103": 27672, "7104": 16679, "7105": 1994, "7106": 22210, "7107": 10020, "7108": 89405, "7109": 38190, "7110": 62070, "7111": 30580, "7112": 52993, "7113": 9683, "7114": 39538, "7115": 564, "7116": 5569, "7117": 172962, "7118": 2265, "7119": 166758, "7120": 28080, "7121": 588, "7122": 178944, "7123": 12702, "7124": 0, "7125": 36660, "7126": 190146, "7127": 570567, "7128": 96957, "7129": 6877, "7130": 62624, "7131": 872, "7132": 3258, "7133": 147561, "7134": 31514, "7135": 56500, "7136": 4364, "7137": 3089, "7138": 81795, "7139": 14628, "7140": 5493, "7141": 13648, "7142": 135, "7143": 42792, "7144": 19099, "7145": 4408, "7146": 33987, "7147": 3880, "7148": 104804, "7149": 23071, "7150": 63101, "7151": 81794, "7152": 366, "7153": 20022, "7154": 19538, "7155": 331, "7156": 37347, "7157": 57423, "7158": 146708, "7159": 45206, "7160": 16414, "7161": 8403, "7162": 24229, "7163": 6980, "7164": 0, "7165": 29884, "7166": 7799, "7167": 34009, "7168": 22551, "7169": 84597, "7170": 239, "7171": 181531, "7172": 115853, "7173": 39984, "7174": 17281, "7175": 5737, "7176": 20540, "7177": 63416, "7178": 42389, "7179": 28964, "7180": 130403, "7181": 471, "7182": 15868, "7183": 2610, "7184": 53240, "7185": 17633, "7186": 61484, "7187": 4077, "7188": 0, "7189": 178881, "7190": 33071, "7191": 15055, "7192": 3628, "7193": 205, "7194": 1187, "7195": 39277, "7196": 40253, "7197": 107781, "7198": 4, "7199": 4274, "7200": 190262, "7201": 59498, "7202": 4716, "7203": 1771, "7204": 26421, "7205": 52526, "7206": 15231, "7207": 126580, "7208": 12221, "7209": 35037, "7210": 863, "7211": 30022, "7212": 11350, "7213": 869, "7214": 14386, "7215": 98112, "7216": 21502, "7217": 294720, "7218": 476, "7219": 274179, "7220": 3061, "7221": 14627, "7222": 1891, "7223": 32005, "7224": 157442, "7225": 17126, "7226": 4812, "7227": 4901, "7228": 176779, "7229": 139254, "7230": 94004, "7231": 21669, "7232": 238, "7233": 1448, "7234": 131768, "7235": 7906, "7236": 22642, "7237": 315, "7238": 65601, "7239": 47353, "7240": 79089, "7241": 156671, "7242": 645, "7243": 11759, "7244": 257680, "7245": 5835, "7246": 2375, "7247": 4811, "7248": 12, "7249": 10718, "7250": 28231, "7251": 68314, "7252": 107639, "7253": 18194, "7254": 111929, "7255": 56, "7256": 1796, "7257": 152507, "7258": 18366, "7259": 226551, "7260": 86601, "7261": 50903, "7262": 95716, "7263": 169959, "7264": 48160, "7265": 32939, "7266": 14745, "7267": 15792, "7268": 966, "7269": 95, "7270": 6858, "7271": 2260, "7272": 3046, "7273": 168938, "7274": 40092, "7275": 67867, "7276": 83682, "7277": 49359, "7278": 74176, "7279": 9809, "7280": 65905, "7281": 2705, "7282": 8643, "7283": 191, "7284": 43532, "7285": 18654, "7286": 147329, "7287": 25821, "7288": 11437, "7289": 42152, "7290": 11449, "7291": 26902, "7292": 50868, "7293": 317, "7294": 95, "7295": 24187, "7296": 68408, "7297": 73802, "7298": 2598, "7299": 52606, "7300": 34468, "7301": 1967, "7302": 98, "7303": 61355, "7304": 3306, "7305": 1943, "7306": 11860, "7307": 7823, "7308": 17775, "7309": 24990, "7310": 12, "7311": 104, "7312": 185116, "7313": 61639, "7314": 51660, "7315": 53610, "7316": 37241, "7317": 267809, "7318": 10281, "7319": 4, "7320": 49307, "7321": 10880, "7322": 2764, "7323": 1686, "7324": 35384, "7325": 76904, "7326": 3861, "7327": 111270, "7328": 64092, "7329": 4196, "7330": 1099, "7331": 39265, "7332": 1036, "7333": 4234, "7334": 553136, "7335": 166, "7336": 4093, "7337": 50334, "7338": 4755, "7339": 5683, "7340": 34029, "7341": 324, "7342": 58100, "7343": 0, "7344": 1660, "7345": 85174, "7346": 22303, "7347": 21361, "7348": 251027, "7349": 17656, "7350": 53994, "7351": 37599, "7352": 138925, "7353": 14229, "7354": 1149, "7355": 15538, "7356": 304, "7357": 83, "7358": 330, "7359": 29229, "7360": 41111, "7361": 2, "7362": 56975, "7363": 21919, "7364": 14800, "7365": 168828, "7366": 13185, "7367": 11700, "7368": 11846, "7369": 59228, "7370": 26988, "7371": 6492, "7372": 9364, "7373": 70058, "7374": 159062, "7375": 61350, "7376": 239902, "7377": 114403, "7378": 330445, "7379": 162, "7380": 283, "7381": 42906, "7382": 30233, "7383": 18624, "7384": 4203, "7385": 5781, "7386": 9216, "7387": 128589, "7388": 14238, "7389": 79593, "7390": 58627, "7391": 11, "7392": 31532, "7393": 28960, "7394": 43889, "7395": 8937, "7396": 0, "7397": 25211, "7398": 11909, "7399": 1621, "7400": 1945, "7401": 0, "7402": 60661, "7403": 185, "7404": 121802, "7405": 9132, "7406": 46684, "7407": 8125, "7408": 31, "7409": 30697, "7410": 31401, "7411": 2, "7412": 2240, "7413": 2150, "7414": 60360, "7415": 155959, "7416": 15686, "7417": 49225, "7418": 311, "7419": 131, "7420": 8323, "7421": 871, "7422": 17024, "7423": 13038, "7424": 11873, "7425": 4764, "7426": 77758, "7427": 19573, "7428": 4590, "7429": 3792, "7430": 8351, "7431": 214, "7432": 26722, "7433": 5782, "7434": 197, "7435": 74013, "7436": 46315, "7437": 4562, "7438": 103351, "7439": 805, "7440": 346739, "7441": 7940, "7442": 26014, "7443": 8564, "7444": 443, "7445": 5750, "7446": 64177, "7447": 8308, "7448": 200504, "7449": 77827, "7450": 48466, "7451": 625, "7452": 24017, "7453": 18044, "7454": 0, "7455": 129499, "7456": 345, "7457": 2665, "7458": 0, "7459": 18717, "7460": 4423, "7461": 29522, "7462": 3917, "7463": 16632, "7464": 635, "7465": 0, "7466": 19115, "7467": 22756, "7468": 37065, "7469": 137, "7470": 55782, "7471": 70879, "7472": 18218, "7473": 33106, "7474": 454, "7475": 7, "7476": 61932, "7477": 122246, "7478": 15343, "7479": 4743, "7480": 34040, "7481": 18849, "7482": 74274, "7483": 7045, "7484": 24971, "7485": 3358, "7486": 5469, "7487": 27713, "7488": 129898, "7489": 0, "7490": 64225, "7491": 60300, "7492": 52653, "7493": 562, "7494": 365222, "7495": 174480, "7496": 146, "7497": 214, "7498": 1160, "7499": 30298, "7500": 2337, "7501": 1792, "7502": 95727, "7503": 12881, "7504": 17452, "7505": 9, "7506": 58, "7507": 34160, "7508": 7679, "7509": 3351, "7510": 8927, "7511": 545457, "7512": 38931, "7513": 6076, "7514": 12382, "7515": 110532, "7516": 68507, "7517": 7717, "7518": 40940, "7519": 10198, "7520": 22118, "7521": 146512, "7522": 6312, "7523": 100553, "7524": 101662, "7525": 36567, "7526": 30675, "7527": 210773, "7528": 231, "7529": 20384, "7530": 19003, "7531": 3480, "7532": 23255, "7533": 10290, "7534": 7737, "7535": 5905, "7536": 6172, "7537": 0, "7538": 36126, "7539": 194965, "7540": 1515, "7541": 31987, "7542": 37375, "7543": 10974, "7544": 12045, "7545": 1574, "7546": 406, "7547": 68304, "7548": 898, "7549": 52289, "7550": 5596, "7551": 5807, "7552": 47880, "7553": 24086, "7554": 6211, "7555": 194, "7556": 2357, "7557": 68260, "7558": 2581, "7559": 55097, "7560": 1043, "7561": 97679, "7562": 589, "7563": 73643, "7564": 8944, "7565": 79985, "7566": 730, "7567": 93669, "7568": 123925, "7569": 981, "7570": 10698, "7571": 4905, "7572": 81102, "7573": 18428, "7574": 9875, "7575": 221, "7576": 14111, "7577": 113294, "7578": 15841, "7579": 125780, "7580": 90432, "7581": 879, "7582": 45046, "7583": 31500, "7584": 67517, "7585": 22956, "7586": 89618, "7587": 3401, "7588": 5733, "7589": 1891, "7590": 11919, "7591": 25708, "7592": 119574, "7593": 24938, "7594": 17, "7595": 17138, "7596": 8403, "7597": 1011, "7598": 15068, "7599": 6125, "7600": 1650, "7601": 4441, "7602": 17877, "7603": 2970, "7604": 209, "7605": 64155, "7606": 55546, "7607": 34557, "7608": 3218, "7609": 4225, "7610": 1333, "7611": 17841, "7612": 743, "7613": 9, "7614": 338, "7615": 267813, "7616": 22205, "7617": 4201, "7618": 60430, "7619": 8018, "7620": 12163, "7621": 11745, "7622": 115835, "7623": 863, "7624": 20504, "7625": 18560, "7626": 74050, "7627": 502, "7628": 39, "7629": 3941, "7630": 54111, "7631": 118817, "7632": 69380, "7633": 314970, "7634": 34875, "7635": 9791, "7636": 433, "7637": 54615, "7638": 0, "7639": 15245, "7640": 6637, "7641": 69553, "7642": 7479, "7643": 4355, "7644": 2979, "7645": 90869, "7646": 31413, "7647": 7083, "7648": 19, "7649": 98778, "7650": 16839, "7651": 100555, "7652": 8828, "7653": 20923, "7654": 141, "7655": 0, "7656": 16251, "7657": 6282, "7658": 50584, "7659": 1351, "7660": 83894, "7661": 13073, "7662": 37101, "7663": 891, "7664": 8, "7665": 1260, "7666": 1292, "7667": 8745, "7668": 26098, "7669": 7242, "7670": 1674, "7671": 35482, "7672": 34981, "7673": 430062, "7674": 4257, "7675": 3435, "7676": 27266, "7677": 67, "7678": 2019, "7679": 75427, "7680": 4528, "7681": 97041, "7682": 147410, "7683": 108902, "7684": 4098, "7685": 51263, "7686": 40016, "7687": 64235, "7688": 515, "7689": 0, "7690": 34585, "7691": 3384, "7692": 25965, "7693": 2304, "7694": 17847, "7695": 8081, "7696": 4429, "7697": 3518, "7698": 37015, "7699": 32088, "7700": 18244, "7701": 43585, "7702": 385, "7703": 4894, "7704": 16439, "7705": 5188, "7706": 121259, "7707": 1740, "7708": 19256, "7709": 29030, "7710": 153, "7711": 555, "7712": 71820, "7713": 95859, "7714": 119168, "7715": 3867, "7716": 0, "7717": 14519, "7718": 84464, "7719": 218, "7720": 23, "7721": 31039, "7722": 1390, "7723": 32214, "7724": 432, "7725": 76035, "7726": 36301, "7727": 66373, "7728": 3186, "7729": 449, "7730": 17173, "7731": 25160, "7732": 33539, "7733": 3909, "7734": 46725, "7735": 84800, "7736": 4209, "7737": 134557, "7738": 55609, "7739": 33061, "7740": 193, "7741": 15326, "7742": 4951, "7743": 21970, "7744": 11247, "7745": 536, "7746": 22114, "7747": 54136, "7748": 18580, "7749": 80915, "7750": 19037, "7751": 61458, "7752": 20558, "7753": 124501, "7754": 51858, "7755": 3809, "7756": 14567, "7757": 169848, "7758": 9884, "7759": 170862, "7760": 55731, "7761": 60367, "7762": 15286, "7763": 50031, "7764": 69303, "7765": 73426, "7766": 6852, "7767": 7185, "7768": 18, "7769": 4611, "7770": 23, "7771": 794, "7772": 1144, "7773": 2205, "7774": 115846, "7775": 3876, "7776": 209, "7777": 9158, "7778": 46834, "7779": 2057, "7780": 0, "7781": 12989, "7782": 35973, "7783": 104648, "7784": 6, "7785": 213166, "7786": 292049, "7787": 1029, "7788": 33575, "7789": 6898, "7790": 1003, "7791": 2176, "7792": 182, "7793": 35520, "7794": 2905, "7795": 63578, "7796": 29052, "7797": 6505, "7798": 7049, "7799": 51378, "7800": 462, "7801": 3821, "7802": 64802, "7803": 2022, "7804": 54372, "7805": 6993, "7806": 17505, "7807": 45529, "7808": 329, "7809": 19385, "7810": 5436, "7811": 51943, "7812": 3248, "7813": 8014, "7814": 67255, "7815": 10937, "7816": 2980, "7817": 31820, "7818": 5820, "7819": 4888, "7820": 173232, "7821": 5607, "7822": 9775, "7823": 8464, "7824": 25235, "7825": 30649, "7826": 3071, "7827": 0, "7828": 4826, "7829": 23181, "7830": 71440, "7831": 4773, "7832": 218236, "7833": 4898, "7834": 36489, "7835": 1699, "7836": 171576, "7837": 47241, "7838": 12, "7839": 8024, "7840": 75863, "7841": 9941, "7842": 52381, "7843": 59699, "7844": 4633, "7845": 85673, "7846": 7, "7847": 14985, "7848": 296660, "7849": 99184, "7850": 117312, "7851": 24253, "7852": 24294, "7853": 163545, "7854": 56164, "7855": 14579, "7856": 78679, "7857": 69576, "7858": 0, "7859": 28249, "7860": 20841, "7861": 900, "7862": 88, "7863": 3, "7864": 2550, "7865": 2677, "7866": 9977, "7867": 1721, "7868": 3059, "7869": 6965, "7870": 0, "7871": 87760, "7872": 57812, "7873": 70363, "7874": 14186, "7875": 340, "7876": 77409, "7877": 112663, "7878": 79979, "7879": 25212, "7880": 34605, "7881": 27946, "7882": 4355, "7883": 1753, "7884": 7, "7885": 60049, "7886": 69791, "7887": 64911, "7888": 37350, "7889": 48391, "7890": 1, "7891": 4610, "7892": 155587, "7893": 49871, "7894": 91295, "7895": 11390, "7896": 46140, "7897": 84305, "7898": 129032, "7899": 7850, "7900": 27356, "7901": 21907, "7902": 16143, "7903": 1684, "7904": 15251, "7905": 183, "7906": 4, "7907": 45383, "7908": 76113, "7909": 3414, "7910": 14198, "7911": 25176, "7912": 66675, "7913": 48038, "7914": 1880, "7915": 16367, "7916": 2081, "7917": 618449, "7918": 25342, "7919": 948, "7920": 66529, "7921": 375, "7922": 40624, "7923": 76968, "7924": 39979, "7925": 1539, "7926": 13978, "7927": 165, "7928": 2593, "7929": 2204, "7930": 6547, "7931": 206, "7932": 48699, "7933": 242412, "7934": 119656, "7935": 186856, "7936": 38370, "7937": 20234, "7938": 27061, "7939": 36598, "7940": 77250, "7941": 4286, "7942": 109742, "7943": 17212, "7944": 5379, "7945": 629, "7946": 52590, "7947": 37487, "7948": 17369, "7949": 5840, "7950": 30339, "7951": 689, "7952": 64976, "7953": 35545, "7954": 254, "7955": 17592, "7956": 2218, "7957": 12139, "7958": 540, "7959": 100169, "7960": 330, "7961": 37442, "7962": 237996, "7963": 42570, "7964": 839802, "7965": 573, "7966": 128721, "7967": 7293, "7968": 627, "7969": 17131, "7970": 4494, "7971": 2701, "7972": 3042, "7973": 30214, "7974": 565, "7975": 263852, "7976": 23937, "7977": 0, "7978": 3762, "7979": 16499, "7980": 26015, "7981": 18530, "7982": 12664, "7983": 68462, "7984": 1493, "7985": 76720, "7986": 35530, "7987": 44, "7988": 9, "7989": 183022, "7990": 13625, "7991": 880, "7992": 47109, "7993": 114, "7994": 1479, "7995": 20620, "7996": 23689, "7997": 54391, "7998": 37316, "7999": 70461, "8000": 162475, "8001": 84552, "8002": 2690, "8003": 88808, "8004": 19806, "8005": 7305, "8006": 145, "8007": 8361, "8008": 98, "8009": 13452, "8010": 52933, "8011": 28998, "8012": 0, "8013": 36756, "8014": 29689, "8015": 10161, "8016": 36814, "8017": 107053, "8018": 0, "8019": 68, "8020": 13133, "8021": 188, "8022": 3453, "8023": 82797, "8024": 26070, "8025": 34844, "8026": 61072, "8027": 2022, "8028": 1564, "8029": 19413, "8030": 247261, "8031": 71188, "8032": 17350, "8033": 107140, "8034": 5300, "8035": 54076, "8036": 62330, "8037": 5700, "8038": 17085, "8039": 30409, "8040": 190142, "8041": 709, "8042": 8919, "8043": 886, "8044": 6975, "8045": 1106, "8046": 36, "8047": 1006, "8048": 977, "8049": 110985, "8050": 14084, "8051": 9, "8052": 44, "8053": 47964, "8054": 19592, "8055": 126407, "8056": 24183, "8057": 547, "8058": 2174, "8059": 80, "8060": 2259, "8061": 23321, "8062": 246835, "8063": 142376, "8064": 28989, "8065": 80009, "8066": 102, "8067": 69486, "8068": 28124, "8069": 5513, "8070": 2675, "8071": 77605, "8072": 5, "8073": 184073, "8074": 14188, "8075": 60319, "8076": 10336, "8077": 49971, "8078": 135529, "8079": 85767, "8080": 64908, "8081": 137897, "8082": 14991, "8083": 121492, "8084": 915, "8085": 32372, "8086": 41550, "8087": 115804, "8088": 10143, "8089": 69689, "8090": 0, "8091": 16912, "8092": 4338, "8093": 175279, "8094": 82113, "8095": 0, "8096": 1883, "8097": 172, "8098": 1404, "8099": 93055, "8100": 6490, "8101": 180316, "8102": 8604, "8103": 68279, "8104": 3135, "8105": 74468, "8106": 22123, "8107": 35143, "8108": 86881, "8109": 13393, "8110": 22348, "8111": 45086, "8112": 23700, "8113": 13406, "8114": 54833, "8115": 20481, "8116": 20818, "8117": 22419, "8118": 31671, "8119": 58560, "8120": 15398, "8121": 39176, "8122": 21922, "8123": 55028, "8124": 7, "8125": 20775, "8126": 0, "8127": 19290, "8128": 145214, "8129": 15144, "8130": 21716, "8131": 7435, "8132": 94224, "8133": 2, "8134": 53936, "8135": 34, "8136": 34231, "8137": 100459, "8138": 3750, "8139": 115719, "8140": 486, "8141": 298, "8142": 6633, "8143": 9809, "8144": 2783, "8145": 16451, "8146": 50498, "8147": 19649, "8148": 1077, "8149": 26499, "8150": 8166, "8151": 487, "8152": 444, "8153": 13, "8154": 1, "8155": 7743, "8156": 3491, "8157": 9173, "8158": 4679, "8159": 0, "8160": 7161, "8161": 16733, "8162": 15695, "8163": 110379, "8164": 39572, "8165": 1962, "8166": 2226, "8167": 18825, "8168": 6973, "8169": 16637, "8170": 1256, "8171": 11279, "8172": 20884, "8173": 0, "8174": 3, "8175": 1229, "8176": 6169, "8177": 91991, "8178": 80152, "8179": 3488, "8180": 22866, "8181": 7961, "8182": 3719, "8183": 10777, "8184": 3462, "8185": 12717, "8186": 2028, "8187": 9416, "8188": 54131, "8189": 28505, "8190": 45702, "8191": 8441} \ No newline at end of file diff --git a/data_generation/vqgan/utils.py b/data_generation/vqgan/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..428d778b208385eb1126e6e6c2a544e23bd98bc5 --- /dev/null +++ b/data_generation/vqgan/utils.py @@ -0,0 +1,57 @@ +import json +import random + +import multiprocess +import numpy as np +import torch +from torch import nn + +from .load import load_model + + +def init_seed(): + # set seed + import torch + random_seed = 1 + random.seed(42) + torch.set_grad_enabled(False) + torch.manual_seed(random_seed) + torch.cuda.manual_seed(random_seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(random_seed) + + +class ParallelWrapper(nn.Module): + def __init__(self, vq_model, func='encode'): + super().__init__() + self.vq_model = vq_model + self.func = func + + def forward(self, x): + return getattr(self.vq_model, self.func)(x) + + +def init_vqgan_encoder(model_name_or_path, device): + init_seed() + vq_model = load_model(model_name_or_path) + vq_model = vq_model.to(device).eval() + + print('vq_model device:', vq_model.device) + + encoder = ParallelWrapper(vq_model) + + return encoder + + +def get_multiprocess(): + multiprocess.set_start_method('spawn', force=True) + torch.utils.data.dataloader.python_multiprocessing = multiprocess + new_multiprocess_ctx = multiprocess.get_context() + return new_multiprocess_ctx + + +def dumps(data): + seqlen = len(data) + saved_bin = str.encode(json.dumps(dict(tokens=data)) + "\n") + return {"bin": saved_bin, "length": seqlen} diff --git a/data_generation/vqgan/vgg.pth b/data_generation/vqgan/vgg.pth new file mode 100644 index 0000000000000000000000000000000000000000..f57dcf5cc764d61c8a460365847fb2137ff0a62d --- /dev/null +++ b/data_generation/vqgan/vgg.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a78928a0af1e5f0fcb1f3b9e8f8c3a2a5a3de244d830ad5c1feddc79b8432868 +size 7289 diff --git a/data_generation/vqgan/vqgan_ckpt/ckpt.pth b/data_generation/vqgan/vqgan_ckpt/ckpt.pth new file mode 100644 index 0000000000000000000000000000000000000000..bd0de7b149acfb8b50f324f69f310a9fd04e554b --- /dev/null +++ b/data_generation/vqgan/vqgan_ckpt/ckpt.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57a97de02a0dd9378afc8b050f0889d8aa21eca0fcd80aadef297823a3d51185 +size 585057981 diff --git a/data_generation/vqgan/vqgan_ckpt/config.json b/data_generation/vqgan/vqgan_ckpt/config.json new file mode 100644 index 0000000000000000000000000000000000000000..174adaaeadade5338583661267f6b600ac08a268 --- /dev/null +++ b/data_generation/vqgan/vqgan_ckpt/config.json @@ -0,0 +1,23 @@ +{ + "_class_name": "VQGANModel", + "_version": "0.0.1", + "attn_resolutions": [], + "channel_mult": [ + 1, + 2, + 2, + 4, + 6 + ], + "commitment_cost": 0.25, + "dropout": 0.0, + "hidden_channels": 128, + "no_attn_mid_block": true, + "num_channels": 3, + "num_embeddings": 8192, + "num_res_blocks": 2, + "quantized_embed_dim": 64, + "resample_with_conv": true, + "resolution": 256, + "z_channels": 64 +} diff --git a/data_generation/vqgan/vqgan_ckpt/gitattributes b/data_generation/vqgan/vqgan_ckpt/gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..c7d9f3332a950355d5a77d85000f05e6f45435ea --- /dev/null +++ b/data_generation/vqgan/vqgan_ckpt/gitattributes @@ -0,0 +1,34 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/data_generation/vqgan/vqgan_ckpt/model.safetensors b/data_generation/vqgan/vqgan_ckpt/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..00b390dfd3a9e0eb887f14241e4033e0a5b971f3 --- /dev/null +++ b/data_generation/vqgan/vqgan_ckpt/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe9eb20c879810c7336d9538099952dbfc4a208116c4a2501274f85044179e7 +size 585007540 diff --git a/data_generation/vqgan/vqgan_ckpt/pytorch_model.bin b/data_generation/vqgan/vqgan_ckpt/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..1c7e3d459052d0b688c2a782083d7ec78761febc --- /dev/null +++ b/data_generation/vqgan/vqgan_ckpt/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fed960b9b88968dcaab28fd0cb28b786693b83b21fa93b1a1ad1177b3f7e9fd1 +size 585077729 diff --git a/dataset/__pycache__/get_vqgan_wds.cpython-37.pyc b/dataset/__pycache__/get_vqgan_wds.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f806f9cb73f1ae328295cb1e300b3c08df7ef610 Binary files /dev/null and b/dataset/__pycache__/get_vqgan_wds.cpython-37.pyc differ diff --git a/dataset/__pycache__/get_vqgan_wds.cpython-39.pyc b/dataset/__pycache__/get_vqgan_wds.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13de79f8be4576e6ddea314951671cc85c8be0bc Binary files /dev/null and b/dataset/__pycache__/get_vqgan_wds.cpython-39.pyc differ diff --git a/dataset/get_vqgan_wds.py b/dataset/get_vqgan_wds.py new file mode 100644 index 0000000000000000000000000000000000000000..1deb3b5d33fdd1044b278919d18c9864e490d66e --- /dev/null +++ b/dataset/get_vqgan_wds.py @@ -0,0 +1,205 @@ +from typing import Optional, List +from dataclasses import dataclass +from io import BytesIO +from PIL import Image as PILImage +from PIL import ImageEnhance +import os +import numpy as np +import datasets +from torchvision import transforms +import time +import random +import torch +try: + import webdataset as wds +except ImportError: + os.system("python3 -m pip install webdataset") + import webdataset as wds + +def handle_exception(ds): + for d in ds: + try: + yield d + except Exception as e: + import traceback + traceback.print_exc() + +import json +import math + +def random_crop_arr(pil_image, image_size, min_crop_frac=0.6, max_crop_frac=1.0): + min_smaller_dim_size = math.ceil(image_size / max_crop_frac) + max_smaller_dim_size = math.ceil(image_size / min_crop_frac) + smaller_dim_size = random.randrange(min_smaller_dim_size, max_smaller_dim_size + 1) + + # We are not on a new enough PIL to support the `reducing_gap` + # argument, which uses BOX downsampling at powers of two first. + # Thus, we do it by hand to improve downsample quality. + while min(*pil_image.size) >= 2 * smaller_dim_size: + pil_image = pil_image.resize( + tuple(x // 2 for x in pil_image.size), resample=PILImage.BOX + ) + + scale = smaller_dim_size / min(*pil_image.size) + pil_image = pil_image.resize( + tuple(round(x * scale) for x in pil_image.size), resample=PILImage.BICUBIC + ) + + arr = np.array(pil_image) + crop_y = random.randrange(arr.shape[0] - image_size + 1) + crop_x = random.randrange(arr.shape[1] - image_size + 1) + return PILImage.fromarray(arr[crop_y : crop_y + image_size, crop_x : crop_x + image_size]) + +# preproc_192 = transforms.Compose([ +# # transforms.Lambda(lambda pil_image: random_crop_arr(pil_image, 192)), +# # transforms.RandomHorizontalFlip(), +# transforms.Resize(192, interpolation=transforms.InterpolationMode.BILINEAR), +# transforms.CenterCrop(192), +# transforms.ToTensor(), +# # transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True) +# ]) + +preproc_256 = transforms.Compose([ + # transforms.Lambda(lambda pil_image: random_crop_arr(pil_image, 256)), + transforms.RandomResizedCrop(256, scale=(0.8, 1.0)), + # transforms.RandomHorizontalFlip(), + transforms.Resize(256, interpolation=transforms.InterpolationMode.BILINEAR), + # transforms.CenterCrop(192), + transforms.ToTensor(), + # transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True) + ]) + + +def decode_image_192(data): + # assert 'rgb.jpg' in data.keys() + rgb = data['rgb.jpg'] + if 'crop.pickle' in data.keys(): + crop = data['crop.pickle'] + else: + crop = None + + rgb = PILImage.open(BytesIO(rgb)).convert("RGB") + if crop is not None: + rgb = np.array(rgb) + rgb = rgb[crop[0][0]:crop[1][0], crop[0][1]:crop[1][1], :] + rgb = PILImage.fromarray(rgb).convert("RGB") + rgb = preproc_192(rgb) + # rgb = proc_func(rgb) + return rgb + +def decode_image_256(data): + # assert 'rgb.jpg' in data.keys() + rgb = data['rgb.jpg'] + if 'crop.pickle' in data.keys(): + crop = data['crop.pickle'] + else: + crop = None + + rgb = PILImage.open(BytesIO(rgb)).convert("RGB") + if crop is not None: + rgb = np.array(rgb) + rgb = rgb[crop[0][0]:crop[1][0], crop[0][1]:crop[1][1], :] + rgb = PILImage.fromarray(rgb).convert("RGB") + rgb = preproc_256(rgb) + # rgb = proc_func(rgb) + return rgb + +def load_webdataset(filepath): + + return wds.WebDataset(filepath).decode().map(decode_image) + + +def get_dataset(urls, multishard=True, shuffle=True, seed=0) -> wds.WebDataset: + if multishard: + nodesplitter = wds.split_by_node + samplesplitter = wds.non_empty + else: + nodesplitter = wds.non_empty + samplesplitter = wds.split_by_node + + # if 'calvin' in urls or 'real_data' in urls: + # ds: wds.WebDataset = ( + # # shard内shuffle (bufsize=100) + # wds.WebDataset(urls, shardshuffle=shuffle, detshuffle=shuffle, nodesplitter=nodesplitter, resampled=True) + # .compose(samplesplitter) + # .decode().map(decode_image_192) + # ) + # else: + ds: wds.WebDataset = ( + # shard内shuffle (bufsize=100) + wds.WebDataset(urls, shardshuffle=shuffle, detshuffle=shuffle, nodesplitter=nodesplitter, resampled=True) + .compose(samplesplitter) + .decode().map(decode_image_256) + ) + if shuffle: + # SimpleShardList: shard之间的shuffle - 只和数据集有关, 需要保持多node一致 + # assert isinstance(ds.pipeline[0], wds.SimpleShardList) + ds.pipeline[0].seed = seed + # wds.detshuffle: shard内部的shuffle - 完全随机 + # assert isinstance(ds.pipeline[3], wds.detshuffle) + ds.pipeline[3].seed = wds.utils.make_seed( + wds.utils.pytorch_worker_seed(), + os.getpid(), + time.time_ns(), + os.urandom(4), + ) + return ds + +def custom_collate_fn(batch): + # print (type(batch)) + # for x in batch: + # print (x.shape) + # batch = [x for x in batch if x is not None] + # # print (type(batch)) + # print (len(batch)) + # return torch.utils.data.dataloader.default_collate(batch) + # data_ = [item[0] for item in batch] + robot_rgb = [item for item in batch if item.shape[1] == 192] + datacomp_rgb = [item for item in batch if item.shape[1] == 256] + + data = {'robot': robot_rgb, 'datacomp': datacomp_rgb} + + return data + + +if __name__ == '__main__': + + ds_list = [] + + + ds1 = get_dataset('/mnt/bn/roboicllq-data1/processed_real/imgs/real_data_img_{0000..0590}.tar', seed=42) + ds_list += [(ds1, 0.2)] + + ds2 = get_dataset('/mnt/bn/roboicllq-data1/processed_real/hand_imgs/real_data_hand_img_{0000..0590}.tar', seed=42) + ds_list += [(ds2, 0.05)] + + ds3 = get_dataset('/mnt/bn/roboicllq-data1/calvin_img/calvin_img_{00000000..00000276}.tar', seed=42) + ds_list += [(ds3, 0.2)] + + + + + with open('/mnt/bn/roboicllq-data1/aligned_robot_ds/calvin/datacomp.json', 'r') as f: + tars = json.load(f) + + ds0 = get_dataset(tars, seed=42) + + ds_list += [(ds0, 0.55)] + + + ds = wds.RandomMix(*zip(*ds_list)) + ds = wds.DataPipeline(ds) + + for d in ds: + print (d.shape) + + # loader = ( + # wds.WebLoader(ds, num_workers=1, batch_size=16, pin_memory=True, collate_fn=custom_collate_fn).with_epoch(50000) + # ) + + # for i, data in enumerate(loader): + # print (i) + # print (type(data)) + + # for k, v in data.items(): + # print (k, len(v)) diff --git a/dist_train_utils.py b/dist_train_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e559dcf742567cf5b81fbda6c9a7e9625b659773 --- /dev/null +++ b/dist_train_utils.py @@ -0,0 +1,97 @@ +import io +import os + +import torch +import torch.distributed as dist +import psutil + + +_print = print + + +# DDP launcher 启动训练脚本时，会传入多机训练相关的环境变量，包括 +# WORLD_SIZE - 进程总数量 +# RANK - 唯一进程 ID，从 0 开始 +# LOCAL_RANK - 一台机器上唯一进程 ID，从 0 开始 +# 例如 2 worker，每个 worker 3 gpu，那么 +# WORLD_SIZE = 2*3=6 +# RANK 取值范围为 0-5 +# LOCAL_RANK 每台机器上的取值范围为 0-2 +def get_world_size(): return int(os.getenv('WORLD_SIZE', 1)) +def get_rank(): return int(os.getenv('RANK', 0)) +def get_local_rank(): return int(os.getenv('LOCAL_RANK', 0)) + +# 是否是分布式训练 +def is_dist(): + return dist.is_available() and dist.is_initialized() and get_world_size() > 1 + +def get_current_memory_gb(): +# 获取当前进程内存占用。 + pid = os.getpid() + p = psutil.Process(pid) + info = p.memory_full_info() + return info.uss / 1024. / 1024. / 1024. + +# 分布式训练使用 print 等方法时，每个进程都会输出，显示效果较差 +# 可以使用如下工具方法，只有 LOCAL_RANK=0 时才会输出 +# all 参数表示强制每个进程输出 +def print(*argc, all=False, **kwargs): + if not is_dist(): + _print(*argc, **kwargs) + return + + if not all and get_local_rank() != 0: + return + + output = io.StringIO() + kwargs['end'] = '' + kwargs['file'] = output + kwargs['flush'] = True + _print(*argc, **kwargs) + + s = output.getvalue() + output.close() + + s = '[rank {}] {}'.format(dist.get_rank(), s) + _print(s) + + +# 通过 all_reduce 计算某数值在所有进程上的平均值 +# 多卡训练时，可以用于计算准确率等指标 +def reduce_mean(tensor, nprocs=None): + if not is_dist(): + return tensor + if not isinstance(tensor, torch.Tensor): + device = torch.cuda.current_device() + rt = torch.tensor(tensor, device=device) + else: + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + nprocs = nprocs if nprocs else dist.get_world_size() + rt = rt / nprocs + if not isinstance(tensor, torch.Tensor): + rt = rt.item() + return rt + + +# 通过 all_reduce 求和 +def reduce_sum(tensor): + if not is_dist(): + return tensor + if not isinstance(tensor, torch.Tensor): + device = torch.cuda.current_device() + rt = torch.tensor(tensor, device=device) + else: + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + if not isinstance(tensor, torch.Tensor): + rt = rt.item() + return rt + + +# 进程间等待，当所有进程都执行该函数后，才会继续向下执行代码 +# 比如 rank=0 的进程保存模型，其他进程等待 rank_0 保存完成后才继续执行后续代码 +def barrier(): + if not is_dist(): + return + dist.barrier() \ No newline at end of file diff --git a/dist_train_vqgan.py b/dist_train_vqgan.py new file mode 100644 index 0000000000000000000000000000000000000000..f027f08788135cc305201e743f73b8038d788e01 --- /dev/null +++ b/dist_train_vqgan.py @@ -0,0 +1,283 @@ +import os +# os.environ['CUDA_LAUNCH_BLOCKING']='1' +import numpy as np +import torch.distributed as dist +import torch.nn as nn +from torch.nn.parallel import DistributedDataParallel as DDP +import argparse +from torch.utils.data import DataLoader +from torch.utils.tensorboard import SummaryWriter +from datetime import datetime +# import cv2 +import torch +from torch.nn.utils import clip_grad_norm_ +import yaml +from dist_train_utils import print, get_world_size, get_rank, get_local_rank, barrier, reduce_sum, reduce_mean +from tqdm import tqdm +from lr_utils import CosineAnnealingWarmupRestarts +import ssl +ssl._create_default_https_context = ssl._create_unverified_context +import webdataset as wds +from dataset.get_vqgan_wds import get_dataset, handle_exception +# from dataset.vq_wds import get_dataset, my_sample_decoder, my_sample_prec +import json +import time +from accelerate import Accelerator +from accelerate.utils import set_seed +from data_generation.vqgan.load import load_model +import random + +parser = argparse.ArgumentParser() +#ddp +parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') +parser.add_argument("--batch_size", type=int, default=16) +parser.add_argument("--epoch", type=int, default=20) +parser.add_argument("--base_lr", type=float, default=4.5e-6) +parser.add_argument("--log_folder", type=str, default='f16') + +args = parser.parse_args() + +set_seed(42) +accelerator = Accelerator() + +# class MyLoader(wds.WebLoader, torch.utils.data.DataLoader): +# pass + +# train_dataloader = MyLoader(train_dataset, ...) + + +# with open('/mnt/bn/robotics-data-hl/jirong/git/incontextrobotics/models/embedding/VQGAN//model.yaml', encoding='utf-8') as f: +# cfg = yaml.load(f, Loader=yaml.FullLoader) + +log_folder = args.log_folder +log_path = os.path.join('/mnt/bn/roboicl-jirong/codebase/DeLVM/logs/', log_folder) +if get_rank() == 0: + if not os.path.exists(log_path): + os.makedirs(log_path) + os.system('chmod -R 777 ' + log_path) +accelerator.wait_for_everyone() + +timestamp = "{0:%Y-%m-%dT%H-%M-%S/}".format(datetime.now()) + +if get_rank() == 0: + WRITER = SummaryWriter(log_path+'/'+timestamp, max_queue=1000) + +print('world_size: {}, rank: {}, local_rank: {}'.format(get_world_size(), get_rank(), get_local_rank()), all=True) + +# dist.init_process_group(backend='nccl', rank = get_rank(), world_size=get_world_size()) +# # assign gpu +# torch.cuda.set_device(get_local_rank()) +# # torch.cuda.set_device(7) +# device = torch.cuda.current_device() +device = accelerator.device + +print('device:', device, all=True) + +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_muse_finetune_calvin_vl_20m_1e-6_256/checkpoint_vq_epoch_48127.tar') +# vqmodel = load_model() +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_1024/checkpoint_vq_epoch_139999.tar') +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_1024/checkpoint_vq_epoch_64999.tar') +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_16384_160m_1e-4_192_disc_50000/checkpoint_vq_epoch_49999.tar') +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_16384_160m_1e-4_192_subset/checkpoint_vq_epoch_549999.tar') +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_16384_160m_3e-4_192/checkpoint_vq_epoch_149999.tar') +# vqmodel = load_model('/mnt/bn/roboicl-jirong/codebase/DeLVM/vqgan_ckpt/ckpt.pth', 8192) +vqmodel = load_model('/mnt/bn/roboicl-jirong/codebase/DeLVM/logs/f16_192_real_calvin_robot_datacomp_1e-5_disc_start_0_weight_0.2_acc_1/checkpoint_vq_epoch_9999.tar', 8192) +# vqmodel = load_model('/mnt/bn/roboicl-jirong/codebase/DeLVM/logs/f16_192_real_calvin_robot_datacomp_1e-5_disc_start_0_weight_0.2_acc_32/checkpoint_vq_epoch_19199.tar', 8192) +# vqmodel = load_model('/mnt/bn/roboicl-jirong/codebase/DeLVM/logs/f16_256_real_calvin_datacomp_1e-5_disc_start_0_weight_0.1/checkpoint_vq_epoch_34999.tar', 8192) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_aug_disc_0.3/checkpoint_vq_epoch_89999.tar', 2048) +# vqmodel = load_model(None, 2048) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_aug_disc_0.3_resume/checkpoint_vq_epoch_23999.tar', 2048) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048/checkpoint_vq_epoch_122499.tar', 2048) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_resume_aug/checkpoint_vq_epoch_97499.tar', 2048) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/git/DeLVM/vqgan_ckpt/ckpt.pth', 8192) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_muse_ego4d_llava_calvin_3e-6_disc_0/checkpoint_vq_epoch_66999.tar', 8192) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_aug_disc_0.3/checkpoint_vq_epoch_89999.tar', 2048) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_aug_disc_0.5_resume/checkpoint_vq_epoch_999.tar', 2048) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_192_real_data_muse_finetune_teacher_2048_resume_aug/checkpoint_vq_epoch_97499.tar', 2048) +# copied_param = [] +# init_param = [] +# for ((name, param), (name_s, param_s)) in zip(vq_model_t.named_parameters(), vqmodel.named_parameters()): +# if param.shape == param_s.shape: +# param_s.data = param.clone().data +# copied_param.append(param_s) +# else: +# init_param.append(name_s) +# # print (name, param.shape, name_s, param_s.shape) +# print ('params coopied from teacher') + +# print (init_param) + +# vqmodel = get_vqmodel(cfg) +# vqmodel = load_model('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_8192_160m_1e-4_muse_pretrained/checkpoint_vq_epoch_49999.tar') +# ckpt = torch.load('/mnt/bn/robotics-data-hl/jirong/rlhf/vq/f16_8192_160m_1e-4_muse_pretrained/checkpoint_vq_epoch_49999.tar', map_location='cpu') +# vqmodel.load_state_dict(ckpt) +vqmodel = nn.SyncBatchNorm.convert_sync_batchnorm(vqmodel) +vqmodel = vqmodel.to(device) + +# with open('/mnt/bn/robotics-data-hl/jirong/git/incontextrobotics/dataset/vqgan_imgs/calvin_only.json', 'r') as f: +# tars = json.load(f) + +ds_list = [] + +# ds0 = get_dataset('/mnt/bn/roboicllq-data1/calvin_img/calvin_img_00000051.tar', seed=42) +# ds_list += [(ds0, 0.05)] + +# print (len(tars), '============') +# ds1 = get_dataset('/mnt/bn/roboicllq-data1/processed_real/imgs/real_data_img_{0000..0590}.tar', seed=42) +# ds_list += [(ds1, 0.1)] + +# ds2 = get_dataset('/mnt/bn/roboicllq-data1/processed_real/hand_imgs/real_data_hand_img_{0000..0590}.tar', seed=42) +# ds_list += [(ds2, 0.02)] + +ds3 = get_dataset('/mnt/bn/roboicllq-data1/calvin_img/calvin_img_{00000000..00000110}.tar', seed=42) +ds_list += [(ds3, 0.1)] + +ds6 = get_dataset('/mnt/bn/roboicllq-data1/calvin_img/hands/calvin_hands_img_{00000000..00000110}.tar', seed=42) +ds_list += [(ds6, 0.02)] + +with open('/mnt/bn/roboicl-jirong/codebase/RoboICL/robot_img.json', 'r') as f: + tars = json.load(f) + +# # # print (len(tars)) +ds4 = get_dataset(tars, seed=42) +ds_list += [(ds4, 0.3)] + +with open('/mnt/bn/roboicllq-data1/aligned_robot_ds/calvin/datacomp.json', 'r') as f: + tars = json.load(f) + +# # # print (len(tars)) +ds0 = get_dataset(tars, seed=42) + +ds_list += [(ds0, 0.68)] + + +ds = wds.RandomMix(*zip(*ds_list)) +ds = wds.DataPipeline(ds) + +loader = ( + wds.WebLoader(ds, num_workers=4, batch_size=args.batch_size, pin_memory=True).with_epoch(args.epoch) + ) + +# loader = MyLoader(dataset=trainset, num_workers=4, batch_size=args.batch_size, pin_memory=True).with_epoch(args.epoch) + +# base_lr = 4.5e-6 * args.batch_size * get_world_size() +base_lr = 1e-5 +# base_lr = 3e-6 +# base_lr = 1e-6 +# base_lr = 1e-3 +# base_lr = 5e-4 +# base_lr = 1e-4 +# base_lr = 5e-5 +opt, _ = vqmodel.configure_optimizers(base_lr) + +# ae_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt[0], T_max=args.epoch, eta_min=base_lr * 0.001) +# disc_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt[1], T_max=args.epoch, eta_min=base_lr * 0.001) +# disc_start = cfg['model']['params']['lossconfig']['params']['disc_start'] +disc_start = 0 + +ae_opt, disc_opt = opt[0], opt[1] +ae_scheduler = CosineAnnealingWarmupRestarts(ae_opt, + first_cycle_steps=args.epoch, + cycle_mult=1, + max_lr=base_lr, + min_lr=base_lr*0.9999999, + warmup_steps=args.epoch/10, + gamma=1) + +disc_scheduler = CosineAnnealingWarmupRestarts(disc_opt, + first_cycle_steps=args.epoch, + cycle_mult=1, + max_lr=base_lr, + min_lr=base_lr*0.9999999, + warmup_steps=args.epoch/10, + gamma=1) + +vqmodel, ae_opt, disc_opt, ae_scheduler, disc_scheduler, loader = accelerator.prepare(vqmodel, ae_opt, disc_opt, ae_scheduler, disc_scheduler, loader) + +print ('global config end ---------------------------------------------------') + +log_iter = 50 + +acc_steps = 1 + +def train_one_epoch(args): + STEP_CNT = 0 + DISC_STEP_CNT = 0 + vqmodel.train() + stat_dict = {} + + for i, data in enumerate(loader): + + batch = data.to(device) + + aeloss, log_dict_ae = vqmodel.module.training_step(batch, 0, device, STEP_CNT) + + if STEP_CNT >= disc_start: + discloss, log_dict_disc = vqmodel.module.training_step(batch, 1, device, STEP_CNT) + + accelerator.backward(aeloss) + # if (STEP_CNT + 1) == 0: + ae_opt.step() + ae_opt.zero_grad() + + if STEP_CNT >= disc_start: + # print('asodkjsaoifdjosjio') + accelerator.backward(discloss) + # if (STEP_CNT + 1) == 0: + disc_opt.step() + disc_opt.zero_grad() + + # STEP_CNT += 1 + + + if (STEP_CNT + 1) % log_iter == 0: + if get_rank() == 0: + for k, v in log_dict_ae.items(): + if k not in stat_dict.keys(): + stat_dict[k] = 0 + stat_dict[k] += v.cpu().item() + + if STEP_CNT >= disc_start: + for k, v in log_dict_disc.items(): + if k not in stat_dict.keys(): + stat_dict[k] = 0 + stat_dict[k] += v.cpu().item() + + for k, v in stat_dict.items(): + WRITER.add_scalar(k, v/log_iter, STEP_CNT*args.batch_size*get_world_size()) + stat_dict[k] = 0 + + WRITER.add_scalar('lr_ae', ae_opt.param_groups[0]['lr'], STEP_CNT*args.batch_size*get_world_size()) + WRITER.add_scalar('lr_disc', disc_opt.param_groups[0]['lr'], STEP_CNT*args.batch_size*get_world_size()) + + STEP_CNT += 1 + # i += 1 + + ae_scheduler.step(STEP_CNT) + if STEP_CNT >= disc_start: + DISC_STEP_CNT += 1 + disc_scheduler.step(STEP_CNT) + + if (STEP_CNT + 1) % (log_iter*1000) == 0: + save_dict = {} + if get_rank() == 0: + + unwrapped_model = accelerator.unwrap_model(vqmodel) + accelerator.save(unwrapped_model.state_dict(), os.path.join(log_path, 'checkpoint_vq_epoch_' +str(STEP_CNT)+'.tar')) + # barrier() + accelerator.wait_for_everyone() + print ('epoch {} train done'.format(STEP_CNT)) + + +def main(args): + + train_one_epoch(args) + +if __name__ == '__main__': + import time + + # train_one_epoch() + main(args) + time.sleep(60) + dist.destroy_process_group() + print ('train done!') diff --git a/dist_train_vqgan.sh b/dist_train_vqgan.sh new file mode 100644 index 0000000000000000000000000000000000000000..bd4b2bfe37a3796a287c09e15f8f6e74371a8b35 --- /dev/null +++ b/dist_train_vqgan.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +export http_proxy=http://bj-rd-proxy.byted.org:3128 +export https_proxy=http://bj-rd-proxy.byted.org:3128 + +# source /mnt/bn/robotics-data-hl/jirong/git/calvin_incontext/bin/activate + +CUR_DIR=$(cd $(dirname $0); pwd) +cd $CUR_DIR + +cd /mnt/bn/roboicl-jirong/codebase/DeLVM + +ports=(`echo $METIS_WORKER_0_PORT | tr ',' ' '`) +port=${ports[0]} + +echo "total workers: ${ARNOLD_WORKER_NUM}" +echo "cur worker id: ${ARNOLD_ID}" +echo "gpus per worker: ${ARNOLD_WORKER_GPU}" +echo "master ip: ${METIS_WORKER_0_HOST}" +echo "master port: ${port}" + +export OMP_NUM_THREADS=8 +export NCCL_IB_DISABLE=0 +export NCCL_IB_GID_INDEX=3 +export NCCL_IB_HCA=${ARNOLD_RDMA_DEVICE} +export NCCL_SOCKET_IFNAME=eth0 +export NCCL_DEBUG=INFO + +# a=4 +# node_rank=$((ARNOLD_ID-a)) + +# script_name=$0 +# bs=$1 +# nw=$2 +# lr=$3 +# folder=$4 + +torchrun \ + --nnodes $ARNOLD_WORKER_NUM \ + --node_rank $ARNOLD_ID \ + --nproc_per_node 8 \ + --master_addr $METIS_WORKER_0_HOST \ + --master_port $port \ + dist_train_vqgan.py --batch_size 16 --epoch 5000000 --log_folder f16_192_real_calvin_robot_datacomp_1e-5_disc_start_0_weight_0.2 + +# torchrun \ +# --nnodes 4 \ +# --node_rank $ARNOLD_ID \ +# --nproc_per_node 8 \ +# --master_addr $METIS_WORKER_0_HOST \ +# --master_port $port \ +# dist_train_vqgan.py --batch_size 30 --epoch 500000 --log_folder f16_192_cofinetune + \ No newline at end of file diff --git a/figs/DeLVM.PNG b/figs/DeLVM.PNG new file mode 100644 index 0000000000000000000000000000000000000000..4246619072e1a8c33f4e32aad41d7ee2f0ee9de7 --- /dev/null +++ b/figs/DeLVM.PNG @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb480391737ead2c3b4bff3c92841b6c6c917d19bff447a08066fcee3a4a009c +size 208305 diff --git a/lr_utils.py b/lr_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7ed99475e33d0551fc60a56d2b9e391a4630d84b --- /dev/null +++ b/lr_utils.py @@ -0,0 +1,89 @@ +import math +import torch +from torch.optim.lr_scheduler import _LRScheduler + +class CosineAnnealingWarmupRestarts(_LRScheduler): + """ + optimizer (Optimizer): Wrapped optimizer. + first_cycle_steps (int): First cycle step size. + cycle_mult(float): Cycle steps magnification. Default: -1. + max_lr(float): First cycle's max learning rate. Default: 0.1. + min_lr(float): Min learning rate. Default: 0.001. + warmup_steps(int): Linear warmup step size. Default: 0. + gamma(float): Decrease rate of max learning rate by cycle. Default: 1. + last_epoch (int): The index of last epoch. Default: -1. + """ + + def __init__(self, + optimizer : torch.optim.Optimizer, + first_cycle_steps : int, + cycle_mult : float = 1., + max_lr : float = 0.1, + min_lr : float = 0.001, + warmup_steps : int = 0, + gamma : float = 1., + last_epoch : int = -1 + ): + assert warmup_steps < first_cycle_steps + + self.first_cycle_steps = first_cycle_steps # first cycle step size + self.cycle_mult = cycle_mult # cycle steps magnification + self.base_max_lr = max_lr # first max learning rate + self.max_lr = max_lr # max learning rate in the current cycle + self.min_lr = min_lr # min learning rate + self.warmup_steps = warmup_steps # warmup step size + self.gamma = gamma # decrease rate of max learning rate by cycle + + self.cur_cycle_steps = first_cycle_steps # first cycle step size + self.cycle = 0 # cycle count + self.step_in_cycle = last_epoch # step size of the current cycle + + super(CosineAnnealingWarmupRestarts, self).__init__(optimizer, last_epoch) + + # set learning rate min_lr + self.init_lr() + + def init_lr(self): + self.base_lrs = [] + for param_group in self.optimizer.param_groups: + param_group['lr'] = self.min_lr + self.base_lrs.append(self.min_lr) + + def get_lr(self): + if self.step_in_cycle == -1: + return self.base_lrs + elif self.step_in_cycle < self.warmup_steps: + return [(self.max_lr - base_lr)*self.step_in_cycle / self.warmup_steps + base_lr for base_lr in self.base_lrs] + else: + # return [self.max_lr] * len(self.base_lrs) + return [base_lr + (self.max_lr - base_lr) \ + * (1 + math.cos(math.pi * (self.step_in_cycle-self.warmup_steps) \ + / (self.cur_cycle_steps - self.warmup_steps))) / 2 + for base_lr in self.base_lrs] + + def step(self, epoch=None): + if epoch is None: + epoch = self.last_epoch + 1 + self.step_in_cycle = self.step_in_cycle + 1 + if self.step_in_cycle >= self.cur_cycle_steps: + self.cycle += 1 + self.step_in_cycle = self.step_in_cycle - self.cur_cycle_steps + self.cur_cycle_steps = int((self.cur_cycle_steps - self.warmup_steps) * self.cycle_mult) + self.warmup_steps + else: + if epoch >= self.first_cycle_steps: + if self.cycle_mult == 1.: + self.step_in_cycle = epoch % self.first_cycle_steps + self.cycle = epoch // self.first_cycle_steps + else: + n = int(math.log((epoch / self.first_cycle_steps * (self.cycle_mult - 1) + 1), self.cycle_mult)) + self.cycle = n + self.step_in_cycle = epoch - int(self.first_cycle_steps * (self.cycle_mult ** n - 1) / (self.cycle_mult - 1)) + self.cur_cycle_steps = self.first_cycle_steps * self.cycle_mult ** (n) + else: + self.cur_cycle_steps = self.first_cycle_steps + self.step_in_cycle = epoch + + self.max_lr = self.base_max_lr * (self.gamma**self.cycle) + self.last_epoch = math.floor(epoch) + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr diff --git a/taming/modules/autoencoder/lpips/vgg.pth b/taming/modules/autoencoder/lpips/vgg.pth new file mode 100644 index 0000000000000000000000000000000000000000..f57dcf5cc764d61c8a460365847fb2137ff0a62d --- /dev/null +++ b/taming/modules/autoencoder/lpips/vgg.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a78928a0af1e5f0fcb1f3b9e8f8c3a2a5a3de244d830ad5c1feddc79b8432868 +size 7289