Add files using upload-large-folder tool

.gitattributes

@@ -1,35 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
+data/babylm_data/* filter=lfs diff=lfs merge=lfs -text
+data/Perturbed_data/* filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+train/checkpoints/

FTP.py

@@ -0,0 +1,316 @@
+import torch
+from torch.optim.optimizer import Optimizer, required
+import copy
+import math
+
+class FTP(object):
+    def __init__(self, k=1.0, exclude_set={}):
+        self.exclude_set = exclude_set
+        self.threshold = torch.nn.Hardtanh(0,1)
+        self.k = k # Gradient annealing factor
+        self.j = 0 # Buffer counter
+
+        # AdamUtil parameteres
+        self.mu = 1e-2
+        self.beta1 = 0.9
+        self.beta2 = 0.999
+        self.t = 1
+        
+        # Buffers
+        self.gamma_buff = []
+        self.first_m_gamma = []
+        self.second_m_gamma = []
+        self.prev_c = []
+        self.prev_scale = []
+    
+    @torch.no_grad()
+    def step(self,name, curr, pre, d_p):
+        if curr.requires_grad and name not in self.exclude_set:  # exclude_set includes those params that not be updated
+            c_t = (curr - d_p) - pre   # (cur - d_p) potential new param/ dp gradient /
+            norms = self._mars_norm(c_t)
+
+            if self.t == 1:
+                gamma = torch.tensor(1e-8).to(norms.device)
+                self._update_buffers(gamma)
+            else:
+                # Get previous values
+                gamma_prev = self.gamma_buff[self.j]
+                c_prev = self.prev_c[self.j]
+                scale_prev = self.prev_scale[self.j]
+
+                # Calculate gradient for gamma
+                gamma_grad = torch.sum(self._dot(curr.grad, c_prev, scale = scale_prev))
+
+                # Anneal positive gradient 
+                if gamma_grad > 0:
+                    gamma_grad = gamma_grad * self.k
+
+                gamma = self._adam_util(gamma_prev, gamma_grad)
+
+                # Clip gamma
+                gamma = self._clip(gamma, norms)
+            
+            # Update
+            denom = 1/norms
+            ratio = gamma * denom
+            new_p = pre + self.threshold(ratio) * c_t 
+
+            # Save updated values
+            self._update_buffers(gamma, c_t, denom)
+            self.j += 1
+            return new_p
+        else:
+            return None
+        
+    def incre_counters(self):
+        self.t += 1
+        self.j = 0
+
+    @torch.no_grad()
+    def _mars_norm(self,tensor):
+        return torch.sum(torch.abs(tensor), dim=tuple(range(1,tensor.dim())), keepdim=True) + 1e-8
+    
+    @torch.no_grad()
+    def _clip(self, constraint, norms):
+        return torch.nn.functional.hardtanh(constraint,1e-8,norms.max())
+        
+    @torch.no_grad()
+    def _dot(self,tensor1,tensor2,scale=1):
+        return torch.sum(torch.mul(tensor1, tensor2),dim=tuple(range(1,tensor1.dim())),keepdim=True)*scale
+    
+    @torch.no_grad()
+    def _adam_util(self,prev, grad):
+        first_moment = self.beta1 * self.first_m_gamma[self.j] + (1-self.beta1) * grad
+        second_moment = self.beta2 * self.second_m_gamma[self.j] + (1-self.beta2) * grad**2
+        self.first_m_gamma[self.j] = first_moment
+        self.second_m_gamma[self.j] = second_moment
+        first_moment = first_moment/(1-self.beta1**self.t)
+        second_moment = second_moment/(1-self.beta2**self.t)
+        return prev  - self.mu * first_moment/(torch.sqrt(second_moment)+1e-8)
+    
+    def _update_buffers(self,gamma,c_t=None,denom=None):
+        if c_t is None:
+            self.first_m_gamma.append(0.0)
+            self.second_m_gamma.append(0.0)
+            self.gamma_buff.append(gamma)
+            self.prev_c.append(0.0)
+            self.prev_scale.append(0.0)
+        else:
+            self.gamma_buff[self.j] = gamma
+            self.prev_c[self.j] = c_t
+            self.prev_scale[self.j] = denom
+
+            
+class SGDP(Optimizer):
+    def __init__(self, params, lr=required, momentum=0, dampening=0,
+                 weight_decay=0, nesterov=False, k=1.0, exclude_set = {}):
+        if lr is not required and lr < 0.0:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if momentum < 0.0:
+            raise ValueError("Invalid momentum value: {}".format(momentum))
+        if weight_decay < 0.0:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+
+        defaults = dict(lr=lr, momentum=momentum, dampening=dampening,
+                        weight_decay=weight_decay, nesterov=nesterov)
+        if nesterov and (momentum <= 0 or dampening != 0):
+            raise ValueError("Nesterov momentum requires a momentum and zero dampening")
+        super(SGDP, self).__init__(params, defaults)
+        self.first_iter_flag = False
+         
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+                    
+
+    def __setstate__(self, state):
+        super(SGDP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('nesterov', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+        for group in self.param_groups:
+            weight_decay = group['weight_decay']
+            momentum = group['momentum']
+            dampening = group['dampening']
+            nesterov = group['nesterov']
+            
+            for p, name, pre in zip(group['params'],group['name'],group['pre']):
+                if p.grad is None:
+                    continue
+                d_p = p.grad
+                if weight_decay != 0:
+                    d_p = d_p.add(p, alpha=weight_decay)
+                if momentum != 0:
+                    param_state = self.state[p]
+                    if 'momentum_buffer' not in param_state:
+                        buf = param_state['momentum_buffer'] = torch.clone(d_p).detach()
+                    else:
+                        buf = param_state['momentum_buffer']
+                        buf.mul_(momentum).add_(d_p, alpha=1 - dampening) 
+                    if nesterov:
+                        d_p = d_p.add(buf, alpha=momentum)
+                    else:
+                        d_p = buf
+
+                # FTP step
+                d_p = group['lr']*d_p
+                new_p = self.ftp.step(name,p,pre,d_p)
+                if new_p is not None :
+                    p.copy_(new_p)
+                else:
+                    p.add_(d_p, alpha=-1)
+        # FTP increment internal counters
+        self.ftp.incre_counters()        
+        return loss
+
+
+class AdamP(Optimizer):
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
+                 weight_decay=0, amsgrad=False, k=1.0, exclude_set={}):
+        if not 0.0 <= lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+        if not 0.0 <= weight_decay:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, amsgrad=amsgrad)
+        super(AdamP, self).__init__(params, defaults)
+
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+
+    def __setstate__(self, state):
+        super(AdamP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('amsgrad', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+
+        for group in self.param_groups:
+            params_with_grad = []
+            grads = []
+            exp_avgs = []
+            exp_avg_sqs = []
+            max_exp_avg_sqs = []
+            state_steps = []
+
+            for p in group['params']:
+                if p.grad is not None:
+                    params_with_grad.append(p)
+                    if p.grad.is_sparse:
+                        raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
+                    grads.append(p.grad)
+
+                    state = self.state[p]
+                    # Lazy state initialization
+                    if len(state) == 0:
+                        state['step'] = 0
+                        # Exponential moving average of gradient values
+                        state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        # Exponential moving average of squared gradient values
+                        state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        if group['amsgrad']:
+                            # Maintains max of all exp. moving avg. of sq. grad. values
+                            state['max_exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+
+                    exp_avgs.append(state['exp_avg'])
+                    exp_avg_sqs.append(state['exp_avg_sq'])
+
+                    if group['amsgrad']:
+                        max_exp_avg_sqs.append(state['max_exp_avg_sq'])
+
+                    # update the steps for each param group update
+                    state['step'] += 1
+                    # record the step after step update
+                    state_steps.append(state['step'])
+
+            beta1, beta2 = group['betas']
+            self.adam(group,
+                   exp_avgs,
+                   exp_avg_sqs,
+                   max_exp_avg_sqs,
+                   state_steps,
+                   group['amsgrad'],
+                   beta1,
+                   beta2,
+                   group['lr'],
+                   group['weight_decay'],
+                   group['eps']
+                   )
+        # FTP increment internal counters            
+        self.ftp.incre_counters()    
+        return loss
+
+    def adam(self, group,
+         exp_avgs,
+         exp_avg_sqs,
+         max_exp_avg_sqs,
+         state_steps, 
+         amsgrad: bool,
+         beta1: float,
+         beta2: float,
+         lr: float,
+         weight_decay: float,
+         eps: float):
+        
+        i = 0
+        for param, name, pre in zip(group['params'],group['name'],group['pre']):
+            if param.grad is None: 
+                continue
+            grad = param.grad
+            exp_avg = exp_avgs[i]
+            exp_avg_sq = exp_avg_sqs[i]
+            step = state_steps[i]
+            if amsgrad:
+                max_exp_avg_sq = max_exp_avg_sqs[i]
+
+            bias_correction1 = 1 - beta1 ** step
+            bias_correction2 = 1 - beta2 ** step
+
+            # Decay the first and second moment running average coefficient
+            exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+            exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+            if amsgrad:
+                # Maintains the maximum of all 2nd moment running avg. till now
+                torch.maximum(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
+                # Use the max. for normalizing running avg. of gradient
+                denom = (max_exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+            else:
+                denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+
+            step_size = lr / bias_correction1
+            i += 1
+            
+            # FTP step
+            d_p = step_size * exp_avg/denom + lr * weight_decay * param
+            new_p = self.ftp.step(name,param,pre,d_p)
+            if new_p is None :
+                new_p = param - d_p
+            param.copy_(new_p)

FTP_update.py

@@ -0,0 +1,323 @@
+import torch
+from torch.optim.optimizer import Optimizer, required
+import copy
+import math
+
+class FTP(object):
+    def __init__(self, k=1.0, exclude_set={}):
+        self.exclude_set = exclude_set
+        self.threshold = torch.nn.Hardtanh(0,1)
+        self.k = k  # Gradient annealing factor
+        self.j = 0  # Buffer counter
+
+        # AdamUtil parameters
+        self.mu = 1e-2
+        self.beta1 = 0.9
+        self.beta2 = 0.999
+        self.t = 1
+        
+        # Buffers
+        self.gamma_buff = []
+        self.first_m_gamma = []
+        self.second_m_gamma = []
+        self.prev_c = []
+        self.prev_scale = []
+    
+    @torch.no_grad()
+    def step(self, name, curr, pre, d_p):
+        if curr.requires_grad and name not in self.exclude_set:  # Exclude set includes those params that not be updated
+            c_t = (curr - d_p) - pre  # Compute potential new param
+            norms = self._mars_norm(c_t)
+
+            # New: Apply spectral normalization to c_t
+            c_t = self._apply_spectral_norm(c_t)
+
+            if self.t == 1:
+                gamma = torch.tensor(1e-8, device=norms.device)
+                self._update_buffers(gamma)
+            else:
+                # Get previous values
+                gamma_prev = self.gamma_buff[self.j]
+                c_prev = self.prev_c[self.j]
+                scale_prev = self.prev_scale[self.j]
+
+                # Calculate gradient for gamma
+                gamma_grad = torch.sum(self._dot(curr.grad, c_prev, scale=scale_prev))
+
+                # Anneal positive gradient 
+                if gamma_grad > 0:
+                    gamma_grad = gamma_grad * self.k
+
+                gamma = self._adam_util(gamma_prev, gamma_grad)
+
+                # Clip gamma
+                gamma = self._clip(gamma, norms)
+            
+            # Update
+            denom = 1/norms
+            ratio = gamma * denom
+            new_p = pre + self.threshold(ratio) * c_t
+
+            # Save updated values
+            self._update_buffers(gamma, c_t, denom)
+            self.j += 1
+            return new_p
+        else:
+            return None
+
+    def _apply_spectral_norm(self, c_t):
+        u, s, vh = torch.linalg.svd(c_t, full_matrices=False)
+        spectral_norm = s.max()
+        return torch.matmul(u, vh) / spectral_norm
+
+    def incre_counters(self):
+        self.t += 1
+        self.j = 0
+
+    @torch.no_grad()
+    def _mars_norm(self, tensor):
+        return torch.sum(torch.abs(tensor), dim=tuple(range(1, tensor.dim())), keepdim=True) + 1e-8
+    
+    @torch.no_grad()
+    def _clip(self, constraint, norms):
+        return torch.nn.functional.hardtanh(constraint, 1e-8, norms.max())
+        
+    @torch.no_grad()
+    def _dot(self, tensor1, tensor2, scale=1):
+        return torch.sum(torch.mul(tensor1, tensor2), dim=tuple(range(1, tensor1.dim())), keepdim=True) * scale
+    
+    @torch.no_grad()
+    def _adam_util(self, prev, grad):
+        first_moment = self.beta1 * self.first_m_gamma[self.j] + (1 - self.beta1) * grad
+        second_moment = self.beta2 * self.second_m_gamma[self.j] + (1 - self.beta2) * grad**2
+        self.first_m_gamma[self.j] = first_moment
+        self.second_m_gamma[self.j] = second_moment
+        first_moment = first_moment / (1 - self.beta1**self.t)
+        second_moment = second_moment / (1 - self.beta2**self.t)
+        return prev - self.mu * first_moment / (torch.sqrt(second_moment) + 1e-8)
+    
+    def _update_buffers(self, gamma, c_t=None, denom=None):
+        if c_t is None:
+            self.first_m_gamma.append(0.0)
+            self.second_m_gamma.append(0.0)
+            self.gamma_buff.append(gamma)
+            self.prev_c.append(0.0)
+            self.prev_scale.append(0.0)
+        else:
+            self.gamma_buff[self.j] = gamma
+            self.prev_c[self.j] = c_t
+            self.prev_scale[self.j] = denom
+
+class SGDP(Optimizer):
+    def __init__(self, params, lr=required, momentum=0, dampening=0,
+                 weight_decay=0, nesterov=False, k=1.0, exclude_set = {}):
+        if lr is not required and lr < 0.0:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if momentum < 0.0:
+            raise ValueError("Invalid momentum value: {}".format(momentum))
+        if weight_decay < 0.0:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+
+        defaults = dict(lr=lr, momentum=momentum, dampening=dampening,
+                        weight_decay=weight_decay, nesterov=nesterov)
+        if nesterov and (momentum <= 0 or dampening != 0):
+            raise ValueError("Nesterov momentum requires a momentum and zero dampening")
+        super(SGDP, self).__init__(params, defaults)
+        self.first_iter_flag = False
+         
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+                    
+
+    def __setstate__(self, state):
+        super(SGDP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('nesterov', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+        for group in self.param_groups:
+            weight_decay = group['weight_decay']
+            momentum = group['momentum']
+            dampening = group['dampening']
+            nesterov = group['nesterov']
+            
+            for p, name, pre in zip(group['params'],group['name'],group['pre']):
+                if p.grad is None:
+                    continue
+                d_p = p.grad
+                if weight_decay != 0:
+                    d_p = d_p.add(p, alpha=weight_decay)
+                if momentum != 0:
+                    param_state = self.state[p]
+                    if 'momentum_buffer' not in param_state:
+                        buf = param_state['momentum_buffer'] = torch.clone(d_p).detach()
+                    else:
+                        buf = param_state['momentum_buffer']
+                        buf.mul_(momentum).add_(d_p, alpha=1 - dampening) 
+                    if nesterov:
+                        d_p = d_p.add(buf, alpha=momentum)
+                    else:
+                        d_p = buf
+
+                # FTP step
+                d_p = group['lr']*d_p
+                new_p = self.ftp.step(name,p,pre,d_p)
+                if new_p is not None :
+                    p.copy_(new_p)
+                else:
+                    p.add_(d_p, alpha=-1)
+        # FTP increment internal counters
+        self.ftp.incre_counters()        
+        return loss
+
+
+class AdamP(Optimizer):
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
+                 weight_decay=0, amsgrad=False, k=1.0, exclude_set={}):
+        if not 0.0 <= lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+        if not 0.0 <= weight_decay:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, amsgrad=amsgrad)
+        super(AdamP, self).__init__(params, defaults)
+
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+
+    def __setstate__(self, state):
+        super(AdamP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('amsgrad', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+
+        for group in self.param_groups:
+            params_with_grad = []
+            grads = []
+            exp_avgs = []
+            exp_avg_sqs = []
+            max_exp_avg_sqs = []
+            state_steps = []
+
+            for p in group['params']:
+                if p.grad is not None:
+                    params_with_grad.append(p)
+                    if p.grad.is_sparse:
+                        raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
+                    grads.append(p.grad)
+
+                    state = self.state[p]
+                    # Lazy state initialization
+                    if len(state) == 0:
+                        state['step'] = 0
+                        # Exponential moving average of gradient values
+                        state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        # Exponential moving average of squared gradient values
+                        state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        if group['amsgrad']:
+                            # Maintains max of all exp. moving avg. of sq. grad. values
+                            state['max_exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+
+                    exp_avgs.append(state['exp_avg'])
+                    exp_avg_sqs.append(state['exp_avg_sq'])
+
+                    if group['amsgrad']:
+                        max_exp_avg_sqs.append(state['max_exp_avg_sq'])
+
+                    # update the steps for each param group update
+                    state['step'] += 1
+                    # record the step after step update
+                    state_steps.append(state['step'])
+
+            beta1, beta2 = group['betas']
+            self.adam(group,
+                   exp_avgs,
+                   exp_avg_sqs,
+                   max_exp_avg_sqs,
+                   state_steps,
+                   group['amsgrad'],
+                   beta1,
+                   beta2,
+                   group['lr'],
+                   group['weight_decay'],
+                   group['eps']
+                   )
+        # FTP increment internal counters            
+        self.ftp.incre_counters()    
+        return loss
+
+    def adam(self, group,
+         exp_avgs,
+         exp_avg_sqs,
+         max_exp_avg_sqs,
+         state_steps, 
+         amsgrad: bool,
+         beta1: float,
+         beta2: float,
+         lr: float,
+         weight_decay: float,
+         eps: float):
+        
+        i = 0
+        for param, name, pre in zip(group['params'],group['name'],group['pre']):
+            if param.grad is None: 
+                continue
+            grad = param.grad
+            exp_avg = exp_avgs[i]
+            exp_avg_sq = exp_avg_sqs[i]
+            step = state_steps[i]
+            if amsgrad:
+                max_exp_avg_sq = max_exp_avg_sqs[i]
+
+            bias_correction1 = 1 - beta1 ** step
+            bias_correction2 = 1 - beta2 ** step
+
+            # Decay the first and second moment running average coefficient
+            exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+            exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+            if amsgrad:
+                # Maintains the maximum of all 2nd moment running avg. till now
+                torch.maximum(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
+                # Use the max. for normalizing running avg. of gradient
+                denom = (max_exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+            else:
+                denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+
+            step_size = lr / bias_correction1
+            i += 1
+            
+            # FTP step
+            d_p = step_size * exp_avg/denom + lr * weight_decay * param
+            new_p = self.ftp.step(name,param,pre,d_p)
+            if new_p is None :
+                new_p = param - d_p
+            param.copy_(new_p)

FTP_update2.py

@@ -0,0 +1,340 @@
+import torch
+from torch.optim.optimizer import Optimizer, required
+import copy
+import math
+
+class FTP(object):
+    def __init__(self, k=1.0, exclude_set={}):
+        self.exclude_set = exclude_set
+        self.threshold = torch.nn.Hardtanh(0,1)
+        self.k = k  # Gradient annealing factor
+        self.j = 0  # Buffer counter
+
+        # AdamUtil parameters
+        self.mu = 1e-2
+        self.beta1 = 0.9
+        self.beta2 = 0.999
+        self.t = 1
+        
+        # Buffers
+        self.gamma_buff = []
+        self.first_m_gamma = []
+        self.second_m_gamma = []
+        self.prev_c = []
+        self.prev_scale = []
+    
+    @torch.no_grad()
+    def step(self, name, curr, pre, d_p):
+        if curr.requires_grad and name not in self.exclude_set:  # Exclude set includes those params that not be updated
+            c_t = (curr - d_p) - pre  # Compute potential new param
+            norms = self._mars_norm(c_t)
+
+            # New: Apply spectral normalization to c_t
+            c_t = self._apply_spectral_norm(c_t)
+
+            if self.t == 1:
+                gamma = torch.tensor(1e-8, device=norms.device)
+                self._update_buffers(gamma)
+            else:
+                # Get previous values
+                gamma_prev = self.gamma_buff[self.j]
+                c_prev = self.prev_c[self.j]
+                scale_prev = self.prev_scale[self.j]
+
+                # Calculate gradient for gamma
+                gamma_grad = torch.sum(self._dot(curr.grad, c_prev, scale=scale_prev))
+
+                # Anneal positive gradient 
+                if gamma_grad > 0:
+                    gamma_grad = gamma_grad * self.k
+
+                gamma = self._adam_util(gamma_prev, gamma_grad)
+
+                # Clip gamma
+                gamma = self._clip(gamma, norms)
+            
+            # Update
+            denom = 1/norms
+            ratio = gamma * denom
+            new_p = pre + self.threshold(ratio) * c_t
+
+            # Save updated values
+            self._update_buffers(gamma, c_t, denom)
+            self.j += 1
+            return new_p
+        else:
+            return None
+
+    def _apply_spectral_norm(self, param, tau=1.0):
+        
+        if param.ndim >= 2:  # Only for 2D matrices
+            u, s, vh = torch.linalg.svd(param, full_matrices=False)
+            s_clipped = torch.clamp(s, max=tau)  # Clip singular values
+            return torch.matmul(u, torch.matmul(torch.diag(s_clipped), vh))
+        return param  # No projection for 1D parameters
+    
+    # def _apply_spectral_norm(self, param, tau=1.0):
+    #     if param.ndim >= 2:  # For 2D or higher-dimensional tensors
+    #         original_shape = param.shape  # Save original shape
+    #         reshaped_param = param.view(original_shape[0], -1)  # Flatten into 2D
+
+    #         u, s, vh = torch.linalg.svd(reshaped_param, full_matrices=False)
+    #         s_clipped = torch.clamp(s, max=tau)  # Clip singular values
+
+    #         # Reconstruct the tensor with clipped singular values
+    #         normalized_param = torch.matmul(u, torch.matmul(torch.diag(s_clipped), vh))
+    #         return normalized_param.view(original_shape)  # Reshape back to original
+    #     return param  # No projection for 1D parameters
+
+
+    def incre_counters(self):
+        self.t += 1
+        self.j = 0
+
+    @torch.no_grad()
+    def _mars_norm(self, tensor):
+        return torch.sum(torch.abs(tensor), dim=tuple(range(1, tensor.dim())), keepdim=True) + 1e-8
+    
+    @torch.no_grad()
+    def _clip(self, constraint, norms):
+        return torch.nn.functional.hardtanh(constraint, 1e-8, norms.max())
+        
+    @torch.no_grad()
+    def _dot(self, tensor1, tensor2, scale=1):
+        return torch.sum(torch.mul(tensor1, tensor2), dim=tuple(range(1, tensor1.dim())), keepdim=True) * scale
+    
+    @torch.no_grad()
+    def _adam_util(self, prev, grad):
+        first_moment = self.beta1 * self.first_m_gamma[self.j] + (1 - self.beta1) * grad
+        second_moment = self.beta2 * self.second_m_gamma[self.j] + (1 - self.beta2) * grad**2
+        self.first_m_gamma[self.j] = first_moment
+        self.second_m_gamma[self.j] = second_moment
+        first_moment = first_moment / (1 - self.beta1**self.t)
+        second_moment = second_moment / (1 - self.beta2**self.t)
+        return prev - self.mu * first_moment / (torch.sqrt(second_moment) + 1e-8)
+    
+    def _update_buffers(self, gamma, c_t=None, denom=None):
+        if c_t is None:
+            self.first_m_gamma.append(0.0)
+            self.second_m_gamma.append(0.0)
+            self.gamma_buff.append(gamma)
+            self.prev_c.append(0.0)
+            self.prev_scale.append(0.0)
+        else:
+            self.gamma_buff[self.j] = gamma
+            self.prev_c[self.j] = c_t
+            self.prev_scale[self.j] = denom
+
+class SGDP(Optimizer):
+    def __init__(self, params, lr=required, momentum=0, dampening=0,
+                 weight_decay=0, nesterov=False, k=1.0, exclude_set = {}):
+        if lr is not required and lr < 0.0:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if momentum < 0.0:
+            raise ValueError("Invalid momentum value: {}".format(momentum))
+        if weight_decay < 0.0:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+
+        defaults = dict(lr=lr, momentum=momentum, dampening=dampening,
+                        weight_decay=weight_decay, nesterov=nesterov)
+        if nesterov and (momentum <= 0 or dampening != 0):
+            raise ValueError("Nesterov momentum requires a momentum and zero dampening")
+        super(SGDP, self).__init__(params, defaults)
+        self.first_iter_flag = False
+         
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+                    
+
+    def __setstate__(self, state):
+        super(SGDP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('nesterov', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+        for group in self.param_groups:
+            weight_decay = group['weight_decay']
+            momentum = group['momentum']
+            dampening = group['dampening']
+            nesterov = group['nesterov']
+            
+            for p, name, pre in zip(group['params'],group['name'],group['pre']):
+                if p.grad is None:
+                    continue
+                d_p = p.grad
+                if weight_decay != 0:
+                    d_p = d_p.add(p, alpha=weight_decay)
+                if momentum != 0:
+                    param_state = self.state[p]
+                    if 'momentum_buffer' not in param_state:
+                        buf = param_state['momentum_buffer'] = torch.clone(d_p).detach()
+                    else:
+                        buf = param_state['momentum_buffer']
+                        buf.mul_(momentum).add_(d_p, alpha=1 - dampening) 
+                    if nesterov:
+                        d_p = d_p.add(buf, alpha=momentum)
+                    else:
+                        d_p = buf
+
+                # FTP step
+                d_p = group['lr']*d_p
+                new_p = self.ftp.step(name,p,pre,d_p)
+                if new_p is not None :
+                    p.copy_(new_p)
+                else:
+                    p.add_(d_p, alpha=-1)
+        # FTP increment internal counters
+        self.ftp.incre_counters()        
+        return loss
+
+
+class AdamP(Optimizer):
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
+                 weight_decay=0, amsgrad=False, k=1.0, exclude_set={}):
+        if not 0.0 <= lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+        if not 0.0 <= weight_decay:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, amsgrad=amsgrad)
+        super(AdamP, self).__init__(params, defaults)
+
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+
+    def __setstate__(self, state):
+        super(AdamP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('amsgrad', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+
+        for group in self.param_groups:
+            params_with_grad = []
+            grads = []
+            exp_avgs = []
+            exp_avg_sqs = []
+            max_exp_avg_sqs = []
+            state_steps = []
+
+            for p in group['params']:
+                if p.grad is not None:
+                    params_with_grad.append(p)
+                    if p.grad.is_sparse:
+                        raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
+                    grads.append(p.grad)
+
+                    state = self.state[p]
+                    # Lazy state initialization
+                    if len(state) == 0:
+                        state['step'] = 0
+                        # Exponential moving average of gradient values
+                        state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        # Exponential moving average of squared gradient values
+                        state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        if group['amsgrad']:
+                            # Maintains max of all exp. moving avg. of sq. grad. values
+                            state['max_exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+
+                    exp_avgs.append(state['exp_avg'])
+                    exp_avg_sqs.append(state['exp_avg_sq'])
+
+                    if group['amsgrad']:
+                        max_exp_avg_sqs.append(state['max_exp_avg_sq'])
+
+                    # update the steps for each param group update
+                    state['step'] += 1
+                    # record the step after step update
+                    state_steps.append(state['step'])
+
+            beta1, beta2 = group['betas']
+            self.adam(group,
+                   exp_avgs,
+                   exp_avg_sqs,
+                   max_exp_avg_sqs,
+                   state_steps,
+                   group['amsgrad'],
+                   beta1,
+                   beta2,
+                   group['lr'],
+                   group['weight_decay'],
+                   group['eps']
+                   )
+        # FTP increment internal counters            
+        self.ftp.incre_counters()    
+        return loss
+
+    def adam(self, group,
+         exp_avgs,
+         exp_avg_sqs,
+         max_exp_avg_sqs,
+         state_steps, 
+         amsgrad: bool,
+         beta1: float,
+         beta2: float,
+         lr: float,
+         weight_decay: float,
+         eps: float):
+        
+        i = 0
+        for param, name, pre in zip(group['params'],group['name'],group['pre']):
+            if param.grad is None: 
+                continue
+            grad = param.grad
+            exp_avg = exp_avgs[i]
+            exp_avg_sq = exp_avg_sqs[i]
+            step = state_steps[i]
+            if amsgrad:
+                max_exp_avg_sq = max_exp_avg_sqs[i]
+
+            bias_correction1 = 1 - beta1 ** step
+            bias_correction2 = 1 - beta2 ** step
+
+            # Decay the first and second moment running average coefficient
+            exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+            exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+            if amsgrad:
+                # Maintains the maximum of all 2nd moment running avg. till now
+                torch.maximum(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
+                # Use the max. for normalizing running avg. of gradient
+                denom = (max_exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+            else:
+                denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+
+            step_size = lr / bias_correction1
+            i += 1
+            
+            # FTP step
+            d_p = step_size * exp_avg/denom + lr * weight_decay * param
+            new_p = self.ftp.step(name,param,pre,d_p)
+            if new_p is None :
+                new_p = param - d_p
+            param.copy_(new_p)

FTP_update3.py

@@ -0,0 +1,329 @@
+import torch
+from torch.optim.optimizer import Optimizer, required
+import copy
+import math
+
+class FTP(object):
+    def __init__(self, k=1.0, exclude_set={}):
+        self.exclude_set = exclude_set
+        self.threshold = torch.nn.Hardtanh(0,1)
+        self.k = k  # Gradient annealing factor
+        self.j = 0  # Buffer counter
+
+        # AdamUtil parameters
+        self.mu = 1e-2
+        self.beta1 = 0.9
+        self.beta2 = 0.999
+        self.t = 1
+        
+        # Buffers
+        self.gamma_buff = []
+        self.first_m_gamma = []
+        self.second_m_gamma = []
+        self.prev_c = []
+        self.prev_scale = []
+    
+    @torch.no_grad()
+    def step(self, name, curr, pre, d_p):
+        if curr.requires_grad and name not in self.exclude_set:  # Exclude set includes those params that not be updated
+            c_t = (curr - d_p) - pre  # Compute potential new param
+            norms = self._mars_norm(c_t, tau=1.0)
+
+            # New: Apply spectral normalization to c_t
+            c_t = self._apply_spectral_norm(c_t)
+
+            if self.t == 1:
+                gamma = torch.tensor(1e-8, device=norms.device)
+                self._update_buffers(gamma)
+            else:
+                # Get previous values
+                gamma_prev = self.gamma_buff[self.j]
+                c_prev = self.prev_c[self.j]
+                scale_prev = self.prev_scale[self.j]
+
+                # Calculate gradient for gamma
+                gamma_grad = torch.sum(self._dot(curr.grad, c_prev, scale=scale_prev))
+
+                # Anneal positive gradient 
+                if gamma_grad > 0:
+                    gamma_grad = gamma_grad * self.k
+
+                gamma = self._adam_util(gamma_prev, gamma_grad)
+
+                # Clip gamma
+                gamma = self._clip(gamma, norms)
+            
+            # Update
+            denom = 1/norms
+            ratio = gamma * denom
+            new_p = pre + self.threshold(ratio) * c_t
+
+            new_p = self._apply_spectral_norm(new_p, tau=2.0)  # tau is the max allowed spectral norm
+
+            # Save updated values
+            self._update_buffers(gamma, c_t, denom)
+            self.j += 1
+            return new_p
+        else:
+            return None
+
+    def _apply_spectral_norm(self, param, tau=1.0):
+        
+        if param.ndim >= 2:  # Only for 2D matrices
+            u, s, vh = torch.linalg.svd(param, full_matrices=False)
+            s_clipped = torch.clamp(s, max=tau)  # Clip singular values
+            return torch.matmul(u, torch.matmul(torch.diag(s_clipped), vh))
+        return param  # No projection for 1D parameters
+    
+
+    def incre_counters(self):
+        self.t += 1
+        self.j = 0
+
+    @torch.no_grad()
+    def _mars_norm(self, tensor):
+        return torch.sum(torch.abs(tensor), dim=tuple(range(1, tensor.dim())), keepdim=True) + 1e-8
+    
+    @torch.no_grad()
+    def _clip(self, constraint, norms):
+        return torch.nn.functional.hardtanh(constraint, 1e-8, norms.max())
+        
+    @torch.no_grad()
+    def _dot(self, tensor1, tensor2, scale=1):
+        return torch.sum(torch.mul(tensor1, tensor2), dim=tuple(range(1, tensor1.dim())), keepdim=True) * scale
+    
+    @torch.no_grad()
+    def _adam_util(self, prev, grad):
+        first_moment = self.beta1 * self.first_m_gamma[self.j] + (1 - self.beta1) * grad
+        second_moment = self.beta2 * self.second_m_gamma[self.j] + (1 - self.beta2) * grad**2
+        self.first_m_gamma[self.j] = first_moment
+        self.second_m_gamma[self.j] = second_moment
+        first_moment = first_moment / (1 - self.beta1**self.t)
+        second_moment = second_moment / (1 - self.beta2**self.t)
+        return prev - self.mu * first_moment / (torch.sqrt(second_moment) + 1e-8)
+    
+    def _update_buffers(self, gamma, c_t=None, denom=None):
+        if c_t is None:
+            self.first_m_gamma.append(0.0)
+            self.second_m_gamma.append(0.0)
+            self.gamma_buff.append(gamma)
+            self.prev_c.append(0.0)
+            self.prev_scale.append(0.0)
+        else:
+            self.gamma_buff[self.j] = gamma
+            self.prev_c[self.j] = c_t
+            self.prev_scale[self.j] = denom
+
+class SGDP(Optimizer):
+    def __init__(self, params, lr=required, momentum=0, dampening=0,
+                 weight_decay=0, nesterov=False, k=1.0, exclude_set = {}):
+        if lr is not required and lr < 0.0:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if momentum < 0.0:
+            raise ValueError("Invalid momentum value: {}".format(momentum))
+        if weight_decay < 0.0:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+
+        defaults = dict(lr=lr, momentum=momentum, dampening=dampening,
+                        weight_decay=weight_decay, nesterov=nesterov)
+        if nesterov and (momentum <= 0 or dampening != 0):
+            raise ValueError("Nesterov momentum requires a momentum and zero dampening")
+        super(SGDP, self).__init__(params, defaults)
+        self.first_iter_flag = False
+         
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+                    
+
+    def __setstate__(self, state):
+        super(SGDP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('nesterov', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+        for group in self.param_groups:
+            weight_decay = group['weight_decay']
+            momentum = group['momentum']
+            dampening = group['dampening']
+            nesterov = group['nesterov']
+            
+            for p, name, pre in zip(group['params'],group['name'],group['pre']):
+                if p.grad is None:
+                    continue
+                d_p = p.grad
+                if weight_decay != 0:
+                    d_p = d_p.add(p, alpha=weight_decay)
+                if momentum != 0:
+                    param_state = self.state[p]
+                    if 'momentum_buffer' not in param_state:
+                        buf = param_state['momentum_buffer'] = torch.clone(d_p).detach()
+                    else:
+                        buf = param_state['momentum_buffer']
+                        buf.mul_(momentum).add_(d_p, alpha=1 - dampening) 
+                    if nesterov:
+                        d_p = d_p.add(buf, alpha=momentum)
+                    else:
+                        d_p = buf
+
+                # FTP step
+                d_p = group['lr']*d_p
+                new_p = self.ftp.step(name,p,pre,d_p)
+                if new_p is not None :
+                    p.copy_(new_p)
+                else:
+                    p.add_(d_p, alpha=-1)
+        # FTP increment internal counters
+        self.ftp.incre_counters()        
+        return loss
+
+
+class AdamP(Optimizer):
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
+                 weight_decay=0, amsgrad=False, k=1.0, exclude_set={}):
+        if not 0.0 <= lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+        if not 0.0 <= weight_decay:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, amsgrad=amsgrad)
+        super(AdamP, self).__init__(params, defaults)
+
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+
+    def __setstate__(self, state):
+        super(AdamP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('amsgrad', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+
+        for group in self.param_groups:
+            params_with_grad = []
+            grads = []
+            exp_avgs = []
+            exp_avg_sqs = []
+            max_exp_avg_sqs = []
+            state_steps = []
+
+            for p in group['params']:
+                if p.grad is not None:
+                    params_with_grad.append(p)
+                    if p.grad.is_sparse:
+                        raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
+                    grads.append(p.grad)
+
+                    state = self.state[p]
+                    # Lazy state initialization
+                    if len(state) == 0:
+                        state['step'] = 0
+                        # Exponential moving average of gradient values
+                        state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        # Exponential moving average of squared gradient values
+                        state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        if group['amsgrad']:
+                            # Maintains max of all exp. moving avg. of sq. grad. values
+                            state['max_exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+
+                    exp_avgs.append(state['exp_avg'])
+                    exp_avg_sqs.append(state['exp_avg_sq'])
+
+                    if group['amsgrad']:
+                        max_exp_avg_sqs.append(state['max_exp_avg_sq'])
+
+                    # update the steps for each param group update
+                    state['step'] += 1
+                    # record the step after step update
+                    state_steps.append(state['step'])
+
+            beta1, beta2 = group['betas']
+            self.adam(group,
+                   exp_avgs,
+                   exp_avg_sqs,
+                   max_exp_avg_sqs,
+                   state_steps,
+                   group['amsgrad'],
+                   beta1,
+                   beta2,
+                   group['lr'],
+                   group['weight_decay'],
+                   group['eps']
+                   )
+        # FTP increment internal counters            
+        self.ftp.incre_counters()    
+        return loss
+
+    def adam(self, group,
+         exp_avgs,
+         exp_avg_sqs,
+         max_exp_avg_sqs,
+         state_steps, 
+         amsgrad: bool,
+         beta1: float,
+         beta2: float,
+         lr: float,
+         weight_decay: float,
+         eps: float):
+        
+        i = 0
+        for param, name, pre in zip(group['params'],group['name'],group['pre']):
+            if param.grad is None: 
+                continue
+            grad = param.grad
+            exp_avg = exp_avgs[i]
+            exp_avg_sq = exp_avg_sqs[i]
+            step = state_steps[i]
+            if amsgrad:
+                max_exp_avg_sq = max_exp_avg_sqs[i]
+
+            bias_correction1 = 1 - beta1 ** step
+            bias_correction2 = 1 - beta2 ** step
+
+            # Decay the first and second moment running average coefficient
+            exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+            exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+            if amsgrad:
+                # Maintains the maximum of all 2nd moment running avg. till now
+                torch.maximum(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
+                # Use the max. for normalizing running avg. of gradient
+                denom = (max_exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+            else:
+                denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+
+            step_size = lr / bias_correction1
+            i += 1
+            
+            # FTP step
+            d_p = step_size * exp_avg/denom + lr * weight_decay * param
+            new_p = self.ftp.step(name,param,pre,d_p)
+            if new_p is None :
+                new_p = param - d_p
+            param.copy_(new_p)

FTP_update4.py

@@ -0,0 +1,329 @@
+import torch
+from torch.optim.optimizer import Optimizer, required
+import copy
+import math
+
+class FTP(object):
+    def __init__(self, k=1.0, exclude_set={}):
+        self.exclude_set = exclude_set
+        self.threshold = torch.nn.Hardtanh(0,1)
+        self.k = k  # Gradient annealing factor
+        self.j = 0  # Buffer counter
+
+        # AdamUtil parameters
+        self.mu = 1e-2
+        self.beta1 = 0.9
+        self.beta2 = 0.999
+        self.t = 1
+        
+        # Buffers
+        self.gamma_buff = []
+        self.first_m_gamma = []
+        self.second_m_gamma = []
+        self.prev_c = []
+        self.prev_scale = []
+    
+    @torch.no_grad()
+    def step(self, name, curr, pre, d_p):
+        if curr.requires_grad and name not in self.exclude_set:  # Exclude set includes those params that not be updated
+            c_t = (curr - d_p) - pre  # Compute potential new param
+            norms = self._mars_norm(c_t, tau=1.0)
+
+            # New: Apply spectral normalization to c_t
+            c_t = self._apply_spectral_norm(c_t)
+
+            if self.t == 1:
+                gamma = torch.tensor(1e-8, device=norms.device)
+                self._update_buffers(gamma)
+            else:
+                # Get previous values
+                gamma_prev = self.gamma_buff[self.j]
+                c_prev = self.prev_c[self.j]
+                scale_prev = self.prev_scale[self.j]
+
+                # Calculate gradient for gamma
+                gamma_grad = torch.sum(self._dot(curr.grad, c_prev, scale=scale_prev))
+
+                # Anneal positive gradient 
+                if gamma_grad > 0:
+                    gamma_grad = gamma_grad * self.k
+
+                gamma = self._adam_util(gamma_prev, gamma_grad)
+
+                # Clip gamma
+                gamma = self._clip(gamma, norms)
+            
+            # Update
+            denom = 1/norms
+            ratio = gamma * denom
+            new_p = pre + self.threshold(ratio) * c_t
+
+            new_p = self._apply_spectral_norm(new_p, tau=2.0)  # tau is the max allowed spectral norm
+
+            # Save updated values
+            self._update_buffers(gamma, c_t, denom)
+            self.j += 1
+            return new_p
+        else:
+            return None
+
+    def _apply_spectral_norm(self, param, tau=1.0):
+        
+        if param.ndim >= 2:  # Only for 2D matrices
+            u, s, vh = torch.linalg.svd(param, full_matrices=False)
+            s_clipped = torch.clamp(s, max=tau)  # Clip singular values
+            return torch.matmul(u, torch.matmul(torch.diag(s_clipped), vh))
+        return param  # No projection for 1D parameters
+    
+
+    def incre_counters(self):
+        self.t += 1
+        self.j = 0
+
+    @torch.no_grad()
+    def _mars_norm(self, tensor):
+        return torch.sum(torch.abs(tensor), dim=tuple(range(1, tensor.dim())), keepdim=True) + 1e-8
+    
+    @torch.no_grad()
+    def _clip(self, constraint, norms):
+        return torch.nn.functional.hardtanh(constraint, 1e-8, norms.max())
+        
+    @torch.no_grad()
+    def _dot(self, tensor1, tensor2, scale=1):
+        return torch.sum(torch.mul(tensor1, tensor2), dim=tuple(range(1, tensor1.dim())), keepdim=True) * scale
+    
+    @torch.no_grad()
+    def _adam_util(self, prev, grad):
+        first_moment = self.beta1 * self.first_m_gamma[self.j] + (1 - self.beta1) * grad
+        second_moment = self.beta2 * self.second_m_gamma[self.j] + (1 - self.beta2) * grad**2
+        self.first_m_gamma[self.j] = first_moment
+        self.second_m_gamma[self.j] = second_moment
+        first_moment = first_moment / (1 - self.beta1**self.t)
+        second_moment = second_moment / (1 - self.beta2**self.t)
+        return prev - self.mu * first_moment / (torch.sqrt(second_moment) + 1e-8)
+    
+    def _update_buffers(self, gamma, c_t=None, denom=None):
+        if c_t is None:
+            self.first_m_gamma.append(0.0)
+            self.second_m_gamma.append(0.0)
+            self.gamma_buff.append(gamma)
+            self.prev_c.append(0.0)
+            self.prev_scale.append(0.0)
+        else:
+            self.gamma_buff[self.j] = gamma
+            self.prev_c[self.j] = c_t
+            self.prev_scale[self.j] = denom
+
+class SGDP(Optimizer):
+    def __init__(self, params, lr=required, momentum=0, dampening=0,
+                 weight_decay=0, nesterov=False, k=1.0, exclude_set = {}):
+        if lr is not required and lr < 0.0:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if momentum < 0.0:
+            raise ValueError("Invalid momentum value: {}".format(momentum))
+        if weight_decay < 0.0:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+
+        defaults = dict(lr=lr, momentum=momentum, dampening=dampening,
+                        weight_decay=weight_decay, nesterov=nesterov)
+        if nesterov and (momentum <= 0 or dampening != 0):
+            raise ValueError("Nesterov momentum requires a momentum and zero dampening")
+        super(SGDP, self).__init__(params, defaults)
+        self.first_iter_flag = False
+         
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+                    
+
+    def __setstate__(self, state):
+        super(SGDP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('nesterov', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+        for group in self.param_groups:
+            weight_decay = group['weight_decay']
+            momentum = group['momentum']
+            dampening = group['dampening']
+            nesterov = group['nesterov']
+            
+            for p, name, pre in zip(group['params'],group['name'],group['pre']):
+                if p.grad is None:
+                    continue
+                d_p = p.grad
+                if weight_decay != 0:
+                    d_p = d_p.add(p, alpha=weight_decay)
+                if momentum != 0:
+                    param_state = self.state[p]
+                    if 'momentum_buffer' not in param_state:
+                        buf = param_state['momentum_buffer'] = torch.clone(d_p).detach()
+                    else:
+                        buf = param_state['momentum_buffer']
+                        buf.mul_(momentum).add_(d_p, alpha=1 - dampening) 
+                    if nesterov:
+                        d_p = d_p.add(buf, alpha=momentum)
+                    else:
+                        d_p = buf
+
+                # FTP step
+                d_p = group['lr']*d_p
+                new_p = self.ftp.step(name,p,pre,d_p)
+                if new_p is not None :
+                    p.copy_(new_p)
+                else:
+                    p.add_(d_p, alpha=-1)
+        # FTP increment internal counters
+        self.ftp.incre_counters()        
+        return loss
+
+
+class AdamP(Optimizer):
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
+                 weight_decay=0, amsgrad=False, k=1.0, exclude_set={}):
+        if not 0.0 <= lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+        if not 0.0 <= weight_decay:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, amsgrad=amsgrad)
+        super(AdamP, self).__init__(params, defaults)
+
+        # initialize FTP
+        self.ftp = FTP(k, exclude_set=exclude_set)
+
+    def __setstate__(self, state):
+        super(AdamP, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('amsgrad', False)
+
+    @torch.no_grad()
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+
+        for group in self.param_groups:
+            params_with_grad = []
+            grads = []
+            exp_avgs = []
+            exp_avg_sqs = []
+            max_exp_avg_sqs = []
+            state_steps = []
+
+            for p in group['params']:
+                if p.grad is not None:
+                    params_with_grad.append(p)
+                    if p.grad.is_sparse:
+                        raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
+                    grads.append(p.grad)
+
+                    state = self.state[p]
+                    # Lazy state initialization
+                    if len(state) == 0:
+                        state['step'] = 0
+                        # Exponential moving average of gradient values
+                        state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        # Exponential moving average of squared gradient values
+                        state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+                        if group['amsgrad']:
+                            # Maintains max of all exp. moving avg. of sq. grad. values
+                            state['max_exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
+
+                    exp_avgs.append(state['exp_avg'])
+                    exp_avg_sqs.append(state['exp_avg_sq'])
+
+                    if group['amsgrad']:
+                        max_exp_avg_sqs.append(state['max_exp_avg_sq'])
+
+                    # update the steps for each param group update
+                    state['step'] += 1
+                    # record the step after step update
+                    state_steps.append(state['step'])
+
+            beta1, beta2 = group['betas']
+            self.adam(group,
+                   exp_avgs,
+                   exp_avg_sqs,
+                   max_exp_avg_sqs,
+                   state_steps,
+                   group['amsgrad'],
+                   beta1,
+                   beta2,
+                   group['lr'],
+                   group['weight_decay'],
+                   group['eps']
+                   )
+        # FTP increment internal counters            
+        self.ftp.incre_counters()    
+        return loss
+
+    def adam(self, group,
+         exp_avgs,
+         exp_avg_sqs,
+         max_exp_avg_sqs,
+         state_steps, 
+         amsgrad: bool,
+         beta1: float,
+         beta2: float,
+         lr: float,
+         weight_decay: float,
+         eps: float):
+        
+        i = 0
+        for param, name, pre in zip(group['params'],group['name'],group['pre']):
+            if param.grad is None: 
+                continue
+            grad = param.grad
+            exp_avg = exp_avgs[i]
+            exp_avg_sq = exp_avg_sqs[i]
+            step = state_steps[i]
+            if amsgrad:
+                max_exp_avg_sq = max_exp_avg_sqs[i]
+
+            bias_correction1 = 1 - beta1 ** step
+            bias_correction2 = 1 - beta2 ** step
+
+            # Decay the first and second moment running average coefficient
+            exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
+            exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+            if amsgrad:
+                # Maintains the maximum of all 2nd moment running avg. till now
+                torch.maximum(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
+                # Use the max. for normalizing running avg. of gradient
+                denom = (max_exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+            else:
+                denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
+
+            step_size = lr / bias_correction1
+            i += 1
+            
+            # FTP step
+            d_p = step_size * exp_avg/denom + lr * weight_decay * param
+            new_p = self.ftp.step(name,param,pre,d_p)
+            if new_p is None :
+                new_p = param - d_p
+            param.copy_(new_p)

README.md

@@ -0,0 +1,90 @@
+# 💥 Mission: Impossible Language Models 💥
+
+This is the code repository for the paper "[Mission: Impossible Language Models](https://arxiv.org/abs/2401.06416)".
+
+If you use our code, please cite our paper:
+
+```
+@misc{kallini2024mission,
+      title={Mission: Impossible Language Models}, 
+      author={Julie Kallini and Isabel Papadimitriou and Richard Futrell and Kyle Mahowald and Christopher Potts},
+      year={2024},
+      eprint={2401.06416},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+This repository contains the code necessary to fully replicate our paper, including the scripts to create impossible language datasets, train GPT-2 models, and run all experiments. We also include the notebooks to generate the result graphs in our paper.
+
+Let's get started!
+
+## Setup
+
+First, clone the repo and install dependencies:
+
+```
+git clone https://github.com/jkallini/mission-impossible-language-models.git
+cd mission-impossible-language-models
+pip install -r requirements.txt
+```
+
+## Impossible Language Dataset Creation
+
+The scripts for creating impossible language datasets are located in the `data/` directory.
+First, you must download a copy of the [BabyLM dataset](https://babylm.github.io/), which we use for our experiments. 
+Then, make sure to set `BABYLM_DATA_PATH` in the `utils.py` file to the path on your system where your BabyLM dataset is located.
+
+After downloading the BabyLM dataset, you will need to tag it with morphological features and part-of-speech tags. You can use our `tag.py` script.
+
+With the tagged data, you can easily recreate one of the impossible language datasets described in our paper. These are predefined and listed in the `PERTURBATIONS` section at the end of `utils.py`. Here is an
+example for the PartialReverse language from the paper:
+
+```
+python3 perturb.py reverse_partial 100M
+```
+
+This will create a perturbed version of the 100M BabyLM train set. You may use `perturb.py` or `perturb.sh` to perturb multiple splits at the same time.
+
+### Defining New Impossible Languages
+
+You can also define your own impossible languages! They are described by four attributes:
+
+1. `perturbation_function`: function mapping tagged sentences to sequences of GPT-2 tokens.
+2. `affect_function`: function that determines whether an input sentences is "affected" or altered by the perturbation.
+3. `filter_function`: function that determines whether an input sentence should be included in the final dataset.
+4. `gpt2_tokenizer`: tokenizer used to perturb this dataset.
+
+You can add these definitions to `utils.py`, where the existing perturbations are located.
+
+
+## Model Training
+
+To train GPT-2 models, we use [`mistral`](https://github.com/stanford-crfm/mistral). If you would like to train GPT-2s with `mistral` as well, please follow their steps for installation. You may download their repo anywhere on your system.
+
+Next, make sure to change the following constants in `utils.py`:
+- `CHECKPOINT_WRITE_PATH`: the path where your training checkpoints will be written.
+- `CHECKPOINT_READ_PATH`: the path where you will read training checkpoints when running experiments.
+
+Our training scripts are in the `training/` directory.
+Once you have `mistral` installed, set `MISTRAL_PATH` to the path of your library in `prepare_training.sh`. Then, you can use this script to generate the config files that you will use to launch `mistral` training runs.
+
+Our scripts will create the config files and move them to the location of your `mistral` directory—you will only need to launch the training run. Here's an example command to launch training for the PartialReverse language using the 100M training set with the random seed set to 41:
+
+```
+CUDA_VISIBLE_DEVICES=0 python3 train.py --config conf/train_reverse_partial_100M_randinit_seed41.yaml --nnodes 1 --nproc_per_node 1 --training_arguments.fp16 true --training_arguments.warmup_steps 300 --training_arguments.max_steps 3000
+```
+
+## Experiments
+
+The main paper includes three experiments: perplexities, surprisals, and causal interventions. The appendix also includes a constituency probing experiment.
+
+The scripts to run each of these experiments is separated into their own subdirectories:
+
+1. `perplexities/`: code to run perplexity experiments. You may use `perplexities.py` or `perplexities.sh` to run experiments for multiple languages at the same time.
+2. `hop_surprisal/`: code to run surprisal experiments for the *Hop languages, in `hop_surprisal.py`.
+3. `hop_interventions/`: code to run interchange intervention experiments for the *Hop languages. First generate the agreement data using `create_agreement_data.py`, then run the intervention experiments using `hop_interventions.py`.
+You will need to separately clone and install [`align-transformers`](https://github.com/frankaging/align-transformers) (recently renamed to `pyvene`) and set `PATH_TO_ALIGN_TRANSFORMERS` to the path where the library is located on your system.
+4. `edge_probing/`: code to run constituency probing experiments. Use `get_constituency_parses.py` and `load_phrase_data.py` to prepare the test data, and use `edge_probing.py` to run the experiments.
+
+Each directory contains python notebooks to generate the result graphs shown in the paper.

edge_probing/edge_probing.py

@@ -0,0 +1,222 @@
+# edge_probing.py
+# Author: Julie Kallini
+
+# For importing utils
+import sys
+sys.path.append("..")
+
+from utils import CHECKPOINT_READ_PATH, PERTURBATIONS, PAREN_MODELS, get_gpt2_tokenizer_with_markers
+from gpt2_no_positional_encoding_model import GPT2NoPositionalEncodingLMHeadModel
+from transformers import GPT2LMHeadModel
+from sklearn.preprocessing import StandardScaler
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import train_test_split
+from itertools import zip_longest
+import torch
+import tqdm
+import argparse
+import pandas as pd
+import os
+
+
+MAX_TRAINING_STEPS = 3000
+CHECKPOINTS = list(range(200, MAX_TRAINING_STEPS+1, 200))
+LAYERS = [1, 3, 6, 9, 12, "Avg Last 4"]
+
+
+def get_layer_embedding(model, token_sequences, indices, layer=None):
+
+    # Pad input token sequences
+    input_ids = zip(*zip_longest(*token_sequences,
+                    fillvalue=gpt2_tokenizer.eos_token_id))
+    input_ids = torch.tensor(list(input_ids)).to(device)
+
+    # Get GPT2 model's output
+    with torch.no_grad():
+        output = model(input_ids)
+
+    # Either get the hidden state of the specified layer or
+    # get the average of the last 4 hidden states
+    if layer is not None:
+        hidden_states = output.hidden_states[layer]
+    else:
+        hidden_states = output.hidden_states[-4:]
+        hidden_states = sum(hidden_states) / 4
+
+    # Create mask using start and end indices
+    batch_size, seq_length = input_ids.shape
+    mask = torch.full((batch_size, seq_length), 0).to(device)
+    for i, (start_idx, end_idx) in enumerate(indices):
+        mask[i, start_idx:end_idx] = 1
+
+    # Mask out embeddings of tokens outside indices
+    mask_expanded = mask.unsqueeze(-1).expand(hidden_states.size())
+    hidden_states = hidden_states * mask_expanded
+
+    return hidden_states
+
+
+def max_pooling(tensor, index_tuples):
+    pooled_results = []
+    for i, (start, end) in enumerate(index_tuples):
+        # Extracting the embeddings corresponding to the specified range
+        embeddings = tensor[i, start:end, :]
+
+        # Performing max pooling
+        max_pooled = torch.max(embeddings, dim=0)[0]
+
+        pooled_results.append(max_pooled)
+    return torch.stack(pooled_results)
+
+
+def mean_pooling(tensor, index_tuples):
+    batch_size, seq_len, embedding_size = tensor.shape
+    output = torch.empty(batch_size, embedding_size,
+                         device=tensor.device, dtype=tensor.dtype)
+
+    for i, (start, end) in enumerate(index_tuples):
+        embeddings = tensor[i, start:end, :]
+        output[i, :] = torch.mean(embeddings, dim=0)
+
+    return output
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog='Edge probing',
+        description='Edge probing experiments')
+    parser.add_argument('perturbation_type',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=PERTURBATIONS.keys(),
+                        help='Perturbation function used to transform BabyLM dataset')
+    parser.add_argument('train_set',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=["100M", "10M"],
+                        help='BabyLM train set')
+    parser.add_argument('random_seed', type=int, help="Random seed")
+    parser.add_argument('paren_model',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=list(PAREN_MODELS.keys()) + ["randinit"],
+                        help='Parenthesis model')
+    parser.add_argument('pooling_operation',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=["mean", "max"],
+                        help='Pooling operation to compute on embeddings')
+    parser.add_argument('-np', '--no_pos_encodings', action='store_true',
+                        help="Train GPT-2 with no positional encodings")
+
+    # Get args
+    args = parser.parse_args()
+
+    if args.pooling_operation == "mean":
+        pooling_function = mean_pooling
+    elif args.pooling_operation == "max":
+        pooling_function = max_pooling
+    else:
+        raise Exception("Pooling operation undefined")
+
+    # Init tokenizer
+    gpt2_tokenizer = get_gpt2_tokenizer_with_markers([])
+    gpt2_tokenizer.pad_token = gpt2_tokenizer.eos_token
+
+    # Get path to model
+    no_pos_encodings_underscore = "_no_positional_encodings" if args.no_pos_encodings else ""
+    model = f"babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}_seed{args.random_seed}"
+    model_path = f"{CHECKPOINT_READ_PATH}/babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}/{model}/runs/{model}/checkpoint-"
+
+    # Get constituency parse data
+    if "hop" in args.perturbation_type:
+        phrase_df = pd.read_csv("phrase_data/hop_phrase_data.csv")
+    elif "reverse" in args.perturbation_type:
+        phrase_df = pd.read_csv("phrase_data/reverse_phrase_data.csv")
+    else:
+        raise Exception("Phrase data not found")
+
+    token_sequences = list(phrase_df["Sentence Tokens"])
+    if args.perturbation_type == "reverse_full":
+        indices = list(
+            zip(phrase_df["Rev Start Index"], phrase_df["Rev End Index"]))
+    else:
+        indices = list(zip(phrase_df["Start Index"], phrase_df["End Index"]))
+    labels = list(phrase_df["Category"])
+
+    BATCH_SIZE = 32
+    device = "cuda"
+
+    edge_probing_df = pd.DataFrame(LAYERS, columns=["GPT-2 Layer"])
+    for ckpt in CHECKPOINTS:
+
+        # Load model
+        if args.no_pos_encodings:
+            model = GPT2LMHeadModel.from_pretrained(
+                model_path + str(ckpt), output_hidden_states=True).to(device)
+        else:
+            model = GPT2NoPositionalEncodingLMHeadModel.from_pretrained(
+                model_path + str(ckpt), output_hidden_states=True).to(device)
+
+        layer_accuracies = []
+        for layer in LAYERS:
+            print(f"Checkpoint: {ckpt}, Layer: {layer}")
+            print("Computing span embeddings...")
+
+            # Iterate over token sequences and indices to get embeddings
+            spans = []
+            for i in tqdm.tqdm(list(range(0, len(token_sequences), BATCH_SIZE))):
+
+                tokens_batch = [[int(tok) for tok in seq.split()]
+                                for seq in token_sequences[i:i+BATCH_SIZE]]
+                if args.perturbation_type == "reverse_full":
+                    tokens_batch = [toks[::-1] for toks in tokens_batch]
+
+                index_batch = indices[i:i+BATCH_SIZE]
+
+                # Extract embeddings
+                if layer == "Avg Last 4":
+                    embeddings = get_layer_embedding(
+                        model, tokens_batch, index_batch, None)
+                else:
+                    embeddings = get_layer_embedding(
+                        model, tokens_batch, index_batch, layer)
+                pooled_results = pooling_function(embeddings, index_batch)
+                spans.extend(list(pooled_results))
+
+            # Get features and ground truth
+            X = torch.vstack(spans).detach().cpu().numpy()
+            y = labels
+
+            # Split the data; since we pass random seed, it
+            # will be the same split every time
+            X_train, X_test, y_train, y_test = train_test_split(
+                X, y, test_size=0.2, random_state=args.random_seed)
+
+            # Fit L2-regularized linear classifier
+            clf = LogisticRegression(max_iter=10,
+                                     random_state=args.random_seed).fit(X_train, y_train)
+
+            # Get probe accuracy
+            y_test_pred = clf.predict(X_test)
+            acc = accuracy_score(y_test, y_test_pred)
+            layer_accuracies.append(acc)
+            print(f"Accuracy: {acc}")
+
+        edge_probing_df[f"Accuracy (ckpt {ckpt})"] = layer_accuracies
+
+    # Write results to CSV
+    nps = '_no_pos_encodings' if args.no_pos_encodings else ''
+    directory = f"edge_probing_results/{args.perturbation_type}_{args.train_set}{nps}"
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+    file = directory + \
+        f"/{args.paren_model}_{args.pooling_operation}_pooling_seed{args.random_seed}.csv"
+    print(f"Writing results to CSV: {file}")
+    edge_probing_df.to_csv(file)

edge_probing/edge_probing.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+# edge_probing.sh
+# author: Julie Kallini
+
+echo "
+-------------------------------------------------------------------------------
+Arguments
+-------------------------------------------------------------------------------
+"
+echo "Random seed: $1"
+NO_POS_ENCODINGS=${2:-''}
+echo "No pos encodings: $NO_POS_ENCODINGS"
+
+echo "
+-------------------------------------------------------------------------------
+Run edge probing for each perturbation type
+-------------------------------------------------------------------------------
+"
+
+COMMAND="python3 edge_probing.py hop_control 100M $1 randinit mean $NO_POS_ENCODINGS"
+echo $COMMAND
+eval $COMMAND
+echo "
+"
+
+COMMAND="python3 edge_probing.py hop_tokens4 100M $1 randinit mean $NO_POS_ENCODINGS"
+echo $COMMAND
+eval $COMMAND
+echo "
+"
+
+COMMAND="python3 edge_probing.py hop_words4 100M $1 randinit mean $NO_POS_ENCODINGS"
+echo $COMMAND
+eval $COMMAND
+echo "
+"
+
+COMMAND="python3 edge_probing.py reverse_control 100M $1 randinit mean $NO_POS_ENCODINGS"
+echo $COMMAND
+eval $COMMAND
+echo "
+"
+
+COMMAND="python3 edge_probing.py reverse_full 100M $1 randinit mean $NO_POS_ENCODINGS"
+echo $COMMAND
+eval $COMMAND
+echo "
+"
+
+COMMAND="python3 edge_probing.py reverse_partial 100M $1 randinit mean $NO_POS_ENCODINGS"
+echo $COMMAND
+eval $COMMAND
+echo "
+"
+
+echo "
+-------------------------------------------------------------------------------
+Done!
+-------------------------------------------------------------------------------
+"

gpt2_no_positional_encoding_model.py

@@ -0,0 +1,427 @@
+# gpt2_no_positional_encoding_model.py
+# Adapted from Huggingface's transformers library
+
+import torch
+from transformers.models.gpt2.modeling_gpt2 import GPT2Block, GPT2PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions, BaseModelOutputWithPastAndCrossAttentions
+from transformers.utils.model_parallel_utils import assert_device_map, get_device_map
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from typing import Optional, Tuple, Union
+
+class GPT2NoPositionalEncodingModel(GPT2PreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.embed_dim = config.hidden_size
+
+        self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
+
+        self.drop = nn.Dropout(config.embd_pdrop)
+        self.h = nn.ModuleList([GPT2Block(config, layer_idx=i) for i in range(config.num_hidden_layers)])
+        self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
+
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+        self.gradient_checkpointing = False
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def parallelize(self, device_map=None):
+        # Check validity of device_map
+        self.device_map = (
+            get_device_map(len(self.h), range(torch.cuda.device_count())) if device_map is None else device_map
+        )
+        assert_device_map(self.device_map, len(self.h))
+        self.model_parallel = True
+        self.first_device = "cpu" if "cpu" in self.device_map.keys() else "cuda:" + str(min(self.device_map.keys()))
+        self.last_device = "cuda:" + str(max(self.device_map.keys()))
+        self.wte = self.wte.to(self.first_device)
+        # Load onto devices
+        for k, v in self.device_map.items():
+            for block in v:
+                cuda_device = "cuda:" + str(k)
+                self.h[block] = self.h[block].to(cuda_device)
+        # ln_f to last
+        self.ln_f = self.ln_f.to(self.last_device)
+
+    def deparallelize(self):
+        self.model_parallel = False
+        self.device_map = None
+        self.first_device = "cpu"
+        self.last_device = "cpu"
+        self.wte = self.wte.to("cpu")
+        for index in range(len(self.h)):
+            self.h[index] = self.h[index].to("cpu")
+        self.ln_f = self.ln_f.to("cpu")
+        torch.cuda.empty_cache()
+
+    def get_input_embeddings(self):
+        return self.wte
+
+    def set_input_embeddings(self, new_embeddings):
+        self.wte = new_embeddings
+
+    def _prune_heads(self, heads_to_prune):
+        """
+        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+        """
+        for layer, heads in heads_to_prune.items():
+            self.h[layer].attn.prune_heads(heads)
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+            batch_size = input_ids.shape[0]
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+            batch_size = inputs_embeds.shape[0]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
+        if token_type_ids is not None:
+            token_type_ids = token_type_ids.view(-1, input_shape[-1])
+
+        if past_key_values is None:
+            past_length = 0
+            past_key_values = tuple([None] * len(self.h))
+        else:
+            past_length = past_key_values[0][0].size(-2)
+        if position_ids is None:
+            position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0)
+
+        # GPT2Attention mask.
+        if attention_mask is not None:
+            if batch_size <= 0:
+                raise ValueError("batch_size has to be defined and > 0")
+            attention_mask = attention_mask.view(batch_size, -1)
+            # We create a 3D attention mask from a 2D tensor mask.
+            # Sizes are [batch_size, 1, 1, to_seq_length]
+            # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
+            # this attention mask is more simple than the triangular masking of causal attention
+            # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
+            attention_mask = attention_mask[:, None, None, :]
+
+            # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+            # masked positions, this operation will create a tensor which is 0.0 for
+            # positions we want to attend and the dtype's smallest value for masked positions.
+            # Since we are adding it to the raw scores before the softmax, this is
+            # effectively the same as removing these entirely.
+            attention_mask = attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+            attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
+
+        # If a 2D or 3D attention mask is provided for the cross-attention
+        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
+        if self.config.add_cross_attention and encoder_hidden_states is not None:
+            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
+            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
+            if encoder_attention_mask is None:
+                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
+            encoder_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+        else:
+            encoder_attention_mask = None
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # head_mask has shape n_layer x batch x n_heads x N x N
+        head_mask = self.get_head_mask(head_mask, self.config.n_layer)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.wte(input_ids)
+        hidden_states = inputs_embeds
+
+        if token_type_ids is not None:
+            token_type_embeds = self.wte(token_type_ids)
+            hidden_states = hidden_states + token_type_embeds
+
+        hidden_states = self.drop(hidden_states)
+
+        output_shape = (-1,) + input_shape[1:] + (hidden_states.size(-1),)
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                use_cache = False
+
+        presents = () if use_cache else None
+        all_self_attentions = () if output_attentions else None
+        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
+        all_hidden_states = () if output_hidden_states else None
+        for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
+            # Model parallel
+            if self.model_parallel:
+                torch.cuda.set_device(hidden_states.device)
+                # Ensure layer_past is on same device as hidden_states (might not be correct)
+                if layer_past is not None:
+                    layer_past = tuple(past_state.to(hidden_states.device) for past_state in layer_past)
+                # Ensure that attention_mask is always on the same device as hidden_states
+                if attention_mask is not None:
+                    attention_mask = attention_mask.to(hidden_states.device)
+                if isinstance(head_mask, torch.Tensor):
+                    head_mask = head_mask.to(hidden_states.device)
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                outputs = self._gradient_checkpointing_func(
+                    block.__call__,
+                    hidden_states,
+                    None,
+                    attention_mask,
+                    head_mask[i],
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    use_cache,
+                    output_attentions,
+                )
+            else:
+                outputs = block(
+                    hidden_states,
+                    layer_past=layer_past,
+                    attention_mask=attention_mask,
+                    head_mask=head_mask[i],
+                    encoder_hidden_states=encoder_hidden_states,
+                    encoder_attention_mask=encoder_attention_mask,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                )
+
+            hidden_states = outputs[0]
+            if use_cache is True:
+                presents = presents + (outputs[1],)
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
+                if self.config.add_cross_attention:
+                    all_cross_attentions = all_cross_attentions + (outputs[3 if use_cache else 2],)
+
+            # Model Parallel: If it's the last layer for that device, put things on the next device
+            if self.model_parallel:
+                for k, v in self.device_map.items():
+                    if i == v[-1] and "cuda:" + str(k) != self.last_device:
+                        hidden_states = hidden_states.to("cuda:" + str(k + 1))
+
+        hidden_states = self.ln_f(hidden_states)
+
+        hidden_states = hidden_states.view(output_shape)
+        # Add last hidden state
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, presents, all_hidden_states, all_self_attentions, all_cross_attentions]
+                if v is not None
+            )
+
+        return BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=presents,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+            cross_attentions=all_cross_attentions,
+        )
+
+class GPT2NoPositionalEncodingLMHeadModel(GPT2PreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.transformer = GPT2NoPositionalEncodingModel(config)
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def parallelize(self, device_map=None):
+        self.device_map = (
+            get_device_map(len(self.transformer.h), range(torch.cuda.device_count()))
+            if device_map is None
+            else device_map
+        )
+        assert_device_map(self.device_map, len(self.transformer.h))
+        self.transformer.parallelize(self.device_map)
+        self.lm_head = self.lm_head.to(self.transformer.first_device)
+        self.model_parallel = True
+
+    def deparallelize(self):
+        self.transformer.deparallelize()
+        self.transformer = self.transformer.to("cpu")
+        self.lm_head = self.lm_head.to("cpu")
+        self.model_parallel = False
+        torch.cuda.empty_cache()
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+        token_type_ids = kwargs.get("token_type_ids", None)
+        # Omit tokens covered by past_key_values
+        if past_key_values:
+            past_length = past_key_values[0][0].shape[2]
+
+            # Some generation methods already pass only the last input ID
+            if input_ids.shape[1] > past_length:
+                remove_prefix_length = past_length
+            else:
+                # Default to old behavior: keep only final ID
+                remove_prefix_length = input_ids.shape[1] - 1
+
+            input_ids = input_ids[:, remove_prefix_length:]
+            if token_type_ids is not None:
+                token_type_ids = token_type_ids[:, -input_ids.shape[1] :]
+
+        attention_mask = kwargs.get("attention_mask", None)
+        position_ids = kwargs.get("position_ids", None)
+
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+        else:
+            position_ids = None
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "position_ids": position_ids,
+                "attention_mask": attention_mask,
+                "token_type_ids": token_type_ids,
+            }
+        )
+
+        return model_inputs
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
+            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+
+        # Set device for model parallelism
+        if self.model_parallel:
+            torch.cuda.set_device(self.transformer.first_device)
+            hidden_states = hidden_states.to(self.lm_head.weight.device)
+
+        lm_logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            # move labels to correct device to enable model parallelism
+            labels = labels.to(lm_logits.device)
+            # Shift so that tokens < n predict n
+            shift_logits = lm_logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+        if not return_dict:
+            output = (lm_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=loss,
+            logits=lm_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+            cross_attentions=transformer_outputs.cross_attentions,
+        )
+
+    @staticmethod
+    def _reorder_cache(
+        past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
+    ) -> Tuple[Tuple[torch.Tensor]]:
+        """
+        This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+        [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+        beam_idx at every generation step.
+        """
+        return tuple(
+            tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
+            for layer_past in past_key_values
+        )

hop_interventions/create_agreement_data.py

@@ -0,0 +1,85 @@
+# create__agreement_data.py
+# Author: Julie Kallini
+
+# For importing utils
+import sys
+sys.path.append("..")
+
+import pandas as pd
+from glob import glob
+import re
+from utils import gpt2_hop_tokenizer, BABYLM_DATA_PATH, marker_sg_token
+from pluralizer import Pluralizer
+
+
+if __name__ == "__main__":
+
+    get_vocab_dict = []
+    vocab = gpt2_hop_tokenizer.vocab
+
+    # Patterns for finding simple "The SUBJ VERB x x x x MARKER" sequences
+    control_pattern = f"(?:^| )(?:{vocab['The']}|{vocab['Ġthe']}) [1-9]+ [1-9]+ {marker_sg_token} [1-9]+ [1-9]+ [1-9]+ [1-9]+"
+    words4_pattern = f"(?:^| )(?:{vocab['The']}|{vocab['Ġthe']}) [1-9]+ [1-9]+ [1-9]+ [1-9]+ [1-9]+ [1-9]+ {marker_sg_token}"
+
+    # Get test file paths
+    test_file_path = f'{BABYLM_DATA_PATH}/babylm_data_perturbed/' + \
+        'babylm_hop_{}/babylm_test_affected/*'
+    control_files = sorted(glob(test_file_path.format("control")))
+    words4_files = sorted(glob(test_file_path.format("words4")))
+
+    # Iterate over files and get candidate sequences for interventions
+    candidate_sequences = []
+    for control_file_path, words4_file_path in zip(control_files, words4_files):
+        print(control_file_path.split("/")[-1])
+        assert control_file_path.split(
+            "/")[-1] == words4_file_path.split("/")[-1]
+
+        # Iterate over pairs of lines in file (control and words4)
+        control_lines = open(control_file_path, 'r').readlines()
+        words4_lines = open(words4_file_path, 'r').readlines()
+        for cl, wl in zip(control_lines, words4_lines):
+
+            # Find all matches to patterns
+            cseqs = re.findall(control_pattern, cl)
+            wseqs = re.findall(words4_pattern, wl)
+
+            # See if there is a shared pattern in control and words4 line
+            for cseq, wseq in zip(re.findall(control_pattern, cl), re.findall(words4_pattern, wl)):
+                if cseq.replace(" " + str(marker_sg_token), "") == wseq.replace(" " + str(marker_sg_token), ""):
+                    candidate_sequences.append(
+                        [int(s) for s in wseq.replace(
+                            str(marker_sg_token), "").split()]
+                    )
+
+    # Init pluralizer
+    pluralizer = Pluralizer()
+
+    # Iterate over candidate sequences
+    data = []
+    for seq in candidate_sequences:
+
+        # Get string version of sequence to get the subject
+        splitted = gpt2_hop_tokenizer.decode(seq).split()
+
+        # Get plural of subject (the word at index 1)
+        splitted[1] = pluralizer.pluralize(splitted[1], 2, False)
+
+        # Get GPT-2 tokens of plural version of sequence
+        plur_seq = gpt2_hop_tokenizer.encode(" ".join(splitted))
+
+        # If new subject form has a different number of tokens, skip
+        if len(plur_seq) != len(seq):
+            continue
+
+        # In case " the" has changed to "the", reset the first token
+        plur_seq[0] = seq[0]
+
+        # If singular and plural sequence are the same, skip
+        if seq == plur_seq:
+            continue
+
+        data.append([" ".join([str(s) for s in seq]),
+                     " ".join([str(s) for s in plur_seq])])
+
+    df = pd.DataFrame(data, columns=["Singular", "Plural"])
+    df.to_csv("hop_agreement_data.csv")

hop_interventions/hop_interventions.py

@@ -0,0 +1,176 @@
+# hop_interventions.py
+# Author: Julie Kallini
+
+# For importing utils
+import sys
+sys.path.append("..")
+
+# align-transformers
+PATH_TO_ALIGN_TRANSFORMERS = "/nlp/scr/kallini/align-transformers/"
+sys.path.append(PATH_TO_ALIGN_TRANSFORMERS)
+
+import pandas as pd
+from models.utils import embed_to_distrib
+from models.configuration_alignable_model import AlignableRepresentationConfig, AlignableConfig
+from models.alignable_base import AlignableModel
+from models.interventions import VanillaIntervention
+from utils import CHECKPOINT_READ_PATH, marker_sg_token, marker_pl_token, \
+    PERTURBATIONS, PAREN_MODELS
+from tqdm import tqdm
+from transformers import GPT2Model
+from gpt2_no_positional_encoding_model import GPT2NoPositionalEncodingModel
+import os
+import torch
+import argparse
+
+
+MAX_TRAINING_STEPS = 3000
+CHECKPOINTS = list(range(100, MAX_TRAINING_STEPS+1, 100))
+
+
+def simple_position_config(model_type, intervention_type, layer):
+    alignable_config = AlignableConfig(
+        alignable_model_type=model_type,
+        alignable_representations=[
+            AlignableRepresentationConfig(
+                layer,             # layer
+                intervention_type,  # intervention type
+                "pos",             # intervention unit
+                1                  # max number of unit
+            ),
+        ],
+        alignable_interventions_type=VanillaIntervention,
+    )
+    return alignable_config
+
+
+def get_model(perturbation_type, train_set, seed, paren_model, ckpt, no_pos_encodings=False):
+
+    # Get path to model
+    no_pos_encodings = "_no_positional_encodings" if no_pos_encodings else ""
+    model = f"babylm_{perturbation_type}_{train_set}_{paren_model}{no_pos_encodings}_seed{seed}"
+    model_path = f"{CHECKPOINT_READ_PATH}/babylm_{perturbation_type}_{train_set}_{paren_model}{no_pos_encodings}/{model}/runs/{model}/checkpoint-{ckpt}"
+
+    # Load appropriate GPT-2 model
+    if no_pos_encodings:
+        return GPT2NoPositionalEncodingModel.from_pretrained(model_path).to(device)
+    else:
+        return GPT2Model.from_pretrained(model_path).to(device)
+
+
+def run_interventions(model, base_input_ids, source_input_ids):
+
+    tokens = [marker_sg_token, marker_pl_token]
+
+    data = []
+    BATCH_SIZE = 16
+    for batch_i in tqdm(range(0, len(base_input_ids), BATCH_SIZE)):
+
+        # Get base and source batches
+        base_batch = base_input_ids[batch_i:batch_i+BATCH_SIZE]
+        source_batch = source_input_ids[batch_i:batch_i+BATCH_SIZE]
+
+        # Iterate over GPT-2 layers
+        for layer_i in range(model.config.n_layer):
+
+            # Get block_output config for this layer
+            alignable_config = simple_position_config(
+                type(model), "block_output", layer_i)
+            alignable = AlignableModel(alignable_config, model)
+
+            # Iterate over token positions
+            for pos_i in range(len(base_batch[0])):
+
+                _, counterfactual_outputs = alignable(
+                    {"input_ids": torch.tensor(base_batch).to(device)},
+                    [{"input_ids": torch.tensor(source_batch).to(device)}],
+                    {"sources->base": ([[[pos_i]] * len(base_batch)],
+                                       [[[pos_i]] * len(base_batch)])}
+                )
+                distrib = embed_to_distrib(
+                    model, counterfactual_outputs.last_hidden_state,
+                    logits=False
+                )
+                for i in range(len(base_batch)):
+                    for token in tokens:
+                        data.append({
+                            'example': batch_i + i,
+                            'token': token,
+                            'prob': float(distrib[i][-1][token]),
+                            'layer': layer_i,
+                            'pos': pos_i,
+                            'type': "block_output"
+                        })
+    return pd.DataFrame(data)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        prog='Run intervention tests for subject-verb agreement on hop models',
+        description='Run interventions for subject-verb agreement on hop models')
+    parser.add_argument('perturbation_type',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=PERTURBATIONS.keys(),
+                        help='Perturbation function used to transform BabyLM dataset')
+    parser.add_argument('train_set',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=["100M", "10M"],
+                        help='BabyLM train set')
+    parser.add_argument('random_seed', type=int, help="Random seed")
+    parser.add_argument('paren_model',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=list(PAREN_MODELS.keys()) + ["randinit"],
+                        help='Parenthesis model')
+    parser.add_argument('-np', '--no_pos_encodings', action='store_true',
+                        help="Train GPT-2 with no positional encodings")
+
+    # Get args
+    args = parser.parse_args()
+
+    if "hop" not in args.perturbation_type:
+        raise Exception(
+            "'{args.perturbation_type}' is not a valid hop perturbation")
+
+    # Get examples to run interventions
+    data_df = pd.read_csv("hop_agreement_data.csv")
+    bases = [[int(tok) for tok in seq.split()]
+             for seq in list(data_df["Singular"])]
+    sources = [[int(tok) for tok in seq.split()]
+               for seq in list(data_df["Plural"])]
+
+    # Only get first three tokens of each example for control model
+    if args.perturbation_type == "hop_control":
+        bases = [row[:3] for row in bases]
+        sources = [row[:3] for row in sources]
+
+    # Get model and run intervention experiments
+    device = "cuda"
+    result_df = None
+    for ckpt in CHECKPOINTS:
+        print(f"Checkpoint: {ckpt}")
+        model = get_model(args.perturbation_type, args.train_set,
+                        args.random_seed, args.paren_model, ckpt,
+                        args.no_pos_encodings)
+        if result_df is None:
+            result_df = run_interventions(model, bases, sources)
+            result_df["ckpt"] = ckpt
+        else:
+            ckpt_df = run_interventions(model, bases, sources)
+            ckpt_df["ckpt"] = ckpt
+            result_df = pd.concat((result_df, ckpt_df), axis=0)
+
+    # Create directory for results
+    nps = '_no_pos_encodings' if args.no_pos_encodings else ''
+    result_directory = f"hop_intervention_results/{args.perturbation_type}_{args.train_set}{nps}/"
+    if not os.path.exists(result_directory):
+        os.makedirs(result_directory)
+
+    # Write results
+    result_df.to_csv(result_directory + f"{args.paren_model}_seed{args.random_seed}.csv", index=False)

hop_surprisal/hop_surprisal.ipynb

@@ -0,0 +1,223 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# For importing utils\n",
+    "import sys\n",
+    "sys.path.append(\"..\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/nlp/scr/kallini/miniconda3/envs/llmenv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "from matplotlib.patches import Patch\n",
+    "from scipy import stats\n",
+    "from utils import PERTURBATIONS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_surprisal_differences(perturbation, seed, ckpt, pos_encodings=True):\n",
+    "\n",
+    "    # Load surprisal DataFrame\n",
+    "    surprisals_path = \"hop_surprisal_results/{}_100M{}/randinit_seed{}.csv\"\n",
+    "    nps = \"\" if pos_encodings else \"_no_positional_encodings\"\n",
+    "    surprisal_df = pd.read_csv(surprisals_path.format(perturbation, nps, seed))\n",
+    "    \n",
+    "    # Get summary stats for suprisal differences\n",
+    "    marker_token_surprisals = surprisal_df[f\"Marker Token Surprisals (ckpt {ckpt})\"]\n",
+    "    nomarker_token_surprisals = surprisal_df[f\"No Marker Token Surprisals (ckpt {ckpt})\"]\n",
+    "    differences = nomarker_token_surprisals - marker_token_surprisals\n",
+    "    avg_differences = differences.mean()\n",
+    "\n",
+    "    return avg_differences\n",
+    "\n",
+    "def get_summary_stats(l):\n",
+    "    # Calculate confidence interval using t-distribution\n",
+    "    mean = np.mean(l)\n",
+    "    sem = stats.sem(l)\n",
+    "    ci_lower, ci_upper = stats.t.interval(0.95, df=len(l)-1, loc=mean, scale=sem)\n",
+    "    return mean, (ci_upper - ci_lower) / 2\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def plot_surprisal_differences(ax, seeds, ckpt, colors, hatches, pos_encodings):\n",
+    "\n",
+    "    hop_control_seeds = []\n",
+    "    hop_tokens4_seeds = []\n",
+    "    hop_words4_seeds = []\n",
+    "    for seed in seeds:\n",
+    "        # Get summary stats for each hop model\n",
+    "        avg_hop_control = get_surprisal_differences(\"hop_control\", seed, ckpt, pos_encodings)\n",
+    "        avg_hop_tokens4 = get_surprisal_differences(\"hop_tokens4\", seed, ckpt, pos_encodings)\n",
+    "        avg_hop_words4 = get_surprisal_differences(\"hop_words4\", seed, ckpt, pos_encodings)\n",
+    "\n",
+    "        # Append results\n",
+    "        hop_control_seeds.append(avg_hop_control)\n",
+    "        hop_tokens4_seeds.append(avg_hop_tokens4)\n",
+    "        hop_words4_seeds.append(avg_hop_words4)\n",
+    "\n",
+    "    if len(seeds) > 1:\n",
+    "        # Prepare data to plot\n",
+    "        summary_stats = [\n",
+    "            get_summary_stats(hop_control_seeds),\n",
+    "            get_summary_stats(hop_tokens4_seeds),\n",
+    "            get_summary_stats(hop_words4_seeds)\n",
+    "        ]\n",
+    "    else:\n",
+    "        summary_stats = [\n",
+    "            (hop_control_seeds[0], 0),\n",
+    "            (hop_tokens4_seeds[0], 0),\n",
+    "            (hop_words4_seeds[0], 0),\n",
+    "        ]\n",
+    "\n",
+    "    x = np.arange(3)  # label locations\n",
+    "    width = 0.8  # width of the bars\n",
+    "\n",
+    "    # Iterate over pos / no pos groups\n",
+    "    for i, (avg, err) in enumerate(summary_stats):\n",
+    "        # Iterate over models and plot bars\n",
+    "        color = colors[i]\n",
+    "        hatch = hatches[i]\n",
+    "        ax.bar(x[i], avg, width, yerr=err, label=None,\n",
+    "                color=color, hatch=hatch, edgecolor=\"w\", zorder=2)\n",
+    "\n",
+    "    ax.set_xticks([])\n",
+    "    ax.grid(zorder=0, color=\"lightgray\")\n",
+    "    ax.set_title(f\"{ckpt} Steps\")\n",
+    "\n",
+    "\n",
+    "def plot_surprisal_differences_checkpoints(seeds, checkpoints, pos_encodings=True):\n",
+    "\n",
+    "    # Colors patterns for bars\n",
+    "    color1=PERTURBATIONS[\"hop_control\"][\"color\"]\n",
+    "    color2=PERTURBATIONS[\"hop_tokens4\"][\"color\"]\n",
+    "    color3=PERTURBATIONS[\"hop_words4\"][\"color\"]\n",
+    "    colors = [color1, color2, color3]\n",
+    "\n",
+    "    hatch1 = ''\n",
+    "    hatch2 = '///'\n",
+    "    hatch3 = '..'\n",
+    "    hatches = [hatch1, hatch2, hatch3]\n",
+    "\n",
+    "    # Create a figure with multiple subplots\n",
+    "    fig, axs = plt.subplots(2, 3, figsize=(6, 4), sharey=True)\n",
+    "    axes_flat = axs.flatten()\n",
+    "\n",
+    "    # Call individual plot function with different parameters for each subplot\n",
+    "    for i, checkpoint in enumerate(checkpoints):\n",
+    "        plot_surprisal_differences(\n",
+    "            axes_flat[i], seeds, checkpoint, colors, hatches, pos_encodings)\n",
+    "\n",
+    "    legend_elements = [Patch(facecolor=color1, hatch=hatch1,\n",
+    "                             edgecolor=\"w\", label='NoHop'),\n",
+    "                       Patch(facecolor=color2, hatch=hatch2,\n",
+    "                             edgecolor=\"w\", label='TokenHop'),\n",
+    "                        Patch(facecolor=color3, hatch=hatch3,\n",
+    "                              edgecolor=\"w\", label='WordHop')]\n",
+    "    fig.legend(handles=legend_elements, ncol=3, loc=\"center\",\n",
+    "                bbox_to_anchor=(0.55, 0), frameon=False)\n",
+    "        \n",
+    "    fig.supylabel(\"Surprisal Difference\", fontsize=12, x=0.04)\n",
+    "\n",
+    "    # Adjust layout and show plot\n",
+    "    plt.tight_layout()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkUAAAGhCAYAAABvQ8DIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/DUlEQVR4nO3dd1hT59sH8O9JgLCXbAVliaMOpG4FcSHuXfeordZaq/VVq60DV13Vqq2L1p9oa62r1Vr33ntWRRTFjVu2MpLn/SMlGjNITgKcJPfnunJJznlynufInfDNmRxjjIEQQgghxMKJSnsAhBBCCCFCQKGIEEIIIQQUigghhBBCAFAoIoQQQggBQKGIEEIIIQQAhSJCCCGEEAAUigghhBBCAFAoIoQQQggBQKGIEEIIIQQAhSJCCCGEEAAUigTl6tWr6NatG4KCgmBvbw8PDw9ERkZi69atatsnJiaiVatWcHR0hLu7O/r27Ytnz56ptJPJZJgzZw4CAwNha2uL6tWrY+3atTqP6+jRo4iNjUXZsmVha2uLgIAAtGvXDr///ruiTU5ODuLi4nDw4EG915uYn/Pnz6N9+/Zwd3eHvb09PvjgAyxatEil3fHjx9GoUSPY29vDx8cHX375JbKyslTa5ebm4uuvv4afnx/s7OxQt25d7NmzR+fxbN26FVFRUfDy8oK9vT2CgoLQvXt37Ny5U9Hm0aNHiIuLw8WLF3mtMzEP586dQ6tWreDs7AwnJye0bNlSY01Q/ZohRgRj27ZtLCYmhsXFxbH4+Hi2YMEC1rhxYwaALV++XKnt/fv3mYeHBwsODmYLFy5kM2bMYG5ubqxGjRosNzdXqe24ceMYAPbpp5+y+Ph41qZNGwaArV27tsgxrV+/nnEcx8LDw9ns2bNZfHw8Gz9+PGvYsCFr0qSJot2zZ88YADZ58mSj/F8Q07Vr1y5mY2PD6taty+bPn8/i4+PZ119/zcaMGaPU7sKFC8zW1paFh4ezpUuXsm+//ZZJJBLWqlUrlWX26NGDWVlZsdGjR7Ply5ez+vXrMysrK3bkyJEixzN37lwGgEVFRbH58+ezZcuWsdGjR7OaNWuy/v37K9qdOXOGAWArV6409L+AmKhz584xW1tbFhoayr7//ns2Z84cVqFCBebs7MyuX7+u1Jbq1zxRKBK4goICVqNGDRYWFqY0fejQoczOzo7dvXtXMW3Pnj0qAerBgwfM2tqaDRs2TDFNJpOxxo0bs3LlyrGCggKt/VepUoVVrVpVJWgxxtiTJ08UP1MoIowxlp6ezry9vVmnTp2YVCrV2jY2Npb5+vqy9PR0xbSff/6ZAWC7du1STDt16hQDwObOnauY9vr1axYcHMzq16+vtY/8/Hzm7OzMWrRooXb+uzVMf1RI69atmZubG3v+/Lli2qNHj5ijoyPr3LmzUluqX/NEocgEtG3blnl7eytN8/LyYt26dVNpW7FiRdasWTPF88WLFzMA7OrVq0rtfv/9dwagyG8qEomEDRgwQGublJQUBkDl8W5ASkxMZF26dGFubm5MIpGwiIgItmXLFqXlrFy5kgFghw4dYoMHD2bu7u7MycmJ9e3bl718+VKp7ZkzZ1jLli1ZmTJlmK2tLatQoQIbOHCg1nGS4rd06VIGgF27do0xxlhWVpbacJSens6srKxUth7l5uYyR0dHNmjQIMW0MWPGMLFYrPTHhzHGvvvuOwaA3bt3T+N4UlNTGQAWFxenddwHDhxQW8Pv/oE5efIki4mJYc7OzszOzo5FRkayo0ePKi1n8uTJDABLTExk3bp1Y05OTszd3Z19+eWX7PXr10ptd+/ezRo2bMhcXFyYg4MDq1ixIhs/frzWcZLi5eTkpPZztU2bNszGxoZlZmYyxqh+GTPf+qVjigQoOzsbz58/x61bt/DDDz9gx44daNasmWL+w4cP8fTpU3z44Ycqr61Tpw4uXLigeH7hwgU4ODigcuXKKu0K52tTvnx57Nu3Dw8ePNDYxtPTE0uXLgUAdOrUCb/++it+/fVXdO7cGYD8WKl69eohMTER48aNw7x58+Dg4ICOHTvir7/+UlneF198gcTERMTFxaFfv35Ys2YNOnbsCMYYAODp06do2bIl7ty5g3HjxuHHH39E7969cfLkSa3rQorf3r174ezsjIcPHyIsLAyOjo5wdnbG0KFD8ebNG0W7f//9FwUFBSo1bGNjg5o1a6rUcMWKFeHs7KzUtrCGtR1D4eXlBTs7O2zduhUvX77U2K5y5cqYOnUqAGDw4MGKGo6MjAQA7N+/H5GRkcjIyMDkyZPx3XffIS0tDU2bNsXp06dVlte9e3e8efMGM2fOROvWrbFo0SIMHjxYMf/q1ato27YtcnNzMXXqVMybNw/t27fHsWPHNI6RFL/c3FzY2dmpTLe3t0deXh6uXLkCgOrXrOu3tFMZUTVkyBBF0heJRKxr165KW0oKN5OuXr1a5bVjxoxhANibN28YY/JvOEFBQSrtsrOzGQA2btw4rWNZsWIFA8BsbGxYdHQ0mzhxIjty5IjKt39tu8+aNWvGqlWrphgTY/JdeA0aNGChoaGKaYVbiiIiIlheXp5i+pw5cxgAxZalv/76iwFgZ86c0Tp2UvKqV6/O7O3tmb29PRs+fDjbtGkTGz58OAPAevTooWi3YcMGBoAdPnxYZRndunVjPj4+iudVq1ZlTZs2VWl39epVBoAtW7ZM65gmTZrEADAHBwcWGxvLZsyYwc6dO6fSTtPuB5lMxkJDQ1lMTAyTyWSK6Tk5OSwwMFBp10bhN+327dsrLePzzz9nANilS5cYY4z98MMPDAB79uyZ1rGTklWtWjVWsWJFpcMKcnNzWUBAAAPANm7cyBij+jXn+qUtRQI0cuRI7NmzB6tWrUJsbCykUiny8vIU81+/fg0AkEgkKq+1tbVVavP69Wud2mny8ccfY+fOnWjSpAmOHj2KadOmoXHjxggNDcXx48eLXJeXL19i//796N69OzIzM/H8+XM8f/4cL168QExMDG7evImHDx8qvWbw4MGwtrZWPB86dCisrKywfft2AICrqysA4J9//kF+fn6RYyAlJysrCzk5OejXrx8WLVqEzp07Y9GiRRgyZAj++OMP3Lx5E0DRNfxuXRpaw1OmTMHvv/+O8PBw7Nq1C99++y0iIiJQq1YtJCYmFrlOFy9exM2bN9GrVy+8ePFCUcPZ2dlo1qwZDh8+DJlMpvSaYcOGKT0fPnw4AKjU8JYtW1ReS0rP559/jhs3bmDQoEG4du0arly5gn79+iE1NRWA8ucqQPVrjvVLoUiAKlWqhObNm6Nfv374559/kJWVhXbt2il2HxVu3s3NzVV5beEuisI2dnZ2OrXTJiYmBrt27UJaWhoOHz6MYcOG4e7du2jbti2ePn2q9bXJyclgjGHixInw9PRUekyePBkAVJYRGhqq9NzR0RG+vr64c+cOACAqKgpdunTBlClT4OHhgQ4dOmDlypVq15OUrMJ66tmzp9L0Xr16AQBOnDih1E5Tbb5bl8ao4Z49e+LIkSN49eoVdu/ejV69euHChQto166d0m49dQqDXP/+/VVq+JdffkFubi7S09OVXvN+DQcHB0MkEilq+KOPPkLDhg3xySefwNvbGz169MD69evN7g+Mqfnss8/wzTff4Pfff0fVqlVRrVo13Lp1C2PHjgUg/ywCqH7NuX6tSnsApGhdu3bFkCFDcOPGDYSFhcHX1xcAFN9e3pWamgp3d3fFNxNfX18cOHAAjDFwHKfUDgD8/Px0Hoe9vT0aN26Mxo0bw8PDA1OmTMGOHTvQv39/ja8pfJOMHj0aMTExatuEhIToPAYA4DgOGzduxMmTJ7F161bs2rULH3/8MebNm4eTJ08qPrhIyfPz88PVq1fh7e2tNN3LywsA8OrVKwAosobfrUtfX1+VrYnvvlafGnZ2dkaLFi3QokULWFtbY9WqVTh16hSioqI0vqawhufOnYuaNWuqbVNUzb373gPkfwgPHz6MAwcOYNu2bdi5cyfWrVuHpk2bYvfu3RCLxTqvEzGuGTNmYPTo0bh69SpcXFxQrVo1fPPNNwCAihUrAqD6Nef6pS1FJqBw82phmi9btiw8PT1x9uxZlbanT59WKvyaNWsiJydHZTPrqVOnFPP5KDzAsPCN/f6bplBQUBAAwNraGs2bN1f7cHJyUnpN4TebQllZWUhNTUWFChWUpterVw8zZszA2bNnsWbNGly9ehV//PEHr/UhxhEREQEAKn8EHj16BEB+UD4AfPDBB7CyslKp4by8PFy8eFGlhm/cuIGMjAyltiVVw8HBwQDkf5A01fC7u3sB1RpOTk6GTCZTqmGRSIRmzZph/vz5uHbtGmbMmIH9+/fjwIEDvNaHGI+bmxsaNWqEatWqAZCfQFCuXDlUqlQJANUvYL71S6FIQNTtisrPz8fq1athZ2eHKlWqKKZ36dIF//zzD+7fv6+Ytm/fPty4cQPdunVTTOvQoQOsra2xZMkSxTTGGJYtW4ayZcuiQYMGWse0b98+tdML9y2HhYUBkG9FAoC0tDSldl5eXmjSpAmWL1+u9luVuitwx8fHKx0rtHTpUhQUFCA2NhaAfGtD4a7EQoUfLLQLrXR1794dALBixQql6b/88gusrKzQpEkTAICLiwuaN2+O3377DZmZmYp2v/76K7KyspRquGvXrpBKpYiPj1dMy83NxcqVK1G3bl34+/trHE9OTo5il937duzYAeBtDTs4OABQreGIiAgEBwfj+++/V3u1YnU1vHjxYqXnP/74IwAoaljdmURUw8K0bt06nDlzBiNHjoRIJP+TSfVrvvVLu88EZMiQIcjIyEBkZCTKli2Lx48fY82aNbh+/TrmzZuntInzm2++wYYNGxAdHY0RI0YgKysLc+fORbVq1TBw4EBFu3LlymHkyJGYO3cu8vPzUbt2bWzevBlHjhzBmjVritzM2aFDBwQGBqJdu3YIDg5GdnY29u7di61bt6J27dpo164dAChC27p161CxYkW4u7vjgw8+wAcffIDFixcrvnV9+umnCAoKwpMnT3DixAk8ePAAly5dUuozLy8PzZo1Q/fu3ZGUlIQlS5agUaNGaN++PQBg1apVWLJkCTp16oTg4GBkZmbi559/hrOzM1q3bm2sXwfhITw8HB9//DH+97//oaCgAFFRUTh48CA2bNiA8ePHK+0qmDFjBho0aICoqCgMHjwYDx48wLx589CyZUu0atVK0a5u3bro1q0bxo8fj6dPnyIkJASrVq3CnTt3VMLX+3JyctCgQQPUq1cPrVq1gr+/P9LS0hTvgY4dOyI8PByA/Bu1q6srli1bBicnJzg4OKBu3boIDAzEL7/8gtjYWFStWhUDBw5E2bJl8fDhQxw4cADOzs4qt+JJSUlB+/bt0apVK5w4cQK//fYbevXqhRo1agAApk6disOHD6NNmzYoX748nj59iiVLlqBcuXJo1KiRsX4dRE+HDx/G1KlT0bJlS5QpUwYnT57EypUr0apVK4wYMUKpLdWvmdZvqZ77RpSsXbuWNW/enHl7ezMrKyvm5ubGmjdvrnKRw0JXrlxhLVu2ZPb29szV1ZX17t2bPX78WKWdVCpl3333HStfvjyzsbFhVatWZb/99pvOY+rRowcLDg5mdnZ2zNbWllWpUoV9++23LCMjQ6nt8ePHWUREBLOxsVE5Pf/WrVusX79+zMfHh1lbW7OyZcuytm3bKk5xZUz14o1ubm7M0dGR9e7dm7148ULR7vz586xnz54sICCASSQS5uXlxdq2bcvOnj2r0zqR4pWXl8fi4uJY+fLlmbW1NQsJCWE//PCD2rZHjhxhDRo0YLa2tszT05MNGzZMpa4Yk18BePTo0czHx4dJJBJWu3ZttnPnziLHkp+fz37++WfWsWNHVr58eSaRSJi9vT0LDw9nc+fOVblS+5YtW1iVKlWYlZWVyunNFy5cYJ07d2ZlypRhEomElS9fnnXv3p3t27dP0abwlOZr166xrl27MicnJ+bm5sa++OILpYvf7du3j3Xo0IH5+fkxGxsb5ufnx3r27Mlu3LhR5DqR4pOcnMxatmzJPDw8mEQiYZUqVWIzZ85Ue0V/xqh+zbF+Ocbe2w9BSClJSEjAwIEDcebMGbUXpiRE6OLi4jBlyhQ8e/YMHh4epT0cQvRC9UvHFBFCCCGEAKBQRAghhBACgEIRIYQQQggAgI4pIoQQQggBbSkihBBCCAFAoYgQQgghBIAFXrxRJpPh0aNHcHJy0nhZdEKMiTGGzMxM+Pn5Ka6IawiqYVKSqH6JqdOnhi0uFD169EjrZdUJKS73799HuXLlDF4O1TApDVS/xNTpUsMWF4oKbz56//59ODs7l/JoiCXIyMiAv7+/yo1v+aIaJiWJ6peYOn1q2OJCUeHmWmdnZ3pDkhJlrF0FVMOkNFD9ElOnSw3TgdaEEEIIIaBQRAghhBACgEIRIYQQQggACkWEEEIIIQAoFBFCCCGEAKBQRAghhBACgEIRIYQQQggACkWEEEIIIQAoFBFCCCGEAKBQRAghhBACgEIRIYQQQggACkWEEEIIIQAEGIoOHz6Mdu3awc/PDxzHYfPmzUrzBwwYAI7jlB6tWrUqncESQgghxGwILhRlZ2ejRo0aWLx4scY2rVq1QmpqquKxdu3aEhwhIYQQQsyRVWkP4H2xsbGIjY3V2kYikcDHx6eERkQIIcQUZWdnw9HREQCQlZUFBweHUh4RETrBhSJdHDx4EF5eXnBzc0PTpk0xffp0lClTRm3b3Nxc5ObmKp5nZGQAAKRSKaRSaYmMl1g2Q+uMapiUJlOu33eXT+8Xy6XP793kQlGrVq3QuXNnBAYG4tatW/jmm28QGxuLEydOQCwWq7SfOXMmpkyZojI9KSlJ8Q2CkOKUlZVl0OuphklpKun6tba2VvtZzkdOTo7i5zt37sDe3t4oy5VKpcjPzzfKskjx06eGOcYYK8axGITjOPz111/o2LGjxja3b99GcHAw9u7di2bNmqnMV/ctxd/fHy9fvoSzs3NxDJsQJRkZGXB3d0d6ejqvmqMaJqWppOuX4ziIRMY53LW4dp/JZDII+E8neY8+NWxyW4reFxQUBA8PDyQnJ6sNRRKJBBKJRGW6WCw22rcRQrQxtM6ohklpKo36Xb58OVJTUw3qFwDy8vIUP8+YMQM2NjYGL9PX1xdDhgwxeDmk5OhTwyYfih48eIAXL17A19e3tIdCiFmhg1RJaUlNTcXdu3cNXs67u7ju3bsHa2trg5dJzJvgQlFWVhaSk5MVz1NSUnDx4kW4u7vD3d0dU6ZMQZcuXeDj44Nbt25h7NixCAkJQUxMTCmOmhBCiNBYW1tjwIABpT0MYkIEF4rOnj2L6OhoxfNRo0YBAPr374+lS5fi8uXLWLVqFdLS0uDn54eWLVti2rRpajfPEkIIIYToSnChqEmTJloPYNu1a1cJjoYQ0yOTyYx2oGpxMYUxEkIsj+BCESHEMCKRyCgHqhbHQaoAHahKCBEuCkWEmCFjHahaeDyGMc4EIoQQoaPt14QQQgghoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIsQHZ2NjiOA8dxyM7OLu3hEEIEikIRIYQQQggoFBFCCCGEAKBQRAghhBACgEIRIYQQQggACkWEEEIIIQAoFBFCCCGEAKBQRAghhBACgEIRIYQQQggACkWEEEKI4NEFSEsGhSJCCCGEEFAoIoQQQggBQKGIECJgTCYt7SEUyRTGSAjRjVVpD4AQQjThRGIU/DkWXKVmEFVuAenhpWBJ+7W+RhTeBeLavSA98ztkFzYBAPJzCxTz8//XC/kS+UcfF9YU4sihkCXugexovPaxBNSCuPlYsPvnId03H5AVgPMIglXnuQauJSFEKAQXig4fPoy5c+fi3LlzSE1NxV9//YWOHTsq5jPGMHnyZPz8889IS0tDw4YNsXTpUoSGhpbeoAkhxUYRiLZOBLv4p9a2osafyQPRgYWQHVn2dkae7O3PT64DNiJwNTvLA9G59ZBtnwqAaR5DaBTEzceAJR+CdOMoQJYPaH0FIcQUCW73WXZ2NmrUqIHFixernT9nzhwsWrQIy5Ytw6lTp+Dg4ICYmBi8efOmhEdKCCkJegWi6BGqgUgNrmZniNtN0z0QdVsElnxYKRARQsyP4LYUxcbGIjY2Vu08xhgWLFiACRMmoEOHDgCA1atXw9vbG5s3b0aPHj1UXpObm4vc3FzF84yMDACAVCqFVErHApDiZ2id6VvDYrHYoP5Kii7/L2KxWL7LzJiBqHp7owcic/4sofpVr6R/5+/2R3+/9KPP/5XgQpE2KSkpePz4MZo3b66Y5uLigrp16+LEiRNqQ9HMmTMxZcoUlelJSUlwdHQs1vESAgBZWVkGvV6fGra1tUVISIhB/ZWUlJQUrVt4C9elyGOI9AhEACBuPcnoW4iKWhdTRvWrXkn/znNychQ/X79+Hfb29iXWt6nTp4ZNKhQ9fvwYAODt7a003dvbWzHvfePHj8eoUaMUzzMyMuDv74+wsDA4OzsX32Ah3xXo4uICAEhPT4eDg0Ox9keEqfCbMV+lWcPFKTAw0OBl6BuIAEB2YRNkB+bAmLvMjLEuQkX1q15J/87fvWBjpUqV6O+JHvSpYZMKRXxIJBJIJBKV6WKxuNg30767/JLojwiTob/30qzh4mTo2PXdZQZ8DwCQ7ZoF2HCa2+oRiDhP+VYNU/49FIXqVz1dxs5kUnAi46xjcf09MeYYhUqf/yuTCkU+Pj4AgCdPnsDX11cx/cmTJ6hZs2YpjYoQUtL0Pqi66TgUhiL5FiL1oUivQORXDeLWk/itALEI8ktKjAF7flt1psgK4majwPnXgnTvHLB757UuS1qrn+Lndy8roY66y1KoHR9dUkKFSYWiwMBA+Pj4YN++fYoQlJGRgVOnTmHo0KGlOzhCSIngdZbZ0dVFLlfvQNRnBdire+B8Kuu9DsRysOe3gcfXlCeKrCHuOh+cfzikG4aD3TykZQkcRK0nQVSp6dtJ/11WQh2Nl6VQJ6yp9vkWSHChKCsrC8nJyYrnKSkpuHjxItzd3REQEICRI0di+vTpCA0NRWBgICZOnAg/Pz+laxkRQswT79Pud83S3pZPIHp2E9K98yAauIbXuhALVRiIQiIh3fClboEoojukG74uetH6vj8iaWPC+wQXis6ePYvo6GjF88ID9Pr374+EhASMHTsW2dnZGDx4MNLS0tCoUSPs3LkTtra2pTVkQkgJEMJ1iJQC0ZpPAfcKPNeGWCS+gWjrRLDLf2tfNJ/3R+IeiKvE8FgR8yW4UNSkSRMwpuUDjOMwdepUTJ06tQRHRQgpTYYGIgcbEfKmVFJta0ggysvR2JYQFYYEouK6cOn5DRSK3iO4K1qXNplMVnSjUmYKYyTEWEThXYS3hYgCEdGHEAPRdtqwoI7gthSVNpFIhOXLlyM1NdXgZeXl5Sl+njFjBmxsbAxepq+vL4YMGWLwcggxFboeNEqBiAiSyEqggYju3KcOhSI1UlNTcffuXYOXk5//9oP23r17sLa2NniZhFga6ZnfKRARkyU/7T6cApGJoFBECBE0bddZASgQEWHj/Gvpftq9lkD07nFxFIiKDx1TRAgxWRSIiNBJ986hLUQmhEIRIcQkCSIQufhqnkcIUMSVqikQCQ2FIkKIyRFEIHKvAKs2qnd/J0Q3FIiEiEIRIcSkCCYQ9UsA8mm3GuFDGIGIC6jFZ/BmjQ60LkbW1tYYMGBAaQ+DELMhqECUm4mCnTNh3XcF/xUiFkgggSg0CuLmY/msgFmjLUWEEJMguEC0eiDwOo33+hBLJKBA1G0R2H1txztZJgpFhBDBE2Qgyn7Of4WIBRJYIEo+DOm++XxWxKxRKCKECBoX1pQCETFxAgxEG0cBsgI+K2PWKBQRQgRNHDmUAhExacIMRJrfH5aMdyjKyMjArFmzEBMTg/DwcJw+fRoA8PLlS8yfPx/JyclGGyQhxHLJEvdQICImS9RoMAUiE8Lr7LMHDx4gKioK9+/fR2hoKK5fv46srCwAgLu7O5YvX467d+9i4cKFRh0sIcTyyI7GgwIRMVWiyi0oEJkQXqFozJgxyMzMxMWLF+Hl5QUvLy+l+R07dsQ///xjlAESQogmFIiI0EkPL6VAZEJ47T7bvXs3vvzyS1SpUgUcx6nMDwoKwv379w0eHCGEaCKIQGTnynv8xDKwpP1a51MgEhZeoej169fw9PTUOD8zM5P3gAghpCiCCEQOHrBqG8d7HQihQCQ8vEJRlSpVcPjwYY3zN2/ejPDwcN6DIoQQTQQTiPqtBKzt+a8IsWhCCEScZwjf4ZstXqFo5MiR+OOPPzB79mykp6cDAGQyGZKTk9G3b1+cOHECX331lVEHSgghggpEEicUbJvMf2WIxRJEIPKrBnHrSXxXwWzxOtC6T58+uHv3LiZMmIBvv/0WANCqVSswxiASifDdd9+hY8eOxhwnIcTCCS4QrR4A2NCWIqIfwQSiPivAXt0D51OZ76qYJd43hP3222/Rt29fbNq0CcnJyZDJZAgODkbnzp0RFBRkzDESQiycIAPRyzuATxW+q0QskKAC0bObkO6dB9HANXxXxyzxDkUAEBAQQLvJCCHFiguoBXHzMcILRKUkOzsbjo6OAICsrCw4ODiU2liI7gQXiNZ8CrhX4Lk25ovXMUXnz5/HkiVLNM5fsmQJLl68yHdMhBCiIG4+lgIRMWmCDETa3h8WjFco+vbbb7F3716N8/fv348JEybwHhQhhBRi989TICImSxTehQKRCeEVis6dO4fGjRtrnN+4cWOcPXuW96AIIaSQdN98CkTEZIlr96JA9J7s7GxwHAeO45CdnV3aw1HCKxRlZmbCykrz4UgikUhxqj4hhBhEVqBxFgUiInTSM79TIDIhvEJRaGgodu/erXH+zp076Qw0QkixEkQgsrblOXpiKWQXNmmdT4FIWHiFokGDBmHbtm0YNWoU0tLSFNPT0tLw1VdfYefOnRg0aJCxxkgIIUoEEYhs7CGOpWMnCX8UiISH1yn5X375JS5evIgFCxZg0aJF8PPzAwA8evQIMpkMffv2pVP1CSHFQjCBqPfP4NwCdB63VAaIeX0NLRlCH5+5EUQgcvE1YA3ME69QxHEcVq5ciX79+mHTpk24ffs2AKBDhw7o0qULmjRpYswxEkIIAIEFIs9QSLdPhVWn2TqNXSwC+h4GXubKnz99o76dvRjwtgNSstTPl77zusbbAbGaPXhetvr1UckF+DWy6HUgxiGIQOReAVZtphiwFubJoIs3RkdHIzo62lhj0UlcXBymTFH+RYaFheH69eslOg5CSMkSXCD6bRCYTKrXOjhZASsbyX9uvQfY/1h5vqMVcLEDUMER+DER+Oq06jJmfQBc+O/n5Awg573/hqY+wJZmhvVBio9gAlG/BCA/B0AZncYtZYCY06lpqTHG1k6DQlFpqVq1qtJ1krSdCUcIMX2CDESP/tX7Nh81ywBW/31oN/RWDSz+DvKwAgAt/NQvo5mvcvukXOX5jbwN74MUD0EFotxMFOycCeu+K3Qa+7uBaHkS8MsN5fmBjsD6/7aRpGQC3Q+qLmNDE6CCk/K0d7d2floRGBzGrw9jbe3klSYYY4iPj8eKFStw+/ZtvHr1SqUNx3EoKNB8Kq0hrKys4OPjUyzLJoQIC+cZAnHsBOEFIh623AOquMp/XnFTdX5iunzrTQs/YPw59cuYdOHtz9czAE6iPH/FTaCpr2F9EOMTXCBaPRBw8tJrHS68ALILgBmXgUfvdXPhpXJdXXip+vqvzgAzI4B/kt9Ou/TqbQ3PuPz2/cG3D0PxCkVjx47F/PnzUbNmTfTp0wdubm7GHpdWN2/ehJ+fH2xtbVG/fn3MnDkTAQHqD3jMzc1Fbu7br1IZGRkAAKlUCqlUddO3WCwunkEbmbqxE2Ey9HdlyTUsFoshbj0J7GmSoAORruvy9A3QZKf2dkXtztr+UPv8hzn8+1C3HlS/6un6OwcEGoiynytCka7rMvi49iBSVO1uvS9/sFz18w2p3UKG1jCvULRq1Sp06dIF69ev5/Nyg9StWxcJCQkICwtDamoqpkyZgsaNG+PKlStwcnJSaT9z5kyVY5AAICkpSXFTxUK2trYICQkptrEbU0pKCt680XAUJRGUrCwNR8zqyFJruHBd2Kt7gg5E+qyL0KlbD6pf9XT9nXNhTSGOHCq8QMRjXUyBoTXMKxS9fv0azZs35/NSg8XGxip+rl69OurWrYvy5ctj/fr1aq+NNH78eIwaNUrxPCMjA/7+/ggLC4Ozs3OJjLk4BAYGlvYQiI4KvxnzZek1LN0xXdCBCCjZ9yMncYA4QfMfVkOoWw+qX/V0/Z0LPRDpsy6mwNAa5hWKmjVrhjNnzmDw4MF8Xm5Urq6uqFixIpKTk9XOl0gkkEgkKtPFYrHJbKZVx5THbmkM/V1ZfA3na9kiKoBABJjP+1HdelD9qqfr2GWJewQdiPRZF2MpzmBvaA3zOnltyZIlOHnyJL777ju8ePGCzyKMJisrC7du3YKvL12EihCLIoRAJKIzX4l2sqPxEHIgIsp4haKwsDDcvn0bEydOhJeXFxwcHODs7Kz0cHFxMfZYAQCjR4/GoUOHcOfOHRw/fhydOnWCWCxGz549i6U/IifkuxoTCySIQGQNcbNRmufrwE4MbIwGzrcDwt3VtxlRBbjWUf6vOrXc5a/fGC1fHp8+etKtKkuFKQciodRuYR/GqmFeX3O6dOkCjiudqzg9ePAAPXv2xIsXL+Dp6YlGjRrh5MmT8PT0LJXxEEJKmFACUdf54Pxr8VwJudblgI7/nTg7thrQ89B73XDAnAj5BenmRMhPR5a9t9FhTDWgurv8EVsO+POu/n2MqGzQahAeBBGI7Fx5j18otVvYxwjV86x44RWKEhISjNM7D3/88Uep9U0IKWVCCkQhkZDunQOrVt/yXBn56c2Z+YCTNXAgVXW+jAFHnwJRPvJ/3/+jAshf162CfDkX1ZwurUsfF18CER68V4PoSRCByMEDVm3jeK+DUGq3sA9j1TDtECeEmAahBaINX4JlPuO5MnK3M4HgTYCbDXArU32bVnuAUGfgpoYTaOJvAPtSgVd5b++rpm8fX5wCTrThtw5EP4IJRP1WAtb2vNdDKLVb2IeDlXFqmPddQu7du4fPPvsMYWFhcHNzw+HDhwEAz58/x5dffokLFy4UsQRCCNGREAPRzUOa2+rhZa7mD3wAyJcB19Lk/2pyK1P9HxVd+yjQsmxiPIIKRBInFGybzH9lIIzaLezDWDXMa0vRtWvX0LhxY8hkMtStWxfJycmKW3p4eHjg6NGjyM7OxooVut1ThRBCNHLxhVXsRLMMRMRyCC4QrR4A2PDfUmSueN/mw9XVFSdPngTHcfDyUr5/Sps2bbBu3TqjDJAQYtms2kwBXr+iQERMliAD0cs7et/Q2BLw2n12+PBhDB06FJ6enmrPQgsICMDDh0XcoIcQQnSRn0OBiJgsLqCWMAMRUYtXKJLJZLC317zZ7dmzZ2qvYEoIIfoq+CfObAKRiAOGhskfYg1XNWnqA0yqAZTV8BFb1l4+v6mP+vliI/RBjEfcfKzZBKKu5c2/dnntPqtVqxa2bduGzz//XGVeQUEB/vjjD9SrV8/gwRFCCF6naZ5nQoEIADoHAF9Xf/t8aZLy/LL2wPYWgJUIaOqr/o7hayKBRt7yA0uDN8nvLP6uwRWBH9/5+OXTBzEedv+8WQQiwDJql9eWovHjx2Pnzp0YOnQorly5AgB48uQJ9u7di5YtWyIxMRHjxo0z6kAJIUSJiQUiYpmk++abRSCyFLy2FMXGxiIhIQEjRoxAfHw8AKBPnz5gjMHZ2RmrV69GZGSkUQdKCCEKgghEHESN9Lsp9p/3gAf//c2Lv6E6/2EO0HoP0NAbWHFT/TJ6HQYGhQLHnqh+035/uXz7IEYkK9A4y9QC0ezL8vo159rlffHGvn37onPnztizZw9u3rwJmUyG4OBgxMTEwMnJSNfbJoSQ9wklELWeBFHlFnoNXcZUdwm8b/9j+UOTRznAtEua50uN0AcpfoIIRNa2eo154135VaY1MYfa1TsU5eTkwN/fH+PGjcOYMWPQsWPHYhgWMQYmk4ITqbnLnoCYwhiJgAgpEEV0h/TwUlhFDeO5MsRSCSIQ2dhDHDuB5xqYL71Dkb29PaysrODg4FAc4yFGxInEKPhzDNjLexA3GwXOvxake+eA3Tuv9XWiRoMhqtwC0sNLwZL2AwDyc99uAs7/Xy/kS+SlIwrvAnHtXpCe+R2yC5u0jyesKcSRQyFL3APZ0XhwHkGw6jzXwLUkFkNogWjrRLDH1wEKRUQPgglEvX8G5xbAcy3MF6/dZ126dMHGjRsxdOhQtdcpIsLBXt6DuNGn4PzDId0wXOddAtKtE8Eu/vl2Vt4711B/ch2wEUHU+DN5IDqwELIjy7SOg6vZWR6Izq2HbPtUAAxq7g9IiHpCDEQX/6SL3xG9CCoQeYZCun0qrDrN5rk25olXKOrRowc+//xzREdH49NPP0WFChVgZ2en0q5WrVoGD5AYRr6FKJzfB74WosafQRw9QvdA1G6aUiAiRGd2rvIPbqEFIiMpIwFctdzw0lr09qaamu4hFewEpOUBLzTcQ6qoPqx43wWT6Epwgei3QWAyKc+1kRNC7Rb2Yawa5hWKmjRpovj5yJEjKvMZY+A4DlKpYf/hxHCcfy3dtxDpGogaDDLpQJSdnQ1HR0cAQFZWFu0KFjirtnGA2MYsA1GQE3CuHeBkDXx+Qv0ZNztbAFE+wKHHQLNdqvMHVwSW1Acy84GIrfI7i+vbx091jbM+RD1BBqJH/xq0pVMotVvYxzkt/0364BWKVq5caZzeSbGT7p1j9A98cdQwkw1ExARZ26MgoY/ZBSIACHeXf+ADQLSv6oe+iAMa/XdryUZe8uey995G0b7yf52sgZruqn9YdOmjprvh60LU4zxDII6dILxAZCCh1G5hH8aqYV6hqH///sbpnRQ77QdV8/vAlx5aDNlZ7cGYAhExloJtk80yEAHA9gfA5ntAkCMwR82wZQwYe07+jTr+huofFUD+ujBn4HYWsOMBvz4WJgKjqhq+PkSVuPUksKdJZhWIAOHUbmEf2x4Yp4Z5X6eoUGpqKp4+fYqQkBDaDWFS9N9lBnwNAJAdXwHYaN6BS4GIGFV6quZ5JhyIAOC1FOh6QHubhdfkD00uvARqbTWsj7W3KRQVF/bqntkFIkA4tVvYR7i7cWqY96FJW7ZsQaVKlVCuXDnUqlULp06dAgA8f/4c4eHh+OuvvwwfHSkmPA6q1vG0Y30CEReg+4H4Ug0H6QmJKYzRrJh4ICKWQbpjutkFInPGa0vR1q1b0blzZ9SvXx+9evVCXFycYp6HhwfKli2LhIQEdOrUyVjjJEbD8yyz7XOKXrI+gSg0CuLmY3UetVgE9D0MXE/X+SUaSd+8/bnxdkCs30Vd1arkAvxKd7YpOQIJRKLwLvzGTyxH/hvN8ygQCQ6vUDR16lRERkbiwIEDePHihVIoAoD69etj+fLlxhgfMSoDTrs/vkL7kvUNRN0Wgd0/Dy5Q99Nerqdrv8S8rtg7p35eegVwEsOXSUqQUALRf9fpIoQXIQQikcFH0JgdXrvPrly5gu7du2uc7+3tjadPn/IeFCkOwrgOkSIQJR+W3z2aEH0IKRBFj4D0zO/81oNYNkEEImuIm43iuQLmi1cosre3R3Z2tsb5t2/fRpkyZXgPihibAAPRxlFa7x6tTVl74GAr+cPPXn2bH+oAVzsC7fzVjEPigI77GK6nMSxorP7kAEP7IMVAaIHowMIib23zPi9bw+uqnb98/g911M+n2hU4oQSirvPB+et3geX4BuZfu7xCUXR0NFatWoWCAtU/ao8fP8bPP/+Mli1bGjw4YhzGCEQONiLkTamEvCmV4PDfmWeGBaJ83uszKBRo5C1/DApVnV/ZBRheGQhzAWZGqF/GrAj5/OGV5e2Low9iREIMREV8YVCnQwDVrkUTUiAKiYR0b9HHir4rvIz51y6vUDRjxgw8ePAAtWvXxvLly8FxHHbt2oUJEyagWrVqYIxh8uTJxh4r4UHUaLAAtxDxD0QAcPQJUCCTP449UZ1/Lxu4kyX/ec8j9cvY/d/0O1ny9sXRBzESa1uzCEQAcPEF1a7FElog2vBlkTcHf58l1C6vo6zCwsJw9OhRjBgxAhMnTgRjDHPnyu923qRJEyxevBgVKlQw5jgJT2pv7qqunYkEIgDY/xgI/m+vxUM1Z7pmFwA1tgABDkCihrPVvjoNxCfJ32TZavbi6dtHOF0RuNiIYyeAcy1n8oEIAM68EF7tkhIgxEB085Det/nosA+4kmbetatTKLp8+TLKly8PF5e327uqVq2KvXv34tWrV0hOToZMJkNQUBA8PT2LZ6SEF+nhpWYViAqpe8O8K7ug6DdNUfON0QcxHOcWAOmvA00+EBWi2rUwLr6wip0ovEDEw9M32mvLHGpXp91n4eHh2LZtm+J506ZNsW/fPgCAm5sbateujbp161IgEiCWtF/rfFMMRMSySLdPNZtARCyPVZspZhGILIVOocjOzg45OW/j28GDB/HkiZodfsSkUCAipoA9S9Y8kwIREbr8HApEJkSn3Wc1atTA/PnzIRaLFbvQzpw5A1tb7ZcC7ty5s+EjJMVCCIGI8wzhO3xCKBARk1DwTxwFIhOiUyhasGABunXrhkGDBgEAOI7DwoULsXDhQo2v4TgOUqnUOKMkRiWIQORXDeLWk/Qadws/zVe07l4BCHQEFl8HstQcwFfdDegUAPx1D7j8SnW+oxUwrBKQkgWsv8OvD1KCBBKIuLCmPAavTAi1G+pswAoQ7V6naZ5n4oFICLVb2IexjjPSKRTVrl0bycnJuHXrFp48eYImTZrgm2++QYsWLYwzClJiBBOI+qwAe3UPnE9lncf+XQRw6hlw6L09t018gN+j5D+XcwCGn3qvPwB7WgJlbIHPKwE+61TXZGYEMLSS/Ocnr/n1QUqIUAJRzc4QRw7lsQJvCaV2l9QzaDUIHyYeiIRSu4V9pOXCKHQKRX///Tc+/PBDhIWFISwsDP3790e7du1Qt67u960ytsWLF2Pu3Ll4/PgxatSogR9//BF16mi4TCYBILBA9OwmpHvnQTRwDd/VUWCah2Y0JdEH0YGQAlG7aZAl7oG4SgyPFZGj2rVQgghEHESNBvNcAfOtXZ1CUadOnfDrr7+iVy/5zQ8PHTqETp06FevAtFm3bh1GjRqFZcuWoW7duliwYAFiYmKQlJQELy+vUhuXkAkuEK35FHCvoNc6fHNO9ZsEIJ/W69DbTazvYwBa7AY6BgCb76lfm/HngAfZ8s24fPsgxUxogejcesjObzAoFAmldj8/+fYbOSlmQglErSdBVJn/3h6h1G5hH9fTjVPDOoUiJycnpKWlKZ7fuXMHWVlZhvfO0/z58/Hpp59i4MCBAIBly5Zh27Zt+N///odx48Yptc3NzUVu7tvtaunp8h2Pr169UnvMk1gshqurq9JrhMTV1RUZGRk6Ha8lFotRYF8WXONYiCO6Q3rgF8gu7wNc1Fw//T9caCTEDT+B7MxmyE78DrhoPhiaK1cD4uiRYNeOQnpoCeBUQXNbjyCIW34Ndv86pHvmAnZlwdmXhZUe63LrKVBdw7H91x8D1wEEWUF9Vb8GNifJf9S0jB3J2udr6yPICsjIgNp1ycjIAAAwnl97qIbLgrkxiKO+AOdVA9It08CePtJax6L6AyAKbQrpjvlgKf9qb1u9Pe/3B1emgl41HGQFSN+rr9KuXQCQ5lD96opX/br892XRzgVWMeOBfCsU/DMRkFprrjdrW4hbjAFn6wvppolg2W80txVZ8Xp/yA6vhji8M+/6FULtFvYR6mycGuaYDq1atmyJq1ev4pNPPoGLiwtGjx6NXr16oVYtzTeT4zgOX331VZED0FdeXh7s7e2xceNGdOzYUTG9f//+SEtLw5YtW5Tax8XFYcqUKUYfByH6un//PsqVK6f366iGiRBQ/RJTp0sN6xSKkpOT0a9fP5w8eVL+Io4rMnEV19lnjx49QtmyZXH8+HHUr19fMX3s2LE4dOgQTp1SPhLr/W8pMpkML1++RJkyZcBxdHgsKX6MMWRmZsLPzw8ikf63G6QaJqWJ6peYOn1qWKfdZyEhITh+/DjevHmDp0+fokKFCliwYAE6dOhglAEXJ4lEAolEojTN1dW1dAZDLNa7t8jRF9UwKW1Uv8TU6VrDet0Q1tbWFgEBAZg8eTKaNm2K8uXL8xqcITw8PCAWi1WuqP3kyRP4+PiU+HgIIYQQYh703xYKYPLkyfjggw+MPRad2NjYICIiQnHvNUC+OXbfvn1Ku9MIIYQQQvSh05aijz/+GBzHIT4+HmKxGB9//HGRr+E4DitWrDB4gOqMGjUK/fv3x4cffog6depgwYIFyM7OVpyNRgghhBCiL51C0f79+yESiSCTySAWi7F///4iD5ArzgPoPvroIzx79gyTJk3C48ePUbNmTezcuRPe3t7F1ichhBBCzJtOZ58RQgghhJg7XscUEUIIIYSYG73OPiuUmZmJo0eP4tatW8jMzISTkxNCQkLQqFEjODo6GnuMhBBCCCHFTq9QJJVKMWHCBPz000/IyclRuoAjx3Gwt7fHiBEjMHXqVF4X+SKEEEIIKS16HVPUo0cPrF+/HlWqVEHPnj3xwQcfwNHREVlZWfj333/x+++/IykpCT179sRvv/1WnOMmhBBCCDEqnUPR3r170bJlSwwbNgwLFy5UuyVIJpNh+PDhWLZsGfbs2YOmTZsafcCEEEIIIcVB51DUr18/nDhxAjdu3NB6ur1MJkNYWBgaNGiAVatWGW2ghBBCCCHFSecDf06fPo3OnTsXef0hkUiEzp07q9yYlRBCCCFEyHQORampqQgJCdGpbUhICFJTU3kPihBCCCGkpOkcirKysuDg4KBTW3t7e2RlZfEeFCGEEEJISdM5FDHGivXWHYQQQgghpUnnA61FIhH8/f3h4uJSZNv09HQ8ePAAUqnU4AESQgghhJQEnS/eGBkZqfOWojJlyiAoKIj3oAghhBBCShrdEJYQQgghBHRDWEIIIYQQABSKCCGEEEIAUCgihBBCCAFAoYgQQgghBACFolKTlZWFyZMno1WrVnB3dwfHcUhISFBpJ5PJkJCQgPbt28Pf3x8ODg744IMPMH36dLx580btslesWIHKlSvD1tYWoaGh+PHHH9W2e/jwIbp37w5XV1c4OzujQ4cOuH37tk7jz8vLw8KFCxEeHg5nZ2e4urqiatWqGDx4MK5fv65od/z4ccTFxSEtLU2n5RLToGv9AsCAAQPAcZzKo1KlSiptZTIZ5syZg8DAQNja2qJ69epYu3at2uUmJiaiVatWcHR0hLu7O/r27Ytnz57pNf4PPvgADg4OKFOmDGrWrIkRI0bg0aNHinbbt29HXFycTsskpuXq1avo1q0bgoKCYG9vDw8PD0RGRmLr1q1q2+tab1TDJo6RUpGSksIAsICAANakSRMGgK1cuVKlXWZmJgPA6tWrx6ZPn87i4+PZwIEDmUgkYk2aNGEymUyp/bJlyxgA1qVLFxYfH8/69u3LALBZs2apLDc0NJR5eXmx2bNns/nz5zN/f39Wrlw59vz58yLH37ZtWyYWi1mfPn3Y4sWL2YIFC9hnn33GypUrp7Qec+fOZQBYSkoKn/8mIlC61i9jjPXv359JJBL266+/Kj3+/vtvlbbjxo1jANinn37K4uPjWZs2bRgAtnbtWqV29+/fZx4eHiw4OJgtXLiQzZgxg7m5ubEaNWqw3NxcrWPPy8tj4eHhzM7Ojn322Wds2bJl7Pvvv2cDBw5kHh4e7MCBA4q2w4YNY/QxaZ62bdvGYmJiWFxcHIuPj2cLFixgjRs3ZgDY8uXLldrqU29Uw6aN/qdKyZs3b1hqaipjjLEzZ85o/KOSm5vLjh07pjJ9ypQpDADbs2ePYlpOTg4rU6YMa9OmjVLb3r17MwcHB/by5UvFtNmzZzMA7PTp04ppiYmJTCwWs/Hjx2sd++nTpxkANmPGDJV5BQUFSqGKQpF50rV+GZOHIgcHhyKX+eDBA2Ztbc2GDRummCaTyVjjxo1ZuXLlWEFBgWL60KFDmZ2dHbt7965i2p49e9T+QXvf+vXrGQC2Zs0alXmvX79m6enpiuf0B8WyFBQUsBo1arCwsDCl6brWG9Ww6dPpf6pChQosMDBQr0dQUFBxj91sFPVHRZ3Lly8zAGzRokWKadu2bWMA2LZt25TaHj9+nAFgv/76q2Ja7dq1We3atVWW27JlSxYcHKy177Vr1zIA7ODBg1rbTZ48mQFQebwbkH799VdWq1YtZmtry9zc3NhHH33E7t27p7ScqKgoVrVqVXb27FlWv359ZmtryypUqMCWLl2q0ueiRYtYlSpVmJ2dHXN1dWURERFqPziI8egaigoKCpQ+rN+3ePFiBoBdvXpVafrvv//OALAjR44opnl5ebFu3bqpLKNixYqsWbNmWsc7c+ZMBoDduXNHa7v+/furrd9CUqmU/fDDD6xKlSpMIpEwLy8vNnjwYKUvH4wxVr58edamTRu2a9cuVqNGDSaRSFjlypXZpk2blNrl5eWxuLg4FhISwiQSCXN3d2cNGzZku3fv1jpOYlxt27Zl3t7eStN0rTeqYdOvYZ2uaB0VFUX3PROYx48fAwA8PDwU0y5cuAAA+PDDD5XaRkREQCQS4cKFC+jTpw9kMhkuX76Mjz/+WGW5derUwe7du5GZmQknJye1fZcvXx4AsGbNGjRs2BBWVurLqHPnzrhx4wbWrl2LH374QTFWT09PAMCMGTMwceJEdO/eHZ988gmePXuGH3/8EZGRkbhw4QJcXV0Vy3r16hVat26N7t27o2fPnli/fj2GDh0KGxsbxXr8/PPP+PLLL9G1a1eMGDECb968weXLl3Hq1Cn06tWryP9TUnxycnLg7OyMnJwcuLm5oWfPnpg9ezYcHR0VbS5cuAAHBwdUrlxZ6bV16tRRzG/UqBEePnyIp0+fqtR5Ydvt27drHUth/a5evRoTJkzQ+Nk2ZMgQPHr0CHv27MGvv/6qdn5CQgIGDhyIL7/8EikpKfjpp59w4cIFHDt2DNbW1oq2N2/exEcffYTPPvsM/fv3x8qVK9GtWzfs3LkTLVq0AADExcVh5syZ+OSTT1CnTh1kZGTg7NmzOH/+vKINMb7s7Gy8fv0a6enp+Pvvv7Fjxw589NFHivn61BvVsBnUcGmnMsJvS1Hz5s2Zs7Mze/XqlWLasGHDmFgsVtve09OT9ejRgzHG2LNnzxgANnXqVJV2hd90rl+/rrFvmUzGoqKiGADm7e3NevbsyRYvXqy0GbiQpt1nd+7cYWKxWGUX3L///susrKyUphf2NW/ePMW03NxcVrNmTebl5cXy8vIYY4x16NCBVa1aVeO4SfEoqn7HjRvHvv76a7Zu3Tq2du1axbfXhg0bsvz8fEW7Nm3aqN3CnJ2dzQCwcePGKfW3evVqlbZjxoxhANibN280jjcnJ4eFhYUxAKx8+fJswIABbMWKFezJkycqbTXtejhy5Ija3Rc7d+5UmV6+fHkGQOlbdXp6OvP19WXh4eGKaTVq1FDZ9U2K35AhQxRbUEQiEevatavSlhJ96o1q2PRrmM4+M0Hfffcd9u7di1mzZiltTXn9+jVsbGzUvsbW1havX79WtAMAiUSitt27bdThOA67du3C9OnT4ebmhrVr12LYsGEoX748PvroI53ONPvzzz8hk8nQvXt3PH/+XPHw8fFBaGgoDhw4oNTeysoKQ4YMUTy3sbHBkCFD8PTpU5w7dw4A4OrqigcPHuDMmTNF9k9KzsyZMzFr1ix0794dPXr0QEJCAmbMmIFjx45h48aNinavX7/WqSYNrV87OzucOnUKY8aMAQAkJCRg0KBB8PX1xfDhw5Gbm1vkOm3YsAEuLi5o0aKFUv1GRETA0dFRpX79/PzQqVMnxXNnZ2f069cPFy5cUGz1dXV1xdWrV3Hz5s0i+yfGM3LkSOzZswerVq1CbGwspFIp8vLyFPP1qTeqYdOvYYNCUX5+Pv79918cPXoUhw8fVnkQ41u3bh0mTJiAQYMGYejQoUrz7OzslN7M73rz5g3s7OwU7QCofeMUnuZf2EYTiUSCb7/9FomJiXj06BHWrl2LevXqYf369fjiiy+KXI+bN2+CMYbQ0FB4enoqPRITE/H06VOl9n5+fnBwcFCaVrFiRQDAnTt3AABff/01HB0dUadOHYSGhmLYsGE4duxYkWMhJe+rr76CSCTC3r17FdPs7Ox0qklj1K+LiwvmzJmDO3fu4M6dO1ixYgXCwsLw008/Ydq0aUWO/+bNm0hPT4eXl5dK/WZlZanUb0hIiMoujvfrd+rUqUhLS0PFihVRrVo1jBkzBpcvXy5yLMQwlSpVQvPmzdGvXz/8888/yMrKQrt27cD+uy2oPvVGNWz6NazTMUXvk8lkGD9+PJYsWYKcnByN7aRSKe+BEVV79uxBv3790KZNGyxbtkxlvq+vL6RSKZ4+fQovLy/F9Ly8PLx48QJ+fn4AAHd3d0gkEqSmpqoso3BaYVtd+Pr6okePHujSpQuqVq2K9evXIyEhQeOxRoC8hjiOw44dOyAWi1Xmv3usia4qV66MpKQk/PPPP9i5cyc2bdqEJUuWYNKkSZgyZYreyyPFx87ODmXKlMHLly8V03x9fXHgwAEwxpQ+fN+vSV9fX6Xp70pNTVXUt67Kly+Pjz/+GJ06dUJQUBDWrFmD6dOna32NTCaDl5cX1qxZo3Z+4XFz+oiMjMStW7ewZcsW7N69G7/88gt++OEHLFu2DJ988oneyyP8dO3aFUOGDMGNGzcQFhamV71RDZt+DfMKRd999x3mzp2LIUOGoFGjRujbty9mz54NV1dXLFmyBBzHYc6cOcYeq0U7deoUOnXqhA8//BDr169XGzhq1qwJADh79ixat26tmH727FnIZDLFfJFIhGrVquHs2bNq+wkKCtJ4kLU21tbWqF69Om7evKnYFabpAMDg4GAwxhAYGKj4tqHNo0ePkJ2drbS16MaNGwCAChUqKKY5ODjgo48+wkcffYS8vDx07twZM2bMwPjx4xWbpUnpy8zMxPPnz5U+eGvWrIlffvkFiYmJqFKlimL6qVOnFPMBoGzZsvD09FRbv6dPn1a005ebmxuCg4Nx5coVxTRt9bt37140bNiwyG/0AJCcnKzyh1Jd/bq7u2PgwIEYOHAgsrKyEBkZibi4OJP5g2IOCndbpaenA9Cv3qiGzaCG+RyIFBwczD766CPGGGPPnz9nHMexffv2McbkB8DWqlWryGvdkLeKOlD12rVrrEyZMqxq1aoqp0q+Kycnh7m7u7O2bdsqTe/Tpw+zt7dnL168UEybNWsWA8DOnDmjmHb9+nUmFovZ119/rXW8N27cUHtQ9atXr5ifnx9zc3NTXI9j6dKlDAC7cOGCUtvk5GQmFotZr169VC5AKZPJlK51pO1Aa09PT8WB1uouOjlmzBgmEolYRkaG1nUi/Gmr39evX6v9vy88mPTPP/9UTLt//77Ga7yULVtW6Rovn332GbOzs1O6fMPevXsZALWXanjXxYsX2bNnz1Sm37lzh9nZ2bHq1asrpn399dcMgNIJDYwxdvDgQQZA7edcfn6+UnttB6nWrFlTMU1d/Xbr1o15eHhoXR/Cj7qDkvPy8litWrWYnZ0dy8zMVEzXtd6ohk2/hnltKXrw4AHGjh0L4O2BYoX7QW1sbNCnTx/Mnz8f3333Hd+sZhF++uknpKWlKS7JvnXrVjx48AAAMHz4cLi4uCAzMxMxMTF49eoVxowZg23btiktIzg4GPXr1wcg3yUxbdo0DBs2DN26dUNMTAyOHDmC3377DTNmzIC7u7vidZ9//jl+/vlntGnTBqNHj4a1tTXmz58Pb29v/N///Z/WcV+6dAm9evVCbGwsGjduDHd3dzx8+BCrVq3Co0ePsGDBAsUusYiICADAt99+ix49esDa2hrt2rVDcHAwpk+fjvHjx+POnTvo2LEjnJyckJKSgr/++guDBw/G6NGjFX36+flh9uzZuHPnDipWrIh169bh4sWLiI+PV5w22rJlS/j4+KBhw4bw9vZGYmIifvrpJ7Rp04bXli+inS71+/jxY4SHh6Nnz56K23rs2rUL27dvR6tWrdChQwfF8sqVK4eRI0di7ty5yM/PR+3atbF582YcOXIEa9asUdrN+s0332DDhg2Ijo7GiBEjkJWVhblz56JatWoYOHCg1nHv2bMHkydPRvv27VGvXj04Ojri9u3b+N///ofc3FylWyIU1u+XX36JmJgYiMVi9OjRA1FRURgyZAhmzpyJixcvomXLlrC2tsbNmzexYcMGLFy4EF27dlUsp2LFihg0aBDOnDkDb29v/O9//8OTJ0+wcuVKRZsqVaqgSZMmiIiIgLu7O86ePYuNGzfqdIwe0d+QIUOQkZGByMhIlC1bFo8fP8aaNWtw/fp1zJs3T2kXvq71RjVsBjXMJ0n5+fkpfWt3cXFhP/74o+L5/PnzdbqCraUrTN/qHoWnsBfeTkHTo3///irLjY+PZ2FhYczGxoYFBwezH374QWVrDGPybzVdu3Zlzs7OzNHRkbVt25bdvHmzyHE/efKEzZo1i0VFRTFfX19mZWXF3NzcWNOmTdnGjRtV2k+bNo2VLVuWiUQildPzN23axBo1asQcHByYg4MDq1SpEhs2bBhLSkpStFF38cby5cuzn376Samf5cuXs8jISFamTBkmkUhYcHAwGzNmjNYLBhL+dKnfV69esT59+rCQkBBmb2/PJBIJq1q1Kvvuu+8UW/jeJZVK2XfffcfKly/PbGxsWNWqVdlvv/2mtv8rV66wli1bMnt7e+bq6sp69+7NHj9+XOS4b9++zSZNmsTq1avHvLy8mJWVFfP09GRt2rRh+/fvV2pbUFDAhg8fzjw9PRnHcSqnNsfHx7OIiAhmZ2fHnJycWLVq1djYsWPZo0ePlP6fCi98V716dSaRSFilSpXYhg0blJY1ffp0VqdOHebq6srs7OxYpUqV2IwZM9T+PxHDrV27ljVv3px5e3srPsOaN2/OtmzZora9rvVGNWzaNcwx9t8h9npo27YtnJycFDe569SpE/7991+sWrUKMpkM/fr1g5+fH535Q4yiSZMmeP78udJ+ckJMRYUKFfDBBx/gn3/+Ke2hEMKLJdUwr1PyBw8ejNzcXMXphDNmzEBaWhoiIyMRFRWFjIwMzJs3z6gDJYQQQggpTryOKWrfvj3at2+veF6lShXcunULBw4cgJWVFRo0aKB0/AohhBBCiNDxCkXquLi4oGPHjsZaHCGEEEJIieJ1TNG9e/dw7949NGrUSDHt0qVLmDdvHnJzc9GzZ08KSIQQQggxKbxCUceOHZGVlaW4RP+TJ09QuXJl5OXlwcnJCU+fPsWGDRvQuXNnow+YEEIIIaQ48DrQ+vTp02jRooXi+erVq/H69WtcunQJDx8+RLNmzfD9998bbZCEEEIIIcWN1zFFL1++VLq31j///IOoqCgEBwcDADp37oxvvvnGOCM0MplMhkePHsHJyUnj5c8JMSbGGDIzM+Hn5weRyKB7MAOgGiYli+qXmDp9aphXKPL09MTdu3cBAGlpaTh58iRmzZqlmF9QUICCggI+iy52jx49gr+/f2kPg1ig+/fvo1y5cgYvh2qYlAaqX2LqdKlhXqGoefPmWLRoEZydnXHw4EHIZDKlA6uvXbsm2KIvvN3D/fv34ezsXMqjIZYgIyMD/v7+RrvVCNUwKUlUv8TU6VPDvELRrFmzcOPGDYwePRo2Njb4/vvvERgYCADIzc3F+vXr0atXLz6LLnaFm2udnZ3pDUlKlLF2FVANk9JA9UtMnS41zCsUeXt749ixY0hPT4ednR1sbGwU82QyGfbt2yfYLUWEEEIIIeoYdPFGFxcXlWl2dnaoUaOGIYslhBBCCClxOoWi1atXAwD69u0LjuMUz4vSr18//iMjhBBCCClBOl28USQSgeM4vH79GjY2NjqdlslxHKRSqVEGaUwZGRlwcXFBeno67c8mJcLYNUc1TEoS1S8xdfrUnE5bilJSUgBAcexQ4XNCCCFEqLKzs+Ho6AgAyMrKgoODQymPiAidTqGofPnyip/z8/ORnp4Od3d3o1yzghBCCCFECPS+PKlIJEJERAT+/PPP4hgPIYQQCyaTyUp7CEUyhTESfvQ++0wsFqN8+fLIzc0tjvEQQgixYCKRCMuXL0dqaqrBy8rLy1P8PGPGDKXLx/Dl6+uLIUOGGLwcIky8TskfPnw4fvrpJwwaNAju7u7GHhMhRADoeAxSWlJTUxW3kjLUgAEDFMskpCi8QpFUKoVEIkFwcDC6du2KChUqwM7OTqkNx3H46quvjDJIQgghhJDixisUjR49WvHzihUr1LahUEQIIYQQU8IrFNEp+YQQQggxN7xC0bun6BNCCCGEmAO9T8l/18uXL7F+/XrMmTMHc+bMwfr16/HixQuDBnT48GG0a9cOfn5+4DgOmzdvVpo/YMAAcByn9GjVqpVBfRaX7OxsxRizs7NLeziEEEII0YL3DWHj4uIwe/ZslVPzbWxsMHbsWEydOpXXcrOzs1GjRg18/PHH6Ny5s9o2rVq1wsqVKxXPJRIJr74IMUcymUynW/GUJlMYIyHE8vAKRdOmTcPUqVPRpk0bfPHFF6hYsSIAICkpCT/99BNmzJgBa2trTJw4Ue9lx8bGIjY2VmsbiUQCHx8fnZaXm5urFNwyMjIAyM+gK+57s727/JLojwiTob93fWtYLBYb5TovxXGNF+DtdV7o/WAaSqN+TQHVr+nQ53fFKxQtW7YM7dq1w5YtW5SmBwYGolWrVmjXrh2WLl3KKxTp4uDBg/Dy8oKbmxuaNm2K6dOno0yZMmrbzpw5E1OmTFGZnpSUpLgGSyFra2sEBwfDyor3BjQl7765xWKx0d7sBQUFuHXrFvLz842yPFK8srKyDHq9PjVsa2uLkJAQo1zn5d36unfvHqytrQ1a3vtSUlLw5s0boy6TGF9p1K8poPo1HfrUMK+//unp6VqP42ndujUOHjzIZ9FFatWqFTp37ozAwEDcunUL33zzDWJjY3HixAm1oWP8+PEYNWqU4nlGRgb8/f0RFham9m65xvqWDRTv1VRN5YODvP1mzJe+NWwqAgMDS3sIRAdUv+pR/ZoOfWqYVyhq2LAhTp06haFDh6qdf+rUKTRs2JDPoovUo0cPxc/VqlVD9erVERwcjIMHD6JZs2Yq7SUSidpjjrRtuTHW1VSL85u2qWxiJob/rvjUsCkw5bFbEqpf9Ux57JZGn98VryMdly1bhhMnTuCrr75CcnIyZDIZZDIZkpOTMXLkSJw8eRLLli3js2i9BQUFwcPDA8nJySXSHyGWwtraGgMGDMCAAQOMvuuMEEKEiNeWourVq0Mmk2HRokVYtGiR4iySwjsHSyQSVK9eXek1HMchPT3dwOGqevDgAV68eAFfX1+jL9tQhX9UCCGEECJ8vEJRly5dwHGcsccCQH5A1LtbfVJSUnDx4kW4u7vD3d0dU6ZMQZcuXeDj44Nbt25h7NixCAkJQUxMTLGMhxBCCCGWgVcoSkhIMPIw3jp79iyio6MVzwsP0Ovfvz+WLl2Ky5cvY9WqVUhLS4Ofnx9atmyJadOm0bWKCCGEEGIQvUNRTk4OGjdujE8//RSfffaZ0QfUpEkTMMY0zt+1a5fR+ySEEEII0ftAa3t7e6SkpBTb7jNCCCGEkNLA6+yzVq1a0RYbQgghhJgVXqFo4sSJuHHjBvr27YujR4/i4cOHePnypcqDEEIIIcRU8DrQumrVqgCAa9eu4ffff9fYju4NQwghhBBTwSsUTZo0iY4pIoQQQohZ4RWK4uLijDwMQgghhJDSxeuYIkIIIYQQc8NrS9HUqVOLbMNxHCZOnMhn8YQQQgghJc7ou884jgNjjEIRIYQQQkwKr91nMplM5VFQUIBbt27hq6++wocffoinT58ae6yEEEIIIcXGaMcUiUQiBAYG4vvvv0doaCiGDx9urEUTQgghhBS7YjnQOjIyEtu3by+ORRNCCCGEFItiCUVnz56FSEQnthFCCCHEdPA60Hr16tVqp6elpeHw4cP4888/8cknnxg0MEIIIYSQksQrFA0YMEDjPA8PD4wbNw6TJk3iOyZCCCGEkBLHKxSlpKSoTOM4Dm5ubnBycjJ4UIQQQgghJY1XKCpfvryxx0EIIYQQUqp4haL3Xb9+HRs2bEBqairCwsIwcOBAODs7G2PRhBBCCCElQudQ9NNPP2HRokU4fvw4PDw8FNO3bt2Kbt26IS8vTzHtxx9/xMmTJ5XaEUIIIYSf7OxsODo6AgCysrLg4OBQyiMyTzqfN//3338jODhYKegUFBTgk08+gVgsxsqVK/Hvv/9i1qxZuHv3LmbMmFEsAyaEEEIIKQ46h6Jr166hXr16StMOHDiAZ8+e4auvvkL//v1RtWpVjB07Ft27d6eLNxJCCCHEpOgcil68eAF/f3+lafv27QPHcejUqZPS9IYNG+LevXvGGSEhhBBCSAnQORR5e3vj8ePHStOOHDkCe3t71KhRQ2m6jY0NbGxsjDNCQgghhJASoHMo+vDDD7Fq1SpkZmYCAK5evYrTp08jJiYGVlbKx2tfv34d5cqVM+5ICSGEEEKKkc6haPLkybh79y5CQ0PRrFkzNGzYEBzHYfz48Spt//rrLzRo0MCoAyWEEEJMCZNJS3sIRWIyWWkPQVB0PiW/WrVq2L9/P2bMmIHbt2+jXr16GD16NCIiIpTaHTx4EPb29ujWrZvRB0sIIYSYCk4kRsGfY8Ce33470c4VVm3jAGt7FGybDKSnal6AtS3EsRPAuQWg4O8Jisn5/+uFfMl7f75FVhA3GwXOvxake+eA3TuvdWyiRoMhqtwCHN28XYleF29s0KABtm3bprVNkyZN8O+//xo0KEIIMSa6xgspLez5beDxNfkTBw9YdZoNiG1QkNAHeHlH8wtt7CHu/TM413KQ/joQ7M6lt/OeXAds3gkzImuIu84H5x8O6YbhYDcPaRkRB1HrSRBVbgHp4aWwihpmyOqZHYqIhBBCSHFz8IBVv5WAxAkFqwfoFog8QyH9bRDYIy0bGgoDUUgkpBu+1C0QRXSHdOtEsKT9PFfGfFEoIjrJzs4Gx3HgOA7Z2dmlPRxCCDEdQgxEF//kuTLmzSj3PiOEEEKIGnau8l1mBgYiBxsR8qZUetuWAlGxoFBECCGEFBOrtnHyY4hoC5FJENzus8OHD6Ndu3bw8/MDx3HYvHmz0nzGGCZNmgRfX1/Y2dmhefPmuHnzZukMlhBCCNHG2p4CkQkRXCjKzs5GjRo1sHjxYrXz58yZg0WLFmHZsmU4deoUHBwcEBMTgzdv3pTwSAkhhBDtCrZNpkBkQgS3+yw2NhaxsbFq5zHGsGDBAkyYMAEdOnQAAKxevRre3t7YvHkzevToUZJDJYQUMyaTghOJS3sYWjGZjK71QjTTdh0iCkSCI7hQpE1KSgoeP36M5s2bK6a5uLigbt26OHHihNpQlJubi9zcXMXzjIwMAIBUKoVUqnq1UbFY2B/AhdSNvaT60/R/R9Qz9P/KkmtYLFZz8Tu8vfCc9PDSIk8rFoV3Aap0UDxXe+G7/3BhTSGOHApZ4h7IjsZrXS4XUAviFl+DE1uZ9fuB6lc9XetXIwEFInOuX0C/9TOpUFR4Q1pvb2+l6epuVlto5syZmDJlisr0pKQkxcXcCtna2iIkJMRIoy1eKSkpJbrLMCcnR/Hz9evXYW9vX2J9m7qsrCyDXm+pNVy4LkoXv3v3wnM6fOCLGn8Gce1ekG6f83bi+xe+K1xyzc7yQHRuPWTbpwJgGpfLhUZB3HwM2L1z4ALrlvj7sSRR/aqna/2qJZBAJArvotO6mDp9atikQhEf48ePx6hRoxTPMzIy4O/vj7CwMDg7O5fiyAwTGBhYov29e22iSpUq0RWB9VD4zZgvquFCen7gN/4M4ugRkB5YCNnxFdqXXLMzxO2m6R6Iui0CSz4M6dGfIfpkXYm/H0sS1a96vH/nQglE/31hAEr+70lJ06eGTSoU+fj4AACePHkCX19fxfQnT56gZs2aal8jkUggkUhUpovFYpPZTKtOSY/93f5M/f+upBn6f0U1DBgUiI4s075kvoFo4yjAK5THupgWql/1eI1dSIEoegSkZ36HuHavEv89lPRtd/RZP5M6OjAwMBA+Pj7Yt2+fYlpGRgZOnTqF+vXrl+LIhMk07tAs/DGS0ifIQCTL57MqxFIJLRAdWAjZhU381sWMCW5LUVZWFpKTkxXPU1JScPHiRbi7uyMgIAAjR47E9OnTERoaisDAQEycOBF+fn7o2LFj6Q1aoDiRGIzJdD9otPlYsPvnId03H5AVKM3Pz337vODvCWCdp4O9ugfpjulAvpZ90S6+sGozBcjPQcE/ccDrtLd9egTBqvNcPqtGLIjioGoKRMRUCTEQHVkG+FThtz5mTHCh6OzZs4iOjlY8L9wX3b9/fyQkJGDs2LHIzs7G4MGDkZaWhkaNGmHnzp2wtbUtrSELmixxD2Qbv4JOB40mH9L8gZ8nU/wobvk12NMkSNd8CuTlqLYt5F4BVrETgdevULB6IJD9XGm25hER8hYFImLSrG2FGYiIWoILRU2aNAFjWj6gOA5Tp07F1KlTS3BUpku+hci4H/js+S1I//qy6EDULwHIzVQbiAjRlfTwUoM/8N+9bxQFIlKSxLETwLmWo0BkIkzqmCJiXPp84HO+VRU/S9cNo0BESkyR1yGiLUREwDi3AApEJoRCkYXSKxD5VYO4x5K3E/Jea16wPoHIzlXvcRPyLgpEROik26dSIDIhFIoskN6BqM8KsOe3il6wPoHIwUN+92hCeBJCIOI8TeNCg6T0sGfJmmdSIBIcCkUWhlcgenZTvstMG30DUb+VgDVdFZvwI4hA5FcN4taT+K4CsXQUiASJQpEF4R2I1nxqvF1mhYFI4iS/ezQhehJMIOqzAuzVPb6rQSyZQAIRF9aUx+DNG4UiC2FYIDLSQdXvBqLVA7TfPZoQNQQViJ7dlF+nixB9CCUQ/XevP11JZUW3KW3GGKPgTsknxmeMQPTuKc0KhgSil3fowmFEL4ILRGs+Bdwr8Fwb/kr6FgnEiIQUiNpNgyxxD8RVYnQauvidTSizLwMb7yrP97IFtjQDrETAhRfA4OOqy4hvAISXAQreCS+NtwPi/y4z2LU88HV1fn1UcgF+jdRpVbSiUGTm5FeqHiOsLUQv7/BdHWKhBBmItL0/CHmf0ALRufWQnd+gcyh614Mc4MJL5Wll3zlENLtAdX7h9PddegVw/90ar56n4X0YikKRmRM3H6v9StX/oUBEhEoU3gXi2r0oEBHTJcRAtH0q4FNZr9WYfVkeVuJvqM57mAO03gM09AZW3FT/+l6HgUGhwOG76ue/u1y+fRiKQpGZY/fPUyAiJo0CETFpIithBiIeN1raeFf71pn9j+UPTR7lANMuASxX/XwpA5YmaR9DUX0YikKRmZPf3JUCETFd0jO/UyAiJkvcbBQ4/3CTD0SWgkKRuZOp2Yn7HwpExBTILmzSOp8CEREyzr8WpBuGUyAyEXRKvoUSRCCytuU5esNkZ2eD4zhwHIfs7OxSGQMxDgpEROike+dQIDIhFIoskCACkY09xLETeK4BIQIJRC6+BqwBsQTs3nktc007EJWRAMFOmudbi4AqrvJ/NQlylC/H0D6sjJRmKBRZGMEEot4/g3ML4LkWxNIJIhC5V4BVmykGrAWxbKYdiIKcgOQuQFJnYHBF9W12tgAud5D/q8nFDvLlBKkJPvr08VNd/ddBHQpFFkRQgcgzFNLtU3muCbFkgglE/RKAfNqtRvgQRiDiAmrxGTwAINwdcLKW/xytZoOpiAMaecl/buQlf67oV+IAcQLD+hQGBwcHOFkDNd0N60Pd6/mgUGQhBBeIfhuk/e7RhKghqECUm4mCf+J0HrvQb5MgpcNNSohAAlFoFMTNx/JZAQDA9gfA5nvA5ZfAnH9V58sYMPYckJQu/1emZihz/pW/fvM9YMcDw/pYmMh7VZTQ2WcWQJCB6NG/dJsPohfBBaLVAwEnL53HLxYBfQ8D19N1fola0jdvf373FgmGMNYtEkhRBBSIui0Cu38eXCC//U6vpUDXA9rbLLwmf2hy4SVQa6tx+gh3B0ZV1d5WFxSKzBznGQJx7AThBSJC9CDIQJT9XK9QBMgDkaG3Jnj3wnfv3iKBCJ3AAlHyYUiP/gzRJ+v4rIzZot1nZk7cehIFImLSuLCmwgxEhOhMgIFo4yit17GzVBSKzBx7dY8CETFp4sihFIiISRNmINL8/rBktPvMzEl3TKdAREyaLHEPBSJiskSNBkNUuQUFIhNBW4rMXf4bzfMoEBETIDsaD3MJRF62wMFW8oefvfo2P9QBrnYE2vmrn9+6rPY+ytob3gcxHnMKRPENDKurdv7y+T/UUT9fCLVLochSCSEQiXTfUCn005kB0xijuTGlQAQAHQKARt7yx6BQ1fmVXYDhlYEwF2BmhPplTA1/+3MlZ9X5g0IN74MYj/TwUrMIRAAQXsawupoVIZ8/vLK8/fuEULu0+8wSCSIQWUPcbJTOQ373dObaZYBF9eTTvzwJnHmh3NZODPzRRP5N448UYN4V5fnvntKcmgN8fEh+6ue79O1j2306pbmkCSIQ2bnqNeaLL4CC/8LzsSeq8+9lA3eygAqOwJ5H6pexL/Xtz/ezofIpfvSJ4X0Q42FJ+7XON5VABBheV7sfyQPNnSx5+/cJoXYpFFkaoQSirvPB+et3NdXC05kvvAS2PZRPe6jhb1vlv4AAByBRzTVh3j2ludbfQI6ad4G+fYQb6WqqRDeCCEQOHrBqG6fXuM+8AII3yX9WV1fZBUCNLZprFwC+PvdOeynAvVe/+x8b3gcpGaYUiACgwz7gShr/uvrqNBCfJA832WpOfBNC7VIosiRCCkQhkZDunQOrVt/yWhVNQaVQdoFubxp1f1SM3QcxLsEEon4rAWsNBz5oYWhdFd4ioTj7IMVPCIGI8wzRa8xP32ivLV3qqqj5pV27dEyRpRBaINrwZRF3jyZElaACkcQJBdsm818ZYrEEEYj8qkHcehLfVTBbFIosgRAD0c1DPFeGWCrBBaLVA4D0VM1tCVFDMIGozwqwV/f4robZolBk7lx8KRARkyfIQKTt/UGIGoIKRM9uyq9jR5RQKDJzVm2mmE0gauGneV73CsDXHwCOGo4Pqu4GTK4h/1cdRyv567tX4N8HKR5cQC2zDkRCqN1QNaf2E+MSXCBa86n269jpQAi1W9iHsWqYPt7NXX6OWQQiAPguAjj1DDj03qmaTXyA36PkP5dzAIafUp7PAdjTEihjC3xeCfBerbrsmRHA0Eryn5+85tcHKR7i5mPBkg+ZZSDSt3Z91qn+eTRG7S6pZ4SVIRoJMhBpe3/oQCi1W9hHWi6MwuS2FMXFxYHjOKVHpUqVSntYglXwT5xZBCJtmPYTcUymD6Ieu3/eLAMRQLVrCUThXcwuEAHmW7smuaWoatWq2Lt3r+K5lZVJrkbJeJ2meZ6JBaJvzql+kwDk03odAgIdgcXXVeczAC12Ax0DgM331H/MjD8HPMgGUrL490GKh3TffLMMRIBwavfzk2+/kRPjEtfuZXaBCBBO7Rb2cT3dODVskmnCysoKPj4+OrXNzc1Fbu7b7WoZGRkAAKlUCqlUqtJeLBYbZ5DFTN3Y36d1XQQUiHRdF21XMF1/R/vrL7+SPwD113nJKgBmX1HzQj36ANSviy7rp43F17BMzVXe/iOUQGTI+1Gf2lXHGLV7U15SVL960PV3Lj3zu+ADEd/6FULtFvZReAFdQ2vYJEPRzZs34efnB1tbW9SvXx8zZ85EQECA2rYzZ87ElClTVKYnJSXB0dFRaZqtrS1CQvS7mFVpSUlJwZs3mg+S07ougghEHESNBhu+LgKjbl2ysrIMWibVsHqCCETWtkZZF6Gg+tWdrr9z2YVNWpdT2oFIn3UxBYbWsMmForp16yIhIQFhYWFITU3FlClT0LhxY1y5cgVOTk4q7cePH49Ro97eYysjIwP+/v4ICwuDs7PpnnIRGBjI74VCCUStJ0FUuQUAA9ZFgNStS+E3Y76ohlUJIhDZ2EMcOwGA+dQw1a/ujPE7F0IgAsynfgHDa9jkQlFsbKzi5+rVq6Nu3booX7481q9fj0GDBqm0l0gkkEgkKtPFYrHJbKZVh9fYhRSIIrpDengprKKGmfTv4X3q1sXQ9aMaViaYQNT7Z3Bu8i3Upvx7eBfVr+4MHbsgApGLLwDzqV/A8Bo2ubPP3ufq6oqKFSsiOTm5tIcibEILRFsnFnn3aG3K2ssfmjhaAZVdtC+jsgvgoOVrgTH6IMYlqEDkGQrp9ql6rwPVLhFEIHKvIL+OnR68bM2/dk0+FGVlZeHWrVvw9fUt7aEIlxAD0cU/ea4M0NQHuNVF/miq5nh7RyvgYgfg347AD3XUL+OHOvL5lzqof4Maow9iXIILRL8NAnum35ex2mWodi2dYAJRvwQgX78z0bY0M//aNblQNHr0aBw6dAh37tzB8ePH0alTJ4jFYvTs2bO0hyZMdq5mFYgAoJE3YCWSPxp6q873dwAq/Hf8pqarYLf8b3oFRyDAoXj6IMYjyECk7f2hQc0yVLuWTFCBKDdTfh07PVhC7ZrcMUUPHjxAz5498eLFC3h6eqJRo0Y4efIkPD09S3togmTVNg4Q25hNIAKAFTeBpr5vf35fYjrwY6L8TTP+nPpljDsnv5rqnkfy9sXRBzEOzjME4tgJJh+IAGDLPaCKq/xnql3LIrhAtHog4OSl1zpceAFkF5h37ZpcKPrjjz9KewimxdoeBQl9zCYQAcDDHKDJTu1tvjqtff7W+/KHsfoovEYGMT5x60lgT5NMPhABwNM3wqtdUvwEGYiyn+sdigYfBy681DzfHGrX5EIR0U/BtslmFYiI5WGv7plFICKWiQtrCnHkUOEFIqKWyR1TRPSUnqp5HgUiYgKkO6ZTICImiwKRaaFQZKkoEBFTka/5SrsUiIjQyRL3UCAyIRSKLJFAApEovAu/8RMCCCMQiegIBKKd7Gg8KBCZDgpFlkYogajxZxDX7qXX0Od8CNipuTCpnRjYGA2cb6f5gOcRVYBrHeX/qlPLXf76jdH8+yAlSBCByBriZqM0z9eBUGq3ZxCv4RMDmXIgEkrtFvZhrBqmUGRJhBSIokdAeuZ3vYYf7QvEllOd3roc0DEAqO4OjK2mpj8OmBMBVHSR/yviVNuMqSZ/fccAfn1wapZJiolQAlHX+eD8a/FcCTkh1K6IA0ZUNmg1CA+CCER2rrzHL5TaLezDWDVMochSCC0QHVhY5N2j35edD1xUczrohZdA5n+fJwfUHFcuY8DRp/Kfjz6VP39f4esyefbBNG8dJ8YkpEAUEgnp3jk8V0ROCLUrY+pfR4qPIAKRg4f8OnY8CaV2C/swVg3TDnFLIMRAdGQZ4KNhm6oG7fcDtzNVp9/OBII3AW42wC018wGg1R4g1Bm4qeFmyfE3gH2pwKs84GUuvz5IMRNaINrwJVjmM54rIyeU2v3iFHCiDb91IPoRTCDqtxKw1nKTsSIIpXYL+3CwMk4N05Yic2dtK8xAxENGnuZ5L3O1h5V8GXAtTf6vJrcy1b8xde2DFCMhBiKt7w/dCaF2C7QsmxiPoAKRxEl+HTsDCKF2C/swVg3TliIzJ46dAM61nMkHImLBXHxhFTvRLAMRsRyCC0SrBwA2/LcUmSvaUmTmOLcACkTEpFm1mUKBiJg0QQYibe8PC0ZbisycdPtUCkTEtOXnUCAiJosLqAVx8zEUiEwEbSkyc+xZsuaZJhqIxBwwNEz+EGs4Fb6pDzCpBlBWw9bhsvby+U19iq8PYhwF/8SZTSASUe1aHHHzsWYTiLqWN//apS1FlspEAxEADK4I/Fjv7fOlScrzy9oD21sAViKgqa/6uy6viQQaecsPzgveJL87syF90F3Hi9HrNM3zTCgQAUDnAODr6m+fC6F2i7orOTEMu3/eLAIRYBm1S1uKLJFAAhEX1pTH4An5j4kFImKZpPvmm0UgshS0pcjSCCUQ1ewMceRQHisgv7aFup8LPcwBWu8BGnoDK26qX0avw8CgUODYE9VvK3z68LbVbx2IgQQRiDiIGg3Wa9h/3gMe/FdvQqldUsxkBRpnmVogmn1ZXr/mXLsUiiyJkAJRu2mQJe6BuEqM3qshZaqbVd+3/7H8ocmjHGDaJeP1QaGoBAklELWeBFHlFnoNXSbA2iWlQxCByFq/D66Nd+VXmdbEHGqXdp9ZCqEFonPr/7t7NCF6EFIgiugO6eGlPFeEWDJBBCIbe4hjJ/BcA/NFocgSCDEQbZ/KY0WIRRNaINo6ESxpP8+VIZZKMIGo98/g3AJ4roX5olBk7kRWAg1EdAdVogchBqIi3h+EvE9QgcgzFFL6cqqCQpGZEzcbZTaByMVG87wyEiDYSfN8axFQxVX+rybBTvLl8O2DFBM7V7MOREKoXSv6S1DsBBeIfhuk/Tp2OhBC7Rb2YawapreCmeP8a5lFIAKALU2BIDVvjiAnILkLkNRZfp0LdXa2AC53kP+rzuCK8tcnd+HfBykeVm3jzDYQCaV2f6rLb/xEN4IMRNreHzoQSu0W9mGsGqZQZOake+eYRSACAAdroKa76vRwd8DJWv5ztK+aMXNAIy/5z4285M/fV/g6J559cBquvkqMwNreLAMRIIzaFXHqX0eMg/MMMbtABAindgv7MFYNUygyc+zeeS1zTScQAcCBVGDHA9Xp2x8Am+8Bl18Cc9S812UMGHsOSEqX/ytTM5Q5/8pfv/kevz4YHSJVbAq2TTbLQAQIo3ZlDFiYaPi6EPXErSeZXSAChFO7hX0Yq4bpOkUWy7QCEQCMPQu8lqpOfy0Fuh7Q/tqF1+QPTS68BGpt1Txflz5IMUlP1TzPhAMRIJzaXXsbGFVVexvCD3t1z+wCESCc2i3sI9zdODVMW4oskjACERdQi8/gCZEz8UBELIN0x3SzC0TmjEKRxRFIIAqNgrj5WD4rQIhgApEovAu/8RPLkf9G8zwKRIJDociiCCgQdVsEdl/b8U6EaCCUQNT4M4hr9+K3DoQIIRCJ6Aia91EoshgCC0TJh+V3jyZEH0IKRNEjID3zO7/1IJZNEIHIGuJmo3iugPmiUGQRBBiINo7SevdobcraAwdbyR9+9urb/FAHuNoRaOevfn47f/n8H+oUXx/EyIQWiA4shOzCJr1WwcuWatfiCSUQdZ0Pzl+/4zrjG5h/7VIosgDCDET5fFYFADAoFGjkLX8MClWdX9kFGF4ZCHMBZkaoX8asCPn84ZXl7YujD2JEQgxERbw/1OkQQLVr0YQUiEIiId07R6/hh5cx/9o12VC0ePFiVKhQAba2tqhbty5Onz5d2kMSJFGjwWYViADg6BOgQCZ/HHuiOv9eNnAnS/7znkfql7H7v+l3suTti6MPYiTWtmYRiADg4guqXYsltEC04csirmOnyhJq1ySPslq3bh1GjRqFZcuWoW7duliwYAFiYmKQlJQELy+v0h6eoIgqtzCrQAQA+x8Dwf/ttXio5kzX7AKgxhYgwAFITFe/jK9OA/FJ8jdZtpq9ePr2EU5XBC424tgJ4FzLmXwgAoAzL4RXu6QECDEQ3TwE+FTRazU67AOupJl37ZrklqL58+fj008/xcCBA1GlShUsW7YM9vb2+N///lfaQxMc6eGlZhWICj3MUf+mKZRdUPSbJjFd/RvTmH0Qw3FuAWYRiApR7VoYF19hBiIenr4x/9o1uS1FeXl5OHfuHMaPH6+YJhKJ0Lx5c5w4cUKlfW5uLnJzcxXP09Pl/5uvXr2CVKp6eWSxWAxXV1el1wiJq6srMjIy1I79fWKxGAX3k8Bc1Oyc/Y+oenuII7pDeuAXyC7vA7S05UIjIW74CWRnNkN24nfAJURz23I1II4eCXbtKKSHlgBOFVTb2JeFlR7rEmQFSG2LbFoqgqyAjAyoXZeMjAwAAON5LxBLr2Hp4WWQZb/RXJsiK4ijvgDnVQPSLdPAnj7SWsei+gMgCm0K6Y75YCn/am9bxPvDXGqY6ld3+tYvi/oaePUCBbtmAlZugIub+sZ2LrCKGQ/kW6Hgn4mA1FpzbVrbQtxiDDhbX0g3TQTj+f4wl/oFjFjDzMQ8fPiQAWDHjx9Xmj5mzBhWp04dlfaTJ09mkG/OoAc9SvVx//59XjVPNUwPITyofulh6g9daphjzLRuZfno0SOULVsWx48fR/369RXTx44di0OHDuHUqVNK7d//liKTyfDy5UuUKVMGHN3anJQAxhgyMzPh5+cHkUj/PdZUw6Q0Uf0SU6dPDZvc7jMPDw+IxWI8eaJ8aPqTJ0/g4+Oj0l4ikUAikShNc3V1Lc4hEqLCxcWF92uphklpo/olpk7XGja5A61tbGwQERGBffv2KabJZDLs27dPacsRIYQQQog+TG5LEQCMGjUK/fv3x4cffog6depgwYIFyM7OxsCBA0t7aIQQQggxUSYZij766CM8e/YMkyZNwuPHj1GzZk3s3LkT3t7epT00QgghhJgokzvQmhBCCCGkOJjcMUWEEEIIIcWBQhEhhBBCCCgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEULMyJ07d8BxHC5evFjaQyFmqEmTJhg5cmRpD4MUIwpFRG8ymUzQ/Q0YMAAcx2HWrFlK0zdv3gyO43RejqYPwISEBLi6uuo1JqFhJfw7ZDKpzm05jtP6iIuLK76B6kBb8DL1P5rSki0LvftbtmwZnJycUFBQoJiWlZUFa2trNGnSRKntwYMHwXEcbt26ZYSRqmfOnxGWyqq0B0BMj0gkwvLly5Gamlrsffn6+mLIkCF6v87W1hazZ8/GkCFD4ObmVgwjM22cSISCbVPBHl7S3MjOFVZt4wBrexRsmwyka/l9W9tCHDsBnFsApNungj1LftuXRxCsOs/VeWzv1tW6deswadIkJCUlKaY5OjrqvCyiH7EI6HsYuJ6uPL2SCzCpBvAgB5h4Ach9L+NKxMC0cKCcPTDlEpD03usBoGcQ0CUA2HQPWHtbvsxfI/UbX3R0NLKysnD27FnUq1cPAHDkyBH4+Pjg1KlTePPmDWxtbQEABw4cQEBAAIKDg/XqgzEGqVQKKyv682iJaEsR4SU1NRV3794t9gff4NW8eXP4+Phg5syZGtts2rQJVatWhUQiQYUKFTBv3jy+/x1YunQpgoODYWNjg7CwMPz6669K8zmOw9KlSxEbGws7OzsEBQVh48aNvPszBvbwEvD4mvpH5lNYtRoPiG1QkNAHSNqnue3LOxA3/z9wruUg/XUg2L9/K81nz2/rNS4fHx/Fw8XFBRzHKZ57eXlh/vz5KFeuHCQSCWrWrImdO3dqXJZUKsXHH3+MSpUq4d69ewCALVu2oFatWrC1tUVQUBCmTJmitOWB4zj88ssv6NSpE+zt7REaGoq///6b1//xq1ev0K9fP7i5ucHe3h6xsbG4efOmYn7hFoXNmzcjNDQUtra2iImJwf3793n1ZwxJGcCFl8qP9gFAqAsQ7Qv42avOL2svnxfqAnQIUJ1/6RUwojJQ3kn+76VXqsFLF2FhYfD19cXBgwcV0w4ePIgOHTogMDAQJ0+eVJoeHR2N3NxcfPnll/Dy8oKtrS0aNWqEM2fOKLXjOA47duxAREQEJBIJjh49iuzsbPTr1w+Ojo7w9fU16PMBMM3PCEtEoYiYJbFYjO+++w4//vgjHjx4oDL/3Llz6N69O3r06IF///0XcXFxmDhxIhISEvTu66+//sKIESPwf//3f7hy5QqGDBmCgQMH4sCBA0rtJk6ciC5duuDSpUvo3bs3evTogcTERL6rWHwcPGDVbyUgcULB6gHAyzua29rYQ9z7Z3CeoZD+Ngjs0b/FOrSFCxdi3rx5+P7773H58mXExMSgffv2SkGjUG5uLrp164aLFy/iyJEjCAgIwJEjR9CvXz+MGDEC165dw/Lly5GQkIAZM2YovXbKlCno3r07Ll++jNatW6N37954+fKl3uMdMGAAzp49i7///hsnTpwAYwytW7dGfn6+ok1OTg5mzJiB1atX49ixY0hLS0OPHj30/88xEsZUpx3477tJZj5wUc1/w4WX8nnvtn2XjAFHn8p/PvpU/pyv6OhopffWgQMH0KRJE0RFRSmmv379GqdOnUJ0dDTGjh2LTZs2YdWqVTh//jxCQkIQExOj8vscN24cZs2ahcTERFSvXh1jxozBoUOHsGXLFuzevRsHDx7E+fPneY3Z7D4jzBjHmLq3ACHaxcXF4e7du8XeT/ny5fU+hmTAgAFIS0vD5s2bUb9+fVSpUgUrVqzA5s2b0alTJzDG0Lt3bzx79gy7d+9WvG7s2LHYtm0brl69CkB+vMDx48dhY2OjtPyCggLY2toiLS0NANCwYUNUrVoV8fHxijbdu3dHdnY2tm3bBkD+LfCzzz7D0qVLFW3q1auHWrVqYcmSJXqtn7Hkx3eRb815V3EEIp8qsB68idcYExISMHLkSMX/ddmyZTFs2DB88803ijZ16tRB7dq1sXjxYty5cweBgYE4cuQI4uLikJubi3/++QcuLi4A5FsQmzVrhvHjxyte/9tvv2Hs2LF49OgRAPnvasKECZg2bRoAIDs7G46OjtixYwdatWql6MPOzg4ikfL3ytevX2P48OFYsGABbt68iYoVK+LYsWNo0KABAODFixfw9/fHqlWr0K1bNyQkJGDgwIE4efIk6tatCwC4fv06KleujFOnTqFOnTq8/t8MUXurPOS8L9gJeJUHvMxV/zp3CeBmA9zKVD/fWgSEOgM3M4B8GRDuDpxpp//4fvnlF0VNvH79Gu7u7nj06BH27t2LZcuW4dChQ9i/fz+aNWuGO3fuIDQ0FAkJCejVqxcAID8/HxUqVMDIkSMxZswYxRalzZs3o0OHDgDkxymVKVMGv/32G7p16wYAePnyJcqVK4fBgwdjwYIFAMz/M8IS0ZYiYtZmz56NVatWqXzbSkxMRMOGDZWmNWzYEDdv3oRU+vaAid69e+PixYtKj6lTp+q0rPf7rF+/vspzQX0LLK4tRCLjHJuRkZGBR48e6fR/3bNnT2RnZ2P37t2KQAQAly5dwtSpU+Ho6Kh4fPrpp0hNTUVOTo6iXfXq1RU/Ozg4wNnZGU+fPlXqY926dSq18eGHHyrmJyYmwsrKShF2AKBMmTIICwtTGq+VlRVq166teF6pUiW4uroKqzYgDzuaAhEgn6cpEAHyIHQtTf6vIZo0aYLs7GycOXMGR44cQcWKFeHp6YmoqCjFcUUHDx5EUFAQ0tPTkZ+fr1Qz1tbWqFOnjsr/77u/u1u3biEvL0/pd+fu7o6wsDCV8VjUZ4QFoCPJiFmLjIxETEwMxo8fjwEDBuj9ehcXF4SEhChN8/LyMtLoBKTYApE1xM1GGXu0RWrdujV+++03nDhxAk2bNlVMz8rKwpQpU9C5c2eV1xQeoAvI/3C+i+M4lbMg/f39VWrDzs7OGMMnWoSEhKBcuXI4cOAAXr16haioKACAn58f/P39cfz4cRw4cEDp964LBwcHXuOxmM8IC0FbiojZmzVrFrZu3YoTJ04oplWuXBnHjh1Tanfs2DFUrFgRYrFYr+VrWlaVKlWUpr17EGjh88qVK+vVV7EozkDUdT44/1pGGaazszP8/Px0+r8eOnQoZs2ahfbt2+PQoUOK6bVq1UJSUhJCQkJUHu/vCjNU5cqVUVBQgFOnTimmvXjxAklJSUrjLSgowNmzZxXPk5KSkJaWJozaEKjo6GgcPHgQBw8eVDoVPzIyEjt27MDp06cRHR2tOLD53ZrJz8/HmTNnVGrmXcHBwbC2tlb63b169Qo3btzgNV6T/4ywILSliPDi6+trMv1Uq1YNvXv3xqJFixTT/u///g+1a9fGtGnT8NFHH+HEiRP46aefeO27HzNmDLp3747w8HA0b94cW7duxZ9//om9e/cqtduwYQM+/PBDNGrUCGvWrMHp06exYsUKg9ePL84jCOz90+5t7AEfDX8s3j/tXibV3FZkBXGzUeD8a0F2+leIG3xslDGPGTMGkydPRnBwMGrWrImVK1fi4sWLWLNmjUrb4cOHQyqVom3bttixYwcaNWqESZMmoW3btggICEDXrl0hEolw6dIlXLlyBdOnTzfKGAuFhoaiQ4cO+PTTT7F8+XI4OTlh3LhxKFu2rOLYFUC+VWr48OFYtGgRrKys8MUXX6BevXqlcjwRID9VXuj9REdHY9iwYcjPz1dsKQKAqKgofPHFF8jLy0N0dDQcHBwwdOhQjBkzBu7u7ggICMCcOXOQk5ODQYMGaVy+o6MjBg0ahDFjxqBMmTLw8vLCt99+yzs4m+pnhEVihOhJKpUKur/+/fuzDh06KE1LSUlhNjY27N2S37hxI6tSpQqztrZmAQEBbO7cuUqviYqKYiNGjFBZ/sqVK5mLi4vStCVLlrCgoCBmbW3NKlasyFavXq00HwBbvHgxa9GiBZNIJKxChQps3bp1eq2XMcmkBSbR3/v/11KplMXFxbGyZcsya2trVqNGDbZjxw7F/JSUFAaAXbhwQTFt3rx5zMnJiR07dowxxtjOnTtZgwYNmJ2dHXN2dmZ16tRh8fHxivYA2F9//aU0DhcXF7Zy5UqNfRR6v2ZevnzJ+vbty1xcXJidnR2LiYlhN27cUFm/TZs2saCgICaRSFjz5s3Z3bt39f/PMoKCkn1rswIZv9cV/g4qVaqkNP3OnTsMAAsLC1NMe/36NRs+fDjz8PBgEomENWzYkJ0+fVox/8CBAwwAe/XqldKyMjMzWZ8+fZi9vT3z9vZmc+bMUfn9mvNnhKWis88IKQEcx+Gvv/5Cx44dS3soREDeP7uOWC76jBAGOqaIEEIIIQQUigghhBBCANDFGwkhhBBCANCWIkIIIYQQABSKCCGEEEIAUCgihBBCCAFAoYgQQgghBACFIkIIIYQQABSKCCGEEEIAUCgihBBCCAFAoYgQQgghBACFIkIIIYQQABSKCCGEEEIAUCgihBBCCAFAoYgQQgghBACFIkIIIYQQABSKCCGEEEIAAP8P7w/uDoRpkpEAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 600x400 with 6 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "CHECKPOINTS = [300, 600, 900, 1200, 1500, 3000]\n",
+    "SEEDS = [0, 14, 41, 53, 96]\n",
+    "\n",
+    "plot_surprisal_differences_checkpoints(seeds=SEEDS, checkpoints=CHECKPOINTS)\n",
+    "plt.savefig(f\"figures/hop_surprisals.pdf\", format=\"pdf\", bbox_inches=\"tight\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkUAAAGhCAYAAABvQ8DIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB+I0lEQVR4nO3dd3wT9f8H8FeS7t0CXdACHZQhoyB7b8qQjYACIgoiIspPEJRREARBEFGm8mUo8mUpfBFB9pKNILIpe5RNNxSafH5/xEZCRpNrSi/J6/l45NHm7nP3+Vz7TvvK3eVOIYQQICIiInJyysIeABEREZEcMBQRERERgaGIiIiICABDEREREREAhiIiIiIiAAxFRERERAAYioiIiIgAMBQRERERAWAoIiIiIgLAUEREREQEgKFIVk6ePImuXbsiKioKXl5eKFq0KBo0aIB169YZbX/69Gm0atUKPj4+CAoKQq9evXD37l2DdhqNBlOmTEHp0qXh4eGBSpUqYdmyZRaPa8+ePUhISEDx4sXh4eGByMhItGvXDj/99JOuTVZWFhITE7Fjxw6rt5scz59//olXXnkFQUFB8PLywksvvYSZM2catNu7dy/q1asHLy8vhIaG4v3330dGRoZBu+zsbHz88ccIDw+Hp6cnatasic2bN1s8nnXr1qFhw4YIDg6Gl5cXoqKi0K1bN2zcuFHX5ubNm0hMTMSxY8ckbTM5hiNHjqBVq1bw8/ODr68vWrRoYbImWL8OSJBsrF+/XrRs2VIkJiaK+fPnixkzZoj69esLAGLevHl6ba9duyaKFi0qoqOjxddffy0mTpwoAgMDReXKlUV2drZe2xEjRggA4u233xbz588Xbdq0EQDEsmXL8hzTihUrhEKhEPHx8eKLL74Q8+fPFyNHjhR169YVjRo10rW7e/euACDGjh1rk58F2a/ff/9duLm5iZo1a4rp06eL+fPni48//lgMGzZMr93Ro0eFh4eHiI+PF3PmzBGffvqpcHd3F61atTJYZ/fu3YWLi4v46KOPxLx580Tt2rWFi4uL2L17d57jmTp1qgAgGjZsKKZPny7mzp0rPvroI1GlShXRp08fXbtDhw4JAGLhwoX5/RGQnTpy5Ijw8PAQsbGx4ssvvxRTpkwRpUqVEn5+fuLMmTN6bVm/jomhSOZycnJE5cqVRVxcnN70gQMHCk9PT3HlyhXdtM2bNxsEqOvXrwtXV1cxaNAg3TSNRiPq168vSpQoIXJycsz2X758eVGhQgWDoCWEELdv39Z9z1BEQgiRmpoqQkJCRMeOHYVarTbbNiEhQYSFhYnU1FTdtO+++04AEL///rtu2oEDBwQAMXXqVN20R48eiejoaFG7dm2zfTx9+lT4+fmJ5s2bG53/bA3znwq1bt1aBAYGinv37umm3bx5U/j4+IhOnTrptWX9OiaGIjvQtm1bERISojctODhYdO3a1aBtmTJlRNOmTXXPZ82aJQCIkydP6rX76aefBIA836m4u7uLN954w2ybS5cuCQAGj2cD0unTp0Xnzp1FYGCgcHd3F9WqVRNr167VW8/ChQsFALFz507Rv39/ERQUJHx9fUWvXr3EgwcP9NoeOnRItGjRQhQpUkR4eHiIUqVKib59+5odJxW8OXPmCADi1KlTQgghMjIyjIaj1NRU4eLiYrD3KDs7W/j4+Ih+/frppg0bNkyoVCq9fz5CCPH5558LAOLq1asmx5OcnCwAiMTERLPj3r59u9EafvYfzP79+0XLli2Fn5+f8PT0FA0aNBB79uzRW8/YsWMFAHH69GnRtWtX4evrK4KCgsT7778vHj16pNd206ZNom7dusLf3194e3uLMmXKiJEjR5odJxUsX19fo39X27RpI9zc3ER6eroQgvUrhOPWL88pkqHMzEzcu3cPFy5cwFdffYUNGzagadOmuvk3btzAnTt38PLLLxssW6NGDRw9elT3/OjRo/D29ka5cuUM2uXON6dkyZLYunUrrl+/brJNsWLFMGfOHABAx44d8cMPP+CHH35Ap06dAGjPlapVqxZOnz6NESNGYNq0afD29kaHDh3wyy+/GKzvvffew+nTp5GYmIjevXtj6dKl6NChA4QQAIA7d+6gRYsWuHz5MkaMGIFvvvkGr732Gvbv3292W6jgbdmyBX5+frhx4wbi4uLg4+MDPz8/DBw4EI8fP9a1+/vvv5GTk2NQw25ubqhSpYpBDZcpUwZ+fn56bXNr2Nw5FMHBwfD09MS6devw4MEDk+3KlSuH8ePHAwD69++vq+EGDRoAALZt24YGDRogLS0NY8eOxeeff46UlBQ0adIEBw8eNFhft27d8PjxY0yaNAmtW7fGzJkz0b9/f938kydPom3btsjOzsb48eMxbdo0vPLKK/jjjz9MjpEKXnZ2Njw9PQ2me3l54cmTJzhx4gQA1q9D129hpzIyNGDAAF3SVyqVokuXLnp7SnJ3ky5ZssRg2WHDhgkA4vHjx0II7TucqKgog3aZmZkCgBgxYoTZsSxYsEAAEG5ubqJx48Zi9OjRYvfu3Qbv/s0dPmvatKmoWLGibkxCaA/h1alTR8TGxuqm5e4pqlatmnjy5Ilu+pQpUwQA3Z6lX375RQAQhw4dMjt2evEqVaokvLy8hJeXlxg8eLBYvXq1GDx4sAAgunfvrmu3cuVKAUDs2rXLYB1du3YVoaGhuucVKlQQTZo0MWh38uRJAUDMnTvX7JjGjBkjAAhvb2+RkJAgJk6cKI4cOWLQztThB41GI2JjY0XLli2FRqPRTc/KyhKlS5fWO7SR+077lVde0VvHu+++KwCIv/76SwghxFdffSUAiLt375odO71YFStWFGXKlNE7rSA7O1tERkYKAGLVqlVCCNavI9cv9xTJ0AcffIDNmzdj8eLFSEhIgFqtxpMnT3TzHz16BABwd3c3WNbDw0OvzaNHjyxqZ8qbb76JjRs3olGjRtizZw8+++wz1K9fH7Gxsdi7d2+e2/LgwQNs27YN3bp1Q3p6Ou7du4d79+7h/v37aNmyJc6fP48bN27oLdO/f3+4urrqng8cOBAuLi747bffAAABAQEAgF9//RVPnz7Ncwz04mRkZCArKwu9e/fGzJkz0alTJ8ycORMDBgzAf//7X5w/fx5A3jX8bF3mt4bHjRuHn376CfHx8fj999/x6aefolq1aqhatSpOnz6d5zYdO3YM58+fR8+ePXH//n1dDWdmZqJp06bYtWsXNBqN3jKDBg3Sez548GAAMKjhtWvXGixLhefdd9/FuXPn0K9fP5w6dQonTpxA7969kZycDED/7yrA+nXE+mUokqGyZcuiWbNm6N27N3799VdkZGSgXbt2usNHubt3s7OzDZbNPUSR28bT09Oidua0bNkSv//+O1JSUrBr1y4MGjQIV65cQdu2bXHnzh2zyyYlJUEIgdGjR6NYsWJ6j7FjxwKAwTpiY2P1nvv4+CAsLAyXL18GADRs2BCdO3fGuHHjULRoUbRv3x4LFy40up30YuXWU48ePfSm9+zZEwCwb98+vXamavPZurRFDffo0QO7d+/Gw4cPsWnTJvTs2RNHjx5Fu3bt9A7rGZMb5Pr06WNQw99//z2ys7ORmpqqt8zzNRwdHQ2lUqmr4VdffRV169bFW2+9hZCQEHTv3h0rVqxwuH8w9uadd97BJ598gp9++gkVKlRAxYoVceHCBQwfPhyA9m8RwPp15Pp1KewBUN66dOmCAQMG4Ny5c4iLi0NYWBgA6N69PCs5ORlBQUG6dyZhYWHYvn07hBBQKBR67QAgPDzc4nF4eXmhfv36qF+/PooWLYpx48Zhw4YN6NOnj8llcl8kH330EVq2bGm0TUxMjMVjAACFQoFVq1Zh//79WLduHX7//Xe8+eabmDZtGvbv36/7w0UvXnh4OE6ePImQkBC96cHBwQCAhw8fAkCeNfxsXYaFhRnsTXx2WWtq2M/PD82bN0fz5s3h6uqKxYsX48CBA2jYsKHJZXJreOrUqahSpYrRNnnV3LOvPUD7j3DXrl3Yvn071q9fj40bN2L58uVo0qQJNm3aBJVKZfE2kW1NnDgRH330EU6ePAl/f39UrFgRn3zyCQCgTJkyAFi/jly/3FNkB3J3r+am+eLFi6NYsWI4fPiwQduDBw/qFX6VKlWQlZVlsJv1wIEDuvlS5J5gmPvCfv5FkysqKgoA4OrqimbNmhl9+Pr66i2T+84mV0ZGBpKTk1GqVCm96bVq1cLEiRNx+PBhLF26FCdPnsR///tfSdtDtlGtWjUAMPgncPPmTQDak/IB4KWXXoKLi4tBDT958gTHjh0zqOFz584hLS1Nr+2LquHo6GgA2n9Ipmr42cO9gGENJyUlQaPR6NWwUqlE06ZNMX36dJw6dQoTJ07Etm3bsH37dknbQ7YTGBiIevXqoWLFigC0HyAoUaIEypYtC4D1Czhu/TIUyYixQ1FPnz7FkiVL4OnpifLly+umd+7cGb/++iuuXbumm7Z161acO3cOXbt21U1r3749XF1dMXv2bN00IQTmzp2L4sWLo06dOmbHtHXrVqPTc48tx8XFAdDuRQKAlJQUvXbBwcFo1KgR5s2bZ/RdlbErcM+fP1/vXKE5c+YgJycHCQkJALR7G3IPJebK/cPCQ2iFq1u3bgCABQsW6E3//vvv4eLigkaNGgEA/P390axZM/z4449IT0/Xtfvhhx+QkZGhV8NdunSBWq3G/PnzddOys7OxcOFC1KxZExERESbHk5WVpTtk97wNGzYA+LeGvb29ARjWcLVq1RAdHY0vv/zS6NWKjdXwrFmz9J5/8803AKCrYWOfJGINy9Py5ctx6NAhfPDBB1Aqtf8yWb+OW788fCYjAwYMQFpaGho0aIDixYvj1q1bWLp0Kc6cOYNp06bp7eL85JNPsHLlSjRu3BhDhgxBRkYGpk6diooVK6Jv3766diVKlMAHH3yAqVOn4unTp6hevTrWrFmD3bt3Y+nSpXnu5mzfvj1Kly6Ndu3aITo6GpmZmdiyZQvWrVuH6tWro127dgCgC23Lly9HmTJlEBQUhJdeegkvvfQSZs2apXvX9fbbbyMqKgq3b9/Gvn37cP36dfz11196fT558gRNmzZFt27dcPbsWcyePRv16tXDK6+8AgBYvHgxZs+ejY4dOyI6Ohrp6en47rvv4Ofnh9atW9vq10ESxMfH480338R//vMf5OTkoGHDhtixYwdWrlyJkSNH6h0qmDhxIurUqYOGDRuif//+uH79OqZNm4YWLVqgVatWunY1a9ZE165dMXLkSNy5cwcxMTFYvHgxLl++bBC+npeVlYU6deqgVq1aaNWqFSIiIpCSkqJ7DXTo0AHx8fEAtO+oAwICMHfuXPj6+sLb2xs1a9ZE6dKl8f333yMhIQEVKlRA3759Ubx4cdy4cQPbt2+Hn5+fwa14Ll26hFdeeQWtWrXCvn378OOPP6Jnz56oXLkyAGD8+PHYtWsX2rRpg5IlS+LOnTuYPXs2SpQogXr16tnq10FW2rVrF8aPH48WLVqgSJEi2L9/PxYuXIhWrVphyJAhem1Zvw5av4X62TfSs2zZMtGsWTMREhIiXFxcRGBgoGjWrJnBRQ5znThxQrRo0UJ4eXmJgIAA8dprr4lbt24ZtFOr1eLzzz8XJUuWFG5ubqJChQrixx9/tHhM3bt3F9HR0cLT01N4eHiI8uXLi08//VSkpaXptd27d6+oVq2acHNzM/h4/oULF0Tv3r1FaGiocHV1FcWLFxdt27bVfcRVCMOLNwYGBgofHx/x2muvifv37+va/fnnn6JHjx4iMjJSuLu7i+DgYNG2bVtx+PBhi7aJCtaTJ09EYmKiKFmypHB1dRUxMTHiq6++Mtp29+7dok6dOsLDw0MUK1ZMDBo0yKCuhNBeAfijjz4SoaGhwt3dXVSvXl1s3Lgxz7E8ffpUfPfdd6JDhw6iZMmSwt3dXXh5eYn4+HgxdepUgyu1r127VpQvX164uLgYfLz56NGjolOnTqJIkSLC3d1dlCxZUnTr1k1s3bpV1yb3I82nTp0SXbp0Eb6+viIwMFC89957ehe/27p1q2jfvr0IDw8Xbm5uIjw8XPTo0UOcO3cuz22igpOUlCRatGghihYtKtzd3UXZsmXFpEmTjF7RXwjWryPWr0KI545DEBWSRYsWoW/fvjh06JDRC1MSyV1iYiLGjRuHu3fvomjRooU9HCKrsH55ThERERERAIYiIiIiIgAMRUREREQAAJ5TRERERATuKSIiIiICwFBEREREBMAJL96o0Whw8+ZN+Pr6mrwsOpEtCSGQnp6O8PBw3RVx84M1TC8S65fsnTU17HSh6ObNm2Yvq05UUK5du4YSJUrkez2sYSoMrF+yd5bUsNOFotybj167dg1+fn6FPBpyBmlpaYiIiDC48a1UrGF6kVi/ZO+sqWGnC0W5u2v9/Pz4gqQXylaHCljDVBhYv2TvLKlhnmhNREREBIYiIiIiIgAMRUREREQAGIqIiIiIADAUEREREQFgKCIiIiICwFBEREREBIChiIiIiAgAQxERERERAIYiIiIiIgAyDEW7du1Cu3btEB4eDoVCgTVr1ujNf+ONN6BQKPQerVq1KpzBEhERkcOQXSjKzMxE5cqVMWvWLJNtWrVqheTkZN1j2bJlL3CERERE5Ihkd0PYhIQEJCQkmG3j7u6O0NDQFzQiIiIicgayC0WW2LFjB4KDgxEYGIgmTZpgwoQJKFKkiNG22dnZyM7O1j1PS0sDAKjVaqjV6hcyXnJu+a0z1jAVphddvwqFAkql7A5i6NFoNBBCFPYwyELW1LDdhaJWrVqhU6dOKF26NC5cuIBPPvkECQkJ2LdvH1QqlUH7SZMmYdy4cQbTz549Cx8fnxcxZHJyGRkZ+VqeNUyF6UXWr4eHB2JiYjBv3jwkJyfnq9+CEhYWhgEDBiApKQmPHz8u7OGQBaypYYWQcdxVKBT45Zdf0KFDB5NtLl68iOjoaGzZsgVNmzY1mG/sXUpERAQePHgAPz+/ghg2kZ60tDQEBQUhNTVVUs2xhqkwvej6ValUSExMxJUrV/I17oJSsmRJJCYmci+tHbGmhu1uT9HzoqKiULRoUSQlJRkNRe7u7nB3dzeYrlKpjO5ZIrK1/NYZa5gKE+vXOHseu7Ox5ncl7wO3Frh+/Tru37+PsLCwwh4KERER2THZ7SnKyMhAUlKS7vmlS5dw7NgxBAUFISgoCOPGjUPnzp0RGhqKCxcuYPjw4YiJiUHLli0LcdRERERk72QXig4fPozGjRvrng8dOhQA0KdPH8yZMwfHjx/H4sWLkZKSgvDwcLRo0QKfffaZ0d2zRERERJaSXShq1KiR2Y86/v777y9wNEREROQs7P6cIiIiIiJbYCgiIiIiAkMREREREQCGIiIiIiIADEVEREREABiKiIiIiAAwFBEREREBYCgiIiIiAsBQRERERASAoYiIiIgIAEMREREREQCGIiIiIiIADEVEREREABiKiIiIiAAwFBEREREBYCgiIiIiAsBQRERERASAoYiIiIgIAEMRkcPRaDSFPYQ82cMYicj5uBT2AIjItpRKJebNm4fk5OTCHopRYWFhGDBgQGEPg4jIAEMRkQNKTk7GlStXCnsYRER2hYfPiIiIiMBQRERERASAoYiIiIgIAEMREREREQCGIiIiIiIADEVEREREABiKiIiIiAAwFBEREREBYCgiIiIiAsBQRERERASAoYiIiIgIgAxD0a5du9CuXTuEh4dDoVBgzZo1evOFEBgzZgzCwsLg6emJZs2a4fz584UzWCIiInIYsgtFmZmZqFy5MmbNmmV0/pQpUzBz5kzMnTsXBw4cgLe3N1q2bInHjx+/4JESERGRI3Ep7AE8LyEhAQkJCUbnCSEwY8YMjBo1Cu3btwcALFmyBCEhIVizZg26d+/+IodKREREDkR2ocicS5cu4datW2jWrJlumr+/P2rWrIl9+/YZDUXZ2dnIzs7WPU9LSwMAqNVqqNXqgh80Ob381pm1NaxSqfLV34vC1599YP0ax/q1H9b8ruwqFN26dQsAEBISojc9JCREN+95kyZNwrhx4wymnz17Fj4+PrYfJNFzMjIy8rW8NTXs4eGBmJiYfPX3oly6dImHve0A69c41q/9sKaG7SoUSTFy5EgMHTpU9zwtLQ0RERGIi4uDn59fIY6MnEXuO2OpHLWGS5cuXdhDIAuwfo1j/doPa2rYrkJRaGgoAOD27dsICwvTTb99+zaqVKlidBl3d3e4u7sbTFepVHazm5bsW37rzFFr2J7H7kxYv8bZ89idjTW/K9l9+syc0qVLIzQ0FFu3btVNS0tLw4EDB1C7du1CHBkRERHZO9ntKcrIyEBSUpLu+aVLl3Ds2DEEBQUhMjISH3zwASZMmIDY2FiULl0ao0ePRnh4ODp06FB4gyYiIiK7J7tQdPjwYTRu3Fj3PPdYdJ8+fbBo0SIMHz4cmZmZ6N+/P1JSUlCvXj1s3LgRHh4ehTVkIiIicgCyC0WNGjWCEMLkfIVCgfHjx2P8+PEvcFRERETk6OzqnCIiIiKigsJQRERERASGIiIiIiIADEVEREREABiKiIiIiAAwFBEREREBYCgiIiIiAsBQRERERASAoYiIiIgIAEMREREREQCGIiIiIiIADEVEREREABiKiIiIKA9qTWGPIG+2GKNL/lfhWDQaDZRKeWdFexgjEZGzExo1FEpVYQ/DLEvHqFICvXYBZ1JfwKAkKOsP/NAg/+thKHqOUqnEvHnzkJycXNhDMSosLAwDBgwo7GEQEVEeFEoVcn4eDkXZplCWaw71rjkQZ7eZXUYZ3xmq6j2hPvQTNEdXm19/XBOoGgyE5vRmaPbMN982sipUzYZDXPsT6q3TAU0OFEWj4NJpqsXbcyYVOPrA4uZ2SXIoSktLw+zZs7F9+3bcuXMH8+bNQ40aNfDgwQMsWrQIr7zyCmJiYmw51hcmOTkZV65cKexhEBGRndMFonWjIY79bLatsv472kC0/Wtods81v94qnbSB6MgKaH4bD0CYbhvbEKpmwyCSdkK9aiigeQqYXcJ5SToGc/36dcTHx2PMmDG4fv06jh8/joyMDABAUFAQ5s2bh2+++camAyUiIrI3VgWixkMsD0TtPrM8EHWdCZG0Sy8QSTG/DrCjFRDuZXz+VzWAkx2AdhHG57eL0M7/qobx+cW9tOvPTx/5JSkUDRs2DOnp6Th27Bh27twJIfR/IR06dMCWLVtsMkAiIiJ7pd41xyECEQDEFwHqhQD9Yg3nlfMHBpcD4vyBSdWMLz+5mnb+4HLa9s/rF6tdf376yC9JoWjTpk14//33Ub58eSgUCoP5UVFRuHbtWr4HR0REZM/yPIfITgIRAORotI8/bhvOu5oJXNYeMMLmm8aX3/TP9MsZ2vbP23M7/33kl6Rzih49eoRixYqZnJ+eni55QERERM7AngIRALTfCpxIAW5kGc7LzAEqrwUivYHTJj6h9uFBYP5ZbbjJzDGcv+0WEP3PueVS+8gvSXuKypcvj127dpmcv2bNGsTHx0seFBERkSOTQyBSFLPuw1B3HhsPK7kyc/IOK6dTjQeiXDey8t9HfkgKRR988AH++9//4osvvkBqqnZ0Go0GSUlJ6NWrF/bt24cPP/zQpgMlIiJyBLIIROEVoWo9RuomOCxJh89ef/11XLlyBaNGjcKnn34KAGjVqhWEEFAqlfj888/RoUMHW46TiIjI7skmEL2+AOLhVShCy0ndFIck+TpFn376KXr16oXVq1cjKSkJGo0G0dHR6NSpE6Kiomw5RiIiIrsnq0B09zzUW6ZB2Xep1M1xSPm6onVkZCQPkxEREeVBdoFo6dtAUCmJW6PVrRRQ2geYdQbIMHKeUKVAoGMk8MtV4PhDw/k+LsCgssClDGDF5fz1YavzjCSFoj///BP79+/Hu+++a3T+7NmzUadOHVSpUiU/YyMiIrJ7sgxET8yczWyBRqHATw2135fwBgYfeK4/AJtbAEU8gHfLAqHLDbdkUjVgYFnt97cfATuf+xi+NX2kZOdrc3QknWj96aefmr0447Zt2zBq1CjJgyIiInIEyvjODheIAEC8gHuEvIg+nidpT9GRI0cwcuRIk/Pr16+PSZMmSR4UERGRI7DqXmZ2EogA7V6dnjv/PbT1PAGg+SagQySw5qrxrRl5BLieqT189vxeImv7OJP6716l/JAUitLT0+HiYnpRpVKp+6g+ERGRs1If+snhAlEuU+cB5Tr+0Pi5RLkycoAvTtimj/gg8+0sJenwWWxsLDZt2mRy/saNG/kJNCIicnqao6vNzrfXQOSoJIWifv36Yf369Rg6dChSUlJ001NSUvDhhx9i48aN6Nevn63GSERE5HAYiORH0uGz999/H8eOHcOMGTMwc+ZMhIeHAwBu3rwJjUaDXr168aP6REREJsgiEPmH5WMLHJOkPUUKhQILFy7E1q1b8c477+Cll17CSy+9hIEDB2Lbtm1YvHgxFAqFrccKAEhMTIRCodB7lC1btkD6IiIisjVZBKKgUnBpM86qcQd7AMW9TM/3cQHK+ZtfRzl/wNvM7pjiXvnvIz/ydfHGxo0bo3HjxrYai8UqVKigd0kAcyd9ExERyYVsAlHvRcDTLABFLB772qbar603a+9o/ywfF+BYe6CUD/DNaeDDg4bLf1UDGFwOuJyhvdv98zeGbRIK/NY8f33kl6Q9RYXNxcUFoaGhukfRokULe0hERERmySoQZacj59dEq8bvotQ+6oYYzovw1oYVAGgebnz5Fv9ML+UDRHobzq8Xkv8+8kvSLhYhBObPn48FCxbg4sWLePjQ8DN3CoUCOTlGrsltA+fPn0d4eDg8PDxQu3ZtTJo0CZGRkUbbZmdnIzv730tdpqWlAQDUajXUarVBe5VKVSBjtjVjYyd5yu/vijVMhYn1a5wlP5dnt0V2gWhJX8A32KptOXpfu3dnwXnD+adTtXtvmodrrz9kzIgj2qtYb75p/LYcC84DTcL+/V5KH8a2xZoalhSKhg8fjunTp6NKlSp4/fXXERgYKGU1ktSsWROLFi1CXFwckpOTMW7cONSvXx8nTpyAr6+vQftJkyZh3DjD46Znz56Fj4+P3jQPDw/ExMQU2Nht6dKlS3j8+HFhD4MskJGRka/lWcNUmFi/xuVVv89uiywDUeY9XSiydFv67wWOPjDdRV6Hs9Zd0z5MuZEFNNpofh159WFsW6ypYUmhaPHixejcuTNWrFghZfF8SUhI0H1fqVIl1KxZEyVLlsSKFSuMXgZg5MiRGDp0qO55WloaIiIiEBcXBz8/vxcy5oJQunTpwh4CWSj3nbFUrGEqTKxf4yytX0VcE6gaDJRfIJKwLfbA2LZYU8OSQtGjR4/QrFkzKYvaXEBAAMqUKYOkpCSj893d3eHu7m4wXaVS2c1uWmPseezOJr+/K9YwFSbWr3GWjl3ugciabbEHxrbFmu2TdKJ106ZNcejQISmL2lxGRgYuXLiAsDBeb4GIiORFc3qzrAMR6ZMUimbPno39+/fj888/x/379209JrM++ugj7Ny5E5cvX8bevXvRsWNHqFQq9OjR44WOg4iIKC+aPfPBQGQ/JIWiuLg4XLx4EaNHj0ZwcDC8vb3h5+en9/D3L5irK12/fh09evRAXFwcunXrhiJFimD//v0oVqxYgfRHRIVHaOT/CTWh0RT2EMhO2XMg8lQBqxoDf7YzfTPWIeWBUx20X42pGqRdflVj7fry00cPG91uVdI5RZ07dy6wK1bn5b///W+h9EtEL55CqULOz8Mg7l38d6J/mPZKvE+ztNdZeZRiegWeAXBpmwi4eiFn/VggNdl0W1cPqBJGQREYCfVv4yHuGj9PEQCgdIGq6VAoIqtBoeLFY8l6sghEngGSx9+6BNDhnyvhDK8I9NipP1+pAKZUA1RK7ddvTgOa53aYDasIVArSPhJKAD9fkd7HEMMPn0si6dW8aNEi2/RORJQHce8icOuU9klQKbgkjAYePcz7D753Ubh0/AJQuSFn0evAg8um27p5QfXad1AElID6h74QN/823VbpClWX6VBExEO9+Qu4tPpU0naR85JFIPIuqn3DINHRB0D6U8DXFdhu5L2GRgB77gANQ7Vfnw9EgHa5rqW06zlm5KP+1vRx7AFQzQbXceZbHCKyD9b+we+9EHD3Rc6SNywLRMViof6xn2WBKKYB1Cvfh0i/K3FjyFnJJhD1Xgi4mrnJWB4upgPRq4FAN+BCuvE2rTYDsX7AeROfiJ9/DtiaDDx8AjzINpxvTR/eLsC+NtK25VmSb/Nx9epVvPPOO4iLi0NgYCB27doFALh37x7ef/99HD16NP+jIyIC5BmIzu803dYItcxPPZL7+ByBrAKRu6/2kHI+PMg2HVYA4KkGOJWi/WrKhXTjgcjaPnJsVL+S9hSdOnUK9evXh0ajQc2aNZGUlKS7pUfRokWxZ88eZGZmYsGCBbYZJRE5L/8w7SEzOw5EgPa8h167gDNGbm9Q2Mr6Az80KOxRODbZBaIlbwBu0vcUOSrJt/kICAjA/v37oVAoEBwcrDe/TZs2WL58uU0GSETOzaXNOMvPIZJpIMp1JtX8bRLIMckyED24DISa+FiYE5N0+GzXrl0YOHAgihUrZvRTaJGRkbhx40a+B0dEhKdZDhGIyDkpIqvKMxCRUZJCkUajgZeX6d1ud+/eNXpZdyIia+X8mugwgUipAAbGaR8qE1c1aRIKjKkMFDfxJ7a4l3Z+k1Dj81U26INsR9VsuMMEoi4lHb92JR0+q1q1KtavX493333XYF5OTg7++9//olatWvkeHBGR2esQ2VEgAoBOkcDHlf59Pues/vziXsBvzQEXJdAkzPgdw5c2AOqFaE8sjV6tvbP4s/qXAb555s+vlD7IdsS1Px0iEAHOUbuS9hSNHDkSGzduxMCBA3HixAkAwO3bt7Flyxa0aNECp0+fxogRI2w6UCIiPXYWiMg5qbdOd4hA5Cwk7SlKSEjAokWLMGTIEMyfPx8A8Prrr0MIAT8/PyxZsgQNGvCjDERUQGQRiBRQ1utv1bB/vgpc/+d/3vxzhvNvZAGtNwN1Q4AF542vo+cuoF8s8Mdtw3faz69Xah9kQ5ock7PsLRB9cVxbv45cu5Iv3tirVy906tQJmzdvxvnz56HRaBAdHY2WLVvC19dG19smInqeXAJR6zFQlmtu1dA1wvCQwPO23dI+TLmZBXz2l+n5ahv0QQVPFoHI1cOqMa+6Yv7Tk45Qu1aHoqysLERERGDEiBEYNmwYOnToUADDIiIyQk6BqFo3qHfNgUvDQRI3hpyVLAKRmxdUCaMkboHjsvqcIi8vL7i4uMDb27sgxkNEZJzcAtG60RBnt0ncGHJWsglEr30HRWCkxK1wXJJOtO7cuTNWrVoFIYzc4Y1kQ2jUhT2EPNnDGEkG5BiIjv0scWPIWckqEBWLhfq38RK3xHFJOqeoe/fuePfdd9G4cWO8/fbbKFWqFDw9PQ3aVa1aNd8DJOkUShU0N09AERwH9ZYpEFf/NNteWa8/lOWaQ71rTp7vgJXxnaGq3hPqQz9Bc3S1+XHENYGqwUBoTm+GZs/8f6cXjYJLp6mWbxA5J88A7d3uHTQQFXEHAszc8NJV+e9NNU3dQyraF0h5Atw3cQ+pvPpwkXwXTLKU7ALRj/3y/aZUDrWb24etalhSKGrUqJHu+927dxvMF0JAoVBAreZegMKmCI6DeuVgi08ateQPvrL+O9pAtP1raHbPNd9/lU7aQHRkBTS/jQfw795F7mckS7i0TQRUbg4ZiKJ8gSPtAF9X4N19xj9xs7E50DAU2HkLaPq74fz+ZYDZtYH0p0C1ddo7i1vbx7c1bbM9ZJwsA9HNv/N1mw+51G5uH0fM/JisISkULVy40Da9U4FTb5li0z/4yvrvQNV4iOWBqN1nRgMRkcVcvZCz6HWHC0QAEB+k/YMPAI3DDP/oKxVAvX9uLVkvWPtc89zLqHGY9quvK1AlyPAfiyV9VAnK/7aQcYpiMVAljJJfIMonudRubh+2qmFJoahPnz626Z0KnPlDZgxEJH8568c6ZCACgN+uA2uuAlE+wBQjw9YIYPgR7Tvq+ecM/6kA2uXi/ICLGcCG69L6+Po0MLRC/reHDKlaj4G4c9ahAhEgn9rN7WP9ddvUsOTrFOVKTk7GnTt3EBMTw0+k2RUGIrITqcmm59lxIAKAR2qgy3bzbb4+pX2YcvQBUHVd/vpYdpGhqKCIh1cdLhAB8qnd3D7ig2xTw5JPTVq7di3Kli2LEiVKoGrVqjhw4AAA4N69e4iPj8cvv/yS/9FRAZFHIFJEWn4ivtrESXpyYg9jdCh2HojIOag3THC4QOTIJO0pWrduHTp16oTatWujZ8+eSExM1M0rWrQoihcvjkWLFqFjx462GifZjEwCUWxDqJoNt3jUKiXQaxdwJtXiRV6osv7AD7yzzYsjk0CkjO8sbfzkPJ4+Nj2PgUh2JIWi8ePHo0GDBti+fTvu37+vF4oAoHbt2pg3b54txkc2JaNA1HUmxLU/oSht+cdezqSav8Q8OQm5BKJ/PoVJJIkcApEy32fQOBxJh89OnDiBbt26mZwfEhKCO3fuSB4UFQSZBaKkXdq7RxNZQ06BqPEQqA/9JG07yLnJIhC5QtV0qMQNcFySQpGXlxcyMzNNzr948SKKFCkieVBkazIMRKuGmr17tDnFvYAdrbSPcC/jbb6qAZzsALSLMD6/XYR2/lc1Cq4PsjG5BaLtX+d54dLnBXuwdp2eXAJRl+lQRFh3geX5dRy/diWFosaNG2Px4sXIyTH8p3br1i189913aNGiRb4HR7Yhz0D0VMqmAAD6xQL1QrSPfrGG88v5A4PLAXH+wKRqxtcxuZp2/uBy2vYF0QfZkBwDUR6vD2PaR7J2nZqcAlFMA6i3TLFq+PFFHL92JYWiiRMn4vr166hevTrmzZsHhUKB33//HaNGjULFihUhhMDYsWNtPVaSQFmvv0MFIgDYcxvI0Wgff9w2nH81E7icof1+803j69j0z/TLGdr2BdEH2Yirh0MEIgA4dp+167TkFohWvp/nrZ+e5wy1K+ksq7i4OOzZswdDhgzB6NGjIYTA1Knae1g1atQIs2bNQqlSpWw5TpLIqlt32EEgAoBtt4Dof45a3DDySdfMHKDyWiDSGzht4tNqHx4E5p/VvsgyjRzFs7aPeF4RuMCoEkZBEVDC7gMRABy6L7/apRdAjoHo/E6rb/PRfitwIsWxa9eiUHT8+HGULFkS/v7/7u+qUKECtmzZgocPHyIpKQkajQZRUVEoVqxYwYyUJFHvmuNQgSiXsRfMszJz8n7R5DXfFn1Q/ikCI6H+oa/dB6JcrF0n4x8Gl4TR8gtEEtx5bL62HKF2LTp8Fh8fj/Xr1+ueN2nSBFu3bgUABAYGonr16qhZsyYDkQzlebd7OwxE5FzUv413mEBEzselzTiHCETOwqJQ5Onpiaysf+Pbjh07cPu2kQN+ZFcYiMgeiLtJpmcyEJHcPc1iILIjFh0+q1y5MqZPnw6VSqU7hHbo0CF4eHiYXa5Tp075HyEVCDkEIkWxGKnDJ2IgIruQ82siA5EdsSgUzZgxA127dkW/fv0AAAqFAl9//TW+/vprk8soFAqo1WrbjJJsShaBKLwiVK3HWDXu5uGmr2jdrRRQ2geYdQbIMHICX6VAoGMk8MtV4PhDw/k+LsCgssClDGDFZWl90Askk0CkiGsiYfD65FC7sX752AAy71GK6Xl2HojkULu5fdjqPCOLQlH16tWRlJSECxcu4Pbt22jUqBE++eQTNG/e3DajoBdGNoHo9QUQD69CEVrO4rF/Xg04cBfY+dyR20ahwE8Ntd+X8AYGH3iuPwCbWwBFPIB3ywKhyw23ZFI1YGBZ7fe3H0nrg14QuQSiKp2gajBQwgb8Sy61O7tWvjaDpLDzQCSX2s3tIyXbBhsFC0PR//73P7z88suIi4tDXFwc+vTpg3bt2qFmTcvvW2Vrs2bNwtSpU3Hr1i1UrlwZ33zzDWrUMHGZTAIgs0B09zzUW6ZB2Xep1M3REaaHZjMvog+ygJwCUbvPoDm9GaryLSVsiBZr10nJIhApoKzXX+IGOG7tWhSKOnbsiB9++AE9e2pvfrhz50507NixQAdmzvLlyzF06FDMnTsXNWvWxIwZM9CyZUucPXsWwcHBhTYuOZNdIFr6NhBUyqpt+OSI4TsJQDut585/d7E+TwBovgnoEAmsuWp8a0YeAa5nanfjSu2DCpjcAtGRFdD8uTJfoUgutfvu/n/fkVMBk0sgaj0GynLSj/bIpXZz+ziTapsatigU+fr6IiUlRff88uXLyMjIyH/vEk2fPh1vv/02+vbtCwCYO3cu1q9fj//85z8YMWKEXtvs7GxkZ/+7Xy01VXvg8eHDh0bPeVKpVAgICNBbRk4CAgKQlpZm0flaKpUKOV7FoaifAFW1blBv/x6a41sBfyPXT/+HIrYBVHXfgubQGmj2/QT4mz4ZWlGiMlSNP4A4tQfqnbMB31Km2xaNgqrFxxDXzkC9eSrgWRwKr+JwsWJbLtwBKpk4t//MLeAMgCgXGK/qR8Cas9pvTa1jQ5L5+eb6iHIB0tJgdFvS0tIAAELi2x7WcHGIQAFVw/egCK4M9drPIO7cNFvHytpvQBnbBOoN0yEu/W2+baVXJL8+FEVKWVXDUS6A+rn6KuzaBQB1FuvXUpLq1/+fN4ue/nBpORJ46oKcX0cDalfT9ebqAVXzYVB4hEG9ejRE5mPTbZUukl4fml1LoIrvJLl+5VC7uX3E+tmmhhXCglYtWrTAyZMn8dZbb8Hf3x8fffQRevbsiapVTd9MTqFQ4MMPP8xzANZ68uQJvLy8sGrVKnTo0EE3vU+fPkhJScHatWv12icmJmLcuHE2HweRta5du4YSJUpYvRxrmOSA9Uv2zpIatigUJSUloXfv3ti/f792IYUiz8RVUJ8+u3nzJooXL469e/eidu3auunDhw/Hzp07ceCA/plYz79L0Wg0ePDgAYoUKQKFgqfHUsETQiA9PR3h4eFQKq2/3SBrmAoT65fsnTU1bNHhs5iYGOzduxePHz/GnTt3UKpUKcyYMQPt27e3yYALkru7O9zd3fWmBQQEFM5gyGk9e4sca7GGqbCxfsneWVrDVt0Q1sPDA5GRkRg7diyaNGmCkiVLShpcfhQtWhQqlcrgitq3b99GaGjoCx8PEREROQbr94UCGDt2LF566SVbj8Uibm5uqFatmu7ea4B2d+zWrVv1DqcRERERWcOiPUVvvvkmFAoF5s+fD5VKhTfffDPPZRQKBRYsWJDvARozdOhQ9OnTBy+//DJq1KiBGTNmIDMzU/dpNCIiIiJrWRSKtm3bBqVSCY1GA5VKhW3btuV5glxBnkD36quv4u7duxgzZgxu3bqFKlWqYOPGjQgJCSmwPomIiMixWfTpMyIiIiJHJ+mcIiIiIiJHY9Wnz3Klp6djz549uHDhAtLT0+Hr64uYmBjUq1cPPj4+th4jERERUYGzKhSp1WqMGjUK3377LbKysvQu4KhQKODl5YUhQ4Zg/Pjxki7yRURERFRYrDqnqHv37lixYgXKly+PHj164KWXXoKPjw8yMjLw999/46effsLZs2fRo0cP/PjjjwU5biIiIiKbsjgUbdmyBS1atMCgQYPw9ddfG90TpNFoMHjwYMydOxebN29GkyZNbD5gIiIiooJgcSjq3bs39u3bh3Pnzpn9uL1Go0FcXBzq1KmDxYsX22ygRERERAXJ4hN/Dh48iE6dOuV5/SGlUolOnToZ3JiViIiISM4sDkXJycmIiYmxqG1MTAySk5MlD4qIiIjoRbM4FGVkZMDb29uitl5eXsjIyJA8KCIiIqIXzeJQJIQo0Ft3EBERERUmi0+0ViqViIiIgL+/f55tU1NTcf36dajV6nwPkIiIiOhFsPjijQ0aNLB4T1GRIkUQFRUleVBERERELxpvCEtEREQE3hCWiIiICABDEREREREAhiIiIiIiAAxFRERERAAYigpNRkYGxo4di1atWiEoKAgKhQKLFi0yaKfRaLBo0SK88soriIiIgLe3N1566SVMmDABjx8/NrruBQsWoFy5cvDw8EBsbCy++eYbo+1u3LiBbt26ISAgAH5+fmjfvj0uXrxo0fifPHmCr7/+GvHx8fDz80NAQAAqVKiA/v3748yZM7p2e/fuRWJiIlJSUixaL9kHS+sXAN544w0oFAqDR9myZQ3aajQaTJkyBaVLl4aHhwcqVaqEZcuWGV3v6dOn0apVK/j4+CAoKAi9evXC3bt3rRr/Sy+9BG9vbxQpUgRVqlTBkCFDcPPmTV273377DYmJiRatk+zLyZMn0bVrV0RFRcHLywtFixZFgwYNsG7dOqPtLa031rCdE1QoLl26JACIyMhI0ahRIwFALFy40KBdenq6ACBq1aolJkyYIObPny/69u0rlEqlaNSokdBoNHrt586dKwCIzp07i/nz54tevXoJAGLy5MkG642NjRXBwcHiiy++ENOnTxcRERGiRIkS4t69e3mOv23btkKlUonXX39dzJo1S8yYMUO88847okSJEnrbMXXqVAFAXLp0ScqPiWTK0voVQog+ffoId3d38cMPP+g9/ve//xm0HTFihAAg3n77bTF//nzRpk0bAUAsW7ZMr921a9dE0aJFRXR0tPj666/FxIkTRWBgoKhcubLIzs42O/YnT56I+Ph44enpKd555x0xd+5c8eWXX4q+ffuKokWLiu3bt+vaDho0SPDPpGNav369aNmypUhMTBTz588XM2bMEPXr1xcAxLx58/TaWlNvrGH7xp9UIXn8+LFITk4WQghx6NAhk/9UsrOzxR9//GEwfdy4cQKA2Lx5s25aVlaWKFKkiGjTpo1e29dee014e3uLBw8e6KZ98cUXAoA4ePCgbtrp06eFSqUSI0eONDv2gwcPCgBi4sSJBvNycnL0QhVDkWOytH6F0IYib2/vPNd5/fp14erqKgYNGqSbptFoRP369UWJEiVETk6ObvrAgQOFp6enuHLlim7a5s2bjf5De96KFSsEALF06VKDeY8ePRKpqam65/yH4lxycnJE5cqVRVxcnN50S+uNNWz/LPpJlSpVSpQuXdqqR1RUVEGP3WHk9U/FmOPHjwsAYubMmbpp69evFwDE+vXr9dru3btXABA//PCDblr16tVF9erVDdbbokULER0dbbbvZcuWCQBix44dZtuNHTtWADB4PBuQfvjhB1G1alXh4eEhAgMDxauvviquXr2qt56GDRuKChUqiMOHD4vatWsLDw8PUapUKTFnzhyDPmfOnCnKly8vPD09RUBAgKhWrZrRPxxkO5aGopycHL0/1s+bNWuWACBOnjypN/2nn34SAMTu3bt104KDg0XXrl0N1lGmTBnRtGlTs+OdNGmSACAuX75stl2fPn2M1m8utVotvvrqK1G+fHnh7u4ugoODRf/+/fXefAghRMmSJUWbNm3E77//LipXrizc3d1FuXLlxOrVq/XaPXnyRCQmJoqYmBjh7u4ugoKCRN26dcWmTZvMjpNsq23btiIkJERvmqX1xhq2/xq26IrWDRs25H3PZObWrVsAgKJFi+qmHT16FADw8ssv67WtVq0alEoljh49itdffx0ajQbHjx/Hm2++abDeGjVqYNOmTUhPT4evr6/RvkuWLAkAWLp0KerWrQsXF+Nl1KlTJ5w7dw7Lli3DV199pRtrsWLFAAATJ07E6NGj0a1bN7z11lu4e/cuvvnmGzRo0ABHjx5FQECAbl0PHz5E69at0a1bN/To0QMrVqzAwIED4ebmptuO7777Du+//z66dOmCIUOG4PHjxzh+/DgOHDiAnj175vkzpYKTlZUFPz8/ZGVlITAwED169MAXX3wBHx8fXZujR4/C29sb5cqV01u2Ro0auvn16tXDjRs3cOfOHYM6z23722+/mR1Lbv0uWbIEo0aNMvm3bcCAAbh58yY2b96MH374wej8RYsWoW/fvnj//fdx6dIlfPvttzh69Cj++OMPuLq66tqeP38er776Kt555x306dMHCxcuRNeuXbFx40Y0b94cAJCYmIhJkybhrbfeQo0aNZCWlobDhw/jzz//1LUh28vMzMSjR4+QmpqK//3vf9iwYQNeffVV3Xxr6o017AA1XNipjKTtKWrWrJnw8/MTDx8+1E0bNGiQUKlURtsXK1ZMdO/eXQghxN27dwUAMX78eIN2ue90zpw5Y7JvjUYjGjZsKACIkJAQ0aNHDzFr1iy93cC5TB0+u3z5slCpVAaH4P7++2/h4uKiNz23r2nTpummZWdniypVqojg4GDx5MkTIYQQ7du3FxUqVDA5bioYedXviBEjxMcffyyWL18uli1bpnv3WrduXfH06VNduzZt2hjdw5yZmSkAiBEjRuj1t2TJEoO2w4YNEwDE48ePTY43KytLxMXFCQCiZMmS4o033hALFiwQt2/fNmhr6tDD7t27jR6+2Lhxo8H0kiVLCgB676pTU1NFWFiYiI+P102rXLmywaFvKngDBgzQ7UFRKpWiS5cuentKrKk31rD91zA/fWaHPv/8c2zZsgWTJ0/W25vy6NEjuLm5GV3Gw8MDjx490rUDAHd3d6Ptnm1jjEKhwO+//44JEyYgMDAQy5Ytw6BBg1CyZEm8+uqrFn3S7Oeff4ZGo0G3bt1w79493SM0NBSxsbHYvn27XnsXFxcMGDBA99zNzQ0DBgzAnTt3cOTIEQBAQEAArl+/jkOHDuXZP704kyZNwuTJk9GtWzd0794dixYtwsSJE/HHH39g1apVunaPHj2yqCbzW7+enp44cOAAhg0bBgBYtGgR+vXrh7CwMAwePBjZ2dl5btPKlSvh7++P5s2b69VvtWrV4OPjY1C/4eHh6Nixo+65n58fevfujaNHj+r2+gYEBODkyZM4f/58nv2T7XzwwQfYvHkzFi9ejISEBKjVajx58kQ335p6Yw3bfw3nKxQ9ffoUf//9N/bs2YNdu3YZPMj2li9fjlGjRqFfv34YOHCg3jxPT0+9F/OzHj9+DE9PT107AEZfOLkf889tY4q7uzs+/fRTnD59Gjdv3sSyZctQq1YtrFixAu+9916e23H+/HkIIRAbG4tixYrpPU6fPo07d+7otQ8PD4e3t7fetDJlygAALl++DAD4+OOP4ePjgxo1aiA2NhaDBg3CH3/8kedY6MX78MMPoVQqsWXLFt00T09Pi2rSFvXr7++PKVOm4PLly7h8+TIWLFiAuLg4fPvtt/jss8/yHP/58+eRmpqK4OBgg/rNyMgwqN+YmBiDQxzP1+/48eORkpKCMmXKoGLFihg2bBiOHz+e51gof8qWLYtmzZqhd+/e+PXXX5GRkYF27dpB/HNbUGvqjTVs/zVs0TlFz9NoNBg5ciRmz56NrKwsk+3UarXkgZGhzZs3o3fv3mjTpg3mzp1rMD8sLAxqtRp37txBcHCwbvqTJ09w//59hIeHAwCCgoLg7u6O5ORkg3XkTstta4mwsDB0794dnTt3RoUKFbBixQosWrTI5LlGgLaGFAoFNmzYAJVKZTD/2XNNLFWuXDmcPXsWv/76KzZu3IjVq1dj9uzZGDNmDMaNG2f1+qjgeHp6okiRInjw4IFuWlhYGLZv3w4hhN4f3+drMiwsTG/6s5KTk3X1bamSJUvizTffRMeOHREVFYWlS5diwoQJZpfRaDQIDg7G0qVLjc7PPW/OGg0aNMCFCxewdu1abNq0Cd9//z2++uorzJ07F2+99ZbV6yNpunTpggEDBuDcuXOIi4uzqt5Yw/Zfw5JC0eeff46pU6diwIABqFevHnr16oUvvvgCAQEBmD17NhQKBaZMmWLrsTq1AwcOoGPHjnj55ZexYsUKo4GjSpUqAIDDhw+jdevWuumHDx+GRqPRzVcqlahYsSIOHz5stJ+oqCiTJ1mb4+rqikqVKuH8+fO6Q2GmTgCMjo6GEAKlS5fWvdsw5+bNm8jMzNTbW3Tu3DkAQKlSpXTTvL298eqrr+LVV1/FkydP0KlTJ0ycOBEjR47U7Zamwpeeno579+7p/eGtUqUKvv/+e5w+fRrly5fXTT9w4IBuPgAUL14cxYoVM1q/Bw8e1LWzVmBgIKKjo3HixAndNHP1u2XLFtStWzfPd/QAkJSUZPCP0lj9BgUFoW/fvujbty8yMjLQoEEDJCYm2s0/FEeQe9gqNTUVgHX1xhp2gBqWciJSdHS0ePXVV4UQQty7d08oFAqxdetWIYT2BNiqVavmea0b+ldeJ6qeOnVKFClSRFSoUMHgo5LPysrKEkFBQaJt27Z6019//XXh5eUl7t+/r5s2efJkAUAcOnRIN+3MmTNCpVKJjz/+2Ox4z507Z/Sk6ocPH4rw8HARGBioux7HnDlzBABx9OhRvbZJSUlCpVKJnj17GlyAUqPR6F3ryNyJ1sWKFdOdaG3sopPDhg0TSqVSpKWlmd0mks5c/T569Mjozz73ZNKff/5ZN+3atWsmr/FSvHhxvWu8vPPOO8LT01Pv8g1btmwRAIxequFZx44dE3fv3jWYfvnyZeHp6SkqVaqkm/bxxx8LAHofaBBCiB07dggARv/OPX36VK+9uZNUq1SpoptmrH67du0qihYtanZ7SBpjJyU/efJEVK1aVXh6eor09HTddEvrjTVs/zUsaU/R9evXMXz4cAD/niiWexzUzc0Nr7/+OqZPn47PP/9calZzCt9++y1SUlJ0l2Rft24drl+/DgAYPHgw/P39kZ6ejpYtW+Lhw4cYNmwY1q9fr7eO6Oho1K5dG4D2kMRnn32GQYMGoWvXrmjZsiV2796NH3/8ERMnTkRQUJBuuXfffRffffcd2rRpg48++giurq6YPn06QkJC8H//939mx/3XX3+hZ8+eSEhIQP369REUFIQbN25g8eLFuHnzJmbMmKE7JFatWjUAwKefforu3bvD1dUV7dq1Q3R0NCZMmICRI0fi8uXL6NChA3x9fXHp0iX88ssv6N+/Pz766CNdn+Hh4fjiiy9w+fJllClTBsuXL8exY8cwf/583cdGW7RogdDQUNStWxchISE4ffo0vv32W7Rp00bSni8yz5L6vXXrFuLj49GjRw/dbT1+//13/Pbbb2jVqhXat2+vW1+JEiXwwQcfYOrUqXj69CmqV6+ONWvWYPfu3Vi6dKneYdZPPvkEK1euROPGjTFkyBBkZGRg6tSpqFixIvr27Wt23Js3b8bYsWPxyiuvoFatWvDx8cHFixfxn//8B9nZ2Xq3RMit3/fffx8tW7aESqVC9+7d0bBhQwwYMACTJk3CsWPH0KJFC7i6uuL8+fNYuXIlvv76a3Tp0kW3njJlyqBfv344dOgQQkJC8J///Ae3b9/GwoULdW3Kly+PRo0aoVq1aggKCsLhw4exatUqi87RI+sNGDAAaWlpaNCgAYoXL45bt25h6dKlOHPmDKZNm6Z3CN/SemMNO0ANS0lS4eHheu/a/f39xTfffKN7Pn36dIuuYOvsctO3sUfuR9hzb6dg6tGnTx+D9c6fP1/ExcUJNzc3ER0dLb766iuDvTFCaN/VdOnSRfj5+QkfHx/Rtm1bcf78+TzHffv2bTF58mTRsGFDERYWJlxcXERgYKBo0qSJWLVqlUH7zz77TBQvXlwolUqDj+evXr1a1KtXT3h7ewtvb29RtmxZMWjQIHH27FldG2MXbyxZsqT49ttv9fqZN2+eaNCggShSpIhwd3cX0dHRYtiwYWYvGEjSWVK/Dx8+FK+//rqIiYkRXl5ewt3dXVSoUEF8/vnnuj18z1Kr1eLzzz8XJUuWFG5ubqJChQrixx9/NNr/iRMnRIsWLYSXl5cICAgQr732mrh161ae47548aIYM2aMqFWrlggODhYuLi6iWLFiok2bNmLbtm16bXNycsTgwYNFsWLFhEKhMPho8/z580W1atWEp6en8PX1FRUrVhTDhw8XN2/e1Ps55V74rlKlSsLd3V2ULVtWrFy5Um9dEyZMEDVq1BABAQHC09NTlC1bVkycONHoz4nyb9myZaJZs2YiJCRE9zesWbNmYu3atUbbW1pvrGH7rmGFEP+cYm+Ftm3bwtfXV3eTu44dO+Lvv//G4sWLodFo0Lt3b4SHh/OTP2QTjRo1wr179/SOkxPZi1KlSuGll17Cr7/+WthDIZLEmWpY0kfy+/fvj+zsbN3HCSdOnIiUlBQ0aNAADRs2RFpaGqZNm2bTgRIREREVJEnnFL3yyit45ZVXdM/Lly+PCxcuYPv27XBxcUGdOnX0zl8hIiIikjtJocgYf39/dOjQwVarIyIiInqhJJ1TdPXqVVy9ehX16tXTTfvrr78wbdo0ZGdno0ePHgxIREREZFckhaIOHTogIyNDd4n+27dvo1y5cnjy5Al8fX1x584drFy5Ep06dbL5gImIiIgKgqQTrQ8ePIjmzZvrni9ZsgSPHj3CX3/9hRs3bqBp06b48ssvbTZIIiIiooIm6ZyiBw8e6N1b69dff0XDhg0RHR0NAOjUqRM++eQT24zQxjQaDW7evAlfX1+Tlz8nsiUhBNLT0xEeHg6lMl/3YAbAGqYXi/VL9s6aGpYUiooVK4YrV64AAFJSUrB//35MnjxZNz8nJwc5OTlSVl3gbt68iYiIiMIeBjmha9euoUSJEvleD2uYCgPrl+ydJTUsKRQ1a9YMM2fOhJ+fH3bs2AGNRqN3YvWpU6dkW/S5t3u4du0a/Pz8Cnk05AzS0tIQERFhs1uNsIbpRWL9kr2zpoYlhaLJkyfj3Llz+Oijj+Dm5oYvv/wSpUuXBgBkZ2djxYoV6Nmzp5RVF7jc3bV+fn58QdILZatDBaxhKgysX7J3ltSwpFAUEhKCP/74A6mpqfD09ISbm5tunkajwdatW2W7p4iIiIjImHxdvNHf399gmqenJypXrpyf1RIRERG9cBaFoiVLlgAAevXqBYVCoXuel969e0sfGREREdELZNHFG5VKJRQKBR49egQ3NzeLPpapUCigVqttMkhbSktLg7+/P1JTU3k8m14IW9cca5heJNYv2Ttras6iPUWXLl0CAN25Q7nPiYiIbEmj0djkekgFyR7GSNJYFIpKliyp+/7p06dITU1FUFCQTa5ZQURElEupVGLevHlITk4u7KEYFRYWhgEDBhT2MKiAWH2itVKpRLVq1TBt2jS8//77BTEmIiJyYsnJyboLBBO9SFbv/1OpVChZsiSys7MLYjxEREREhULSQdHBgwdj/vz5ePDgga3HQ0RERFQoJF2nSK1Ww93dHdHR0ejSpQtKlSoFT09PvTYKhQIffvihTQZJREREVNAkhaKPPvpI9/2CBQuMtmEoIiIiInsi6fDZpUuX8nxcvHhR0oB27dqFdu3aITw8HAqFAmvWrNGb/8Ybb0ChUOg9WrVqJakvIkek0WgKewh5socxEpHzkbSn6NmP6NtaZmYmKleujDfffBOdOnUy2qZVq1ZYuHCh7rm7u3uBjYfI3vAjzURE0uTr3mcPHjzAli1bcPnyZQBAqVKl0LRpUxQpUkTyOhMSEpCQkGC2jbu7O0JDQyX3YY49XJTLHsZIhYsfaSYisp7kUJSYmIgvvvjC4KP5bm5uGD58OMaPH5/vwZmyY8cOBAcHIzAwEE2aNMGECRNMBrHs7Gy9MaalpQHQnixu7DYkKpXKLt5ly/EWKmRcfn9XUmrYHrCG7QPr1zjWr/2w5nclKRR99tlnGD9+PNq0aYP33nsPZcqUAQCcPXsW3377LSZOnAhXV1eMHj1ayurNatWqFTp16oTSpUvjwoUL+OSTT5CQkIB9+/YZfTFNmjQJ48aNM5h+9uxZ+Pj46E3z8PBATEyMXbzLvnTpEh4/flzYwyALZGRk5Gt5KTVsD1jD9oH1axzr135YU8MW3RD2ecWLF8fLL7+MtWvXGp3frl07HDlyBDdv3rR21fqDUyjwyy+/oEOHDibbXLx4EdHR0diyZQuaNm1qMN/Yu5SIiAg8ePDA6I3hVCoVEhMTZRuKSpYsicTERL5LsSNpaWkICgqSfANM1jAVJtavPtav/bGmhiXtKUpNTTX7ia/WrVtjx44dUlZttaioKBQtWhRJSUlGQ5G7u7vRE7FVKpXd7KY1xp7H7mzy+7tiDVNhYv0aZ89jdzbW/K4kna1bt25dHDhwwOT8AwcOoG7dulJWbbXr16/j/v37CAsLeyH9ERERkWOSFIrmzp2Lffv24cMPP0RSUhI0Gg00Gg2SkpLwwQcfYP/+/Zg7d66kAWVkZODYsWM4duwYAO1x22PHjuHq1avIyMjAsGHDsH//fly+fBlbt25F+/btERMTg5YtW0rqj4iIiAiQePisUqVK0Gg0mDlzJmbOnKn7eHjuBdnc3d1RqVIlvWUUCgVSU1PzXPfhw4fRuHFj3fOhQ4cCAPr06YM5c+bg+PHjWLx4MVJSUhAeHo4WLVrgs88+47WKiIiIKF8khaLOnTtDoVDYeiwAgEaNGsHcud+///57gfRLREREzk1SKFq0aJGNh0FERERUuKw+pygrKwvVqlWTfM4QERERkRxZHYq8vLxw6dKlAjt8RkRERFQYJH36rFWrVjy3h4iIiByKpFA0evRonDt3Dr169cKePXtw48YNPHjwwOBBREREZC8knWhdoUIFAMCpU6fw008/mWzHy6ATERGRvZAUisaMGcNzioiIiMihSApFiYmJNh4GERGRYxEaNRRKed8jzR7G+CJJCkVERERknkKpQs7PwwCFEqrWYyAeXoV6wwTg6WPTC/mHwaXNOOBpFnJ+TQQepZhu6xkAl7aJgKsXctaPBVKTTbd19YAqYRQUgZFQ/zYe4m4SFEWj4NJpqsStc0ySQtH48ePzbKNQKDB69GgpqyciInIMCiVUCaMg7pyFeunbwJMs022DSsElYTTw6CFylvQFMu+ZbutdFC4dvwBUbshZ9Drw4LLptm5eUL32HRQBJaD+oS/Ezb8BAKbvHeG8bH74TKFQQAjBUERERE5P1XqM5YGo9yIgO92yQNR7IeDui5wlb1gWiIrFQv1jP10gIuMkfSRfo9EYPHJycnDhwgV8+OGHePnll3Hnzh1bj5WIiMiuiIdXGYjsiKRQZHRFSiVKly6NL7/8ErGxsRg8eLCtVk1ERGSX1BsmMBDZEZuFomc1aNAAv/32W0GsmoiIyH6YO6magUh2CiQUHT58GEplgayaiIjI/skhECn5AfTnSfqJLFmyxOj0lJQU7Nq1Cz///DPeeuutfA2MiIjIIckiELlC1XSoxA1wXJJC0RtvvGFyXtGiRTFixAiMGTNG6piIiIgck1wCUZfpUERUlbgRjktSKLp06ZLBNIVCgcDAQPj6+uZ7UERERA5HToEopgHUW6bApdWnEjfGMUkKRSVLlrT1OIiIiByX3ALRyvch0u9aPHy1AFQyv+WpWgOo8nk6s03Osjpz5gxWrlyJ5ORkxMXFoW/fvvDz87PFqomIiOybHAPR+Z1AaHmLN0GlAIYcAJLSgDsmPlDnqQJCPYFLGabXU9oHuPUIeKQ2Pj/YQ/vV2j7K+gM/NDC/DZawOBR9++23mDlzJvbu3YuiRYvqpq9btw5du3bFkydPdNO++eYb7N+/X68dERGR0/EP0966Q26BSIK9d4CjDyQtqpPf5QuaxTua/ve//yE6Olov6OTk5OCtt96CSqXCwoUL8ffff2Py5Mm4cuUKJk6cWCADJiIishcubcY5RCByFhaHolOnTqFWrVp607Zv3467d+/iww8/RJ8+fVChQgUMHz4c3bp148UbiSjfhMbEPnYZsYcxUiF6msVAZEcsPnx2//59RERE6E3bunUrFAoFOnbsqDe9bt26+Pnnn20zQiJyWgqlCjk/D4O4d9FwptIFqqZDoYioCvWWKRBX/zS7LmW9/lCWaw71rjkQZ7eZbxvfGarqPaE+9BM0R1ebHl/RKLh0mmrRtpBzyvk1kYHIjlgcikJCQnDr1i29abt374aXlxcqV66sN93NzQ1ubm62GSEROTVx7yJw65T+RN11VuKhXjk4jz/4Cihbj9EGonWjIY6Zf8OmrP+ONhBt/xqa3XPNDy6uiWUbQc7rUYrpeXYeiLqV0p44PesMkJFjOL9SINAxEvjlKnD8oeF8HxdgUFntSdMrLuevj9Op+diQZ1gcil5++WUsXrwYgwcPhq+vL06ePImDBw+iffv2cHHRX82ZM2dQokQJ24yQiOhZVv3B/ycQVetmeSBqPMSiQKSo0gmqBgMlbAAR7D4QNQoFfmqo/b6ENzD4gP58BYDNLYAiHsC7ZYHQ5YB4bh2TqgEDy2q/v/0I2Hlbeh8p2TbYKFhxTtHYsWNx5coVxMbGomnTpqhbty4UCgVGjhxp0PaXX35BnTp1bDNCIqJccgpE7T6D5vRmCRtBTk8WgUgBZb3+EjcAEM8nnALwIvp4nsV7iipWrIht27Zh4sSJuHjxImrVqoWPPvoI1apV02u3Y8cOeHl5oWvXrjYfLBE5MbkFoiMroPlzJVTlW0rYGHJacglE/xxSlmrnbaDnzn8PbT1PAGi+CegQCay5ariXCABGHgGuZ2oPnz2/l8jaPs6k/rtXKT+sunhjnTp1sH79erNtGjVqhL//NvOLIyKylhwD0W/jgdByEjaGnJacAlG1blDvmgOXhoMkbozp84ByHX9o/FyiXBk5wBcnbNNHfJD5dpbK5wWxiYgKmNJFnoHI6HtfIhPkFojWjc7zU5jOiKGIiGRN1XQoAxHZNzkGojxeH86KoYiIZE0RUZWBiOyXZwADkR2RXSjatWsX2rVrh/DwcCgUCqxZs0ZvvhACY8aMQVhYGDw9PdGsWTOcP3++cAZLRAVOvWWKQwQitUbyoi+E3Mdnr1zaJjpMIAr2AIp7mZ7v4wKU8ze/jnL+gLeZs5mLe+W/j/yw6kTrFyEzMxOVK1fGm2++iU6dOhnMnzJlCmbOnInFixejdOnSGD16NFq2bIlTp07Bw8OjEEZMRAXJ/JWq7SMQAYBKCUw+Dnz0kvb5+/uBQ/f123ipgGWNgHAv4L+XgGlGTkL9v5eA7qWBm1lA9x2GdxuvXgSYWcu6Pmx1h3EywtULOYtet/tABABrm2q/tt4MbNO/ljN8XIBj7YFSPsA3p4EPDxou/1UNYHA54HIGUHktkPncxRibhAK/Nc9fH/klu1CUkJCAhIQEo/OEEJgxYwZGjRqF9u3bAwCWLFmCkJAQrFmzBt27d3+RQyWiQmU/gShXkDvg8s/++RAv4OhzO7nL+WvDCgBUCTJ+R/HcT9mEewEPnxheybddRP77INvJWT/WIQIR8G9d1Q0xDCwR3tqwAgDNw40v3+Kf6aV8gEhvw9qtF5L/PvJLdqHInEuXLuHWrVto1qyZbpq/vz9q1qyJffv2GQ1F2dnZyM7+91KXaWlpAAC1Wg212vBGjiqVqgBGbnvGxk7ylN/flTPXsOltkUcgUkRWBWD5tqy9CpQP0D5fYOSo/+lU7Tvg5uHaa7gYM+KI9krAm28av7XBgvNAkzBpfRjbDtavcRbXb2qy6QYyCUSWbsvR+9q9O3KsXXPbYk0N21Uoyr33WkhIiN50Y/dlyzVp0iSMGzfOYPrZs2fh4+OjN83DwwMxMTE2Gm3BunTpEh4/flzYwyALZGRk5Gt5Z61h09sik0AU2xCqZsOt2pY7j4FGG80OIc9DAuuuaR+m3MiS3oex7WD9Gie9fv8hk0Bkzbb032t+z2Jh1m6u/NawXYUiKUaOHImhQ4fqnqelpSEiIgJxcXHw8/MrxJHlT+nSpQt7CGSh3HfGUrGGnyWjQNR1JsS1P6EoXdNhXo/GtoP1a1y+fucyCUTK+M4AHOv/SX5r2K5CUWhoKADg9u3bCAsL002/ffs2qlSpYnQZd3d3uLu7G0xXqVR2s5vWGEvGLjRqKJTy3kZ7GGN+5bfOnLmG9cksECXtgnrPd1C+tdyufw/PMrYdrF/jJI9dLoGo/jtQVe8JwH4OWVoivzVsV6GodOnSCA0NxdatW3UhKC0tDQcOHMDAgbxb9fMUShVyfh4GFCkFVYOB0JzeDM2e+eaXiawKVbPhENf+hHrrdECTY7ptsRioWo+BeHgV6g0TgKdmDuf5h8GlzTjgaRZyfk0EHqVAUTQKLp2mStw6ci4yDESrhgLBsVI2hpyVnAJR4yFQH/pJF4xIS3ahKCMjA0lJSbrnly5dwrFjxxAUFITIyEh88MEHmDBhAmJjY3UfyQ8PD0eHDh0Kb9BylhuILD5HYhhE0k7tH3zNU9NtwytClTAK4s5ZqJe+DTzJMj2GoFJwSRgNPHqInCV9gcx7gNmREOmTZSAy8/ogMiC3QLT9a2jO72Ioeo7sLt54+PBhxMfHIz4+HgAwdOhQxMfHY8yYMQCA4cOHY/Dgwejfvz+qV6+OjIwMbNy4kdcoMsGqQGThH3xFeEWoXl8Acfe8ZYGo9yIgO10vEBFZSlmvv8MGIk8VsKox8Gc70ze0HFIeONVB+9WYqkHa5Vc11q5PSh89oiQNnywlx0CUx+sjL3Kp3dw+bFXDsgtFjRo1ghDC4LFo0SIAgEKhwPjx43Hr1i08fvwYW7ZsQZkyZQp30DKmOb2ZgYjsmrJcc4cMRADQugTQIRKoFAQMr2g4X6kAplQDyvhrvyoVhm2GVdQu3yESSCghrY8h5fK/LWSCq4fDBSJAPrWb24etalh2oYhsS3sOEQMR2S/1rjkOGYgA7ceb0/9Z5XYjl7PRCGDPHe33e+5onz8vd7n0p8AxIx+XtqQPY8uRbagSRjlcIALkU7u5fdiqhmV3ThG9OAxEZA/E2W1m59trIAKAi+lA9Gog0A24kG68TavNQKwfcN7Ep4rnnwO2Jmuvbv0g23C+JX28dwDY10baNpB5isBIqH/o61CBCJBP7eb24e1imxpmKHJSsghEngGSx08E2HcgyvUg2/g/hFxPNcCpFPPrMPUPw9I+cngz2AKj/m28wwWiXHKo3dw+TJ1zZC0ePnNCsghE3kW1d48mkkgOgUhRzD6uvkyFR9xNMj3TjgORo2IocjKyCUS9FwKuXtI3hJyaLAJReEWoWo+Rugnk7BiIZImhyInIKhC5+2rvHk1kJdkEotcXQDy8KnUzyJnJJBAp4ppIGLxjYyhyErILREveMH/3aCIjZBWI7p7XXsndmvErgIFx2ofKyEeUAaBJKDCmMlDcxI7U4l7a+U1Cjc9X2aAPKkByCURVOkHVwLo7QXQp6fi1yxOtnYAsA9GDy0CoiSt6ERkhu0C09G0gqJRV29ApEvi40r/P55zVn1/cC/itOeCiBJqEGb9j+NIGQL0Q7cnR0au1dxZ/Vv8ywDe18tcHFRA5BaJ2n0FzejNU5VtaPHxnqF3uKXJwisiq8gxERFaQZSAy9/ogep7cAtGRFXneC9MZcU+Rg1M1G275vcxkHIjUGkAl8whvD2O0R8r4zlBV7+kQgejnq8D1fxadf85w/o0soPVmoG4IsOC88XX03AX0iwX+uG34Tvv59Urtg2xMjoHot/FAqHWXgf7iuLZ+Hbl2GYocnLj2p90HIkAbNnrtAs6kSl5FgSrrD/zQoLBH4ZgcJRAB2ivwPn9I4HnbbmkfptzMAj77y/R8tQ36IBtSusgzEEm4JfeqK9qrTJviCLXLUOTg1Fun230gynUm1fwLkhyT+tBPDhGIyDmpmg6FIiLe7gORs2AocnSaHJOz7CkQkfPSHF1tdj4DEcmZIqIq1CsHMxDZCZ4B4aRkEYhcPSSOnkiLgYjkTr1lCgORHWEockKyCERuXlAljLJq3P5upucVcQeifU3Pd1UC5QO0X02J9tWuR2of9GLJIhD5h+VjC7TkULsu/E9QYMTVP83Mte9AJIfaze3DVjXMl4KTkU0geu07KAIjrRr72iZAlJEXR5QvkNQZONtJe50LYzY2B4631341pn8Z7fJJnaX3QS+OLAJRUCm4tBmXj62QT+1+W1Pa+Ck/7DsQyaV2c/uwVQ0zFDkRWQWiYrFQ/zbeqvF7uwJVjNwJOT4I8HXVft/YyBt3pQKoF6z9vl6w9vnzcpfzldiHwsTVV8n2ZBOIei8CnubvsJocalepML4cFSR5BCJFZFUpgwcgn9rN7cNWNcxQ5CRkF4h+7Gf+7tFGbE8GNlw3nP7bdWDNVeD4A2DK34bzNQIYfgQ4m6r9qjHyN2LK39rl11yV1ofgofoXQlaBKDsdOb8mSt4WQB61qxHA16fztRlkFZkEotiGUDUbLmUDAMindnP7sFUN89NnTkCWgejm31bf5mP4YeCR2nD6IzXQZbv5Zb8+pX2YcvQBUHWd6fmW9EEFS3aBaElfwDdY+gZBPrW77CIwtIL5NmQLMgpEXWdCXPsTitLSjjvJpXZz+4gPsk0Nc0+Rg1MUi5FnICKygiwDkbnXB5EBmQWipF3a69iRHoYiB6dqPYaBiOyaIq4JAxHZORkGolVDzV7HzlkxFDk48fAqAxHZNVWDgQxEZNfkGYhMvz6cGc8pcnDqDRMYiMiuaU5vZiAiu6Ws1x/Kcs0ZiOwE9xQ5uqePTc+z00BU3AvY0Ur7CPcy3uarGsDJDkC7COPz20Vo539Vo+D6INvQ7JkPRwlEwR6sXWfjSIFofh3Hr12GImclh0CklLajsl8sUC9E++gXazi/nD8wuBwQ5w9MqmZ8HZOraecPLqdtXxB9UMGzp0AEAO0jWbvORr1rjkMEIgCIL+L4tctQ5IxkEYhcoWo6VNLw99wGcjTaxx+3DedfzQQuZ2i/33zT+Do2/TP9coa2fUH0QQVLFoHIM8CqMR+7z9p1NuLsNrPz7SUQAc5RuzynyNnIJRB1mQ5FhLSrqW67BUT/c+P0G0b+t2XmAJXXApHewOlU4+v48CAw/6z2RZZp5AMY1vYRzysCv1CyCETeReHSNtGqcR+6L7/apcJjT4EIANpvBU6kOHbtMhQ5EzkFopgGUG+ZApdWn0raFGMvmGdl5uT9oslrvi36INuTTSDqvRBwNXHigxmsXQLkEYgUxWKsGvOdx+ZryxFql4fPnIXcAtHK9/O4ezSRIVkFIndf5KwfK31jyGnJIhCFV4Sq9Ripm+CwGIqcgRwD0fmdEjeGnJXsAtGSN4DUZMnbQ85JNoHo9QUQD69K3QyHxVDk6PzDGIjI7skyEJl7fRAZIatAdPe89jp2pIehyMG5tBnnMIGoebjped1KAR+/BPiYOEuuUiAwtrL2qzE+Ltrlu5WS3gcVDEVkVYcORHKo3Vg/KwZMksguEC192/x17Cwgh9rN7cNWNWx3f94TExMxbtw4vWlxcXE4c+ZMIY1I5p5mOUQgAoDPqwEH7gI7n/uoZqNQ4KeG2u9LeAODD+jPVwDY3AIo4gG8WxYIXW74J2ZSNWBgWe33tx9J64MKhqrZcIiknQ4ZiORSu7Nr2WBjyCRZBiJzrw8LyKV2c/tIyc7X5ujYXSgCgAoVKmDLli265y4udrkZL0TOr4kOEYjMEab/ZthVH2ScuPanQwYigLXrDJTxnaGq3tOhAhHguLVrl2nCxcUFoaGhhT0M+/AoxfQ8OwtEnxwxfCcBaKf13AmU9gFmGdlhKAA03wR0iATWXDX+Z2bkEeB6JnApQ3ofVDDUW6c7ZCAC5FO77+7/9x052ZYjBiJAPrWb28eZVNvUsF2GovPnzyM8PBweHh6oXbs2Jk2ahMjISKNts7OzkZ397361tLQ0AIBarYZarTZor1KpCmbQNmZs7M8zuy0yCkSWbou5K5iuuGx++eMPtQ9TMnKAL06YX0defQDGt8WS7TPH6WtYY+Qqb/+QSyDKz+tRDrV7XltSrF8rWPo7Vx/6SfaBSGr9yqF2c/vIvYBufmvY7kJRzZo1sWjRIsTFxSE5ORnjxo1D/fr1ceLECfj6+hq0nzRpksE5SABw9uxZ+Pj46E3z8PBATIx1F7MqLJcuXcLjx6ZPkjO7LbIIRAoo6/XP/7bIjLFtycjIyNc6WcPGySIQuXrYZFvkgvVrOUt/55qjq82up7ADkTXbYg/yW8N2F4oSEhJ031eqVAk1a9ZEyZIlsWLFCvTr18+g/ciRIzF06L/32EpLS0NERATi4uLg52e/H7koXbq0tAXlEohaj4GyXHMA+dgWGTK2LbnvjKViDRuSRSBy84IqYRQAx6lh1q/lbPE7l0MgAhynfoH817DdhaLnBQQEoEyZMkhKSjI6393dHe7u7gbTVSqV3eymNUbS2OUUiKp1g3rXHLg0HGTXv4fnGduW/G4fa1ifbALRa99BEag9bG/Pv4dnsX4tl9+xyyIQ+YcBcJz6BfJfw3Z/naKMjAxcuHABYWFhhT0UeZNbIFo3Os+7R5tT3Ev7MMXHBSjnb34d5fwBbzNvC2zRB9mWrAJRsViofxtv9TawdkkWgSiolPY6dlYI9nD82rW7UPTRRx9h586duHz5Mvbu3YuOHTtCpVKhR48ehT00+ZJjIDr2s8SNAZqEAhc6ax9NjHwI0ccFONYe+LsD8FUN4+v4qoZ2/l/tjb9AbdEH2ZbsAtGP/SDuGt9DbUr1IqxdZyebQNR7EfDUuk+irW3q+LVrd6Ho+vXr6NGjB+Li4tCtWzcUKVIE+/fvR7FixQp7aPLkGeBQgQgA6oUALkrto26I4fwIb6DUP+dvmroKdot/ppfyASK9C6YPsh1ZBiJzrw8TqhRh7TozWQWi7HTtdeys4Ay1a3fnFP33v/8t7CHYFZe2iYDKzWECEQAsOA80Cfv3++edTgW+Oa190Yw8YnwdI45or6a6+aa2fUH0QbahKBYDVcIouw9EALD2KlA+QPs9a9e5yC4QLekL+AZbtQ1H7wOZOY5du3YXishKrl7IWfS6wwQiALiRBTTaaL7NhwfNz193TfuwVR+518gg21O1HgNx56zdByIAuPNYfrVLBU+WgSjzntWhqP9e4OgD0/MdoXYZihxczvqxDhWIyPmIh1cdIhCRc1LENYGqwUD5BSIyyu7OKSIrpSabnsdARHZAvWECAxHZLQYi+8JQ5KwYiMhePDV9pV0GIpI7zenNDER2hKHIGckkECnjO0sbPxEgj0Ck5BkIZJ5mz3wwENkPhiJnI5dAVP8dqKr3tGroU14GPI1cmNRTBaxqDPzZzvQJz0PKA6c6aL8aUzVIu/yqxtL7oBdIFoHIFaqmQ03Pt4BcardHlKThUz7ZcyCSS+3m9mGrGmYociZyCkSNh0B96Cerht84DEgoYTi9dQmgQyRQKQgYXtFIfwpgSjWgjL/2q1Jh2GZYRe3yHSKl9aEwsk4qIHIJRF2mQxFRVeJGaMmhdpUKYEi5fG0GSSCLQOQZIHn8cqnd3D5sVcMMRc5CboFo+9d53j36eZlPgWNGPg569AGQ/s/fk+1GzivXCGDPHe33e+5onz8vd7l0iX0I03vHyZbkFIhiGkC9ZYrEDdGSQ+1qhPHlqODIIhB5F9Vex04iudRubh+2qmEeEHcGcgxEu+cCoSb2qZrwyjbgYrrh9IvpQPRqINANuGBkPgC02gzE+gHnTdwsef45YGsy8PAJ8CBbWh9UwOQWiFa+D5F+V+LGaMmldt87AOxrI20byDqyCUS9FwKuZm4ylge51G5uH94utqlh7ilydK4e8gxEEqQ9MT3vQbb5sPJUA5xK0X415UK68RempX1QAZJjIDL7+rCcHGo3x8y6yXZkFYjcfbXXscsHOdRubh+2qmHuKXJwqoRRUASUsPtARE7MPwwuCaMdMhCR85BdIFryBuAmfU+Ro+KeIgenCIxkICK75tJmHAMR2TVZBiJzrw8nxj1FDk7923gGIrJvT7MYiMhuKSKrQtVsGAORneCeIgcn7iaZnmmngUilAAbGaR8qEx+FbxIKjKkMFDexd7i4l3Z+k9CC64NsI+fXRIcJRErWrtNRNRvuMIGoS0nHr13uKXJWdhqIAKB/GeCbWv8+n3NWf35xL+C35oCLEmgSZvyuy0sbAPVCtCfnRa/W3p05P33wruMF6FGK6Xl2FIgAoFMk8HGlf5/LoXbzuis55Y+49qdDBCLAOWqXe4qckUwCkSKuiYTBE/3DzgIROSf11ukOEYicBfcUORu5BKIqnaBqMFDCBmivbWHs+1w3soDWm4G6IcCC88bX0XMX0C8W+OO24bsVKX2EeFi3DZRPsghECijr9bdq2D9fBa7/U29yqV0qYJock7PsLRB9cVxbv45cuwxFzkROgajdZ9Cc3gxV+ZZWb4ZaGO5Wfd62W9qHKTezgM/+sl0fDEUvkFwCUesxUJZrbtXQNTKsXSocsghErtb94Vp1RXuVaVMcoXZ5+MxZyC0QHVnxz92jiawgp0BUrRvUu+ZI3BByZrIIRG5eUCWMkrgFjouhyBnIMRD9Nl7ChpBTk1sgWjca4uw2iRtDzko2gei176AIjJS4FY6LocjRKV1kGoh4B1WyghwDUR6vD6LnySoQFYuFmm9ODTAUOThV06EOE4j83UzPK+IORPuanu+qBMoHaL+aEu2rXY/UPqiAeAY4dCCSQ+268D9BgZNdIPqxn/nr2FlADrWb24etapgvBQeniKjqEIEIANY2AaKMvDiifIGkzsDZTtrrXBizsTlwvL32qzH9y2iXT+osvQ8qGC5tEx02EMmldr+tKW38ZBlZBiJzrw8LyKV2c/uwVQ0zFDk49ZYpDhGIAMDbFagSZDg9PgjwddV+3zjMyJgVQL1g7ff1grXPn5e7nK/EPhQmrr5KNuDq5ZCBCJBH7SoVxpcj21AUi3G4QATIp3Zz+7BVDTMUOThx9U8zc+0nEAHA9mRgw3XD6b9dB9ZcBY4/AKYYea1rBDD8CHA2VftVY2QoU/7WLr/mqrQ+BE+RKjA568c6ZCAC5FG7GgF8fTr/20LGqVqPcbhABMindnP7sFUN8zpFTsu+AhEADD8MPFIbTn+kBrpsN7/s16e0D1OOPgCqrjM935I+qICkJpueZ8eBCJBP7S67CAytYL4NSSMeXnW4QATIp3Zz+4gPsk0Nc0+RU5JHIFJEVpUyeCItOw9E5BzUGyY4XCByZAxFTkcmgSi2IVTNhkvZACLZBCJlfGdp4yfn8fSx6XkMRLLDUORUZBSIus6EuGbufCciE+QSiOq/A1X1ntK2gUgOgUjJM2iex1DkNGQWiJJ2ae8eTWQNOQWixkOgPvSTtO0g5yaLQOQKVdOhEjfAcTEUOQUZBqJVQ83ePdqc4l7AjlbaR7iX8TZf1QBOdgDaRRif3y5CO/+rGgXXB9mY3ALR9q+hObraqk0I9mDtOj25BKIu06GIsO68zvl1HL92GYqcgDwD0VMpmwIA6BcL1AvRPvrFGs4v5w8MLgfE+QOTqhlfx+Rq2vmDy2nbF0QfZENyDER5vD6MaR/J2nVqcgpEMQ2g3jLFquHHF3H82rXbUDRr1iyUKlUKHh4eqFmzJg4ePFjYQ5IlZb3+DhWIAGDPbSBHo338cdtw/tVM4HKG9vvNN42vY9M/0y9naNsXRB9kI64eDhGIAODYfdau05JbIFr5fh7XsTPkDLVrl2dZLV++HEOHDsXcuXNRs2ZNzJgxAy1btsTZs2cRHBxc2MOTFWW55g4ViABg2y0g+p+jFjeMfNI1MweovBaI9AZOpxpfx4cHgflntS+yTCNH8aztI55XBC4wqoRRUASUsPtABACH7suvdukFkGMgOr8TCC1v1Wa03wqcSHHs2rXLPUXTp0/H22+/jb59+6J8+fKYO3cuvLy88J///KewhyY76l1zHCoQ5bqRZfxFkyszJ+8XzelU4y9MW/ZB+acIjHSIQJSLtetk/MPkGYgkuPPY8WvX7vYUPXnyBEeOHMHIkSN105RKJZo1a4Z9+/YZtM/OzkZ2drbueWqq9qf58OFDqNWGl0dWqVQICAjQW0ZOAgICkJaWZnTsz1OpVMi5dhbC38jB2X8oK70CVbVuUG//HprjWwEzbRWxDaCq+xY0h9ZAs+8nwD/GdNsSlaFq/AHEqT1Q75wN+JYybONVHC5WbEuUC6D2yLNpoYhyAdLSYHRb0tLSAABC4r1AnL2G1bvmQpP52HRtKl2gavgeFMGVoV77GcSdm2brWFn7DShjm0C9YTrEpb/Nt83j9eEoNcz6tZy19Ssafgw8vI+c3ycBLoGAf6Dxxp7+cGk5EnjqgpxfRwNqV9O16eoBVfNhUHiEQb16NITE14ej1C9gwxoWdubGjRsCgNi7d6/e9GHDhokaNWoYtB87dqyAdncGH3wU6uPatWuSap41zIccHqxfPuz9YUkNK4Swr1tZ3rx5E8WLF8fevXtRu3Zt3fThw4dj586dOHDggF7759+laDQaPHjwAEWKFIGCtzanF0AIgfT0dISHh0OptP6INWuYChPrl+ydNTVsd4fPihYtCpVKhdu39U9Nv337NkJDQw3au7u7w93dXW9aQEBAQQ6RyIC/v7/kZVnDVNhYv2TvLK1huzvR2s3NDdWqVcPWrVt10zQaDbZu3aq354iIiIjIGna3pwgAhg4dij59+uDll19GjRo1MGPGDGRmZqJv376FPTQiIiKyU3YZil599VXcvXsXY8aMwa1bt1ClShVs3LgRISEhhT00IiIislN2d6I1ERERUUGwu3OKiIiIiAoCQxERERERGIqIiIiIADAUEREREQFgKCIiIiICwFBEREREBIChiIiIiAgAQxERERERAIYiIiIiIgAMRUREREQAGIqIyIFcvnwZCoUCx44dK+yhkANq1KgRPvjgg8IeBhUghiKymkajkXV/b7zxBhQKBSZPnqw3fc2aNVAoFBavx9QfwEWLFiEgIMCqMcmNeMG/Q6FRW9xWoVCYfSQmJhbcQC1gLnjZ+z9N9YstC6v7mzt3Lnx9fZGTk6OblpGRAVdXVzRq1Eiv7Y4dO6BQKHDhwgUbjNQ4R/4b4axcCnsAZH+USiXmzZuH5OTkAu8rLCwMAwYMsHo5Dw8PfPHFFxgwYAACAwMLYGT2TaFUImf9eIgbf5lu5BkAl7aJgKsXctaPBVLN/L5dPaBKGAVFYCTUv42HuJv0b19Fo+DSaarFY3u2rpYvX44xY8bg7Nmzumk+Pj4Wr4uso1ICvXYBZ1L1p5f1B8ZUBq5nAaOPAtnPZVx3FfBZPFDCCxj3F3D2ueUBoEcU0DkSWH0VWHZRu84fGlg3vsaNGyMjIwOHDx9GrVq1AAC7d+9GaGgoDhw4gMePH8PDwwMAsH37dkRGRiI6OtqqPoQQUKvVcHHhv0dnxD1FJElycjKuXLlS4A+pwatZs2YIDQ3FpEmTTLZZvXo1KlSoAHd3d5QqVQrTpk2T+uPAnDlzEB0dDTc3N8TFxeGHH37Qm69QKDBnzhwkJCTA09MTUVFRWLVqleT+bEHc+Au4dcr4I/0OXFqNBFRuyFn0OnB2q+m2Dy5D1ez/oAgoAfUPfSH+/p/efHHvolXjCg0N1T38/f2hUCh0z4ODgzF9+nSUKFEC7u7uqFKlCjZu3GhyXWq1Gm+++SbKli2Lq1evAgDWrl2LqlWrwsPDA1FRURg3bpzengeFQoHvv/8eHTt2hJeXF2JjY/G///1P0s/44cOH6N27NwIDA+Hl5YWEhAScP39eNz93j8KaNWsQGxsLDw8PtGzZEteuXZPUny2cTQOOPtB/vBIJxPoDjcOAcC/D+cW9tPNi/YH2kYbz/3oIDCkHlPTVfv3roWHwskRcXBzCwsKwY8cO3bQdO3agffv2KF26NPbv3683vXHjxsjOzsb777+P4OBgeHh4oF69ejh06JBeO4VCgQ0bNqBatWpwd3fHnj17kJmZid69e8PHxwdhYWH5+vsA2OffCGfEUEQOSaVS4fPPP8c333yD69evG8w/cuQIunXrhu7du+Pvv/9GYmIiRo8ejUWLFlnd1y+//IIhQ4bg//7v/3DixAkMGDAAffv2xfbt2/XajR49Gp07d8Zff/2F1157Dd27d8fp06elbmLB8S4Kl94LAXdf5Cx5A3hw2XRbNy+oXvsOimKxUP/YD+Lm3wU6tK+//hrTpk3Dl19+iePHj6Nly5Z45ZVX9IJGruzsbHTt2hXHjh3D7t27ERkZid27d6N3794YMmQITp06hXnz5mHRokWYOHGi3rLjxo1Dt27dcPz4cbRu3RqvvfYaHjx4YPV433jjDRw+fBj/+9//sG/fPggh0Lp1azx9+lTXJisrCxMnTsSSJUvwxx9/ICUlBd27d7f+h2MjQhhO2/7Pe5P0p8AxIz+Gow+0855t+yyNAPbc0X6/5472uVSNGzfWe21t374djRo1QsOGDXXTHz16hAMHDqBx48YYPnw4Vq9ejcWLF+PPP/9ETEwMWrZsafD7HDFiBCZPnozTp0+jUqVKGDZsGHbu3Im1a9di06ZN2LFjB/78809JY3a4vxEOTCGEsZcAkXmJiYm4cuVKgfdTsmRJq88heeONN5CSkoI1a9agdu3aKF++PBYsWIA1a9agY8eOEELgtddew927d7Fp0ybdcsOHD8f69etx8uRJANrzBfbu3Qs3Nze99efk5MDDwwMpKSkAgLp166JChQqYP3++rk23bt2QmZmJ9evXA9C+C3znnXcwZ84cXZtatWqhatWqmD17tlXbZytP53fW7s15VkEEotDycO2/WtIYFy1ahA8++ED3sy5evDgGDRqETz75RNemRo0aqF69OmbNmoXLly+jdOnS2L17NxITE5GdnY1ff/0V/v7+ALR7EJs2bYqRI0fqlv/xxx8xfPhw3Lx5E4D2dzVq1Ch89tlnAIDMzEz4+Phgw4YNaNWqla4PT09PKJX67ysfPXqEwYMHY8aMGTh//jzKlCmDP/74A3Xq1AEA3L9/HxEREVi8eDG6du2KRYsWoW/fvti/fz9q1qwJADhz5gzKlSuHAwcOoEaNGpJ+bvlRfZ025Dwv2hd4+AR4kG18uSB3INANuJBufL6rEoj1A86nAU81QHwQcKid9eP7/vvvdTXx6NEjBAUF4ebNm9iyZQvmzp2LnTt3Ytu2bWjatCkuX76M2NhYLFq0CD179gQAPH36FKVKlcIHH3yAYcOG6fYorVmzBu3btwegPU+pSJEi+PHHH9G1a1cAwIMHD1CiRAn0798fM2bMAOD4fyOcEfcUkUP74osvsHjxYoN3W6dPn0bdunX1ptWtWxfnz5+HWv3vCROvvfYajh07pvcYP368Ret6vs/atWsbPJfVu8CC2kOktM25GWlpabh586ZFP+sePXogMzMTmzZt0gUiAPjrr78wfvx4+Pj46B5vv/02kpOTkZWVpWtXqVIl3ffe3t7w8/PDnTt39PpYvny5QW28/PLLuvmnT5+Gi4uLLuwAQJEiRRAXF6c3XhcXF1SvXl33vGzZsggICJBXbUAbdkwFIkA7z1QgArRB6FSK9mt+NGrUCJmZmTh06BB2796NMmXKoFixYmjYsKHuvKIdO3YgKioKqampePr0qV7NuLq6okaNGgY/32d/dxcuXMCTJ0/0fndBQUGIi4szGI9T/Y1wAjyTjBxagwYN0LJlS4wcORJvvPGG1cv7+/sjJiZGb1pwcLCNRicjBRaIXKFqOtTWo81T69at8eOPP2Lfvn1o0qSJbnpGRgbGjRuHTp06GSyTe4IuoP3H+SyFQmHwKciIiAiD2vD09LTF8MmMmJgYlChRAtu3b8fDhw/RsGFDAEB4eDgiIiKwd+9ebN++Xe/3bglvb29J43GavxFOgnuKyOFNnjwZ69atw759+3TTypUrhz/++EOv3R9//IEyZcpApVJZtX5T6ypfvrzetGdPAs19Xq5cOav6KhAFGYi6TIcioqpNhunn54fw8HCLftYDBw7E5MmT8corr2Dnzp266VWrVsXZs2cRExNj8Hj+UFh+lStXDjk5OThw4IBu2v3793H27Fm98ebk5ODw4cO652fPnkVKSoo8akOmGjdujB07dmDHjh16H8Vv0KABNmzYgIMHD6Jx48a6E5ufrZmnT5/i0KFDBjXzrOjoaLi6uur97h4+fIhz585JGq/d/41wItxTRJKEhYXZTT8VK1bEa6+9hpkzZ+qm/d///R+qV6+Ozz77DK+++ir27duHb7/9VtKx+2HDhqFbt26Ij49Hs2bNsG7dOvz888/YsmWLXruVK1fi5ZdfRr169bB06VIcPHgQCxYsyPf2SaUoGgXx/Mfu3byAUBP/LJ7/2L1Gbbqt0gWqpkOhiKgKzcEfoKrzpk3GPGzYMIwdOxbR0dGoUqUKFi5ciGPHjmHp0qUGbQcPHgy1Wo22bdtiw4YNqFevHsaMGYO2bdsiMjISXbp0gVKpxF9//YUTJ05gwoQJNhljrtjYWLRv3x5vv/025s2bB19fX4wYMQLFixfXnbsCaPdKDR48GDNnzoSLiwvee+891KpVq1DOJwK0H5WXez+NGzfGoEGD8PTpU92eIgBo2LAh3nvvPTx58gSNGzeGt7c3Bg4ciGHDhiEoKAiRkZGYMmUKsrKy0K9fP5Pr9/HxQb9+/TBs2DAUKVIEwcHB+PTTTyUHZ3v9G+GUBJGV1Gq1rPvr06ePaN++vd60S5cuCTc3N/Fsya9atUqUL19euLq6isjISDF16lS9ZRo2bCiGDBlisP6FCxcKf39/vWmzZ88WUVFRwtXVVZQpU0YsWbJEbz4AMWvWLNG8eXPh7u4uSpUqJZYvX27VdtmSRp1jF/09/7NWq9UiMTFRFC9eXLi6uorKlSuLDRs26OZfunRJABBHjx7VTZs2bZrw9fUVf/zxhxBCiI0bN4o6deoIT09P4efnJ2rUqCHmz5+vaw9A/PLLL3rj8Pf3FwsXLjTZR67na+bBgweiV69ewt/fX3h6eoqWLVuKc+fOGWzf6tWrRVRUlHB3dxfNmjUTV65csf6HZQM5L/alLXI00pbL/R2ULVtWb/rly5cFABEXF6eb9ujRIzF48GBRtGhR4e7uLurWrSsOHjyom799+3YBQDx8+FBvXenp6eL1118XXl5eIiQkREyZMsXg9+vIfyOcFT99RvQCKBQK/PLLL+jQoUNhD4Vk5PlP15Hz4t8IeeA5RURERERgKCIiIiICwIs3EhEREQHgniIiIiIiAAxFRERERAAYioiIiIgAMBQRERERAWAoIiIiIgLAUEREREQEgKGIiIiICABDEREREREAhiIiIiIiAAxFRERERAAYioiIiIgAMBQRERERAWAoIiIiIgLAUEREREQEAPh//1rsSgNxj0AAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 600x400 with 6 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "plot_surprisal_differences_checkpoints(seeds=[53], checkpoints=CHECKPOINTS, pos_encodings=False)\n",
+    "plt.savefig(f\"figures/hop_surprisals_no_pos_encodings.pdf\", format=\"pdf\", bbox_inches=\"tight\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "babyenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

hop_surprisal/hop_surprisal.py

@@ -0,0 +1,193 @@
+# hop_surprisal.py
+# Author: Julie Kallini
+
+# For importing utils
+import sys
+sys.path.append("..")
+
+import os
+import torch
+import pandas as pd
+import tqdm
+import argparse
+from numpy.random import default_rng
+from transformers import GPT2LMHeadModel
+from gpt2_no_positional_encoding_model import GPT2NoPositionalEncodingLMHeadModel
+from itertools import zip_longest
+from glob import glob
+from utils import CHECKPOINT_READ_PATH, PERTURBATIONS, PAREN_MODELS, \
+    BABYLM_DATA_PATH, gpt2_hop_tokenizer, \
+    marker_sg_token, marker_pl_token, compute_surprisals
+
+
+MAX_TRAINING_STEPS = 3000
+CHECKPOINTS = list(range(100, MAX_TRAINING_STEPS+1, 100))
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        prog='Get marker token surprisals for hop verb perturbations',
+        description='Marker token surprisals')
+    parser.add_argument('perturbation_type',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=PERTURBATIONS.keys(),
+                        help='Perturbation function used to transform BabyLM dataset')
+    parser.add_argument('train_set',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=["100M", "10M"],
+                        help='BabyLM train set')
+    parser.add_argument('random_seed', type=int, help="Random seed")
+    parser.add_argument('paren_model',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=list(PAREN_MODELS.keys()) + ["randinit"],
+                        help='Parenthesis model')
+    parser.add_argument('-np', '--no_pos_encodings', action='store_true',
+                        help="Train GPT-2 with no positional encodings")
+
+    # Get args
+    args = parser.parse_args()
+    no_pos_encodings_underscore = "_no_positional_encodings" if args.no_pos_encodings else ""
+
+    if "hop" not in args.perturbation_type:
+        raise Exception(
+            "'{args.perturbation_type}' is not a valid hop perturbation")
+
+    # Get path to model
+    model = f"babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}_seed{args.random_seed}"
+    model_path = f"{CHECKPOINT_READ_PATH}/babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}/{model}/runs/{model}/checkpoint-"
+
+    # Get perturbed test files
+    test_files = sorted(glob(BABYLM_DATA_PATH +
+        "/babylm_data_perturbed/babylm_{}/babylm_test_affected/*".format(args.perturbation_type)))
+
+    EOS_TOKEN = gpt2_hop_tokenizer.eos_token_id
+    FILE_SAMPLE_SIZE = 1000
+    MAX_SEQ_LEN = 1024
+    rng = default_rng(args.random_seed)
+
+    marker_token_sequences = []
+    nomarker_token_sequences = []
+    target_indices = []
+
+    # Iterate over data files to get surprisal data
+    print("Sampling BabyLM affected test files to extract surprisals...")
+    for test_file in test_files:
+        print(test_file)
+
+        # Get tokens from test file (+ eos token), and subsample
+        f = open(test_file, 'r')
+        file_token_sequences = [
+            [int(s) for s in l.split()] + [EOS_TOKEN] for l in f.readlines()]
+        file_token_sequences = [
+            toks for toks in file_token_sequences if len(toks) < MAX_SEQ_LEN]
+        sample_indices = rng.choice(
+            list(range(len(file_token_sequences))), FILE_SAMPLE_SIZE, replace=False)
+        file_token_sequences = [file_token_sequences[i]
+                                for i in sample_indices]
+
+        file_target_indices = []
+        file_nomarker_token_sequences = []
+        for tokens in file_token_sequences:
+            # Find index of first marker token for surprisal target
+            target_index = None
+            for idx in range(len(tokens)):
+                if tokens[idx] in (marker_sg_token, marker_pl_token):
+                    target_index = idx
+                    break
+            assert (target_index is not None)
+
+            # Make a version of tokens with marker removed at surprisal target
+            nomarker_tokens = tokens.copy()
+            nomarker_tokens.pop(target_index)
+            assert (tokens[:target_index] == nomarker_tokens[:target_index])
+            assert (tokens[target_index] in (marker_sg_token, marker_pl_token))
+            assert (tokens[target_index+1] == nomarker_tokens[target_index])
+
+            file_target_indices.append(target_index)
+            file_nomarker_token_sequences.append(nomarker_tokens)
+
+        marker_token_sequences.extend(file_token_sequences)
+        nomarker_token_sequences.extend(file_nomarker_token_sequences)
+        target_indices.extend(file_target_indices)
+
+    # For logging/debugging, include decoded sentence
+    marker_sents = [gpt2_hop_tokenizer.decode(
+        toks) for toks in marker_token_sequences]
+    nomarker_sents = [gpt2_hop_tokenizer.decode(
+        toks) for toks in nomarker_token_sequences]
+
+    surprisal_df = pd.DataFrame({
+        "Sentences with Marker": marker_sents,
+        "Sentences without Marker": nomarker_sents,
+    })
+
+    BATCH_SIZE = 32
+    device = "cuda"
+    for ckpt in CHECKPOINTS:
+        print(f"Checkpoint: {ckpt}")
+
+        # Load model
+        if args.no_pos_encodings:
+            model = GPT2NoPositionalEncodingLMHeadModel.from_pretrained(
+                model_path + str(ckpt)).to(device)
+        else:
+            model = GPT2LMHeadModel.from_pretrained(
+                model_path + str(ckpt)).to(device)
+
+        # Init lists for tracking correct/wrong surprisals for each example
+        marker_token_surprisals = []
+        nomarker_token_surprisals = []
+
+        # Iterate over data in batches
+        for i in tqdm.tqdm(range(0, len(marker_token_sequences), BATCH_SIZE)):
+
+            # Extract data for batch and pad the sequences
+            marker_batch = marker_token_sequences[i:i+BATCH_SIZE]
+            correct_batch_padded = zip(
+                *zip_longest(*marker_batch, fillvalue=gpt2_hop_tokenizer.eos_token_id))
+            marker_batch_tensors = torch.tensor(
+                list(correct_batch_padded)).to(device)
+
+            # Do the same for wrong batch
+            nomarker_batch = nomarker_token_sequences[i:i+BATCH_SIZE]
+            nomarker_batch_padded = zip(
+                *zip_longest(*nomarker_batch, fillvalue=gpt2_hop_tokenizer.eos_token_id))
+            nomarker_batch_tensors = torch.tensor(
+                list(nomarker_batch_padded)).to(device)
+
+            # Get target indices in a batch
+            targets_batch = target_indices[i:i+BATCH_SIZE]
+
+            # Compute marker/nomarker surprisals for batches
+            marker_surprisal_sequences = compute_surprisals(
+                model, marker_batch_tensors)
+            nomarker_surprisal_sequences = compute_surprisals(
+                model, nomarker_batch_tensors)
+
+            # Extract surprisals for target token
+            for marker_seq, nomarker_seq, idx in \
+                    zip(marker_surprisal_sequences, nomarker_surprisal_sequences, targets_batch):
+                marker_token_surprisals.append(marker_seq[idx])
+                nomarker_token_surprisals.append(nomarker_seq[idx])
+
+        # Add surprisals to df
+        ckpt_df = pd.DataFrame(
+            list(zip(marker_token_surprisals, nomarker_token_surprisals)),
+            columns=[f'Marker Token Surprisals (ckpt {ckpt})',
+                     f'No Marker Token Surprisals (ckpt {ckpt})']
+        )
+        surprisal_df = pd.concat((surprisal_df, ckpt_df), axis=1)
+
+    # Write results to CSV
+    directory = f"hop_surprisal_results/{args.perturbation_type}_{args.train_set}{no_pos_encodings_underscore}"
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    file = directory + f"/{args.paren_model}_seed{args.random_seed}.csv"
+    print(f"Writing results to CSV: {file}")
+    surprisal_df.to_csv(file)

hop_surprisal/hop_surprisal_gpt2.py

@@ -0,0 +1,204 @@
+# hop_surprisal.py
+# Author: Yaning
+
+# For importing utils
+import sys
+sys.path.append("..")
+
+import os
+import torch
+import pandas as pd
+import tqdm
+import argparse
+from numpy.random import default_rng
+
+from itertools import zip_longest
+from glob import glob
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling
+from utils_gpt2 import CHECKPOINT_READ_PATH, PERTURBATIONS, PAREN_MODELS, \
+    BABYLM_DATA_PATH, gpt2_hop_tokenizer, \
+    marker_sg_token, marker_pl_token, compute_surprisals
+
+
+MODEL_NAME = "GPT2-Hop"
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+MAX_TRAINING_STEPS = 1500
+# CHECKPOINTS = list(range(500, MAX_TRAINING_STEPS+1, 500))
+CHECKPOINTS = [2241]    # control 2241 
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        prog='Get marker token surprisals for hop verb perturbations',
+        description='Marker token surprisals')
+    parser.add_argument('perturbation_type',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=PERTURBATIONS.keys(),
+                        help='Perturbation function used to transform BabyLM dataset')
+    parser.add_argument('train_set',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=["100M", "10M"],
+                        help='BabyLM train set')
+    
+    parser.add_argument('random_seed', type=int, help="Random seed")
+
+    # Get args
+    args = parser.parse_args()
+
+    if "hop" not in args.perturbation_type:
+        raise Exception(
+            "'{args.perturbation_type}' is not a valid hop perturbation")
+
+    # Get path to model
+    # model = f"babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}_seed{args.random_seed}"
+    # model_path = f"{CHECKPOINT_READ_PATH}/babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}/{model}/runs/{model}/checkpoint-"
+
+    model_path = f'../train/checkpoints/{MODEL_NAME}/babylm_{args.perturbation_type}_10M_seed0/runs/checkpoint-'
+
+    # Get perturbed test files
+    # test_files = sorted(glob(BABYLM_DATA_PATH +
+    #     "/babylm_data_perturbed/babylm_{}/babylm_test_affected/*".format(args.perturbation_type)))
+    test_files = sorted(glob("../data/Perturbed_data/gpt2/babylm_{}/babylm_test_affected/*".format(args.perturbation_type)))
+    print("test_files:", test_files)
+
+    EOS_TOKEN = gpt2_hop_tokenizer.eos_token_id
+    print("EOS_TOKEN:", EOS_TOKEN)
+    FILE_SAMPLE_SIZE = 500
+    MAX_SEQ_LEN = 512
+    rng = default_rng(args.random_seed)
+
+    marker_token_sequences = []
+    nomarker_token_sequences = []
+    target_indices = []
+
+    # Iterate over data files to get surprisal data
+    print("Sampling BabyLM affected test files to extract surprisals...")
+    for test_file in test_files:
+        print(test_file)
+
+        # Get tokens from test file (+ eos token), and subsample
+        f = open(test_file, 'r')
+        file_token_sequences = [
+            [int(s) for s in l.split()] + [EOS_TOKEN] for l in f.readlines()]
+        file_token_sequences = [
+            toks for toks in file_token_sequences if len(toks) < MAX_SEQ_LEN]
+        sample_indices = rng.choice(
+            list(range(len(file_token_sequences))), FILE_SAMPLE_SIZE, replace=False)
+        file_token_sequences = [file_token_sequences[i]
+                                for i in sample_indices]
+
+        file_target_indices = []
+        file_nomarker_token_sequences = []
+        for tokens in file_token_sequences:
+            # Find index of first marker token for surprisal target
+            target_index = None
+            for idx in range(len(tokens)):
+                # print("marker_sg_token", marker_sg_token)
+                # print("marker_pl_token", marker_pl_token)
+                if tokens[idx] in (marker_sg_token, marker_pl_token):
+                    target_index = idx
+                    break
+            assert (target_index is not None)
+
+            # Make a version of tokens with marker removed at surprisal target
+            nomarker_tokens = tokens.copy()
+            nomarker_tokens.pop(target_index)
+            assert (tokens[:target_index] == nomarker_tokens[:target_index])
+            assert (tokens[target_index] in (marker_sg_token, marker_pl_token))
+            assert (tokens[target_index+1] == nomarker_tokens[target_index])
+
+            file_target_indices.append(target_index)
+            file_nomarker_token_sequences.append(nomarker_tokens)
+
+        marker_token_sequences.extend(file_token_sequences)
+        nomarker_token_sequences.extend(file_nomarker_token_sequences)
+        target_indices.extend(file_target_indices)
+
+    # For logging/debugging, include decoded sentence
+    marker_sents = [gpt2_hop_tokenizer.decode(
+        toks) for toks in marker_token_sequences]
+    nomarker_sents = [gpt2_hop_tokenizer.decode(
+        toks) for toks in nomarker_token_sequences]
+
+    surprisal_df = pd.DataFrame({
+        "Sentences with Marker": marker_sents,
+        "Sentences without Marker": nomarker_sents,
+    })
+
+    # BATCH_SIZE = 32
+    BATCH_SIZE = 32
+    for ckpt in CHECKPOINTS:
+        print(f"Checkpoint: {ckpt}")
+
+        model = AutoModelForCausalLM.from_pretrained(model_path + str(ckpt)).to(device)
+        # model = AutoModelForCausalLM.from_pretrained(model_path + str(ckpt))
+        orig_vocab_size = model.config.vocab_size
+        model.resize_token_embeddings(len(gpt2_hop_tokenizer))
+        # Init lists for tracking correct/wrong surprisals for each example
+        marker_token_surprisals = []
+        nomarker_token_surprisals = []
+
+        # Iterate over data in batches
+        for i in tqdm.tqdm(range(0, len(marker_token_sequences), BATCH_SIZE)):
+
+            # Extract data for batch and pad the sequences
+            marker_batch = marker_token_sequences[i:i+BATCH_SIZE]
+            correct_batch_padded = zip(
+                *zip_longest(*marker_batch, fillvalue=gpt2_hop_tokenizer.eos_token_id))
+            # marker_batch_tensors = torch.tensor(
+            #     list(correct_batch_padded))
+            marker_batch_tensors = torch.tensor(
+                list(correct_batch_padded)).to(device)
+
+            # Do the same for wrong batch
+            nomarker_batch = nomarker_token_sequences[i:i+BATCH_SIZE]
+            nomarker_batch_padded = zip(
+                *zip_longest(*nomarker_batch, fillvalue=gpt2_hop_tokenizer.eos_token_id))
+            # nomarker_batch_tensors = torch.tensor(
+            #     list(nomarker_batch_padded))
+            nomarker_batch_tensors = torch.tensor(
+                list(nomarker_batch_padded)).to(device)
+
+            # Get target indices in a batch
+            targets_batch = target_indices[i:i+BATCH_SIZE]
+
+            # Compute marker/nomarker surprisals for batches
+            marker_surprisal_sequences = compute_surprisals(
+                model, marker_batch_tensors)
+            nomarker_surprisal_sequences = compute_surprisals(
+                model, nomarker_batch_tensors)
+
+            # Extract surprisals for target token
+            for marker_seq, nomarker_seq, idx in \
+                    zip(marker_surprisal_sequences, nomarker_surprisal_sequences, targets_batch):
+                marker_token_surprisals.append(marker_seq[idx])
+                nomarker_token_surprisals.append(nomarker_seq[idx])
+
+        # Add surprisals to df
+        ckpt_df = pd.DataFrame(
+            list(zip(marker_token_surprisals, nomarker_token_surprisals)),
+            columns=[f'Marker Token Surprisals (ckpt {ckpt})',
+                     f'No Marker Token Surprisals (ckpt {ckpt})']
+        )
+        surprisal_df = pd.concat((surprisal_df, ckpt_df), axis=1)
+
+    # Write results to CSV
+    directory = os.path.join(
+        "hop_surprisal_results", 
+        MODEL_NAME, 
+        f"{args.perturbation_type}_{args.train_set}"
+    )
+
+    os.makedirs(directory, exist_ok=True)
+
+    filename = f"{args.perturbation_type}_{args.train_set}_seed{args.random_seed}.csv"
+    file_path = os.path.join(directory, filename)
+
+    print(f"The file saves as to: {file_path}")
+    surprisal_df.to_csv(file_path, index=False)
+

hop_surprisal/hop_surprisal_llama3.py

@@ -0,0 +1,205 @@
+wwwwwww# hop_surprisal.py
+# Author: Yaning
+
+# For importing utils
+import sys
+sys.path.append("..")
+
+import os
+import torch
+import pandas as pd
+import tqdm
+import argparse
+from numpy.random import default_rng
+
+from itertools import zip_longest
+from glob import glob
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling
+from utils_llama import CHECKPOINT_READ_PATH, PERTURBATIONS, PAREN_MODELS, \
+    BABYLM_DATA_PATH, gpt2_hop_tokenizer, \
+    marker_sg_token, marker_pl_token, compute_surprisals
+
+
+# MODEL_NAME = "GPT-2"
+MODEL_NAME = "meta-llama/Llama-3.2-3B"
+
+MAX_TRAINING_STEPS = 1500
+# CHECKPOINTS = list(range(500, MAX_TRAINING_STEPS+1, 500))
+CHECKPOINTS = [2376]
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        prog='Get marker token surprisals for hop verb perturbations',
+        description='Marker token surprisals')
+    parser.add_argument('perturbation_type',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=PERTURBATIONS.keys(),
+                        help='Perturbation function used to transform BabyLM dataset')
+    parser.add_argument('train_set',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=["100M", "10M"],
+                        help='BabyLM train set')
+    
+    parser.add_argument('random_seed', type=int, help="Random seed")
+
+    # Get args
+    args = parser.parse_args()
+
+    if "hop" not in args.perturbation_type:
+        raise Exception(
+            "'{args.perturbation_type}' is not a valid hop perturbation")
+
+    # Get path to model
+    # model = f"babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}_seed{args.random_seed}"
+    # model_path = f"{CHECKPOINT_READ_PATH}/babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}/{model}/runs/{model}/checkpoint-"
+
+    model_path = f'../train/checkpoints/{MODEL_NAME}/babylm_{args.perturbation_type}_10M_seed0/runs/checkpoint-'
+
+    # Get perturbed test files
+    # test_files = sorted(glob(BABYLM_DATA_PATH +
+    #     "/babylm_data_perturbed/babylm_{}/babylm_test_affected/*".format(args.perturbation_type)))
+    test_files = sorted(glob("../data/Perturbed_data/gpt2/babylm_{}/babylm_test_affected/*".format(args.perturbation_type)))
+    print("test_files:", test_files)
+
+    EOS_TOKEN = gpt2_hop_tokenizer.eos_token_id
+    print("EOS_TOKEN:", EOS_TOKEN)
+    FILE_SAMPLE_SIZE = 500
+    MAX_SEQ_LEN = 512
+    rng = default_rng(args.random_seed)
+
+    marker_token_sequences = []
+    nomarker_token_sequences = []
+    target_indices = []
+
+    # Iterate over data files to get surprisal data
+    print("Sampling BabyLM affected test files to extract surprisals...")
+    for test_file in test_files:
+        print(test_file)
+
+        # Get tokens from test file (+ eos token), and subsample
+        f = open(test_file, 'r')
+        file_token_sequences = [
+            [int(s) for s in l.split()] + [EOS_TOKEN] for l in f.readlines()]
+        file_token_sequences = [
+            toks for toks in file_token_sequences if len(toks) < MAX_SEQ_LEN]
+        sample_indices = rng.choice(
+            list(range(len(file_token_sequences))), FILE_SAMPLE_SIZE, replace=False)
+        file_token_sequences = [file_token_sequences[i]
+                                for i in sample_indices]
+
+        file_target_indices = []
+        file_nomarker_token_sequences = []
+        for tokens in file_token_sequences:
+            # Find index of first marker token for surprisal target
+            target_index = None
+            for idx in range(len(tokens)):
+                # print("marker_sg_token", marker_sg_token)
+                # print("marker_pl_token", marker_pl_token)
+                if tokens[idx] in (marker_sg_token, marker_pl_token):
+                    target_index = idx
+                    break
+            assert (target_index is not None)
+
+            # Make a version of tokens with marker removed at surprisal target
+            nomarker_tokens = tokens.copy()
+            nomarker_tokens.pop(target_index)
+            assert (tokens[:target_index] == nomarker_tokens[:target_index])
+            assert (tokens[target_index] in (marker_sg_token, marker_pl_token))
+            assert (tokens[target_index+1] == nomarker_tokens[target_index])
+
+            file_target_indices.append(target_index)
+            file_nomarker_token_sequences.append(nomarker_tokens)
+
+        marker_token_sequences.extend(file_token_sequences)
+        nomarker_token_sequences.extend(file_nomarker_token_sequences)
+        target_indices.extend(file_target_indices)
+
+    # For logging/debugging, include decoded sentence
+    marker_sents = [gpt2_hop_tokenizer.decode(
+        toks) for toks in marker_token_sequences]
+    nomarker_sents = [gpt2_hop_tokenizer.decode(
+        toks) for toks in nomarker_token_sequences]
+
+    surprisal_df = pd.DataFrame({
+        "Sentences with Marker": marker_sents,
+        "Sentences without Marker": nomarker_sents,
+    })
+
+    # BATCH_SIZE = 32
+    BATCH_SIZE = 32
+    device = "cuda"
+    for ckpt in CHECKPOINTS:
+        print(f"Checkpoint: {ckpt}")
+
+        # model = AutoModelForCausalLM.from_pretrained(model_path + str(ckpt)).to(device)
+        model = AutoModelForCausalLM.from_pretrained(model_path + str(ckpt))
+        orig_vocab_size = model.config.vocab_size
+        model.resize_token_embeddings(len(gpt2_hop_tokenizer))
+        # Init lists for tracking correct/wrong surprisals for each example
+        marker_token_surprisals = []
+        nomarker_token_surprisals = []
+
+        # Iterate over data in batches
+        for i in tqdm.tqdm(range(0, len(marker_token_sequences), BATCH_SIZE)):
+
+            # Extract data for batch and pad the sequences
+            marker_batch = marker_token_sequences[i:i+BATCH_SIZE]
+            correct_batch_padded = zip(
+                *zip_longest(*marker_batch, fillvalue=gpt2_hop_tokenizer.eos_token_id))
+            marker_batch_tensors = torch.tensor(
+                list(correct_batch_padded))
+            # marker_batch_tensors = torch.tensor(
+            #     list(correct_batch_padded)).to(device)
+
+            # Do the same for wrong batch
+            nomarker_batch = nomarker_token_sequences[i:i+BATCH_SIZE]
+            nomarker_batch_padded = zip(
+                *zip_longest(*nomarker_batch, fillvalue=gpt2_hop_tokenizer.eos_token_id))
+            nomarker_batch_tensors = torch.tensor(
+                list(nomarker_batch_padded))
+            # nomarker_batch_tensors = torch.tensor(
+            #     list(nomarker_batch_padded)).to(device)
+
+            # Get target indices in a batch
+            targets_batch = target_indices[i:i+BATCH_SIZE]
+
+            # Compute marker/nomarker surprisals for batches
+            marker_surprisal_sequences = compute_surprisals(
+                model, marker_batch_tensors)
+            nomarker_surprisal_sequences = compute_surprisals(
+                model, nomarker_batch_tensors)
+
+            # Extract surprisals for target token
+            for marker_seq, nomarker_seq, idx in \
+                    zip(marker_surprisal_sequences, nomarker_surprisal_sequences, targets_batch):
+                marker_token_surprisals.append(marker_seq[idx])
+                nomarker_token_surprisals.append(nomarker_seq[idx])
+
+        # Add surprisals to df
+        ckpt_df = pd.DataFrame(
+            list(zip(marker_token_surprisals, nomarker_token_surprisals)),
+            columns=[f'Marker Token Surprisals (ckpt {ckpt})',
+                     f'No Marker Token Surprisals (ckpt {ckpt})']
+        )
+        surprisal_df = pd.concat((surprisal_df, ckpt_df), axis=1)
+
+    # Write results to CSV
+    directory = os.path.join(
+        "hop_surprisal_results", 
+        MODEL_NAME, 
+        f"{args.perturbation_type}_{args.train_set}"
+    )
+
+    os.makedirs(directory, exist_ok=True)
+
+    filename = f"{args.perturbation_type}_{args.train_set}_seed{args.random_seed}.csv"
+    file_path = os.path.join(directory, filename)
+
+    print(f"The file saves as to: {file_path}")
+    surprisal_df.to_csv(file_path, index=False)
+

impossible_llm.yaml

@@ -0,0 +1,154 @@
+name: impossible_llm
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - ca-certificates=2024.7.2=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.15=h5eee18b_0
+  - pip=24.2=py39h06a4308_0
+  - python=3.9.19=h955ad1f_1
+  - readline=8.2=h5eee18b_0
+  - setuptools=72.1.0=py39h06a4308_0
+  - sqlite=3.45.3=h5eee18b_0
+  - tk=8.6.14=h39e8969_0
+  - wheel=0.44.0=py39h06a4308_0
+  - xz=5.4.6=h5eee18b_1
+  - zlib=1.2.13=h5eee18b_1
+  - pip:
+      - accelerate==0.34.2
+      - aiohappyeyeballs==2.4.2
+      - aiohttp==3.10.6
+      - aiosignal==1.3.1
+      - asttokens==2.2.1
+      - async-timeout==4.0.3
+      - attrs==24.2.0
+      - backcall==0.2.0
+      - black==24.8.0
+      - blessed==1.20.0
+      - certifi==2023.11.17
+      - charset-normalizer==3.3.2
+      - click==8.1.7
+      - cmake==3.30.3
+      - comm==0.1.2
+      - contourpy==1.2.0
+      - cycler==0.12.1
+      - data==0.4
+      - datasets==3.0.1
+      - debugpy==1.6.7
+      - decorator==5.1.1
+      - dill==0.3.8
+      - emoji==2.8.0
+      - exceptiongroup==1.1.0
+      - executing==1.2.0
+      - filelock==3.12.2
+      - fonttools==4.45.1
+      - frozenlist==1.4.1
+      - fsspec==2023.10.0
+      - funcsigs==1.0.2
+      - future==0.18.3
+      - gmpy2==2.1.0
+      - gpustat==1.1.1
+      - huggingface-hub==0.25.0
+      - idna==3.6
+      - importlib-metadata==6.0.0
+      - importlib-resources==6.4.5
+      - iniconfig==2.0.0
+      - ipykernel==6.23.1
+      - ipython==8.0.0
+      - jedi==0.18.2
+      - jinja2==3.1.2
+      - joblib==1.3.2
+      - jupyter-client==8.1.0
+      - jupyter-core==5.1.0
+      - kiwisolver==1.4.5
+      - latex==0.7.0
+      - latexcodec==1.0.0
+      - lit==18.1.8
+      - markupsafe==2.1.2
+      - matplotlib==3.8.2
+      - matplotlib-inline==0.1.6
+      - mizani==0.9.3
+      - mpmath==1.2.1
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - mypy-extensions==1.0.0
+      - nest-asyncio==1.5.6
+      - networkx==2.8.6
+      - nltk==3.8.1
+      - numpy==1.26.2
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-ml-py==12.560.30
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - packaging==23.0
+      - pandas==2.1.3
+      - parso==0.8.3
+      - pathspec==0.12.1
+      - patsy==0.5.3
+      - peft==0.13.0
+      - pexpect==4.8.0
+      - pickleshare==0.7.5
+      - pillow==10.1.0
+      - platformdirs==2.5.2
+      - plotnine==0.12.4
+      - pluggy==1.3.0
+      - pluralizer==1.2.0
+      - prompt-toolkit==3.0.30
+      - protobuf==4.25.1
+      - psutil==5.9.1
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - pyarrow==17.0.0
+      - pygments==2.15.0
+      - pyparsing==3.1.1
+      - pytest==7.4.3
+      - python-dateutil==2.8.2
+      - pytz==2023.3.post1
+      - pyyaml==6.0.1
+      - pyzmq==23.0.0
+      - regex==2023.10.3
+      - requests==2.32.3
+      - safetensors==0.4.5
+      - scikit-learn==1.3.2
+      - scipy==1.11.4
+      - seaborn==0.13.0
+      - sentencepiece==0.2.0
+      - shutilwhich==1.1.0
+      - six==1.16.0
+      - stack-data==0.6.0
+      - stanza==1.9.2
+      - statsmodels==0.14.0
+      - sympy==1.11.1
+      - tempdir==0.7.1
+      - threadpoolctl==3.2.0
+      - tokenizers==0.20.0
+      - tomli==2.0.1
+      - torch==2.0.0
+      - tornado==6.2
+      - tqdm==4.66.5
+      - traitlets==5.7.1
+      - transformers==4.45.1
+      - triton==2.0.0
+      - typing-extensions==4.6.0
+      - tzdata==2023.3
+      - urllib3==2.1.0
+      - wcwidth==0.2.5
+      - xxhash==3.5.0
+      - yarl==1.13.0
+      - zipp==3.12.0
+prefix: /home/yiren/new_ssd2/chunhui/miniconda/envs/impossible_llm

impossible_llm_update.yaml

@@ -0,0 +1,162 @@
+name: impossible_llm
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - ca-certificates=2024.7.2=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.15=h5eee18b_0
+  - pip=24.2=py39h06a4308_0
+  - python=3.9.19=h955ad1f_1
+  - readline=8.2=h5eee18b_0
+  - setuptools=72.1.0=py39h06a4308_0
+  - sqlite=3.45.3=h5eee18b_0
+  - tk=8.6.14=h39e8969_0
+  - wheel=0.44.0=py39h06a4308_0
+  - xz=5.4.6=h5eee18b_1
+  - zlib=1.2.13=h5eee18b_1
+  - pip:
+      - accelerate==1.0.0
+      - aiohappyeyeballs==2.4.2
+      - aiohttp==3.10.6
+      - aiosignal==1.3.1
+      - annotated-types==0.7.0
+      - asttokens==2.2.1
+      - async-timeout==4.0.3
+      - attrs==24.2.0
+      - backcall==0.2.0
+      - black==24.8.0
+      - blessed==1.20.0
+      - certifi==2023.11.17
+      - charset-normalizer==3.3.2
+      - click==8.1.7
+      - cmake==3.30.3
+      - comm==0.1.2
+      - contourpy==1.2.0
+      - cycler==0.12.1
+      - data==0.4
+      - datasets==3.0.1
+      - debugpy==1.6.7
+      - decorator==5.1.1
+      - deepspeed==0.15.2
+      - dill==0.3.8
+      - emoji==2.8.0
+      - exceptiongroup==1.1.0
+      - executing==1.2.0
+      - filelock==3.12.2
+      - fonttools==4.45.1
+      - frozenlist==1.4.1
+      - fsspec==2023.10.0
+      - funcsigs==1.0.2
+      - future==0.18.3
+      - gmpy2==2.1.0
+      - gpustat==1.1.1
+      - hjson==3.1.0
+      - huggingface-hub==0.25.0
+      - idna==3.6
+      - importlib-metadata==6.0.0
+      - importlib-resources==6.4.5
+      - iniconfig==2.0.0
+      - ipykernel==6.23.1
+      - ipython==8.0.0
+      - jedi==0.18.2
+      - jinja2==3.1.2
+      - joblib==1.3.2
+      - jupyter-client==8.1.0
+      - jupyter-core==5.1.0
+      - kiwisolver==1.4.5
+      - latex==0.7.0
+      - latexcodec==1.0.0
+      - lit==18.1.8
+      - markupsafe==2.1.2
+      - matplotlib==3.8.2
+      - matplotlib-inline==0.1.6
+      - mizani==0.9.3
+      - mpmath==1.2.1
+      - msgpack==1.1.0
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - mypy-extensions==1.0.0
+      - nest-asyncio==1.5.6
+      - networkx==2.8.6
+      - ninja==1.11.1.1
+      - nltk==3.8.1
+      - numpy==1.26.2
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-ml-py==12.560.30
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - packaging==23.0
+      - pandas==2.1.3
+      - parso==0.8.3
+      - pathspec==0.12.1
+      - patsy==0.5.3
+      - peft==0.13.0
+      - pexpect==4.8.0
+      - pickleshare==0.7.5
+      - pillow==10.1.0
+      - platformdirs==2.5.2
+      - plotnine==0.12.4
+      - pluggy==1.3.0
+      - pluralizer==1.2.0
+      - prompt-toolkit==3.0.30
+      - protobuf==4.25.1
+      - psutil==5.9.1
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - py-cpuinfo==9.0.0
+      - pyarrow==17.0.0
+      - pydantic==2.9.2
+      - pydantic-core==2.23.4
+      - pygments==2.15.0
+      - pyparsing==3.1.1
+      - pytest==7.4.3
+      - python-dateutil==2.8.2
+      - pytz==2023.3.post1
+      - pyyaml==6.0.1
+      - pyzmq==23.0.0
+      - regex==2023.10.3
+      - requests==2.32.3
+      - safetensors==0.4.5
+      - scikit-learn==1.3.2
+      - scipy==1.11.4
+      - seaborn==0.13.0
+      - sentencepiece==0.2.0
+      - shutilwhich==1.1.0
+      - six==1.16.0
+      - stack-data==0.6.0
+      - stanza==1.9.2
+      - statsmodels==0.14.0
+      - sympy==1.11.1
+      - tempdir==0.7.1
+      - threadpoolctl==3.2.0
+      - tokenizers==0.20.0
+      - tomli==2.0.1
+      - torch==2.0.0
+      - tornado==6.2
+      - tqdm==4.66.5
+      - traitlets==5.7.1
+      - transformers==4.45.1
+      - triton==2.0.0
+      - typing-extensions==4.12.2
+      - tzdata==2023.3
+      - urllib3==2.1.0
+      - wcwidth==0.2.5
+      - xxhash==3.5.0
+      - yarl==1.13.0
+      - zipp==3.12.0
+prefix: /home/yiren/new_ssd2/chunhui/miniconda/envs/impossible_llm

perplexities/demo.py

@@ -0,0 +1,82 @@
+import torch
+import sys
+import argparse
+import os
+sys.path.append("..")
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling
+from datasets import load_dataset
+from numpy.random import default_rng
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+MODEL_NAME_SAVE = "Llama-3.2-3B"
+FILE_SAMPLE_SIZE = 5
+
+def get_perplexities(model, eval_dataset, batch_size):
+    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+    training_args = TrainingArguments(
+        output_dir="./tmp_trainer",
+        per_device_eval_batch_size=batch_size,
+        fp16=True,
+        report_to="none"
+    )
+
+    trainer = Trainer(model=model, args=training_args, eval_dataset=eval_dataset, data_collator=data_collator)
+    eval_results = trainer.evaluate()
+    print("eval_results:", eval_results)  
+    loss = eval_results['eval_loss']
+    perplexity = torch.exp(torch.tensor(loss)).item()
+    return perplexity
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Calculate perplexity on test dataset.")
+
+    parser.add_argument('perturbation', 
+                        type=str, 
+                        default='reverse_full', 
+                        nargs='?', 
+                        help='Type of perturbation to use.')
+    parser.add_argument('train_set', 
+                        type=str, 
+                        default='test', 
+                        nargs='?', 
+                        help='Dataset size for training.')
+    parser.add_argument('batch_size', 
+                        type=int, 
+                        default=4, 
+                        nargs='?', 
+                        help='Batch size for evaluation.')
+    parser.add_argument('seed', 
+                        type=int, 
+                        default=0, 
+                        nargs='?', 
+                        help='Random seed.')
+
+    args = parser.parse_args()
+
+    dataset_name = f"babylm_{args.perturbation}_{args.train_set}_seed{args.seed}"
+    dataset = load_dataset('../train/babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+    test_dataset = dataset['test']  # Load test dataset
+    print(test_dataset)
+    
+    checkpoint_path = f'../train/checkpoints/{MODEL_NAME_SAVE}/babylm_{args.perturbation}_10M_seed0/runs/checkpoint-450'
+    
+    rng = default_rng(args.seed)
+    indices = rng.choice(len(test_dataset), FILE_SAMPLE_SIZE, replace=False)
+    sampled_test_dataset = test_dataset.select(indices)
+
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)
+    model = AutoModelForCausalLM.from_pretrained(checkpoint_path)
+    
+    model.eval()
+    if torch.cuda.is_available():
+        model.to('cuda')
+
+    def tokenize_function(examples):
+        return tokenizer(examples['text'], padding="max_length", truncation=True, max_length=1024)
+
+    tokenized_test = sampled_test_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+
+
+    perplexity = get_perplexities(model, tokenized_test, 1)
+    print(f"Perplexity on test set: {perplexity}")

perplexities/gpt2_mask.py

@@ -0,0 +1,231 @@
+import torch
+import sys
+import argparse
+import os
+import pandas as pd
+import matplotlib.pyplot as plt
+from tqdm import tqdm
+import gc
+from numpy.random import default_rng
+sys.path.append("..")
+from transformers import (
+    AutoTokenizer, 
+    AutoModelForCausalLM,
+    Trainer, 
+    TrainingArguments, 
+    DataCollatorForLanguageModeling
+)
+from datasets import load_dataset
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+class LayerMasker:
+    def __init__(self, model):
+        self.model = model
+        self.original_forwards = {}  # Store original forward methods
+        
+    def _mask_ffn(self, layer, mask_strength=1.0):
+        """Correctly replace FFN forward propagation"""
+        layer_id = id(layer.mlp)
+        if layer_id not in self.original_forwards:
+            self.original_forwards[layer_id] = layer.mlp.forward  # Store original function
+            
+        def masked_forward(x):
+            return self.original_forwards[layer_id](x) * (1 - mask_strength)
+        
+        layer.mlp.forward = masked_forward  # Override with masked function
+    
+    def _mask_attn(self, layer, mask_strength=1.0):
+        """Correctly replace Attention forward propagation""" 
+        layer_id = id(layer.attn)
+        if layer_id not in self.original_forwards:
+            self.original_forwards[layer_id] = layer.attn.forward  # Store original function
+            
+        def masked_forward(hidden_states, **kwargs):
+            output = self.original_forwards[layer_id](hidden_states, **kwargs)
+            masked_output = (output[0] * (1 - mask_strength),) + output[1:]
+            return masked_output
+        
+        layer.attn.forward = masked_forward  # Override with masked function
+    
+    def mask_layer(self, layer_idx, module_type="ffn", mask_strength=1.0):
+        layer = self.model.transformer.h[layer_idx]
+        if module_type == "ffn":
+            self._mask_ffn(layer, mask_strength)
+        elif module_type == "attn":
+            self._mask_attn(layer, mask_strength)
+        else:
+            raise ValueError(f"Invalid module type: {module_type}")
+    
+    def reset(self):
+        """Restore all modified forward methods"""
+        for layer in self.model.transformer.h:
+            layer_id = id(layer.mlp)
+            if layer_id in self.original_forwards:
+                layer.mlp.forward = self.original_forwards[layer_id]
+                
+            layer_id = id(layer.attn)
+            if layer_id in self.original_forwards:
+                layer.attn.forward = self.original_forwards[layer_id]
+        
+        self.original_forwards.clear()
+
+@torch.no_grad()  # Disable gradient computation
+def get_perplexities(model, eval_dataset, batch_size):
+    """Compute perplexity for the dataset"""
+    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+    training_args = TrainingArguments(
+        output_dir="./tmp_trainer",
+        per_device_eval_batch_size=batch_size,
+        fp16=torch.cuda.is_available(),
+        report_to="none"
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        eval_dataset=eval_dataset,
+        data_collator=data_collator
+    )
+    
+    eval_results = trainer.evaluate()
+    loss = eval_results['eval_loss']
+    return torch.exp(torch.tensor(loss)).item()
+
+def run_mask_experiment(model, tokenized_data, args):
+    """Run layer masking experiment"""
+    results = []
+    masker = LayerMasker(model)
+    
+    # Compute baseline perplexity
+    baseline_ppl = get_perplexities(model, tokenized_data, args.batch_size)
+    print(f"Baseline Perplexity: {baseline_ppl:.2f}")
+    
+    # Layer masking experiments
+    print("\nRunning layer masking experiments:")
+    for layer_idx in tqdm(range(model.config.n_layer), desc="Layers"):
+        for module_type in ["ffn", "attn"]:
+            # Apply masking
+            masker.mask_layer(layer_idx, module_type)
+            
+            # Compute perplexity
+            ppl = get_perplexities(model, tokenized_data, args.batch_size)
+            delta = ppl - baseline_ppl
+
+            gc.collect()            
+            torch.cuda.empty_cache()  
+            
+            results.append({
+                "layer": layer_idx,
+                "module": module_type,
+                "perplexity": ppl,
+                "delta": delta
+            })
+            
+            # Reset the model
+            masker.reset()
+    
+    return results, baseline_ppl
+
+def visualize_results(results_df, output_dir):
+    """Visualize layer impact results"""
+    plt.figure(figsize=(12, 6))
+    
+    # Plot FFN curve
+    ffn_data = results_df[results_df["module"] == "ffn"]
+    plt.plot(ffn_data["layer"], ffn_data["delta"], 
+             marker='o', linestyle='-', linewidth=2, markersize=8,
+             color='#1f77b4', label='FFN')
+    
+    # Plot Attention curve
+    attn_data = results_df[results_df["module"] == "attn"]
+    plt.plot(attn_data["layer"], attn_data["delta"], 
+             marker='s', linestyle='--', linewidth=2, markersize=8,
+             color='#ff7f0e', label='Attention')
+    
+    plt.xlabel("Layer Index", fontsize=12)
+    plt.ylabel("Δ Perplexity (log scale)", fontsize=12)
+    plt.title("GPT-2 Layer Masking Impact Analysis", fontsize=14)
+    plt.legend(fontsize=10)
+    plt.grid(True, alpha=0.3)
+    plt.xticks(range(0, results_df["layer"].max()+1))
+    
+    # Set y-axis to log scale
+    plt.yscale('log')
+
+    plot_path = os.path.join(output_dir, f"{args.perturbation}_layer_impact.png")
+    plt.savefig(plot_path, bbox_inches='tight', dpi=300)
+    plt.close()
+
+
+if __name__ == "__main__":
+    # Argument configuration
+    parser = argparse.ArgumentParser(description="GPT-2 Layer Masking Experiment")
+    parser.add_argument('--perturbation', type=str, default="hop_sg2pl",
+                       help="Perturbation type (default: hop_sg2pl)")
+    parser.add_argument('--train_set', type=str, default="10M",
+                       help="Training set size (default: 10M)")
+    parser.add_argument('--checkpoint_path', type=str, default="checkpoint-2736",
+                       help="Model checkpoint path (default: checkpoint-2736)")
+    parser.add_argument('--batch_size', type=int, default=3,
+                       help="Evaluation batch size (default: 3)")
+    parser.add_argument('--seed', type=int, default=0,
+                       help="Random seed (default: 0)")
+    args = parser.parse_args()
+
+    # Initialize output directory
+    output_dir = f"mask_results/GPT2-MaskResults"
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Load dataset
+    dataset_name = f"babylm_{args.perturbation}_{args.train_set}_seed{args.seed}"
+    dataset = load_dataset('../train/babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+    test_dataset = dataset['test']
+    
+    # Data sampling
+    rng = default_rng(args.seed)
+    indices = rng.choice(len(test_dataset), size=500, replace=False)
+    sampled_test = test_dataset.select(indices)
+
+    # Load model and tokenizer
+    checkpoint_path = f"../train/checkpoints/GPT-2/babylm_{args.perturbation}_10M_seed0/runs/{args.checkpoint_path}"
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)
+    model = AutoModelForCausalLM.from_pretrained(checkpoint_path, torch_dtype=torch.float16).eval()
+    if torch.cuda.is_available():
+        model = model.cuda()
+
+    # Data preprocessing
+    def tokenize_fn(examples):
+        return tokenizer(
+            examples["text"],
+            padding="max_length",
+            truncation=True,
+            max_length=512,
+            return_tensors="pt"
+        )
+    
+    tokenized_test = sampled_test.map(
+        tokenize_fn, 
+        batched=True, 
+        remove_columns=["text"],
+        desc="Tokenizing"
+    )
+
+    # Run experiment
+    results, baseline_ppl = run_mask_experiment(model, tokenized_test, args)
+    results_df = pd.DataFrame(results)
+    
+    # Save results
+    output_file = os.path.join(output_dir, f"mask_results_{args.perturbation}.csv")
+    results_df.to_csv(output_file, index=False)
+    
+    # Generate visualization
+    visualize_results(results_df, output_dir)
+    
+    # Print key results
+    print(f"\nExperiment completed. Results saved to {output_dir}")
+    print(f"Baseline Perplexity: {baseline_ppl:.2f}")
+    print("Layer impact results:")
+    print(results_df[["layer", "module", "delta"]].to_string(index=False))
+

perplexities/gpt2_noise.py

@@ -0,0 +1,203 @@
+# Activation Steering
+
+import torch
+import sys
+import argparse
+import os
+import pandas as pd
+import matplotlib.pyplot as plt
+from tqdm import tqdm
+from numpy.random import default_rng
+sys.path.append("..")
+from transformers import (
+    AutoTokenizer, 
+    AutoModelForCausalLM,
+    Trainer, 
+    TrainingArguments, 
+    DataCollatorForLanguageModeling
+)
+from datasets import load_dataset
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+class LayerNoiseInjector:
+
+    def __init__(self, model):
+        self.model = model
+        self.hooks = []  # Store registered hooks
+        self.noise_std = 0.1  # Noise standard deviation (adjustable)
+    
+    
+    def _add_noise_hook(self, layer_idx):
+        def hook(module, input, output):
+            if isinstance(output, tuple):
+                modified_output = list(output)
+                noise = torch.randn_like(modified_output[0]) * self.noise_std
+                modified_output[0] = modified_output[0] + noise
+                return tuple(modified_output)
+            else:
+                noise = torch.randn_like(output) * self.noise_std
+                return output + noise
+        
+        handle = self.model.transformer.h[layer_idx].register_forward_hook(hook)
+        self.hooks.append(handle)
+        
+    
+    def inject_noise(self, layer_idx):
+        """Inject noise to specified layer"""
+        self._add_noise_hook(layer_idx)
+    
+    def reset(self):
+        """Remove all noise hooks"""
+        for handle in self.hooks:
+            handle.remove()
+        self.hooks.clear()
+
+@torch.no_grad()  # Disable gradient calculation
+def get_perplexities(model, eval_dataset, batch_size):
+    """Calculate dataset perplexity"""
+    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+    training_args = TrainingArguments(
+        output_dir="./tmp_trainer",
+        per_device_eval_batch_size=batch_size,
+        fp16=torch.cuda.is_available(),
+        report_to="none"
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        eval_dataset=eval_dataset,
+        data_collator=data_collator
+    )
+    
+    eval_results = trainer.evaluate()
+    loss = eval_results['eval_loss']
+    return torch.exp(torch.tensor(loss)).item()
+
+# Data preprocessing
+def tokenize_fn(examples):
+    return tokenizer(
+        examples["text"],
+        padding="max_length",
+        truncation=True,
+        max_length=512,
+        return_tensors="pt"
+    )
+
+
+def run_noise_experiment(model, tokenized_data, args):
+    """Execute layer-wise noise injection experiment"""
+    results = []
+    noise_injector = LayerNoiseInjector(model)
+    
+    # Calculate baseline perplexity
+    baseline_ppl = get_perplexities(model, tokenized_data, args.batch_size)
+    print(f"Baseline Perplexity: {baseline_ppl:.2f}")
+    
+    # Layer-wise noise injection experiment
+    print("\nRunning layer noise injection experiments:")
+    for layer_idx in tqdm(range(model.config.n_layer), desc="Layers"):
+        # Inject noise
+        noise_injector.inject_noise(layer_idx)
+        
+        # Calculate perplexity
+        ppl = get_perplexities(model, tokenized_data, args.batch_size)
+        delta = ppl - baseline_ppl
+        
+        results.append({
+            "layer": layer_idx,
+            "perplexity": ppl,
+            "delta": delta
+        })
+        
+        # Reset model
+        noise_injector.reset()
+    
+    return results, baseline_ppl
+
+def visualize_noise_results(results_df, output_dir):
+    """Visualize noise injection impact results"""
+    plt.figure(figsize=(12, 6))
+    
+    # Plot delta perplexity curve
+    plt.plot(results_df["layer"], results_df["delta"], 
+             marker='o', linestyle='-', linewidth=2, markersize=8,
+             color='#2ca02c', label='Activation Noise')
+    
+    plt.xlabel("Layer Index", fontsize=12)
+    plt.ylabel("Δ Perplexity (log scale)", fontsize=12)
+    plt.title("GPT-2 Layer Noise Injection Impact Analysis", fontsize=14)
+    plt.legend(fontsize=10)
+    plt.grid(True, alpha=0.3)
+    plt.xticks(range(0, results_df["layer"].max()+1))
+    
+    # Set y-axis to log scale
+    plt.yscale('log')
+
+    plot_path = os.path.join(output_dir, f"{args.perturbation}_noise_impact.png")
+    plt.savefig(plot_path, bbox_inches='tight', dpi=300)
+    plt.close()
+
+if __name__ == "__main__":
+    # Parameter configuration
+    parser = argparse.ArgumentParser(description="GPT-2 Layer Noise Injection Experiment")
+    parser.add_argument('--perturbation', type=str, default="hop_sg2pl",
+                       help="Perturbation type (default: hop_sg2pl)")
+    parser.add_argument('--train_set', type=str, default="10M",
+                       help="Training set size (default: 10M)")
+    parser.add_argument('--checkpoint_path', type=str, default="checkpoint-2736",
+                       help="Model checkpoint path (default: checkpoint-2736)")
+    parser.add_argument('--batch_size', type=int, default=3,
+                       help="Evaluation batch size (default: 3)")
+    parser.add_argument('--seed', type=int, default=0,
+                       help="Random seed (default: 0)")
+    parser.add_argument('--noise_std', type=float, default=0.1,
+                       help="Noise standard deviation (default: 0.1)")
+    args = parser.parse_args()
+
+    # Initialize output directory
+    output_dir = f"noise_results/GPT2-NoiseResults"
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Load dataset
+    dataset_name = f"babylm_{args.perturbation}_{args.train_set}_seed{args.seed}"
+    dataset = load_dataset('../train/babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+    test_dataset = dataset['test']
+    
+    # Data sampling
+    rng = default_rng(args.seed)
+    indices = rng.choice(len(test_dataset), size=500, replace=False)
+    sampled_test = test_dataset.select(indices)
+
+    # Load model and tokenizer
+    checkpoint_path = f"../train/checkpoints/GPT-2/babylm_{args.perturbation}_10M_seed0/runs/{args.checkpoint_path}"
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)
+    model = AutoModelForCausalLM.from_pretrained(checkpoint_path, torch_dtype=torch.float16).eval()
+    if torch.cuda.is_available():
+        model = model.cuda()
+    
+    tokenized_test = sampled_test.map(
+        tokenize_fn, 
+        batched=True, 
+        remove_columns=["text"],
+        desc="Tokenizing"
+    )
+
+    # Run noise injection experiment
+    results, baseline_ppl = run_noise_experiment(model, tokenized_test, args)
+    results_df = pd.DataFrame(results)
+    
+    # Save results
+    output_file = os.path.join(output_dir, f"noise_results_{args.perturbation}.csv")
+    results_df.to_csv(output_file, index=False)
+    
+    # Generate visualization
+    visualize_noise_results(results_df, output_dir)
+    
+    # Print key results
+    print(f"\nExperiment completed. Results saved to {output_dir}")
+    print(f"Baseline Perplexity: {baseline_ppl:.2f}")
+    print("Layer impact results:")
+    print(results_df[["layer", "delta"]].to_string(index=False))

perplexities/gpt2_remove_layers.py

@@ -0,0 +1,141 @@
+import torch
+import sys
+import argparse
+import os
+sys.path.append("..")
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling
+from datasets import load_dataset
+from numpy.random import default_rng
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+# MODEL_NAME = "Llama-3.2-3B"
+# ori_model_name = "meta-llama/Llama-3.2-3B"
+# MODEL_NAME_SAVE = "Llama-3.2-3B-500-Remove"
+
+MODEL_NAME = "GPT-2"
+ori_model_name = "gpt2"
+MODEL_NAME_SAVE = "GPT2-500-Remove-layers"
+
+FILE_SAMPLE_SIZE = 500
+
+
+def remove_layers(original_model, exp_model, i, j):
+
+    original_blocks = original_model.transformer.h
+    exp_blocks = exp_model.transformer.h
+
+    print("len(original_blocks):", len(original_blocks))
+    print("len(exp_blocks):", len(exp_blocks))
+
+    # Ensure the indices are valid
+    if i < 0 or (i + j) > len(exp_blocks) or j < 0:
+        raise ValueError(f"Invalid block indices: i={i}, i+j={i+j}. Must satisfy 0 <= i and i+j <= {len(exp_blocks)}.")
+
+    # Replace the parameters of the blocks from i to i+j
+    for idx in range(i, i + j):
+        print(f"Replacing parameters of Transformer block {idx}...")
+        original_block = original_blocks[idx]
+        exp_blocks[idx].load_state_dict(original_block.state_dict())
+
+    return exp_model
+
+
+
+def get_perplexities(model, eval_dataset, batch_size):
+    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+    training_args = TrainingArguments(
+        output_dir="./tmp_trainer",
+        per_device_eval_batch_size=batch_size,
+        fp16=True,
+        report_to="none"
+    )
+
+    trainer = Trainer(model=model, args=training_args, eval_dataset=eval_dataset, data_collator=data_collator)
+    eval_results = trainer.evaluate()
+    print("eval_results:", eval_results)  
+    loss = eval_results['eval_loss']
+    perplexity = torch.exp(torch.tensor(loss)).item()
+    return perplexity
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Calculate perplexity on test dataset.")
+
+    parser.add_argument('perturbation', 
+                        type=str, 
+                        default='reverse_full', 
+                        nargs='?', 
+                        help='Type of perturbation to use.')
+    parser.add_argument('train_set', 
+                        type=str, 
+                        default='test', 
+                        nargs='?', 
+                        help='Dataset size for training.')
+    parser.add_argument('checkpoint_path', 
+                        type=str, 
+                        default='checkpoint-100', 
+                        nargs='?', 
+                        help='Dataset size for training.')
+    parser.add_argument('batch_size', 
+                        type=int, 
+                        default=4, 
+                        nargs='?', 
+                        help='Batch size for evaluation.')
+    parser.add_argument('seed', 
+                        type=int, 
+                        default=0, 
+                        nargs='?', 
+                        help='Random seed.')
+    parser.add_argument('remove_layer', 
+                        type=int, 
+                        default=1, 
+                        nargs='?', 
+                        help='Layer index to remove')
+    args = parser.parse_args()
+
+    dataset_name = f"babylm_{args.perturbation}_{args.train_set}_seed{args.seed}"
+    dataset = load_dataset('../train/babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+    test_dataset = dataset['test']  # Load test dataset
+    print(test_dataset)
+    
+    
+    checkpoint_path = f'../train/checkpoints/{MODEL_NAME}/babylm_{args.perturbation}_10M_seed0/runs/{args.checkpoint_path}'
+    
+    rng = default_rng(args.seed)
+    indices = rng.choice(len(test_dataset), FILE_SAMPLE_SIZE, replace=False)
+    sampled_test_dataset = test_dataset.select(indices)
+
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)
+    model = AutoModelForCausalLM.from_pretrained(checkpoint_path)
+    model_ori = AutoModelForCausalLM.from_pretrained(ori_model_name)
+    model = remove_layers(model_ori, model, args.remove_layer, 4)
+    print(model)
+    model.eval()
+    # if torch.cuda.is_available():
+    #     model.to('cuda')
+
+    def tokenize_function(examples):
+        return tokenizer(examples['text'], padding="max_length", truncation=True, max_length=1024)
+
+    tokenized_test = sampled_test_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+
+
+    perplexity = get_perplexities(model, tokenized_test, args.batch_size)
+    
+    # Save the result to the specified directory
+    output_directory = f"perplexities_results/{MODEL_NAME_SAVE}"
+    os.makedirs(output_directory, exist_ok=True)
+
+    # Construct the output file path
+    output_file = os.path.join(output_directory, f"{args.perturbation}_{args.batch_size}_{args.seed}.csv")
+
+    # Write the header to the CSV file if it doesn't exist
+    if not os.path.exists(output_file):
+        with open(output_file, 'w') as f:
+            print("Writing header to CSV...")
+            f.write(f"checkpoint_path, perplexity\n")
+
+    # Append the perplexity result to the CSV file
+    with open(output_file, 'a') as f:  # Open in append mode
+        print("Appending result to CSV...")
+        f.write(f"{args.checkpoint_path}-{args.remove_layer+1}, {perplexity}\n")

perplexities/gpt_mask.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+CUDA_VISIBLE_DEVICES=1,2 torchrun --nproc_per_node=2 --master_port=22235 gpt2_mask.py --perturbation shuffle_nondeterministic --train_set 10M --checkpoint_path checkpoint-2736 --batch_size 3 --seed 0
+
+CUDA_VISIBLE_DEVICES=1,2 torchrun --nproc_per_node=2 --master_port=22235 gpt2_mask.py --perturbation shuffle_nondeterministic --train_set 10M --checkpoint_path checkpoint-2736 --batch_size 3 --seed 0
+
+CUDA_VISIBLE_DEVICES=1,2 torchrun --nproc_per_node=2 --master_port=22235 gpt2_mask.py --perturbation hop_words4 --train_set 10M --checkpoint_path checkpoint-2376 --batch_size 3 --seed 0
+
+CUDA_VISIBLE_DEVICES=1,2 torchrun --nproc_per_node=2 --master_port=22235 gpt2_mask.py --perturbation hop_control --train_set 10M --checkpoint_path checkpoint-2409 --batch_size 3 --seed 0

perplexities/gpt_noise.sh

@@ -0,0 +1,7 @@
+CUDA_VISIBLE_DEVICES=2,3 torchrun --nproc_per_node=2 --master_port=25235 gpt2_noise.py --perturbation shuffle_nondeterministic --train_set 10M --checkpoint_path checkpoint-2736 --batch_size 3 --seed 0 --noise_std 0.1
+
+CUDA_VISIBLE_DEVICES=2,3 torchrun --nproc_per_node=2 --master_port=25236 gpt2_noise.py --perturbation shuffle_control --train_set 10M --checkpoint_path checkpoint-2736 --batch_size 3 --seed 0 --noise_std 0.1
+
+CUDA_VISIBLE_DEVICES=2,3 torchrun --nproc_per_node=2 --master_port=25237 gpt2_noise.py --perturbation hop_words4 --train_set 10M --checkpoint_path checkpoint-2376 --batch_size 3 --seed 0 --noise_std 0.1
+
+CUDA_VISIBLE_DEVICES=2,3 torchrun --nproc_per_node=2 --master_port=25238 gpt2_noise.py --perturbation hop_control --train_set 10M --checkpoint_path checkpoint-2409 --batch_size 3 --seed 0 --noise_std 0.1

perplexities/llama3.2_remove_layers.py

@@ -0,0 +1,135 @@
+import torch
+import sys
+import argparse
+import os
+sys.path.append("..")
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling
+from datasets import load_dataset
+from numpy.random import default_rng
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+MODEL_NAME = "Llama-3.2-3B"
+ori_model_name = "meta-llama/Llama-3.2-3B"
+MODEL_NAME_SAVE = "Llama-3.2-3B-500-Remove-layers"
+
+
+FILE_SAMPLE_SIZE = 500
+
+
+def remove_layers(original_model, exp_model, i, j):
+
+    original_layers = original_model.model.layers
+    exp_layers = exp_model.model.layers
+
+    print("len(layers):", len(original_layers))
+    # Ensure the indices are valid
+    if i < 0 or (i + j) > len(exp_layers) or j < 0:
+        raise ValueError(f"Invalid layer indices: i={i}, i+j={i+j}. Must satisfy 0 <= i and i+j <= {len(exp_layers)}.")
+
+    # Replace the parameters of the layers from i to i+j
+    for idx in range(i, i + j):
+        print(f"Replacing parameters of LlamaDecoderLayer {idx}...")
+        original_layer = original_layers[idx]
+        exp_layers[idx].load_state_dict(original_layer.state_dict())
+
+    return exp_model
+
+
+def get_perplexities(model, eval_dataset, batch_size):
+    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+    training_args = TrainingArguments(
+        output_dir="./tmp_trainer",
+        per_device_eval_batch_size=batch_size,
+        fp16=True,
+        report_to="none"
+    )
+
+    trainer = Trainer(model=model, args=training_args, eval_dataset=eval_dataset, data_collator=data_collator)
+    eval_results = trainer.evaluate()
+    print("eval_results:", eval_results)  
+    loss = eval_results['eval_loss']
+    perplexity = torch.exp(torch.tensor(loss)).item()
+    return perplexity
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Calculate perplexity on test dataset.")
+
+    parser.add_argument('perturbation', 
+                        type=str, 
+                        default='reverse_full', 
+                        nargs='?', 
+                        help='Type of perturbation to use.')
+    parser.add_argument('train_set', 
+                        type=str, 
+                        default='test', 
+                        nargs='?', 
+                        help='Dataset size for training.')
+    parser.add_argument('checkpoint_path', 
+                        type=str, 
+                        default='checkpoint-100', 
+                        nargs='?', 
+                        help='Dataset size for training.')
+    parser.add_argument('batch_size', 
+                        type=int, 
+                        default=4, 
+                        nargs='?', 
+                        help='Batch size for evaluation.')
+    parser.add_argument('seed', 
+                        type=int, 
+                        default=0, 
+                        nargs='?', 
+                        help='Random seed.')
+    parser.add_argument('remove_layer', 
+                        type=int, 
+                        default=1, 
+                        nargs='?', 
+                        help='Layer index to remove')
+    args = parser.parse_args()
+
+    dataset_name = f"babylm_{args.perturbation}_{args.train_set}_seed{args.seed}"
+    dataset = load_dataset('../train/babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+    test_dataset = dataset['test']  # Load test dataset
+    print(test_dataset)
+    
+    
+    checkpoint_path = f'../train/checkpoints/{MODEL_NAME}/babylm_{args.perturbation}_10M_seed0/runs/{args.checkpoint_path}'
+    
+    rng = default_rng(args.seed)
+    indices = rng.choice(len(test_dataset), FILE_SAMPLE_SIZE, replace=False)
+    sampled_test_dataset = test_dataset.select(indices)
+
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)
+    model = AutoModelForCausalLM.from_pretrained(checkpoint_path)
+    model_ori = AutoModelForCausalLM.from_pretrained(ori_model_name)
+    model = remove_layers(model_ori, model, args.remove_layer, 9)
+    print(model)
+    model.eval()
+    # if torch.cuda.is_available():
+    #     model.to('cuda')
+
+    def tokenize_function(examples):
+        return tokenizer(examples['text'], padding="max_length", truncation=True, max_length=1024)
+
+    tokenized_test = sampled_test_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+
+
+    perplexity = get_perplexities(model, tokenized_test, args.batch_size)
+    
+    # Save the result to the specified directory
+    output_directory = f"perplexities_results/{MODEL_NAME_SAVE}"
+    os.makedirs(output_directory, exist_ok=True)
+
+    # Construct the output file path
+    output_file = os.path.join(output_directory, f"{args.perturbation}_{args.batch_size}_{args.seed}.csv")
+
+    # Write the header to the CSV file if it doesn't exist
+    if not os.path.exists(output_file):
+        with open(output_file, 'w') as f:
+            print("Writing header to CSV...")
+            f.write(f"checkpoint_path, perplexity\n")
+
+    # Append the perplexity result to the CSV file
+    with open(output_file, 'a') as f:  # Open in append mode
+        print("Appending result to CSV...")
+        f.write(f"{args.checkpoint_path}-{args.remove_layer+1}, {perplexity}\n")

perplexities/perplexities.py

@@ -0,0 +1,179 @@
+# perplexities.py
+# Author: Julie Kallini
+
+# For importing utils
+import sys
+sys.path.append("..")
+
+from transformers import GPT2LMHeadModel
+from gpt2_no_positional_encoding_model import GPT2NoPositionalEncodingLMHeadModel
+from utils import CHECKPOINT_READ_PATH, PERTURBATIONS, BABYLM_DATA_PATH, \
+    PAREN_MODELS, gpt2_original_tokenizer
+from tqdm import tqdm
+from glob import glob
+from numpy.random import default_rng
+import pandas as pd
+import torch
+import itertools
+import argparse
+import os
+
+
+MAX_TRAINING_STEPS = 3000
+CHECKPOINTS = list(range(100, MAX_TRAINING_STEPS+1, 100))
+
+
+def create_attention_mask(token_lists):
+    seq_length = max([len(i) for i in token_lists])
+    batch_size = len(token_lists)
+    mask = torch.full((batch_size, seq_length), 0)
+
+    for i, tokens in enumerate(token_lists):
+        mask[i, 0:len(tokens)] = 1
+
+    return mask
+
+def create_input_ids(token_lists, pad_token_id):
+    padded = zip(*itertools.zip_longest(*token_lists, fillvalue=pad_token_id))
+    return torch.tensor(list(padded))
+
+
+def get_perplexities(model, token_lists, pad_token_id, device="cuda"):
+
+    # Prepare data
+    input_ids = create_input_ids(token_lists, pad_token_id).to(device)
+    labels = input_ids.clone()  # GPT-2 uses input as labels for CLM task
+    attention_mask = create_attention_mask(token_lists).to(device)
+
+    # Forward pass
+    outputs = model(input_ids=input_ids, labels=labels,
+                    attention_mask=attention_mask)
+
+    # The "shifted" nature of labels in GPT-2 (next token prediction)
+    # Shift logits, labels, and attention mask by one position
+    shift_logits = outputs.logits[..., :-1, :].contiguous()
+    shift_labels = labels[..., 1:].contiguous()
+    shift_attention_mask = attention_mask[..., 1:].contiguous()
+
+    # Instantiate loss function with no reduction
+    loss_fct = torch.nn.CrossEntropyLoss(reduction='none')
+
+    # Calculate per-token loss
+    loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)),
+                    shift_labels.view(-1))
+
+    # Reshape back to the original batch size and sequence length
+    loss = loss.view(shift_labels.size())
+
+    # Apply the attention mask - only calculate loss where mask is 1
+    loss = loss * shift_attention_mask
+
+    # Sum the loss over the sequence length, get per-example perplexity
+    per_example_loss = loss.sum(dim=1) / shift_attention_mask.sum(dim=1)
+    return torch.exp(per_example_loss).tolist()
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        prog='Edge probing',
+        description='Edge probing experiments')
+    parser.add_argument('perturbation_type',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=PERTURBATIONS.keys(),
+                        help='Perturbation function used to transform BabyLM dataset')
+    parser.add_argument('test_perturbation_type',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=PERTURBATIONS.keys(),
+                        help='Perturbation function used to transform test BabyLM dataset')
+    parser.add_argument('train_set',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=["100M", "10M"],
+                        help='BabyLM train set')
+    parser.add_argument('random_seed', type=int, help="Random seed")
+    parser.add_argument('paren_model',
+                        default='all',
+                        const='all',
+                        nargs='?',
+                        choices=list(PAREN_MODELS.keys()) + ["randinit"],
+                        help='Parenthesis model')
+    parser.add_argument('-np', '--no_pos_encodings', action='store_true',
+                        help="Train GPT-2 with no positional encodings")
+
+    # Get args
+    args = parser.parse_args()
+    no_pos_encodings_underscore = "_no_positional_encodings" if args.no_pos_encodings else ""
+
+    # Get path to model
+    model = f"babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}_seed{args.random_seed}"
+    model_path = f"{CHECKPOINT_READ_PATH}/babylm_{args.perturbation_type}_{args.train_set}_{args.paren_model}{no_pos_encodings_underscore}/{model}/runs/{model}/checkpoint-"
+
+    # Get perturbed test files
+    test_files = sorted(glob(
+        f"{BABYLM_DATA_PATH}/babylm_data_perturbed/babylm_{args.test_perturbation_type}/babylm_test_affected/*"))
+
+    FILE_SAMPLE_SIZE = 1000
+    rng = default_rng(args.random_seed)
+
+    # Iterate over data files to get perplexity data
+    print("Sampling BabyLM affected test files to extract surprisals...")
+    token_sequences = []
+    for test_file in test_files:
+        print(test_file)
+
+        # Get tokens from test file and subsample
+        f = open(test_file, 'r')
+        file_token_sequences = [
+            [int(s) for s in l.split()] for l in f.readlines()]
+        sample_indices = rng.choice(
+            list(range(len(file_token_sequences))), FILE_SAMPLE_SIZE, replace=False)
+        file_token_sequences = [file_token_sequences[i]
+                                for i in sample_indices]
+        token_sequences.extend(file_token_sequences)
+
+    # For logging/debugging, include decoded sentence
+    test_sents = [gpt2_original_tokenizer.decode(
+        toks) for toks in token_sequences]
+
+    ppl_df = pd.DataFrame({
+        "Sentences": test_sents
+    })
+
+    BATCH_SIZE = 8
+    device = "cuda"
+    for ckpt in CHECKPOINTS:
+        print(f"Checkpoint: {ckpt}")
+
+        # Load model
+        if args.no_pos_encodings:
+            model = GPT2NoPositionalEncodingLMHeadModel.from_pretrained(
+                model_path + str(ckpt)).to(device)
+        else:
+            model = GPT2LMHeadModel.from_pretrained(
+            model_path + str(ckpt)).to(device)
+
+        # Get perplexities
+        perplexities = []
+        for i in tqdm(range(0, len(token_sequences), BATCH_SIZE)):
+            batch = token_sequences[i:i+BATCH_SIZE]
+            ppls = get_perplexities(
+                model, batch, gpt2_original_tokenizer.eos_token_id)
+            perplexities.extend(ppls)
+
+        # Add ppls to df
+        ppl_df[f'Perplexities (ckpt {ckpt})'] = perplexities
+
+    # Write results to CSV
+    directory = f"perplexity_results/{args.perturbation_type}_{args.train_set}{no_pos_encodings_underscore}"
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    file = directory + \
+        f"/{args.paren_model}_seed{args.random_seed}_test_{args.test_perturbation_type}.csv"
+    print(f"Writing results to CSV: {file}")
+    ppl_df.to_csv(file)

perplexities/perplexities_llama.py

@@ -0,0 +1,108 @@
+import torch
+import sys
+import argparse
+import os
+sys.path.append("..")
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling
+from datasets import load_dataset
+from numpy.random import default_rng
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+# MODEL_NAME = "Llama-3.2-1B"
+# MODEL_NAME = "Llama-3.2-1B-500"
+
+MODEL_NAME = "GPT-2"
+MODEL_NAME_SAVE = "GPT2-500"
+FILE_SAMPLE_SIZE = 500
+
+def get_perplexities(model, eval_dataset, batch_size):
+    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+    training_args = TrainingArguments(
+        output_dir="./tmp_trainer",
+        per_device_eval_batch_size=batch_size,
+        fp16=True,
+        report_to="none"
+    )
+
+    trainer = Trainer(model=model, args=training_args, eval_dataset=eval_dataset, data_collator=data_collator)
+    eval_results = trainer.evaluate()
+    print("eval_results:", eval_results)  
+    loss = eval_results['eval_loss']
+    perplexity = torch.exp(torch.tensor(loss)).item()
+    return perplexity
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Calculate perplexity on test dataset.")
+
+    parser.add_argument('perturbation', 
+                        type=str, 
+                        default='reverse_full', 
+                        nargs='?', 
+                        help='Type of perturbation to use.')
+    parser.add_argument('train_set', 
+                        type=str, 
+                        default='test', 
+                        nargs='?', 
+                        help='Dataset size for training.')
+    parser.add_argument('checkpoint_path', 
+                        type=str, 
+                        default='checkpoint-100', 
+                        nargs='?', 
+                        help='Dataset size for training.')
+    parser.add_argument('batch_size', 
+                        type=int, 
+                        default=4, 
+                        nargs='?', 
+                        help='Batch size for evaluation.')
+    parser.add_argument('seed', 
+                        type=int, 
+                        default=0, 
+                        nargs='?', 
+                        help='Random seed.')
+    args = parser.parse_args()
+
+    dataset_name = f"babylm_{args.perturbation}_{args.train_set}_seed{args.seed}"
+    dataset = load_dataset('../train/babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+    test_dataset = dataset['test']  # Load test dataset
+    print(test_dataset)
+    
+    
+    checkpoint_path = f'../train/checkpoints/{MODEL_NAME}/babylm_{args.perturbation}_10M_seed0/runs/{args.checkpoint_path}'
+    
+    rng = default_rng(args.seed)
+    indices = rng.choice(len(test_dataset), FILE_SAMPLE_SIZE, replace=False)
+    sampled_test_dataset = test_dataset.select(indices)
+
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)
+    model = AutoModelForCausalLM.from_pretrained(checkpoint_path)
+    
+    model.eval()
+    if torch.cuda.is_available():
+        model.to('cuda')
+
+    def tokenize_function(examples):
+        return tokenizer(examples['text'], padding="max_length", truncation=True, max_length=1024)
+
+    tokenized_test = sampled_test_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+
+
+    perplexity = get_perplexities(model, tokenized_test, args.batch_size)
+    
+    # Save the result to the specified directory
+    output_directory = f"perplexities_results/{MODEL_NAME_SAVE}"
+    os.makedirs(output_directory, exist_ok=True)
+
+    # Construct the output file path
+    output_file = os.path.join(output_directory, f"{args.perturbation}_{args.batch_size}_{args.seed}.csv")
+
+    # Write the header to the CSV file if it doesn't exist
+    if not os.path.exists(output_file):
+        with open(output_file, 'w') as f:
+            print("Writing header to CSV...")
+            f.write("checkpoint_path, perplexity\n")
+
+    # Append the perplexity result to the CSV file
+    with open(output_file, 'a') as f:  # Open in append mode
+        print("Appending result to CSV...")
+        f.write(f"{args.checkpoint_path}, {perplexity}\n")

requirements.txt

@@ -0,0 +1,94 @@
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.1.1
+cycler==0.12.1
+data==0.4
+emoji==2.8.0
+fonttools==4.45.1
+fsspec==2023.10.0
+funcsigs==1.0.2
+future==0.18.3
+huggingface-hub==0.19.4
+idna==3.6
+iniconfig==2.0.0
+joblib==1.3.2
+kiwisolver==1.4.5
+latex==0.7.0
+matplotlib==3.7.5
+mizani==0.9.3
+nltk==3.8.1
+numpy==1.24.4
+pandas==2.0.3
+patsy==0.5.3
+Pillow==10.1.0
+plotnine==0.12.4
+pluggy==1.3.0
+pluralizer==1.2.0
+protobuf==4.25.1
+pyparsing==3.1.1
+pytest==7.4.3
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.10.3
+requests==2.31.0
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.10.1
+seaborn==0.13.0
+shutilwhich==1.1.0
+stanza==1.6.1
+statsmodels==0.14.0
+tempdir==0.7.1
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+tomli==2.0.1
+torch==2.0.0
+tqdm==4.66.1
+transformers==4.35.2
+triton==2.0.0
+tzdata==2023.3
+urllib3==2.1.0
+asttokens==2.0.5
+comm==0.1.2
+debugpy==1.6.7
+decorator==5.1.1
+exceptiongroup==1.1.0
+executing==1.2.0
+filelock==3.12.2
+gmpy2==2.1.0
+importlib-metadata==6.0.0
+ipykernel==6.23.1
+ipython==8.0.0
+jedi==0.18.2
+Jinja2==3.1.2
+jupyter_client==8.1.0
+jupyter_core==5.1.0
+latexcodec==1.0.0
+MarkupSafe==2.1.2
+matplotlib-inline==0.1.6
+mpmath==1.2.1
+nest-asyncio==1.5.6
+networkx==2.8.6
+packaging==23.0
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+platformdirs==2.5.2
+prompt-toolkit==3.0.30
+psutil==5.9.1
+ptyprocess==0.7.0
+pure-eval==0.2.2
+Pygments==2.15.0
+python-dateutil==2.8.2
+pyzmq==23.0.0
+six==1.16.0
+stack-data==0.6.0
+sympy==1.11.1
+tornado==6.2
+traitlets==5.7.1
+typing_extensions==4.6.0
+wcwidth==0.2.5
+zipp==3.12.0
+
+

requirements_1.txt

@@ -0,0 +1,51 @@
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.2.0
+cycler==0.12.1
+data==0.4
+emoji==2.8.0
+fonttools==4.45.1
+fsspec==2023.10.0
+funcsigs==1.0.2
+future==0.18.3
+huggingface-hub==0.19.4
+idna==3.6
+iniconfig==2.0.0
+joblib==1.3.2
+kiwisolver==1.4.5
+latex==0.7.0
+matplotlib==3.8.2
+mizani==0.9.3
+nltk==3.8.1
+numpy==1.26.2
+pandas==2.1.3
+patsy==0.5.3
+Pillow==10.1.0
+plotnine==0.12.4
+pluggy==1.3.0
+pluralizer==1.2.0
+protobuf==4.25.1
+pyparsing==3.1.1
+pytest==7.4.3
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.10.3
+requests==2.31.0
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.11.4
+seaborn==0.13.0
+shutilwhich==1.1.0
+stanza==1.6.1
+statsmodels==0.14.0
+tempdir==0.7.1
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+tomli==2.0.1
+torch==2.0.0
+tqdm==4.66.1
+transformers==4.35.2
+triton==2.0.0
+tzdata==2023.3
+urllib3==2.1.0