alsubari commited on
Commit
0835cc1
·
verified ·
1 Parent(s): 3e6404f

Sync from GitHub via hub-sync

Browse files
Files changed (1) hide show
  1. src/dytr/core/model.py +13 -7
src/dytr/core/model.py CHANGED
@@ -603,7 +603,7 @@ class DynamicTransformer(nn.Module):
603
  self.ewc_penalties[task_name] = ewc
604
  print(f" ✓ EWC stored for: {task_name}")
605
 
606
- def save_model(self, path: str):
607
  """Save the entire model to disk including EWC and Replay Buffer."""
608
  self.config.tasks = self.current_tasks
609
 
@@ -641,7 +641,7 @@ class DynamicTransformer(nn.Module):
641
  print(f" Saved EWC for tasks: {list(ewc_data.keys())}")
642
 
643
  @classmethod
644
- def load_model(cls, path: str, device: Optional[str] = None):
645
  """Load a saved model from disk with full state restoration."""
646
  try:
647
  if not device:
@@ -665,6 +665,7 @@ class DynamicTransformer(nn.Module):
665
  # Load model weights with strict=False (handles rotary cache)
666
  model.load_state_dict(state['model_state'], strict=False)
667
 
 
668
  # Restore training state
669
  model.current_tasks = state.get('current_tasks', {})
670
  model.training_step = state.get('training_step', 0)
@@ -707,7 +708,7 @@ class DynamicTransformer(nn.Module):
707
  import traceback
708
  traceback.print_exc()
709
  return None
710
- def save_model_old(self, path: str):
711
  """Save the entire model to disk."""
712
  self.config.tasks = self.current_tasks
713
  state = {
@@ -723,23 +724,28 @@ class DynamicTransformer(nn.Module):
723
  torch.save(state, path)
724
 
725
  @classmethod
726
- def load_model_old(cls, path: str, device: Optional[str] = None):
727
  """Load a saved model from disk."""
728
  try:
729
  if not device:
730
  device = "cuda" if torch.cuda.is_available() else "cpu"
731
-
732
  state = torch.load(path, map_location=device, weights_only=False)
733
  config = state["config"]
734
  config.device = device
735
-
 
 
 
 
 
 
 
736
  model = cls(config)
737
  model.load_state_dict(state["model_state"], strict=False)
738
  model.current_tasks = state.get("current_tasks", {})
739
  model.training_step = state.get("training_step", 0)
740
  model.best_val_loss = state.get("best_val_loss", float("inf"))
741
  model.steps_without_improvement = state.get("steps_without_improvement", 0)
742
-
743
  return model.to(config.device)
744
  except Exception as e:
745
  print(f"Error loading model: {e}")
 
603
  self.ewc_penalties[task_name] = ewc
604
  print(f" ✓ EWC stored for: {task_name}")
605
 
606
+ def save_model_all(self, path: str):
607
  """Save the entire model to disk including EWC and Replay Buffer."""
608
  self.config.tasks = self.current_tasks
609
 
 
641
  print(f" Saved EWC for tasks: {list(ewc_data.keys())}")
642
 
643
  @classmethod
644
+ def load_model_all(cls, path: str, device: Optional[str] = None):
645
  """Load a saved model from disk with full state restoration."""
646
  try:
647
  if not device:
 
665
  # Load model weights with strict=False (handles rotary cache)
666
  model.load_state_dict(state['model_state'], strict=False)
667
 
668
+
669
  # Restore training state
670
  model.current_tasks = state.get('current_tasks', {})
671
  model.training_step = state.get('training_step', 0)
 
708
  import traceback
709
  traceback.print_exc()
710
  return None
711
+ def save_model(self, path: str):
712
  """Save the entire model to disk."""
713
  self.config.tasks = self.current_tasks
714
  state = {
 
724
  torch.save(state, path)
725
 
726
  @classmethod
727
+ def load_model(cls, path: str, device: Optional[str] = None):
728
  """Load a saved model from disk."""
729
  try:
730
  if not device:
731
  device = "cuda" if torch.cuda.is_available() else "cpu"
 
732
  state = torch.load(path, map_location=device, weights_only=False)
733
  config = state["config"]
734
  config.device = device
735
+ # Detect and fix max_seq_len mismatch for rotary cache
736
+ if 'model_state' in state:
737
+ for key in state['model_state'].keys():
738
+ if 'rotary.cos_cached' in key:
739
+ saved_seq_len = state['model_state'][key].shape[0]
740
+ if saved_seq_len != config.max_seq_len:
741
+ config.max_seq_len = saved_seq_len
742
+ break
743
  model = cls(config)
744
  model.load_state_dict(state["model_state"], strict=False)
745
  model.current_tasks = state.get("current_tasks", {})
746
  model.training_step = state.get("training_step", 0)
747
  model.best_val_loss = state.get("best_val_loss", float("inf"))
748
  model.steps_without_improvement = state.get("steps_without_improvement", 0)
 
749
  return model.to(config.device)
750
  except Exception as e:
751
  print(f"Error loading model: {e}")