Sync from GitHub via hub-sync
Browse files- src/dytr/core/model.py +13 -7
src/dytr/core/model.py
CHANGED
|
@@ -603,7 +603,7 @@ class DynamicTransformer(nn.Module):
|
|
| 603 |
self.ewc_penalties[task_name] = ewc
|
| 604 |
print(f" ✓ EWC stored for: {task_name}")
|
| 605 |
|
| 606 |
-
def
|
| 607 |
"""Save the entire model to disk including EWC and Replay Buffer."""
|
| 608 |
self.config.tasks = self.current_tasks
|
| 609 |
|
|
@@ -641,7 +641,7 @@ class DynamicTransformer(nn.Module):
|
|
| 641 |
print(f" Saved EWC for tasks: {list(ewc_data.keys())}")
|
| 642 |
|
| 643 |
@classmethod
|
| 644 |
-
def
|
| 645 |
"""Load a saved model from disk with full state restoration."""
|
| 646 |
try:
|
| 647 |
if not device:
|
|
@@ -665,6 +665,7 @@ class DynamicTransformer(nn.Module):
|
|
| 665 |
# Load model weights with strict=False (handles rotary cache)
|
| 666 |
model.load_state_dict(state['model_state'], strict=False)
|
| 667 |
|
|
|
|
| 668 |
# Restore training state
|
| 669 |
model.current_tasks = state.get('current_tasks', {})
|
| 670 |
model.training_step = state.get('training_step', 0)
|
|
@@ -707,7 +708,7 @@ class DynamicTransformer(nn.Module):
|
|
| 707 |
import traceback
|
| 708 |
traceback.print_exc()
|
| 709 |
return None
|
| 710 |
-
def
|
| 711 |
"""Save the entire model to disk."""
|
| 712 |
self.config.tasks = self.current_tasks
|
| 713 |
state = {
|
|
@@ -723,23 +724,28 @@ class DynamicTransformer(nn.Module):
|
|
| 723 |
torch.save(state, path)
|
| 724 |
|
| 725 |
@classmethod
|
| 726 |
-
def
|
| 727 |
"""Load a saved model from disk."""
|
| 728 |
try:
|
| 729 |
if not device:
|
| 730 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 731 |
-
|
| 732 |
state = torch.load(path, map_location=device, weights_only=False)
|
| 733 |
config = state["config"]
|
| 734 |
config.device = device
|
| 735 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 736 |
model = cls(config)
|
| 737 |
model.load_state_dict(state["model_state"], strict=False)
|
| 738 |
model.current_tasks = state.get("current_tasks", {})
|
| 739 |
model.training_step = state.get("training_step", 0)
|
| 740 |
model.best_val_loss = state.get("best_val_loss", float("inf"))
|
| 741 |
model.steps_without_improvement = state.get("steps_without_improvement", 0)
|
| 742 |
-
|
| 743 |
return model.to(config.device)
|
| 744 |
except Exception as e:
|
| 745 |
print(f"Error loading model: {e}")
|
|
|
|
| 603 |
self.ewc_penalties[task_name] = ewc
|
| 604 |
print(f" ✓ EWC stored for: {task_name}")
|
| 605 |
|
| 606 |
+
def save_model_all(self, path: str):
|
| 607 |
"""Save the entire model to disk including EWC and Replay Buffer."""
|
| 608 |
self.config.tasks = self.current_tasks
|
| 609 |
|
|
|
|
| 641 |
print(f" Saved EWC for tasks: {list(ewc_data.keys())}")
|
| 642 |
|
| 643 |
@classmethod
|
| 644 |
+
def load_model_all(cls, path: str, device: Optional[str] = None):
|
| 645 |
"""Load a saved model from disk with full state restoration."""
|
| 646 |
try:
|
| 647 |
if not device:
|
|
|
|
| 665 |
# Load model weights with strict=False (handles rotary cache)
|
| 666 |
model.load_state_dict(state['model_state'], strict=False)
|
| 667 |
|
| 668 |
+
|
| 669 |
# Restore training state
|
| 670 |
model.current_tasks = state.get('current_tasks', {})
|
| 671 |
model.training_step = state.get('training_step', 0)
|
|
|
|
| 708 |
import traceback
|
| 709 |
traceback.print_exc()
|
| 710 |
return None
|
| 711 |
+
def save_model(self, path: str):
|
| 712 |
"""Save the entire model to disk."""
|
| 713 |
self.config.tasks = self.current_tasks
|
| 714 |
state = {
|
|
|
|
| 724 |
torch.save(state, path)
|
| 725 |
|
| 726 |
@classmethod
|
| 727 |
+
def load_model(cls, path: str, device: Optional[str] = None):
|
| 728 |
"""Load a saved model from disk."""
|
| 729 |
try:
|
| 730 |
if not device:
|
| 731 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
|
|
| 732 |
state = torch.load(path, map_location=device, weights_only=False)
|
| 733 |
config = state["config"]
|
| 734 |
config.device = device
|
| 735 |
+
# Detect and fix max_seq_len mismatch for rotary cache
|
| 736 |
+
if 'model_state' in state:
|
| 737 |
+
for key in state['model_state'].keys():
|
| 738 |
+
if 'rotary.cos_cached' in key:
|
| 739 |
+
saved_seq_len = state['model_state'][key].shape[0]
|
| 740 |
+
if saved_seq_len != config.max_seq_len:
|
| 741 |
+
config.max_seq_len = saved_seq_len
|
| 742 |
+
break
|
| 743 |
model = cls(config)
|
| 744 |
model.load_state_dict(state["model_state"], strict=False)
|
| 745 |
model.current_tasks = state.get("current_tasks", {})
|
| 746 |
model.training_step = state.get("training_step", 0)
|
| 747 |
model.best_val_loss = state.get("best_val_loss", float("inf"))
|
| 748 |
model.steps_without_improvement = state.get("steps_without_improvement", 0)
|
|
|
|
| 749 |
return model.to(config.device)
|
| 750 |
except Exception as e:
|
| 751 |
print(f"Error loading model: {e}")
|