refactor: update training code, history and remove large checkpoint to fix storage limit

.gitignore

@@ -30,7 +30,7 @@ data/cache/
 
 # Models
 checkpoints/*.pt
-!checkpoints/best.pt
+# !checkpoints/best.pt
 *.pth
 *.ckpt
 !artifacts/*.json

outputs/training_history.json

@@ -1,92 +1,92 @@
 {
   "train_epoch_1": {
-    "summarization_loss": 6.064953740309393,
-    "summarization_rouge_like": 0.12524601610475034,
-    "emotion_loss": 0.36621941694599003,
-    "emotion_f1": 0.1745172753442244,
-    "topic_loss": 0.4304544466319557,
-    "topic_accuracy": 0.8456603510727222,
+    "summarization_loss": 5.023585737518827,
+    "summarization_rouge_like": 0.19371884805954312,
+    "emotion_loss": 0.0821188951971249,
+    "emotion_f1": 0.865718169566,
+    "topic_loss": 0.24917707448061954,
+    "topic_accuracy": 0.9192776539426024,
     "epoch": 1.0
   },
   "val_epoch_1": {
-    "summarization_loss": 5.418145108123086,
-    "summarization_rouge_like": 0.1430367610601125,
-    "emotion_loss": 0.23809201324292817,
-    "emotion_f1": 0.6853306461493459,
-    "topic_loss": 0.34334531054694933,
-    "topic_accuracy": 0.8990873728306403,
+    "summarization_loss": 3.7266472615858954,
+    "summarization_rouge_like": 0.2827026719016518,
+    "emotion_loss": 0.14450823713558134,
+    "emotion_f1": 0.9086874146293125,
+    "topic_loss": 0.21787223087735602,
+    "topic_accuracy": 0.9326002393776182,
     "epoch": 1.0
   },
   "train_epoch_2": {
-    "summarization_loss": 5.313167212766071,
-    "summarization_rouge_like": 0.15972691839488215,
-    "emotion_loss": 0.10728189612031838,
-    "emotion_f1": 0.8470616280293525,
-    "topic_loss": 0.25546857879703205,
-    "topic_accuracy": 0.9178218166620229,
+    "summarization_loss": 3.398382334982861,
+    "summarization_rouge_like": 0.31421210196164595,
+    "emotion_loss": 0.008744604070504772,
+    "emotion_f1": 0.9922616565848632,
+    "topic_loss": 0.12368396144345378,
+    "topic_accuracy": 0.9631060183895236,
     "epoch": 2.0
   },
   "val_epoch_2": {
-    "summarization_loss": 5.047249109141489,
-    "summarization_rouge_like": 0.161552796098159,
-    "emotion_loss": 0.14593230016643607,
-    "emotion_f1": 0.8951226884843632,
-    "topic_loss": 0.31189994352884165,
-    "topic_accuracy": 0.9135996409335727,
+    "summarization_loss": 2.728874285017067,
+    "summarization_rouge_like": 0.3867885960963845,
+    "emotion_loss": 0.20949344621063382,
+    "emotion_f1": 0.9095850804121747,
+    "topic_loss": 0.2887416907434674,
+    "topic_accuracy": 0.9329742669060442,
     "epoch": 2.0
   },
   "train_epoch_3": {
-    "summarization_loss": 5.0241737005072205,
-    "summarization_rouge_like": 0.1736184152131748,
-    "emotion_loss": 0.033076984120505394,
-    "emotion_f1": 0.9598094899305961,
-    "topic_loss": 0.2030463524970495,
-    "topic_accuracy": 0.9360720256338813,
+    "summarization_loss": 2.699047506134568,
+    "summarization_rouge_like": 0.38349341261349945,
+    "emotion_loss": 0.005096756787117961,
+    "emotion_f1": 0.9953213525834805,
+    "topic_loss": 0.07009015341349616,
+    "topic_accuracy": 0.9802800222903316,
     "epoch": 3.0
   },
   "val_epoch_3": {
-    "summarization_loss": 4.856048575138204,
-    "summarization_rouge_like": 0.17376463823910304,
-    "emotion_loss": 0.2033885457060806,
-    "emotion_f1": 0.8938759300324961,
-    "topic_loss": 0.3384457681798222,
-    "topic_accuracy": 0.9156941950927588,
+    "summarization_loss": 2.354555403451446,
+    "summarization_rouge_like": 0.4275408038759501,
+    "emotion_loss": 0.20089952317384335,
+    "emotion_f1": 0.9075279304326329,
+    "topic_loss": 0.4845805834182202,
+    "topic_accuracy": 0.9298324356672651,
     "epoch": 3.0
   },
   "train_epoch_4": {
-    "summarization_loss": 4.838652187544311,
-    "summarization_rouge_like": 0.18201329144700432,
-    "emotion_loss": 0.016137643483381023,
-    "emotion_f1": 0.9827882665002137,
-    "topic_loss": 0.16494447479392424,
-    "topic_accuracy": 0.9490770409584842,
+    "summarization_loss": 2.3750830047009015,
+    "summarization_rouge_like": 0.4200744394095619,
+    "emotion_loss": 0.0037049090056492364,
+    "emotion_f1": 0.9962315410599798,
+    "topic_loss": 0.042221361385891144,
+    "topic_accuracy": 0.9888652828085818,
     "epoch": 4.0
   },
   "val_epoch_4": {
-    "summarization_loss": 4.713183320688674,
-    "summarization_rouge_like": 0.18237954041396615,
-    "emotion_loss": 0.23604378891068725,
-    "emotion_f1": 0.8973917873015594,
-    "topic_loss": 0.33224777920524323,
-    "topic_accuracy": 0.922277079593058,
+    "summarization_loss": 2.198225014299636,
+    "summarization_rouge_like": 0.444635960654823,
+    "emotion_loss": 0.20359252842952202,
+    "emotion_f1": 0.9163175773506461,
+    "topic_loss": 0.5501026207833392,
+    "topic_accuracy": 0.9272890484739676,
     "epoch": 4.0
   },
   "train_epoch_5": {
-    "summarization_loss": 4.698010115737734,
-    "summarization_rouge_like": 0.1875389010633369,
-    "emotion_loss": 0.010522338807794362,
-    "emotion_f1": 0.9898056571265721,
-    "topic_loss": 0.1306824073892107,
-    "topic_accuracy": 0.9599958205628308,
+    "summarization_loss": 2.186419085976007,
+    "summarization_rouge_like": 0.4416556068282783,
+    "emotion_loss": 0.0030099891204739266,
+    "emotion_f1": 0.9964672148443591,
+    "topic_loss": 0.03006078401232904,
+    "topic_accuracy": 0.9925606018389523,
     "epoch": 5.0
   },
   "val_epoch_5": {
-    "summarization_loss": 4.615742755178775,
-    "summarization_rouge_like": 0.18968598032145229,
-    "emotion_loss": 0.2739744641244897,
-    "emotion_f1": 0.8967185377860626,
-    "topic_loss": 0.39091844442157797,
-    "topic_accuracy": 0.9225014961101137,
+    "summarization_loss": 2.114973693461849,
+    "summarization_rouge_like": 0.4553148986859889,
+    "emotion_loss": 0.2197709748711572,
+    "emotion_f1": 0.9121534032496345,
+    "topic_loss": 0.6607796598369469,
+    "topic_accuracy": 0.931178934769599,
     "epoch": 5.0
   }
 }

requirements.txt

@@ -1,15 +1,23 @@
 # requirements.txt
 torch>=2.0.0
-transformers>=4.40.0
-scikit-learn>=1.4.0
+transformers>=4.30.0
+datasets>=2.14.0
+tokenizers>=0.13.0
 numpy>=1.24.0
 pandas>=2.0.0
+scikit-learn>=1.3.0
+matplotlib>=3.7.0
+seaborn>=0.12.0
+nltk>=3.8.0
+tqdm>=4.65.0
+pyyaml>=6.0
+omegaconf>=2.3.0
+tensorboard>=2.13.0
+gradio>=3.35.0
+requests>=2.31.0
+kaggle>=1.5.12
 streamlit>=1.25.0
 plotly>=5.18.0
-fastapi>=0.110.0
-datasets>=4.4.0
-gradio>=4.0.0
-seaborn
-pytest
-matplotlib
-rouge-score>=0.1.2
+faiss-cpu==1.9.0; platform_system != "Windows"
+faiss-cpu==1.9.0; platform_system == "Windows"
+huggingface_hub>=0.19.0

src/inference/factory.py

@@ -6,6 +6,7 @@ from typing import Tuple
 
 import torch
 
+from ..data.preprocessing import TextPreprocessor
 from ..data.tokenization import Tokenizer, TokenizerConfig
 from ..models.factory import ModelConfig, build_multitask_model, load_model_config
 from ..utils.io import load_state
@@ -45,24 +46,23 @@ def create_inference_pipeline(
             )
 
     tokenizer = Tokenizer(resolved_tokenizer_config)
+    
+    # Default to base config if not specified (checkpoint was trained with base config)
+    if model_config_path is None:
+        model_config_path = Path(__file__).resolve().parent.parent.parent / "configs" / "model" / "base.yaml"
+    
     model_config = load_model_config(model_config_path)
     model = build_multitask_model(
         tokenizer,
         num_emotions=labels.emotion_size,
         num_topics=labels.topic_size,
         config=model_config,
+        load_pretrained=False,
     )
+    
+    # Load checkpoint - weights will load separately since factory doesn't tie them
     load_state(model, str(checkpoint))
 
-    # Tie weights manually to ensure decoder output projection matches embeddings
-    # This fixes issues where the output projection might be untrained or mismatched
-    decoder = getattr(model, "decoder", None)
-    output_projection = getattr(decoder, "output_projection", None) if decoder is not None else None
-    embedding = getattr(decoder, "embedding", None) if decoder is not None else None
-
-    if output_projection is not None and embedding is not None:
-        output_projection.weight = embedding.weight
-
     if isinstance(device, torch.device):
         device_str = str(device)
     else:
@@ -80,5 +80,6 @@ def create_inference_pipeline(
         emotion_labels=labels.emotion,
         topic_labels=labels.topic,
         device=device,
+        preprocessor=TextPreprocessor(tokenizer=tokenizer, lowercase=tokenizer.config.lower),
     )
     return pipeline, labels

src/models/factory.py

@@ -69,7 +69,8 @@ def _load_pretrained_weights(encoder: TransformerEncoder, decoder: TransformerDe
     # Load encoder weights
     print("Transferring encoder weights...")
     encoder.embedding.weight.data.copy_(bart.encoder.embed_tokens.weight.data)
-    encoder.pos_encoder.pe.data.copy_(bart.encoder.embed_positions.weight.data.unsqueeze(0))
+    # Skip positional encoding - BART uses learned positions, I use sinusoidal
+    # implementation will work fine with sinusoidal encodings
     
     for i, (custom_layer, bart_layer) in enumerate(zip(encoder.layers, bart.encoder.layers)):
         # Self-attention
@@ -88,19 +89,22 @@ def _load_pretrained_weights(encoder: TransformerEncoder, decoder: TransformerDe
         custom_layer.norm2.weight.data.copy_(bart_layer.final_layer_norm.weight.data)
         custom_layer.norm2.bias.data.copy_(bart_layer.final_layer_norm.bias.data)
         
-        # FFN
-        custom_layer.ffn.fc1.weight.data.copy_(bart_layer.fc1.weight.data)
-        custom_layer.ffn.fc1.bias.data.copy_(bart_layer.fc1.bias.data)
-        custom_layer.ffn.fc2.weight.data.copy_(bart_layer.fc2.weight.data)
-        custom_layer.ffn.fc2.bias.data.copy_(bart_layer.fc2.bias.data)
+        # FFN - use linear1/linear2 
+        custom_layer.ffn.linear1.weight.data.copy_(bart_layer.fc1.weight.data)
+        custom_layer.ffn.linear1.bias.data.copy_(bart_layer.fc1.bias.data)
+        custom_layer.ffn.linear2.weight.data.copy_(bart_layer.fc2.weight.data)
+        custom_layer.ffn.linear2.bias.data.copy_(bart_layer.fc2.bias.data)
     
-    encoder.final_norm.weight.data.copy_(bart.encoder.layernorm_embedding.weight.data)
-    encoder.final_norm.bias.data.copy_(bart.encoder.layernorm_embedding.bias.data)
+    # BART has layernorm_embedding at the input, I have final_norm at output
+    # Copy it to final_norm - not a perfect match but close enough for transfer learning
+    if hasattr(bart.encoder, 'layernorm_embedding'):
+        encoder.final_norm.weight.data.copy_(bart.encoder.layernorm_embedding.weight.data)
+        encoder.final_norm.bias.data.copy_(bart.encoder.layernorm_embedding.bias.data)
     
     # Load decoder weights
     print("Transferring decoder weights...")
     decoder.embedding.weight.data.copy_(bart.decoder.embed_tokens.weight.data)
-    decoder.pos_encoder.pe.data.copy_(bart.decoder.embed_positions.weight.data.unsqueeze(0))
+    # Skip positional encoding - BART uses learned positions, we use sinusoidal
     
     for i, (custom_layer, bart_layer) in enumerate(zip(decoder.layers, bart.decoder.layers)):
         # Self-attention
@@ -131,14 +135,16 @@ def _load_pretrained_weights(encoder: TransformerEncoder, decoder: TransformerDe
         custom_layer.norm3.weight.data.copy_(bart_layer.final_layer_norm.weight.data)
         custom_layer.norm3.bias.data.copy_(bart_layer.final_layer_norm.bias.data)
         
-        # FFN
-        custom_layer.ffn.fc1.weight.data.copy_(bart_layer.fc1.weight.data)
-        custom_layer.ffn.fc1.bias.data.copy_(bart_layer.fc1.bias.data)
-        custom_layer.ffn.fc2.weight.data.copy_(bart_layer.fc2.weight.data)
-        custom_layer.ffn.fc2.bias.data.copy_(bart_layer.fc2.bias.data)
+        # FFN - use linear1/linear2 (not fc1/fc2)
+        custom_layer.ffn.linear1.weight.data.copy_(bart_layer.fc1.weight.data)
+        custom_layer.ffn.linear1.bias.data.copy_(bart_layer.fc1.bias.data)
+        custom_layer.ffn.linear2.weight.data.copy_(bart_layer.fc2.weight.data)
+        custom_layer.ffn.linear2.bias.data.copy_(bart_layer.fc2.bias.data)
     
-    decoder.final_norm.weight.data.copy_(bart.decoder.layernorm_embedding.weight.data)
-    decoder.final_norm.bias.data.copy_(bart.decoder.layernorm_embedding.bias.data)
+    # BART has layernorm_embedding at the input, we have final_norm at output
+    if hasattr(bart.decoder, 'layernorm_embedding'):
+        decoder.final_norm.weight.data.copy_(bart.decoder.layernorm_embedding.weight.data)
+        decoder.final_norm.bias.data.copy_(bart.decoder.layernorm_embedding.bias.data)
     
     print("Pretrained weights loaded successfully!")
 
@@ -149,8 +155,17 @@ def build_multitask_model(
     num_emotions: int,
     num_topics: int,
     config: ModelConfig | None = None,
+    load_pretrained: bool | None = None,
 ) -> MultiTaskModel:
-    """Construct the multitask transformer with heads for the three tasks."""
+    """Construct the multitask transformer with heads for the three tasks.
+    
+    Args:
+        tokenizer: Tokenizer for vocabulary size and pad token
+        num_emotions: Number of emotion classes
+        num_topics: Number of topic classes
+        config: Model architecture configuration
+        load_pretrained: Override config.use_pretrained (for inference to skip loading)
+    """
 
     cfg = config or ModelConfig()
     if not isinstance(num_emotions, int) or num_emotions <= 0:
@@ -179,10 +194,14 @@ def build_multitask_model(
         pad_token_id=tokenizer.pad_token_id,
     )
     
-    # Load pretrained weights if requested
-    if cfg.use_pretrained:
+    # Load pretrained weights if requested (but allow override for inference)
+    should_load = cfg.use_pretrained if load_pretrained is None else load_pretrained
+    if should_load:
         _load_pretrained_weights(encoder, decoder, cfg.pretrained_model_name)
 
+    # NOTE: Weight tying disabled because the current checkpoint was trained without it
+    # For NEW training runs, uncomment this line to enable proper weight tying:
+    # decoder.output_projection.weight = decoder.embedding.weight
     
     model = MultiTaskModel(encoder=encoder, decoder=decoder, decoder_outputs_logits=True)
     model.add_head(

start_training.bat

@@ -0,0 +1,4 @@
+@echo off
+cd /d C:\Users\olive\OneDrive\Desktop\LexiMind\LexiMind
+call C:\Users\olive\OneDrive\Desktop\LexiMind\.venv\Scripts\activate.bat
+python scripts\train.py --training-config configs\training\default.yaml --model-config configs\model\base.yaml --data-config configs\data\datasets.yaml --device cuda > logs\training_live.log 2>&1