Reduce max sequence length to 250 as PhoBERT allows maximum 256 tokens

distill_bert_to_lstm.py

@@ -44,7 +44,7 @@ def main():
     # BERT model arguments
     parser.add_argument("--bert_model", type=str, default="bert-base-uncased", help="BERT model to use")
     parser.add_argument("--bert_model_path", type=str, required=True, help="Path to saved BERT model weights")
-    parser.add_argument("--max_seq_length", type=int, default=512, help="Maximum sequence length")
+    parser.add_argument("--max_seq_length", type=int, default=250, help="Maximum sequence length (e.g., 250 for PhoBERT as PhoBERT allows max_position_embeddings=258)")
     
     # LSTM model arguments
     parser.add_argument("--embedding_dim", type=int, default=300, help="Dimension of word embeddings in LSTM")

inference_example.py

@@ -11,7 +11,7 @@ if __name__ == "__main__":
     parser.add_argument("--data_path", type=str, required=True, help="Path to the dataset")
     parser.add_argument("--bert_model", type=str, default="bert-base-uncased", help="Pre-trained BERT model name")
     parser.add_argument("--model_path", type=str, required=True, help="Path to the trained model")
-    parser.add_argument("--max_seq_length", type=int, default=512, help="Maximum sequence length for BERT")
+    parser.add_argument("--max_seq_length", type=int, default=250, help="Maximum sequence length for BERT (e.g., 250 for PhoBERT as PhoBERT allows max_position_embeddings=258)")
     parser.add_argument("--batch_size", type=int, default=32, help="Batch size for training and evaluation")
     parser.add_argument("--num_classes", type=int, required=True, help="Number of classes for classification")
     parser.add_argument("--text_column", type=str, default="text", help="Column name for text data")

inference_lstm.py

@@ -16,7 +16,7 @@ if __name__ == "__main__":
     parser.add_argument("--data_path", type=str, required=True, help="Path to the dataset")
     parser.add_argument("--bert_model", type=str, default="bert-base-uncased", help="BERT model name or path used for distillation (as we'll use its tokenizer)")
     parser.add_argument("--model_path", type=str, required=True, help="Path to the trained model")
-    parser.add_argument("--max_seq_length", type=int, default=512, help="Maximum sequence length for LSTM")
+    parser.add_argument("--max_seq_length", type=int, default=250, help="Maximum sequence length for LSTM")
     parser.add_argument("--batch_size", type=int, default=32, help="Batch size for training and evaluation")
     parser.add_argument("--num_classes", type=int, required=True, help="Number of classes for classification")
     parser.add_argument("--text_column", type=str, default="text", help="Column name for text data")

train.py

@@ -40,7 +40,7 @@ def main():
     parser.add_argument("--bert_model", type=str, default="bert-base-uncased", 
                         help="BERT model to use (e.g., bert-base-uncased, bert-large-uncased)")
     parser.add_argument("--num_classes", type=int, required=True, help="Number of classes to predict")
-    parser.add_argument("--max_length", type=int, default=512, help="Maximum sequence length")
+    parser.add_argument("--max_length", type=int, default=250, help="Maximum sequence length (PhoBERT has 258 max_position_embeddings so we choose 250)")
     parser.add_argument("--dropout", type=float, default=0.1, help="Dropout probability")
     
     # Training arguments