testing, removed obsolete files.

model_loader.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python3
-"""Install CalamanCy model properly"""
-
-import sys
-import subprocess
-
-def install_calamancy_model():
-    """Install the Filipino NLP model"""
-    try:
-        import calamancy
-        print("CalamanCy found, loading model...")
-        # This will download and cache the model
-        nlp = calamancy.load("tl_calamancy_md-0.2.0")
-        print("✅ Model loaded successfully!")
-        return True
-    except Exception as e:
-        print(f"Error loading model: {e}")
-        print("Attempting to install via spacy...")
-        try:
-            subprocess.run([
-                sys.executable, "-m", "spacy", "download", "tl_calamancy_md"
-            ], check=True)
-            print("✅ Model installed via spacy!")
-            return True
-        except Exception as e2:
-            print(f"Installation failed: {e2}")
-            return False
-
-if __name__ == "__main__":
-    success = install_calamancy_model()
-    sys.exit(0 if success else 1)

models/Corpus_cleaner.py

@@ -4,11 +4,11 @@ import re
 
 input_file = "corpus.csv"
 output_file = "corpus_clean.csv"
-expected_cols = 3  # adjust based on your CSV structure
+expected_cols = 3  
 
 clean_rows = []
 
-# Step 1: Read the CSV manually to handle bad lines
+# Read the CSV manually to handle bad lines
 with open(input_file, encoding="utf-8") as f:
     reader = csv.reader(f)
     header = next(reader)
@@ -28,13 +28,15 @@ with open(input_file, encoding="utf-8") as f:
                 # Skip rows with too few columns
                 print(f"Skipped line {i} (too few columns): {row}")
 
-# Step 2: Convert to DataFrame
+# Convert to DataFrame
 df = pd.DataFrame(clean_rows[1:], columns=clean_rows[0])
 
-# Step 3: Clean the text column for NLP
-# Replace 'text' with the column name containing your text
+# Clean the text column for NLP
+
+# assign the column containing the sentences to text_col
 text_col = 'text'  
 
+# clean sentences -- uniformed
 def clean_text(text):
     text = text.lower()  # lowercase
     text = re.sub(r'\s+', ' ', text)  # remove extra whitespace
@@ -44,7 +46,7 @@ def clean_text(text):
 
 df[text_col] = df[text_col].apply(clean_text)
 
-# Step 4: Save clean CSV
+# Save clean CSV
 df.to_csv(output_file, index=False, encoding="utf-8")
 print(f"Clean CSV saved to {output_file}")
 

models/feature_extractor.py

@@ -3,7 +3,7 @@ import sys
 
 import pandas as pd
 
-# Support running this file both as a module and as a direct script.
+# Support running the file both as a module and as a direct script.
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 BACKEND_DIR = os.path.abspath(os.path.join(CURRENT_DIR, ".."))
 if BACKEND_DIR not in sys.path:
@@ -11,7 +11,6 @@ if BACKEND_DIR not in sys.path:
 
 from feature_core import extract_features, load_nlp_model  # noqa: E402
 
-
 # Load corpus
 _df = pd.read_csv("corpus_with_group.csv")
 

models/random_forest.py

@@ -1,3 +1,5 @@
+# THIS FILE IS NOT CRUCIAL TO THE CURRENT SYSTEM. USED ONLY AS A BENCHMARK.
+
 import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
@@ -46,7 +48,7 @@ preprocessor = ColumnTransformer(
     remainder="passthrough"
 )
 
-# First, let's split the data
+# split the data
 X_train, X_test, y_train, y_test = train_test_split(
     X, y, test_size=0.2, random_state=42, stratify=y
 )
@@ -56,7 +58,7 @@ print(f"Test set size: {len(X_test)}")
 print(f"Training set class distribution: {np.bincount(y_train)}")
 print(f"Test set class distribution: {np.bincount(y_test)}")
 
-# Train a model with your best params from previous run
+# Train a model with best params from previous run
 print("\n=== ANALYZING FEATURE IMPORTANCE ===")
 best_params = {
     'max_depth': 10,
@@ -152,7 +154,7 @@ smaller_grid = {
     "classifier__class_weight": ["balanced"]
 }
 
-cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)  # Using 3-fold for speed
+cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)  # Used 5-fold based on planned methodology
 f1_macro_scorer = make_scorer(f1_score, average="macro")
 
 grid_search = GridSearchCV(
@@ -218,7 +220,7 @@ for i, (name, score) in enumerate(results.items()):
 plt.ylim(0, 1)
 plt.show()
 
-# Option 3: Try ensemble of best models
+# Try ensemble of best models
 print("\n=== OPTION 3: ENSEMBLE MODEL ===")
 
 # Create a few different models