Upload folder using huggingface_hub

scripts/train_from_files.py

@@ -24,6 +24,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from training.train_regression import ResolutionTimePredictor
 from training.train_nlp import SeverityClassifier, IssueTypeClassifier, SimpleSummarizer
 from training.train_tfidf_classifier import train_tfidf_classifier
+from data.root_cause_service import RootCauseService
 
 def clean_header(header):
     """Normalize headers to match what training scripts expect"""
@@ -55,14 +56,34 @@ def load_local_csv(filepath, sheet_name):
     return data
 
 def main():
-    # File paths
-    cgo_path = "/Users/nrzngr/Desktop/ai-model/Acc Data 2 - Irregularity Report - Manual for Dashboard - CGO (1).csv"
-    non_cargo_path = "/Users/nrzngr/Desktop/ai-model/Acc Data 2 - Irregularity Report - Manual for Dashboard - NON CARGO.csv"
-    
-    # Load and combine data
-    cgo_data = load_local_csv(cgo_path, "CGO")
-    non_cargo_data = load_local_csv(non_cargo_path, "NON CARGO")
-    all_data = cgo_data + non_cargo_data
+    # Preferred local cache path (single combined file)
+    cache_path = os.path.join(os.path.dirname(__file__), "..", "data", "training_data_cache.csv")
+    cache_path = os.path.abspath(cache_path)
+
+    all_data = []
+
+    if os.path.exists(cache_path):
+        logger.info(f"Using cached training data: {cache_path}")
+        df = pd.read_csv(cache_path)
+        df = df.fillna("")
+        all_data = df.to_dict(orient="records")
+        # Ensure _row_id and _sheet_name exist for downstream components
+        for i, r in enumerate(all_data):
+            if "_row_id" not in r:
+                r["_row_id"] = f"ALL_{i+2}"
+            if "_sheet_name" not in r:
+                r["_sheet_name"] = "ALL"
+    else:
+        # Fallback: separate CGO and NON CARGO CSVs (if provided locally)
+        cgo_path = "/Users/nrzngr/Desktop/ai-model/Acc Data 2 - Irregularity Report - Manual for Dashboard - CGO (1).csv"
+        non_cargo_path = "/Users/nrzngr/Desktop/ai-model/Acc Data 2 - Irregularity Report - Manual for Dashboard - NON CARGO.csv"
+        cgo_data = []
+        non_cargo_data = []
+        if os.path.exists(cgo_path):
+            cgo_data = load_local_csv(cgo_path, "CGO")
+        if os.path.exists(non_cargo_path):
+            non_cargo_data = load_local_csv(non_cargo_path, "NON CARGO")
+        all_data = cgo_data + non_cargo_data
     
     logger.info(f"Total records for training: {len(all_data)}")
     
@@ -80,6 +101,12 @@ def main():
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     predictor.save(os.path.join(model_dir_reg, f"resolution_predictor_{timestamp}.pkl"))
     predictor.save(os.path.join(model_dir_reg, "resolution_predictor_latest.pkl"))
+    # Save metrics JSON alongside latest
+    try:
+        with open(os.path.join(model_dir_reg, "resolution_predictor_latest_metrics.json"), "w") as f:
+            json.dump(reg_metrics, f, indent=2, default=str)
+    except Exception as e:
+        logger.warning(f"Failed to save regression metrics JSON: {e}")
     
     # 2. Train NLP BERT Models
     # logger.info("\n" + "="*30 + " Training NLP BERT Models " + "="*30)
@@ -123,6 +150,14 @@ def main():
         "total_samples": len(all_data)
     }
     
+    # 5. Train Root Cause Classifier (TF-IDF + LogisticRegression)
+    try:
+        rc_service = RootCauseService()
+        rc_metrics = rc_service.train_from_data(all_data)
+        summary_metrics["root_cause"] = rc_metrics
+    except Exception as e:
+        logger.warning(f"Failed training root cause classifier: {e}")
+
     with open("models/training_summary.json", "w") as f:
         json.dump(summary_metrics, f, indent=2, default=str)