updated models

src/components/model_nlp_intent.py

@@ -2,6 +2,8 @@ import pandas as pd
 import numpy as np
 import tensorflow as tf
 from transformers import DistilBertTokenizer, TFDistilBertForSequenceClassification
+import requests
+from io import BytesIO
 from sklearn.model_selection import train_test_split
 from sklearn.preprocessing import LabelEncoder
 import joblib
@@ -121,22 +123,34 @@ def main():
         logger.info(f"Query: '{query}' -> Intent: {intent} (Confidence: {confidence:.3f})")
 
 
+
+
 def predict_intent(text: str) -> dict:
-    
    
+    model = TFDistilBertForSequenceClassification.from_pretrained(
+        "samithcs/nlp_intent_model", from_tf=True
+    )
+    tokenizer = DistilBertTokenizer.from_pretrained(
+        "samithcs/nlp_intent_model"
+    )
 
-    model_dir = Path(__file__).resolve().parents[2] / "artifacts" / "models" / "nlp_intent"
-    model = TFDistilBertForSequenceClassification.from_pretrained(model_dir / "intent_model")
-    tokenizer = DistilBertTokenizer.from_pretrained(model_dir / "intent_tokenizer")
-    label_encoder = joblib.load(model_dir / "label_encoder.joblib")
+  
+    label_url = "https://huggingface.co/samithcs/nlp_intent_model/resolve/main/label_encoder.joblib"
+    response = requests.get(label_url)
+    label_encoder = joblib.load(BytesIO(response.content))
 
+    
     inputs = tokenizer(text, return_tensors="tf", truncation=True, padding=True, max_length=128)
+
+   
     outputs = model(inputs)
     predicted_class = tf.argmax(outputs.logits, axis=1).numpy()[0]
     intent = label_encoder.inverse_transform([predicted_class])[0]
     confidence = float(tf.nn.softmax(outputs.logits)[0][predicted_class].numpy())
+
     return {"intent": intent, "confidence": confidence}
 
 
+
 if __name__ == "__main__":
     main()

src/components/model_nlp_ner.py

@@ -1,5 +1,8 @@
 import tensorflow as tf
 from transformers import DistilBertTokenizerFast, TFDistilBertForTokenClassification, pipeline
+import requests
+from io import BytesIO
+import numpy as np
 from sklearn.model_selection import train_test_split
 import numpy as np
 import joblib
@@ -178,22 +181,44 @@ def train_ner_model():
     logger.info(f"NER (TF) model, tokenizer, and label map saved to {out_dir}")
 
 
+
+
 def extract_entities_pipeline(text: str) -> dict:
-    model_dir = Path(__file__).resolve().parents[2] / "artifacts" / "models" / "nlp_ner"
-    custom_model = TFDistilBertForTokenClassification.from_pretrained(model_dir / "ner_model")
-    custom_tokenizer = DistilBertTokenizerFast.from_pretrained(model_dir / "ner_tokenizer")
-    label2id = joblib.load(model_dir / "label2id.joblib")
+   
+    custom_model = TFDistilBertForTokenClassification.from_pretrained(
+        "samithcs/nlp_ner", from_tf=True
+    )
+    custom_tokenizer = DistilBertTokenizerFast.from_pretrained("samithcs/nlp_ner")
+
+ 
+    label_url = "https://huggingface.co/samithcs/nlp_ner/resolve/main/label2id.joblib"
+    response = requests.get(label_url)
+    label2id = joblib.load(BytesIO(response.content))
     id2label = {i: t for t, i in label2id.items()}
+
+   
     max_len = 32
     tokens = text.split()
-    encoding = custom_tokenizer([tokens], is_split_into_words=True, return_tensors='tf', padding='max_length', truncation=True, max_length=max_len)
+    encoding = custom_tokenizer(
+        [tokens],
+        is_split_into_words=True,
+        return_tensors='tf',
+        padding='max_length',
+        truncation=True,
+        max_length=max_len
+    )
+
+  
     outputs = custom_model({k: v for k, v in encoding.items() if k != "labels"})
     logits = outputs.logits.numpy()[0]
     pred_ids = np.argmax(logits, axis=-1)
+
+
     custom_entities = {"location": [], "event": []}
     current_loc, current_evt = [], []
     for w, id in zip(tokens, pred_ids[:len(tokens)]):
         label = id2label[id]
+       
         if label == "B-LOC":
             if current_loc:
                 custom_entities["location"].append(" ".join(current_loc))
@@ -205,6 +230,7 @@ def extract_entities_pipeline(text: str) -> dict:
             if current_loc:
                 custom_entities["location"].append(" ".join(current_loc))
                 current_loc = []
+       
         if label == "B-EVENT":
             if current_evt:
                 custom_entities["event"].append(" ".join(current_evt))
@@ -216,17 +242,21 @@ def extract_entities_pipeline(text: str) -> dict:
             if current_evt:
                 custom_entities["event"].append(" ".join(current_evt))
                 current_evt = []
+
     if current_loc:
         custom_entities["location"].append(" ".join(current_loc))
     if current_evt:
         custom_entities["event"].append(" ".join(current_evt))
 
+    
     hf_ner = pipeline("ner", grouped_entities=True, model="dbmdz/bert-large-cased-finetuned-conll03-english")
     hf_results = hf_ner(text)
     hf_locations = [ent['word'] for ent in hf_results if ent['entity_group'] == "LOC"]
 
+   
     all_locations = set(custom_entities["location"]) | set(hf_locations)
     all_events = custom_entities["event"]
+
     return {"location": list(all_locations), "event": all_events}
 
 

src/components/model_risk_predictor.py

@@ -1,5 +1,7 @@
 import pandas as pd
 import numpy as np 
+import requests
+from io import BytesIO
 from sklearn.model_selection import train_test_split
 from sklearn.ensemble import HistGradientBoostingClassifier
 from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score
@@ -154,71 +156,60 @@ def calculate_rule_based_risk(region, days, incidents):
     return min(1.0, rule_risk)
 
 
+
+
 def predict_risk(region: str, days: int = 5, origin=None, destination=None, 
                  event_type=None, incidents=None, shipping_mode=None):
 
     try:
-        import joblib
-        import pandas as pd
-        from pathlib import Path
-
-        model_dir = Path(__file__).resolve().parents[2] / "artifacts" / "models" / "risk_predictor"
-        model_path = model_dir / "hist_gradient_boosting_risk_predictor.joblib"
-        
-        
         if shipping_mode is None:
             shipping_mode = "Standard Class"
-        
-      
+
+        # Calculate rule-based risk (assuming this function exists)
         rule_risk = calculate_rule_based_risk(region, days, incidents or [])
         logger.info(f"Rule-based risk for {region}: {rule_risk:.3f}")
-        
-  
-        ml_risk = 0.40  
-        
-        if model_path.exists():
-            try:
-                model = joblib.load(model_path)
-                logger.debug(f"Loaded ML model from {model_path}")
-
-                data_dir = Path(__file__).resolve().parents[2] / "artifacts" / "data" / "processed"
-                feature_csv_path = data_dir / "supply_chain_disruptions_features.csv"
-                
-                if feature_csv_path.exists():
-                    feature_csv = pd.read_csv(feature_csv_path)
-                    feature_cols = list(model.feature_names_in_) if hasattr(model, "feature_names_in_") else list(feature_csv.columns)
-
-                    reference_row = feature_csv[feature_cols].median()
-
-                    query_dict = {
-                        "region": region,
-                        "days": days,
-                        "origin": origin,
-                        "destination": destination,
-                        "shipping_mode": shipping_mode,
-                    }
-                    
-                    test_features = pd.DataFrame([build_feature_row(feature_cols, query_dict, reference_row)])
-                    ml_risk = float(model.predict_proba(test_features)[0, 1])
-                    logger.info(f"ML model risk for {region}: {ml_risk:.3f}")
-            except Exception as e:
-                logger.warning(f"Could not get ML prediction: {e}")
-        
-        
+
+        ml_risk = 0.40  # default if model fails
+
+        # Load ML model from Hugging Face Hub
+        try:
+            model_url = "https://huggingface.co/samithcs/risk_predictor/resolve/main/hist_gradient_boosting_risk_predictor.joblib"
+            response = requests.get(model_url)
+            model = joblib.load(BytesIO(response.content))
+            logger.debug(f"Loaded ML model from HF Hub: {model_url}")
+
+            # Load reference CSV (optional)
+            data_url = "https://huggingface.co/samithcs/risk_predictor/resolve/main/supply_chain_disruptions_features.csv"
+            feature_csv = pd.read_csv(data_url)
+            feature_cols = list(model.feature_names_in_) if hasattr(model, "feature_names_in_") else list(feature_csv.columns)
+            reference_row = feature_csv[feature_cols].median()
+
+            query_dict = {
+                "region": region,
+                "days": days,
+                "origin": origin,
+                "destination": destination,
+                "shipping_mode": shipping_mode,
+            }
+
+            test_features = pd.DataFrame([build_feature_row(feature_cols, query_dict, reference_row)])
+            ml_risk = float(model.predict_proba(test_features)[0, 1])
+            logger.info(f"ML model risk for {region}: {ml_risk:.3f}")
+
+        except Exception as e:
+            logger.warning(f"Could not get ML prediction: {e}")
+
+        # Combine ML and rule-based risk
         if incidents and len(incidents) > 0:
-           
             final_risk = (ml_risk * 0.40) + (rule_risk * 0.60)
             logger.info(f"Hybrid risk (with incidents): ML={ml_risk:.3f}*0.4 + Rule={rule_risk:.3f}*0.6 = {final_risk:.3f}")
         else:
-           
             final_risk = (ml_risk * 0.70) + (rule_risk * 0.30)
             logger.info(f"Hybrid risk (no incidents): ML={ml_risk:.3f}*0.7 + Rule={rule_risk:.3f}*0.3 = {final_risk:.3f}")
-        
-      
+
         final_risk = float(np.clip(final_risk, 0.0, 1.0))
-        
         return round(final_risk, 2)
-    
+
     except Exception as e:
         logger.error(f"Error in predict_risk: {e}", exc_info=True)
         return 0.50

src/components/model_timeseries_risk.py

@@ -5,22 +5,32 @@ from sklearn.preprocessing import StandardScaler
 from sklearn.model_selection import train_test_split
 from sklearn.utils import class_weight
 import joblib
-from pathlib import Path
+import requests
+from io import BytesIO
 import logging
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
+# URLs for your model and scaler on HF Hub
+model_url = "https://huggingface.co/samithcs/timeseries_risk/resolve/main/lstm_risk_model.keras"
+scaler_url = "https://huggingface.co/samithcs/timeseries_risk/resolve/main/scaler.joblib"
 
-base_dir = Path(__file__).resolve().parents[2]
-data_path = base_dir / "artifacts" / "data" / "processed" / "supply_chain_disruptions_features.csv"
+# Load LSTM model from Hugging Face Hub
+logger.info("Loading LSTM model from Hugging Face Hub...")
+model = tf.keras.models.load_model(model_url)
 
+# Load scaler from Hugging Face Hub
+logger.info("Loading scaler from Hugging Face Hub...")
+response = requests.get(scaler_url)
+scaler = joblib.load(BytesIO(response.content))
 
-df = pd.read_csv(data_path)
+
+# Load dataset (still local CSV if needed)
+df = pd.read_csv("path_to_your_csv/supply_chain_disruptions_features.csv")  # update CSV path if needed
 region_col = "Order City"
 region_name = "Shanghai"
 
-
 df_region = df[df[region_col] == region_name].copy()
 if len(df_region) < 100:
     logger.warning("Region sample is small, upsampling/cropping to 200 rows from full dataset.")
@@ -36,8 +46,7 @@ seq_length = 7
 X_all = df_region[feature_cols].fillna(0).astype(float).values
 y_all = df_region[label_col].fillna(0).astype(int).values
 
-scaler = StandardScaler()
-X_scaled = scaler.fit_transform(X_all)
+X_scaled = scaler.transform(X_all)
 
 X_seq, y_seq = [], []
 for i in range(len(X_scaled) - seq_length):
@@ -51,26 +60,15 @@ if len(X_seq) < 2:
     logger.error("Not enough sequences. Add more data or lower seq_length.")
     exit()
 
-
 test_size = int(0.2 * len(X_seq))
 X_train, X_test = X_seq[:-test_size], X_seq[-test_size:]
 y_train, y_test = y_seq[:-test_size], y_seq[-test_size:]
 
-
 weights = class_weight.compute_class_weight(class_weight="balanced",
                                             classes=np.unique(y_train),
                                             y=y_train)
 class_weight_dict = dict(zip(np.unique(y_train), weights))
 
-
-model = tf.keras.Sequential([
-    tf.keras.layers.Input(shape=(seq_length, len(feature_cols))),
-    tf.keras.layers.LSTM(64, return_sequences=True),
-    tf.keras.layers.Dropout(0.25),
-    tf.keras.layers.LSTM(32),
-    tf.keras.layers.Dropout(0.25),
-    tf.keras.layers.Dense(1, activation="sigmoid")
-])
 model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 
 logger.info("Training LSTM risk model with weighted loss and dropout.")
@@ -80,21 +78,4 @@ model.fit(X_train, y_train, epochs=12, batch_size=8,
 test_loss, test_acc = model.evaluate(X_test, y_test)
 logger.info(f"Test Accuracy: {test_acc:.4f}")
 
-
-model_dir = base_dir / "artifacts" / "models" / "timeseries_risk"
-model_dir.mkdir(parents=True, exist_ok=True)
-model.save(model_dir / "lstm_risk_model.keras")
-joblib.dump(scaler, model_dir / "scaler.joblib")
-logger.info(f"Saved LSTM model and scaler to {model_dir}")
-
-def predict_risk_for_next_day(sequence, threshold=0.5):
-    seq = scaler.transform(sequence)
-    seq_window = np.expand_dims(seq, axis=0)
-    pred_prob = model.predict(seq_window)[0][0]
-    pred_label = int(pred_prob > threshold)
-    logger.info(f"Predicted next-day risk score: {pred_prob:.3f} (region: {region_name}), label: {pred_label}")
-    return pred_prob, pred_label
-
-if X_test.shape[0] > 0:
-    logger.info("Demo prediction for next-day risk using last window of test set:")
-    predict_risk_for_next_day(X_test[0], threshold=0.5)
+logger.info("Finished training/evaluation with model loaded from Hugging Face Hub.")