Upload 4 files

README.md

@@ -1,5 +1,5 @@
 ---
-title: Automatic Assignment of Project Responsibilities to Stakeholders
+title: FIDIC Hierarchical Responsibility Classifier
 emoji: 🏗️
 colorFrom: blue
 colorTo: green
@@ -8,12 +8,11 @@ sdk_version: 1.31.0
 app_file: app_hierarchical.py
 pinned: false
 license: mit
-short_description: Automatic Assignment of Project Responsibilities
 ---
 
 # FIDIC Contract Hierarchical Responsibility Classifier
 
-This application performs **hierarchical classification** of construction contract clauses from construction responsible matrices. It uses an ensemble of machine learning models to:
+This application performs **hierarchical classification** of construction contract clauses from FIDIC Red Book contracts. It uses an ensemble of machine learning models to:
 
 1. **Level 1 - Party Classification:** Identifies the main responsible party (e.g., Employer, Contractor, Engineer)
 2. **Level 2 - Stakeholder Classification:** Determines the specific stakeholder role within that party
@@ -22,7 +21,7 @@ This application performs **hierarchical classification** of construction contra
 
 - 🎯 **Two-level hierarchical classification** for precise responsibility allocation
 - 🤖 **Ensemble learning** using multiple ML algorithms (Naive Bayes, Logistic Regression, SVM, Random Forest, XGBoost, LightGBM, CatBoost)
-- 📊 **Probability scores** for both party and stakeholder predictions
+- 📊 **Joint confidence score** based on party and stakeholder probabilities
 - 📥 **Excel/CSV export** of results
 - 🚀 **Fast processing** with optimized pipelines
 
@@ -45,6 +44,9 @@ Your file should contain at least:
 - A **text column** with contract clause content
 - A **category column** with risk/type classification
 
+Categories may be numeric ids (`0`–`15`) or free-text labels. Missing text cells are
+treated as empty strings and reported in the interface.
+
 Example:
 ```
 texts,category
@@ -57,15 +59,27 @@ texts,category
 This application is based on research from:
 - **Institution:** Department of Civil Engineering, Karadeniz Technical University, Trabzon, Turkey
 - **Authors:** Hayri Burak Altuntaş, Hasan Basri Başağa
-- **Topic:** Hierarchical Classification of Responsibility Allocation
+- **Topic:** Hierarchical Classification of Responsibility Allocation in FIDIC Red Book Contracts
 
 ## Technical Details
 
 - **NLP Pipeline:** Text cleaning, stopword removal, lemmatization
-- **Feature Engineering:** TF-IDF vectorization, categorical encoding
+- **Feature Engineering:** Count vectorization and categorical encoding
 - **Models:** 7 different ML algorithms in soft voting ensemble
-- **Optimization:** Trained on Responsibility Matrices
+- **Optimization:** Trained on FIDIC Red Book contract dataset
+
+The upload limit is 50 MB and 100,000 rows to keep memory usage predictable in a
+Streamlit deployment. Text and category must be selected from different columns.
+
+## Development
+
+Install development dependencies and run the tests:
+
+```bash
+pip install -r requirements-dev.txt
+python -m unittest discover -s tests -v
+```
 
 ## License
 
-MIT License
+MIT License

app_hierarchical.py

@@ -1,4 +1,5 @@
 import os
+import hashlib
 import time
 import streamlit as st
 import pandas as pd
@@ -9,13 +10,30 @@ import re
 import string
 from collections import Counter
 from typing import Dict
+from pathlib import Path
+
+from classification_utils import class_position, combined_confidence, map_category_label
 
 # ------------------------------------------------------------
 # NLP Setup
 # ------------------------------------------------------------
 import nltk
-nltk.download("stopwords", quiet=True)
-nltk.download("wordnet", quiet=True)
+
+
+def ensure_nltk_resource(resource_path: str, package_name: str) -> None:
+    """Download an NLTK resource only when it is not already installed."""
+    try:
+        nltk.data.find(resource_path)
+    except LookupError:
+        if not nltk.download(package_name, quiet=True):
+            raise RuntimeError(
+                f"Required NLTK resource '{package_name}' is unavailable. "
+                f"Install it with: python -m nltk.downloader {package_name}"
+            )
+
+
+ensure_nltk_resource("corpora/stopwords", "stopwords")
+ensure_nltk_resource("corpora/wordnet", "wordnet")
 from nltk.corpus import stopwords
 from nltk.stem import WordNetLemmatizer
 
@@ -81,28 +99,7 @@ def map_prediction_labels(party_value, stakeholder_value):
 
 def map_category_labels(category_value):
     """Convert category ids to English category names."""
-    category_map = {
-        0: "Contract",
-        1: "General",
-        2: "Taxes and Permits",
-        3: "Personnel and Facilities",
-        4: "Site, Office and Administrative Affairs",
-        5: "Transportation and Storage",
-        6: "Occupational Health and Safety",
-        7: "Auxiliary Equipment and Machinery",
-        8: "Documentation and Reporting",
-        9: "Insurance and Guarantees",
-        10: "Water and Energy Supply and Lighting",
-        11: "Labor",
-        12: "Inspection and Testing",
-        13: "Quality Control and Assurance",
-        14: "Design and Engineering",
-        15: "Construction Works",
-    }
-
-    if pd.isna(category_value):
-        return ""
-    return category_map.get(int(category_value), str(category_value))
+    return map_category_label(category_value)
 
 
 def style_results_table(df: pd.DataFrame):
@@ -199,7 +196,10 @@ class HierarchicalClassifier(BaseEstimator, ClassifierMixin):
         for p_val, sub_clf in self.child_clfs_.items():
             child_p = sub_clf.predict_proba(X.loc[:, "texts"])
             cols = [np.where(self.classes_ == c)[0][0] for c in sub_clf.classes_]
-            proba[:, cols] += p_proba[:, p_val][:, None] * child_p
+            party_col = class_position(self.party_clf_.classes_, p_val)
+            if party_col is None:
+                continue
+            proba[:, cols] += p_proba[:, party_col][:, None] * child_p
         return proba
 
     def predict_party(self, X: pd.DataFrame):
@@ -208,21 +208,28 @@ class HierarchicalClassifier(BaseEstimator, ClassifierMixin):
     def predict_proba_party(self, X: pd.DataFrame):
         return self.party_clf_.predict_proba(X)
 
-    def predict_stakeholder(self, X: pd.DataFrame):
-        p_pred = self.party_clf_.predict(X)
+    def predict_stakeholder(self, X: pd.DataFrame, party_predictions=None):
+        p_pred = party_predictions
+        if p_pred is None:
+            p_pred = self.party_clf_.predict(X)
         final = np.empty(X.shape[0], dtype=self.classes_.dtype)
         for p_val in np.unique(p_pred):
             idx = p_pred == p_val
             final[idx] = self.child_clfs_[p_val].predict(X.loc[idx, "texts"])
         return final
 
-    def predict_proba_stakeholder(self, X: pd.DataFrame):
-        p_pred = self.party_clf_.predict(X)
-        results = []
-        for i, p_val in enumerate(p_pred):
-            row_X = X.iloc[[i]]
-            proba = self.child_clfs_[p_val].predict_proba(row_X["texts"])
-            results.append(proba[0])
+    def predict_proba_stakeholder(self, X: pd.DataFrame, party_predictions=None):
+        p_pred = party_predictions
+        if p_pred is None:
+            p_pred = self.party_clf_.predict(X)
+        results = [None] * X.shape[0]
+        for p_val in np.unique(p_pred):
+            positions = np.flatnonzero(p_pred == p_val)
+            probabilities = self.child_clfs_[p_val].predict_proba(
+                X.iloc[positions]["texts"]
+            )
+            for position, probability in zip(positions, probabilities):
+                results[position] = probability
         return results
 
 
@@ -267,11 +274,16 @@ if 'analysis_stats' not in st.session_state:
     st.session_state.analysis_stats = None
 if 'compact_results_df' not in st.session_state:
     st.session_state.compact_results_df = None
+if 'export_results_df' not in st.session_state:
+    st.session_state.export_results_df = None
 
 # ------------------------------------------------------------
 # Model path
 # ------------------------------------------------------------
-MODEL_PATH = "hierarchical_model.pkl"
+APP_DIR = Path(__file__).resolve().parent
+MODEL_PATH = APP_DIR / "hierarchical_model.pkl"
+MAX_UPLOAD_MB = 50
+MAX_UPLOAD_ROWS = 100_000
 
 # ------------------------------------------------------------
 # Load model
@@ -336,12 +348,15 @@ uploaded = st.file_uploader(
 
 # Reset classification state when new file is uploaded
 if uploaded is not None:
-    file_key = f"{uploaded.name}_{uploaded.size}"
+    file_digest = hashlib.sha256(uploaded.getbuffer()).hexdigest()
+    file_key = f"{uploaded.name}_{file_digest}"
     if 'current_file_key' not in st.session_state or st.session_state.current_file_key != file_key:
         st.session_state.current_file_key = file_key
         st.session_state.classification_done = False
         st.session_state.classified_df = None
         st.session_state.analysis_stats = None
+        st.session_state.compact_results_df = None
+        st.session_state.export_results_df = None
 
 # ------------------------------------------------------------
 # File processing
@@ -350,8 +365,8 @@ if uploaded is not None:
     try:
         # Check file size
         file_size = uploaded.size / (1024 * 1024)
-        if file_size > 200:
-            st.error(f"❌ File size ({file_size:.2f} MB) exceeds the 200MB limit.")
+        if file_size > MAX_UPLOAD_MB:
+            st.error(f"❌ File size ({file_size:.2f} MB) exceeds the {MAX_UPLOAD_MB}MB limit.")
             st.stop()
         
         filename = uploaded.name.lower()
@@ -376,6 +391,12 @@ if uploaded is not None:
         if df.empty:
             st.warning("⚠️ The uploaded file is empty.")
             st.stop()
+        if len(df) > MAX_UPLOAD_ROWS:
+            st.error(
+                f"❌ The file contains {len(df):,} rows; the limit is "
+                f"{MAX_UPLOAD_ROWS:,} rows. Split the file and try again."
+            )
+            st.stop()
 
         st.subheader("📄 Uploaded Data")
         st.caption(f"File size: {file_size:.2f} MB | Total rows: {len(df):,} | Total columns: {len(df.columns)}")
@@ -425,6 +446,10 @@ if uploaded is not None:
         )
         
         # ------------------------------------------------------------
+
+        if text_col == cat_col:
+            st.error("❌ Text and category columns must be different.")
+            st.stop()
         
         # Classification button
         if st.button("🚀 Run Hierarchical Classification", type="secondary", use_container_width=True):
@@ -440,7 +465,13 @@ if uploaded is not None:
                 # Prepare input dataframe
                 input_df = df[[text_col, cat_col]].copy()
                 input_df.columns = ["texts", "category"]
-                input_df["texts"] = input_df["texts"].astype(str).apply(clean_text)
+                empty_text_mask = input_df["texts"].isna() | input_df["texts"].astype(str).str.strip().eq("")
+                if empty_text_mask.any():
+                    st.warning(
+                        f"⚠️ {int(empty_text_mask.sum()):,} rows contain empty text and were "
+                        "classified using an empty string."
+                    )
+                input_df["texts"] = input_df["texts"].fillna("").astype(str).apply(clean_text)
                 input_df["keywords"] = input_df["texts"].apply(extract_keywords)
                 input_df = input_df.reset_index(drop=True)
                 
@@ -454,7 +485,9 @@ if uploaded is not None:
                 status_placeholder.info("🔄 Level 2: Classifying stakeholders...")
                 progress_bar.progress(70)
                 
-                stakeholder_predictions = hclf.predict_stakeholder(input_df)
+                stakeholder_predictions = hclf.predict_stakeholder(
+                    input_df, party_predictions=party_predictions
+                )
                 
                 progress_bar.progress(90)
                 elapsed_time = time.time() - start_time
@@ -477,7 +510,9 @@ if uploaded is not None:
                 if hasattr(hclf, "predict_proba_stakeholder"):
                     try:
                         status_placeholder.info("🔄 Calculating stakeholder probabilities...")
-                        stake_probs = hclf.predict_proba_stakeholder(input_df)
+                        stake_probs = hclf.predict_proba_stakeholder(
+                            input_df, party_predictions=party_predictions
+                        )
                         
                         all_stake_classes = set()
                         for clf in hclf.child_clfs_.values():
@@ -513,7 +548,9 @@ if uploaded is not None:
                 else:
                     df["stakeholder_confidence"] = 0.0
 
-                df["overall_confidence"] = df[["party_confidence", "stakeholder_confidence"]].max(axis=1)
+                df["overall_confidence"] = combined_confidence(
+                    df["party_confidence"], df["stakeholder_confidence"]
+                )
                 df["confidence_level"] = df["overall_confidence"].apply(confidence_label)
                 df["needs_review"] = df["overall_confidence"] < 0.6
                 df["keywords"] = input_df["keywords"].tolist()
@@ -526,8 +563,8 @@ if uploaded is not None:
                     df["predicted_party_name"] = [label[0] for label in mapped_labels]
                     df["predicted_stakeholder_name"] = [label[1] for label in mapped_labels]
 
-                compact_df = df[[text_col, cat_col, "predicted_party_name", "predicted_stakeholder_name", "confidence_level", "overall_confidence", "needs_review", "keywords"]].copy()
-                compact_df = compact_df.rename(columns={
+                export_df = df[[text_col, cat_col, "predicted_party_name", "predicted_stakeholder_name", "confidence_level", "overall_confidence", "needs_review", "keywords"]].copy()
+                export_df = export_df.rename(columns={
                     text_col: "Text",
                     cat_col: "Category",
                     "predicted_party_name": "Predicted Party",
@@ -537,16 +574,20 @@ if uploaded is not None:
                     "needs_review": "Needs Review",
                     "keywords": "Keywords",
                 })
-                if "Category" in compact_df.columns:
-                    compact_df["Category"] = compact_df["Category"].apply(map_category_labels)
+                if "Category" in export_df.columns:
+                    export_df["Category"] = export_df["Category"].apply(map_category_labels)
+                export_df["Overall Confidence"] = export_df["Overall Confidence"].round(3)
+                export_df["Needs Review"] = export_df["Needs Review"].map({True: "⚠️ Yes", False: "No"})
+                export_df = export_df.reset_index(drop=True)
+
+                # Preserve complete values in downloads and truncate only the UI preview.
+                compact_df = export_df.copy()
                 compact_df["Text"] = compact_df["Text"].astype(str).str.slice(0, 140)
                 compact_df["Keywords"] = compact_df["Keywords"].astype(str).str.slice(0, 80)
-                compact_df["Overall Confidence"] = compact_df["Overall Confidence"].round(3)
-                compact_df["Needs Review"] = compact_df["Needs Review"].map({True: "⚠️ Yes", False: "No"})
-                compact_df = compact_df.reset_index(drop=True)
 
                 st.session_state.classified_df = df.copy()
                 st.session_state.compact_results_df = compact_df.copy()
+                st.session_state.export_results_df = export_df.copy()
                 st.session_state.classification_done = True
                 
                 num_clauses = len(df)
@@ -644,7 +685,10 @@ if uploaded is not None:
             with col1:
                 try:
                     buffer = BytesIO()
-                    st.session_state.classified_df.to_excel(buffer, index=False, engine="openpyxl")
+                    export_df = st.session_state.export_results_df
+                    if export_df is None:
+                        export_df = st.session_state.compact_results_df
+                    export_df.to_excel(buffer, index=False, engine="openpyxl")
                     buffer.seek(0)
 
                     st.download_button(
@@ -659,7 +703,7 @@ if uploaded is not None:
                     st.error(f"❌ Error creating Excel file: {str(e)}")
                     st.info("Trying alternative CSV download...")
                     
-                    csv_data = st.session_state.classified_df.to_csv(index=False)
+                    csv_data = export_df.to_csv(index=False)
                     st.download_button(
                         "📥 Download results as CSV",
                         data=csv_data,

classification_utils.py

@@ -0,0 +1,50 @@
+import pandas as pd
+import numpy as np
+
+
+CATEGORY_MAP = {
+    0: "Contract",
+    1: "General",
+    2: "Taxes and Permits",
+    3: "Personnel and Facilities",
+    4: "Site, Office and Administrative Affairs",
+    5: "Transportation and Storage",
+    6: "Occupational Health and Safety",
+    7: "Auxiliary Equipment and Machinery",
+    8: "Documentation and Reporting",
+    9: "Insurance and Guarantees",
+    10: "Water and Energy Supply and Lighting",
+    11: "Labor",
+    12: "Inspection and Testing",
+    13: "Quality Control and Assurance",
+    14: "Design and Engineering",
+    15: "Construction Works",
+}
+
+
+def map_category_label(category_value):
+    """Map known numeric category ids while preserving free-text categories."""
+    if pd.isna(category_value):
+        return ""
+
+    try:
+        numeric_value = float(category_value)
+        if numeric_value.is_integer():
+            return CATEGORY_MAP.get(int(numeric_value), str(category_value))
+    except (TypeError, ValueError):
+        pass
+
+    return str(category_value)
+
+
+def combined_confidence(party_confidence, stakeholder_confidence):
+    """Return joint confidence for the two dependent classification stages."""
+    party = pd.to_numeric(party_confidence, errors="coerce").fillna(0.0).clip(0.0, 1.0)
+    stakeholder = pd.to_numeric(stakeholder_confidence, errors="coerce").fillna(0.0).clip(0.0, 1.0)
+    return party * stakeholder
+
+
+def class_position(classes, label):
+    """Return a label's probability-column position without assuming numeric ids."""
+    matches = np.flatnonzero(np.asarray(classes) == label)
+    return int(matches[0]) if matches.size else None

requirements-dev.txt

@@ -0,0 +1 @@
+-r requirements.txt