2025.03.17.post1

.github/workflows/ossar.yml

@@ -31,6 +31,10 @@ jobs:
     runs-on: windows-latest
 
     steps:
+
+    - name: Enable long paths in Git
+      run: git config --system core.longpaths true
+
     - name: Checkout repository
       uses: actions/checkout@v4
 

app_main_hf.py

@@ -1,13 +1,30 @@
 import streamlit as st
 import os
+import asyncio
 import sys
+
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+else:
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        asyncio.set_event_loop(asyncio.new_event_loop())
+
+
 import joblib
+import importlib
 import importlib.util
 
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), )))
 
+from imports import *
+
+
+
+
 from dashboard import show_dashboard
-from sentiment_analysis.sentiment_analysis import show_sentiment_analysis
+from sentiment_analysis.sentiment_analysis_main import show_sentiment_analysis
 from emotion_analysis import show_emotion_analysis
 # from text_transformation import show_text_transformation
 

imports.py

@@ -9,5 +9,11 @@ import json
 import gc
 import psutil
 import os
+import importlib
 import importlib.util
-import sys
+import asyncio
+import sys
+import pytorch_lightning as pl
+
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), )))

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tachygraphy-microtext-analysis-and-normalization"
-version = "2025.03.16.post3"
+version = "2025.03.17.post1"
 description = ""
 authors = [
     { name = "Archisman Karmakar", email = "92569441+ArchismanKarmakar@users.noreply.github.com" },
@@ -192,6 +192,8 @@ dependencies = [
     "zict (>=3.0.0,<4.0.0)",
     "zipp (>=3.21.0,<4.0.0)",
     "zstandard (>=0.23.0,<0.24.0)",
+    "asyncio (>=3.4.3,<4.0.0)",
+    "pytorch-lightning (>=2.5.0.post0,<3.0.0)",
 ]
 
 

pyprojectOLD.toml

@@ -0,0 +1,202 @@
+[project]
+name = "tachygraphy-microtext-analysis-and-normalization"
+version = "2025.03.16.post3"
+description = ""
+authors = [
+    { name = "Archisman Karmakar", email = "92569441+ArchismanKarmakar@users.noreply.github.com" },
+]
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "numpy (==2.1.3)",
+    "torch (>=2.6.0,<3.0.0)",
+    "torchvision (>=0.21.0,<0.22.0)",
+    "tensorflow (>=2.19.0,<3.0.0)",
+    "pandas (>=2.2.3,<3.0.0)",
+    "openpyxl (>=3.1.5,<4.0.0)",
+    "streamlit (>=1.43.2,<2.0.0)",
+    "transformers (>=4.49.0,<5.0.0)",
+    "datasets (>=3.4.0,<4.0.0)",
+    "autograd (>=1.7.0,<2.0.0)",
+    "ipykernel (>=6.29.5,<7.0.0)",
+    "matplotlib (>=3.10.1,<4.0.0)",
+    "plotly (>=6.0.0,<7.0.0)",
+    "importlib (>=1.0.4,<2.0.0)",
+    "joblib (>=1.4.2,<2.0.0)",
+    "accelerate (>=1.5.2,<2.0.0)",
+    "peft (>=0.14.0,<0.15.0)",
+    "mpi4py (>=4.0.3,<5.0.0)",
+    "tiktoken (>=0.9.0,<0.10.0)",
+    "sentencepiece (>=0.2.0,<0.3.0)",
+    "bs4 (>=0.0.2,<0.0.3)",
+    "emoji (>=2.14.1,<3.0.0)",
+    "safetensors (>=0.5.3,<0.6.0)",
+    "lxml (>=5.3.1,<6.0.0)",
+    "evaluate (>=0.4.3,<0.5.0)",
+    "scipy (>=1.15.2,<2.0.0)",
+    "sacrebleu (>=2.5.1,<3.0.0)",
+    "rouge-score (>=0.1.2,<0.2.0)",
+    "scikit-learn (>=1.6.1,<2.0.0)",
+    "regex (>=2024.11.6,<2025.0.0)",
+    "pywin32 (>=309,<310)",
+    "pycurl (>=7.45.6,<8.0.0)",
+    "pillow (>=11.1.0,<12.0.0)",
+    "diffusers (>=0.32.2,<0.33.0)",
+    "onnx (>=1.17.0,<2.0.0)",
+    "keras (>=3.9.0,<4.0.0)",
+    "h2o (>=3.46.0.6,<4.0.0.0)",
+    "distributed (>=2025.2.0,<2026.0.0)",
+    "absl-py (>=2.1.0,<3.0.0)",
+    "aiohappyeyeballs (>=2.6.1,<3.0.0)",
+    "aiohttp (>=3.11.13,<4.0.0)",
+    "aiosignal (>=1.3.2,<2.0.0)",
+    "altair (>=5.5.0,<6.0.0)",
+    "anyio (>=4.8.0,<5.0.0)",
+    "asttokens (>=3.0.0,<4.0.0)",
+    "astunparse (>=1.6.3,<2.0.0)",
+    "attrs (>=25.3.0,<26.0.0)",
+    "beautifulsoup4 (>=4.13.3,<5.0.0)",
+    "blinker (>=1.9.0,<2.0.0)",
+    "build (>=1.2.2.post1,<2.0.0)",
+    "cachecontrol (>=0.14.2,<0.15.0)",
+    "cachetools (>=5.5.2,<6.0.0)",
+    "certifi (>=2025.1.31,<2026.0.0)",
+    "charset-normalizer (>=3.4.1,<4.0.0)",
+    "click (>=8.1.8,<9.0.0)",
+    "cloudpickle (>=3.1.1,<4.0.0)",
+    "colorama (>=0.4.6,<0.5.0)",
+    "comm (>=0.2.2,<0.3.0)",
+    "contourpy (>=1.3.1,<2.0.0)",
+    "cycler (>=0.12.1,<0.13.0)",
+    "dask (>=2025.2.0,<2026.0.0)",
+    "debugpy (>=1.8.13,<2.0.0)",
+    "decorator (>=5.2.1,<6.0.0)",
+    "distlib (>=0.3.9,<0.4.0)",
+    "dulwich (>=0.22.8,<0.23.0)",
+    "et-xmlfile (>=2.0.0,<3.0.0)",
+    "executing (>=2.2.0,<3.0.0)",
+    "fastjsonschema (>=2.21.1,<3.0.0)",
+    "filelock (>=3.18.0,<4.0.0)",
+    "findpython (>=0.6.3,<0.7.0)",
+    "flatbuffers (>=25.2.10,<26.0.0)",
+    "fonttools (>=4.56.0,<5.0.0)",
+    "frozenlist (>=1.5.0,<2.0.0)",
+    "gast (>=0.6.0,<0.7.0)",
+    "gitdb (>=4.0.12,<5.0.0)",
+    "gitpython (>=3.1.44,<4.0.0)",
+    "google-pasta (>=0.2.0,<0.3.0)",
+    "grpcio (>=1.71.0,<2.0.0)",
+    "h11 (>=0.14.0,<0.15.0)",
+    "h5py (>=3.13.0,<4.0.0)",
+    "httpcore (>=1.0.7,<2.0.0)",
+    "httpx (>=0.28.1,<0.29.0)",
+    "huggingface-hub (>=0.29.3,<0.30.0)",
+    "idna (>=3.10,<4.0)",
+    "importlib-metadata (>=8.6.1,<9.0.0)",
+    "installer (>=0.7.0,<0.8.0)",
+    "ipython (>=9.0.2,<10.0.0)",
+    "ipython-pygments-lexers (>=1.1.1,<2.0.0)",
+    "jaraco-classes (>=3.4.0,<4.0.0)",
+    "jaraco-context (>=6.0.1,<7.0.0)",
+    "jaraco-functools (>=4.1.0,<5.0.0)",
+    "jedi (>=0.19.2,<0.20.0)",
+    "jinja2 (>=3.1.6,<4.0.0)",
+    "jsonschema (>=4.23.0,<5.0.0)",
+    "jsonschema-specifications (>=2024.10.1,<2025.0.0)",
+    "jupyter-client (>=8.6.3,<9.0.0)",
+    "jupyter-core (>=5.7.2,<6.0.0)",
+    "kagglehub (>=0.3.10,<0.4.0)",
+    "keyring (>=25.6.0,<26.0.0)",
+    "kiwisolver (>=1.4.8,<2.0.0)",
+    "libclang (>=18.1.1,<19.0.0)",
+    "locket (>=1.0.0,<2.0.0)",
+    "markdown (>=3.7,<4.0)",
+    "markdown-it-py (>=3.0.0,<4.0.0)",
+    "markupsafe (>=3.0.2,<4.0.0)",
+    "matplotlib-inline (>=0.1.7,<0.2.0)",
+    "mdurl (>=0.1.2,<0.2.0)",
+    "ml-dtypes (>=0.5.1,<0.6.0)",
+    "more-itertools (>=10.6.0,<11.0.0)",
+    "mpmath (>=1.3.0,<2.0.0)",
+    "msgpack (>=1.1.0,<2.0.0)",
+    "multidict (>=6.1.0,<7.0.0)",
+    "namex (>=0.0.8,<0.0.9)",
+    "narwhals (>=1.30.0,<2.0.0)",
+    "nest-asyncio (>=1.6.0,<2.0.0)",
+    "networkx (>=3.4.2,<4.0.0)",
+    "nltk (>=3.9.1,<4.0.0)",
+    "opt-einsum (>=3.4.0,<4.0.0)",
+    "optree (>=0.14.1,<0.15.0)",
+    "packaging (>=24.2,<25.0)",
+    "parso (>=0.8.4,<0.9.0)",
+    "partd (>=1.4.2,<2.0.0)",
+    "pbs-installer (>=2025.3.11,<2026.0.0)",
+    "pkginfo (>=1.12.1.2,<2.0.0.0)",
+    "platformdirs (>=4.3.6,<5.0.0)",
+    "portalocker (>=3.1.1,<4.0.0)",
+    "prompt-toolkit (>=3.0.50,<4.0.0)",
+    "propcache (>=0.3.0,<0.4.0)",
+    "psutil (>=7.0.0,<8.0.0)",
+    "pure-eval (>=0.2.3,<0.3.0)",
+    "pyarrow (>=19.0.1,<20.0.0)",
+    "pydeck (>=0.9.1,<0.10.0)",
+    "pygments (>=2.19.1,<3.0.0)",
+    "pyparsing (>=3.2.1,<4.0.0)",
+    "pyproject-hooks (>=1.2.0,<2.0.0)",
+    "python-dateutil (>=2.9.0.post0,<3.0.0)",
+    "pytz (>=2025.1,<2026.0)",
+    "pywin32-ctypes (>=0.2.3,<0.3.0)",
+    "pyyaml (>=6.0.2,<7.0.0)",
+    "pyzmq (>=26.3.0,<27.0.0)",
+    "rapidfuzz (>=3.12.2,<4.0.0)",
+    "referencing (>=0.36.2,<0.37.0)",
+    "requests (>=2.32.3,<3.0.0)",
+    "requests-toolbelt (>=1.0.0,<2.0.0)",
+    "rich (>=13.9.4,<14.0.0)",
+    "rpds-py (>=0.23.1,<0.24.0)",
+    "sentence-transformers (>=3.4.1,<4.0.0)",
+    "setuptools (>=76.0.0,<77.0.0)",
+    "shellingham (>=1.5.4,<2.0.0)",
+    "six (>=1.17.0,<2.0.0)",
+    "smmap (>=5.0.2,<6.0.0)",
+    "sniffio (>=1.3.1,<2.0.0)",
+    "sortedcontainers (>=2.4.0,<3.0.0)",
+    "soupsieve (>=2.6,<3.0)",
+    "stack-data (>=0.6.3,<0.7.0)",
+    "tabulate (>=0.9.0,<0.10.0)",
+    "tblib (>=3.0.0,<4.0.0)",
+    "tenacity (>=9.0.0,<10.0.0)",
+    "tensorboard (>=2.19.0,<3.0.0)",
+    "tensorboard-data-server (>=0.7.2,<0.8.0)",
+    "termcolor (>=2.5.0,<3.0.0)",
+    "threadpoolctl (>=3.6.0,<4.0.0)",
+    "tokenizers (>=0.21.1,<0.22.0)",
+    "toml (>=0.10.2,<0.11.0)",
+    "tomlkit (>=0.13.2,<0.14.0)",
+    "toolz (>=1.0.0,<2.0.0)",
+    "tornado (>=6.4.2,<7.0.0)",
+    "tqdm (>=4.67.1,<5.0.0)",
+    "traitlets (>=5.14.3,<6.0.0)",
+    "trove-classifiers (>=2025.3.13.13,<2026.0.0.0)",
+    "typing-extensions (>=4.12.2,<5.0.0)",
+    "tzdata (>=2025.1,<2026.0)",
+    "urllib3 (>=2.3.0,<3.0.0)",
+    "virtualenv (>=20.29.3,<21.0.0)",
+    "watchdog (>=6.0.0,<7.0.0)",
+    "wcwidth (>=0.2.13,<0.3.0)",
+    "werkzeug (>=3.1.3,<4.0.0)",
+    "wheel (>=0.45.1,<0.46.0)",
+    "wrapt (>=1.17.2,<2.0.0)",
+    "xxhash (>=3.5.0,<4.0.0)",
+    "yarl (>=1.18.3,<2.0.0)",
+    "zict (>=3.0.0,<4.0.0)",
+    "zipp (>=3.21.0,<4.0.0)",
+    "zstandard (>=0.23.0,<0.24.0)",
+    "asyncio (>=3.4.3,<4.0.0)",
+    "pytorch-lightning (>=2.5.0.post0,<3.0.0)",
+]
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -1,13 +1,14 @@
 absl-py==2.1.0 ; python_version >= "3.12"
 accelerate==1.5.2 ; python_version >= "3.12"
 aiohappyeyeballs==2.6.1 ; python_version >= "3.12"
-aiohttp==3.11.13 ; python_version >= "3.12"
+aiohttp==3.11.14 ; python_version >= "3.12"
 aiosignal==1.3.2 ; python_version >= "3.12"
 altair==5.5.0 ; python_version >= "3.12"
-anyio==4.8.0 ; python_version >= "3.12"
+anyio==4.9.0 ; python_version >= "3.12"
 appnope==0.1.4 ; python_version >= "3.12" and platform_system == "Darwin"
 asttokens==3.0.0 ; python_version >= "3.12"
 astunparse==1.6.3 ; python_version >= "3.12"
+asyncio==3.4.3 ; python_version >= "3.12"
 attrs==25.3.0 ; python_version >= "3.12"
 autograd==1.7.0 ; python_version >= "3.12"
 beautifulsoup4==4.13.3 ; python_version >= "3.12"
@@ -80,6 +81,7 @@ keras==3.9.0 ; python_version >= "3.12"
 keyring==25.6.0 ; python_version >= "3.12"
 kiwisolver==1.4.8 ; python_version >= "3.12"
 libclang==18.1.1 ; python_version >= "3.12"
+lightning-utilities==0.14.1 ; python_version >= "3.12"
 locket==1.0.0 ; python_version >= "3.12"
 lxml==5.3.1 ; python_version >= "3.12"
 markdown-it-py==3.0.0 ; python_version >= "3.12"
@@ -144,9 +146,10 @@ pygments==2.19.1 ; python_version >= "3.12"
 pyparsing==3.2.1 ; python_version >= "3.12"
 pyproject-hooks==1.2.0 ; python_version >= "3.12"
 python-dateutil==2.9.0.post0 ; python_version >= "3.12"
+pytorch-lightning==2.5.0.post0 ; python_version >= "3.12"
 pytz==2025.1 ; python_version >= "3.12"
-# pywin32-ctypes==0.2.3 ; python_version >= "3.12"  ## ONLY FOR WINDOWS
-# pywin32==309 ; python_version >= "3.12"  ## ONLY FOR WINDOWS
+pywin32-ctypes==0.2.3 ; python_version >= "3.12"
+pywin32==309 ; python_version >= "3.12"
 pyyaml==6.0.2 ; python_version >= "3.12"
 pyzmq==26.3.0 ; python_version >= "3.12"
 rapidfuzz==3.12.2 ; python_version >= "3.12"
@@ -188,6 +191,7 @@ toml==0.10.2 ; python_version >= "3.12"
 tomlkit==0.13.2 ; python_version >= "3.12"
 toolz==1.0.0 ; python_version >= "3.12"
 torch==2.6.0 ; python_version >= "3.12"
+torchmetrics==1.6.3 ; python_version >= "3.12"
 torchvision==0.21.0 ; python_version >= "3.12"
 tornado==6.4.2 ; python_version >= "3.12"
 tqdm==4.67.1 ; python_version >= "3.12"

sentiment_analysis/hmv_cfg_base_stage1/imports.py

@@ -9,5 +9,8 @@ import json
 import gc
 import psutil
 import os
+import importlib
 import importlib.util
-import sys
+import asyncio
+import sys
+import pytorch_lightning as pl

sentiment_analysis/{sentiment_analysis.py → sentiment_analysis_main.py}

@@ -1,296 +1,296 @@
-from imports import *
-import importlib.util
-import os
-import sys
-import joblib
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), )))
-
-# from hmv_cfg_base_stage1.model1 import load_model as load_model1
-# from hmv_cfg_base_stage1.model1 import predict as predict1
-
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-CONFIG_STAGE1 = os.path.join(BASE_DIR, "config", "stage1_models.json")
-LOADERS_STAGE1 = os.path.join(BASE_DIR, "hmv-cfg-base-stage1")
-
-# Load the model and tokenizer
-# model_name = "tachygraphy-microtrext-norm-org/DeBERTa-v3-seqClassfication-LV1-SentimentPolarities-Batch8"
-# tokenizer = AutoTokenizer.from_pretrained(model_name)
-# model = AutoModel.from_pretrained(model_name)
-
-SENTIMENT_POLARITY_LABELS = [
-    "negative", "neutral", "positive"
-]
-
-current_model = None
-current_tokenizer = None
-
-# Enabling Resource caching
-@st.cache_resource
-
-def load_model_config():
-    with open(CONFIG_STAGE1, "r") as f:
-        model_data = json.load(f)
-
-    model_options = {v["name"]: v for v in model_data.values()}  # Extract names for dropdown
-    return model_data, model_options
-
-MODEL_DATA, MODEL_OPTIONS = load_model_config()
-
-
-
-# def load_model():
-#     model = DebertaV2ForSequenceClassification.from_pretrained(model_name)
-#     tokenizer = DebertaV2Tokenizer.from_pretrained(model_name)
-#     return model, tokenizer
-
-
-# ✅ Dynamically Import Model Functions
-def import_from_module(module_name, function_name):
-    try:
-        module = importlib.import_module(module_name)
-        return getattr(module, function_name)
-    except (ModuleNotFoundError, AttributeError) as e:
-        st.error(f"❌ Import Error: {e}")
-        return None
-
-
-def free_memory():
-    #  """Free up CPU & GPU memory before loading a new model."""
-    global current_model, current_tokenizer
-
-    if current_model is not None:
-        del current_model  # Delete the existing model
-        current_model = None  # Reset reference
-
-    if current_tokenizer is not None:
-        del current_tokenizer  # Delete the tokenizer
-        current_tokenizer = None
-
-    gc.collect()  # Force garbage collection for CPU memory
-
-    if torch.cuda.is_available():  
-        torch.cuda.empty_cache()  # Free GPU memory
-        torch.cuda.ipc_collect()  # Clean up PyTorch GPU cache
-
-    # If running on CPU, reclaim memory using OS-level commands
-    try:
-        if torch.cuda.is_available() is False:
-            psutil.virtual_memory()  # Refresh memory stats
-    except Exception as e:
-        print(f"Memory cleanup error: {e}")
-
-
-def load_selected_model(model_name):
-    global current_model, current_tokenizer
-
-    free_memory()
-
-    # st.write("DEBUG: Available Models:", MODEL_OPTIONS.keys())  # ✅ See available models
-    # st.write("DEBUG: Selected Model:", MODEL_OPTIONS[model_name])  # ✅ Check selected model
-    # st.write("DEBUG: Model Name:", model_name)  # ✅ Check selected model
-
-    if model_name not in MODEL_OPTIONS:
-        st.error(f"⚠️ Model '{model_name}' not found in config!")
-        return None, None, None
-
-    model_info = MODEL_OPTIONS[model_name]
-    hf_location = model_info["hf_location"]
-
-    model_module = model_info["module_path"]
-    load_function = model_info["load_function"]
-    predict_function = model_info["predict_function"]
-
-    load_model_func = import_from_module(model_module, load_function)
-    predict_func = import_from_module(model_module, predict_function)
-
-    if load_model_func is None or predict_func is None:
-        st.error("❌ Model functions could not be loaded!")
-        return None, None, None
-
-    model, tokenizer = load_model_func()
-    
-    current_model, current_tokenizer = model, tokenizer
-    return model, tokenizer, predict_func
-
-# def load_selected_model(model_name):
-#     # """Load model and tokenizer based on user selection."""
-#     global current_model, current_tokenizer
-
-#     # Free memory before loading a new model
-#     free_memory()
-
-#     if model_name not in MODEL_OPTIONS:
-#         st.error(f"⚠️ Model '{model_name}' not found in config!")
-#         return None, None
-
-#     model_info = MODEL_OPTIONS[model_name]
-#     hf_location = model_info["hf_location"]
-
-#     model_module = model_info["module_path"]
-#     # load_function = "load_model"
-#     # predict_function = "predict"
-
-#     load_function = model_info["load_function"]
-#     predict_function = model_info["predict_function"]
-
-#     # tokenizer_class = globals()[model_info["tokenizer_class"]]
-#     # model_class = globals()[model_info["model_class"]]
-
-#     # tokenizer = tokenizer_class.from_pretrained(hf_location)
-
-
-#     load_model_func = import_from_module(model_module, load_function)
-#     predict_func = import_from_module(model_module, predict_function)
-
-#     # # Load model
-#     # if model_info["type"] == "custom_checkpoint" or model_info["type"] == "custom_model":
-#     #     model = torch.load(hf_location, map_location="cpu")  # Load PyTorch model
-#     # elif model_info["type"] == "hf_automodel_finetuned_dbt3":
-#     #     tokenizer_class = globals()[model_info["tokenizer_class"]]
-#     #     model_class = globals()[model_info["model_class"]]
-#     #     tokenizer = tokenizer_class.from_pretrained(hf_location)
-#     #     model = model_class.from_pretrained(hf_location,
-#     #                                         problem_type=model_info["problem_type"],
-#     #                                         num_labels=model_info["num_labels"]
-#     #     )
-#     # else:
-#     #     st.error("Invalid model selection")
-#     #     return None, None
-    
-
-#     if load_model_func is None or predict_func is None:
-#         st.error("❌ Model functions could not be loaded!")
-#         return None, None
-
-#     # current_model, current_tokenizer = model, tokenizer  # Store references
-#     # return model, tokenizer
-
-#     model, tokenizer = load_model_func(hf_location)
-    
-#     current_model, current_tokenizer = model, tokenizer
-#     return model, tokenizer, predict_func
-
-
-
-def predict(text, model, tokenizer, device, max_len=128):
-    # Tokenize and pad the input text
-    inputs = tokenizer(
-        text,
-        add_special_tokens=True,
-        padding=True,
-        truncation=False,
-        return_tensors="pt",
-        return_token_type_ids=False,
-    ).to(device)  # Move input tensors to the correct device
-
-    with torch.no_grad():
-        outputs = model(**inputs)
-
-    # Apply sigmoid activation (for BCEWithLogitsLoss)
-    probabilities = outputs.logits.cpu().numpy()
-
-    return probabilities 
-
-# def show_sentiment_analysis():
-
-    # Add your sentiment analysis code here
-
-    # user_input = st.text_input("Enter text for sentiment analysis:")
-    # user_input = st.text_area("Enter text for sentiment analysis:", height=200)
-    # user_input = st.text_area("Enter text for sentiment analysis:", max_chars=500)
-
-def show_sentiment_analysis():
-    st.title("Stage 1: Sentiment Polarity Analysis")
-    st.write("This section will handle sentiment analysis.")
-
-    if "selected_model" not in st.session_state:
-        st.session_state.selected_model = list(MODEL_OPTIONS.keys())[0]  # Default selection
-
-    if "clear_output" not in st.session_state:
-        st.session_state.clear_output = False
-
-    st.selectbox("Choose a model:", list(MODEL_OPTIONS.keys()), key="selected_model")
-
-    selected_model = st.session_state.selected_model
-
-    if selected_model not in MODEL_OPTIONS:
-        st.error(f"❌ Selected model '{selected_model}' not found!")
-        st.stop()
-
-    st.session_state.clear_output = True  # Reset output when model changes
-
-
-    # st.write("DEBUG: Available Models:", MODEL_OPTIONS.keys())  # ✅ See available models
-    # st.write("DEBUG: Selected Model:", MODEL_OPTIONS[selected_model])  # ✅ Check selected model
-
-
-    user_input = st.text_input("Enter text for sentiment analysis:")
-
-    if user_input:
-        # Make prediction
-
-        # model, tokenizer = load_model()
-        # model, tokenizer = load_selected_model(selected_model)
-
-        model, tokenizer, predict_func = load_selected_model(selected_model)
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-        if model is None:
-            st.error("⚠️ Error: Model failed to load! Check model selection or configuration.")
-            st.stop()
-
-        model.to(device)
-
-        # predictions = predict(user_input, model, tokenizer, device)
-
-        predictions = predict_func(user_input, model, tokenizer, device)
-
-        # Squeeze predictions to remove extra dimensions
-        predictions_array = predictions.squeeze()
-
-        # Convert to binary predictions (argmax)
-        binary_predictions = np.zeros_like(predictions_array)
-        max_indices = np.argmax(predictions_array)
-        binary_predictions[max_indices] = 1
-
-        # Display raw predictions
-        st.write(f"**Predicted Sentiment Scores:** {predictions_array}")
-
-        # Display binary classification result
-        st.write(f"**Predicted Sentiment:**")
-        st.write(f"**NEGATIVE:** {binary_predictions[0]}, **NEUTRAL:** {binary_predictions[1]}, **POSITIVE:** {binary_predictions[2]}")
-        # st.write(f"**NEUTRAL:** {binary_predictions[1]}")
-        # st.write(f"**POSITIVE:** {binary_predictions[2]}")
-
-        # 1️⃣ **Polar Plot (Plotly)**
-        sentiment_polarities = predictions_array.tolist()
-        fig_polar = px.line_polar(
-            pd.DataFrame(dict(r=sentiment_polarities, theta=SENTIMENT_POLARITY_LABELS)), 
-            r='r', theta='theta', line_close=True
-        )
-        st.plotly_chart(fig_polar)
-
-        # 2️⃣ **Normalized Horizontal Bar Chart (Matplotlib)**
-        normalized_predictions = predictions_array / predictions_array.sum()
-
-        fig, ax = plt.subplots(figsize=(8, 2))
-        left = 0
-        for i in range(len(normalized_predictions)):
-            ax.barh(0, normalized_predictions[i], color=plt.cm.tab10(i), left=left, label=SENTIMENT_POLARITY_LABELS[i])
-            left += normalized_predictions[i]
-
-        # Configure the chart
-        ax.set_xlim(0, 1)
-        ax.set_yticks([])
-        ax.set_xticks(np.arange(0, 1.1, 0.1))
-        ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=len(SENTIMENT_POLARITY_LABELS))
-        plt.title("Sentiment Polarity Prediction Distribution")
-
-        # Display in Streamlit
-        st.pyplot(fig)
-        
-
-
-if __name__ == "__main__":
+from imports import *
+import importlib.util
+import os
+import sys
+import joblib
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), )))
+
+# from hmv_cfg_base_stage1.model1 import load_model as load_model1
+# from hmv_cfg_base_stage1.model1 import predict as predict1
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+CONFIG_STAGE1 = os.path.join(BASE_DIR, "config", "stage1_models.json")
+LOADERS_STAGE1 = os.path.join(BASE_DIR, "hmv-cfg-base-stage1")
+
+# Load the model and tokenizer
+# model_name = "tachygraphy-microtrext-norm-org/DeBERTa-v3-seqClassfication-LV1-SentimentPolarities-Batch8"
+# tokenizer = AutoTokenizer.from_pretrained(model_name)
+# model = AutoModel.from_pretrained(model_name)
+
+SENTIMENT_POLARITY_LABELS = [
+    "negative", "neutral", "positive"
+]
+
+current_model = None
+current_tokenizer = None
+
+# Enabling Resource caching
+@st.cache_resource
+
+def load_model_config():
+    with open(CONFIG_STAGE1, "r") as f:
+        model_data = json.load(f)
+
+    model_options = {v["name"]: v for v in model_data.values()}  # Extract names for dropdown
+    return model_data, model_options
+
+MODEL_DATA, MODEL_OPTIONS = load_model_config()
+
+
+
+# def load_model():
+#     model = DebertaV2ForSequenceClassification.from_pretrained(model_name)
+#     tokenizer = DebertaV2Tokenizer.from_pretrained(model_name)
+#     return model, tokenizer
+
+
+# ✅ Dynamically Import Model Functions
+def import_from_module(module_name, function_name):
+    try:
+        module = importlib.import_module(module_name)
+        return getattr(module, function_name)
+    except (ModuleNotFoundError, AttributeError) as e:
+        st.error(f"❌ Import Error: {e}")
+        return None
+
+
+def free_memory():
+    #  """Free up CPU & GPU memory before loading a new model."""
+    global current_model, current_tokenizer
+
+    if current_model is not None:
+        del current_model  # Delete the existing model
+        current_model = None  # Reset reference
+
+    if current_tokenizer is not None:
+        del current_tokenizer  # Delete the tokenizer
+        current_tokenizer = None
+
+    gc.collect()  # Force garbage collection for CPU memory
+
+    if torch.cuda.is_available():  
+        torch.cuda.empty_cache()  # Free GPU memory
+        torch.cuda.ipc_collect()  # Clean up PyTorch GPU cache
+
+    # If running on CPU, reclaim memory using OS-level commands
+    try:
+        if torch.cuda.is_available() is False:
+            psutil.virtual_memory()  # Refresh memory stats
+    except Exception as e:
+        print(f"Memory cleanup error: {e}")
+
+
+def load_selected_model(model_name):
+    global current_model, current_tokenizer
+
+    free_memory()
+
+    # st.write("DEBUG: Available Models:", MODEL_OPTIONS.keys())  # ✅ See available models
+    # st.write("DEBUG: Selected Model:", MODEL_OPTIONS[model_name])  # ✅ Check selected model
+    # st.write("DEBUG: Model Name:", model_name)  # ✅ Check selected model
+
+    if model_name not in MODEL_OPTIONS:
+        st.error(f"⚠️ Model '{model_name}' not found in config!")
+        return None, None, None
+
+    model_info = MODEL_OPTIONS[model_name]
+    hf_location = model_info["hf_location"]
+
+    model_module = model_info["module_path"]
+    load_function = model_info["load_function"]
+    predict_function = model_info["predict_function"]
+
+    load_model_func = import_from_module(model_module, load_function)
+    predict_func = import_from_module(model_module, predict_function)
+
+    if load_model_func is None or predict_func is None:
+        st.error("❌ Model functions could not be loaded!")
+        return None, None, None
+
+    model, tokenizer = load_model_func()
+    
+    current_model, current_tokenizer = model, tokenizer
+    return model, tokenizer, predict_func
+
+# def load_selected_model(model_name):
+#     # """Load model and tokenizer based on user selection."""
+#     global current_model, current_tokenizer
+
+#     # Free memory before loading a new model
+#     free_memory()
+
+#     if model_name not in MODEL_OPTIONS:
+#         st.error(f"⚠️ Model '{model_name}' not found in config!")
+#         return None, None
+
+#     model_info = MODEL_OPTIONS[model_name]
+#     hf_location = model_info["hf_location"]
+
+#     model_module = model_info["module_path"]
+#     # load_function = "load_model"
+#     # predict_function = "predict"
+
+#     load_function = model_info["load_function"]
+#     predict_function = model_info["predict_function"]
+
+#     # tokenizer_class = globals()[model_info["tokenizer_class"]]
+#     # model_class = globals()[model_info["model_class"]]
+
+#     # tokenizer = tokenizer_class.from_pretrained(hf_location)
+
+
+#     load_model_func = import_from_module(model_module, load_function)
+#     predict_func = import_from_module(model_module, predict_function)
+
+#     # # Load model
+#     # if model_info["type"] == "custom_checkpoint" or model_info["type"] == "custom_model":
+#     #     model = torch.load(hf_location, map_location="cpu")  # Load PyTorch model
+#     # elif model_info["type"] == "hf_automodel_finetuned_dbt3":
+#     #     tokenizer_class = globals()[model_info["tokenizer_class"]]
+#     #     model_class = globals()[model_info["model_class"]]
+#     #     tokenizer = tokenizer_class.from_pretrained(hf_location)
+#     #     model = model_class.from_pretrained(hf_location,
+#     #                                         problem_type=model_info["problem_type"],
+#     #                                         num_labels=model_info["num_labels"]
+#     #     )
+#     # else:
+#     #     st.error("Invalid model selection")
+#     #     return None, None
+    
+
+#     if load_model_func is None or predict_func is None:
+#         st.error("❌ Model functions could not be loaded!")
+#         return None, None
+
+#     # current_model, current_tokenizer = model, tokenizer  # Store references
+#     # return model, tokenizer
+
+#     model, tokenizer = load_model_func(hf_location)
+    
+#     current_model, current_tokenizer = model, tokenizer
+#     return model, tokenizer, predict_func
+
+
+
+def predict(text, model, tokenizer, device, max_len=128):
+    # Tokenize and pad the input text
+    inputs = tokenizer(
+        text,
+        add_special_tokens=True,
+        padding=True,
+        truncation=False,
+        return_tensors="pt",
+        return_token_type_ids=False,
+    ).to(device)  # Move input tensors to the correct device
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+
+    # Apply sigmoid activation (for BCEWithLogitsLoss)
+    probabilities = outputs.logits.cpu().numpy()
+
+    return probabilities 
+
+# def show_sentiment_analysis():
+
+    # Add your sentiment analysis code here
+
+    # user_input = st.text_input("Enter text for sentiment analysis:")
+    # user_input = st.text_area("Enter text for sentiment analysis:", height=200)
+    # user_input = st.text_area("Enter text for sentiment analysis:", max_chars=500)
+
+def show_sentiment_analysis():
+    st.title("Stage 1: Sentiment Polarity Analysis")
+    st.write("This section will handle sentiment analysis.")
+
+    if "selected_model" not in st.session_state:
+        st.session_state.selected_model = list(MODEL_OPTIONS.keys())[0]  # Default selection
+
+    if "clear_output" not in st.session_state:
+        st.session_state.clear_output = False
+
+    st.selectbox("Choose a model:", list(MODEL_OPTIONS.keys()), key="selected_model")
+
+    selected_model = st.session_state.selected_model
+
+    if selected_model not in MODEL_OPTIONS:
+        st.error(f"❌ Selected model '{selected_model}' not found!")
+        st.stop()
+
+    st.session_state.clear_output = True  # Reset output when model changes
+
+
+    # st.write("DEBUG: Available Models:", MODEL_OPTIONS.keys())  # ✅ See available models
+    # st.write("DEBUG: Selected Model:", MODEL_OPTIONS[selected_model])  # ✅ Check selected model
+
+
+    user_input = st.text_input("Enter text for sentiment analysis:")
+
+    if user_input:
+        # Make prediction
+
+        # model, tokenizer = load_model()
+        # model, tokenizer = load_selected_model(selected_model)
+
+        model, tokenizer, predict_func = load_selected_model(selected_model)
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        if model is None:
+            st.error("⚠️ Error: Model failed to load! Check model selection or configuration.")
+            st.stop()
+
+        model.to(device)
+
+        # predictions = predict(user_input, model, tokenizer, device)
+
+        predictions = predict_func(user_input, model, tokenizer, device)
+
+        # Squeeze predictions to remove extra dimensions
+        predictions_array = predictions.squeeze()
+
+        # Convert to binary predictions (argmax)
+        binary_predictions = np.zeros_like(predictions_array)
+        max_indices = np.argmax(predictions_array)
+        binary_predictions[max_indices] = 1
+
+        # Display raw predictions
+        st.write(f"**Predicted Sentiment Scores:** {predictions_array}")
+
+        # Display binary classification result
+        st.write(f"**Predicted Sentiment:**")
+        st.write(f"**NEGATIVE:** {binary_predictions[0]}, **NEUTRAL:** {binary_predictions[1]}, **POSITIVE:** {binary_predictions[2]}")
+        # st.write(f"**NEUTRAL:** {binary_predictions[1]}")
+        # st.write(f"**POSITIVE:** {binary_predictions[2]}")
+
+        # 1️⃣ **Polar Plot (Plotly)**
+        sentiment_polarities = predictions_array.tolist()
+        fig_polar = px.line_polar(
+            pd.DataFrame(dict(r=sentiment_polarities, theta=SENTIMENT_POLARITY_LABELS)), 
+            r='r', theta='theta', line_close=True
+        )
+        st.plotly_chart(fig_polar)
+
+        # 2️⃣ **Normalized Horizontal Bar Chart (Matplotlib)**
+        normalized_predictions = predictions_array / predictions_array.sum()
+
+        fig, ax = plt.subplots(figsize=(8, 2))
+        left = 0
+        for i in range(len(normalized_predictions)):
+            ax.barh(0, normalized_predictions[i], color=plt.cm.tab10(i), left=left, label=SENTIMENT_POLARITY_LABELS[i])
+            left += normalized_predictions[i]
+
+        # Configure the chart
+        ax.set_xlim(0, 1)
+        ax.set_yticks([])
+        ax.set_xticks(np.arange(0, 1.1, 0.1))
+        ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=len(SENTIMENT_POLARITY_LABELS))
+        plt.title("Sentiment Polarity Prediction Distribution")
+
+        # Display in Streamlit
+        st.pyplot(fig)
+        
+
+
+if __name__ == "__main__":
     show_sentiment_analysis()