diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..53e70de9acba84482afa13cf82e1cb64dbdd0854 --- /dev/null +++ b/Makefile @@ -0,0 +1,46 @@ +.PHONY: install dev api frontend worker test lint typecheck security coverage docker-up docker-down clean + +# ── Setup ──────────────────────────────────────────────────────────────── +install: + pip install . + +dev: + pip install -e ".[dev]" + +# ── Services ───────────────────────────────────────────────────────────── +api: + uvicorn api.main:app --host 0.0.0.0 --port 8000 --reload + +frontend: + streamlit run frontend/Home.py --server.port 8501 + +worker: + celery -A api.tasks worker --loglevel=info + +# ── Quality gates ──────────────────────────────────────────────────────── +test: + PYTHONPATH=src pytest + +lint: + PYTHONPATH=src ruff check api src/absa tests + +typecheck: + PYTHONPATH=src mypy api src/absa + +security: + bandit -r api src/absa + +coverage: + PYTHONPATH=src pytest --cov=api --cov=absa --cov-report=term-missing + +# ── Docker ─────────────────────────────────────────────────────────────── +docker-up: + docker compose -f docker/docker-compose.yml up -d + +docker-down: + docker compose -f docker/docker-compose.yml down + +# ── Cleanup ────────────────────────────────────────────────────────────── +clean: + find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true + rm -rf .pytest_cache .mypy_cache .ruff_cache .coverage coverage.xml htmlcov diff --git a/absa/__init__.py b/api/__init__.py similarity index 100% rename from absa/__init__.py rename to api/__init__.py diff --git a/absa/data/__init__.py b/api/core/__init__.py similarity index 100% rename from absa/data/__init__.py rename to api/core/__init__.py diff --git a/api/main.py b/api/main.py new file mode 100644 index 0000000000000000000000000000000000000000..03019d7836fde60b54924f42a2c33f8be0788169 --- /dev/null +++ b/api/main.py @@ -0,0 +1,56 @@ +import os +from contextlib import asynccontextmanager + +from dotenv import load_dotenv +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from slowapi import Limiter, _rate_limit_exceeded_handler +from slowapi.errors import RateLimitExceeded +from slowapi.util import get_remote_address + +load_dotenv() + +from api.middleware.dependencies import engine # noqa: E402 +from api.middleware.metrics import instrumentator # noqa: E402 +from api.routes import predict, results # noqa: E402 +from api.schemas.db_models import Base # noqa: E402 +from api.services.absa_pipeline import pipeline # noqa: E402 + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Startup + print("Initializing Database tables...") + Base.metadata.create_all(bind=engine) + + print("Loading Models...") + pipeline.load_models() + + yield + # Shutdown + print("Shutting down...") + + +limiter = Limiter(key_func=get_remote_address) + +app = FastAPI( + title="Multilingual ABSA API", + description="Aspect-Based Sentiment Analysis for English and Hindi", + version="1.0.0", + lifespan=lifespan, +) + +app.state.limiter = limiter +app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) + +app.add_middleware( + CORSMiddleware, + allow_origins=os.getenv("CORS_ORIGINS", "http://localhost:8000,http://localhost:8501").split(","), + allow_credentials=True, + allow_methods=["GET", "POST"], + allow_headers=["*"], +) +app.include_router(predict.router, tags=["Predict"]) +app.include_router(results.router, tags=["System"]) + +instrumentator.instrument(app).expose(app, endpoint="/metrics") diff --git a/absa/evaluation/__init__.py b/api/middleware/__init__.py similarity index 100% rename from absa/evaluation/__init__.py rename to api/middleware/__init__.py diff --git a/app/middleware/dependencies.py b/api/middleware/dependencies.py similarity index 100% rename from app/middleware/dependencies.py rename to api/middleware/dependencies.py diff --git a/app/middleware/metrics.py b/api/middleware/metrics.py similarity index 100% rename from app/middleware/metrics.py rename to api/middleware/metrics.py diff --git a/absa/models/__init__.py b/api/routes/__init__.py similarity index 100% rename from absa/models/__init__.py rename to api/routes/__init__.py diff --git a/app/routes/predict.py b/api/routes/predict.py similarity index 94% rename from app/routes/predict.py rename to api/routes/predict.py index 23141061df2de678104eee7855fc909ba5d284d1..4af0ded561585f1f82cefd0f65502e414e5c8240 100644 --- a/app/routes/predict.py +++ b/api/routes/predict.py @@ -7,11 +7,11 @@ import tempfile import time import re -from app.schemas.schemas import ReviewInput, PredictionResponse, BatchJobResponse -from app.schemas.db_models import Review, AspectResult, BatchJob -from app.middleware.dependencies import get_db -from app.services.absa_pipeline import pipeline -from app.tasks.batch_tasks import process_batch +from api.schemas.schemas import ReviewInput, PredictionResponse, BatchJobResponse +from api.schemas.db_models import Review, AspectResult, BatchJob +from api.middleware.dependencies import get_db +from api.services.absa_pipeline import pipeline +from api.tasks.batch_tasks import process_batch router = APIRouter() diff --git a/app/routes/results.py b/api/routes/results.py similarity index 100% rename from app/routes/results.py rename to api/routes/results.py diff --git a/absa/training/__init__.py b/api/schemas/__init__.py similarity index 100% rename from absa/training/__init__.py rename to api/schemas/__init__.py diff --git a/app/schemas/db_models.py b/api/schemas/db_models.py similarity index 100% rename from app/schemas/db_models.py rename to api/schemas/db_models.py diff --git a/app/schemas/schemas.py b/api/schemas/schemas.py similarity index 100% rename from app/schemas/schemas.py rename to api/schemas/schemas.py diff --git a/absa/utils/__init__.py b/api/services/__init__.py similarity index 100% rename from absa/utils/__init__.py rename to api/services/__init__.py diff --git a/app/services/absa_pipeline.py b/api/services/absa_pipeline.py similarity index 69% rename from app/services/absa_pipeline.py rename to api/services/absa_pipeline.py index 769ee531a72ab2b5c67a578645502b9acca166d0..2d0f69b63e82efb3deac6ac9e0824559950a84f1 100644 --- a/app/services/absa_pipeline.py +++ b/api/services/absa_pipeline.py @@ -12,20 +12,21 @@ Strategy: import os import re -import time import threading +import time from pathlib import Path -from typing import List, Tuple +from typing import List, Optional, Tuple + import numpy as np -from app.schemas.schemas import PredictionResponse, AspectSentiment -from app.services.lang_service import lang_service +from api.schemas.schemas import AspectSentiment, PredictionResponse +from api.services.lang_service import lang_service # ── Optional heavy imports (ONNX custom models) ─────────────────────────────── try: from optimum.onnxruntime import ( - ORTModelForTokenClassification, ORTModelForSequenceClassification, + ORTModelForTokenClassification, ) from transformers import AutoTokenizer @@ -143,6 +144,114 @@ ASPECT_PHRASES: List[str] = sorted( reverse=True, ) +# ── Fallback aspect nouns ───────────────────────────────────────────────────── +# Used when the fixed lexicon misses a comment (e.g. food/service/experience). +# These are the nouns we anchor near sentiment words when no lexicon aspect +# matches. English + transliterated + Devanagari. +GENERAL_ASPECT_NOUNS: set[str] = { + # Experience / service + "food", + "taste", + "flavor", + "flavour", + "service", + "staff", + "experience", + "product", + "quality", + "delivery", + "packaging", + "price", + "cost", + "value", + "design", + "look", + "feel", + "build", + "material", + "comfort", + "durability", + "cleanliness", + "hygiene", + "room", + "location", + "ambience", + "menu", + "portion", + "hotel", + "restaurant", + "support", + "response", + "warranty", + "atmosphere", + "purchase", + "order", + "buy", + # Devices / electronics + "phone", + "smartphone", + "laptop", + "tablet", + "headphones", + "earphones", + "earbuds", + "watch", + "device", + "camera", + "screen", + "display", + "battery", + "speaker", + "speakers", + "keyboard", + "mouse", + "monitor", + "processor", + "performance", + "speed", + "sound", + "audio", + "picture", + "photo", + "video", + "signal", + "network", + "call", + "app", + "software", + "interface", + "ui", + "features", + "battery life", + "charging", + "processor", + "ram", + "memory", + # Hindi transliterated + "khana", + "swad", + "delivery", + "experience", + "speed", + "sound", + "signal", + # Devanagari + "खाना", + "सेवा", + "गुणवत्ता", + "कीमत", + "डिज़ाइन", + "उत्पाद", + "अनुभव", + "बैटरी", + "कैमरा", + "प्रदर्शन", + "आवाज़", + "स्पीड", + "डिलीवरी", + "स्वाद", +} + # ── Sentiment lexicon ───────────────────────────────────────────────────────── POSITIVE_WORDS = { "excellent", @@ -182,6 +291,11 @@ POSITIVE_WORDS = { "recommended", "worth", "affordable", + "best", + "better", + "awesome", + "favorite", + "favourite", "value", "effective", "efficient", @@ -215,6 +329,18 @@ POSITIVE_WORDS = { "shandar", "zabardast", "mast", + # Hindi positive (Devanagari) + "अच्छा", + "बढ़िया", + "शानदार", + "ज़बरदस्त", + "मस्त", + "पसंद", + "उत्तम", + "उम्दा", + "सुंदर", + "बेहतरीन", + "अद्भुत", } NEGATIVE_WORDS = { @@ -271,6 +397,10 @@ NEGATIVE_WORDS = { "broken", "defective", "faulty", + "worst", + "worse", + "useless", + "pathetic", "average", "ordinary", "basic", @@ -281,6 +411,14 @@ NEGATIVE_WORDS = { "bura", "ganda", "faltu", + # Hindi negative (Devanagari) + "खराब", + "बेकार", + "बुरा", + "घटिया", + "फालतू", + "निराश", + "सस्ता", } NEGATION_WORDS = { @@ -322,12 +460,34 @@ INTENSIFIERS = { } +# Unicode-aware token pattern. `\w` alone misses Devanagari vowel signs +# (combining marks), which would split "खाना" into stray characters, so the +# Devanagari block is included explicitly. +WORD_RE = re.compile(r"[\w\u0900-\u097f'-]+") + +# Split a review into sentiment clauses on punctuation and conjunctions, so a +# sentence like "food was great but service was terrible" is scored per-clause. +# Includes English + transliterated + Devanagari conjunctions. The Devanagari +# ones need whitespace around them, otherwise "या" matches inside "बढ़िया". +_CLAUSE_SPLIT_RE = re.compile( + r"[.!?;,]|\b(?:but|and|yet|however|although|though|while|whereas|because|" + r"since|so|or|nor|lekin|par|magar|aur|kintu|va|ya)\b|" + r"(?<=\s)(?:लेकिन|और|पर|मगर|किंतु)(?=\s)" +) + + +def _split_clauses(text: str) -> List[str]: + """Split a review into clauses (punctuation + conjunction boundaries).""" + parts = [p for p in _CLAUSE_SPLIT_RE.split(text) if p and p.strip()] + return parts or [text] + + def _score_sentence(sentence: str) -> Tuple[float, float]: """ Return (positive_score, negative_score) for a sentence. Handles negation (3-word window) and intensifiers. """ - words = re.findall(r"\b[\w'-]+\b", sentence.lower()) + words = WORD_RE.findall(sentence.lower()) pos, neg = 0.0, 0.0 i = 0 while i < len(words): @@ -399,10 +559,8 @@ class ABSAPipeline: self.aspect_model = ORTModelForTokenClassification.from_pretrained( hf_repo_id, subfolder="aspect_extraction_int8" ) - self.sentiment_model = ( - ORTModelForSequenceClassification.from_pretrained( - hf_repo_id, subfolder="sentiment_int8" - ) + self.sentiment_model = ORTModelForSequenceClassification.from_pretrained( + hf_repo_id, subfolder="sentiment_int8" ) print("Custom ONNX models loaded.") except Exception as e: @@ -420,14 +578,8 @@ class ABSAPipeline: try: print(f"Loading custom ONNX models from {model_path_base}") self.tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base") - self.aspect_model = ORTModelForTokenClassification.from_pretrained( - str(aspect_path) - ) - self.sentiment_model = ( - ORTModelForSequenceClassification.from_pretrained( - str(sentiment_path) - ) - ) + self.aspect_model = ORTModelForTokenClassification.from_pretrained(str(aspect_path)) + self.sentiment_model = ORTModelForSequenceClassification.from_pretrained(str(sentiment_path)) print("Custom ONNX models loaded.") except Exception as e: print(f"Custom model load skipped: {e}") @@ -437,7 +589,7 @@ class ABSAPipeline: self.is_loaded = True - def predict(self, text: str, requested_lang: str = None) -> PredictionResponse: + def predict(self, text: str, requested_lang: Optional[str] = None) -> PredictionResponse: start = time.time() detected_lang = lang_service.detect_language(text) actual_lang = requested_lang or detected_lang @@ -461,14 +613,17 @@ class ABSAPipeline: # ── Custom ONNX path ────────────────────────────────────────────────────── def _predict_onnx(self, text: str) -> List[AspectSentiment]: - inputs = self.tokenizer( - text, return_tensors="pt", truncation=True, max_length=128 - ) + assert self.aspect_model is not None + assert self.sentiment_model is not None + assert self.tokenizer is not None + inputs = self.tokenizer(text, return_tensors="pt", truncation=True, max_length=128) logits = self.aspect_model(**inputs).logits[0].detach().numpy() preds = np.argmax(logits, axis=1) tokens = self.tokenizer.convert_ids_to_tokens(inputs["input_ids"][0]) - raw, current, start_idx = [], [], -1 + raw: List[Tuple[str, int, int]] = [] + current: List[str] = [] + start_idx = -1 skip = { self.tokenizer.cls_token, self.tokenizer.sep_token, @@ -519,17 +674,26 @@ class ABSAPipeline: def _predict_rule_based(self, text: str) -> List[AspectSentiment]: text_lower = text.lower() - sentences = re.split(r"(?<=[.!?])\s+", text) + clauses = _split_clauses(text) found_aspects = self._extract_aspects(text_lower) + # Tier 2: no lexicon aspect matched — anchor nouns near sentiment words. + if not found_aspects: + found_aspects = self._extract_targets_from_sentiment(text_lower) + + # Tier 3: still nothing — fall back to review-level sentiment on a + # generic aspect so every input produces a useful result. + if not found_aspects: + found_aspects = [("Overall", 0, len(text))] + results = [] for aspect_label, start_char, end_char in found_aspects: - # Find the sentence(s) mentioning this aspect for focused scoring + # Score only the clause(s) mentioning this aspect so that mixed + # sentences ("food was great but service was terrible") don't + # cancel each other into neutral. aspect_lower = aspect_label.lower() - context_sentences = [s for s in sentences if aspect_lower in s.lower()] or [ - text - ] - context = " ".join(context_sentences) + context_clauses = [c for c in clauses if aspect_lower in c.lower()] or [text] + context = " ".join(context_clauses) pos, neg = _score_sentence(context) @@ -550,6 +714,33 @@ class ABSAPipeline: ) return results + def _extract_targets_from_sentiment(self, text_lower: str) -> List[Tuple[str, int, int]]: + """Fallback aspect extraction: nouns found near sentiment words. + + Handles cases like "The food was great" (noun before the sentiment + word) and "great camera" (noun after it), plus Devanagari text. + """ + tokens = [(m.start(), m.end(), m.group()) for m in WORD_RE.finditer(text_lower)] + sent_words = POSITIVE_WORDS | NEGATIVE_WORDS + + targets: List[Tuple[str, int, int]] = [] + seen_ranges: List[Tuple[int, int]] = [] + + for i, (_, _, token) in enumerate(tokens): + if token not in sent_words: + continue + window = tokens[max(0, i - 4) : i] + tokens[i + 1 : i + 5] + for s, e, noun in window: + if noun in GENERAL_ASPECT_NOUNS: + if any(s >= r0 and e <= r1 for r0, r1 in seen_ranges): + continue + targets.append((noun.title(), s, e)) + seen_ranges.append((s, e)) + break + + targets.sort(key=lambda x: x[1]) + return targets + def _extract_aspects(self, text_lower: str) -> List[Tuple[str, int, int]]: """Find aspect keyword matches; return (label, start, end) sorted by position.""" found: List[Tuple[str, int, int]] = [] diff --git a/app/services/lang_service.py b/api/services/lang_service.py similarity index 100% rename from app/services/lang_service.py rename to api/services/lang_service.py diff --git a/app/tasks/__init__.py b/api/tasks/__init__.py similarity index 100% rename from app/tasks/__init__.py rename to api/tasks/__init__.py diff --git a/app/tasks/batch_tasks.py b/api/tasks/batch_tasks.py similarity index 95% rename from app/tasks/batch_tasks.py rename to api/tasks/batch_tasks.py index 7582716e99c37a0f3b3c4e6ef796063d6e4b02a5..ab18c94d974df13226b31ede9957d973a4d1c0eb 100644 --- a/app/tasks/batch_tasks.py +++ b/api/tasks/batch_tasks.py @@ -1,7 +1,7 @@ -from app.tasks import celery_app -from app.services.absa_pipeline import pipeline -from app.middleware.dependencies import SessionLocal -from app.schemas.db_models import BatchJob, AspectResult, Review +from api.tasks import celery_app +from api.services.absa_pipeline import pipeline +from api.middleware.dependencies import SessionLocal +from api.schemas.db_models import BatchJob, AspectResult, Review import pandas as pd import os import csv diff --git a/app/core/templates.py b/app/core/templates.py deleted file mode 100644 index 35a412593be9328ad178c90e9e31dd8cedc21115..0000000000000000000000000000000000000000 --- a/app/core/templates.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Centralized Jinja2Templates instance. - -Kept in api/app/core/ so every page-rendering router imports from one place, -avoiding multiple conflicting Template objects pointing at the same directory. - -WHY THIS FILE EXISTS --------------------- -FastAPI's Jinja2Templates must be initialised with a directory path. -Centralising it here means that when Phase 3-5 routers add fragment endpoints -they import `templates` from here — no duplication, no divergence. -""" -from __future__ import annotations - -from pathlib import Path - -from fastapi import Request -from fastapi.templating import Jinja2Templates - -# Resolve relative to this file: -# api/app/core/templates.py → api/app/templates/ -_TEMPLATE_DIR: Path = Path(__file__).parent.parent / "templates" - -templates = Jinja2Templates(directory=str(_TEMPLATE_DIR)) - - -# ── Global template context processor ───────────────────────────────────────── -# Ensures every template rendered via this instance always has access to -# csrf_token — even partial/fragment templates that don't go through _base_ctx. - -def _csrf_processor(request: Request) -> dict: # type: ignore[no-redef] - from app.middleware.csrf import generate_csrf_token - return {"csrf_token": generate_csrf_token()} - - -templates.context_processors.append(_csrf_processor) # type: ignore[arg-type] diff --git a/app/main.py b/app/main.py deleted file mode 100644 index 02edaa1bb68ab1a754b4721d4a1929589fb8bd2b..0000000000000000000000000000000000000000 --- a/app/main.py +++ /dev/null @@ -1,73 +0,0 @@ -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from contextlib import asynccontextmanager -from dotenv import load_dotenv -from slowapi import Limiter, _rate_limit_exceeded_handler -from slowapi.util import get_remote_address -from slowapi.errors import RateLimitExceeded - -import os -from fastapi.staticfiles import StaticFiles -from pathlib import Path - -load_dotenv() - -from app.routes import predict, results # noqa: E402 -from app.routes import pages # noqa: E402 Phase 2: Jinja2 page routes -from app.middleware.metrics import instrumentator # noqa: E402 -from app.services.absa_pipeline import pipeline # noqa: E402 -from app.schemas.db_models import Base # noqa: E402 -from app.middleware.dependencies import engine # noqa: E402 - - -@asynccontextmanager -async def lifespan(app: FastAPI): - # Startup - print("Initializing Database tables...") - Base.metadata.create_all(bind=engine) - - print("Loading Models...") - pipeline.load_models() - - yield - # Shutdown - print("Shutting down...") - - -limiter = Limiter(key_func=get_remote_address) - -app = FastAPI( - title="Multilingual ABSA API", - description="Aspect-Based Sentiment Analysis for English and Hindi", - version="1.0.0", - lifespan=lifespan, -) - -app.state.limiter = limiter -app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) - -from app.middleware.csrf import CSRFMiddleware # noqa: E402 - -app.add_middleware( - CORSMiddleware, - allow_origins=os.getenv("CORS_ORIGINS", "http://localhost:3000,http://localhost:8000").split(","), - allow_credentials=True, - allow_methods=["GET", "POST"], - allow_headers=["*"], -) -app.add_middleware(CSRFMiddleware) # Skips /api/* routes; protects HTMX form endpoints -app.include_router(predict.router, tags=["Predict"]) -app.include_router(results.router, tags=["System"]) - -instrumentator.instrument(app).expose(app, endpoint="/metrics") - -# Jinja2 / HTMX frontend routing and static files -# Note: StaticFiles is mounted after instrumentator so prometheus -# ignores it for /metrics, although this might still log /static requests. - -app.include_router(pages.router) # include_in_schema=False is set on the router itself - -# Resolve path relative to this file so it works regardless of CWD. -_STATIC_DIR = Path(__file__).parent / "static" -_STATIC_DIR.mkdir(parents=True, exist_ok=True) # idempotent safety guard -app.mount("/static", StaticFiles(directory=str(_STATIC_DIR)), name="static") diff --git a/app/middleware/csrf.py b/app/middleware/csrf.py deleted file mode 100644 index bcd3066a6193ffdcd2471b342d8e05532a139b19..0000000000000000000000000000000000000000 --- a/app/middleware/csrf.py +++ /dev/null @@ -1,59 +0,0 @@ -import os -import re -from typing import Optional -from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.requests import Request -from starlette.responses import Response - -_CSRF_SECRET = os.getenv("CSRF_SECRET", "unsafe-default-change-in-production") -_CSRF_SALT = "csrf-token" -_SAFE_METHODS = {"GET", "HEAD", "OPTIONS", "TRACE"} -_EXEMPT_PATHS = {"/metrics", "/health", "/info", "/docs", "/openapi.json"} - -_serializer = URLSafeTimedSerializer(_CSRF_SECRET, salt=_CSRF_SALT) - - -def generate_csrf_token() -> str: - return _serializer.dumps("csrf") - - -def validate_csrf_token(token: str, max_age: int = 3600) -> bool: - try: - _serializer.loads(token, max_age=max_age) - return True - except (BadSignature, SignatureExpired): - return False - - -class CSRFMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - path = request.url.path - needs_csrf = request.method in {"POST"} and path.endswith("/fragment") - is_html_page = request.method in _SAFE_METHODS and not path.startswith("/api/") and not path.startswith("/static/") and path not in _EXEMPT_PATHS - - if needs_csrf: - csrf_cookie = request.cookies.get("csrf_token", "") - csrf_header = request.headers.get("X-CSRF-Token", "") - token = csrf_header or csrf_cookie - - if token and not validate_csrf_token(str(token)): - from fastapi.responses import HTMLResponse - return HTMLResponse( - content="

403: CSRF validation failed

Invalid or expired token. Please refresh the page.

", - status_code=403, - ) - - response: Response = await call_next(request) - - if is_html_page: - response.set_cookie( - key="csrf_token", - value=generate_csrf_token(), - max_age=3600, - secure=False, - httponly=True, - samesite="lax", - ) - - return response diff --git a/app/routes/pages.py b/app/routes/pages.py deleted file mode 100644 index 7c0a33cc29706eb7de1056beea707caffe9d3bbe..0000000000000000000000000000000000000000 --- a/app/routes/pages.py +++ /dev/null @@ -1,327 +0,0 @@ -""" -Page routes — Jinja2/HTMX web frontend. - -WHY THIS FILE EXISTS --------------------- -All HTML-serving GET routes live here, completely separate from the JSON REST -routes in routes/predict.py and routes/results.py. This boundary means: - - • REST routes never return HTML accidentally. - • Page routes never appear in the OpenAPI schema (include_in_schema=False). - • Future phases add HTMX fragment endpoints alongside these page routes - without touching any existing API code. - -WHAT THIS FILE DOES (Phase 2) ------------------------------- -Registers four GET routes that render placeholder Jinja2 templates: - GET / → redirect to /predict - GET /predict → pages/predict.html - GET /batch → pages/batch.html - GET /monitor → pages/monitor.html - -No business logic. No inference. No database queries. -The routes exist only to prove the template rendering infrastructure works. - -HTMX fragment endpoints (POST /predict/fragment, GET /batch/progress/{id}, -GET /monitor/health-partial) will be added in Phases 3-5. -""" -from __future__ import annotations - -from fastapi import APIRouter, Request, Depends, Form -from fastapi.responses import HTMLResponse -from sqlalchemy.orm import Session - -from app.core.templates import templates -from app.middleware.csrf import generate_csrf_token -from app.middleware.dependencies import get_db - -# include_in_schema=False keeps these HTML routes out of the OpenAPI / Swagger UI. -router = APIRouter(include_in_schema=False) - -# ── Navigation structure ─────────────────────────────────────────────────────── -# Mirrors the NAV constant in the React Sidebar.jsx so sidebar rendering is -# driven from a single Python list rather than hard-coded in every template. -_NAV_ITEMS: list[dict[str, str]] = [ - {"path": "/predict", "icon": "psychology", "label": "Predictor"}, - {"path": "/batch", "icon": "cloud_upload", "label": "Batch Analytics"}, - {"path": "/monitor", "icon": "monitoring", "label": "System Health"}, -] - - -def _base_ctx(request: Request, page_title: str, **extra: object) -> dict: - """ - Build the Jinja2 template context that base.html expects. - - Every page renderer calls this so the sidebar and header always receive - the nav items and the current path (for active-link highlighting). - Also includes CSRF token for HTMX form submissions. - """ - from app.middleware.csrf import generate_csrf_token - return { - "request": request, # required by Jinja2Templates - "page_title": page_title, - "nav_items": _NAV_ITEMS, - "current_path": request.url.path, - "csrf_token": generate_csrf_token(), - **extra, - } - - -# ── Routes ───────────────────────────────────────────────────────────────────── - -@router.get("/", response_class=HTMLResponse) -async def index(request: Request) -> HTMLResponse: - """Root → serve the Predict page (same behaviour as React's Navigate redirect).""" - return templates.TemplateResponse( - "pages/predict.html", - _base_ctx(request, "Live Predictor"), - ) - - -@router.get("/predict", response_class=HTMLResponse) -async def predict_page(request: Request) -> HTMLResponse: - """ - Jinja2 Live Predictor page. - Phase 2: renders the layout shell with a placeholder content block. - Phase 3: the content block will contain the HTMX predict form + result panel. - """ - return templates.TemplateResponse( - "pages/predict.html", - _base_ctx(request, "Live Predictor"), - ) - - -@router.get("/batch", response_class=HTMLResponse) -async def batch_page(request: Request) -> HTMLResponse: - """ - Jinja2 Batch Analytics page. - Phase 2: placeholder. - Phase 4: file upload form + progress polling. - """ - return templates.TemplateResponse( - "pages/batch.html", - _base_ctx(request, "Batch Analytics"), - ) - - -from app.routes.results import health_check - -@router.get("/monitor", response_class=HTMLResponse) -async def monitor_page(request: Request) -> HTMLResponse: - """ - Jinja2 System Monitor page. - Phase 5: live health status + performance metrics. - """ - try: - health = await health_check() - ctx = _base_ctx(request, "System Monitor", health=health, error=None) - except Exception: - ctx = _base_ctx(request, "System Monitor", health=None, error="Service temporarily unavailable") - - return templates.TemplateResponse("pages/monitor.html", ctx) -# ── SSE Endpoint for batch progress ────────────────────────────────────────── - -import asyncio -import json -from sse_starlette.sse import EventSourceResponse - -@router.get("/api/batch/progress/{job_id}") -async def batch_progress_sse(job_id: str, db: Session = Depends(get_db)): - """ - SSE endpoint for live batch progress updates. - Clients connect via EventSource and receive progress events every 2 seconds. - """ - async def event_generator(): - try: - import re - if not re.match(r'^[a-fA-F0-9\-]{36}$', job_id): - yield {"event": "error", "data": json.dumps({"detail": "Invalid job ID"})} - return - - while True: - job = await get_batch_status(job_id, db) - data = { - "job_id": job.job_id, - "status": job.status, - "total_reviews": job.total_reviews, - "processed": job.processed, - "result_url": job.result_url, - } - yield {"event": "progress", "data": json.dumps(data)} - - if job.status in ("completed", "failed"): - yield {"event": job.status, "data": json.dumps(data)} - break - - await asyncio.sleep(2) - except Exception: - yield {"event": "error", "data": json.dumps({"detail": "Failed to fetch job progress"})} - - return EventSourceResponse(event_generator()) - -# ── Phase 3 HTMX Endpoints ─────────────────────────────────────────────────── - -from app.schemas.schemas import ReviewInput -from app.routes.predict import predict as predict_json - -@router.post("/predict/fragment", response_class=HTMLResponse) -async def predict_fragment( - request: Request, - text: str = Form(...), - language: str = Form("auto"), - db: Session = Depends(get_db) -) -> HTMLResponse: - """ - Phase 3: HTMX partial for the Predict page. - Calls the EXACT SAME prediction logic as the JSON API. - """ - try: - if language == "auto": - language = None - - prediction = await predict_json(ReviewInput(text=text, language=language), db) - return templates.TemplateResponse( - "partials/predict_result.html", - {"request": request, "result": prediction, "error": None} - ) - except Exception: - return templates.TemplateResponse( - "partials/predict_result.html", - {"request": request, "result": None, "error": "Analysis failed. Please try again."} - ) - -# ── Phase 4 HTMX Endpoints ─────────────────────────────────────────────────── - -from fastapi import UploadFile, File -from app.routes.predict import predict_batch, get_batch_status - -@router.post("/batch/fragment", response_class=HTMLResponse) -async def batch_fragment( - request: Request, - file: UploadFile = File(...), - db: Session = Depends(get_db) -) -> HTMLResponse: - """ - Phase 4: HTMX partial for starting a batch job. - """ - try: - response = await predict_batch(file, db) - return templates.TemplateResponse( - "partials/batch_progress.html", - {"request": request, "job": response, "error": None} - ) - except Exception: - return templates.TemplateResponse( - "partials/batch_progress.html", - {"request": request, "job": None, "error": "Batch processing failed. Please try again."} - ) - -@router.get("/batch/progress/{job_id}", response_class=HTMLResponse) -async def batch_progress_fragment( - request: Request, - job_id: str, - db: Session = Depends(get_db) -) -> HTMLResponse: - """ - Phase 4: HTMX partial for polling batch job status. - """ - try: - job = await get_batch_status(job_id, db) - return templates.TemplateResponse( - "partials/batch_progress.html", - {"request": request, "job": job, "error": None} - ) - except Exception: - return templates.TemplateResponse( - "partials/batch_progress.html", - {"request": request, "job": None, "error": "Failed to retrieve job status."} - ) - -# ── Phase 5 HTMX Endpoints ─────────────────────────────────────────────────── - -@router.get("/monitor/health-partial", response_class=HTMLResponse) -async def monitor_health_fragment(request: Request) -> HTMLResponse: - """ - Phase 5: HTMX partial for polling the system health status. - Uses the exact same health logic as the JSON API. - """ - try: - health = await health_check() - return templates.TemplateResponse( - "partials/monitor_health.html", - {"request": request, "health": health, "error": None} - ) - except Exception: - return templates.TemplateResponse( - "partials/monitor_health.html", - {"request": request, "health": None, "error": "Health check failed. Service may be unavailable."} - ) - -import pandas as pd -from pathlib import Path -import json - -@router.get("/batch/charts/{job_id}", response_class=HTMLResponse) -async def batch_charts_fragment(request: Request, job_id: str) -> HTMLResponse: - """ - Phase 5.6: HTMX partial for rendering charts. - Parses the generated CSV and passes JSON directly to the template for Chart.js. - """ - try: - file_path = Path(f"data/results/{job_id}.csv") - if not file_path.exists(): - return templates.TemplateResponse("partials/batch_charts.html", {"request": request, "error": "CSV not found"}) - - df = pd.read_csv(file_path) - - lang_pie = [] - if "language" in df.columns: - counts = df["language"].value_counts().to_dict() - lang_pie = [{"name": str(k), "value": int(v)} for k, v in counts.items()] - - aspect_heat = [] - if "aspect" in df.columns and "sentiment" in df.columns: - # Group by aspect and sentiment - grouped = df.groupby(["aspect", "sentiment"]).size().unstack(fill_value=0) - for aspect, row in grouped.iterrows(): - if pd.isna(aspect) or not aspect: - continue - aspect_heat.append({ - "aspect": str(aspect), - "positive": int(row.get("positive", 0)), - "negative": int(row.get("negative", 0)), - "neutral": int(row.get("neutral", 0)), - "conflict": int(row.get("conflict", 0)) - }) - - sent_line = [] - if "sentiment" in df.columns: - df_sent = df[df["sentiment"].notna()] - n = len(df_sent) - # Create 7 chunks for the line chart - chunk_size = max(1, n // 7) if n > 0 else 1 - for i in range(7): - chunk = df_sent.iloc[i*chunk_size : (i+1)*chunk_size] - if chunk.empty: - break - counts = chunk["sentiment"].value_counts().to_dict() - sent_line.append({ - "name": f"Batch {i+1}", - "positive": int(counts.get("positive", 0)), - "negative": int(counts.get("negative", 0)), - "neutral": int(counts.get("neutral", 0)), - "conflict": int(counts.get("conflict", 0)) - }) - - return templates.TemplateResponse( - "partials/batch_charts.html", - { - "request": request, - "language_pie": json.dumps(lang_pie), - "aspect_heatmap": json.dumps(aspect_heat), - "sentiment_chart": json.dumps(sent_line), - "error": None - } - ) - except Exception: - return templates.TemplateResponse("partials/batch_charts.html", {"request": request, "error": "An unexpected error occurred while generating charts."}) diff --git a/app/static/css/app.css b/app/static/css/app.css deleted file mode 100644 index 2172e8c7d1908ceb84c74a1afa199df148bf1c5d..0000000000000000000000000000000000000000 --- a/app/static/css/app.css +++ /dev/null @@ -1,497 +0,0 @@ -/* - * SentimentAI — Application Design System - * - * WHY THIS FILE EXISTS - * -------------------- - * The React dashboard used Tailwind's @apply directive to define component - * classes (badge-positive, card, btn-primary, etc.) inside index.css. Those - * @apply rules require a compiled Tailwind build step which we are eliminating. - * - * This file replaces index.css + the @apply rules with equivalent plain CSS. - * Tailwind utility classes (bg-*, text-*, flex, etc.) are still available via - * the CDN Play CDN loaded in base.html — this file only contains component-level - * classes that the CDN cannot generate from the HTML scan. - * - * Design token values are taken verbatim from dashboard/tailwind.config.js. - * Do not change token values here without updating the Tailwind CDN config in - * base.html — they must stay in sync. - * - * SECTIONS - * -------- - * 1. Base / Reset - * 2. Material Symbols Outlined icon font - * 3. Scrollbar - * 4. Focus ring - * 5. Layout helpers (glass-panel, sidebar, overlay) - * 6. Navigation (nav-item, nav-item-active) - * 7. Badges (badge-positive, -negative, -neutral, -processing, -error) - * 8. Highlights (highlight-positive, -negative, -neutral) - * 9. Cards (card, card-low, stat-card) - * 10. Form controls (input-base, textarea reset) - * 11. Button (btn-primary) - * 12. Drag-and-drop (drag-active) - * 13. HTMX (htmx-indicator) - * 14. Animations (keyframes + helper classes) - * 15. Toast notices - */ - -/* ── 1. Base / Reset ────────────────────────────────────────────────────────── */ - -html { - color-scheme: dark; - scroll-behavior: smooth; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} - -body { - background-color: #0b1326; - color: #dae2fd; - font-family: 'Inter', ui-sans-serif, system-ui, sans-serif; - min-height: 100vh; - margin: 0; -} - -*, -*::before, -*::after { - box-sizing: border-box; -} - -/* ── 2. Material Symbols Outlined ───────────────────────────────────────────── */ -/* - * Mirrors the class defined in React's index.css exactly. - * The font itself is loaded via Google Fonts CDN in base.html. - */ -.material-symbols-outlined { - font-family: 'Material Symbols Outlined'; - font-weight: normal; - font-style: normal; - font-size: 20px; - line-height: 1; - letter-spacing: normal; - text-transform: none; - display: inline-block; - white-space: nowrap; - word-wrap: normal; - direction: ltr; - -webkit-font-smoothing: antialiased; - user-select: none; - vertical-align: middle; -} - -/* ── 3. Scrollbar ────────────────────────────────────────────────────────────── */ - -::-webkit-scrollbar { width: 6px; height: 6px; } -::-webkit-scrollbar-track { background: transparent; } -::-webkit-scrollbar-thumb { background: rgba(255, 255, 255, 0.12); border-radius: 3px; } -::-webkit-scrollbar-thumb:hover { background: rgba(255, 255, 255, 0.22); } - -/* ── 4. Focus ring ──────────────────────────────────────────────────────────── */ - -:focus-visible { - outline: 2px solid #c0c1ff; - outline-offset: 2px; -} - -/* ── 5. Layout helpers ──────────────────────────────────────────────────────── */ - -.glass-panel { - background-color: rgba(23, 31, 51, 0.75); - backdrop-filter: blur(12px); - -webkit-backdrop-filter: blur(12px); - border: 1px solid rgba(255, 255, 255, 0.06); -} - -/* Sidebar slide-in / slide-out on mobile */ -.sidebar { - transform: translateX(-100%); - transition: transform 250ms ease-out; -} -.sidebar.sidebar--open { - transform: translateX(0); -} -@media (min-width: 768px) { - .sidebar { - transform: translateX(0); - } -} - -/* Mobile overlay (backdrop) */ -.sidebar-overlay { - display: none; - position: fixed; - inset: 0; - background-color: rgba(0, 0, 0, 0.6); - backdrop-filter: blur(4px); - -webkit-backdrop-filter: blur(4px); - z-index: 40; -} -.sidebar-overlay.sidebar-overlay--visible { - display: block; -} - -/* ── 6. Navigation ──────────────────────────────────────────────────────────── */ - -.nav-item { - display: flex; - align-items: center; - gap: 12px; - padding: 10px 12px; - border-radius: 8px; - color: #c7c4d7; - font-size: 14px; - line-height: 20px; - font-weight: 500; - text-decoration: none; - cursor: pointer; - transition: color 150ms ease, background-color 150ms ease; -} -.nav-item:hover { - background-color: rgba(255, 255, 255, 0.05); - color: #dae2fd; -} - -.nav-item-active { - display: flex; - align-items: center; - gap: 12px; - padding: 10px 12px; - border-radius: 8px; - color: #c0c1ff; - background-color: rgba(255, 255, 255, 0.07); - border-right: 2px solid #c0c1ff; - font-size: 14px; - line-height: 20px; - font-weight: 600; - text-decoration: none; -} - -/* ── 7. Badges ──────────────────────────────────────────────────────────────── */ -/* - * All badges share the same structural CSS. The colour variant is applied - * via the class suffix. Each badge is intentionally uppercase + monospace - * to match the React component styling. - */ - -.badge-base { - display: inline-flex; - align-items: center; - gap: 6px; - padding: 2px 8px; - border-radius: 9999px; - font-family: 'JetBrains Mono', ui-monospace, monospace; - font-size: 11px; - line-height: 16px; - font-weight: 500; - letter-spacing: 0.06em; - text-transform: uppercase; -} - -.badge-positive { - display: inline-flex; - align-items: center; - gap: 6px; - padding: 2px 8px; - border-radius: 9999px; - background-color: rgba(78, 222, 163, 0.10); - color: #4edea3; - border: 1px solid rgba(78, 222, 163, 0.25); - font-family: 'JetBrains Mono', ui-monospace, monospace; - font-size: 11px; - line-height: 16px; - font-weight: 500; - letter-spacing: 0.06em; - text-transform: uppercase; -} - -.badge-negative { - display: inline-flex; - align-items: center; - gap: 6px; - padding: 2px 8px; - border-radius: 9999px; - background-color: rgba(255, 180, 171, 0.10); - color: #ffb4ab; - border: 1px solid rgba(255, 180, 171, 0.25); - font-family: 'JetBrains Mono', ui-monospace, monospace; - font-size: 11px; - line-height: 16px; - font-weight: 500; - letter-spacing: 0.06em; - text-transform: uppercase; -} - -.badge-neutral { - display: inline-flex; - align-items: center; - gap: 6px; - padding: 2px 8px; - border-radius: 9999px; - background-color: rgba(144, 143, 160, 0.10); - color: #c7c4d7; - border: 1px solid rgba(144, 143, 160, 0.25); - font-family: 'JetBrains Mono', ui-monospace, monospace; - font-size: 11px; - line-height: 16px; - font-weight: 500; - letter-spacing: 0.06em; - text-transform: uppercase; -} - -.badge-processing { - display: inline-flex; - align-items: center; - gap: 6px; - padding: 2px 8px; - border-radius: 9999px; - background-color: rgba(192, 193, 255, 0.10); - color: #c0c1ff; - border: 1px solid rgba(192, 193, 255, 0.25); - font-family: 'JetBrains Mono', ui-monospace, monospace; - font-size: 11px; - line-height: 16px; - font-weight: 500; - letter-spacing: 0.06em; - text-transform: uppercase; - animation: pulse-badge 2s cubic-bezier(0.4, 0, 0.6, 1) infinite; -} - -.badge-error { - display: inline-flex; - align-items: center; - gap: 6px; - padding: 2px 8px; - border-radius: 9999px; - background-color: rgba(255, 180, 171, 0.10); - color: #ffb4ab; - border: 1px solid rgba(255, 180, 171, 0.25); - font-family: 'JetBrains Mono', ui-monospace, monospace; - font-size: 11px; - line-height: 16px; - font-weight: 500; - letter-spacing: 0.06em; - text-transform: uppercase; -} - -/* Dot inside badge */ -.badge-dot { - width: 6px; - height: 6px; - border-radius: 9999px; - flex-shrink: 0; - display: inline-block; -} -.badge-dot--positive { background-color: #4edea3; } -.badge-dot--negative { background-color: #ffb4ab; } -.badge-dot--neutral { background-color: #908fa0; } -.badge-dot--primary { background-color: #c0c1ff; } -.badge-dot--error { background-color: #ffb4ab; } - -/* ── 8. Highlights (annotated text) ─────────────────────────────────────────── */ -/* - * Applied by the server-side annotated-text builder (Phase 3) to wrap - * aspect spans inside the review text. - */ - -.highlight-positive { - background-color: rgba(78, 222, 163, 0.15); - color: #4edea3; - border: 1px solid rgba(78, 222, 163, 0.30); - border-radius: 4px; - padding: 0 4px; - margin: 0 2px; - font-weight: 500; -} - -.highlight-negative { - background-color: rgba(255, 180, 171, 0.15); - color: #ffb4ab; - border: 1px solid rgba(255, 180, 171, 0.30); - border-radius: 4px; - padding: 0 4px; - margin: 0 2px; - font-weight: 500; -} - -.highlight-neutral { - background-color: rgba(144, 143, 160, 0.15); - color: #c7c4d7; - border: 1px solid rgba(144, 143, 160, 0.25); - border-radius: 4px; - padding: 0 4px; - margin: 0 2px; - font-weight: 500; -} - -/* ── 9. Cards ────────────────────────────────────────────────────────────────── */ - -.card { - background-color: #171f33; - border-radius: 12px; - border: 1px solid rgba(255, 255, 255, 0.08); - padding: 24px; -} - -.card-low { - background-color: #131b2e; - border-radius: 12px; - border: 1px solid rgba(255, 255, 255, 0.06); - padding: 24px; -} - -.stat-card { - background-color: #171f33; - border-radius: 12px; - border: 1px solid rgba(255, 255, 255, 0.08); - padding: 24px; - position: relative; - overflow: hidden; -} - -/* ── 10. Form controls ──────────────────────────────────────────────────────── */ - -.input-base { - background-color: #0b1326; - border: 1px solid rgba(255, 255, 255, 0.12); - border-radius: 8px; - padding: 8px 12px; - font-size: 14px; - line-height: 20px; - color: #dae2fd; - width: 100%; - transition: border-color 150ms ease, box-shadow 150ms ease; - appearance: none; - -webkit-appearance: none; -} - -.input-base::placeholder { - color: rgba(199, 196, 215, 0.60); -} - -.input-base:focus { - outline: none; - border-color: #c0c1ff; - box-shadow: 0 0 0 1px rgba(192, 193, 255, 0.40); -} - -/* Select arrow */ -select.input-base { - background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='8' viewBox='0 0 12 8'%3E%3Cpath d='M1 1l5 5 5-5' stroke='%23c7c4d7' stroke-width='1.5' fill='none' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E"); - background-repeat: no-repeat; - background-position: right 10px center; - padding-right: 32px; - cursor: pointer; -} - -/* Textarea */ -textarea.input-base { - resize: vertical; - font-family: inherit; - line-height: 1.6; -} - -/* ── 11. Button — primary ────────────────────────────────────────────────────── */ - -.btn-primary { - display: inline-flex; - align-items: center; - justify-content: center; - gap: 8px; - background-color: #c0c1ff; - color: #1000a9; - font-family: 'JetBrains Mono', ui-monospace, monospace; - font-size: 12px; - line-height: 16px; - font-weight: 500; - letter-spacing: 0.05em; - padding: 10px 20px; - border-radius: 8px; - border: none; - cursor: pointer; - text-decoration: none; - transition: filter 150ms ease, transform 150ms ease; - white-space: nowrap; -} -.btn-primary:hover { filter: brightness(1.10); } -.btn-primary:active { transform: scale(0.98); } -.btn-primary:disabled, -.btn-primary[disabled] { - opacity: 0.50; - cursor: not-allowed; - transform: none; - pointer-events: none; -} - -/* ── 12. Drag-and-drop ──────────────────────────────────────────────────────── */ - -.drag-active { - border-color: rgba(192, 193, 255, 0.70) !important; - background-color: rgba(192, 193, 255, 0.04) !important; -} - -/* ── 13. HTMX indicators ────────────────────────────────────────────────────── */ -/* - * HTMX adds .htmx-request to the element that triggered the request. - * Elements with .htmx-indicator are hidden by default and shown during request. - */ -.htmx-indicator { display: none; } -.htmx-request .htmx-indicator { display: flex; } -.htmx-request.htmx-indicator { display: flex; } - -/* Progress bar fill animation */ -.progress-bar { - transition: width 500ms ease; -} - -/* ── 14. Animations ──────────────────────────────────────────────────────────── */ - -@keyframes fadeIn { - from { opacity: 0; } - to { opacity: 1; } -} - -@keyframes slideIn { - from { opacity: 0; transform: translateY(8px); } - to { opacity: 1; transform: translateY(0); } -} - -@keyframes pulse-badge { - 0%, 100% { opacity: 1; } - 50% { opacity: 0.6; } -} - -@keyframes pulse-dot { - 0%, 100% { opacity: 1; } - 50% { opacity: 0.4; } -} - -@keyframes spin { - from { transform: rotate(0deg); } - to { transform: rotate(360deg); } -} - -.animate-fade-in { animation: fadeIn 0.20s ease-out; } -.animate-slide-in { animation: slideIn 0.25s ease-out; } -.animate-spin { animation: spin 1s linear infinite; } -.animate-pulse-slow { animation: pulse-dot 3s cubic-bezier(0.4, 0, 0.6, 1) infinite; } - -/* ── 15. Toast notifications ─────────────────────────────────────────────────── */ -/* - * Toasts are managed by Alpine.js appState(). - * Base styles here; position and z-index are set with Tailwind utilities in base.html. - */ -.toast { - padding: 12px 16px; - border-radius: 8px; - border: 1px solid rgba(255, 255, 255, 0.08); - background-color: #222a3d; - color: #dae2fd; - font-size: 14px; - max-width: 380px; - pointer-events: auto; - transition: opacity 150ms ease, transform 150ms ease; -} -.toast--success { border-color: rgba(78, 222, 163, 0.30); color: #4edea3; } -.toast--error { border-color: rgba(255, 180, 171, 0.30); color: #ffb4ab; } -.toast--info { border-color: rgba(192, 193, 255, 0.20); color: #dae2fd; } diff --git a/app/templates/base.html b/app/templates/base.html deleted file mode 100644 index e0550d157d530aab97d2579ab1c722a5809cf495..0000000000000000000000000000000000000000 --- a/app/templates/base.html +++ /dev/null @@ -1,449 +0,0 @@ - - - - - - {% block title %}{{ page_title }}{% endblock %} — SentimentAI - - - - {# ── Fonts ────────────────────────────────────────────────────────────────── #} - {# Inter replaces Geist (same design language, available on Google Fonts CDN). #} - {# JetBrains Mono is used verbatim from the original tailwind.config.js. #} - - - - - {# ── Material Symbols Outlined icon font ──────────────────────────────────── #} - {# Variable-font version so FILL and wght axes are available (matching React). #} - - - {# - ── Tailwind CSS (Play CDN) ─────────────────────────────────────────────── - The CDN version scans the DOM at runtime and generates utilities on demand. - This eliminates the npm build step. Custom design tokens from the original - tailwind.config.js are provided in the tailwind.config object below. - - NOTE: The config - - - {# ── Custom design system CSS ──────────────────────────────────────────────── #} - {# Component classes that use @apply in the React version are written as plain #} - {# CSS in app.css (since the Play CDN does not process @apply in external CSS). #} - - - {# ── Per-page extra head content ──────────────────────────────────────────── #} - {% block head %}{% endblock %} - - - - - {# ── Toast notification container ────────────────────────────────────────── #} - {# Managed by Alpine.js appState(). HTMX error events dispatch to this. #} -
- -
- - {# ── Page wrapper ─────────────────────────────────────────────────────────── #} -
- - {# ── Mobile sidebar overlay ──────────────────────────────────────────────── #} - - - {# ── Sidebar ─────────────────────────────────────────────────────────────── #} - {# - The sidebar is always visible on desktop (md:translate-x-0) and slides in - on mobile when sidebarOpen is true. Alpine.js toggles the translation via - :class binding. The safelist in tailwind.config ensures translate-x-0 and - -translate-x-full are always generated by the CDN. - #} - - - {# ── Main area (right of sidebar) ────────────────────────────────────────── #} -
- - {# Top header bar #} -
- -
- {# Mobile hamburger #} - - - {# Mobile brand (hidden on desktop) #} -
- - SentimentAI -
- - {# Desktop page title (hidden on mobile) #} - -
- - {# Right action group #} -
- - - - - {# Avatar placeholder #} - -
-
- - {# Page content area #} -
-
- {% block content %}{% endblock %} -
-
- - {# Footer #} - - -
{# /main area #} -
{# /page wrapper #} - - {# ── JavaScript ────────────────────────────────────────────────────────────── #} - - {# - Alpine.js state — defined BEFORE the defer script so appState() is in scope - when Alpine.js initialises after DOMContentLoaded. - #} - - - {# HTMX — loaded deferred so it does not block rendering #} - - - {# Alpine.js v3 — loaded deferred; appState() above is already in global scope #} - - - {# ── Per-page extra scripts ───────────────────────────────────────────────── #} - {% block scripts %}{% endblock %} - - - diff --git a/app/templates/macros/ui.html b/app/templates/macros/ui.html deleted file mode 100644 index da0e2b1066d93899f2ef1c8525e640c33ad40137..0000000000000000000000000000000000000000 --- a/app/templates/macros/ui.html +++ /dev/null @@ -1,178 +0,0 @@ -{# - macros/ui.html — Reusable Jinja2 macros for the SentimentAI dashboard. - - WHY THIS FILE EXISTS - -------------------- - The React components (Sidebar.jsx, Monitor.jsx, Analytics.jsx) defined small - helper components (MSIcon, StatusBadge, HealthChip, InfoRow) that were reused - across pages. Jinja2 macros provide the equivalent pattern: define once, - import anywhere. - - USAGE IN TEMPLATES - ------------------ - {% from "macros/ui.html" import ms_icon, status_badge, health_chip, sentiment_badge %} - - Each macro produces self-contained HTML with no JavaScript dependencies. - CSS classes reference app.css component classes (badge-*, highlight-*) and - Tailwind utilities from the CDN. -#} - -{# ── Material Symbols Outlined icon ───────────────────────────────────────── #} -{# - ms_icon(name, filled, size, cls) - --------------------------------- - Renders a Material Symbols Outlined icon span. - - Parameters - ---------- - name : str — icon name e.g. "psychology", "cloud_upload", "bolt" - filled : bool — whether to use FILL=1 (solid) or FILL=0 (outlined) - size : int — font-size in px (default 20) - cls : str — extra CSS classes appended to the span -#} -{% macro ms_icon(name, filled=False, size=20, cls='') %} - -{% endmacro %} - - -{# ── Status badge ─────────────────────────────────────────────────────────── #} -{# - status_badge(status) - --------------------- - Renders a coloured badge for a batch job status. - - status values: "completed" | "processing" | "queued" | "failed" -#} -{% macro status_badge(status) %} -{% if status == 'completed' %} - - - Completed - -{% elif status == 'processing' %} - - - Processing - -{% elif status == 'queued' %} - - - Queued - -{% elif status == 'failed' %} - - - Failed - -{% else %} -{{ status }} -{% endif %} -{% endmacro %} - - -{# ── Health chip ──────────────────────────────────────────────────────────── #} -{# - health_chip(ok) - ---------------- - Renders a "Healthy" or "Degraded" status chip. - - ok : bool — True if the API health check returned status=="ok" -#} -{% macro health_chip(ok) %} -{% if ok %} - - - Healthy - -{% else %} - - - Degraded - -{% endif %} -{% endmacro %} - - -{# ── Sentiment badge ──────────────────────────────────────────────────────── #} -{# - sentiment_badge(sentiment) - --------------------------- - Renders the sentiment label for an aspect card. - - sentiment values: "positive" | "negative" | "neutral" | "conflict" -#} -{% macro sentiment_badge(sentiment) %} -{% if sentiment == 'positive' %} - - - {{ sentiment }} - -{% elif sentiment == 'negative' %} - - - {{ sentiment }} - -{% elif sentiment == 'conflict' %} - - - {{ sentiment }} - -{% else %} - - - {{ sentiment }} - -{% endif %} -{% endmacro %} - - -{# ── Info row (used on Monitor page) ─────────────────────────────────────── #} -{# - info_row(label, value, value_cls) - ----------------------------------- - Renders a labelled key-value cell inside the Model Configuration card. -#} -{% macro info_row(label, value, value_cls='') %} -
-

{{ label }}

-

{{ value }}

-
-{% endmacro %} - - -{# ── Loaded badge (aspect/sentiment model status) ────────────────────────── #} -{% macro loaded_badge() %} -
- {{ ms_icon('check_circle', filled=False, size=16, cls='text-[#4edea3]') }} - Loaded -
-{% endmacro %} - - -{# ── Skeleton placeholder (used during HTMX loading states) ─────────────── #} -{# - skeleton(height, width_cls) - ---------------------------- - Renders a pulsing skeleton placeholder matching the React Skeleton component. -#} -{% macro skeleton(height='h-4', width_cls='w-full') %} -
-{% endmacro %} - - -{# ── Empty-state panel ────────────────────────────────────────────────────── #} -{# - empty_state(icon, message) - --------------------------- - Centred icon + message for panels with no data yet. -#} -{% macro empty_state(icon='psychology', message='No data yet') %} -
- -

{{ message }}

-
-{% endmacro %} diff --git a/app/templates/pages/batch.html b/app/templates/pages/batch.html deleted file mode 100644 index ce3f55e7c81a9813b725e44322e3b117fa15a604..0000000000000000000000000000000000000000 --- a/app/templates/pages/batch.html +++ /dev/null @@ -1,124 +0,0 @@ -{% extends "base.html" %} -{% from "macros/ui.html" import ms_icon, status_badge, empty_state %} - -{% block title %}Batch Analytics{% endblock %} -{% block description %}Upload a CSV of reviews for bulk aspect-based sentiment analysis.{% endblock %} - -{% block content %} -{# - Phase 2: Placeholder layout for the Batch Analytics page. - - What this page will contain after Phase 4: - - Upload zone : drag-and-drop CSV input (HTML5 + Alpine.js drag events) - - Progress card: job status + progress bar polling via hx-trigger="every 2s" - - Recent table : DB-queried list of past batch jobs - - Charts : post-completion AspectHeatmap, LanguagePie, SentimentChart - (Chart.js, rendered with server-provided JSON data) - - The page structure below mirrors Analytics.jsx exactly so Phase 4 only needs to - replace placeholder content with functional forms and HTMX targets. -#} -
- - {# ── Page header ─────────────────────────────────────────────────────────── #} -
-

Batch Analytics

-

- Upload a CSV of reviews for bulk aspect-based sentiment analysis. -

-
- - {# ── Upload zone ──────────────────────────────────────────────────────────── #} -
-
- -
- - {{ ms_icon('cloud_upload', size=48, cls='text-[#c7c4d7] mb-4') }} -

-

- Must contain a text column. - Maximum 10,000 rows. Files are deleted after analysis. -

-
- - - -
- -
-
- - {# ── Recent batches table placeholder ─────────────────────────────────────── #} -
-

Recent Batches

-
-
- - - - - - - - - - - {# Phase 2: static mock rows matching React's Analytics.jsx mock data #} - {% for row in [ - {'name': 'q3_customer_feedback.csv', 'rows': '4,250', 'status': 'completed', 'date': 'Today, 14:32'}, - {'name': 'product_launch_tweets.csv', 'rows': '8,912', 'status': 'processing', 'date': 'Today, 14:15'}, - {'name': 'corrupted_export_09.csv', 'rows': '—', 'status': 'failed', 'date': 'Yesterday'}, - ] %} - - - - - - - {% endfor %} - -
FilenameRowsStatusDate
-
- {{ ms_icon('description', size=16, cls='text-[#c7c4d7]') }} - {{ row.name }} -
- 		{{ row.rows }}{{ status_badge(row.status) }}{{ row.date }}
-
-
-
- -
-{% endblock %} diff --git a/app/templates/pages/monitor.html b/app/templates/pages/monitor.html deleted file mode 100644 index b3f9c982d4e117e0403d513caecdd9897c30493c..0000000000000000000000000000000000000000 --- a/app/templates/pages/monitor.html +++ /dev/null @@ -1,161 +0,0 @@ -{% extends "base.html" %} -{% from "macros/ui.html" import ms_icon, health_chip, info_row, loaded_badge %} - -{% block title %}System Monitor{% endblock %} -{% block description %}Real-time API health, model metadata, and request statistics.{% endblock %} - -{% block content %} -{# - Phase 2: Placeholder layout for the System Monitor page. - - What this page will contain after Phase 5: - - API Status card : live health chip polled via hx-trigger="every 30s" - - Model Configuration : architecture, languages, model status - - Performance Metrics : stat cards with SVG sparklines - - Recent Activity table : last N API requests - - All section structures below match Monitor.jsx exactly so Phase 5 only needs to - add live data and HTMX polling attributes. -#} -
- - {# ── Page header ─────────────────────────────────────────────────────────── #} -
-
-

System Monitor

-

- Real-time API health, model metadata, and request statistics. -

-
-
- - -
-
- - {# ── Health + Model config ────────────────────────────────────────────────── #} -
- - {# API Status card #} -
-
-
- {{ ms_icon('monitor_heart', size=24, cls='text-[#4edea3]') }} -
-
-

API Status

-

Core Inference Engine

-
-
- - {# Health status — HTMX polling target in Phase 5 #} - {% include "partials/monitor_health.html" %} -
- - {# Model Configuration card #} -
-
-
- {{ ms_icon('memory', size=24, cls='text-[#c0c1ff]') }} -
-
-

Model Configuration

-

Loaded ONNX Graphs

-
-
-
- {{ info_row('Architecture', 'XLM-RoBERTa (INT8)') }} - {{ info_row('Supported Languages', 'English, Hindi, Hinglish') }} -
-

Aspect Extraction

- {{ loaded_badge() }} -
-
-

Sentiment Classification

- {{ loaded_badge() }} -
-
-
-
- - {# ── Performance metrics (stat cards) ─────────────────────────────────────── #} -
-

Performance Metrics

-
- - {# Macro-style stat card — defined inline since it's used only here #} - {% for stat in [ - {'icon': 'database', 'label': 'Total Requests Today', 'value': '12.4k', 'sub': '↑ 8% vs yesterday', 'color': '#c0c1ff', 'positive': true}, - {'icon': 'bolt', 'label': 'Avg Latency (P95)', 'value': '145ms', 'sub': 'Well within SLA', 'color': '#4edea3', 'positive': true}, - {'icon': 'warning', 'label': 'Error Rate', 'value': '0.2%', 'sub': 'Last 24 hours', 'color': '#ffb4ab', 'positive': false}, - ] %} -
- {# Sparkline gradient background #} -
- -
-
-
- - {{ stat.label }} -
-

{{ stat.value }}

-

{{ stat.sub }}

-
-
- {% endfor %} - -
-
- - {# ── Recent endpoint activity ──────────────────────────────────────────────── #} -
-

Recent Endpoint Activity

-
- {% for req in [ - {'method': 'POST', 'path': '/predict', 'status': 200, 'time': '3.5ms', 'ago': '2s ago'}, - {'method': 'GET', 'path': '/health', 'status': 200, 'time': '0.8ms', 'ago': '5s ago'}, - {'method': 'POST', 'path': '/batch', 'status': 202, 'time': '12.1ms', 'ago': '1m ago'}, - {'method': 'GET', 'path': '/status/abc12', 'status': 200, 'time': '1.2ms', 'ago': '1m ago'}, - {'method': 'POST', 'path': '/predict', 'status': 500, 'time': '23ms', 'ago': '3m ago'}, - ] %} -
- - {{ req.method }} - - {{ req.path }} - - {{ req.status }} - - {{ req.time }} - -
- {% endfor %} -
- - -
- -
-{% endblock %} diff --git a/app/templates/pages/predict.html b/app/templates/pages/predict.html deleted file mode 100644 index 6a25ab581de197d3370fb46a50b855f3ed1e9ffa..0000000000000000000000000000000000000000 --- a/app/templates/pages/predict.html +++ /dev/null @@ -1,107 +0,0 @@ -{% extends "base.html" %} -{% from "macros/ui.html" import ms_icon, empty_state %} - -{% block title %}Live Predictor{% endblock %} -{% block description %}Analyze aspect-based sentiment in English and Hindi text in real time.{% endblock %} - -{% block content %} -{# - Phase 2: Placeholder layout for the Predict page. - - What this page will contain after Phase 3: - - Left panel : textarea input + language selector + Analyze button - - Right panel : annotated result text + aspect cards (HTMX swap target) - - POST /predict/fragment → returns partials/predict_result.html - - The outer grid, headings, and card shells are already correct here so Phase 3 - only needs to fill in the form and wire up HTMX — no structural changes. -#} -
- - {# ── Page header ─────────────────────────────────────────────────────────── #} -
-

Live Sentiment Predictor

-

- Enter text to analyze its aspects and sentiments in real-time. - The model automatically identifies the language and extracts key phrases. -

-
- - {# ── Two-column layout (mirrors LivePredictor.jsx structure) ─────────────── #} -
- - {# ── Left: Input panel ───────────────────────────────────────────────── #} -
-
- -
-

- Analyze Input -

-
- - -
-
- -
- - -
- ⌘ Enter to analyze - 0 / 512 -
-
- - - -
-
- - {# ── Right: Results panel ─────────────────────────────────────────────── #} -
-
- -
-

- Analysis Results -

-
- - {# Phase 2 placeholder — replaced by HTMX partial in Phase 3 #} -
- {{ empty_state('psychology', 'Enter a review and click Analyze to see results') }} -
- -
-
- -
{# /grid #} - - - -
-{% endblock %} diff --git a/app/templates/partials/batch_charts.html b/app/templates/partials/batch_charts.html deleted file mode 100644 index 9f7f524a12cce2a9d94aed7752996708d0208e3d..0000000000000000000000000000000000000000 --- a/app/templates/partials/batch_charts.html +++ /dev/null @@ -1,114 +0,0 @@ -{% if error %} -
-

Failed to load charts: {{ error }}

-
-{% else %} - -
- - {# Aspect Heatmap (Stacked Bar) #} -
-

Top Aspects by Sentiment

-
- -
-
- - {# Language Pie #} -
-

Language Distribution

-
- -
-
- - {# Sentiment Chart (Line) #} -
-

Sentiment Over Time (Row Chunks)

-
- -
-
- -
- - -{% endif %} diff --git a/app/templates/partials/batch_progress.html b/app/templates/partials/batch_progress.html deleted file mode 100644 index 2adcf8b822ab942ec881785b5cc52bde4a7f3ae0..0000000000000000000000000000000000000000 --- a/app/templates/partials/batch_progress.html +++ /dev/null @@ -1,51 +0,0 @@ -{% from "macros/ui.html" import ms_icon %} - -
- {% if error %} -
- {{ ms_icon('error', size=48, cls='opacity-80') }} -

{{ error }}

-
- {% else %} -
-
-

Batch Job: {{ job.job_id }}

-

Status: {{ job.status }}

-
- {% if job.status == 'completed' %} - - {{ ms_icon('download', size=18) }} - Download Results - - {% elif job.status == 'failed' %} - {{ ms_icon('error', size=16) }} Failed - {% else %} -
- autorenew - Processing... -
- {% endif %} -
- -
- {% set percent = (job.processed / job.total_reviews * 100) if job.total_reviews > 0 else 0 %} -
-
- -
- {{ job.processed }} processed - {{ job.total_reviews }} total -
- - {% if job.status in ['queued', 'processing'] %} -
- {% elif job.status == 'completed' %} -
-
- analytics - Generating charts... -
-
- {% endif %} - {% endif %} -
diff --git a/app/templates/partials/monitor_health.html b/app/templates/partials/monitor_health.html deleted file mode 100644 index 3aa191e978f9f4366fff60b894cc811381570e7e..0000000000000000000000000000000000000000 --- a/app/templates/partials/monitor_health.html +++ /dev/null @@ -1,19 +0,0 @@ -{% from "macros/ui.html" import health_chip %} - -
- Current state: - - {% if error %} - - - Service Unavailable - - {% else %} - {{ health_chip(health.status == 'ok') }} - {% endif %} - Updating... -
diff --git a/app/templates/partials/predict_result.html b/app/templates/partials/predict_result.html deleted file mode 100644 index 91d9a3b6aecaedebc7bccd0f62a429f8237e4af6..0000000000000000000000000000000000000000 --- a/app/templates/partials/predict_result.html +++ /dev/null @@ -1,101 +0,0 @@ -{% from "macros/ui.html" import ms_icon, sentiment_badge, empty_state %} - -{% if error %} -
- {{ ms_icon('error', size=48, cls='opacity-80') }} -

{{ error }}

-
-{% elif result %} - {# Macro to highlight text based on aspect positions #} - {% macro render_annotated_text(text, aspects) %} - {# - For Phase 3 we do a simplified highlight. - In a real template, we'd slice the text by start_pos/end_pos. - For now, we just print the text, as doing complex string slicing in Jinja is hard. - Wait, we can pass an 'annotated_text' pre-computed from the router, but the prompt says "do NOT duplicate business logic". - Let's just output the text, or if possible, use JS or a simple replace. - Actually, the user expects "highlight-positive" etc. from the CSS we wrote in Phase 2. - #} -
{{ text }}
- {% endmacro %} - -
- - {# ── Header metrics ── #} -
-
-
- {{ ms_icon('language', size=16) }} - {{ result.detected_language | upper }} -
-
- {{ ms_icon('timer', size=16) }} - {{ result.processing_time_ms | round }}ms -
-
-
- - {# ── Annotated Text ── #} -
-
{{ result.text }}
-
- - {# ── Aspects List ── #} -
-

Detected Aspects

- {% if result.aspects %} -
- {% for aspect in result.aspects %} -
- {{ aspect.aspect }} -
- conf: {{ "%.2f"|format(aspect.confidence) }} - {{ sentiment_badge(aspect.sentiment) }} -
-
- {% endfor %} -
- {% else %} -
-

No aspects detected in this text.

-
- {% endif %} -
- - {# ── Client-side Text Highlighting ── #} - {# Uses DOM API to safely highlight aspect spans — never innerHTML with user data #} - - -
-{% else %} - {{ empty_state('psychology', 'Enter a review and click Analyze to see results') }} -{% endif %} diff --git a/docker/Dockerfile b/docker/Dockerfile index 62ea45fd752ca7609979445fb323c10483cdca14..2c80af27b9f111a671b155474b00ccedc5bd022d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,14 +2,16 @@ FROM python:3.11-slim AS builder WORKDIR /app -COPY requirements.txt . +COPY pyproject.toml . +COPY src ./src +COPY api ./api RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ git \ && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir --prefix=/install -r requirements.txt +RUN pip install --no-cache-dir --prefix=/install . # Stage 2: Runtime FROM python:3.11-slim @@ -20,13 +22,15 @@ WORKDIR /app COPY --from=builder /install /usr/local # Copy application code -COPY app /app/app +COPY api /app/api +COPY src /app/src COPY scripts /app/scripts -COPY absa /app/absa COPY docker /app/docker COPY .env.example /app/.env.example # .env is injected via docker-compose environment vars — no need to COPY it +ENV PYTHONPATH=/app/src + # Add a non-root user RUN adduser --disabled-password --gecos "" absauser \ && chown -R absauser /app @@ -35,4 +39,4 @@ USER absauser EXPOSE 8000 -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/Dockerfile.prod b/docker/Dockerfile.prod index 723da770e535a6da0d7422592a47414ce80a7c3a..899d9bf8d2bc46ef019c760c9cb0d1f06d1ef6f6 100644 --- a/docker/Dockerfile.prod +++ b/docker/Dockerfile.prod @@ -1,17 +1,19 @@ FROM python:3.11-slim as builder WORKDIR /app -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +COPY pyproject.toml . +COPY src ./src +COPY api ./api +RUN pip install --no-cache-dir . FROM python:3.11-slim as runtime WORKDIR /app COPY --from=builder /usr/local/lib/python3.11 /usr/local/lib/python3.11 COPY --from=builder /usr/local/bin /usr/local/bin -COPY app/ ./app/ -COPY absa/ ./absa/ -ENV PYTHONPATH=/app +COPY api/ ./api/ +COPY src/ ./src/ +ENV PYTHONPATH=/app/src ENV MODEL_SOURCE=huggingface_hub RUN useradd -m appuser && chown -R appuser /app USER appuser EXPOSE 8000 -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"] +CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 07e32b71d0faa08b781260332e13ff4ea463cf82..76f1731036ff08a39420cb69762e0d67746e9b3a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -25,7 +25,7 @@ services: context: ../ dockerfile: Dockerfile container_name: absa-worker - command: ["celery", "-A", "app.tasks", "worker", "--loglevel=info"] + command: ["celery", "-A", "api.tasks", "worker", "--loglevel=info"] environment: - DATABASE_URL=${DATABASE_URL} - REDIS_URL=${REDIS_URL} diff --git a/docs/HTMX_MIGRATION.md b/docs/HTMX_MIGRATION.md deleted file mode 100644 index 8114ac19a61b85b4e5cdd107047276b1533aabb2..0000000000000000000000000000000000000000 --- a/docs/HTMX_MIGRATION.md +++ /dev/null @@ -1,92 +0,0 @@ -# HTMX Migration - -## Summary - -The Streamlit frontend (`streamlit_app/`) has been fully replaced with an HTMX + Jinja2 frontend served directly by the FastAPI backend. This eliminates the separate Streamlit server, reduces resource usage, and provides a single unified server for both UI and API. - -## What Changed - -| Before | After | -|--------|-------| -| Streamlit frontend at `streamlit_app/` | HTMX + Jinja2 at `api/app/templates/` | -| Two servers: FastAPI (8000) + Streamlit (8501) | Single FastAPI server (8000) | -| `streamlit run streamlit_app/Home.py` | `uvicorn app.main:app --reload` | -| Plotly for charts | Chart.js (client-side rendering) | -| Streamlit state management | Alpine.js + HTMX for interactivity | - -## Architecture - -``` -User Browser - ↕ HTMX / Alpine.js -FastAPI (port 8000) - ├── /predict → Predict page (HTML via Jinja2) - ├── /batch → Batch upload page (HTML) - ├── /monitor → System monitor page (HTML) - ├── /docs → Swagger UI (unchanged) - ├── /api/predict → JSON API (unchanged) - ├── /api/batch → JSON API (unchanged) - ├── /predict/fragment → HTMX fragment (HTML partial) - ├── /batch/fragment → HTMX fragment (HTML partial) - └── /api/batch/progress/{job_id} → SSE endpoint -``` - -## Frontend Stack - -- **HTMX 2.0.3** - AJAX, CSS transitions, WebSocket/SSE -- **Alpine.js 3.14** - Reactive UI state (toasts, sidebar) -- **Tailwind CSS (Play CDN)** - Utility-first CSS -- **Chart.js** - Client-side charts (batch results) -- **Material Symbols** - Icon font -- **itsdangerous** - CSRF protection - -## CSRF Protection - -All HTMX form endpoints require a CSRF token. The token is: -- Set as an `HttpOnly` cookie on every GET response -- Injected into `` in `base.html` -- Automatically attached to HTMX requests via `htmx:configRequest` event handler -- Validated by `CSRFMiddleware` for all non-GET, non-API requests - -## File Structure - -``` -api/app/ -├── templates/ -│ ├── base.html # Base layout with nav, sidebar, toast system -│ ├── pages/ -│ │ ├── predict.html # Single review prediction form -│ │ ├── batch.html # CSV upload + progress + results -│ │ └── monitor.html # Health stats + performance metrics -│ ├── partials/ -│ │ ├── predict_result.html # Prediction result card -│ │ ├── batch_progress.html # Batch job progress bar -│ │ ├── batch_charts.html # Chart.js charts -│ │ └── monitor_health.html # Health status chip -│ └── macros/ -│ └── ui.html # Reusable components (badges, icons, etc.) -├── static/ -│ └── css/ -│ └── app.css # Design system components -├── core/ -│ └── templates.py # Centralized Jinja2Templates instance -├── middleware/ -│ └── csrf.py # CSRF protection middleware -└── main.py # FastAPI app entry point -``` - -## Deleted Files - -- `streamlit_app/` (entire directory) -- `config/docker/Dockerfile.streamlit` -- Streamlit service in `config/docker/docker-compose.yml` -- Streamlit dependency from `requirements.txt` -- Plotly dependency from `requirements.txt` - -## Verification - -- All page routes return HTML at `/predict`, `/batch`, `/monitor` -- JSON API endpoints remain at `/api/predict`, `/api/batch` -- Swagger UI at `/docs` is unchanged -- HTMX endpoints return HTML fragments (no page reload) -- SSE endpoint for live batch progress at `/api/batch/progress/{job_id}` diff --git a/dvc.yaml b/dvc.yaml index d9a1c583449ccbadd0f1fa65f45b9d63923cc35c..dd43678d6937953eaf5b66855471343688fba243 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -1,11 +1,11 @@ stages: preprocess_semeval: - cmd: PYTHONPATH=. python absa/data/dataset.py + cmd: PYTHONPATH=src python -m absa.data.dataset deps: - - absa/data/dataset.py - - absa/data/preprocess.py - - absa/data/lang_detect.py - - absa/data/transliterate.py + - src/absa/data/dataset.py + - src/absa/data/preprocess.py + - src/absa/data/lang_detect.py + - src/absa/data/transliterate.py - data/raw/semeval_restaurants - data/raw/semeval_laptops outs: @@ -13,12 +13,12 @@ stages: - data/processed/semeval_test.jsonl preprocess_hindi: - cmd: PYTHONPATH=. python absa/data/hindi_loader.py + cmd: PYTHONPATH=src python -m absa.data.hindi_loader deps: - - absa/data/hindi_loader.py - - absa/data/preprocess.py - - absa/data/lang_detect.py - - absa/data/transliterate.py + - src/absa/data/hindi_loader.py + - src/absa/data/preprocess.py + - src/absa/data/lang_detect.py + - src/absa/data/transliterate.py - data/raw/amazon_hindi/hindi_sentiment.jsonl outs: - data/processed/amazon_hindi.jsonl diff --git a/frontend/Home.py b/frontend/Home.py new file mode 100644 index 0000000000000000000000000000000000000000..0c4a95fc2ab0b08a0db2902cd5627ea4925e0c49 --- /dev/null +++ b/frontend/Home.py @@ -0,0 +1,57 @@ +"""Multilingual ABSA — Streamlit entry point. + +Navigation split: + • Analysis → the only screen regular users see (input comment → results). + • Admin → application status, batch analytics, system monitor. +""" + +from __future__ import annotations + +import streamlit as st +from ui import apply_theme + +st.set_page_config( + page_title="Multilingual ABSA", + page_icon="🌍", + layout="wide", + initial_sidebar_state="expanded", +) + +apply_theme() + +analyzer = st.Page( + "views/predict.py", + title="Sentiment Analyzer", + icon="💬", + url_path="predict", + default=True, +) + +admin_overview = st.Page( + "views/admin/overview.py", + title="Overview", + icon="📊", + url_path="admin", +) +admin_batch = st.Page( + "views/admin/batch.py", + title="Batch Analytics", + icon="📁", + url_path="batch", +) +admin_monitor = st.Page( + "views/admin/monitor.py", + title="System Monitor", + icon="🩺", + url_path="monitor", +) + +pg = st.navigation( + { + "Analysis": [analyzer], + "Admin": [admin_overview, admin_batch, admin_monitor], + }, + position="sidebar", +) + +pg.run() diff --git a/app/__init__.py b/frontend/__init__.py similarity index 100% rename from app/__init__.py rename to frontend/__init__.py diff --git a/frontend/absa_client.py b/frontend/absa_client.py new file mode 100644 index 0000000000000000000000000000000000000000..dfa3e078c6ea641886b2ff5ac7fa8f404749a28e --- /dev/null +++ b/frontend/absa_client.py @@ -0,0 +1,84 @@ +"""HTTP client for the Multilingual ABSA FastAPI backend. + +The Streamlit frontend never imports the ML pipeline directly — it talks to +the running FastAPI service (``API_BASE_URL``, default ``http://localhost:8000``) +over plain HTTP. This keeps the dashboard a thin, deployable UI layer. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +import httpx +import streamlit as st + +API_BASE_URL: str = os.getenv("API_BASE_URL", "http://localhost:8000") + +_TIMEOUT: float = float(os.getenv("API_TIMEOUT", "60")) + + +class APIClient: + """Thin wrapper around the ABSA REST API with Streamlit-friendly errors.""" + + def __init__(self, base_url: str = API_BASE_URL, transport: Optional[httpx.BaseTransport] = None) -> None: + self.base_url = base_url.rstrip("/") + self._client = httpx.Client( + base_url=self.base_url, + timeout=_TIMEOUT, + follow_redirects=True, + transport=transport, + ) + + # ── Helpers ──────────────────────────────────────────────────────────── + + def _request(self, method: str, path: str, **kwargs: Any) -> Optional[Any]: + try: + response = self._client.request(method, path, **kwargs) + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as exc: + detail = exc.response.text + st.error(f"API error ({exc.response.status_code}): {detail}") + return None + except httpx.HTTPError as exc: + st.error(f"Cannot reach the ABSA API at `{self.base_url}` — is it running?\n\n{exc}") + return None + + def close(self) -> None: + self._client.close() + + # ── Endpoints ────────────────────────────────────────────────────────── + + def get_health(self) -> Optional[dict[str, str]]: + return self._request("GET", "/health") + + def get_info(self) -> Optional[dict[str, str]]: + return self._request("GET", "/info") + + def predict(self, text: str, language: str = "auto") -> Optional[dict[str, Any]]: + payload = {"text": text, "language": language if language != "auto" else None} + return self._request("POST", "/predict", json=payload) + + def upload_batch(self, file: Any) -> Optional[dict[str, Any]]: + files = {"file": (file.name, file.getvalue(), "text/csv")} + return self._request("POST", "/batch", files=files) + + def get_batch_status(self, job_id: str) -> Optional[dict[str, Any]]: + return self._request("GET", f"/status/{job_id}") + + def download_result(self, job_id: str) -> Optional[bytes]: + """Fetch the generated CSV bytes for a completed batch job.""" + try: + response = self._client.get(f"/download/{job_id}") + response.raise_for_status() + return response.content + except httpx.HTTPError as exc: + st.error(f"Failed to download results: {exc}") + return None + + +@st.cache_resource(show_spinner=False) +def get_client() -> APIClient: + """Return a process-wide cached API client (reused across reruns).""" + return APIClient() diff --git a/frontend/ui.py b/frontend/ui.py new file mode 100644 index 0000000000000000000000000000000000000000..b27c12246fc717151ee24127c33e3743dfa17e19 --- /dev/null +++ b/frontend/ui.py @@ -0,0 +1,251 @@ +"""Shared UI helpers for the Streamlit dashboard: theme, cards, badges.""" + +from __future__ import annotations + +import os +from typing import Iterable + +import streamlit as st + +SENTIMENT_COLORS: dict[str, str] = { + "positive": "#16a34a", + "negative": "#dc2626", + "neutral": "#64748b", + "conflict": "#9333ea", +} + +LANGUAGE_LABELS: dict[str, str] = { + "en": "English", + "hi": "Hindi", + "hinglish": "Hinglish", + "auto": "Auto-detect", +} + +_SAMPLE_REVIEWS: list[dict[str, str]] = [ + { + "label": "Battery + Screen (EN)", + "text": "The battery life is amazing but the screen is too dim.", + }, + { + "label": "Camera & Service (EN)", + "text": "Great camera quality, though the delivery was terribly slow.", + }, + { + "label": "Sound (HI)", + "text": "आवाज़ बहुत साफ़ है और बेस भी बढ़िया है।", + }, + { + "label": "Hinglish Mix", + "text": "Phone ka design badhiya hai lekin battery life kharab hai.", + }, +] + + +def apply_theme() -> None: + """Inject custom CSS: modern gradient hero, cards, badges, spacing.""" + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + + +def hero(title: str, subtitle: str) -> None: + """Render the gradient hero banner.""" + st.markdown( + f""" +
+

{title}

+

{subtitle}

+
+ """, + unsafe_allow_html=True, + ) + + +def feature_card(icon: str, title: str, description: str, accent: str = "#6366f1") -> None: + """Render a clickable-style feature card.""" + st.markdown( + f""" +
+
{icon}
+
{title}
+
{description}
+
+ """, + unsafe_allow_html=True, + ) + + +def sentiment_badge(sentiment: str) -> str: + """HTML for a colored sentiment badge.""" + color = SENTIMENT_COLORS.get(sentiment, "#64748b") + return f'{sentiment}' + + +def aspect_card(aspect: str, sentiment: str, confidence: float) -> str: + """HTML for a single aspect result card with a confidence bar.""" + color = SENTIMENT_COLORS.get(sentiment, "#64748b") + pct = max(0.0, min(100.0, float(confidence) * 100.0)) + return ( + f'
' + f'
{aspect}   {sentiment_badge(sentiment)}
' + f'
Confidence: {confidence:.2f}
' + f'
' + f"
" + ) + + +def render_aspects(aspects: Iterable[dict]) -> None: + """Render a list of aspect dicts as styled cards.""" + items = list(aspects) + if not items: + st.info("No aspects detected in this text.") + return + html = "".join( + aspect_card( + str(a.get("aspect", "N/A")), + str(a.get("sentiment", "neutral")), + float(a.get("confidence", 0.0)), + ) + for a in items + ) + st.markdown(html, unsafe_allow_html=True) + + +def language_options() -> list[str]: + return ["auto", "en", "hi", "hinglish"] + + +def sample_reviews() -> list[dict[str, str]]: + return _SAMPLE_REVIEWS + + +def require_admin() -> bool: + """Gate admin pages behind an optional password (env `ADMIN_PASSWORD`). + + When `ADMIN_PASSWORD` is empty the admin section is open; otherwise a + lock screen is shown until the correct password is entered. Callers should + `st.stop()` when this returns ``False``. + """ + password = os.getenv("ADMIN_PASSWORD", "") + if not password: + return True + if st.session_state.get("admin_ok"): + return True + + st.markdown( + """ +
+
🔒
+

Admin access

+

This section is restricted. Enter the admin + password to view application status and features.

+
+ """, + unsafe_allow_html=True, + ) + candidate = st.text_input("Admin password", type="password", key="admin_password") + if st.button("Unlock", type="primary", key="admin_unlock"): + if candidate == password: + st.session_state["admin_ok"] = True + st.rerun() + else: + st.error("Incorrect password.") + return False diff --git a/frontend/views/admin/batch.py b/frontend/views/admin/batch.py new file mode 100644 index 0000000000000000000000000000000000000000..4125dcd0bb3cec8459c1c9cded2ef7b809225ae2 --- /dev/null +++ b/frontend/views/admin/batch.py @@ -0,0 +1,118 @@ +"""Admin — batch analytics: CSV upload, live job progress, result download.""" + +from __future__ import annotations + +import io +import time + +import pandas as pd +import streamlit as st +from absa_client import get_client +from ui import apply_theme, hero, require_admin + +apply_theme() + +if not require_admin(): + st.stop() + +client = get_client() + +hero( + "Admin · Batch Analytics 📁", + "Upload a CSV of product reviews, queue an async batch job on the worker, " + "track progress live, and download the full annotated results.", +) + +col_up, col_hint = st.columns([2, 1], gap="large") + +with col_up: + uploaded = st.file_uploader("Upload a CSV file", type=["csv"], accept_multiple_files=False) + +with col_hint: + st.markdown( + """ +
+
Required format
+
A CSV with a text column. Limits:
+ +
+ """, + unsafe_allow_html=True, + ) + +df_preview = None +if uploaded is not None: + try: + df_preview = pd.read_csv(io.BytesIO(uploaded.getvalue())) + st.success(f"Loaded {len(df_preview)} rows with columns: {', '.join(df_preview.columns)}") + if "text" not in df_preview.columns: + st.error("CSV must contain a **text** column.") + df_preview = None + else: + st.dataframe(df_preview.head(5), use_container_width=True, hide_index=True) + except Exception as exc: + st.error(f"Could not parse CSV: {exc}") + + if df_preview is not None: + start = st.button("🚀 Start Batch Processing", type="primary", use_container_width=True) + + if start: + with st.spinner("Uploading and queuing job…"): + job = client.upload_batch(uploaded) + if job: + st.session_state["batch_job_id"] = job.get("job_id") + st.rerun() + +job_id = st.session_state.get("batch_job_id") +if job_id: + st.markdown("---") + st.subheader(f"Job progress — `{job_id[:8]}…`") + + progress = st.progress(0.0) + status = st.status("Queued…", expanded=True) + + while True: + job = client.get_batch_status(job_id) + if job is None: + st.error("Failed to fetch job status.") + break + + total = max(int(job.get("total_reviews") or 0), 1) + processed = int(job.get("processed") or 0) + ratio = min(processed / total, 1.0) + progress.progress(ratio) + + label = { + "queued": "Queued — waiting for a worker…", + "processing": f"Processing {processed}/{total} reviews…", + "completed": f"Completed — {processed}/{total} reviews analyzed ✅", + "failed": "Job failed ❌", + }.get(job.get("status"), job.get("status", "…")) + status.update( + label=label, + state="running" if job.get("status") in ("queued", "processing") else "complete", + ) + + if job.get("status") in ("completed", "failed"): + break + time.sleep(2) + + c1, c2, c3 = st.columns(3) + c1.metric("Total Reviews", job.get("total_reviews")) + c2.metric("Processed", job.get("processed")) + c3.metric("Status", job.get("status")) + + if job.get("status") == "completed": + result_bytes = client.download_result(job_id) + if result_bytes is not None: + st.download_button( + "⬇️ Download results (CSV)", + data=result_bytes, + file_name=f"absa_results_{job_id}.csv", + mime="text/csv", + type="primary", + use_container_width=True, + ) diff --git a/frontend/views/admin/monitor.py b/frontend/views/admin/monitor.py new file mode 100644 index 0000000000000000000000000000000000000000..6754832f42e387b6613f6e769778feb175d92491 --- /dev/null +++ b/frontend/views/admin/monitor.py @@ -0,0 +1,53 @@ +"""Admin — system monitor: health checks, service metadata, diagnostics.""" + +from __future__ import annotations + +import streamlit as st +from absa_client import get_client +from ui import apply_theme, hero, require_admin + +apply_theme() + +if not require_admin(): + st.stop() + +client = get_client() + +hero( + "Admin · System Monitor 🩺", + "Live health checks, model metadata, and API configuration for the ABSA service.", +) + +auto_refresh = st.toggle("Auto-refresh every 5s", value=False) + +col_h, col_i = st.columns(2) +with col_h: + health = client.get_health() + if health: + st.success("### API is healthy ✅") + for key, value in health.items(): + st.markdown(f"- **{key}**: `{value}`") + else: + st.error("### API unreachable ❌") + +with col_i: + info = client.get_info() + if info: + st.info("### Service info") + for key, value in info.items(): + st.markdown(f"- **{key}**: `{value}`") + +st.markdown("---") + +col_a, col_b, col_c = st.columns(3) +col_a.metric("Endpoint", client.base_url) +col_b.metric("Timeout (s)", "60") +col_c.metric("Languages", "en · hi · hinglish") + +st.caption( + "Tip: run the API with `uvicorn api.main:app --port 8000` and this " + "dashboard with `streamlit run streamlit_app/Home.py`." +) + +if auto_refresh: + st.rerun() diff --git a/frontend/views/admin/overview.py b/frontend/views/admin/overview.py new file mode 100644 index 0000000000000000000000000000000000000000..e82e9dbc6ea3bbdb7fea89f86879e0d054082ef0 --- /dev/null +++ b/frontend/views/admin/overview.py @@ -0,0 +1,98 @@ +"""Admin — overview: application status and feature details.""" + +from __future__ import annotations + +import streamlit as st +from absa_client import get_client +from ui import apply_theme, feature_card, hero, require_admin + +apply_theme() + +if not require_admin(): + st.stop() + +client = get_client() + +hero( + "Admin · Application Overview 📊", + "Status of the ABSA platform, model metadata, and everything available in this deployment.", +) + +# ── Live status strip ───────────────────────────────────────────────────────── +info = client.get_info() +health = client.get_health() +api_online = health is not None + +col1, col2, col3, col4 = st.columns(4) +col1.metric( + "API Status", + "Online" if api_online else "Offline", + delta="●" if api_online else "○", + delta_color="normal" if api_online else "off", +) +col2.metric( + "Database", + "connected" if api_online else "unknown", +) +col3.metric("Model", (info or {}).get("model_name", "xlm-roberta-base-absa")) +col4.metric("API Base URL", client.base_url) + +st.markdown("---") + +# ── Detailed status ─────────────────────────────────────────────────────────── +col_h, col_i = st.columns(2, gap="large") + +with col_h: + st.subheader("Health check") + if health: + for key, value in health.items(): + st.markdown(f"- **{key}**: `{value}`") + else: + st.error("API unreachable — start `uvicorn api.main:app --port 8000`.") + +with col_i: + st.subheader("Service info") + if info: + for key, value in info.items(): + st.markdown(f"- **{key}**: `{value}`") + else: + st.warning("No service info available.") + +st.markdown("---") + +# ── Feature details ─────────────────────────────────────────────────────────── +st.subheader("What's available") +col_a, col_b, col_c = st.columns(3) +with col_a: + feature_card( + "💬", + "Sentiment Analyzer", + "The public view — users paste a review and get aspect-level sentiment.", + ) +with col_b: + feature_card( + "📁", + "Batch Analytics", + "Upload a CSV of reviews, run async jobs, and download annotated results.", + ) +with col_c: + feature_card( + "🩺", + "System Monitor", + "Health checks, service metadata, and auto-refresh diagnostics.", + ) + +st.markdown( + """ +
+
REST API
+
+ All dashboard features are backed by the JSON API at /docs: +
+ POST /predict · POST /batch · + GET /status/{job_id} · GET /health · + GET /info · GET /metrics +
+ """, + unsafe_allow_html=True, +) diff --git a/frontend/views/predict.py b/frontend/views/predict.py new file mode 100644 index 0000000000000000000000000000000000000000..6d00632e2b2e8ec5bc4e4395b4af15e764b2c272 --- /dev/null +++ b/frontend/views/predict.py @@ -0,0 +1,67 @@ +"""User-facing view — the only screen regular users see. + +Type/paste a review, pick a language, get aspect-level sentiment back. +No dashboards, no metrics, no administration here. +""" + +from __future__ import annotations + +import streamlit as st +from absa_client import get_client +from ui import LANGUAGE_LABELS, apply_theme, language_options, render_aspects + +client = get_client() +apply_theme() + +st.markdown( + """ +
+

💬 Sentiment Analyzer

+

+ Paste a review or comment in English, Hindi, or Hinglish and get + aspect-level sentiment instantly. +

+
+ """, + unsafe_allow_html=True, +) + +# ── Input ───────────────────────────────────────────────────────────────────── +text = st.text_area( + "Your comment", + height=170, + placeholder="e.g. The camera is amazing but the battery drains too fast.", + key="review_text", +) + +col_lang, col_btn = st.columns([1, 2], gap="medium") +with col_lang: + language = st.selectbox( + "Language", + language_options(), + index=0, + format_func=lambda code: LANGUAGE_LABELS[code], + ) +with col_btn: + st.caption("Auto-detects the language if unsure.") + analyze = st.button( + "Analyze Sentiment", + type="primary", + use_container_width=True, + disabled=not text.strip(), + ) + +st.markdown("---") + +# ── Results ─────────────────────────────────────────────────────────────────── +if analyze and text.strip(): + with st.spinner("Analyzing…"): + result = client.predict(text.strip(), language) + + if result: + aspects = result.get("aspects") or [] + + st.subheader("Aspects found") + render_aspects(aspects) +else: + st.info("Enter a review above and press **Analyze Sentiment** to get started.") diff --git a/ml/notebooks/01_data_exploration.ipynb b/notebooks/01_data_exploration.ipynb similarity index 100% rename from ml/notebooks/01_data_exploration.ipynb rename to notebooks/01_data_exploration.ipynb diff --git a/ml/notebooks/03_model_comparison.ipynb b/notebooks/03_model_comparison.ipynb similarity index 100% rename from ml/notebooks/03_model_comparison.ipynb rename to notebooks/03_model_comparison.ipynb diff --git a/ml/notebooks/03_train_colab.ipynb b/notebooks/03_train_colab.ipynb similarity index 100% rename from ml/notebooks/03_train_colab.ipynb rename to notebooks/03_train_colab.ipynb diff --git a/ml/notebooks/04_qlora_colab.ipynb b/notebooks/04_qlora_colab.ipynb similarity index 100% rename from ml/notebooks/04_qlora_colab.ipynb rename to notebooks/04_qlora_colab.ipynb diff --git a/ml/notebooks/08_final_evaluation.ipynb b/notebooks/08_final_evaluation.ipynb similarity index 100% rename from ml/notebooks/08_final_evaluation.ipynb rename to notebooks/08_final_evaluation.ipynb diff --git a/pyproject.toml b/pyproject.toml index 8c35c91b4edc7fcf8e4d858102cc75f8470f06cc..c337fbe9d52c059b6f108d072314ed3d1e4a0c22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] requires = ["setuptools>=68.0"] -build-backend = "setuptools.backends._legacy:_Backend" +build-backend = "setuptools.build_meta" [project] name = "multilingual-absa" -version = "2.0.0" +version = "2.1.0" description = "Multilingual Aspect-Based Sentiment Analysis for English, Hindi, and Hinglish" readme = "README.md" license = {text = "MIT"} @@ -20,12 +20,67 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Text Processing :: Linguistic", ] +dependencies = [ + # ML / NLP + "transformers>=4.41.0", + "datasets>=2.19.0", + "torch>=2.3.0", + "onnxruntime>=1.18.0", + "optimum[onnxruntime]>=1.19.0", + "peft>=0.10.0", + "fasttext-predict>=0.9.2.4", + "indic-nlp-library @ git+https://github.com/anoopkunchukuttan/indic_nlp_library.git", + "nlpaug>=1.1.11", + # Data & Metrics + "scikit-learn>=1.4.2", + "pandas>=2.2.3", + "numpy>=1.26.4", + "seqeval>=1.2.2", + # Experiment tracking & pipelines + "mlflow>=2.15.0", + "dvc>=3.51.1", + "evidently>=0.4.30", + # API + "fastapi>=0.115.0", + "uvicorn>=0.29.0", + "python-multipart>=0.0.9", + "slowapi>=0.1.9", + "pydantic>=2.7.1", + "python-dotenv>=1.0.1", + "prometheus-fastapi-instrumentator>=7.0.0", + # Async workers + "celery>=5.4.0", + "redis>=5.0.4", + # Database + "psycopg2-binary>=2.9.9", + # Frontend (Streamlit dashboard — calls the FastAPI backend over HTTP) + "streamlit>=1.37.0", + "httpx>=0.28.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.2.0", + "pytest-asyncio>=0.23.7", + "pytest-cov>=5.0.0", + "coverage>=7.5.0", + "ruff>=0.6.0", + "mypy>=1.10.0", + "bandit>=1.7.9", + "radon>=6.0.1", + "scalene>=1.5.30", +] + +[tool.setuptools.packages.find] +where = ["src", "."] +include = ["absa*", "api*"] [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["test_*.py"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" +addopts = "--cov=api --cov=absa --cov-report=term-missing --cov-report=xml" [tool.ruff] target-version = "py310" @@ -33,7 +88,31 @@ line-length = 120 [tool.ruff.lint] select = ["E", "F", "I", "N", "W"] +extend-ignore = ["N999"] # module file name style + +[tool.mypy] +python_version = "3.10" +warn_return_any = true +warn_unused_configs = true +ignore_missing_imports = true +check_untyped_defs = true +exclude = [ + "tests/", + "scripts/", + "notebooks/", + "docs/", +] + +[tool.bandit] +exclude_dirs = ["tests", "scripts"] +targets = ["api", "absa"] +skips = ["B101"] # allow assert (pytest usage) [tool.coverage.run] -source = ["app", "absa"] +source = ["api", "absa"] omit = ["*/tests/*", "*/__pycache__/*"] + +[tool.coverage.report] +show_missing = true +skip_covered = true +fail_under = 24 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b26b99460396dea243a6ffa589ca6250aff5fafb..0000000000000000000000000000000000000000 --- a/requirements.txt +++ /dev/null @@ -1,54 +0,0 @@ -# ML / NLP — Pinned for reproducibility -transformers>=4.41.0 -datasets>=2.19.0 -torch>=2.3.0 -onnxruntime>=1.18.0 -optimum[onnxruntime]>=1.19.0 -peft>=0.10.0 -fasttext-predict>=0.9.2.4 -indic-nlp-library @ git+https://github.com/anoopkunchukuttan/indic_nlp_library.git -nlpaug>=1.1.11 - -# Data & Metrics -scikit-learn>=1.4.2 -pandas>=2.2.3 # CVE-2024-42992 fix -numpy>=1.26.4 -seqeval>=1.2.2 - -# MLflow & DVC -mlflow>=2.15.0 # Security improvements -dvc>=3.51.1 -evidently>=0.4.30 - -# API — Latest with fixes -fastapi>=0.115.0 # CVE-2024-24762 fix -jinja2>=3.1.5 # CVE-2024-56326 fix -uvicorn>=0.29.0 -python-multipart>=0.0.9 -slowapi>=0.1.9 # Rate limiting - -# Async Workers -celery>=5.4.0 -redis>=5.0.4 - -# Database -psycopg2-binary>=2.9.9 # Dev only; use psycopg2 (source) in production - -# Validation -pydantic>=2.7.1 -python-dotenv>=1.0.1 - -# Testing -pytest>=8.2.0 -httpx>=0.28.0 # Security fixes - -# Monitoring -prometheus-fastapi-instrumentator>=7.0.0 -slowapi>=0.1.9 - -# API Layer additions -sse-starlette>=2.1.3,<3.0.0 # Server-Sent Events for live progress (avoid starlette conflict) -itsdangerous>=2.2.0 # CSRF protection for HTMX forms - -# Testing -pytest-asyncio>=0.23.7 # Async test support diff --git a/scripts/download_data.py b/scripts/download_data.py index 4ef5db541abdeac095b2a88866675e692f597e7f..67277e4c5a0d393e815cb554c26b73141ae6b929 100644 --- a/scripts/download_data.py +++ b/scripts/download_data.py @@ -1,7 +1,7 @@ import os import urllib.request from datasets import load_dataset -from absa.config import DATA_DIR, RAW_DIR, FASTTEXT_MODEL_PATH +from absa.utils.config import DATA_DIR, RAW_DIR, FASTTEXT_MODEL_PATH def download_fasttext(): url = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz" diff --git a/scripts/generate_notebooks.py b/scripts/generate_notebooks.py index bb066a1ac28fb64ddc6f9d9694f6f15f6e77eeba..4b4188b90b83007db3daf500e1ef9b6add5305bf 100644 --- a/scripts/generate_notebooks.py +++ b/scripts/generate_notebooks.py @@ -46,14 +46,14 @@ def create_notebook(filename: str, cells_content: list): def main(): colab_cells = [ ("# Google Colab Training Notebook\n\nThis notebook is intended to be run on Google Colab with a T4 GPU. It clones the repo, installs dependencies, and runs the training scripts.", "markdown"), - ("!git clone https://github.com/Aryanmishra-dev/Multilingual-Absa.git\n%cd Multilingual-Absa\n!pip install -r requirements.txt", "code"), + ("!git clone https://github.com/Aryanmishra-dev/Multilingual-Absa.git\n%cd Multilingual-Absa\n!pip install .", "code"), ("# Mount Google Drive to save models and MLflow logs persistently\nfrom google.colab import drive\ndrive.mount('/content/drive')", "code"), ("# Create symlinks or copy data if needed\n# Assuming data is in the repo for now\n!mkdir -p /content/drive/MyDrive/ABSA_models", "code"), - ("# Prepare dataset\n!PYTHONPATH=. python absa/data/hf_dataset.py", "code"), - ("# Run Aspect Extraction Training\n!PYTHONPATH=. python absa/models/train_aspect_extraction.py", "code"), - ("# Run Sentiment Classification Training\n!PYTHONPATH=. python absa/models/train_sentiment.py", "code"), - ("# Run Baseline as well\n!PYTHONPATH=. python absa/models/baseline.py", "code"), - ("# Cross-lingual Evaluation\n!PYTHONPATH=. python absa/evaluation/cross_lingual_eval.py", "code"), + ("# Prepare dataset\n!PYTHONPATH=src python -m absa.data.hf_dataset", "code"), + ("# Run Aspect Extraction Training\n!PYTHONPATH=src python -m absa.models.train_aspect_extraction", "code"), + ("# Run Sentiment Classification Training\n!PYTHONPATH=src python -m absa.models.train_sentiment", "code"), + ("# Run Baseline as well\n!PYTHONPATH=src python -m absa.models.baseline", "code"), + ("# Cross-lingual Evaluation\n!PYTHONPATH=src python -m absa.evaluation.cross_lingual_eval", "code"), ("# Copy models back to Drive\n!cp -r models/* /content/drive/MyDrive/ABSA_models/\n!cp -r mlflow /content/drive/MyDrive/ABSA_models/", "code") ] diff --git a/scripts/init_db.py b/scripts/init_db.py index 3b0d2ea6fc68a3b93799112c9b7e2bb7ecc82b22..ebf07ffb541981f75dfbb6534353c34fc5cbbb9c 100644 --- a/scripts/init_db.py +++ b/scripts/init_db.py @@ -5,7 +5,7 @@ from dotenv import load_dotenv import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from app.schemas.db_models import Base +from api.schemas.db_models import Base def init_db(): load_dotenv() diff --git a/app/core/__init__.py b/src/absa/__init__.py similarity index 100% rename from app/core/__init__.py rename to src/absa/__init__.py diff --git a/app/middleware/__init__.py b/src/absa/data/__init__.py similarity index 100% rename from app/middleware/__init__.py rename to src/absa/data/__init__.py diff --git a/absa/data/augmentation.py b/src/absa/data/augmentation.py similarity index 100% rename from absa/data/augmentation.py rename to src/absa/data/augmentation.py diff --git a/absa/data/bio_tagger.py b/src/absa/data/bio_tagger.py similarity index 100% rename from absa/data/bio_tagger.py rename to src/absa/data/bio_tagger.py diff --git a/absa/data/dataset.py b/src/absa/data/dataset.py similarity index 100% rename from absa/data/dataset.py rename to src/absa/data/dataset.py diff --git a/absa/data/hf_dataset.py b/src/absa/data/hf_dataset.py similarity index 100% rename from absa/data/hf_dataset.py rename to src/absa/data/hf_dataset.py diff --git a/absa/data/hindi_loader.py b/src/absa/data/hindi_loader.py similarity index 100% rename from absa/data/hindi_loader.py rename to src/absa/data/hindi_loader.py diff --git a/absa/data/lang_detect.py b/src/absa/data/lang_detect.py similarity index 100% rename from absa/data/lang_detect.py rename to src/absa/data/lang_detect.py diff --git a/absa/data/preprocess.py b/src/absa/data/preprocess.py similarity index 100% rename from absa/data/preprocess.py rename to src/absa/data/preprocess.py diff --git a/absa/data/transliterate.py b/src/absa/data/transliterate.py similarity index 100% rename from absa/data/transliterate.py rename to src/absa/data/transliterate.py diff --git a/app/routes/__init__.py b/src/absa/evaluation/__init__.py similarity index 100% rename from app/routes/__init__.py rename to src/absa/evaluation/__init__.py diff --git a/absa/evaluation/benchmark_latency.py b/src/absa/evaluation/benchmark_latency.py similarity index 100% rename from absa/evaluation/benchmark_latency.py rename to src/absa/evaluation/benchmark_latency.py diff --git a/absa/evaluation/cross_lingual_eval.py b/src/absa/evaluation/cross_lingual_eval.py similarity index 100% rename from absa/evaluation/cross_lingual_eval.py rename to src/absa/evaluation/cross_lingual_eval.py diff --git a/absa/evaluation/final_eval.py b/src/absa/evaluation/final_eval.py similarity index 100% rename from absa/evaluation/final_eval.py rename to src/absa/evaluation/final_eval.py diff --git a/app/schemas/__init__.py b/src/absa/models/__init__.py similarity index 100% rename from app/schemas/__init__.py rename to src/absa/models/__init__.py diff --git a/absa/models/baseline.py b/src/absa/models/baseline.py similarity index 100% rename from absa/models/baseline.py rename to src/absa/models/baseline.py diff --git a/absa/models/export_onnx.py b/src/absa/models/export_onnx.py similarity index 100% rename from absa/models/export_onnx.py rename to src/absa/models/export_onnx.py diff --git a/absa/models/train_aspect_extraction.py b/src/absa/models/train_aspect_extraction.py similarity index 100% rename from absa/models/train_aspect_extraction.py rename to src/absa/models/train_aspect_extraction.py diff --git a/absa/models/train_joint_absa.py b/src/absa/models/train_joint_absa.py similarity index 100% rename from absa/models/train_joint_absa.py rename to src/absa/models/train_joint_absa.py diff --git a/absa/models/train_multilingual.py b/src/absa/models/train_multilingual.py similarity index 100% rename from absa/models/train_multilingual.py rename to src/absa/models/train_multilingual.py diff --git a/absa/models/train_qlora.py b/src/absa/models/train_qlora.py similarity index 100% rename from absa/models/train_qlora.py rename to src/absa/models/train_qlora.py diff --git a/absa/models/train_sentiment.py b/src/absa/models/train_sentiment.py similarity index 100% rename from absa/models/train_sentiment.py rename to src/absa/models/train_sentiment.py diff --git a/app/services/__init__.py b/src/absa/training/__init__.py similarity index 100% rename from app/services/__init__.py rename to src/absa/training/__init__.py diff --git a/absa/training/mlflow_utils.py b/src/absa/training/mlflow_utils.py similarity index 100% rename from absa/training/mlflow_utils.py rename to src/absa/training/mlflow_utils.py diff --git a/src/absa/utils/__init__.py b/src/absa/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/absa/utils/config.py b/src/absa/utils/config.py similarity index 92% rename from absa/utils/config.py rename to src/absa/utils/config.py index 2c29561f47bca30ea076fe01c9cd64fb463ad051..e340ca98d627893db716fc4ed10afc71114ae4d4 100644 --- a/absa/utils/config.py +++ b/src/absa/utils/config.py @@ -1,6 +1,6 @@ from pathlib import Path -ROOT_DIR = Path(__file__).parent.parent.parent +ROOT_DIR = Path(__file__).parent.parent.parent.parent DATA_DIR = ROOT_DIR / "data" RAW_DIR = DATA_DIR / "raw" PROCESSED_DIR = DATA_DIR / "processed" diff --git a/tests/api/test_api.py b/tests/api/test_api.py index 56bd48d81f320aedecb67cf2b7218864bf5e1cff..5a3084cc2883f748475ad5ea2fce958ba682cbe4 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -4,7 +4,7 @@ import os os.environ["DATABASE_URL"] = "sqlite:///./tests/fixtures/test.db" import unittest.mock as mock -from app.main import app +from api.main import app import json import io @@ -44,7 +44,7 @@ def test_batch_upload(): with TestClient(app) as client: csv_content = "text\nThe food was great\nTerrible service" files = {"file": ("test.csv", io.BytesIO(csv_content.encode("utf-8")), "text/csv")} - with mock.patch("app.routes.predict.process_batch.delay") as mock_delay: + with mock.patch("api.routes.predict.process_batch.delay") as mock_delay: response = client.post("/batch", files=files) assert response.status_code == 200 data = response.json() diff --git a/tests/unit/test_pipeline.py b/tests/unit/test_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..dfd846aab2672d7df38af0b9225551336b9384c3 --- /dev/null +++ b/tests/unit/test_pipeline.py @@ -0,0 +1,108 @@ +"""Unit tests for the rule-based ABSA pipeline aspect extraction + fallbacks.""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "sqlite:///./tests/fixtures/test.db") + +from api.services.absa_pipeline import pipeline + + +def _aspects(text: str) -> list[dict]: + result = pipeline.predict(text, "en") + return [a.model_dump() if hasattr(a, "model_dump") else dict(a) for a in result.aspects] + + +def _sentiment_of(aspects: list[dict], aspect: str) -> str | None: + for a in aspects: + if a["aspect"].lower() == aspect.lower(): + return a["sentiment"] + return None + + +def test_food_and_service_extracted_from_sentiment_words(): + aspects = _aspects("The food was great but the service was terrible.") + labels = [a["aspect"].lower() for a in aspects] + assert "food" in labels + assert "service" in labels + assert _sentiment_of(aspects, "food") == "positive" + assert _sentiment_of(aspects, "service") == "negative" + + +def test_noun_after_sentiment_word_extracted(): + aspects = _aspects("Great camera quality and a lovely experience overall.") + labels = [a["aspect"].lower() for a in aspects] + # "camera quality" is in the lexicon; "experience" must come from the fallback. + assert any("camera" in label or "experience" in label for label in labels) + + +def test_product_target_fallback(): + aspects = _aspects("I absolutely love this product.") + assert _sentiment_of(aspects, "product") == "positive" + + +def test_generic_overall_fallback_when_no_aspect_noun(): + aspects = _aspects("Absolutely terrible, do not recommend.") + assert aspects, "should always return at least one aspect" + assert _sentiment_of(aspects, "overall") == "negative" + + +def test_positive_bare_comment_returns_result(): + aspects = _aspects("Amazing!") + assert aspects, "should always return at least one aspect" + assert _sentiment_of(aspects, "overall") == "positive" + + +def test_neutral_text_returns_overall_neutral(): + aspects = _aspects("Hello there, just checking.") + assert aspects + assert _sentiment_of(aspects, "overall") == "neutral" + + +def test_devanagari_comment_gets_aspect_and_sentiment(): + aspects = _aspects("खाना बहुत अच्छा था।") + assert aspects, "Devanagari comment should still produce a result" + assert any(a["aspect"].lower() in ("खाना", "overall") for a in aspects) + assert _sentiment_of(aspects, "खाना") == "positive" + + +def test_empty_text_never_crashes(): + aspects = _aspects("") + assert aspects # generic fallback keeps the response non-empty + + +def test_lexicon_still_used_first(): + aspects = _aspects("The battery life is amazing but the screen is too dim.") + labels = [a["aspect"].lower() for a in aspects] + assert "battery life" in labels or "battery" in labels + assert _sentiment_of(aspects, "battery life") == "positive" + assert _sentiment_of(aspects, "screen") == "negative" + + +def test_purchase_target_with_strong_negation(): + aspects = _aspects("Worst purchase ever, do not buy.") + assert _sentiment_of(aspects, "purchase") == "negative" + + +def test_devanagari_mixed_clauses_split_on_lekin(): + aspects = _aspects("खाना बढ़िया था लेकिन सेवा खराब थी।") + assert _sentiment_of(aspects, "खाना") == "positive" + assert _sentiment_of(aspects, "सेवा") == "negative" + + +def test_hinglish_mixed_clauses_split_on_lekin(): + aspects = _aspects("The phone ka design badhiya hai lekin battery life kharab hai.") + assert _sentiment_of(aspects, "design") == "positive" + assert _sentiment_of(aspects, "battery life") == "negative" + + +def test_devanagari_word_never_split_inside(): + # Regression: "बढ़िया" used to be split by the "या" clause separator. + aspects = _aspects("खाना बहुत बढ़िया था।") + assert _sentiment_of(aspects, "खाना") == "positive" + + +def test_experience_positive(): + aspects = _aspects("Great experience overall, will order again.") + assert _sentiment_of(aspects, "experience") == "positive" diff --git a/tests/web/test_pages.py b/tests/web/test_pages.py deleted file mode 100644 index e9574f75415bcaad2897f9af23634836767f38c8..0000000000000000000000000000000000000000 --- a/tests/web/test_pages.py +++ /dev/null @@ -1,338 +0,0 @@ -""" -tests/web/test_pages.py — Phase 2 smoke tests. - -PURPOSE -------- -Verify that the new Jinja2 page infrastructure: - 1. Renders all three pages without raising an exception (HTTP 200). - 2. Returns HTML content (not JSON). - 3. Contains the expected page titles / identifiers. - 4. Does NOT break any existing JSON API endpoint. - 5. Serves static CSS from the /static mount. - -These tests are strictly additive — they do not modify or replace -any tests in tests/api/test_api.py. - -IMPORTANT: DATABASE_URL must be set before importing the app because -api/app/middleware/dependencies.py reads it at module-import time. -""" -import os - -# Must be set before any app import (same pattern as tests/api/test_api.py) -os.environ.setdefault("DATABASE_URL", "sqlite:///./tests/fixtures/test.db") - -from unittest import mock - -import pytest -from fastapi.testclient import TestClient - -from app.main import app - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _html_client() -> TestClient: - """Return a TestClient that triggers the lifespan (model loading).""" - return TestClient(app) - - -# --------------------------------------------------------------------------- -# Page rendering tests -# --------------------------------------------------------------------------- - -class TestPageRoutes: - """Smoke tests: every GET page route returns 200 HTML.""" - - def test_root_redirects_to_predict(self): - """GET / should return the Predict page (200, HTML).""" - with _html_client() as client: - response = client.get("/", follow_redirects=True) - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - - def test_predict_page_renders(self): - with _html_client() as client: - response = client.get("/predict") - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "SentimentAI" in response.text - assert "Live Predictor" in response.text - - def test_batch_page_renders(self): - with _html_client() as client: - response = client.get("/batch") - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "SentimentAI" in response.text - assert "Batch Analytics" in response.text - - def test_monitor_page_renders(self): - with _html_client() as client: - response = client.get("/monitor") - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "SentimentAI" in response.text - assert "System Monitor" in response.text - - def test_sidebar_nav_items_present(self): - """All three nav links must appear in every page.""" - with _html_client() as client: - for path in ("/predict", "/batch", "/monitor"): - response = client.get(path) - assert response.status_code == 200 - # Check all nav labels are present - assert "Predictor" in response.text - assert "Batch Analytics" in response.text - assert "System Health" in response.text - - def test_predict_active_state(self): - """/predict page must mark Predictor nav item as active.""" - with _html_client() as client: - response = client.get("/predict") - assert "nav-item-active" in response.text - # The active item must contain the predictor icon - assert "psychology" in response.text - - def test_base_template_includes_htmx(self): - """HTMX CDN script must be present in every page.""" - with _html_client() as client: - response = client.get("/predict") - assert "htmx.org" in response.text - - def test_base_template_includes_alpinejs(self): - """Alpine.js CDN script must be present in every page.""" - with _html_client() as client: - response = client.get("/predict") - assert "alpinejs" in response.text - - def test_base_template_includes_tailwind(self): - """Tailwind CDN script must be present in every page.""" - with _html_client() as client: - response = client.get("/predict") - assert "cdn.tailwindcss.com" in response.text - - -# --------------------------------------------------------------------------- -# Static file tests -# --------------------------------------------------------------------------- - -class TestStaticFiles: - """Verify the /static mount serves files correctly.""" - - def test_css_file_served(self): - with _html_client() as client: - response = client.get("/static/css/app.css") - assert response.status_code == 200 - assert "text/css" in response.headers["content-type"] - # Spot-check for key design system classes - assert "badge-positive" in response.text - assert "btn-primary" in response.text - - def test_static_missing_file_returns_404(self): - with _html_client() as client: - response = client.get("/static/does-not-exist.css") - assert response.status_code == 404 - - -# --------------------------------------------------------------------------- -# Regression tests — existing API must be unaffected -# --------------------------------------------------------------------------- - -class TestExistingAPIUnchanged: - """ - Re-run the core API assertions to prove Phase 2 changes introduced - zero regressions. These mirror tests/api/test_api.py in spirit. - """ -class TestPredictFragment: - """Verify the new HTMX predict endpoint works exactly like the JSON one.""" - - def test_predict_fragment_english(self): - with _html_client() as client: - response = client.post( - "/predict/fragment", - data={"text": "The food was great but service was slow.", "language": "en"}, - ) - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "Detected Aspects" in response.text - # the response might say "No aspects detected" or show the list, but both are valid HTML - assert "food" in response.text # text should be in the annotated container - - def test_predict_fragment_auto(self): - with _html_client() as client: - response = client.post( - "/predict/fragment", - data={"text": "The food was great but service was slow.", "language": "auto"}, - ) - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "EN" in response.text or "English" in response.text or "en" in response.text - - def test_predict_fragment_empty_text(self): - with _html_client() as client: - # Form submission missing 'text' should trigger FastAPI 422 - response = client.post("/predict/fragment", data={"language": "en"}) - # Note: Depending on FastAPI validation, missing Form field might be 422 - assert response.status_code == 422 - def test_health_endpoint_still_returns_json(self): - with _html_client() as client: - response = client.get("/health") - assert response.status_code == 200 - data = response.json() - assert data["status"] == "ok" - # Verify content-type is JSON (not HTML) - assert "application/json" in response.headers["content-type"] - - def test_info_endpoint_still_returns_json(self): - with _html_client() as client: - response = client.get("/info") - assert response.status_code == 200 - data = response.json() - assert "model_name" in data - assert "supported_languages" in data - - def test_predict_endpoint_still_returns_json(self): - """POST /predict must still return PredictionResponse JSON.""" - with _html_client() as client: - response = client.post( - "/predict", - json={"text": "The sound quality is excellent.", "language": "en"}, - ) - assert response.status_code == 200 - data = response.json() - # Schema check — all required fields must be present - assert "text" in data - assert "language" in data - assert "detected_language" in data - assert "aspects" in data - assert "processing_time_ms" in data - # Content-type must be JSON - assert "application/json" in response.headers["content-type"] - - def test_openapi_schema_page_routes_hidden(self): - """Page GET routes must NOT appear in the OpenAPI schema.""" - with _html_client() as client: - response = client.get("/openapi.json") - assert response.status_code == 200 - schema = response.json() - paths = schema.get("paths", {}) - # None of the page routes should be in the schema - assert "/predict" not in paths or all( - method == "post" for method in paths.get("/predict", {}) - ), "GET /predict should not appear in OpenAPI schema" - assert "/" not in paths, "GET / should not appear in OpenAPI schema" - assert "/batch" not in paths or all( - method == "post" for method in paths.get("/batch", {}) - ), "GET /batch should not appear in OpenAPI schema" - assert "/monitor" not in paths, "GET /monitor should not appear in OpenAPI schema" - -class TestBatchFragments: - """Verify Phase 4 Batch Analytics HTMX endpoints.""" - - def test_batch_fragment_upload(self): - with _html_client() as client: - files = {"file": ("test.csv", b"text\nThis is great\nThis is bad", "text/csv")} - with mock.patch("app.routes.predict.process_batch.delay"): - response = client.post("/batch/fragment", files=files) - - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "Batch Job:" in response.text - assert "queued" in response.text.lower() or "processing" in response.text.lower() or "completed" in response.text.lower() - - def test_batch_fragment_invalid_file(self): - with _html_client() as client: - files = {"file": ("test.txt", b"text\nThis is great", "text/plain")} - with mock.patch("app.routes.predict.process_batch.delay"): - response = client.post("/batch/fragment", files=files) - - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "Only CSV files are allowed" in response.text or "error" in response.text.lower() - - def test_batch_progress_polling(self): - # 1. upload file to get job_id - with _html_client() as client: - files = {"file": ("test.csv", b"text\nHello", "text/csv")} - with mock.patch("app.routes.predict.process_batch.delay"): - response1 = client.post("/batch/fragment", files=files) - import re - match = re.search(r'([a-f0-9-]+)', response1.text) - assert match, "Could not find job_id in HTML" - job_id = match.group(1) - - # 2. poll progress - response2 = client.get(f"/batch/progress/{job_id}") - assert response2.status_code == 200 - assert "text/html" in response2.headers["content-type"] - assert job_id in response2.text - - def test_download_endpoint_exists(self): - with _html_client() as client: - response = client.get("/results/download/fake-job-id") - assert response.status_code == 404 - - def test_batch_charts_endpoint_handles_missing_file(self): - with _html_client() as client: - response = client.get("/batch/charts/fake-job-id") - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "CSV not found" in response.text - - def test_batch_charts_endpoint_valid_file(self, tmp_path): - import pandas as pd - from pathlib import Path - - # Create a mock CSV for a fake job - job_id = "test-job-charts" - test_file = Path(f"data/results/{job_id}.csv") - test_file.parent.mkdir(parents=True, exist_ok=True) - - df = pd.DataFrame({ - "text": ["hello", "world"], - "language": ["en", "hi"], - "aspect": ["food", "service"], - "sentiment": ["positive", "negative"] - }) - df.to_csv(test_file, index=False) - - try: - with _html_client() as client: - response = client.get(f"/batch/charts/{job_id}") - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "chart.js" in response.text.lower() - assert "languageChart" in response.text - finally: - if test_file.exists(): - test_file.unlink() - -class TestMonitorFragments: - """Verify Phase 5 System Monitor HTMX endpoints.""" - - def test_monitor_health_fragment(self): - with _html_client() as client: - response = client.get("/monitor/health-partial") - - assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] - assert "Current state:" in response.text - assert "Healthy" in response.text or "Degraded" in response.text - - def test_monitor_page_initial_render(self): - with _html_client() as client: - response = client.get("/monitor") - - assert response.status_code == 200 - assert "System Monitor" in response.text - assert "hx-get=\"/monitor/health-partial\"" in response.text - assert "hx-trigger=\"every 30s\"" in response.text - assert "Current state:" in response.text - - def test_monitor_fragment_error_handling(self): - # We can't easily mock the error in the test suite without patching, - # but we can verify the template renders cleanly and doesn't 500 - # if the health_check fails by manually rendering it. - pass diff --git a/tests/web/test_streamlit_app.py b/tests/web/test_streamlit_app.py new file mode 100644 index 0000000000000000000000000000000000000000..34c7ec0928cece96ee016b36adedff795c814f18 --- /dev/null +++ b/tests/web/test_streamlit_app.py @@ -0,0 +1,120 @@ +"""Smoke tests for the Streamlit dashboard pages via streamlit AppTest. + +These verify each page script runs without raising, and that key UI elements +render. The API base URL is pointed at a dead port so health/network calls fail +fast and deterministically (pages degrade gracefully). +""" + +from __future__ import annotations + +import os +import sys + +os.environ["API_BASE_URL"] = "http://127.0.0.1:1" + +import pytest +from streamlit.testing.v1 import AppTest + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Real `streamlit run` adds the app dir to sys.path; AppTest does not, so mimic it. +APP_DIR = os.path.join(ROOT, "frontend") +if APP_DIR not in sys.path: + sys.path.insert(0, APP_DIR) + +APP_FILES = { + "home": "frontend/Home.py", + "predict": "frontend/views/predict.py", + "overview": "frontend/views/admin/overview.py", + "batch": "frontend/views/admin/batch.py", + "monitor": "frontend/views/admin/monitor.py", +} + + +@pytest.mark.parametrize("page", list(APP_FILES.keys())) +def test_page_runs_without_exception(page: str): + at = AppTest.from_file(os.path.join(ROOT, APP_FILES[page])) + at.run(timeout=30) + assert not at.exception, f"{page} raised: {at.exception}" + + +# ── Public user view (Sentiment Analyzer) ───────────────────────────────────── + + +def test_predict_is_clean_input_to_results_view(): + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["predict"])) + at.run(timeout=30) + rendered = " ".join(str(m.value) for m in at.markdown) + assert "Sentiment Analyzer" in rendered + assert len(at.text_area) >= 1 + assert any(b.label == "Analyze Sentiment" for b in at.button) + + +def test_predict_analyze_button_disabled_for_empty_text(): + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["predict"])) + at.run(timeout=30) + analyze = next(b for b in at.button if b.label == "Analyze Sentiment") + assert analyze.disabled + + +def test_home_navigation_lands_on_analyzer(): + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["home"])) + at.run(timeout=30) + assert not at.exception + rendered = " ".join(str(m.value) for m in at.markdown) + assert "Sentiment Analyzer" in rendered + + +# ── Admin views ─────────────────────────────────────────────────────────────── + + +def test_admin_overview_shows_status_metrics(): + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["overview"])) + at.run(timeout=30) + labels = [m.label for m in at.metric] + assert "API Status" in labels + assert "Model" in labels + + +def test_admin_overview_lists_features(): + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["overview"])) + at.run(timeout=30) + rendered = " ".join(str(m.value) for m in at.markdown) + assert "Batch Analytics" in rendered + assert "System Monitor" in rendered + + +def test_admin_batch_page_has_csv_uploader(): + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["batch"])) + at.run(timeout=30) + assert len(at.get("file_uploader")) >= 1 + + +def test_admin_monitor_handles_api_unreachable(): + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["monitor"])) + at.run(timeout=30) + assert not at.exception + rendered = " ".join(str(m.value) for m in at.markdown) + errors = " ".join(str(e.value) for e in at.error) + assert "unreachable" in (rendered + errors).lower() + + +def test_admin_locked_without_password(monkeypatch): + monkeypatch.setenv("ADMIN_PASSWORD", "secret") + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["overview"])) + at.run(timeout=30) + assert not at.exception + labels = [m.label for m in at.metric] + assert "API Status" not in labels + assert any(t.label == "Admin password" for t in at.text_input) + + +def test_admin_unlocked_with_correct_password(monkeypatch): + monkeypatch.setenv("ADMIN_PASSWORD", "secret") + at = AppTest.from_file(os.path.join(ROOT, APP_FILES["overview"])) + at.run(timeout=30) + at.text_input(key="admin_password").set_value("secret") + at.button(key="admin_unlock").click().run(timeout=30) + assert not at.exception + labels = [m.label for m in at.metric] + assert "API Status" in labels diff --git a/tests/web/test_streamlit_client.py b/tests/web/test_streamlit_client.py new file mode 100644 index 0000000000000000000000000000000000000000..0985d44ecbd9ce26de853481b1b71c99cb150b0b --- /dev/null +++ b/tests/web/test_streamlit_client.py @@ -0,0 +1,108 @@ +"""Unit tests for the Streamlit API client (httpx MockTransport, no network).""" + +from __future__ import annotations + +import httpx +import pytest + +from frontend.absa_client import APIClient + + +def _client(handler) -> APIClient: + return APIClient(base_url="http://testserver", transport=httpx.MockTransport(handler)) + + +def _json_handler(payload, status: int = 200): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(status, json=payload, request=request) + + return handler + + +def test_health_returns_json(): + client = _client(_json_handler({"status": "ok", "model": "loaded"})) + assert client.get_health() == {"status": "ok", "model": "loaded"} + + +def test_info_returns_json(): + client = _client(_json_handler({"model_name": "xlm-roberta-base-absa"})) + assert client.get_info()["model_name"] == "xlm-roberta-base-absa" + + +def test_predict_payload_language_auto_sends_null(): + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["json"] = request.content + return httpx.Response(200, json={"aspects": []}, request=request) + + client = _client(handler) + client.predict("Great phone", "auto") + import json + + assert json.loads(captured["json"]) == {"text": "Great phone", "language": None} + + +def test_predict_payload_explicit_language(): + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["json"] = request.content + return httpx.Response(200, json={"aspects": []}, request=request) + + client = _client(handler) + client.predict("बढ़िया फोन", "hi") + import json + + assert json.loads(captured["json"]) == {"text": "बढ़िया फोन", "language": "hi"} + + +def test_upload_batch_sends_csv_file(): + captured = {} + + class FakeFile: + name = "reviews.csv" + getvalue = lambda self: b"text\nGreat\n" # noqa: E731 + + def handler(request: httpx.Request) -> httpx.Response: + captured["content_type"] = request.headers.get("content-type", "") + return httpx.Response(200, json={"job_id": "abc-123", "status": "queued"}, request=request) + + client = _client(handler) + result = client.upload_batch(FakeFile()) + assert result["job_id"] == "abc-123" + assert "multipart/form-data" in captured["content_type"] + + +def test_get_batch_status_parses(): + client = _client(_json_handler({"job_id": "x", "status": "completed", "processed": 5})) + status = client.get_batch_status("x") + assert status["status"] == "completed" + assert status["processed"] == 5 + + +def test_download_result_returns_bytes(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b"text,sentiment\nok,positive\n", request=request) + + client = _client(handler) + assert client.download_result("abc") == b"text,sentiment\nok,positive\n" + + +def test_http_error_returns_none(): + client = _client(_json_handler({"detail": "boom"}, status=500)) + assert client.get_health() is None + + +def test_network_error_returns_none(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("refused", request=request) + + client = _client(handler) + assert client.get_health() is None + + +@pytest.mark.parametrize("status", ["queued", "processing", "completed", "failed"]) +def test_status_passthrough(status: str): + client = _client(_json_handler({"status": status})) + assert client.get_batch_status("x")["status"] == status