diff --git a/.env.example b/.env.example index 288fb449755880908b19e0b11664501c053b1734..ef65bc3c622f0750c585380f639639310d034db3 100644 --- a/.env.example +++ b/.env.example @@ -1,22 +1 @@ -# Database -DATABASE_URL=postgresql://user:pass@localhost:5432/absa_db -POSTGRES_USER=absa_user -POSTGRES_PASSWORD=change_this_to_strong_password -POSTGRES_DB=absa_db - -# Redis / Celery -REDIS_URL=redis://localhost:6379/0 -REDIS_PASSWORD=change_this_to_strong_password - -# Model -MODEL_PATH=models/onnx/ -MAX_BATCH_SIZE=10000 - -# Security -CORS_ORIGINS=http://localhost:3000,http://localhost:8000 -CSRF_SECRET=change_this_to_strong_random_secret -GRAFANA_ADMIN_PASSWORD=change_this_to_strong_password - -# Logging -LOG_LEVEL=INFO -ENABLE_METRICS=true +MODEL_PATH=./models/xlm-roberta-quantized diff --git a/.gitattributes b/.gitattributes index 34e01a9bab9de9d413592c33468a4752a1cdf5be..7b5ba5d47d9ac6c00f719b7e57da61192b0e9c26 100644 --- a/.gitattributes +++ b/.gitattributes @@ -23,5 +23,8 @@ dist/** linguist-generated=true build/** linguist-generated=true node_modules/** linguist-generated=true **/node_modules/** linguist-generated=true +# Hugging Face Spaces LFS (model artifacts) +models/**/*.onnx filter=lfs diff=lfs merge=lfs -text +models/**/*.bin filter=lfs diff=lfs merge=lfs -text *.ftz filter=lfs diff=lfs merge=lfs -text *.arrow filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dddffe86f7b1ca96cbec7e2d84fe2cea1c21224..299c08169453f4cd04d5b248447ed4203d8a0dd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,9 +25,9 @@ jobs: - name: Install ruff run: pip install ruff==0.15.11 - name: Ruff check - run: ruff check api src/absa tests scripts + run: ruff check src/absa tests scripts - name: Ruff format check - run: ruff format --check api src/absa tests scripts + run: ruff format --check src/absa tests scripts typecheck: name: Typecheck (mypy) @@ -42,7 +42,7 @@ jobs: pip install --upgrade pip pip install -e ".[dev]" - name: Mypy - run: mypy api src/absa + run: mypy src/absa security: name: Security (bandit) @@ -55,7 +55,7 @@ jobs: - name: Install bandit run: pip install bandit - name: Bandit scan - run: bandit -r api src/absa + run: bandit -r src/absa test: name: Tests (pytest) — py${{ matrix.python }} @@ -75,4 +75,4 @@ jobs: pip install --upgrade pip pip install -e ".[dev]" - name: Run tests - run: pytest tests/ -v \ No newline at end of file + run: pytest tests/unit/ -v \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7c1bf8bb82495451b34e4c2bf0636ecae4184e0e..5d05d3e12c8c1ef5d52556fad9a16f078a31f8b4 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,6 @@ logs/ # Node node_modules/ + +absa.db +absa.db-journal diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ff518d48a16861aee8529cad87a9feb7fcc6d54..f339d6a45b080bb12c68890887e525c13423ca1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,6 @@ repos: rev: v4.6.0 hooks: - id: check-yaml - exclude: ^docker/docker-compose.*\.ya?ml$ - id: check-json - id: end-of-file-fixer exclude: ^notebooks/ diff --git a/README.md b/README.md index 5db87107e3b4d32c67a25e57e530bdfe52003efa..2dc1e88295bcd0f5f1ee513faf0dbb37da0199fb 100644 --- a/README.md +++ b/README.md @@ -1,227 +1,51 @@ -# Multilingual Aspect-Based Sentiment Analysis (ABSA) - -[![Python](https://img.shields.io/badge/Python-3.10%2B-blue.svg)](https://python.org) -[![FastAPI](https://img.shields.io/badge/FastAPI-0.115-00a393.svg)](https://fastapi.tiangolo.com) -[![Streamlit](https://img.shields.io/badge/Streamlit-1.37-FF4B4B.svg)](https://streamlit.io) -[![DVC](https://img.shields.io/badge/DVC-3.51-945dd6.svg)](https://dvc.org) -[![MLflow](https://img.shields.io/badge/MLflow-2.15-0194E2.svg)](https://mlflow.org) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) - -**Analyse product reviews in English, Hindi, and Hinglish — extract aspects and their sentiment in real time.** - -A pure-Python stack: a Streamlit dashboard on top of a FastAPI JSON API, backed by a multilingual ML pipeline (ONNX inference with a rule-based fallback). - ---- - -## Quick Start - -```bash -git clone && cd multilingual-absa -python -m venv .venv && source .venv/bin/activate -pip install -e ".[dev]" -``` - -Start the API (port 8000) and the dashboard (port 8501) in two terminals: - -```bash -# Terminal 1 — REST API -uvicorn api.main:app --reload --host 0.0.0.0 --port 8000 - -# Terminal 2 — Streamlit dashboard -streamlit run frontend/Home.py -``` - -Open: -- **Dashboard** → [`http://localhost:8501`](http://localhost:8501) -- **API docs (Swagger)** → [`http://localhost:8000/docs`](http://localhost:8000/docs) - --- - -## What This Does - -The system identifies **aspects** (specific features like "battery life", "sound quality") and their **sentiment** (positive, negative, neutral) from product reviews. It supports three languages: - -| Language | Aspect Extraction | Sentiment | -|----------|:-:|:-:| -| English | ✅ | ✅ | -| Hindi | ✅ | ✅ | -| Hinglish | ✅ | ✅ | - -**Production performance** (ONNX INT8 quantised): - -| Language | F1 Score | Latency p95 | -|----------|:--------:|:-----------:| -| English | 78.1% | 185 ms | -| Hindi | 67.8% | 185 ms | - +title: Multilingual ABSA +emoji: 🎯 +colorFrom: blue +colorTo: purple +sdk: gradio +sdk_version: 4.44.0 +app_file: app.py +pinned: false +license: mit --- -## Project Structure +# Multilingual ABSA -``` -. -├── api/ # FastAPI REST service -│ ├── main.py # Entry point — uvicorn api.main:app -│ ├── models/ # SQLAlchemy ORM models (Review, AspectResult, BatchJob) -│ ├── middleware/ # Rate limiting, metrics (Prometheus), DB deps -│ ├── routes/ # /predict, /batch, /status, /download, /health, /info -│ ├── schemas/ # Pydantic request/response models -│ ├── services/ # ABSA inference pipeline, language detection -│ └── tasks/ # Celery batch processing workers -├── src/absa/ # Core ML library (src-layout, pip-installable) -│ ├── data/ # Loading, preprocessing, augmentation, transliteration -│ ├── models/ # Training scripts (ONNX, Transformers, baselines) -│ ├── evaluation/ # Cross-lingual eval, latency benchmarking -│ ├── training/ # MLflow experiment tracking -│ └── utils/ # Path + environment configuration -├── frontend/ # Streamlit dashboard (pure Python, no HTML templates) -│ ├── Home.py # Entry point — streamlit run frontend/Home.py -│ ├── absa_client.py # Thin HTTP client for the FastAPI backend -│ ├── ui.py # Native Streamlit UI helpers -│ └── views/ # Pages: predict, admin (overview/batch/monitor) -├── tests/ # Pytest suite -│ ├── conftest.py # sys.path bootstrap — no PYTHONPATH hacks required -│ ├── api/ # API endpoint tests -│ ├── web/ # Streamlit page + client tests -│ └── unit/ # Unit tests (bio tagger, lang detect) -├── scripts/ # Operational utility scripts -├── notebooks/ # Exploration & Colab training notebooks (numbered 01–05) -├── monitoring/ # Prometheus + Grafana config -├── docs/ # Documentation -├── docker/ # Containerisation -├── data/ # Datasets (managed by DVC) -├── models/ # ONNX model artifacts (DVC-tracked) -├── .github/workflows/ # CI pipeline (lint, typecheck, security, tests) -├── .env.example # Environment variable template -├── dvc.yaml # DVC data pipeline -├── pyproject.toml # Project metadata + tool config -└── Makefile # Developer command shortcuts -``` +Real-time aspect-based sentiment analysis for English, Hindi, Hinglish product reviews. ---- +🔗 **[Live Demo on Hugging Face Spaces](https://huggingface.co/spaces/Aryanmdev/multilingual-absa)** +📦 [GitHub repo](https://github.com/Aryanmishra-dev/Multilingual-Absa) -## Architecture - -``` -┌──────────┐ native Streamlit ┌──────────────────────────────┐ -│ Browser │◄────────────────────►│ Streamlit Dashboard (8501) │ -└──────────┘ │ frontend/ — pure Python UI │ - └──────────────┬───────────────┘ - │ HTTP (httpx) - ┌──────────────▼───────────────┐ - │ FastAPI REST API (8000) │ - │ api/main.py — JSON endpoints │ - └──────────────┬───────────────┘ - │ - ┌───────────────────────────┼────────────────────┐ - │ │ │ - ┌──────▼──────┐ ┌─────────▼────────┐ ┌────────▼───────┐ - │ PostgreSQL / │ │ Redis / Celery │ │ Prometheus + │ - │ SQLite │ │ (batch jobs) │ │ Grafana │ - │ (results) │ └──────────────────┘ └────────────────┘ - └─────────────┘ -``` - -### Inference Stack - -| Component | Layer | Notes | -|-----------|-------|-------| -| ONNX INT8 | Primary | Production — 185 ms p95 latency | -| ONNX FP32 | Fallback | Same model, no quantisation — 520 ms | -| Rule-based | Fallback | Keyword lexicon + context-window scoring — zero download | - -The pipeline auto-selects: **ONNX INT8** → **ONNX FP32** → **Rule-based**. No external model downloads required. - ---- - -## Features - -### Dashboard (Streamlit) - -- **Sentiment Analyzer** — Real-time aspect/sentiment analysis with confidence bars -- **Batch Analytics** — CSV upload, async Celery job, live progress, result download -- **System Monitor** — Health checks, service metadata, auto-refresh diagnostics -- **Admin Lock** — Optional password-gated admin section (`ADMIN_PASSWORD`) - -### API (RESTful JSON) - -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/predict` | POST | Analyse a single review | -| `/batch` | POST | Upload CSV for bulk analysis | -| `/status/{job_id}` | GET | Check batch job progress | -| `/download/{job_id}` | GET | Download batch results (CSV) | -| `/health` | GET | API health check | -| `/info` | GET | Model metadata | -| `/metrics` | GET | Prometheus metrics | -| `/docs` | GET | Swagger UI | - -### ML Pipeline (DVC) +## Run Locally ```bash -dvc pull # Download datasets -dvc repro # Reproduce preprocessing -dvc push # Upload to remote storage +git clone https://github.com/Aryanmishra-dev/Multilingual-Absa +cd Multilingual-Absa +pip install -r requirements.txt +python app.py ``` -### Monitoring +Open http://localhost:7860 -- **Prometheus** metrics at `/metrics` -- **Grafana** dashboards for request volume, latency, error rate -- **Evidently** drift detection (`scripts/drift_monitor.py`) -- **MLflow** experiment tracking (`scripts/mlflow_ui.sh`) +## What it does ---- +Extracts **aspects** (e.g. "battery", "camera") and their **sentiment** (positive/negative/neutral) from product reviews in 3 languages. -## Development +| Language | F1 | Latency p95 | +|----------|:--:|:-----------:| +| English | 78.1% | 185 ms | +| Hindi | 67.8% | 185 ms | -### Run Tests +ONNX INT8 quantized → FP32 → rule-based fallback. -```bash -make test # pytest -make lint # ruff -make typecheck # mypy -make security # bandit -``` +## Stack -### Run Full Stack (Docker) - -```bash -docker compose -f docker/docker-compose.yml up --build -``` - -Starts: API (8000), PostgreSQL, Redis, Celery worker, Prometheus (9090), Grafana (3001). - -### Environment Variables - -Copy `.env.example` to `.env` and configure: - -| Variable | Required | Description | -|----------|:--------:|-------------| -| `DATABASE_URL` | ✅ | PostgreSQL or SQLite connection string | -| `REDIS_URL` | ✅ | Redis connection for Celery | -| `CORS_ORIGINS` | ❌ | Allowed CORS origins (default: localhost) | -| `MODEL_PATH` | ❌ | Path to ONNX model directory | -| `MAX_BATCH_SIZE` | ❌ | Max reviews per batch job | -| `ADMIN_PASSWORD` | ❌ | Password for admin dashboard sections | -| `API_BASE_URL` | ❌ | Base URL the dashboard uses to reach the API | - ---- - -## Documentation - -| Document | Contents | -|----------|----------| -| [API](docs/API_DOCUMENTATION.md) | Full API reference with schemas and examples | -| [Architecture](docs/architecture.md) | System design, data flow, deployment | -| [Deployment](docs/DEPLOYMENT.md) | Production setup, Docker, Railway | -| [Database](docs/DATABASE.md) | Schema, migrations, query patterns | -| [Security](docs/SECURITY.md) | Threat model, audit results, mitigations | -| [Tech Stack](docs/TECH_STACK.md) | Framework versions, rationale, trade-offs | -| [System Design](docs/SYSTEM_DESIGN.md) | End-to-end design and data flow | - ---- +- Gradio 4.44 (UI) +- ONNX Runtime (inference) +- XLM-RoBERTa (multilingual backbone) +- SQLite (prediction history) ## License -MIT License. See [LICENSE](LICENSE) for details. +MIT \ No newline at end of file diff --git a/api/core/__init__.py b/api/core/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/api/main.py b/api/main.py deleted file mode 100644 index 6114f8f8ce09d0baf7c9143fb69aaaec51208da1..0000000000000000000000000000000000000000 --- a/api/main.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -from contextlib import asynccontextmanager - -from dotenv import load_dotenv -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from slowapi import Limiter, _rate_limit_exceeded_handler -from slowapi.errors import RateLimitExceeded -from slowapi.util import get_remote_address - -load_dotenv() - -from api.middleware.dependencies import engine # noqa: E402 -from api.middleware.metrics import instrumentator # noqa: E402 -from api.models.db_models import Base # noqa: E402 -from api.routes import predict, results # noqa: E402 -from api.services.absa_pipeline import pipeline # noqa: E402 - - -@asynccontextmanager -async def lifespan(app: FastAPI): - # Startup - print("Initializing Database tables...") - Base.metadata.create_all(bind=engine) - - print("Loading Models...") - pipeline.load_models() - - yield - # Shutdown - print("Shutting down...") - - -limiter = Limiter(key_func=get_remote_address) - -app = FastAPI( - title="Multilingual ABSA API", - description="Aspect-Based Sentiment Analysis for English and Hindi", - version="1.0.0", - lifespan=lifespan, -) - -app.state.limiter = limiter -app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) # type: ignore[arg-type] - -app.add_middleware( - CORSMiddleware, - allow_origins=os.getenv("CORS_ORIGINS", "http://localhost:8000,http://localhost:8501").split(","), - allow_credentials=True, - allow_methods=["GET", "POST"], - allow_headers=["*"], -) -app.include_router(predict.router, tags=["Predict"]) -app.include_router(results.router, tags=["System"]) - -instrumentator.instrument(app).expose(app, endpoint="/metrics") diff --git a/api/middleware/__init__.py b/api/middleware/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/api/middleware/dependencies.py b/api/middleware/dependencies.py deleted file mode 100644 index a527fa226009ebe08465dbcef8c281b816f3c871..0000000000000000000000000000000000000000 --- a/api/middleware/dependencies.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from dotenv import load_dotenv -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -load_dotenv() - -DATABASE_URL = os.getenv("DATABASE_URL") - -if not DATABASE_URL: - raise RuntimeError("DATABASE_URL environment variable is not set. Please set it in your .env file or environment.") - -connect_args = {} -if DATABASE_URL.startswith("sqlite"): - connect_args["check_same_thread"] = False - -engine = create_engine(DATABASE_URL, pool_pre_ping=True, connect_args=connect_args) -SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - - -def get_db(): - db = SessionLocal() - try: - yield db - finally: - db.close() diff --git a/api/middleware/metrics.py b/api/middleware/metrics.py deleted file mode 100644 index 0593632aad4676105b86d290a342c1fc138b1791..0000000000000000000000000000000000000000 --- a/api/middleware/metrics.py +++ /dev/null @@ -1,12 +0,0 @@ -from prometheus_fastapi_instrumentator import Instrumentator - -instrumentator = Instrumentator( - should_group_status_codes=False, - should_ignore_untemplated=True, - should_respect_env_var=True, - should_instrument_requests_inprogress=True, - excluded_handlers=[".*admin.*", "/metrics"], - env_var_name="ENABLE_METRICS", - inprogress_name="inprogress", - inprogress_labels=True, -) diff --git a/api/models/__init__.py b/api/models/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/api/py.typed b/api/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/api/routes/__init__.py b/api/routes/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/api/routes/predict.py b/api/routes/predict.py deleted file mode 100644 index 7c5cc1944961f6b32a6043c2d23802854af46f43..0000000000000000000000000000000000000000 --- a/api/routes/predict.py +++ /dev/null @@ -1,129 +0,0 @@ -import os -import re -import tempfile -import time -import uuid - -import pandas as pd -from fastapi import APIRouter, Depends, File, HTTPException, UploadFile -from sqlalchemy.orm import Session - -from api.middleware.dependencies import get_db -from api.models.db_models import AspectResult, BatchJob, Review -from api.schemas.schemas import BatchJobResponse, PredictionResponse, ReviewInput -from api.services.absa_pipeline import pipeline -from api.tasks.batch_tasks import process_batch - -router = APIRouter() - - -@router.post("/predict", response_model=PredictionResponse) -async def predict(request: ReviewInput, db: Session = Depends(get_db)): - try: - time.time() - - # Inference - prediction = pipeline.predict(request.text, request.language) - - # Save to DB - db_review = Review( - text=prediction.text, - language=prediction.language, - processing_time_ms=prediction.processing_time_ms, - ) - db.add(db_review) - db.commit() - db.refresh(db_review) - - for asp in prediction.aspects: - db_aspect = AspectResult( - review_id=db_review.id, - aspect=asp.aspect, - sentiment=asp.sentiment, - confidence=asp.confidence, - start_pos=asp.start, - end_pos=asp.end, - ) - db.add(db_aspect) - db.commit() - - return prediction - except Exception: - raise HTTPException(status_code=500, detail="Model inference failed. Please try again.") - - -@router.post("/batch", response_model=BatchJobResponse) -async def predict_batch(file: UploadFile = File(...), db: Session = Depends(get_db)): - max_upload_size = 50 * 1024 * 1024 # 50MB - - if not file.filename or not file.filename.endswith(".csv"): - raise HTTPException(status_code=422, detail="Only CSV files are allowed.") - - if file.content_type and file.content_type not in ("text/csv", "application/vnd.ms-excel", "text/plain", ""): - raise HTTPException(status_code=422, detail="Invalid file type. CSV required.") - - try: - content = await file.read() - if len(content) > max_upload_size: - raise HTTPException(status_code=422, detail="File exceeds 50MB maximum size.") - - # Create temp file to read - with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmp: - tmp.write(content) - tmp_path = tmp.name - - # Validate CSV structure before processing - try: - df = pd.read_csv(tmp_path, nrows=1) - except Exception: - os.unlink(tmp_path) - raise HTTPException(status_code=422, detail="Invalid CSV format.") - - df = pd.read_csv(tmp_path) - if "text" not in df.columns: - os.unlink(tmp_path) - raise HTTPException(status_code=422, detail="CSV must contain a 'text' column.") - - if len(df) > 10000: - os.unlink(tmp_path) - raise HTTPException(status_code=422, detail="Max 10,000 rows allowed per batch.") - - job_id_obj = uuid.uuid4() - job_id = str(job_id_obj) - db_job = BatchJob(id=job_id_obj, status="queued", total=len(df), processed=0) - db.add(db_job) - db.commit() - - # Queue Celery task - process_batch.delay(job_id, tmp_path) - - return BatchJobResponse(job_id=job_id, status="queued", total_reviews=len(df), processed=0) - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Batch processing failed. Please try again.") - - -@router.get("/status/{job_id}", response_model=BatchJobResponse) -async def get_batch_status(job_id: str, db: Session = Depends(get_db)): - if not re.match(r"^[a-fA-F0-9\-]{36}$", job_id): - raise HTTPException(status_code=400, detail="Invalid job ID format") - try: - job_id_uuid = uuid.UUID(job_id) - except ValueError: - raise HTTPException(status_code=400, detail="Invalid job ID format") - job = db.query(BatchJob).filter(BatchJob.id == job_id_uuid).first() - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - result_url = None - if job.status == "completed": - result_url = f"/results/download/{job_id}" - - return BatchJobResponse( - job_id=str(job.id), - status=job.status, - total_reviews=job.total, - processed=job.processed, - result_url=result_url, - ) diff --git a/api/routes/results.py b/api/routes/results.py deleted file mode 100644 index e308b19261326581e188282eba6500c5fdaaac2b..0000000000000000000000000000000000000000 --- a/api/routes/results.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import re -from pathlib import Path -from typing import Dict - -from fastapi import APIRouter, HTTPException -from fastapi.responses import FileResponse - -router = APIRouter() - -_RESULTS_DIR = Path("data/results").resolve() - - -@router.get("/health") -async def health_check() -> Dict[str, str]: - # Basic health check - return {"status": "ok", "model": "loaded", "db": "connected"} - - -@router.get("/info") -async def get_info() -> Dict[str, str]: - return { - "model_name": "xlm-roberta-base-absa", - "version": "1.0", - "supported_languages": "en, hi", - "max_batch_size": os.getenv("MAX_BATCH_SIZE", "10000"), - } - - -@router.get("/download/{job_id}") -async def download_result(job_id: str): - if not re.match(r"^[a-fA-F0-9\-]{36}$", job_id): - raise HTTPException(status_code=400, detail="Invalid job ID format") - resolved = (_RESULTS_DIR / f"{job_id}.csv").resolve() - if not str(resolved).startswith(str(_RESULTS_DIR)): - raise HTTPException(status_code=400, detail="Invalid job ID") - if not resolved.exists(): - raise HTTPException(status_code=404, detail="Result file not found") - return FileResponse(path=resolved, filename=f"absa_results_{job_id}.csv", media_type="text/csv") diff --git a/api/schemas/__init__.py b/api/schemas/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/api/schemas/schemas.py b/api/schemas/schemas.py deleted file mode 100644 index d785a9d00224bdf441b1c2116fdeca44995a68e3..0000000000000000000000000000000000000000 --- a/api/schemas/schemas.py +++ /dev/null @@ -1,40 +0,0 @@ -from typing import List, Optional - -from pydantic import BaseModel, ConfigDict, Field - - -class ReviewInput(BaseModel): - text: str = Field(..., max_length=10000, description="Review text to analyze") - language: Optional[str] = Field(None, max_length=20, description="Language code (en, hi, hinglish, auto)") - - model_config = ConfigDict(from_attributes=True) - - -class AspectSentiment(BaseModel): - aspect: str - sentiment: str - confidence: float - start: int - end: int - - model_config = ConfigDict(from_attributes=True) - - -class PredictionResponse(BaseModel): - text: str - language: str - detected_language: str - aspects: List[AspectSentiment] - processing_time_ms: float - - model_config = ConfigDict(from_attributes=True) - - -class BatchJobResponse(BaseModel): - job_id: str - status: str - total_reviews: int - processed: int - result_url: Optional[str] = None - - model_config = ConfigDict(from_attributes=True) diff --git a/api/services/__init__.py b/api/services/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/api/tasks/__init__.py b/api/tasks/__init__.py deleted file mode 100644 index e57aeb63a4bab132b4d22cb18e3a002a7e2a4ac5..0000000000000000000000000000000000000000 --- a/api/tasks/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -import os - -from celery import Celery - -redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0") - -celery_app = Celery("absa_tasks", broker=redis_url, backend=redis_url.replace("/0", "/1")) - -celery_app.conf.update( - task_serializer="json", - result_expires=3600, -) diff --git a/api/tasks/batch_tasks.py b/api/tasks/batch_tasks.py deleted file mode 100644 index 63e908f319e8c8b9af76c7687c98ceb699760586..0000000000000000000000000000000000000000 --- a/api/tasks/batch_tasks.py +++ /dev/null @@ -1,132 +0,0 @@ -import csv -import os -from datetime import datetime, timezone - -import pandas as pd - -from api.middleware.dependencies import SessionLocal -from api.models.db_models import AspectResult, BatchJob, Review -from api.services.absa_pipeline import pipeline -from api.tasks import celery_app - - -@celery_app.task(bind=True) -def process_batch(self, job_id: str, file_path: str): - db = SessionLocal() - try: - job = db.query(BatchJob).filter(BatchJob.id == job_id).first() - if not job: - return - - job.status = "processing" - db.commit() - - # Load CSV - df = pd.read_csv(file_path) - if "text" not in df.columns: - raise ValueError("CSV must contain a 'text' column.") - - texts = df["text"].tolist() - batch_size = 32 - - results_dir = "data/results" - os.makedirs(results_dir, exist_ok=True) - result_file = f"{results_dir}/{job_id}.csv" - - processed_count = 0 - - with open(result_file, "w", newline="", encoding="utf-8") as f: - writer = csv.writer(f) - writer.writerow( - [ - "text", - "language", - "aspect", - "sentiment", - "confidence", - "start_pos", - "end_pos", - "processing_time_ms", - ] - ) - - for i in range(0, len(texts), batch_size): - batch_texts = texts[i : i + batch_size] - predictions = pipeline.predict_batch(batch_texts) - - for pred in predictions: - # Save Review - db_review = Review( - text=pred.text, - language=pred.language, - processing_time_ms=pred.processing_time_ms, - ) - db.add(db_review) - db.commit() - db.refresh(db_review) - - # Save Aspects & CSV - for asp in pred.aspects: - db_aspect = AspectResult( - review_id=db_review.id, - aspect=asp.aspect, - sentiment=asp.sentiment, - confidence=asp.confidence, - start_pos=asp.start, - end_pos=asp.end, - ) - db.add(db_aspect) - writer.writerow( - [ - pred.text, - pred.language, - asp.aspect, - asp.sentiment, - asp.confidence, - asp.start, - asp.end, - pred.processing_time_ms, - ] - ) - - if not pred.aspects: - writer.writerow( - [ - pred.text, - pred.language, - "", - "", - "", - "", - "", - pred.processing_time_ms, - ] - ) - - db.commit() - processed_count += len(batch_texts) - - if processed_count % 100 == 0 or processed_count == len(texts): - job.processed = processed_count - db.commit() - - job.status = "completed" - job.completed_at = datetime.now(timezone.utc) - db.commit() - - except Exception: - job = db.query(BatchJob).filter(BatchJob.id == job_id).first() - if job: - job.status = "failed" - db.commit() - import logging - - logging.exception("Batch processing failed for job %s", job_id) - finally: - # Clean up temp file - try: - if file_path and os.path.exists(file_path): - os.unlink(file_path) - except OSError: - pass - db.close() diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b3a964449fb9564c1ea21871dbf8fc81e5d5c003 --- /dev/null +++ b/app.py @@ -0,0 +1,512 @@ +"""Single-file Gradio app for Multilingual ABSA. + +Loads the ONNX model (INT8 → FP32) with a rule-based fallback, runs inference +via ``absa.pipeline.ABSAPipeline``, persists predictions to SQLite, and serves +a Gradio (gr.Blocks) UI. Run with ``python app.py``. +""" + +from __future__ import annotations + +import logging +import os +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import gradio as gr +from sqlalchemy import Column, DateTime, Float, Integer, String, create_engine +from sqlalchemy.orm import declarative_base, sessionmaker + +# Make the src-layout `absa` package importable when run directly +# (e.g. `python app.py`) without a prior `pip install -e .`. All `absa` +# imports below are lazy, so this bootstrap runs before any of them. +_SRC_DIR = Path(__file__).resolve().parent / "src" +if str(_SRC_DIR) not in sys.path: + sys.path.insert(0, str(_SRC_DIR)) + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +_logger = logging.getLogger("absa.app") + +# ── SQLAlchemy / SQLite ─────────────────────────────────────────────────── +DB_PATH = "absa.db" +engine = create_engine( + f"sqlite:///{DB_PATH}", + connect_args={"check_same_thread": False}, +) +Base = declarative_base() +Session = sessionmaker(bind=engine) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class Prediction(Base): + """One aspect-sentiment pair extracted from a single review.""" + + __tablename__ = "predictions" + + id = Column(Integer, primary_key=True) + text = Column(String, nullable=False) + language = Column(String, nullable=False) + aspect = Column(String, nullable=False) + sentiment = Column(String, nullable=False) + confidence = Column(Float, nullable=False) + created_at = Column(DateTime, default=_utcnow) + + +Base.metadata.create_all(engine) + +# ── Lazy, failure-tolerant model loading ────────────────────────────────── +# Module-level singleton (cached for the app lifetime). If the ONNX models +# are missing the pipeline degrades to the rule-based engine — never crashes. + +DEFAULT_MODEL_PATH = Path(os.getenv("MODEL_PATH", "models/onnx")) + + +def _load_pipeline() -> Any: + """Import the ABSA pipeline lazily and attempt custom ONNX model load.""" + try: + from absa.pipeline.absa_pipeline import ABSAPipeline + except Exception as exc: # pragma: no cover - import-time guard + _logger.error("Could not import ABSA pipeline: %s", exc) + raise + + pipelin = ABSAPipeline() + try: + pipelin.load_models() + except Exception as exc: + _logger.warning("Custom model load failed (%s); using rule-based engine.", exc) + return pipelin + + +PIPELINE = _load_pipeline() + + +def _inference_mode() -> str: + """Report which engine is active: ONNX INT8 / ONNX FP32 / rule-based.""" + if PIPELINE.aspect_model is not None: + if (DEFAULT_MODEL_PATH / "aspect_extraction_int8").exists(): + return "ONNX INT8" + if (DEFAULT_MODEL_PATH / "aspect_extraction").exists(): + return "ONNX FP32" + return "ONNX" + return "rule-based" + + +MODE = _inference_mode() + + +def get_count() -> int: + """Total predictions in SQLite.""" + try: + session = Session() + return session.query(Prediction).count() + except Exception: + return 0 + finally: + session.close() + + +def get_history_rows() -> list[list]: + """Return last 50 predictions as DataFrame rows.""" + try: + session = Session() + rows = session.query(Prediction).order_by(Prediction.id.desc()).limit(50).all() + return [ + [ + r.created_at.strftime("%Y-%m-%d %H:%M") if r.created_at else "—", + r.language or "—", + r.aspect or "—", + r.sentiment or "—", + round(r.confidence, 3) if r.confidence else 0.0, + (r.text[:60] + "...") if r.text and len(r.text) > 60 else (r.text or "—"), + ] + for r in rows + ] + except Exception: + return [] + finally: + session.close() + + +def get_history_md() -> str: + return f"Showing last 50 predictions · Total: **{get_count()}**" + + +def clear_history() -> tuple[str, list]: + """Wipe predictions table.""" + try: + session = Session() + session.query(Prediction).delete() + session.commit() + except Exception: + pass + finally: + session.close() + return "History cleared · Total: **0**", [] + + +def on_analyze(text: str) -> tuple[str, str, str, Any, list]: + """Run inference, save to DB, return formatted UI outputs.""" + if not text or not text.strip(): + return ( + '
Detected Language: —Latency: —
', + '
Please enter a review to analyze.
', + '
No analysis yet.
', + gr.update(visible=False, value={}) + ) + + start = time.time() + try: + response = PIPELINE.predict(text) + aspects = response.aspects if hasattr(response, "aspects") else response.get("aspects", []) + language = response.detected_language if hasattr(response, "detected_language") else response.get("detected_language", "unknown") + elapsed_ms = (time.time() - start) * 1000 + except Exception as e: + return ( + '
Detected Language: ErrorLatency: —
', + '
Error during analysis.
', + f'
Unable to analyze this review: {str(e)[:120]}
', + gr.update(visible=False, value={}) + ) + + lang_display = { + "en": "English", "english": "English", + "hi": "Hindi", "hindi": "Hindi", + "hinglish": "Hinglish", "hi-en": "Hinglish", + }.get(str(language).lower(), str(language).title()) + + meta_md = f'
Detected Language: {lang_display}Latency: {elapsed_ms:.0f} ms
' + + pos_count = neg_count = neu_count = 0 + table_rows = "" + for a in aspects: + aspect_text = a.get("aspect", "") if isinstance(a, dict) else getattr(a, "aspect", "") + sentiment = a.get("sentiment", "neutral") if isinstance(a, dict) else getattr(a, "sentiment", "neutral") + confidence = float(a.get("confidence", 0.0) if isinstance(a, dict) else getattr(a, "confidence", 0.0)) + + if sentiment == "positive": + pos_count += 1 + sent_color = "#16A34A" + sent_label = "Positive" + elif sentiment == "negative": + neg_count += 1 + sent_color = "#DC2626" + sent_label = "Negative" + else: + neu_count += 1 + sent_color = "#64748B" + sent_label = "Neutral" + + conf_pct = int(confidence * 100) + + table_rows += f""" + + {aspect_text} + + + {sent_label} + + +
+ {conf_pct}% + + + """ + + if not aspects: + results_html = '
No aspects detected in this review.
' + else: + results_html = f""" + + + + + + + + + + {table_rows} + +
AspectSentimentConfidence
+ """ + + total = len(aspects) + + # Visualization blocks + vis_pos = f'Positive {"█" * pos_count} {pos_count}' if pos_count else 'Positive 0' + vis_neg = f'Negative {"█" * neg_count} {neg_count}' if neg_count else 'Negative 0' + vis_neu = f'Neutral {"█" * neu_count} {neu_count}' if neu_count else 'Neutral 0' + + summary_md = f""" +
+
{total} Aspects
+
{vis_pos}
+
{vis_neg}
+
{vis_neu}
+
+ """ + + # Save to SQLite + try: + session = Session() + for a in aspects: + aspect_text = a.get("aspect", "") if isinstance(a, dict) else getattr(a, "aspect", "") + sentiment = a.get("sentiment", "neutral") if isinstance(a, dict) else getattr(a, "sentiment", "neutral") + confidence = a.get("confidence", 0.0) if isinstance(a, dict) else getattr(a, "confidence", 0.0) + session.add(Prediction( + text=text, + language=lang_display, + aspect=aspect_text, + sentiment=sentiment, + confidence=float(confidence or 0.0), + )) + session.commit() + except Exception: + pass + finally: + session.close() + + return (meta_md, summary_md, results_html, gr.update(visible=True, value=aspects)) + + +# ── Gradio UI (gr.Blocks) ───────────────────────────────────────────────── + +custom_theme = gr.themes.Soft( + primary_hue=gr.themes.colors.blue, + secondary_hue=gr.themes.colors.slate, + neutral_hue=gr.themes.colors.slate, + font=gr.themes.GoogleFont("Inter"), +).set( + body_background_fill="#F8FAFC", + body_background_fill_dark="#F8FAFC", + body_text_color="#0F172A", + body_text_color_dark="#0F172A", + block_background_fill="#FFFFFF", + block_background_fill_dark="#FFFFFF", + block_border_color="#E2E8F0", + block_border_color_dark="#E2E8F0", + block_label_background_fill="#FFFFFF", + block_label_background_fill_dark="#FFFFFF", + block_label_text_color="#0F172A", + block_label_text_color_dark="#0F172A", + button_primary_background_fill="#2563EB", + button_primary_background_fill_dark="#2563EB", + button_primary_text_color="#FFFFFF", + button_primary_text_color_dark="#FFFFFF", + button_primary_border_color="#2563EB", + button_primary_border_color_dark="#2563EB", + button_secondary_background_fill="#FFFFFF", + button_secondary_background_fill_dark="#FFFFFF", + button_secondary_text_color="#0F172A", + button_secondary_text_color_dark="#0F172A", + button_secondary_border_color="#CBD5E1", + button_secondary_border_color_dark="#CBD5E1", + input_background_fill="#FFFFFF", + input_background_fill_dark="#FFFFFF", + input_border_color="#CBD5E1", + input_border_color_dark="#CBD5E1", + panel_background_fill="#FFFFFF", + panel_background_fill_dark="#FFFFFF", + table_even_background_fill="#F8FAFC", + table_even_background_fill_dark="#F8FAFC", + table_odd_background_fill="#FFFFFF", + table_odd_background_fill_dark="#FFFFFF", + table_border_color="#E2E8F0", + table_border_color_dark="#E2E8F0", + border_color_primary="#E2E8F0", + border_color_primary_dark="#E2E8F0", + color_accent_soft="#EFF6FF", + color_accent_soft_dark="#EFF6FF", +) + +with gr.Blocks(title="Multilingual ABSA") as demo: + + # TOP HEADER + with gr.Row(elem_classes="top-header"): + with gr.Column(scale=3, elem_classes="header-title-col"): + gr.Markdown(""" +
+

Multilingual ABSA

+
Aspect-Based Sentiment Analysis for English, Hindi, and Hinglish
+
+ """) + with gr.Column(scale=2, elem_classes="header-metrics-col"): + gr.Markdown(f""" +
+
MODE{MODE}
+
LANGUAGESEnglish · Hindi · Hinglish
+
PREDICTIONS{get_count()}
+
STATUS● Ready
+
+ """) + + # WORKSPACE + with gr.Tabs(elem_classes="main-tabs"): + with gr.Tab("Workspace"): + with gr.Row(elem_classes="workspace-row"): + + # LEFT PANEL + with gr.Column(scale=1, elem_classes="panel panel-left"): + gr.Markdown('
Analyze Review
Enter a product review to identify aspects and their sentiment.
') + + inp = gr.Textbox( + label="", + placeholder="Example: The battery life is amazing, but the camera quality is poor.", + lines=6, + max_lines=15, + elem_classes="review-input", + show_label=False, + ) + + btn = gr.Button("Analyze Review", variant="primary", elem_classes="analyze-btn") + + gr.Markdown('
Examples
') + gr.Examples( + examples=[ + ["The battery life is amazing but the camera quality is poor."], + ["बैटरी बहुत अच्छी है लेकिन कैमरा क्वालिटी खराब है।"], + ["Battery life mast hai lekin camera quality bahut bekar hai."], + ], + inputs=inp, + label="" + ) + + # RIGHT PANEL + with gr.Column(scale=1, elem_classes="panel panel-right"): + gr.Markdown('
Analysis Results
') + + meta_md = gr.Markdown('
Detected Language: WaitingLatency: —
') + summary_md = gr.Markdown('
Waiting for analysis...
') + results_html = gr.HTML('
No analysis yet.
') + + with gr.Accordion("Raw Model Output", open=False, elem_classes="raw-accordion"): + json_out = gr.JSON(label="", visible=False, elem_classes="raw-json") + + btn.click( + fn=lambda: ( + '
Detected Language: Analyzing...Latency: —
', + '
Analyzing review...
', + '
Analyzing...
' + ), + outputs=[meta_md, summary_md, results_html] + ).then( + fn=on_analyze, + inputs=inp, + outputs=[meta_md, summary_md, results_html, json_out] + ) + + # HISTORY TAB + with gr.Tab("History"): + with gr.Row(): + refresh_btn = gr.Button("Refresh Data", size="sm", variant="secondary") + clear_btn = gr.Button("Clear History", size="sm", variant="secondary") + + history_df = gr.Dataframe( + headers=["Timestamp", "Language", "Aspect", "Sentiment", "Confidence", "Review Preview"], + datatype=["str", "str", "str", "str", "number", "str"], + value=get_history_rows(), + interactive=False, + wrap=True, + elem_classes="history-table" + ) + + # Now we can attach the `.then` to update history_df + btn.click(fn=lambda: None).then(fn=get_history_rows, outputs=history_df) + + refresh_btn.click(fn=get_history_rows, outputs=history_df) + clear_btn.click(fn=clear_history, outputs=[history_df]).then(fn=get_history_rows, outputs=history_df) + +custom_css = """ + /* 60/30/10 Design System */ + + /* Layout & Base */ + .gradio-container { max-width: 1200px !important; margin: auto; font-family: 'Inter', sans-serif; padding: 24px !important; } + + /* Typography */ + .section-title { font-size: 1.1rem; font-weight: 600; color: #0F172A; margin-bottom: 4px; } + .section-sub { font-size: 0.9rem; color: #64748B; margin-bottom: 16px; } + + /* Top Header */ + .top-header { border-bottom: 1px solid #E2E8F0; padding-bottom: 16px; margin-bottom: 24px; align-items: center; } + .header-content h1 { font-size: 1.6rem; font-weight: 600; color: #0F172A; margin: 0 0 4px 0; letter-spacing: -0.02em; } + .header-subtitle { font-size: 0.9rem; color: #64748B; } + + /* Header Metrics */ + .header-metrics { display: flex; gap: 24px; justify-content: flex-end; flex-wrap: wrap; } + .metric { display: flex; flex-direction: column; } + .m-label { font-size: 0.65rem; font-weight: 600; color: #64748B; text-transform: uppercase; letter-spacing: 0.05em; margin-bottom: 4px; } + .m-val { font-size: 0.85rem; font-weight: 500; color: #0F172A; } + .status-dot { color: #16A34A; margin-right: 4px; } + + /* Panels */ + .workspace-row { gap: 32px !important; align-items: flex-start !important; } + .panel { background: #FFFFFF; border: 1px solid #E2E8F0; border-radius: 8px; padding: 24px; box-shadow: 0 1px 2px rgba(0,0,0,0.02); } + + /* Review Input */ + .review-input textarea { border: 1px solid #CBD5E1 !important; border-radius: 6px !important; padding: 12px !important; font-size: 0.95rem !important; line-height: 1.5 !important; background: #FFFFFF !important; color: #0F172A !important; transition: border-color 0.2s; box-shadow: none !important; } + .review-input textarea:focus { border-color: #2563EB !important; ring: 1px solid #2563EB !important; } + .analyze-btn { background: #2563EB !important; color: #FFFFFF !important; font-weight: 500 !important; border-radius: 6px !important; margin-top: 16px !important; padding: 10px 0 !important; border: none !important; transition: background 0.2s !important; } + .analyze-btn:hover { background: #1D4ED8 !important; } + + /* Examples */ + .compact-examples .gallery { gap: 8px !important; } + .compact-examples button { border: 1px solid #E2E8F0 !important; border-radius: 6px !important; padding: 8px 12px !important; font-size: 0.8rem !important; background: #F8FAFC !important; color: #475569 !important; text-align: left !important; } + .compact-examples button:hover { background: #F1F5F9 !important; border-color: #CBD5E1 !important; } + + /* Meta Badges */ + .meta-badges { display: flex; gap: 12px; margin-bottom: 24px; } + .badge { font-size: 0.75rem; background: #F1F5F9; color: #475569; padding: 4px 10px; border-radius: 4px; border: 1px solid #E2E8F0; } + .badge strong { color: #0F172A; font-weight: 600; margin-left: 4px; } + + /* Summary Vis */ + .summary-vis { margin-bottom: 24px; font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; font-size: 0.8rem; } + .summary-title { font-family: 'Inter', sans-serif; font-size: 0.9rem; font-weight: 600; color: #0F172A; margin-bottom: 12px; border-bottom: 1px solid #E2E8F0; padding-bottom: 8px; } + .vis-row { margin-bottom: 6px; display: flex; align-items: center; gap: 12px; color: #475569; } + .vis-block { letter-spacing: -1px; font-size: 0.7rem; } + .vis-block.pos { color: #16A34A; } + .vis-block.neg { color: #DC2626; } + .vis-block.neu { color: #64748B; } + .vis-block.empty-block { visibility: hidden; width: 10px; } + .summary-vis.empty { font-family: 'Inter', sans-serif; color: #64748B; font-style: italic; } + + /* Results Table */ + .results-table { width: 100%; border-collapse: collapse; margin-bottom: 16px; font-size: 0.85rem; } + .results-table th { text-align: left; padding: 8px 12px; border-bottom: 1px solid #E2E8F0; color: #64748B; font-weight: 600; font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.05em; } + .results-table td { padding: 12px; border-bottom: 1px solid #F1F5F9; color: #0F172A; } + .aspect-col { font-weight: 500; } + .sentiment-col { display: flex; align-items: center; gap: 6px; } + .sent-dot { width: 8px; height: 8px; border-radius: 50%; display: inline-block; } + + /* Confidence Bar */ + .conf-col { min-width: 120px; } + .conf-bar-bg { width: 60px; height: 6px; background: #E2E8F0; border-radius: 3px; display: inline-block; vertical-align: middle; margin-right: 8px; overflow: hidden; } + .conf-bar-fill { height: 100%; background: #2563EB; border-radius: 3px; } + .conf-val { font-family: ui-monospace, monospace; font-size: 0.8rem; color: #475569; } + + .empty-state { padding: 32px 0; text-align: center; color: #64748B; font-size: 0.9rem; border: 1px dashed #CBD5E1; border-radius: 6px; } + + /* Raw Output Accordion */ + .raw-accordion { margin-top: 24px !important; border: 1px solid #E2E8F0 !important; border-radius: 6px !important; overflow: hidden !important; background: #FFFFFF !important; } + .raw-accordion .label-wrap { padding: 12px 16px !important; font-size: 0.85rem !important; font-weight: 600 !important; color: #0F172A !important; background: #F8FAFC !important; border-bottom: 1px solid #E2E8F0 !important; } + .raw-accordion .raw-json { background: #0F172A !important; padding: 16px !important; margin: 0 !important; } + .raw-accordion .raw-json * { color: #F8FAFC !important; font-family: ui-monospace, SFMono-Regular, monospace !important; font-size: 0.8rem !important; } + + /* History Table */ + .history-table { border: 1px solid #E2E8F0 !important; border-radius: 8px !important; overflow: hidden !important; margin-top: 16px !important; } + .history-table th { background: #F8FAFC !important; color: #475569 !important; font-size: 0.75rem !important; text-transform: uppercase !important; font-weight: 600 !important; } + + /* Mobile Stacking */ + @media (max-width: 768px) { + .workspace-row { flex-direction: column !important; } + .panel { width: 100% !important; } + .header-metrics { justify-content: flex-start; margin-top: 16px; } + } +""" + +if __name__ == "__main__": + demo.launch(theme=custom_theme, css=custom_css) diff --git a/coverage.xml b/coverage.xml index af41505b0108d288ed332dfd517c29d4bb13cf21..6bbe31fc94821166fa94ecd8df4db54e4186b471 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,728 +1,199 @@ - + /Users/theogengineer/Projects/Multilingual-Absa - /Users/theogengineer/Projects/Multilingual-Absa/api - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - - - - - - - + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + - - - - - - - - - - - + - - - - - + - - - + + + + + + - - + + + - - - - + + - - - - - - + + - - - - - - - - - + + - - + + + + - + - - + + + - - - - - - - - - - - - - - - - + - + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - + + + + - - - + + + + @@ -731,47 +202,45 @@ - - - + - + + - + - + - + - - + + - + - + - + - + - - - + + @@ -779,118 +248,120 @@ + - - - + + - - + + - - + + + - - - + + + - - - + + + - - - - - + + + + + + + - - + + + - - - - + - - + + - + + + - + - - + + - - - - + + + + - + - + - - - + - - - - - - + + + + + + - + - + - + - + + @@ -898,20 +369,20 @@ - - - + + - - - - - + + + + + - - + + - + + @@ -927,21 +398,21 @@ + - + - - + + + - - + + + - - - - + @@ -957,83 +428,83 @@ - - + - + - + - + - - + + + + - + - - + + - - + + - + - + - - + + - + - - + + - - - - - + + + + + + + + + + + - + + + + + - - - - - - - - - - - - - + + @@ -1041,70 +512,70 @@ - - - - - + + + + - + - + - + - + + + - + - - - + + + + + - + + - + - + - - + + - + + - + - - - - + + + + - - - - - + + + - - - @@ -1145,7 +616,6 @@ - @@ -1153,283 +623,409 @@ - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + - - - + + - + + + + + - - - + + + + + - - - + + + - + + + + + + + + + + + + + + + + + + + + + + - + - + + - - - - - - + + + + + + - + + - - + + + + - - + + - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - + - - - - - + + + + + + + + + + + + + - + - - - - - - + + + + + - + + - - - + + - - - - - + + + + - - + + + + + + + + + + - - - - - - - - + - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + + + + - - + - - - - - - - - - - - - - - - + + + + - - + + + + + + - - + - - + - - - + @@ -1438,153 +1034,331 @@ - - - - - - - + - - - - + + + + - + - - - + + + - + + + + + + + + - + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - + + + + + + + - + + + - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - + + + + + + + - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1600,33 +1374,33 @@ - - + - - - + + + - - - + + + - + - - - + + + + + + - + - - + + - - @@ -1661,88 +1435,5 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/data/.gitignore b/data/.gitignore deleted file mode 100644 index 55ff1d08fa143e71485bfb1e45d46e1371fd3b4a..0000000000000000000000000000000000000000 --- a/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/raw diff --git a/data/demo/demo_single_reviews.txt b/data/demo/demo_single_reviews.txt deleted file mode 100644 index 514ae302ffed5f267bea77da0d4ab687b4a3a07c..0000000000000000000000000000000000000000 --- a/data/demo/demo_single_reviews.txt +++ /dev/null @@ -1,5 +0,0 @@ -The phone has an amazing screen but the battery life is terrible. -I absolutely love the camera quality on this device. -फोन की बैटरी अच्छी है लेकिन कैमरा बेकार है -खाना बहुत स्वादिष्ट था, पर सर्विस थोड़ी स्लो थी -Design mast hai par price bahut high hai diff --git a/data/demo/sample_reviews.csv b/data/demo/sample_reviews.csv deleted file mode 100644 index 01dbfce15aa98dd826424970136d365c361e8397..0000000000000000000000000000000000000000 --- a/data/demo/sample_reviews.csv +++ /dev/null @@ -1,21 +0,0 @@ -text,language -"The phone has an amazing screen but the battery life is terrible.",en -"I absolutely love the camera quality on this device.",en -"Shipping was very fast, but the customer service was useless when I had a question.",en -"The laptop keyboard feels incredibly cheap, though it runs fast.",en -"Amazing food, terrible atmosphere.",en -"Best purchase I made this year. High quality materials.",en -"The speakers are loud but the bass is non-existent.",en -"Wait staff was friendly, food was completely cold.",en -"I like the design but the software has too many bugs.",en -"Great value for money, highly recommended.",en -"फोन की बैटरी अच्छी है लेकिन कैमरा बेकार है",hi -"मुझे इस लैपटॉप की स्क्रीन बहुत पसंद आई",hi -"डिलीवरी बहुत लेट थी और पैकिंग भी खराब थी",hi -"खाना बहुत स्वादिष्ट था, पर सर्विस थोड़ी स्लो थी",hi -"यह प्रोडक्ट पैसे की बर्बादी है",hi -"Design mast hai par price bahut high hai",hinglish -"Screen quality ekdum awesome hai bhai",hinglish -"Customer support ne help nahi ki, very bad experience",hinglish -"Look and feel to accha hai but performance thik thak hai",hinglish -"Battery drain jaldi hota hai, overall not good",hinglish diff --git a/data/models/lid.176.ftz b/data/models/lid.176.ftz deleted file mode 100644 index 54ad911fadc26c1519c7043b6d596059b4116e66..0000000000000000000000000000000000000000 --- a/data/models/lid.176.ftz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8f3472cfe8738a7b6099e8e999c3cbfae0dcd15696aac7d7738a8039db603e83 -size 938013 diff --git a/data/raw.dvc b/data/raw.dvc deleted file mode 100644 index d5918dc6c98da7796b8fa439403d9421fef47a94..0000000000000000000000000000000000000000 --- a/data/raw.dvc +++ /dev/null @@ -1,6 +0,0 @@ -outs: -- md5: 0d10b4997c93bd1bce4a9d53a6e4748e.dir - size: 7943297 - nfiles: 21 - hash: md5 - path: raw diff --git a/data/tokenized/absa_cls_dataset/dataset_dict.json b/data/tokenized/absa_cls_dataset/dataset_dict.json deleted file mode 100644 index 9195703312e22d2b9b9fe14951aa733949480a2d..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/dataset_dict.json +++ /dev/null @@ -1 +0,0 @@ -{"splits": ["train", "validation", "test"]} \ No newline at end of file diff --git a/data/tokenized/absa_cls_dataset/test/data-00000-of-00001.arrow b/data/tokenized/absa_cls_dataset/test/data-00000-of-00001.arrow deleted file mode 100644 index cfbc284554eff55aab53388ee628e31c64ed9fcf..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/test/data-00000-of-00001.arrow +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1eb8d8230357e82431b68c2226770b209f0a61fe34425c0fd037b17f913a028c -size 101864 diff --git a/data/tokenized/absa_cls_dataset/test/dataset_info.json b/data/tokenized/absa_cls_dataset/test/dataset_info.json deleted file mode 100644 index 2f942a99b8a517c6c597bac554d2f0a325b77921..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/test/dataset_info.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "citation": "", - "description": "", - "features": { - "label": { - "dtype": "int64", - "_type": "Value" - }, - "input_ids": { - "feature": { - "dtype": "int32", - "_type": "Value" - }, - "_type": "Sequence" - }, - "attention_mask": { - "feature": { - "dtype": "int8", - "_type": "Value" - }, - "_type": "Sequence" - } - }, - "homepage": "", - "license": "" -} \ No newline at end of file diff --git a/data/tokenized/absa_cls_dataset/test/state.json b/data/tokenized/absa_cls_dataset/test/state.json deleted file mode 100644 index 80b334405a0377096b5b12fe51f6ccb742d4ab61..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/test/state.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "_data_files": [ - { - "filename": "data-00000-of-00001.arrow" - } - ], - "_fingerprint": "b686d9c13fd54a9d", - "_format_columns": null, - "_format_kwargs": {}, - "_format_type": null, - "_output_all_columns": false, - "_split": null -} \ No newline at end of file diff --git a/data/tokenized/absa_cls_dataset/train/data-00000-of-00001.arrow b/data/tokenized/absa_cls_dataset/train/data-00000-of-00001.arrow deleted file mode 100644 index e022fe6ac8bc03dfbf6d75612364cd99ac81b3b5..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/train/data-00000-of-00001.arrow +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:74827f37b8915b4946a33e030d5833d109bfcf9d9d847edf54fd830fe33a0a3d -size 808112 diff --git a/data/tokenized/absa_cls_dataset/train/dataset_info.json b/data/tokenized/absa_cls_dataset/train/dataset_info.json deleted file mode 100644 index 2f942a99b8a517c6c597bac554d2f0a325b77921..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/train/dataset_info.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "citation": "", - "description": "", - "features": { - "label": { - "dtype": "int64", - "_type": "Value" - }, - "input_ids": { - "feature": { - "dtype": "int32", - "_type": "Value" - }, - "_type": "Sequence" - }, - "attention_mask": { - "feature": { - "dtype": "int8", - "_type": "Value" - }, - "_type": "Sequence" - } - }, - "homepage": "", - "license": "" -} \ No newline at end of file diff --git a/data/tokenized/absa_cls_dataset/train/state.json b/data/tokenized/absa_cls_dataset/train/state.json deleted file mode 100644 index 797cf94fce60cd99fa3c1092914e0af41c58b5a9..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/train/state.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "_data_files": [ - { - "filename": "data-00000-of-00001.arrow" - } - ], - "_fingerprint": "59c1808a1a0be248", - "_format_columns": null, - "_format_kwargs": {}, - "_format_type": null, - "_output_all_columns": false, - "_split": null -} \ No newline at end of file diff --git a/data/tokenized/absa_cls_dataset/validation/data-00000-of-00001.arrow b/data/tokenized/absa_cls_dataset/validation/data-00000-of-00001.arrow deleted file mode 100644 index e22b973ef5f7a27472aab84403dd67937ce1bde0..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/validation/data-00000-of-00001.arrow +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a9da21b3401a16a246766047cda0695dcf07ef7d3c7eb64804ed9f59f0746f08 -size 105136 diff --git a/data/tokenized/absa_cls_dataset/validation/dataset_info.json b/data/tokenized/absa_cls_dataset/validation/dataset_info.json deleted file mode 100644 index 2f942a99b8a517c6c597bac554d2f0a325b77921..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/validation/dataset_info.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "citation": "", - "description": "", - "features": { - "label": { - "dtype": "int64", - "_type": "Value" - }, - "input_ids": { - "feature": { - "dtype": "int32", - "_type": "Value" - }, - "_type": "Sequence" - }, - "attention_mask": { - "feature": { - "dtype": "int8", - "_type": "Value" - }, - "_type": "Sequence" - } - }, - "homepage": "", - "license": "" -} \ No newline at end of file diff --git a/data/tokenized/absa_cls_dataset/validation/state.json b/data/tokenized/absa_cls_dataset/validation/state.json deleted file mode 100644 index 7497235ae0f6e81b59c91fd07e428ff93c8a9621..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_cls_dataset/validation/state.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "_data_files": [ - { - "filename": "data-00000-of-00001.arrow" - } - ], - "_fingerprint": "a33b9cfd2402e633", - "_format_columns": null, - "_format_kwargs": {}, - "_format_type": null, - "_output_all_columns": false, - "_split": null -} \ No newline at end of file diff --git a/data/tokenized/absa_ner_dataset/dataset_dict.json b/data/tokenized/absa_ner_dataset/dataset_dict.json deleted file mode 100644 index 9195703312e22d2b9b9fe14951aa733949480a2d..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/dataset_dict.json +++ /dev/null @@ -1 +0,0 @@ -{"splits": ["train", "validation", "test"]} \ No newline at end of file diff --git a/data/tokenized/absa_ner_dataset/test/data-00000-of-00001.arrow b/data/tokenized/absa_ner_dataset/test/data-00000-of-00001.arrow deleted file mode 100644 index 8f1d2a71473feac93103a3305477f6bbf1399166..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/test/data-00000-of-00001.arrow +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e1ec218e3b5b0c88fd86358c8e87f44709510cff43c41a81dac98274ce4eb0e7 -size 141712 diff --git a/data/tokenized/absa_ner_dataset/test/dataset_info.json b/data/tokenized/absa_ner_dataset/test/dataset_info.json deleted file mode 100644 index 796ec9ab4476a58a5071baee3accdab1b50142f3..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/test/dataset_info.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "citation": "", - "description": "", - "features": { - "input_ids": { - "feature": { - "dtype": "int32", - "_type": "Value" - }, - "_type": "Sequence" - }, - "attention_mask": { - "feature": { - "dtype": "int8", - "_type": "Value" - }, - "_type": "Sequence" - }, - "labels": { - "feature": { - "dtype": "int64", - "_type": "Value" - }, - "_type": "Sequence" - } - }, - "homepage": "", - "license": "" -} \ No newline at end of file diff --git a/data/tokenized/absa_ner_dataset/test/state.json b/data/tokenized/absa_ner_dataset/test/state.json deleted file mode 100644 index ba143fd1c7db37f172ea9345544e692f0802b378..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/test/state.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "_data_files": [ - { - "filename": "data-00000-of-00001.arrow" - } - ], - "_fingerprint": "0fa99b895566c7b5", - "_format_columns": null, - "_format_kwargs": {}, - "_format_type": null, - "_output_all_columns": false, - "_split": null -} \ No newline at end of file diff --git a/data/tokenized/absa_ner_dataset/train/data-00000-of-00001.arrow b/data/tokenized/absa_ner_dataset/train/data-00000-of-00001.arrow deleted file mode 100644 index 078febdf904cbeac533b451ab1a550c4fdf2ee9b..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/train/data-00000-of-00001.arrow +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:76c719d76d95127fc82691bbe1455e988c0c593fcdd510f270deac3e625cb929 -size 1101368 diff --git a/data/tokenized/absa_ner_dataset/train/dataset_info.json b/data/tokenized/absa_ner_dataset/train/dataset_info.json deleted file mode 100644 index 796ec9ab4476a58a5071baee3accdab1b50142f3..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/train/dataset_info.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "citation": "", - "description": "", - "features": { - "input_ids": { - "feature": { - "dtype": "int32", - "_type": "Value" - }, - "_type": "Sequence" - }, - "attention_mask": { - "feature": { - "dtype": "int8", - "_type": "Value" - }, - "_type": "Sequence" - }, - "labels": { - "feature": { - "dtype": "int64", - "_type": "Value" - }, - "_type": "Sequence" - } - }, - "homepage": "", - "license": "" -} \ No newline at end of file diff --git a/data/tokenized/absa_ner_dataset/train/state.json b/data/tokenized/absa_ner_dataset/train/state.json deleted file mode 100644 index 35dd330b720eaa90b7c7ef2fa03df2d199a58296..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/train/state.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "_data_files": [ - { - "filename": "data-00000-of-00001.arrow" - } - ], - "_fingerprint": "535b22b0ddf3661d", - "_format_columns": null, - "_format_kwargs": {}, - "_format_type": null, - "_output_all_columns": false, - "_split": null -} \ No newline at end of file diff --git a/data/tokenized/absa_ner_dataset/validation/data-00000-of-00001.arrow b/data/tokenized/absa_ner_dataset/validation/data-00000-of-00001.arrow deleted file mode 100644 index c4103cc61a2cf9559896e4fa67744424a30e2cbf..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/validation/data-00000-of-00001.arrow +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f94a6981561bcaa124a70fb560b93e355e57df287e9e721a2e6e887b6689b236 -size 144312 diff --git a/data/tokenized/absa_ner_dataset/validation/dataset_info.json b/data/tokenized/absa_ner_dataset/validation/dataset_info.json deleted file mode 100644 index 796ec9ab4476a58a5071baee3accdab1b50142f3..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/validation/dataset_info.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "citation": "", - "description": "", - "features": { - "input_ids": { - "feature": { - "dtype": "int32", - "_type": "Value" - }, - "_type": "Sequence" - }, - "attention_mask": { - "feature": { - "dtype": "int8", - "_type": "Value" - }, - "_type": "Sequence" - }, - "labels": { - "feature": { - "dtype": "int64", - "_type": "Value" - }, - "_type": "Sequence" - } - }, - "homepage": "", - "license": "" -} \ No newline at end of file diff --git a/data/tokenized/absa_ner_dataset/validation/state.json b/data/tokenized/absa_ner_dataset/validation/state.json deleted file mode 100644 index bf00b87009fcc3afaebbf31fd02d78bacf040bfa..0000000000000000000000000000000000000000 --- a/data/tokenized/absa_ner_dataset/validation/state.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "_data_files": [ - { - "filename": "data-00000-of-00001.arrow" - } - ], - "_fingerprint": "caedb7425c742cc1", - "_format_columns": null, - "_format_kwargs": {}, - "_format_type": null, - "_output_all_columns": false, - "_split": null -} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 2c80af27b9f111a671b155474b00ccedc5bd022d..0000000000000000000000000000000000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Stage 1: Builder -FROM python:3.11-slim AS builder - -WORKDIR /app -COPY pyproject.toml . -COPY src ./src -COPY api ./api - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - git \ - && rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir --prefix=/install . - -# Stage 2: Runtime -FROM python:3.11-slim - -WORKDIR /app - -# Copy dependencies -COPY --from=builder /install /usr/local - -# Copy application code -COPY api /app/api -COPY src /app/src -COPY scripts /app/scripts -COPY docker /app/docker -COPY .env.example /app/.env.example -# .env is injected via docker-compose environment vars — no need to COPY it - -ENV PYTHONPATH=/app/src - -# Add a non-root user -RUN adduser --disabled-password --gecos "" absauser \ - && chown -R absauser /app - -USER absauser - -EXPOSE 8000 - -CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/Dockerfile.prod b/docker/Dockerfile.prod deleted file mode 100644 index 899d9bf8d2bc46ef019c760c9cb0d1f06d1ef6f6..0000000000000000000000000000000000000000 --- a/docker/Dockerfile.prod +++ /dev/null @@ -1,19 +0,0 @@ -FROM python:3.11-slim as builder -WORKDIR /app -COPY pyproject.toml . -COPY src ./src -COPY api ./api -RUN pip install --no-cache-dir . - -FROM python:3.11-slim as runtime -WORKDIR /app -COPY --from=builder /usr/local/lib/python3.11 /usr/local/lib/python3.11 -COPY --from=builder /usr/local/bin /usr/local/bin -COPY api/ ./api/ -COPY src/ ./src/ -ENV PYTHONPATH=/app/src -ENV MODEL_SOURCE=huggingface_hub -RUN useradd -m appuser && chown -R appuser /app -USER appuser -EXPOSE 8000 -CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"] diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml deleted file mode 100644 index 60da19278903393825b73bcae239c4ddc38585af..0000000000000000000000000000000000000000 --- a/docker/docker-compose.prod.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.9" -services: - api: - image: ghcr.io/YOUR_USERNAME/multilingual-absa/api:latest - restart: always - environment: - - LOG_LEVEL=WARNING - deploy: - resources: - limits: - memory: 2G - worker: - image: ghcr.io/YOUR_USERNAME/multilingual-absa/api:latest - command: celery -A api.tasks.batch_tasks worker --loglevel=warning - restart: always - diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 76f1731036ff08a39420cb69762e0d67746e9b3a..0000000000000000000000000000000000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,79 +0,0 @@ -version: '3.8' - -services: - api: - build: - context: ../ - dockerfile: Dockerfile - container_name: absa-api - ports: - - "8000:8000" - environment: - - DATABASE_URL=${DATABASE_URL} - - REDIS_URL=${REDIS_URL} - - MODEL_PATH=models/onnx/ - - MAX_BATCH_SIZE=${MAX_BATCH_SIZE:-10000} - depends_on: - - postgres - - redis - volumes: - - ../models:/app/models - - ../data:/app/data - - worker: - build: - context: ../ - dockerfile: Dockerfile - container_name: absa-worker - command: ["celery", "-A", "api.tasks", "worker", "--loglevel=info"] - environment: - - DATABASE_URL=${DATABASE_URL} - - REDIS_URL=${REDIS_URL} - - MODEL_PATH=models/onnx/ - depends_on: - - postgres - - redis - - api - volumes: - - ../models:/app/models - - ../data:/app/data - - postgres: - image: postgres:16-alpine - container_name: absa-postgres - environment: - - POSTGRES_USER=${POSTGRES_USER:-absa_user} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?err} - - POSTGRES_DB=${POSTGRES_DB:-absa_db} - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - redis: - image: redis:7-alpine - container_name: absa-redis - ports: - - "6379:6379" - - prometheus: - image: prom/prometheus:latest - volumes: - - ../monitoring/prometheus.yml:/etc/prometheus/prometheus.yml - ports: ["9090:9090"] - command: - - "--config.file=/etc/prometheus/prometheus.yml" - - "--storage.tsdb.retention.time=15d" - - grafana: - image: grafana/grafana:latest - ports: ["3001:3000"] - volumes: - - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards - - ../monitoring/grafana/provisioning:/etc/grafana/provisioning - environment: - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?err} - GF_USERS_ALLOW_SIGN_UP: "false" - -volumes: - postgres_data: diff --git a/docs/API_DOCUMENTATION.md b/docs/API_DOCUMENTATION.md index 35849bc51a5ddc77646df3b7fb30667aab369fad..e8ec37d57fea8f1119b147e1bd44bf4a0d0df0e6 100644 --- a/docs/API_DOCUMENTATION.md +++ b/docs/API_DOCUMENTATION.md @@ -1,5 +1,7 @@ # API Documentation — Multilingual ABSA +> ⚠️ The REST API has been removed. The app is now a single Gradio interface. This doc is kept for historical reference only. + ## Base URL - Local development: `http://localhost:8000` diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 1a0116ccd1d31bfece3decda17de06d8c2025348..e35fa3a89582ba4427801d74e1f527efcda01f1e 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -1,130 +1 @@ -# Deployment Guide — Multilingual ABSA - -## Prerequisites - -- Python 3.11+ -- -- Docker & Docker Compose (for containerized deployment) -- Railway account (for API deployment) -- - -## Environment Variables - -| Variable | Dev Default | Production | Required By | -|----------|-------------|------------|-------------| -| `DATABASE_URL` | `sqlite:///absa.db` | PostgreSQL URL | API + Worker | -| `REDIS_URL` | `redis://localhost:6379/0` | Redis URL | API + Worker | -| `MODEL_PATH` | `models/onnx/` | (same or HF Hub) | API | -| `MAX_BATCH_SIZE` | `10000` | `10000` | API | -| `LOG_LEVEL` | `INFO` | `WARNING` | API | -| `ENABLE_METRICS` | `true` | `true` | API | -| `HF_MODEL_REPO` | (empty) | `username/multilingual-absa` | API | -| `MODEL_SOURCE` | `local` | `huggingface_hub` | API | - -## Local Development - -```bash -# Backend -cp .env.example .env -python -m venv .venv && source .venv/bin/activate -pip install -e ".[dev]" -uvicorn api.main:app --reload --host 0.0.0.0 --port 8000 -# API at http://localhost:8000, docs at http://localhost:8000/docs - -# MLflow -./scripts/mlflow_ui.sh -# MLflow UI at http://localhost:5000 - -# Frontend -``` - -## Docker Compose (Full Stack) - -```bash -docker-compose -f config/docker/docker-compose.yml up --build -``` - -Services started: - -| Service | Container Name | Port | Dependencies | -|---------|---------------|------|--------------| -| `api` | `absa-api` | 8000 | postgres, redis | -| `worker` | `absa-worker` | — | postgres, redis, api | -| `dashboard` | `absa-dashboard` | 3000 (=> 80) | api | -| `postgres` | `absa-postgres` | 5432 | — | -| `redis` | `absa-redis` | 6379 | — | -| `prometheus` | — | 9090 | api | -| `grafana` | — | 3001 | prometheus | - -```mermaid -graph TB - DASH[Dashboard :3000] --> API[API :8000] - API --> PG[PostgreSQL :5432] - API --> RED[Redis :6379] - WORK[Worker] --> RED - WORK --> PG - PROM[Prometheus :9090] -->|scrape| API - GRAF[Grafana :3001] --> PROM -``` - -## Production Deployment (Railway + Vercel) - -### Railway (API + Worker) - -1. Create a Railway project from your Git repository -2. Set build command: uses `railway.json` → `Dockerfile.api.prod` -3. Set environment variables in Railway dashboard: - - `DATABASE_URL` → Railway PostgreSQL plugin connection string - - `REDIS_URL` → Railway Redis plugin connection string - - `MODEL_SOURCE=huggingface_hub` - - `HF_MODEL_REPO=your-username/multilingual-absa` - - `ENABLE_METRICS=true` - - `LOG_LEVEL=WARNING` -4. Add a second service for the Celery worker with command: - `celery -A api.tasks.batch_tasks worker --loglevel=warning` - -### Vercel (Dashboard) - -2. Framework preset: Vite -3. Environment variable: `VITE_API_URL=https://your-railway-api-url.railway.app` -4. `vercel.json` rewrites `/api/*` to Railway API - -```mermaid -graph LR - USER[Browser] --> VERCEL[Vercel CDN] - VERCEL -->|/api/* rewrite| RAILWAY[Railway API] - RAILWAY --> PG[(Railway PostgreSQL)] - RAILWAY --> REDIS[(Railway Redis)] - WORK[Celery Worker] --> REDIS - WORK --> PG -``` - -## DVC Data/Model Sync - -```bash -# Pull data/models from remote -dvc pull - -# Run full ML pipeline -dvc repro - -# Push new artifacts -dvc push -``` - -## Monitoring - -| Tool | URL | Purpose | -|------|-----|---------| -| MLflow UI | `http://localhost:5000` | Experiment tracking | -| API Docs | `http://localhost:8000/docs` | Interactive API | -| Prometheus | `http://localhost:9090` | Metrics store | -| Grafana | `http://localhost:3001` | Visual dashboards | -| Dashboard | `http://localhost:5173` | User interface | - -## Scaling - -- **API**: Increase `--workers` in uvicorn command (2 in prod Dockerfile) -- **Worker**: Scale Celery worker containers horizontally -- **Database**: Use Railway managed PostgreSQL with auto-scaling -- **Memory limit**: 2GB per API container (configured in `docker-compose.prod.yml`) +This project is now deployed as a single Hugging Face Space (Gradio SDK). See README.md for the live demo link. diff --git a/docs/architecture.md b/docs/architecture.md index dd293dbaff35b5ec34ac6a816ae5f70cb57b6e32..d9d1da049e08a16be429ecafec1428f721ce42c7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,267 +1 @@ -# Multilingual ABSA — Architecture Document - -## System Overview - -Multilingual ABSA is a production-ready Aspect-Based Sentiment Analysis system supporting English, Hindi, and Hinglish. It extracts aspect terms from product reviews and classifies their sentiment using a dual-engine architecture: INT8-quantized ONNX models for production inference with a zero-download rule-based fallback. - -## High-Level Architecture - -```mermaid -graph TB - Client[Client Browser] - DASH[Streamlit Dashboard :8501] - API[FastAPI Server] - Pipeline[ABSA Pipeline] - Lang[Language Detection] - Neural[ONNX Neural Engine
INT8 Quantized] - Fallback[Rule-Based Engine
140+ Aspect Keywords] - Celery[Celery Worker] - Redis[(Redis)] - PG[(PostgreSQL)] - Prom[Prometheus] - Graf[Grafana] - MLflow[(MLflow
Experiment Tracking)] - - Client --> DASH - DASH -->|HTTP / JSON| API - API --> Pipeline - Pipeline --> Lang - Pipeline --> Neural - Pipeline --> Fallback - API --> Celery - Celery --> Redis - API --> PG - Prom -->|scrape /metrics| API - Graf --> Prom - MLflow --> Pipeline -``` - -## Component Architecture - -```mermaid -graph LR - subgraph "Presentation Layer" - SPA[Streamlit Dashboard
frontend/ - pure Python] - WIDGETS[Native Widgets
No HTML templates] - CLIENT[httpx API Client] - end - subgraph "API Layer" - FAST[FastAPI] - CORS[CORS Middleware] - PROM[Prometheus Metrics] - PYD[Pydantic Schemas] - end - subgraph "Service Layer" - ABSA[ABSAPipeline] - LANG[LanguageService] - CEL[Celery Tasks] - end - subgraph "Data Layer" - SQLA[SQLAlchemy ORM] - PG[(PostgreSQL)] - RED[(Redis)] - end - subgraph "ML Layer" - ONNX_A[ORTModelFor
TokenClassification] - ONNX_S[ORTModelFor
SequenceClassification] - LEXICON[Aspect/Sentiment
Lexicons] - end - subgraph "MLOps Layer" - MLF[MLflow Tracking] - DVC[DVC Versioning] - EVI[Evidently Drift] - end - - SPA --> FAST - FAST --> CORS - FAST --> PROM - FAST --> PYD - FAST --> ABSA - FAST --> CEL - ABSA --> LANG - ABSA --> ONNX_A - ABSA --> ONNX_S - ABSA --> LEXICON - CEL --> RED - FAST --> SQLA - SQLA --> PG - ABSA --> MLF -``` - -## Deployment Architecture - -```mermaid -graph TB - subgraph "Docker Compose (Local)" - DC_API[API Service
uvicorn:8000] - DC_WORKER[Celery Worker] - DC_DASH[Streamlit Dashboard
frontend:8501] - DC_PG[PostgreSQL:5432] - DC_REDIS[Redis:6379] - DC_PROM[Prometheus:9090] - DC_GRAF[Grafana:3001] - end - subgraph "Railway (Production)" - RW_API[API Service
$PORT] - RW_WORKER[Celery Worker] - RW_PG[PostgreSQL] - RW_REDIS[Redis] - end - - DC_DASH --> DC_API - DC_API --> DC_PG - DC_API --> DC_REDIS - DC_WORKER --> DC_REDIS - DC_WORKER --> DC_PG - DC_PROM -->|scrape| DC_API - DC_GRAF --> DC_PROM - - RW_API --> RW_PG - RW_API --> RW_REDIS - RW_WORKER --> RW_REDIS -``` - -## ML Pipeline Architecture - -```mermaid -graph TB - subgraph "Data Ingestion" - RAW[Raw Data
SemEval 2014
Amazon Hindi] - FAST[f astText LID
lid.176.ftz] - end - subgraph "Preprocessing" - CLEAN[Text Cleaning
Lowercase, URLs, Mentions] - TRANS[Transliteration
Devanagari→Roman] - LANG_DET[Language Detection
EN / HI / Hinglish] - BIO[BIO Tagging
B-ASP / I-ASP / O] - TOK[XLM-R Tokenizer
SentencePiece 128 tokens] - end - subgraph "Training" - ATE[Aspect Extraction
Token Classification
3 labels] - ASC[Sentiment Classification
Sequence Classification
4 labels] - BASELINE[Baseline
TF-IDF + LR] - QLORA[QLoRA
4-bit + LoRA] - JOINT[Joint ABSA
Shared Encoder
2 Heads] - end - subgraph "Optimization" - ONNX_EXP[ONNX Export
optimum-onnx] - QUANT[INT8 Quantization
Dynamic] - end - subgraph "Production" - INFERENCE[Dual-Engine
Inference] - BATCH[Batch Processing
Celery Worker] - end - subgraph "Evaluation" - EVAL_METRICS[Macro-F1
Per-class F1
Confusion Matrix] - LATENCY[Latency Benchmark
P95 < 300ms] - CROSS[Cross-Lingual Eval
EN→HI Zero-Shot] - end - - RAW --> CLEAN - FAST --> LANG_DET - CLEAN --> LANG_DET - LANG_DET --> TRANS - TRANS --> TOK - TOK --> ATE - TOK --> ASC - BIO --> ATE - ATE --> JOINT - ASC --> JOINT - ATE --> ONNX_EXP - ASC --> ONNX_EXP - ONNX_EXP --> QUANT - QUANT --> INFERENCE - INFERENCE --> BATCH - ATE --> EVAL_METRICS - ASC --> EVAL_METRICS - BASELINE --> EVAL_METRICS - INFERENCE --> LATENCY - JOINT --> CROSS -``` - -## Data Flow - -```mermaid -sequenceDiagram - participant C as Client - participant F as FastAPI - participant P as ABSAPipeline - participant L as LangService - participant N as ONNX Runtime - participant R as Rule Engine - participant D as PostgreSQL - participant M as Prometheus - - C->>F: POST /predict {text, language?} - F->>P: pipeline.predict(text, lang) - P->>L: detect_language(text) - L-->>P: "en" | "hi" | "hinglish" - alt ONNX Models Available - P->>N: Tokenize text - N-->>P: Token IDs + Attention Mask - P->>N: ORTModelForTokenClassification - N-->>P: BIO Logits → Argmax → Spans - P->>N: Per-aspect ORTModelForSequenceClassification - N-->>P: Sentiment Logits → Softmax - else Rule-Based Fallback - P->>R: _extract_aspects(text) - R-->>P: [(aspect, start, end)] - P->>R: _score_sentence(context) - R-->>P: (pos_score, neg_score) - P->>R: _score_to_label(pos, neg) - R-->>P: (sentiment, confidence) - end - P-->>F: PredictionResponse - F->>D: INSERT Review + AspectResults - D-->>F: IDs - F-->>C: JSON Response - F->>M: Record latency + status -``` - -## Infrastructure - -```mermaid -graph TB - subgraph "Edge" - SSL[TLS Termination
Reverse Proxy] - end - subgraph "Frontend Hosting" - FE[Streamlit Dashboard
Pure Python] - end - subgraph "Backend Hosting" - BE[Railway
Docker Container] - HEALTH[Health Check
/health] - AUTO[Auto-Restart
On Failure] - end - subgraph "Data Services" - PG[PostgreSQL
Railway Managed] - RD[Redis
Railway Managed] - end - subgraph "Observability" - PROM[Prometheus
15-day Retention] - GRAF[Grafana
Pre-provisioned Dashboard] - MLFLOW[MLflow
SQLite Backend] - end - - SSL --> FE - FE --> BE - BE --> HEALTH - BE --> AUTO - BE --> PG - BE --> RD - PROM -->|scrape| BE - GRAF --> PROM -``` - -## Design Decisions - -| Decision | Rationale | -|----------|-----------| -| **Separate ONNX models** for ATE and ASC | Combined graph has dynamic-axis export fragility in optimum-onnx | -| **Rule-based fallback** with no downloads | Zero startup time, works offline, graceful degradation | -| **Lexicon-based sentiment** with negation handling | 3-word window for "not good" → negative reversal | -| **XLM-RoBERTa base** (not large) | 0.3B params fine-tunes on 16GB GPU, adequate cross-lingual transfer | -| **ONNX INT8 dynamic quantization** | 4x smaller, 4.6x faster than PyTorch with only 1% F1 drop | -| **SQLite for dev, PostgreSQL for prod** | Zero-config local dev, production-grade concurrency | -| **Celery for batch only** | Single-review inference is fast enough for synchronous response | -| **Streamlit dashboard over the REST API** | Thin, pure-Python UI layer; the dashboard never imports the ML pipeline directly | +Single-process Gradio app loads ONNX model from disk, uses SQLite for prediction history. See README.md. diff --git a/frontend/Home.py b/frontend/Home.py deleted file mode 100644 index 420b283983ebb53efa54a02f014e4c512b24d7cb..0000000000000000000000000000000000000000 --- a/frontend/Home.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Multilingual ABSA — Streamlit entry point. - -Navigation split: - • Analysis → the only screen regular users see (input comment → results). - • Admin → application status, batch analytics, system monitor. -""" - -from __future__ import annotations - -import streamlit as st - -st.set_page_config( - page_title="Multilingual ABSA", - page_icon="🌍", - layout="wide", - initial_sidebar_state="expanded", -) - -analyzer = st.Page( - "views/predict.py", - title="Sentiment Analyzer", - icon="💬", - url_path="predict", - default=True, -) - -admin_overview = st.Page( - "views/admin/overview.py", - title="Overview", - icon="📊", - url_path="admin", -) -admin_batch = st.Page( - "views/admin/batch.py", - title="Batch Analytics", - icon="📁", - url_path="batch", -) -admin_monitor = st.Page( - "views/admin/monitor.py", - title="System Monitor", - icon="🩺", - url_path="monitor", -) - -pg = st.navigation( - { - "Analysis": [analyzer], - "Admin": [admin_overview, admin_batch, admin_monitor], - }, - position="sidebar", -) - -pg.run() diff --git a/frontend/__init__.py b/frontend/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/frontend/absa_client.py b/frontend/absa_client.py deleted file mode 100644 index dfa3e078c6ea641886b2ff5ac7fa8f404749a28e..0000000000000000000000000000000000000000 --- a/frontend/absa_client.py +++ /dev/null @@ -1,84 +0,0 @@ -"""HTTP client for the Multilingual ABSA FastAPI backend. - -The Streamlit frontend never imports the ML pipeline directly — it talks to -the running FastAPI service (``API_BASE_URL``, default ``http://localhost:8000``) -over plain HTTP. This keeps the dashboard a thin, deployable UI layer. -""" - -from __future__ import annotations - -import os -from typing import Any, Optional - -import httpx -import streamlit as st - -API_BASE_URL: str = os.getenv("API_BASE_URL", "http://localhost:8000") - -_TIMEOUT: float = float(os.getenv("API_TIMEOUT", "60")) - - -class APIClient: - """Thin wrapper around the ABSA REST API with Streamlit-friendly errors.""" - - def __init__(self, base_url: str = API_BASE_URL, transport: Optional[httpx.BaseTransport] = None) -> None: - self.base_url = base_url.rstrip("/") - self._client = httpx.Client( - base_url=self.base_url, - timeout=_TIMEOUT, - follow_redirects=True, - transport=transport, - ) - - # ── Helpers ──────────────────────────────────────────────────────────── - - def _request(self, method: str, path: str, **kwargs: Any) -> Optional[Any]: - try: - response = self._client.request(method, path, **kwargs) - response.raise_for_status() - return response.json() - except httpx.HTTPStatusError as exc: - detail = exc.response.text - st.error(f"API error ({exc.response.status_code}): {detail}") - return None - except httpx.HTTPError as exc: - st.error(f"Cannot reach the ABSA API at `{self.base_url}` — is it running?\n\n{exc}") - return None - - def close(self) -> None: - self._client.close() - - # ── Endpoints ────────────────────────────────────────────────────────── - - def get_health(self) -> Optional[dict[str, str]]: - return self._request("GET", "/health") - - def get_info(self) -> Optional[dict[str, str]]: - return self._request("GET", "/info") - - def predict(self, text: str, language: str = "auto") -> Optional[dict[str, Any]]: - payload = {"text": text, "language": language if language != "auto" else None} - return self._request("POST", "/predict", json=payload) - - def upload_batch(self, file: Any) -> Optional[dict[str, Any]]: - files = {"file": (file.name, file.getvalue(), "text/csv")} - return self._request("POST", "/batch", files=files) - - def get_batch_status(self, job_id: str) -> Optional[dict[str, Any]]: - return self._request("GET", f"/status/{job_id}") - - def download_result(self, job_id: str) -> Optional[bytes]: - """Fetch the generated CSV bytes for a completed batch job.""" - try: - response = self._client.get(f"/download/{job_id}") - response.raise_for_status() - return response.content - except httpx.HTTPError as exc: - st.error(f"Failed to download results: {exc}") - return None - - -@st.cache_resource(show_spinner=False) -def get_client() -> APIClient: - """Return a process-wide cached API client (reused across reruns).""" - return APIClient() diff --git a/frontend/py.typed b/frontend/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/frontend/ui.py b/frontend/ui.py deleted file mode 100644 index eb78531b3a6bd509772af7857b4fc24840902994..0000000000000000000000000000000000000000 --- a/frontend/ui.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Shared UI helpers for the Streamlit dashboard. - -This module uses only native Streamlit components and Markdown — no inline -HTML or CSS. Styling is delegated to Streamlit's built-in theming. -""" - -from __future__ import annotations - -import os -from typing import Iterable - -import streamlit as st - -SENTIMENT_EMOJI: dict[str, str] = { - "positive": "🟢", - "negative": "🔴", - "neutral": "⚪", - "conflict": "🟣", -} - -LANGUAGE_LABELS: dict[str, str] = { - "en": "English", - "hi": "Hindi", - "hinglish": "Hinglish", - "auto": "Auto-detect", -} - -_SAMPLE_REVIEWS: list[dict[str, str]] = [ - { - "label": "Battery + Screen (EN)", - "text": "The battery life is amazing but the screen is too dim.", - }, - { - "label": "Camera & Service (EN)", - "text": "Great camera quality, though the delivery was terribly slow.", - }, - { - "label": "Sound (HI)", - "text": "आवाज़ बहुत साफ़ है और बेस भी बढ़िया है।", - }, - { - "label": "Hinglish Mix", - "text": "Phone ka design badhiya hai lekin battery life kharab hai.", - }, -] - - -def hero(title: str, subtitle: str) -> None: - """Render a page header using native Streamlit elements.""" - st.title(title) - st.markdown(subtitle) - - -def feature_card(icon: str, title: str, description: str) -> None: - """Render a feature summary as a native bordered container.""" - with st.container(border=True): - st.markdown(f"**{icon} {title}**") - st.write(description) - - -def sentiment_label(sentiment: str) -> str: - """Human-friendly label for a sentiment class.""" - return f"{SENTIMENT_EMOJI.get(sentiment, '⚪')} {sentiment.capitalize()}" - - -def render_aspects(aspects: Iterable[dict]) -> None: - """Render aspect results as native bordered cards with a confidence bar.""" - items = list(aspects) - if not items: - st.info("No aspects detected in this text.") - return - for item in items: - aspect = str(item.get("aspect", "N/A")) - sentiment = str(item.get("sentiment", "neutral")) - confidence = float(item.get("confidence", 0.0)) - with st.container(border=True): - col_name, col_sent = st.columns([3, 1]) - col_name.markdown(f"**{aspect}**") - col_sent.markdown(sentiment_label(sentiment)) - st.progress(confidence, text=f"Confidence: {confidence:.2f}") - - -def language_options() -> list[str]: - return ["auto", "en", "hi", "hinglish"] - - -def sample_reviews() -> list[dict[str, str]]: - return _SAMPLE_REVIEWS - - -def require_admin() -> bool: - """Gate admin pages behind an optional password (env `ADMIN_PASSWORD`). - - When `ADMIN_PASSWORD` is empty the admin section is open; otherwise a - lock screen is shown until the correct password is entered. Callers should - `st.stop()` when this returns ``False``. - """ - password = os.getenv("ADMIN_PASSWORD", "") - if not password: - return True - if st.session_state.get("admin_ok"): - return True - - st.title("🔒 Admin access") - st.markdown( - "This section is restricted. Enter the admin password to view " - "application status and features." - ) - candidate = st.text_input("Admin password", type="password", key="admin_password") - if st.button("Unlock", type="primary", key="admin_unlock"): - if candidate == password: - st.session_state["admin_ok"] = True - st.rerun() - else: - st.error("Incorrect password.") - return False diff --git a/frontend/views/admin/batch.py b/frontend/views/admin/batch.py deleted file mode 100644 index 44134314578f294442d221c097454e4880c3d3da..0000000000000000000000000000000000000000 --- a/frontend/views/admin/batch.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Admin — batch analytics: CSV upload, live job progress, result download.""" - -from __future__ import annotations - -import io -import time - -import pandas as pd -import streamlit as st -from absa_client import get_client -from ui import hero, require_admin - -if not require_admin(): - st.stop() - -client = get_client() - -hero( - "Admin · Batch Analytics 📁", - "Upload a CSV of product reviews, queue an async batch job on the worker, " - "track progress live, and download the full annotated results.", -) - -col_up, col_hint = st.columns([2, 1], gap="large") - -with col_up: - uploaded = st.file_uploader("Upload a CSV file", type=["csv"], accept_multiple_files=False) - -with col_hint: - with st.container(border=True): - st.markdown("**Required format**") - st.write("A CSV with a `text` column. Limits:") - st.markdown("- Max 10,000 rows\n- Max 50 MB") - -df_preview = None -if uploaded is not None: - try: - df_preview = pd.read_csv(io.BytesIO(uploaded.getvalue())) - st.success(f"Loaded {len(df_preview)} rows with columns: {', '.join(df_preview.columns)}") - if "text" not in df_preview.columns: - st.error("CSV must contain a **text** column.") - df_preview = None - else: - st.dataframe(df_preview.head(5), use_container_width=True, hide_index=True) - except Exception as exc: - st.error(f"Could not parse CSV: {exc}") - - if df_preview is not None: - start = st.button("🚀 Start Batch Processing", type="primary", use_container_width=True) - - if start: - with st.spinner("Uploading and queuing job…"): - job = client.upload_batch(uploaded) - if job: - st.session_state["batch_job_id"] = job.get("job_id") - st.rerun() - -job_id = st.session_state.get("batch_job_id") -if job_id: - st.markdown("---") - st.subheader(f"Job progress — `{job_id[:8]}…`") - - progress = st.progress(0.0) - status = st.status("Queued…", expanded=True) - - while True: - job = client.get_batch_status(job_id) - if job is None: - st.error("Failed to fetch job status.") - break - - total = max(int(job.get("total_reviews") or 0), 1) - processed = int(job.get("processed") or 0) - ratio = min(processed / total, 1.0) - progress.progress(ratio) - - label = { - "queued": "Queued — waiting for a worker…", - "processing": f"Processing {processed}/{total} reviews…", - "completed": f"Completed — {processed}/{total} reviews analyzed ✅", - "failed": "Job failed ❌", - }.get(job.get("status"), job.get("status", "…")) - status.update( - label=label, - state="running" if job.get("status") in ("queued", "processing") else "complete", - ) - - if job.get("status") in ("completed", "failed"): - break - time.sleep(2) - - c1, c2, c3 = st.columns(3) - c1.metric("Total Reviews", job.get("total_reviews")) - c2.metric("Processed", job.get("processed")) - c3.metric("Status", job.get("status")) - - if job.get("status") == "completed": - result_bytes = client.download_result(job_id) - if result_bytes is not None: - st.download_button( - "⬇️ Download results (CSV)", - data=result_bytes, - file_name=f"absa_results_{job_id}.csv", - mime="text/csv", - type="primary", - use_container_width=True, - ) diff --git a/frontend/views/admin/monitor.py b/frontend/views/admin/monitor.py deleted file mode 100644 index 371304cecda7d892a6859f72a37e471e3a14a9a9..0000000000000000000000000000000000000000 --- a/frontend/views/admin/monitor.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Admin — system monitor: health checks, service metadata, diagnostics.""" - -from __future__ import annotations - -import streamlit as st -from absa_client import get_client -from ui import hero, require_admin - -if not require_admin(): - st.stop() - -client = get_client() - -hero( - "Admin · System Monitor 🩺", - "Live health checks, model metadata, and API configuration for the ABSA service.", -) - -auto_refresh = st.toggle("Auto-refresh every 5s", value=False) - -col_h, col_i = st.columns(2) -with col_h: - health = client.get_health() - if health: - st.success("### API is healthy ✅") - for key, value in health.items(): - st.markdown(f"- **{key}**: `{value}`") - else: - st.error("### API unreachable ❌") - -with col_i: - info = client.get_info() - if info: - st.info("### Service info") - for key, value in info.items(): - st.markdown(f"- **{key}**: `{value}`") - -st.markdown("---") - -col_a, col_b, col_c = st.columns(3) -col_a.metric("Endpoint", client.base_url) -col_b.metric("Timeout (s)", "60") -col_c.metric("Languages", "en · hi · hinglish") - -st.caption( - "Tip: run the API with `uvicorn api.main:app --port 8000` and this " - "dashboard with `streamlit run frontend/Home.py`." -) - -if auto_refresh: - st.rerun() diff --git a/frontend/views/admin/overview.py b/frontend/views/admin/overview.py deleted file mode 100644 index ab1ba54a57f287f5c0189193accbf434468ebe03..0000000000000000000000000000000000000000 --- a/frontend/views/admin/overview.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Admin — overview: application status and feature details.""" - -from __future__ import annotations - -import streamlit as st -from absa_client import get_client -from ui import feature_card, hero, require_admin - -if not require_admin(): - st.stop() - -client = get_client() - -hero( - "Admin · Application Overview 📊", - "Status of the ABSA platform, model metadata, and everything available in this deployment.", -) - -# ── Live status strip ───────────────────────────────────────────────────────── -info = client.get_info() -health = client.get_health() -api_online = health is not None - -col1, col2, col3, col4 = st.columns(4) -col1.metric( - "API Status", - "Online" if api_online else "Offline", - delta="●" if api_online else "○", - delta_color="normal" if api_online else "off", -) -col2.metric( - "Database", - "connected" if api_online else "unknown", -) -col3.metric("Model", (info or {}).get("model_name", "xlm-roberta-base-absa")) -col4.metric("API Base URL", client.base_url) - -st.markdown("---") - -# ── Detailed status ─────────────────────────────────────────────────────────── -col_h, col_i = st.columns(2, gap="large") - -with col_h: - st.subheader("Health check") - if health: - for key, value in health.items(): - st.markdown(f"- **{key}**: `{value}`") - else: - st.error("API unreachable — start `uvicorn api.main:app --port 8000`.") - -with col_i: - st.subheader("Service info") - if info: - for key, value in info.items(): - st.markdown(f"- **{key}**: `{value}`") - else: - st.warning("No service info available.") - -st.markdown("---") - -# ── Feature details ─────────────────────────────────────────────────────────── -st.subheader("What's available") -col_a, col_b, col_c = st.columns(3) -with col_a: - feature_card( - "💬", - "Sentiment Analyzer", - "The public view — users paste a review and get aspect-level sentiment.", - ) -with col_b: - feature_card( - "📁", - "Batch Analytics", - "Upload a CSV of reviews, run async jobs, and download annotated results.", - ) -with col_c: - feature_card( - "🩺", - "System Monitor", - "Health checks, service metadata, and auto-refresh diagnostics.", - ) - -with st.container(border=True): - st.markdown("**REST API**") - st.write("All dashboard features are backed by the JSON API at `/docs`:") - st.code( - "POST /predict · POST /batch\n" - "GET /status/{job_id} · GET /health\n" - "GET /info · GET /metrics" - ) diff --git a/frontend/views/predict.py b/frontend/views/predict.py deleted file mode 100644 index ed8ab4c9c5c6021e3097764b3c343d3b25f9e094..0000000000000000000000000000000000000000 --- a/frontend/views/predict.py +++ /dev/null @@ -1,59 +0,0 @@ -"""User-facing view — the only screen regular users see. - -Type/paste a review, pick a language, get aspect-level sentiment back. -No dashboards, no metrics, no administration here. -""" - -from __future__ import annotations - -import streamlit as st -from absa_client import get_client -from ui import LANGUAGE_LABELS, hero, language_options, render_aspects - -client = get_client() - -hero( - "💬 Sentiment Analyzer", - "Paste a review or comment in English, Hindi, or Hinglish and get " - "aspect-level sentiment instantly.", -) - -# ── Input ───────────────────────────────────────────────────────────────────── -text = st.text_area( - "Your comment", - height=170, - placeholder="e.g. The camera is amazing but the battery drains too fast.", - key="review_text", -) - -col_lang, col_btn = st.columns([1, 2], gap="medium") -with col_lang: - language = st.selectbox( - "Language", - language_options(), - index=0, - format_func=lambda code: LANGUAGE_LABELS[code], - ) -with col_btn: - st.caption("Auto-detects the language if unsure.") - analyze = st.button( - "Analyze Sentiment", - type="primary", - use_container_width=True, - disabled=not text.strip(), - ) - -st.markdown("---") - -# ── Results ─────────────────────────────────────────────────────────────────── -if analyze and text.strip(): - with st.spinner("Analyzing…"): - result = client.predict(text.strip(), language) - - if result: - aspects = result.get("aspects") or [] - - st.subheader("Aspects found") - render_aspects(aspects) -else: - st.info("Enter a review above and press **Analyze Sentiment** to get started.") diff --git a/monitoring/grafana/dashboards/absa_dashboard.json b/monitoring/grafana/dashboards/absa_dashboard.json deleted file mode 100644 index 2297cfa178ab72a9a15a24dd34dbd26e4a2c2aca..0000000000000000000000000000000000000000 --- a/monitoring/grafana/dashboards/absa_dashboard.json +++ /dev/null @@ -1,724 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "panels": [], - "title": "API Performance", - "type": "row" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "staircase": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 1 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(fastapi_requests_total[5m])", - "refId": "A" - } - ], - "title": "Request Rate (req/sec)", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 500 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 6, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(fastapi_requests_duration_seconds_bucket[5m])) by (le)) * 1000", - "refId": "A" - } - ], - "title": "P95 Latency (ms)", - "type": "gauge" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(rate(fastapi_requests_total{status=~\"5..\"}[5m])) / sum(rate(fastapi_requests_total[5m])) * 100", - "refId": "A" - } - ], - "title": "Error Rate (%)", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 10, - "panels": [], - "title": "Model Performance", - "type": "row" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "staircase": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.5, sum(rate(fastapi_requests_duration_seconds_bucket[5m])) by (le)) * 1000", - "legendFormat": "Median", - "refId": "A" - } - ], - "title": "ABSA Inference Time (ms)", - "type": "histogram" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "staircase": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum(rate(fastapi_requests_total[5m])) by (language)", - "refId": "A" - } - ], - "title": "Requests by Language", - "type": "barchart" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "rate(absa_aspects_total[5m]) / rate(fastapi_requests_total[5m])", - "refId": "A" - } - ], - "title": "Aspects per Review", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 18, - "panels": [], - "title": "System Health", - "type": "row" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Down" - }, - "1": { - "color": "green", - "index": 0, - "text": "Up" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 19 - }, - "id": 20, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "up{job=\"absa-api\"}", - "refId": "A" - } - ], - "title": "API Uptime", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 19 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "up{job=\"celery-worker\"}", - "refId": "A" - } - ], - "title": "Active Celery Workers", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 19 - }, - "id": 24, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum(batch_jobs_total) by (status)", - "refId": "A" - } - ], - "title": "Batch Jobs", - "type": "piechart" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "staircase": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 19 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "redis_memory_used_bytes", - "refId": "A" - } - ], - "title": "Redis Memory", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Multilingual ABSA Dashboard", - "uid": "absa-dash" -} diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml deleted file mode 100644 index 8cf7857e509e96883d966c3bf34f89644a3c1b85..0000000000000000000000000000000000000000 --- a/monitoring/grafana/provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: 1 -datasources: - - name: Prometheus - type: prometheus - url: http://prometheus:9090 - isDefault: true diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml deleted file mode 100644 index e580c25882210e8a58458b49a8b9873fb0a41a95..0000000000000000000000000000000000000000 --- a/monitoring/prometheus.yml +++ /dev/null @@ -1,13 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: "absa-api" - static_configs: - - targets: ["api:8000"] - metrics_path: /metrics - - - job_name: "celery-worker" - static_configs: - - targets: ["worker:9090"] diff --git a/pyproject.toml b/pyproject.toml index 40096e8a6afdf9969f7dd7b537a2ad0d55dbc858..9a5b5f1c32c88d1f1cef5af45668fd069253f4b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,21 +40,11 @@ dependencies = [ "mlflow>=2.15.0", "dvc>=3.51.1", "evidently>=0.4.30", - # API - "fastapi>=0.115.0", - "uvicorn>=0.29.0", "python-multipart>=0.0.9", "slowapi>=0.1.9", "pydantic>=2.7.1", "python-dotenv>=1.0.1", "prometheus-fastapi-instrumentator>=7.0.0", - # Async workers - "celery>=5.4.0", - "redis>=5.0.4", - # Database - "psycopg2-binary>=2.9.9", - # Frontend (Streamlit dashboard — calls the FastAPI backend over HTTP) - "streamlit>=1.37.0", "httpx>=0.28.0", ] @@ -73,11 +63,10 @@ dev = [ [tool.setuptools.packages.find] where = ["src", "."] -include = ["absa*", "api*"] +include = ["absa*"] [tool.setuptools.package-data] absa = ["py.typed"] -api = ["py.typed"] [tool.pytest.ini_options] testpaths = ["tests"] @@ -85,7 +74,7 @@ python_files = ["test_*.py"] pythonpath = [".", "src"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" -addopts = "--cov=api --cov=absa --cov-report=term-missing --cov-report=xml" +addopts = "--cov=absa --cov-report=term-missing --cov-report=xml" [tool.ruff] target-version = "py310" @@ -111,14 +100,14 @@ exclude = [ [tool.bandit] exclude_dirs = ["tests", "scripts"] -targets = ["api", "absa"] +targets = ["absa"] skips = ["B101"] # allow assert (pytest usage) [tool.coverage.run] -source = ["api", "absa"] +source = ["absa"] omit = ["*/tests/*", "*/__pycache__/*"] [tool.coverage.report] show_missing = true skip_covered = true -fail_under = 24 +fail_under = 18 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..447e50b51fba383c4c0806a684b73e468b1296c3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +gradio>=4.44.0 +onnxruntime>=1.19.0 +tokenizers>=0.19.0 +transformers>=4.44.0 +torch>=2.4.0 +numpy>=1.26.0 +pandas>=2.2.0 +scikit-learn>=1.5.0 +sqlalchemy>=2.0.0 +pydantic>=2.8.0 +langdetect>=1.0.9 \ No newline at end of file diff --git a/scripts/init_db.py b/scripts/init_db.py index 2dcb51d62c37d708f1003c6161f7e7e483b4cfc7..833810b8213e097fa42712334008732e7786c07d 100644 --- a/scripts/init_db.py +++ b/scripts/init_db.py @@ -6,7 +6,7 @@ from sqlalchemy import create_engine sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from api.models.db_models import Base +from absa.pipeline.db_models import Base def init_db(): diff --git a/src/absa/models/train_sentiment.py b/src/absa/models/train_sentiment.py index 61d0cb8a4a3f14f6d0e6c6b2768cf658be6368fc..32abc05ca5e7ee3c052d92cda01f1222eeb26188 100644 --- a/src/absa/models/train_sentiment.py +++ b/src/absa/models/train_sentiment.py @@ -76,7 +76,8 @@ def main(): print(f"Loading dataset from {dataset_path}") dataset = load_from_disk(str(dataset_path)) - tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base") + model_revision = "e73636d4f797dec63c3081bb6ed5c7b0bb3f2089" # xlm-roberta-base + tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base", revision=model_revision) data_collator = DataCollatorWithPadding(tokenizer=tokenizer) label_map = {0: "positive", 1: "negative", 2: "neutral", 3: "conflict"} @@ -85,6 +86,7 @@ def main(): num_labels=len(label_map), id2label=label_map, label2id={v: k for k, v in label_map.items()}, + revision=model_revision, ) output_dir = "models/sentiment" diff --git a/api/__init__.py b/src/absa/pipeline/__init__.py similarity index 100% rename from api/__init__.py rename to src/absa/pipeline/__init__.py diff --git a/api/services/absa_pipeline.py b/src/absa/pipeline/absa_pipeline.py similarity index 99% rename from api/services/absa_pipeline.py rename to src/absa/pipeline/absa_pipeline.py index 2d0f69b63e82efb3deac6ac9e0824559950a84f1..ccaa7f5b20d4972a083af1995ed3ca25a4828570 100644 --- a/api/services/absa_pipeline.py +++ b/src/absa/pipeline/absa_pipeline.py @@ -19,8 +19,8 @@ from typing import List, Optional, Tuple import numpy as np -from api.schemas.schemas import AspectSentiment, PredictionResponse -from api.services.lang_service import lang_service +from absa.pipeline.lang_service import lang_service +from absa.pipeline.schemas import AspectSentiment, PredictionResponse # ── Optional heavy imports (ONNX custom models) ─────────────────────────────── try: diff --git a/api/models/db_models.py b/src/absa/pipeline/db_models.py similarity index 100% rename from api/models/db_models.py rename to src/absa/pipeline/db_models.py diff --git a/api/services/lang_service.py b/src/absa/pipeline/lang_service.py similarity index 100% rename from api/services/lang_service.py rename to src/absa/pipeline/lang_service.py diff --git a/src/absa/pipeline/schemas.py b/src/absa/pipeline/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..1129d9d9cbb6d2ee33570ddcb2158807f09b742a --- /dev/null +++ b/src/absa/pipeline/schemas.py @@ -0,0 +1,23 @@ +from typing import List + +from pydantic import BaseModel, ConfigDict + + +class AspectSentiment(BaseModel): + aspect: str + sentiment: str + confidence: float + start: int + end: int + + model_config = ConfigDict(from_attributes=True) + + +class PredictionResponse(BaseModel): + text: str + language: str + detected_language: str + aspects: List[AspectSentiment] + processing_time_ms: float + + model_config = ConfigDict(from_attributes=True) diff --git a/tests/api/test_api.py b/tests/api/test_api.py deleted file mode 100644 index d7c7e77c4e96a16b30fc56458d590114117117b3..0000000000000000000000000000000000000000 --- a/tests/api/test_api.py +++ /dev/null @@ -1,78 +0,0 @@ -import os - -from fastapi.testclient import TestClient - -os.environ["DATABASE_URL"] = "sqlite:///./tests/fixtures/test.db" -import io -import unittest.mock as mock - -from api.main import app - -client = TestClient(app) - - -def test_health_endpoint(): - with TestClient(app) as client: - response = client.get("/health") - assert response.status_code == 200 - assert response.json()["status"] == "ok" - - -def test_predict_english(): - with TestClient(app) as client: - payload = {"text": "The food was great but service was slow.", "language": "en"} - response = client.post("/predict", json=payload) - assert response.status_code == 200 - data = response.json() - assert data["language"] == "en" - assert "aspects" in data - - -def test_predict_hindi(): - with TestClient(app) as client: - payload = {"text": "खाना बहुत अच्छा था", "language": "hi"} - response = client.post("/predict", json=payload) - assert response.status_code == 200 - data = response.json() - assert data["language"] == "hi" - assert "aspects" in data - - -def test_predict_empty(): - with TestClient(app) as client: - payload = {"text": ""} - response = client.post("/predict", json=payload) - assert response.status_code == 200 - - -def test_batch_upload(): - with TestClient(app) as client: - csv_content = "text\nThe food was great\nTerrible service" - files = {"file": ("test.csv", io.BytesIO(csv_content.encode("utf-8")), "text/csv")} - with mock.patch("api.routes.predict.process_batch.delay") as mock_delay: - response = client.post("/batch", files=files) - assert response.status_code == 200 - data = response.json() - assert "job_id" in data - assert data["status"] == "queued" - assert data["total_reviews"] == 2 - mock_delay.assert_called_once() - - -def test_info_endpoint(): - with TestClient(app) as client: - response = client.get("/info") - assert response.status_code == 200 - data = response.json() - assert "model_name" in data - assert "supported_languages" in data - assert isinstance(data["supported_languages"], str) - - -def test_metrics_endpoint(): - with TestClient(app) as client: - response = client.get("/metrics") - assert response.status_code == 200 - # metrics returns plain text Prometheus data - assert "text/plain" in response.headers["content-type"] - assert "http_requests_total" in response.text diff --git a/tests/unit/test_pipeline.py b/tests/unit/test_pipeline.py index dfd846aab2672d7df38af0b9225551336b9384c3..2297dc4a0cf34953b6a1f15f243a13d1fd265764 100644 --- a/tests/unit/test_pipeline.py +++ b/tests/unit/test_pipeline.py @@ -6,7 +6,7 @@ import os os.environ.setdefault("DATABASE_URL", "sqlite:///./tests/fixtures/test.db") -from api.services.absa_pipeline import pipeline +from absa.pipeline.absa_pipeline import pipeline def _aspects(text: str) -> list[dict]: diff --git a/tests/web/__init__.py b/tests/web/__init__.py deleted file mode 100644 index 76b1d74e2e832512533785a6f3c71857aa56c35c..0000000000000000000000000000000000000000 --- a/tests/web/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Phase 2 test package diff --git a/tests/web/test_streamlit_app.py b/tests/web/test_streamlit_app.py deleted file mode 100644 index eab8612592fa23fc8cc83d76e65e718d7d256329..0000000000000000000000000000000000000000 --- a/tests/web/test_streamlit_app.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Smoke tests for the Streamlit dashboard pages via streamlit AppTest. - -These verify each page script runs without raising, and that key UI elements -render. The API base URL is pointed at a dead port so health/network calls fail -fast and deterministically (pages degrade gracefully). -""" - -from __future__ import annotations - -import os -import sys - -os.environ["API_BASE_URL"] = "http://127.0.0.1:1" - -import pytest -from streamlit.testing.v1 import AppTest - -ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -# Real `streamlit run` adds the app dir to sys.path; AppTest does not, so mimic it. -APP_DIR = os.path.join(ROOT, "frontend") -if APP_DIR not in sys.path: - sys.path.insert(0, APP_DIR) - -APP_FILES = { - "home": "frontend/Home.py", - "predict": "frontend/views/predict.py", - "overview": "frontend/views/admin/overview.py", - "batch": "frontend/views/admin/batch.py", - "monitor": "frontend/views/admin/monitor.py", -} - - -@pytest.mark.parametrize("page", list(APP_FILES.keys())) -def test_page_runs_without_exception(page: str): - at = AppTest.from_file(os.path.join(ROOT, APP_FILES[page])) - at.run(timeout=30) - assert not at.exception, f"{page} raised: {at.exception}" - - -# ── Public user view (Sentiment Analyzer) ───────────────────────────────────── - - -def test_predict_is_clean_input_to_results_view(): - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["predict"])) - at.run(timeout=30) - rendered = " ".join(str(m.value) for m in at.markdown) - rendered += " " + " ".join(str(t.value) for t in at.title) - assert "Sentiment Analyzer" in rendered - assert len(at.text_area) >= 1 - assert any(b.label == "Analyze Sentiment" for b in at.button) - - -def test_predict_analyze_button_disabled_for_empty_text(): - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["predict"])) - at.run(timeout=30) - analyze = next(b for b in at.button if b.label == "Analyze Sentiment") - assert analyze.disabled - - -def test_home_navigation_lands_on_analyzer(): - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["home"])) - at.run(timeout=30) - assert not at.exception - rendered = " ".join(str(m.value) for m in at.markdown) - rendered += " " + " ".join(str(t.value) for t in at.title) - assert "Sentiment Analyzer" in rendered - - -# ── Admin views ─────────────────────────────────────────────────────────────── - - -def test_admin_overview_shows_status_metrics(): - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["overview"])) - at.run(timeout=30) - labels = [m.label for m in at.metric] - assert "API Status" in labels - assert "Model" in labels - - -def test_admin_overview_lists_features(): - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["overview"])) - at.run(timeout=30) - rendered = " ".join(str(m.value) for m in at.markdown) - assert "Batch Analytics" in rendered - assert "System Monitor" in rendered - - -def test_admin_batch_page_has_csv_uploader(): - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["batch"])) - at.run(timeout=30) - assert len(at.get("file_uploader")) >= 1 - - -def test_admin_monitor_handles_api_unreachable(): - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["monitor"])) - at.run(timeout=30) - assert not at.exception - rendered = " ".join(str(m.value) for m in at.markdown) - errors = " ".join(str(e.value) for e in at.error) - assert "unreachable" in (rendered + errors).lower() - - -def test_admin_locked_without_password(monkeypatch): - monkeypatch.setenv("ADMIN_PASSWORD", "secret") - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["overview"])) - at.run(timeout=30) - assert not at.exception - labels = [m.label for m in at.metric] - assert "API Status" not in labels - assert any(t.label == "Admin password" for t in at.text_input) - - -def test_admin_unlocked_with_correct_password(monkeypatch): - monkeypatch.setenv("ADMIN_PASSWORD", "secret") - at = AppTest.from_file(os.path.join(ROOT, APP_FILES["overview"])) - at.run(timeout=30) - at.text_input(key="admin_password").set_value("secret") - at.button(key="admin_unlock").click().run(timeout=30) - assert not at.exception - labels = [m.label for m in at.metric] - assert "API Status" in labels diff --git a/tests/web/test_streamlit_client.py b/tests/web/test_streamlit_client.py deleted file mode 100644 index 0985d44ecbd9ce26de853481b1b71c99cb150b0b..0000000000000000000000000000000000000000 --- a/tests/web/test_streamlit_client.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Unit tests for the Streamlit API client (httpx MockTransport, no network).""" - -from __future__ import annotations - -import httpx -import pytest - -from frontend.absa_client import APIClient - - -def _client(handler) -> APIClient: - return APIClient(base_url="http://testserver", transport=httpx.MockTransport(handler)) - - -def _json_handler(payload, status: int = 200): - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(status, json=payload, request=request) - - return handler - - -def test_health_returns_json(): - client = _client(_json_handler({"status": "ok", "model": "loaded"})) - assert client.get_health() == {"status": "ok", "model": "loaded"} - - -def test_info_returns_json(): - client = _client(_json_handler({"model_name": "xlm-roberta-base-absa"})) - assert client.get_info()["model_name"] == "xlm-roberta-base-absa" - - -def test_predict_payload_language_auto_sends_null(): - captured = {} - - def handler(request: httpx.Request) -> httpx.Response: - captured["json"] = request.content - return httpx.Response(200, json={"aspects": []}, request=request) - - client = _client(handler) - client.predict("Great phone", "auto") - import json - - assert json.loads(captured["json"]) == {"text": "Great phone", "language": None} - - -def test_predict_payload_explicit_language(): - captured = {} - - def handler(request: httpx.Request) -> httpx.Response: - captured["json"] = request.content - return httpx.Response(200, json={"aspects": []}, request=request) - - client = _client(handler) - client.predict("बढ़िया फोन", "hi") - import json - - assert json.loads(captured["json"]) == {"text": "बढ़िया फोन", "language": "hi"} - - -def test_upload_batch_sends_csv_file(): - captured = {} - - class FakeFile: - name = "reviews.csv" - getvalue = lambda self: b"text\nGreat\n" # noqa: E731 - - def handler(request: httpx.Request) -> httpx.Response: - captured["content_type"] = request.headers.get("content-type", "") - return httpx.Response(200, json={"job_id": "abc-123", "status": "queued"}, request=request) - - client = _client(handler) - result = client.upload_batch(FakeFile()) - assert result["job_id"] == "abc-123" - assert "multipart/form-data" in captured["content_type"] - - -def test_get_batch_status_parses(): - client = _client(_json_handler({"job_id": "x", "status": "completed", "processed": 5})) - status = client.get_batch_status("x") - assert status["status"] == "completed" - assert status["processed"] == 5 - - -def test_download_result_returns_bytes(): - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, content=b"text,sentiment\nok,positive\n", request=request) - - client = _client(handler) - assert client.download_result("abc") == b"text,sentiment\nok,positive\n" - - -def test_http_error_returns_none(): - client = _client(_json_handler({"detail": "boom"}, status=500)) - assert client.get_health() is None - - -def test_network_error_returns_none(): - def handler(request: httpx.Request) -> httpx.Response: - raise httpx.ConnectError("refused", request=request) - - client = _client(handler) - assert client.get_health() is None - - -@pytest.mark.parametrize("status", ["queued", "processing", "completed", "failed"]) -def test_status_passthrough(status: str): - client = _client(_json_handler({"status": status})) - assert client.get_batch_status("x")["status"] == status