Spaces:
Sleeping
Sleeping
alexchilton Copilot commited on
Commit ·
6242ddb
1
Parent(s): b84cfce
Initial deployment: Sentiment & Topic Analysis Dashboard
Browse filesFull-stack app with FastAPI backend + React TypeScript frontend.
ML-powered multilingual sentiment analysis, topic clustering,
anomaly detection, and interactive visualizations.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This view is limited to 50 files because it contains too many changes. See raw diff
- Dockerfile +51 -0
- README.md +25 -1
- backend/Dockerfile +21 -0
- backend/app/__init__.py +0 -0
- backend/app/api/__init__.py +0 -0
- backend/app/api/analysis.py +290 -0
- backend/app/api/export.py +75 -0
- backend/app/api/health.py +42 -0
- backend/app/api/webhooks.py +107 -0
- backend/app/core/__init__.py +0 -0
- backend/app/core/config.py +103 -0
- backend/app/core/logging.py +81 -0
- backend/app/core/middleware.py +39 -0
- backend/app/core/security.py +63 -0
- backend/app/core/telemetry.py +46 -0
- backend/app/main.py +103 -0
- backend/app/models/__init__.py +0 -0
- backend/app/models/schemas.py +240 -0
- backend/app/services/__init__.py +0 -0
- backend/app/services/analysis_pipeline.py +338 -0
- backend/app/services/anomaly_detection.py +133 -0
- backend/app/services/data_quality.py +62 -0
- backend/app/services/export.py +131 -0
- backend/app/services/file_processing.py +184 -0
- backend/app/services/language_detection.py +58 -0
- backend/app/services/notifications.py +92 -0
- backend/app/services/redis_client.py +83 -0
- backend/app/services/sentiment.py +180 -0
- backend/app/services/topic_clustering.py +220 -0
- backend/app/utils/__init__.py +0 -0
- backend/pyproject.toml +12 -0
- backend/requirements.txt +54 -0
- backend/tests/__init__.py +0 -0
- backend/tests/conftest.py +73 -0
- backend/tests/test_api.py +118 -0
- backend/tests/test_services.py +276 -0
- demo_data/demo_feedback.csv +501 -0
- demo_data/demo_feedback.json +0 -0
- demo_data/feedback_feb2024.csv +51 -0
- demo_data/feedback_jan2024.csv +51 -0
- demo_data/feedback_mar2024.csv +51 -0
- frontend/Dockerfile +19 -0
- frontend/eslint.config.js +26 -0
- frontend/index.html +13 -0
- frontend/nginx.conf +18 -0
- frontend/package-lock.json +0 -0
- frontend/package.json +51 -0
- frontend/src/App.tsx +33 -0
- frontend/src/__mocks__/handlers.ts +185 -0
- frontend/src/__tests__/components.test.tsx +115 -0
Dockerfile
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM node:20-alpine AS frontend-build
|
| 2 |
+
|
| 3 |
+
WORKDIR /frontend
|
| 4 |
+
COPY frontend/package.json frontend/package-lock.json* ./
|
| 5 |
+
RUN npm ci
|
| 6 |
+
COPY frontend/ .
|
| 7 |
+
RUN npm run build
|
| 8 |
+
|
| 9 |
+
FROM python:3.11-slim
|
| 10 |
+
|
| 11 |
+
WORKDIR /app
|
| 12 |
+
|
| 13 |
+
# Create non-root user (required by HF Spaces)
|
| 14 |
+
RUN useradd -m -u 1000 appuser
|
| 15 |
+
|
| 16 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 17 |
+
build-essential \
|
| 18 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 19 |
+
|
| 20 |
+
COPY backend/requirements.txt .
|
| 21 |
+
|
| 22 |
+
# Install CPU-only torch first (much smaller), skip problematic deps
|
| 23 |
+
RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu && \
|
| 24 |
+
pip install --no-cache-dir \
|
| 25 |
+
$(grep -v '^#' requirements.txt | grep -v '^$' | grep -v '^torch==' | grep -v 'pycld3' | grep -v 'weasyprint' | tr '\n' ' ') && \
|
| 26 |
+
pip install --no-cache-dir reportlab
|
| 27 |
+
|
| 28 |
+
COPY backend/ .
|
| 29 |
+
|
| 30 |
+
# Copy frontend build into backend static dir
|
| 31 |
+
COPY --from=frontend-build /frontend/dist /app/static
|
| 32 |
+
|
| 33 |
+
# Copy demo data
|
| 34 |
+
COPY demo_data/ /app/demo_data/
|
| 35 |
+
|
| 36 |
+
RUN mkdir -p uploads model_cache data && \
|
| 37 |
+
chown -R appuser:appuser /app
|
| 38 |
+
|
| 39 |
+
ENV PORT=7860
|
| 40 |
+
ENV APP_ENV=production
|
| 41 |
+
ENV LOG_LEVEL=INFO
|
| 42 |
+
ENV TRANSFORMERS_CACHE=/app/model_cache
|
| 43 |
+
ENV SENTENCE_TRANSFORMERS_HOME=/app/model_cache
|
| 44 |
+
ENV HF_HOME=/app/model_cache
|
| 45 |
+
|
| 46 |
+
USER appuser
|
| 47 |
+
|
| 48 |
+
EXPOSE 7860
|
| 49 |
+
|
| 50 |
+
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1"]
|
| 51 |
+
|
README.md
CHANGED
|
@@ -5,6 +5,30 @@ colorFrom: green
|
|
| 5 |
colorTo: blue
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
colorTo: blue
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
| 8 |
+
app_port: 7860
|
| 9 |
---
|
| 10 |
|
| 11 |
+
# 📊 Sentiment & Topic Analysis Dashboard
|
| 12 |
+
|
| 13 |
+
Upload CSV, JSON, or Excel files containing customer feedback, support tickets, or reviews — get instant multilingual sentiment analysis, topic clustering, anomaly detection, and interactive visualizations.
|
| 14 |
+
|
| 15 |
+
## Features
|
| 16 |
+
- **Multilingual sentiment analysis** using `cardiffnlp/twitter-xlm-roberta-base-sentiment`
|
| 17 |
+
- **Dynamic topic clustering** with BERTopic (HDBSCAN + UMAP)
|
| 18 |
+
- **Interactive force-directed** topic cluster graph
|
| 19 |
+
- **Sentiment trend charts** with confidence intervals
|
| 20 |
+
- **Data quality dashboard** flagging low-confidence predictions, mixed languages, duplicates
|
| 21 |
+
- **Comparison mode** to contrast time periods or segments
|
| 22 |
+
- **Export** to CSV, JSON, or PDF
|
| 23 |
+
- **Dark mode** support
|
| 24 |
+
|
| 25 |
+
## Usage
|
| 26 |
+
1. Upload a file with text data (CSV, JSON, Excel)
|
| 27 |
+
2. Wait for analysis to complete (~30s for 50 entries)
|
| 28 |
+
3. Explore the dashboard tabs: Overview, Data Quality, Compare
|
| 29 |
+
|
| 30 |
+
**API Key**: Use `dev-key-1` (pre-configured in the UI)
|
| 31 |
+
|
| 32 |
+
## Tech Stack
|
| 33 |
+
- **Backend**: FastAPI, PyTorch, Transformers, BERTopic
|
| 34 |
+
- **Frontend**: React, TypeScript, Recharts, D3.js
|
backend/Dockerfile
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
|
| 5 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 6 |
+
build-essential \
|
| 7 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 8 |
+
|
| 9 |
+
COPY requirements.txt .
|
| 10 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 11 |
+
|
| 12 |
+
COPY . .
|
| 13 |
+
|
| 14 |
+
RUN mkdir -p uploads model_cache data
|
| 15 |
+
|
| 16 |
+
EXPOSE 8000
|
| 17 |
+
|
| 18 |
+
HEALTHCHECK --interval=30s --timeout=10s --retries=3 \
|
| 19 |
+
CMD python -c "import httpx; r = httpx.get('http://localhost:8000/health/live'); r.raise_for_status()"
|
| 20 |
+
|
| 21 |
+
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"]
|
backend/app/__init__.py
ADDED
|
File without changes
|
backend/app/api/__init__.py
ADDED
|
File without changes
|
backend/app/api/analysis.py
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Upload and analysis API endpoints."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import uuid
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, Query, UploadFile
|
| 9 |
+
|
| 10 |
+
from app.core.config import settings
|
| 11 |
+
from app.core.logging import get_logger
|
| 12 |
+
from app.core.security import get_api_key
|
| 13 |
+
from app.models.schemas import (
|
| 14 |
+
AnalysisResult,
|
| 15 |
+
AnalysisStatus,
|
| 16 |
+
ComparisonRequest,
|
| 17 |
+
ComparisonResult,
|
| 18 |
+
FilterParams,
|
| 19 |
+
JobStatus,
|
| 20 |
+
TopicInfo,
|
| 21 |
+
)
|
| 22 |
+
from app.services.analysis_pipeline import (
|
| 23 |
+
filter_entries,
|
| 24 |
+
get_all_jobs,
|
| 25 |
+
get_job,
|
| 26 |
+
run_analysis,
|
| 27 |
+
)
|
| 28 |
+
from app.services.file_processing import parse_file
|
| 29 |
+
|
| 30 |
+
logger = get_logger(__name__)
|
| 31 |
+
router = APIRouter(prefix="/api/v1", tags=["analysis"])
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@router.post("/upload", response_model=JobStatus)
|
| 35 |
+
async def upload_file(
|
| 36 |
+
background_tasks: BackgroundTasks,
|
| 37 |
+
file: UploadFile = File(...),
|
| 38 |
+
source: Optional[str] = Query(None, description="Data source label"),
|
| 39 |
+
api_key: str = Depends(get_api_key),
|
| 40 |
+
):
|
| 41 |
+
"""Upload a file for analysis. Supports CSV, JSON, Excel, ZIP."""
|
| 42 |
+
if not file.filename:
|
| 43 |
+
raise HTTPException(status_code=400, detail="No filename provided")
|
| 44 |
+
|
| 45 |
+
content = await file.read()
|
| 46 |
+
size_mb = len(content) / (1024 * 1024)
|
| 47 |
+
|
| 48 |
+
if size_mb > settings.max_upload_size_mb:
|
| 49 |
+
raise HTTPException(
|
| 50 |
+
status_code=413,
|
| 51 |
+
detail=f"File too large ({size_mb:.1f}MB). Maximum: {settings.max_upload_size_mb}MB",
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
try:
|
| 55 |
+
entries = parse_file(content, file.filename, source)
|
| 56 |
+
except ValueError as exc:
|
| 57 |
+
raise HTTPException(status_code=400, detail=str(exc))
|
| 58 |
+
|
| 59 |
+
if not entries:
|
| 60 |
+
raise HTTPException(status_code=400, detail="No valid entries found in the uploaded file")
|
| 61 |
+
|
| 62 |
+
job_id = uuid.uuid4().hex[:12]
|
| 63 |
+
logger.info("upload_received", job_id=job_id, filename=file.filename, entries=len(entries), size_mb=round(size_mb, 2))
|
| 64 |
+
|
| 65 |
+
background_tasks.add_task(run_analysis, entries, job_id)
|
| 66 |
+
|
| 67 |
+
from datetime import datetime
|
| 68 |
+
|
| 69 |
+
return JobStatus(
|
| 70 |
+
job_id=job_id,
|
| 71 |
+
status=AnalysisStatus.PENDING,
|
| 72 |
+
progress=0.0,
|
| 73 |
+
message=f"Processing {len(entries)} entries from {file.filename}",
|
| 74 |
+
created_at=datetime.utcnow(),
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
@router.post("/upload/chunked", response_model=JobStatus)
|
| 79 |
+
async def upload_chunked(
|
| 80 |
+
background_tasks: BackgroundTasks,
|
| 81 |
+
file: UploadFile = File(...),
|
| 82 |
+
chunk_index: int = Query(0, ge=0),
|
| 83 |
+
total_chunks: int = Query(1, ge=1),
|
| 84 |
+
upload_id: Optional[str] = Query(None),
|
| 85 |
+
source: Optional[str] = Query(None),
|
| 86 |
+
api_key: str = Depends(get_api_key),
|
| 87 |
+
):
|
| 88 |
+
"""Chunked upload for files >10MB."""
|
| 89 |
+
from pathlib import Path
|
| 90 |
+
|
| 91 |
+
upload_id = upload_id or uuid.uuid4().hex[:12]
|
| 92 |
+
chunk_dir = settings.upload_path / f"chunks_{upload_id}"
|
| 93 |
+
chunk_dir.mkdir(parents=True, exist_ok=True)
|
| 94 |
+
|
| 95 |
+
content = await file.read()
|
| 96 |
+
chunk_path = chunk_dir / f"chunk_{chunk_index:04d}"
|
| 97 |
+
chunk_path.write_bytes(content)
|
| 98 |
+
|
| 99 |
+
logger.info("chunk_received", upload_id=upload_id, chunk=chunk_index, total=total_chunks)
|
| 100 |
+
|
| 101 |
+
if chunk_index + 1 < total_chunks:
|
| 102 |
+
from datetime import datetime
|
| 103 |
+
|
| 104 |
+
return JobStatus(
|
| 105 |
+
job_id=upload_id,
|
| 106 |
+
status=AnalysisStatus.PENDING,
|
| 107 |
+
progress=chunk_index / total_chunks,
|
| 108 |
+
message=f"Received chunk {chunk_index + 1}/{total_chunks}",
|
| 109 |
+
created_at=datetime.utcnow(),
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
# All chunks received — reassemble
|
| 113 |
+
chunks = sorted(chunk_dir.glob("chunk_*"))
|
| 114 |
+
combined = b"".join(c.read_bytes() for c in chunks)
|
| 115 |
+
|
| 116 |
+
# Clean up chunks
|
| 117 |
+
for c in chunks:
|
| 118 |
+
c.unlink()
|
| 119 |
+
chunk_dir.rmdir()
|
| 120 |
+
|
| 121 |
+
try:
|
| 122 |
+
filename = file.filename or "upload.csv"
|
| 123 |
+
entries = parse_file(combined, filename, source)
|
| 124 |
+
except ValueError as exc:
|
| 125 |
+
raise HTTPException(status_code=400, detail=str(exc))
|
| 126 |
+
|
| 127 |
+
if not entries:
|
| 128 |
+
raise HTTPException(status_code=400, detail="No valid entries found")
|
| 129 |
+
|
| 130 |
+
background_tasks.add_task(run_analysis, entries, upload_id)
|
| 131 |
+
|
| 132 |
+
from datetime import datetime
|
| 133 |
+
|
| 134 |
+
return JobStatus(
|
| 135 |
+
job_id=upload_id,
|
| 136 |
+
status=AnalysisStatus.PROCESSING,
|
| 137 |
+
progress=0.0,
|
| 138 |
+
message=f"All chunks received. Processing {len(entries)} entries.",
|
| 139 |
+
created_at=datetime.utcnow(),
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
@router.get("/jobs", response_model=list[JobStatus])
|
| 144 |
+
async def list_jobs(api_key: str = Depends(get_api_key)):
|
| 145 |
+
"""List all analysis jobs."""
|
| 146 |
+
jobs = get_all_jobs()
|
| 147 |
+
return [
|
| 148 |
+
JobStatus(
|
| 149 |
+
job_id=j.job_id,
|
| 150 |
+
status=j.status,
|
| 151 |
+
progress=1.0 if j.status == AnalysisStatus.COMPLETED else 0.5,
|
| 152 |
+
message="",
|
| 153 |
+
created_at=j.created_at,
|
| 154 |
+
completed_at=j.completed_at,
|
| 155 |
+
)
|
| 156 |
+
for j in jobs
|
| 157 |
+
]
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
@router.get("/jobs/{job_id}", response_model=AnalysisResult)
|
| 161 |
+
async def get_job_result(job_id: str, api_key: str = Depends(get_api_key)):
|
| 162 |
+
"""Get analysis results for a specific job."""
|
| 163 |
+
job = get_job(job_id)
|
| 164 |
+
if not job:
|
| 165 |
+
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
| 166 |
+
return job
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
@router.get("/jobs/{job_id}/status", response_model=JobStatus)
|
| 170 |
+
async def get_job_status(job_id: str, api_key: str = Depends(get_api_key)):
|
| 171 |
+
"""Get status of an analysis job."""
|
| 172 |
+
job = get_job(job_id)
|
| 173 |
+
if not job:
|
| 174 |
+
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
| 175 |
+
return JobStatus(
|
| 176 |
+
job_id=job.job_id,
|
| 177 |
+
status=job.status,
|
| 178 |
+
progress=1.0 if job.status == AnalysisStatus.COMPLETED else 0.5,
|
| 179 |
+
message="",
|
| 180 |
+
created_at=job.created_at,
|
| 181 |
+
completed_at=job.completed_at,
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
@router.post("/jobs/{job_id}/filter")
|
| 186 |
+
async def filter_job_results(
|
| 187 |
+
job_id: str,
|
| 188 |
+
filters: FilterParams,
|
| 189 |
+
api_key: str = Depends(get_api_key),
|
| 190 |
+
):
|
| 191 |
+
"""Filter analysis results."""
|
| 192 |
+
job = get_job(job_id)
|
| 193 |
+
if not job:
|
| 194 |
+
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
| 195 |
+
if job.status != AnalysisStatus.COMPLETED:
|
| 196 |
+
raise HTTPException(status_code=400, detail="Analysis not yet completed")
|
| 197 |
+
|
| 198 |
+
filtered = filter_entries(
|
| 199 |
+
job.entries,
|
| 200 |
+
date_from=filters.date_from,
|
| 201 |
+
date_to=filters.date_to,
|
| 202 |
+
sentiment_min=filters.sentiment_min,
|
| 203 |
+
sentiment_max=filters.sentiment_max,
|
| 204 |
+
topics=filters.topics,
|
| 205 |
+
languages=filters.languages,
|
| 206 |
+
sources=filters.sources,
|
| 207 |
+
search_text=filters.search_text,
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
# Paginate
|
| 211 |
+
start = (filters.page - 1) * filters.page_size
|
| 212 |
+
end = start + filters.page_size
|
| 213 |
+
|
| 214 |
+
return {
|
| 215 |
+
"total": len(filtered),
|
| 216 |
+
"page": filters.page,
|
| 217 |
+
"page_size": filters.page_size,
|
| 218 |
+
"entries": filtered[start:end],
|
| 219 |
+
}
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
@router.post("/jobs/{job_id}/compare", response_model=ComparisonResult)
|
| 223 |
+
async def compare_segments(
|
| 224 |
+
job_id: str,
|
| 225 |
+
comparison: ComparisonRequest,
|
| 226 |
+
api_key: str = Depends(get_api_key),
|
| 227 |
+
):
|
| 228 |
+
"""Compare two data segments from the same job."""
|
| 229 |
+
job = get_job(job_id)
|
| 230 |
+
if not job:
|
| 231 |
+
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
| 232 |
+
if job.status != AnalysisStatus.COMPLETED:
|
| 233 |
+
raise HTTPException(status_code=400, detail="Analysis not yet completed")
|
| 234 |
+
|
| 235 |
+
from collections import Counter
|
| 236 |
+
|
| 237 |
+
import numpy as np
|
| 238 |
+
|
| 239 |
+
from app.models.schemas import AnalysisSummary, SentimentLabel
|
| 240 |
+
|
| 241 |
+
seg_a_entries = filter_entries(
|
| 242 |
+
job.entries, **comparison.segment_a.model_dump(exclude={"page", "page_size"})
|
| 243 |
+
)
|
| 244 |
+
seg_b_entries = filter_entries(
|
| 245 |
+
job.entries, **comparison.segment_b.model_dump(exclude={"page", "page_size"})
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
def make_summary(entries):
|
| 249 |
+
if not entries:
|
| 250 |
+
return AnalysisSummary(
|
| 251 |
+
total_entries=0, avg_sentiment=0.5,
|
| 252 |
+
dominant_sentiment=SentimentLabel.NEUTRAL,
|
| 253 |
+
num_topics=0, top_topics=[], languages_detected=[],
|
| 254 |
+
)
|
| 255 |
+
sentiments = [e.sentiment for e in entries]
|
| 256 |
+
topic_counts = Counter(e.topic_id for e in entries)
|
| 257 |
+
return AnalysisSummary(
|
| 258 |
+
total_entries=len(entries),
|
| 259 |
+
avg_sentiment=round(float(np.mean([s.score for s in sentiments])), 4),
|
| 260 |
+
dominant_sentiment=SentimentLabel(
|
| 261 |
+
Counter(s.label.value for s in sentiments).most_common(1)[0][0]
|
| 262 |
+
),
|
| 263 |
+
num_topics=len(set(e.topic_id for e in entries) - {-1}),
|
| 264 |
+
top_topics=[
|
| 265 |
+
TopicInfo(topic_id=tid, label=f"Topic {tid}", keywords=[], size=cnt)
|
| 266 |
+
for tid, cnt in topic_counts.most_common(5) if tid != -1
|
| 267 |
+
],
|
| 268 |
+
languages_detected=list(set(e.language.language for e in entries)),
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
sum_a = make_summary(seg_a_entries)
|
| 272 |
+
sum_b = make_summary(seg_b_entries)
|
| 273 |
+
|
| 274 |
+
topics_a = set(e.topic_id for e in seg_a_entries) - {-1}
|
| 275 |
+
topics_b = set(e.topic_id for e in seg_b_entries) - {-1}
|
| 276 |
+
|
| 277 |
+
return ComparisonResult(
|
| 278 |
+
segment_a=sum_a,
|
| 279 |
+
segment_b=sum_b,
|
| 280 |
+
sentiment_delta=round(sum_b.avg_sentiment - sum_a.avg_sentiment, 4),
|
| 281 |
+
topic_changes=[],
|
| 282 |
+
new_topics=[
|
| 283 |
+
TopicInfo(topic_id=t, label=f"Topic {t}", keywords=[], size=0)
|
| 284 |
+
for t in topics_b - topics_a
|
| 285 |
+
],
|
| 286 |
+
disappeared_topics=[
|
| 287 |
+
TopicInfo(topic_id=t, label=f"Topic {t}", keywords=[], size=0)
|
| 288 |
+
for t in topics_a - topics_b
|
| 289 |
+
],
|
| 290 |
+
)
|
backend/app/api/export.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Export API endpoints."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from fastapi import APIRouter, Depends, HTTPException
|
| 6 |
+
from fastapi.responses import Response
|
| 7 |
+
|
| 8 |
+
from app.core.security import get_api_key
|
| 9 |
+
from app.models.schemas import AnalysisStatus, ExportFormat, FilterParams
|
| 10 |
+
from app.services.analysis_pipeline import filter_entries, get_job
|
| 11 |
+
from app.services.export import export_entries
|
| 12 |
+
|
| 13 |
+
router = APIRouter(prefix="/api/v1", tags=["export"])
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
CONTENT_TYPES = {
|
| 17 |
+
ExportFormat.CSV: "text/csv",
|
| 18 |
+
ExportFormat.JSON: "application/json",
|
| 19 |
+
ExportFormat.PDF: "application/pdf",
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
FILE_EXTENSIONS = {
|
| 23 |
+
ExportFormat.CSV: "csv",
|
| 24 |
+
ExportFormat.JSON: "json",
|
| 25 |
+
ExportFormat.PDF: "pdf",
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@router.post("/jobs/{job_id}/export")
|
| 30 |
+
async def export_results(
|
| 31 |
+
job_id: str,
|
| 32 |
+
fmt: ExportFormat = ExportFormat.CSV,
|
| 33 |
+
filters: FilterParams | None = None,
|
| 34 |
+
api_key: str = Depends(get_api_key),
|
| 35 |
+
):
|
| 36 |
+
"""Export filtered analysis results."""
|
| 37 |
+
job = get_job(job_id)
|
| 38 |
+
if not job:
|
| 39 |
+
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
| 40 |
+
if job.status != AnalysisStatus.COMPLETED:
|
| 41 |
+
raise HTTPException(status_code=400, detail="Analysis not yet completed")
|
| 42 |
+
|
| 43 |
+
entries = job.entries
|
| 44 |
+
if filters:
|
| 45 |
+
entries = filter_entries(
|
| 46 |
+
entries,
|
| 47 |
+
date_from=filters.date_from,
|
| 48 |
+
date_to=filters.date_to,
|
| 49 |
+
sentiment_min=filters.sentiment_min,
|
| 50 |
+
sentiment_max=filters.sentiment_max,
|
| 51 |
+
topics=filters.topics,
|
| 52 |
+
languages=filters.languages,
|
| 53 |
+
sources=filters.sources,
|
| 54 |
+
search_text=filters.search_text,
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
summary = None
|
| 58 |
+
if job.summary:
|
| 59 |
+
summary = {
|
| 60 |
+
"Total Entries": job.summary.total_entries,
|
| 61 |
+
"Average Sentiment": job.summary.avg_sentiment,
|
| 62 |
+
"Dominant Sentiment": job.summary.dominant_sentiment.value,
|
| 63 |
+
"Topics Found": job.summary.num_topics,
|
| 64 |
+
"Languages": ", ".join(job.summary.languages_detected),
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
content = export_entries(entries, fmt, summary)
|
| 68 |
+
|
| 69 |
+
return Response(
|
| 70 |
+
content=content,
|
| 71 |
+
media_type=CONTENT_TYPES[fmt],
|
| 72 |
+
headers={
|
| 73 |
+
"Content-Disposition": f"attachment; filename=analysis_{job_id}.{FILE_EXTENSIONS[fmt]}"
|
| 74 |
+
},
|
| 75 |
+
)
|
backend/app/api/health.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Health and system endpoints."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import time
|
| 6 |
+
|
| 7 |
+
from fastapi import APIRouter
|
| 8 |
+
|
| 9 |
+
from app.core.config import settings
|
| 10 |
+
from app.models.schemas import HealthResponse
|
| 11 |
+
from app.services.redis_client import check_redis_health
|
| 12 |
+
from app.services.sentiment import is_model_available
|
| 13 |
+
|
| 14 |
+
router = APIRouter(tags=["system"])
|
| 15 |
+
|
| 16 |
+
_start_time = time.time()
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@router.get("/health", response_model=HealthResponse)
|
| 20 |
+
async def health_check():
|
| 21 |
+
redis_ok = await check_redis_health()
|
| 22 |
+
return HealthResponse(
|
| 23 |
+
status="healthy" if redis_ok else "degraded",
|
| 24 |
+
version="1.0.0",
|
| 25 |
+
models_loaded=is_model_available(),
|
| 26 |
+
redis_connected=redis_ok,
|
| 27 |
+
uptime_seconds=round(time.time() - _start_time, 2),
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@router.get("/health/live")
|
| 32 |
+
async def liveness():
|
| 33 |
+
return {"status": "alive"}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@router.get("/health/ready")
|
| 37 |
+
async def readiness():
|
| 38 |
+
redis_ok = await check_redis_health()
|
| 39 |
+
if not redis_ok:
|
| 40 |
+
from fastapi import HTTPException
|
| 41 |
+
raise HTTPException(status_code=503, detail="Redis not available")
|
| 42 |
+
return {"status": "ready"}
|
backend/app/api/webhooks.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Webhook and SSE endpoints for real-time ingestion."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import asyncio
|
| 6 |
+
import json
|
| 7 |
+
from datetime import datetime
|
| 8 |
+
|
| 9 |
+
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
| 10 |
+
from sse_starlette.sse import EventSourceResponse
|
| 11 |
+
|
| 12 |
+
from app.core.logging import get_logger
|
| 13 |
+
from app.core.security import get_api_key, verify_webhook_signature
|
| 14 |
+
from app.models.schemas import AnalysisStatus, FeedbackEntry, JobStatus, WebhookPayload
|
| 15 |
+
from app.services.analysis_pipeline import run_analysis
|
| 16 |
+
from app.services.redis_client import subscribe_events
|
| 17 |
+
|
| 18 |
+
logger = get_logger(__name__)
|
| 19 |
+
router = APIRouter(prefix="/api/v1", tags=["realtime"])
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@router.post("/webhooks/ingest", response_model=JobStatus)
|
| 23 |
+
async def webhook_ingest(
|
| 24 |
+
request: Request,
|
| 25 |
+
background_tasks: BackgroundTasks,
|
| 26 |
+
):
|
| 27 |
+
"""Receive data via webhook with Stripe-style signature verification."""
|
| 28 |
+
body = await request.body()
|
| 29 |
+
signature = request.headers.get("X-Signature", "")
|
| 30 |
+
timestamp = request.headers.get("X-Timestamp", "")
|
| 31 |
+
|
| 32 |
+
if not verify_webhook_signature(body, signature, timestamp):
|
| 33 |
+
raise HTTPException(status_code=401, detail="Invalid webhook signature")
|
| 34 |
+
|
| 35 |
+
try:
|
| 36 |
+
payload = WebhookPayload.model_validate_json(body)
|
| 37 |
+
except Exception as exc:
|
| 38 |
+
raise HTTPException(status_code=400, detail=f"Invalid payload: {exc}")
|
| 39 |
+
|
| 40 |
+
if not payload.data:
|
| 41 |
+
raise HTTPException(status_code=400, detail="No data entries in payload")
|
| 42 |
+
|
| 43 |
+
import uuid
|
| 44 |
+
|
| 45 |
+
job_id = uuid.uuid4().hex[:12]
|
| 46 |
+
logger.info("webhook_received", job_id=job_id, event=payload.event_type, entries=len(payload.data))
|
| 47 |
+
|
| 48 |
+
entries = [
|
| 49 |
+
FeedbackEntry(
|
| 50 |
+
id=e.id,
|
| 51 |
+
text=e.text,
|
| 52 |
+
source=payload.source or e.source or "webhook",
|
| 53 |
+
timestamp=e.timestamp or datetime.utcnow(),
|
| 54 |
+
metadata=e.metadata,
|
| 55 |
+
)
|
| 56 |
+
for e in payload.data
|
| 57 |
+
]
|
| 58 |
+
|
| 59 |
+
background_tasks.add_task(run_analysis, entries, job_id)
|
| 60 |
+
|
| 61 |
+
return JobStatus(
|
| 62 |
+
job_id=job_id,
|
| 63 |
+
status=AnalysisStatus.PENDING,
|
| 64 |
+
progress=0.0,
|
| 65 |
+
message=f"Webhook: processing {len(entries)} entries",
|
| 66 |
+
created_at=datetime.utcnow(),
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@router.get("/events/analysis")
|
| 71 |
+
async def analysis_events(request: Request, api_key: str = Depends(get_api_key)):
|
| 72 |
+
"""Server-Sent Events stream for live analysis updates."""
|
| 73 |
+
|
| 74 |
+
async def event_generator():
|
| 75 |
+
try:
|
| 76 |
+
async for data in subscribe_events("analysis_updates"):
|
| 77 |
+
if await request.is_disconnected():
|
| 78 |
+
break
|
| 79 |
+
yield {
|
| 80 |
+
"event": "analysis_update",
|
| 81 |
+
"data": json.dumps(data),
|
| 82 |
+
}
|
| 83 |
+
except asyncio.CancelledError:
|
| 84 |
+
pass
|
| 85 |
+
except Exception as exc:
|
| 86 |
+
logger.error("sse_error", error=str(exc))
|
| 87 |
+
|
| 88 |
+
return EventSourceResponse(event_generator())
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
@router.get("/events/anomalies")
|
| 92 |
+
async def anomaly_events(request: Request, api_key: str = Depends(get_api_key)):
|
| 93 |
+
"""SSE stream for anomaly alerts."""
|
| 94 |
+
|
| 95 |
+
async def event_generator():
|
| 96 |
+
try:
|
| 97 |
+
async for data in subscribe_events("anomaly_alerts"):
|
| 98 |
+
if await request.is_disconnected():
|
| 99 |
+
break
|
| 100 |
+
yield {
|
| 101 |
+
"event": "anomaly_alert",
|
| 102 |
+
"data": json.dumps(data),
|
| 103 |
+
}
|
| 104 |
+
except asyncio.CancelledError:
|
| 105 |
+
pass
|
| 106 |
+
|
| 107 |
+
return EventSourceResponse(event_generator())
|
backend/app/core/__init__.py
ADDED
|
File without changes
|
backend/app/core/config.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Application configuration using pydantic-settings."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import List
|
| 7 |
+
|
| 8 |
+
from pydantic import field_validator
|
| 9 |
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class Settings(BaseSettings):
|
| 13 |
+
model_config = SettingsConfigDict(
|
| 14 |
+
env_file=".env",
|
| 15 |
+
env_file_encoding="utf-8",
|
| 16 |
+
case_sensitive=False,
|
| 17 |
+
extra="ignore",
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
# Application
|
| 21 |
+
app_name: str = "TopicAnalysis"
|
| 22 |
+
app_env: str = "development"
|
| 23 |
+
debug: bool = False
|
| 24 |
+
secret_key: str = "change-me-in-production"
|
| 25 |
+
api_key_header: str = "X-API-Key"
|
| 26 |
+
allowed_api_keys: List[str] = ["dev-key-1"]
|
| 27 |
+
|
| 28 |
+
# Server
|
| 29 |
+
backend_host: str = "0.0.0.0"
|
| 30 |
+
backend_port: int = 8000
|
| 31 |
+
frontend_url: str = "http://localhost:3000"
|
| 32 |
+
cors_origins: List[str] = ["http://localhost:3000", "http://localhost:8080"]
|
| 33 |
+
|
| 34 |
+
# Redis
|
| 35 |
+
redis_url: str = "redis://localhost:6379/0"
|
| 36 |
+
|
| 37 |
+
# File Upload
|
| 38 |
+
max_upload_size_mb: int = 500
|
| 39 |
+
chunk_size_mb: int = 10
|
| 40 |
+
upload_dir: str = "./uploads"
|
| 41 |
+
|
| 42 |
+
# ML Models
|
| 43 |
+
sentiment_model: str = "cardiffnlp/twitter-xlm-roberta-base-sentiment"
|
| 44 |
+
embedding_model: str = "paraphrase-multilingual-MiniLM-L12-v2"
|
| 45 |
+
model_cache_dir: str = "./model_cache"
|
| 46 |
+
model_load_timeout: int = 120
|
| 47 |
+
|
| 48 |
+
# Rate Limiting
|
| 49 |
+
rate_limit_per_minute: int = 60
|
| 50 |
+
rate_limit_burst: int = 10
|
| 51 |
+
|
| 52 |
+
# Anomaly Detection
|
| 53 |
+
anomaly_rolling_window: int = 50
|
| 54 |
+
anomaly_sentiment_threshold: float = 1.5
|
| 55 |
+
anomaly_topic_spike_threshold: float = 3.0
|
| 56 |
+
|
| 57 |
+
# Notifications
|
| 58 |
+
slack_webhook_url: str = ""
|
| 59 |
+
notification_email_from: str = ""
|
| 60 |
+
notification_email_to: str = ""
|
| 61 |
+
smtp_host: str = ""
|
| 62 |
+
smtp_port: int = 587
|
| 63 |
+
smtp_user: str = ""
|
| 64 |
+
smtp_password: str = ""
|
| 65 |
+
|
| 66 |
+
# Webhook
|
| 67 |
+
webhook_secret: str = "whsec_change-me"
|
| 68 |
+
|
| 69 |
+
# Observability
|
| 70 |
+
otel_exporter_otlp_endpoint: str = "http://localhost:4317"
|
| 71 |
+
otel_service_name: str = "topic-analysis"
|
| 72 |
+
log_level: str = "INFO"
|
| 73 |
+
log_format: str = "json"
|
| 74 |
+
|
| 75 |
+
# Database
|
| 76 |
+
database_url: str = "sqlite:///./data/analysis.db"
|
| 77 |
+
|
| 78 |
+
@field_validator("allowed_api_keys", mode="before")
|
| 79 |
+
@classmethod
|
| 80 |
+
def parse_api_keys(cls, v: str | list) -> list:
|
| 81 |
+
if isinstance(v, str):
|
| 82 |
+
return [k.strip() for k in v.split(",") if k.strip()]
|
| 83 |
+
return v
|
| 84 |
+
|
| 85 |
+
@field_validator("cors_origins", mode="before")
|
| 86 |
+
@classmethod
|
| 87 |
+
def parse_cors(cls, v: str | list) -> list:
|
| 88 |
+
if isinstance(v, str):
|
| 89 |
+
return [o.strip() for o in v.split(",") if o.strip()]
|
| 90 |
+
return v
|
| 91 |
+
|
| 92 |
+
@property
|
| 93 |
+
def upload_path(self) -> Path:
|
| 94 |
+
p = Path(self.upload_dir)
|
| 95 |
+
p.mkdir(parents=True, exist_ok=True)
|
| 96 |
+
return p
|
| 97 |
+
|
| 98 |
+
@property
|
| 99 |
+
def is_production(self) -> bool:
|
| 100 |
+
return self.app_env == "production"
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
settings = Settings()
|
backend/app/core/logging.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Structured logging with correlation IDs."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
import sys
|
| 7 |
+
import uuid
|
| 8 |
+
from contextvars import ContextVar
|
| 9 |
+
|
| 10 |
+
import structlog
|
| 11 |
+
|
| 12 |
+
correlation_id_var: ContextVar[str] = ContextVar("correlation_id", default="")
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def get_correlation_id() -> str:
|
| 16 |
+
cid = correlation_id_var.get()
|
| 17 |
+
if not cid:
|
| 18 |
+
cid = uuid.uuid4().hex[:16]
|
| 19 |
+
correlation_id_var.set(cid)
|
| 20 |
+
return cid
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def add_correlation_id(
|
| 24 |
+
logger: structlog.types.WrappedLogger,
|
| 25 |
+
method_name: str,
|
| 26 |
+
event_dict: dict,
|
| 27 |
+
) -> dict:
|
| 28 |
+
event_dict["correlation_id"] = get_correlation_id()
|
| 29 |
+
return event_dict
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def setup_logging(log_level: str = "INFO", log_format: str = "json") -> None:
|
| 33 |
+
shared_processors: list = [
|
| 34 |
+
structlog.contextvars.merge_contextvars,
|
| 35 |
+
add_correlation_id,
|
| 36 |
+
structlog.stdlib.add_log_level,
|
| 37 |
+
structlog.stdlib.add_logger_name,
|
| 38 |
+
structlog.processors.TimeStamper(fmt="iso"),
|
| 39 |
+
structlog.processors.StackInfoRenderer(),
|
| 40 |
+
structlog.processors.UnicodeDecoder(),
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
if log_format == "json":
|
| 44 |
+
renderer = structlog.processors.JSONRenderer()
|
| 45 |
+
else:
|
| 46 |
+
renderer = structlog.dev.ConsoleRenderer()
|
| 47 |
+
|
| 48 |
+
structlog.configure(
|
| 49 |
+
processors=[
|
| 50 |
+
*shared_processors,
|
| 51 |
+
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
|
| 52 |
+
],
|
| 53 |
+
logger_factory=structlog.stdlib.LoggerFactory(),
|
| 54 |
+
wrapper_class=structlog.stdlib.BoundLogger,
|
| 55 |
+
cache_logger_on_first_use=True,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
formatter = structlog.stdlib.ProcessorFormatter(
|
| 59 |
+
processors=[
|
| 60 |
+
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
|
| 61 |
+
renderer,
|
| 62 |
+
],
|
| 63 |
+
foreign_pre_chain=shared_processors,
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
handler = logging.StreamHandler(sys.stdout)
|
| 67 |
+
handler.setFormatter(formatter)
|
| 68 |
+
|
| 69 |
+
root = logging.getLogger()
|
| 70 |
+
root.handlers.clear()
|
| 71 |
+
root.addHandler(handler)
|
| 72 |
+
root.setLevel(getattr(logging, log_level.upper(), logging.INFO))
|
| 73 |
+
|
| 74 |
+
for name in ("uvicorn", "uvicorn.access", "uvicorn.error"):
|
| 75 |
+
lg = logging.getLogger(name)
|
| 76 |
+
lg.handlers.clear()
|
| 77 |
+
lg.propagate = True
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger:
|
| 81 |
+
return structlog.get_logger(name)
|
backend/app/core/middleware.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastAPI middleware for correlation IDs, request logging, and error handling."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import time
|
| 6 |
+
import uuid
|
| 7 |
+
|
| 8 |
+
from fastapi import FastAPI, Request, Response
|
| 9 |
+
from starlette.middleware.base import BaseHTTPMiddleware
|
| 10 |
+
|
| 11 |
+
from app.core.logging import correlation_id_var, get_logger
|
| 12 |
+
|
| 13 |
+
logger = get_logger(__name__)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class CorrelationIdMiddleware(BaseHTTPMiddleware):
|
| 17 |
+
async def dispatch(self, request: Request, call_next):
|
| 18 |
+
cid = request.headers.get("X-Correlation-ID", uuid.uuid4().hex[:16])
|
| 19 |
+
correlation_id_var.set(cid)
|
| 20 |
+
start = time.perf_counter()
|
| 21 |
+
|
| 22 |
+
response: Response = await call_next(request)
|
| 23 |
+
|
| 24 |
+
duration_ms = round((time.perf_counter() - start) * 1000, 2)
|
| 25 |
+
response.headers["X-Correlation-ID"] = cid
|
| 26 |
+
response.headers["X-Response-Time-Ms"] = str(duration_ms)
|
| 27 |
+
|
| 28 |
+
logger.info(
|
| 29 |
+
"request_completed",
|
| 30 |
+
method=request.method,
|
| 31 |
+
path=request.url.path,
|
| 32 |
+
status_code=response.status_code,
|
| 33 |
+
duration_ms=duration_ms,
|
| 34 |
+
)
|
| 35 |
+
return response
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def register_middleware(app: FastAPI) -> None:
|
| 39 |
+
app.add_middleware(CorrelationIdMiddleware)
|
backend/app/core/security.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Security utilities: API key validation, webhook signature verification, rate limiting."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import hmac
|
| 7 |
+
import time
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
from fastapi import HTTPException, Request, Security
|
| 11 |
+
from fastapi.security import APIKeyHeader
|
| 12 |
+
from slowapi import Limiter
|
| 13 |
+
from slowapi.util import get_remote_address
|
| 14 |
+
|
| 15 |
+
from app.core.config import settings
|
| 16 |
+
|
| 17 |
+
api_key_header = APIKeyHeader(name=settings.api_key_header, auto_error=False)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def get_api_key(api_key: Optional[str] = Security(api_key_header)) -> str:
|
| 21 |
+
if not api_key or api_key not in settings.allowed_api_keys:
|
| 22 |
+
raise HTTPException(status_code=403, detail="Invalid or missing API key")
|
| 23 |
+
return api_key
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _key_func(request: Request) -> str:
|
| 27 |
+
api_key = request.headers.get(settings.api_key_header, "")
|
| 28 |
+
if api_key:
|
| 29 |
+
return api_key
|
| 30 |
+
return get_remote_address(request)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
limiter = Limiter(key_func=_key_func, default_limits=[f"{settings.rate_limit_per_minute}/minute"])
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def verify_webhook_signature(payload: bytes, signature: str, timestamp: str) -> bool:
|
| 37 |
+
"""Verify Stripe-style webhook signature (t=timestamp,v1=signature)."""
|
| 38 |
+
if not signature or not timestamp:
|
| 39 |
+
return False
|
| 40 |
+
|
| 41 |
+
try:
|
| 42 |
+
ts = int(timestamp)
|
| 43 |
+
except (ValueError, TypeError):
|
| 44 |
+
return False
|
| 45 |
+
|
| 46 |
+
if abs(time.time() - ts) > 300:
|
| 47 |
+
return False
|
| 48 |
+
|
| 49 |
+
signed_payload = f"{timestamp}.{payload.decode('utf-8')}"
|
| 50 |
+
expected = hmac.new(
|
| 51 |
+
settings.webhook_secret.encode("utf-8"),
|
| 52 |
+
signed_payload.encode("utf-8"),
|
| 53 |
+
hashlib.sha256,
|
| 54 |
+
).hexdigest()
|
| 55 |
+
|
| 56 |
+
parts = signature.split(",")
|
| 57 |
+
for part in parts:
|
| 58 |
+
if part.startswith("v1="):
|
| 59 |
+
sig_value = part[3:]
|
| 60 |
+
if hmac.compare_digest(expected, sig_value):
|
| 61 |
+
return True
|
| 62 |
+
|
| 63 |
+
return False
|
backend/app/core/telemetry.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""OpenTelemetry and Prometheus instrumentation setup."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from fastapi import FastAPI
|
| 6 |
+
from prometheus_fastapi_instrumentator import Instrumentator
|
| 7 |
+
|
| 8 |
+
from app.core.config import settings
|
| 9 |
+
from app.core.logging import get_logger
|
| 10 |
+
|
| 11 |
+
logger = get_logger(__name__)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def setup_telemetry(app: FastAPI) -> None:
|
| 15 |
+
"""Initialize OpenTelemetry tracing and Prometheus metrics."""
|
| 16 |
+
# Prometheus metrics
|
| 17 |
+
Instrumentator(
|
| 18 |
+
should_group_status_codes=True,
|
| 19 |
+
should_ignore_untemplated=True,
|
| 20 |
+
excluded_handlers=["/health", "/metrics"],
|
| 21 |
+
).instrument(app).expose(app, endpoint="/metrics")
|
| 22 |
+
|
| 23 |
+
# OpenTelemetry — only in production to avoid dev noise
|
| 24 |
+
if settings.is_production:
|
| 25 |
+
try:
|
| 26 |
+
from opentelemetry import trace
|
| 27 |
+
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
|
| 28 |
+
OTLPSpanExporter,
|
| 29 |
+
)
|
| 30 |
+
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
| 31 |
+
from opentelemetry.sdk.resources import Resource
|
| 32 |
+
from opentelemetry.sdk.trace import TracerProvider
|
| 33 |
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
| 34 |
+
|
| 35 |
+
resource = Resource.create({"service.name": settings.otel_service_name})
|
| 36 |
+
provider = TracerProvider(resource=resource)
|
| 37 |
+
exporter = OTLPSpanExporter(endpoint=settings.otel_exporter_otlp_endpoint)
|
| 38 |
+
provider.add_span_processor(BatchSpanProcessor(exporter))
|
| 39 |
+
trace.set_tracer_provider(provider)
|
| 40 |
+
|
| 41 |
+
FastAPIInstrumentor.instrument_app(app)
|
| 42 |
+
logger.info("opentelemetry_initialized")
|
| 43 |
+
except ImportError:
|
| 44 |
+
logger.warning("opentelemetry_not_available", detail="Install opentelemetry packages")
|
| 45 |
+
except Exception as exc:
|
| 46 |
+
logger.error("opentelemetry_setup_failed", error=str(exc))
|
backend/app/main.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastAPI application entry point."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
from contextlib import asynccontextmanager
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
from fastapi import FastAPI, Request
|
| 10 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 11 |
+
from fastapi.responses import FileResponse, JSONResponse
|
| 12 |
+
from fastapi.staticfiles import StaticFiles
|
| 13 |
+
from slowapi import _rate_limit_exceeded_handler
|
| 14 |
+
from slowapi.errors import RateLimitExceeded
|
| 15 |
+
|
| 16 |
+
from app.api import analysis, export, health, webhooks
|
| 17 |
+
from app.core.config import settings
|
| 18 |
+
from app.core.logging import get_logger, setup_logging
|
| 19 |
+
from app.core.middleware import register_middleware
|
| 20 |
+
from app.core.security import limiter
|
| 21 |
+
from app.core.telemetry import setup_telemetry
|
| 22 |
+
from app.services.redis_client import close_redis
|
| 23 |
+
|
| 24 |
+
setup_logging(settings.log_level, settings.log_format)
|
| 25 |
+
logger = get_logger(__name__)
|
| 26 |
+
|
| 27 |
+
STATIC_DIR = Path(__file__).parent.parent / "static"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@asynccontextmanager
|
| 31 |
+
async def lifespan(app: FastAPI):
|
| 32 |
+
logger.info(
|
| 33 |
+
"application_starting",
|
| 34 |
+
app_name=settings.app_name,
|
| 35 |
+
env=settings.app_env,
|
| 36 |
+
)
|
| 37 |
+
settings.upload_path # Ensure upload directory exists
|
| 38 |
+
yield
|
| 39 |
+
logger.info("application_shutting_down")
|
| 40 |
+
await close_redis()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
app = FastAPI(
|
| 44 |
+
title=settings.app_name,
|
| 45 |
+
description="Sentiment & Topic Analysis Dashboard API",
|
| 46 |
+
version="1.0.0",
|
| 47 |
+
lifespan=lifespan,
|
| 48 |
+
docs_url="/docs" if not settings.is_production else None,
|
| 49 |
+
redoc_url="/redoc" if not settings.is_production else None,
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
# Middleware
|
| 53 |
+
app.state.limiter = limiter
|
| 54 |
+
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
| 55 |
+
|
| 56 |
+
app.add_middleware(
|
| 57 |
+
CORSMiddleware,
|
| 58 |
+
allow_origins=settings.cors_origins,
|
| 59 |
+
allow_credentials=True,
|
| 60 |
+
allow_methods=["*"],
|
| 61 |
+
allow_headers=["*"],
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
register_middleware(app)
|
| 65 |
+
setup_telemetry(app)
|
| 66 |
+
|
| 67 |
+
# Routes
|
| 68 |
+
app.include_router(health.router)
|
| 69 |
+
app.include_router(analysis.router)
|
| 70 |
+
app.include_router(export.router)
|
| 71 |
+
app.include_router(webhooks.router)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@app.exception_handler(Exception)
|
| 75 |
+
async def global_exception_handler(request: Request, exc: Exception):
|
| 76 |
+
from app.core.logging import get_correlation_id
|
| 77 |
+
|
| 78 |
+
logger.error(
|
| 79 |
+
"unhandled_exception",
|
| 80 |
+
path=request.url.path,
|
| 81 |
+
error=str(exc),
|
| 82 |
+
exc_info=True,
|
| 83 |
+
)
|
| 84 |
+
return JSONResponse(
|
| 85 |
+
status_code=500,
|
| 86 |
+
content={
|
| 87 |
+
"detail": "Internal server error",
|
| 88 |
+
"correlation_id": get_correlation_id(),
|
| 89 |
+
},
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# Serve frontend static files in production (when static/ dir exists)
|
| 94 |
+
if STATIC_DIR.is_dir():
|
| 95 |
+
app.mount("/assets", StaticFiles(directory=str(STATIC_DIR / "assets")), name="static-assets")
|
| 96 |
+
|
| 97 |
+
@app.get("/{full_path:path}")
|
| 98 |
+
async def serve_spa(full_path: str):
|
| 99 |
+
"""Serve the SPA index.html for all non-API routes."""
|
| 100 |
+
file_path = STATIC_DIR / full_path
|
| 101 |
+
if file_path.is_file():
|
| 102 |
+
return FileResponse(str(file_path))
|
| 103 |
+
return FileResponse(str(STATIC_DIR / "index.html"))
|
backend/app/models/__init__.py
ADDED
|
File without changes
|
backend/app/models/schemas.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pydantic schemas for all data models."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
from enum import Enum
|
| 7 |
+
from typing import Any, Dict, List, Optional
|
| 8 |
+
|
| 9 |
+
from pydantic import BaseModel, Field
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
# --- Enums ---
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class SentimentLabel(str, Enum):
|
| 16 |
+
POSITIVE = "positive"
|
| 17 |
+
NEGATIVE = "negative"
|
| 18 |
+
NEUTRAL = "neutral"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ExportFormat(str, Enum):
|
| 22 |
+
CSV = "csv"
|
| 23 |
+
JSON = "json"
|
| 24 |
+
PDF = "pdf"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class AnomalyType(str, Enum):
|
| 28 |
+
SENTIMENT_DROP = "sentiment_drop"
|
| 29 |
+
TOPIC_SPIKE = "topic_spike"
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class AnalysisStatus(str, Enum):
|
| 33 |
+
PENDING = "pending"
|
| 34 |
+
PROCESSING = "processing"
|
| 35 |
+
COMPLETED = "completed"
|
| 36 |
+
FAILED = "failed"
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# --- Request Models ---
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class FeedbackEntry(BaseModel):
|
| 43 |
+
id: Optional[str] = None
|
| 44 |
+
text: str = Field(..., min_length=1, max_length=50000)
|
| 45 |
+
source: Optional[str] = None
|
| 46 |
+
timestamp: Optional[datetime] = None
|
| 47 |
+
metadata: Optional[Dict[str, Any]] = None
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class AnalysisRequest(BaseModel):
|
| 51 |
+
entries: List[FeedbackEntry] = Field(..., min_items=1)
|
| 52 |
+
options: Optional[AnalysisOptions] = None
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class AnalysisOptions(BaseModel):
|
| 56 |
+
min_cluster_size: int = Field(default=5, ge=2, le=100)
|
| 57 |
+
min_samples: int = Field(default=3, ge=1, le=50)
|
| 58 |
+
detect_anomalies: bool = True
|
| 59 |
+
language_filter: Optional[str] = None
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class FilterParams(BaseModel):
|
| 63 |
+
date_from: Optional[datetime] = None
|
| 64 |
+
date_to: Optional[datetime] = None
|
| 65 |
+
sentiment_min: Optional[float] = Field(default=None, ge=-1.0, le=1.0)
|
| 66 |
+
sentiment_max: Optional[float] = Field(default=None, ge=-1.0, le=1.0)
|
| 67 |
+
topics: Optional[List[int]] = None
|
| 68 |
+
languages: Optional[List[str]] = None
|
| 69 |
+
sources: Optional[List[str]] = None
|
| 70 |
+
search_text: Optional[str] = None
|
| 71 |
+
page: int = Field(default=1, ge=1)
|
| 72 |
+
page_size: int = Field(default=50, ge=1, le=500)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class ComparisonRequest(BaseModel):
|
| 76 |
+
segment_a: FilterParams
|
| 77 |
+
segment_b: FilterParams
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class WebhookPayload(BaseModel):
|
| 81 |
+
event_type: str
|
| 82 |
+
data: List[FeedbackEntry]
|
| 83 |
+
source: Optional[str] = None
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class AnomalyThresholds(BaseModel):
|
| 87 |
+
sentiment_threshold: float = Field(default=1.5, ge=0.1, le=5.0)
|
| 88 |
+
topic_spike_threshold: float = Field(default=3.0, ge=1.0, le=10.0)
|
| 89 |
+
rolling_window: int = Field(default=50, ge=10, le=1000)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# --- Response Models ---
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class SentimentResult(BaseModel):
|
| 96 |
+
label: SentimentLabel
|
| 97 |
+
score: float = Field(..., ge=0.0, le=1.0)
|
| 98 |
+
confidence: float = Field(..., ge=0.0, le=1.0)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class LanguageResult(BaseModel):
|
| 102 |
+
language: str
|
| 103 |
+
confidence: float = Field(..., ge=0.0, le=1.0)
|
| 104 |
+
method: str # "langdetect" or "cld3"
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
class TopicInfo(BaseModel):
|
| 108 |
+
topic_id: int
|
| 109 |
+
label: str
|
| 110 |
+
keywords: List[str]
|
| 111 |
+
size: int
|
| 112 |
+
representative_docs: List[str] = Field(default_factory=list)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class AnalyzedEntry(BaseModel):
|
| 116 |
+
id: str
|
| 117 |
+
text: str
|
| 118 |
+
source: Optional[str] = None
|
| 119 |
+
timestamp: Optional[datetime] = None
|
| 120 |
+
sentiment: SentimentResult
|
| 121 |
+
language: LanguageResult
|
| 122 |
+
topic_id: int
|
| 123 |
+
topic_label: str
|
| 124 |
+
embedding: Optional[List[float]] = None
|
| 125 |
+
metadata: Optional[Dict[str, Any]] = None
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
class TopicCluster(BaseModel):
|
| 129 |
+
topic_id: int
|
| 130 |
+
label: str
|
| 131 |
+
keywords: List[str]
|
| 132 |
+
size: int
|
| 133 |
+
avg_sentiment: float
|
| 134 |
+
sentiment_distribution: Dict[str, int]
|
| 135 |
+
languages: Dict[str, int]
|
| 136 |
+
representative_docs: List[str]
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
class SentimentTrend(BaseModel):
|
| 140 |
+
period: str
|
| 141 |
+
avg_sentiment: float
|
| 142 |
+
count: int
|
| 143 |
+
positive: int
|
| 144 |
+
negative: int
|
| 145 |
+
neutral: int
|
| 146 |
+
confidence_lower: float
|
| 147 |
+
confidence_upper: float
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
class TopicLink(BaseModel):
|
| 151 |
+
source: int
|
| 152 |
+
target: int
|
| 153 |
+
weight: float
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class TopicGraph(BaseModel):
|
| 157 |
+
nodes: List[TopicCluster]
|
| 158 |
+
links: List[TopicLink]
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class DataQualityReport(BaseModel):
|
| 162 |
+
total_entries: int
|
| 163 |
+
low_confidence_count: int
|
| 164 |
+
low_confidence_entries: List[str]
|
| 165 |
+
mixed_language_count: int
|
| 166 |
+
mixed_language_entries: List[str]
|
| 167 |
+
duplicate_count: int
|
| 168 |
+
duplicate_entries: List[str]
|
| 169 |
+
avg_confidence: float
|
| 170 |
+
language_distribution: Dict[str, int]
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
class AnomalyAlert(BaseModel):
|
| 174 |
+
id: str
|
| 175 |
+
type: AnomalyType
|
| 176 |
+
severity: str
|
| 177 |
+
message: str
|
| 178 |
+
detected_at: datetime
|
| 179 |
+
details: Dict[str, Any]
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
class AnalysisResult(BaseModel):
|
| 183 |
+
job_id: str
|
| 184 |
+
status: AnalysisStatus
|
| 185 |
+
created_at: datetime
|
| 186 |
+
completed_at: Optional[datetime] = None
|
| 187 |
+
total_entries: int
|
| 188 |
+
entries: List[AnalyzedEntry] = Field(default_factory=list)
|
| 189 |
+
topics: List[TopicCluster] = Field(default_factory=list)
|
| 190 |
+
sentiment_trends: List[SentimentTrend] = Field(default_factory=list)
|
| 191 |
+
topic_graph: Optional[TopicGraph] = None
|
| 192 |
+
data_quality: Optional[DataQualityReport] = None
|
| 193 |
+
anomalies: List[AnomalyAlert] = Field(default_factory=list)
|
| 194 |
+
summary: Optional[AnalysisSummary] = None
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
class AnalysisSummary(BaseModel):
|
| 198 |
+
total_entries: int
|
| 199 |
+
avg_sentiment: float
|
| 200 |
+
dominant_sentiment: SentimentLabel
|
| 201 |
+
num_topics: int
|
| 202 |
+
top_topics: List[TopicInfo]
|
| 203 |
+
languages_detected: List[str]
|
| 204 |
+
date_range: Optional[Dict[str, str]] = None
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
class ComparisonResult(BaseModel):
|
| 208 |
+
segment_a: AnalysisSummary
|
| 209 |
+
segment_b: AnalysisSummary
|
| 210 |
+
sentiment_delta: float
|
| 211 |
+
topic_changes: List[Dict[str, Any]]
|
| 212 |
+
new_topics: List[TopicInfo]
|
| 213 |
+
disappeared_topics: List[TopicInfo]
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
class JobStatus(BaseModel):
|
| 217 |
+
job_id: str
|
| 218 |
+
status: AnalysisStatus
|
| 219 |
+
progress: float = Field(default=0.0, ge=0.0, le=1.0)
|
| 220 |
+
message: str = ""
|
| 221 |
+
created_at: datetime
|
| 222 |
+
completed_at: Optional[datetime] = None
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
class HealthResponse(BaseModel):
|
| 226 |
+
status: str
|
| 227 |
+
version: str
|
| 228 |
+
models_loaded: bool
|
| 229 |
+
redis_connected: bool
|
| 230 |
+
uptime_seconds: float
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
class ErrorResponse(BaseModel):
|
| 234 |
+
detail: str
|
| 235 |
+
correlation_id: Optional[str] = None
|
| 236 |
+
code: Optional[str] = None
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
# Fix forward references
|
| 240 |
+
AnalysisRequest.model_rebuild()
|
backend/app/services/__init__.py
ADDED
|
File without changes
|
backend/app/services/analysis_pipeline.py
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Analysis pipeline orchestrator — coordinates all ML services."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import uuid
|
| 6 |
+
from collections import Counter
|
| 7 |
+
from datetime import datetime
|
| 8 |
+
from typing import Any, Dict, List, Optional
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
|
| 12 |
+
from app.core.logging import get_logger
|
| 13 |
+
from app.models.schemas import (
|
| 14 |
+
AnalysisResult,
|
| 15 |
+
AnalysisSummary,
|
| 16 |
+
AnalysisStatus,
|
| 17 |
+
AnalyzedEntry,
|
| 18 |
+
FeedbackEntry,
|
| 19 |
+
SentimentLabel,
|
| 20 |
+
SentimentTrend,
|
| 21 |
+
TopicInfo,
|
| 22 |
+
)
|
| 23 |
+
from app.services.anomaly_detection import run_anomaly_detection
|
| 24 |
+
from app.services.data_quality import analyze_data_quality
|
| 25 |
+
from app.services.language_detection import detect_languages_batch
|
| 26 |
+
from app.services.notifications import notify_anomalies
|
| 27 |
+
from app.services.redis_client import publish_event
|
| 28 |
+
from app.services.sentiment import (
|
| 29 |
+
analyze_sentiment,
|
| 30 |
+
get_fallback_sentiment,
|
| 31 |
+
is_model_available,
|
| 32 |
+
)
|
| 33 |
+
from app.services.topic_clustering import (
|
| 34 |
+
build_topic_graph,
|
| 35 |
+
cluster_topics,
|
| 36 |
+
compute_embeddings,
|
| 37 |
+
is_embedding_model_available,
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
logger = get_logger(__name__)
|
| 41 |
+
|
| 42 |
+
# In-memory job store (production would use a database)
|
| 43 |
+
_jobs: Dict[str, AnalysisResult] = {}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
async def run_analysis(
|
| 47 |
+
entries: list[FeedbackEntry],
|
| 48 |
+
job_id: Optional[str] = None,
|
| 49 |
+
detect_anomalies: bool = True,
|
| 50 |
+
min_cluster_size: Optional[int] = None,
|
| 51 |
+
min_samples: Optional[int] = None,
|
| 52 |
+
) -> AnalysisResult:
|
| 53 |
+
"""Run the full analysis pipeline."""
|
| 54 |
+
job_id = job_id or uuid.uuid4().hex[:12]
|
| 55 |
+
now = datetime.utcnow()
|
| 56 |
+
|
| 57 |
+
result = AnalysisResult(
|
| 58 |
+
job_id=job_id,
|
| 59 |
+
status=AnalysisStatus.PROCESSING,
|
| 60 |
+
created_at=now,
|
| 61 |
+
total_entries=len(entries),
|
| 62 |
+
)
|
| 63 |
+
_jobs[job_id] = result
|
| 64 |
+
|
| 65 |
+
await publish_event("analysis_updates", {"job_id": job_id, "status": "processing", "progress": 0.0})
|
| 66 |
+
|
| 67 |
+
try:
|
| 68 |
+
import time as _time
|
| 69 |
+
|
| 70 |
+
texts = [e.text for e in entries]
|
| 71 |
+
logger.info("pipeline_started", job_id=job_id, entry_count=len(texts),
|
| 72 |
+
sample_text=texts[0][:100] if texts else "")
|
| 73 |
+
|
| 74 |
+
# Step 1: Language detection
|
| 75 |
+
t0 = _time.time()
|
| 76 |
+
logger.info("pipeline_step", step="language_detection", count=len(texts))
|
| 77 |
+
languages = detect_languages_batch(texts)
|
| 78 |
+
lang_counts = {}
|
| 79 |
+
for l in languages:
|
| 80 |
+
lang_counts[l.language] = lang_counts.get(l.language, 0) + 1
|
| 81 |
+
logger.info("language_detection_complete", elapsed=round(_time.time() - t0, 2),
|
| 82 |
+
language_distribution=str(lang_counts))
|
| 83 |
+
await publish_event("analysis_updates", {"job_id": job_id, "status": "processing", "progress": 0.2})
|
| 84 |
+
|
| 85 |
+
# Step 2: Sentiment analysis
|
| 86 |
+
t0 = _time.time()
|
| 87 |
+
model_available = is_model_available()
|
| 88 |
+
logger.info("pipeline_step", step="sentiment_analysis", count=len(texts),
|
| 89 |
+
model_available=model_available)
|
| 90 |
+
if model_available:
|
| 91 |
+
sentiments = await analyze_sentiment(texts)
|
| 92 |
+
else:
|
| 93 |
+
logger.warning("sentiment_model_unavailable_using_fallback",
|
| 94 |
+
reason="ML model could not be loaded — using keyword fallback")
|
| 95 |
+
sentiments = [get_fallback_sentiment(t) for t in texts]
|
| 96 |
+
|
| 97 |
+
# Log sentiment distribution
|
| 98 |
+
sent_dist = {}
|
| 99 |
+
scores = [s.score for s in sentiments]
|
| 100 |
+
for s in sentiments:
|
| 101 |
+
sent_dist[s.label.value] = sent_dist.get(s.label.value, 0) + 1
|
| 102 |
+
logger.info("sentiment_analysis_complete",
|
| 103 |
+
elapsed=round(_time.time() - t0, 2),
|
| 104 |
+
distribution=str(sent_dist),
|
| 105 |
+
avg_score=round(sum(scores) / len(scores), 4) if scores else 0,
|
| 106 |
+
min_score=round(min(scores), 4) if scores else 0,
|
| 107 |
+
max_score=round(max(scores), 4) if scores else 0,
|
| 108 |
+
sample_label=sentiments[0].label.value if sentiments else "none",
|
| 109 |
+
sample_score=sentiments[0].score if sentiments else 0)
|
| 110 |
+
await publish_event("analysis_updates", {"job_id": job_id, "status": "processing", "progress": 0.4})
|
| 111 |
+
|
| 112 |
+
# Step 3: Embeddings + Topic Clustering
|
| 113 |
+
t0 = _time.time()
|
| 114 |
+
logger.info("pipeline_step", step="topic_clustering", count=len(texts))
|
| 115 |
+
topic_assignments = [-1] * len(texts)
|
| 116 |
+
clusters = []
|
| 117 |
+
topic_graph = None
|
| 118 |
+
reduced_embeddings = None
|
| 119 |
+
|
| 120 |
+
if is_embedding_model_available() and len(texts) >= 5:
|
| 121 |
+
embeddings = await compute_embeddings(texts)
|
| 122 |
+
topic_assignments, clusters, reduced_embeddings = await cluster_topics(
|
| 123 |
+
texts, embeddings, min_cluster_size, min_samples
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
# Enrich clusters with sentiment/language data
|
| 127 |
+
for cluster in clusters:
|
| 128 |
+
indices = [i for i, t in enumerate(topic_assignments) if t == cluster.topic_id]
|
| 129 |
+
if indices:
|
| 130 |
+
cluster_sentiments = [sentiments[i] for i in indices]
|
| 131 |
+
cluster.avg_sentiment = round(
|
| 132 |
+
np.mean([s.score for s in cluster_sentiments]), 4
|
| 133 |
+
)
|
| 134 |
+
cluster.sentiment_distribution = dict(
|
| 135 |
+
Counter(s.label.value for s in cluster_sentiments)
|
| 136 |
+
)
|
| 137 |
+
cluster.languages = dict(
|
| 138 |
+
Counter(languages[i].language for i in indices)
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
topic_graph = build_topic_graph(clusters, embeddings, topic_assignments)
|
| 142 |
+
else:
|
| 143 |
+
logger.warning("topic_clustering_skipped", reason="model unavailable or too few entries")
|
| 144 |
+
|
| 145 |
+
await publish_event("analysis_updates", {"job_id": job_id, "status": "processing", "progress": 0.7})
|
| 146 |
+
|
| 147 |
+
# Step 4: Build analyzed entries
|
| 148 |
+
analyzed_entries = []
|
| 149 |
+
for i, entry in enumerate(entries):
|
| 150 |
+
topic_id = topic_assignments[i]
|
| 151 |
+
topic_label = "Uncategorized"
|
| 152 |
+
for c in clusters:
|
| 153 |
+
if c.topic_id == topic_id:
|
| 154 |
+
topic_label = c.label
|
| 155 |
+
break
|
| 156 |
+
|
| 157 |
+
analyzed_entries.append(
|
| 158 |
+
AnalyzedEntry(
|
| 159 |
+
id=entry.id or uuid.uuid4().hex[:12],
|
| 160 |
+
text=entry.text,
|
| 161 |
+
source=entry.source,
|
| 162 |
+
timestamp=entry.timestamp,
|
| 163 |
+
sentiment=sentiments[i],
|
| 164 |
+
language=languages[i],
|
| 165 |
+
topic_id=topic_id,
|
| 166 |
+
topic_label=topic_label,
|
| 167 |
+
metadata=entry.metadata,
|
| 168 |
+
)
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
# Step 5: Sentiment trends
|
| 172 |
+
trends = _compute_sentiment_trends(analyzed_entries)
|
| 173 |
+
|
| 174 |
+
# Step 6: Data quality
|
| 175 |
+
data_quality = analyze_data_quality(analyzed_entries)
|
| 176 |
+
|
| 177 |
+
# Step 7: Anomaly detection
|
| 178 |
+
anomalies = []
|
| 179 |
+
if detect_anomalies and len(sentiments) >= 20:
|
| 180 |
+
anomalies = run_anomaly_detection(sentiments, topic_assignments)
|
| 181 |
+
if anomalies:
|
| 182 |
+
await notify_anomalies(anomalies)
|
| 183 |
+
|
| 184 |
+
await publish_event("analysis_updates", {"job_id": job_id, "status": "processing", "progress": 0.9})
|
| 185 |
+
|
| 186 |
+
# Build summary
|
| 187 |
+
sentiment_counts = Counter(s.label.value for s in sentiments)
|
| 188 |
+
dominant = max(sentiment_counts, key=sentiment_counts.get) if sentiment_counts else "neutral"
|
| 189 |
+
top_topics = [
|
| 190 |
+
TopicInfo(
|
| 191 |
+
topic_id=c.topic_id,
|
| 192 |
+
label=c.label,
|
| 193 |
+
keywords=c.keywords,
|
| 194 |
+
size=c.size,
|
| 195 |
+
)
|
| 196 |
+
for c in sorted(clusters, key=lambda c: c.size, reverse=True)[:5]
|
| 197 |
+
if c.topic_id != -1
|
| 198 |
+
]
|
| 199 |
+
|
| 200 |
+
summary = AnalysisSummary(
|
| 201 |
+
total_entries=len(entries),
|
| 202 |
+
avg_sentiment=round(np.mean([s.score for s in sentiments]), 4),
|
| 203 |
+
dominant_sentiment=SentimentLabel(dominant),
|
| 204 |
+
num_topics=len([c for c in clusters if c.topic_id != -1]),
|
| 205 |
+
top_topics=top_topics,
|
| 206 |
+
languages_detected=list(set(l.language for l in languages if l.language != "unknown")),
|
| 207 |
+
date_range=_get_date_range(entries),
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
# Final result
|
| 211 |
+
result.status = AnalysisStatus.COMPLETED
|
| 212 |
+
result.completed_at = datetime.utcnow()
|
| 213 |
+
result.entries = analyzed_entries
|
| 214 |
+
result.topics = clusters
|
| 215 |
+
result.sentiment_trends = trends
|
| 216 |
+
result.topic_graph = topic_graph
|
| 217 |
+
result.data_quality = data_quality
|
| 218 |
+
result.anomalies = anomalies
|
| 219 |
+
result.summary = summary
|
| 220 |
+
_jobs[job_id] = result
|
| 221 |
+
|
| 222 |
+
await publish_event("analysis_updates", {
|
| 223 |
+
"job_id": job_id,
|
| 224 |
+
"status": "completed",
|
| 225 |
+
"progress": 1.0,
|
| 226 |
+
"total_entries": len(entries),
|
| 227 |
+
})
|
| 228 |
+
|
| 229 |
+
logger.info("analysis_completed", job_id=job_id, entries=len(entries), topics=len(clusters))
|
| 230 |
+
return result
|
| 231 |
+
|
| 232 |
+
except Exception as exc:
|
| 233 |
+
result.status = AnalysisStatus.FAILED
|
| 234 |
+
_jobs[job_id] = result
|
| 235 |
+
await publish_event("analysis_updates", {"job_id": job_id, "status": "failed", "error": str(exc)})
|
| 236 |
+
logger.error("analysis_failed", job_id=job_id, error=str(exc))
|
| 237 |
+
raise
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _compute_sentiment_trends(entries: list[AnalyzedEntry]) -> list[SentimentTrend]:
|
| 241 |
+
"""Compute sentiment trends over time periods."""
|
| 242 |
+
dated = [e for e in entries if e.timestamp]
|
| 243 |
+
if not dated:
|
| 244 |
+
return [_single_period_trend(entries, "all")]
|
| 245 |
+
|
| 246 |
+
dated.sort(key=lambda e: e.timestamp)
|
| 247 |
+
|
| 248 |
+
# Determine grouping: daily if span > 7 days, else hourly
|
| 249 |
+
span = (dated[-1].timestamp - dated[0].timestamp).days
|
| 250 |
+
if span > 30:
|
| 251 |
+
fmt = "%Y-%m"
|
| 252 |
+
elif span > 7:
|
| 253 |
+
fmt = "%Y-%m-%d"
|
| 254 |
+
else:
|
| 255 |
+
fmt = "%Y-%m-%d %H:00"
|
| 256 |
+
|
| 257 |
+
groups: dict[str, list[AnalyzedEntry]] = {}
|
| 258 |
+
for e in dated:
|
| 259 |
+
key = e.timestamp.strftime(fmt)
|
| 260 |
+
groups.setdefault(key, []).append(e)
|
| 261 |
+
|
| 262 |
+
trends = []
|
| 263 |
+
for period, group_entries in groups.items():
|
| 264 |
+
trends.append(_single_period_trend(group_entries, period))
|
| 265 |
+
|
| 266 |
+
return trends
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _single_period_trend(entries: list[AnalyzedEntry], period: str) -> SentimentTrend:
|
| 270 |
+
scores = [e.sentiment.score for e in entries]
|
| 271 |
+
mean = np.mean(scores) if scores else 0.5
|
| 272 |
+
std = np.std(scores) if scores else 0
|
| 273 |
+
n = len(scores)
|
| 274 |
+
se = std / np.sqrt(n) if n > 0 else 0
|
| 275 |
+
|
| 276 |
+
return SentimentTrend(
|
| 277 |
+
period=period,
|
| 278 |
+
avg_sentiment=round(float(mean), 4),
|
| 279 |
+
count=n,
|
| 280 |
+
positive=sum(1 for e in entries if e.sentiment.label == SentimentLabel.POSITIVE),
|
| 281 |
+
negative=sum(1 for e in entries if e.sentiment.label == SentimentLabel.NEGATIVE),
|
| 282 |
+
neutral=sum(1 for e in entries if e.sentiment.label == SentimentLabel.NEUTRAL),
|
| 283 |
+
confidence_lower=round(float(max(0, mean - 1.96 * se)), 4),
|
| 284 |
+
confidence_upper=round(float(min(1, mean + 1.96 * se)), 4),
|
| 285 |
+
)
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def _get_date_range(entries: list[FeedbackEntry]) -> dict[str, str] | None:
|
| 289 |
+
dated = [e.timestamp for e in entries if e.timestamp]
|
| 290 |
+
if not dated:
|
| 291 |
+
return None
|
| 292 |
+
return {
|
| 293 |
+
"start": min(dated).isoformat(),
|
| 294 |
+
"end": max(dated).isoformat(),
|
| 295 |
+
}
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def get_job(job_id: str) -> AnalysisResult | None:
|
| 299 |
+
return _jobs.get(job_id)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def get_all_jobs() -> list[AnalysisResult]:
|
| 303 |
+
return list(_jobs.values())
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def filter_entries(
|
| 307 |
+
entries: list[AnalyzedEntry],
|
| 308 |
+
date_from=None,
|
| 309 |
+
date_to=None,
|
| 310 |
+
sentiment_min=None,
|
| 311 |
+
sentiment_max=None,
|
| 312 |
+
topics=None,
|
| 313 |
+
languages=None,
|
| 314 |
+
sources=None,
|
| 315 |
+
search_text=None,
|
| 316 |
+
) -> list[AnalyzedEntry]:
|
| 317 |
+
"""Apply filters to analyzed entries."""
|
| 318 |
+
result = entries
|
| 319 |
+
|
| 320 |
+
if date_from:
|
| 321 |
+
result = [e for e in result if e.timestamp and e.timestamp >= date_from]
|
| 322 |
+
if date_to:
|
| 323 |
+
result = [e for e in result if e.timestamp and e.timestamp <= date_to]
|
| 324 |
+
if sentiment_min is not None:
|
| 325 |
+
result = [e for e in result if e.sentiment.score >= sentiment_min]
|
| 326 |
+
if sentiment_max is not None:
|
| 327 |
+
result = [e for e in result if e.sentiment.score <= sentiment_max]
|
| 328 |
+
if topics:
|
| 329 |
+
result = [e for e in result if e.topic_id in topics]
|
| 330 |
+
if languages:
|
| 331 |
+
result = [e for e in result if e.language.language in languages]
|
| 332 |
+
if sources:
|
| 333 |
+
result = [e for e in result if e.source in sources]
|
| 334 |
+
if search_text:
|
| 335 |
+
search_lower = search_text.lower()
|
| 336 |
+
result = [e for e in result if search_lower in e.text.lower()]
|
| 337 |
+
|
| 338 |
+
return result
|
backend/app/services/anomaly_detection.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Anomaly detection: sentiment drift and topic spikes."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import uuid
|
| 6 |
+
from datetime import datetime
|
| 7 |
+
from typing import List, Optional
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
from app.core.config import settings
|
| 12 |
+
from app.core.logging import get_logger
|
| 13 |
+
from app.models.schemas import AnomalyAlert, AnomalyType, SentimentResult
|
| 14 |
+
|
| 15 |
+
logger = get_logger(__name__)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def detect_sentiment_anomalies(
|
| 19 |
+
sentiments: list[SentimentResult],
|
| 20 |
+
window: Optional[int] = None,
|
| 21 |
+
threshold: Optional[float] = None,
|
| 22 |
+
) -> list[AnomalyAlert]:
|
| 23 |
+
"""Detect when sentiment drops below rolling average - threshold * std."""
|
| 24 |
+
window = window or settings.anomaly_rolling_window
|
| 25 |
+
threshold = threshold or settings.anomaly_sentiment_threshold
|
| 26 |
+
|
| 27 |
+
if len(sentiments) < window:
|
| 28 |
+
return []
|
| 29 |
+
|
| 30 |
+
scores = np.array([s.score for s in sentiments])
|
| 31 |
+
alerts = []
|
| 32 |
+
|
| 33 |
+
for i in range(window, len(scores)):
|
| 34 |
+
window_slice = scores[i - window : i]
|
| 35 |
+
mean = np.mean(window_slice)
|
| 36 |
+
std = np.std(window_slice)
|
| 37 |
+
|
| 38 |
+
if std == 0:
|
| 39 |
+
continue
|
| 40 |
+
|
| 41 |
+
z_score = (scores[i] - mean) / std
|
| 42 |
+
|
| 43 |
+
if z_score < -threshold:
|
| 44 |
+
alerts.append(
|
| 45 |
+
AnomalyAlert(
|
| 46 |
+
id=uuid.uuid4().hex[:12],
|
| 47 |
+
type=AnomalyType.SENTIMENT_DROP,
|
| 48 |
+
severity="high" if z_score < -2 * threshold else "medium",
|
| 49 |
+
message=f"Sentiment dropped to {scores[i]:.3f} (rolling avg: {mean:.3f}, z-score: {z_score:.2f})",
|
| 50 |
+
detected_at=datetime.utcnow(),
|
| 51 |
+
details={
|
| 52 |
+
"index": i,
|
| 53 |
+
"value": float(scores[i]),
|
| 54 |
+
"rolling_mean": float(mean),
|
| 55 |
+
"rolling_std": float(std),
|
| 56 |
+
"z_score": float(z_score),
|
| 57 |
+
},
|
| 58 |
+
)
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
return alerts
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def detect_topic_spikes(
|
| 65 |
+
topic_assignments: list[int],
|
| 66 |
+
window: Optional[int] = None,
|
| 67 |
+
threshold: Optional[float] = None,
|
| 68 |
+
) -> list[AnomalyAlert]:
|
| 69 |
+
"""Detect unusual spikes in topic frequency."""
|
| 70 |
+
window = window or settings.anomaly_rolling_window
|
| 71 |
+
threshold = threshold or settings.anomaly_topic_spike_threshold
|
| 72 |
+
|
| 73 |
+
if len(topic_assignments) < window:
|
| 74 |
+
return []
|
| 75 |
+
|
| 76 |
+
alerts = []
|
| 77 |
+
unique_topics = set(topic_assignments)
|
| 78 |
+
|
| 79 |
+
for topic_id in unique_topics:
|
| 80 |
+
if topic_id == -1:
|
| 81 |
+
continue
|
| 82 |
+
|
| 83 |
+
occurrences = [1 if t == topic_id else 0 for t in topic_assignments]
|
| 84 |
+
|
| 85 |
+
for i in range(window, len(occurrences)):
|
| 86 |
+
window_slice = occurrences[i - window : i]
|
| 87 |
+
mean = np.mean(window_slice)
|
| 88 |
+
std = np.std(window_slice)
|
| 89 |
+
|
| 90 |
+
if std == 0:
|
| 91 |
+
continue
|
| 92 |
+
|
| 93 |
+
# Check for spike in last 10% of window
|
| 94 |
+
recent = occurrences[max(0, i - window // 10) : i]
|
| 95 |
+
recent_rate = np.mean(recent) if recent else 0
|
| 96 |
+
|
| 97 |
+
z_score = (recent_rate - mean) / std if std > 0 else 0
|
| 98 |
+
|
| 99 |
+
if z_score > threshold:
|
| 100 |
+
alerts.append(
|
| 101 |
+
AnomalyAlert(
|
| 102 |
+
id=uuid.uuid4().hex[:12],
|
| 103 |
+
type=AnomalyType.TOPIC_SPIKE,
|
| 104 |
+
severity="high" if z_score > 2 * threshold else "medium",
|
| 105 |
+
message=f"Topic {topic_id} spike detected (rate: {recent_rate:.3f}, avg: {mean:.3f})",
|
| 106 |
+
detected_at=datetime.utcnow(),
|
| 107 |
+
details={
|
| 108 |
+
"topic_id": topic_id,
|
| 109 |
+
"recent_rate": float(recent_rate),
|
| 110 |
+
"rolling_mean": float(mean),
|
| 111 |
+
"z_score": float(z_score),
|
| 112 |
+
},
|
| 113 |
+
)
|
| 114 |
+
)
|
| 115 |
+
break # One alert per topic
|
| 116 |
+
|
| 117 |
+
return alerts
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def run_anomaly_detection(
|
| 121 |
+
sentiments: list[SentimentResult],
|
| 122 |
+
topic_assignments: list[int],
|
| 123 |
+
thresholds: Optional[dict] = None,
|
| 124 |
+
) -> list[AnomalyAlert]:
|
| 125 |
+
"""Run all anomaly detection checks."""
|
| 126 |
+
window = thresholds.get("rolling_window") if thresholds else None
|
| 127 |
+
sent_thresh = thresholds.get("sentiment_threshold") if thresholds else None
|
| 128 |
+
topic_thresh = thresholds.get("topic_spike_threshold") if thresholds else None
|
| 129 |
+
|
| 130 |
+
alerts = []
|
| 131 |
+
alerts.extend(detect_sentiment_anomalies(sentiments, window, sent_thresh))
|
| 132 |
+
alerts.extend(detect_topic_spikes(topic_assignments, window, topic_thresh))
|
| 133 |
+
return alerts
|
backend/app/services/data_quality.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Data quality analysis: low confidence, mixed language, duplicate detection."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from collections import Counter
|
| 6 |
+
from typing import List
|
| 7 |
+
|
| 8 |
+
from app.models.schemas import AnalyzedEntry, DataQualityReport
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def analyze_data_quality(entries: list[AnalyzedEntry]) -> DataQualityReport:
|
| 12 |
+
"""Generate data quality report from analyzed entries."""
|
| 13 |
+
if not entries:
|
| 14 |
+
return DataQualityReport(
|
| 15 |
+
total_entries=0,
|
| 16 |
+
low_confidence_count=0,
|
| 17 |
+
low_confidence_entries=[],
|
| 18 |
+
mixed_language_count=0,
|
| 19 |
+
mixed_language_entries=[],
|
| 20 |
+
duplicate_count=0,
|
| 21 |
+
duplicate_entries=[],
|
| 22 |
+
avg_confidence=0.0,
|
| 23 |
+
language_distribution={},
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
# Low confidence predictions (< 0.5)
|
| 27 |
+
low_conf = [e for e in entries if e.sentiment.confidence < 0.5]
|
| 28 |
+
low_conf_ids = [e.id for e in low_conf[:50]]
|
| 29 |
+
|
| 30 |
+
# Mixed language: entries where detected language differs from majority
|
| 31 |
+
lang_counts = Counter(e.language.language for e in entries)
|
| 32 |
+
majority_lang = lang_counts.most_common(1)[0][0] if lang_counts else "unknown"
|
| 33 |
+
mixed_lang = [
|
| 34 |
+
e for e in entries
|
| 35 |
+
if e.language.language != majority_lang and e.language.language != "unknown"
|
| 36 |
+
]
|
| 37 |
+
mixed_lang_ids = [e.id for e in mixed_lang[:50]]
|
| 38 |
+
|
| 39 |
+
# Duplicate detection via text similarity (exact and near-duplicates)
|
| 40 |
+
seen_texts: dict[str, str] = {}
|
| 41 |
+
duplicate_ids = []
|
| 42 |
+
for e in entries:
|
| 43 |
+
normalized = e.text.strip().lower()[:200]
|
| 44 |
+
if normalized in seen_texts:
|
| 45 |
+
duplicate_ids.append(e.id)
|
| 46 |
+
else:
|
| 47 |
+
seen_texts[normalized] = e.id
|
| 48 |
+
|
| 49 |
+
# Average confidence
|
| 50 |
+
avg_conf = sum(e.sentiment.confidence for e in entries) / len(entries)
|
| 51 |
+
|
| 52 |
+
return DataQualityReport(
|
| 53 |
+
total_entries=len(entries),
|
| 54 |
+
low_confidence_count=len(low_conf),
|
| 55 |
+
low_confidence_entries=low_conf_ids,
|
| 56 |
+
mixed_language_count=len(mixed_lang),
|
| 57 |
+
mixed_language_entries=mixed_lang_ids,
|
| 58 |
+
duplicate_count=len(duplicate_ids),
|
| 59 |
+
duplicate_entries=duplicate_ids[:50],
|
| 60 |
+
avg_confidence=round(avg_conf, 4),
|
| 61 |
+
language_distribution=dict(lang_counts),
|
| 62 |
+
)
|
backend/app/services/export.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Export service: CSV, JSON, PDF report generation."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import csv
|
| 6 |
+
import io
|
| 7 |
+
import json
|
| 8 |
+
from typing import List
|
| 9 |
+
|
| 10 |
+
from app.core.logging import get_logger
|
| 11 |
+
from app.models.schemas import AnalyzedEntry, ExportFormat
|
| 12 |
+
|
| 13 |
+
logger = get_logger(__name__)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def export_csv(entries: list[AnalyzedEntry]) -> bytes:
|
| 17 |
+
output = io.StringIO()
|
| 18 |
+
writer = csv.writer(output)
|
| 19 |
+
writer.writerow([
|
| 20 |
+
"id", "text", "source", "timestamp", "sentiment_label",
|
| 21 |
+
"sentiment_score", "confidence", "language", "topic_id", "topic_label",
|
| 22 |
+
])
|
| 23 |
+
for e in entries:
|
| 24 |
+
writer.writerow([
|
| 25 |
+
e.id, e.text, e.source or "", e.timestamp or "",
|
| 26 |
+
e.sentiment.label.value, e.sentiment.score, e.sentiment.confidence,
|
| 27 |
+
e.language.language, e.topic_id, e.topic_label,
|
| 28 |
+
])
|
| 29 |
+
return output.getvalue().encode("utf-8")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def export_json(entries: list[AnalyzedEntry]) -> bytes:
|
| 33 |
+
data = [
|
| 34 |
+
{
|
| 35 |
+
"id": e.id,
|
| 36 |
+
"text": e.text,
|
| 37 |
+
"source": e.source,
|
| 38 |
+
"timestamp": e.timestamp.isoformat() if e.timestamp else None,
|
| 39 |
+
"sentiment": {
|
| 40 |
+
"label": e.sentiment.label.value,
|
| 41 |
+
"score": e.sentiment.score,
|
| 42 |
+
"confidence": e.sentiment.confidence,
|
| 43 |
+
},
|
| 44 |
+
"language": {
|
| 45 |
+
"language": e.language.language,
|
| 46 |
+
"confidence": e.language.confidence,
|
| 47 |
+
},
|
| 48 |
+
"topic_id": e.topic_id,
|
| 49 |
+
"topic_label": e.topic_label,
|
| 50 |
+
}
|
| 51 |
+
for e in entries
|
| 52 |
+
]
|
| 53 |
+
return json.dumps(data, indent=2, default=str).encode("utf-8")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def export_pdf(entries: list[AnalyzedEntry], summary: dict | None = None) -> bytes:
|
| 57 |
+
"""Generate a PDF report using reportlab."""
|
| 58 |
+
try:
|
| 59 |
+
from reportlab.lib import colors
|
| 60 |
+
from reportlab.lib.pagesizes import A4, letter
|
| 61 |
+
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
| 62 |
+
from reportlab.lib.units import inch
|
| 63 |
+
from reportlab.platypus import (
|
| 64 |
+
Paragraph,
|
| 65 |
+
SimpleDocTemplate,
|
| 66 |
+
Spacer,
|
| 67 |
+
Table,
|
| 68 |
+
TableStyle,
|
| 69 |
+
)
|
| 70 |
+
except ImportError:
|
| 71 |
+
logger.error("reportlab_not_installed")
|
| 72 |
+
raise ImportError(
|
| 73 |
+
"PDF export requires reportlab. Install it with: pip install reportlab"
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
buffer = io.BytesIO()
|
| 77 |
+
doc = SimpleDocTemplate(buffer, pagesize=A4)
|
| 78 |
+
styles = getSampleStyleSheet()
|
| 79 |
+
elements = []
|
| 80 |
+
|
| 81 |
+
# Title
|
| 82 |
+
title_style = ParagraphStyle("Title", parent=styles["Title"], fontSize=18)
|
| 83 |
+
elements.append(Paragraph("Topic Analysis Report", title_style))
|
| 84 |
+
elements.append(Spacer(1, 12))
|
| 85 |
+
|
| 86 |
+
# Summary
|
| 87 |
+
if summary:
|
| 88 |
+
elements.append(Paragraph("Summary", styles["Heading2"]))
|
| 89 |
+
for key, val in summary.items():
|
| 90 |
+
elements.append(Paragraph(f"<b>{key}:</b> {val}", styles["Normal"]))
|
| 91 |
+
elements.append(Spacer(1, 12))
|
| 92 |
+
|
| 93 |
+
# Data table
|
| 94 |
+
elements.append(Paragraph("Analysis Results", styles["Heading2"]))
|
| 95 |
+
table_data = [["ID", "Sentiment", "Score", "Language", "Topic"]]
|
| 96 |
+
|
| 97 |
+
for e in entries[:500]: # Limit for PDF
|
| 98 |
+
table_data.append([
|
| 99 |
+
e.id[:8],
|
| 100 |
+
e.sentiment.label.value,
|
| 101 |
+
f"{e.sentiment.score:.2f}",
|
| 102 |
+
e.language.language,
|
| 103 |
+
e.topic_label[:30],
|
| 104 |
+
])
|
| 105 |
+
|
| 106 |
+
table = Table(table_data, colWidths=[60, 70, 50, 60, 180])
|
| 107 |
+
table.setStyle(TableStyle([
|
| 108 |
+
("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#1a1a2e")),
|
| 109 |
+
("TEXTCOLOR", (0, 0), (-1, 0), colors.white),
|
| 110 |
+
("ALIGN", (0, 0), (-1, -1), "LEFT"),
|
| 111 |
+
("FONTSIZE", (0, 0), (-1, 0), 10),
|
| 112 |
+
("FONTSIZE", (0, 1), (-1, -1), 8),
|
| 113 |
+
("BOTTOMPADDING", (0, 0), (-1, 0), 8),
|
| 114 |
+
("GRID", (0, 0), (-1, -1), 0.5, colors.grey),
|
| 115 |
+
("ROWBACKGROUNDS", (0, 1), (-1, -1), [colors.white, colors.HexColor("#f5f5f5")]),
|
| 116 |
+
]))
|
| 117 |
+
|
| 118 |
+
elements.append(table)
|
| 119 |
+
doc.build(elements)
|
| 120 |
+
return buffer.getvalue()
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def export_entries(entries: list[AnalyzedEntry], fmt: ExportFormat, summary: dict | None = None) -> bytes:
|
| 124 |
+
if fmt == ExportFormat.CSV:
|
| 125 |
+
return export_csv(entries)
|
| 126 |
+
elif fmt == ExportFormat.JSON:
|
| 127 |
+
return export_json(entries)
|
| 128 |
+
elif fmt == ExportFormat.PDF:
|
| 129 |
+
return export_pdf(entries, summary)
|
| 130 |
+
else:
|
| 131 |
+
raise ValueError(f"Unsupported export format: {fmt}")
|
backend/app/services/file_processing.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""File processing service: handles CSV, JSON, Excel, ZIP with chunked uploads."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import io
|
| 6 |
+
import json
|
| 7 |
+
import uuid
|
| 8 |
+
import zipfile
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import List
|
| 11 |
+
|
| 12 |
+
import pandas as pd
|
| 13 |
+
|
| 14 |
+
from app.core.config import settings
|
| 15 |
+
from app.core.logging import get_logger
|
| 16 |
+
from app.models.schemas import FeedbackEntry
|
| 17 |
+
|
| 18 |
+
logger = get_logger(__name__)
|
| 19 |
+
|
| 20 |
+
SUPPORTED_EXTENSIONS = {".csv", ".json", ".xlsx", ".xls", ".zip"}
|
| 21 |
+
TEXT_COLUMN_CANDIDATES = [
|
| 22 |
+
"text", "content", "message", "body", "feedback", "review",
|
| 23 |
+
"comment", "description", "note", "summary", "title",
|
| 24 |
+
"Text", "Content", "Message", "Body", "Feedback", "Review",
|
| 25 |
+
]
|
| 26 |
+
TIMESTAMP_COLUMN_CANDIDATES = [
|
| 27 |
+
"timestamp", "date", "created_at", "created", "time", "datetime",
|
| 28 |
+
"Timestamp", "Date", "Created", "CreatedAt",
|
| 29 |
+
]
|
| 30 |
+
SOURCE_COLUMN_CANDIDATES = [
|
| 31 |
+
"source", "channel", "platform", "origin", "category", "type",
|
| 32 |
+
"Source", "Channel", "Platform",
|
| 33 |
+
]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _find_column(df: pd.DataFrame, candidates: list[str]) -> str | None:
|
| 37 |
+
for col in candidates:
|
| 38 |
+
if col in df.columns:
|
| 39 |
+
return col
|
| 40 |
+
for col in df.columns:
|
| 41 |
+
for candidate in candidates:
|
| 42 |
+
if candidate.lower() in col.lower():
|
| 43 |
+
return col
|
| 44 |
+
return None
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _df_to_entries(df: pd.DataFrame, source: str | None = None) -> list[FeedbackEntry]:
|
| 48 |
+
text_col = _find_column(df, TEXT_COLUMN_CANDIDATES)
|
| 49 |
+
if not text_col:
|
| 50 |
+
if len(df.columns) == 1:
|
| 51 |
+
text_col = df.columns[0]
|
| 52 |
+
else:
|
| 53 |
+
raise ValueError(
|
| 54 |
+
f"No text column found. Expected one of: {TEXT_COLUMN_CANDIDATES}. "
|
| 55 |
+
f"Found columns: {list(df.columns)}"
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
ts_col = _find_column(df, TIMESTAMP_COLUMN_CANDIDATES)
|
| 59 |
+
src_col = _find_column(df, SOURCE_COLUMN_CANDIDATES)
|
| 60 |
+
|
| 61 |
+
entries = []
|
| 62 |
+
other_cols = [c for c in df.columns if c not in {text_col, ts_col, src_col}]
|
| 63 |
+
|
| 64 |
+
for _, row in df.iterrows():
|
| 65 |
+
text = str(row[text_col]).strip()
|
| 66 |
+
if not text or text == "nan":
|
| 67 |
+
continue
|
| 68 |
+
|
| 69 |
+
ts = None
|
| 70 |
+
if ts_col and pd.notna(row.get(ts_col)):
|
| 71 |
+
try:
|
| 72 |
+
ts = pd.to_datetime(row[ts_col])
|
| 73 |
+
except Exception:
|
| 74 |
+
pass
|
| 75 |
+
|
| 76 |
+
src = source
|
| 77 |
+
if src_col and pd.notna(row.get(src_col)):
|
| 78 |
+
src = str(row[src_col])
|
| 79 |
+
|
| 80 |
+
metadata = {}
|
| 81 |
+
for col in other_cols:
|
| 82 |
+
val = row.get(col)
|
| 83 |
+
if pd.notna(val):
|
| 84 |
+
metadata[col] = str(val) if not isinstance(val, (int, float, bool)) else val
|
| 85 |
+
|
| 86 |
+
entries.append(
|
| 87 |
+
FeedbackEntry(
|
| 88 |
+
id=uuid.uuid4().hex[:12],
|
| 89 |
+
text=text,
|
| 90 |
+
source=src,
|
| 91 |
+
timestamp=ts,
|
| 92 |
+
metadata=metadata if metadata else None,
|
| 93 |
+
)
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
return entries
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def parse_csv(content: bytes, source: str | None = None) -> list[FeedbackEntry]:
|
| 100 |
+
for encoding in ("utf-8", "latin-1", "cp1252"):
|
| 101 |
+
try:
|
| 102 |
+
df = pd.read_csv(io.BytesIO(content), encoding=encoding)
|
| 103 |
+
return _df_to_entries(df, source)
|
| 104 |
+
except UnicodeDecodeError:
|
| 105 |
+
continue
|
| 106 |
+
raise ValueError("Unable to decode CSV file with supported encodings")
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def parse_json(content: bytes, source: str | None = None) -> list[FeedbackEntry]:
|
| 110 |
+
data = json.loads(content.decode("utf-8"))
|
| 111 |
+
|
| 112 |
+
if isinstance(data, list):
|
| 113 |
+
if all(isinstance(item, str) for item in data):
|
| 114 |
+
return [
|
| 115 |
+
FeedbackEntry(id=uuid.uuid4().hex[:12], text=item, source=source)
|
| 116 |
+
for item in data
|
| 117 |
+
if item.strip()
|
| 118 |
+
]
|
| 119 |
+
df = pd.DataFrame(data)
|
| 120 |
+
return _df_to_entries(df, source)
|
| 121 |
+
elif isinstance(data, dict):
|
| 122 |
+
if "data" in data:
|
| 123 |
+
df = pd.DataFrame(data["data"])
|
| 124 |
+
elif "entries" in data:
|
| 125 |
+
df = pd.DataFrame(data["entries"])
|
| 126 |
+
elif "results" in data:
|
| 127 |
+
df = pd.DataFrame(data["results"])
|
| 128 |
+
else:
|
| 129 |
+
df = pd.DataFrame([data])
|
| 130 |
+
return _df_to_entries(df, source)
|
| 131 |
+
|
| 132 |
+
raise ValueError("Unsupported JSON structure")
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def parse_excel(content: bytes, source: str | None = None) -> list[FeedbackEntry]:
|
| 136 |
+
df = pd.read_excel(io.BytesIO(content), engine="openpyxl")
|
| 137 |
+
return _df_to_entries(df, source)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def parse_zip(content: bytes, source: str | None = None) -> list[FeedbackEntry]:
|
| 141 |
+
all_entries = []
|
| 142 |
+
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
| 143 |
+
for name in zf.namelist():
|
| 144 |
+
if name.startswith("__MACOSX") or name.startswith("."):
|
| 145 |
+
continue
|
| 146 |
+
ext = Path(name).suffix.lower()
|
| 147 |
+
inner = zf.read(name)
|
| 148 |
+
file_source = source or Path(name).stem
|
| 149 |
+
try:
|
| 150 |
+
if ext == ".csv":
|
| 151 |
+
all_entries.extend(parse_csv(inner, file_source))
|
| 152 |
+
elif ext == ".json":
|
| 153 |
+
all_entries.extend(parse_json(inner, file_source))
|
| 154 |
+
elif ext in (".xlsx", ".xls"):
|
| 155 |
+
all_entries.extend(parse_excel(inner, file_source))
|
| 156 |
+
else:
|
| 157 |
+
logger.warning("skipping_unsupported_file_in_zip", filename=name)
|
| 158 |
+
except Exception as exc:
|
| 159 |
+
logger.error("error_processing_zip_entry", filename=name, error=str(exc))
|
| 160 |
+
return all_entries
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def parse_file(content: bytes, filename: str, source: str | None = None) -> list[FeedbackEntry]:
|
| 164 |
+
ext = Path(filename).suffix.lower()
|
| 165 |
+
if ext not in SUPPORTED_EXTENSIONS:
|
| 166 |
+
raise ValueError(f"Unsupported file format: {ext}. Supported: {SUPPORTED_EXTENSIONS}")
|
| 167 |
+
|
| 168 |
+
parsers = {
|
| 169 |
+
".csv": parse_csv,
|
| 170 |
+
".json": parse_json,
|
| 171 |
+
".xlsx": parse_excel,
|
| 172 |
+
".xls": parse_excel,
|
| 173 |
+
".zip": parse_zip,
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
return parsers[ext](content, source)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
async def save_upload(content: bytes, filename: str) -> Path:
|
| 180 |
+
upload_dir = settings.upload_path
|
| 181 |
+
safe_name = f"{uuid.uuid4().hex[:8]}_{Path(filename).name}"
|
| 182 |
+
file_path = upload_dir / safe_name
|
| 183 |
+
file_path.write_bytes(content)
|
| 184 |
+
return file_path
|
backend/app/services/language_detection.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Language detection with langdetect primary and cld3 fallback."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from app.core.logging import get_logger
|
| 6 |
+
from app.models.schemas import LanguageResult
|
| 7 |
+
|
| 8 |
+
logger = get_logger(__name__)
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def detect_language(text: str) -> LanguageResult:
|
| 12 |
+
"""Detect language using langdetect with cld3 fallback."""
|
| 13 |
+
if not text or len(text.strip()) < 3:
|
| 14 |
+
return LanguageResult(language="unknown", confidence=0.0, method="none")
|
| 15 |
+
|
| 16 |
+
# Primary: langdetect
|
| 17 |
+
try:
|
| 18 |
+
from langdetect import DetectorFactory, detect_langs
|
| 19 |
+
|
| 20 |
+
DetectorFactory.seed = 42
|
| 21 |
+
results = detect_langs(text)
|
| 22 |
+
if results:
|
| 23 |
+
top = results[0]
|
| 24 |
+
return LanguageResult(
|
| 25 |
+
language=str(top.lang),
|
| 26 |
+
confidence=round(top.prob, 4),
|
| 27 |
+
method="langdetect",
|
| 28 |
+
)
|
| 29 |
+
except Exception as exc:
|
| 30 |
+
logger.debug("langdetect_failed", error=str(exc))
|
| 31 |
+
|
| 32 |
+
# Fallback: cld3
|
| 33 |
+
try:
|
| 34 |
+
import cld3
|
| 35 |
+
|
| 36 |
+
result = cld3.get_language(text)
|
| 37 |
+
if result and result.is_reliable:
|
| 38 |
+
return LanguageResult(
|
| 39 |
+
language=result.language,
|
| 40 |
+
confidence=round(result.probability, 4),
|
| 41 |
+
method="cld3",
|
| 42 |
+
)
|
| 43 |
+
elif result:
|
| 44 |
+
return LanguageResult(
|
| 45 |
+
language=result.language,
|
| 46 |
+
confidence=round(result.probability, 4),
|
| 47 |
+
method="cld3",
|
| 48 |
+
)
|
| 49 |
+
except ImportError:
|
| 50 |
+
logger.warning("cld3_not_available", detail="Install pycld3 for fallback detection")
|
| 51 |
+
except Exception as exc:
|
| 52 |
+
logger.debug("cld3_failed", error=str(exc))
|
| 53 |
+
|
| 54 |
+
return LanguageResult(language="unknown", confidence=0.0, method="none")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def detect_languages_batch(texts: list[str]) -> list[LanguageResult]:
|
| 58 |
+
return [detect_language(t) for t in texts]
|
backend/app/services/notifications.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Notification service for anomaly alerts (email + Slack webhook)."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import smtplib
|
| 6 |
+
from email.mime.text import MIMEText
|
| 7 |
+
from typing import List
|
| 8 |
+
|
| 9 |
+
import httpx
|
| 10 |
+
|
| 11 |
+
from app.core.config import settings
|
| 12 |
+
from app.core.logging import get_logger
|
| 13 |
+
from app.models.schemas import AnomalyAlert
|
| 14 |
+
|
| 15 |
+
logger = get_logger(__name__)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
async def send_slack_notification(alerts: list[AnomalyAlert]) -> bool:
|
| 19 |
+
if not settings.slack_webhook_url:
|
| 20 |
+
logger.debug("slack_webhook_not_configured")
|
| 21 |
+
return False
|
| 22 |
+
|
| 23 |
+
blocks = []
|
| 24 |
+
for alert in alerts[:10]:
|
| 25 |
+
emoji = "🔴" if alert.severity == "high" else "🟡"
|
| 26 |
+
blocks.append(
|
| 27 |
+
{
|
| 28 |
+
"type": "section",
|
| 29 |
+
"text": {
|
| 30 |
+
"type": "mrkdwn",
|
| 31 |
+
"text": f"{emoji} *{alert.type.value}* ({alert.severity})\n{alert.message}",
|
| 32 |
+
},
|
| 33 |
+
}
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
payload = {
|
| 37 |
+
"text": f"🚨 {len(alerts)} anomaly alert(s) detected",
|
| 38 |
+
"blocks": [
|
| 39 |
+
{
|
| 40 |
+
"type": "header",
|
| 41 |
+
"text": {"type": "plain_text", "text": f"🚨 {len(alerts)} Anomaly Alert(s)"},
|
| 42 |
+
},
|
| 43 |
+
*blocks,
|
| 44 |
+
],
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
try:
|
| 48 |
+
async with httpx.AsyncClient(timeout=10) as client:
|
| 49 |
+
resp = await client.post(settings.slack_webhook_url, json=payload)
|
| 50 |
+
resp.raise_for_status()
|
| 51 |
+
logger.info("slack_notification_sent", alert_count=len(alerts))
|
| 52 |
+
return True
|
| 53 |
+
except Exception as exc:
|
| 54 |
+
logger.error("slack_notification_failed", error=str(exc))
|
| 55 |
+
return False
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
async def send_email_notification(alerts: list[AnomalyAlert]) -> bool:
|
| 59 |
+
if not all([settings.smtp_host, settings.notification_email_from, settings.notification_email_to]):
|
| 60 |
+
logger.debug("email_notification_not_configured")
|
| 61 |
+
return False
|
| 62 |
+
|
| 63 |
+
body_lines = []
|
| 64 |
+
for alert in alerts:
|
| 65 |
+
body_lines.append(f"[{alert.severity.upper()}] {alert.type.value}: {alert.message}")
|
| 66 |
+
body_lines.append(f" Detected at: {alert.detected_at.isoformat()}")
|
| 67 |
+
body_lines.append("")
|
| 68 |
+
|
| 69 |
+
msg = MIMEText("\n".join(body_lines))
|
| 70 |
+
msg["Subject"] = f"Topic Analysis: {len(alerts)} anomaly alert(s)"
|
| 71 |
+
msg["From"] = settings.notification_email_from
|
| 72 |
+
msg["To"] = settings.notification_email_to
|
| 73 |
+
|
| 74 |
+
try:
|
| 75 |
+
with smtplib.SMTP(settings.smtp_host, settings.smtp_port) as server:
|
| 76 |
+
server.starttls()
|
| 77 |
+
if settings.smtp_user:
|
| 78 |
+
server.login(settings.smtp_user, settings.smtp_password)
|
| 79 |
+
server.send_message(msg)
|
| 80 |
+
logger.info("email_notification_sent", alert_count=len(alerts))
|
| 81 |
+
return True
|
| 82 |
+
except Exception as exc:
|
| 83 |
+
logger.error("email_notification_failed", error=str(exc))
|
| 84 |
+
return False
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
async def notify_anomalies(alerts: list[AnomalyAlert]) -> None:
|
| 88 |
+
if not alerts:
|
| 89 |
+
return
|
| 90 |
+
|
| 91 |
+
await send_slack_notification(alerts)
|
| 92 |
+
await send_email_notification(alerts)
|
backend/app/services/redis_client.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Redis client for caching and SSE broadcast."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from typing import Any, AsyncIterator, Optional
|
| 7 |
+
|
| 8 |
+
import redis.asyncio as aioredis
|
| 9 |
+
|
| 10 |
+
from app.core.config import settings
|
| 11 |
+
from app.core.logging import get_logger
|
| 12 |
+
|
| 13 |
+
logger = get_logger(__name__)
|
| 14 |
+
|
| 15 |
+
_redis: Optional[aioredis.Redis] = None
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
async def get_redis() -> aioredis.Redis:
|
| 19 |
+
global _redis
|
| 20 |
+
if _redis is None:
|
| 21 |
+
_redis = aioredis.from_url(
|
| 22 |
+
settings.redis_url,
|
| 23 |
+
decode_responses=True,
|
| 24 |
+
max_connections=20,
|
| 25 |
+
)
|
| 26 |
+
return _redis
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
async def close_redis() -> None:
|
| 30 |
+
global _redis
|
| 31 |
+
if _redis:
|
| 32 |
+
await _redis.aclose()
|
| 33 |
+
_redis = None
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
async def cache_get(key: str) -> Optional[Any]:
|
| 37 |
+
try:
|
| 38 |
+
r = await get_redis()
|
| 39 |
+
val = await r.get(f"cache:{key}")
|
| 40 |
+
return json.loads(val) if val else None
|
| 41 |
+
except Exception as exc:
|
| 42 |
+
logger.warning("redis_cache_get_failed", key=key, error=str(exc))
|
| 43 |
+
return None
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
async def cache_set(key: str, value: Any, ttl: int = 300) -> None:
|
| 47 |
+
try:
|
| 48 |
+
r = await get_redis()
|
| 49 |
+
await r.setex(f"cache:{key}", ttl, json.dumps(value, default=str))
|
| 50 |
+
except Exception as exc:
|
| 51 |
+
logger.warning("redis_cache_set_failed", key=key, error=str(exc))
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
async def publish_event(channel: str, data: dict) -> None:
|
| 55 |
+
try:
|
| 56 |
+
r = await get_redis()
|
| 57 |
+
await r.publish(channel, json.dumps(data, default=str))
|
| 58 |
+
except Exception as exc:
|
| 59 |
+
logger.warning("redis_publish_failed", channel=channel, error=str(exc))
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
async def subscribe_events(channel: str) -> AsyncIterator[dict]:
|
| 63 |
+
r = await get_redis()
|
| 64 |
+
pubsub = r.pubsub()
|
| 65 |
+
await pubsub.subscribe(channel)
|
| 66 |
+
try:
|
| 67 |
+
async for message in pubsub.listen():
|
| 68 |
+
if message["type"] == "message":
|
| 69 |
+
try:
|
| 70 |
+
yield json.loads(message["data"])
|
| 71 |
+
except json.JSONDecodeError:
|
| 72 |
+
continue
|
| 73 |
+
finally:
|
| 74 |
+
await pubsub.unsubscribe(channel)
|
| 75 |
+
await pubsub.aclose()
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
async def check_redis_health() -> bool:
|
| 79 |
+
try:
|
| 80 |
+
r = await get_redis()
|
| 81 |
+
return await r.ping()
|
| 82 |
+
except Exception:
|
| 83 |
+
return False
|
backend/app/services/sentiment.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sentiment analysis using cardiffnlp/twitter-xlm-roberta-base-sentiment."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import asyncio
|
| 6 |
+
import time
|
| 7 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 8 |
+
from typing import List, Optional
|
| 9 |
+
|
| 10 |
+
from app.core.config import settings
|
| 11 |
+
from app.core.logging import get_logger
|
| 12 |
+
from app.models.schemas import SentimentLabel, SentimentResult
|
| 13 |
+
|
| 14 |
+
logger = get_logger(__name__)
|
| 15 |
+
|
| 16 |
+
_model = None
|
| 17 |
+
_tokenizer = None
|
| 18 |
+
_executor = ThreadPoolExecutor(max_workers=2)
|
| 19 |
+
|
| 20 |
+
LABEL_MAP = {
|
| 21 |
+
"negative": SentimentLabel.NEGATIVE,
|
| 22 |
+
"neutral": SentimentLabel.NEUTRAL,
|
| 23 |
+
"positive": SentimentLabel.POSITIVE,
|
| 24 |
+
"LABEL_0": SentimentLabel.NEGATIVE,
|
| 25 |
+
"LABEL_1": SentimentLabel.NEUTRAL,
|
| 26 |
+
"LABEL_2": SentimentLabel.POSITIVE,
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _load_model():
|
| 31 |
+
global _model, _tokenizer
|
| 32 |
+
if _model is not None:
|
| 33 |
+
return
|
| 34 |
+
|
| 35 |
+
try:
|
| 36 |
+
from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
| 37 |
+
|
| 38 |
+
model_name = settings.sentiment_model
|
| 39 |
+
logger.info("loading_sentiment_model", model=model_name)
|
| 40 |
+
t0 = time.time()
|
| 41 |
+
|
| 42 |
+
_tokenizer = AutoTokenizer.from_pretrained(
|
| 43 |
+
model_name,
|
| 44 |
+
cache_dir=settings.model_cache_dir,
|
| 45 |
+
)
|
| 46 |
+
logger.info("tokenizer_loaded", model=model_name, elapsed=round(time.time() - t0, 2))
|
| 47 |
+
|
| 48 |
+
_model = AutoModelForSequenceClassification.from_pretrained(
|
| 49 |
+
model_name,
|
| 50 |
+
cache_dir=settings.model_cache_dir,
|
| 51 |
+
)
|
| 52 |
+
_model.eval()
|
| 53 |
+
label_config = getattr(_model.config, "id2label", {})
|
| 54 |
+
logger.info(
|
| 55 |
+
"sentiment_model_loaded",
|
| 56 |
+
model=model_name,
|
| 57 |
+
elapsed=round(time.time() - t0, 2),
|
| 58 |
+
model_labels=str(label_config),
|
| 59 |
+
num_labels=getattr(_model.config, "num_labels", "unknown"),
|
| 60 |
+
)
|
| 61 |
+
except Exception as exc:
|
| 62 |
+
logger.error("sentiment_model_load_failed", error=str(exc), exc_type=type(exc).__name__)
|
| 63 |
+
raise
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _predict_batch_sync(texts: list[str]) -> list[SentimentResult]:
|
| 67 |
+
import torch
|
| 68 |
+
from scipy.special import softmax
|
| 69 |
+
|
| 70 |
+
_load_model()
|
| 71 |
+
|
| 72 |
+
results = []
|
| 73 |
+
batch_size = 32
|
| 74 |
+
t0 = time.time()
|
| 75 |
+
|
| 76 |
+
for i in range(0, len(texts), batch_size):
|
| 77 |
+
batch = texts[i : i + batch_size]
|
| 78 |
+
truncated = [t[:512] for t in batch]
|
| 79 |
+
|
| 80 |
+
inputs = _tokenizer(
|
| 81 |
+
truncated,
|
| 82 |
+
padding=True,
|
| 83 |
+
truncation=True,
|
| 84 |
+
max_length=512,
|
| 85 |
+
return_tensors="pt",
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
with torch.no_grad():
|
| 89 |
+
outputs = _model(**inputs)
|
| 90 |
+
|
| 91 |
+
scores = outputs.logits.detach().numpy()
|
| 92 |
+
|
| 93 |
+
for j, score_row in enumerate(scores):
|
| 94 |
+
probs = softmax(score_row)
|
| 95 |
+
label_idx = int(probs.argmax())
|
| 96 |
+
# Use model's own id2label mapping (0=negative, 1=neutral, 2=positive)
|
| 97 |
+
id2label = {0: SentimentLabel.NEGATIVE, 1: SentimentLabel.NEUTRAL, 2: SentimentLabel.POSITIVE}
|
| 98 |
+
label = id2label.get(label_idx, SentimentLabel.NEUTRAL)
|
| 99 |
+
confidence = float(probs[label_idx])
|
| 100 |
+
|
| 101 |
+
# Sentiment score: -1 (negative) to +1 (positive)
|
| 102 |
+
sentiment_score = float(probs[2] - probs[0])
|
| 103 |
+
|
| 104 |
+
results.append(
|
| 105 |
+
SentimentResult(
|
| 106 |
+
label=label,
|
| 107 |
+
score=round(max(0, min(1, (sentiment_score + 1) / 2)), 4),
|
| 108 |
+
confidence=round(confidence, 4),
|
| 109 |
+
)
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
# Log first batch for debugging
|
| 113 |
+
if i == 0 and len(results) > 0:
|
| 114 |
+
sample = results[0]
|
| 115 |
+
logger.info(
|
| 116 |
+
"sentiment_first_batch_sample",
|
| 117 |
+
text_preview=truncated[0][:80],
|
| 118 |
+
label=sample.label.value,
|
| 119 |
+
score=sample.score,
|
| 120 |
+
confidence=sample.confidence,
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
elapsed = round(time.time() - t0, 2)
|
| 124 |
+
logger.info(
|
| 125 |
+
"sentiment_batch_complete",
|
| 126 |
+
total_texts=len(texts),
|
| 127 |
+
elapsed_seconds=elapsed,
|
| 128 |
+
texts_per_second=round(len(texts) / max(elapsed, 0.001), 1),
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
return results
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
async def analyze_sentiment(texts: list[str]) -> list[SentimentResult]:
|
| 135 |
+
"""Analyze sentiment for a batch of texts asynchronously."""
|
| 136 |
+
logger.info("analyze_sentiment_called", count=len(texts), using="ml_model")
|
| 137 |
+
loop = asyncio.get_event_loop()
|
| 138 |
+
return await loop.run_in_executor(_executor, _predict_batch_sync, texts)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def analyze_sentiment_sync(texts: list[str]) -> list[SentimentResult]:
|
| 142 |
+
"""Synchronous sentiment analysis."""
|
| 143 |
+
logger.info("analyze_sentiment_sync_called", count=len(texts))
|
| 144 |
+
return _predict_batch_sync(texts)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
_models_available: Optional[bool] = None
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def is_model_available() -> bool:
|
| 151 |
+
"""Check if ML model is available. Re-checks on each call until successful."""
|
| 152 |
+
global _models_available
|
| 153 |
+
if _models_available is True:
|
| 154 |
+
return True
|
| 155 |
+
# Always retry if previously failed — deps may have been installed since last check
|
| 156 |
+
try:
|
| 157 |
+
_load_model()
|
| 158 |
+
_models_available = True
|
| 159 |
+
logger.info("model_availability_check", available=True)
|
| 160 |
+
except Exception as exc:
|
| 161 |
+
_models_available = False
|
| 162 |
+
logger.warning("model_availability_check", available=False, error=str(exc))
|
| 163 |
+
return _models_available
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def get_fallback_sentiment(text: str) -> SentimentResult:
|
| 167 |
+
"""Simple keyword-based fallback when ML model unavailable."""
|
| 168 |
+
logger.debug("using_fallback_sentiment", text_preview=text[:60])
|
| 169 |
+
text_lower = text.lower()
|
| 170 |
+
positive_words = {"good", "great", "excellent", "love", "amazing", "happy", "best", "wonderful", "fantastic"}
|
| 171 |
+
negative_words = {"bad", "terrible", "awful", "hate", "worst", "horrible", "poor", "disappointing", "angry"}
|
| 172 |
+
|
| 173 |
+
pos = sum(1 for w in text_lower.split() if w in positive_words)
|
| 174 |
+
neg = sum(1 for w in text_lower.split() if w in negative_words)
|
| 175 |
+
|
| 176 |
+
if pos > neg:
|
| 177 |
+
return SentimentResult(label=SentimentLabel.POSITIVE, score=0.7, confidence=0.3)
|
| 178 |
+
elif neg > pos:
|
| 179 |
+
return SentimentResult(label=SentimentLabel.NEGATIVE, score=0.3, confidence=0.3)
|
| 180 |
+
return SentimentResult(label=SentimentLabel.NEUTRAL, score=0.5, confidence=0.3)
|
backend/app/services/topic_clustering.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Topic clustering using BERTopic with HDBSCAN + UMAP."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import asyncio
|
| 6 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 7 |
+
from typing import List, Optional, Tuple
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
from app.core.config import settings
|
| 12 |
+
from app.core.logging import get_logger
|
| 13 |
+
from app.models.schemas import TopicCluster, TopicGraph, TopicInfo, TopicLink
|
| 14 |
+
|
| 15 |
+
logger = get_logger(__name__)
|
| 16 |
+
|
| 17 |
+
_embedding_model = None
|
| 18 |
+
_executor = ThreadPoolExecutor(max_workers=2)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _load_embedding_model():
|
| 22 |
+
global _embedding_model
|
| 23 |
+
if _embedding_model is not None:
|
| 24 |
+
return
|
| 25 |
+
|
| 26 |
+
try:
|
| 27 |
+
from sentence_transformers import SentenceTransformer
|
| 28 |
+
|
| 29 |
+
model_name = settings.embedding_model
|
| 30 |
+
logger.info("loading_embedding_model", model=model_name)
|
| 31 |
+
_embedding_model = SentenceTransformer(
|
| 32 |
+
model_name,
|
| 33 |
+
cache_folder=settings.model_cache_dir,
|
| 34 |
+
)
|
| 35 |
+
logger.info("embedding_model_loaded", model=model_name)
|
| 36 |
+
except Exception as exc:
|
| 37 |
+
logger.error("embedding_model_load_failed", error=str(exc))
|
| 38 |
+
raise
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _compute_embeddings(texts: list[str]) -> np.ndarray:
|
| 42 |
+
_load_embedding_model()
|
| 43 |
+
return _embedding_model.encode(
|
| 44 |
+
texts,
|
| 45 |
+
show_progress_bar=False,
|
| 46 |
+
batch_size=64,
|
| 47 |
+
normalize_embeddings=True,
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _adaptive_params(n_docs: int) -> dict:
|
| 52 |
+
"""Adapt HDBSCAN/UMAP parameters to data volume."""
|
| 53 |
+
if n_docs < 20:
|
| 54 |
+
return {"min_cluster_size": 2, "min_samples": 1, "n_neighbors": 3, "n_components": 2}
|
| 55 |
+
elif n_docs < 100:
|
| 56 |
+
return {"min_cluster_size": 3, "min_samples": 2, "n_neighbors": 5, "n_components": 3}
|
| 57 |
+
elif n_docs < 500:
|
| 58 |
+
return {"min_cluster_size": 5, "min_samples": 3, "n_neighbors": 10, "n_components": 5}
|
| 59 |
+
elif n_docs < 2000:
|
| 60 |
+
return {"min_cluster_size": 10, "min_samples": 5, "n_neighbors": 15, "n_components": 5}
|
| 61 |
+
else:
|
| 62 |
+
return {"min_cluster_size": 15, "min_samples": 8, "n_neighbors": 15, "n_components": 10}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _cluster_topics_sync(
|
| 66 |
+
texts: list[str],
|
| 67 |
+
embeddings: Optional[np.ndarray] = None,
|
| 68 |
+
min_cluster_size: Optional[int] = None,
|
| 69 |
+
min_samples: Optional[int] = None,
|
| 70 |
+
) -> Tuple[list[int], list[TopicCluster], Optional[np.ndarray]]:
|
| 71 |
+
from bertopic import BERTopic
|
| 72 |
+
from hdbscan import HDBSCAN
|
| 73 |
+
from sklearn.feature_extraction.text import CountVectorizer
|
| 74 |
+
from umap import UMAP
|
| 75 |
+
|
| 76 |
+
if embeddings is None:
|
| 77 |
+
embeddings = _compute_embeddings(texts)
|
| 78 |
+
|
| 79 |
+
params = _adaptive_params(len(texts))
|
| 80 |
+
mcs = min_cluster_size or params["min_cluster_size"]
|
| 81 |
+
ms = min_samples or params["min_samples"]
|
| 82 |
+
|
| 83 |
+
umap_model = UMAP(
|
| 84 |
+
n_neighbors=params["n_neighbors"],
|
| 85 |
+
n_components=params["n_components"],
|
| 86 |
+
min_dist=0.0,
|
| 87 |
+
metric="cosine",
|
| 88 |
+
random_state=42,
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
hdbscan_model = HDBSCAN(
|
| 92 |
+
min_cluster_size=mcs,
|
| 93 |
+
min_samples=ms,
|
| 94 |
+
metric="euclidean",
|
| 95 |
+
prediction_data=True,
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
vectorizer = CountVectorizer(
|
| 99 |
+
stop_words="english",
|
| 100 |
+
max_features=10000,
|
| 101 |
+
ngram_range=(1, 2),
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
topic_model = BERTopic(
|
| 105 |
+
umap_model=umap_model,
|
| 106 |
+
hdbscan_model=hdbscan_model,
|
| 107 |
+
vectorizer_model=vectorizer,
|
| 108 |
+
calculate_probabilities=True,
|
| 109 |
+
verbose=False,
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
topics, probs = topic_model.fit_transform(texts, embeddings)
|
| 113 |
+
|
| 114 |
+
topic_info = topic_model.get_topic_info()
|
| 115 |
+
clusters = []
|
| 116 |
+
|
| 117 |
+
for _, row in topic_info.iterrows():
|
| 118 |
+
tid = int(row["Topic"])
|
| 119 |
+
if tid == -1:
|
| 120 |
+
label = "Uncategorized"
|
| 121 |
+
keywords = []
|
| 122 |
+
else:
|
| 123 |
+
topic_words = topic_model.get_topic(tid)
|
| 124 |
+
keywords = [w for w, _ in topic_words[:10]] if topic_words else []
|
| 125 |
+
label = " | ".join(keywords[:3]) if keywords else f"Topic {tid}"
|
| 126 |
+
|
| 127 |
+
indices = [i for i, t in enumerate(topics) if t == tid]
|
| 128 |
+
rep_docs = [texts[i][:200] for i in indices[:3]]
|
| 129 |
+
|
| 130 |
+
clusters.append(
|
| 131 |
+
TopicCluster(
|
| 132 |
+
topic_id=tid,
|
| 133 |
+
label=label,
|
| 134 |
+
keywords=keywords,
|
| 135 |
+
size=int(row.get("Count", len(indices))),
|
| 136 |
+
avg_sentiment=0.0,
|
| 137 |
+
sentiment_distribution={"positive": 0, "negative": 0, "neutral": 0},
|
| 138 |
+
languages={},
|
| 139 |
+
representative_docs=rep_docs,
|
| 140 |
+
)
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
# Get 2D coordinates for visualization
|
| 144 |
+
reduced = None
|
| 145 |
+
if len(texts) > 2:
|
| 146 |
+
try:
|
| 147 |
+
umap_2d = UMAP(n_components=2, random_state=42, metric="cosine")
|
| 148 |
+
reduced = umap_2d.fit_transform(embeddings)
|
| 149 |
+
except Exception:
|
| 150 |
+
pass
|
| 151 |
+
|
| 152 |
+
return topics, clusters, reduced
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def build_topic_graph(clusters: list[TopicCluster], embeddings: np.ndarray, topics: list[int]) -> TopicGraph:
|
| 156 |
+
"""Build force-directed graph from topic clusters."""
|
| 157 |
+
from sklearn.metrics.pairwise import cosine_similarity
|
| 158 |
+
|
| 159 |
+
unique_topics = list({c.topic_id for c in clusters if c.topic_id != -1})
|
| 160 |
+
links = []
|
| 161 |
+
|
| 162 |
+
if len(unique_topics) > 1:
|
| 163 |
+
centroids = []
|
| 164 |
+
for tid in unique_topics:
|
| 165 |
+
indices = [i for i, t in enumerate(topics) if t == tid]
|
| 166 |
+
if indices:
|
| 167 |
+
centroid = embeddings[indices].mean(axis=0)
|
| 168 |
+
centroids.append(centroid)
|
| 169 |
+
else:
|
| 170 |
+
centroids.append(np.zeros(embeddings.shape[1]))
|
| 171 |
+
|
| 172 |
+
sim_matrix = cosine_similarity(np.array(centroids))
|
| 173 |
+
|
| 174 |
+
for i, t1 in enumerate(unique_topics):
|
| 175 |
+
for j, t2 in enumerate(unique_topics):
|
| 176 |
+
if i < j and sim_matrix[i][j] > 0.1:
|
| 177 |
+
links.append(
|
| 178 |
+
TopicLink(
|
| 179 |
+
source=t1,
|
| 180 |
+
target=t2,
|
| 181 |
+
weight=round(float(sim_matrix[i][j]), 4),
|
| 182 |
+
)
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
return TopicGraph(nodes=clusters, links=links)
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
async def cluster_topics(
|
| 189 |
+
texts: list[str],
|
| 190 |
+
embeddings: Optional[np.ndarray] = None,
|
| 191 |
+
min_cluster_size: Optional[int] = None,
|
| 192 |
+
min_samples: Optional[int] = None,
|
| 193 |
+
) -> Tuple[list[int], list[TopicCluster], Optional[np.ndarray]]:
|
| 194 |
+
loop = asyncio.get_event_loop()
|
| 195 |
+
return await loop.run_in_executor(
|
| 196 |
+
_executor, _cluster_topics_sync, texts, embeddings, min_cluster_size, min_samples
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
async def compute_embeddings(texts: list[str]) -> np.ndarray:
|
| 201 |
+
loop = asyncio.get_event_loop()
|
| 202 |
+
return await loop.run_in_executor(_executor, _compute_embeddings, texts)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
_embedding_available: Optional[bool] = None
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def is_embedding_model_available() -> bool:
|
| 209 |
+
"""Check if embedding model is available. Re-checks on each call until successful."""
|
| 210 |
+
global _embedding_available
|
| 211 |
+
if _embedding_available is True:
|
| 212 |
+
return True
|
| 213 |
+
try:
|
| 214 |
+
_load_embedding_model()
|
| 215 |
+
_embedding_available = True
|
| 216 |
+
logger.info("embedding_model_availability", available=True)
|
| 217 |
+
except Exception as exc:
|
| 218 |
+
_embedding_available = False
|
| 219 |
+
logger.warning("embedding_model_availability", available=False, error=str(exc))
|
| 220 |
+
return _embedding_available
|
backend/app/utils/__init__.py
ADDED
|
File without changes
|
backend/pyproject.toml
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[tool.ruff]
|
| 2 |
+
target-version = "py311"
|
| 3 |
+
line-length = 120
|
| 4 |
+
|
| 5 |
+
[tool.ruff.lint]
|
| 6 |
+
select = ["E", "F", "W", "I", "N", "UP", "B", "SIM"]
|
| 7 |
+
ignore = ["E501"]
|
| 8 |
+
|
| 9 |
+
[tool.pytest.ini_options]
|
| 10 |
+
testpaths = ["tests"]
|
| 11 |
+
asyncio_mode = "auto"
|
| 12 |
+
filterwarnings = ["ignore::DeprecationWarning"]
|
backend/requirements.txt
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Backend dependencies — pinned versions
|
| 2 |
+
fastapi==0.115.6
|
| 3 |
+
uvicorn[standard]==0.34.0
|
| 4 |
+
pydantic==2.10.4
|
| 5 |
+
pydantic-settings==2.7.1
|
| 6 |
+
python-multipart==0.0.20
|
| 7 |
+
aiofiles==24.1.0
|
| 8 |
+
httpx==0.28.1
|
| 9 |
+
redis[hiredis]==5.2.1
|
| 10 |
+
sse-starlette==2.2.1
|
| 11 |
+
|
| 12 |
+
# ML / NLP
|
| 13 |
+
torch==2.5.1
|
| 14 |
+
transformers==4.47.1
|
| 15 |
+
sentence-transformers==3.3.1
|
| 16 |
+
bertopic==0.16.4
|
| 17 |
+
hdbscan==0.8.40
|
| 18 |
+
umap-learn==0.5.7
|
| 19 |
+
scikit-learn==1.6.1
|
| 20 |
+
langdetect==1.0.9
|
| 21 |
+
pycld3==0.22
|
| 22 |
+
|
| 23 |
+
# Data handling
|
| 24 |
+
pandas==2.2.3
|
| 25 |
+
openpyxl==3.1.5
|
| 26 |
+
xlrd==2.0.1
|
| 27 |
+
numpy==1.26.4
|
| 28 |
+
|
| 29 |
+
# Export
|
| 30 |
+
reportlab==4.2.5
|
| 31 |
+
weasyprint==63.1
|
| 32 |
+
|
| 33 |
+
# Observability
|
| 34 |
+
opentelemetry-api==1.29.0
|
| 35 |
+
opentelemetry-sdk==1.29.0
|
| 36 |
+
opentelemetry-instrumentation-fastapi==0.50b0
|
| 37 |
+
opentelemetry-exporter-otlp==1.29.0
|
| 38 |
+
prometheus-client==0.21.1
|
| 39 |
+
prometheus-fastapi-instrumentator==7.0.2
|
| 40 |
+
structlog==24.4.0
|
| 41 |
+
|
| 42 |
+
# Security
|
| 43 |
+
python-jose[cryptography]==3.3.0
|
| 44 |
+
slowapi==0.1.9
|
| 45 |
+
|
| 46 |
+
# Testing
|
| 47 |
+
pytest==8.3.4
|
| 48 |
+
pytest-asyncio==0.25.0
|
| 49 |
+
pytest-cov==6.0.0
|
| 50 |
+
httpx==0.28.1
|
| 51 |
+
|
| 52 |
+
# Utilities
|
| 53 |
+
python-dotenv==1.0.1
|
| 54 |
+
tenacity==9.0.0
|
backend/tests/__init__.py
ADDED
|
File without changes
|
backend/tests/conftest.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pytest configuration and shared fixtures."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
from unittest.mock import AsyncMock, MagicMock, patch
|
| 7 |
+
|
| 8 |
+
import pytest
|
| 9 |
+
from fastapi.testclient import TestClient
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@pytest.fixture(autouse=True)
|
| 13 |
+
def mock_env():
|
| 14 |
+
os.environ["ALLOWED_API_KEYS"] = '["test-key"]'
|
| 15 |
+
os.environ["REDIS_URL"] = "redis://localhost:6379/0"
|
| 16 |
+
os.environ["APP_ENV"] = "testing"
|
| 17 |
+
os.environ["LOG_FORMAT"] = "console"
|
| 18 |
+
os.environ["CORS_ORIGINS"] = '["http://localhost:3000"]'
|
| 19 |
+
yield
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@pytest.fixture
|
| 23 |
+
def api_headers():
|
| 24 |
+
return {"X-API-Key": "test-key"}
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@pytest.fixture
|
| 28 |
+
def mock_redis():
|
| 29 |
+
with patch("app.services.redis_client.get_redis") as mock:
|
| 30 |
+
redis_mock = AsyncMock()
|
| 31 |
+
redis_mock.ping.return_value = True
|
| 32 |
+
redis_mock.get.return_value = None
|
| 33 |
+
redis_mock.setex.return_value = True
|
| 34 |
+
redis_mock.publish.return_value = 1
|
| 35 |
+
mock.return_value = redis_mock
|
| 36 |
+
yield redis_mock
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@pytest.fixture
|
| 40 |
+
def mock_sentiment():
|
| 41 |
+
with patch("app.services.sentiment._load_model"):
|
| 42 |
+
with patch("app.services.sentiment.is_model_available", return_value=False):
|
| 43 |
+
yield
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@pytest.fixture
|
| 47 |
+
def mock_embeddings():
|
| 48 |
+
with patch("app.services.topic_clustering._load_embedding_model"):
|
| 49 |
+
with patch("app.services.topic_clustering.is_embedding_model_available", return_value=False):
|
| 50 |
+
yield
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@pytest.fixture
|
| 54 |
+
def client(mock_redis, mock_sentiment, mock_embeddings):
|
| 55 |
+
from app.main import app
|
| 56 |
+
with TestClient(app) as c:
|
| 57 |
+
yield c
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@pytest.fixture
|
| 61 |
+
def sample_csv_content():
|
| 62 |
+
return b"text,source,timestamp\nGreat product!,survey,2024-01-01\nTerrible service,email,2024-01-02\nOkay experience,chat,2024-01-03\n"
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
@pytest.fixture
|
| 66 |
+
def sample_json_content():
|
| 67 |
+
import json
|
| 68 |
+
data = [
|
| 69 |
+
{"text": "Love this product!", "source": "app", "timestamp": "2024-01-01"},
|
| 70 |
+
{"text": "Not happy with the service", "source": "email", "timestamp": "2024-01-02"},
|
| 71 |
+
{"text": "It works fine", "source": "web", "timestamp": "2024-01-03"},
|
| 72 |
+
]
|
| 73 |
+
return json.dumps(data).encode("utf-8")
|
backend/tests/test_api.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for API endpoints."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import io
|
| 6 |
+
import json
|
| 7 |
+
from unittest.mock import AsyncMock, patch
|
| 8 |
+
|
| 9 |
+
import pytest
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class TestHealthEndpoints:
|
| 13 |
+
def test_health(self, client, api_headers):
|
| 14 |
+
resp = client.get("/health")
|
| 15 |
+
assert resp.status_code == 200
|
| 16 |
+
data = resp.json()
|
| 17 |
+
assert data["status"] in ("healthy", "degraded")
|
| 18 |
+
assert "version" in data
|
| 19 |
+
assert "uptime_seconds" in data
|
| 20 |
+
|
| 21 |
+
def test_liveness(self, client):
|
| 22 |
+
resp = client.get("/health/live")
|
| 23 |
+
assert resp.status_code == 200
|
| 24 |
+
assert resp.json()["status"] == "alive"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class TestUploadEndpoints:
|
| 28 |
+
def test_upload_csv(self, client, api_headers, sample_csv_content):
|
| 29 |
+
with patch("app.api.analysis.run_analysis", new_callable=AsyncMock) as mock_run:
|
| 30 |
+
mock_run.return_value = None
|
| 31 |
+
resp = client.post(
|
| 32 |
+
"/api/v1/upload",
|
| 33 |
+
files={"file": ("test.csv", io.BytesIO(sample_csv_content), "text/csv")},
|
| 34 |
+
headers=api_headers,
|
| 35 |
+
)
|
| 36 |
+
assert resp.status_code == 200
|
| 37 |
+
data = resp.json()
|
| 38 |
+
assert "job_id" in data
|
| 39 |
+
assert data["status"] == "pending"
|
| 40 |
+
|
| 41 |
+
def test_upload_json(self, client, api_headers, sample_json_content):
|
| 42 |
+
with patch("app.api.analysis.run_analysis", new_callable=AsyncMock):
|
| 43 |
+
resp = client.post(
|
| 44 |
+
"/api/v1/upload",
|
| 45 |
+
files={"file": ("test.json", io.BytesIO(sample_json_content), "application/json")},
|
| 46 |
+
headers=api_headers,
|
| 47 |
+
)
|
| 48 |
+
assert resp.status_code == 200
|
| 49 |
+
|
| 50 |
+
def test_upload_unsupported_format(self, client, api_headers):
|
| 51 |
+
resp = client.post(
|
| 52 |
+
"/api/v1/upload",
|
| 53 |
+
files={"file": ("test.txt", io.BytesIO(b"hello"), "text/plain")},
|
| 54 |
+
headers=api_headers,
|
| 55 |
+
)
|
| 56 |
+
assert resp.status_code == 400
|
| 57 |
+
assert "Unsupported" in resp.json()["detail"]
|
| 58 |
+
|
| 59 |
+
def test_upload_no_api_key(self, client, sample_csv_content):
|
| 60 |
+
resp = client.post(
|
| 61 |
+
"/api/v1/upload",
|
| 62 |
+
files={"file": ("test.csv", io.BytesIO(sample_csv_content), "text/csv")},
|
| 63 |
+
)
|
| 64 |
+
assert resp.status_code == 403
|
| 65 |
+
|
| 66 |
+
def test_upload_invalid_api_key(self, client, sample_csv_content):
|
| 67 |
+
resp = client.post(
|
| 68 |
+
"/api/v1/upload",
|
| 69 |
+
files={"file": ("test.csv", io.BytesIO(sample_csv_content), "text/csv")},
|
| 70 |
+
headers={"X-API-Key": "wrong-key"},
|
| 71 |
+
)
|
| 72 |
+
assert resp.status_code == 403
|
| 73 |
+
|
| 74 |
+
def test_upload_empty_file(self, client, api_headers):
|
| 75 |
+
resp = client.post(
|
| 76 |
+
"/api/v1/upload",
|
| 77 |
+
files={"file": ("test.csv", io.BytesIO(b"text\n"), "text/csv")},
|
| 78 |
+
headers=api_headers,
|
| 79 |
+
)
|
| 80 |
+
assert resp.status_code == 400
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class TestJobEndpoints:
|
| 84 |
+
def test_list_jobs(self, client, api_headers):
|
| 85 |
+
resp = client.get("/api/v1/jobs", headers=api_headers)
|
| 86 |
+
assert resp.status_code == 200
|
| 87 |
+
assert isinstance(resp.json(), list)
|
| 88 |
+
|
| 89 |
+
def test_get_nonexistent_job(self, client, api_headers):
|
| 90 |
+
resp = client.get("/api/v1/jobs/nonexistent", headers=api_headers)
|
| 91 |
+
assert resp.status_code == 404
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class TestWebhookEndpoints:
|
| 95 |
+
def test_webhook_invalid_signature(self, client):
|
| 96 |
+
payload = json.dumps({
|
| 97 |
+
"event_type": "feedback",
|
| 98 |
+
"data": [{"text": "test feedback"}],
|
| 99 |
+
})
|
| 100 |
+
resp = client.post(
|
| 101 |
+
"/api/v1/webhooks/ingest",
|
| 102 |
+
content=payload,
|
| 103 |
+
headers={
|
| 104 |
+
"Content-Type": "application/json",
|
| 105 |
+
"X-Signature": "v1=invalid",
|
| 106 |
+
"X-Timestamp": "0",
|
| 107 |
+
},
|
| 108 |
+
)
|
| 109 |
+
assert resp.status_code == 401
|
| 110 |
+
|
| 111 |
+
def test_webhook_missing_signature(self, client):
|
| 112 |
+
payload = json.dumps({"event_type": "feedback", "data": [{"text": "test"}]})
|
| 113 |
+
resp = client.post(
|
| 114 |
+
"/api/v1/webhooks/ingest",
|
| 115 |
+
content=payload,
|
| 116 |
+
headers={"Content-Type": "application/json"},
|
| 117 |
+
)
|
| 118 |
+
assert resp.status_code == 401
|
backend/tests/test_services.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for core services with mocked ML inference."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from unittest.mock import patch
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import pytest
|
| 10 |
+
|
| 11 |
+
from app.models.schemas import FeedbackEntry, SentimentLabel, SentimentResult
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class TestLanguageDetection:
|
| 15 |
+
def test_detect_english(self):
|
| 16 |
+
from app.services.language_detection import detect_language
|
| 17 |
+
result = detect_language("This is a test sentence in English")
|
| 18 |
+
assert result.language in ("en", "unknown")
|
| 19 |
+
assert result.confidence >= 0.0
|
| 20 |
+
|
| 21 |
+
def test_detect_empty_text(self):
|
| 22 |
+
from app.services.language_detection import detect_language
|
| 23 |
+
result = detect_language("")
|
| 24 |
+
assert result.language == "unknown"
|
| 25 |
+
assert result.confidence == 0.0
|
| 26 |
+
|
| 27 |
+
def test_detect_short_text(self):
|
| 28 |
+
from app.services.language_detection import detect_language
|
| 29 |
+
result = detect_language("hi")
|
| 30 |
+
assert result.language == "unknown"
|
| 31 |
+
|
| 32 |
+
def test_batch_detection(self):
|
| 33 |
+
from app.services.language_detection import detect_languages_batch
|
| 34 |
+
results = detect_languages_batch(["Hello world", "Bonjour le monde", ""])
|
| 35 |
+
assert len(results) == 3
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class TestSentiment:
|
| 39 |
+
def test_fallback_sentiment_positive(self):
|
| 40 |
+
from app.services.sentiment import get_fallback_sentiment
|
| 41 |
+
result = get_fallback_sentiment("This is great and amazing!")
|
| 42 |
+
assert result.label == SentimentLabel.POSITIVE
|
| 43 |
+
|
| 44 |
+
def test_fallback_sentiment_negative(self):
|
| 45 |
+
from app.services.sentiment import get_fallback_sentiment
|
| 46 |
+
result = get_fallback_sentiment("This is terrible and awful")
|
| 47 |
+
assert result.label == SentimentLabel.NEGATIVE
|
| 48 |
+
|
| 49 |
+
def test_fallback_sentiment_neutral(self):
|
| 50 |
+
from app.services.sentiment import get_fallback_sentiment
|
| 51 |
+
result = get_fallback_sentiment("The weather is cloudy today")
|
| 52 |
+
assert result.label == SentimentLabel.NEUTRAL
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class TestFileProcessing:
|
| 56 |
+
def test_parse_csv(self):
|
| 57 |
+
from app.services.file_processing import parse_csv
|
| 58 |
+
content = b"text,source\nHello world,test\nGoodbye world,test\n"
|
| 59 |
+
entries = parse_csv(content)
|
| 60 |
+
assert len(entries) == 2
|
| 61 |
+
assert entries[0].text == "Hello world"
|
| 62 |
+
|
| 63 |
+
def test_parse_json_array(self):
|
| 64 |
+
from app.services.file_processing import parse_json
|
| 65 |
+
data = [{"text": "entry 1"}, {"text": "entry 2"}]
|
| 66 |
+
entries = parse_json(json.dumps(data).encode())
|
| 67 |
+
assert len(entries) == 2
|
| 68 |
+
|
| 69 |
+
def test_parse_json_string_array(self):
|
| 70 |
+
from app.services.file_processing import parse_json
|
| 71 |
+
data = ["feedback one", "feedback two"]
|
| 72 |
+
entries = parse_json(json.dumps(data).encode())
|
| 73 |
+
assert len(entries) == 2
|
| 74 |
+
|
| 75 |
+
def test_parse_json_with_wrapper(self):
|
| 76 |
+
from app.services.file_processing import parse_json
|
| 77 |
+
data = {"data": [{"text": "entry 1"}]}
|
| 78 |
+
entries = parse_json(json.dumps(data).encode())
|
| 79 |
+
assert len(entries) == 1
|
| 80 |
+
|
| 81 |
+
def test_parse_csv_missing_text_column(self):
|
| 82 |
+
from app.services.file_processing import parse_csv
|
| 83 |
+
content = b"name,age\nJohn,30\n"
|
| 84 |
+
# Should fall back to first column or raise
|
| 85 |
+
try:
|
| 86 |
+
entries = parse_csv(content)
|
| 87 |
+
assert len(entries) >= 0
|
| 88 |
+
except ValueError:
|
| 89 |
+
pass
|
| 90 |
+
|
| 91 |
+
def test_unsupported_format(self):
|
| 92 |
+
from app.services.file_processing import parse_file
|
| 93 |
+
with pytest.raises(ValueError, match="Unsupported"):
|
| 94 |
+
parse_file(b"content", "file.txt")
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class TestAnomalyDetection:
|
| 98 |
+
def test_no_anomalies_stable(self):
|
| 99 |
+
from app.services.anomaly_detection import detect_sentiment_anomalies
|
| 100 |
+
sentiments = [
|
| 101 |
+
SentimentResult(label=SentimentLabel.NEUTRAL, score=0.5, confidence=0.9)
|
| 102 |
+
for _ in range(100)
|
| 103 |
+
]
|
| 104 |
+
alerts = detect_sentiment_anomalies(sentiments)
|
| 105 |
+
assert len(alerts) == 0
|
| 106 |
+
|
| 107 |
+
def test_detects_sentiment_drop(self):
|
| 108 |
+
from app.services.anomaly_detection import detect_sentiment_anomalies
|
| 109 |
+
sentiments = [
|
| 110 |
+
SentimentResult(label=SentimentLabel.POSITIVE, score=0.8, confidence=0.9)
|
| 111 |
+
for _ in range(60)
|
| 112 |
+
]
|
| 113 |
+
sentiments.append(
|
| 114 |
+
SentimentResult(label=SentimentLabel.NEGATIVE, score=0.1, confidence=0.9)
|
| 115 |
+
)
|
| 116 |
+
alerts = detect_sentiment_anomalies(sentiments, window=50, threshold=1.5)
|
| 117 |
+
assert len(alerts) > 0
|
| 118 |
+
assert alerts[0].type.value == "sentiment_drop"
|
| 119 |
+
|
| 120 |
+
def test_too_few_entries(self):
|
| 121 |
+
from app.services.anomaly_detection import detect_sentiment_anomalies
|
| 122 |
+
sentiments = [
|
| 123 |
+
SentimentResult(label=SentimentLabel.NEUTRAL, score=0.5, confidence=0.9)
|
| 124 |
+
for _ in range(5)
|
| 125 |
+
]
|
| 126 |
+
alerts = detect_sentiment_anomalies(sentiments, window=50)
|
| 127 |
+
assert len(alerts) == 0
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
class TestDataQuality:
|
| 131 |
+
def test_empty_entries(self):
|
| 132 |
+
from app.services.data_quality import analyze_data_quality
|
| 133 |
+
report = analyze_data_quality([])
|
| 134 |
+
assert report.total_entries == 0
|
| 135 |
+
|
| 136 |
+
def test_quality_report(self):
|
| 137 |
+
from app.models.schemas import AnalyzedEntry, LanguageResult
|
| 138 |
+
from app.services.data_quality import analyze_data_quality
|
| 139 |
+
|
| 140 |
+
entries = [
|
| 141 |
+
AnalyzedEntry(
|
| 142 |
+
id="1", text="Great product", source="test",
|
| 143 |
+
sentiment=SentimentResult(label=SentimentLabel.POSITIVE, score=0.9, confidence=0.95),
|
| 144 |
+
language=LanguageResult(language="en", confidence=0.99, method="langdetect"),
|
| 145 |
+
topic_id=0, topic_label="Topic 0",
|
| 146 |
+
),
|
| 147 |
+
AnalyzedEntry(
|
| 148 |
+
id="2", text="Mauvais service", source="test",
|
| 149 |
+
sentiment=SentimentResult(label=SentimentLabel.NEGATIVE, score=0.2, confidence=0.4),
|
| 150 |
+
language=LanguageResult(language="fr", confidence=0.85, method="langdetect"),
|
| 151 |
+
topic_id=1, topic_label="Topic 1",
|
| 152 |
+
),
|
| 153 |
+
]
|
| 154 |
+
|
| 155 |
+
report = analyze_data_quality(entries)
|
| 156 |
+
assert report.total_entries == 2
|
| 157 |
+
assert report.low_confidence_count == 1
|
| 158 |
+
assert report.mixed_language_count == 1
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class TestExport:
|
| 162 |
+
def test_export_csv(self):
|
| 163 |
+
from app.models.schemas import AnalyzedEntry, LanguageResult
|
| 164 |
+
from app.services.export import export_csv
|
| 165 |
+
|
| 166 |
+
entries = [
|
| 167 |
+
AnalyzedEntry(
|
| 168 |
+
id="1", text="Test", source="test",
|
| 169 |
+
sentiment=SentimentResult(label=SentimentLabel.POSITIVE, score=0.9, confidence=0.95),
|
| 170 |
+
language=LanguageResult(language="en", confidence=0.99, method="langdetect"),
|
| 171 |
+
topic_id=0, topic_label="Topic 0",
|
| 172 |
+
),
|
| 173 |
+
]
|
| 174 |
+
result = export_csv(entries)
|
| 175 |
+
assert b"id" in result
|
| 176 |
+
assert b"Test" in result
|
| 177 |
+
|
| 178 |
+
def test_export_json(self):
|
| 179 |
+
from app.models.schemas import AnalyzedEntry, LanguageResult
|
| 180 |
+
from app.services.export import export_json
|
| 181 |
+
|
| 182 |
+
entries = [
|
| 183 |
+
AnalyzedEntry(
|
| 184 |
+
id="1", text="Test", source="test",
|
| 185 |
+
sentiment=SentimentResult(label=SentimentLabel.POSITIVE, score=0.9, confidence=0.95),
|
| 186 |
+
language=LanguageResult(language="en", confidence=0.99, method="langdetect"),
|
| 187 |
+
topic_id=0, topic_label="Topic 0",
|
| 188 |
+
),
|
| 189 |
+
]
|
| 190 |
+
result = export_json(entries)
|
| 191 |
+
data = json.loads(result)
|
| 192 |
+
assert len(data) == 1
|
| 193 |
+
assert data[0]["text"] == "Test"
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _ml_available() -> bool:
|
| 197 |
+
try:
|
| 198 |
+
import torch # noqa: F401
|
| 199 |
+
import transformers # noqa: F401
|
| 200 |
+
return True
|
| 201 |
+
except ImportError:
|
| 202 |
+
return False
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
@pytest.mark.skipif(
|
| 206 |
+
not _ml_available(),
|
| 207 |
+
reason="ML models not installed — skipping real model tests",
|
| 208 |
+
)
|
| 209 |
+
class TestRealSentimentModel:
|
| 210 |
+
"""Diagnostic tests using the real ML model (not mocked)."""
|
| 211 |
+
|
| 212 |
+
def test_model_loads(self):
|
| 213 |
+
from app.services import sentiment
|
| 214 |
+
sentiment._load_model()
|
| 215 |
+
assert sentiment._model is not None
|
| 216 |
+
|
| 217 |
+
def test_positive_english(self):
|
| 218 |
+
from app.services.sentiment import analyze_sentiment_sync
|
| 219 |
+
results = analyze_sentiment_sync(["I love this product, it is amazing!"])
|
| 220 |
+
assert len(results) == 1
|
| 221 |
+
assert results[0].label == SentimentLabel.POSITIVE
|
| 222 |
+
assert results[0].score > 0.7
|
| 223 |
+
assert results[0].confidence > 0.5
|
| 224 |
+
|
| 225 |
+
def test_negative_english(self):
|
| 226 |
+
from app.services.sentiment import analyze_sentiment_sync
|
| 227 |
+
results = analyze_sentiment_sync(["This is terrible, worst experience ever."])
|
| 228 |
+
assert len(results) == 1
|
| 229 |
+
assert results[0].label == SentimentLabel.NEGATIVE
|
| 230 |
+
assert results[0].score < 0.3
|
| 231 |
+
assert results[0].confidence > 0.5
|
| 232 |
+
|
| 233 |
+
def test_neutral_english(self):
|
| 234 |
+
from app.services.sentiment import analyze_sentiment_sync
|
| 235 |
+
results = analyze_sentiment_sync(["The order was delivered on Tuesday."])
|
| 236 |
+
assert len(results) == 1
|
| 237 |
+
assert results[0].score > 0.3
|
| 238 |
+
assert results[0].score < 0.7
|
| 239 |
+
|
| 240 |
+
def test_multilingual_german(self):
|
| 241 |
+
from app.services.sentiment import analyze_sentiment_sync
|
| 242 |
+
results = analyze_sentiment_sync(["Ich bin sehr zufrieden mit dem Service!"])
|
| 243 |
+
assert results[0].label == SentimentLabel.POSITIVE
|
| 244 |
+
assert results[0].score > 0.7
|
| 245 |
+
|
| 246 |
+
def test_multilingual_spanish_negative(self):
|
| 247 |
+
from app.services.sentiment import analyze_sentiment_sync
|
| 248 |
+
results = analyze_sentiment_sync(["Este producto es horrible, no funciona."])
|
| 249 |
+
assert results[0].label == SentimentLabel.NEGATIVE
|
| 250 |
+
assert results[0].score < 0.3
|
| 251 |
+
|
| 252 |
+
def test_batch_produces_varied_scores(self):
|
| 253 |
+
from app.services.sentiment import analyze_sentiment_sync
|
| 254 |
+
texts = [
|
| 255 |
+
"I love this!",
|
| 256 |
+
"This is terrible.",
|
| 257 |
+
"The weather is normal today.",
|
| 258 |
+
"Best purchase I ever made!",
|
| 259 |
+
"Worst customer service.",
|
| 260 |
+
]
|
| 261 |
+
results = analyze_sentiment_sync(texts)
|
| 262 |
+
scores = [r.score for r in results]
|
| 263 |
+
assert not all(s == 0.5 for s in scores), f"All scores are 0.5: {scores}"
|
| 264 |
+
assert max(scores) - min(scores) > 0.3, f"Score spread too narrow: {scores}"
|
| 265 |
+
|
| 266 |
+
def test_scores_not_all_neutral(self):
|
| 267 |
+
from app.services.sentiment import analyze_sentiment_sync
|
| 268 |
+
texts = [
|
| 269 |
+
"Amazing fantastic wonderful product",
|
| 270 |
+
"Horrible terrible awful experience",
|
| 271 |
+
"Normal everyday standard thing",
|
| 272 |
+
]
|
| 273 |
+
results = analyze_sentiment_sync(texts)
|
| 274 |
+
labels = [r.label for r in results]
|
| 275 |
+
assert SentimentLabel.NEUTRAL not in labels or len(set(labels)) > 1, \
|
| 276 |
+
f"All labels are neutral: {labels}"
|
demo_data/demo_feedback.csv
ADDED
|
@@ -0,0 +1,501 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
id,text,source,timestamp,rating
|
| 2 |
+
0550d633e0fc,Worst customer service I've ever encountered. Score: 1/5.,app_store,2024-01-01T13:02:00,1
|
| 3 |
+
173ef1403ddb,El servicio al cliente fue increíblemente útil.,support_ticket,2024-01-01T22:34:00,4
|
| 4 |
+
e6b5dff85469,Ich liebe dieses Produkt! Bester Kauf den ich je gemacht habe.,support_ticket,2024-01-01T05:44:00,5
|
| 5 |
+
1a1d71db74d0,カスタマーサービスが非常に親切で助かりました。 スコア: 5/10。,app_store,2024-01-02T19:16:00,5
|
| 6 |
+
e5668ef2ba52,Functional product. Does what it says. Overall rating: 3/5.,survey,2024-01-02T21:14:00,3
|
| 7 |
+
d2ab38a62733,"Standard service, met expectations but didn't exceed them.",twitter,2024-01-02T11:10:00,3
|
| 8 |
+
1e7e92790b1b,Qualité médiocre. Cassé après deux semaines d'utilisation.,email,2024-01-03T19:40:00,1
|
| 9 |
+
1dfe556ae589,"The new feature update is amazing, exactly what I needed.",app_store,2024-01-03T01:14:00,5
|
| 10 |
+
96a99d0e0c89,Excellent value for money. Exceeded my expectations. Overall rating: 5/5.,chat,2024-01-03T08:08:00,5
|
| 11 |
+
7b11f298ec63,Average experience. Delivery was on time.,web_form,2024-01-04T04:32:00,3
|
| 12 |
+
f88afbed6282,Average experience. Delivery was on time.,play_store,2024-01-04T13:38:00,3
|
| 13 |
+
e897b47af36f,Very satisfied with the experience. Highly recommend!,play_store,2024-01-04T00:43:00,5
|
| 14 |
+
a23f0c3ca118,Really impressed with the build quality and design. Score: 5/5.,chat,2024-01-05T09:53:00,4
|
| 15 |
+
dabb18cc10d7,El producto funciona como se describe. Nada especial.,play_store,2024-01-05T15:01:00,3
|
| 16 |
+
c5236a2867ef,Customer service was incredibly helpful and resolved my issue quickly. Overall rating: 5/5.,email,2024-01-06T21:30:00,5
|
| 17 |
+
89d5feeb2dbe,製品は説明通りに動作します。特別なものはありません。,twitter,2024-01-06T12:42:00,3
|
| 18 |
+
bc23021bfd04,Excellent value for money. Exceeded my expectations. Overall rating: 4/5.,app_store,2024-01-06T22:40:00,4
|
| 19 |
+
606cc9cf0904,Das Produkt funktioniert wie beschrieben. Nichts Besonderes.,web_form,2024-01-07T08:42:00,3
|
| 20 |
+
8296a3f28b29,Schlechte Qualität. Nach zwei Wochen kaputt gegangen.,support_ticket,2024-01-07T15:51:00,1
|
| 21 |
+
3a739e818bfc,The team went above and beyond to help me. Outstanding!,play_store,2024-01-07T23:03:00,4
|
| 22 |
+
670c149d339a,Das Produkt kam beschädigt an und der Support war nutzlos. Note: 2/10.,web_form,2024-01-08T05:17:00,1
|
| 23 |
+
54e348820065,"Ausgezeichnete Qualität, schneller Versand. Sehr empfehlenswert! Bewertung: 4/5.",web_form,2024-01-08T13:31:00,4
|
| 24 |
+
5327d77f7b35,Product works as described. Nothing special.,web_form,2024-01-08T08:59:00,3
|
| 25 |
+
3393cd2932cc,Qualité médiocre. Cassé après deux semaines d'utilisation.,chat,2024-01-09T15:09:00,1
|
| 26 |
+
b927e7bdc05f,El producto funciona como se describe. Nada especial.,web_form,2024-01-09T01:03:00,3
|
| 27 |
+
888b9177995c,Average experience. Delivery was on time. Overall rating: 3/5.,app_store,2024-01-10T18:15:00,3
|
| 28 |
+
d81bdf9f2357,Mala calidad. Se rompió después de dos semanas.,twitter,2024-01-10T06:42:00,2
|
| 29 |
+
da135c9331b5,"Excellente qualité, livraison rapide. Je recommande vivement !",web_form,2024-01-10T02:00:00,4
|
| 30 |
+
8fd0b472807f,El servicio al cliente fue increíblemente útil.,chat,2024-01-11T02:56:00,5
|
| 31 |
+
92af5b48a7c7,"Ausgezeichnete Qualität, schneller Versand. Sehr empfehlenswert!",app_store,2024-01-11T20:33:00,5
|
| 32 |
+
b2c2c0cc6bbb,Product works as described. Nothing special.,email,2024-01-11T23:35:00,3
|
| 33 |
+
b634861b9043,Le produit est arrivé endommagé et le support était inutile.,web_form,2024-01-12T08:32:00,1
|
| 34 |
+
1d1538f28fd5,Very satisfied with the experience. Highly recommend! Overall rating: 5/5.,support_ticket,2024-01-12T23:28:00,4
|
| 35 |
+
86c5d6ce1088,Ich liebe dieses Produkt! Bester Kauf den ich je gemacht habe.,email,2024-01-12T11:37:00,5
|
| 36 |
+
a0a53f1616dd,Absolutely love this product! Best purchase I've made.,chat,2024-01-13T06:43:00,4
|
| 37 |
+
778d0f82ac8f,Das Produkt funktioniert wie beschrieben. Nichts Besonderes.,chat,2024-01-13T23:09:00,3
|
| 38 |
+
fb224973241b,It's okay for the price point. Nothing to complain about.,chat,2024-01-13T13:51:00,3
|
| 39 |
+
4ec684fa590c,Product works as described. Nothing special.,web_form,2024-01-14T07:12:00,3
|
| 40 |
+
e800bce99714,Das Produkt funktioniert wie beschrieben. Nichts Besonderes. Note: 3/10.,survey,2024-01-14T02:49:00,3
|
| 41 |
+
343d312dd42b,Schreckliche Erfahrung. 3 Wochen gewartet ohne Ergebnis.,survey,2024-01-15T03:56:00,2
|
| 42 |
+
36f59b9ff17f,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",play_store,2024-01-15T10:27:00,5
|
| 43 |
+
b0d51f1b0512,J'adore ce produit ! Le meilleur achat que j'ai fait.,chat,2024-01-15T17:43:00,5
|
| 44 |
+
aae9126e971c,"Excellente qualité, livraison rapide. Je recommande vivement !",play_store,2024-01-16T21:54:00,5
|
| 45 |
+
e8dc1b963a6f,Worst customer service I've ever encountered.,support_ticket,2024-01-16T06:26:00,2
|
| 46 |
+
46e6d7b29beb,Der Kundenservice war unglaublich hilfreich. Note: 5/10.,chat,2024-01-16T14:43:00,5
|
| 47 |
+
f7c6833d7219,Not worth the price. Very disappointing quality. Overall rating: 2/5.,app_store,2024-01-17T04:01:00,2
|
| 48 |
+
c531f9434103,Expérience terrible. J'ai attendu 3 semaines pour rien.,chat,2024-01-17T20:36:00,2
|
| 49 |
+
163491235d98,Absolutely love this product! Best purchase I've made.,support_ticket,2024-01-17T03:49:00,4
|
| 50 |
+
a259a173d317,Expérience moyenne. Livraison à temps. Note: 3/5.,twitter,2024-01-18T14:42:00,3
|
| 51 |
+
f400e865ee2f,El producto funciona como se describe. Nada especial.,survey,2024-01-18T15:28:00,3
|
| 52 |
+
a6f2276573c5,El producto funciona como se describe. Nada especial. Calificación: 3/10.,play_store,2024-01-19T10:20:00,3
|
| 53 |
+
51757444a3b3,Le service client était incroyablement utile et efficace. Évaluation: 5/10.,support_ticket,2024-01-19T01:13:00,5
|
| 54 |
+
7d35f60f1579,Expérience moyenne. Livraison à temps.,support_ticket,2024-01-19T11:19:00,3
|
| 55 |
+
bf1be5d15e60,Really impressed with the build quality and design. Would rate 5/10.,app_store,2024-01-20T04:39:00,5
|
| 56 |
+
54063714b971,Really impressed with the build quality and design. Overall rating: 5/5.,twitter,2024-01-20T14:20:00,5
|
| 57 |
+
9a502b79557d,Das Produkt funktioniert wie beschrieben. Nichts Besonderes.,app_store,2024-01-20T15:01:00,3
|
| 58 |
+
8a0b4fdaa6c3,Qualité médiocre. Cassé après deux semaines d'utilisation. Note: 1/5.,chat,2024-01-21T19:09:00,2
|
| 59 |
+
140d1db71e91,Experiencia terrible. Esperé 3 semanas sin resultado.,twitter,2024-01-21T08:49:00,1
|
| 60 |
+
9975ccf52bc0,It's okay for the price point. Nothing to complain about. Would rate 3/10.,play_store,2024-01-21T22:12:00,3
|
| 61 |
+
bc467cbf5958,Terrible experience. Waited 3 weeks for delivery that never came.,app_store,2024-01-22T18:50:00,2
|
| 62 |
+
a0336571974a,Not worth the price. Very disappointing quality. Would rate 2/10.,web_form,2024-01-22T11:40:00,2
|
| 63 |
+
c3f31a575d3d,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",web_form,2024-01-22T17:49:00,5
|
| 64 |
+
90c35decddb2,Experiencia promedio. La entrega fue puntual.,web_form,2024-01-23T02:17:00,3
|
| 65 |
+
94e74a08a7bf,"Excellente qualité, livraison rapide. Je recommande vivement ! Note: 5/5.",survey,2024-01-23T11:51:00,5
|
| 66 |
+
dfe9e2a10c94,Expérience moyenne. Livraison à temps.,play_store,2024-01-24T16:12:00,3
|
| 67 |
+
b6fb94eb1892,Très satisfait de l'expérience. Hautement recommandé !,app_store,2024-01-24T15:28:00,4
|
| 68 |
+
c8b70629ff7e,"Excellente qualité, livraison rapide. Je recommande vivement !",twitter,2024-01-24T11:30:00,5
|
| 69 |
+
01f86c46c244,Very satisfied with the experience. Highly recommend! Overall rating: 4/5.,email,2024-01-25T23:34:00,4
|
| 70 |
+
f56d52e51b18,Worst customer service I've ever encountered.,chat,2024-01-25T06:18:00,1
|
| 71 |
+
dfd3e9757cbc,Le service client était incroyablement utile et efficace. Évaluation: 4/10.,app_store,2024-01-25T03:55:00,5
|
| 72 |
+
4d976b6f6a88,Absolutely love this product! Best purchase I've made. Would rate 5/10.,app_store,2024-01-26T15:04:00,4
|
| 73 |
+
bf4450bf2dd7,製品は説明通りに動作します。特別なものはありません。,support_ticket,2024-01-26T03:35:00,3
|
| 74 |
+
a1099b2043a5,The team went above and beyond to help me. Outstanding!,play_store,2024-01-26T19:03:00,4
|
| 75 |
+
6539bc5b1089,"The new feature update is amazing, exactly what I needed. Would rate 4/10.",web_form,2024-01-27T14:44:00,5
|
| 76 |
+
733c121b417e,Très satisfait de l'expérience. Hautement recommandé ! Évaluation: 4/10.,chat,2024-01-27T03:34:00,4
|
| 77 |
+
ac1cca4b7bc4,Average experience. Delivery was on time.,survey,2024-01-28T19:47:00,3
|
| 78 |
+
2060c03dd652,The team went above and beyond to help me. Outstanding!,web_form,2024-01-28T16:34:00,5
|
| 79 |
+
24c8a0d9693f,"Ausgezeichnete Qualität, schneller Versand. Sehr empfehlenswert!",play_store,2024-01-28T08:01:00,4
|
| 80 |
+
f4867eaece55,Expérience moyenne. Livraison à temps.,web_form,2024-01-29T05:30:00,3
|
| 81 |
+
3c19c87c4a10,"Standard service, met expectations but didn't exceed them.",support_ticket,2024-01-29T15:22:00,3
|
| 82 |
+
ed52a1a6d205,Schreckliche Erfahrung. 3 Wochen gewartet ohne Ergebnis.,support_ticket,2024-01-29T15:18:00,2
|
| 83 |
+
4f7928992601,Customer service was incredibly helpful and resolved my issue quickly.,app_store,2024-01-30T12:55:00,5
|
| 84 |
+
f198df0da3a0,Muy satisfecho con la experiencia. ¡Lo recomiendo!,survey,2024-01-30T01:13:00,4
|
| 85 |
+
dc1b9aa72d14,J'adore ce produit ! Le meilleur achat que j'ai fait. Note: 4/5.,play_store,2024-01-30T17:26:00,4
|
| 86 |
+
40946150cc51,"Great quality, fast shipping. Will definitely order again.",survey,2024-01-31T22:18:00,5
|
| 87 |
+
2beccf55e2df,Functional product. Does what it says. Would rate 3/10.,survey,2024-01-31T13:21:00,3
|
| 88 |
+
dd7733e046a7,Schlechte Qualität. Nach zwei Wochen kaputt gegangen.,email,2024-01-31T15:55:00,2
|
| 89 |
+
0843fb436b8d,Muy satisfecho con la experiencia. ¡Lo recomiendo!,chat,2024-02-01T10:55:00,4
|
| 90 |
+
75baf2ad5cf2,J'adore ce produit ! Le meilleur achat que j'ai fait.,support_ticket,2024-02-01T15:45:00,5
|
| 91 |
+
b78ef5099494,Excellent value for money. Exceeded my expectations.,web_form,2024-02-02T14:06:00,4
|
| 92 |
+
6d6a2a6b90f8,Le service client était incroyablement utile et efficace. Note: 5/5.,support_ticket,2024-02-02T10:54:00,5
|
| 93 |
+
5ad7a5d5d629,Product works as described. Nothing special. Would rate 3/10.,play_store,2024-02-02T03:48:00,3
|
| 94 |
+
628981826b8b,製品が破損して届きました。サポートも役に立ちませんでした。,twitter,2024-02-03T01:18:00,2
|
| 95 |
+
82d9e7af73b4,El servicio al cliente fue increíblemente útil. Calificación: 4/10.,chat,2024-02-03T18:09:00,5
|
| 96 |
+
4d93ae3dd21c,Très satisfait de l'expérience. Hautement recommandé !,email,2024-02-03T21:34:00,4
|
| 97 |
+
b676359c603a,製品は説明通りに動作します。特別なものはありません。,survey,2024-02-04T13:44:00,3
|
| 98 |
+
b70f2085da3e,Really impressed with the build quality and design. Score: 5/5.,support_ticket,2024-02-04T09:01:00,5
|
| 99 |
+
5d084a7ecc0b,J'adore ce produit ! Le meilleur achat que j'ai fait.,survey,2024-02-04T23:31:00,4
|
| 100 |
+
32678c956496,Very satisfied with the experience. Highly recommend!,email,2024-02-05T23:49:00,4
|
| 101 |
+
50250461bc18,Excellent value for money. Exceeded my expectations.,survey,2024-02-05T02:58:00,5
|
| 102 |
+
2cfe16ffcac1,Excellent value for money. Exceeded my expectations.,email,2024-02-06T08:53:00,5
|
| 103 |
+
64b362a374c5,It's okay for the price point. Nothing to complain about.,email,2024-02-06T14:04:00,3
|
| 104 |
+
f8cf2040bdc3,Das Produkt funktioniert wie beschrieben. Nichts Besonderes. Bewertung: 3/5.,survey,2024-02-06T14:23:00,3
|
| 105 |
+
c5a4102c0ec3,Absolutely love this product! Best purchase I've made.,chat,2024-02-07T17:20:00,4
|
| 106 |
+
e02fb7dcd6c1,"Standard service, met expectations but didn't exceed them. Would rate 3/10.",chat,2024-02-07T03:06:00,3
|
| 107 |
+
d847a34ad2a9,Très satisfait de l'expérience. Hautement recommandé !,support_ticket,2024-02-07T23:09:00,4
|
| 108 |
+
9c53a13f87f4,ひどい経験でした。3週間待っても届きませんでした。 スコア: 1/10。,twitter,2024-02-08T07:54:00,2
|
| 109 |
+
2fdecc6a7d5e,カスタマーサービスが非常に親切で助かりました。 スコア: 5/10。,chat,2024-02-08T02:42:00,4
|
| 110 |
+
c5ccf0df4480,Average experience. Delivery was on time.,chat,2024-02-08T06:04:00,3
|
| 111 |
+
6c260231024c,Absolutely love this product! Best purchase I've made. Overall rating: 4/5.,app_store,2024-02-09T03:42:00,4
|
| 112 |
+
c542220e734d,"Excellente qualité, livraison rapide. Je recommande vivement ! Évaluation: 5/10.",play_store,2024-02-09T16:07:00,5
|
| 113 |
+
18138fbc57b6,El producto funciona como se describe. Nada especial.,app_store,2024-02-09T07:39:00,3
|
| 114 |
+
075c6e01e4f8,Misleading product description. Nothing like advertised.,play_store,2024-02-10T13:43:00,2
|
| 115 |
+
b9c98afc431e,"Great quality, fast shipping. Will definitely order again. Score: 5/5.",web_form,2024-02-10T16:18:00,5
|
| 116 |
+
20b0b9cb0def,Functional product. Does what it says.,web_form,2024-02-11T06:27:00,3
|
| 117 |
+
ebfecc937650,The team went above and beyond to help me. Outstanding!,twitter,2024-02-11T10:27:00,5
|
| 118 |
+
ac1a64402be9,Expérience moyenne. Livraison à temps. Note: 3/5.,twitter,2024-02-11T23:47:00,3
|
| 119 |
+
9cbd8bbf44e9,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",support_ticket,2024-02-12T13:22:00,4
|
| 120 |
+
9c56d6667025,Poor quality materials. Broke after two weeks of use. Would rate 1/10.,play_store,2024-02-12T17:23:00,1
|
| 121 |
+
5c1d11a5dd01,Das Produkt funktioniert wie beschrieben. Nichts Besonderes.,survey,2024-02-12T21:53:00,3
|
| 122 |
+
18ff62b0b951,"Excellente qualité, livraison rapide. Je recommande vivement !",email,2024-02-13T05:55:00,5
|
| 123 |
+
8c7784e2e007,¡Me encanta este producto! La mejor compra que he hecho. Calificación: 4/10.,twitter,2024-02-13T10:10:00,5
|
| 124 |
+
f7115d9153c8,El producto funciona como se describe. Nada especial.,app_store,2024-02-13T04:10:00,3
|
| 125 |
+
d68d8f0b4219,Really impressed with the build quality and design.,play_store,2024-02-14T06:48:00,5
|
| 126 |
+
a49854c53956,Très satisfait de l'expérience. Hautement recommandé !,chat,2024-02-14T09:02:00,5
|
| 127 |
+
c4d95d92ddee,"The new feature update is amazing, exactly what I needed.",twitter,2024-02-15T08:18:00,4
|
| 128 |
+
389f5a9171f1,"Great quality, fast shipping. Will definitely order again.",twitter,2024-02-15T22:14:00,4
|
| 129 |
+
021d3dbea95a,Product works as described. Nothing special.,twitter,2024-02-15T02:55:00,3
|
| 130 |
+
ce92f5aa1131,Terrible experience. Waited 3 weeks for delivery that never came.,web_form,2024-02-16T10:10:00,1
|
| 131 |
+
247d1dd5de5a,The team went above and beyond to help me. Outstanding!,support_ticket,2024-02-16T16:05:00,5
|
| 132 |
+
04dc02158007,J'adore ce produit ! Le meilleur achat que j'ai fait.,chat,2024-02-16T03:33:00,5
|
| 133 |
+
b4413ca52dda,Das Produkt kam beschädigt an und der Support war nutzlos. Bewertung: 2/5.,play_store,2024-02-17T19:09:00,2
|
| 134 |
+
a05c3f9d5e29,Muy satisfecho con la experiencia. ¡Lo recomiendo!,twitter,2024-02-17T18:41:00,5
|
| 135 |
+
b9a9d3e33dc0,Very satisfied with the experience. Highly recommend!,twitter,2024-02-17T18:03:00,4
|
| 136 |
+
fed9888473f7,Excellent value for money. Exceeded my expectations.,play_store,2024-02-18T02:41:00,5
|
| 137 |
+
2ca433075686,素晴らしい品質です。強くお勧めします!,chat,2024-02-18T14:58:00,5
|
| 138 |
+
ac04fd57afc3,カスタマーサービスが非常に親切で助かりました。 評価: 5/5。,web_form,2024-02-18T07:45:00,5
|
| 139 |
+
da14a011772e,カスタマーサービスが非常に親切で助かりました。,app_store,2024-02-19T10:43:00,5
|
| 140 |
+
1cdd5b8c0c40,Muy satisfecho con la experiencia. ¡Lo recomiendo! Puntuación: 4/5.,support_ticket,2024-02-19T11:19:00,4
|
| 141 |
+
8da171ee35f4,Excellent value for money. Exceeded my expectations.,play_store,2024-02-20T21:44:00,5
|
| 142 |
+
5d5d109b06c0,"The new feature update is amazing, exactly what I needed.",chat,2024-02-20T15:12:00,4
|
| 143 |
+
42101be2f892,Ich liebe dieses Produkt! Bester Kauf den ich je gemacht habe.,app_store,2024-02-20T17:53:00,5
|
| 144 |
+
aee5fd8241b0,"Great quality, fast shipping. Will definitely order again.",email,2024-02-21T19:51:00,5
|
| 145 |
+
9720c191019c,Le service client était incroyablement utile et efficace.,web_form,2024-02-21T19:18:00,4
|
| 146 |
+
bf2323afae55,Der Kundenservice war unglaublich hilfreich.,web_form,2024-02-21T06:23:00,5
|
| 147 |
+
dd7462f688fe,Experiencia promedio. La entrega fue puntual.,email,2024-02-22T06:51:00,3
|
| 148 |
+
a14a777be24b,Schlechte Qualität. Nach zwei Wochen kaputt gegangen.,play_store,2024-02-22T22:54:00,1
|
| 149 |
+
d5c68a8124fe,Functional product. Does what it says. Would rate 3/10.,app_store,2024-02-22T11:59:00,3
|
| 150 |
+
063783a4c96d,The team went above and beyond to help me. Outstanding!,app_store,2024-02-23T11:14:00,5
|
| 151 |
+
eb64c0add977,El producto funciona como se describe. Nada especial. Calificación: 3/10.,app_store,2024-02-23T14:39:00,3
|
| 152 |
+
23566e75a4a4,Schlechte Qualität. Nach zwei Wochen kaputt gegangen.,support_ticket,2024-02-24T19:33:00,2
|
| 153 |
+
5f69f5c7df59,Poor quality materials. Broke after two weeks of use.,play_store,2024-02-24T19:29:00,2
|
| 154 |
+
302e4e5e6d4e,Poor quality materials. Broke after two weeks of use.,app_store,2024-02-24T09:27:00,1
|
| 155 |
+
169f965e3cb6,Terrible experience. Waited 3 weeks for delivery that never came.,app_store,2024-02-25T11:04:00,2
|
| 156 |
+
73c9f76a8afa,Excellent value for money. Exceeded my expectations. Would rate 4/10.,play_store,2024-02-25T07:46:00,5
|
| 157 |
+
7e3b9ab6df95,El servicio al cliente fue increíblemente útil.,web_form,2024-02-25T21:29:00,4
|
| 158 |
+
ac10cdfed3bd,Très satisfait de l'expérience. Hautement recommandé ! Note: 4/5.,survey,2024-02-26T10:59:00,5
|
| 159 |
+
aa424db54c37,Le produit est arrivé endommagé et le support était inutile. Note: 2/5.,support_ticket,2024-02-26T11:27:00,2
|
| 160 |
+
66bf64e93e6c,Misleading product description. Nothing like advertised.,play_store,2024-02-26T16:17:00,1
|
| 161 |
+
77ec32f577db,製品が破損して届きました。サポートも役に立ちませんでした。,play_store,2024-02-27T10:06:00,2
|
| 162 |
+
b79aac27202a,El peor servicio al cliente que he experimentado. Puntuación: 2/5.,twitter,2024-02-27T23:55:00,2
|
| 163 |
+
6caaf83f597c,Le produit est arrivé endommagé et le support était inutile. Note: 2/5.,web_form,2024-02-27T04:54:00,1
|
| 164 |
+
526d8f64594d,"Excellente qualité, livraison rapide. Je recommande vivement !",email,2024-02-28T18:02:00,5
|
| 165 |
+
d508b22b559b,Qualité médiocre. Cassé après deux semaines d'utilisation. Évaluation: 2/10.,app_store,2024-02-28T07:07:00,2
|
| 166 |
+
7679a8a35fc1,製品が破損して届きました。サポートも役に立ちませんでした。 スコア: 2/10。,web_form,2024-02-29T22:58:00,1
|
| 167 |
+
bc36cbb607e6,"Great quality, fast shipping. Will definitely order again.",twitter,2024-02-29T14:05:00,5
|
| 168 |
+
ae2f775d7121,Worst customer service I've ever encountered.,email,2024-02-29T22:45:00,1
|
| 169 |
+
2c27fcd963d8,"Great quality, fast shipping. Will definitely order again. Score: 5/5.",web_form,2024-03-01T18:23:00,4
|
| 170 |
+
ab62a3f12541,Really impressed with the build quality and design.,play_store,2024-03-01T13:52:00,4
|
| 171 |
+
e5a8841b80fc,Not worth the price. Very disappointing quality. Score: 1/5.,play_store,2024-03-01T01:53:00,2
|
| 172 |
+
126ea062cbd8,"The new feature update is amazing, exactly what I needed.",survey,2024-03-02T16:57:00,4
|
| 173 |
+
d0d07e36a0b8,カスタマーサービスが非常に親切で助かりました。 評価: 5/5。,twitter,2024-03-02T05:42:00,4
|
| 174 |
+
4aeee428251e,素晴らしい品質です。強くお勧めします! スコア: 4/10。,survey,2024-03-02T20:34:00,5
|
| 175 |
+
4e085be965de,Schreckliche Erfahrung. 3 Wochen gewartet ohne Ergebnis. Note: 1/10.,web_form,2024-03-03T13:30:00,2
|
| 176 |
+
dc35614e7f48,Not worth the price. Very disappointing quality.,email,2024-03-03T11:25:00,1
|
| 177 |
+
6391048b938d,Worst customer service I've ever encountered.,app_store,2024-03-03T04:06:00,2
|
| 178 |
+
614860c7bc7f,Average experience. Delivery was on time.,twitter,2024-03-04T18:46:00,3
|
| 179 |
+
0ba94e4aae2c,El peor servicio al cliente que he experimentado.,play_store,2024-03-04T15:01:00,2
|
| 180 |
+
2cd21a3b6407,It's okay for the price point. Nothing to complain about. Would rate 3/10.,play_store,2024-03-05T17:00:00,3
|
| 181 |
+
a7771b767fca,Expérience moyenne. Livraison à temps.,chat,2024-03-05T12:03:00,3
|
| 182 |
+
6626e8341b30,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",play_store,2024-03-05T19:43:00,4
|
| 183 |
+
d67316930082,Expérience terrible. J'ai attendu 3 semaines pour rien.,email,2024-03-06T11:48:00,2
|
| 184 |
+
7d73508b37a0,The software crashes constantly. Very frustrating.,web_form,2024-03-06T15:33:00,1
|
| 185 |
+
a6ca4b5e29b7,Experiencia terrible. Esperé 3 semanas sin resultado.,survey,2024-03-06T19:47:00,2
|
| 186 |
+
4a8db1e38a00,Product arrived damaged and customer support was unhelpful.,support_ticket,2024-03-07T21:07:00,1
|
| 187 |
+
a9c8ba0f3417,Poor quality materials. Broke after two weeks of use.,email,2024-03-07T18:30:00,1
|
| 188 |
+
f2affdb0a287,Functional product. Does what it says.,app_store,2024-03-07T03:12:00,3
|
| 189 |
+
5f9d2ff7ec45,Poor quality materials. Broke after two weeks of use.,web_form,2024-03-08T19:58:00,1
|
| 190 |
+
a63a8fffc3ed,El peor servicio al cliente que he experimentado.,web_form,2024-03-08T10:11:00,2
|
| 191 |
+
5399e5852a49,Schlechte Qualität. Nach zwei Wochen kaputt gegangen.,web_form,2024-03-09T22:16:00,1
|
| 192 |
+
bdfb9dacf3d3,Worst customer service I've ever encountered.,web_form,2024-03-09T22:13:00,2
|
| 193 |
+
361d4fb8d236,ひどい���験でした。3週間待っても届きませんでした。 評価: 2/5。,survey,2024-03-09T09:02:00,2
|
| 194 |
+
9acdae4d45dd,Expérience moyenne. Livraison à temps. Note: 3/5.,chat,2024-03-10T19:47:00,3
|
| 195 |
+
2d110d201206,Experiencia terrible. Esperé 3 semanas sin resultado. Puntuación: 2/5.,email,2024-03-10T21:49:00,1
|
| 196 |
+
50076c00863c,ひどい経験でした。3週間待っても届きませんでした。,survey,2024-03-10T06:48:00,2
|
| 197 |
+
2b666b3d4da8,製品が破損して届きました。サポートも役に立ちませんでした。,app_store,2024-03-11T07:34:00,2
|
| 198 |
+
2ece8489ed9b,Product works as described. Nothing special.,web_form,2024-03-11T12:50:00,3
|
| 199 |
+
4b4d830d2a88,Functional product. Does what it says.,web_form,2024-03-11T10:18:00,3
|
| 200 |
+
59b23f162bdd,Average experience. Delivery was on time.,web_form,2024-03-12T10:10:00,3
|
| 201 |
+
ec5cbb02072a,Very satisfied with the experience. Highly recommend!,support_ticket,2024-03-12T06:11:00,5
|
| 202 |
+
01805b045f78,Le service client était incroyablement utile et efficace.,chat,2024-03-13T15:20:00,4
|
| 203 |
+
49386ec69c94,Worst customer service I've ever encountered.,app_store,2024-03-13T09:37:00,2
|
| 204 |
+
656f19b49e10,Terrible experience. Waited 3 weeks for delivery that never came.,survey,2024-03-13T05:25:00,2
|
| 205 |
+
f32ecdbf3bf2,The software crashes constantly. Very frustrating.,twitter,2024-03-14T14:47:00,2
|
| 206 |
+
9b0946c02e90,Das Produkt funktioniert wie beschrieben. Nichts Besonderes.,play_store,2024-03-14T21:48:00,3
|
| 207 |
+
22705c465bbf,¡Me encanta este producto! La mejor compra que he hecho.,email,2024-03-14T16:12:00,4
|
| 208 |
+
d936f17c9fe1,Average experience. Delivery was on time.,web_form,2024-03-15T15:05:00,3
|
| 209 |
+
11cd45dc8595,Absolutely love this product! Best purchase I've made.,survey,2024-03-15T21:51:00,4
|
| 210 |
+
a426493fdb35,Poor quality materials. Broke after two weeks of use.,twitter,2024-03-15T01:27:00,1
|
| 211 |
+
54f59417dfaf,"Standard service, met expectations but didn't exceed them. Score: 3/5.",support_ticket,2024-03-16T23:41:00,3
|
| 212 |
+
8807cfab33fa,ひどい経験でした。3週間待っても届きませんでした。,email,2024-03-16T21:42:00,1
|
| 213 |
+
9599268f98b3,El servicio al cliente fue increíblemente útil.,app_store,2024-03-16T07:53:00,4
|
| 214 |
+
b32102273fab,Expérience terrible. J'ai attendu 3 semaines pour rien.,web_form,2024-03-17T07:15:00,1
|
| 215 |
+
db01f92bc44d,製品が破損して届きました。サポートも役に立ちませんでした。 評価: 1/5。,app_store,2024-03-17T02:06:00,1
|
| 216 |
+
5b83380eb993,Misleading product description. Nothing like advertised. Score: 1/5.,twitter,2024-03-18T09:14:00,1
|
| 217 |
+
5f23203cfaf4,Poor quality materials. Broke after two weeks of use.,email,2024-03-18T08:03:00,2
|
| 218 |
+
12101320cd4f,The team went above and beyond to help me. Outstanding!,support_ticket,2024-03-18T10:44:00,5
|
| 219 |
+
81e1839285f2,Misleading product description. Nothing like advertised. Would rate 2/10.,support_ticket,2024-03-19T01:35:00,1
|
| 220 |
+
413b7c7fe009,Not worth the price. Very disappointing quality.,twitter,2024-03-19T14:54:00,1
|
| 221 |
+
1733d7255f2c,Absolutely love this product! Best purchase I've made.,chat,2024-03-19T06:51:00,5
|
| 222 |
+
1cc03469e91f,Product works as described. Nothing special. Overall rating: 3/5.,chat,2024-03-20T22:38:00,3
|
| 223 |
+
545d0bb313e0,Expérience terrible. J'ai attendu 3 semaines pour rien.,survey,2024-03-20T20:22:00,2
|
| 224 |
+
9217d6bb48e9,Terrible experience. Waited 3 weeks for delivery that never came.,twitter,2024-03-20T23:30:00,2
|
| 225 |
+
d5cec0823ea3,Poor quality materials. Broke after two weeks of use.,support_ticket,2024-03-21T00:16:00,1
|
| 226 |
+
5c8873f259db,"The new feature update is amazing, exactly what I needed.",twitter,2024-03-21T20:43:00,5
|
| 227 |
+
73d70d5125b3,Experiencia promedio. La entrega fue puntual.,twitter,2024-03-22T23:50:00,3
|
| 228 |
+
c2e558b70544,Product arrived damaged and customer support was unhelpful.,app_store,2024-03-22T02:52:00,2
|
| 229 |
+
bd26ada51971,¡Me encanta este producto! La mejor compra que he hecho.,web_form,2024-03-22T00:13:00,5
|
| 230 |
+
e564a7c66c93,Qualité médiocre. Cassé après deux semaines d'utilisation.,app_store,2024-03-23T21:31:00,2
|
| 231 |
+
72db904b73ab,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",support_ticket,2024-03-23T09:01:00,4
|
| 232 |
+
f3ec3339069f,El producto funciona como se describe. Nada especial. Calificación: 3/10.,support_ticket,2024-03-23T02:23:00,3
|
| 233 |
+
26eff9c842d7,Product works as described. Nothing special.,app_store,2024-03-24T20:48:00,3
|
| 234 |
+
dc1d6ba29517,El producto llegó dañado y el soporte no ayudó.,play_store,2024-03-24T22:10:00,2
|
| 235 |
+
e7aa1ef201a2,Terrible experience. Waited 3 weeks for delivery that never came.,chat,2024-03-24T01:14:00,2
|
| 236 |
+
d0b623ad00a5,Very satisfied with the experience. Highly recommend!,app_store,2024-03-25T18:42:00,4
|
| 237 |
+
3ae049d27d41,"Great quality, fast shipping. Will definitely order again.",support_ticket,2024-03-25T22:52:00,4
|
| 238 |
+
89d0c21beda2,The team went above and beyond to help me. Outstanding!,support_ticket,2024-03-25T17:21:00,4
|
| 239 |
+
a375efd736b8,製品が破損して届きました。サポートも役に立ちませんでした。 スコア: 1/10。,survey,2024-03-26T10:01:00,2
|
| 240 |
+
d18ac8e50ba5,El peor servicio al cliente que he experimentado.,support_ticket,2024-03-26T21:37:00,2
|
| 241 |
+
e5cedce66157,Le produit fonctionne comme décrit. Rien de spécial. Évaluation: 3/10.,play_store,2024-03-27T11:06:00,3
|
| 242 |
+
e7f1427704dc,Le produit est arrivé endommagé et le support était inutile. Évaluation: 2/10.,play_store,2024-03-27T03:01:00,1
|
| 243 |
+
b74a85c0071c,Expérience terrible. J'ai attendu 3 semaines pour rien. Note: 2/5.,email,2024-03-27T18:33:00,1
|
| 244 |
+
b867a1a840c4,Really impressed with the build quality and design. Would rate 4/10.,play_store,2024-03-28T09:30:00,5
|
| 245 |
+
404c92898c7e,Expérience moyenne. Livraison à temps.,play_store,2024-03-28T06:29:00,3
|
| 246 |
+
be88c46c038e,The team went above and beyond to help me. Outstanding!,twitter,2024-03-28T10:28:00,5
|
| 247 |
+
18b023c1103a,Experiencia terrible. Esperé 3 semanas sin resultado.,twitter,2024-03-29T05:25:00,2
|
| 248 |
+
b6a80b36aae3,Expérience moyenne. Livraison à temps.,play_store,2024-03-29T12:23:00,3
|
| 249 |
+
86a1de9c3563,El peor servicio al cliente que he experimentado.,survey,2024-03-29T06:28:00,2
|
| 250 |
+
887449793d2d,Worst customer service I've ever encountered. Overall rating: 2/5.,web_form,2024-03-30T14:38:00,2
|
| 251 |
+
b693dcc2f5c9,Expérience terrible. J'ai attendu 3 semaines pour rien.,play_store,2024-03-30T17:38:00,1
|
| 252 |
+
fdb477af36db,Product arrived damaged and customer support was unhelpful.,support_ticket,2024-03-31T20:43:00,2
|
| 253 |
+
fa057f188161,"The new feature update is amazing, exactly what I needed.",chat,2024-03-31T16:35:00,5
|
| 254 |
+
f42ca40c1391,Qualité médiocre. Cassé après deux semaines d'utilisation.,app_store,2024-03-31T04:22:00,1
|
| 255 |
+
a34f61b66c0b,製品が破損して届きました。サポートも役に立ちませんでした。 スコア: 2/10。,chat,2024-04-01T14:50:00,2
|
| 256 |
+
aa21d354df9f,Misleading product description. Nothing like advertised.,play_store,2024-04-01T09:02:00,2
|
| 257 |
+
4c24c3d87472,Le produit est arrivé endommagé et le support était inutile.,email,2024-04-01T23:15:00,2
|
| 258 |
+
4e19280e0617,製品が破損して届きました。サポートも役に立ちませんでした。,email,2024-04-02T20:11:00,2
|
| 259 |
+
bc4c02db08a7,El servicio al cliente fue increíblemente útil.,app_store,2024-04-02T13:24:00,4
|
| 260 |
+
f3846d7596d4,Le produit est arrivé endommagé et le support était inutile.,twitter,2024-04-02T17:11:00,1
|
| 261 |
+
fb30ab40f582,Misleading product description. Nothing like advertised.,support_ticket,2024-04-03T20:20:00,1
|
| 262 |
+
3cb99c3724bd,Schlechte Qualität. Nach zwei Wochen kaputt gegangen. Note: 1/10.,survey,2024-04-03T03:15:00,1
|
| 263 |
+
5b4e15892a35,The app is full of bugs. Each update makes it worse.,support_ticket,2024-04-03T04:36:00,1
|
| 264 |
+
cf9df54106d4,Experiencia terrible. Esperé 3 semanas sin resultado.,email,2024-04-04T15:18:00,2
|
| 265 |
+
4fe1947761d1,Experiencia promedio. La entrega fue puntual.,chat,2024-04-04T15:08:00,3
|
| 266 |
+
9d053f2d608c,Misleading product description. Nothing like advertised.,app_store,2024-04-05T02:36:00,2
|
| 267 |
+
81f76698d94e,Very satisfied with the experience. Highly recommend! Score: 5/5.,twitter,2024-04-05T03:17:00,4
|
| 268 |
+
322d70094e7b,製品が破損して届きました。サポートも役に立ちませんでした。,chat,2024-04-05T20:48:00,1
|
| 269 |
+
7f607caac54a,Terrible experience. Waited 3 weeks for delivery that never came.,chat,2024-04-06T03:49:00,2
|
| 270 |
+
dc80149104c9,Experiencia promedio. La entrega fue puntual. Puntuación: 3/5.,survey,2024-04-06T18:22:00,3
|
| 271 |
+
dff2acdd415b,Functional product. Does what it says. Overall rating: 3/5.,email,2024-04-06T08:55:00,3
|
| 272 |
+
ba2038d093d8,"The new feature update is amazing, exactly what I needed.",web_form,2024-04-07T21:25:00,4
|
| 273 |
+
ab8d1e151121,Terrible experience. Waited 3 weeks for delivery that never came.,survey,2024-04-07T02:25:00,2
|
| 274 |
+
f584e5b57190,Product works as described. Nothing special.,web_form,2024-04-07T23:05:00,3
|
| 275 |
+
59b356c778b5,"Standard service, met expectations but didn't exceed them.",twitter,2024-04-08T15:17:00,3
|
| 276 |
+
6a4150e88d00,Misleading product description. Nothing like advertised.,app_store,2024-04-08T21:48:00,1
|
| 277 |
+
c8ea06ef9bce,Das Produkt kam beschädigt an und der Support war nutzlos. Bewertung: 2/5.,email,2024-04-09T13:12:00,1
|
| 278 |
+
91d27827fe8f,Das Produkt funktioniert wie beschrieben. Nichts Besonderes.,twitter,2024-04-09T08:05:00,3
|
| 279 |
+
2e25ae4c6ca2,Poor quality materials. Broke after two weeks of use.,play_store,2024-04-09T02:45:00,1
|
| 280 |
+
02f843e52619,Poor quality materials. Broke after two weeks of use.,app_store,2024-04-10T14:00:00,1
|
| 281 |
+
a57a13fde7c1,Average experience. Delivery was on time.,play_store,2024-04-10T07:24:00,3
|
| 282 |
+
5ab13cadd7a8,Das Produkt kam beschädigt an und der Support war nutzlos.,email,2024-04-10T04:41:00,1
|
| 283 |
+
1ad7a81621b8,Not worth the price. Very disappointing quality.,email,2024-04-11T20:47:00,2
|
| 284 |
+
f3a9c5b14225,Product works as described. Nothing special. Would rate 3/10.,email,2024-04-11T10:50:00,3
|
| 285 |
+
fcc9d208c2ca,Worst customer service I've ever encountered.,survey,2024-04-11T22:01:00,1
|
| 286 |
+
580e6abb0f9b,Product arrived damaged and customer support was unhelpful.,support_ticket,2024-04-12T07:31:00,1
|
| 287 |
+
2e7f346e325c,El producto llegó dañado y el soporte no ayudó.,support_ticket,2024-04-12T10:08:00,2
|
| 288 |
+
271077900307,Misleading product description. Nothing like advertised.,play_store,2024-04-12T02:01:00,2
|
| 289 |
+
ff393310ba05,ひどい経験でした。3週間待っても届きませんでした。,support_ticket,2024-04-13T20:45:00,2
|
| 290 |
+
73d3b30c0681,El servicio al cliente fue increíblemente útil. Puntuación: 5/5.,support_ticket,2024-04-13T17:20:00,4
|
| 291 |
+
cb138b5884f9,Le produit est arrivé endommagé et le support était inutile. Évaluation: 2/10.,app_store,2024-04-14T19:38:00,1
|
| 292 |
+
28c3aea3ee75,J'adore ce produit ! Le meilleur achat que j'ai fait. Évaluation: 5/10.,survey,2024-04-14T04:47:00,4
|
| 293 |
+
75df13da99b3,Terrible experience. Waited 3 weeks for delivery that never came. Score: 2/5.,web_form,2024-04-14T11:03:00,2
|
| 294 |
+
10f96758c1a8,Qualité médiocre. Cassé après deux semaines d'utilisation.,app_store,2024-04-15T03:14:00,1
|
| 295 |
+
03c96e160bc7,The app is full of bugs. Each update makes it worse.,app_store,2024-04-15T05:59:00,1
|
| 296 |
+
9a1f59c27bab,Not worth the price. Very disappointing quality. Score: 2/5.,app_store,2024-04-15T21:57:00,1
|
| 297 |
+
a9015f11104c,製品は説明通りに動作します。特別なものはありません。 スコア: 3/10。,web_form,2024-04-16T02:35:00,3
|
| 298 |
+
12741663077a,Mala calidad. Se rompió después de dos semanas.,twitter,2024-04-16T07:29:00,1
|
| 299 |
+
14100a5c8d69,Experiencia terrible. Esperé 3 semanas sin resultado. Calificación: 1/10.,app_store,2024-04-16T20:53:00,1
|
| 300 |
+
69252e0bbf8d,It's okay for the price point. Nothing to complain about. Overall rating: 3/5.,survey,2024-04-17T04:45:00,3
|
| 301 |
+
501c386ac0cc,Das Produkt funktioniert wie beschrieben. Nichts Besonderes.,play_store,2024-04-17T11:29:00,3
|
| 302 |
+
76dc7fa1c3b5,Product arrived damaged and customer support was unhelpful.,survey,2024-04-18T04:15:00,2
|
| 303 |
+
2cd63c85fdfb,It's okay for the price point. Nothing to complain about.,play_store,2024-04-18T08:14:00,3
|
| 304 |
+
e09a987e999c,El producto funciona como se describe. Nada especial. Calificación: 3/10.,twitter,2024-04-18T03:34:00,3
|
| 305 |
+
d2db59bbc719,Terrible experience. Waited 3 weeks for delivery that never came. Would rate 1/10.,chat,2024-04-19T04:05:00,2
|
| 306 |
+
b7c8cfe17f76,Expérience terrible. J'ai attendu 3 semaines pour rien.,support_ticket,2024-04-19T05:01:00,1
|
| 307 |
+
0d98b2509fdb,"Great quality, fast shipping. Will definitely order again. Overall rating: 4/5.",play_store,2024-04-19T12:23:00,5
|
| 308 |
+
afb29e245af2,Really impressed with the build quality and design.,twitter,2024-04-20T02:57:00,4
|
| 309 |
+
a92617a33970,Functional product. Does what it says.,web_form,2024-04-20T23:25:00,3
|
| 310 |
+
f01979c7e3dd,Poor quality materials. Broke after two weeks of use.,web_form,2024-04-20T08:17:00,1
|
| 311 |
+
9ee8188eec34,Mala calidad. Se rompió después de dos semanas.,app_store,2024-04-21T17:30:00,2
|
| 312 |
+
4d25d21b5d54,Poor quality materials. Broke after two weeks of use.,app_store,2024-04-21T17:50:00,2
|
| 313 |
+
68cec6a195e0,Expérience terrible. J'ai attendu 3 semaines pour rien.,support_ticket,2024-04-21T16:20:00,1
|
| 314 |
+
62a6a3ef328b,¡Me encanta este producto! La mejor compra que he hecho.,web_form,2024-04-22T08:03:00,5
|
| 315 |
+
9db15c8adf74,Average experience. Delivery was on time. Would rate 3/10.,chat,2024-04-22T05:11:00,3
|
| 316 |
+
ddd461ac0e83,It's okay for the price point. Nothing to complain about.,chat,2024-04-23T18:09:00,3
|
| 317 |
+
a8995de033da,Product arrived damaged and customer support was unhelpful. Overall rating: 1/5.,chat,2024-04-23T10:58:00,2
|
| 318 |
+
cf0609c1cfcb,ひどい経験でした。3週間待っても届きませんでした。 スコア: 1/10。,support_ticket,2024-04-23T02:23:00,2
|
| 319 |
+
0b46244aad20,The software crashes constantly. Very frustrating.,web_form,2024-04-24T13:50:00,1
|
| 320 |
+
973f981ca8ee,Mala calidad. Se rompió después de dos semanas. Calificación: 2/10.,app_store,2024-04-24T04:11:00,1
|
| 321 |
+
a7fcf4aa8eb4,Poor quality materials. Broke after two weeks of use.,web_form,2024-04-24T03:59:00,2
|
| 322 |
+
bc6817c71804,Not worth the price. Very disappointing quality. Overall rating: 2/5.,chat,2024-04-25T08:13:00,2
|
| 323 |
+
eecc004d8f5d,Product arrived damaged and customer support was unhelpful. Would rate 2/10.,support_ticket,2024-04-25T15:56:00,2
|
| 324 |
+
822bd418d317,"The new feature update is amazing, exactly what I needed. Overall rating: 4/5.",play_store,2024-04-25T10:16:00,4
|
| 325 |
+
5a608e492af1,Terrible experience. Waited 3 weeks for delivery that never came.,web_form,2024-04-26T02:21:00,2
|
| 326 |
+
2e2078c0d315,"Standard service, met expectations but didn't exceed them. Overall rating: 3/5.",survey,2024-04-26T13:44:00,3
|
| 327 |
+
9992b76025c3,Misleading product description. Nothing like advertised. Overall rating: 1/5.,web_form,2024-04-27T09:52:00,1
|
| 328 |
+
9b9167666561,Very satisfied with the experience. Highly recommend!,play_store,2024-04-27T15:54:00,5
|
| 329 |
+
cacc430f759e,Schreckliche Erfahrung. 3 Wochen gewartet ohne Ergebnis. Note: 1/10.,web_form,2024-04-27T11:11:00,2
|
| 330 |
+
4d2f814dbaa1,Worst customer service I've ever encountered.,play_store,2024-04-28T04:48:00,2
|
| 331 |
+
f494ca790f4b,Functional product. Does what it says. Score: 3/5.,web_form,2024-04-28T21:55:00,3
|
| 332 |
+
15b6dbaade39,Product arrived damaged and customer support was unhelpful.,email,2024-04-28T16:51:00,2
|
| 333 |
+
d2d7176a11e3,製品が破損して届きました。サポートも役に立ちませんでした。,support_ticket,2024-04-29T04:02:00,2
|
| 334 |
+
a50f1450851b,Das Produkt kam beschädigt an und der Support war nutzlos.,play_store,2024-04-29T08:20:00,2
|
| 335 |
+
e1e0d4d62c30,Mala calidad. Se rompió después de dos semanas.,support_ticket,2024-04-29T00:50:00,2
|
| 336 |
+
40efc5eb438f,¡Me encanta este producto! La mejor compra que he hecho.,play_store,2024-04-30T18:49:00,5
|
| 337 |
+
8242512ea8e0,"Standard service, met expectations but didn't exceed them.",email,2024-04-30T15:02:00,3
|
| 338 |
+
50d18a94c572,Muy satisfecho con la experiencia. ¡Lo recomiendo! Calificación: 5/10.,twitter,2024-04-30T08:57:00,4
|
| 339 |
+
7d217031f852,It's okay for the price point. Nothing to complain about.,play_store,2024-05-01T22:43:00,3
|
| 340 |
+
48b0b5be77e4,Expérience terrible. J'ai attendu 3 semaines pour rien. Évaluation: 1/10.,support_ticket,2024-05-01T06:58:00,2
|
| 341 |
+
ea09df97e12f,El producto funciona como se describe. Nada especial. Puntuación: 3/5.,chat,2024-05-02T06:56:00,3
|
| 342 |
+
1d220675c4a0,Excellent value for money. Exceeded my expectations.,web_form,2024-05-02T08:23:00,5
|
| 343 |
+
c61f1802c573,Misleading product description. Nothing like advertised.,web_form,2024-05-02T05:55:00,2
|
| 344 |
+
5cddb2374982,Schlechte Qualität. Nach zwei Wochen kaputt gegangen.,app_store,2024-05-03T02:18:00,1
|
| 345 |
+
33af4f2edc2b,Experiencia terrible. Esperé 3 semanas sin resultado.,chat,2024-05-03T20:14:00,2
|
| 346 |
+
c9cd5f4f05f3,Schlechte Qualität. Nach zwei Wochen kaputt gegangen. Bewertung: 1/5.,survey,2024-05-03T08:38:00,2
|
| 347 |
+
c52d31c416e8,Das Produkt funktioniert wie beschrieben. Nichts Besonderes. Bewertung: 3/5.,twitter,2024-05-04T00:32:00,3
|
| 348 |
+
2730ef05e7fa,Not worth the price. Very disappointing quality.,survey,2024-05-04T20:20:00,2
|
| 349 |
+
10165b116545,Experiencia terrible. Esperé 3 semanas sin resultado.,twitter,2024-05-04T19:25:00,1
|
| 350 |
+
5fb507265130,ひどい経験でした。3週間待っても届きませんでした。,app_store,2024-05-05T12:24:00,2
|
| 351 |
+
623e6a443af5,Misleading product description. Nothing like advertised.,survey,2024-05-05T21:22:00,1
|
| 352 |
+
c87974c8b9a9,Very satisfied with the experience. Highly recommend!,email,2024-05-05T17:48:00,5
|
| 353 |
+
9d1cbde7fbf4,Very satisfied with the experience. Highly recommend!,survey,2024-05-06T10:23:00,5
|
| 354 |
+
7cf995866844,Functional product. Does what it says.,web_form,2024-05-06T19:04:00,3
|
| 355 |
+
250d6266ed15,It's okay for the price point. Nothing to complain about.,chat,2024-05-07T04:08:00,3
|
| 356 |
+
843d41f74d2d,Das Produkt funktioniert wie beschrieben. Nichts Besonderes. Note: 3/10.,twitter,2024-05-07T20:11:00,3
|
| 357 |
+
0f21bc3ec57b,"Great quality, fast shipping. Will definitely order again.",play_store,2024-05-07T05:31:00,4
|
| 358 |
+
7d02a4ee50ef,Misleading product description. Nothing like advertised.,chat,2024-05-08T01:34:00,1
|
| 359 |
+
170b42fb7a19,Le produit fonctionne comme décrit. Rien de spécial. Évaluation: 3/10.,play_store,2024-05-08T09:08:00,3
|
| 360 |
+
ff7859d5527c,Der Kundenservice war unglaublich hilfreich.,twitter,2024-05-08T14:46:00,5
|
| 361 |
+
061fdaedff5f,カスタマーサービスが非常に親切で助かりました。,support_ticket,2024-05-09T05:58:00,4
|
| 362 |
+
1891677b908a,この製品が大好きです!最高の買い物でした。,email,2024-05-09T18:32:00,4
|
| 363 |
+
0647b75ffb0c,Expérience terrible. J'ai attendu 3 semaines pour rien.,app_store,2024-05-09T08:49:00,1
|
| 364 |
+
150aa21654f3,Terrible experience. Waited 3 weeks for delivery that never came.,survey,2024-05-10T12:07:00,1
|
| 365 |
+
aa3faf994758,J'adore ce produit ! Le meilleur achat que j'ai fait.,survey,2024-05-10T21:06:00,4
|
| 366 |
+
1aa8fb020790,Functional product. Does what it says. Overall rating: 3/5.,survey,2024-05-11T19:12:00,3
|
| 367 |
+
150dcc6d227d,Product arrived damaged and customer support was unhelpful.,web_form,2024-05-11T06:44:00,1
|
| 368 |
+
8d7976b8ccfa,J'adore ce produit ! Le meilleur achat que j'ai fait.,email,2024-05-11T21:31:00,4
|
| 369 |
+
6f858c35935c,Worst customer service I've ever encountered.,survey,2024-05-12T14:18:00,1
|
| 370 |
+
848dcd163944,Absolutely love this product! Best purchase I've made.,twitter,2024-05-12T11:51:00,5
|
| 371 |
+
bd84f35ea401,J'adore ce produit ! Le meilleur achat que j'ai fait. Évaluation: 4/10.,survey,2024-05-12T14:39:00,4
|
| 372 |
+
1f2d3e7e9212,"Standard service, met expectations but didn't exceed them.",support_ticket,2024-05-13T14:17:00,3
|
| 373 |
+
b7b60593dd21,Le produit est arrivé endommagé et le support était inutile. Évaluation: 1/10.,web_form,2024-05-13T18:30:00,2
|
| 374 |
+
e0323f0ac8cb,El producto funciona como se describe. Nada especial.,support_ticket,2024-05-13T15:07:00,3
|
| 375 |
+
fa31b2965135,J'adore ce produit ! Le meilleur achat que j'ai fait.,chat,2024-05-14T21:33:00,5
|
| 376 |
+
7c7942ffd19b,Très satisfait de l'expérience. Hautement recommandé !,support_ticket,2024-05-14T21:41:00,4
|
| 377 |
+
5d5374f10b70,素晴らしい品質です。強くお勧めします!,email,2024-05-15T15:55:00,4
|
| 378 |
+
bace1e085071,Very satisfied with the experience. Highly recommend! Overall rating: 4/5.,web_form,2024-05-15T02:29:00,5
|
| 379 |
+
bcb499bcc7aa,Product works as described. Nothing special.,app_store,2024-05-15T23:41:00,3
|
| 380 |
+
8f2c98d4218a,¡Me encanta este producto! La mejor compra que he hecho.,survey,2024-05-16T19:39:00,4
|
| 381 |
+
c6ffab859e8f,Product works as described. Nothing special.,web_form,2024-05-16T14:21:00,3
|
| 382 |
+
065b8c632edd,Le service client était incroyablement utile et efficace. Note: 5/5.,email,2024-05-16T01:57:00,5
|
| 383 |
+
33be87d724fd,Expérience moyenne. Livraison à temps. Évaluation: 3/10.,chat,2024-05-17T06:29:00,3
|
| 384 |
+
6be00bdb9004,Functional product. Does what it says. Would rate 3/10.,app_store,2024-05-17T00:33:00,3
|
| 385 |
+
8caf1b96d74f,Le produit fonctionne comme décrit. Rien de spécial.,email,2024-05-17T05:46:00,3
|
| 386 |
+
18434f96b724,Mala calidad. Se rompió después de dos semanas.,chat,2024-05-18T12:26:00,2
|
| 387 |
+
1e772c657344,製品は説明通りに動作します。特別なものはありません。,survey,2024-05-18T04:26:00,3
|
| 388 |
+
14e97151cbb4,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",email,2024-05-18T21:08:00,5
|
| 389 |
+
ac485b5e58b9,Experiencia promedio. La entrega fue puntual.,support_ticket,2024-05-19T17:18:00,3
|
| 390 |
+
3590608515ea,Absolutely love this product! Best purchase I've made. Overall rating: 4/5.,survey,2024-05-19T01:44:00,4
|
| 391 |
+
d5dc0e77691e,"The new feature update is amazing, exactly what I needed.",chat,2024-05-20T17:24:00,5
|
| 392 |
+
e62786e9542c,It's okay for the price point. Nothing to complain about.,survey,2024-05-20T01:00:00,3
|
| 393 |
+
f471d8302e37,製品は説明通りに動作します。特別なものはありません。,chat,2024-05-20T17:47:00,3
|
| 394 |
+
36a7285498a9,この製品が大好きです!最高の買い物でした。 評価: 4/5。,survey,2024-05-21T13:17:00,4
|
| 395 |
+
a6216310aa51,Not worth the price. Very disappointing quality.,email,2024-05-21T04:02:00,1
|
| 396 |
+
5780155cd18a,Experiencia promedio. La entrega fue puntual.,twitter,2024-05-21T18:35:00,3
|
| 397 |
+
3771c1095a70,素晴らしい品質です。強くお勧めします! 評価: 4/5。,chat,2024-05-22T11:14:00,5
|
| 398 |
+
11acc24032cd,Really impressed with the build quality and design.,play_store,2024-05-22T20:00:00,5
|
| 399 |
+
d9cb669457ef,El servicio al cliente fue increíblemente útil.,twitter,2024-05-22T21:56:00,5
|
| 400 |
+
8af6fe2e3d27,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo. Puntuación: 5/5.",chat,2024-05-23T09:57:00,5
|
| 401 |
+
b933d6cd797e,Functional product. Does what it says.,chat,2024-05-23T20:04:00,3
|
| 402 |
+
8bb6d99a448e,"Standard service, met expectations but didn't exceed them.",email,2024-05-24T02:47:00,3
|
| 403 |
+
490f5a8625ac,Terrible experience. Waited 3 weeks for delivery that never came. Would rate 1/10.,web_form,2024-05-24T13:24:00,2
|
| 404 |
+
421bb398eafd,Customer service was incredibly helpful and resolved my issue quickly.,play_store,2024-05-24T20:46:00,4
|
| 405 |
+
5057dad31f32,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",play_store,2024-05-25T09:38:00,5
|
| 406 |
+
121990128874,Ich liebe dieses Produkt! Bester Kauf den ich je gemacht habe.,email,2024-05-25T17:14:00,5
|
| 407 |
+
07f58e9bb0df,It's okay for the price point. Nothing to complain about.,email,2024-05-25T00:35:00,3
|
| 408 |
+
a933cf020589,¡Me encanta este producto! La mejor compra que he hecho.,twitter,2024-05-26T23:47:00,4
|
| 409 |
+
8ad329d31baa,Das Produkt funktioniert wie beschrieben. Nichts Besonderes. Bewertung: 3/5.,app_store,2024-05-26T05:42:00,3
|
| 410 |
+
4dd9b74f599f,Customer service was incredibly helpful and resolved my issue quickly. Overall rating: 5/5.,chat,2024-05-26T21:41:00,4
|
| 411 |
+
c91e26498430,ひどい経験でした。3週間待っても届きませんでした。,survey,2024-05-27T10:39:00,2
|
| 412 |
+
d12cc8db5170,製品は説明通りに動作します。特別なものはありません。 スコア: 3/10。,twitter,2024-05-27T11:08:00,3
|
| 413 |
+
43bbef2f1e6f,Très satisfait de l'expérience. Hautement recommandé !,play_store,2024-05-27T05:41:00,5
|
| 414 |
+
7773474acd14,Product works as described. Nothing special.,app_store,2024-05-28T00:54:00,3
|
| 415 |
+
72702c18d353,Experiencia promedio. La entrega fue puntual.,support_ticket,2024-05-28T04:23:00,3
|
| 416 |
+
d739055b9726,¡Me encanta este producto! La mejor compra que he hecho.,chat,2024-05-29T16:00:00,4
|
| 417 |
+
a82b33b2364b,"Great quality, fast shipping. Will definitely order again.",app_store,2024-05-29T15:22:00,4
|
| 418 |
+
c4627cd85c9c,"The new feature update is amazing, exactly what I needed. Would rate 4/10.",app_store,2024-05-29T14:56:00,4
|
| 419 |
+
1f60c6d4d727,素晴らしい品質です。強くお勧めします!,email,2024-05-30T23:16:00,5
|
| 420 |
+
072c5c1d6a13,J'adore ce produit ! Le meilleur achat que j'ai fait.,chat,2024-05-30T12:58:00,5
|
| 421 |
+
242fc15d6ffa,"Standard service, met expectations but didn't exceed them.",email,2024-05-30T00:32:00,3
|
| 422 |
+
1517985d27df,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo. Puntuación: 4/5.",support_ticket,2024-05-31T21:14:00,5
|
| 423 |
+
df95218b0d2d,"The new feature update is amazing, exactly what I needed.",support_ticket,2024-05-31T20:53:00,4
|
| 424 |
+
9900e7759b57,El servicio al cliente fue increíblemente útil. Puntuación: 4/5.,email,2024-05-31T08:59:00,5
|
| 425 |
+
ae3d97b9d8ad,Expérience moyenne. Livraison à temps.,web_form,2024-06-01T06:28:00,3
|
| 426 |
+
e300c723cbf5,"Great quality, fast shipping. Will definitely order again.",survey,2024-06-01T05:39:00,4
|
| 427 |
+
d1a826c685e2,El producto funciona como se describe. Nada especial. Puntuación: 3/5.,support_ticket,2024-06-02T06:33:00,3
|
| 428 |
+
e1219697ba88,素晴らしい品質です。強くお勧めします! 評価: 4/5。,app_store,2024-06-02T19:34:00,5
|
| 429 |
+
1e4649494e8a,The team went above and beyond to help me. Outstanding! Score: 5/5.,play_store,2024-06-02T23:58:00,5
|
| 430 |
+
59e70c5d398c,J'adore ce produit ! Le meilleur achat que j'ai fait. Note: 5/5.,app_store,2024-06-03T22:47:00,4
|
| 431 |
+
110dc92921a9,Excellent value for money. Exceeded my expectations.,support_ticket,2024-06-03T03:09:00,5
|
| 432 |
+
1851486dcfa2,It's okay for the price point. Nothing to complain about.,chat,2024-06-03T21:25:00,3
|
| 433 |
+
b4904ca108e9,Muy satisfecho con la experiencia. ¡Lo recomiendo! Puntuación: 5/5.,chat,2024-06-04T21:07:00,5
|
| 434 |
+
d1cabaaad146,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",app_store,2024-06-04T19:44:00,4
|
| 435 |
+
ef3746b1d1a2,Product works as described. Nothing special.,support_ticket,2024-06-04T15:59:00,3
|
| 436 |
+
c24c6da72f48,Absolutely love this product! Best purchase I've made.,survey,2024-06-05T12:39:00,4
|
| 437 |
+
687f221e3dee,Poor quality materials. Broke after two weeks of use.,chat,2024-06-05T16:18:00,1
|
| 438 |
+
3e9ebfbbb39b,Expérience terrible. J'ai attendu 3 semaines pour rien. Évaluation: 1/10.,support_ticket,2024-06-05T12:22:00,1
|
| 439 |
+
72c13722c41e,J'adore ce produit ! Le meilleur achat que j'ai fait.,twitter,2024-06-06T17:18:00,5
|
| 440 |
+
fd59306b1ae1,カスタマーサービスが非常に親切で助かりました。,survey,2024-06-06T18:00:00,4
|
| 441 |
+
a28b9b5eccb8,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",survey,2024-06-07T03:51:00,5
|
| 442 |
+
079f3bc59e94,El producto funciona como se describe. Nada especial.,web_form,2024-06-07T14:43:00,3
|
| 443 |
+
0db2c1f911b2,"The new feature update is amazing, exactly what I needed.",twitter,2024-06-07T07:51:00,5
|
| 444 |
+
7ae50654b135,Very satisfied with the experience. Highly recommend! Overall rating: 4/5.,email,2024-06-08T17:59:00,5
|
| 445 |
+
7a3807dc2020,Der Kundenservice war unglaublich hilfreich. Bewertung: 4/5.,survey,2024-06-08T23:15:00,4
|
| 446 |
+
a5de481553b5,Excellent value for money. Exceeded my expectations. Score: 4/5.,app_store,2024-06-08T21:31:00,4
|
| 447 |
+
57787a914331,Very satisfied with the experience. Highly recommend!,support_ticket,2024-06-09T16:17:00,4
|
| 448 |
+
ee556d2224d9,Très satisfait de l'expérience. Hautement recommandé !,twitter,2024-06-09T01:32:00,5
|
| 449 |
+
b4dc9d0bd3d9,Really impressed with the build quality and design.,chat,2024-06-09T00:00:00,5
|
| 450 |
+
878c2117c4d9,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo. Calificación: 5/10.",twitter,2024-06-10T19:21:00,4
|
| 451 |
+
86dbaa244266,Excellent value for money. Exceeded my expectations.,web_form,2024-06-10T12:17:00,4
|
| 452 |
+
7e548aa56f4d,Le produit fonctionne comme décrit. Rien de spécial.,support_ticket,2024-06-11T07:35:00,3
|
| 453 |
+
8ea6dfdaac77,この製品が大好きです!最高の買い物でした。,support_ticket,2024-06-11T00:11:00,5
|
| 454 |
+
d9475fac1fae,Das Produkt funktioniert wie beschrieben. Nichts Besonderes.,app_store,2024-06-11T09:16:00,3
|
| 455 |
+
a13566e1668a,Worst customer service I've ever encountered.,web_form,2024-06-12T20:06:00,2
|
| 456 |
+
ace332d6c3b8,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",twitter,2024-06-12T13:07:00,4
|
| 457 |
+
a0e253c884d2,It's okay for the price point. Nothing to complain about.,web_form,2024-06-12T15:49:00,3
|
| 458 |
+
ff25c26bfd16,The software crashes constantly. Very frustrating.,twitter,2024-06-13T20:51:00,2
|
| 459 |
+
4e3f46de5fbd,Average experience. Delivery was on time.,survey,2024-06-13T10:20:00,3
|
| 460 |
+
124c46968421,Excellent value for money. Exceeded my expectations.,support_ticket,2024-06-13T16:28:00,4
|
| 461 |
+
e4df0d9d1acb,Average experience. Delivery was on time.,play_store,2024-06-14T23:44:00,3
|
| 462 |
+
5c0ef4a958e7,Really impressed with the build quality and design.,support_ticket,2024-06-14T00:49:00,4
|
| 463 |
+
465dd702ec31,The team went above and beyond to help me. Outstanding!,survey,2024-06-14T07:47:00,5
|
| 464 |
+
996ee71a2644,Ich liebe dieses Produkt! Bester Kauf den ich je gemacht habe. Note: 5/10.,survey,2024-06-15T17:59:00,5
|
| 465 |
+
ebcd9c32c2a6,Le produit fonctionne comme décrit. Rien de spécial.,support_ticket,2024-06-15T14:40:00,3
|
| 466 |
+
a8393d640f33,Customer service was incredibly helpful and resolved my issue quickly.,survey,2024-06-16T11:26:00,4
|
| 467 |
+
2a1ba35ee8cd,El servicio al cliente fue increíblemente útil.,app_store,2024-06-16T15:34:00,5
|
| 468 |
+
a1e0925b2831,"Excellente qualité, livraison rapide. Je recommande vivement !",twitter,2024-06-16T00:35:00,4
|
| 469 |
+
c8cae4d4ad37,Average experience. Delivery was on time. Would rate 3/10.,chat,2024-06-17T02:27:00,3
|
| 470 |
+
1bdaadfd3324,Mala calidad. Se rompió después de dos semanas.,email,2024-06-17T02:11:00,1
|
| 471 |
+
dda4878e3eac,Mala calidad. Se rompió después de dos semanas.,twitter,2024-06-17T15:12:00,1
|
| 472 |
+
10b1d9086a4a,El servicio al cliente fue increíblemente útil.,email,2024-06-18T09:17:00,5
|
| 473 |
+
edc3a7cdd2a3,Really impressed with the build quality and design. Would rate 5/10.,app_store,2024-06-18T14:44:00,5
|
| 474 |
+
c856ddf1fe22,製品は説明通りに動作します。特別なものはありません。 評価: 3/5。,survey,2024-06-18T06:57:00,3
|
| 475 |
+
4f90d45e5c9f,Le service client était incroyablement utile et efficace.,app_store,2024-06-19T14:15:00,4
|
| 476 |
+
f05290b914ca,Poor quality materials. Broke after two weeks of use.,twitter,2024-06-19T21:19:00,1
|
| 477 |
+
0d7a07c75daa,El producto llegó dañado y el soporte no ayudó. Puntuación: 1/5.,play_store,2024-06-20T21:13:00,2
|
| 478 |
+
69152cbaec4f,Very satisfied with the experience. Highly recommend!,chat,2024-06-20T12:50:00,4
|
| 479 |
+
d444eb504b6a,Qualité médiocre. Cassé après deux semaines d'utilisation.,play_store,2024-06-20T11:49:00,1
|
| 480 |
+
9c980249696d,"Excellente qualité, livraison rapide. Je recommande vivement !",app_store,2024-06-21T20:03:00,5
|
| 481 |
+
4f9e1eeea954,Le produit est arrivé endommagé et le support était inutile.,twitter,2024-06-21T10:35:00,1
|
| 482 |
+
c96542d7c843,"Great quality, fast shipping. Will definitely order again. Score: 4/5.",web_form,2024-06-21T09:34:00,4
|
| 483 |
+
027765270430,Really impressed with the build quality and design. Would rate 4/10.,survey,2024-06-22T09:52:00,5
|
| 484 |
+
6121ca8f872b,Muy satisfecho con la experiencia. ¡Lo recomiendo!,twitter,2024-06-22T00:39:00,4
|
| 485 |
+
07c2908a3633,Qualité médiocre. Cassé après deux semaines d'utilisation.,web_form,2024-06-22T06:33:00,2
|
| 486 |
+
e9efb0b27602,¡Me encanta este producto! La mejor compra que he hecho. Puntuación: 4/5.,play_store,2024-06-23T01:12:00,5
|
| 487 |
+
5ea983b64b10,Muy satisfecho con la experiencia. ¡Lo recomiendo!,survey,2024-06-23T16:39:00,4
|
| 488 |
+
162c104bcfa7,J'adore ce produit ! Le meilleur achat que j'ai fait.,web_form,2024-06-23T06:41:00,4
|
| 489 |
+
a8e091488caa,Functional product. Does what it says.,twitter,2024-06-24T13:52:00,3
|
| 490 |
+
ebd01bcb41d3,"Excellente qualité, livraison rapide. Je recommande vivement ! Note: 4/5.",chat,2024-06-24T17:24:00,4
|
| 491 |
+
3ae192e2c3ae,ひどい経験でした。3週間待っても届きませんでした。,twitter,2024-06-25T20:36:00,2
|
| 492 |
+
54729aa27653,Average experience. Delivery was on time.,email,2024-06-25T16:21:00,3
|
| 493 |
+
cac2f6371e53,Experiencia promedio. La entrega fue puntual.,support_ticket,2024-06-25T13:48:00,3
|
| 494 |
+
761cbb4a85d5,Worst customer service I've ever encountered.,support_ticket,2024-06-26T14:28:00,1
|
| 495 |
+
188492cb059b,El producto funciona como se describe. Nada especial.,survey,2024-06-26T09:00:00,3
|
| 496 |
+
5fe5909e7053,It's okay for the price point. Nothing to complain about.,email,2024-06-26T16:35:00,3
|
| 497 |
+
8623f34647f1,Very satisfied with the experience. Highly recommend!,chat,2024-06-27T18:15:00,5
|
| 498 |
+
443844832f8a,"The new feature update is amazing, exactly what I needed.",web_form,2024-06-27T23:20:00,4
|
| 499 |
+
fcfcc04a776a,Le produit fonctionne comme décrit. Rien de spécial.,support_ticket,2024-06-27T09:02:00,3
|
| 500 |
+
1676a3c79ef9,"Standard service, met expectations but didn't exceed them.",chat,2024-06-28T03:55:00,3
|
| 501 |
+
d1d26b71afe9,"Excelente calidad, envío rápido. Definitivamente pediré de nuevo.",chat,2024-06-28T04:20:00,4
|
demo_data/demo_feedback.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
demo_data/feedback_feb2024.csv
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
text,source,timestamp,language
|
| 2 |
+
La actualización más reciente mejoró mucho el rendimiento. ¡Genial!,twitter,2024-02-01 06:43:48,Spanish
|
| 3 |
+
Deux semaines sans réponse du support. C'est inadmissible.,support_ticket,2024-02-01 16:28:13,French
|
| 4 |
+
Seit zwei Wochen keine Antwort vom Support. Das ist inakzeptabel.,support_ticket,2024-02-02 08:09:53,German
|
| 5 |
+
サポートに問い合わせて2週間経ちますが、まだ返答がありません。,twitter,2024-02-03 11:01:03,Japanese
|
| 6 |
+
Seit zwei Wochen keine Antwort vom Support. Das ist inakzeptabel.,chat,2024-02-04 16:42:07,German
|
| 7 |
+
Das neue Design ist gewöhnungsbedürftig. Bin noch unentschieden.,twitter,2024-02-05 12:11:30,German
|
| 8 |
+
"Viel zu viele Werbeanzeigen, selbst in der bezahlten Version. Nicht empfehlenswert.",chat,2024-02-05 13:47:30,German
|
| 9 |
+
Das letzte Update hat viele nützliche Funktionen gebracht. Sehr zufrieden!,support_ticket,2024-02-06 04:50:05,German
|
| 10 |
+
El nuevo diseño es diferente. Aún no sé si me gusta más que el anterior.,twitter,2024-02-06 12:17:41,Spanish
|
| 11 |
+
The app crashes every time I try to export a report. Completely unusable.,chat,2024-02-07 02:12:47,English
|
| 12 |
+
The app crashes every time I try to export a report. Completely unusable.,chat,2024-02-07 10:48:49,English
|
| 13 |
+
"Decent product overall. Nothing exceptional, but it gets the job done.",support_ticket,2024-02-08 16:04:05,English
|
| 14 |
+
La qualité du produit est bien en dessous de ce qui était annoncé.,support_ticket,2024-02-08 16:52:03,French
|
| 15 |
+
The app crashes every time I try to export a report. Completely unusable.,chat,2024-02-09 07:41:30,English
|
| 16 |
+
I was charged twice for my subscription and nobody seems to care.,twitter,2024-02-09 13:46:33,English
|
| 17 |
+
Das Produkt war bei Lieferung beschädigt und die Rückgabe ist kompliziert.,chat,2024-02-09 20:24:45,German
|
| 18 |
+
"Viel zu viele Werbeanzeigen, selbst in der bezahlten Version. Nicht empfehlenswert.",twitter,2024-02-09 23:15:43,German
|
| 19 |
+
Der Kundenservice hat mir innerhalb einer Stunde geholfen. Top!,twitter,2024-02-10 00:03:06,German
|
| 20 |
+
最新のアップデート後、バッテリーの消耗が激しくなりました。非常に困っています。,twitter,2024-02-10 02:29:06,Japanese
|
| 21 |
+
"Die App stürzt ständig ab, seit dem letzten Update. Sehr enttäuschend.",support_ticket,2024-02-10 16:09:18,German
|
| 22 |
+
Perdí todos mis datos tras la migración. Estoy muy decepcionado.,twitter,2024-02-10 17:17:27,Spanish
|
| 23 |
+
The new design is different. Not sure if I prefer it over the old one yet.,twitter,2024-02-10 22:38:40,English
|
| 24 |
+
アプリが頻繁にクラッシュして仕事になりません。早急に修正してください。,chat,2024-02-13 00:27:02,Japanese
|
| 25 |
+
Der Kundenservice hat mir innerhalb einer Stunde geholfen. Top!,chat,2024-02-13 12:10:58,German
|
| 26 |
+
Works as expected. Would appreciate more customization options in future updates.,chat,2024-02-13 19:28:58,English
|
| 27 |
+
"Exactly what I was looking for. Simple, elegant, and powerful.",chat,2024-02-14 06:13:41,English
|
| 28 |
+
基本的な機能は問題なく使えますが、もう少し高度な機能が欲しいです。,support_ticket,2024-02-14 06:57:59,Japanese
|
| 29 |
+
La qualité du produit est bien en dessous de ce qui était annoncé.,twitter,2024-02-15 04:01:04,French
|
| 30 |
+
I was charged twice for my subscription and nobody seems to care.,support_ticket,2024-02-16 01:51:51,English
|
| 31 |
+
アプリが頻繁にクラッシュして仕事になりません。早急に修正してください。,support_ticket,2024-02-16 14:11:36,Japanese
|
| 32 |
+
Connection drops constantly. I can't rely on this for my business anymore.,support_ticket,2024-02-16 19:43:50,English
|
| 33 |
+
Deux semaines sans réponse du support. C'est inadmissible.,twitter,2024-02-17 12:05:49,French
|
| 34 |
+
Way too many ads. I'm paying for premium and still seeing banner ads everywhere.,support_ticket,2024-02-17 20:05:25,English
|
| 35 |
+
基本的な機能は問題なく使えますが、もう少し高度な機能が欲しいです。,support_ticket,2024-02-18 07:12:48,Japanese
|
| 36 |
+
Works as expected. Would appreciate more customization options in future updates.,chat,2024-02-18 13:55:16,English
|
| 37 |
+
アプリが頻繁にクラッシュして仕事になりません。早急に修正してください。,support_ticket,2024-02-19 09:27:42,Japanese
|
| 38 |
+
普通の製品です。特に不満はありませんが、特筆すべき点もありません。,chat,2024-02-19 14:28:16,Japanese
|
| 39 |
+
The latest update fixed every issue I had. Developers really listen to feedback.,support_ticket,2024-02-22 11:53:47,English
|
| 40 |
+
Das Produkt war bei Lieferung beschädigt und die Rückgabe ist kompliziert.,twitter,2024-02-22 17:38:10,German
|
| 41 |
+
Best customer experience I've had in years. The support team truly cares.,chat,2024-02-23 14:57:38,English
|
| 42 |
+
"Ganz okay. Nichts Besonderes, aber es erfüllt seinen Zweck.",support_ticket,2024-02-23 20:00:44,German
|
| 43 |
+
Rapport qualité-prix imbattable. Je suis client fidèle depuis un an.,chat,2024-02-24 00:26:22,French
|
| 44 |
+
"Die App stürzt ständig ab, seit dem letzten Update. Sehr enttäuschend.",twitter,2024-02-24 16:42:15,German
|
| 45 |
+
普通の製品です。特に不満はありませんが、特��すべき点もありません。,twitter,2024-02-26 01:23:17,Japanese
|
| 46 |
+
Great value for the price. I've recommended it to all my coworkers.,chat,2024-02-26 01:35:21,English
|
| 47 |
+
El servicio al cliente fue excelente. Resolvieron mi problema en minutos.,chat,2024-02-26 13:04:52,Spanish
|
| 48 |
+
Honestly disappointed. The features advertised on the website don't actually exist.,chat,2024-02-28 05:42:31,English
|
| 49 |
+
I was charged twice for my subscription and nobody seems to care.,twitter,2024-02-28 06:29:34,English
|
| 50 |
+
基本的な機能は問題なく使えますが、もう少し高度な機能が欲しいです。,support_ticket,2024-02-29 07:26:20,Japanese
|
| 51 |
+
"Viel zu viele Werbeanzeigen, selbst in der bezahlten Version. Nicht empfehlenswert.",twitter,2024-02-29 12:45:41,German
|
demo_data/feedback_jan2024.csv
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
text,source,timestamp,language
|
| 2 |
+
This product has completely changed how I manage my daily workflow. Five stars!,email,2024-01-01 02:24:52,English
|
| 3 |
+
Hervorragende Qualität zum fairen Preis. Kann ich nur weiterempfehlen.,app_store,2024-01-02 19:05:46,German
|
| 4 |
+
La mise à jour a vraiment amélioré les performances. Bravo à l'équipe !,survey,2024-01-02 23:15:52,French
|
| 5 |
+
Shipping was lightning fast and the packaging was eco-friendly. Impressed!,app_store,2024-01-02 23:38:06,English
|
| 6 |
+
Das Produkt war bei Lieferung beschädigt und die Rückgabe ist kompliziert.,email,2024-01-03 01:14:39,German
|
| 7 |
+
"The app works fine for basic tasks, but lacks some advanced features I need.",app_store,2024-01-03 02:08:44,English
|
| 8 |
+
La mise à jour a vraiment amélioré les performances. Bravo à l'équipe !,app_store,2024-01-03 22:10:42,French
|
| 9 |
+
Das neue Design ist gewöhnungsbedürftig. Bin noch unentschieden.,app_store,2024-01-04 23:31:23,German
|
| 10 |
+
"Exactly what I was looking for. Simple, elegant, and powerful.",email,2024-01-05 10:17:47,English
|
| 11 |
+
Muy contento con mi compra. Lo recomiendo sin dudarlo.,survey,2024-01-06 07:19:37,Spanish
|
| 12 |
+
El servicio al cliente fue excelente. Resolvieron mi problema en minutos.,email,2024-01-06 16:30:54,Spanish
|
| 13 |
+
Bin absolut begeistert von der App. Läuft stabil und sieht super aus.,app_store,2024-01-06 22:32:34,German
|
| 14 |
+
Hervorragende Qualität zum fairen Preis. Kann ich nur weiterempfehlen.,survey,2024-01-06 23:03:45,German
|
| 15 |
+
The new design is different. Not sure if I prefer it over the old one yet.,survey,2024-01-07 19:50:30,English
|
| 16 |
+
The search function is broken. It returns completely irrelevant results every time.,survey,2024-01-08 19:03:07,English
|
| 17 |
+
"L'application fonctionne correctement pour les tâches simples, sans plus.",survey,2024-01-10 06:02:34,French
|
| 18 |
+
カスタマーサポートの対応が素晴らしかったです。すぐに問題が解決しました。,survey,2024-01-11 01:15:58,Japanese
|
| 19 |
+
Le nouveau design est différent. Je ne suis pas encore sûr de l'apprécier.,survey,2024-01-11 13:05:25,French
|
| 20 |
+
Great value for the price. I've recommended it to all my coworkers.,app_store,2024-01-12 11:24:52,English
|
| 21 |
+
"Es un producto aceptable. Cumple su función, aunque no destaca en nada.",app_store,2024-01-12 14:11:30,Spanish
|
| 22 |
+
Le service client a été irréprochable. Problème résolu en un clin d'œil.,email,2024-01-12 18:18:53,French
|
| 23 |
+
カスタマーサポートの対応が素晴らしかったです。すぐに問題が解決しました。,app_store,2024-01-12 18:48:49,Japanese
|
| 24 |
+
Le temps de chargement est insupportable. Plusieurs minutes pour ouvrir une page.,email,2024-01-13 08:51:00,French
|
| 25 |
+
"Trop de publicités intrusives. J'ai payé pour la version premium, c'est scandaleux.",survey,2024-01-13 09:09:13,French
|
| 26 |
+
La qualité du produit est bien en dessous de ce qui était annoncé.,email,2024-01-13 17:51:35,French
|
| 27 |
+
J'ai perdu toutes mes données sans aucun avertissement. Très déçu.,survey,2024-01-15 15:40:01,French
|
| 28 |
+
"Die App funktioniert gut für grundlegende Aufgaben, aber es fehlen einige Funktionen.",email,2024-01-16 01:19:00,German
|
| 29 |
+
After the latest update the battery drain is insane. Please fix this ASAP.,survey,2024-01-16 07:10:32,English
|
| 30 |
+
Great value for the price. I've recommended it to all my coworkers.,app_store,2024-01-16 22:31:55,English
|
| 31 |
+
I've been using it for a month now. It's okay but I'm still evaluating alternatives.,email,2024-01-17 08:35:20,English
|
| 32 |
+
The free tier is generous enough for my needs. Might upgrade soon though!,survey,2024-01-17 16:59:56,English
|
| 33 |
+
Really appreciate the attention to detail in the UI. Everything just works.,app_store,2024-01-17 17:11:23,English
|
| 34 |
+
La actualización más reciente mejoró mucho el rendimiento. ¡Genial!,app_store,2024-01-18 02:43:05,Spanish
|
| 35 |
+
Honestly disappointed. The features advertised on the website don't actually exist.,app_store,2024-01-19 13:36:52,English
|
| 36 |
+
Application fantastique ! L'interface est claire et agréable à utiliser.,app_store,2024-01-20 10:49:10,French
|
| 37 |
+
Bin absolut begeistert von der App. Läuft stabil und sieht super aus.,app_store,2024-01-21 05:11:44,German
|
| 38 |
+
Bin absolut begeistert von der App. Läuft stabil und sieht super aus.,survey,2024-01-21 06:10:14,German
|
| 39 |
+
品質が非常に高く、価格以上の価値があります。大満足です。,app_store,2024-01-21 07:38:46,Japanese
|
| 40 |
+
"Me encanta esta aplicación, es muy fácil de usar y funciona de maravilla.",survey,2024-01-23 08:13:00,Spanish
|
| 41 |
+
"Me encanta esta aplicación, es muy fácil de usar y funciona de maravilla.",email,2024-01-23 13:47:26,Spanish
|
| 42 |
+
Bin absolut begeistert von der App. Läuft stabil und sieht super aus.,app_store,2024-01-23 16:32:20,German
|
| 43 |
+
Llevo dos semanas esperando respuesta del soporte técnico. Inaceptable.,survey,2024-01-24 09:55:01,Spanish
|
| 44 |
+
"Exactly what I was looking for. Simple, elegant, and powerful.",app_store,2024-01-24 23:09:45,English
|
| 45 |
+
Best customer experience I've had in years. The support team truly cares.,email,2024-01-26 01:10:05,English
|
| 46 |
+
Application fantastique ! L'interface est claire et agréable à utiliser.,app_store,2024-01-26 11:25:15,French
|
| 47 |
+
Best customer experience I've had in years. The support team truly cares.,email,2024-01-28 05:46:07,English
|
| 48 |
+
Le nouveau design est différent. Je ne suis pas encore sûr de l'apprécier.,email,2024-01-28 21:18:45,French
|
| 49 |
+
"Me encanta esta aplicación, es muy fácil de usar y funciona de maravilla.",app_store,2024-01-29 17:13:00,Spanish
|
| 50 |
+
"Delivery was on time. Product matches the description, nothing more nothing less.",survey,2024-01-30 00:52:14,English
|
| 51 |
+
Le temps de chargement est insupportable. Plusieurs minutes pour ouvrir une page.,email,2024-01-31 03:05:04,French
|
demo_data/feedback_mar2024.csv
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
text,source,timestamp,language
|
| 2 |
+
"Livraison dans les temps. Le produit correspond à la description, sans surprise.",play_store,2024-03-01 04:23:12,French
|
| 3 |
+
Connection drops constantly. I can't rely on this for my business anymore.,email,2024-03-02 14:45:06,English
|
| 4 |
+
基本的な機能は問題なく使えますが、もう少し高度な機能が欲しいです。,email,2024-03-03 03:03:07,Japanese
|
| 5 |
+
Muy contento con mi compra. Lo recomiendo sin dudarlo.,web_form,2024-03-03 15:41:45,Spanish
|
| 6 |
+
It's a solid tool for beginners. Power users might find it a bit limited.,play_store,2024-03-04 06:31:15,English
|
| 7 |
+
Deux semaines sans réponse du support. C'est inadmissible.,play_store,2024-03-04 08:25:31,French
|
| 8 |
+
Honestly disappointed. The features advertised on the website don't actually exist.,web_form,2024-03-05 06:57:00,English
|
| 9 |
+
Absolutely love this app! The interface is so intuitive and responsive.,email,2024-03-05 17:22:20,English
|
| 10 |
+
アップデートで動作がさらに快適になりました。開発チームに感謝します。,email,2024-03-05 17:49:31,Japanese
|
| 11 |
+
"Livraison dans les temps. Le produit correspond à la description, sans surprise.",email,2024-03-06 02:47:35,French
|
| 12 |
+
アップデートで動作がさらに快適になりました。開発チームに感謝します。,email,2024-03-07 03:33:26,Japanese
|
| 13 |
+
I've been using it for a month now. It's okay but I'm still evaluating alternatives.,web_form,2024-03-07 16:59:24,English
|
| 14 |
+
L'application plante à chaque ouverture depuis la dernière mise à jour.,play_store,2024-03-07 20:52:03,French
|
| 15 |
+
Honestly disappointed. The features advertised on the website don't actually exist.,email,2024-03-09 08:56:28,English
|
| 16 |
+
"Delivery was on time. Product matches the description, nothing more nothing less.",web_form,2024-03-09 10:09:21,English
|
| 17 |
+
Great value for the price. I've recommended it to all my coworkers.,play_store,2024-03-09 15:01:22,English
|
| 18 |
+
"Delivery was on time. Product matches the description, nothing more nothing less.",play_store,2024-03-11 03:33:01,English
|
| 19 |
+
The search function is broken. It returns completely irrelevant results every time.,web_form,2024-03-13 01:18:34,English
|
| 20 |
+
Shipping was lightning fast and the packaging was eco-friendly. Impressed!,play_store,2024-03-13 01:46:14,English
|
| 21 |
+
Das Produkt war bei Lieferung beschädigt und die Rückgabe ist kompliziert.,play_store,2024-03-14 12:04:02,German
|
| 22 |
+
Producto de gran calidad. Superó todas mis expectativas.,email,2024-03-15 01:35:10,Spanish
|
| 23 |
+
J'ai perdu toutes mes données sans aucun avertissement. Très déçu.,web_form,2024-03-16 03:14:33,French
|
| 24 |
+
Le temps de chargement est insupportable. Plusieurs minutes pour ouvrir une page.,email,2024-03-16 05:10:15,French
|
| 25 |
+
"L'application fonctionne correctement pour les tâches simples, sans plus.",play_store,2024-03-17 12:21:44,French
|
| 26 |
+
This product has completely changed how I manage my daily workflow. Five stars!,email,2024-03-17 13:23:16,English
|
| 27 |
+
It's a solid tool for beginners. Power users might find it a bit limited.,play_store,2024-03-17 16:26:09,English
|
| 28 |
+
The onboarding process was seamless. I was up and running in minutes.,play_store,2024-03-17 19:42:09,English
|
| 29 |
+
"Exactly what I was looking for. Simple, elegant, and powerful.",web_form,2024-03-18 11:35:35,English
|
| 30 |
+
Le nouveau design est différent. Je ne suis pas encore sûr de l'apprécier.,email,2024-03-19 00:40:37,French
|
| 31 |
+
The search function is broken. It returns completely irrelevant results every time.,email,2024-03-19 22:05:11,English
|
| 32 |
+
L'application plante à chaque ouverture depuis la dernière mise à jour.,email,2024-03-21 01:46:19,French
|
| 33 |
+
新しいデザインは慣れが必要です。前の方が良かったかもしれません。,email,2024-03-21 13:16:17,Japanese
|
| 34 |
+
"Delivery was on time. Product matches the description, nothing more nothing less.",email,2024-03-21 23:56:46,English
|
| 35 |
+
"Es un producto aceptable. Cumple su función, aunque no destaca en nada.",web_form,2024-03-22 14:42:41,Spanish
|
| 36 |
+
The free tier is generous enough for my needs. Might upgrade soon though!,web_form,2024-03-22 21:34:32,English
|
| 37 |
+
"Lost all my data after the migration. No warning, no backup option. Furious.",email,2024-03-23 06:27:27,English
|
| 38 |
+
The new design is different. Not sure if I prefer it over the old one yet.,web_form,2024-03-23 08:38:07,English
|
| 39 |
+
"Trop de publicités intrusives. J'ai payé pour la version premium, c'est scandaleux.",email,2024-03-24 12:04:27,French
|
| 40 |
+
The free tier is generous enough for my needs. Might upgrade soon though!,play_store,2024-03-24 18:34:19,English
|
| 41 |
+
Shipping was lightning fast and the packaging was eco-friendly. Impressed!,play_store,2024-03-25 17:35:13,English
|
| 42 |
+
The UI redesign is awful. Everything I need is now buried under three menus.,email,2024-03-25 23:18:45,English
|
| 43 |
+
アップデートで動作がさらに快適になりました。開発チームに感謝します。,play_store,2024-03-26 21:10:24,Japanese
|
| 44 |
+
Absolutely love this app! The interface is so intuitive and responsive.,play_store,2024-03-27 09:06:31,English
|
| 45 |
+
Très satisfait de la qualité du produit. Je le recommande vivement.,play_store,2024-03-27 09:40:27,French
|
| 46 |
+
Le temps de chargement est insupportable. Plusieurs minutes pour ouvrir une page.,web_form,2024-03-27 13:24:30,French
|
| 47 |
+
The onboarding process was seamless. I was up and running in minutes.,web_form,2024-03-27 20:56:13,English
|
| 48 |
+
Customer support responded within an hour and solved my issue on the first try.,email,2024-03-28 06:47:58,English
|
| 49 |
+
Bin absolut begeistert von der App. Läuft stabil und sieht super aus.,play_store,2024-03-28 07:56:00,German
|
| 50 |
+
このアプリは本当に使いやすくて、毎日愛用しています。おすすめです!,web_form,2024-03-30 10:30:39,Japanese
|
| 51 |
+
Works as expected. Would appreciate more customization options in future updates.,play_store,2024-03-31 21:29:36,English
|
frontend/Dockerfile
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM node:20-alpine AS build
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
|
| 5 |
+
COPY package.json package-lock.json* ./
|
| 6 |
+
RUN npm ci
|
| 7 |
+
|
| 8 |
+
COPY . .
|
| 9 |
+
RUN npm run build
|
| 10 |
+
|
| 11 |
+
FROM nginx:alpine
|
| 12 |
+
|
| 13 |
+
COPY --from=build /app/dist /usr/share/nginx/html
|
| 14 |
+
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
| 15 |
+
|
| 16 |
+
EXPOSE 80
|
| 17 |
+
|
| 18 |
+
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
|
| 19 |
+
CMD wget -qO- http://localhost:80/ || exit 1
|
frontend/eslint.config.js
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import js from '@eslint/js';
|
| 2 |
+
import globals from 'globals';
|
| 3 |
+
import reactHooks from 'eslint-plugin-react-hooks';
|
| 4 |
+
import reactRefresh from 'eslint-plugin-react-refresh';
|
| 5 |
+
import tseslint from 'typescript-eslint';
|
| 6 |
+
|
| 7 |
+
export default tseslint.config(
|
| 8 |
+
{ ignores: ['dist', 'coverage'] },
|
| 9 |
+
{
|
| 10 |
+
extends: [js.configs.recommended, ...tseslint.configs.recommended],
|
| 11 |
+
files: ['**/*.{ts,tsx}'],
|
| 12 |
+
languageOptions: {
|
| 13 |
+
ecmaVersion: 2020,
|
| 14 |
+
globals: globals.browser,
|
| 15 |
+
},
|
| 16 |
+
plugins: {
|
| 17 |
+
'react-hooks': reactHooks,
|
| 18 |
+
'react-refresh': reactRefresh,
|
| 19 |
+
},
|
| 20 |
+
rules: {
|
| 21 |
+
...reactHooks.configs.recommended.rules,
|
| 22 |
+
'react-refresh/only-export-components': ['warn', { allowConstantExport: true }],
|
| 23 |
+
'@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }],
|
| 24 |
+
},
|
| 25 |
+
}
|
| 26 |
+
);
|
frontend/index.html
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8" />
|
| 5 |
+
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
| 6 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
| 7 |
+
<title>Topic Analysis Dashboard</title>
|
| 8 |
+
</head>
|
| 9 |
+
<body>
|
| 10 |
+
<div id="root"></div>
|
| 11 |
+
<script type="module" src="/src/main.tsx"></script>
|
| 12 |
+
</body>
|
| 13 |
+
</html>
|
frontend/nginx.conf
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
server {
|
| 2 |
+
listen 80;
|
| 3 |
+
server_name localhost;
|
| 4 |
+
root /usr/share/nginx/html;
|
| 5 |
+
index index.html;
|
| 6 |
+
|
| 7 |
+
location / {
|
| 8 |
+
try_files $uri $uri/ /index.html;
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
location /assets/ {
|
| 12 |
+
expires 1y;
|
| 13 |
+
add_header Cache-Control "public, immutable";
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
gzip on;
|
| 17 |
+
gzip_types text/plain text/css application/json application/javascript text/xml;
|
| 18 |
+
}
|
frontend/package-lock.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
frontend/package.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "topic-analysis-frontend",
|
| 3 |
+
"private": true,
|
| 4 |
+
"version": "1.0.0",
|
| 5 |
+
"type": "module",
|
| 6 |
+
"scripts": {
|
| 7 |
+
"dev": "vite",
|
| 8 |
+
"build": "tsc -b && vite build",
|
| 9 |
+
"lint": "eslint .",
|
| 10 |
+
"format": "prettier --write 'src/**/*.{ts,tsx,css}'",
|
| 11 |
+
"preview": "vite preview",
|
| 12 |
+
"test": "vitest",
|
| 13 |
+
"test:coverage": "vitest --coverage"
|
| 14 |
+
},
|
| 15 |
+
"dependencies": {
|
| 16 |
+
"react": "18.3.1",
|
| 17 |
+
"react-dom": "18.3.1",
|
| 18 |
+
"react-router-dom": "7.1.1",
|
| 19 |
+
"recharts": "2.15.0",
|
| 20 |
+
"d3-force": "3.0.0",
|
| 21 |
+
"d3-selection": "3.0.0",
|
| 22 |
+
"d3-zoom": "3.0.0",
|
| 23 |
+
"lucide-react": "0.468.0",
|
| 24 |
+
"clsx": "2.1.1",
|
| 25 |
+
"date-fns": "4.1.0"
|
| 26 |
+
},
|
| 27 |
+
"devDependencies": {
|
| 28 |
+
"@eslint/js": "9.17.0",
|
| 29 |
+
"@testing-library/jest-dom": "6.6.3",
|
| 30 |
+
"@testing-library/react": "16.1.0",
|
| 31 |
+
"@testing-library/user-event": "14.5.2",
|
| 32 |
+
"@types/d3-force": "3.0.10",
|
| 33 |
+
"@types/d3-selection": "3.0.11",
|
| 34 |
+
"@types/d3-zoom": "3.0.8",
|
| 35 |
+
"@types/react": "18.3.18",
|
| 36 |
+
"@types/react-dom": "18.3.5",
|
| 37 |
+
"@vitejs/plugin-react": "4.3.4",
|
| 38 |
+
"eslint": "9.17.0",
|
| 39 |
+
"eslint-plugin-react-hooks": "5.1.0",
|
| 40 |
+
"eslint-plugin-react-refresh": "0.4.16",
|
| 41 |
+
"globals": "15.14.0",
|
| 42 |
+
"jsdom": "25.0.1",
|
| 43 |
+
"msw": "2.7.0",
|
| 44 |
+
"prettier": "3.4.2",
|
| 45 |
+
"typescript": "5.7.2",
|
| 46 |
+
"typescript-eslint": "8.18.2",
|
| 47 |
+
"vite": "6.0.5",
|
| 48 |
+
"vitest": "2.1.8",
|
| 49 |
+
"@vitest/coverage-v8": "2.1.8"
|
| 50 |
+
}
|
| 51 |
+
}
|
frontend/src/App.tsx
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { BrowserRouter, Routes, Route } from 'react-router-dom';
|
| 2 |
+
import { Sidebar } from './components/layout/Sidebar';
|
| 3 |
+
import { DashboardPage } from './pages/DashboardPage';
|
| 4 |
+
import { UploadPage } from './pages/UploadPage';
|
| 5 |
+
import { DataQualityPage } from './pages/DataQualityPage';
|
| 6 |
+
import { ComparePage } from './pages/ComparePage';
|
| 7 |
+
import { SettingsPage } from './pages/SettingsPage';
|
| 8 |
+
import { AnalysisProvider } from './hooks/useAnalysis';
|
| 9 |
+
import { useTheme } from './hooks/useTheme';
|
| 10 |
+
import './styles/globals.css';
|
| 11 |
+
|
| 12 |
+
export default function App() {
|
| 13 |
+
const { theme, toggleTheme } = useTheme();
|
| 14 |
+
|
| 15 |
+
return (
|
| 16 |
+
<BrowserRouter>
|
| 17 |
+
<AnalysisProvider>
|
| 18 |
+
<div className="app-layout">
|
| 19 |
+
<Sidebar theme={theme} onToggleTheme={toggleTheme} />
|
| 20 |
+
<main className="main-content">
|
| 21 |
+
<Routes>
|
| 22 |
+
<Route path="/" element={<DashboardPage />} />
|
| 23 |
+
<Route path="/upload" element={<UploadPage />} />
|
| 24 |
+
<Route path="/quality" element={<DataQualityPage />} />
|
| 25 |
+
<Route path="/compare" element={<ComparePage />} />
|
| 26 |
+
<Route path="/settings" element={<SettingsPage />} />
|
| 27 |
+
</Routes>
|
| 28 |
+
</main>
|
| 29 |
+
</div>
|
| 30 |
+
</AnalysisProvider>
|
| 31 |
+
</BrowserRouter>
|
| 32 |
+
);
|
| 33 |
+
}
|
frontend/src/__mocks__/handlers.ts
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { http, HttpResponse } from 'msw';
|
| 2 |
+
import { setupServer } from 'msw/node';
|
| 3 |
+
import type { AnalysisResult, JobStatus } from '../types';
|
| 4 |
+
|
| 5 |
+
const mockJobStatus: JobStatus = {
|
| 6 |
+
job_id: 'test-job-1',
|
| 7 |
+
status: 'completed',
|
| 8 |
+
progress: 1.0,
|
| 9 |
+
message: 'Analysis complete',
|
| 10 |
+
created_at: '2024-01-01T00:00:00Z',
|
| 11 |
+
completed_at: '2024-01-01T00:05:00Z',
|
| 12 |
+
};
|
| 13 |
+
|
| 14 |
+
const mockAnalysisResult: AnalysisResult = {
|
| 15 |
+
job_id: 'test-job-1',
|
| 16 |
+
status: 'completed',
|
| 17 |
+
created_at: '2024-01-01T00:00:00Z',
|
| 18 |
+
completed_at: '2024-01-01T00:05:00Z',
|
| 19 |
+
total_entries: 3,
|
| 20 |
+
entries: [
|
| 21 |
+
{
|
| 22 |
+
id: '1',
|
| 23 |
+
text: 'Great product!',
|
| 24 |
+
source: 'survey',
|
| 25 |
+
timestamp: '2024-01-01T00:00:00Z',
|
| 26 |
+
sentiment: { label: 'positive', score: 0.9, confidence: 0.95 },
|
| 27 |
+
language: { language: 'en', confidence: 0.99, method: 'langdetect' },
|
| 28 |
+
topic_id: 0,
|
| 29 |
+
topic_label: 'Product Quality',
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
id: '2',
|
| 33 |
+
text: 'Terrible service',
|
| 34 |
+
source: 'email',
|
| 35 |
+
timestamp: '2024-01-02T00:00:00Z',
|
| 36 |
+
sentiment: { label: 'negative', score: 0.2, confidence: 0.88 },
|
| 37 |
+
language: { language: 'en', confidence: 0.98, method: 'langdetect' },
|
| 38 |
+
topic_id: 1,
|
| 39 |
+
topic_label: 'Customer Service',
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
id: '3',
|
| 43 |
+
text: 'It works fine',
|
| 44 |
+
source: 'chat',
|
| 45 |
+
timestamp: '2024-01-03T00:00:00Z',
|
| 46 |
+
sentiment: { label: 'neutral', score: 0.5, confidence: 0.7 },
|
| 47 |
+
language: { language: 'en', confidence: 0.95, method: 'langdetect' },
|
| 48 |
+
topic_id: 0,
|
| 49 |
+
topic_label: 'Product Quality',
|
| 50 |
+
},
|
| 51 |
+
],
|
| 52 |
+
topics: [
|
| 53 |
+
{
|
| 54 |
+
topic_id: 0,
|
| 55 |
+
label: 'Product Quality',
|
| 56 |
+
keywords: ['product', 'quality', 'great'],
|
| 57 |
+
size: 2,
|
| 58 |
+
avg_sentiment: 0.7,
|
| 59 |
+
sentiment_distribution: { positive: 1, neutral: 1, negative: 0 },
|
| 60 |
+
languages: { en: 2 },
|
| 61 |
+
representative_docs: ['Great product!', 'It works fine'],
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
topic_id: 1,
|
| 65 |
+
label: 'Customer Service',
|
| 66 |
+
keywords: ['service', 'support', 'help'],
|
| 67 |
+
size: 1,
|
| 68 |
+
avg_sentiment: 0.2,
|
| 69 |
+
sentiment_distribution: { positive: 0, neutral: 0, negative: 1 },
|
| 70 |
+
languages: { en: 1 },
|
| 71 |
+
representative_docs: ['Terrible service'],
|
| 72 |
+
},
|
| 73 |
+
],
|
| 74 |
+
sentiment_trends: [
|
| 75 |
+
{
|
| 76 |
+
period: '2024-01-01',
|
| 77 |
+
avg_sentiment: 0.9,
|
| 78 |
+
count: 1,
|
| 79 |
+
positive: 1,
|
| 80 |
+
negative: 0,
|
| 81 |
+
neutral: 0,
|
| 82 |
+
confidence_lower: 0.8,
|
| 83 |
+
confidence_upper: 1.0,
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
period: '2024-01-02',
|
| 87 |
+
avg_sentiment: 0.2,
|
| 88 |
+
count: 1,
|
| 89 |
+
positive: 0,
|
| 90 |
+
negative: 1,
|
| 91 |
+
neutral: 0,
|
| 92 |
+
confidence_lower: 0.1,
|
| 93 |
+
confidence_upper: 0.3,
|
| 94 |
+
},
|
| 95 |
+
],
|
| 96 |
+
topic_graph: {
|
| 97 |
+
nodes: [
|
| 98 |
+
{
|
| 99 |
+
topic_id: 0,
|
| 100 |
+
label: 'Product Quality',
|
| 101 |
+
keywords: ['product'],
|
| 102 |
+
size: 2,
|
| 103 |
+
avg_sentiment: 0.7,
|
| 104 |
+
sentiment_distribution: { positive: 1, neutral: 1 },
|
| 105 |
+
languages: { en: 2 },
|
| 106 |
+
representative_docs: [],
|
| 107 |
+
},
|
| 108 |
+
],
|
| 109 |
+
links: [],
|
| 110 |
+
},
|
| 111 |
+
data_quality: {
|
| 112 |
+
total_entries: 3,
|
| 113 |
+
low_confidence_count: 0,
|
| 114 |
+
low_confidence_entries: [],
|
| 115 |
+
mixed_language_count: 0,
|
| 116 |
+
mixed_language_entries: [],
|
| 117 |
+
duplicate_count: 0,
|
| 118 |
+
duplicate_entries: [],
|
| 119 |
+
avg_confidence: 0.843,
|
| 120 |
+
language_distribution: { en: 3 },
|
| 121 |
+
},
|
| 122 |
+
anomalies: [],
|
| 123 |
+
summary: {
|
| 124 |
+
total_entries: 3,
|
| 125 |
+
avg_sentiment: 0.533,
|
| 126 |
+
dominant_sentiment: 'positive',
|
| 127 |
+
num_topics: 2,
|
| 128 |
+
top_topics: [
|
| 129 |
+
{ topic_id: 0, label: 'Product Quality', keywords: ['product'], size: 2 },
|
| 130 |
+
{ topic_id: 1, label: 'Customer Service', keywords: ['service'], size: 1 },
|
| 131 |
+
],
|
| 132 |
+
languages_detected: ['en'],
|
| 133 |
+
date_range: { start: '2024-01-01', end: '2024-01-03' },
|
| 134 |
+
},
|
| 135 |
+
};
|
| 136 |
+
|
| 137 |
+
export const handlers = [
|
| 138 |
+
http.get('/api/v1/jobs', () => {
|
| 139 |
+
return HttpResponse.json([mockJobStatus]);
|
| 140 |
+
}),
|
| 141 |
+
|
| 142 |
+
http.get('/api/v1/jobs/:jobId', () => {
|
| 143 |
+
return HttpResponse.json(mockAnalysisResult);
|
| 144 |
+
}),
|
| 145 |
+
|
| 146 |
+
http.get('/api/v1/jobs/:jobId/status', () => {
|
| 147 |
+
return HttpResponse.json(mockJobStatus);
|
| 148 |
+
}),
|
| 149 |
+
|
| 150 |
+
http.post('/api/v1/upload', () => {
|
| 151 |
+
return HttpResponse.json(mockJobStatus);
|
| 152 |
+
}),
|
| 153 |
+
|
| 154 |
+
http.post('/api/v1/jobs/:jobId/filter', () => {
|
| 155 |
+
return HttpResponse.json({
|
| 156 |
+
total: 3,
|
| 157 |
+
page: 1,
|
| 158 |
+
entries: mockAnalysisResult.entries,
|
| 159 |
+
});
|
| 160 |
+
}),
|
| 161 |
+
|
| 162 |
+
http.post('/api/v1/jobs/:jobId/compare', () => {
|
| 163 |
+
return HttpResponse.json({
|
| 164 |
+
segment_a: mockAnalysisResult.summary,
|
| 165 |
+
segment_b: mockAnalysisResult.summary,
|
| 166 |
+
sentiment_delta: 0.1,
|
| 167 |
+
topic_changes: [],
|
| 168 |
+
new_topics: [],
|
| 169 |
+
disappeared_topics: [],
|
| 170 |
+
});
|
| 171 |
+
}),
|
| 172 |
+
|
| 173 |
+
http.get('/health', () => {
|
| 174 |
+
return HttpResponse.json({
|
| 175 |
+
status: 'healthy',
|
| 176 |
+
version: '1.0.0',
|
| 177 |
+
models_loaded: true,
|
| 178 |
+
redis_connected: true,
|
| 179 |
+
uptime_seconds: 100,
|
| 180 |
+
});
|
| 181 |
+
}),
|
| 182 |
+
];
|
| 183 |
+
|
| 184 |
+
export const server = setupServer(...handlers);
|
| 185 |
+
export { mockAnalysisResult, mockJobStatus };
|
frontend/src/__tests__/components.test.tsx
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest';
|
| 2 |
+
import { render, screen } from '@testing-library/react';
|
| 3 |
+
import { BrowserRouter } from 'react-router-dom';
|
| 4 |
+
import { server } from '../__mocks__/handlers';
|
| 5 |
+
import { Sidebar } from '../components/layout/Sidebar';
|
| 6 |
+
import { Alert } from '../components/common/Alert';
|
| 7 |
+
import { Skeleton } from '../components/common/Skeleton';
|
| 8 |
+
import { DataQualityPanel } from '../components/quality/DataQualityPanel';
|
| 9 |
+
import type { DataQualityReport } from '../types';
|
| 10 |
+
|
| 11 |
+
beforeAll(() => server.listen());
|
| 12 |
+
afterEach(() => server.resetHandlers());
|
| 13 |
+
afterAll(() => server.close());
|
| 14 |
+
|
| 15 |
+
describe('Sidebar', () => {
|
| 16 |
+
it('renders navigation items', () => {
|
| 17 |
+
render(
|
| 18 |
+
<BrowserRouter>
|
| 19 |
+
<Sidebar theme="dark" onToggleTheme={() => {}} />
|
| 20 |
+
</BrowserRouter>,
|
| 21 |
+
);
|
| 22 |
+
|
| 23 |
+
expect(screen.getByText('Dashboard')).toBeInTheDocument();
|
| 24 |
+
expect(screen.getByText('Upload Data')).toBeInTheDocument();
|
| 25 |
+
expect(screen.getByText('Data Quality')).toBeInTheDocument();
|
| 26 |
+
expect(screen.getByText('Compare')).toBeInTheDocument();
|
| 27 |
+
});
|
| 28 |
+
|
| 29 |
+
it('renders theme toggle button', () => {
|
| 30 |
+
render(
|
| 31 |
+
<BrowserRouter>
|
| 32 |
+
<Sidebar theme="dark" onToggleTheme={() => {}} />
|
| 33 |
+
</BrowserRouter>,
|
| 34 |
+
);
|
| 35 |
+
|
| 36 |
+
expect(screen.getByText('Light Mode')).toBeInTheDocument();
|
| 37 |
+
});
|
| 38 |
+
|
| 39 |
+
it('shows dark mode text when theme is light', () => {
|
| 40 |
+
render(
|
| 41 |
+
<BrowserRouter>
|
| 42 |
+
<Sidebar theme="light" onToggleTheme={() => {}} />
|
| 43 |
+
</BrowserRouter>,
|
| 44 |
+
);
|
| 45 |
+
|
| 46 |
+
expect(screen.getByText('Dark Mode')).toBeInTheDocument();
|
| 47 |
+
});
|
| 48 |
+
});
|
| 49 |
+
|
| 50 |
+
describe('Alert', () => {
|
| 51 |
+
it('renders danger alert', () => {
|
| 52 |
+
render(<Alert type="danger" message="Something went wrong" />);
|
| 53 |
+
expect(screen.getByText('Something went wrong')).toBeInTheDocument();
|
| 54 |
+
});
|
| 55 |
+
|
| 56 |
+
it('renders success alert', () => {
|
| 57 |
+
render(<Alert type="success" message="Operation completed" />);
|
| 58 |
+
expect(screen.getByText('Operation completed')).toBeInTheDocument();
|
| 59 |
+
});
|
| 60 |
+
|
| 61 |
+
it('calls onDismiss when close button clicked', async () => {
|
| 62 |
+
const onDismiss = vi.fn();
|
| 63 |
+
render(<Alert type="warning" message="Warning" onDismiss={onDismiss} />);
|
| 64 |
+
|
| 65 |
+
const dismissBtn = screen.getByLabelText('Dismiss');
|
| 66 |
+
dismissBtn.click();
|
| 67 |
+
expect(onDismiss).toHaveBeenCalledOnce();
|
| 68 |
+
});
|
| 69 |
+
});
|
| 70 |
+
|
| 71 |
+
describe('Skeleton', () => {
|
| 72 |
+
it('renders loading skeleton', () => {
|
| 73 |
+
render(<Skeleton variant="text" count={3} />);
|
| 74 |
+
const skeletons = screen.getAllByRole('status');
|
| 75 |
+
expect(skeletons).toHaveLength(3);
|
| 76 |
+
});
|
| 77 |
+
});
|
| 78 |
+
|
| 79 |
+
describe('DataQualityPanel', () => {
|
| 80 |
+
const mockReport: DataQualityReport = {
|
| 81 |
+
total_entries: 100,
|
| 82 |
+
low_confidence_count: 5,
|
| 83 |
+
low_confidence_entries: ['1', '2', '3', '4', '5'],
|
| 84 |
+
mixed_language_count: 3,
|
| 85 |
+
mixed_language_entries: ['6', '7', '8'],
|
| 86 |
+
duplicate_count: 2,
|
| 87 |
+
duplicate_entries: ['9', '10'],
|
| 88 |
+
avg_confidence: 0.85,
|
| 89 |
+
language_distribution: { en: 80, es: 12, fr: 8 },
|
| 90 |
+
};
|
| 91 |
+
|
| 92 |
+
it('renders quality stats', () => {
|
| 93 |
+
render(<DataQualityPanel report={mockReport} />);
|
| 94 |
+
|
| 95 |
+
expect(screen.getByText('5')).toBeInTheDocument();
|
| 96 |
+
expect(screen.getByText('3')).toBeInTheDocument();
|
| 97 |
+
expect(screen.getByText('2')).toBeInTheDocument();
|
| 98 |
+
});
|
| 99 |
+
|
| 100 |
+
it('shows health score', () => {
|
| 101 |
+
render(<DataQualityPanel report={mockReport} />);
|
| 102 |
+
expect(screen.getByText('Data Health Score')).toBeInTheDocument();
|
| 103 |
+
});
|
| 104 |
+
|
| 105 |
+
it('displays language distribution', () => {
|
| 106 |
+
render(<DataQualityPanel report={mockReport} />);
|
| 107 |
+
expect(screen.getByText('en: 80')).toBeInTheDocument();
|
| 108 |
+
expect(screen.getByText('es: 12')).toBeInTheDocument();
|
| 109 |
+
});
|
| 110 |
+
|
| 111 |
+
it('shows warning when issues exist', () => {
|
| 112 |
+
render(<DataQualityPanel report={mockReport} />);
|
| 113 |
+
expect(screen.getByText(/data quality issue/i)).toBeInTheDocument();
|
| 114 |
+
});
|
| 115 |
+
});
|