logger and main classes for fe and be
Browse files- app.py +72 -0
- main.py +23 -0
- src/fe_handler.py +25 -0
- src/logger.py +46 -0
app.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
from contextlib import asynccontextmanager
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from dotenv import load_dotenv
|
| 5 |
+
from fastapi import FastAPI
|
| 6 |
+
from src.models.hf_download import download_all
|
| 7 |
+
from src.pipelines.predict_pipeline import PredictPipeline
|
| 8 |
+
from src.pipelines.predict_all_pipeline import PredictAllPipeline
|
| 9 |
+
from src.pipelines.fasttext_pipeline import FastTextPipeline
|
| 10 |
+
from src.api.health import router as health_router
|
| 11 |
+
from src.api.predict import router as predict_router
|
| 12 |
+
from src.api.predict_all import router as predict_all_router
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
load_dotenv()
|
| 16 |
+
log = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
MODELS_ROOT = Path("models")
|
| 19 |
+
VALID_MODES = ("marker", "qa_m", "qa_b")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@asynccontextmanager
|
| 23 |
+
async def lifespan(app: FastAPI):
|
| 24 |
+
downloaded = download_all()
|
| 25 |
+
|
| 26 |
+
all_pipeline = PredictAllPipeline()
|
| 27 |
+
|
| 28 |
+
for mode in VALID_MODES:
|
| 29 |
+
local_dir = downloaded.get(mode)
|
| 30 |
+
if local_dir is None:
|
| 31 |
+
local_dir = MODELS_ROOT / mode
|
| 32 |
+
|
| 33 |
+
onnx_path = local_dir / "model.onnx"
|
| 34 |
+
if not onnx_path.exists():
|
| 35 |
+
log.warning(f"{mode} not available — skipping")
|
| 36 |
+
continue
|
| 37 |
+
|
| 38 |
+
log.info(f"Loading PredictPipeline ({mode})")
|
| 39 |
+
app.state.__dict__[f"pipeline_{mode}"] = PredictPipeline(
|
| 40 |
+
onnx_path=onnx_path,
|
| 41 |
+
mode=mode,
|
| 42 |
+
model_name=mode,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
all_pipeline.add_model(mode, onnx_path, mode)
|
| 46 |
+
|
| 47 |
+
fasttext_path = MODELS_ROOT / "fasttext" / "model.bin"
|
| 48 |
+
if fasttext_path.exists():
|
| 49 |
+
log.info("Loading FastTextPipeline")
|
| 50 |
+
app.state.pipeline_fasttext = FastTextPipeline(fasttext_path, "fasttext")
|
| 51 |
+
all_pipeline.add_fasttext("fasttext", fasttext_path)
|
| 52 |
+
else:
|
| 53 |
+
log.warning("fastText model not available — skipping")
|
| 54 |
+
|
| 55 |
+
app.state.all_models_pipeline = all_pipeline
|
| 56 |
+
available = list(all_pipeline.models.keys()) + list(all_pipeline.fasttext_models.keys())
|
| 57 |
+
log.info(f"Available models: {available}")
|
| 58 |
+
|
| 59 |
+
yield
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
app = FastAPI(
|
| 63 |
+
title="Entity Sentiment Classification API",
|
| 64 |
+
description="Classify sentiment (positive, neutral, negative) for entities in text.",
|
| 65 |
+
version="1.0.0",
|
| 66 |
+
lifespan=lifespan,
|
| 67 |
+
root_path="/api",
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
app.include_router(health_router)
|
| 71 |
+
app.include_router(predict_router)
|
| 72 |
+
app.include_router(predict_all_router)
|
main.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import streamlit as st
|
| 2 |
+
|
| 3 |
+
st.set_page_config(
|
| 4 |
+
page_title="Entity Sentiment Classification",
|
| 5 |
+
layout="centered",
|
| 6 |
+
initial_sidebar_state="collapsed",
|
| 7 |
+
)
|
| 8 |
+
|
| 9 |
+
st.markdown(
|
| 10 |
+
"""
|
| 11 |
+
<style>
|
| 12 |
+
[data-testid="collapsedControl"] { display: none; }
|
| 13 |
+
</style>
|
| 14 |
+
""",
|
| 15 |
+
unsafe_allow_html=True,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
home = st.Page("pages/welcome.py", title="Welcome", default=True)
|
| 19 |
+
config = st.Page("pages/config.py", title="Configuration")
|
| 20 |
+
result = st.Page("pages/result.py", title="Results")
|
| 21 |
+
|
| 22 |
+
pg = st.navigation([home, config, result], position="hidden")
|
| 23 |
+
pg.run()
|
src/fe_handler.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
import requests
|
| 4 |
+
|
| 5 |
+
API_BASE_URL = os.environ.get("API_BASE_URL", "http://localhost:8000")
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def call_predict(samples: list[dict]) -> list[dict]:
|
| 9 |
+
response = requests.post(f"{API_BASE_URL}/predict", json=samples)
|
| 10 |
+
response.raise_for_status()
|
| 11 |
+
return response.json()
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def call_predict_all(samples: list[dict]) -> dict[str, list[dict]]:
|
| 15 |
+
response = requests.post(f"{API_BASE_URL}/predict-all-models", json=samples)
|
| 16 |
+
response.raise_for_status()
|
| 17 |
+
return response.json()
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def check_health() -> bool:
|
| 21 |
+
try:
|
| 22 |
+
response = requests.get(f"{API_BASE_URL}/health", timeout=3)
|
| 23 |
+
return response.status_code == 200
|
| 24 |
+
except requests.ConnectionError:
|
| 25 |
+
return False
|
src/logger.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import time
|
| 3 |
+
from datetime import datetime, timezone
|
| 4 |
+
import requests
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
BETTERSTACK_URL = "https://s2383648.eu-fsn-3.betterstackdata.com"
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def log_to_betterstack(
|
| 11 |
+
endpoint_name: str,
|
| 12 |
+
original_text: str,
|
| 13 |
+
formatted_text: str | dict,
|
| 14 |
+
model_name: str,
|
| 15 |
+
time_elapsed: float,
|
| 16 |
+
) -> None:
|
| 17 |
+
token = os.environ.get("BETTERSTACK_SOURCE_TOKEN")
|
| 18 |
+
if not token:
|
| 19 |
+
return
|
| 20 |
+
|
| 21 |
+
payload = {
|
| 22 |
+
"dt": datetime.now(timezone.utc).isoformat(),
|
| 23 |
+
"message": f"Endpoint called: {endpoint_name}",
|
| 24 |
+
"endpoint_name": endpoint_name,
|
| 25 |
+
"model_name": model_name,
|
| 26 |
+
"request": original_text,
|
| 27 |
+
"response": formatted_text,
|
| 28 |
+
"time_elapsed": round(time_elapsed, 4),
|
| 29 |
+
"request_date": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
|
| 30 |
+
"level": "info"
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
try:
|
| 34 |
+
resp = requests.post(
|
| 35 |
+
BETTERSTACK_URL,
|
| 36 |
+
json=payload,
|
| 37 |
+
headers={
|
| 38 |
+
"Authorization": f"Bearer {token}",
|
| 39 |
+
"Content-Type": "application/json",
|
| 40 |
+
},
|
| 41 |
+
timeout=5,
|
| 42 |
+
)
|
| 43 |
+
if resp.status_code >= 300:
|
| 44 |
+
print(f"[betterstack] {resp.status_code}: {resp.text}")
|
| 45 |
+
except Exception as e:
|
| 46 |
+
print(f"[betterstack] exception: {e}")
|