Spaces:
Running
Running
Saibalaji Namburi commited on
Commit Β·
15c00a9
1
Parent(s): 392346c
fix(lint): resolve ruff code formatting and unused import warnings
Browse files- src/agent/nodes/classify_agent.py +0 -1
- src/api/routers/metrics.py +0 -1
- src/api/routers/triage.py +0 -1
- src/api/store.py +1 -1
- src/db/repository.py +1 -3
- src/rag/graph_rag.py +1 -1
- src/rag/hybrid_retriever.py +1 -1
- src/rag/llm_client.py +1 -1
- src/rag/multilingual.py +2 -3
- src/rag/router.py +0 -2
- src/responsible_ai/audit_log.py +0 -1
- src/responsible_ai/privacy_vault.py +1 -3
- src/streaming/bronze_consumer.py +1 -2
- src/streaming/bronze_to_silver.py +2 -2
- src/streaming/data_loader.py +4 -4
- src/streaming/minio_setup.py +1 -1
- src/streaming/triage_consumer.py +0 -1
src/agent/nodes/classify_agent.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
| 1 |
import mlflow
|
| 2 |
import os
|
| 3 |
-
import re
|
| 4 |
import structlog
|
| 5 |
from typing import List
|
| 6 |
from src.agent.state import AgentState
|
|
|
|
| 1 |
import mlflow
|
| 2 |
import os
|
|
|
|
| 3 |
import structlog
|
| 4 |
from typing import List
|
| 5 |
from src.agent.state import AgentState
|
src/api/routers/metrics.py
CHANGED
|
@@ -32,7 +32,6 @@ WHY COUNTERS AND HISTOGRAMS?
|
|
| 32 |
|
| 33 |
from __future__ import annotations
|
| 34 |
|
| 35 |
-
import time
|
| 36 |
|
| 37 |
from fastapi import APIRouter
|
| 38 |
from fastapi.responses import PlainTextResponse
|
|
|
|
| 32 |
|
| 33 |
from __future__ import annotations
|
| 34 |
|
|
|
|
| 35 |
|
| 36 |
from fastapi import APIRouter
|
| 37 |
from fastapi.responses import PlainTextResponse
|
src/api/routers/triage.py
CHANGED
|
@@ -7,7 +7,6 @@ Replaces the in-memory triage_store with TicketRepository.
|
|
| 7 |
|
| 8 |
from __future__ import annotations
|
| 9 |
|
| 10 |
-
import asyncio
|
| 11 |
from datetime import datetime, timezone
|
| 12 |
from typing import Any
|
| 13 |
from uuid import UUID, uuid4
|
|
|
|
| 7 |
|
| 8 |
from __future__ import annotations
|
| 9 |
|
|
|
|
| 10 |
from datetime import datetime, timezone
|
| 11 |
from typing import Any
|
| 12 |
from uuid import UUID, uuid4
|
src/api/store.py
CHANGED
|
@@ -29,7 +29,7 @@ import time
|
|
| 29 |
from collections import defaultdict
|
| 30 |
from datetime import datetime
|
| 31 |
from typing import Any
|
| 32 |
-
from uuid import
|
| 33 |
|
| 34 |
from src.api.models import TriageResultResponse, TriageStatus
|
| 35 |
|
|
|
|
| 29 |
from collections import defaultdict
|
| 30 |
from datetime import datetime
|
| 31 |
from typing import Any
|
| 32 |
+
from uuid import uuid4
|
| 33 |
|
| 34 |
from src.api.models import TriageResultResponse, TriageStatus
|
| 35 |
|
src/db/repository.py
CHANGED
|
@@ -39,11 +39,10 @@ ROW LEVEL SECURITY:
|
|
| 39 |
from __future__ import annotations
|
| 40 |
|
| 41 |
import os
|
| 42 |
-
from dataclasses import dataclass
|
| 43 |
from datetime import datetime, timezone
|
| 44 |
from typing import Any
|
| 45 |
import uuid
|
| 46 |
-
from uuid import UUID
|
| 47 |
|
| 48 |
from supabase import AsyncClient, acreate_client
|
| 49 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -178,7 +177,6 @@ class TicketRepository:
|
|
| 178 |
if self.use_in_memory or os.getenv("APP_ENV") == "test":
|
| 179 |
return self.tenant_id
|
| 180 |
|
| 181 |
-
import uuid
|
| 182 |
try:
|
| 183 |
uuid.UUID(self.tenant_id)
|
| 184 |
return self.tenant_id
|
|
|
|
| 39 |
from __future__ import annotations
|
| 40 |
|
| 41 |
import os
|
| 42 |
+
from dataclasses import dataclass
|
| 43 |
from datetime import datetime, timezone
|
| 44 |
from typing import Any
|
| 45 |
import uuid
|
|
|
|
| 46 |
|
| 47 |
from supabase import AsyncClient, acreate_client
|
| 48 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 177 |
if self.use_in_memory or os.getenv("APP_ENV") == "test":
|
| 178 |
return self.tenant_id
|
| 179 |
|
|
|
|
| 180 |
try:
|
| 181 |
uuid.UUID(self.tenant_id)
|
| 182 |
return self.tenant_id
|
src/rag/graph_rag.py
CHANGED
|
@@ -47,7 +47,7 @@ Run demo:
|
|
| 47 |
import logging
|
| 48 |
from collections import defaultdict
|
| 49 |
from dataclasses import dataclass, field
|
| 50 |
-
from typing import Optional
|
| 51 |
|
| 52 |
logger = logging.getLogger(__name__)
|
| 53 |
|
|
|
|
| 47 |
import logging
|
| 48 |
from collections import defaultdict
|
| 49 |
from dataclasses import dataclass, field
|
| 50 |
+
from typing import Optional
|
| 51 |
|
| 52 |
logger = logging.getLogger(__name__)
|
| 53 |
|
src/rag/hybrid_retriever.py
CHANGED
|
@@ -37,7 +37,7 @@ Run standalone demo:
|
|
| 37 |
import hashlib
|
| 38 |
import logging
|
| 39 |
import time
|
| 40 |
-
from dataclasses import dataclass
|
| 41 |
from typing import Optional
|
| 42 |
|
| 43 |
logger = logging.getLogger(__name__)
|
|
|
|
| 37 |
import hashlib
|
| 38 |
import logging
|
| 39 |
import time
|
| 40 |
+
from dataclasses import dataclass
|
| 41 |
from typing import Optional
|
| 42 |
|
| 43 |
logger = logging.getLogger(__name__)
|
src/rag/llm_client.py
CHANGED
|
@@ -26,7 +26,7 @@ Environment variables:
|
|
| 26 |
import os
|
| 27 |
import time
|
| 28 |
import logging
|
| 29 |
-
from dataclasses import dataclass
|
| 30 |
from typing import Optional, Any
|
| 31 |
|
| 32 |
try:
|
|
|
|
| 26 |
import os
|
| 27 |
import time
|
| 28 |
import logging
|
| 29 |
+
from dataclasses import dataclass
|
| 30 |
from typing import Optional, Any
|
| 31 |
|
| 32 |
try:
|
src/rag/multilingual.py
CHANGED
|
@@ -37,7 +37,6 @@ write in French. Routing all tickets through an English-only model:
|
|
| 37 |
|
| 38 |
import logging
|
| 39 |
import re
|
| 40 |
-
from typing import Optional
|
| 41 |
|
| 42 |
logger = logging.getLogger(__name__)
|
| 43 |
|
|
@@ -94,7 +93,7 @@ def detect_language(text: str, min_text_length: int = 20) -> str:
|
|
| 94 |
return "en" # too short for reliable detection
|
| 95 |
|
| 96 |
try:
|
| 97 |
-
from langdetect import detect
|
| 98 |
lang = detect(text)
|
| 99 |
# langdetect returns e.g. "zh-cn" β normalize to first part
|
| 100 |
lang = lang.split("-")[0].lower()
|
|
@@ -116,7 +115,7 @@ def detect_language_with_confidence(text: str) -> tuple[str, float]:
|
|
| 116 |
return "en", 1.0
|
| 117 |
|
| 118 |
try:
|
| 119 |
-
from langdetect import detect_langs
|
| 120 |
results = detect_langs(text)
|
| 121 |
if results:
|
| 122 |
top = results[0]
|
|
|
|
| 37 |
|
| 38 |
import logging
|
| 39 |
import re
|
|
|
|
| 40 |
|
| 41 |
logger = logging.getLogger(__name__)
|
| 42 |
|
|
|
|
| 93 |
return "en" # too short for reliable detection
|
| 94 |
|
| 95 |
try:
|
| 96 |
+
from langdetect import detect
|
| 97 |
lang = detect(text)
|
| 98 |
# langdetect returns e.g. "zh-cn" β normalize to first part
|
| 99 |
lang = lang.split("-")[0].lower()
|
|
|
|
| 115 |
return "en", 1.0
|
| 116 |
|
| 117 |
try:
|
| 118 |
+
from langdetect import detect_langs
|
| 119 |
results = detect_langs(text)
|
| 120 |
if results:
|
| 121 |
top = results[0]
|
src/rag/router.py
CHANGED
|
@@ -40,8 +40,6 @@ Run standalone demo:
|
|
| 40 |
"""
|
| 41 |
|
| 42 |
import logging
|
| 43 |
-
import os
|
| 44 |
-
import time
|
| 45 |
from dataclasses import dataclass, field
|
| 46 |
from typing import Literal, Optional
|
| 47 |
from collections import defaultdict
|
|
|
|
| 40 |
"""
|
| 41 |
|
| 42 |
import logging
|
|
|
|
|
|
|
| 43 |
from dataclasses import dataclass, field
|
| 44 |
from typing import Literal, Optional
|
| 45 |
from collections import defaultdict
|
src/responsible_ai/audit_log.py
CHANGED
|
@@ -20,7 +20,6 @@ Run standalone check:
|
|
| 20 |
import hashlib
|
| 21 |
import json
|
| 22 |
import logging
|
| 23 |
-
import os
|
| 24 |
from datetime import datetime, timezone
|
| 25 |
from pathlib import Path
|
| 26 |
from typing import Literal
|
|
|
|
| 20 |
import hashlib
|
| 21 |
import json
|
| 22 |
import logging
|
|
|
|
| 23 |
from datetime import datetime, timezone
|
| 24 |
from pathlib import Path
|
| 25 |
from typing import Literal
|
src/responsible_ai/privacy_vault.py
CHANGED
|
@@ -39,10 +39,8 @@ Run standalone demo:
|
|
| 39 |
python -m src.responsible_ai.privacy_vault
|
| 40 |
"""
|
| 41 |
|
| 42 |
-
import os
|
| 43 |
import re
|
| 44 |
import sqlite3
|
| 45 |
-
import hashlib
|
| 46 |
from datetime import datetime, timezone
|
| 47 |
from pathlib import Path
|
| 48 |
from typing import Optional
|
|
@@ -438,5 +436,5 @@ if __name__ == "__main__":
|
|
| 438 |
except PermissionError as e:
|
| 439 |
print(f" BLOCKED: {e}")
|
| 440 |
|
| 441 |
-
print(
|
| 442 |
print("=" * 60)
|
|
|
|
| 39 |
python -m src.responsible_ai.privacy_vault
|
| 40 |
"""
|
| 41 |
|
|
|
|
| 42 |
import re
|
| 43 |
import sqlite3
|
|
|
|
| 44 |
from datetime import datetime, timezone
|
| 45 |
from pathlib import Path
|
| 46 |
from typing import Optional
|
|
|
|
| 436 |
except PermissionError as e:
|
| 437 |
print(f" BLOCKED: {e}")
|
| 438 |
|
| 439 |
+
print("\nAudit log written to: logs/vault_audit/audit_trail.jsonl")
|
| 440 |
print("=" * 60)
|
src/streaming/bronze_consumer.py
CHANGED
|
@@ -15,7 +15,6 @@ import argparse
|
|
| 15 |
import io
|
| 16 |
import json
|
| 17 |
import signal
|
| 18 |
-
import sys
|
| 19 |
import time
|
| 20 |
from collections import defaultdict
|
| 21 |
from datetime import datetime, timezone
|
|
@@ -165,7 +164,7 @@ def main(batch_size: int = 500, timeout_seconds: int = 30):
|
|
| 165 |
print(f" Parquet files : {files_written}")
|
| 166 |
print(f" Duration : {elapsed:.1f}s")
|
| 167 |
print(f" Sink : s3://{BUCKET}/bronze/")
|
| 168 |
-
print(
|
| 169 |
print(f"{'=' * 60}")
|
| 170 |
|
| 171 |
|
|
|
|
| 15 |
import io
|
| 16 |
import json
|
| 17 |
import signal
|
|
|
|
| 18 |
import time
|
| 19 |
from collections import defaultdict
|
| 20 |
from datetime import datetime, timezone
|
|
|
|
| 164 |
print(f" Parquet files : {files_written}")
|
| 165 |
print(f" Duration : {elapsed:.1f}s")
|
| 166 |
print(f" Sink : s3://{BUCKET}/bronze/")
|
| 167 |
+
print(" Browse MinIO : http://localhost:9001")
|
| 168 |
print(f"{'=' * 60}")
|
| 169 |
|
| 170 |
|
src/streaming/bronze_to_silver.py
CHANGED
|
@@ -267,9 +267,9 @@ def main(source: str = "tickets", limit: int = None, use_vault: bool = True):
|
|
| 267 |
if use_vault and _VAULT_AVAILABLE:
|
| 268 |
from src.responsible_ai.privacy_vault import CryptographicPrivacyVault
|
| 269 |
vault = CryptographicPrivacyVault()
|
| 270 |
-
print(
|
| 271 |
else:
|
| 272 |
-
print(
|
| 273 |
|
| 274 |
# ββ 4. Clean + mask PII βββββββββββββββββββββββββββββββββββ
|
| 275 |
print(f"\n[4/5] Cleaning and masking PII in {len(all_records):,} records...")
|
|
|
|
| 267 |
if use_vault and _VAULT_AVAILABLE:
|
| 268 |
from src.responsible_ai.privacy_vault import CryptographicPrivacyVault
|
| 269 |
vault = CryptographicPrivacyVault()
|
| 270 |
+
print(" Privacy Vault: ENABLED (AES-256, silver_version=2.0)")
|
| 271 |
else:
|
| 272 |
+
print(" Privacy Vault: DISABLED (legacy Presidio masking, silver_version=1.0)")
|
| 273 |
|
| 274 |
# ββ 4. Clean + mask PII βββββββββββββββββββββββββββββββββββ
|
| 275 |
print(f"\n[4/5] Cleaning and masking PII in {len(all_records):,} records...")
|
src/streaming/data_loader.py
CHANGED
|
@@ -45,7 +45,7 @@ import uuid
|
|
| 45 |
from datetime import datetime, timezone
|
| 46 |
|
| 47 |
from confluent_kafka import Producer
|
| 48 |
-
from datasets import load_dataset
|
| 49 |
from tqdm import tqdm
|
| 50 |
|
| 51 |
# ββ Config ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -286,10 +286,10 @@ def main(limit: int = None, delay: float = 0.0, sources: str = "all"):
|
|
| 286 |
|
| 287 |
# ββ 1. Download datasets ββββββββββββββββββββββββββββββββββ
|
| 288 |
print("\n[1/3] Downloading datasets from Hugging Face...")
|
| 289 |
-
|
| 290 |
all_rows, source_counts = load_sources(sources)
|
| 291 |
|
| 292 |
-
print(
|
| 293 |
for src, count in source_counts.items():
|
| 294 |
print(f" {src:18s}: {count:,} rows")
|
| 295 |
total = len(all_rows)
|
|
@@ -351,7 +351,7 @@ def main(limit: int = None, delay: float = 0.0, sources: str = "all"):
|
|
| 351 |
print(f" Duration : {elapsed:.1f}s ({len(all_rows)/elapsed:.0f} msg/s)")
|
| 352 |
print(f" Topic : {TOPIC}")
|
| 353 |
print(f" Sources : {sources}")
|
| 354 |
-
print(
|
| 355 |
for lang, count in sorted(lang_counts.items()):
|
| 356 |
print(f" {lang}: {count:,} rows")
|
| 357 |
print(f"{'=' * 65}")
|
|
|
|
| 45 |
from datetime import datetime, timezone
|
| 46 |
|
| 47 |
from confluent_kafka import Producer
|
| 48 |
+
from datasets import load_dataset
|
| 49 |
from tqdm import tqdm
|
| 50 |
|
| 51 |
# ββ Config ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 286 |
|
| 287 |
# ββ 1. Download datasets ββββββββββββββββββββββββββββββββββ
|
| 288 |
print("\n[1/3] Downloading datasets from Hugging Face...")
|
| 289 |
+
time.time()
|
| 290 |
all_rows, source_counts = load_sources(sources)
|
| 291 |
|
| 292 |
+
print("\n Source breakdown:")
|
| 293 |
for src, count in source_counts.items():
|
| 294 |
print(f" {src:18s}: {count:,} rows")
|
| 295 |
total = len(all_rows)
|
|
|
|
| 351 |
print(f" Duration : {elapsed:.1f}s ({len(all_rows)/elapsed:.0f} msg/s)")
|
| 352 |
print(f" Topic : {TOPIC}")
|
| 353 |
print(f" Sources : {sources}")
|
| 354 |
+
print(" Language breakdown:")
|
| 355 |
for lang, count in sorted(lang_counts.items()):
|
| 356 |
print(f" {lang}: {count:,} rows")
|
| 357 |
print(f"{'=' * 65}")
|
src/streaming/minio_setup.py
CHANGED
|
@@ -59,7 +59,7 @@ def setup():
|
|
| 59 |
print(f" [OK] {BUCKET}/{prefix}")
|
| 60 |
|
| 61 |
print(f"\nLakehouse structure ready at MinIO: {MINIO_ENDPOINT}")
|
| 62 |
-
print(
|
| 63 |
|
| 64 |
|
| 65 |
if __name__ == "__main__":
|
|
|
|
| 59 |
print(f" [OK] {BUCKET}/{prefix}")
|
| 60 |
|
| 61 |
print(f"\nLakehouse structure ready at MinIO: {MINIO_ENDPOINT}")
|
| 62 |
+
print("Browse at: http://localhost:9001 (user: minioadmin / pass: minioadmin)")
|
| 63 |
|
| 64 |
|
| 65 |
if __name__ == "__main__":
|
src/streaming/triage_consumer.py
CHANGED
|
@@ -3,7 +3,6 @@ import json
|
|
| 3 |
import logging
|
| 4 |
import os
|
| 5 |
import signal
|
| 6 |
-
import sys
|
| 7 |
from confluent_kafka import Consumer, KafkaError
|
| 8 |
from src.api.routers.triage import _run_triage
|
| 9 |
from src.db.repository import TicketRepository, TicketRecord
|
|
|
|
| 3 |
import logging
|
| 4 |
import os
|
| 5 |
import signal
|
|
|
|
| 6 |
from confluent_kafka import Consumer, KafkaError
|
| 7 |
from src.api.routers.triage import _run_triage
|
| 8 |
from src.db.repository import TicketRepository, TicketRecord
|