Upload folder using huggingface_hub
Browse files- .env +3 -3
- app/api/routes.py +10 -2
- app/config.py +5 -0
- app/utils/cache.py +7 -2
- tests/test_bulk_async.py +43 -0
- tests/test_ml_async.py +4 -1
.env
CHANGED
|
@@ -28,12 +28,12 @@ DATABASE_URL=sqlite+aiosqlite:///./emailval.db
|
|
| 28 |
# NOTE: free-tier databases support ONLY db 0 0 -- one database per logical store.
|
| 29 |
|
| 30 |
# Main cache -- db 0 of credible-kangaroo-120640
|
| 31 |
-
REDIS_URL=rediss://default:
|
| 32 |
REDIS_CACHE_TTL=86400
|
| 33 |
# Celery broker -- db 0 of immortal-gar-118475
|
| 34 |
-
CELERY_BROKER_URL=rediss://default:
|
| 35 |
# Celery result backend -- db 0 of exciting-gannet-119131
|
| 36 |
-
CELERY_RESULT_BACKEND=rediss://default:
|
| 37 |
|
| 38 |
# --- CORS ---
|
| 39 |
CORS_ORIGINS=["http://localhost:5173","http://localhost:3000","http://127.0.0.1:5173","https://sitcod3-bridge-modul-x-api.hf.space","https://uranusmediaconnection-dot.github.io","https://frontend-j6yuq34u0-sitcod3lab-2797s-projects.vercel.app","https://frontend-grm2qel63-sitcod3lab-2797s-projects.vercel.app","https://frontend-seven-rose-23.vercel.app"]
|
|
|
|
| 28 |
# NOTE: free-tier databases support ONLY db 0 0 -- one database per logical store.
|
| 29 |
|
| 30 |
# Main cache -- db 0 of credible-kangaroo-120640
|
| 31 |
+
REDIS_URL=rediss://default:gQAAAAAAA4vQAAIgcDEwZGE3NTU5MTU1ZjY0OTZlOWUxN2QwMjY1ZTlhMzViNA@new-parrot-232400.upstash.io:6379
|
| 32 |
REDIS_CACHE_TTL=86400
|
| 33 |
# Celery broker -- db 0 of immortal-gar-118475
|
| 34 |
+
CELERY_BROKER_URL=rediss://default:gQAAAAAAA4vQAAIgcDEwZGE3NTU5MTU1ZjY0OTZlOWUxN2QwMjY1ZTlhMzViNA@new-parrot-232400.upstash.io:6379
|
| 35 |
# Celery result backend -- db 0 of exciting-gannet-119131
|
| 36 |
+
CELERY_RESULT_BACKEND=rediss://default:gQAAAAAAA4vQAAIgcDEwZGE3NTU5MTU1ZjY0OTZlOWUxN2QwMjY1ZTlhMzViNA@new-parrot-232400.upstash.io:6379
|
| 37 |
|
| 38 |
# --- CORS ---
|
| 39 |
CORS_ORIGINS=["http://localhost:5173","http://localhost:3000","http://127.0.0.1:5173","https://sitcod3-bridge-modul-x-api.hf.space","https://uranusmediaconnection-dot.github.io","https://frontend-j6yuq34u0-sitcod3lab-2797s-projects.vercel.app","https://frontend-grm2qel63-sitcod3lab-2797s-projects.vercel.app","https://frontend-seven-rose-23.vercel.app"]
|
app/api/routes.py
CHANGED
|
@@ -21,6 +21,14 @@ from ..db import repo
|
|
| 21 |
_background_tasks: set[asyncio.Task] = set()
|
| 22 |
_bulk_semaphore = asyncio.Semaphore(settings.bulk_concurrency)
|
| 23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
router = APIRouter()
|
| 25 |
|
| 26 |
|
|
@@ -159,7 +167,8 @@ async def validate_bulk(request: BulkEmailRequest):
|
|
| 159 |
# .delay() would block retrying the Redis connection.
|
| 160 |
try:
|
| 161 |
from ..utils.cache import redis_available
|
| 162 |
-
use_celery =
|
|
|
|
| 163 |
print(f"[BULK] Redis available: {use_celery}")
|
| 164 |
except Exception as e:
|
| 165 |
print(f"[BULK] Redis check failed: {str(e)}")
|
|
@@ -167,7 +176,6 @@ async def validate_bulk(request: BulkEmailRequest):
|
|
| 167 |
|
| 168 |
if use_celery:
|
| 169 |
try:
|
| 170 |
-
from ..tasks.celery_tasks import bulk_validate_task
|
| 171 |
task = bulk_validate_task.delay(request.emails, request.webhook_url, request.deep)
|
| 172 |
# Record the job in Supabase so it can be tracked across restarts
|
| 173 |
if settings.enable_db_persistence:
|
|
|
|
| 21 |
_background_tasks: set[asyncio.Task] = set()
|
| 22 |
_bulk_semaphore = asyncio.Semaphore(settings.bulk_concurrency)
|
| 23 |
|
| 24 |
+
# Celery task handle (module level so the routing is testable/patchable).
|
| 25 |
+
# Import failure degrades to the background-task path; Celery itself is only
|
| 26 |
+
# required when celery_bulk_enabled=True.
|
| 27 |
+
try:
|
| 28 |
+
from ..tasks.celery_tasks import bulk_validate_task # noqa: E402
|
| 29 |
+
except Exception: # pragma: no cover - celery not installed
|
| 30 |
+
bulk_validate_task = None
|
| 31 |
+
|
| 32 |
router = APIRouter()
|
| 33 |
|
| 34 |
|
|
|
|
| 167 |
# .delay() would block retrying the Redis connection.
|
| 168 |
try:
|
| 169 |
from ..utils.cache import redis_available
|
| 170 |
+
use_celery = (settings.celery_bulk_enabled
|
| 171 |
+
and await redis_available())
|
| 172 |
print(f"[BULK] Redis available: {use_celery}")
|
| 173 |
except Exception as e:
|
| 174 |
print(f"[BULK] Redis check failed: {str(e)}")
|
|
|
|
| 176 |
|
| 177 |
if use_celery:
|
| 178 |
try:
|
|
|
|
| 179 |
task = bulk_validate_task.delay(request.emails, request.webhook_url, request.deep)
|
| 180 |
# Record the job in Supabase so it can be tracked across restarts
|
| 181 |
if settings.enable_db_persistence:
|
app/config.py
CHANGED
|
@@ -63,6 +63,11 @@ class Settings(BaseSettings):
|
|
| 63 |
# gateway list (dead hotspot IPs) makes every lookup burn the full timeout
|
| 64 |
# and fail — see dns_check.build_resolver.
|
| 65 |
dns_nameservers: list[str] = ["1.1.1.1", "8.8.8.8"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
max_bulk_emails: int = 10000
|
| 67 |
rate_limit_per_minute: int = 60
|
| 68 |
bulk_sync_chunk_size: int = 50 # Process this many emails at a time in sync mode
|
|
|
|
| 63 |
# gateway list (dead hotspot IPs) makes every lookup burn the full timeout
|
| 64 |
# and fail — see dns_check.build_resolver.
|
| 65 |
dns_nameservers: list[str] = ["1.1.1.1", "8.8.8.8"]
|
| 66 |
+
# Bulk validation only routes through Celery on hosts that actually run a
|
| 67 |
+
# worker. The HF Space runs uvicorn alone (Dockerfile CMD), so a reachable
|
| 68 |
+
# Redis must not by itself select the Celery branch — jobs would sit in
|
| 69 |
+
# the queue forever. Opt in on the VPS deployment.
|
| 70 |
+
celery_bulk_enabled: bool = False
|
| 71 |
max_bulk_emails: int = 10000
|
| 72 |
rate_limit_per_minute: int = 60
|
| 73 |
bulk_sync_chunk_size: int = 50 # Process this many emails at a time in sync mode
|
app/utils/cache.py
CHANGED
|
@@ -77,8 +77,13 @@ async def get_redis():
|
|
| 77 |
url,
|
| 78 |
encoding="utf-8",
|
| 79 |
decode_responses=True,
|
| 80 |
-
|
| 81 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
)
|
| 83 |
await _redis_client.ping()
|
| 84 |
except Exception:
|
|
|
|
| 77 |
url,
|
| 78 |
encoding="utf-8",
|
| 79 |
decode_responses=True,
|
| 80 |
+
# The TLS handshake to Upstash measures ~750-850 ms on some
|
| 81 |
+
# networks; a 2 s cap killed the very first PING at exactly
|
| 82 |
+
# the timeout (observed live), which made every cache op pay
|
| 83 |
+
# the full budget and trip the breaker. 5 s connects reliably
|
| 84 |
+
# while the breaker still fails fast on real outages.
|
| 85 |
+
socket_connect_timeout=5,
|
| 86 |
+
socket_timeout=5,
|
| 87 |
)
|
| 88 |
await _redis_client.ping()
|
| 89 |
except Exception:
|
tests/test_bulk_async.py
CHANGED
|
@@ -99,6 +99,49 @@ class BulkAsyncTests(unittest.TestCase):
|
|
| 99 |
self.assertEqual(body["status"], "completed")
|
| 100 |
self.assertEqual(len(body["results"]), 1)
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
if __name__ == "__main__":
|
| 104 |
unittest.main()
|
|
|
|
| 99 |
self.assertEqual(body["status"], "completed")
|
| 100 |
self.assertEqual(len(body["results"]), 1)
|
| 101 |
|
| 102 |
+
def test_redis_available_but_flag_off_uses_background_path(self):
|
| 103 |
+
"""Redis being reachable must NOT switch bulk to Celery on its own.
|
| 104 |
+
|
| 105 |
+
The deployed Space runs uvicorn only (Dockerfile CMD) — no Celery
|
| 106 |
+
worker. If a live Redis alone selected the Celery branch, every bulk
|
| 107 |
+
job would sit in the queue forever and the Results page would show
|
| 108 |
+
"processing" indefinitely. The explicit flag (default off) is the
|
| 109 |
+
opt-in for hosts that actually run a worker.
|
| 110 |
+
"""
|
| 111 |
+
with (
|
| 112 |
+
patch("app.utils.cache.redis_available",
|
| 113 |
+
new=AsyncMock(return_value=True)),
|
| 114 |
+
patch("app.api.routes.settings.enable_db_persistence", False),
|
| 115 |
+
patch("app.api.routes.settings.celery_bulk_enabled", False),
|
| 116 |
+
):
|
| 117 |
+
r = client.post("/api/v1/validate/bulk",
|
| 118 |
+
json={"emails": ["a@b.test"]})
|
| 119 |
+
|
| 120 |
+
self.assertEqual(r.status_code, 200)
|
| 121 |
+
body = r.json()
|
| 122 |
+
self.assertNotEqual(body["task_id"], "sync")
|
| 123 |
+
self.assertIn("status", body)
|
| 124 |
+
|
| 125 |
+
def test_flag_on_and_redis_available_uses_celery(self):
|
| 126 |
+
from app.tasks.celery_tasks import bulk_validate_task # noqa: F401
|
| 127 |
+
|
| 128 |
+
fake_task = MagicMock()
|
| 129 |
+
fake_task.id = "celery-123"
|
| 130 |
+
with (
|
| 131 |
+
patch("app.utils.cache.redis_available",
|
| 132 |
+
new=AsyncMock(return_value=True)),
|
| 133 |
+
patch("app.api.routes.settings.enable_db_persistence", False),
|
| 134 |
+
patch("app.api.routes.settings.celery_bulk_enabled", True),
|
| 135 |
+
patch("app.api.routes.bulk_validate_task.delay",
|
| 136 |
+
return_value=fake_task) as delay_mock,
|
| 137 |
+
):
|
| 138 |
+
r = client.post("/api/v1/validate/bulk",
|
| 139 |
+
json={"emails": ["a@b.test"]})
|
| 140 |
+
|
| 141 |
+
self.assertEqual(r.status_code, 200)
|
| 142 |
+
self.assertEqual(r.json()["task_id"], "celery-123")
|
| 143 |
+
delay_mock.assert_called_once()
|
| 144 |
+
|
| 145 |
|
| 146 |
if __name__ == "__main__":
|
| 147 |
unittest.main()
|
tests/test_ml_async.py
CHANGED
|
@@ -94,7 +94,10 @@ class NonBlockingInferenceTests(unittest.IsolatedAsyncioTestCase):
|
|
| 94 |
predict_async(m, ["c [SEP] d.test"]),
|
| 95 |
)
|
| 96 |
elapsed = time.perf_counter() - t0
|
| 97 |
-
|
|
|
|
|
|
|
|
|
|
| 98 |
|
| 99 |
|
| 100 |
if __name__ == "__main__":
|
|
|
|
| 94 |
predict_async(m, ["c [SEP] d.test"]),
|
| 95 |
)
|
| 96 |
elapsed = time.perf_counter() - t0
|
| 97 |
+
# Serial execution would take ~0.8 s (two full sleeps); anything
|
| 98 |
+
# comfortably below that proves overlap. 0.7 leaves scheduling slack
|
| 99 |
+
# on a loaded machine while still failing on serialisation.
|
| 100 |
+
self.assertLess(elapsed, 0.7, f"inferences serialised ({elapsed:.2f}s)")
|
| 101 |
|
| 102 |
|
| 103 |
if __name__ == "__main__":
|