sitcod3 commited on
Commit
d92dd26
·
verified ·
1 Parent(s): f4a3981

Upload folder using huggingface_hub

Browse files
.env CHANGED
@@ -28,12 +28,12 @@ DATABASE_URL=sqlite+aiosqlite:///./emailval.db
28
  # NOTE: free-tier databases support ONLY db 0 0 -- one database per logical store.
29
 
30
  # Main cache -- db 0 of credible-kangaroo-120640
31
- REDIS_URL=rediss://default:gQAAAAAAAddAAQIgcDIwNGY2YmFkNThiMDI0ODhlODMxY2RkMjk3NmE5MThmOA@credible-kangaroo-120640.upstash.io:6379/0?ssl_cert_reqs=none
32
  REDIS_CACHE_TTL=86400
33
  # Celery broker -- db 0 of immortal-gar-118475
34
- CELERY_BROKER_URL=rediss://default:gQAAAAAAAc7LAQIgcDFlNWNlNjEyZDlhN2I0YzdmOTEzNjk1ODQ3OTk5MTcyOA@immortal-gar-118475.upstash.io:6379/0?ssl_cert_reqs=none
35
  # Celery result backend -- db 0 of exciting-gannet-119131
36
- CELERY_RESULT_BACKEND=rediss://default:gQAAAAAAAdFbAQIgcDFhMGZjMjc3YzAxNGQ0ZDM1YWNhN2RlZTI2ZWI1NGY5ZQ@exciting-gannet-119131.upstash.io:6379/0?ssl_cert_reqs=none
37
 
38
  # --- CORS ---
39
  CORS_ORIGINS=["http://localhost:5173","http://localhost:3000","http://127.0.0.1:5173","https://sitcod3-bridge-modul-x-api.hf.space","https://uranusmediaconnection-dot.github.io","https://frontend-j6yuq34u0-sitcod3lab-2797s-projects.vercel.app","https://frontend-grm2qel63-sitcod3lab-2797s-projects.vercel.app","https://frontend-seven-rose-23.vercel.app"]
 
28
  # NOTE: free-tier databases support ONLY db 0 0 -- one database per logical store.
29
 
30
  # Main cache -- db 0 of credible-kangaroo-120640
31
+ REDIS_URL=rediss://default:gQAAAAAAA4vQAAIgcDEwZGE3NTU5MTU1ZjY0OTZlOWUxN2QwMjY1ZTlhMzViNA@new-parrot-232400.upstash.io:6379
32
  REDIS_CACHE_TTL=86400
33
  # Celery broker -- db 0 of immortal-gar-118475
34
+ CELERY_BROKER_URL=rediss://default:gQAAAAAAA4vQAAIgcDEwZGE3NTU5MTU1ZjY0OTZlOWUxN2QwMjY1ZTlhMzViNA@new-parrot-232400.upstash.io:6379
35
  # Celery result backend -- db 0 of exciting-gannet-119131
36
+ CELERY_RESULT_BACKEND=rediss://default:gQAAAAAAA4vQAAIgcDEwZGE3NTU5MTU1ZjY0OTZlOWUxN2QwMjY1ZTlhMzViNA@new-parrot-232400.upstash.io:6379
37
 
38
  # --- CORS ---
39
  CORS_ORIGINS=["http://localhost:5173","http://localhost:3000","http://127.0.0.1:5173","https://sitcod3-bridge-modul-x-api.hf.space","https://uranusmediaconnection-dot.github.io","https://frontend-j6yuq34u0-sitcod3lab-2797s-projects.vercel.app","https://frontend-grm2qel63-sitcod3lab-2797s-projects.vercel.app","https://frontend-seven-rose-23.vercel.app"]
app/api/routes.py CHANGED
@@ -21,6 +21,14 @@ from ..db import repo
21
  _background_tasks: set[asyncio.Task] = set()
22
  _bulk_semaphore = asyncio.Semaphore(settings.bulk_concurrency)
23
 
 
 
 
 
 
 
 
 
24
  router = APIRouter()
25
 
26
 
@@ -159,7 +167,8 @@ async def validate_bulk(request: BulkEmailRequest):
159
  # .delay() would block retrying the Redis connection.
160
  try:
161
  from ..utils.cache import redis_available
162
- use_celery = await redis_available()
 
163
  print(f"[BULK] Redis available: {use_celery}")
164
  except Exception as e:
165
  print(f"[BULK] Redis check failed: {str(e)}")
@@ -167,7 +176,6 @@ async def validate_bulk(request: BulkEmailRequest):
167
 
168
  if use_celery:
169
  try:
170
- from ..tasks.celery_tasks import bulk_validate_task
171
  task = bulk_validate_task.delay(request.emails, request.webhook_url, request.deep)
172
  # Record the job in Supabase so it can be tracked across restarts
173
  if settings.enable_db_persistence:
 
21
  _background_tasks: set[asyncio.Task] = set()
22
  _bulk_semaphore = asyncio.Semaphore(settings.bulk_concurrency)
23
 
24
+ # Celery task handle (module level so the routing is testable/patchable).
25
+ # Import failure degrades to the background-task path; Celery itself is only
26
+ # required when celery_bulk_enabled=True.
27
+ try:
28
+ from ..tasks.celery_tasks import bulk_validate_task # noqa: E402
29
+ except Exception: # pragma: no cover - celery not installed
30
+ bulk_validate_task = None
31
+
32
  router = APIRouter()
33
 
34
 
 
167
  # .delay() would block retrying the Redis connection.
168
  try:
169
  from ..utils.cache import redis_available
170
+ use_celery = (settings.celery_bulk_enabled
171
+ and await redis_available())
172
  print(f"[BULK] Redis available: {use_celery}")
173
  except Exception as e:
174
  print(f"[BULK] Redis check failed: {str(e)}")
 
176
 
177
  if use_celery:
178
  try:
 
179
  task = bulk_validate_task.delay(request.emails, request.webhook_url, request.deep)
180
  # Record the job in Supabase so it can be tracked across restarts
181
  if settings.enable_db_persistence:
app/config.py CHANGED
@@ -63,6 +63,11 @@ class Settings(BaseSettings):
63
  # gateway list (dead hotspot IPs) makes every lookup burn the full timeout
64
  # and fail — see dns_check.build_resolver.
65
  dns_nameservers: list[str] = ["1.1.1.1", "8.8.8.8"]
 
 
 
 
 
66
  max_bulk_emails: int = 10000
67
  rate_limit_per_minute: int = 60
68
  bulk_sync_chunk_size: int = 50 # Process this many emails at a time in sync mode
 
63
  # gateway list (dead hotspot IPs) makes every lookup burn the full timeout
64
  # and fail — see dns_check.build_resolver.
65
  dns_nameservers: list[str] = ["1.1.1.1", "8.8.8.8"]
66
+ # Bulk validation only routes through Celery on hosts that actually run a
67
+ # worker. The HF Space runs uvicorn alone (Dockerfile CMD), so a reachable
68
+ # Redis must not by itself select the Celery branch — jobs would sit in
69
+ # the queue forever. Opt in on the VPS deployment.
70
+ celery_bulk_enabled: bool = False
71
  max_bulk_emails: int = 10000
72
  rate_limit_per_minute: int = 60
73
  bulk_sync_chunk_size: int = 50 # Process this many emails at a time in sync mode
app/utils/cache.py CHANGED
@@ -77,8 +77,13 @@ async def get_redis():
77
  url,
78
  encoding="utf-8",
79
  decode_responses=True,
80
- socket_connect_timeout=2,
81
- socket_timeout=2
 
 
 
 
 
82
  )
83
  await _redis_client.ping()
84
  except Exception:
 
77
  url,
78
  encoding="utf-8",
79
  decode_responses=True,
80
+ # The TLS handshake to Upstash measures ~750-850 ms on some
81
+ # networks; a 2 s cap killed the very first PING at exactly
82
+ # the timeout (observed live), which made every cache op pay
83
+ # the full budget and trip the breaker. 5 s connects reliably
84
+ # while the breaker still fails fast on real outages.
85
+ socket_connect_timeout=5,
86
+ socket_timeout=5,
87
  )
88
  await _redis_client.ping()
89
  except Exception:
tests/test_bulk_async.py CHANGED
@@ -99,6 +99,49 @@ class BulkAsyncTests(unittest.TestCase):
99
  self.assertEqual(body["status"], "completed")
100
  self.assertEqual(len(body["results"]), 1)
101
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
102
 
103
  if __name__ == "__main__":
104
  unittest.main()
 
99
  self.assertEqual(body["status"], "completed")
100
  self.assertEqual(len(body["results"]), 1)
101
 
102
+ def test_redis_available_but_flag_off_uses_background_path(self):
103
+ """Redis being reachable must NOT switch bulk to Celery on its own.
104
+
105
+ The deployed Space runs uvicorn only (Dockerfile CMD) — no Celery
106
+ worker. If a live Redis alone selected the Celery branch, every bulk
107
+ job would sit in the queue forever and the Results page would show
108
+ "processing" indefinitely. The explicit flag (default off) is the
109
+ opt-in for hosts that actually run a worker.
110
+ """
111
+ with (
112
+ patch("app.utils.cache.redis_available",
113
+ new=AsyncMock(return_value=True)),
114
+ patch("app.api.routes.settings.enable_db_persistence", False),
115
+ patch("app.api.routes.settings.celery_bulk_enabled", False),
116
+ ):
117
+ r = client.post("/api/v1/validate/bulk",
118
+ json={"emails": ["a@b.test"]})
119
+
120
+ self.assertEqual(r.status_code, 200)
121
+ body = r.json()
122
+ self.assertNotEqual(body["task_id"], "sync")
123
+ self.assertIn("status", body)
124
+
125
+ def test_flag_on_and_redis_available_uses_celery(self):
126
+ from app.tasks.celery_tasks import bulk_validate_task # noqa: F401
127
+
128
+ fake_task = MagicMock()
129
+ fake_task.id = "celery-123"
130
+ with (
131
+ patch("app.utils.cache.redis_available",
132
+ new=AsyncMock(return_value=True)),
133
+ patch("app.api.routes.settings.enable_db_persistence", False),
134
+ patch("app.api.routes.settings.celery_bulk_enabled", True),
135
+ patch("app.api.routes.bulk_validate_task.delay",
136
+ return_value=fake_task) as delay_mock,
137
+ ):
138
+ r = client.post("/api/v1/validate/bulk",
139
+ json={"emails": ["a@b.test"]})
140
+
141
+ self.assertEqual(r.status_code, 200)
142
+ self.assertEqual(r.json()["task_id"], "celery-123")
143
+ delay_mock.assert_called_once()
144
+
145
 
146
  if __name__ == "__main__":
147
  unittest.main()
tests/test_ml_async.py CHANGED
@@ -94,7 +94,10 @@ class NonBlockingInferenceTests(unittest.IsolatedAsyncioTestCase):
94
  predict_async(m, ["c [SEP] d.test"]),
95
  )
96
  elapsed = time.perf_counter() - t0
97
- self.assertLess(elapsed, 0.55, f"inferences serialised ({elapsed:.2f}s)")
 
 
 
98
 
99
 
100
  if __name__ == "__main__":
 
94
  predict_async(m, ["c [SEP] d.test"]),
95
  )
96
  elapsed = time.perf_counter() - t0
97
+ # Serial execution would take ~0.8 s (two full sleeps); anything
98
+ # comfortably below that proves overlap. 0.7 leaves scheduling slack
99
+ # on a loaded machine while still failing on serialisation.
100
+ self.assertLess(elapsed, 0.7, f"inferences serialised ({elapsed:.2f}s)")
101
 
102
 
103
  if __name__ == "__main__":