Upload folder using huggingface_hub

app/config.py

@@ -59,6 +59,10 @@ class Settings(BaseSettings):
     # Validation
     smtp_timeout: int = 10
     dns_timeout: int = 5
+    # Public resolvers used instead of the host's own resolver config. A stale
+    # gateway list (dead hotspot IPs) makes every lookup burn the full timeout
+    # and fail — see dns_check.build_resolver.
+    dns_nameservers: list[str] = ["1.1.1.1", "8.8.8.8"]
     max_bulk_emails: int = 10000
     rate_limit_per_minute: int = 60
     bulk_sync_chunk_size: int = 50  # Process this many emails at a time in sync mode

app/validator/dns_check.py

@@ -1,19 +1,41 @@
+import asyncio
+
 import dns.resolver
 import dns.asyncresolver
 import dns.exception
+from ..config import settings
 from ..models import DNSResult
 
 DKIM_SELECTORS = ['default', 'google', 'selector1', 'selector2', 'dkim', 'k1']
 
 
+def build_resolver(timeout: float) -> dns.asyncresolver.Resolver:
+    """Resolver that never trusts a broken system resolver config.
+
+    Diagnosed live: the host's resolver list held three stale gateway IPs
+    (192.168.3.1 / 192.168.1.1 / 172.20.10.1) from old networks. dnspython tried
+    each in turn, each timing out, so EVERY lookup burned the full lifetime and
+    returned a timeout — even for domains whose MX answered in 56 ms via a
+    public resolver. That produced 42s requests and mass false-invalid verdicts.
+    """
+    resolver = dns.asyncresolver.Resolver(configure=not settings.dns_nameservers)
+    if settings.dns_nameservers:
+        resolver.nameservers = list(settings.dns_nameservers)
+    resolver.timeout = timeout
+    resolver.lifetime = timeout
+    return resolver
+
+
 async def check_dns(domain: str, timeout: int = 5) -> DNSResult:
     """
     Layer 2 & 3: Async DNS + MX Record validation.
-    Also checks SPF, DMARC, DKIM authentication records.
+    Also checks SPF, DMARC and DKIM authentication records.
+
+    The authentication lookups are independent of one another and are issued
+    concurrently: the DKIM selector list alone is six sequential lookups, which
+    multiplied every request's latency when run one after another.
     """
-    resolver = dns.asyncresolver.Resolver()
-    resolver.timeout = timeout
-    resolver.lifetime = timeout
+    resolver = build_resolver(timeout)
 
     # ---- MX Records --------------------------------------------------
     mx_records = []
@@ -97,59 +119,66 @@ async def check_dns(domain: str, timeout: int = 5) -> DNSResult:
             error_code="no_mail_records",
         )
 
-    # ---- SPF Record ---------------------------------------------------
-    has_spf = False
-    spf_record = None
+    # ---- SPF / DMARC / DKIM (independent of each other -> concurrent) ----
+    has_spf, spf_record = await _lookup_spf(resolver, domain)
+    (has_dmarc, dmarc_record), (has_dkim, _) = await asyncio.gather(
+        _lookup_dmarc(resolver, domain),
+        _lookup_dkim(resolver, domain),
+    )
+
+    return DNSResult(
+        domain_exists=domain_exists,
+        has_mx_records=has_mx,
+        mx_records=mx_records,
+        a_fallback=a_fallback,
+        has_spf=has_spf,
+        spf_record=spf_record,
+        has_dmarc=has_dmarc,
+        dmarc_record=dmarc_record,
+        has_dkim=has_dkim
+    )
+
+
+async def _lookup_spf(resolver, domain: str) -> tuple[bool, str | None]:
     try:
         txt_answers = await resolver.resolve(domain, 'TXT')
         for record in txt_answers:
             txt = b''.join(record.strings).decode('utf-8', errors='replace')
             if txt.startswith('v=spf1'):
-                has_spf = True
-                spf_record = txt
-                break
+                return True, txt
     except Exception:
         pass
+    return False, None
 
-    # ---- DMARC Record -------------------------------------------------
-    has_dmarc = False
-    dmarc_record = None
+
+async def _lookup_dmarc(resolver, domain: str) -> tuple[bool, str | None]:
     try:
-        dmarc_domain = f"_dmarc.{domain}"
-        dmarc_answers = await resolver.resolve(dmarc_domain, 'TXT')
-        for record in dmarc_answers:
+        answers = await resolver.resolve(f"_dmarc.{domain}", 'TXT')
+        for record in answers:
             txt = b''.join(record.strings).decode('utf-8', errors='replace')
             if 'DMARC1' in txt:
-                has_dmarc = True
-                dmarc_record = txt
-                break
+                return True, txt
     except Exception:
         pass
+    return False, None
+
+
+async def _lookup_dkim(resolver, domain: str) -> tuple[bool, str | None]:
+    """Probe the known selectors concurrently, first hit wins."""
 
-    # ---- DKIM Record --------------------------------------------------
-    has_dkim = False
-    for selector in DKIM_SELECTORS:
+    async def probe(selector: str):
         try:
-            dkim_domain = f"{selector}._domainkey.{domain}"
-            dkim_answers = await resolver.resolve(dkim_domain, 'TXT')
-            for record in dkim_answers:
+            answers = await resolver.resolve(f"{selector}._domainkey.{domain}", 'TXT')
+            for record in answers:
                 txt = b''.join(record.strings).decode('utf-8', errors='replace')
                 if 'DKIM1' in txt or 'k=rsa' in txt or 'p=' in txt:
-                    has_dkim = True
-                    break
-            if has_dkim:
-                break
+                    return txt
         except Exception:
-            continue
+            return None
+        return None
 
-    return DNSResult(
-        domain_exists=domain_exists,
-        has_mx_records=has_mx,
-        mx_records=mx_records,
-        a_fallback=a_fallback,
-        has_spf=has_spf,
-        spf_record=spf_record,
-        has_dmarc=has_dmarc,
-        dmarc_record=dmarc_record,
-        has_dkim=has_dkim
-    )
+    results = await asyncio.gather(*(probe(s) for s in DKIM_SELECTORS))
+    for txt in results:
+        if txt:
+            return True, txt
+    return False, None

tests/test_dns_concurrency.py

@@ -0,0 +1,78 @@
+import asyncio
+import time
+import unittest
+from unittest.mock import patch
+
+from app.validator import dns_check
+
+
+class FakeAnswer:
+    def __init__(self, text: str):
+        self.strings = [text.encode()]
+
+
+class SlowResolver:
+    """Every query takes 0.1s — 8 sequential queries cost ~0.8s."""
+
+    timeout = 5
+    lifetime = 5
+
+    async def resolve(self, domain, rtype):
+        await asyncio.sleep(0.1)
+        if rtype == "MX":
+            return [type("MX", (), {"exchange": "mx.example.test.", "preference": 10})()]
+        if rtype == "TXT" and domain.startswith("_dmarc."):
+            return [FakeAnswer("v=DMARC1; p=none")]
+        if rtype == "TXT" and domain.startswith("default._domainkey."):
+            return [FakeAnswer("v=DKIM1; k=rsa; p=abc")]
+        if rtype == "TXT":
+            return [FakeAnswer("v=spf1 include:example.test -all")]
+        raise dns_check.dns.resolver.NoAnswer()
+
+
+class DnsAuthConcurrencyTests(unittest.IsolatedAsyncioTestCase):
+    """SPF / DMARC / DKIM are independent DNS queries and must not run serially.
+
+    Measured live: one request took 42.4s while the same address took 2.45s
+    once caches were warm — the shape of ~8 sequential queries each paying a
+    5s timeout. The DKIM selector list alone is 6 sequential lookups.
+    """
+
+    async def test_auth_record_lookups_run_concurrently(self):
+        with patch.object(dns_check.dns.asyncresolver, "Resolver",
+                          return_value=SlowResolver()):
+            t0 = time.perf_counter()
+            result = await dns_check.check_dns("example.test")
+            elapsed = time.perf_counter() - t0
+
+        self.assertTrue(result.has_mx_records)
+        self.assertTrue(result.has_spf)
+        self.assertTrue(result.has_dmarc)
+        self.assertTrue(result.has_dkim)
+        # Sequential = 1 (MX) + 1 (SPF) + 1 (DMARC) + 6 (DKIM) = 0.9s.
+        # Concurrent should be far below that.
+        self.assertLess(
+            elapsed, 0.5,
+            f"auth lookups ran sequentially ({elapsed:.2f}s for 9 queries)",
+        )
+
+    async def test_still_returns_a_result_when_auth_records_are_absent(self):
+        class Bare(SlowResolver):
+            async def resolve(self, domain, rtype):
+                await asyncio.sleep(0.01)
+                if rtype == "MX":
+                    return [type("MX", (), {"exchange": "mx.example.test.",
+                                            "preference": 10})()]
+                raise dns_check.dns.resolver.NoAnswer()
+
+        with patch.object(dns_check.dns.asyncresolver, "Resolver", return_value=Bare()):
+            result = await dns_check.check_dns("example.test")
+
+        self.assertTrue(result.has_mx_records)
+        self.assertFalse(result.has_spf)
+        self.assertFalse(result.has_dmarc)
+        self.assertFalse(result.has_dkim)  # absence is a verdict, not an error
+
+
+if __name__ == "__main__":
+    unittest.main()