Upload folder using huggingface_hub

__pycache__/server.cpython-310.pyc

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c97a8de2cad0418387614cfc47d4309539f6a69dd5a32f0ed78d66fd339e09a
-size 126116
+oid sha256:60a16111c476fdd2fa4b44436a4e38eed423d6d9656d9823088fd54a3258f9d8
+size 141610

dashboard.css

@@ -995,7 +995,7 @@ body.theme-light .profile-submenu .profile-dropdown__item { color: var(--text-pr
 
 .flow-canvas {
   position: relative;
-  min-width: 1000px;
+  min-width: 1100px;
   height: 580px;
   border-radius: 18px;
   background:
@@ -1235,6 +1235,13 @@ body.theme-light .profile-submenu .profile-dropdown__item { color: var(--text-pr
   font: 12px/1.55 'JetBrains Mono', monospace;
 }
 
+#flow-api-table td,
+#flow-register td {
+  max-width: 280px;
+  white-space: normal;
+  word-break: break-word;
+}
+
 .tool-timeline {
   display: grid;
   gap: 10px;

dashboard.html

@@ -371,12 +371,16 @@
             <input type="checkbox" id="flow-cookies" checked>
             <span>Cookies</span>
           </label>
-          <label class="ai-leak-toggle">
-            <input type="checkbox" id="flow-ai" checked>
-            <span>AI</span>
-          </label>
-          <button class="btn btn--primary" id="flow-scan-btn"><i data-lucide="route"></i> Map Data Flow</button>
-        </div>
+          <label class="ai-leak-toggle">
+            <input type="checkbox" id="flow-ai" checked>
+            <span>AI</span>
+          </label>
+          <label class="ai-leak-toggle">
+            <input type="checkbox" id="flow-runtime" checked>
+            <span>Runtime</span>
+          </label>
+          <button class="btn btn--primary" id="flow-scan-btn"><i data-lucide="route"></i> Map Data Flow</button>
+        </div>
         <div class="flow-progress" id="flow-progress" style="display:none;">
           <span></span>
           <div>Scanning forms, cookies, trackers, processors, and public PII paths...</div>
@@ -425,12 +429,18 @@
           </div>
 
           <div class="flow-side">
-            <div class="chart-card">
-              <div class="chart-card__header">
-                <h3><i data-lucide="fingerprint"></i> PII Types</h3>
-              </div>
-              <div class="flow-chip-list" id="flow-pii-types"></div>
-            </div>
+            <div class="chart-card">
+              <div class="chart-card__header">
+                <h3><i data-lucide="server-cog"></i> Services</h3>
+              </div>
+              <div class="flow-chip-list" id="flow-services"></div>
+            </div>
+            <div class="chart-card">
+              <div class="chart-card__header">
+                <h3><i data-lucide="fingerprint"></i> PII Types</h3>
+              </div>
+              <div class="flow-chip-list" id="flow-pii-types"></div>
+            </div>
             <div class="chart-card">
               <div class="chart-card__header">
                 <h3><i data-lucide="shield-alert"></i> Controls</h3>
@@ -438,9 +448,29 @@
               <ol class="flow-remediation" id="flow-remediation"></ol>
             </div>
           </div>
-        </div>
-
-        <div class="chart-card">
+        </div>
+
+        <div class="chart-card">
+          <div class="chart-card__header">
+            <h3><i data-lucide="radio-tower"></i> API Calls & Provider Evidence</h3>
+            <span class="badge" id="flow-api-count">0 calls</span>
+          </div>
+          <div class="table-wrap">
+            <table class="data-table">
+              <thead>
+                <tr>
+                  <th>Host</th>
+                  <th>Type</th>
+                  <th>Observed In</th>
+                  <th>Evidence</th>
+                </tr>
+              </thead>
+              <tbody id="flow-api-table"></tbody>
+            </table>
+          </div>
+        </div>
+
+        <div class="chart-card">
           <div class="chart-card__header">
             <h3><i data-lucide="list-tree"></i> Flow Register</h3>
             <span class="badge" id="flow-register-count">0 rows</span>

dashboard.js

@@ -804,31 +804,56 @@ function initDataFlowVisualizer() {
     const columns = {
       subject: 60,
       client: 230,
-      first_party: 405,
-      storage: 405,
-      processor: 625,
-      third_party: 625,
-      third_party_domain: 625,
-      high_risk_processor: 625,
-      governance: 810,
-      exposure: 810,
+      first_party: 390,
+      api_endpoint: 565,
+      external_api: 565,
+      database: 740,
+      storage: 740,
+      processor: 740,
+      third_party: 740,
+      third_party_domain: 740,
+      runtime_service: 740,
+      high_risk_processor: 740,
+      governance: 915,
+      exposure: 915,
     };
     const x = columns[node.kind] || 625;
     const y = 70 + (groupIndex * 92);
     return { x, y };
   }
 
+  function nodeColumn(node) {
+    const columns = {
+      subject: 60,
+      client: 230,
+      first_party: 390,
+      api_endpoint: 565,
+      external_api: 565,
+      database: 740,
+      storage: 740,
+      processor: 740,
+      third_party: 740,
+      third_party_domain: 740,
+      runtime_service: 740,
+      high_risk_processor: 740,
+      governance: 915,
+      exposure: 915,
+    };
+    return columns[node.kind] || 625;
+  }
+
   function renderFlowMap(data) {
     const nodes = data.nodes || [];
     const edges = data.edges || [];
     const groupCounts = {};
     const positions = {};
     nodes.forEach((node, index) => {
-      const key = node.kind || 'processor';
+      const key = String(nodeColumn(node));
       const groupIndex = groupCounts[key] || 0;
       groupCounts[key] = groupIndex + 1;
       positions[node.id] = nodePosition(node, index, groupIndex);
     });
+    const canvasHeight = Math.max(640, Math.max(...Object.values(groupCounts), 1) * 92 + 180);
 
     const lineSvg = edges.map(edge => {
       const from = positions[edge.source];
@@ -854,7 +879,7 @@ function initDataFlowVisualizer() {
 
     mapEl.innerHTML = `
       <div class="flow-canvas">
-        <svg viewBox="0 0 1000 580" preserveAspectRatio="none" aria-hidden="true">
+        <svg viewBox="0 0 1100 ${canvasHeight}" preserveAspectRatio="none" aria-hidden="true">
           <defs>
             <marker id="flow-arrow" markerWidth="10" markerHeight="10" refX="8" refY="3" orient="auto" markerUnits="strokeWidth">
               <path d="M0,0 L0,6 L8,3 z" fill="currentColor"></path>
@@ -864,6 +889,7 @@ function initDataFlowVisualizer() {
         </svg>
         ${nodeHtml}
       </div>`;
+    mapEl.querySelector('.flow-canvas').style.height = canvasHeight + 'px';
   }
 
   function renderFlowReport(data) {
@@ -882,8 +908,18 @@ function initDataFlowVisualizer() {
     document.getElementById('flow-register-count').textContent = `${edges.length} rows`;
 
     const piiTypes = summary.pii_types || [];
+    const services = data.services || [];
+    const apiCalls = data.api_calls || [];
+    document.getElementById('flow-api-count').textContent = `${apiCalls.length} calls`;
     document.getElementById('flow-pii-types').innerHTML = piiTypes.map(type => `<span>${escapeFlowHtml(type)}</span>`).join('');
+    document.getElementById('flow-services').innerHTML = services.length ? services.slice(0, 18).map(service => `<span>${escapeFlowHtml(service.name)} · ${escapeFlowHtml(service.category)}</span>`).join('') : '<span>No provider signals found in public surface</span>';
     document.getElementById('flow-remediation').innerHTML = (data.remediation || []).map(item => `<li>${escapeFlowHtml(item)}</li>`).join('');
+    document.getElementById('flow-api-table').innerHTML = apiCalls.slice(0, 50).map(call => `<tr>
+      <td>${escapeFlowHtml(call.host || '')}</td>
+      <td>${escapeFlowHtml(call.first_party ? 'First-party API' : 'External API')}</td>
+      <td>${escapeFlowHtml(call.asset_type || call.confidence || '')}</td>
+      <td>${escapeFlowHtml((call.evidence || call.url || '').slice(0, 160))}</td>
+    </tr>`).join('');
     document.getElementById('flow-register').innerHTML = edges.map(edge => {
       const from = (data.nodes || []).find(n => n.id === edge.source);
       const to = (data.nodes || []).find(n => n.id === edge.target);
@@ -922,6 +958,8 @@ function initDataFlowVisualizer() {
           include_trackers: document.getElementById('flow-trackers')?.checked !== false,
           include_cookies: document.getElementById('flow-cookies')?.checked !== false,
           include_ai: document.getElementById('flow-ai')?.checked !== false,
+          include_runtime: document.getElementById('flow-runtime')?.checked !== false,
+          include_source_maps: true,
         }),
       });
       const data = await res.json();

server.py

@@ -2628,6 +2628,9 @@ class DataFlowVisualizeRequest(BaseModel):
     include_cookies: bool = True
     include_trackers: bool = True
     include_ai: bool = True
+    include_runtime: bool = True
+    include_source_maps: bool = True
+    repo_url: Optional[str] = None
 
 
 class DPDPQuickCheckRequest(BaseModel):
@@ -2663,8 +2666,319 @@ def _flow_risk(score: int) -> str:
     return "low"
 
 
+SERVICE_PROVIDER_SIGNATURES = [
+    {"name": "Supabase", "category": "database", "patterns": ["supabase.co", "supabase.com", "createclient(", "@supabase/supabase-js", "/rest/v1", "/auth/v1"], "data": ["database rows", "auth tokens", "PII records"]},
+    {"name": "Firebase / Firestore", "category": "database", "patterns": ["firebaseio.com", "firestore.googleapis.com", "firebaseapp.com", "identitytoolkit.googleapis.com", "firebase/auth"], "data": ["user profile", "auth identity", "documents"]},
+    {"name": "MongoDB Atlas Data API", "category": "database", "patterns": ["data.mongodb-api.com", "mongodb+srv://", "realm.mongodb.com"], "data": ["documents", "database records"]},
+    {"name": "Neon Postgres", "category": "database", "patterns": ["neon.tech", "neon database", "postgresql://", "DATABASE_URL"], "data": ["SQL rows", "PII records"]},
+    {"name": "PlanetScale", "category": "database", "patterns": ["planetscale.com", "pscale_pw_", "mysql://"], "data": ["SQL rows", "PII records"]},
+    {"name": "Upstash Redis", "category": "database", "patterns": ["upstash.io", "UPSTASH_REDIS", "redis://"], "data": ["cache keys", "session data"]},
+    {"name": "Hasura", "category": "database_api", "patterns": ["hasura.app", "/v1/graphql", "x-hasura"], "data": ["GraphQL records", "PII records"]},
+    {"name": "Appwrite", "category": "database_api", "patterns": ["appwrite.io", "/v1/databases", "appwrite"], "data": ["database documents", "auth identity"]},
+    {"name": "Convex", "category": "database_api", "patterns": ["convex.cloud", "convex.site", "convex/react"], "data": ["application records"]},
+    {"name": "Clerk", "category": "auth", "patterns": ["clerk.accounts.dev", "clerk.com", "@clerk/", "__clerk"], "data": ["identity", "session", "email"]},
+    {"name": "Auth0", "category": "auth", "patterns": ["auth0.com", "auth0", "/oauth/token"], "data": ["identity", "session", "email"]},
+    {"name": "NextAuth", "category": "auth", "patterns": ["/api/auth", "next-auth", "authjs"], "data": ["session", "identity"]},
+    {"name": "Stripe", "category": "payment", "patterns": ["js.stripe.com", "api.stripe.com", "stripe.confirm", "stripe.redirecttocheckout"], "data": ["payment metadata", "billing contact"]},
+    {"name": "Razorpay", "category": "payment", "patterns": ["checkout.razorpay.com", "api.razorpay.com", "razorpay"], "data": ["payment metadata", "billing contact"]},
+    {"name": "Paddle", "category": "payment", "patterns": ["paddle.com", "paddle.js"], "data": ["billing contact", "subscription metadata"]},
+    {"name": "Google Analytics", "category": "analytics", "patterns": ["googletagmanager.com", "google-analytics.com", "gtag(", "G-"], "data": ["page view", "device identifiers", "events"]},
+    {"name": "Vercel Analytics", "category": "analytics", "patterns": ["va.vercel-scripts.com", "vercel analytics", "_vercel/insights"], "data": ["page view", "performance events", "device metadata"]},
+    {"name": "PostHog", "category": "analytics", "patterns": ["posthog.com", "posthog-js", "posthog.capture"], "data": ["product events", "user identifiers"]},
+    {"name": "Mixpanel", "category": "analytics", "patterns": ["mixpanel.com", "mixpanel.track"], "data": ["product events", "user identifiers"]},
+    {"name": "Segment", "category": "analytics", "patterns": ["segment.com", "analytics.identify", "analytics.track"], "data": ["event stream", "traits"]},
+    {"name": "Sentry", "category": "error_monitoring", "patterns": ["sentry.io", "Sentry.init", "@sentry/"], "data": ["errors", "user context", "stack traces"]},
+    {"name": "LogRocket", "category": "session_replay", "patterns": ["logrocket.com", "LogRocket.init"], "data": ["session replay", "user events"]},
+    {"name": "Intercom", "category": "support", "patterns": ["intercom.io", "intercomcdn.com", "Intercom("], "data": ["support identity", "messages"]},
+    {"name": "Zendesk", "category": "support", "patterns": ["zendesk.com", "zdassets.com", "zE("], "data": ["support identity", "tickets"]},
+    {"name": "OpenAI", "category": "ai", "patterns": ["api.openai.com", "/v1/chat/completions", "/v1/responses", "openai"], "data": ["prompt text", "user message", "metadata"]},
+    {"name": "Anthropic", "category": "ai", "patterns": ["api.anthropic.com", "claude-", "anthropic"], "data": ["prompt text", "user message", "metadata"]},
+    {"name": "Google Gemini", "category": "ai", "patterns": ["generativelanguage.googleapis.com", "gemini-"], "data": ["prompt text", "user message", "metadata"]},
+    {"name": "Pinecone", "category": "vector_db", "patterns": ["pinecone.io", "pinecone", "pcsk_"], "data": ["embeddings", "document chunks", "metadata"]},
+    {"name": "Qdrant", "category": "vector_db", "patterns": ["qdrant", "qdrant.tech"], "data": ["embeddings", "document chunks", "metadata"]},
+    {"name": "Cloudinary", "category": "storage", "patterns": ["cloudinary.com", "res.cloudinary.com"], "data": ["uploaded files", "media metadata"]},
+    {"name": "AWS S3", "category": "storage", "patterns": ["amazonaws.com", ".s3.", "s3.amazonaws.com"], "data": ["files", "exports", "media"]},
+]
+
+
+def _service_node_kind(category: str) -> str:
+    if category in {"database", "database_api", "vector_db"}:
+        return "database"
+    if category in {"auth", "payment", "ai", "storage", "support"}:
+        return "processor"
+    if category in {"analytics", "error_monitoring", "session_replay"}:
+        return "third_party"
+    return "service"
+
+
+def _service_risk(category: str) -> str:
+    if category in {"database", "database_api", "vector_db", "ai", "session_replay"}:
+        return "high"
+    if category in {"auth", "payment", "analytics", "support", "storage"}:
+        return "medium"
+    return "low"
+
+
+def _absolute_url(base_url: str, value: str) -> str:
+    from urllib.parse import urljoin
+    value = (value or "").strip()
+    if not value:
+        return ""
+    if value.startswith("//"):
+        return "https:" + value
+    return urljoin(base_url, value)
+
+
+def _extract_public_service_map(url: str, include_source_maps: bool = True, max_assets: int = 36) -> dict:
+    from urllib.parse import urljoin, urlparse
+
+    target_url = url.strip()
+    if not target_url.startswith(("http://", "https://")):
+        target_url = "https://" + target_url
+    parsed = urlparse(target_url)
+    base_origin = f"{parsed.scheme}://{parsed.netloc}"
+    headers = {
+        "User-Agent": "Mozilla/5.0 RedactAI-ServiceMap/1.0",
+        "Accept": "text/html,application/javascript,text/plain,*/*",
+    }
+    assets = []
+    errors = []
+
+    def add_asset(asset_url, asset_type, text, status=200, content_type=""):
+        assets.append({
+            "url": asset_url,
+            "type": asset_type,
+            "text": text[:1_000_000],
+            "status": status,
+            "content_type": content_type,
+        })
+
+    def fetch(asset_url, asset_type, max_bytes=1_000_000):
+        try:
+            resp = http_requests.get(asset_url, headers=headers, timeout=14, allow_redirects=True)
+            text = resp.text[:max_bytes]
+            add_asset(str(resp.url), asset_type, text, resp.status_code, resp.headers.get("content-type", ""))
+            return text, resp
+        except Exception as exc:
+            errors.append({"url": asset_url, "error": str(exc)[:160]})
+            return "", None
+
+    html, resp = fetch(target_url, "html", 1_200_000)
+    if resp is None:
+        return {"assets": [], "api_calls": [], "services": [], "errors": errors}
+
+    soup = BeautifulSoup(html, "html.parser")
+    discovered = []
+    for script in soup.find_all("script"):
+        src = script.get("src")
+        if src:
+            discovered.append((_absolute_url(target_url, src), "javascript"))
+        else:
+            inline = script.string or script.get_text() or ""
+            if inline.strip():
+                add_asset(target_url + "#inline-script", "inline_script", inline)
+    for link in soup.find_all("link", href=True):
+        rel = " ".join(link.get("rel") or []).lower()
+        href = link.get("href")
+        if "preload" in rel or "modulepreload" in rel or str(href).endswith((".js", ".mjs", ".map")):
+            discovered.append((_absolute_url(target_url, href), "linked_asset"))
+
+    seen = {asset["url"] for asset in assets}
+    for asset_url, asset_type in discovered[:max_assets]:
+        if asset_url in seen:
+            continue
+        seen.add(asset_url)
+        text, asset_resp = fetch(asset_url, asset_type)
+        if include_source_maps and text:
+            for map_match in re.finditer(r"sourceMappingURL=([^\s*]+)", text, re.IGNORECASE):
+                map_url = urljoin(asset_url, map_match.group(1).strip())
+                if map_url in seen or len(assets) >= max_assets + 12:
+                    continue
+                seen.add(map_url)
+                map_text, map_resp = fetch(map_url, "source_map", 1_500_000)
+                if map_resp is not None and map_resp.status_code < 400:
+                    try:
+                        source_map = json.loads(map_text)
+                        sources = source_map.get("sources") or []
+                        for idx, source_text in enumerate(source_map.get("sourcesContent") or []):
+                            if isinstance(source_text, str) and source_text.strip():
+                                source_name = sources[idx] if idx < len(sources) else f"source-{idx}"
+                                add_asset(f"{map_url}#{source_name}", "source_map_source", source_text, 200, "text/source")
+                    except Exception:
+                        pass
+
+    api_calls = []
+    services = []
+    service_seen = set()
+    api_patterns = [
+        r"""(?i)\bfetch\(\s*["']([^"']{2,240})["']""",
+        r"""(?i)\baxios\.(?:get|post|put|patch|delete)\(\s*["']([^"']{2,240})["']""",
+        r"""(?i)\b(?:baseURL|apiUrl|apiURL|endpoint|url)\s*[:=]\s*["']([^"']{2,240})["']""",
+        r"""https?://[A-Za-z0-9._~:/?#\[\]@!$&'()*+,;=%-]{6,240}""",
+    ]
+    for asset in assets:
+        text = asset.get("text") or ""
+        lower = text.lower()
+        for pattern_index, pattern in enumerate(api_patterns):
+            for match in re.finditer(pattern, text):
+                raw = match.group(1) if match.groups() else match.group(0)
+                if raw.startswith(("data:", "blob:", "javascript:", "#")):
+                    continue
+                full = _absolute_url(base_origin, raw)
+                if not full.startswith(("http://", "https://", "/")):
+                    continue
+                if full.lower().split("?", 1)[0].endswith((".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico", ".css", ".woff", ".woff2", ".ttf", ".mp4", ".mp3")):
+                    continue
+                host = urlparse(full).netloc or parsed.netloc
+                if not host:
+                    continue
+                evidence = _text_window(text, match.start(), match.end(), radius=55)
+                url_lower = (full + " " + evidence).lower()
+                provider_like = any(any(p.lower() in url_lower for p in signature["patterns"]) for signature in SERVICE_PROVIDER_SIGNATURES)
+                api_path_like = any(token in url_lower for token in ["/api/", "/api?", "/graphql", "/rest/", "/rpc/", "/auth/", "/v1/", "/v2/", "/trpc", "/functions/", "/checkout", "/webhook"])
+                path = urlparse(full).path or ""
+                host_lower = host.lower()
+                if host_lower.endswith(("w3.org", "schema.org", "mozilla.org")):
+                    continue
+                if path in {"", "/"} and host_lower.endswith(parsed.netloc.lower()):
+                    continue
+                if path in {"", "/"} and not provider_like and not api_path_like:
+                    continue
+                if pattern_index == 3 and not provider_like and not api_path_like:
+                    continue
+                api_id = hashlib.sha256((full + asset["url"]).encode("utf-8", errors="ignore")).hexdigest()[:14]
+                if not any(call["id"] == api_id for call in api_calls):
+                    api_calls.append({
+                        "id": api_id,
+                        "url": full[:300],
+                        "host": host,
+                        "first_party": host.lower().endswith(parsed.netloc.lower()),
+                        "asset": asset["url"],
+                        "asset_type": asset["type"],
+                        "evidence": evidence[:240],
+                        "confidence": "static_js",
+                    })
+
+        for signature in SERVICE_PROVIDER_SIGNATURES:
+            matched = [p for p in signature["patterns"] if p.lower() in lower or p.lower() in asset["url"].lower()]
+            if not matched:
+                continue
+            key = (signature["name"], signature["category"])
+            if key in service_seen:
+                continue
+            service_seen.add(key)
+            services.append({
+                "name": signature["name"],
+                "category": signature["category"],
+                "matched": matched[:4],
+                "asset": asset["url"],
+                "asset_type": asset["type"],
+                "data_types": signature["data"],
+                "confidence": "public_bundle" if asset["type"] != "html" else "public_html",
+            })
+
+    for call in api_calls:
+        call_text = (call["url"] + " " + call["evidence"]).lower()
+        for signature in SERVICE_PROVIDER_SIGNATURES:
+            if any(p.lower() in call_text for p in signature["patterns"]):
+                key = (signature["name"], signature["category"])
+                if key not in service_seen:
+                    service_seen.add(key)
+                    services.append({
+                        "name": signature["name"],
+                        "category": signature["category"],
+                        "matched": [call["host"]],
+                        "asset": call["asset"],
+                        "asset_type": "api_call",
+                        "data_types": signature["data"],
+                        "confidence": "api_url",
+                    })
+
+    return {
+        "assets": [{"url": a["url"], "type": a["type"], "status": a["status"], "content_type": a["content_type"]} for a in assets],
+        "api_calls": api_calls[:120],
+        "services": services,
+        "errors": errors[:10],
+    }
+
+
+def _extract_runtime_service_map(url: str, seconds: int = 8) -> dict:
+    import asyncio
+    import threading
+
+    try:
+        asyncio.get_running_loop()
+        in_async_loop = True
+    except RuntimeError:
+        in_async_loop = False
+
+    if not in_async_loop:
+        return _extract_runtime_service_map_sync(url, seconds)
+
+    holder = {}
+
+    def worker():
+        holder["result"] = _extract_runtime_service_map_sync(url, seconds)
+
+    thread = threading.Thread(target=worker, daemon=True)
+    thread.start()
+    thread.join(timeout=max(12, min(seconds, 20) + 8))
+    if thread.is_alive():
+        return {"available": True, "requests": [], "services": [], "error": "Runtime capture timed out"}
+    return holder.get("result", {"available": False, "requests": [], "services": [], "reason": "Runtime capture did not return"})
+
+
+def _extract_runtime_service_map_sync(url: str, seconds: int = 8) -> dict:
+    try:
+        from playwright.sync_api import sync_playwright
+    except Exception:
+        return {"available": False, "requests": [], "services": [], "reason": "Playwright is not installed on this host"}
+
+    target_url = url.strip()
+    if not target_url.startswith(("http://", "https://")):
+        target_url = "https://" + target_url
+    requests_seen = []
+    services = []
+    service_seen = set()
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            try:
+                page = browser.new_page()
+
+                def on_request(request):
+                    req_url = request.url
+                    requests_seen.append({"url": req_url, "method": request.method, "resource_type": request.resource_type})
+                    low = req_url.lower()
+                    for signature in SERVICE_PROVIDER_SIGNATURES:
+                        if any(pattern.lower() in low for pattern in signature["patterns"]):
+                            key = (signature["name"], signature["category"])
+                            if key not in service_seen:
+                                service_seen.add(key)
+                                services.append({
+                                    "name": signature["name"],
+                                    "category": signature["category"],
+                                    "matched": [request.url[:160]],
+                                    "asset": request.url,
+                                    "asset_type": "runtime_request",
+                                    "data_types": signature["data"],
+                                    "confidence": "runtime_observed",
+                                })
+
+                page.on("request", on_request)
+                page.goto(target_url, wait_until="domcontentloaded", timeout=max(5000, min(seconds, 20) * 1000))
+                page.wait_for_timeout(max(1500, min(seconds, 10) * 1000))
+            finally:
+                browser.close()
+    except Exception as exc:
+        return {"available": True, "requests": requests_seen[:80], "services": services, "error": str(exc)[:220]}
+    return {"available": True, "requests": requests_seen[:120], "services": services}
+
+
 def _build_data_flow_visualization(scan_report: dict, req: DataFlowVisualizeRequest) -> dict:
     domain = scan_report.get("domain") or scan_report.get("url", "product")
+    public_services = _extract_public_service_map(scan_report.get("url") or req.url, req.include_source_maps)
+    runtime_services = _extract_runtime_service_map(scan_report.get("url") or req.url) if req.include_runtime else {"available": False, "requests": [], "services": [], "reason": "Runtime capture disabled"}
     pii_inputs = scan_report.get("pii_collection", {}).get("inputs", []) or []
     trackers = scan_report.get("trackers", {}).get("items", []) or []
     tracker_categories = scan_report.get("trackers", {}).get("categories", {}) or {}
@@ -2675,6 +2989,9 @@ def _build_data_flow_visualization(scan_report: dict, req: DataFlowVisualizeRequ
     cookie_summary = scan_report.get("cookies", {}).get("summary", {}) or {}
     compliance = scan_report.get("compliance", {}) or {}
     blacklight = scan_report.get("blacklight", {}) or {}
+    detected_services = public_services.get("services", []) + runtime_services.get("services", [])
+    api_calls = public_services.get("api_calls", [])
+    runtime_requests = runtime_services.get("requests", [])
 
     nodes = []
     edges = []
@@ -2701,6 +3018,7 @@ def _build_data_flow_visualization(scan_report: dict, req: DataFlowVisualizeRequ
             "risk": risk,
             "evidence": evidence[:260],
             "control": control,
+            "confidence": "observed" if "runtime" in evidence.lower() else "inferred",
         })
 
     collected_types = sorted({(item.get("type") or "personal_data").replace("_", " ") for item in pii_inputs})
@@ -2717,6 +3035,80 @@ def _build_data_flow_visualization(scan_report: dict, req: DataFlowVisualizeRequ
     add_node("product", domain, "first_party", _flow_risk(scan_report.get("risk_score", 0)), "First-party product surface")
     add_node("policy", "Notice, Consent, Retention", "governance", "low" if compliance.get("cookie_consent") and compliance.get("privacy_policy") else "high", "DPDP/GDPR-style control plane")
 
+    for call in api_calls[:40]:
+        call_host = call.get("host") or "api"
+        path_hint = call.get("url", "").split(call_host, 1)[-1][:42] if call_host in call.get("url", "") else call.get("url", "")[:42]
+        node_id = "api_" + call["id"]
+        node_label = ("First-party API " if call.get("first_party") else "External API ") + call_host
+        add_node(
+            node_id,
+            node_label,
+            "api_endpoint" if call.get("first_party") else "external_api",
+            "medium" if call.get("first_party") else "high",
+            path_hint or call.get("confidence", "api call"),
+        )
+        add_edge(
+            "browser",
+            node_id,
+            "calls API",
+            ["request metadata", "possible form data", "session identifiers"],
+            "medium" if call.get("first_party") else "high",
+            f"{call.get('confidence')} {call.get('evidence', '')}",
+            "Review request payloads, auth, rate limits, and whether PII is sent to this endpoint.",
+        )
+
+    service_seen_ids = set()
+    for service in detected_services:
+        service_key = f"{service.get('name')}:{service.get('category')}"
+        if service_key in service_seen_ids:
+            continue
+        service_seen_ids.add(service_key)
+        category = service.get("category", "service")
+        node_id = "service_" + hashlib.sha256(service_key.encode("utf-8", errors="ignore")).hexdigest()[:10]
+        kind = _service_node_kind(category)
+        risk = _service_risk(category)
+        add_node(
+            node_id,
+            service.get("name", "External service"),
+            kind,
+            risk,
+            f"{category} - {service.get('confidence', 'detected')}",
+        )
+        source = "browser" if service.get("confidence") in {"runtime_observed", "api_url", "public_html"} else "product"
+        if category in {"database", "database_api", "vector_db"} and service.get("confidence") in {"public_bundle", "source_map_source", "api_url"}:
+            source = "product"
+        add_edge(
+            source,
+            node_id,
+            "connects to " + category.replace("_", " "),
+            service.get("data_types") or ["application data"],
+            risk,
+            f"{service.get('confidence')} via {service.get('asset', '')[:140]} matched {', '.join(service.get('matched', [])[:3])}",
+            "Confirm whether this provider receives PII, whether access is server-only, and whether keys/tokens are protected.",
+        )
+
+    if runtime_requests:
+        runtime_hosts = {}
+        from urllib.parse import urlparse
+        for item in runtime_requests:
+            host = urlparse(item.get("url", "")).netloc
+            if host and not host.endswith(domain):
+                runtime_hosts[host] = runtime_hosts.get(host, 0) + 1
+        for host, count in sorted(runtime_hosts.items(), key=lambda pair: pair[1], reverse=True)[:15]:
+            node_id = "runtime_" + hashlib.sha256(host.encode("utf-8", errors="ignore")).hexdigest()[:10]
+            if any(node["id"] == node_id for node in nodes):
+                continue
+            add_node(node_id, host, "runtime_service", "medium", f"{count} runtime request(s)")
+            add_edge(
+                "browser",
+                node_id,
+                "loads/calls at runtime",
+                ["IP address", "user agent", "referrer", "event metadata"],
+                "medium",
+                f"runtime observed {count} request(s)",
+                "Classify this host as a vendor, CDN, API, analytics, or storage provider and document purpose.",
+            )
+
     add_edge(
         "user",
         "browser",
@@ -2846,6 +3238,10 @@ def _build_data_flow_visualization(scan_report: dict, req: DataFlowVisualizeRequ
         remediation.append("Remove visible PII from public content, metadata, caches, and example payloads.")
     if ai_endpoints:
         remediation.append("Redact PII before LLM prompts and route AI processing through a monitored backend.")
+    if detected_services:
+        remediation.append("Review every detected provider, API host, and database signal for data category, purpose, auth, retention, and contract owner.")
+    if not any(service.get("category") in {"database", "database_api", "vector_db"} for service in detected_services):
+        remediation.append("No database was publicly observable. To map private DB calls, connect a GitHub repo, AppMap/OpenTelemetry traces, or backend logs.")
     if not compliance.get("privacy_policy"):
         remediation.append("Publish a discoverable privacy notice that maps purposes, retention, sharing, and user rights.")
     if not remediation:
@@ -2864,13 +3260,25 @@ def _build_data_flow_visualization(scan_report: dict, req: DataFlowVisualizeRequ
             "flows": len(edges),
             "pii_types": collected_types,
             "processors": len([n for n in nodes if n["kind"] in {"third_party", "third_party_domain", "processor", "high_risk_processor"}]),
+            "api_calls": len(api_calls),
+            "runtime_requests": len(runtime_requests),
+            "services_detected": len(detected_services),
+            "databases_detected": len([s for s in detected_services if s.get("category") in {"database", "database_api", "vector_db"}]),
+            "runtime_capture": runtime_services,
             "high_risk_flows": len([e for e in edges if e["risk"] in {"high", "critical"}]),
             "tracker_categories": tracker_categories,
             "cookie_summary": cookie_summary,
         },
         "nodes": nodes,
         "edges": edges,
+        "api_calls": api_calls,
+        "services": detected_services,
+        "assets": public_services.get("assets", []),
         "remediation": remediation,
+        "limitations": [
+            "Public URL scans can only observe browser-visible services, public bundles, source maps, metadata, and runtime network requests.",
+            "Private backend-to-database calls require source repository analysis, AppMap/OpenTelemetry traces, logs, or cloud account integration.",
+        ],
         "source_report": {
             "risk_factors": scan_report.get("risk_factors", []),
             "dpdp": scan_report.get("dpdp", {}),