Add DPDP policy control matrix

__pycache__/server.cpython-310.pyc

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5df4d6fb8be71cc7c0fbc66169cb50ec19bca9c4c2c025fb5f5dfa8bd6bd45eb
-size 145960
+oid sha256:13079c3489f54c20fc3d1d8607ac76848be9a8065fae61fbe98b229b29782a62
+size 155172

dashboard.css

@@ -1180,6 +1180,14 @@ body.theme-light .profile-submenu .profile-dropdown__item { color: var(--text-pr
   margin-bottom: 16px;
 }
 
+.tool-score--compact {
+  padding: 14px 16px;
+}
+
+.tool-score--compact strong {
+  font-size: 22px;
+}
+
 .tool-score strong {
   color: var(--text-primary);
   font: 800 30px/1 'JetBrains Mono', monospace;
@@ -1202,6 +1210,12 @@ body.theme-light .profile-submenu .profile-dropdown__item { color: var(--text-pr
   margin-top: 16px;
 }
 
+.tool-section-title {
+  color: var(--text-primary);
+  font-size: 16px;
+  margin: 4px 0 12px;
+}
+
 .tool-result-item {
   padding: 14px;
   border: 1px solid rgba(255,255,255,0.08);

dashboard.js

@@ -1641,11 +1641,23 @@ function initProductTools() {
       };
       const evidenceLine = check => (check.evidence || []).slice(0, 2).map(e => `<span>${escapeFlowHtml(e)}</span>`).join('');
       const processors = (data.detected_processors || []).slice(0, 8).map(p => `<span class="badge badge--warning">${escapeFlowHtml(p.name || 'Provider')} · ${escapeFlowHtml(p.category || 'processor')}</span>`).join('');
+      const policy = data.policy_control_matrix || {};
+      const policyControls = (policy.controls || []).slice(0, 12).map(c => `
+        <div class="tool-result-item">
+          <strong>${statusBadge(c.status)} ${escapeFlowHtml(c.title || c.id)}</strong>
+          <span>${escapeFlowHtml(c.act || '')}</span>
+          <span>${escapeFlowHtml(c.rules || '')}</span>
+          ${(c.evidence || []).slice(0, 1).map(e => `<span>${escapeFlowHtml(e)}</span>`).join('')}
+        </div>`).join('');
       render(out, `
         <div class="tool-score">
           <strong>${escapeFlowHtml(data.grade)}</strong>
           <span>${escapeFlowHtml(data.verdict)} · ${escapeFlowHtml(data.score)}% · ${escapeFlowHtml(data.overall_risk || 'Review')} risk</span>
         </div>
+        <div class="tool-score tool-score--compact">
+          <strong>${escapeFlowHtml(policy.grade || '-')}</strong>
+          <span>Policy evidence · ${escapeFlowHtml(policy.score ?? 'n/a')}% · ${escapeFlowHtml((policy.coverage && `${policy.coverage.pass || 0} pass / ${policy.coverage.review || 0} review / ${policy.coverage.fail || 0} fail`) || 'no policy matrix')}</span>
+        </div>
         ${processors ? `<div class="tool-result-item"><strong>Detected processors / providers</strong><span>${processors}</span></div>` : ''}
         <div class="tool-grid">${(data.checks || []).map(c => `
           <div class="tool-result-item">
@@ -1655,6 +1667,7 @@ function initProductTools() {
             <span>${escapeFlowHtml(c.section || '')}</span>
           </div>`).join('')}</div>
         <div class="tool-result-list">
+          ${policyControls ? `<div><h3 class="tool-section-title">Policy Control Matrix</h3><div class="tool-grid">${policyControls}</div></div>` : ''}
           <div class="tool-result-item"><strong>Priority actions</strong>${(data.priority_actions || []).slice(0, 4).map(a => `<span>${escapeFlowHtml(a)}</span>`).join('')}</div>
           <div class="tool-result-item"><strong>Scanner limitations</strong>${(data.limitations || []).map(a => `<span>${escapeFlowHtml(a)}</span>`).join('')}</div>
         </div>

server.py

@@ -3293,6 +3293,191 @@ def _simple_grade(score: int) -> str:
     return "F"
 
 
+DPDP_POLICY_CONTROL_CATALOG = [
+    {
+        "id": "notice_plain_language",
+        "title": "Standalone notice in clear language",
+        "act": "DPDP Act Section 5",
+        "rules": "DPDP Rules 2025 - notice requirements",
+        "keywords": ["privacy notice", "privacy policy", "personal data", "collect", "purpose", "processing"],
+        "required_hits": 3,
+        "weight": 10,
+        "fix": "Make the notice standalone, plain-English, and separate from unrelated terms.",
+    },
+    {
+        "id": "itemized_personal_data",
+        "title": "Itemized personal data categories",
+        "act": "DPDP Act Section 5",
+        "rules": "DPDP Rules 2025 - itemized data collection notice",
+        "keywords": ["name", "email", "phone", "address", "payment", "device", "location", "personal information", "personal data we collect"],
+        "required_hits": 2,
+        "weight": 9,
+        "fix": "List each data category collected, grouped by product workflow.",
+    },
+    {
+        "id": "purpose_specificity",
+        "title": "Specific purpose for each collection",
+        "act": "DPDP Act Section 5 and Section 7",
+        "rules": "DPDP Rules 2025 - purpose description in notice",
+        "keywords": ["purpose", "to provide", "to process", "to improve", "to communicate", "for marketing", "for analytics", "services enabled"],
+        "required_hits": 2,
+        "weight": 10,
+        "fix": "Map every data category to a specific purpose and service enabled by processing.",
+    },
+    {
+        "id": "consent_withdrawal",
+        "title": "Consent withdrawal and preference management",
+        "act": "DPDP Act Section 6(4)",
+        "rules": "DPDP Rules 2025 - withdrawal comparable to giving consent",
+        "keywords": ["withdraw consent", "revoke consent", "manage consent", "manage preferences", "cookie settings", "opt out", "unsubscribe"],
+        "required_hits": 1,
+        "weight": 12,
+        "fix": "Provide a persistent preference link and make withdrawal as easy as giving consent.",
+    },
+    {
+        "id": "data_principal_rights",
+        "title": "Access, correction, erasure, grievance, and nomination rights",
+        "act": "DPDP Act Sections 11, 12, 13, and 14",
+        "rules": "DPDP Rules 2025 - rights request handling",
+        "keywords": ["access your data", "correct", "correction", "erase", "erasure", "delete your data", "grievance", "complaint", "nominate"],
+        "required_hits": 3,
+        "weight": 13,
+        "fix": "Add a rights section covering access, correction, deletion, grievance, and nomination workflows.",
+    },
+    {
+        "id": "grievance_contact",
+        "title": "Grievance/contact channel and escalation",
+        "act": "DPDP Act Section 13 and Section 8(7)",
+        "rules": "DPDP Rules 2025 - complaint and Board communication link",
+        "keywords": ["grievance officer", "grievance", "privacy@", "dpo@", "data protection officer", "complaint", "data protection board"],
+        "required_hits": 1,
+        "weight": 12,
+        "fix": "Publish a privacy contact, grievance process, response path, and escalation route.",
+    },
+    {
+        "id": "retention_deletion",
+        "title": "Retention schedule and deletion policy",
+        "act": "DPDP Act Section 8(6)",
+        "rules": "DPDP Rules 2025 - retention/deletion accountability",
+        "keywords": ["retention", "retain", "how long", "delete", "deletion", "erasure", "storage period", "no longer necessary"],
+        "required_hits": 2,
+        "weight": 12,
+        "fix": "State retention periods by data category and explain deletion triggers.",
+    },
+    {
+        "id": "security_safeguards",
+        "title": "Reasonable security safeguards",
+        "act": "DPDP Act Section 8(5)",
+        "rules": "DPDP Rules 2025 - security safeguards and breach duties",
+        "keywords": ["security", "safeguards", "encryption", "access control", "confidentiality", "incident", "breach", "unauthorized"],
+        "required_hits": 2,
+        "weight": 10,
+        "fix": "Describe encryption, access controls, audit logging, incident response, and vendor security safeguards.",
+    },
+    {
+        "id": "breach_notification",
+        "title": "Breach notification process",
+        "act": "DPDP Act Section 8(6) and security obligations",
+        "rules": "DPDP Rules 2025 - personal data breach notice",
+        "keywords": ["data breach", "security breach", "personal data breach", "notify", "notification", "incident response", "data protection board"],
+        "required_hits": 2,
+        "weight": 9,
+        "fix": "Document breach notification to affected users and the Data Protection Board.",
+    },
+    {
+        "id": "children_data",
+        "title": "Children's data posture",
+        "act": "DPDP Act Section 9",
+        "rules": "DPDP Rules 2025 - verifiable parental consent",
+        "keywords": ["child", "children", "minor", "under 18", "parental consent", "guardian", "age verification"],
+        "required_hits": 1,
+        "weight": 7,
+        "fix": "State whether children can use the service; if yes, document parental consent and no tracking/profiling controls.",
+    },
+    {
+        "id": "processor_vendor_sharing",
+        "title": "Processors, vendors, and sharing purposes",
+        "act": "DPDP Act Section 8 - accountability",
+        "rules": "DPDP Rules 2025 - fiduciary accountability",
+        "keywords": ["third party", "service provider", "processor", "vendor", "affiliate", "share", "sub-processor", "analytics"],
+        "required_hits": 2,
+        "weight": 10,
+        "fix": "List processor categories, sharing purposes, safeguards, and contract ownership.",
+    },
+    {
+        "id": "cross_border_transfer",
+        "title": "Cross-border transfer disclosure",
+        "act": "DPDP Act Section 16",
+        "rules": "DPDP Rules 2025 - cross-border transfer restrictions",
+        "keywords": ["transfer", "outside india", "cross-border", "international", "global", "countries", "jurisdiction"],
+        "required_hits": 1,
+        "weight": 6,
+        "fix": "Disclose whether personal data is transferred outside India and how transfer restrictions are handled.",
+    },
+]
+
+
+def _snippet_for_keywords(text: str, keywords: list, max_snippets: int = 3) -> list:
+    snippets = []
+    if not text:
+        return snippets
+    compact = re.sub(r"\s+", " ", text).strip()
+    lower = compact.lower()
+    for keyword in keywords:
+        idx = lower.find(keyword.lower())
+        if idx == -1:
+            continue
+        start = max(0, idx - 90)
+        end = min(len(compact), idx + len(keyword) + 140)
+        snippet = compact[start:end].strip()
+        if snippet and snippet not in snippets:
+            snippets.append(snippet)
+        if len(snippets) >= max_snippets:
+            break
+    return snippets
+
+
+def _analyze_dpdp_policy_text(policy_text: str, privacy_url: Optional[str] = None) -> dict:
+    text = re.sub(r"\s+", " ", (policy_text or "")).strip()
+    lower = text.lower()
+    controls = []
+    total_weight = sum(control["weight"] for control in DPDP_POLICY_CONTROL_CATALOG)
+    earned = 0
+    for control in DPDP_POLICY_CONTROL_CATALOG:
+        matched = sorted({kw for kw in control["keywords"] if kw.lower() in lower})
+        passed = len(matched) >= control["required_hits"]
+        partial = bool(matched) and not passed
+        if passed:
+            earned += control["weight"]
+        elif partial:
+            earned += control["weight"] * 0.35
+        snippets = _snippet_for_keywords(text, matched or control["keywords"], max_snippets=3)
+        controls.append({
+            "id": control["id"],
+            "title": control["title"],
+            "status": "pass" if passed else "review" if partial else "fail",
+            "act": control["act"],
+            "rules": control["rules"],
+            "matched_terms": matched[:8],
+            "evidence": snippets,
+            "weight": control["weight"],
+            "fix": control["fix"],
+        })
+    score = round((earned / total_weight) * 100) if total_weight else 0
+    return {
+        "privacy_url": privacy_url,
+        "text_length": len(text),
+        "score": score,
+        "grade": _simple_grade(score),
+        "controls": controls,
+        "coverage": {
+            "pass": sum(1 for c in controls if c["status"] == "pass"),
+            "review": sum(1 for c in controls if c["status"] == "review"),
+            "fail": sum(1 for c in controls if c["status"] == "fail"),
+        },
+    }
+
+
 def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) -> dict:
     compliance = report.get("compliance", {}) or {}
     dpdp = report.get("dpdp", {}) or {}
@@ -3303,6 +3488,8 @@ def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) ->
     exposed = report.get("exposed_pii", {}) or {}
     headers = compliance.get("security_headers", {}) or {}
     blacklight = report.get("blacklight", {}) or {}
+    policy_analysis = compliance.get("policy_analysis", {}) or {}
+    policy_controls = {control.get("id"): control for control in (policy_analysis.get("controls") or [])}
     service_map = service_map or {}
     services = service_map.get("services", []) or []
     api_calls = service_map.get("api_calls", []) or []
@@ -3331,6 +3518,23 @@ def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) ->
     def header_pass(name: str) -> bool:
         return bool((headers.get(name) or {}).get("present")) and (headers.get(name) or {}).get("rating") in {"pass", "warn"}
 
+    def policy_status(control_id: str) -> str:
+        return (policy_controls.get(control_id) or {}).get("status", "fail")
+
+    def policy_pass(control_id: str) -> bool:
+        return policy_status(control_id) == "pass"
+
+    def policy_evidence(control_id: str, fallback: str) -> list:
+        control = policy_controls.get(control_id) or {}
+        snippets = control.get("evidence") or []
+        terms = control.get("matched_terms") or []
+        evidence = snippets[:2]
+        if terms:
+            evidence.append("Matched terms: " + ", ".join(terms[:6]))
+        if not evidence:
+            evidence.append(fallback)
+        return evidence
+
     def make_check(
         check_id: str,
         label: str,
@@ -3360,20 +3564,21 @@ def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) ->
     checks = []
 
     privacy_url = compliance.get("privacy_policy_url")
+    notice_quality = policy_pass("notice_plain_language") and policy_pass("itemized_personal_data") and policy_pass("purpose_specificity")
     checks.append(make_check(
         "privacy_notice",
-        "Privacy notice is discoverable",
-        "pass" if compliance.get("privacy_policy") and privacy_url else "fail",
+        "Privacy notice is discoverable and meaningfully itemized",
+        "pass" if compliance.get("privacy_policy") and privacy_url and notice_quality else "review" if compliance.get("privacy_policy") and privacy_url else "fail",
         "DPDP Act Section 5 - Notice",
         "high",
         "A Data Principal should be able to find a clear notice explaining what personal data is collected and why.",
-        "Publish a visible privacy notice link in the footer, signup, checkout, and every form that collects personal data.",
-        [f"Privacy URL: {privacy_url}" if privacy_url else "No privacy policy URL discovered on the scanned page"],
+        "Publish a visible privacy notice and itemize personal data categories, purpose, goods/services enabled, rights, and complaint links.",
+        [f"Privacy URL: {privacy_url}" if privacy_url else "No privacy policy URL discovered on the scanned page"] + policy_evidence("notice_plain_language", "No clear notice evidence found")[:1],
         16,
         "high" if privacy_url else "medium",
     ))
 
-    consent_ready = bool(compliance.get("cookie_consent")) and dpdp_pass("consent_mechanism")
+    consent_ready = bool(compliance.get("cookie_consent")) and (dpdp_pass("consent_mechanism") or policy_pass("consent_withdrawal"))
     consent_status = "pass" if consent_ready else "fail" if (tracker_count or non_essential_processors) else "review"
     checks.append(make_check(
         "consent_mechanism",
@@ -3395,44 +3600,56 @@ def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) ->
     checks.append(make_check(
         "consent_withdrawal",
         "Consent withdrawal or preference management is disclosed",
-        "pass" if dpdp_pass("consent_withdrawal") else "fail" if (tracker_count or non_essential_processors) else "review",
+        "pass" if dpdp_pass("consent_withdrawal") or policy_pass("consent_withdrawal") else "fail" if (tracker_count or non_essential_processors) else "review",
         "DPDP Act Section 6(4) - Consent withdrawal",
         "high",
         "Users should have an easy way to withdraw consent or change preferences after accepting.",
         "Add a persistent cookie/privacy preferences link and describe withdrawal steps in the privacy notice.",
-        ["Policy text matched withdrawal language" if dpdp_pass("consent_withdrawal") else "No withdrawal/preference evidence found in public text"],
+        policy_evidence("consent_withdrawal", "No withdrawal/preference evidence found in public text"),
         12,
     ))
 
     checks.append(make_check(
         "grievance_redressal",
         "Grievance/contact path is published",
-        "pass" if dpdp_pass("grievance_officer") else "fail",
+        "pass" if dpdp_pass("grievance_officer") or policy_pass("grievance_contact") else "fail",
         "DPDP Act Section 13 and Section 8(7) - Grievance redressal",
         "high",
         "A user should know who to contact for privacy complaints, correction, erasure, and escalation.",
         "Add a privacy contact or grievance officer section with email, response process, and escalation path.",
-        ["Grievance/DPO language found" if dpdp_pass("grievance_officer") else "No grievance officer, DPO, privacy@, or redressal signal found"],
+        policy_evidence("grievance_contact", "No grievance officer, DPO, privacy@, or redressal signal found"),
         12,
     ))
 
     checks.append(make_check(
         "retention_and_erasure",
         "Retention, deletion, correction, and erasure rights are disclosed",
-        "pass" if dpdp_pass("data_retention_policy") else "fail",
+        "pass" if dpdp_pass("data_retention_policy") and policy_pass("data_principal_rights") else "review" if dpdp_pass("data_retention_policy") or policy_pass("data_principal_rights") or policy_pass("retention_deletion") else "fail",
         "DPDP Act Section 8(6), Section 11, and Section 12",
         "high",
         "Users should know how long data is kept and how they can request correction or deletion.",
         "Document retention periods by data category and add correction/deletion request instructions.",
-        ["Retention/deletion language found" if dpdp_pass("data_retention_policy") else "No retention period, deletion, or erasure evidence found"],
+        policy_evidence("retention_deletion", "No retention period, deletion, or erasure evidence found") + policy_evidence("data_principal_rights", "")[:1],
         12,
     ))
 
+    checks.append(make_check(
+        "rights_request_workflow",
+        "Data Principal rights workflow is operationally described",
+        "pass" if policy_pass("data_principal_rights") else "review" if policy_status("data_principal_rights") == "review" else "fail",
+        "DPDP Act Sections 11, 12, 13, and 14",
+        "high",
+        "A policy should tell users how to access, correct, erase, complain, and nominate a representative.",
+        "Add a rights request workflow with channel, expected response path, identity verification, and nomination language.",
+        policy_evidence("data_principal_rights", "No access/correction/erasure/grievance/nomination workflow evidence found"),
+        10,
+    ))
+
     vendor_status = "pass" if processor_count == 0 and tracker_count == 0 else "review"
     checks.append(make_check(
         "processor_inventory",
         "Third-party processors and API destinations are inventoried",
-        vendor_status,
+        "pass" if vendor_status == "pass" and policy_pass("processor_vendor_sharing") else "review",
         "DPDP Act Section 8 - Data Fiduciary accountability",
         "medium" if vendor_status == "review" else "low",
         "Every analytics, database, payment, support, AI, and auth provider needs a purpose, data category, contract owner, and retention note.",
@@ -3440,11 +3657,24 @@ def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) ->
         [
             f"Detected processors/providers: {', '.join(sorted({p.get('name') for p in detected_processors if p.get('name')}))}" if detected_processors else "No third-party processor signal detected",
             f"API calls discovered: {len(api_calls)}" if api_calls else "",
-        ],
+        ] + policy_evidence("processor_vendor_sharing", "")[:1],
         12,
         "medium" if processor_count else "high",
     ))
 
+    checks.append(make_check(
+        "cross_border_transfer",
+        "Cross-border transfer position is disclosed",
+        "pass" if policy_pass("cross_border_transfer") else "review",
+        "DPDP Act Section 16 and DPDP Rules 2025 transfer restrictions",
+        "medium",
+        "Users and auditors should know whether data leaves India and which safeguards/restrictions apply.",
+        "Add a cross-border transfer statement, countries/regions where practical, and controls for restricted transfers.",
+        policy_evidence("cross_border_transfer", "No cross-border transfer disclosure found; manual review needed"),
+        6,
+        "low",
+    ))
+
     checks.append(make_check(
         "public_pii_exposure",
         "No obvious personal data is exposed on public pages",
@@ -3462,7 +3692,7 @@ def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) ->
     checks.append(make_check(
         "secure_transport_headers",
         "HTTPS and browser privacy/security headers are configured",
-        "pass" if browser_security_ok else "fail",
+        "pass" if browser_security_ok and (policy_pass("security_safeguards") or policy_status("security_safeguards") == "review") else "review" if browser_security_ok else "fail",
         "DPDP Act Section 8(5) - Reasonable security safeguards",
         "medium",
         "Transport security and browser headers reduce accidental disclosure and client-side abuse.",
@@ -3471,10 +3701,22 @@ def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) ->
             f"HTTPS: {bool(compliance.get('https'))}",
             f"HSTS: {(headers.get('strict-transport-security') or {}).get('note', 'missing')}",
             f"Referrer-Policy: {(headers.get('referrer-policy') or {}).get('note', 'missing')}",
-        ],
+        ] + policy_evidence("security_safeguards", "")[:1],
         8,
     ))
 
+    checks.append(make_check(
+        "breach_notification",
+        "Personal data breach notification process is documented",
+        "pass" if dpdp_pass("breach_notification") or policy_pass("breach_notification") else "fail",
+        "DPDP Act security obligations and DPDP Rules 2025 breach notice",
+        "high",
+        "A real audit needs evidence that affected users and the Data Protection Board can be notified when a breach occurs.",
+        "Add breach notification language, incident response owner, timelines, and Data Protection Board reporting path.",
+        policy_evidence("breach_notification", "No personal data breach notification evidence found"),
+        9,
+    ))
+
     child_status = "pass" if dpdp_pass("children_protection") else "review"
     checks.append(make_check(
         "children_data",
@@ -3530,10 +3772,12 @@ def _dpdp_quick_from_report(report: dict, service_map: Optional[dict] = None) ->
         "manual_review": review[:4],
         "detected_processors": detected_processors,
         "evidence_register": evidence_register,
+        "policy_control_matrix": policy_analysis,
         "plain_english": [
             f"We found {pii.get('pii_input_count', 0)} personal-data input signal(s).",
             f"We found {tracker_count} tracker signal(s) and {processor_count} processor/provider signal(s).",
             f"Public PII exposure count: {exposed.get('count', 0)}.",
+            f"Policy evidence score: {policy_analysis.get('score', 'not available')}%.",
             f"DPDP text-evidence checks passed: {dpdp.get('score', 'not available')} of {dpdp.get('total_checks', 'unknown')}.",
         ],
         "priority_actions": [
@@ -4345,8 +4589,10 @@ async def scan_url(req: URLScanRequest):
     except Exception as e:
         print(f"[!] Privacy policy page fetch failed: {e}")
 
-    # Combine homepage + privacy page for DPDP analysis
-    combined_compliance_text = page_lower + " " + privacy_page_text
+    # Combine visible homepage text + privacy page text for DPDP analysis.
+    # Use clean text rather than raw HTML so evidence snippets are audit-readable.
+    combined_compliance_text = visible_text.lower() + " " + privacy_page_text
+    dpdp_policy_analysis = _analyze_dpdp_policy_text(combined_compliance_text, privacy_url_found)
 
     # ---- 15. DPDP ACT 2023 (India) COMPLIANCE CHECKS ----
     # Based on the Digital Personal Data Protection Act, 2023
@@ -4644,6 +4890,7 @@ async def scan_url(req: URLScanRequest):
             "https": is_https,
             "privacy_policy": has_privacy_policy,
             "privacy_policy_url": privacy_url_found,
+            "policy_analysis": dpdp_policy_analysis,
             "cookie_consent": has_cookie_consent,
             "cmp_platforms": cmp_detected,
             "terms_of_service": has_terms,