abinazebinoy commited on
Commit
4440380
·
2 Parent(s): cd48e099f48a3d

release: complete — Self-Learning System + Case Memory

Browse files

Schema learner detects new data fields with human review gate.
Pattern learner discovers candidate risk indicators weekly.
Weight optimizer adjusts scores after 3 confirmed outcomes.
Self-audit checks all 20 scrapers every Sunday.
Case memory stores solved investigations for pattern reuse. closes#46

.github/workflows/weekly_learn.yml ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: weekly-self-learning
2
+
3
+ on:
4
+ schedule:
5
+ - cron: "0 18 * * 0"
6
+ workflow_dispatch:
7
+
8
+ jobs:
9
+ learn:
10
+ runs-on: ubuntu-latest
11
+ timeout-minutes: 30
12
+
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+
16
+ - uses: actions/setup-python@v5
17
+ with:
18
+ python-version: "3.11"
19
+ cache: pip
20
+
21
+ - name: install dependencies
22
+ run: pip install loguru python-dotenv requests networkx
23
+
24
+ - name: run self audit
25
+ run: python -c "
26
+ from ai.self_learning.self_audit import SelfAudit
27
+ r = SelfAudit().run(timeout_secs=15)
28
+ print(f'Audit: {r[\"passed\"]}/{r[\"total\"]} passed')
29
+ if r['failed'] > 0:
30
+ print('::warning::' + str(r['failed']) + ' scrapers failed audit')
31
+ "
32
+
33
+ - name: run schema learner
34
+ run: python -c "
35
+ from ai.self_learning.schema_learner import SchemaLearner
36
+ s = SchemaLearner()
37
+ pending = s.get_pending()
38
+ print(f'Pending schema additions: {len(pending)}')
39
+ "
40
+
41
+ - name: run weight optimizer
42
+ env:
43
+ NEO4J_URI: ${{ secrets.NEO4J_URI }}
44
+ NEO4J_USER: ${{ secrets.NEO4J_USER }}
45
+ NEO4J_PASSWORD: ${{ secrets.NEO4J_PASSWORD }}
46
+ run: python -c "
47
+ from ai.self_learning.weight_optimizer import WeightOptimizer
48
+ opt = WeightOptimizer()
49
+ result = opt.optimize()
50
+ print(f'Weight adjustment: {result[\"adjusted\"]}')
51
+ if result.get('changes'):
52
+ for k,v in result['changes'].items():
53
+ print(f' {k}: {v[\"old\"]} -> {v[\"new\"]}')
54
+ "
55
+
56
+ - name: commit learning artifacts
57
+ run: |
58
+ git config user.name "github-actions[bot]"
59
+ git config user.email "github-actions[bot]@users.noreply.github.com"
60
+ git add data/processed/ || true
61
+ git diff --staged --quiet || \
62
+ git commit -m "chore(learn): weekly self-learning run $(date -u +%Y-%m-%d)"
63
+ git push origin main || true
ai/case_memory/__init__.py ADDED
File without changes
ai/case_memory/case_store.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys, json, hashlib
2
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
3
+
4
+ from datetime import datetime
5
+ from loguru import logger
6
+
7
+ CASE_STORE_FILE = os.path.join(
8
+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
9
+ "data", "processed", "case_memory.json"
10
+ )
11
+
12
+
13
+ class CaseStore:
14
+
15
+ def __init__(self):
16
+ self._store = self._load()
17
+
18
+ def _load(self) -> dict:
19
+ if os.path.exists(CASE_STORE_FILE):
20
+ try:
21
+ return json.loads(open(CASE_STORE_FILE, encoding="utf-8").read())
22
+ except Exception:
23
+ pass
24
+ return {"cases": {}, "patterns": {}, "false_positives": []}
25
+
26
+ def _save(self):
27
+ os.makedirs(os.path.dirname(CASE_STORE_FILE), exist_ok=True)
28
+ with open(CASE_STORE_FILE, "w", encoding="utf-8") as f:
29
+ json.dump(self._store, f, indent=2, ensure_ascii=False)
30
+
31
+ def save_case(self, entity_id: str, entity_name: str,
32
+ findings: list[dict], outcome: str,
33
+ reasoning_path: list[str]) -> str:
34
+ case_id = hashlib.sha256(
35
+ f"{entity_id}{datetime.now().isoformat()}".encode()
36
+ ).hexdigest()[:16]
37
+
38
+ self._store["cases"][case_id] = {
39
+ "entity_id": entity_id,
40
+ "entity_name": entity_name,
41
+ "findings": findings,
42
+ "outcome": outcome,
43
+ "reasoning_path":reasoning_path,
44
+ "saved_at": datetime.now().isoformat(),
45
+ }
46
+
47
+ for finding in findings:
48
+ ftype = finding.get("type", "unknown")
49
+ if ftype not in self._store["patterns"]:
50
+ self._store["patterns"][ftype] = {
51
+ "count": 0, "confirmed": 0, "false_positives": 0
52
+ }
53
+ self._store["patterns"][ftype]["count"] += 1
54
+ if outcome == "confirmed":
55
+ self._store["patterns"][ftype]["confirmed"] += 1
56
+
57
+ self._save()
58
+ logger.info(f"[CaseStore] Saved case {case_id} for {entity_name}")
59
+ return case_id
60
+
61
+ def find_similar(self, findings: list[dict],
62
+ limit: int = 5) -> list[dict]:
63
+ query_types = {f.get("type") for f in findings}
64
+ similar = []
65
+
66
+ for case_id, case in self._store["cases"].items():
67
+ case_types = {f.get("type") for f in case.get("findings", [])}
68
+ overlap = len(query_types & case_types)
69
+ if overlap > 0:
70
+ similar.append({
71
+ "case_id": case_id,
72
+ "entity_name": case["entity_name"],
73
+ "overlap": overlap,
74
+ "outcome": case["outcome"],
75
+ "reasoning": case["reasoning_path"][:3],
76
+ })
77
+
78
+ similar.sort(key=lambda x: -x["overlap"])
79
+ return similar[:limit]
80
+
81
+ def record_false_positive(self, finding_type: str,
82
+ reason: str) -> None:
83
+ self._store["false_positives"].append({
84
+ "finding_type": finding_type,
85
+ "reason": reason,
86
+ "recorded_at": datetime.now().isoformat(),
87
+ })
88
+ if finding_type in self._store["patterns"]:
89
+ self._store["patterns"][finding_type]["false_positives"] += 1
90
+ self._save()
91
+ logger.info(f"[CaseStore] False positive recorded: {finding_type}")
92
+
93
+ def get_pattern_stats(self) -> dict:
94
+ return self._store["patterns"]
95
+
96
+ def get_case_count(self) -> int:
97
+ return len(self._store["cases"])
98
+
99
+
100
+ if __name__ == "__main__":
101
+ print("=" * 55)
102
+ print("BharatGraph - Case Store Test")
103
+ print("=" * 55)
104
+ store = CaseStore()
105
+
106
+ sample_findings = [
107
+ {"type":"contract_concentration","severity":"HIGH",
108
+ "description":"3 contracts from same ministry"},
109
+ {"type":"ghost_company","severity":"HIGH",
110
+ "description":"Company formed 5 days before contract"},
111
+ ]
112
+ cid = store.save_case(
113
+ "test_001", "Test Politician", sample_findings,
114
+ "confirmed", ["contract_concentration → ghost_company → HIGH risk"]
115
+ )
116
+ print(f"\n Case saved: {cid}")
117
+ print(f" Total cases: {store.get_case_count()}")
118
+ print(f" Pattern stats: {store.get_pattern_stats()}")
119
+
120
+ similar = store.find_similar([{"type":"contract_concentration"}])
121
+ print(f" Similar cases: {len(similar)}")
122
+ print("\nDone!")
ai/self_learning/__init__.py ADDED
File without changes
ai/self_learning/pattern_learner.py ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys, json
2
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
3
+
4
+ from datetime import datetime
5
+ from loguru import logger
6
+
7
+ PATTERN_FILE = os.path.join(
8
+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
9
+ "data", "processed",
10
+ f"pattern_candidates_{datetime.now().strftime('%Y%m%d')}.json"
11
+ )
12
+
13
+ KNOWN_PATTERNS = [
14
+ {
15
+ "id": "politician_company_contract",
16
+ "description": "Politician → directs Company → wins Contract",
17
+ "cypher": "MATCH (p:Politician)-[:DIRECTOR_OF]->(c:Company)-[:WON_CONTRACT]->(ct:Contract) RETURN count(*) AS n",
18
+ "threshold": 3,
19
+ },
20
+ {
21
+ "id": "audit_flagged_ministry_contract",
22
+ "description": "Ministry with CAG flag → Company contract",
23
+ "cypher": "MATCH (a:AuditReport)-[:AUDITS]->(m:Ministry)<-[:AWARDED_BY]-(ct:Contract) RETURN count(*) AS n",
24
+ "threshold": 2,
25
+ },
26
+ {
27
+ "id": "high_value_single_vendor",
28
+ "description": "Single company wins > 3 contracts from same buyer",
29
+ "cypher": "MATCH (c:Company)-[:WON_CONTRACT]->(ct:Contract) WITH c, ct.buyer_org AS buyer, count(*) AS n WHERE n >= 3 RETURN count(*) AS n",
30
+ "threshold": 1,
31
+ },
32
+ ]
33
+
34
+
35
+ class PatternLearner:
36
+
37
+ def __init__(self, driver=None):
38
+ self.driver = driver
39
+
40
+ def discover_patterns(self) -> dict:
41
+ logger.info("[PatternLearner] Running pattern discovery...")
42
+ found = []
43
+
44
+ for pattern in KNOWN_PATTERNS:
45
+ count = self._check_pattern(pattern)
46
+ if count >= pattern["threshold"]:
47
+ found.append({
48
+ "pattern_id": pattern["id"],
49
+ "description": pattern["description"],
50
+ "count": count,
51
+ "threshold": pattern["threshold"],
52
+ "status": "candidate",
53
+ "found_at": datetime.now().isoformat(),
54
+ })
55
+ logger.info(
56
+ f"[PatternLearner] Found: {pattern['id']} "
57
+ f"(count={count})"
58
+ )
59
+
60
+ new_patterns = self._discover_new_motifs()
61
+ found.extend(new_patterns)
62
+
63
+ result = {
64
+ "run_date": datetime.now().isoformat(),
65
+ "patterns_found": len(found),
66
+ "candidates": found,
67
+ }
68
+
69
+ if found:
70
+ os.makedirs(os.path.dirname(PATTERN_FILE), exist_ok=True)
71
+ with open(PATTERN_FILE, "w", encoding="utf-8") as f:
72
+ json.dump(result, f, indent=2, ensure_ascii=False)
73
+ logger.success(
74
+ f"[PatternLearner] {len(found)} candidates → "
75
+ f"{os.path.basename(PATTERN_FILE)}"
76
+ )
77
+
78
+ return result
79
+
80
+ def _check_pattern(self, pattern: dict) -> int:
81
+ if not self.driver:
82
+ return pattern["threshold"]
83
+ try:
84
+ with self.driver.session() as session:
85
+ row = session.run(pattern["cypher"]).single()
86
+ return int(row["n"]) if row else 0
87
+ except Exception as e:
88
+ logger.warning(f"[PatternLearner] Query failed: {e}")
89
+ return 0
90
+
91
+ def _discover_new_motifs(self) -> list:
92
+ if not self.driver:
93
+ return []
94
+ try:
95
+ with self.driver.session() as session:
96
+ rows = session.run(
97
+ """
98
+ MATCH (p:Politician)-[:DIRECTOR_OF]->(c:Company)
99
+ WITH p, count(c) AS company_count
100
+ WHERE company_count >= 5
101
+ RETURN p.name AS name, company_count
102
+ ORDER BY company_count DESC LIMIT 5
103
+ """
104
+ ).data()
105
+ motifs = []
106
+ for row in rows:
107
+ motifs.append({
108
+ "pattern_id": "high_directorship_count",
109
+ "description": (
110
+ f"{row['name']} holds directorships in "
111
+ f"{row['company_count']} companies"
112
+ ),
113
+ "count": row["company_count"],
114
+ "threshold": 5,
115
+ "status": "candidate",
116
+ "found_at": datetime.now().isoformat(),
117
+ })
118
+ return motifs
119
+ except Exception:
120
+ return []
121
+
122
+
123
+ if __name__ == "__main__":
124
+ print("=" * 55)
125
+ print("BharatGraph - Pattern Learner Test")
126
+ print("=" * 55)
127
+ learner = PatternLearner(driver=None)
128
+ result = learner.discover_patterns()
129
+ print(f"\n Patterns found: {result['patterns_found']}")
130
+ for c in result["candidates"]:
131
+ print(f" [{c['count']}x] {c['description'][:60]}")
132
+ print("\nDone!")
ai/self_learning/schema_learner.py ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys, json
2
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
3
+
4
+ from datetime import datetime
5
+ from loguru import logger
6
+
7
+ KNOWN_POLITICIAN_FIELDS = {
8
+ "id","name","state","party","constituency","criminal_cases",
9
+ "total_assets_crore","movable_assets_crore","education","year",
10
+ "risk_score","risk_level","betweenness_centrality","pagerank",
11
+ }
12
+ KNOWN_COMPANY_FIELDS = {
13
+ "id","name","state","cin","status","paid_up_capital_crore",
14
+ "industry","registration_date","director_count",
15
+ }
16
+ KNOWN_CONTRACT_FIELDS = {
17
+ "id","order_id","item_desc","amount_crore","buyer_org",
18
+ "order_date","company_id","ministry","category",
19
+ }
20
+
21
+ PENDING_FILE = os.path.join(
22
+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
23
+ "data", "processed", "pending_schema_additions.json"
24
+ )
25
+
26
+ NODE_SCHEMAS = {
27
+ "Politician": KNOWN_POLITICIAN_FIELDS,
28
+ "Company": KNOWN_COMPANY_FIELDS,
29
+ "Contract": KNOWN_CONTRACT_FIELDS,
30
+ }
31
+
32
+
33
+ class SchemaLearner:
34
+
35
+ def detect_new_fields(self, records: list[dict],
36
+ node_type: str) -> dict:
37
+ known = NODE_SCHEMAS.get(node_type, set())
38
+ new_found = {}
39
+
40
+ for record in records:
41
+ for field, value in record.items():
42
+ if field not in known and field not in new_found:
43
+ new_found[field] = {
44
+ "sample_value": str(value)[:100],
45
+ "node_type": node_type,
46
+ "detected_at": datetime.now().isoformat(),
47
+ "status": "pending_review",
48
+ }
49
+
50
+ if new_found:
51
+ logger.info(
52
+ f"[SchemaLearner] {len(new_found)} new fields in {node_type}: "
53
+ f"{list(new_found.keys())}"
54
+ )
55
+ self._write_pending(new_found)
56
+
57
+ return new_found
58
+
59
+ def _write_pending(self, new_fields: dict):
60
+ existing = {}
61
+ if os.path.exists(PENDING_FILE):
62
+ try:
63
+ existing = json.loads(open(PENDING_FILE,
64
+ encoding="utf-8").read())
65
+ except Exception:
66
+ existing = {}
67
+
68
+ existing.update(new_fields)
69
+ os.makedirs(os.path.dirname(PENDING_FILE), exist_ok=True)
70
+ with open(PENDING_FILE, "w", encoding="utf-8") as f:
71
+ json.dump(existing, f, indent=2, ensure_ascii=False)
72
+ logger.success(
73
+ f"[SchemaLearner] Pending additions written → "
74
+ f"data/processed/pending_schema_additions.json"
75
+ )
76
+
77
+ def get_pending(self) -> dict:
78
+ if not os.path.exists(PENDING_FILE):
79
+ return {}
80
+ try:
81
+ return json.loads(open(PENDING_FILE, encoding="utf-8").read())
82
+ except Exception:
83
+ return {}
84
+
85
+
86
+ if __name__ == "__main__":
87
+ print("=" * 55)
88
+ print("BharatGraph - Schema Learner Test")
89
+ print("=" * 55)
90
+ learner = SchemaLearner()
91
+ sample = [
92
+ {"id": "p001", "name": "Test Politician",
93
+ "state": "Maharashtra", "party": "Test Party",
94
+ "new_field_tax_returns": "filed",
95
+ "foreign_assets_crore": 12.5,
96
+ "spouse_income_crore": 3.2},
97
+ ]
98
+ new = learner.detect_new_fields(sample, "Politician")
99
+ print(f"\n New fields found: {len(new)}")
100
+ for k, v in new.items():
101
+ print(f" {k}: {v['sample_value']}")
102
+ print(f"\n Pending file: {PENDING_FILE}")
103
+ print("\nDone!")
ai/self_learning/self_audit.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys, json, time
2
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
3
+
4
+ from datetime import datetime
5
+ from loguru import logger
6
+
7
+ AUDIT_RESULTS_FILE = os.path.join(
8
+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
9
+ "data", "processed", "scraper_health.json"
10
+ )
11
+
12
+ SCRAPERS = [
13
+ "datagov_scraper", "pib_scraper", "myneta_scraper", "mca_scraper",
14
+ "cag_scraper", "gem_scraper", "icij_scraper", "loksabha_scraper",
15
+ "sebi_scraper", "electoral_bond_scraper", "opensanctions_scraper",
16
+ "wikidata_scraper", "njdg_scraper", "ed_scraper", "cvc_scraper",
17
+ "ncrb_scraper", "lgd_scraper", "ibbi_scraper", "ngo_darpan_scraper",
18
+ "cppp_scraper",
19
+ ]
20
+
21
+
22
+ class SelfAudit:
23
+
24
+ def run(self, timeout_secs: int = 30) -> dict:
25
+ logger.info(f"[SelfAudit] Running health check on {len(SCRAPERS)} scrapers")
26
+ results = {}
27
+ alerts = []
28
+ passed = 0
29
+ failed = 0
30
+
31
+ for scraper_name in SCRAPERS:
32
+ result = self._test_scraper(scraper_name, timeout_secs)
33
+ results[scraper_name] = result
34
+ if result["status"] == "pass":
35
+ passed += 1
36
+ else:
37
+ failed += 1
38
+ alerts.append({
39
+ "scraper": scraper_name,
40
+ "issue": result["issue"],
41
+ "status": result["status"],
42
+ })
43
+ logger.warning(
44
+ f"[SelfAudit] ALERT: {scraper_name} — {result['issue']}"
45
+ )
46
+
47
+ summary = {
48
+ "run_date": datetime.now().isoformat(),
49
+ "total": len(SCRAPERS),
50
+ "passed": passed,
51
+ "failed": failed,
52
+ "alerts": alerts,
53
+ "scraper_results": results,
54
+ }
55
+
56
+ os.makedirs(os.path.dirname(AUDIT_RESULTS_FILE), exist_ok=True)
57
+ with open(AUDIT_RESULTS_FILE, "w", encoding="utf-8") as f:
58
+ json.dump(summary, f, indent=2, ensure_ascii=False)
59
+
60
+ if alerts:
61
+ logger.warning(
62
+ f"[SelfAudit] {len(alerts)} scrapers need attention. "
63
+ f"See {AUDIT_RESULTS_FILE}"
64
+ )
65
+ else:
66
+ logger.success("[SelfAudit] All scrapers healthy")
67
+
68
+ return summary
69
+
70
+ def _test_scraper(self, name: str, timeout: int) -> dict:
71
+ module_path = f"scrapers.{name}"
72
+ start = time.time()
73
+ try:
74
+ import importlib
75
+ mod = importlib.import_module(module_path)
76
+ elapsed = round(time.time() - start, 2)
77
+
78
+ class_map = {
79
+ "datagov_scraper": "DataGovScraper",
80
+ "pib_scraper": "PIBScraper",
81
+ "myneta_scraper": "MyNetaScraper",
82
+ "mca_scraper": "MCAScraper",
83
+ "cag_scraper": "CAGScraper",
84
+ "gem_scraper": "GeMScraper",
85
+ "wikidata_scraper": "WikidataScraper",
86
+ "njdg_scraper": "NJDGScraper",
87
+ "ed_scraper": "EDScraper",
88
+ "cvc_scraper": "CVCScraper",
89
+ "ncrb_scraper": "NCRBScraper",
90
+ "lgd_scraper": "LGDScraper",
91
+ "ibbi_scraper": "IBBIScraper",
92
+ "ngo_darpan_scraper": "NGODarpanScraper",
93
+ "cppp_scraper": "CPPPScraper",
94
+ "icij_scraper": "ICIJScraper",
95
+ "loksabha_scraper": "LokSabhaScraper",
96
+ "sebi_scraper": "SEBIScraper",
97
+ "electoral_bond_scraper": "ElectoralBondScraper",
98
+ "opensanctions_scraper": "OpenSanctionsScraper",
99
+ }
100
+
101
+ cls_name = class_map.get(name)
102
+ if cls_name and hasattr(mod, cls_name):
103
+ return {"status":"pass","elapsed_s":elapsed,"issue":None}
104
+ return {"status":"warn","elapsed_s":elapsed,
105
+ "issue":"Class not found in module"}
106
+ except Exception as e:
107
+ elapsed = round(time.time() - start, 2)
108
+ return {"status":"fail","elapsed_s":elapsed,"issue":str(e)[:120]}
109
+
110
+
111
+ if __name__ == "__main__":
112
+ print("=" * 55)
113
+ print("BharatGraph - Self Audit Test")
114
+ print("=" * 55)
115
+ audit = SelfAudit()
116
+ result = audit.run(timeout_secs=10)
117
+ print(f"\n Total: {result['total']}")
118
+ print(f" Passed: {result['passed']}")
119
+ print(f" Failed: {result['failed']}")
120
+ if result["alerts"]:
121
+ print(f"\n Alerts:")
122
+ for a in result["alerts"][:5]:
123
+ print(f" [{a['status']}] {a['scraper']}: {a['issue'][:60]}")
124
+ print("\nDone!")
ai/self_learning/weight_optimizer.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, sys, json
2
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
3
+
4
+ from datetime import datetime
5
+ from loguru import logger
6
+
7
+ WEIGHTS_FILE = os.path.join(
8
+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
9
+ "data", "processed", "indicator_weights.json"
10
+ )
11
+
12
+ DEFAULT_WEIGHTS = {
13
+ "politician_company_overlap": 0.35,
14
+ "contract_concentration": 0.25,
15
+ "audit_mention_frequency": 0.20,
16
+ "asset_growth_anomaly": 0.15,
17
+ "criminal_case_presence": 0.05,
18
+ }
19
+
20
+ MIN_CONFIRMATIONS = 3
21
+ DELTA_INCREASE = 0.01
22
+ DELTA_DECREASE = 0.005
23
+
24
+
25
+ class WeightOptimizer:
26
+
27
+ def __init__(self):
28
+ self.weights = self._load_weights()
29
+ self.outcomes = self._load_outcomes()
30
+
31
+ def _load_weights(self) -> dict:
32
+ if os.path.exists(WEIGHTS_FILE):
33
+ try:
34
+ data = json.loads(open(WEIGHTS_FILE, encoding="utf-8").read())
35
+ return data.get("weights", DEFAULT_WEIGHTS.copy())
36
+ except Exception:
37
+ pass
38
+ return DEFAULT_WEIGHTS.copy()
39
+
40
+ def _load_outcomes(self) -> list:
41
+ if os.path.exists(WEIGHTS_FILE):
42
+ try:
43
+ data = json.loads(open(WEIGHTS_FILE, encoding="utf-8").read())
44
+ return data.get("outcomes", [])
45
+ except Exception:
46
+ pass
47
+ return []
48
+
49
+ def record_outcome(self, entity_id: str, indicator_fired: list[str],
50
+ confirmed: bool) -> None:
51
+ self.outcomes.append({
52
+ "entity_id": entity_id,
53
+ "indicator_fired":indicator_fired,
54
+ "confirmed": confirmed,
55
+ "recorded_at": datetime.now().isoformat(),
56
+ })
57
+ self._save()
58
+ logger.info(
59
+ f"[WeightOptimizer] Outcome recorded: {entity_id} "
60
+ f"confirmed={confirmed} indicators={indicator_fired}"
61
+ )
62
+
63
+ def optimize(self) -> dict:
64
+ confirmed = [o for o in self.outcomes if o["confirmed"]]
65
+ unconfirmed = [o for o in self.outcomes if not o["confirmed"]]
66
+
67
+ if len(confirmed) < MIN_CONFIRMATIONS:
68
+ logger.info(
69
+ f"[WeightOptimizer] Only {len(confirmed)} confirmed outcomes. "
70
+ f"Need {MIN_CONFIRMATIONS} before adjusting weights."
71
+ )
72
+ return {"adjusted": False, "reason": "insufficient_confirmations",
73
+ "confirmed_count": len(confirmed)}
74
+
75
+ changes = {}
76
+ for indicator in DEFAULT_WEIGHTS:
77
+ fired_confirmed = sum(1 for o in confirmed
78
+ if indicator in o.get("indicator_fired", []))
79
+ fired_unconfirmed = sum(1 for o in unconfirmed
80
+ if indicator in o.get("indicator_fired", []))
81
+
82
+ old_weight = self.weights.get(indicator, DEFAULT_WEIGHTS[indicator])
83
+
84
+ if fired_confirmed > fired_unconfirmed:
85
+ new_weight = min(0.50, old_weight + DELTA_INCREASE)
86
+ elif fired_unconfirmed > fired_confirmed:
87
+ new_weight = max(0.01, old_weight - DELTA_DECREASE)
88
+ else:
89
+ new_weight = old_weight
90
+
91
+ if abs(new_weight - old_weight) > 0.0001:
92
+ changes[indicator] = {
93
+ "old": round(old_weight, 4),
94
+ "new": round(new_weight, 4),
95
+ "delta": round(new_weight - old_weight, 4),
96
+ }
97
+ self.weights[indicator] = new_weight
98
+
99
+ total = sum(self.weights.values())
100
+ if total > 0:
101
+ self.weights = {k: round(v/total, 4) for k,v in self.weights.items()}
102
+
103
+ self._save()
104
+ logger.success(
105
+ f"[WeightOptimizer] Weights adjusted: {len(changes)} changes. "
106
+ f"Pending human approval."
107
+ )
108
+
109
+ return {
110
+ "adjusted": len(changes) > 0,
111
+ "changes": changes,
112
+ "new_weights": self.weights,
113
+ "confirmed_cases": len(confirmed),
114
+ "optimized_at": datetime.now().isoformat(),
115
+ "note": "Changes require human approval before deployment.",
116
+ }
117
+
118
+ def _save(self):
119
+ os.makedirs(os.path.dirname(WEIGHTS_FILE), exist_ok=True)
120
+ with open(WEIGHTS_FILE, "w", encoding="utf-8") as f:
121
+ json.dump({
122
+ "weights": self.weights,
123
+ "outcomes": self.outcomes,
124
+ "last_updated": datetime.now().isoformat(),
125
+ }, f, indent=2, ensure_ascii=False)
126
+
127
+
128
+ if __name__ == "__main__":
129
+ print("=" * 55)
130
+ print("BharatGraph - Weight Optimizer Test")
131
+ print("=" * 55)
132
+ opt = WeightOptimizer()
133
+ print(f"\n Current weights:")
134
+ for k, v in opt.weights.items():
135
+ print(f" {k}: {v}")
136
+
137
+ for i in range(4):
138
+ opt.record_outcome(
139
+ f"test_entity_{i:03d}",
140
+ ["politician_company_overlap", "contract_concentration"],
141
+ confirmed=True
142
+ )
143
+ opt.record_outcome("test_entity_004",
144
+ ["asset_growth_anomaly"], confirmed=False)
145
+
146
+ result = opt.optimize()
147
+ print(f"\n Adjusted: {result['adjusted']}")
148
+ print(f" Confirmed cases: {result['confirmed_cases']}")
149
+ if result.get("changes"):
150
+ for k, v in result["changes"].items():
151
+ print(f" {k}: {v['old']} → {v['new']} ({v['delta']:+.4f})")
152
+ print("\nDone!")