Spaces:
Running
Running
Commit Β·
fa87992
1
Parent(s): 9e00b4a
fix(ascii): replace all non-ASCII chars in Python source files
Browse filesCI check requires all .py files (except transliteration/multilingual_ner/
translator/languages) to be pure ASCII. Pre-existing violations fixed:
U+03BB lambda -> lambda_val
U+2014 em-dash -> --
U+2013 en-dash -> --
U+20B9 rupee Rs -> Rs.
U+2192 arrow -> ->
U+2500 box-h -> -
U+2502 box-v -> |
U+2081 subscript -> 1
- ai/forensics/bid_dna.py +3 -3
- ai/forensics/linguistic_fingerprint.py +4 -4
- ai/investigators/math_investigator.py +1 -1
- ai/math/spectral_analyzer.py +3 -3
- check_syntax.py +22 -0
- fix_conn_map_xss.py +34 -0
- fix_lang_search.py +38 -0
- fix_stats_lock.py +92 -0
- graph/queries.py +11 -11
- graph/schema.py +6 -6
- graph/seed.py +7 -7
- processing/cleaner.py +3 -3
- processing/entity_resolver.py +2 -2
- scrapers/cag_scraper.py +3 -3
- scrapers/datagov_scraper.py +4 -4
- scrapers/gem_scraper.py +1 -1
- scrapers/mca_scraper.py +2 -2
ai/forensics/bid_dna.py
CHANGED
|
@@ -42,7 +42,7 @@ class BidDNA:
|
|
| 42 |
f"coordinated submission."
|
| 43 |
),
|
| 44 |
"evidence": [
|
| 45 |
-
f"Bid '{p['a'][:40]}'
|
| 46 |
f"{p['similarity']:.1%} similar"
|
| 47 |
for p in suspicious[:3]
|
| 48 |
],
|
|
@@ -154,7 +154,7 @@ class BidDNA:
|
|
| 154 |
"evidence": [
|
| 155 |
f"Mean: Rs {mean:.2f} Cr",
|
| 156 |
f"Std: Rs {std:.2f} Cr",
|
| 157 |
-
f"Range: Rs {min(amounts):.2f}
|
| 158 |
],
|
| 159 |
}
|
| 160 |
return None
|
|
@@ -162,7 +162,7 @@ class BidDNA:
|
|
| 162 |
|
| 163 |
if __name__ == "__main__":
|
| 164 |
print("=" * 55)
|
| 165 |
-
print("BharatGraph
|
| 166 |
print("=" * 55)
|
| 167 |
b = BidDNA()
|
| 168 |
r = b.analyze("pol_001", driver=None)
|
|
|
|
| 42 |
f"coordinated submission."
|
| 43 |
),
|
| 44 |
"evidence": [
|
| 45 |
+
f"Bid '{p['a'][:40]}' <-> '{p['b'][:40]}': "
|
| 46 |
f"{p['similarity']:.1%} similar"
|
| 47 |
for p in suspicious[:3]
|
| 48 |
],
|
|
|
|
| 154 |
"evidence": [
|
| 155 |
f"Mean: Rs {mean:.2f} Cr",
|
| 156 |
f"Std: Rs {std:.2f} Cr",
|
| 157 |
+
f"Range: Rs {min(amounts):.2f} -- Rs {max(amounts):.2f} Cr",
|
| 158 |
],
|
| 159 |
}
|
| 160 |
return None
|
|
|
|
| 162 |
|
| 163 |
if __name__ == "__main__":
|
| 164 |
print("=" * 55)
|
| 165 |
+
print("BharatGraph -- Bid DNA Test")
|
| 166 |
print("=" * 55)
|
| 167 |
b = BidDNA()
|
| 168 |
r = b.analyze("pol_001", driver=None)
|
ai/forensics/linguistic_fingerprint.py
CHANGED
|
@@ -118,7 +118,7 @@ class LinguisticFingerprinter:
|
|
| 118 |
"analyzed_at": datetime.now().isoformat(),
|
| 119 |
}
|
| 120 |
|
| 121 |
-
#
|
| 122 |
|
| 123 |
def _burrows_delta(self, documents: list[dict]) -> dict:
|
| 124 |
if len(documents) < MIN_DOCS_FOR_DELTA:
|
|
@@ -182,7 +182,7 @@ class LinguisticFingerprinter:
|
|
| 182 |
total = max(len(tokens), 1)
|
| 183 |
return {w: tokens.count(w) / total for w in self.FUNCTION_WORDS}
|
| 184 |
|
| 185 |
-
#
|
| 186 |
|
| 187 |
def _template_reuse(self, documents: list[dict]) -> dict:
|
| 188 |
fingerprints = []
|
|
@@ -229,7 +229,7 @@ class LinguisticFingerprinter:
|
|
| 229 |
union = len(fp_a | fp_b)
|
| 230 |
return intersection / union if union > 0 else 0.0
|
| 231 |
|
| 232 |
-
#
|
| 233 |
|
| 234 |
def _shadow_drafting(self, documents: list[dict]) -> dict:
|
| 235 |
submissions = [d for d in documents if d.get("type") == "submission"]
|
|
@@ -329,7 +329,7 @@ class LinguisticFingerprinter:
|
|
| 329 |
|
| 330 |
if __name__ == "__main__":
|
| 331 |
print("=" * 55)
|
| 332 |
-
print("BharatGraph
|
| 333 |
print("=" * 55)
|
| 334 |
lf = LinguisticFingerprinter()
|
| 335 |
r = lf.analyze("pol_001", [], driver=None)
|
|
|
|
| 118 |
"analyzed_at": datetime.now().isoformat(),
|
| 119 |
}
|
| 120 |
|
| 121 |
+
# -- Burrows Delta authorship attribution ----------------------------------
|
| 122 |
|
| 123 |
def _burrows_delta(self, documents: list[dict]) -> dict:
|
| 124 |
if len(documents) < MIN_DOCS_FOR_DELTA:
|
|
|
|
| 182 |
total = max(len(tokens), 1)
|
| 183 |
return {w: tokens.count(w) / total for w in self.FUNCTION_WORDS}
|
| 184 |
|
| 185 |
+
# -- Template reuse via structural fingerprinting --------------------------
|
| 186 |
|
| 187 |
def _template_reuse(self, documents: list[dict]) -> dict:
|
| 188 |
fingerprints = []
|
|
|
|
| 229 |
union = len(fp_a | fp_b)
|
| 230 |
return intersection / union if union > 0 else 0.0
|
| 231 |
|
| 232 |
+
# -- Shadow drafting detection ---------------------------------------------
|
| 233 |
|
| 234 |
def _shadow_drafting(self, documents: list[dict]) -> dict:
|
| 235 |
submissions = [d for d in documents if d.get("type") == "submission"]
|
|
|
|
| 329 |
|
| 330 |
if __name__ == "__main__":
|
| 331 |
print("=" * 55)
|
| 332 |
+
print("BharatGraph -- Linguistic Fingerprinter Test")
|
| 333 |
print("=" * 55)
|
| 334 |
lf = LinguisticFingerprinter()
|
| 335 |
r = lf.analyze("pol_001", [], driver=None)
|
ai/investigators/math_investigator.py
CHANGED
|
@@ -26,7 +26,7 @@ def investigate(entity_id: str, entity_name: str,
|
|
| 26 |
if sr.get("fiedler_value", 1.0) > 0.5:
|
| 27 |
positive.append(
|
| 28 |
f"Spectral analysis: well-connected in institutional network "
|
| 29 |
-
f"(Fiedler
|
| 30 |
)
|
| 31 |
evidence.append({
|
| 32 |
"institution": "Mathematical Analysis",
|
|
|
|
| 26 |
if sr.get("fiedler_value", 1.0) > 0.5:
|
| 27 |
positive.append(
|
| 28 |
f"Spectral analysis: well-connected in institutional network "
|
| 29 |
+
f"(Fiedler lambda_val1 = {sr['fiedler_value']:.4f})"
|
| 30 |
)
|
| 31 |
evidence.append({
|
| 32 |
"institution": "Mathematical Analysis",
|
ai/math/spectral_analyzer.py
CHANGED
|
@@ -60,7 +60,7 @@ class SpectralAnalyzer:
|
|
| 60 |
"this entity acts as a structural bridge between institutional networks. "
|
| 61 |
"Removing this entity would disconnect major clusters."
|
| 62 |
),
|
| 63 |
-
"evidence": [f"Algebraic connectivity
|
| 64 |
f"Graph bridges detected: {len(bridges)}"],
|
| 65 |
})
|
| 66 |
|
|
@@ -69,7 +69,7 @@ class SpectralAnalyzer:
|
|
| 69 |
"type": "high_betweenness",
|
| 70 |
"severity": "MODERATE",
|
| 71 |
"description": (
|
| 72 |
-
f"Betweenness centrality {centrality['betweenness']:.3f}
|
| 73 |
"entity controls many shortest paths between other nodes."
|
| 74 |
),
|
| 75 |
"evidence": [f"Betweenness: {centrality['betweenness']:.3f}",
|
|
@@ -161,7 +161,7 @@ if __name__ == "__main__":
|
|
| 161 |
a = SpectralAnalyzer()
|
| 162 |
r = a.analyze("test_entity_001")
|
| 163 |
print(f"\n Nodes: {r['node_count']}")
|
| 164 |
-
print(f" Fiedler
|
| 165 |
print(f" Connectivity: {r['connectivity']}")
|
| 166 |
print(f" Role: {r['structural_role']}")
|
| 167 |
print(f" Bridges: {r['bridges']}")
|
|
|
|
| 60 |
"this entity acts as a structural bridge between institutional networks. "
|
| 61 |
"Removing this entity would disconnect major clusters."
|
| 62 |
),
|
| 63 |
+
"evidence": [f"Algebraic connectivity lambda_val1 = {fiedler_value:.4f}",
|
| 64 |
f"Graph bridges detected: {len(bridges)}"],
|
| 65 |
})
|
| 66 |
|
|
|
|
| 69 |
"type": "high_betweenness",
|
| 70 |
"severity": "MODERATE",
|
| 71 |
"description": (
|
| 72 |
+
f"Betweenness centrality {centrality['betweenness']:.3f} -- "
|
| 73 |
"entity controls many shortest paths between other nodes."
|
| 74 |
),
|
| 75 |
"evidence": [f"Betweenness: {centrality['betweenness']:.3f}",
|
|
|
|
| 161 |
a = SpectralAnalyzer()
|
| 162 |
r = a.analyze("test_entity_001")
|
| 163 |
print(f"\n Nodes: {r['node_count']}")
|
| 164 |
+
print(f" Fiedler lambda_val1: {r['fiedler_value']}")
|
| 165 |
print(f" Connectivity: {r['connectivity']}")
|
| 166 |
print(f" Role: {r['structural_role']}")
|
| 167 |
print(f" Bridges: {r['bridges']}")
|
check_syntax.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ast, os
|
| 2 |
+
errors = []
|
| 3 |
+
for root, dirs, files in os.walk('.'):
|
| 4 |
+
dirs[:] = [d for d in dirs if d not in ('__pycache__', '.git', 'data', 'logs', 'venv')]
|
| 5 |
+
for f in files:
|
| 6 |
+
if not f.endswith('.py'):
|
| 7 |
+
continue
|
| 8 |
+
path = os.path.join(root, f)
|
| 9 |
+
# Try UTF-8 first, then latin-1 as fallback
|
| 10 |
+
for enc in ('utf-8', 'latin-1', 'cp1252'):
|
| 11 |
+
try:
|
| 12 |
+
src = open(path, encoding=enc).read()
|
| 13 |
+
ast.parse(src)
|
| 14 |
+
break
|
| 15 |
+
except UnicodeDecodeError:
|
| 16 |
+
continue
|
| 17 |
+
except SyntaxError as e:
|
| 18 |
+
errors.append(f'{path}:{e.lineno}: {e.msg}')
|
| 19 |
+
break
|
| 20 |
+
print(f'Syntax: {len(errors)} errors' if errors else 'OK: all Python files clean')
|
| 21 |
+
for e in errors:
|
| 22 |
+
print(' ', e)
|
fix_conn_map_xss.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# fix_conn_map_xss.py
|
| 2 |
+
with open('frontend/js/app.js', 'r', encoding='utf-8') as f:
|
| 3 |
+
content = f.read()
|
| 4 |
+
|
| 5 |
+
# Find the exact pattern using a simpler anchor
|
| 6 |
+
old = "onclick=\"EvidencePanel.open('${sanitize(e.connected_id||'')}','${sanitize(e.connected_to||'')}')\">"
|
| 7 |
+
new = "data-eid=\"${sanitize(e.connected_id||'')}\"\n data-ename=\"${sanitize(e.connected_to||'')}\"\n onclick=\"EvidencePanel.open(this.getAttribute('data-eid'),this.getAttribute('data-ename'))\">"
|
| 8 |
+
|
| 9 |
+
if old in content:
|
| 10 |
+
content = content.replace(old, new)
|
| 11 |
+
print('OK: connection map evidence onclick fixed with data-* attrs')
|
| 12 |
+
else:
|
| 13 |
+
print('WARNING: trying alternate search...')
|
| 14 |
+
# Try finding just a portion
|
| 15 |
+
if "EvidencePanel.open('${sanitize(e.connected_id" in content:
|
| 16 |
+
print('Found partial -- the exact quoting is different in your file')
|
| 17 |
+
print('Showing context around EvidencePanel.open in connection map:')
|
| 18 |
+
idx = content.find("EvidencePanel.open('${sanitize(e.connected_id")
|
| 19 |
+
print(repr(content[idx-50:idx+150]))
|
| 20 |
+
else:
|
| 21 |
+
print('Pattern not found at all')
|
| 22 |
+
|
| 23 |
+
# Fix data-eid on Find Shortest Path button (unsanitized entityId)
|
| 24 |
+
old2 = 'data-eid="${entityId}"'
|
| 25 |
+
new2 = 'data-eid="${sanitize(entityId)}"'
|
| 26 |
+
if old2 in content:
|
| 27 |
+
content = content.replace(old2, new2)
|
| 28 |
+
print('OK: path-finder data-eid sanitized')
|
| 29 |
+
else:
|
| 30 |
+
print('INFO: path-finder data-eid pattern not found (may already be sanitized)')
|
| 31 |
+
|
| 32 |
+
with open('frontend/js/app.js', 'w', encoding='utf-8') as f:
|
| 33 |
+
f.write(content)
|
| 34 |
+
print('Done')
|
fix_lang_search.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# fix_lang_search.py
|
| 2 |
+
with open('frontend/js/app.js', 'r', encoding='utf-8') as f:
|
| 3 |
+
content = f.read()
|
| 4 |
+
|
| 5 |
+
# LOGIC-3: preserve language in filter buttons
|
| 6 |
+
# Find the filter button onclick and append lang param
|
| 7 |
+
import re
|
| 8 |
+
|
| 9 |
+
# Pattern: onclick="Router.navigate('/search?q=${...}${...}')"
|
| 10 |
+
old_onclick = "onclick=\"Router.navigate('/search?q=${encodeURIComponent(query)}${t!==\"All\"?\"&type=\"+t.toLowerCase():\"\"}')\">"
|
| 11 |
+
new_onclick = "onclick=\"Router.navigate('/search?q=${encodeURIComponent(query)}${t!==\\'All\\'?\"&type=\"+t.toLowerCase():\"\"}'+(State.language&&State.language!=='en'?'&lang='+State.language:''))\">"
|
| 12 |
+
|
| 13 |
+
if old_onclick in content:
|
| 14 |
+
content = content.replace(old_onclick, new_onclick)
|
| 15 |
+
print('OK: lang param added to filter buttons')
|
| 16 |
+
else:
|
| 17 |
+
print('WARNING: filter onclick not found, trying simpler replace...')
|
| 18 |
+
# Simpler: just find the navigate call inside the filter map
|
| 19 |
+
idx = content.find("Router.navigate('/search?q=${encodeURIComponent(query)}")
|
| 20 |
+
if idx != -1:
|
| 21 |
+
print('Found at index', idx)
|
| 22 |
+
print('Context:', repr(content[idx:idx+120]))
|
| 23 |
+
else:
|
| 24 |
+
print('Not found at all')
|
| 25 |
+
|
| 26 |
+
# LOGIC-3: preserve language in search button
|
| 27 |
+
old_btn = "if (q) Router.navigate(`/search?q=${encodeURIComponent(q)}`);"
|
| 28 |
+
new_btn = "const _l=State.language&&State.language!=='en'?'&lang='+State.language:'';\n if (q) Router.navigate(`/search?q=${encodeURIComponent(q)}${_l}`);"
|
| 29 |
+
|
| 30 |
+
replaced = 0
|
| 31 |
+
while old_btn in content:
|
| 32 |
+
content = content.replace(old_btn, new_btn, 1)
|
| 33 |
+
replaced += 1
|
| 34 |
+
print(f'OK: search button lang param added ({replaced} replacements)')
|
| 35 |
+
|
| 36 |
+
with open('frontend/js/app.js', 'w', encoding='utf-8') as f:
|
| 37 |
+
f.write(content)
|
| 38 |
+
print('Done')
|
fix_stats_lock.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# fix_stats_lock.py -- fixes BACKEND-2: add threading lock for stats cache
|
| 2 |
+
import re
|
| 3 |
+
|
| 4 |
+
with open('api/main.py', 'r', encoding='utf-8') as f:
|
| 5 |
+
content = f.read()
|
| 6 |
+
|
| 7 |
+
# First: check what actually exists in the file around stats cache
|
| 8 |
+
idx = content.find('_stats_cache')
|
| 9 |
+
if idx == -1:
|
| 10 |
+
print('ERROR: _stats_cache not found in main.py at all')
|
| 11 |
+
exit()
|
| 12 |
+
|
| 13 |
+
# Show context
|
| 14 |
+
print('Current stats cache area:')
|
| 15 |
+
print(repr(content[idx-10:idx+200]))
|
| 16 |
+
print()
|
| 17 |
+
|
| 18 |
+
# Add threading import + lock after _stats_cache = None line
|
| 19 |
+
# Find the module-level _stats_cache declaration
|
| 20 |
+
import_added = False
|
| 21 |
+
if '_stats_lock' not in content:
|
| 22 |
+
# Add threading lock after _STATS_TTL line
|
| 23 |
+
old_ttl = '_STATS_TTL = 60.0'
|
| 24 |
+
if old_ttl in content:
|
| 25 |
+
content = content.replace(
|
| 26 |
+
old_ttl,
|
| 27 |
+
old_ttl + '\nimport threading as _th\n_stats_lock = _th.Lock()'
|
| 28 |
+
)
|
| 29 |
+
import_added = True
|
| 30 |
+
print('OK: _stats_lock added after _STATS_TTL')
|
| 31 |
+
else:
|
| 32 |
+
# Try alternate TTL variable name
|
| 33 |
+
for candidate in ['_STATS_TTL = 60.0', '_STATS_TTL=60.0']:
|
| 34 |
+
if candidate in content:
|
| 35 |
+
content = content.replace(
|
| 36 |
+
candidate,
|
| 37 |
+
candidate + '\nimport threading as _th\n_stats_lock = _th.Lock()'
|
| 38 |
+
)
|
| 39 |
+
import_added = True
|
| 40 |
+
print(f'OK: lock added after {candidate}')
|
| 41 |
+
break
|
| 42 |
+
if not import_added:
|
| 43 |
+
print('WARNING: could not find _STATS_TTL -- adding lock near _stats_cache')
|
| 44 |
+
content = content.replace(
|
| 45 |
+
'_stats_cache = None',
|
| 46 |
+
'_stats_cache = None\nimport threading as _th\n_stats_lock = _th.Lock()',
|
| 47 |
+
1
|
| 48 |
+
)
|
| 49 |
+
import_added = True
|
| 50 |
+
|
| 51 |
+
# Add double-check pattern inside get_stats
|
| 52 |
+
# Find the early return and add lock before the Neo4j call
|
| 53 |
+
old_early_return = ' if _stats_cache is not None and'
|
| 54 |
+
if old_early_return in content:
|
| 55 |
+
# Find the full early-return block
|
| 56 |
+
start = content.find(old_early_return)
|
| 57 |
+
# Find end of that if block (the return statement)
|
| 58 |
+
end = content.find('\n', content.find('return _stats_cache', start)) + 1
|
| 59 |
+
early_block = content[start:end]
|
| 60 |
+
print('Found early return block:')
|
| 61 |
+
print(repr(early_block))
|
| 62 |
+
|
| 63 |
+
# After the early return, add lock + double-check
|
| 64 |
+
old_driver_line = ' driver = get_driver()'
|
| 65 |
+
if old_driver_line in content:
|
| 66 |
+
content = content.replace(
|
| 67 |
+
old_driver_line,
|
| 68 |
+
' # BACKEND-2 FIX: lock prevents two concurrent requests both\n'
|
| 69 |
+
' # seeing _stats_cache is None and both running the full graph scan\n'
|
| 70 |
+
' with _stats_lock:\n'
|
| 71 |
+
' import time as _tc\n'
|
| 72 |
+
' if _stats_cache is not None and (_tc.monotonic() - _stats_cached_at) < _STATS_TTL:\n'
|
| 73 |
+
' return _stats_cache\n'
|
| 74 |
+
' driver = get_driver()',
|
| 75 |
+
1
|
| 76 |
+
)
|
| 77 |
+
print('OK: double-check lock added before get_driver()')
|
| 78 |
+
else:
|
| 79 |
+
print('WARNING: "driver = get_driver()" not found in get_stats')
|
| 80 |
+
else:
|
| 81 |
+
print('WARNING: early return not found')
|
| 82 |
+
|
| 83 |
+
with open('api/main.py', 'w', encoding='utf-8') as f:
|
| 84 |
+
f.write(content)
|
| 85 |
+
|
| 86 |
+
# Verify syntax
|
| 87 |
+
import ast
|
| 88 |
+
try:
|
| 89 |
+
ast.parse(content)
|
| 90 |
+
print('SYNTAX OK')
|
| 91 |
+
except SyntaxError as e:
|
| 92 |
+
print(f'SYNTAX ERROR at line {e.lineno}: {e.msg}')
|
graph/queries.py
CHANGED
|
@@ -18,7 +18,7 @@ from loguru import logger
|
|
| 18 |
|
| 19 |
QUERIES = {
|
| 20 |
|
| 21 |
-
#
|
| 22 |
"politician_company_contracts": {
|
| 23 |
"description": "Find politicians linked to companies that won govt contracts",
|
| 24 |
"cypher": """
|
|
@@ -37,7 +37,7 @@ QUERIES = {
|
|
| 37 |
"risk_level": "HIGH",
|
| 38 |
},
|
| 39 |
|
| 40 |
-
#
|
| 41 |
"repeated_contract_winners": {
|
| 42 |
"description": "Companies that won multiple contracts - concentration risk",
|
| 43 |
"cypher": """
|
|
@@ -53,7 +53,7 @@ QUERIES = {
|
|
| 53 |
"risk_level": "MEDIUM",
|
| 54 |
},
|
| 55 |
|
| 56 |
-
#
|
| 57 |
"ministry_audit_flags": {
|
| 58 |
"description": "Ministries most flagged by CAG audit reports",
|
| 59 |
"cypher": """
|
|
@@ -68,7 +68,7 @@ QUERIES = {
|
|
| 68 |
"risk_level": "MEDIUM",
|
| 69 |
},
|
| 70 |
|
| 71 |
-
#
|
| 72 |
"scheme_irregularities": {
|
| 73 |
"description": "Government schemes with largest CAG-flagged irregularities",
|
| 74 |
"cypher": """
|
|
@@ -83,7 +83,7 @@ QUERIES = {
|
|
| 83 |
"risk_level": "HIGH",
|
| 84 |
},
|
| 85 |
|
| 86 |
-
#
|
| 87 |
"politicians_with_cases": {
|
| 88 |
"description": "Politicians with declared criminal cases",
|
| 89 |
"cypher": """
|
|
@@ -100,7 +100,7 @@ QUERIES = {
|
|
| 100 |
"risk_level": "MEDIUM",
|
| 101 |
},
|
| 102 |
|
| 103 |
-
#
|
| 104 |
"politician_profile": {
|
| 105 |
"description": "Full graph profile for a named politician",
|
| 106 |
"cypher": """
|
|
@@ -122,7 +122,7 @@ QUERIES = {
|
|
| 122 |
"params": {"name": "politician name to search"},
|
| 123 |
},
|
| 124 |
|
| 125 |
-
#
|
| 126 |
"high_value_contracts": {
|
| 127 |
"description": "All contracts above threshold crore value",
|
| 128 |
"cypher": """
|
|
@@ -140,7 +140,7 @@ QUERIES = {
|
|
| 140 |
"params": {"min_crore": 1.0},
|
| 141 |
},
|
| 142 |
|
| 143 |
-
#
|
| 144 |
"node_counts": {
|
| 145 |
"description": "Count of all node types in the graph",
|
| 146 |
"cypher": """
|
|
@@ -151,7 +151,7 @@ QUERIES = {
|
|
| 151 |
"risk_level": "INFO",
|
| 152 |
},
|
| 153 |
|
| 154 |
-
#
|
| 155 |
"relationship_counts": {
|
| 156 |
"description": "Count of all relationship types in the graph",
|
| 157 |
"cypher": """
|
|
@@ -220,11 +220,11 @@ class QueryRunner:
|
|
| 220 |
def print_query_library():
|
| 221 |
"""Print all available queries."""
|
| 222 |
print("=" * 55)
|
| 223 |
-
print(" BharatGraph
|
| 224 |
print("=" * 55)
|
| 225 |
for name, q in QUERIES.items():
|
| 226 |
risk = q["risk_level"]
|
| 227 |
-
emoji = {"HIGH": "
|
| 228 |
print(f"\n {emoji} {name}")
|
| 229 |
print(f" {q['description']}")
|
| 230 |
if "params" in q:
|
|
|
|
| 18 |
|
| 19 |
QUERIES = {
|
| 20 |
|
| 21 |
+
# -- Core corruption pattern ---------------------------
|
| 22 |
"politician_company_contracts": {
|
| 23 |
"description": "Find politicians linked to companies that won govt contracts",
|
| 24 |
"cypher": """
|
|
|
|
| 37 |
"risk_level": "HIGH",
|
| 38 |
},
|
| 39 |
|
| 40 |
+
# -- Repeated contract winners -------------------------
|
| 41 |
"repeated_contract_winners": {
|
| 42 |
"description": "Companies that won multiple contracts - concentration risk",
|
| 43 |
"cypher": """
|
|
|
|
| 53 |
"risk_level": "MEDIUM",
|
| 54 |
},
|
| 55 |
|
| 56 |
+
# -- Ministry audit flags ------------------------------
|
| 57 |
"ministry_audit_flags": {
|
| 58 |
"description": "Ministries most flagged by CAG audit reports",
|
| 59 |
"cypher": """
|
|
|
|
| 68 |
"risk_level": "MEDIUM",
|
| 69 |
},
|
| 70 |
|
| 71 |
+
# -- Scheme irregularities -----------------------------
|
| 72 |
"scheme_irregularities": {
|
| 73 |
"description": "Government schemes with largest CAG-flagged irregularities",
|
| 74 |
"cypher": """
|
|
|
|
| 83 |
"risk_level": "HIGH",
|
| 84 |
},
|
| 85 |
|
| 86 |
+
# -- Politicians with criminal cases -------------------
|
| 87 |
"politicians_with_cases": {
|
| 88 |
"description": "Politicians with declared criminal cases",
|
| 89 |
"cypher": """
|
|
|
|
| 100 |
"risk_level": "MEDIUM",
|
| 101 |
},
|
| 102 |
|
| 103 |
+
# -- Full profile: one politician ----------------------
|
| 104 |
"politician_profile": {
|
| 105 |
"description": "Full graph profile for a named politician",
|
| 106 |
"cypher": """
|
|
|
|
| 122 |
"params": {"name": "politician name to search"},
|
| 123 |
},
|
| 124 |
|
| 125 |
+
# -- High value contracts ------------------------------
|
| 126 |
"high_value_contracts": {
|
| 127 |
"description": "All contracts above threshold crore value",
|
| 128 |
"cypher": """
|
|
|
|
| 140 |
"params": {"min_crore": 1.0},
|
| 141 |
},
|
| 142 |
|
| 143 |
+
# -- Node counts (health check) ------------------------
|
| 144 |
"node_counts": {
|
| 145 |
"description": "Count of all node types in the graph",
|
| 146 |
"cypher": """
|
|
|
|
| 151 |
"risk_level": "INFO",
|
| 152 |
},
|
| 153 |
|
| 154 |
+
# -- Relationship counts (health check) ----------------
|
| 155 |
"relationship_counts": {
|
| 156 |
"description": "Count of all relationship types in the graph",
|
| 157 |
"cypher": """
|
|
|
|
| 220 |
def print_query_library():
|
| 221 |
"""Print all available queries."""
|
| 222 |
print("=" * 55)
|
| 223 |
+
print(" BharatGraph -- Cypher Query Library")
|
| 224 |
print("=" * 55)
|
| 225 |
for name, q in QUERIES.items():
|
| 226 |
risk = q["risk_level"]
|
| 227 |
+
emoji = {"HIGH": "?", "MEDIUM": "?", "INFO": "?"}.get(risk, "?")
|
| 228 |
print(f"\n {emoji} {name}")
|
| 229 |
print(f" {q['description']}")
|
| 230 |
if "params" in q:
|
graph/schema.py
CHANGED
|
@@ -18,7 +18,7 @@ This is the schema that makes BharatGraph powerful:
|
|
| 18 |
- Query: Show audit reports flagging the same ministry
|
| 19 |
"""
|
| 20 |
|
| 21 |
-
#
|
| 22 |
# Each dict defines the properties a node of that type can have.
|
| 23 |
|
| 24 |
NODE_SCHEMAS = {
|
|
@@ -146,7 +146,7 @@ NODE_SCHEMAS = {
|
|
| 146 |
}
|
| 147 |
|
| 148 |
|
| 149 |
-
#
|
| 150 |
|
| 151 |
RELATIONSHIP_SCHEMAS = {
|
| 152 |
|
|
@@ -206,10 +206,10 @@ RELATIONSHIP_SCHEMAS = {
|
|
| 206 |
}
|
| 207 |
|
| 208 |
|
| 209 |
-
#
|
| 210 |
# Run these once when setting up a new Neo4j database.
|
| 211 |
|
| 212 |
-
#
|
| 213 |
# This powers instant search across all labels and fields simultaneously.
|
| 214 |
FULLTEXT_INDEX_QUERY = (
|
| 215 |
# BUG-19 FIX: expanded from 8 to 16 node types + 14 searchable fields.
|
|
@@ -253,12 +253,12 @@ SETUP_QUERIES = [
|
|
| 253 |
def print_schema():
|
| 254 |
"""Print a human-readable summary of the graph schema."""
|
| 255 |
print("=" * 55)
|
| 256 |
-
print(" BharatGraph
|
| 257 |
print("=" * 55)
|
| 258 |
print(f"\nNode types ({len(NODE_SCHEMAS)}):")
|
| 259 |
for label, schema in NODE_SCHEMAS.items():
|
| 260 |
props = len(schema["properties"])
|
| 261 |
-
print(f" ({label})
|
| 262 |
print(f" {props} properties, indexes on: {schema['indexes']}")
|
| 263 |
print(f"\nRelationship types ({len(RELATIONSHIP_SCHEMAS)}):")
|
| 264 |
for rel, schema in RELATIONSHIP_SCHEMAS.items():
|
|
|
|
| 18 |
- Query: Show audit reports flagging the same ministry
|
| 19 |
"""
|
| 20 |
|
| 21 |
+
# -- Node Labels -------------------------------------------
|
| 22 |
# Each dict defines the properties a node of that type can have.
|
| 23 |
|
| 24 |
NODE_SCHEMAS = {
|
|
|
|
| 146 |
}
|
| 147 |
|
| 148 |
|
| 149 |
+
# -- Relationship Types ------------------------------------
|
| 150 |
|
| 151 |
RELATIONSHIP_SCHEMAS = {
|
| 152 |
|
|
|
|
| 206 |
}
|
| 207 |
|
| 208 |
|
| 209 |
+
# -- Cypher constraint + index statements -----------------
|
| 210 |
# Run these once when setting up a new Neo4j database.
|
| 211 |
|
| 212 |
+
# -- Full-text index (run once) -----------------------------------------------
|
| 213 |
# This powers instant search across all labels and fields simultaneously.
|
| 214 |
FULLTEXT_INDEX_QUERY = (
|
| 215 |
# BUG-19 FIX: expanded from 8 to 16 node types + 14 searchable fields.
|
|
|
|
| 253 |
def print_schema():
|
| 254 |
"""Print a human-readable summary of the graph schema."""
|
| 255 |
print("=" * 55)
|
| 256 |
+
print(" BharatGraph -- Neo4j Schema")
|
| 257 |
print("=" * 55)
|
| 258 |
print(f"\nNode types ({len(NODE_SCHEMAS)}):")
|
| 259 |
for label, schema in NODE_SCHEMAS.items():
|
| 260 |
props = len(schema["properties"])
|
| 261 |
+
print(f" ({label}) -- {schema['description'][:50]}")
|
| 262 |
print(f" {props} properties, indexes on: {schema['indexes']}")
|
| 263 |
print(f"\nRelationship types ({len(RELATIONSHIP_SCHEMAS)}):")
|
| 264 |
for rel, schema in RELATIONSHIP_SCHEMAS.items():
|
graph/seed.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
BharatGraph
|
| 3 |
Loads sample nodes AND relationships into Neo4j for demonstration.
|
| 4 |
Real data comes from POST /admin/pipeline which runs all 21 scrapers.
|
| 5 |
"""
|
|
@@ -12,7 +12,7 @@ from graph.loader import GraphLoader
|
|
| 12 |
|
| 13 |
load_dotenv()
|
| 14 |
|
| 15 |
-
#
|
| 16 |
SAMPLE_POLITICIANS = [
|
| 17 |
{"id":"pol_001","name":"Narendra Modi","state":"Gujarat","party":"BJP",
|
| 18 |
"constituency":"Varanasi","criminal_cases":0,"criminal_case_count":"0",
|
|
@@ -46,7 +46,7 @@ SAMPLE_POLITICIANS = [
|
|
| 46 |
"total_assets":33.87,"education":"BA History","year":2024,"source":"myneta","scraped_at":"2026-05-04T05:45:50Z"},
|
| 47 |
]
|
| 48 |
|
| 49 |
-
#
|
| 50 |
SAMPLE_COMPANIES = [
|
| 51 |
{"id":"co_001","name":"Adani Enterprises Limited","state":"Gujarat",
|
| 52 |
"cin":"L51100GJ1993PLC019067","status":"Active","paid_up_capital_crore":112.0,"source":"mca","scraped_at":"2026-05-04T05:45:50Z"},
|
|
@@ -64,7 +64,7 @@ SAMPLE_COMPANIES = [
|
|
| 64 |
"cin":"L74899DL1993GOI054155","status":"Active","paid_up_capital_crore":1258.0,"source":"mca","scraped_at":"2026-05-04T05:45:50Z"},
|
| 65 |
]
|
| 66 |
|
| 67 |
-
#
|
| 68 |
SAMPLE_CONTRACTS = [
|
| 69 |
{"id":"ct_001","order_id":"GEM/2024/B/001","item_desc":"Road Construction Supplies",
|
| 70 |
"amount_crore":45.5,"buyer_org":"Ministry of Road Transport",
|
|
@@ -88,7 +88,7 @@ SAMPLE_CONTRACTS = [
|
|
| 88 |
"seller_name":"Adani Enterprises Limited","seller_name_raw":"Adani Enterprises Limited","source":"gem","scraped_at":"2026-05-04T05:45:50Z"},
|
| 89 |
]
|
| 90 |
|
| 91 |
-
#
|
| 92 |
SAMPLE_AUDIT_REPORTS = [
|
| 93 |
{"id":"ar_001","title":"Performance Audit of Pradhan Mantri Gram Sadak Yojana 2023",
|
| 94 |
"year":2023,"ministry":"Ministry of Rural Development","state":"National",
|
|
@@ -104,9 +104,9 @@ SAMPLE_AUDIT_REPORTS = [
|
|
| 104 |
"amount_crore":892.0,"url":"https://cag.gov.in/reports/2022","source":"cag","scraped_at":"2026-05-04T05:45:50Z"},
|
| 105 |
]
|
| 106 |
|
| 107 |
-
#
|
| 108 |
SAMPLE_DIRECTOR_LINKS = [
|
| 109 |
-
# Keys must be name_a / name_b
|
| 110 |
{"name_a": "Amit Shah", "name_b": "Adani Enterprises Limited", "score": 0.92},
|
| 111 |
{"name_a": "Narendra Modi", "name_b": "ONGC Limited", "score": 0.88},
|
| 112 |
{"name_a": "Anurag Thakur", "name_b": "Tata Consultancy Services", "score": 0.81},
|
|
|
|
| 1 |
"""
|
| 2 |
+
BharatGraph -- Seed Data
|
| 3 |
Loads sample nodes AND relationships into Neo4j for demonstration.
|
| 4 |
Real data comes from POST /admin/pipeline which runs all 21 scrapers.
|
| 5 |
"""
|
|
|
|
| 12 |
|
| 13 |
load_dotenv()
|
| 14 |
|
| 15 |
+
# -- Politicians (ECI / MyNeta) -------------------------------------------------
|
| 16 |
SAMPLE_POLITICIANS = [
|
| 17 |
{"id":"pol_001","name":"Narendra Modi","state":"Gujarat","party":"BJP",
|
| 18 |
"constituency":"Varanasi","criminal_cases":0,"criminal_case_count":"0",
|
|
|
|
| 46 |
"total_assets":33.87,"education":"BA History","year":2024,"source":"myneta","scraped_at":"2026-05-04T05:45:50Z"},
|
| 47 |
]
|
| 48 |
|
| 49 |
+
# -- Companies (MCA21) ----------------------------------------------------------
|
| 50 |
SAMPLE_COMPANIES = [
|
| 51 |
{"id":"co_001","name":"Adani Enterprises Limited","state":"Gujarat",
|
| 52 |
"cin":"L51100GJ1993PLC019067","status":"Active","paid_up_capital_crore":112.0,"source":"mca","scraped_at":"2026-05-04T05:45:50Z"},
|
|
|
|
| 64 |
"cin":"L74899DL1993GOI054155","status":"Active","paid_up_capital_crore":1258.0,"source":"mca","scraped_at":"2026-05-04T05:45:50Z"},
|
| 65 |
]
|
| 66 |
|
| 67 |
+
# -- Contracts (GeM) ------------------------------------------------------------
|
| 68 |
SAMPLE_CONTRACTS = [
|
| 69 |
{"id":"ct_001","order_id":"GEM/2024/B/001","item_desc":"Road Construction Supplies",
|
| 70 |
"amount_crore":45.5,"buyer_org":"Ministry of Road Transport",
|
|
|
|
| 88 |
"seller_name":"Adani Enterprises Limited","seller_name_raw":"Adani Enterprises Limited","source":"gem","scraped_at":"2026-05-04T05:45:50Z"},
|
| 89 |
]
|
| 90 |
|
| 91 |
+
# -- Audit Reports (CAG) --------------------------------------------------------
|
| 92 |
SAMPLE_AUDIT_REPORTS = [
|
| 93 |
{"id":"ar_001","title":"Performance Audit of Pradhan Mantri Gram Sadak Yojana 2023",
|
| 94 |
"year":2023,"ministry":"Ministry of Rural Development","state":"National",
|
|
|
|
| 104 |
"amount_crore":892.0,"url":"https://cag.gov.in/reports/2022","source":"cag","scraped_at":"2026-05-04T05:45:50Z"},
|
| 105 |
]
|
| 106 |
|
| 107 |
+
# -- Politician-Company Director links (entity resolution) ----------------------
|
| 108 |
SAMPLE_DIRECTOR_LINKS = [
|
| 109 |
+
# Keys must be name_a / name_b -- these match graph/loader.py load_politician_company_links()
|
| 110 |
{"name_a": "Amit Shah", "name_b": "Adani Enterprises Limited", "score": 0.92},
|
| 111 |
{"name_a": "Narendra Modi", "name_b": "ONGC Limited", "score": 0.88},
|
| 112 |
{"name_a": "Anurag Thakur", "name_b": "Tata Consultancy Services", "score": 0.81},
|
processing/cleaner.py
CHANGED
|
@@ -77,7 +77,7 @@ class NameCleaner:
|
|
| 77 |
"""
|
| 78 |
Parse Indian currency strings to float in crore.
|
| 79 |
'150 Cr' -> 150.0, '500 lakh' -> 5.0,
|
| 80 |
-
'1500000' -> 1.5 (raw rupees), '
|
| 81 |
"""
|
| 82 |
if not amount_str:
|
| 83 |
return 0.0
|
|
@@ -88,7 +88,7 @@ class NameCleaner:
|
|
| 88 |
except ValueError:
|
| 89 |
pass
|
| 90 |
# Remove currency symbols and commas
|
| 91 |
-
s = re.sub(r"[
|
| 92 |
s = re.sub(r"^rs\.?", "", s, flags=re.IGNORECASE)
|
| 93 |
# Detect suffix
|
| 94 |
if re.search(r"crore|cr", s, re.IGNORECASE):
|
|
@@ -192,7 +192,7 @@ if __name__ == "__main__":
|
|
| 192 |
print(f" '{n}' -> '{c.clean_company_name(n)}'")
|
| 193 |
|
| 194 |
print("\n[3] Amounts (to crore):")
|
| 195 |
-
for a in ["150 Cr","500 lakh","1500000","Rs. 2,50,00,000","
|
| 196 |
print(f" '{a}' -> {c.clean_amount(a)} Cr")
|
| 197 |
|
| 198 |
print("\n[4] States:")
|
|
|
|
| 77 |
"""
|
| 78 |
Parse Indian currency strings to float in crore.
|
| 79 |
'150 Cr' -> 150.0, '500 lakh' -> 5.0,
|
| 80 |
+
'1500000' -> 1.5 (raw rupees), 'Rs.75cr' -> 75.0
|
| 81 |
"""
|
| 82 |
if not amount_str:
|
| 83 |
return 0.0
|
|
|
|
| 88 |
except ValueError:
|
| 89 |
pass
|
| 90 |
# Remove currency symbols and commas
|
| 91 |
+
s = re.sub(r"[Rs.,\s]", "", s)
|
| 92 |
s = re.sub(r"^rs\.?", "", s, flags=re.IGNORECASE)
|
| 93 |
# Detect suffix
|
| 94 |
if re.search(r"crore|cr", s, re.IGNORECASE):
|
|
|
|
| 192 |
print(f" '{n}' -> '{c.clean_company_name(n)}'")
|
| 193 |
|
| 194 |
print("\n[3] Amounts (to crore):")
|
| 195 |
+
for a in ["150 Cr","500 lakh","1500000","Rs. 2,50,00,000","Rs.75cr"]:
|
| 196 |
print(f" '{a}' -> {c.clean_amount(a)} Cr")
|
| 197 |
|
| 198 |
print("\n[4] States:")
|
processing/entity_resolver.py
CHANGED
|
@@ -248,7 +248,7 @@ class EntityResolver:
|
|
| 248 |
logger.success(f"[Resolver] Saved {len(matches)} matches to {filepath}")
|
| 249 |
|
| 250 |
|
| 251 |
-
#
|
| 252 |
if __name__ == "__main__":
|
| 253 |
print("=" * 55)
|
| 254 |
print("BharatGraph - Entity Resolver Test")
|
|
@@ -268,7 +268,7 @@ if __name__ == "__main__":
|
|
| 268 |
]
|
| 269 |
for a, b in pairs:
|
| 270 |
score = resolver.similarity_score(a, b)
|
| 271 |
-
match = "
|
| 272 |
print(f" {match} ({score:.2f}) '{a}' vs '{b}'")
|
| 273 |
|
| 274 |
print("\n[2] Deduplication within dataset:")
|
|
|
|
| 248 |
logger.success(f"[Resolver] Saved {len(matches)} matches to {filepath}")
|
| 249 |
|
| 250 |
|
| 251 |
+
# -- Run directly to test ---------------------------------
|
| 252 |
if __name__ == "__main__":
|
| 253 |
print("=" * 55)
|
| 254 |
print("BharatGraph - Entity Resolver Test")
|
|
|
|
| 268 |
]
|
| 269 |
for a, b in pairs:
|
| 270 |
score = resolver.similarity_score(a, b)
|
| 271 |
+
match = "? MATCH" if score >= 0.6 else "? no match"
|
| 272 |
print(f" {match} ({score:.2f}) '{a}' vs '{b}'")
|
| 273 |
|
| 274 |
print("\n[2] Deduplication within dataset:")
|
scrapers/cag_scraper.py
CHANGED
|
@@ -198,11 +198,11 @@ class CAGScraper(BaseScraper):
|
|
| 198 |
r for r in reports
|
| 199 |
if float(r.get("amount_crore", 0) or 0) >= min_crore
|
| 200 |
]
|
| 201 |
-
logger.info(f"[CAG] Reports >=
|
| 202 |
return flagged
|
| 203 |
|
| 204 |
|
| 205 |
-
#
|
| 206 |
if __name__ == "__main__":
|
| 207 |
print("=" * 60)
|
| 208 |
print("BharatGraph - CAG Audit Report Scraper Test")
|
|
@@ -221,7 +221,7 @@ if __name__ == "__main__":
|
|
| 221 |
print(f" URL: {r.get('url', 'N/A')[:60]}")
|
| 222 |
print(f" Alert kws: {r.get('alert_keywords', [])}")
|
| 223 |
if r.get("amount_crore"):
|
| 224 |
-
print(f" Amount:
|
| 225 |
|
| 226 |
print("\n[2] Filtering high-value irregularities (>=10 Cr)...")
|
| 227 |
big = scraper.get_high_value_irregularities(reports, min_crore=10.0)
|
|
|
|
| 198 |
r for r in reports
|
| 199 |
if float(r.get("amount_crore", 0) or 0) >= min_crore
|
| 200 |
]
|
| 201 |
+
logger.info(f"[CAG] Reports >= Rs.{min_crore}Cr: {len(flagged)}")
|
| 202 |
return flagged
|
| 203 |
|
| 204 |
|
| 205 |
+
# -- Run directly to test ------------------------------------------------------
|
| 206 |
if __name__ == "__main__":
|
| 207 |
print("=" * 60)
|
| 208 |
print("BharatGraph - CAG Audit Report Scraper Test")
|
|
|
|
| 221 |
print(f" URL: {r.get('url', 'N/A')[:60]}")
|
| 222 |
print(f" Alert kws: {r.get('alert_keywords', [])}")
|
| 223 |
if r.get("amount_crore"):
|
| 224 |
+
print(f" Amount: Rs.{r.get('amount_crore')} Crore")
|
| 225 |
|
| 226 |
print("\n[2] Filtering high-value irregularities (>=10 Cr)...")
|
| 227 |
big = scraper.get_high_value_irregularities(reports, min_crore=10.0)
|
scrapers/datagov_scraper.py
CHANGED
|
@@ -86,7 +86,7 @@ class DataGovScraper(BaseScraper):
|
|
| 86 |
return self.get_json(url, params=params)
|
| 87 |
|
| 88 |
|
| 89 |
-
#
|
| 90 |
if __name__ == "__main__":
|
| 91 |
print("=" * 60)
|
| 92 |
print("BharatGraph - DataGov Scraper Test")
|
|
@@ -100,12 +100,12 @@ if __name__ == "__main__":
|
|
| 100 |
)
|
| 101 |
|
| 102 |
if data:
|
| 103 |
-
print(f"
|
| 104 |
print(f" Fields: {list(data.get('fields', [{}])[0].keys()) if data.get('fields') else 'N/A'}")
|
| 105 |
else:
|
| 106 |
-
print("
|
| 107 |
|
| 108 |
print("\n[2] Fetching all configured datasets...")
|
| 109 |
all_data = scraper.fetch_all_datasets(save=True)
|
| 110 |
-
print(f"
|
| 111 |
print("\nDone! Check data/samples/ folder for output files.")
|
|
|
|
| 86 |
return self.get_json(url, params=params)
|
| 87 |
|
| 88 |
|
| 89 |
+
# -- Run directly to test ------------------------------------------------------
|
| 90 |
if __name__ == "__main__":
|
| 91 |
print("=" * 60)
|
| 92 |
print("BharatGraph - DataGov Scraper Test")
|
|
|
|
| 100 |
)
|
| 101 |
|
| 102 |
if data:
|
| 103 |
+
print(f" ? Success! Total records available: {data.get('total', '?')}")
|
| 104 |
print(f" Fields: {list(data.get('fields', [{}])[0].keys()) if data.get('fields') else 'N/A'}")
|
| 105 |
else:
|
| 106 |
+
print(" ? Failed (check internet connection)")
|
| 107 |
|
| 108 |
print("\n[2] Fetching all configured datasets...")
|
| 109 |
all_data = scraper.fetch_all_datasets(save=True)
|
| 110 |
+
print(f" ? Fetched {len(all_data)} datasets")
|
| 111 |
print("\nDone! Check data/samples/ folder for output files.")
|
scrapers/gem_scraper.py
CHANGED
|
@@ -214,7 +214,7 @@ class GeMScraper(BaseScraper):
|
|
| 214 |
return result
|
| 215 |
|
| 216 |
|
| 217 |
-
#
|
| 218 |
if __name__ == "__main__":
|
| 219 |
print("=" * 60)
|
| 220 |
print("BharatGraph - GeM Contracts Scraper Test")
|
|
|
|
| 214 |
return result
|
| 215 |
|
| 216 |
|
| 217 |
+
# -- Run directly to test -----------------------------------------------------
|
| 218 |
if __name__ == "__main__":
|
| 219 |
print("=" * 60)
|
| 220 |
print("BharatGraph - GeM Contracts Scraper Test")
|
scrapers/mca_scraper.py
CHANGED
|
@@ -4,7 +4,7 @@ Fetches company and director data from India's corporate registry.
|
|
| 4 |
Sources:
|
| 5 |
- data.gov.in (MCA company master dataset - free, public)
|
| 6 |
- MCA21 portal snapshots
|
| 7 |
-
This links politicians
|
| 8 |
"""
|
| 9 |
|
| 10 |
import json
|
|
@@ -169,7 +169,7 @@ class MCAScraper(BaseScraper):
|
|
| 169 |
return results
|
| 170 |
|
| 171 |
|
| 172 |
-
#
|
| 173 |
if __name__ == "__main__":
|
| 174 |
print("=" * 60)
|
| 175 |
print("BharatGraph - MCA Scraper Test")
|
|
|
|
| 4 |
Sources:
|
| 5 |
- data.gov.in (MCA company master dataset - free, public)
|
| 6 |
- MCA21 portal snapshots
|
| 7 |
+
This links politicians -> companies -> contracts in the graph.
|
| 8 |
"""
|
| 9 |
|
| 10 |
import json
|
|
|
|
| 169 |
return results
|
| 170 |
|
| 171 |
|
| 172 |
+
# -- Run directly to test ------------------------------------------------------
|
| 173 |
if __name__ == "__main__":
|
| 174 |
print("=" * 60)
|
| 175 |
print("BharatGraph - MCA Scraper Test")
|