File size: 4,123 Bytes
6993919 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | #!/usr/bin/env python3
"""Convert and ingest additional arxiv papers into RAG."""
import json
import os
import sys
import pymupdf4llm
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
PAPERS_DIR = os.path.dirname(os.path.abspath(__file__)).rstrip("/scripts") + "/data/papers"
with open(os.path.join(PAPERS_DIR, "papers_metadata.json")) as f:
existing = json.load(f)
new_papers = {
"2109.05940": {
"title": "Smart Contract Vulnerability Detection: Graph Feature + Expert Pattern Fusion",
"authors": [],
"year": 2021,
"arxiv_id": "2109.05940",
"attack_types": ["vulnerability", "static-analysis"],
"collection": "forensic_reports",
"description": "Improving smart contract vulnerability detection using graph features and expert patterns.",
},
"2006.06116": {
"title": "Temporal-Amount Snapshot MultiGraph for Ethereum Transaction Tracking",
"authors": [],
"year": 2020,
"arxiv_id": "2006.06116",
"attack_types": ["transaction-graph", "tracking"],
"collection": "forensic_reports",
"description": "Multi-graph snapshot approach for tracking Ethereum transactions.",
},
"1905.00346": {
"title": "Reentrancy Vulnerability Identification in Ethereum Smart Contracts",
"authors": [],
"year": 2019,
"arxiv_id": "1905.00346",
"attack_types": ["reentrancy", "vulnerability"],
"collection": "forensic_reports",
"description": "Detection of reentrancy vulnerabilities in Ethereum smart contracts.",
},
"2207.01247": {
"title": "How to Design Tokenomics: Framework from 20+ Projects",
"authors": [],
"year": 2022,
"arxiv_id": "2207.01247",
"attack_types": ["tokenomics", "framework"],
"collection": "market_intel",
"description": "Tokenomics design framework from analyzing 20+ blockchain projects.",
},
"2202.05146": {
"title": "Empirical Evidence from Governance Token Distributions",
"authors": [],
"year": 2022,
"arxiv_id": "2202.05146",
"attack_types": ["governance", "token-distribution"],
"collection": "market_intel",
"description": "Empirical analysis of governance token distributions and DAO implications.",
},
"2004.02968": {
"title": "Ponzi Scheme Detection in Ethereum Transaction Network",
"authors": [],
"year": 2020,
"arxiv_id": "2004.02968",
"attack_types": ["ponzi", "detection"],
"collection": "forensic_reports",
"description": "Graph-based Ponzi scheme detection in Ethereum transactions.",
},
"2104.02368": {
"title": "EtherClue: Digital Investigation of Attacks on Ethereum Smart Contracts",
"authors": [],
"year": 2021,
"arxiv_id": "2104.02368",
"attack_types": ["forensics", "attack-detection"],
"collection": "forensic_reports",
"description": "EtherClue digital forensic investigation of smart contract attacks.",
},
}
converted = 0
for arxiv_id, info in new_papers.items():
pdf_path = os.path.join(PAPERS_DIR, f"{arxiv_id}.pdf")
txt_path = os.path.join(PAPERS_DIR, f"{arxiv_id}.txt")
if not os.path.exists(pdf_path):
print(f"SKIP {arxiv_id}: PDF not found")
continue
if arxiv_id in existing:
print(f"SKIP {arxiv_id}: already in metadata")
continue
sz = os.path.getsize(pdf_path)
if sz < 5000:
print(f"INVALID {arxiv_id}: PDF too small ({sz} bytes)")
continue
try:
text = pymupdf4llm.to_markdown(pdf_path)
with open(txt_path, "w") as f:
f.write(text)
words = len(text.split())
print(f"OK {arxiv_id}: {words} words")
existing[arxiv_id] = info
converted += 1
except Exception as e:
print(f"ERROR {arxiv_id}: {e}")
with open(os.path.join(PAPERS_DIR, "papers_metadata.json"), "w") as f:
json.dump(existing, f, indent=2)
print(f"Converted: {converted}, Total papers: {len(existing)}")
|