File size: 2,460 Bytes
bde2f3a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | #!/usr/bin/env python3
"""Search Semantic Scholar for paper arxiv IDs with rate limit handling."""
import json
import sys
import time
import urllib.request
def search_s2(query, limit=3, retries=3):
"""Search Semantic Scholar API with retries."""
q = "+".join(query.split())
url = f"https://api.semanticscholar.org/graph/v1/paper/search?query={q}&limit={limit}&fields=title,externalIds"
for attempt in range(retries):
try:
req = urllib.request.Request(url, headers={"User-Agent": "RMI-Research-Bot/1.0"})
resp = urllib.request.urlopen(req, timeout=25)
data = json.loads(resp.read())
results = []
for paper in data.get("data", []):
title = paper.get("title", "")
ids = paper.get("externalIds", {})
arxiv_id = ids.get("ArXiv", "")
results.append((arxiv_id, title))
return results
except urllib.error.HTTPError as e:
if e.code == 429:
wait = 10 * (attempt + 1)
print(f" Rate limited, waiting {wait}s...", flush=True)
time.sleep(wait)
else:
print(f" HTTP Error {e}", file=sys.stderr)
return []
except Exception as e:
print(f" ERROR: {e}", file=sys.stderr)
return []
return []
searches = [
("smart_vuln", "Smart Contract Vulnerability Detection Graph Feature Expert Pattern"),
("temporal_eth", "Temporal-Amount Snapshot MultiGraph Ethereum"),
("reentrancy", "Reentrancy Vulnerability Identification Ethereum Smart Contracts"),
("tokenomics", "How to Design Tokenomics"),
("vesting", "Token Vesting Cryptocurrency Blockchain"),
("governance", "Governance Token Distributions Empirical"),
]
found = {}
for key, q in searches:
print(f"Searching: {q}", flush=True)
results = search_s2(q)
if results:
for arxiv_id, title in results:
mark = "✓" if arxiv_id else "✗ (no arxiv)"
print(f" {mark} {arxiv_id or 'N/A'} | {title[:120]}")
if arxiv_id:
found[key] = arxiv_id
break
else:
print(" NO RESULTS")
time.sleep(5)
print(f"\n=== FOUND {len(found)}/{len(searches)} ===")
for k, v in found.items():
print(f" {k}: {v}")
with open("/root/backend/data/papers/found_ids.json", "w") as f:
json.dump(found, f, indent=2)
|