Buckets:
| """Bucket B: Tencent SkillHub (skillhub.cn) own skills (source=community; the clawhub-mirror | |
| part is skipped as we already have ClawHub). 1) listing -> jsonl. 2) per skill full bundle zip via | |
| /api/v1/download (302 -> Tencent COS). API calls throttled to ~4/s; CDN fetches 4 threads.""" | |
| import json, time, sys, io, zipfile, threading, requests | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| OUT = Path(sys.argv[1] if len(sys.argv) > 1 else "data/raw/tencent_skillhub") | |
| SOURCE = sys.argv[2] if len(sys.argv) > 2 else "community" | |
| (OUT / "zips").mkdir(parents=True, exist_ok=True) | |
| B = "https://api.skillhub.cn" | |
| S = requests.Session(); S.headers["User-Agent"] = "skills-db-research/0.1 (dataset building)" | |
| lock = threading.Lock(); last = [0.0] | |
| def api(url, **kw): | |
| for i in range(8): | |
| with lock: | |
| w = last[0] + 0.25 - time.time() | |
| if w > 0: time.sleep(w) | |
| last[0] = time.time() | |
| try: | |
| r = S.get(url, timeout=60, **kw) | |
| except requests.RequestException: | |
| time.sleep(2 ** i); continue | |
| if r.status_code == 429 or r.status_code >= 500: | |
| time.sleep(int(r.headers.get("retry-after", 2 ** i))); continue | |
| return r | |
| return None | |
| lst = OUT / f"listing_{SOURCE}.jsonl" | |
| if not lst.exists(): | |
| page, n, tmp = 1, 0, lst.with_suffix(".tmp") | |
| with open(tmp, "w") as f: | |
| while True: | |
| d = api(f"{B}/api/skills", params={"page": page, "pageSize": 100, "sortBy": "downloads", "source": SOURCE}).json()["data"] | |
| sk = d["skills"] | |
| for s in sk: f.write(json.dumps(s, ensure_ascii=False) + "\n") | |
| n += len(sk); page += 1 | |
| if page % 50 == 0: print("listed", n, "of", d.get("total"), flush=True) | |
| if not sk: break | |
| tmp.rename(lst); print("listing done", n, flush=True) | |
| seen, items = set(), [] | |
| for l in open(lst): | |
| s = json.loads(l); h = s["namespace"]["handle"]; key = (h, s["slug"]) | |
| if key not in seen: seen.add(key); items.append(s) | |
| done = {p.name for p in (OUT / "zips").glob("*.zip")} | |
| todo = [s for s in items if f"{s['namespace']['handle']}__{s['slug']}.zip" not in done] | |
| print(f"{len(items)} unique, {len(todo)} to fetch", flush=True) | |
| fails = open(OUT / "failures.jsonl", "a") | |
| def fetch(s): | |
| name = f"{s['namespace']['handle']}__{s['slug']}.zip" | |
| r = api(f"{B}/api/v1/download", params={"slug": s["slug"], "namespace": s["namespace"]["handle"]}, allow_redirects=False) | |
| loc = r.headers.get("location") if r is not None else None | |
| if not loc: | |
| fails.write(json.dumps({"key": name, "status": getattr(r, "status_code", None)}) + "\n"); return | |
| for i in range(5): | |
| try: | |
| z = requests.get(loc, timeout=120) | |
| if z.status_code == 200 and z.content[:2] == b"PK": | |
| zipfile.ZipFile(io.BytesIO(z.content)).namelist() | |
| (OUT / "zips" / name).write_bytes(z.content); return | |
| except Exception: pass | |
| time.sleep(2 ** i) | |
| fails.write(json.dumps({"key": name, "loc": loc}) + "\n") | |
| with ThreadPoolExecutor(4) as ex: | |
| for i, _ in enumerate(ex.map(fetch, todo)): | |
| if i % 1000 == 0: print("fetched", i, "/", len(todo), flush=True); fails.flush() | |
| print("done", flush=True) | |
Xet Storage Details
- Size:
- 3.26 kB
- Xet hash:
- e887717bdd740160988730e12401d4427ffe245bb5ece011f84e92b497b71185
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.