| |
| """Round-2 crawl list: inner pages (articles, docs, product and app pages) of the sites crawled in round 1. |
| |
| Round 1 crawled home pages only; the extension mostly meets inner pages. One same-site link per light or |
| native-dark-mode site (a second one for some), preferring content-like paths, plus a curated list of web apps. |
| """ |
| import argparse, glob, json, random, re |
| from urllib.parse import urlparse |
|
|
| CONTENT = re.compile(r"/(blog|news|article|articles|post|posts|docs?|documentation|help|support|guide|guides|learn|wiki|product|products|" |
| r"shop|store|category|categories|topic|topics|forum|community|about|pricing|features|faq|story|stories|events?|" |
| r"courses?|recipes?|reviews?|search|tag|tags|catalog|collections?|services?)(/|$)", re.I) |
|
|
|
|
| def score(u): |
| p = urlparse(u).path.rstrip("/") |
| segs = [s for s in p.split("/") if s] |
| s = 1.0 + (2.0 if CONTENT.search(p + "/") else 0) + (1.0 if len(segs) >= 2 else 0) + (0.5 if re.search(r"[-_][a-z]+[-_]", p) else 0) |
| if re.search(r"\.(php|aspx?)$", p) and len(segs) <= 1: |
| s -= 0.5 |
| if re.search(r"/(privacy|terms|legal|cookie|imprint|impressum|contact|careers?|jobs|sitemap)", p, re.I): |
| s -= 1.5 |
| return max(0.1, s) |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--index", required=True, help="glob of round-1 crawl index files") |
| ap.add_argument("--extra", default="", help="curated URL list to add") |
| ap.add_argument("--out", required=True) |
| ap.add_argument("--second", type=float, default=0.3) |
| args = ap.parse_args() |
| rnd = random.Random(7) |
| out, seen = [], set() |
| n = {"light": 0, "native": 0} |
| for f in sorted(glob.glob(args.index)): |
| for line in open(f): |
| try: |
| r = json.loads(line) |
| except Exception: |
| continue |
| kind = r.get("kind") |
| if kind not in ("light", "native_media", "native_class") or not r.get("links"): |
| continue |
| home = (r.get("final_url") or r["url"]).rstrip("/") |
| links = [l for l in r["links"] if l.rstrip("/") != home and l not in seen] |
| if not links: |
| continue |
| k = 2 if (kind == "light" and rnd.random() < args.second and len(links) > 1) else 1 |
| for _ in range(k): |
| w = [score(l) for l in links] |
| pick = rnd.choices(links, weights=w)[0] |
| links.remove(pick) |
| seen.add(pick) |
| out.append(pick) |
| n["light" if kind == "light" else "native"] += 1 |
| if not links: |
| break |
| if args.extra: |
| for u in open(args.extra): |
| u = u.strip() |
| if u and not u.startswith("#") and u not in seen: |
| seen.add(u) |
| out.append(u) |
| rnd.shuffle(out) |
| open(args.out, "w").write("\n".join(out) + "\n") |
| print(len(out), n) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
