#!/usr/bin/env python3
"""Round-2 crawl list: inner pages (articles, docs, product and app pages) of the sites crawled in round 1.

Round 1 crawled home pages only; the extension mostly meets inner pages. One same-site link per light or
native-dark-mode site (a second one for some), preferring content-like paths, plus a curated list of web apps.
"""
import argparse, glob, json, random, re
from urllib.parse import urlparse

CONTENT = re.compile(r"/(blog|news|article|articles|post|posts|docs?|documentation|help|support|guide|guides|learn|wiki|product|products|"
                     r"shop|store|category|categories|topic|topics|forum|community|about|pricing|features|faq|story|stories|events?|"
                     r"courses?|recipes?|reviews?|search|tag|tags|catalog|collections?|services?)(/|$)", re.I)


def score(u):
    p = urlparse(u).path.rstrip("/")
    segs = [s for s in p.split("/") if s]
    s = 1.0 + (2.0 if CONTENT.search(p + "/") else 0) + (1.0 if len(segs) >= 2 else 0) + (0.5 if re.search(r"[-_][a-z]+[-_]", p) else 0)
    if re.search(r"\.(php|aspx?)$", p) and len(segs) <= 1:
        s -= 0.5
    if re.search(r"/(privacy|terms|legal|cookie|imprint|impressum|contact|careers?|jobs|sitemap)", p, re.I):
        s -= 1.5
    return max(0.1, s)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--index", required=True, help="glob of round-1 crawl index files")
    ap.add_argument("--extra", default="", help="curated URL list to add")
    ap.add_argument("--out", required=True)
    ap.add_argument("--second", type=float, default=0.3)
    args = ap.parse_args()
    rnd = random.Random(7)
    out, seen = [], set()
    n = {"light": 0, "native": 0}
    for f in sorted(glob.glob(args.index)):
        for line in open(f):
            try:
                r = json.loads(line)
            except Exception:
                continue
            kind = r.get("kind")
            if kind not in ("light", "native_media", "native_class") or not r.get("links"):
                continue
            home = (r.get("final_url") or r["url"]).rstrip("/")
            links = [l for l in r["links"] if l.rstrip("/") != home and l not in seen]
            if not links:
                continue
            k = 2 if (kind == "light" and rnd.random() < args.second and len(links) > 1) else 1
            for _ in range(k):
                w = [score(l) for l in links]
                pick = rnd.choices(links, weights=w)[0]
                links.remove(pick)
                seen.add(pick)
                out.append(pick)
                n["light" if kind == "light" else "native"] += 1
                if not links:
                    break
    if args.extra:
        for u in open(args.extra):
            u = u.strip()
            if u and not u.startswith("#") and u not in seen:
                seen.add(u)
                out.append(u)
    rnd.shuffle(out)
    open(args.out, "w").write("\n".join(out) + "\n")
    print(len(out), n)


if __name__ == "__main__":
    main()