Spaces:
Running
Running
| """One-off: ask the model for subdomains of each top-level domain and write static/domains.json.""" | |
| import json, os, re, sys | |
| sys.path.insert(0, "app") | |
| from concurrent.futures import ThreadPoolExecutor | |
| from engine import Endpoint | |
| DOMAINS = ["Academic education", "Agriculture", "Animal and nature", "Architecture and design", "Art", "Career and workplace", "Code", "Cooking", | |
| "Corporate and business", "Culture", "Data analysis and visualization", "Dating", "Entertainment", "Fashion and beauty", "Fitness and sports", | |
| "Games", "Geography", "Governance", "History", "How-to", "HR", "Language", "Legal", "Market analysis", "Marketing", "Math", "Medical", "Music", | |
| "News", "Parenting and family", "Personal finance", "Personal growth", "Product advice", "Religion", "Roleplay", "Science", "Social media", | |
| "Sports", "Technology", "Transportation", "Travel", "Writing and editing"] | |
| ep = Endpoint(api_key=open(os.path.expanduser("~/.cache/huggingface/token")).read().strip()) | |
| def one(d): | |
| for _ in range(3): | |
| t = ep.chat(f'List 12 distinct subdomains of the domain "{d}" that a training dataset could cover. Return ONLY a JSON array of short names (2 to 4 words each), no commentary.', | |
| max_tokens=400, temperature=0.4) | |
| m = re.search(r"\[.*\]", t, re.S) | |
| try: | |
| arr = [s.strip() for s in json.loads(m.group(0)) if isinstance(s, str) and 2 < len(s) < 40] | |
| if len(arr) >= 8: | |
| return d, arr[:12] | |
| except Exception: | |
| pass | |
| return d, [] | |
| with ThreadPoolExecutor(6) as ex: | |
| res = dict(ex.map(one, DOMAINS)) | |
| bad = [d for d, v in res.items() if not v] | |
| json.dump([{"name": d, "subdomains": res[d]} for d in DOMAINS], open("static/domains.json", "w"), indent=1, ensure_ascii=False) | |
| print(len(DOMAINS), "domains;", "failed:", bad); print(res["Medical"][:6], res["Code"][:6]) | |