"""Build the synthetic bilingual training corpus for the humanize model. Methodology ----------- Supervision comes from the conservative rule engine in ``src/rules.py`` (derived from the lynote-ai reference projects). We generate: 1. **AI -> human**: formulaic clause combinations rewritten by the rules, 2. **Human -> human (identity)**: clean prose unchanged, so the model learns not to rewrite already-human text, 3. **Mixed**: clean prose with one injected AI cliché that must be removed, 4. **Protected spans**: examples containing numbers, URLs and quoted text. Every input maps to exactly one deterministic target (one-to-one), which keeps the training signal consistent. Outputs are seeded JSONL files with fields ``input_text``, ``output_text``, ``lang``. """ import argparse import json import random import re import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from src.rules import EN_PHRASES, ZH_PHRASES, humanize_rules, protect # -------------------------------------------------------------------------- # Seed prose (curated, human-flavored). # -------------------------------------------------------------------------- EN_SEEDS = [ "I walked the dog this morning and the sky was grey and low.", "Halfway through the park it started raining, which I hadn't expected, so we ran home.", "The dog seemed to enjoy it more than me.", "When we got back I made coffee and watched the rain run down the window.", "Thanks for sending the report over, the numbers look good.", "Could you double-check the retention section before we share it?", "Let's grab coffee this week and talk through next steps.", "The conference was exhausting, but a couple of the sessions were genuinely useful.", "I came home with a notebook full of scribbles and a dozen open browser tabs.", "The sky cleared by evening and we went for a long walk.", "I've been testing the new tool for a few days and it mostly works.", "The docs are clear enough, though the install steps could be shorter.", "We tried the new approach on a small project and it held up fine.", "My laptop died in the middle of the meeting, which was awkward.", "She asked me to review her draft and I found a few typos.", "The traffic was bad this morning, so I took the subway instead.", "We spent the afternoon fixing bugs and ordering pizza.", "I think the plan works, but we should test it with real users first.", "The garden looks great after the rain.", "He replied faster than I expected, which was a nice surprise.", "The new hire seems sharp and asks good questions.", "I read the whole chapter on the train and highlighted half of it.", "We moved the deadline to Friday so nobody has to work the weekend.", "The cafe near the office makes a decent flat white.", "My phone died, so I missed your call, sorry about that.", "She asked me to review her draft and I found a few typos. It was a nice read.", "I think the plan works, but we should test it with real users first. We can start next week.", ] ZH_SEEDS = [ "今天早上我带狗去公园，出门时天阴阴的。", "走到一半忽然下起雨，我们只好跑回家，狗倒是挺高兴。", "回来以后我煮了杯咖啡，坐在窗边看了会儿雨。", "你要的资料我发你邮箱了，附件里有两个版本。", "你看一下哪个合适，有问题随时叫我。", "下午我跟老王聊了聊，他觉得可以，但预算要再压一压。", "今天加班到九点，回来路上吃了碗牛肉面。", "老板认得我，多给了几片牛肉。", "猫蹲在门口等我，尾巴翘得老高。", "周末去了趟公园，人特别多，树荫下倒是凉快。", "家里的路由器坏了，折腾了一晚上也没修好。", "这个月房租涨了两百块，心里有点不爽。", "我在楼下便利店买了瓶水，顺便取了个快递。", "下周开会的内容我还没准备完，今晚得加个班。", "邻居家的狗最近晚上老叫，吵得人睡不着。", "上次说的那个方案，客户反馈还不错，就是工期太紧。", "我今天试着做了道红烧肉，味道还行，就是有点咸。", "地铁口新开了一家奶茶店，排队的人特别多。", "天气预报说明天降温，出门记得多穿点。", "我把旧手机的照片都备份到电脑上了。", "新来的同事挺靠谱的，交办的事情都能按时完成。", "超市打折的时候我囤了不少纸巾和洗衣液。", "这个项目其实不难，主要是沟通起来比较费劲。", "楼下的早餐店换了老板，油条没有以前好吃了。", "我在网上买了个书架，装了一个小时才装好。", "妈妈打电话来说家里的桂花开了，让我回去看看。", "这个周末打算去爬山，好久没运动了。", "我把会议纪要整理好发到群里了，大家记得看。", "打印机又卡纸了，修了半小时才弄好。", "晚上的风挺大的，窗户被吹得呼呼响。", "上个月的项目结项了，团队一起吃了顿饭庆祝。", "小区里的银杏树黄了，特别好看。", "我最近在学做菜，最拿手的是西红柿炒蛋。", "公交车晚点了二十分钟，差点迟到。", "这个月业绩不错，老板说要给大家发奖金。", "我把书还回图书馆了，顺便借了两本新的。", "朋友推荐了一家火锅店，说是味道特别好。", "早上起晚了，来不及吃早饭，现在有点饿。", "新买的耳机用着不错，降噪效果明显。", "我把房间收拾了一遍，看着舒服多了。", ] # -------------------------------------------------------------------------- # Formulaic clause pools. Combinations of these produce many unique inputs; # the rule engine maps each one to a deterministic human-flavored target. # -------------------------------------------------------------------------- EN_SUBJECTS = [ "artificial intelligence", "digital transformation", "cloud computing", "data analytics", "the new platform", "our organization", "this initiative", "remote work", "machine learning", "automation", "the latest release", "the training program", "our partnership", "the revised policy", ] EN_CLAUSES = [ "it is important to note that {s} plays a crucial role in business success", "it is worth noting that leveraging {s} serves as a testament to our commitment to innovation", "our comprehensive approach to {s} fosters collaboration and drives growth", "the seamless integration of {s} enhances efficiency and delivers a robust solution", "embracing {s} is no longer optional but essential for long-term competitiveness", "{s} requires high-quality data to function effectively", "our groundbreaking platform leverages cutting-edge technology to provide a seamless experience", "the state-of-the-art solution plays a crucial role in improving operational efficiency", "it is essential to delve into {s} in order to remain competitive", "fostering a holistic approach to {s} is paramount", "{s} transforms the way we work in a significant way", "this game-changing innovation will revolutionize the industry", ] EN_CONNECTORS = ["Moreover,", "Furthermore,", "Additionally,", "In conclusion,"] ZH_SUBJECTS = [ "数字化转型", "人工智能", "这套方案", "新的平台", "我们的团队", "在线办公", "自动化流程", "数据治理", "这次改革", "人才培养", ] ZH_CLAUSES = [ "值得注意的是，{s}发挥着重要作用", "在当今快速发展的时代，{s}是企业发展的重要保障", "综上所述，我们通过赋能团队来助力企业实现降本增效", "这一方案形成了完整的业务闭环，无缝连接各个环节", "由此可见，{s}已经成为企业发展的必然趋势", "总而言之，企业应当积极拥抱新技术，从而在激烈的竞争中保持优势", "{s}对于企业来说至关重要", "我们需要不断提升效率，持续优化流程", "这一举措实现了闭环管理，进一步提升了整体效率", "数字化转型为{s}提供了新的发展机遇", "相关部门应该加快制定实施细则，确保各项措施落地见效", "这一政策有助于优化资源配置，推动产业转型升级", "我们要坚持以用户为中心，不断完善产品服务体系", "项目团队克服了诸多困难，最终按期完成了交付任务", "该平台整合了线上线下资源，覆盖了多个业务场景", "通过数据驱动的方式，我们能够更好地洞察市场变化", "本次会议明确了下一阶段的工作重点和发展方向", "这套体系有效降低了运营成本，提高了响应速度", "从长远来看，{s}将深刻改变行业的竞争格局", "各方应加强沟通协作，共同推进相关工作的开展", ] PROTECTED_SEEDS = [ "The license key is ABC-12345 and the server is at https://api.lynote.ai/v1/detect.", "The report shows a 23.5% increase in Q3, up from 18.2% last year.", "Run `pip install humanize-text==1.4.2` and check /usr/local/bin/humanize.", "The budget is $12,400 for this quarter, as noted in \"Project Lynote\".", "The API accepts 500 requests per minute at https://hub.example.com and returns a 99.9% success rate.", "The server rebooted at 03:47 and restored 1.2 TB of data within 10 minutes.", "版本号是 v2.1.3，接口地址是 https://api.lynote.ai，成本下降了 15%。", "配置文件放在 /etc/lynote/config.toml，端口是 8000，超时 30 秒。", "上个月的收入是 85000 元，比预期多了 12%，主要来自 https://lynote.ai。", "服务器在 03:47 重启，恢复了 1.2 TB 的数据，耗时 10 分钟。", ] def _ai_en(rng: random.Random, n: int) -> list[str]: out = [] subjects = EN_SUBJECTS while len(out) < n: s = rng.choice(subjects) k = rng.randint(2, 4) clauses = [c.format(s=s) for c in rng.sample(EN_CLAUSES, k)] clauses[0] = clauses[0][0].upper() + clauses[0][1:] text = clauses[0] for c in clauses[1:]: text += ". " + rng.choice(EN_CONNECTORS) + " " + c text += "." if text not in out: out.append(text) return out def _ai_zh(rng: random.Random, n: int) -> list[str]: out = [] while len(out) < n: s = rng.choice(ZH_SUBJECTS) k = rng.randint(2, 4) clauses = [c.format(s=s) for c in rng.sample(ZH_CLAUSES, k)] text = "，".join(clauses[:-1]) + "，" + clauses[-1] + "。" if k > 1 else clauses[0] + "。" if text not in out: out.append(text) return out def _inject_cliche_en(seed: str) -> str: phrase = random.choice(list(EN_PHRASES.keys())) seed = seed.rstrip(".") return f"{phrase}, {seed}." def _inject_cliche_zh(seed: str) -> str: phrase = random.choice(list(ZH_PHRASES.keys())) return f"{phrase}，{seed}" def build( seed: int = 42, n_en_ai: int = 8_000, n_zh_ai: int = 6_000, n_mixed: int = 1_000, n_protected: int = 300, ) -> list[dict]: rng = random.Random(seed) pairs: list[dict] = [] # 1. AI -> human. for lang, ai_texts, _seeds in ( ("en", _ai_en(rng, n_en_ai), EN_SEEDS), ("zh", _ai_zh(rng, n_zh_ai), ZH_SEEDS), ): for ai in ai_texts: human = humanize_rules(ai, lang) if human == ai: continue pairs.append({"input_text": ai, "output_text": human, "lang": lang}) # 2. Identity pairs. for lang, seeds in (("en", EN_SEEDS), ("zh", ZH_SEEDS)): for s in seeds: pairs.append({"input_text": s, "output_text": s, "lang": lang}) # 3. Mixed: clean prose + injected cliche -> cleaned prose. for _ in range(n_mixed): lang = rng.choice(["en", "zh"]) seed = rng.choice(EN_SEEDS if lang == "en" else ZH_SEEDS) noisy = _inject_cliche_en(seed) if lang == "en" else _inject_cliche_zh(seed) cleaned = humanize_rules(noisy, lang) if cleaned != noisy: pairs.append({"input_text": noisy, "output_text": cleaned, "lang": lang}) # 3b. Extra Chinese mixed pairs (identity/copy grounding). for _ in range(n_mixed): seed = rng.choice(ZH_SEEDS) noisy = _inject_cliche_zh(seed) cleaned = humanize_rules(noisy, "zh") if cleaned != noisy: pairs.append({"input_text": noisy, "output_text": cleaned, "lang": "zh"}) # 4. Protected spans (contextualized so dedup keeps variety). en_prefixes = [ "It is important to note that ", "It is worth noting that ", "Moreover, ", "Additionally, ", "In conclusion, ", "", ] en_suffixes = [ " This is a robust and seamless solution.", " Overall, this approach is a game-changer.", " In conclusion, this matters for our work.", " Furthermore, this initiative drives growth.", "", ] zh_prefixes = ["值得注意的是，", "由此可见，", "综上所述，", "在当今快速发展的时代，", ""] zh_suffixes = ["这一方案发挥了重要作用。", "这就是完整的业务闭环。", "综上所述，我们需要持续优化。", ""] protected_srcs = [] for seed in PROTECTED_SEEDS: if re.search(r"[\u4e00-\u9fff]", seed): protected_srcs.extend( f"{p}{seed}{s}" for p in zh_prefixes for s in zh_suffixes ) else: protected_srcs.extend( f"{p}{seed}{s}" for p in en_prefixes for s in en_suffixes ) for src in protected_srcs: lang = "zh" if re.search(r"[\u4e00-\u9fff]", src) else "en" cleaned = humanize_rules(src, lang) if cleaned != src: # Raw pair (URLs/numbers visible). pairs.append({"input_text": src, "output_text": cleaned, "lang": lang}) # Masked pair: identical placeholder spans on both sides, matching # inference-time protection so the model learns to copy them. masked_src, _ = protect(src, lang) masked_cleaned, _ = protect(cleaned, lang) pairs.append( {"input_text": masked_src, "output_text": masked_cleaned, "lang": lang} ) seen, unique = set(), [] for p in pairs: if p["input_text"] in seen: continue seen.add(p["input_text"]) unique.append(p) return unique def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--out", default="data", help="output directory") parser.add_argument("--seed", type=int, default=42) parser.add_argument("--val-size", type=int, default=500) parser.add_argument("--test-size", type=int, default=1_000) args = parser.parse_args() out = Path(args.out) out.mkdir(parents=True, exist_ok=True) pairs = build(seed=args.seed) rng = random.Random(args.seed) rng.shuffle(pairs) test = pairs[: args.test_size] val = pairs[args.test_size : args.test_size + args.val_size] train = pairs[args.test_size + args.val_size :] for split, items in (("train", train), ("val", val), ("test", test)): with open(out / f"{split}.jsonl", "w", encoding="utf-8") as f: for p in items: f.write(json.dumps(p, ensure_ascii=False) + "\n") print(f"{split}: {len(items)} pairs") print("test langs:", sorted({p['lang'] for p in test})) if __name__ == "__main__": main()