| """Build the synthetic bilingual training corpus for the humanize model. |
| |
| Methodology |
| ----------- |
| Supervision comes from the conservative rule engine in ``src/rules.py`` |
| (derived from the lynote-ai reference projects). We generate: |
| |
| 1. **AI -> human**: formulaic clause combinations rewritten by the rules, |
| 2. **Human -> human (identity)**: clean prose unchanged, so the model learns |
| not to rewrite already-human text, |
| 3. **Mixed**: clean prose with one injected AI cliché that must be removed, |
| 4. **Protected spans**: examples containing numbers, URLs and quoted text. |
| |
| Every input maps to exactly one deterministic target (one-to-one), which |
| keeps the training signal consistent. Outputs are seeded JSONL files with |
| fields ``input_text``, ``output_text``, ``lang``. |
| """ |
|
|
| import argparse |
| import json |
| import random |
| import re |
| import sys |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) |
| from src.rules import EN_PHRASES, ZH_PHRASES, humanize_rules, protect |
|
|
| |
| |
| |
|
|
| EN_SEEDS = [ |
| "I walked the dog this morning and the sky was grey and low.", |
| "Halfway through the park it started raining, which I hadn't expected, so we ran home.", |
| "The dog seemed to enjoy it more than me.", |
| "When we got back I made coffee and watched the rain run down the window.", |
| "Thanks for sending the report over, the numbers look good.", |
| "Could you double-check the retention section before we share it?", |
| "Let's grab coffee this week and talk through next steps.", |
| "The conference was exhausting, but a couple of the sessions were genuinely useful.", |
| "I came home with a notebook full of scribbles and a dozen open browser tabs.", |
| "The sky cleared by evening and we went for a long walk.", |
| "I've been testing the new tool for a few days and it mostly works.", |
| "The docs are clear enough, though the install steps could be shorter.", |
| "We tried the new approach on a small project and it held up fine.", |
| "My laptop died in the middle of the meeting, which was awkward.", |
| "She asked me to review her draft and I found a few typos.", |
| "The traffic was bad this morning, so I took the subway instead.", |
| "We spent the afternoon fixing bugs and ordering pizza.", |
| "I think the plan works, but we should test it with real users first.", |
| "The garden looks great after the rain.", |
| "He replied faster than I expected, which was a nice surprise.", |
| "The new hire seems sharp and asks good questions.", |
| "I read the whole chapter on the train and highlighted half of it.", |
| "We moved the deadline to Friday so nobody has to work the weekend.", |
| "The cafe near the office makes a decent flat white.", |
| "My phone died, so I missed your call, sorry about that.", |
| "She asked me to review her draft and I found a few typos. It was a nice read.", |
| "I think the plan works, but we should test it with real users first. We can start next week.", |
| ] |
|
|
| ZH_SEEDS = [ |
| "今天早上我带狗去公园，出门时天阴阴的。", |
| "走到一半忽然下起雨，我们只好跑回家，狗倒是挺高兴。", |
| "回来以后我煮了杯咖啡，坐在窗边看了会儿雨。", |
| "你要的资料我发你邮箱了，附件里有两个版本。", |
| "你看一下哪个合适，有问题随时叫我。", |
| "下午我跟老王聊了聊，他觉得可以，但预算要再压一压。", |
| "今天加班到九点，回来路上吃了碗牛肉面。", |
| "老板认得我，多给了几片牛肉。", |
| "猫蹲在门口等我，尾巴翘得老高。", |
| "周末去了趟公园，人特别多，树荫下倒是凉快。", |
| "家里的路由器坏了，折腾了一晚上也没修好。", |
| "这个月房租涨了两百块，心里有点不爽。", |
| "我在楼下便利店买了瓶水，顺便取了个快递。", |
| "下周开会的内容我还没准备完，今晚得加个班。", |
| "邻居家的狗最近晚上老叫，吵得人睡不着。", |
| "上次说的那个方案，客户反馈还不错，就是工期太紧。", |
| "我今天试着做了道红烧肉，味道还行，就是有点咸。", |
| "地铁口新开了一家奶茶店，排队的人特别多。", |
| "天气预报说明天降温，出门记得多穿点。", |
| "我把旧手机的照片都备份到电脑上了。", |
| "新来的同事挺靠谱的，交办的事情都能按时完成。", |
| "超市打折的时候我囤了不少纸巾和洗衣液。", |
| "这个项目其实不难，主要是沟通起来比较费劲。", |
| "楼下的早餐店换了老板，油条没有以前好吃了。", |
| "我在网上买了个书架，装了一个小时才装好。", |
| "妈妈打电话来说家里的桂花开了，让我回去看看。", |
| "这个周末打算去爬山，好久没运动了。", |
| "我把会议纪要整理好发到群里了，大家记得看。", |
| "打印机又卡纸了，修了半小时才弄好。", |
| "晚上的风挺大的，窗户被吹得呼呼响。", |
| "上个月的项目结项了，团队一起吃了顿饭庆祝。", |
| "小区里的银杏树黄了，特别好看。", |
| "我最近在学做菜，最拿手的是西红柿炒蛋。", |
| "公交车晚点了二十分钟，差点迟到。", |
| "这个月业绩不错，老板说要给大家发奖金。", |
| "我把书还回图书馆了，顺便借了两本新的。", |
| "朋友推荐了一家火锅店，说是味道特别好。", |
| "早上起晚了，来不及吃早饭，现在有点饿。", |
| "新买的耳机用着不错，降噪效果明显。", |
| "我把房间收拾了一遍，看着舒服多了。", |
| ] |
|
|
| |
| |
| |
| |
|
|
| EN_SUBJECTS = [ |
| "artificial intelligence", "digital transformation", "cloud computing", |
| "data analytics", "the new platform", "our organization", "this initiative", |
| "remote work", "machine learning", "automation", "the latest release", |
| "the training program", "our partnership", "the revised policy", |
| ] |
|
|
| EN_CLAUSES = [ |
| "it is important to note that {s} plays a crucial role in business success", |
| "it is worth noting that leveraging {s} serves as a testament to our commitment to innovation", |
| "our comprehensive approach to {s} fosters collaboration and drives growth", |
| "the seamless integration of {s} enhances efficiency and delivers a robust solution", |
| "embracing {s} is no longer optional but essential for long-term competitiveness", |
| "{s} requires high-quality data to function effectively", |
| "our groundbreaking platform leverages cutting-edge technology to provide a seamless experience", |
| "the state-of-the-art solution plays a crucial role in improving operational efficiency", |
| "it is essential to delve into {s} in order to remain competitive", |
| "fostering a holistic approach to {s} is paramount", |
| "{s} transforms the way we work in a significant way", |
| "this game-changing innovation will revolutionize the industry", |
| ] |
|
|
| EN_CONNECTORS = ["Moreover,", "Furthermore,", "Additionally,", "In conclusion,"] |
|
|
| ZH_SUBJECTS = [ |
| "数字化转型", "人工智能", "这套方案", "新的平台", "我们的团队", |
| "在线办公", "自动化流程", "数据治理", "这次改革", "人才培养", |
| ] |
|
|
| ZH_CLAUSES = [ |
| "值得注意的是，{s}发挥着重要作用", |
| "在当今快速发展的时代，{s}是企业发展的重要保障", |
| "综上所述，我们通过赋能团队来助力企业实现降本增效", |
| "这一方案形成了完整的业务闭环，无缝连接各个环节", |
| "由此可见，{s}已经成为企业发展的必然趋势", |
| "总而言之，企业应当积极拥抱新技术，从而在激烈的竞争中保持优势", |
| "{s}对于企业来说至关重要", |
| "我们需要不断提升效率，持续优化流程", |
| "这一举措实现了闭环管理，进一步提升了整体效率", |
| "数字化转型为{s}提供了新的发展机遇", |
| "相关部门应该加快制定实施细则，确保各项措施落地见效", |
| "这一政策有助于优化资源配置，推动产业转型升级", |
| "我们要坚持以用户为中心，不断完善产品服务体系", |
| "项目团队克服了诸多困难，最终按期完成了交付任务", |
| "该平台整合了线上线下资源，覆盖了多个业务场景", |
| "通过数据驱动的方式，我们能够更好地洞察市场变化", |
| "本次会议明确了下一阶段的工作重点和发展方向", |
| "这套体系有效降低了运营成本，提高了响应速度", |
| "从长远来看，{s}将深刻改变行业的竞争格局", |
| "各方应加强沟通协作，共同推进相关工作的开展", |
| ] |
|
|
| PROTECTED_SEEDS = [ |
| "The license key is ABC-12345 and the server is at https://api.lynote.ai/v1/detect.", |
| "The report shows a 23.5% increase in Q3, up from 18.2% last year.", |
| "Run `pip install humanize-text==1.4.2` and check /usr/local/bin/humanize.", |
| "The budget is $12,400 for this quarter, as noted in \"Project Lynote\".", |
| "The API accepts 500 requests per minute at https://hub.example.com and returns a 99.9% success rate.", |
| "The server rebooted at 03:47 and restored 1.2 TB of data within 10 minutes.", |
| "版本号是 v2.1.3，接口地址是 https://api.lynote.ai，成本下降了 15%。", |
| "配置文件放在 /etc/lynote/config.toml，端口是 8000，超时 30 秒。", |
| "上个月的收入是 85000 元，比预期多了 12%，主要来自 https://lynote.ai。", |
| "服务器在 03:47 重启，恢复了 1.2 TB 的数据，耗时 10 分钟。", |
| ] |
|
|
|
|
| def _ai_en(rng: random.Random, n: int) -> list[str]: |
| out = [] |
| subjects = EN_SUBJECTS |
| while len(out) < n: |
| s = rng.choice(subjects) |
| k = rng.randint(2, 4) |
| clauses = [c.format(s=s) for c in rng.sample(EN_CLAUSES, k)] |
| clauses[0] = clauses[0][0].upper() + clauses[0][1:] |
| text = clauses[0] |
| for c in clauses[1:]: |
| text += ". " + rng.choice(EN_CONNECTORS) + " " + c |
| text += "." |
| if text not in out: |
| out.append(text) |
| return out |
|
|
|
|
| def _ai_zh(rng: random.Random, n: int) -> list[str]: |
| out = [] |
| while len(out) < n: |
| s = rng.choice(ZH_SUBJECTS) |
| k = rng.randint(2, 4) |
| clauses = [c.format(s=s) for c in rng.sample(ZH_CLAUSES, k)] |
| text = "，".join(clauses[:-1]) + "，" + clauses[-1] + "。" if k > 1 else clauses[0] + "。" |
| if text not in out: |
| out.append(text) |
| return out |
|
|
|
|
| def _inject_cliche_en(seed: str) -> str: |
| phrase = random.choice(list(EN_PHRASES.keys())) |
| seed = seed.rstrip(".") |
| return f"{phrase}, {seed}." |
|
|
|
|
| def _inject_cliche_zh(seed: str) -> str: |
| phrase = random.choice(list(ZH_PHRASES.keys())) |
| return f"{phrase}，{seed}" |
|
|
|
|
| def build( |
| seed: int = 42, |
| n_en_ai: int = 8_000, |
| n_zh_ai: int = 6_000, |
| n_mixed: int = 1_000, |
| n_protected: int = 300, |
| ) -> list[dict]: |
| rng = random.Random(seed) |
| pairs: list[dict] = [] |
|
|
| |
| for lang, ai_texts, _seeds in ( |
| ("en", _ai_en(rng, n_en_ai), EN_SEEDS), |
| ("zh", _ai_zh(rng, n_zh_ai), ZH_SEEDS), |
| ): |
| for ai in ai_texts: |
| human = humanize_rules(ai, lang) |
| if human == ai: |
| continue |
| pairs.append({"input_text": ai, "output_text": human, "lang": lang}) |
|
|
| |
| for lang, seeds in (("en", EN_SEEDS), ("zh", ZH_SEEDS)): |
| for s in seeds: |
| pairs.append({"input_text": s, "output_text": s, "lang": lang}) |
|
|
| |
| for _ in range(n_mixed): |
| lang = rng.choice(["en", "zh"]) |
| seed = rng.choice(EN_SEEDS if lang == "en" else ZH_SEEDS) |
| noisy = _inject_cliche_en(seed) if lang == "en" else _inject_cliche_zh(seed) |
| cleaned = humanize_rules(noisy, lang) |
| if cleaned != noisy: |
| pairs.append({"input_text": noisy, "output_text": cleaned, "lang": lang}) |
|
|
| |
| for _ in range(n_mixed): |
| seed = rng.choice(ZH_SEEDS) |
| noisy = _inject_cliche_zh(seed) |
| cleaned = humanize_rules(noisy, "zh") |
| if cleaned != noisy: |
| pairs.append({"input_text": noisy, "output_text": cleaned, "lang": "zh"}) |
|
|
| |
| en_prefixes = [ |
| "It is important to note that ", "It is worth noting that ", |
| "Moreover, ", "Additionally, ", "In conclusion, ", "", |
| ] |
| en_suffixes = [ |
| " This is a robust and seamless solution.", |
| " Overall, this approach is a game-changer.", |
| " In conclusion, this matters for our work.", |
| " Furthermore, this initiative drives growth.", |
| "", |
| ] |
| zh_prefixes = ["值得注意的是，", "由此可见，", "综上所述，", "在当今快速发展的时代，", ""] |
| zh_suffixes = ["这一方案发挥了重要作用。", "这就是完整的业务闭环。", "综上所述，我们需要持续优化。", ""] |
| protected_srcs = [] |
| for seed in PROTECTED_SEEDS: |
| if re.search(r"[\u4e00-\u9fff]", seed): |
| protected_srcs.extend( |
| f"{p}{seed}{s}" for p in zh_prefixes for s in zh_suffixes |
| ) |
| else: |
| protected_srcs.extend( |
| f"{p}{seed}{s}" for p in en_prefixes for s in en_suffixes |
| ) |
| for src in protected_srcs: |
| lang = "zh" if re.search(r"[\u4e00-\u9fff]", src) else "en" |
| cleaned = humanize_rules(src, lang) |
| if cleaned != src: |
| |
| pairs.append({"input_text": src, "output_text": cleaned, "lang": lang}) |
| |
| |
| masked_src, _ = protect(src, lang) |
| masked_cleaned, _ = protect(cleaned, lang) |
| pairs.append( |
| {"input_text": masked_src, "output_text": masked_cleaned, "lang": lang} |
| ) |
|
|
| seen, unique = set(), [] |
| for p in pairs: |
| if p["input_text"] in seen: |
| continue |
| seen.add(p["input_text"]) |
| unique.append(p) |
| return unique |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--out", default="data", help="output directory") |
| parser.add_argument("--seed", type=int, default=42) |
| parser.add_argument("--val-size", type=int, default=500) |
| parser.add_argument("--test-size", type=int, default=1_000) |
| args = parser.parse_args() |
|
|
| out = Path(args.out) |
| out.mkdir(parents=True, exist_ok=True) |
| pairs = build(seed=args.seed) |
| rng = random.Random(args.seed) |
| rng.shuffle(pairs) |
|
|
| test = pairs[: args.test_size] |
| val = pairs[args.test_size : args.test_size + args.val_size] |
| train = pairs[args.test_size + args.val_size :] |
|
|
| for split, items in (("train", train), ("val", val), ("test", test)): |
| with open(out / f"{split}.jsonl", "w", encoding="utf-8") as f: |
| for p in items: |
| f.write(json.dumps(p, ensure_ascii=False) + "\n") |
| print(f"{split}: {len(items)} pairs") |
|
|
| print("test langs:", sorted({p['lang'] for p in test})) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
