| | |
| | import argparse |
| | import os |
| | import random |
| | import sys |
| |
|
| |
|
| | if __name__ == "__main__": |
| | parser = argparse.ArgumentParser() |
| | parser.add_argument("paths", nargs="+", type=str, help="Input CoNLL-U files") |
| | parser.add_argument("--ratio", default=0.95, type=float, help="Train/dev ratio") |
| | parser.add_argument("--train", default=None, type=str, help="Train file to append to") |
| | parser.add_argument("--dev", default=None, type=str, help="Dev file to append to") |
| | parser.add_argument("--seed", default=42, type=int, help="Random seed") |
| | args = parser.parse_args() |
| |
|
| | train, dev = [], [] |
| |
|
| | for path in args.paths: |
| | print("Loading {}".format(path), file=sys.stderr) |
| |
|
| | with open(path, "r", encoding="utf-8") as input_file: |
| | blocks, block = [], [] |
| | header = "# newdoc id = {}\n".format(os.path.basename(path)) |
| | for line in input_file: |
| | if line.startswith("# variant "): continue |
| | block.append(line) |
| | if not line.rstrip("\n"): |
| | block = "".join(block) |
| | if "greek.expression" not in block: |
| | blocks.append(block) |
| | block = [] |
| | assert not block |
| |
|
| | random.Random(args.seed).shuffle(blocks) |
| | train.extend([header] + blocks[:int(args.ratio * len(blocks))]) |
| | if args.ratio < 1: |
| | dev.extend([header] + blocks[int(args.ratio * len(blocks)):]) |
| |
|
| | for path, blocks in [(args.train, train), (args.dev, dev)]: |
| | if not blocks: continue |
| |
|
| | with open(path, "a", encoding="utf-8") as output_file: |
| | print("".join(blocks), end="", file=output_file) |
| |
|
