File size: 3,708 Bytes
295d147 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | import argparse
import json
from datetime import datetime, timezone
from pathlib import Path
DEFAULT_EXTENSIONS = {".cs", ".js", ".md", ".txt"}
SKIP_DIRS = {
".git",
"Library",
"Temp",
"Obj",
"Build",
"Builds",
"Logs",
"UserSettings",
}
def should_skip(path):
return path.suffix.lower() == ".meta" or any(part in SKIP_DIRS for part in path.parts)
def iter_files(root, extensions):
for path in sorted(root.rglob("*")):
if not path.is_file() or should_skip(path):
continue
if path.suffix.lower() in extensions:
yield path
def file_type_for(path):
name = path.name.lower()
suffix = path.suffix.lower()
if name.endswith(".build.cs") or name.endswith(".target.cs"):
return "unreal-build-csharp"
if suffix == ".cs":
return "csharp"
if suffix in {".cpp", ".cc", ".cxx"}:
return "cpp"
if suffix in {".h", ".hpp", ".hh"}:
return "cpp-header"
if suffix == ".js":
return "javascript"
return "notes"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--scripts-dir", required=True)
parser.add_argument("--out", default="data/clean/mmorpg_project_scripts.txt")
parser.add_argument("--metadata-out", default="data/metadata/mmorpg_project_scripts.jsonl")
parser.add_argument("--max-file-chars", type=int, default=12000)
parser.add_argument("--extension", action="append")
parser.add_argument("--domain-label", default="Unity MMORPG project")
parser.add_argument("--source-label", default="Scripts")
args = parser.parse_args()
root = Path(args.scripts_dir).resolve()
if not root.exists():
raise SystemExit(f"Scripts directory not found: {root}")
extensions = {ext.lower() if ext.startswith(".") else f".{ext.lower()}" for ext in args.extension or []}
if not extensions:
extensions = DEFAULT_EXTENSIONS
documents = []
included = 0
skipped_empty = 0
total_chars = 0
for path in iter_files(root, extensions):
rel = path.relative_to(root).as_posix()
text = path.read_text(encoding="utf-8", errors="ignore").strip()
if len(text) < 20:
skipped_empty += 1
continue
if len(text) > args.max_file_chars:
text = text[: args.max_file_chars].rsplit("\n", 1)[0].strip()
documents.append(
f"Domain: {args.domain_label}\n"
f"Source: {args.source_label}/{rel}\n"
f"Type: {file_type_for(path)}\n\n"
f"{text}\n"
)
included += 1
total_chars += len(text)
out_path = Path(args.out)
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text("\n\n<|eos|>\n\n".join(documents) + "\n", encoding="utf-8")
metadata = {
"source": "local Unity MMORPG project scripts",
"scripts_dir": str(root),
"domain_label": args.domain_label,
"source_label": args.source_label,
"output": str(out_path),
"extensions": sorted(extensions),
"included_files": included,
"skipped_empty_files": skipped_empty,
"total_chars": total_chars,
"prepared_at": datetime.now(timezone.utc).isoformat(),
}
metadata_path = Path(args.metadata_out)
metadata_path.parent.mkdir(parents=True, exist_ok=True)
metadata_path.write_text(json.dumps(metadata, ensure_ascii=False) + "\n", encoding="utf-8")
print(f"included files: {included}")
print(f"skipped empty: {skipped_empty}")
print(f"wrote {out_path} ({out_path.stat().st_size / 1024 / 1024:.2f} MiB)")
print(f"wrote {metadata_path}")
if __name__ == "__main__":
main()
|