make project runnable and endpoint-ready

run_pipeline.py

@@ -0,0 +1,123 @@
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+def run(cmd):
+    print("Running:", " ".join(cmd))
+    result = subprocess.run(cmd, check=False)
+    if result.returncode != 0:
+        raise SystemExit(result.returncode)
+
+
+def flag_present(flag_name):
+    return flag_name in sys.argv
+
+
+def usable_repo_id(repo_id):
+    if not repo_id:
+        return ""
+    placeholders = ("your-username/", "your-user/", "username/")
+    return "" if repo_id.startswith(placeholders) else repo_id
+
+
+def apply_config_defaults(args):
+    config_path = Path("training_config.json")
+    if not config_path.exists():
+        return args
+
+    with config_path.open("r", encoding="utf-8") as f:
+        cfg = json.load(f)
+
+    if not flag_present("--model-name"):
+        args.model_name = cfg.get("model_name", args.model_name)
+    if not flag_present("--dataset-size"):
+        args.dataset_size = cfg.get("dataset_size", args.dataset_size)
+    if not flag_present("--train-file"):
+        args.train_file = cfg.get("train_file", args.train_file)
+    if not flag_present("--output-dir"):
+        args.output_dir = cfg.get("output_dir", args.output_dir)
+    if not flag_present("--hf-repo"):
+        args.hf_repo = usable_repo_id(cfg.get("hf_repo_id", args.hf_repo))
+    if not flag_present("--epochs"):
+        args.epochs = cfg.get("epochs", args.epochs)
+    if not flag_present("--batch-size"):
+        args.batch_size = cfg.get("batch_size", args.batch_size)
+    if not flag_present("--learning-rate"):
+        args.learning_rate = cfg.get("learning_rate", args.learning_rate)
+    if not flag_present("--max-length"):
+        args.max_length = cfg.get("max_length", args.max_length)
+    if not flag_present("--max-train-samples"):
+        args.max_train_samples = cfg.get("max_train_samples", args.max_train_samples)
+    if not flag_present("--use-4bit"):
+        args.use_4bit = cfg.get("use_4bit", args.use_4bit)
+    return args
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset-size", type=int, default=8000)
+    parser.add_argument("--train-file", type=str, default="train.json")
+    parser.add_argument("--output-dir", type=str, default="model")
+    parser.add_argument("--model-name", type=str, default="Qwen/Qwen2.5-Coder-0.5B-Instruct")
+    parser.add_argument("--epochs", type=float, default=1)
+    parser.add_argument("--batch-size", type=int, default=2)
+    parser.add_argument("--learning-rate", type=float, default=2e-4)
+    parser.add_argument("--max-length", type=int, default=512)
+    parser.add_argument("--max-train-samples", type=int, default=0)
+    parser.add_argument("--use-4bit", action="store_true")
+    parser.add_argument("--hf-repo", type=str, default="")
+    parser.add_argument("--skip-generate", action="store_true")
+    parser.add_argument("--skip-train", action="store_true")
+    parser.add_argument("--skip-upload", action="store_true")
+    args = parser.parse_args()
+    args = apply_config_defaults(args)
+
+    if not (5000 <= args.dataset_size <= 10000):
+        raise ValueError("dataset-size must be between 5000 and 10000")
+
+    if not args.skip_generate:
+        run([sys.executable, "generate_dataset.py", "--size", str(args.dataset_size), "--out", args.train_file])
+
+    if not args.skip_train:
+        train_cmd = [
+            sys.executable,
+            "finetune_coding_llm_colab.py",
+            "--dataset-size",
+            str(args.dataset_size),
+            "--train-file",
+            args.train_file,
+            "--output-dir",
+            args.output_dir,
+            "--model-name",
+            args.model_name,
+            "--epochs",
+            str(args.epochs),
+            "--batch-size",
+            str(args.batch_size),
+            "--learning-rate",
+            str(args.learning_rate),
+            "--max-length",
+            str(args.max_length),
+            "--max-train-samples",
+            str(args.max_train_samples),
+            "--skip-dataset-gen",
+        ]
+        if args.use_4bit:
+            train_cmd.append("--use-4bit")
+        run(train_cmd)
+    else:
+        print("Skipping training stage (--skip-train).")
+
+    if not args.skip_upload:
+        if not args.hf_repo:
+            raise ValueError("Pass --hf-repo when upload is enabled, or use --skip-upload")
+        run([sys.executable, "upload_to_hf.py", "--model-dir", args.output_dir, "--repo-id", args.hf_repo])
+
+    print("Pipeline completed.")
+
+
+if __name__ == "__main__":
+    main()