pipeline: pass explicit --with deps to child jobs (--no-project suppressed PEP 723 and killed the merge in 3s)
Browse files- pipeline.py +11 -2
pipeline.py
CHANGED
|
@@ -43,6 +43,15 @@ RESULT = {"train": "NOT_RUN", "merge": "NOT_RUN", "gguf": "NOT_RUN",
|
|
| 43 |
"adapter": ADAPTER, "merged": MERGED, "gguf_repo": GGUF, "quant": QUANT}
|
| 44 |
|
| 45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
def log(msg):
|
| 47 |
print("[supervisor %s] %s" % (time.strftime("%H:%M:%S"), msg), flush=True)
|
| 48 |
|
|
@@ -175,7 +184,7 @@ def main():
|
|
| 175 |
log("adapter %s is live" % ADAPTER)
|
| 176 |
|
| 177 |
merge_job = launch(
|
| 178 |
-
["uv", "run"
|
| 179 |
HUB + "/" + NS + "/securecoder-scripts/resolve/main/merge_securecoder.py",
|
| 180 |
"--adapter", ADAPTER, "--output-repo", MERGED],
|
| 181 |
"a100-large", 3 * 3600, "securecoder-v2-merge")
|
|
@@ -189,7 +198,7 @@ def main():
|
|
| 189 |
log("merged repo %s is live" % MERGED)
|
| 190 |
|
| 191 |
gguf_job = launch(
|
| 192 |
-
["uv", "run"
|
| 193 |
HUB + "/" + NS + "/securecoder-scripts/resolve/main/quantise_securecoder.py",
|
| 194 |
"--merged-repo", MERGED, "--gguf-repo", GGUF, "--quant", QUANT],
|
| 195 |
"cpu-performance", 6 * 3600, "securecoder-v2-gguf")
|
|
|
|
| 43 |
"adapter": ADAPTER, "merged": MERGED, "gguf_repo": GGUF, "quant": QUANT}
|
| 44 |
|
| 45 |
|
| 46 |
+
# Explicit dependencies for the child jobs. `uv run --no-project` was the cause of a
|
| 47 |
+
# silent 3-second merge failure: --no-project suppresses PEP 723 inline metadata, so torch
|
| 48 |
+
# and friends were never installed. Keep in sync with the `# dependencies = [...]` header
|
| 49 |
+
# of merge_securecoder.py / quantise_securecoder.py.
|
| 50 |
+
DEPS = ["--with", "torch", "--with", "transformers", "--with", "peft",
|
| 51 |
+
"--with", "huggingface_hub", "--with", "accelerate",
|
| 52 |
+
"--with", "safetensors", "--with", "unsloth"]
|
| 53 |
+
|
| 54 |
+
|
| 55 |
def log(msg):
|
| 56 |
print("[supervisor %s] %s" % (time.strftime("%H:%M:%S"), msg), flush=True)
|
| 57 |
|
|
|
|
| 184 |
log("adapter %s is live" % ADAPTER)
|
| 185 |
|
| 186 |
merge_job = launch(
|
| 187 |
+
["uv", "run"] + DEPS + [
|
| 188 |
HUB + "/" + NS + "/securecoder-scripts/resolve/main/merge_securecoder.py",
|
| 189 |
"--adapter", ADAPTER, "--output-repo", MERGED],
|
| 190 |
"a100-large", 3 * 3600, "securecoder-v2-merge")
|
|
|
|
| 198 |
log("merged repo %s is live" % MERGED)
|
| 199 |
|
| 200 |
gguf_job = launch(
|
| 201 |
+
["uv", "run"] + DEPS + [
|
| 202 |
HUB + "/" + NS + "/securecoder-scripts/resolve/main/quantise_securecoder.py",
|
| 203 |
"--merged-repo", MERGED, "--gguf-repo", GGUF, "--quant", QUANT],
|
| 204 |
"cpu-performance", 6 * 3600, "securecoder-v2-gguf")
|