Update TabArena EC2 runner scripts (ec2_run.sh)

benchmark/tabarena/ec2_run.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# FelaTab x TabArena full benchmark driver for a plain Linux box (EC2 c7i.8xlarge, Ubuntu 24.04).
+#
+# Replaces the unreliable HF Jobs path (hf_job.sh). Runs setup once, then one or more
+# benchmark phases (e.g. lite shakedown -> full) sharing a single results cache, so the
+# full run resumes on top of the lite shakedown instead of recomputing split 0.
+#
+# Env (all set by launch_ec2.py user-data):
+#   HF_TOKEN           write token for lowdown-labs (artifact upload)            [required]
+#   PHASES             comma list: any of lite,full        [default: lite,full]
+#   RUN_NAME           results cache + eval dir name       [default: felatab_ec2]
+#   DATASETS           optional csv shard of dataset names (full mode only)
+#   WITH_LGBM          1 = also run the LightGBM cross-check [default: 0]
+#   SELF_TERMINATE     1 = poweroff when done (instance shutdown behavior = terminate) [default: 1]
+set -euxo pipefail
+
+export HOME="${HOME:-/root}"
+WORK=/work
+PHASES="${PHASES:-lite,full}"
+RUN_NAME="${RUN_NAME:-felatab_ec2}"
+DATASETS="${DATASETS:-}"
+WITH_LGBM="${WITH_LGBM:-0}"
+SELF_TERMINATE="${SELF_TERMINATE:-1}"
+RESULTS_REPO="lowdown-labs/fela-tab-tabarena-results"
+
+export PATH="$HOME/.local/bin:$PATH"
+command -v uv >/dev/null || { curl -LsSf https://astral.sh/uv/install.sh | sh; export PATH="$HOME/.local/bin:$PATH"; }
+
+mkdir -p "$WORK/tmp" "$WORK/logs" && cd "$WORK"
+
+upload() {  # upload <phase-label>
+    "$WORK/bootvenv/bin/python" - <<EOF
+from huggingface_hub import HfApi
+api = HfApi()
+api.create_repo("$RESULTS_REPO", repo_type="dataset", exist_ok=True)
+for src, dst in [
+    ("$WORK/logs/$1.log", "logs/${RUN_NAME}_$1.log"),
+    ("$WORK/fela-tab/benchmark/tabarena/eval/$RUN_NAME", "eval/$RUN_NAME"),
+    ("$WORK/fela-tab/benchmark/tabarena/experiments/$RUN_NAME/data", "raw/$RUN_NAME"),
+]:
+    try:
+        if src.endswith(".log"):
+            api.upload_file(path_or_fileobj=src, path_in_repo=dst, repo_id="$RESULTS_REPO", repo_type="dataset")
+        else:
+            api.upload_folder(folder_path=src, path_in_repo=dst, repo_id="$RESULTS_REPO", repo_type="dataset")
+        print("uploaded", dst)
+    except Exception as e:
+        print("upload failed", dst, e)
+EOF
+}
+
+finish() {  # finish <exit-code>
+    sync || true
+    if [ "$SELF_TERMINATE" = "1" ]; then poweroff; fi
+    exit "$1"
+}
+
+# --- one-time setup ---------------------------------------------------------
+uv venv --seed "$WORK/bootvenv" >/dev/null
+uv pip install --python "$WORK/bootvenv/bin/python" -q huggingface_hub
+
+# 1) FelaTab repo (model + benchmark code) from HF
+"$WORK/bootvenv/bin/python" - <<'EOF'
+from huggingface_hub import snapshot_download
+snapshot_download(
+    "lowdown-labs/fela-tab",
+    allow_patterns=[
+        "benchmark/**", "modeling.py", "configuration_felatab.py",
+        "config*.json", "model_small_int8.safetensors", "model_big_int8.safetensors",
+        "model_small.safetensors", "model_big.safetensors",
+    ],
+    local_dir="/work/fela-tab",
+)
+print("fela-tab snapshot ready")
+EOF
+
+# 2) TabArena in its own uv venv (Python 3.12 + pre-release AutoGluon)
+[ -d tabarena ] || git clone --depth 1 https://github.com/autogluon/tabarena.git
+cd tabarena
+uv venv --seed --python 3.12 .venv
+# CPU torch first so nothing pulls the multi-GB CUDA wheels
+uv pip install --python .venv/bin/python -q torch --index-url https://download.pytorch.org/whl/cpu
+uv pip install --python .venv/bin/python -q --prerelease=allow -e "./packages/tabarena[benchmark]"
+uv pip install --python .venv/bin/python -q safetensors tabulate "botocore[crt]"
+
+# --- benchmark phases --------------------------------------------------------
+cd "$WORK/fela-tab/benchmark/tabarena"
+SUCCESS=0
+IFS=',' read -ra PHASE_LST <<< "$PHASES"
+for PHASE in "${PHASE_LST[@]}"; do
+    if [ "$PHASE" = "full" ]; then SUB_ARG="--full"; else SUB_ARG="--subset lite"; fi
+    DS_ARG=""
+    if [ -n "$DATASETS" ] && [ "$PHASE" = "full" ]; then DS_ARG="--datasets $DATASETS"; fi
+    set +e
+    TMPDIR="$WORK/tmp" AWS_CONFIG_FILE=/dev/null AWS_EC2_METADATA_DISABLED=true WITH_LGBM="$WITH_LGBM" \
+      "$WORK/tabarena/.venv/bin/python" -u run_tabarena.py $SUB_ARG $DS_ARG --run-name "$RUN_NAME" \
+      2>&1 | tee "$WORK/logs/$PHASE.log"
+    RC=${PIPESTATUS[0]}
+    set -e
+    # count cumulative successes from the runner's progress lines
+    SUCCESS=$(grep -oP '\d+(?= success \|)' "$WORK/logs/$PHASE.log" | tail -1 || true)
+    SUCCESS=${SUCCESS:-0}
+    echo "phase=$PHASE rc=$RC successes=$SUCCESS"
+    upload "$PHASE" || true
+    if [ "$RC" != "0" ] || [ "$SUCCESS" = "0" ]; then
+        echo "phase $PHASE failed (rc=$RC successes=$SUCCESS); aborting before any later phase"
+        finish 1
+    fi
+done
+
+echo "ALL_PHASES_DONE"
+finish 0