social-data-attribution-w2/scripts/slurm/dedup/progress.sh

#!/usr/bin/env bash
set -euo pipefail

DEDUP_DIR="${DEDUP_DIR:-/storage/ice-shared/cs7634/staff/TDA/soc-90}"
WORK_ROOT="${WORK_ROOT:-/storage/ice-shared/cs7634/staff/TDA/soc-90/unique_docs_work}"
MANIFEST="${DEDUP_DIR}/shard_manifest_6t.txt"
IDS_DIR="${DEDUP_DIR}/ids"
KEEP_DIR="${DEDUP_DIR}/keep"
BLOOM_FILE="${DEDUP_DIR}/unique_6t_ids.bloom"
FILTERED_DIR="${DEDUP_DIR}/filtered"
POOL_MANIFEST="${POOL_MANIFEST:-scripts/slurm/shard_manifest_pool.txt}"
POOL_SHARED_MANIFEST="${POOL_SHARED_MANIFEST:-scripts/slurm/shard_manifest_pool_shared.txt}"
MIX_NONPOOL_MANIFEST="${MIX_NONPOOL_MANIFEST:-scripts/slurm/shard_manifest_6t_nonpool.txt}"
PREFLIGHT_FILE="${PREFLIGHT_FILE:-$WORK_ROOT/preflight.json}"
export PREFLIGHT_FILE

echo "=== Legacy 6T Dedup Progress ==="
echo ""

if [ -f "$MANIFEST" ]; then
    TOTAL_SHARDS=$(wc -l < "$MANIFEST" | tr -d ' ')
    echo "Phase 1 (Enumerate):  $TOTAL_SHARDS shards in manifest"
else
    TOTAL_SHARDS=65718
    echo "Phase 1 (Enumerate):  manifest not found"
fi

if [ -d "$IDS_DIR" ]; then
    GZ_COUNT=$(find "$IDS_DIR" -name "*.ids.gz" | wc -l | tr -d ' ')
    TSV_COUNT=$(find "$IDS_DIR" -name "*.ids.tsv" | wc -l | tr -d ' ')
    echo "Phase 2 (Extract):    $GZ_COUNT / $TOTAL_SHARDS .ids.gz files"
    [ "$TSV_COUNT" -gt 0 ] && echo "  legacy .ids.tsv:    $TSV_COUNT"
    TAR_FILE="${DEDUP_DIR}/ids_all.tar"
    if [ -f "$TAR_FILE" ]; then
        TAR_SIZE=$(du -sh "$TAR_FILE" | cut -f1)
        echo "  backup tarball:     $TAR_SIZE"
    fi
else
    echo "Phase 2 (Extract):    ids directory not found"
fi

if [ -d "$KEEP_DIR" ]; then
    BUCKET_COUNT=$(find "$KEEP_DIR" -name "bucket_*.tsv" | wc -l | tr -d ' ')
    echo "Phase 3 (Dedup):      $BUCKET_COUNT / 256 bucket files"
else
    echo "Phase 3 (Dedup):      keep directory not found"
fi

DEDUP_REPORT="${DEDUP_DIR}/dedup_stats_report.json"
if [ -f "$DEDUP_REPORT" ]; then
    echo "Phase 3.5 (Summary):  ready"
else
    echo "Phase 3.5 (Summary):  missing"
fi

if [ -f "$BLOOM_FILE" ]; then
    BLOOM_SIZE=$(du -sh "$BLOOM_FILE" | cut -f1)
    echo "Phase 4 (Bloom):      $BLOOM_SIZE"
else
    echo "Phase 4 (Bloom):      missing"
fi

if [ -d "$FILTERED_DIR" ]; then
    FILTERED_COUNT=$(find "$FILTERED_DIR" -name "*.parquet" | wc -l | tr -d ' ')
    FILTERED_SIZE=$(du -sh "$FILTERED_DIR" | cut -f1)
    echo "Phase 5 (Filter):     $FILTERED_COUNT parquet files ($FILTERED_SIZE)"
else
    echo "Phase 5 (Filter):     not started"
fi

echo ""
echo "=== SOC-127 Materialization Progress ==="
echo ""

[ -f "$POOL_MANIFEST" ] && echo "Pool manifest:        $(wc -l < "$POOL_MANIFEST" | tr -d ' ') shards" || echo "Pool manifest:        missing"
[ -f "$POOL_SHARED_MANIFEST" ] && echo "Pool shared manifest: $(wc -l < "$POOL_SHARED_MANIFEST" | tr -d ' ') shards" || echo "Pool shared manifest: missing"
[ -f "$MIX_NONPOOL_MANIFEST" ] && echo "Mix non-pool manifest:$(wc -l < "$MIX_NONPOOL_MANIFEST" | tr -d ' ') shards" || echo "Mix non-pool manifest: missing"

if [ -f "$PREFLIGHT_FILE" ]; then
    python3 - <<'PY'
import json
import os

with open(os.environ["PREFLIGHT_FILE"], encoding="utf-8") as handle:
    payload = json.load(handle)
status = "passed" if payload.get("all_required_approved") else "blocked"
approved = ", ".join(payload.get("approved_families", []))
blocked = ", ".join(payload.get("blocked_families", []))
print(f"Preflight:            {status}")
print(f"  approved:           {approved or 'none'}")
print(f"  blocked:            {blocked or 'none'}")
PY
else
    echo "Preflight:            missing"
fi

[ -d "$WORK_ROOT/pool" ] && echo "Pool outputs:         $(find "$WORK_ROOT/pool" -name '*.done' | wc -l | tr -d ' ') done markers" || echo "Pool outputs:         not started"
[ -d "$WORK_ROOT/mix_nonpool_raw" ] && echo "Mix raw outputs:      $(find "$WORK_ROOT/mix_nonpool_raw" -name '*.done' | wc -l | tr -d ' ') done markers" || echo "Mix raw outputs:      not started"
[ -d "$WORK_ROOT/mix_nonpool_final" ] && echo "Mix final buckets:    $(find "$WORK_ROOT/mix_nonpool_final" -name 'bucket_*.jsonl.zst' | wc -l | tr -d ' ') bucket files" || echo "Mix final buckets:    not started"
[ -f "$WORK_ROOT/manifest.parquet" ] && echo "Manifest:             ready" || echo "Manifest:             missing"
[ -f "$WORK_ROOT/stats.json" ] && echo "Stats:                ready" || echo "Stats:                missing"
[ -f "$WORK_ROOT/README.md" ] && echo "README:               ready" || echo "README:               missing"

echo ""
echo "=== Active SLURM Jobs ==="
squeue -u "$USER" -n "extract_ids,backup_ids,dedup_bucket,dedup_report,build_bloom,filter_sample,preflight_materialize,materialize_pool,materialize_mix,merge_mix,finalize_unique,upload_unique" --format="%.10i %.20j %.8T %.10M %.6D %R" 2>/dev/null || echo "squeue unavailable"

echo ""
echo "=== Disk Usage (NFS) ==="
du -sh "$DEDUP_DIR" 2>/dev/null || echo "$DEDUP_DIR not found"
pace-quota 2>/dev/null | grep -A1 "scratch" || true

GLOBUS="${HOME}/.local/bin/globus"
if [ -x "$GLOBUS" ]; then
    echo ""
    echo "=== Globus Transfers (recent) ==="
    "$GLOBUS" task list --limit 3 2>/dev/null || echo "globus unavailable"
fi