File size: 3,866 Bytes
c7ae99a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
"""Full training pipeline: convert -> tokenizer (<=2h) -> prep -> generator (<=2h).

    python code/run_cmu.py --amc <folder with the CMU .asf/.amc files> --work <empty work folder>

Each step logs to <work>/runs/<step>.log.
<work>/runs/STATUS.json = current step / state; <work>/runs/ALERT.txt = something needs attention.
Hang guard: a step whose log has not grown for --stall-min minutes is killed and reported.
Resumable: steps listed in runs/STATUS.json as done are skipped (use --from to force a start step).
Checkpoints are written to <work>/ckpt (cmu_tok_v1_H.pt, cmu_gen_v1.pt).
"""
import os, sys, json, time, subprocess, argparse

CODE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(CODE)
PY = sys.executable
W = RUNS = STEPS = None


def setup(amc, work):
    global W, RUNS, STEPS
    W = os.path.abspath(work)
    RUNS = os.path.join(W, "runs")
    STEPS = [
        ("convert", [os.path.join(CODE, "cmu_to272.py"), amc, os.path.join(REPO, "data", "cmu_files.csv"),
                     os.path.join(W, "data", "motion_raw", "h272")]),
        ("tokenizer", [os.path.join(CODE, "tok_cmu.py"), "--seconds", "7200", "--tag", "cmu_tok_v1"]),
        ("prep", [os.path.join(CODE, "gen_prep.py")]),
        ("generator", [os.path.join(CODE, "gen_ar.py"), "--seconds", "7200", "--tag", "cmu_gen_v1"]),
    ]


def now():
    return time.strftime("%Y-%m-%d %H:%M:%S")


def status(**kw):
    p = os.path.join(RUNS, "STATUS.json")
    st = json.load(open(p, encoding="utf-8")) if os.path.exists(p) else {"done": []}
    st.update(kw, updated=now())
    json.dump(st, open(p, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
    return st


def alert(msg):
    with open(os.path.join(RUNS, "ALERT.txt"), "a", encoding="utf-8") as f:
        f.write("[%s] %s\n" % (now(), msg))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--from", dest="start", default=None)
    ap.add_argument("--stall-min", type=float, default=30)
    ap.add_argument("--amc", required=True, help="folder with the CMU .asf/.amc files (allasfamc.zip, extracted)")
    ap.add_argument("--work", required=True, help="work folder (data/, ckpt/, runs/ are created here)")
    a = ap.parse_args()
    setup(a.amc, a.work)
    os.makedirs(RUNS, exist_ok=True)
    os.makedirs(os.path.join(W, "ckpt"), exist_ok=True)
    env = dict(os.environ, PARTLOCK_DIR=W, PYTHONIOENCODING="utf-8", PYTHONUNBUFFERED="1")
    st = status(pid=os.getpid(), state="running")
    names = [n for n, _ in STEPS]
    first = names.index(a.start) if a.start else 0
    for name, args in STEPS[first:]:
        if not a.start and name in st.get("done", []):
            continue
        log = os.path.join(RUNS, name + ".log")
        status(step=name, step_started=now(), state="running")
        with open(log, "w", encoding="utf-8") as lf:
            lf.write("[%s] start %s\n" % (now(), " ".join(args)))
            lf.flush()
            p = subprocess.Popen([PY] + args, stdout=lf, stderr=subprocess.STDOUT, env=env, cwd=CODE)
            last_size, last_change = 0, time.time()
            while p.poll() is None:
                time.sleep(30)
                sz = os.path.getsize(log)
                if sz != last_size:
                    last_size, last_change = sz, time.time()
                elif time.time() - last_change > a.stall_min * 60:
                    p.kill()
                    alert("%s: no log output for %d min -> killed (hang)" % (name, a.stall_min))
                    break
            rc = p.wait()
        if rc != 0:
            alert("%s failed (exit %s). see runs/%s.log" % (name, rc, name))
            status(state="failed", failed_step=name, exit_code=rc)
            return
        st = status(done=st.get("done", []) + [name], last_done=name)
    status(state="all done", step=None)


if __name__ == "__main__":
    main()