File size: 3,329 Bytes
387a4cf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
"""Public interface for data preparation, resumable search, evaluation and reporting."""

import argparse
import functools
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
import json
from pathlib import Path
import subprocess
import sys
from dotenv import load_dotenv
from . import campaign as study
from .data import prepare
from .endpoints import freeze
from .reporting import report


def main(argv=None):
    parser = argparse.ArgumentParser(prog="rqgm-study", description=__doc__)
    parser.add_argument("--project", type=Path, default=Path.cwd())
    commands = parser.add_subparsers(dest="action", required=True)
    for name in ("prepare", "preflight", "report", "selftest"):
        commands.add_parser(name)
    run = commands.add_parser("run")
    run.add_argument("--arm", choices=(*study.ARMS, "fork-native"), required=True)
    run.add_argument("--seed", type=int, required=True)
    run.add_argument("--stage", choices=("pilot", "confirm"), required=True)
    run.add_argument("--budget", type=int, required=True)
    run.add_argument("--stop-at", type=int)
    ev = commands.add_parser("evaluate")
    ev.add_argument("--split", choices=("pilot", "test"), required=True)
    ev.add_argument("--name", required=True)
    commands.add_parser("freeze").add_argument(
        "--allow-incomplete", action="store_true"
    )
    commands.add_parser("campaign").add_argument("--publish", action="store_true")
    commands.add_parser("release").add_argument("--message", required=True)
    serve = commands.add_parser("serve")
    serve.add_argument("--port", default=8765, type=int)
    args = parser.parse_args(argv)
    project = args.project.resolve()
    load_dotenv(project / ".env")
    if hasattr(sys.stdout, "reconfigure"):
        sys.stdout.reconfigure(
            encoding="utf-8", errors="backslashreplace", line_buffering=True
        )
    match args.action:
        case "prepare":
            result = {k: v["sha256"] for k, v in prepare(project)["splits"].items()}
        case "preflight":
            result = study.preflight(project)
        case "run":
            result = study.search(
                project, args.stage, args.seed, args.arm, args.budget, args.stop_at
            )
        case "evaluate":
            result = study.evaluate(project, args.split, args.name)
        case "freeze":
            result = freeze(project, args.allow_incomplete)["entries_sha"]
        case "campaign":
            result = study.campaign(project, args.publish)
        case "release":
            result = study.publish(project, args.message)
        case "report":
            result = report(project)
        case "selftest":
            raise SystemExit(
                subprocess.run(
                    [sys.executable, "-m", "pytest", "-q"], cwd=project
                ).returncode
            )
        case "serve":
            handler = functools.partial(
                SimpleHTTPRequestHandler, directory=str(project / "docs")
            )
            print(f"Dashboard at http://127.0.0.1:{args.port}/", flush=True)
            with ThreadingHTTPServer(("127.0.0.1", args.port), handler) as server:
                server.serve_forever()
            return
    print(json.dumps(result, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()