File size: 2,802 Bytes
6374a33 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | from __future__ import annotations
import argparse
import json
import resource
import time
from pathlib import Path
import numpy as np
import skops.io as sio
from threadpoolctl import threadpool_limits
from banking_intent_error_predictor.training import (
SOURCE_REVISION,
download,
fit_once,
load_rows,
sha256_file,
)
def main() -> None:
parser = argparse.ArgumentParser(description="Reproduce the reviewed release")
parser.add_argument("--cache-dir", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--reference", type=Path, required=True)
args = parser.parse_args()
args.output_dir.mkdir(parents=True, exist_ok=False)
train_rows = load_rows(download(args.cache_dir, "train.csv"))
test_rows = load_rows(download(args.cache_dir, "test.csv"))
download(args.cache_dir, "categories.json")
wall_start = time.perf_counter()
cpu_start = time.process_time()
with threadpool_limits(limits=8):
result = fit_once(train_rows, test_rows)
reference = result["reproduction_reference"]
with np.load(args.reference, allow_pickle=False) as expected:
exact = all(
np.array_equal(reference[key], expected[key]) for key in expected.files
)
if not exact:
raise RuntimeError("reproduced decision scores differ from v1.0.0")
primary_path = args.output_dir / "primary_baseline.skops"
candidate_path = args.output_dir / "model.skops"
sio.dump(
{
"features": result["models"]["feature_extractor"],
"classifier": result["models"]["primary"],
"labels": result["labels"],
},
primary_path,
)
sio.dump(
{
"classifier": result["models"]["candidate"],
"labels": result["labels"],
"review_rate": 0.20,
},
candidate_path,
)
for path in (primary_path, candidate_path):
if sio.get_untrusted_types(file=path):
raise RuntimeError(f"reproduced artifact requires untrusted types: {path}")
report = {
"dataset_revision": SOURCE_REVISION,
"reference_outputs_exact": exact,
"primary_artifact_sha256": sha256_file(primary_path),
"candidate_artifact_sha256": sha256_file(candidate_path),
"wall_seconds": time.perf_counter() - wall_start,
"cpu_seconds": time.process_time() - cpu_start,
"thread_limit": 8,
"peak_rss_bytes": int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss),
}
(args.output_dir / "reproduction.json").write_text(
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
print(json.dumps(report, indent=2, sort_keys=True))
if __name__ == "__main__":
main()
|