Download runtime/tools/cpu_mem.py from SupersonicLabs/Julia-2-Flash: direct link, hf CLI and curl.
- Browser
- Download file 2.38 kB
-
https://huggingface.co/SupersonicLabs/Julia-2-Flash/resolve/main/runtime/tools/cpu_mem.py
- Command line
-
hf download hf://SupersonicLabs/Julia-2-Flash/runtime/tools/cpu_mem.py
-
curl -L -o cpu_mem.py https://huggingface.co/SupersonicLabs/Julia-2-Flash/resolve/main/runtime/tools/cpu_mem.py
2.38 kB
| """Where does the CPU memory go? RSS after import, after load, and after each text phase of tools.bench. | |
| The peak RSS (ru_maxrss) only grows, so the phase that raises it is the phase that sets the peak. | |
| python -m tools.cpu_mem [--precision int4e4s] [--out file.json] | |
| """ | |
| import argparse, json, os, resource, sys, time | |
| from pathlib import Path | |
| import torch | |
| from tools.bench import QS, REPO, STATE | |
| def main(): | |
| ap = argparse.ArgumentParser(); ap.add_argument("--precision", default="bf16"); ap.add_argument("--out") | |
| a = ap.parse_args() | |
| from julia2 import Runtime | |
| scale = 1 if sys.platform == "darwin" else 1024 # ru_maxrss: bytes on macOS, KiB on Linux | |
| gb = lambda x: round(x / 1e9, 3) | |
| snap = lambda: {"rss_gb": gb(int(os.popen(f"ps -o rss= -p {os.getpid()}").read()) * 1024), "peak_gb": gb(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * scale)} | |
| res = {"precision": a.precision, "threads": torch.get_num_threads(), "import": snap()} | |
| rt = Runtime(REPO, device="cpu", precision=a.precision, modalities=("text",)) | |
| res["load"] = snap() | |
| seen, by = set(), {} | |
| for mod_name, m in rt.trunk.named_modules(): | |
| for n, t in list(m.named_parameters(recurse=False)) + list(m.named_buffers(recurse=False)) + [(k, v) for k, v in vars(m).items() if torch.is_tensor(v)]: | |
| if t.data_ptr() in seen: | |
| continue | |
| seen.add(t.data_ptr()) | |
| key = f"{type(m).__name__}.{n}:{str(t.dtype).replace('torch.', '')}" | |
| by[key] = by.get(key, 0) + t.numel() * t.element_size() | |
| res["trunk_tensors_gb"] = {k: gb(v) for k, v in sorted(by.items(), key=lambda kv: -kv[1]) if v > 5e6} | |
| one = {"action": QS["action"]} | |
| phases = {"text_1q": lambda: rt.predict_questions(STATE, one), | |
| "text_6q": lambda: rt.predict_questions(STATE, QS, shared=True), | |
| "batch64": lambda: rt.predict_batch([{"state": STATE + f" Case {i}.", "questions": one} for i in range(64)], shared=False)} | |
| for name, fn in phases.items(): | |
| fn(); res[name] = snap() | |
| if name != "batch64": # the first call compiles; time the second | |
| t0 = time.perf_counter(); fn(); res[name]["ms"] = round((time.perf_counter() - t0) * 1e3, 1) | |
| print(json.dumps(res, indent=1)) | |
| if a.out: | |
| Path(a.out).write_text(json.dumps(res, indent=1)) | |
| if __name__ == "__main__": | |
| main() | |