Julia-2-Flash / runtime /tools /cpu_mem.py
vianahnz's picture
Julia 2 Flash
b61b4c4 verified
Raw History Blame Contribute Delete
2.38 kB
"""Where does the CPU memory go? RSS after import, after load, and after each text phase of tools.bench.
The peak RSS (ru_maxrss) only grows, so the phase that raises it is the phase that sets the peak.
python -m tools.cpu_mem [--precision int4e4s] [--out file.json]
"""
import argparse, json, os, resource, sys, time
from pathlib import Path
import torch
from tools.bench import QS, REPO, STATE
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--precision", default="bf16"); ap.add_argument("--out")
a = ap.parse_args()
from julia2 import Runtime
scale = 1 if sys.platform == "darwin" else 1024 # ru_maxrss: bytes on macOS, KiB on Linux
gb = lambda x: round(x / 1e9, 3)
snap = lambda: {"rss_gb": gb(int(os.popen(f"ps -o rss= -p {os.getpid()}").read()) * 1024), "peak_gb": gb(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * scale)}
res = {"precision": a.precision, "threads": torch.get_num_threads(), "import": snap()}
rt = Runtime(REPO, device="cpu", precision=a.precision, modalities=("text",))
res["load"] = snap()
seen, by = set(), {}
for mod_name, m in rt.trunk.named_modules():
for n, t in list(m.named_parameters(recurse=False)) + list(m.named_buffers(recurse=False)) + [(k, v) for k, v in vars(m).items() if torch.is_tensor(v)]:
if t.data_ptr() in seen:
continue
seen.add(t.data_ptr())
key = f"{type(m).__name__}.{n}:{str(t.dtype).replace('torch.', '')}"
by[key] = by.get(key, 0) + t.numel() * t.element_size()
res["trunk_tensors_gb"] = {k: gb(v) for k, v in sorted(by.items(), key=lambda kv: -kv[1]) if v > 5e6}
one = {"action": QS["action"]}
phases = {"text_1q": lambda: rt.predict_questions(STATE, one),
"text_6q": lambda: rt.predict_questions(STATE, QS, shared=True),
"batch64": lambda: rt.predict_batch([{"state": STATE + f" Case {i}.", "questions": one} for i in range(64)], shared=False)}
for name, fn in phases.items():
fn(); res[name] = snap()
if name != "batch64": # the first call compiles; time the second
t0 = time.perf_counter(); fn(); res[name]["ms"] = round((time.perf_counter() - t0) * 1e3, 1)
print(json.dumps(res, indent=1))
if a.out:
Path(a.out).write_text(json.dumps(res, indent=1))
if __name__ == "__main__":
main()