Nekochu commited on
Commit
b57f6e2
·
1 Parent(s): 19a7c6b

fix OOM: use mmap loading + assign (no memory copy)

Browse files
Files changed (1) hide show
  1. app.py +9 -2
app.py CHANGED
@@ -48,7 +48,12 @@ def _load_generator():
48
  print("[init] Loading pre-quantized INT8 DiT...")
49
  t0 = time.time()
50
 
51
- # Build model structure, then quantize empty model, then load pre-quantized weights
 
 
 
 
 
52
  step1x_params = Step1XParams(
53
  in_channels=64, out_channels=64, vec_in_dim=768, context_in_dim=4096,
54
  hidden_size=3072, mlp_ratio=4.0, num_heads=24, depth=19,
@@ -56,7 +61,9 @@ def _load_generator():
56
  )
57
  dit = Step1XEdit(step1x_params).float()
58
  dit = torch.quantization.quantize_dynamic(dit, {torch.nn.Linear}, dtype=torch.qint8)
59
- dit.load_state_dict(torch.load(dit_path, map_location="cpu", weights_only=False))
 
 
60
  print(f"[init] DiT loaded in {time.time()-t0:.0f}s")
61
 
62
  # Load VAE
 
48
  print("[init] Loading pre-quantized INT8 DiT...")
49
  t0 = time.time()
50
 
51
+ # Use mmap to avoid doubling memory during load
52
+ print("[init] Loading with mmap (memory-mapped, no copy)...")
53
+ sd = torch.load(dit_path, map_location="cpu", weights_only=False, mmap=True)
54
+ print(f"[init] State dict mmap'd ({len(sd)} keys)")
55
+
56
+ # Build quantized model structure and load weights via assign (no copy)
57
  step1x_params = Step1XParams(
58
  in_channels=64, out_channels=64, vec_in_dim=768, context_in_dim=4096,
59
  hidden_size=3072, mlp_ratio=4.0, num_heads=24, depth=19,
 
61
  )
62
  dit = Step1XEdit(step1x_params).float()
63
  dit = torch.quantization.quantize_dynamic(dit, {torch.nn.Linear}, dtype=torch.qint8)
64
+ dit.load_state_dict(sd, strict=False, assign=True)
65
+ del sd
66
+ gc.collect()
67
  print(f"[init] DiT loaded in {time.time()-t0:.0f}s")
68
 
69
  # Load VAE