kaupane commited on
Commit
7ee2cb8
·
verified ·
1 Parent(s): ea64714

Default MSRoPE to the shared scaling factor

Browse files
Files changed (1) hide show
  1. src/models/dit_blocks.py +10 -7
src/models/dit_blocks.py CHANGED
@@ -267,13 +267,13 @@ def set_real_rope(model: nn.Module, enabled: bool) -> None:
267
 
268
 
269
  def set_uniform_rope_scaling(model: nn.Module, enabled: bool) -> None:
270
- """Scale both spatial axes by one shared factor instead of per-axis ones.
271
 
272
- Per-axis factors (the DyPE default) normalize a rectangular canvas to the
273
- square training window in phase space, so content dilates along the longer
274
- axis by the aspect ratio. One shared max factor preserves the canvas
275
- aspect in phase space at the cost of a shorter phase span on the short
276
- axis. Instance-local launch policy; not checkpoint state.
277
  """
278
  if type(enabled) is not bool:
279
  raise ValueError("uniform RoPE scaling policy must be boolean")
@@ -288,6 +288,9 @@ class MSRoPE(nn.Module):
288
  The 896p training buckets reach 1184 pixels on either axis, or 74 patches
289
  at 16 pixels per patch. Training always uses the original frequencies.
290
  DyPE follows https://arxiv.org/html/2510.20766v2, equations 9 and 13–17.
 
 
 
291
  """
292
 
293
  def __init__(
@@ -311,7 +314,7 @@ class MSRoPE(nn.Module):
311
  # 0-based from the corner. Text starts at max(height, width) on both axes.
312
  self.centered = centered
313
  # Launch policy set by set_uniform_rope_scaling; not checkpoint state.
314
- self.uniform_scaling = False
315
 
316
  # Precompute frequency tables for both spatial axes
317
  pos_index = torch.arange(2560) # maximum position index
 
267
 
268
 
269
  def set_uniform_rope_scaling(model: nn.Module, enabled: bool) -> None:
270
+ """Scale both spatial axes by one shared factor (default) or per-axis ones.
271
 
272
+ Per-axis factors (the DyPE paper's behavior) normalize a rectangular
273
+ canvas to the square training window in phase space, so content dilates
274
+ along the longer axis by the aspect ratio. One shared max factor preserves
275
+ the canvas aspect in phase space at the cost of a shorter phase span on
276
+ the short axis. Instance-local launch policy; not checkpoint state.
277
  """
278
  if type(enabled) is not bool:
279
  raise ValueError("uniform RoPE scaling policy must be boolean")
 
288
  The 896p training buckets reach 1184 pixels on either axis, or 74 patches
289
  at 16 pixels per patch. Training always uses the original frequencies.
290
  DyPE follows https://arxiv.org/html/2510.20766v2, equations 9 and 13–17.
291
+ Rectangular canvases default to one shared scaling factor so the phase
292
+ aspect matches the canvas aspect; set_uniform_rope_scaling(model, False)
293
+ restores the paper's per-axis factors.
294
  """
295
 
296
  def __init__(
 
314
  # 0-based from the corner. Text starts at max(height, width) on both axes.
315
  self.centered = centered
316
  # Launch policy set by set_uniform_rope_scaling; not checkpoint state.
317
+ self.uniform_scaling = True
318
 
319
  # Precompute frequency tables for both spatial axes
320
  pos_index = torch.arange(2560) # maximum position index