Default MSRoPE to the shared scaling factor
Browse files- src/models/dit_blocks.py +10 -7
src/models/dit_blocks.py
CHANGED
|
@@ -267,13 +267,13 @@ def set_real_rope(model: nn.Module, enabled: bool) -> None:
|
|
| 267 |
|
| 268 |
|
| 269 |
def set_uniform_rope_scaling(model: nn.Module, enabled: bool) -> None:
|
| 270 |
-
"""Scale both spatial axes by one shared factor
|
| 271 |
|
| 272 |
-
Per-axis factors (the DyPE
|
| 273 |
-
square training window in phase space, so content dilates
|
| 274 |
-
axis by the aspect ratio. One shared max factor preserves
|
| 275 |
-
aspect in phase space at the cost of a shorter phase span on
|
| 276 |
-
axis. Instance-local launch policy; not checkpoint state.
|
| 277 |
"""
|
| 278 |
if type(enabled) is not bool:
|
| 279 |
raise ValueError("uniform RoPE scaling policy must be boolean")
|
|
@@ -288,6 +288,9 @@ class MSRoPE(nn.Module):
|
|
| 288 |
The 896p training buckets reach 1184 pixels on either axis, or 74 patches
|
| 289 |
at 16 pixels per patch. Training always uses the original frequencies.
|
| 290 |
DyPE follows https://arxiv.org/html/2510.20766v2, equations 9 and 13–17.
|
|
|
|
|
|
|
|
|
|
| 291 |
"""
|
| 292 |
|
| 293 |
def __init__(
|
|
@@ -311,7 +314,7 @@ class MSRoPE(nn.Module):
|
|
| 311 |
# 0-based from the corner. Text starts at max(height, width) on both axes.
|
| 312 |
self.centered = centered
|
| 313 |
# Launch policy set by set_uniform_rope_scaling; not checkpoint state.
|
| 314 |
-
self.uniform_scaling =
|
| 315 |
|
| 316 |
# Precompute frequency tables for both spatial axes
|
| 317 |
pos_index = torch.arange(2560) # maximum position index
|
|
|
|
| 267 |
|
| 268 |
|
| 269 |
def set_uniform_rope_scaling(model: nn.Module, enabled: bool) -> None:
|
| 270 |
+
"""Scale both spatial axes by one shared factor (default) or per-axis ones.
|
| 271 |
|
| 272 |
+
Per-axis factors (the DyPE paper's behavior) normalize a rectangular
|
| 273 |
+
canvas to the square training window in phase space, so content dilates
|
| 274 |
+
along the longer axis by the aspect ratio. One shared max factor preserves
|
| 275 |
+
the canvas aspect in phase space at the cost of a shorter phase span on
|
| 276 |
+
the short axis. Instance-local launch policy; not checkpoint state.
|
| 277 |
"""
|
| 278 |
if type(enabled) is not bool:
|
| 279 |
raise ValueError("uniform RoPE scaling policy must be boolean")
|
|
|
|
| 288 |
The 896p training buckets reach 1184 pixels on either axis, or 74 patches
|
| 289 |
at 16 pixels per patch. Training always uses the original frequencies.
|
| 290 |
DyPE follows https://arxiv.org/html/2510.20766v2, equations 9 and 13–17.
|
| 291 |
+
Rectangular canvases default to one shared scaling factor so the phase
|
| 292 |
+
aspect matches the canvas aspect; set_uniform_rope_scaling(model, False)
|
| 293 |
+
restores the paper's per-axis factors.
|
| 294 |
"""
|
| 295 |
|
| 296 |
def __init__(
|
|
|
|
| 314 |
# 0-based from the corner. Text starts at max(height, width) on both axes.
|
| 315 |
self.centered = centered
|
| 316 |
# Launch policy set by set_uniform_rope_scaling; not checkpoint state.
|
| 317 |
+
self.uniform_scaling = True
|
| 318 |
|
| 319 |
# Precompute frequency tables for both spatial axes
|
| 320 |
pos_index = torch.arange(2560) # maximum position index
|