Remove DeMemWM raw training fallback
Browse files
algorithms/dememwm/df_video.py
CHANGED
|
@@ -719,6 +719,12 @@ class DeMemWMMinecraft(DiffusionForcingBase):
|
|
| 719 |
Returns:
|
| 720 |
dict: A dictionary containing the training loss.
|
| 721 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 722 |
preprocessed = self._preprocess_batch(batch)
|
| 723 |
if isinstance(preprocessed, Mapping):
|
| 724 |
xs = preprocessed["latents"]
|
|
@@ -778,64 +784,10 @@ class DeMemWMMinecraft(DiffusionForcingBase):
|
|
| 778 |
self.log("training/loss", loss.cpu())
|
| 779 |
|
| 780 |
return {"loss": loss}
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
|
| 784 |
-
if self.relative_embedding:
|
| 785 |
-
input_pose_condition = []
|
| 786 |
-
frame_idx_list = []
|
| 787 |
-
for i in range(self.n_frames):
|
| 788 |
-
input_pose_condition.append(
|
| 789 |
-
convert_to_plucker(
|
| 790 |
-
torch.cat([c2w_mat[i:i + 1], c2w_mat[-self.memory_condition_length:]]).clone(),
|
| 791 |
-
0,
|
| 792 |
-
focal_length=self.focal_length,
|
| 793 |
-
image_height=xs.shape[-2],image_width=xs.shape[-1]
|
| 794 |
-
).to(xs.dtype)
|
| 795 |
-
)
|
| 796 |
-
frame_idx_list.append(
|
| 797 |
-
torch.cat([
|
| 798 |
-
frame_idx[i:i + 1] - frame_idx[i:i + 1],
|
| 799 |
-
frame_idx[-self.memory_condition_length:] - frame_idx[i:i + 1]
|
| 800 |
-
]).clone()
|
| 801 |
-
)
|
| 802 |
-
input_pose_condition = torch.cat(input_pose_condition)
|
| 803 |
-
frame_idx_list = torch.cat(frame_idx_list)
|
| 804 |
-
else:
|
| 805 |
-
input_pose_condition = convert_to_plucker(
|
| 806 |
-
c2w_mat, 0, focal_length=self.focal_length
|
| 807 |
-
).to(xs.dtype)
|
| 808 |
-
frame_idx_list = frame_idx
|
| 809 |
-
else:
|
| 810 |
-
input_pose_condition = pose_conditions.to(xs.dtype)
|
| 811 |
-
frame_idx_list = None
|
| 812 |
-
|
| 813 |
-
xs = self.encode(xs)
|
| 814 |
-
|
| 815 |
-
noise_levels = self._generate_noise_levels(xs)
|
| 816 |
-
|
| 817 |
-
if self.memory_condition_length:
|
| 818 |
-
noise_levels[-self.memory_condition_length:] = self.diffusion_model.stabilization_level
|
| 819 |
-
conditions[-self.memory_condition_length:] *= 0
|
| 820 |
-
|
| 821 |
-
_, loss = self.diffusion_model(
|
| 822 |
-
xs,
|
| 823 |
-
conditions,
|
| 824 |
-
input_pose_condition,
|
| 825 |
-
noise_levels=noise_levels,
|
| 826 |
-
reference_length=self.memory_condition_length,
|
| 827 |
-
frame_idx=frame_idx_list
|
| 828 |
)
|
| 829 |
-
|
| 830 |
-
if self.memory_condition_length:
|
| 831 |
-
loss = loss[:-self.memory_condition_length]
|
| 832 |
-
|
| 833 |
-
loss = self.reweight_loss(loss, None)
|
| 834 |
-
|
| 835 |
-
if batch_idx % 20 == 0:
|
| 836 |
-
self.log("training/loss", loss.cpu())
|
| 837 |
-
|
| 838 |
-
return {"loss": loss}
|
| 839 |
|
| 840 |
def on_validation_epoch_start(self) -> None:
|
| 841 |
self._reset_metric_accumulators()
|
|
|
|
| 719 |
Returns:
|
| 720 |
dict: A dictionary containing the training loss.
|
| 721 |
"""
|
| 722 |
+
if not isinstance(batch, Mapping):
|
| 723 |
+
raise TypeError(
|
| 724 |
+
"DeMemWM training requires the latent dict batch contract "
|
| 725 |
+
"from video_minecraft_dememwm_latent; raw tuple batches are unsupported."
|
| 726 |
+
)
|
| 727 |
+
|
| 728 |
preprocessed = self._preprocess_batch(batch)
|
| 729 |
if isinstance(preprocessed, Mapping):
|
| 730 |
xs = preprocessed["latents"]
|
|
|
|
| 784 |
self.log("training/loss", loss.cpu())
|
| 785 |
|
| 786 |
return {"loss": loss}
|
| 787 |
+
raise TypeError(
|
| 788 |
+
"DeMemWM _preprocess_batch must return the latent dict contract; "
|
| 789 |
+
"legacy raw training tuples are unsupported."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 790 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 791 |
|
| 792 |
def on_validation_epoch_start(self) -> None:
|
| 793 |
self._reset_metric_accumulators()
|
tests/test_dememwm_latent_dataset.py
CHANGED
|
@@ -302,6 +302,16 @@ class DeMemWMLatentDatasetTests(unittest.TestCase):
|
|
| 302 |
self.assertEqual(harness.logged[0][0], "training/loss")
|
| 303 |
self.assertEqual(float(harness.logged[0][1]), 2.0)
|
| 304 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
def test_memory_noise_helpers_route_shift_and_keep_validation_clean(self):
|
| 306 |
from algorithms.dememwm.df_video import _apply_memory_route_masks, _memory_noise_levels_for_streams
|
| 307 |
|
|
|
|
| 302 |
self.assertEqual(harness.logged[0][0], "training/loss")
|
| 303 |
self.assertEqual(float(harness.logged[0][1]), 2.0)
|
| 304 |
|
| 305 |
+
def test_training_step_rejects_raw_tuple_batches(self):
|
| 306 |
+
from algorithms.dememwm.df_video import DeMemWMMinecraft
|
| 307 |
+
|
| 308 |
+
class Harness:
|
| 309 |
+
def _preprocess_batch(self, batch):
|
| 310 |
+
raise AssertionError("raw batches must be rejected before preprocessing")
|
| 311 |
+
|
| 312 |
+
with self.assertRaisesRegex(TypeError, "latent dict batch contract"):
|
| 313 |
+
DeMemWMMinecraft.training_step(Harness(), (None, None, None, None), 0)
|
| 314 |
+
|
| 315 |
def test_memory_noise_helpers_route_shift_and_keep_validation_clean(self):
|
| 316 |
from algorithms.dememwm.df_video import _apply_memory_route_masks, _memory_noise_levels_for_streams
|
| 317 |
|