Fail fast for DeMemWM interactive path

algorithms/dememwm/df_video.py

@@ -1164,137 +1164,7 @@ class DeMemWMMinecraft(DiffusionForcingBase):
     @torch.no_grad()
     def interactive(self, first_frame, new_actions, first_pose, device,
                     memory_latent_frames, memory_actions, memory_poses, memory_c2w, memory_frame_idx):
-    
-        memory_condition_length = self.memory_condition_length
-
-        if memory_latent_frames is None:
-            first_frame = torch.from_numpy(first_frame)
-            new_actions = torch.from_numpy(new_actions)
-            first_pose = torch.from_numpy(first_pose)
-            first_frame_encode = self.encode(first_frame[None, None].to(device))
-            memory_latent_frames = first_frame_encode.cpu()
-            memory_actions = new_actions[None, None].to(device)
-            memory_poses = first_pose[None, None].to(device)
-            new_c2w_mat = euler_to_camera_to_world_matrix(first_pose)
-            memory_c2w = new_c2w_mat[None, None].to(device)
-            memory_frame_idx = torch.tensor([[0]]).to(device)
-            return first_frame.cpu().numpy(), memory_latent_frames.cpu().numpy(), memory_actions.cpu().numpy(), memory_poses.cpu().numpy(), memory_c2w.cpu().numpy(), memory_frame_idx.cpu().numpy()
-        else:
-            memory_latent_frames = torch.from_numpy(memory_latent_frames)
-            memory_actions = torch.from_numpy(memory_actions).to(device)
-            memory_poses = torch.from_numpy(memory_poses).to(device)
-            memory_c2w = torch.from_numpy(memory_c2w).to(device)
-            memory_frame_idx = torch.from_numpy(memory_frame_idx).to(device)
-            new_actions = new_actions.to(device)
-
-        curr_frame = 0
-        batch_size = 1
-        horizon = self.next_frame_length
-        n_frames = curr_frame + horizon
-        # context
-        n_context_frames = len(memory_latent_frames)
-        xs_pred = memory_latent_frames[:n_context_frames].clone()
-        curr_frame += n_context_frames
-
-        pbar = tqdm(total=n_frames, initial=curr_frame, desc="Sampling")
-
-        new_pose_condition_list = []
-        last_frame = xs_pred[-1].clone()
-        last_pose_condition = memory_poses[-1].clone()
-        curr_actions = new_actions.clone()
-        for hi in range(len(new_actions)):
-            last_pose_condition[:,3:] = last_pose_condition[:,3:] // 15
-            new_pose_condition_offset = self.pose_prediction_model(last_frame.to(device), curr_actions[None, hi], last_pose_condition)
-            new_pose_condition_offset[:,3:] = torch.round(new_pose_condition_offset[:,3:])
-            new_pose_condition = last_pose_condition + new_pose_condition_offset
-            new_pose_condition[:,3:] = new_pose_condition[:,3:] * 15
-            new_pose_condition[:,3:] %= 360
-            last_pose_condition = new_pose_condition.clone()
-            new_pose_condition_list.append(new_pose_condition[None])
-        new_pose_condition_list = torch.cat(new_pose_condition_list, 0)
-        
-        ai = 0
-        while ai < len(new_actions):
-            next_horizon = min(horizon, len(new_actions) - ai)
-            last_frame = xs_pred[-1].clone()
-            curr_actions = new_actions[ai:ai+next_horizon].clone()
-
-            new_pose_condition = new_pose_condition_list[ai:ai+next_horizon].clone()
-
-            new_c2w_mat = euler_to_camera_to_world_matrix(new_pose_condition)
-            memory_poses = torch.cat([memory_poses, new_pose_condition])
-            memory_actions = torch.cat([memory_actions, curr_actions[:, None]])
-            memory_c2w = torch.cat([memory_c2w, new_c2w_mat])
-            new_indices = memory_frame_idx[-1,0] + torch.arange(next_horizon, device=memory_frame_idx.device) + 1
-
-            memory_frame_idx = torch.cat([memory_frame_idx, new_indices[:, None]])
-
-            conditions = memory_actions.clone()
-            pose_conditions = memory_poses.clone()
-            c2w_mat = memory_c2w .clone()
-            frame_idx = memory_frame_idx.clone()
-
-            # generation on frame
-            scheduling_matrix = self._generate_scheduling_matrix(next_horizon)
-            chunk = torch.randn((next_horizon, batch_size, *xs_pred.shape[2:])).to(xs_pred.device)
-            chunk = torch.clamp(chunk, -self.clip_noise, self.clip_noise)
-
-            xs_pred = torch.cat([xs_pred, chunk], 0)
-
-            # sliding window: only input the last n_tokens frames
-            start_frame = max(0, curr_frame - self.n_tokens)
-
-            pbar.set_postfix(
-                {
-                    "start": start_frame,
-                    "end": curr_frame + next_horizon,
-                }
-            )
-
-            # Handle condition similarity logic
-            if memory_condition_length:
-                random_idx = self._generate_condition_indices(
-                    curr_frame, memory_condition_length, xs_pred, pose_conditions, frame_idx, next_horizon
-                )
-                
-                # random_idx = np.unique(random_idx)[:, None]
-                # memory_condition_length = len(random_idx)
-                xs_pred = torch.cat([xs_pred, xs_pred[random_idx[:, range(xs_pred.shape[1])], range(xs_pred.shape[1])].clone()], 0)
-
-            # Prepare input conditions and pose conditions
-            input_condition, input_pose_condition, frame_idx_list = self._prepare_conditions(
-                start_frame, curr_frame, next_horizon, conditions, pose_conditions, c2w_mat, frame_idx, random_idx,
-                image_width=first_frame.shape[-1], image_height=first_frame.shape[-2]
-            )
-
-            # Perform sampling for each step in the scheduling matrix
-            for m in range(scheduling_matrix.shape[0] - 1):
-                from_noise_levels, to_noise_levels = self._prepare_noise_levels(
-                    scheduling_matrix, m, curr_frame, batch_size, memory_condition_length
-                )
-
-                xs_pred[start_frame:] = self.diffusion_model.sample_step(
-                    xs_pred[start_frame:].to(input_condition.device),
-                    input_condition,
-                    input_pose_condition,
-                    from_noise_levels[start_frame:],
-                    to_noise_levels[start_frame:],
-                    current_frame=curr_frame,
-                    mode="validation",
-                    reference_length=memory_condition_length,
-                    frame_idx=frame_idx_list
-                ).cpu()
-
-
-            if memory_condition_length:
-                xs_pred = xs_pred[:-memory_condition_length]
-
-            curr_frame += next_horizon
-            pbar.update(next_horizon)
-            ai += next_horizon
-
-        memory_latent_frames = torch.cat([memory_latent_frames, xs_pred[n_context_frames:]])
-        xs_pred = self.decode(xs_pred[n_context_frames:].to(device)).cpu()
-
-        return xs_pred.cpu().numpy(), memory_latent_frames.cpu().numpy(), memory_actions.cpu().numpy(), \
-            memory_poses.cpu().numpy(), memory_c2w.cpu().numpy(), memory_frame_idx.cpu().numpy()
+        raise NotImplementedError(
+            "DeMemWM interactive generation is unsupported until it uses the packed "
+            "[target][anchor][dynamic][revisit] frame-memory API."
+        )

tests/test_dememwm_latent_dataset.py

@@ -596,6 +596,12 @@ class DeMemWMLatentDatasetTests(unittest.TestCase):
         self.assertEqual(calls[2]["kwargs"]["frame_memory_masks"]["revisit"].tolist(), [[True]])
         self.assertAlmostEqual(float(loss), 1.0 / 3.0)
 
+    def test_interactive_generation_fails_until_packed_memory_path_exists(self):
+        from algorithms.dememwm.df_video import DeMemWMMinecraft
+
+        with self.assertRaisesRegex(NotImplementedError, "packed .* frame-memory API"):
+            DeMemWMMinecraft.interactive(object(), None, None, None, None, None, None, None, None, None)
+
     def test_dataset_returns_target_anchor_dynamic_revisit_contract(self):
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)