| { |
| "baseline_summary": { |
| "baseline_heads": "minimal and neural MLP heads", |
| "current_use": "task design, data-contract validation, case studies, and baseline comparison", |
| "split": "chronological single-episode split for public-sample diagnostics", |
| "task_count": 12 |
| }, |
| "directions": [ |
| { |
| "code": "A", |
| "counts": { |
| "diagnostic": 0, |
| "direct": 2, |
| "proxy": 2, |
| "total_links": 4 |
| }, |
| "current_readout": "The sample supports hand trajectory forecasting and contact/object probes, but it does not yet include a full body/shape model or multi-person priors.", |
| "current_status": "partially implemented", |
| "extension_tasks": [ |
| { |
| "current_limit": "This is a motion-energy proxy, not a SMPL/MANO body model or a generative motion prior.", |
| "family": "classification", |
| "id": "body_motion_intensity", |
| "metric_name": "macro-F1", |
| "name": "Body and Hand Motion Intensity" |
| } |
| ], |
| "focus": "Human/hand/body motion, deformation priors, human-object interaction, affordance modeling.", |
| "id": "human_motion", |
| "name": "Human Modeling & Motion Understanding", |
| "next_steps": [ |
| "Add SMPL/SMPL-X or MANO-style body/hand parameter targets where available.", |
| "Train sequence models over multi-episode motion trajectories instead of isolated windows.", |
| "Evaluate affordance prediction on held-out objects and held-out episodes." |
| ], |
| "preferred_background": "Human pose/shape estimation, SMPL-style models, motion capture, or motion generation.", |
| "task_ids": [ |
| "timeline_action", |
| "hand_trajectory_forecast", |
| "contact_prediction", |
| "object_relevance" |
| ], |
| "tasks": [ |
| { |
| "architecture_family": "multiclass classifier", |
| "case_study": "In the coffee-making sample, if the 20-frame window is during a pouring moment, the task asks the model to output an action such as Pour coffee or Pour milk into coffee.", |
| "current_limit": "Chronological single-episode split creates unseen future action classes.", |
| "direction_roles": { |
| "A": "proxy", |
| "C": "direct" |
| }, |
| "display_name": "Action Recognition", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "timeline_action", |
| "input": "One 20-frame window represented by the current feature vector: video/audio/depth summaries, pose, SLAM/camera pose, motion capture, IMU, calibration, and language-derived context.", |
| "input_short": "20-frame multimodal window", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.05, |
| "name": "macro-F1", |
| "neural_mlp": 0.0148 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "current action class", |
| "primary_direction": "C", |
| "process_short": "window features -> action label builder -> classifier", |
| "research_name": "Egocentric Action Recognition", |
| "why": "Reads egocentric sensor state as the current human action; also provides a weak human-motion readout." |
| }, |
| { |
| "architecture_family": "continuous regressor", |
| "case_study": "When the hand is moving toward a cup or bottle, the model predicts the future 3D hand-joint path.", |
| "current_limit": "Forecasting is window-level and not yet a full sequence or policy model.", |
| "direction_roles": { |
| "A": "direct", |
| "C": "proxy" |
| }, |
| "display_name": "Hand Trajectory Forecasting", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/hand_trajectory_forecast/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/hand_trajectory_forecast/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "forecast", |
| "id": "hand_trajectory_forecast", |
| "input": "The current all-modality window vector at time t.", |
| "input_short": "current multimodal window", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "lower", |
| "key": "mpjpe", |
| "minimal": 0.8647, |
| "name": "MPJPE", |
| "neural_mlp": 0.1079 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "video", |
| "depth", |
| "pose_slam", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "future hand-joint trajectory", |
| "primary_direction": "A", |
| "process_short": "current features -> future mocap target -> regression head", |
| "research_name": "3D Hand Motion Forecasting", |
| "why": "Directly predicts human hand motion and supports hand-object interaction modeling." |
| }, |
| { |
| "architecture_family": "binary classifier", |
| "case_study": "During manipulation, the hand may touch a cup, table, or bottle. The task asks whether any contact is happening.", |
| "current_limit": "The public sample is degenerate for this target because one class dominates.", |
| "direction_roles": { |
| "A": "direct", |
| "C": "proxy" |
| }, |
| "display_name": "Contact State Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "contact_prediction", |
| "input": "Non-contact and non-caption feature blocks, so the answer is not directly leaked from the target labels.", |
| "input_short": "non-contact, non-caption features", |
| "metric": { |
| "better_baseline": "tie", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 1.0, |
| "name": "macro-F1", |
| "neural_mlp": 1.0 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "video", |
| "depth", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "contact or no contact", |
| "primary_direction": "A", |
| "process_short": "feature filter -> contact target -> binary classifier", |
| "research_name": "Human-Object Contact Prediction", |
| "why": "Targets physical interaction state, a core affordance and manipulation signal." |
| }, |
| { |
| "architecture_family": "multi-label classifier", |
| "case_study": "If the person is pouring milk into coffee, relevant objects may include milk, cup, coffee, or container-like items.", |
| "current_limit": "Object labels are language-derived and sparse in one episode.", |
| "direction_roles": { |
| "A": "proxy", |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Object Relevance Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/object_relevance/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/object_relevance/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/object_relevance/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/object_relevance/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "supervised", |
| "id": "object_relevance", |
| "input": "Non-caption feature blocks, so the model must infer objects from sensors rather than copying the caption words.", |
| "input_short": "non-caption multimodal features", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "micro_f1", |
| "minimal": 0.1803, |
| "name": "micro-F1", |
| "neural_mlp": 0.1679 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "relevant object set", |
| "primary_direction": "C", |
| "process_short": "object vocabulary -> multi-hot labels -> sigmoid heads", |
| "research_name": "Object-Centric Interaction Recognition", |
| "why": "Connects egocentric activity to manipulated objects and early object-centric state." |
| } |
| ] |
| }, |
| { |
| "code": "B", |
| "counts": { |
| "diagnostic": 1, |
| "direct": 0, |
| "proxy": 2, |
| "total_links": 3 |
| }, |
| "current_readout": "The current suite checks cross-modal alignment and depth/video reconstruction proxies; it does not yet train a renderer or reconstruct geometry.", |
| "current_status": "proxy tasks only", |
| "extension_tasks": [ |
| { |
| "current_limit": "This checks calibrated multi-view signal, but it is still feature retrieval, not NeRF, Gaussian Splatting, or novel-view synthesis.", |
| "family": "retrieval", |
| "id": "multi_view_consistency_retrieval", |
| "metric_name": "MRR", |
| "name": "Multi-View Consistency Retrieval" |
| } |
| ], |
| "focus": "Multi-view dynamic scene reconstruction, NeRF/Gaussian Splatting, novel-view synthesis.", |
| "id": "reconstruction_rendering", |
| "name": "3D/4D Reconstruction & Neural Rendering", |
| "next_steps": [ |
| "Use calibrated multi-view video plus SLAM pose to build per-episode camera trajectories.", |
| "Add depth-supervised point clouds, TSDF, Gaussian Splatting, or NeRF baselines.", |
| "Evaluate novel-view synthesis and temporal consistency across held-out views/time." |
| ], |
| "preferred_background": "3D reconstruction, neural rendering, camera calibration, and bundle adjustment.", |
| "task_ids": [ |
| "cross_modal_retrieval", |
| "modality_reconstruction", |
| "misalignment_detection" |
| ], |
| "tasks": [ |
| { |
| "architecture_family": "two-tower retrieval head", |
| "case_study": "Use motion, IMU, and camera-pose signals from a pouring moment to retrieve the matching depth/video representation for that same moment.", |
| "current_limit": "Retrieval shows an alignment signal, not geometric reconstruction.", |
| "direction_roles": { |
| "B": "proxy", |
| "C": "diagnostic", |
| "D": "proxy" |
| }, |
| "display_name": "Cross-Modal Retrieval", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/cross_modal_retrieval/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/cross_modal_retrieval/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "retrieval", |
| "id": "cross_modal_retrieval", |
| "input": "Query side: motion, IMU, and camera/pose features. Candidate side: depth and video features.", |
| "input_short": "motion/IMU/pose query; depth/video candidates", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "mrr", |
| "minimal": 0.2693, |
| "name": "MRR", |
| "neural_mlp": 0.13 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "pose_slam", |
| "depth", |
| "video" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "ranked visual windows", |
| "primary_direction": "C", |
| "process_short": "modality split -> projection -> nearest-neighbor ranker", |
| "research_name": "Multimodal Representation Retrieval", |
| "why": "Tests whether synchronized modalities identify the same 4D moment, a prerequisite for reconstruction and world modeling." |
| }, |
| { |
| "architecture_family": "feature regressor", |
| "case_study": "Given motion, IMU, and camera-pose signals while the hand moves, predict the matching depth/video feature vector.", |
| "current_limit": "Feature-vector reconstruction is not pixel, depth-map, mesh, NeRF, or Gaussian reconstruction.", |
| "direction_roles": { |
| "B": "proxy", |
| "D": "proxy" |
| }, |
| "display_name": "Cross-Modal Reconstruction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/modality_reconstruction/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/modality_reconstruction/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "forecast", |
| "id": "modality_reconstruction", |
| "input": "Motion, IMU, and camera/pose features as input; depth/video features as the regression target.", |
| "input_short": "motion, IMU, and camera/pose features", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "r2", |
| "minimal": -0.0153, |
| "name": "R2", |
| "neural_mlp": -0.0102 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "pose_slam", |
| "depth", |
| "video" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "reconstructed depth/video vector", |
| "primary_direction": "B", |
| "process_short": "source-target split -> scaler -> regression head", |
| "research_name": "Modality Feature Reconstruction", |
| "why": "Predicts visual/depth state from non-target sensors as a weak reconstruction/world-model objective." |
| }, |
| { |
| "architecture_family": "pairwise classifier", |
| "case_study": "Motion from a pouring moment is paired with video/depth from several windows later. The task asks the model to detect that mismatch.", |
| "current_limit": "Synthetic shifts diagnose alignment but do not solve calibration or mapping.", |
| "direction_roles": { |
| "B": "diagnostic", |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "display_name": "Multimodal Synchronization Detection", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/misalignment_detection/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/misalignment_detection/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/misalignment_detection/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/misalignment_detection/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "misalignment_detection", |
| "input": "A motion-side feature group and a visual/depth-side feature group, either aligned or artificially shifted.", |
| "input_short": "motion-side and visual/depth-side feature groups", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "f1", |
| "minimal": 0.5052, |
| "name": "F1", |
| "neural_mlp": 0.7153 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "video", |
| "depth", |
| "pose_slam" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "aligned or shifted", |
| "primary_direction": "C", |
| "process_short": "aligned/shifted pairs -> feature combiner -> binary classifier", |
| "research_name": "Cross-Modal Misalignment Detection", |
| "why": "Detects temporal desynchronization, a key data-quality gate for multimodal reconstruction and world models." |
| } |
| ] |
| }, |
| { |
| "code": "C", |
| "counts": { |
| "diagnostic": 3, |
| "direct": 6, |
| "proxy": 2, |
| "total_links": 11 |
| }, |
| "current_readout": "Most of the 12 tasks directly target egocentric action, task state, interaction, grounding, and alignment.", |
| "current_status": "strongest implemented track", |
| "extension_tasks": [ |
| { |
| "current_limit": "This is an action-structure probe inside one episode, not a general intent model across homes, people, or tasks.", |
| "family": "regression", |
| "id": "action_phase_progress", |
| "metric_name": "MAE", |
| "name": "Action Phase Progress Estimation" |
| } |
| ], |
| "focus": "Egocentric action and intention understanding, hand-object interaction, gaze/attention modeling, task structure modeling.", |
| "id": "egocentric_interaction", |
| "name": "Egocentric Vision & Interaction", |
| "next_steps": [ |
| "Move from single-episode chronological splits to held-out-episode splits.", |
| "Use the audio signal with stronger multimodal backbones for action, intent, and grounding.", |
| "Evaluate long-horizon task success prediction and action-conditioned generation." |
| ], |
| "preferred_background": "Video understanding, action recognition, or egocentric vision.", |
| "task_ids": [ |
| "timeline_action", |
| "timeline_subtask", |
| "transition_detection", |
| "next_action", |
| "hand_trajectory_forecast", |
| "contact_prediction", |
| "object_relevance", |
| "caption_grounding", |
| "cross_modal_retrieval", |
| "temporal_order", |
| "misalignment_detection" |
| ], |
| "tasks": [ |
| { |
| "architecture_family": "multiclass classifier", |
| "case_study": "In the coffee-making sample, if the 20-frame window is during a pouring moment, the task asks the model to output an action such as Pour coffee or Pour milk into coffee.", |
| "current_limit": "Chronological single-episode split creates unseen future action classes.", |
| "direction_roles": { |
| "A": "proxy", |
| "C": "direct" |
| }, |
| "display_name": "Action Recognition", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "timeline_action", |
| "input": "One 20-frame window represented by the current feature vector: video/audio/depth summaries, pose, SLAM/camera pose, motion capture, IMU, calibration, and language-derived context.", |
| "input_short": "20-frame multimodal window", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.05, |
| "name": "macro-F1", |
| "neural_mlp": 0.0148 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "current action class", |
| "primary_direction": "C", |
| "process_short": "window features -> action label builder -> classifier", |
| "research_name": "Egocentric Action Recognition", |
| "why": "Reads egocentric sensor state as the current human action; also provides a weak human-motion readout." |
| }, |
| { |
| "architecture_family": "multiclass classifier", |
| "case_study": "A pouring action may belong to a broader subtask such as preparing or pouring a drink. The model predicts that broader stage instead of a fine action.", |
| "current_limit": "Single-episode ordering makes future subtasks hard to generalize.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Procedure Step Recognition", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "timeline_subtask", |
| "input": "The same all-modality window vector used by action recognition.", |
| "input_short": "20-frame multimodal window", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.0506, |
| "name": "macro-F1", |
| "neural_mlp": 0.0281 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "current procedure step", |
| "primary_direction": "C", |
| "process_short": "window features -> subtask label builder -> classifier", |
| "research_name": "Temporal Subtask Recognition", |
| "why": "Segments egocentric task state and provides a first proxy for symbolic world/task state." |
| }, |
| { |
| "architecture_family": "binary classifier", |
| "case_study": "When the demonstrator changes from preparing to pouring, the model should flag a boundary instead of a steady action window.", |
| "current_limit": "Boundary class is sparse, so accuracy alone is misleading.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "diagnostic" |
| }, |
| "display_name": "Action Boundary Detection", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "transition_detection", |
| "input": "One all-modality window vector plus labels derived from action-change timestamps.", |
| "input_short": "current window with boundary target", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.6118, |
| "name": "macro-F1", |
| "neural_mlp": 0.5862 |
| }, |
| "modalities": [ |
| "video", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "boundary or steady", |
| "primary_direction": "C", |
| "process_short": "action changes -> boundary labels -> binary classifier", |
| "research_name": "Temporal Action Segmentation", |
| "why": "Localizes egocentric task boundaries and diagnoses temporal state changes." |
| }, |
| { |
| "architecture_family": "future-label classifier", |
| "case_study": "If a window shows the person preparing to pour, the target can be the action 20 frames later, such as the start of pouring.", |
| "current_limit": "Unseen future labels dominate the single-episode chronological test.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Next-Action Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "next_action", |
| "input": "The current all-modality window vector at time t.", |
| "input_short": "current window at time t", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.0593, |
| "name": "macro-F1", |
| "neural_mlp": 0.0419 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "action at t+20 frames", |
| "primary_direction": "C", |
| "process_short": "current features -> future label shift -> classifier", |
| "research_name": "Short-Horizon Intention Prediction", |
| "why": "Tests action intention/task-flow prediction from egocentric context." |
| }, |
| { |
| "architecture_family": "continuous regressor", |
| "case_study": "When the hand is moving toward a cup or bottle, the model predicts the future 3D hand-joint path.", |
| "current_limit": "Forecasting is window-level and not yet a full sequence or policy model.", |
| "direction_roles": { |
| "A": "direct", |
| "C": "proxy" |
| }, |
| "display_name": "Hand Trajectory Forecasting", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/hand_trajectory_forecast/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/hand_trajectory_forecast/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "forecast", |
| "id": "hand_trajectory_forecast", |
| "input": "The current all-modality window vector at time t.", |
| "input_short": "current multimodal window", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "lower", |
| "key": "mpjpe", |
| "minimal": 0.8647, |
| "name": "MPJPE", |
| "neural_mlp": 0.1079 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "video", |
| "depth", |
| "pose_slam", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "future hand-joint trajectory", |
| "primary_direction": "A", |
| "process_short": "current features -> future mocap target -> regression head", |
| "research_name": "3D Hand Motion Forecasting", |
| "why": "Directly predicts human hand motion and supports hand-object interaction modeling." |
| }, |
| { |
| "architecture_family": "binary classifier", |
| "case_study": "During manipulation, the hand may touch a cup, table, or bottle. The task asks whether any contact is happening.", |
| "current_limit": "The public sample is degenerate for this target because one class dominates.", |
| "direction_roles": { |
| "A": "direct", |
| "C": "proxy" |
| }, |
| "display_name": "Contact State Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "contact_prediction", |
| "input": "Non-contact and non-caption feature blocks, so the answer is not directly leaked from the target labels.", |
| "input_short": "non-contact, non-caption features", |
| "metric": { |
| "better_baseline": "tie", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 1.0, |
| "name": "macro-F1", |
| "neural_mlp": 1.0 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "video", |
| "depth", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "contact or no contact", |
| "primary_direction": "A", |
| "process_short": "feature filter -> contact target -> binary classifier", |
| "research_name": "Human-Object Contact Prediction", |
| "why": "Targets physical interaction state, a core affordance and manipulation signal." |
| }, |
| { |
| "architecture_family": "multi-label classifier", |
| "case_study": "If the person is pouring milk into coffee, relevant objects may include milk, cup, coffee, or container-like items.", |
| "current_limit": "Object labels are language-derived and sparse in one episode.", |
| "direction_roles": { |
| "A": "proxy", |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Object Relevance Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/object_relevance/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/object_relevance/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/object_relevance/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/object_relevance/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "supervised", |
| "id": "object_relevance", |
| "input": "Non-caption feature blocks, so the model must infer objects from sensors rather than copying the caption words.", |
| "input_short": "non-caption multimodal features", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "micro_f1", |
| "minimal": 0.1803, |
| "name": "micro-F1", |
| "neural_mlp": 0.1679 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "relevant object set", |
| "primary_direction": "C", |
| "process_short": "object vocabulary -> multi-hot labels -> sigmoid heads", |
| "research_name": "Object-Centric Interaction Recognition", |
| "why": "Connects egocentric activity to manipulated objects and early object-centric state." |
| }, |
| { |
| "architecture_family": "retrieval ranker", |
| "case_study": "A query like Pour milk into coffee should rank the windows from the actual pouring moment higher than unrelated windows.", |
| "current_limit": "Bag-of-objects language features are too weak for rich grounding.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Language Grounding", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/caption_grounding/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/caption_grounding/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "retrieval", |
| "id": "caption_grounding", |
| "input": "Caption/object/interaction query features and a set of candidate sensor-window features.", |
| "input_short": "text-like query and candidate windows", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "mrr", |
| "minimal": 0.016, |
| "name": "MRR", |
| "neural_mlp": 0.0168 |
| }, |
| "modalities": [ |
| "language", |
| "video", |
| "depth", |
| "pose_slam" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "ranked matching moments", |
| "primary_direction": "C", |
| "process_short": "query features -> candidate index -> cosine ranker", |
| "research_name": "Language-to-Moment Grounding", |
| "why": "Grounds language annotation into egocentric sensor time and task state." |
| }, |
| { |
| "architecture_family": "two-tower retrieval head", |
| "case_study": "Use motion, IMU, and camera-pose signals from a pouring moment to retrieve the matching depth/video representation for that same moment.", |
| "current_limit": "Retrieval shows an alignment signal, not geometric reconstruction.", |
| "direction_roles": { |
| "B": "proxy", |
| "C": "diagnostic", |
| "D": "proxy" |
| }, |
| "display_name": "Cross-Modal Retrieval", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/cross_modal_retrieval/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/cross_modal_retrieval/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "retrieval", |
| "id": "cross_modal_retrieval", |
| "input": "Query side: motion, IMU, and camera/pose features. Candidate side: depth and video features.", |
| "input_short": "motion/IMU/pose query; depth/video candidates", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "mrr", |
| "minimal": 0.2693, |
| "name": "MRR", |
| "neural_mlp": 0.13 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "pose_slam", |
| "depth", |
| "video" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "ranked visual windows", |
| "primary_direction": "C", |
| "process_short": "modality split -> projection -> nearest-neighbor ranker", |
| "research_name": "Multimodal Representation Retrieval", |
| "why": "Tests whether synchronized modalities identify the same 4D moment, a prerequisite for reconstruction and world modeling." |
| }, |
| { |
| "architecture_family": "pairwise classifier", |
| "case_study": "If window A shows reaching and window B shows pouring, the model should distinguish A then B from B then A.", |
| "current_limit": "Only local adjacent ordering, not long-horizon causal modeling.", |
| "direction_roles": { |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "display_name": "Temporal Order Verification", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/temporal_order/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/temporal_order/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/temporal_order/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/temporal_order/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "temporal_order", |
| "input": "A pair of adjacent window vectors, plus their difference vector.", |
| "input_short": "two adjacent windows plus difference vector", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "f1", |
| "minimal": 0.54, |
| "name": "F1", |
| "neural_mlp": 0.852 |
| }, |
| "modalities": [ |
| "video", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "correct or reversed", |
| "primary_direction": "C", |
| "process_short": "pair builder -> feature combiner -> binary classifier", |
| "research_name": "Temporal Order Verification", |
| "why": "Checks whether features encode local time direction and task progression." |
| }, |
| { |
| "architecture_family": "pairwise classifier", |
| "case_study": "Motion from a pouring moment is paired with video/depth from several windows later. The task asks the model to detect that mismatch.", |
| "current_limit": "Synthetic shifts diagnose alignment but do not solve calibration or mapping.", |
| "direction_roles": { |
| "B": "diagnostic", |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "display_name": "Multimodal Synchronization Detection", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/misalignment_detection/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/misalignment_detection/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/misalignment_detection/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/misalignment_detection/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "misalignment_detection", |
| "input": "A motion-side feature group and a visual/depth-side feature group, either aligned or artificially shifted.", |
| "input_short": "motion-side and visual/depth-side feature groups", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "f1", |
| "minimal": 0.5052, |
| "name": "F1", |
| "neural_mlp": 0.7153 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "video", |
| "depth", |
| "pose_slam" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "aligned or shifted", |
| "primary_direction": "C", |
| "process_short": "aligned/shifted pairs -> feature combiner -> binary classifier", |
| "research_name": "Cross-Modal Misalignment Detection", |
| "why": "Detects temporal desynchronization, a key data-quality gate for multimodal reconstruction and world models." |
| } |
| ] |
| }, |
| { |
| "code": "D", |
| "counts": { |
| "diagnostic": 3, |
| "direct": 0, |
| "proxy": 6, |
| "total_links": 9 |
| }, |
| "current_readout": "The current tasks probe temporal structure, object relevance, cross-modal retrieval, and modality prediction, but they do not yet build persistent maps or scene graphs.", |
| "current_status": "early proxy tasks", |
| "extension_tasks": [ |
| { |
| "current_limit": "This is a compact world-model proxy; it does not build a persistent map, scene graph, or object permanence model.", |
| "family": "forecast", |
| "id": "ego_motion_forecast", |
| "metric_name": "MAE", |
| "name": "Short-Horizon Ego-Motion Forecasting" |
| } |
| ], |
| "focus": "Long-term consistent 3D/4D scene mapping, scene graphs, object- and space-centric representations, spatial reasoning.", |
| "id": "world_modeling", |
| "name": "Scene Reconstruction & World Modeling", |
| "next_steps": [ |
| "Convert windows into persistent object/scene-state nodes with timestamps and camera poses.", |
| "Add map consistency, object permanence, and spatial relation prediction tasks.", |
| "Train held-out-episode world models that predict future observations and task state." |
| ], |
| "preferred_background": "Large-scale mapping, semantic reconstruction, or agent world models.", |
| "task_ids": [ |
| "timeline_subtask", |
| "transition_detection", |
| "next_action", |
| "object_relevance", |
| "caption_grounding", |
| "cross_modal_retrieval", |
| "modality_reconstruction", |
| "temporal_order", |
| "misalignment_detection" |
| ], |
| "tasks": [ |
| { |
| "architecture_family": "multiclass classifier", |
| "case_study": "A pouring action may belong to a broader subtask such as preparing or pouring a drink. The model predicts that broader stage instead of a fine action.", |
| "current_limit": "Single-episode ordering makes future subtasks hard to generalize.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Procedure Step Recognition", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "timeline_subtask", |
| "input": "The same all-modality window vector used by action recognition.", |
| "input_short": "20-frame multimodal window", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.0506, |
| "name": "macro-F1", |
| "neural_mlp": 0.0281 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "current procedure step", |
| "primary_direction": "C", |
| "process_short": "window features -> subtask label builder -> classifier", |
| "research_name": "Temporal Subtask Recognition", |
| "why": "Segments egocentric task state and provides a first proxy for symbolic world/task state." |
| }, |
| { |
| "architecture_family": "binary classifier", |
| "case_study": "When the demonstrator changes from preparing to pouring, the model should flag a boundary instead of a steady action window.", |
| "current_limit": "Boundary class is sparse, so accuracy alone is misleading.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "diagnostic" |
| }, |
| "display_name": "Action Boundary Detection", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "transition_detection", |
| "input": "One all-modality window vector plus labels derived from action-change timestamps.", |
| "input_short": "current window with boundary target", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.6118, |
| "name": "macro-F1", |
| "neural_mlp": 0.5862 |
| }, |
| "modalities": [ |
| "video", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "boundary or steady", |
| "primary_direction": "C", |
| "process_short": "action changes -> boundary labels -> binary classifier", |
| "research_name": "Temporal Action Segmentation", |
| "why": "Localizes egocentric task boundaries and diagnoses temporal state changes." |
| }, |
| { |
| "architecture_family": "future-label classifier", |
| "case_study": "If a window shows the person preparing to pour, the target can be the action 20 frames later, such as the start of pouring.", |
| "current_limit": "Unseen future labels dominate the single-episode chronological test.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Next-Action Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "next_action", |
| "input": "The current all-modality window vector at time t.", |
| "input_short": "current window at time t", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.0593, |
| "name": "macro-F1", |
| "neural_mlp": 0.0419 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "action at t+20 frames", |
| "primary_direction": "C", |
| "process_short": "current features -> future label shift -> classifier", |
| "research_name": "Short-Horizon Intention Prediction", |
| "why": "Tests action intention/task-flow prediction from egocentric context." |
| }, |
| { |
| "architecture_family": "multi-label classifier", |
| "case_study": "If the person is pouring milk into coffee, relevant objects may include milk, cup, coffee, or container-like items.", |
| "current_limit": "Object labels are language-derived and sparse in one episode.", |
| "direction_roles": { |
| "A": "proxy", |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Object Relevance Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/object_relevance/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/object_relevance/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/object_relevance/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/object_relevance/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "supervised", |
| "id": "object_relevance", |
| "input": "Non-caption feature blocks, so the model must infer objects from sensors rather than copying the caption words.", |
| "input_short": "non-caption multimodal features", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "micro_f1", |
| "minimal": 0.1803, |
| "name": "micro-F1", |
| "neural_mlp": 0.1679 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "relevant object set", |
| "primary_direction": "C", |
| "process_short": "object vocabulary -> multi-hot labels -> sigmoid heads", |
| "research_name": "Object-Centric Interaction Recognition", |
| "why": "Connects egocentric activity to manipulated objects and early object-centric state." |
| }, |
| { |
| "architecture_family": "retrieval ranker", |
| "case_study": "A query like Pour milk into coffee should rank the windows from the actual pouring moment higher than unrelated windows.", |
| "current_limit": "Bag-of-objects language features are too weak for rich grounding.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Language Grounding", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/caption_grounding/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/caption_grounding/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "retrieval", |
| "id": "caption_grounding", |
| "input": "Caption/object/interaction query features and a set of candidate sensor-window features.", |
| "input_short": "text-like query and candidate windows", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "mrr", |
| "minimal": 0.016, |
| "name": "MRR", |
| "neural_mlp": 0.0168 |
| }, |
| "modalities": [ |
| "language", |
| "video", |
| "depth", |
| "pose_slam" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "ranked matching moments", |
| "primary_direction": "C", |
| "process_short": "query features -> candidate index -> cosine ranker", |
| "research_name": "Language-to-Moment Grounding", |
| "why": "Grounds language annotation into egocentric sensor time and task state." |
| }, |
| { |
| "architecture_family": "two-tower retrieval head", |
| "case_study": "Use motion, IMU, and camera-pose signals from a pouring moment to retrieve the matching depth/video representation for that same moment.", |
| "current_limit": "Retrieval shows an alignment signal, not geometric reconstruction.", |
| "direction_roles": { |
| "B": "proxy", |
| "C": "diagnostic", |
| "D": "proxy" |
| }, |
| "display_name": "Cross-Modal Retrieval", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/cross_modal_retrieval/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/cross_modal_retrieval/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "retrieval", |
| "id": "cross_modal_retrieval", |
| "input": "Query side: motion, IMU, and camera/pose features. Candidate side: depth and video features.", |
| "input_short": "motion/IMU/pose query; depth/video candidates", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "mrr", |
| "minimal": 0.2693, |
| "name": "MRR", |
| "neural_mlp": 0.13 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "pose_slam", |
| "depth", |
| "video" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "ranked visual windows", |
| "primary_direction": "C", |
| "process_short": "modality split -> projection -> nearest-neighbor ranker", |
| "research_name": "Multimodal Representation Retrieval", |
| "why": "Tests whether synchronized modalities identify the same 4D moment, a prerequisite for reconstruction and world modeling." |
| }, |
| { |
| "architecture_family": "feature regressor", |
| "case_study": "Given motion, IMU, and camera-pose signals while the hand moves, predict the matching depth/video feature vector.", |
| "current_limit": "Feature-vector reconstruction is not pixel, depth-map, mesh, NeRF, or Gaussian reconstruction.", |
| "direction_roles": { |
| "B": "proxy", |
| "D": "proxy" |
| }, |
| "display_name": "Cross-Modal Reconstruction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/modality_reconstruction/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/modality_reconstruction/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "forecast", |
| "id": "modality_reconstruction", |
| "input": "Motion, IMU, and camera/pose features as input; depth/video features as the regression target.", |
| "input_short": "motion, IMU, and camera/pose features", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "r2", |
| "minimal": -0.0153, |
| "name": "R2", |
| "neural_mlp": -0.0102 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "pose_slam", |
| "depth", |
| "video" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "reconstructed depth/video vector", |
| "primary_direction": "B", |
| "process_short": "source-target split -> scaler -> regression head", |
| "research_name": "Modality Feature Reconstruction", |
| "why": "Predicts visual/depth state from non-target sensors as a weak reconstruction/world-model objective." |
| }, |
| { |
| "architecture_family": "pairwise classifier", |
| "case_study": "If window A shows reaching and window B shows pouring, the model should distinguish A then B from B then A.", |
| "current_limit": "Only local adjacent ordering, not long-horizon causal modeling.", |
| "direction_roles": { |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "display_name": "Temporal Order Verification", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/temporal_order/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/temporal_order/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/temporal_order/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/temporal_order/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "temporal_order", |
| "input": "A pair of adjacent window vectors, plus their difference vector.", |
| "input_short": "two adjacent windows plus difference vector", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "f1", |
| "minimal": 0.54, |
| "name": "F1", |
| "neural_mlp": 0.852 |
| }, |
| "modalities": [ |
| "video", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "correct or reversed", |
| "primary_direction": "C", |
| "process_short": "pair builder -> feature combiner -> binary classifier", |
| "research_name": "Temporal Order Verification", |
| "why": "Checks whether features encode local time direction and task progression." |
| }, |
| { |
| "architecture_family": "pairwise classifier", |
| "case_study": "Motion from a pouring moment is paired with video/depth from several windows later. The task asks the model to detect that mismatch.", |
| "current_limit": "Synthetic shifts diagnose alignment but do not solve calibration or mapping.", |
| "direction_roles": { |
| "B": "diagnostic", |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "display_name": "Multimodal Synchronization Detection", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/misalignment_detection/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/misalignment_detection/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/misalignment_detection/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/misalignment_detection/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "misalignment_detection", |
| "input": "A motion-side feature group and a visual/depth-side feature group, either aligned or artificially shifted.", |
| "input_short": "motion-side and visual/depth-side feature groups", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "f1", |
| "minimal": 0.5052, |
| "name": "F1", |
| "neural_mlp": 0.7153 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "video", |
| "depth", |
| "pose_slam" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "aligned or shifted", |
| "primary_direction": "C", |
| "process_short": "aligned/shifted pairs -> feature combiner -> binary classifier", |
| "research_name": "Cross-Modal Misalignment Detection", |
| "why": "Detects temporal desynchronization, a key data-quality gate for multimodal reconstruction and world models." |
| } |
| ] |
| } |
| ], |
| "foundation_model_plan": { |
| "decision": { |
| "external_reasoning_reference": "Gemini Robotics", |
| "first_policy_branch_candidates": [ |
| "OpenVLA / OpenVLA-OFT", |
| "openpi pi0/pi0.5", |
| "NVIDIA GR00T" |
| ], |
| "first_world_model_branch": "Cosmos 3", |
| "immediate_trainable_backbone": "Qwen3-Omni" |
| }, |
| "evaluation_additions": [ |
| { |
| "metrics": [ |
| "JSON validity", |
| "macro-F1", |
| "accuracy", |
| "micro-F1" |
| ], |
| "model_families": [ |
| "Qwen3-Omni", |
| "Gemini Robotics reference" |
| ], |
| "target": "structured_task_prediction" |
| }, |
| { |
| "metrics": [ |
| "retrieval rank", |
| "temporal consistency", |
| "feature reconstruction", |
| "qualitative visual inspection" |
| ], |
| "model_families": [ |
| "Cosmos 3" |
| ], |
| "target": "future_state_prediction" |
| }, |
| { |
| "metrics": [ |
| "transition accuracy", |
| "contact accuracy", |
| "next-action accuracy" |
| ], |
| "model_families": [ |
| "Cosmos 3", |
| "OpenVLA", |
| "openpi", |
| "GR00T" |
| ], |
| "target": "action_conditioned_dynamics" |
| }, |
| { |
| "metrics": [ |
| "held-out episode metrics", |
| "held-out session metrics", |
| "leakage audit" |
| ], |
| "model_families": [ |
| "all trainable branches" |
| ], |
| "target": "cross_episode_generalization" |
| } |
| ], |
| "execution_order": [ |
| { |
| "action": "Stage at least 32 valid Xperience-10M episodes with held-out episode split.", |
| "name": "Data gate", |
| "step": 1 |
| }, |
| { |
| "action": "Run Qwen3-Omni LoRA to establish the full train/eval loop.", |
| "name": "First held-out baseline", |
| "step": 2 |
| }, |
| { |
| "action": "Run 3-8 episode dry runs for Qwen3-Omni prompt/LoRA, Cosmos 3 preprocessing, and one policy candidate.", |
| "name": "Model-selection dry run", |
| "step": 3 |
| }, |
| { |
| "action": "Promote Cosmos 3 if future-window/action-conditioned preprocessing fits storage and compute.", |
| "name": "World-model branch", |
| "step": 4 |
| }, |
| { |
| "action": "Promote OpenVLA/openpi/GR00T after action target conversion and retargeting artifacts are traceable.", |
| "name": "Policy branch", |
| "step": 5 |
| }, |
| { |
| "action": "Publish branch results only with real manifests, predictions, metrics, and qualitative examples.", |
| "name": "Publication rule", |
| "step": 6 |
| } |
| ], |
| "model_families": [ |
| { |
| "best_role": "First selected-episode multimodal LoRA pilot and structured task predictor.", |
| "category": "omni_instruction_model", |
| "current_decision": "keep_as_first_pilot", |
| "entry_condition": "Selected episodes staged with held-out episode split.", |
| "family": "Qwen3-Omni", |
| "openness": "open_weights_available_from_official_hf_repo", |
| "priority": 1, |
| "public_source": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct", |
| "xperience10m_fit": [ |
| "RGB/fisheye video, embedded audio, and language prompts can enter directly.", |
| "Depth, pose/SLAM, mocap, contacts, and IMU enter through the existing sensor bridge.", |
| "Matches current task outputs: labels, structured JSON, captions, and short decisions." |
| ] |
| }, |
| { |
| "best_role": "Embodied world modeling, action generation, future-window prediction, and synthetic-data expansion.", |
| "category": "world_foundation_model", |
| "current_decision": "add_as_first_world_model_branch_after_data_gate", |
| "entry_condition": "Multi-episode data plus enough storage/compute for generated or latent video-state outputs.", |
| "family": "Cosmos 3", |
| "openness": "track_official_nvidia_release_and_available_weights", |
| "priority": 2, |
| "public_source": "https://www.nvidia.com/en-us/ai/cosmos/", |
| "xperience10m_fit": [ |
| "Uses video streams as visual state.", |
| "Uses pose/SLAM, depth, mocap, IMU, and language as physical-world conditioning signals.", |
| "Better aligned with prediction/generation objectives than simple label classification." |
| ] |
| }, |
| { |
| "best_role": "Humanoid action understanding, retargeting, contact/action prediction, and embodied skill transfer.", |
| "category": "humanoid_policy_foundation_model", |
| "current_decision": "track_as_humanoid_policy_branch", |
| "entry_condition": "Retargeting artifact and action-space definition exist.", |
| "family": "NVIDIA GR00T", |
| "openness": "track_official_nvidia_release_and_tooling", |
| "priority": 3, |
| "public_source": "https://developer.nvidia.com/isaac/gr00t", |
| "xperience10m_fit": [ |
| "Hand/body mocap and contact cues can be retargeted into humanoid state/action targets.", |
| "Egocentric video plus human motion can support affordance and interaction tasks." |
| ] |
| }, |
| { |
| "best_role": "Open robot-policy baseline after observations and action labels are converted into a VLA format.", |
| "category": "vision_language_action_policy", |
| "current_decision": "candidate_after_action_space_design", |
| "entry_condition": "Window-to-action-token conversion is implemented and audited.", |
| "family": "OpenVLA / OpenVLA-OFT", |
| "openness": "open_project_and_weights", |
| "priority": 4, |
| "public_source": "https://openvla.github.io/", |
| "xperience10m_fit": [ |
| "Good candidate when each window is expressed as visual observation, instruction/context, and action token.", |
| "Requires an explicit action target; current human egocentric labels are not robot controls by default." |
| ] |
| }, |
| { |
| "best_role": "Action-chunking, policy fine-tuning, and embodiment-transfer experiments.", |
| "category": "robot_policy_model", |
| "current_decision": "candidate_policy_branch", |
| "entry_condition": "Action target and train/eval protocol exist for at least 64 episodes.", |
| "family": "openpi pi0/pi0.5", |
| "openness": "open_source_policy_training_stack", |
| "priority": 5, |
| "public_source": "https://github.com/Physical-Intelligence/openpi", |
| "xperience10m_fit": [ |
| "Useful once hand trajectories, contacts, or retargeted body motion are converted into policy targets.", |
| "Better for policy branch than for current structured task JSON outputs." |
| ] |
| }, |
| { |
| "best_role": "Qualitative reasoning reference, annotation helper, and external comparison when API access exists.", |
| "category": "closed_embodied_reasoning_reference", |
| "current_decision": "external_reference_only", |
| "entry_condition": "API/access exists and outputs are logged separately from trainable model metrics.", |
| "family": "Gemini Robotics", |
| "openness": "closed_or_limited_access", |
| "priority": 6, |
| "public_source": "https://deepmind.google/discover/blog/gemini-robotics-brings-ai-into-the-physical-world/", |
| "xperience10m_fit": [ |
| "Can help reason over egocentric scenes and task descriptions.", |
| "Not a local fine-tune target for this repo." |
| ] |
| }, |
| { |
| "best_role": "Cheaper policy baselines for observation-to-action experiments.", |
| "category": "lightweight_robot_policy_baselines", |
| "current_decision": "optional_baseline_after_data_staging", |
| "entry_condition": "Action labels and baseline protocol exist.", |
| "family": "Octo / SmolVLA-style lightweight policies", |
| "openness": "open_projects", |
| "priority": 7, |
| "public_source": "https://github.com/huggingface/lerobot", |
| "xperience10m_fit": [ |
| "Useful after action target design.", |
| "Less directly omni-modal than Qwen3-Omni or Cosmos 3." |
| ] |
| } |
| ], |
| "source_links": [ |
| { |
| "label": "Qwen3-Omni official HF model", |
| "url": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct" |
| }, |
| { |
| "label": "NVIDIA Cosmos", |
| "url": "https://www.nvidia.com/en-us/ai/cosmos/" |
| }, |
| { |
| "label": "NVIDIA Isaac GR00T", |
| "url": "https://developer.nvidia.com/isaac/gr00t" |
| }, |
| { |
| "label": "OpenVLA", |
| "url": "https://openvla.github.io/" |
| }, |
| { |
| "label": "openpi", |
| "url": "https://github.com/Physical-Intelligence/openpi" |
| }, |
| { |
| "label": "Gemini Robotics", |
| "url": "https://deepmind.google/discover/blog/gemini-robotics-brings-ai-into-the-physical-world/" |
| }, |
| { |
| "label": "Octo", |
| "url": "https://octo-models.github.io/" |
| }, |
| { |
| "label": "LeRobot / SmolVLA", |
| "url": "https://github.com/huggingface/lerobot" |
| } |
| ], |
| "status": "planning_artifact" |
| }, |
| "generated_at_utc": "2026-06-03T14:43:22+00:00", |
| "omni_plan": { |
| "adapter": "LoRA rank 16, alpha 32, dropout 0.05", |
| "backbone": "Qwen/Qwen3-Omni-30B-A3B-Instruct", |
| "evaluation": [ |
| "JSON validity", |
| "action macro-F1", |
| "subtask accuracy", |
| "transition accuracy", |
| "next-action accuracy", |
| "contact accuracy", |
| "object micro-F1", |
| "held-out episode count" |
| ], |
| "first_pilot": "32 held-out-episode pilot after valid episodes are staged", |
| "training_unit": "episode-level split, window-level supervised examples" |
| }, |
| "phases": [ |
| { |
| "completion_evidence": [ |
| "PROJECT_STATUS.md", |
| "EVALUATION_PROTOCOL.md", |
| "RESEARCH_TAKEAWAYS.md", |
| "docs/data/summary_metrics.json", |
| "results/episode_task_suite/summary_report.json" |
| ], |
| "deliverables": [ |
| "1161 aligned windows", |
| "12 task contracts", |
| "minimal baseline heads", |
| "neural MLP heads", |
| "modality atlas", |
| "task walkthroughs", |
| "derived figures" |
| ], |
| "entry_condition": "One public Xperience-10M sample episode is available.", |
| "id": "public_sample_task_lab", |
| "name": "Public-Sample Task Lab", |
| "reader_takeaway": "The public sample supports task design, feature contracts, walkthroughs, and baseline comparisons.", |
| "stage": "now", |
| "status": "implemented" |
| }, |
| { |
| "completion_evidence": [ |
| "results/omni_finetune/DATA_ACCESS_STATUS.md", |
| "results/omni_finetune/MULTI_EPISODE_ACCESS_STATUS.md", |
| "results/omni_finetune/source_discovery.json" |
| ], |
| "deliverables": [ |
| "128 selected episodes", |
| "episode manifest", |
| "missing-view manifest", |
| "held-out episode split", |
| "source-discovery report" |
| ], |
| "entry_condition": "Full-dataset access and enough storage for selected episodes.", |
| "id": "multi_episode_data_staging", |
| "name": "Multi-Episode Data Staging", |
| "reader_takeaway": "The next scale decision is data staging, with train/test separation at the episode level.", |
| "stage": "scale_up", |
| "status": "active" |
| }, |
| { |
| "completion_evidence": [ |
| "dataset_manifest.json", |
| "training_metadata.json", |
| "progress.jsonl", |
| "metrics.json", |
| "predictions.jsonl", |
| "RUN_REPORT.md" |
| ], |
| "deliverables": [ |
| "dataset JSONL/media manifests", |
| "LoRA adapter checkpoint", |
| "progress logs", |
| "held-out predictions", |
| "metrics", |
| "confusion matrices", |
| "run report" |
| ], |
| "entry_condition": "Selected episodes are staged locally with no train/test episode leakage.", |
| "id": "qwen3_omni_lora_pilot", |
| "name": "Qwen3-Omni LoRA Pilot", |
| "reader_takeaway": "The first omni-model pilot should establish a complete held-out-episode training and evaluation loop.", |
| "stage": "omni", |
| "status": "next" |
| }, |
| { |
| "completion_evidence": [ |
| "FOUNDATION_MODEL_PLAN.md", |
| "docs/data/foundation_model_plan.json", |
| "research_roadmap_interactive.json" |
| ], |
| "deliverables": [ |
| "backbone registry", |
| "Cosmos 3 world-model branch plan", |
| "Qwen3-Omni LoRA baseline plan", |
| "OpenVLA/openpi/GR00T policy-branch candidates", |
| "model-specific evaluation additions" |
| ], |
| "entry_condition": "The selected relay is staged or a 3-8 episode dry run is staged for preprocessing checks.", |
| "id": "foundation_model_selection_matrix", |
| "name": "Foundation-Model Selection Matrix", |
| "reader_takeaway": "Qwen3-Omni remains the first trainable held-out pilot; Cosmos 3 is the first world-model branch; VLA/policy models wait for explicit action targets.", |
| "stage": "omni", |
| "status": "next" |
| }, |
| { |
| "completion_evidence": [ |
| "held-out metrics by session", |
| "held-out metrics by task", |
| "held-out metrics by modality", |
| "ablation tables", |
| "qualitative error analysis" |
| ], |
| "deliverables": [ |
| "split-by-session metrics", |
| "modality ablations", |
| "calibration/object/language error analysis", |
| "missing-view sensitivity analysis" |
| ], |
| "entry_condition": "The selected-episode pilot trains and evaluates cleanly.", |
| "id": "robustness_run_64_128_episode", |
| "name": "64-128 Episode Robustness Run", |
| "reader_takeaway": "The robustness run tests whether the pilot conclusions survive broader sessions and missing modalities.", |
| "stage": "future", |
| "status": "planned" |
| }, |
| { |
| "completion_evidence": [ |
| "task-specific held-out evaluations", |
| "qualitative inspection", |
| "updated model cards" |
| ], |
| "deliverables": [ |
| "Cosmos 3 future-window or action-conditioned world-model probe", |
| "OpenVLA/openpi/GR00T action-policy baseline", |
| "audio/video/depth/pose/mocap conditioning audit", |
| "affordance and object-interaction tasks", |
| "synthetic-data usefulness test" |
| ], |
| "entry_condition": "Enough multi-episode data, compute budget, and model-specific action/world-state targets.", |
| "id": "foundation_world_model_extensions", |
| "name": "Cosmos 3 and Policy-Model Extensions", |
| "reader_takeaway": "The long-term direction is richer multimodal representation learning for embodied-AI reasoning, with model branches chosen by task fit rather than by a single default backbone.", |
| "stage": "future", |
| "status": "planned" |
| } |
| ], |
| "scale_up": { |
| "access_status": "Full-dataset access is granted; selected multi-episode relay is in progress.", |
| "candidate_scan_top_level_sessions": 802, |
| "estimated_bytes": 298188841943, |
| "exclude": [ |
| "visualization.rrd" |
| ], |
| "selection_strategy": "stratified_round_robin_by_top_level_session", |
| "status": "selected_relay_in_progress", |
| "target_episodes": 128, |
| "valid_candidates": 12102 |
| }, |
| "scope": { |
| "feature_blocks": 18, |
| "feature_dim": 8546, |
| "num_frames": 5821, |
| "num_windows": 1161, |
| "sample_episode_count": 1, |
| "stride_frames": 5, |
| "warning": "These walkthroughs explain task contracts on one public sample episode; cross-episode performance requires held-out episodes.", |
| "window_frames": 20 |
| }, |
| "source_files": [ |
| "docs/data/research_directions.json", |
| "docs/data/task_walkthroughs.json", |
| "docs/data/research_roadmap.json", |
| "docs/data/foundation_model_plan.json", |
| "docs/data/summary_metrics.json", |
| "docs/data/research_direction_extensions.json", |
| "results/episode_task_suite/summary_report.json", |
| "results/episode_task_suite/feature_manifest.json" |
| ], |
| "tasks": [ |
| { |
| "architecture_family": "multiclass classifier", |
| "case_study": "In the coffee-making sample, if the 20-frame window is during a pouring moment, the task asks the model to output an action such as Pour coffee or Pour milk into coffee.", |
| "current_limit": "Chronological single-episode split creates unseen future action classes.", |
| "direction_roles": { |
| "A": "proxy", |
| "C": "direct" |
| }, |
| "display_name": "Action Recognition", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_action/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_action/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "timeline_action", |
| "input": "One 20-frame window represented by the current feature vector: video/audio/depth summaries, pose, SLAM/camera pose, motion capture, IMU, calibration, and language-derived context.", |
| "input_short": "20-frame multimodal window", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.05, |
| "name": "macro-F1", |
| "neural_mlp": 0.0148 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "current action class", |
| "primary_direction": "C", |
| "process_short": "window features -> action label builder -> classifier", |
| "research_name": "Egocentric Action Recognition", |
| "why": "Reads egocentric sensor state as the current human action; also provides a weak human-motion readout." |
| }, |
| { |
| "architecture_family": "multiclass classifier", |
| "case_study": "A pouring action may belong to a broader subtask such as preparing or pouring a drink. The model predicts that broader stage instead of a fine action.", |
| "current_limit": "Single-episode ordering makes future subtasks hard to generalize.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Procedure Step Recognition", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/timeline_subtask/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/timeline_subtask/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "timeline_subtask", |
| "input": "The same all-modality window vector used by action recognition.", |
| "input_short": "20-frame multimodal window", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.0506, |
| "name": "macro-F1", |
| "neural_mlp": 0.0281 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "current procedure step", |
| "primary_direction": "C", |
| "process_short": "window features -> subtask label builder -> classifier", |
| "research_name": "Temporal Subtask Recognition", |
| "why": "Segments egocentric task state and provides a first proxy for symbolic world/task state." |
| }, |
| { |
| "architecture_family": "binary classifier", |
| "case_study": "When the demonstrator changes from preparing to pouring, the model should flag a boundary instead of a steady action window.", |
| "current_limit": "Boundary class is sparse, so accuracy alone is misleading.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "diagnostic" |
| }, |
| "display_name": "Action Boundary Detection", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/transition_detection/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/transition_detection/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "transition_detection", |
| "input": "One all-modality window vector plus labels derived from action-change timestamps.", |
| "input_short": "current window with boundary target", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.6118, |
| "name": "macro-F1", |
| "neural_mlp": 0.5862 |
| }, |
| "modalities": [ |
| "video", |
| "pose_slam", |
| "motion_capture", |
| "inertial", |
| "language" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "boundary or steady", |
| "primary_direction": "C", |
| "process_short": "action changes -> boundary labels -> binary classifier", |
| "research_name": "Temporal Action Segmentation", |
| "why": "Localizes egocentric task boundaries and diagnoses temporal state changes." |
| }, |
| { |
| "architecture_family": "future-label classifier", |
| "case_study": "If a window shows the person preparing to pour, the target can be the action 20 frames later, such as the start of pouring.", |
| "current_limit": "Unseen future labels dominate the single-episode chronological test.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Next-Action Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/next_action/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/next_action/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "next_action", |
| "input": "The current all-modality window vector at time t.", |
| "input_short": "current window at time t", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 0.0593, |
| "name": "macro-F1", |
| "neural_mlp": 0.0419 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "action at t+20 frames", |
| "primary_direction": "C", |
| "process_short": "current features -> future label shift -> classifier", |
| "research_name": "Short-Horizon Intention Prediction", |
| "why": "Tests action intention/task-flow prediction from egocentric context." |
| }, |
| { |
| "architecture_family": "continuous regressor", |
| "case_study": "When the hand is moving toward a cup or bottle, the model predicts the future 3D hand-joint path.", |
| "current_limit": "Forecasting is window-level and not yet a full sequence or policy model.", |
| "direction_roles": { |
| "A": "direct", |
| "C": "proxy" |
| }, |
| "display_name": "Hand Trajectory Forecasting", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/hand_trajectory_forecast/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/hand_trajectory_forecast/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "forecast", |
| "id": "hand_trajectory_forecast", |
| "input": "The current all-modality window vector at time t.", |
| "input_short": "current multimodal window", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "lower", |
| "key": "mpjpe", |
| "minimal": 0.8647, |
| "name": "MPJPE", |
| "neural_mlp": 0.1079 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "video", |
| "depth", |
| "pose_slam", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "future hand-joint trajectory", |
| "primary_direction": "A", |
| "process_short": "current features -> future mocap target -> regression head", |
| "research_name": "3D Hand Motion Forecasting", |
| "why": "Directly predicts human hand motion and supports hand-object interaction modeling." |
| }, |
| { |
| "architecture_family": "binary classifier", |
| "case_study": "During manipulation, the hand may touch a cup, table, or bottle. The task asks whether any contact is happening.", |
| "current_limit": "The public sample is degenerate for this target because one class dominates.", |
| "direction_roles": { |
| "A": "direct", |
| "C": "proxy" |
| }, |
| "display_name": "Contact State Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/predictions.csv", |
| "label": "Neural predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/contact_prediction/confusion_matrix.csv", |
| "label": "Confusion matrix" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/contact_prediction/confusion_matrix.csv", |
| "label": "Neural confusion matrix" |
| } |
| ], |
| "family": "supervised", |
| "id": "contact_prediction", |
| "input": "Non-contact and non-caption feature blocks, so the answer is not directly leaked from the target labels.", |
| "input_short": "non-contact, non-caption features", |
| "metric": { |
| "better_baseline": "tie", |
| "direction": "higher", |
| "key": "macro_f1", |
| "minimal": 1.0, |
| "name": "macro-F1", |
| "neural_mlp": 1.0 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "video", |
| "depth", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "contact or no contact", |
| "primary_direction": "A", |
| "process_short": "feature filter -> contact target -> binary classifier", |
| "research_name": "Human-Object Contact Prediction", |
| "why": "Targets physical interaction state, a core affordance and manipulation signal." |
| }, |
| { |
| "architecture_family": "multi-label classifier", |
| "case_study": "If the person is pouring milk into coffee, relevant objects may include milk, cup, coffee, or container-like items.", |
| "current_limit": "Object labels are language-derived and sparse in one episode.", |
| "direction_roles": { |
| "A": "proxy", |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Object Relevance Prediction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/object_relevance/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/object_relevance/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/object_relevance/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/object_relevance/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "supervised", |
| "id": "object_relevance", |
| "input": "Non-caption feature blocks, so the model must infer objects from sensors rather than copying the caption words.", |
| "input_short": "non-caption multimodal features", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "micro_f1", |
| "minimal": 0.1803, |
| "name": "micro-F1", |
| "neural_mlp": 0.1679 |
| }, |
| "modalities": [ |
| "video", |
| "depth", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "relevant object set", |
| "primary_direction": "C", |
| "process_short": "object vocabulary -> multi-hot labels -> sigmoid heads", |
| "research_name": "Object-Centric Interaction Recognition", |
| "why": "Connects egocentric activity to manipulated objects and early object-centric state." |
| }, |
| { |
| "architecture_family": "retrieval ranker", |
| "case_study": "A query like Pour milk into coffee should rank the windows from the actual pouring moment higher than unrelated windows.", |
| "current_limit": "Bag-of-objects language features are too weak for rich grounding.", |
| "direction_roles": { |
| "C": "direct", |
| "D": "proxy" |
| }, |
| "display_name": "Language Grounding", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/caption_grounding/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/caption_grounding/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "retrieval", |
| "id": "caption_grounding", |
| "input": "Caption/object/interaction query features and a set of candidate sensor-window features.", |
| "input_short": "text-like query and candidate windows", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "mrr", |
| "minimal": 0.016, |
| "name": "MRR", |
| "neural_mlp": 0.0168 |
| }, |
| "modalities": [ |
| "language", |
| "video", |
| "depth", |
| "pose_slam" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "ranked matching moments", |
| "primary_direction": "C", |
| "process_short": "query features -> candidate index -> cosine ranker", |
| "research_name": "Language-to-Moment Grounding", |
| "why": "Grounds language annotation into egocentric sensor time and task state." |
| }, |
| { |
| "architecture_family": "two-tower retrieval head", |
| "case_study": "Use motion, IMU, and camera-pose signals from a pouring moment to retrieve the matching depth/video representation for that same moment.", |
| "current_limit": "Retrieval shows an alignment signal, not geometric reconstruction.", |
| "direction_roles": { |
| "B": "proxy", |
| "C": "diagnostic", |
| "D": "proxy" |
| }, |
| "display_name": "Cross-Modal Retrieval", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/cross_modal_retrieval/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/cross_modal_retrieval/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "retrieval", |
| "id": "cross_modal_retrieval", |
| "input": "Query side: motion, IMU, and camera/pose features. Candidate side: depth and video features.", |
| "input_short": "motion/IMU/pose query; depth/video candidates", |
| "metric": { |
| "better_baseline": "minimal", |
| "direction": "higher", |
| "key": "mrr", |
| "minimal": 0.2693, |
| "name": "MRR", |
| "neural_mlp": 0.13 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "pose_slam", |
| "depth", |
| "video" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "ranked visual windows", |
| "primary_direction": "C", |
| "process_short": "modality split -> projection -> nearest-neighbor ranker", |
| "research_name": "Multimodal Representation Retrieval", |
| "why": "Tests whether synchronized modalities identify the same 4D moment, a prerequisite for reconstruction and world modeling." |
| }, |
| { |
| "architecture_family": "feature regressor", |
| "case_study": "Given motion, IMU, and camera-pose signals while the hand moves, predict the matching depth/video feature vector.", |
| "current_limit": "Feature-vector reconstruction is not pixel, depth-map, mesh, NeRF, or Gaussian reconstruction.", |
| "direction_roles": { |
| "B": "proxy", |
| "D": "proxy" |
| }, |
| "display_name": "Cross-Modal Reconstruction", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/modality_reconstruction/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/modality_reconstruction/metrics.json", |
| "label": "Neural metrics" |
| } |
| ], |
| "family": "forecast", |
| "id": "modality_reconstruction", |
| "input": "Motion, IMU, and camera/pose features as input; depth/video features as the regression target.", |
| "input_short": "motion, IMU, and camera/pose features", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "r2", |
| "minimal": -0.0153, |
| "name": "R2", |
| "neural_mlp": -0.0102 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "pose_slam", |
| "depth", |
| "video" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "reconstructed depth/video vector", |
| "primary_direction": "B", |
| "process_short": "source-target split -> scaler -> regression head", |
| "research_name": "Modality Feature Reconstruction", |
| "why": "Predicts visual/depth state from non-target sensors as a weak reconstruction/world-model objective." |
| }, |
| { |
| "architecture_family": "pairwise classifier", |
| "case_study": "If window A shows reaching and window B shows pouring, the model should distinguish A then B from B then A.", |
| "current_limit": "Only local adjacent ordering, not long-horizon causal modeling.", |
| "direction_roles": { |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "display_name": "Temporal Order Verification", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/temporal_order/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/temporal_order/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/temporal_order/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/temporal_order/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "temporal_order", |
| "input": "A pair of adjacent window vectors, plus their difference vector.", |
| "input_short": "two adjacent windows plus difference vector", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "f1", |
| "minimal": 0.54, |
| "name": "F1", |
| "neural_mlp": 0.852 |
| }, |
| "modalities": [ |
| "video", |
| "pose_slam", |
| "motion_capture", |
| "inertial" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "correct or reversed", |
| "primary_direction": "C", |
| "process_short": "pair builder -> feature combiner -> binary classifier", |
| "research_name": "Temporal Order Verification", |
| "why": "Checks whether features encode local time direction and task progression." |
| }, |
| { |
| "architecture_family": "pairwise classifier", |
| "case_study": "Motion from a pouring moment is paired with video/depth from several windows later. The task asks the model to detect that mismatch.", |
| "current_limit": "Synthetic shifts diagnose alignment but do not solve calibration or mapping.", |
| "direction_roles": { |
| "B": "diagnostic", |
| "C": "diagnostic", |
| "D": "diagnostic" |
| }, |
| "display_name": "Multimodal Synchronization Detection", |
| "evidence_links": [ |
| { |
| "href": "data/task_walkthroughs.json", |
| "label": "Task walkthrough" |
| }, |
| { |
| "href": "single_episode_explorer.html", |
| "label": "Single-episode explorer" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/misalignment_detection/metrics.json", |
| "label": "Minimal metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/misalignment_detection/metrics.json", |
| "label": "Neural metrics" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/misalignment_detection/predictions.csv", |
| "label": "Minimal predictions" |
| }, |
| { |
| "href": "https://github.com/ChaoYue0307/ropedia-xperience-10m-task-suite/blob/main/results/episode_task_suite/neural_mlp/misalignment_detection/predictions.csv", |
| "label": "Neural predictions" |
| } |
| ], |
| "family": "diagnostic", |
| "id": "misalignment_detection", |
| "input": "A motion-side feature group and a visual/depth-side feature group, either aligned or artificially shifted.", |
| "input_short": "motion-side and visual/depth-side feature groups", |
| "metric": { |
| "better_baseline": "neural_mlp", |
| "direction": "higher", |
| "key": "f1", |
| "minimal": 0.5052, |
| "name": "F1", |
| "neural_mlp": 0.7153 |
| }, |
| "modalities": [ |
| "motion_capture", |
| "inertial", |
| "video", |
| "depth", |
| "pose_slam" |
| ], |
| "module_summary": "input window -> feature/target builder -> baseline head -> evaluator -> artifact files", |
| "output_short": "aligned or shifted", |
| "primary_direction": "C", |
| "process_short": "aligned/shifted pairs -> feature combiner -> binary classifier", |
| "research_name": "Cross-Modal Misalignment Detection", |
| "why": "Detects temporal desynchronization, a key data-quality gate for multimodal reconstruction and world models." |
| } |
| ], |
| "title": "Interactive Research Roadmap" |
| } |
|
|
