File size: 13,193 Bytes
31e3087 eeac43c bfcf156 31e3087 bfcf156 31e3087 45c1706 31e3087 fc9e8cf 31e3087 04c0bde 31e3087 bfcf156 31e3087 eeac43c 31e3087 9371cfb 31e3087 04c0bde 31e3087 bfcf156 31e3087 04c0bde 31e3087 bfcf156 31e3087 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 | {
"title": "Xperience-10M Foundation Model Plan",
"status": "planning_artifact",
"current_boundary": "A final held-out multi-episode Qwen3-Omni diagnostic result is verified in this repo and meets the strict-JSON target, but it is not a strong action/subtask model result. The current foundation-model work should treat it as the baseline train/eval/package loop before Qwen action/subtask improvements, Cosmos-style world modeling, or policy/VLA branches.",
"backbone_registry": {
"config_dir": "configs/omni_backbones",
"validator": "scripts/omni/backbone_registry.py --validate --json",
"extension_contract": "OMNI_MODEL_EXTENSION_CONTRACT.md",
"implemented_backbone": "qwen3_omni_lora",
"planned_backbones": [
"cosmos_world_model",
"policy_vla_branch"
]
},
"decision": {
"immediate_trainable_backbone": "Qwen3-Omni",
"first_world_model_branch": "Cosmos 3",
"first_policy_branch_candidates": [
"OpenVLA / OpenVLA-OFT",
"openpi pi0/pi0.5",
"NVIDIA GR00T"
],
"external_reasoning_reference": "Gemini Robotics",
"long_term_native_pretraining_goal": "Xperience Embodied Foundation Model"
},
"future_pretraining_goal": {
"name": "Xperience Embodied Foundation Model",
"status": "future_planning_goal",
"role": "Domain-specific embodied foundation model pretrained on full Xperience-10M if full-corpus data, storage, and compute become available.",
"not_current_result": true,
"document": "XPERIENCE_EMBODIED_FOUNDATION_MODEL_PRETRAINING.md",
"entry_conditions": [
"Selected multi-episode Qwen3-Omni pilot trains and evaluates cleanly.",
"Scaling from 128 episodes to thousands of episodes shows measurable value.",
"Full-corpus storage, derived-shard storage, and fast active-cache capacity are available.",
"Distributed training, checkpoint/restart, and provenance tracking are reliable.",
"Evaluation covers held-out episodes, sessions, activities, objects, and missing-modality robustness."
],
"target_modules": [
"multi-view video encoder",
"audio encoder",
"depth and geometry encoder",
"pose/SLAM encoder",
"hand/body mocap encoder",
"IMU encoder",
"language encoder/decoder",
"temporal fusion transformer",
"task heads and decoders"
],
"pretraining_objectives": [
"masked multimodal modeling",
"cross-modal contrastive alignment",
"future-state prediction",
"ego-motion and hand-motion forecasting",
"action and procedure prediction",
"language grounding and captioning",
"contact and affordance prediction",
"optional policy-style targets after action conversion"
],
"hardware_ranges": [
{
"goal": "0.3B-1B pilot",
"compute": "8-32 modern 80GB-class data-center GPUs",
"use": "prove objectives and data loaders"
},
{
"goal": "1B-3B domain model",
"compute": "32-128 GPUs",
"use": "research-scale Xperience representation learning"
},
{
"goal": "3B-7B full-corpus domain model",
"compute": "128-512 GPUs",
"use": "first realistic full Xperience-native foundation model"
},
{
"goal": "30B-class omni model from scratch",
"compute": "512-2000+ GPUs",
"use": "lab-scale project after scaling curves justify cost"
}
]
},
"model_families": [
{
"priority": 1,
"family": "Qwen3-Omni",
"category": "omni_instruction_model",
"openness": "open_weights_available_from_official_hf_repo",
"best_role": "First selected-episode multimodal LoRA pilot and structured task predictor.",
"xperience10m_fit": [
"RGB/fisheye video, embedded audio, and language prompts can enter directly.",
"Depth, pose/SLAM, mocap, contacts, and IMU enter through the existing sensor bridge.",
"Matches current task outputs: labels, structured JSON, captions, and short decisions."
],
"current_decision": "keep_as_first_pilot",
"entry_condition": "Selected episodes prepared with held-out episode split.",
"public_source": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct"
},
{
"priority": 2,
"family": "Cosmos 3",
"category": "world_foundation_model",
"openness": "track_official_nvidia_release_and_available_weights",
"best_role": "Embodied world modeling, action generation, future-window prediction, and synthetic-data expansion.",
"xperience10m_fit": [
"Uses video streams as visual state.",
"Uses pose/SLAM, depth, mocap, IMU, and language as physical-world conditioning signals.",
"Better aligned with prediction/generation objectives than simple label classification."
],
"current_decision": "add_as_first_world_model_branch_after_data_gate",
"entry_condition": "Multi-episode data plus enough storage/compute for generated or latent video-state outputs.",
"public_source": "https://www.nvidia.com/en-us/ai/cosmos/"
},
{
"priority": 3,
"family": "NVIDIA GR00T",
"category": "humanoid_policy_foundation_model",
"openness": "track_official_nvidia_release_and_tooling",
"best_role": "Humanoid action understanding, retargeting, contact/action prediction, and embodied skill transfer.",
"xperience10m_fit": [
"Hand/body mocap and contact cues can be retargeted into humanoid state/action targets.",
"Egocentric video plus human motion can support affordance and interaction tasks."
],
"current_decision": "track_as_humanoid_policy_branch",
"entry_condition": "Retargeting artifact and action-space definition exist.",
"public_source": "https://developer.nvidia.com/isaac/gr00t"
},
{
"priority": 4,
"family": "OpenVLA / OpenVLA-OFT",
"category": "vision_language_action_policy",
"openness": "open_project_and_weights",
"best_role": "Open robot-policy baseline after observations and action labels are converted into a VLA format.",
"xperience10m_fit": [
"Good candidate when each window is expressed as visual observation, instruction/context, and action token.",
"Requires an explicit action target; current human egocentric labels are not robot controls by default."
],
"current_decision": "candidate_after_action_space_design",
"entry_condition": "Window-to-action-token conversion is implemented and checked.",
"public_source": "https://openvla.github.io/"
},
{
"priority": 5,
"family": "openpi pi0/pi0.5",
"category": "robot_policy_model",
"openness": "open_source_policy_training_stack",
"best_role": "Action-chunking, policy fine-tuning, and embodiment-transfer experiments.",
"xperience10m_fit": [
"Useful once hand trajectories, contacts, or retargeted body motion are converted into policy targets.",
"Better for policy branch than for current structured task JSON outputs."
],
"current_decision": "candidate_policy_branch",
"entry_condition": "Action target and train/eval protocol exist for at least 64 episodes.",
"public_source": "https://github.com/Physical-Intelligence/openpi"
},
{
"priority": 6,
"family": "Gemini Robotics",
"category": "closed_embodied_reasoning_reference",
"openness": "closed_or_limited_access",
"best_role": "Qualitative reasoning reference, annotation helper, and external comparison when API access exists.",
"xperience10m_fit": [
"Can help reason over egocentric scenes and task descriptions.",
"Not a local fine-tune target for this repo."
],
"current_decision": "external_reference_only",
"entry_condition": "API/access exists and outputs are logged separately from trainable model metrics.",
"public_source": "https://deepmind.google/discover/blog/gemini-robotics-brings-ai-into-the-physical-world/"
},
{
"priority": 7,
"family": "Octo / SmolVLA-style lightweight policies",
"category": "lightweight_robot_policy_baselines",
"openness": "open_projects",
"best_role": "Cheaper policy baselines for observation-to-action experiments.",
"xperience10m_fit": [
"Useful after action target design.",
"Less directly omni-modal than Qwen3-Omni or Cosmos 3."
],
"current_decision": "optional_baseline_after_data_staging",
"entry_condition": "Action labels and baseline protocol exist.",
"public_source": "https://github.com/huggingface/lerobot"
},
{
"priority": 8,
"family": "Xperience Embodied Foundation Model",
"category": "xperience_native_pretraining_goal",
"openness": "future project-specific model if full-corpus access and compute exist",
"best_role": "Domain model over synchronized embodied experience.",
"xperience10m_fit": [
"Uses the full aligned modality stack rather than treating sensors as auxiliary metadata.",
"Targets temporal embodied representation learning across perception, motion, geometry, audio, and language.",
"Can become the shared pretraining backbone for Qwen-style instruction tasks, Cosmos-style world modeling, and policy/action branches."
],
"current_decision": "future_goal_after_scaling_evidence",
"entry_condition": "Full-corpus data path, PB-scale storage, multi-node compute, and positive smaller-run scaling evidence.",
"public_source": "XPERIENCE_EMBODIED_FOUNDATION_MODEL_PRETRAINING.md"
}
],
"execution_order": [
{
"step": 1,
"name": "Data gate",
"action": "Stage at least 32 valid Xperience-10M episodes with held-out episode split."
},
{
"step": 2,
"name": "First held-out baseline",
"action": "Run Qwen3-Omni action/subtask error analysis and targeted reruns to improve the verified diagnostic baseline."
},
{
"step": 3,
"name": "Model-selection dry run",
"action": "Run 3-8 episode dry runs for Qwen3-Omni prompt/LoRA, Cosmos 3 preprocessing, and one policy candidate."
},
{
"step": 4,
"name": "World-model branch",
"action": "Promote Cosmos 3 if future-window/action-conditioned preprocessing fits storage and compute."
},
{
"step": 5,
"name": "Policy branch",
"action": "Promote OpenVLA/openpi/GR00T after action target conversion and retargeting artifacts are traceable."
},
{
"step": 6,
"name": "Publishing threshold",
"action": "Publish branch results only with real manifests, predictions, metrics, and qualitative examples."
},
{
"step": 7,
"name": "Xperience-native pretraining",
"action": "Start a from-scratch Xperience Embodied Foundation Model only after smaller scaling stages, full-corpus storage, multi-node compute, and held-out evaluation protocols are in place."
}
],
"evaluation_additions": [
{
"target": "structured_task_prediction",
"metrics": [
"JSON validity",
"macro-F1",
"accuracy",
"micro-F1"
],
"model_families": [
"Qwen3-Omni",
"Gemini Robotics reference"
]
},
{
"target": "future_state_prediction",
"metrics": [
"retrieval rank",
"temporal consistency",
"feature reconstruction",
"qualitative visual inspection"
],
"model_families": [
"Cosmos 3"
]
},
{
"target": "action_conditioned_dynamics",
"metrics": [
"transition accuracy",
"contact accuracy",
"next-action accuracy"
],
"model_families": [
"Cosmos 3",
"OpenVLA",
"openpi",
"GR00T"
]
},
{
"target": "cross_episode_generalization",
"metrics": [
"held-out episode metrics",
"held-out session metrics",
"leakage checks"
],
"model_families": [
"all trainable branches"
]
}
],
"source_links": [
{
"label": "Qwen3-Omni official HF model",
"url": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct"
},
{
"label": "NVIDIA Cosmos",
"url": "https://www.nvidia.com/en-us/ai/cosmos/"
},
{
"label": "NVIDIA Isaac GR00T",
"url": "https://developer.nvidia.com/isaac/gr00t"
},
{
"label": "OpenVLA",
"url": "https://openvla.github.io/"
},
{
"label": "openpi",
"url": "https://github.com/Physical-Intelligence/openpi"
},
{
"label": "Gemini Robotics",
"url": "https://deepmind.google/discover/blog/gemini-robotics-brings-ai-into-the-physical-world/"
},
{
"label": "Octo",
"url": "https://octo-models.github.io/"
},
{
"label": "LeRobot / SmolVLA",
"url": "https://github.com/huggingface/lerobot"
},
{
"label": "Xperience Embodied Foundation Model pretraining plan",
"url": "XPERIENCE_EMBODIED_FOUNDATION_MODEL_PRETRAINING.md"
}
]
}
|