| {"contact": {"url": "https://adaptive-machine-patterns.com/embodied-ai-workbench/", "home": "https://adaptive-machine-patterns.com/"}, "backends": [{"id": "teacher", "label": "Teacher", "capability": "teacher", "note": "Scripted; not AI. It makes the training examples."}, {"id": "amp-tadpole", "label": "AMP Tadpole", "capability": "tadpole_live", "note": "AMP's 6.5M-parameter model, running in your browser."}, {"id": "smolvla", "label": "AMP SmolVLA", "capability": "smolvla_live", "note": "Hugging Face's 450M-parameter SmolVLA, fine-tuned by AMP.", "why": "it's a 450M-parameter model with a 2 GB backbone, which Pro downloads the first time you choose it.", "blurb": "Drive the arm with Hugging Face's SmolVLA, fine-tuned by AMP on the benchmark, and look inside it pass by pass. It opens and closes the gripper and returns home; reaching for the objects is beyond it, and the trace shows why: each camera reaches it as 8 x 8 tokens, smaller than the objects."}, {"id": "cosmos3-edge", "label": "NVIDIA Cosmos3-Edge", "capability": "cosmos3_live", "soon": true, "note": "4 billion parameters. Coming soon in Pro.", "why": "it's a 4-billion-parameter model, about 8 GB to download, and it needs a capable GPU.", "blurb": "Run NVIDIA's Cosmos3-Edge world-model policy on the benchmark, live, with weights you download from NVIDIA."}, {"id": "vla-adapter", "label": "VLA-Adapter", "capability": "vla_adapter_live", "soon": true, "note": "1.6B parameters. Coming soon in Pro.", "why": "it has 1.6 billion parameters, about 3 GB to download, and it needs a GPU.", "blurb": "Run OpenHelix's VLA-Adapter (a 0.5B language model, 1.6B parameters in all) on the benchmark and look inside it."}], "catalogue": [{"id": "own_models", "label": "Load your own PyTorch models", "capability": "load_package", "note": "Drive the arm with them, score them on the benchmark and look inside, layer by layer.", "why": "the demo runs a fixed, converted copy of AMP Tadpole; your model runs in PyTorch on your own machine.", "blurb": "Package your model (model.py and its weights): it drives the arm, is scored on the benchmark, and you can look inside, layer by layer, every block described automatically. Record a trace at any moment and see inside it on exactly what it saw."}, {"id": "lerobot", "label": "LeRobot models from Hugging Face", "capability": "lerobot", "note": "Paste a model card link. Pro shows its size and licence, downloads it, and opens it up. Models trained for this arm drive it.", "why": "LeRobot models run on PyTorch and are hundreds of megabytes, too much for a web page.", "blurb": "Paste a Hugging Face link or choose a folder. Models trained for this arm drive it and are scored on the benchmark; models made for other robots open up layer by layer, and diffusion and flow models such as SmolVLA pass by pass, with the answer after every pass (tested: ACT, SmolVLA, Diffusion Policy)."}, {"id": "tools", "label": "Compare runs, and the motion lens", "capability": "extra_tools", "note": "Two runs side by side, storey by storey, and the model's habits averaged over 300 moments.", "why": "the lens averages the model's gradients over 300 moments, which takes PyTorch and real computing power.", "blurb": "Compare two runs or two checkpoints storey by storey. The motion lens is the model's Jacobian, averaged over 300 moments: which tokens push the arm, and which way."}, {"id": "capture", "label": "Save pictures, videos and traces", "capability": "capture", "note": "Light versions for print, videos of the calculation, and the raw traces for your own analysis.", "blurb": "Save the views and the calculation as pictures (light ones too, for print) and videos, and the traces themselves for your own analysis."}, {"id": "straight_tadpole", "label": "AMP Tadpole, straight-path variant", "capability": "straight_tadpole", "note": "A second Tadpole that moves the gripper in straighter lines.", "blurb": "A second AMP Tadpole, fine-tuned so the gripper moves in straighter lines instead of copying the teacher's swoops: same network, same inputs. Run it beside the original and compare them inside."}, {"id": "train", "label": "Train and fine-tune models", "capability": "train", "soon": true, "note": "AMP Tadpole, your own models, and LeRobot models such as SmolVLA and ACT, or NVIDIA Cosmos3-Edge, on unlimited benchmark data. Coming soon.", "why": "training takes hours of GPU time, and these models have up to 4 billion parameters (Cosmos3-Edge needs a large GPU).", "blurb": "Fine-tune AMP Tadpole, or train your own model, LeRobot models such as SmolVLA and ACT, or NVIDIA Cosmos3-Edge, on scenes generated from seeds with the teacher's examples: as much data as you like, and reproducible. Then run them live on the arm, score them on the benchmark, and compare them with AMP Tadpole, layer by layer."}], "features": [{"id": "inspect_moment", "label": "Record trace", "capability": "trace_any", "why": "the demo runs a compact, converted copy of AMP Tadpole that doesn't expose its insides; Pro traces the PyTorch model itself.", "blurb": "Pause at any moment of any scene and look inside the model on exactly what it saw. The Demo shows prepared tour scenes only."}, {"id": "load_model", "label": "Load your own model\u2026", "capability": "load_package", "why": "the demo runs a fixed, converted copy of AMP Tadpole; your model runs in PyTorch on your own machine.", "blurb": "Bring your own PyTorch model as a package (model.py + safetensors). It joins the backend list, is scored on the benchmark, and is traced automatically for the inside view."}, {"id": "compare", "label": "Compare two runs", "capability": "compare", "blurb": "Two models, or two checkpoints, side by side on the same scene."}, {"id": "lens", "label": "Motion lens", "capability": "lens", "why": "the lens averages the model's gradients over 300 moments, which takes PyTorch and real computing power.", "blurb": "See which pixels and which words moved the arm: the model's motion traced back to its inputs."}, {"id": "train", "label": "Train / fine-tune", "capability": "train", "blurb": "Train on unlimited, seed-generated data from the benchmark, and fine-tune AMP Tadpole or your own model."}, {"id": "checkpoints", "label": "AMP Tadpole checkpoints", "capability": "checkpoints", "blurb": "Every AMP Tadpole checkpoint, including the path-efficient one."}, {"id": "export_images", "label": "Export images", "capability": "export_images", "blurb": "Save the camera views and the model views as images."}, {"id": "export_traces", "label": "Export traces", "capability": "export_traces", "blurb": "Save traces of the model's internals for your own analysis."}, {"id": "export_film", "label": "Export film", "capability": "export_film", "blurb": "Record fly-throughs of the model and runs of the arm as video."}, {"id": "desktop", "label": "Desktop app, your GPU", "capability": "desktop", "blurb": "Runs offline on your own machine and GPU."}], "sizes": {"start_mb": 61, "scene_mb": 24, "phone_scene_mb": 21, "listen_mb": 47}} |