| { |
| "source": { |
| "shared_windows": "results/episode_task_suite/shared_windows.npz", |
| "windows_csv": "results/episode_task_suite/windows.csv", |
| "feature_manifest": "results/episode_task_suite/feature_manifest.json" |
| }, |
| "dataset_scope": { |
| "sample_episode_count": 1, |
| "num_windows": 1161, |
| "feature_dim": 8546, |
| "first_start_frame": 0, |
| "last_end_frame": 5819, |
| "warning": "Single public sample episode; these extension probes validate task design and pipeline mechanics, not cross-episode generalization." |
| }, |
| "baselines": { |
| "minimal": "Ridge classifiers/regressors/projections plus cosine retrieval on the committed feature tensor.", |
| "neural_mlp": "Small one-hidden-layer PyTorch MLP heads using the same inputs, targets, chronological split, and evaluator." |
| }, |
| "run_config": { |
| "train_fraction": 0.7, |
| "ridge_l2": 10.0, |
| "seed": 7, |
| "future_windows": 4, |
| "neural_epochs": 25, |
| "neural_hidden_dim": 128, |
| "neural_batch_size": 128, |
| "skip_neural": false |
| }, |
| "task_specs": { |
| "body_motion_intensity": { |
| "direction": "A", |
| "direction_name": "Human Modeling & Motion Understanding", |
| "name": "Body and Hand Motion Intensity", |
| "family": "classification", |
| "case_study": "A window with a fast reach or pour should be classified as high motion; a steady holding window should be low motion.", |
| "input": "Current non-mocap feature blocks: video, audio, depth, camera pose/rotation, IMU, SLAM, calibration, and language context.", |
| "middle_process": "Compute the target from hand/body joint changes between neighboring windows, hide the mocap blocks from the input, then classify high versus low motion using the train-set median as the threshold.", |
| "output": "Binary label: high_motion or low_motion.", |
| "minimal_baseline": "Ridge classifier on standardized non-mocap features.", |
| "neural_baseline": "One-hidden-layer MLP binary classifier on the same input features.", |
| "metric_name": "macro-F1", |
| "metric_key": "macro_f1", |
| "metric_direction": "higher", |
| "current_limit": "This is a motion-energy proxy, not a SMPL/MANO body model or a generative motion prior." |
| }, |
| "multi_view_consistency_retrieval": { |
| "direction": "B", |
| "direction_name": "3D/4D Reconstruction & Neural Rendering", |
| "name": "Multi-View Consistency Retrieval", |
| "family": "retrieval", |
| "case_study": "Given the fisheye camera features for a pouring moment, retrieve the synchronized stereo-left view from the same time window.", |
| "input": "Query side: fisheye_cam0 video feature block. Candidate side: stereo_left video feature block from held-out windows.", |
| "middle_process": "Learn a projection from one camera-view feature space into another, then rank held-out candidate windows by cosine similarity.", |
| "output": "Ranked candidate windows; the correct synchronized view should rank near the top.", |
| "minimal_baseline": "Ridge projection followed by cosine nearest-neighbor retrieval.", |
| "neural_baseline": "One-hidden-layer MLP projection followed by the same cosine retrieval evaluator.", |
| "metric_name": "MRR", |
| "metric_key": "mrr", |
| "metric_direction": "higher", |
| "current_limit": "This checks calibrated multi-view signal, but it is still feature retrieval, not NeRF, Gaussian Splatting, or novel-view synthesis." |
| }, |
| "action_phase_progress": { |
| "direction": "C", |
| "direction_name": "Egocentric Vision & Interaction", |
| "name": "Action Phase Progress Estimation", |
| "family": "regression", |
| "case_study": "Inside a Pour coffee action segment, estimate whether the current window is near the beginning, middle, or end of that action.", |
| "input": "Current non-caption multimodal feature vector, so the label text cannot be copied directly from the language block.", |
| "middle_process": "Convert contiguous action-label runs into a normalized 0-to-1 progress target, train on earlier windows, and regress progress for later windows.", |
| "output": "A scalar progress value between 0.0 and 1.0 for the current action segment.", |
| "minimal_baseline": "Ridge regressor on standardized non-caption features.", |
| "neural_baseline": "One-hidden-layer MLP regressor on the same input features.", |
| "metric_name": "MAE", |
| "metric_key": "mae", |
| "metric_direction": "lower", |
| "current_limit": "This is an action-structure probe inside one episode, not a general intent model across homes, people, or tasks." |
| }, |
| "ego_motion_forecast": { |
| "direction": "D", |
| "direction_name": "Scene Reconstruction & World Modeling", |
| "name": "Short-Horizon Ego-Motion Forecasting", |
| "family": "forecast", |
| "case_study": "From the current sensors, predict how the camera translation will change over the next 20 frames while the wearer moves through the scene.", |
| "input": "Current multimodal features excluding the camera-translation block and caption text.", |
| "middle_process": "Build a future target from camera-translation difference at a four-window horizon, then regress that future ego-motion delta from current sensors.", |
| "output": "A future camera-translation delta vector.", |
| "minimal_baseline": "Ridge regressor with a 20-frame forecast horizon.", |
| "neural_baseline": "One-hidden-layer MLP regressor with the same horizon and split.", |
| "metric_name": "MAE", |
| "metric_key": "mae", |
| "metric_direction": "lower", |
| "current_limit": "This is a compact world-model proxy; it does not build a persistent map, scene graph, or object permanence model." |
| } |
| }, |
| "tasks": { |
| "body_motion_intensity": { |
| "train_windows": 812, |
| "test_windows": 348, |
| "target_threshold_train_median": 0.476467490196228, |
| "input_dim": 6425, |
| "target_source": "hand/body joint delta between neighboring windows", |
| "minimal": { |
| "accuracy": 0.7758620689655172, |
| "macro_f1": 0.7658385093167701, |
| "positive_rate_true": 0.35919540229885055, |
| "positive_rate_pred": 0.4339080459770115, |
| "num_test": 348 |
| }, |
| "neural_mlp": { |
| "accuracy": 0.8304597701149425, |
| "macro_f1": 0.8254423029509534, |
| "positive_rate_true": 0.35919540229885055, |
| "positive_rate_pred": 0.47126436781609193, |
| "num_test": 348 |
| }, |
| "neural_training": { |
| "available": true, |
| "epochs": 25, |
| "hidden_dim": 128, |
| "loss_history": [ |
| 0.37322977610996794, |
| 0.22245765099384515, |
| 0.1382973729242832, |
| 0.10363741681493562, |
| 0.0795709453523159, |
| 0.06539858697817244, |
| 0.055655122610735776, |
| 0.043255199022187385, |
| 0.03558319240001035, |
| 0.031631215764530776, |
| 0.029465350402711796, |
| 0.024383640274625695, |
| 0.02020622924740972, |
| 0.016222001351599624, |
| 0.018758724778523587, |
| 0.013199950316049196, |
| 0.014794624612432689, |
| 0.01013119441452505, |
| 0.009688532855040554, |
| 0.008956241283767622, |
| 0.006733611014469761, |
| 0.006677041435843618, |
| 0.0067647325489761795, |
| 0.005346325666556511, |
| 0.004052691048084588 |
| ] |
| } |
| }, |
| "multi_view_consistency_retrieval": { |
| "train_windows": 813, |
| "test_windows": 348, |
| "query_block": "video_fisheye_cam0", |
| "target_block": "video_stereo_left", |
| "query_dim": 686, |
| "target_dim": 686, |
| "minimal": { |
| "mrr": 0.5533982515335083, |
| "top1": 0.41954022988505746, |
| "top5": 0.7068965517241379, |
| "top10": 0.8304597701149425, |
| "median_rank": 2.0, |
| "num_test": 348 |
| }, |
| "neural_mlp": { |
| "mrr": 0.34691643714904785, |
| "top1": 0.23275862068965517, |
| "top5": 0.46264367816091956, |
| "top10": 0.5890804597701149, |
| "median_rank": 7.0, |
| "num_test": 348 |
| }, |
| "neural_training": { |
| "available": true, |
| "epochs": 25, |
| "hidden_dim": 128, |
| "loss_history": [ |
| 0.9800718805740094, |
| 0.8296866191855803, |
| 0.7029420470986126, |
| 0.6089339927846948, |
| 0.5426930648814268, |
| 0.49323386093200644, |
| 0.45315542230600214, |
| 0.4240272395578551, |
| 0.3964498403400806, |
| 0.37567753094588696, |
| 0.3599070675332021, |
| 0.3417643405048023, |
| 0.32952829051721577, |
| 0.31516501450450657, |
| 0.3070896395824639, |
| 0.29752101269888553, |
| 0.287490411878072, |
| 0.2791558311654193, |
| 0.2707079971921693, |
| 0.2669465311998811, |
| 0.2603630047442728, |
| 0.2501040017656148, |
| 0.24714160980920216, |
| 0.24146720613060843, |
| 0.23866056472173036 |
| ] |
| } |
| }, |
| "action_phase_progress": { |
| "train_windows": 813, |
| "test_windows": 348, |
| "input_dim": 7650, |
| "target_source": "normalized position inside contiguous action-label runs", |
| "minimal": { |
| "mse": 0.16943013668060303, |
| "mae": 0.32674381136894226, |
| "r2": -1.0236103208433347, |
| "num_test": 348 |
| }, |
| "neural_mlp": { |
| "mse": 0.14226463437080383, |
| "mae": 0.301545649766922, |
| "r2": -0.6991557278041094, |
| "num_test": 348 |
| }, |
| "neural_training": { |
| "available": true, |
| "epochs": 25, |
| "hidden_dim": 128, |
| "loss_history": [ |
| 3.236165899632162, |
| 1.2893786148831414, |
| 0.8036823107014429, |
| 0.5113777590120557, |
| 0.42643894586909153, |
| 0.37627028047965166, |
| 0.30304253713524504, |
| 0.25041163572526065, |
| 0.20774717810409096, |
| 0.18116216590630319, |
| 0.16409150619165191, |
| 0.14362229159397394, |
| 0.1277421933001991, |
| 0.1232111468672899, |
| 0.11924667946013724, |
| 0.11960234325310401, |
| 0.09951645682988572, |
| 0.08239271907825459, |
| 0.08965909919102401, |
| 0.07973466079503408, |
| 0.07939992471154765, |
| 0.07085797808340408, |
| 0.08331163055269188, |
| 0.068286436959313, |
| 0.07575550977814241 |
| ] |
| } |
| }, |
| "ego_motion_forecast": { |
| "train_windows": 810, |
| "test_windows": 347, |
| "forecast_horizon_windows": 4, |
| "forecast_horizon_frames": 20, |
| "input_dim": 7629, |
| "target_dim": 21, |
| "target_source": "future minus current camera_translation feature block", |
| "minimal": { |
| "mse": 3.3189830780029297, |
| "mae": 0.16999860107898712, |
| "r2": -5674.96718626448, |
| "num_test": 347 |
| }, |
| "neural_mlp": { |
| "mse": 0.547074019908905, |
| "mae": 0.0833469107747078, |
| "r2": -934.5800418396838, |
| "num_test": 347 |
| }, |
| "neural_training": { |
| "available": true, |
| "epochs": 25, |
| "hidden_dim": 128, |
| "loss_history": [ |
| 1.034793225188314, |
| 0.6070329941349265, |
| 0.459870958622591, |
| 0.3616479166495947, |
| 0.2902924293353234, |
| 0.23913239262722158, |
| 0.1907091121982645, |
| 0.17297036942140556, |
| 0.15034288657300265, |
| 0.13807891327657817, |
| 0.13084740807980666, |
| 0.12387588925567675, |
| 0.12085371225336451, |
| 0.11271689225126195, |
| 0.10728766140010622, |
| 0.10044601888936243, |
| 0.09365091108613544, |
| 0.09090288755150489, |
| 0.08592776887946658, |
| 0.0805281208805096, |
| 0.08014915316929051, |
| 0.07900124887625376, |
| 0.07897219589830917, |
| 0.07575527565714754, |
| 0.0774567014273302 |
| ] |
| } |
| } |
| } |
| } |
