Upload nano_WaveGen

.gitattributes

@@ -43,3 +43,4 @@ EMS-superquadric_fitting_inference/src/EMS/__pycache__/EMS_recovery.Distance-286
 EMS-superquadric_fitting_inference/src/EMS/__pycache__/EMS_recovery.SwitchCost-265.py312.1.nbc filter=lfs diff=lfs merge=lfs -text
 EMS-superquadric_fitting_inference/src/EMS/__pycache__/EMS_recovery.SimilarityCandidates-138.py312.1.nbc filter=lfs diff=lfs merge=lfs -text
 EMS-superquadric_fitting_inference/src/EMS/__pycache__/EMS_recovery.CostFunc-307.py311.1.nbc filter=lfs diff=lfs merge=lfs -text
+nano_WaveGen/checkpoints_text2wave/losses.png filter=lfs diff=lfs merge=lfs -text

nano_WaveGen/.gitignore

@@ -0,0 +1,45 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.pyc
+
+# PyTorch
+*.pth
+*.pt
+checkpoint*.pth
+best_model*.pth
+final_model.pth
+
+# Logs and outputs
+logs/
+*.log
+core_space/
+outputs/
+
+# Jupyter Notebook
+.ipynb_checkpoints/
+
+# Environment
+.env
+venv/
+
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Temporary files
+*.tmp
+*.bak
+
+data/movi_a_128x128
+wandb
+*.npz

nano_WaveGen/checkpoints_text2wave/best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a83304beb70d3e77c15709df1d486ce35e2a32dbebc13285914d763a54e74ee
+size 926425298

nano_WaveGen/checkpoints_text2wave/latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f5e09c520f8fde415c6f95a7347dd0d3fa49f03fb1342da95f04a436c736567
+size 926458332

nano_WaveGen/checkpoints_text2wave/training_log.txt

@@ -0,0 +1,9 @@
+2025-12-05 18:40:09 ============================================================
+2025-12-05 18:40:09 Dataset Information:
+2025-12-05 18:40:09 - Training samples: 100
+2025-12-05 18:40:09 - Batch size: 24
+2025-12-05 18:40:09 - Steps per epoch (full dataset): 5
+2025-12-05 18:40:09 - Total training steps: 50000
+2025-12-05 18:40:09 - Will traverse dataset: 10000.00 times
+2025-12-05 18:40:09 ============================================================
+2025-12-05 18:41:07 New best checkpoint at step 5: train_loss=2085.487305

nano_WaveGen/checkpoints_text2wave/training_stats.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc87b2676cd86e2d19f72c320931e39eeba832d8e385b937c4f10c7261a752cf
+size 3496

nano_WaveGen/configs/accelerate_config.yaml

@@ -0,0 +1,15 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: fp16
+num_machines: 1
+num_processes: 8
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

nano_WaveGen/configs/default.yaml

@@ -0,0 +1,154 @@
+# configs/default.yaml
+
+# ==================== 核心配置 ====================
+
+# 训练配置
+training:
+  batch_size: 24  # Reduced for faster first step (8 per GPU)
+  
+  # 基于步数的训练（v30新增）
+  max_steps: 50000  # 总训练步数（可以根据需要调整）
+  
+  # 时序预测配置
+  max_history_frames: 3  # 最大历史帧数（1-3帧）
+  bidirectional_training: true  # 双向预测训练（从中间帧开始）
+  # max_prediction_frames: 1  # <deprecated> 由模型覆盖整段序列，这里保留用于兼容
+  initial_teacher_forcing_prob: 0.5  # 初始锚点帧教师强制概率
+  initial_frame:
+    strategy: "middle"  # 可选: middle | fixed | random
+    offset: 0  # 在基础策略上的偏移
+  random_history_sampling: true  # 是否在推理时随机选择0-历史帧数量作为参考帧
+  freeze_static_from_anchor: true  # 是否在预测序列时固定0-5维的静态参数沿用锚点帧
+  multi_sample_attempts: 5  # 每个样本的随机尝试次数（仅训练时启用）
+  decoder_noise_std: 0.2  # 解码阶段加入的高斯噪声标准差（0表示禁用）
+  frame_rate: 8.0  # MOVi 样本在预处理阶段统一为 8fps，速度计算与训练假设保持一致
+  
+  # 调试和日志配置
+  debug_print_interval: 1  # 每多少步打印一次调试信息（损失值等）
+  log_interval: 50  # 每多少步记录一次日志
+  
+  learning_rate: 0.001  # Standard T5 learning rate
+  gradient_clip_val: 1.0  # Gradient clipping value
+  
+  # GPU配置
+  use_multi_gpu: true  # 启用多GPU训练
+  gpu_list: [0, 1, 2, 3, 4, 5, 6, 7]  # 可用GPU列表
+  use_free_gpus: true  # 自动选择空闲GPU
+  
+  # 保存训练的模型和生成结果 配置
+  evaluation:
+    max_batches: 0  # 例如0禁用验证，只比较训练 loss。改成 None 就是验证全量；任何正整数则限制评估批次数。
+
+  save_generation:
+    enabled: true #保存训练的模型
+    save_gt: true  # 是否保存GT数据
+    fixed_samples: 5  # 固定样本数量，用于对比
+    save_interval: 100  # 每100步保存一次
+    save_dir: "core_space"  # 保存目录
+
+
+# ==================== Text2Wave 配置 ====================
+
+# 模型设置
+text2wave_model:
+  # 原始模型: google/long-t5-tglobal-base
+  model_name: "google/t5-v1_1-small"
+
+# 损失函数配置
+loss:
+  
+  # 损失权重
+  weights:
+    wave_loss: 4.0  # 波损失（超二次元参数）权重
+    wave_contrastive_loss: 2.0  # 序列级对比损失权重
+    world_info_loss: 0.5  # 世界信息损失（相机，缩放，时间）权重
+    controllable_info_loss: 0.1  # 可控制信息损失（质量，摩擦，弹性）权重
+    pla_loss: 3.0  # 最小作用量约束损失权重
+
+  wave_contrastive:
+    temperature: 0.2  # 对比分布温度
+
+# 数据配置
+data:
+  # MOVi数据集配置
+  num_workers: 32  # 数据加载线程数
+  max_sequences: 100  # 最大序列数，-1表示使用所有数据，设置较小值用于快速测试
+
+physics:
+  gravity: 9.81  # 自由落体重力加速度（单位：m/s^2）
+  collision_buffer: 1.05  # 判定碰撞时的半径放大系数
+
+# ==================== Wave2Pixel 配置 ====================
+
+# 网格配置
+grid:
+  size: 64  # 3D网格分辨率
+  prob_threshold: 0.5
+
+# 世界坐标系配置
+world_coordinate_system:
+  enabled: true  # 是否启用世界坐标系
+  world_scale: 10.0  # 世界坐标范围 ±10米
+  voxel_size: 0.05   # 体素大小 5cm
+  near_plane: 0.1    # 近平面距离
+  far_plane: 50.0    # 远平面距离
+  predict_world_scale: true  # 让模型预测世界缩放比例
+  world_scale_loss_weight: 0.1  # 世界缩放比例损失权重 - 增加到1.0以加快学习
+
+# 相机配置
+camera:
+  default_view: "front"
+  fov: 60
+  near: 0.1
+  far: 100.0
+  # 世界坐标系中的相机位置
+  views:
+    front: [0, 0, 2]
+    back: [0, 0, -2]
+    left: [-2, 0, 0]
+    right: [2, 0, 0]
+    top: [0, 2, 0]
+    bottom: [0, -2, 0]
+  # 相机旋转角度 (pitch, yaw, roll)
+  view_rotations:
+    front: [0, 0, 0]
+    back: [0, 3.14159, 0]  # 180度旋转
+    left: [0, -1.5708, 0]  # -90度旋转
+    right: [0, 1.5708, 0]  # 90度旋转
+    top: [-1.5708, 0, 0]   # -90度俯视
+    bottom: [1.5708, 0, 0] # 90度仰视
+
+# 生成配置
+generation:
+  mode: "image"  # "image" 或 "video"
+  time:
+    start: 0.0
+    end: 12.0
+    fps: 30  # 外部观察频率
+    timestep: 0.0  # 用于单帧图像生成
+  compute_wsf: false  # 是否默认计算完整WSF场
+  output_dir: "core_space"  # 默认输出目录
+
+# 输出格式配置
+output:
+  format: "triple_channel"  # 可选: "complex", "dual_channel", "triple_channel"
+  third_channel: "amplitude"  # 如果format为"triple_channel"，第三通道的内容: "amplitude", "phase", "none"
+
+# Wave2Pixel相关的模型组件
+model:
+  wave_encoder:
+    hidden_dim: 256
+    dropout: 0.1
+  
+  feature_extractor:
+    input_dim: 4  # 实部、虚部、振幅、相位
+    hidden_dim: 64
+    output_dim: 32
+    dropout: 0.1
+
+  # 重命名为pixel_net以匹配代码中的使用
+  pixel_net:
+    channels: [32, 64, 128, 64, 4]  # 最后4通道: RGB + 概率
+    kernel_size: 3
+    padding: 1
+    dropout: 0.1

nano_WaveGen/core_space/20251205_184253_step5_text2wave/batch_statistics.json

@@ -0,0 +1,151 @@
+{
+  "batch_size": 5,
+  "num_frames": 24,
+  "timestamp": "2025-12-05T18:42:58.362258",
+  "objects_per_frame": [
+    {
+      "frame": 0,
+      "mean_objects": 3.4,
+      "max_objects": 5,
+      "min_objects": 2
+    },
+    {
+      "frame": 1,
+      "mean_objects": 3.6,
+      "max_objects": 6,
+      "min_objects": 2
+    },
+    {
+      "frame": 2,
+      "mean_objects": 3.8,
+      "max_objects": 6,
+      "min_objects": 3
+    },
+    {
+      "frame": 3,
+      "mean_objects": 3.6,
+      "max_objects": 6,
+      "min_objects": 2
+    },
+    {
+      "frame": 4,
+      "mean_objects": 4.0,
+      "max_objects": 7,
+      "min_objects": 3
+    },
+    {
+      "frame": 5,
+      "mean_objects": 3.8,
+      "max_objects": 7,
+      "min_objects": 2
+    },
+    {
+      "frame": 6,
+      "mean_objects": 4.0,
+      "max_objects": 7,
+      "min_objects": 3
+    },
+    {
+      "frame": 7,
+      "mean_objects": 3.6,
+      "max_objects": 6,
+      "min_objects": 2
+    },
+    {
+      "frame": 8,
+      "mean_objects": 4.2,
+      "max_objects": 8,
+      "min_objects": 3
+    },
+    {
+      "frame": 9,
+      "mean_objects": 3.6,
+      "max_objects": 7,
+      "min_objects": 2
+    },
+    {
+      "frame": 10,
+      "mean_objects": 3.4,
+      "max_objects": 5,
+      "min_objects": 2
+    },
+    {
+      "frame": 11,
+      "mean_objects": 3.6,
+      "max_objects": 7,
+      "min_objects": 2
+    },
+    {
+      "frame": 12,
+      "mean_objects": 3.4,
+      "max_objects": 6,
+      "min_objects": 2
+    },
+    {
+      "frame": 13,
+      "mean_objects": 3.8,
+      "max_objects": 7,
+      "min_objects": 2
+    },
+    {
+      "frame": 14,
+      "mean_objects": 2.6,
+      "max_objects": 6,
+      "min_objects": 0
+    },
+    {
+      "frame": 15,
+      "mean_objects": 3.2,
+      "max_objects": 6,
+      "min_objects": 1
+    },
+    {
+      "frame": 16,
+      "mean_objects": 2.8,
+      "max_objects": 7,
+      "min_objects": 1
+    },
+    {
+      "frame": 17,
+      "mean_objects": 2.8,
+      "max_objects": 5,
+      "min_objects": 1
+    },
+    {
+      "frame": 18,
+      "mean_objects": 3.0,
+      "max_objects": 6,
+      "min_objects": 1
+    },
+    {
+      "frame": 19,
+      "mean_objects": 2.8,
+      "max_objects": 5,
+      "min_objects": 1
+    },
+    {
+      "frame": 20,
+      "mean_objects": 2.4,
+      "max_objects": 4,
+      "min_objects": 0
+    },
+    {
+      "frame": 21,
+      "mean_objects": 3.0,
+      "max_objects": 5,
+      "min_objects": 1
+    },
+    {
+      "frame": 22,
+      "mean_objects": 2.8,
+      "max_objects": 5,
+      "min_objects": 1
+    },
+    {
+      "frame": 23,
+      "mean_objects": 3.0,
+      "max_objects": 5,
+      "min_objects": 1
+    }
+  ]
+}

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/error_statistics.json

@@ -0,0 +1,4 @@
+{
+  "object_mae": 1.520950198173523,
+  "world_mae": 3.048270600940062
+}

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/info.txt

@@ -0,0 +1,10 @@
+Text: large yellow metal cylinder, small yellow metal cylinder, small gray metal sphere
+Generated at step: 5
+Number of frames: 24
+Sequence: sample_00000
+
+--- Model Output Summary ---
+Max objects: 10
+Object parameters: 15 (exists + shape[2] + scale[3] + translation[3] + rotation[3] + velocity[3])
+World parameters: 8 (camera_pos[3] + camera_quat[4] + scene_scale[1])
+Physics parameters: 3 (mass + friction + restitution)

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/Full_Sample_Data_for_Learning_Target.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a99d3bd56b39f896e3f1febab643edf5c312fbc73d82073da95288a8b61a0ee
+size 30977

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/camera_trajectory.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4fd4cdb82402db1d9b33317c378ad09319a97c25c846d6d7435e4f7ff1d9786
+size 450

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/depth/depth_merge.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f17e9fda8898016fa785a6d002e2fb865c4945f38a486be7520263cf4704e02f
+size 2225336

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/file_manifest.txt

@@ -0,0 +1,38 @@
+Original sequence: sample_00000
+Data split: validation
+Original path: ../data/movi_a_128x128/validation/sample_00000
+Copied at: 2025-12-05T18:42:54.238918
+
+Files included:
+- Full_Sample_Data_for_Learning_Target.npz
+- camera_trajectory.npz
+- depth/depth_merge.npz
+- metadata.json
+- normal/normal_merge.npz
+- object_coordinates/object_coordinates_merge.npz
+- point_clouds/point_clouds_merge.npz
+- rgb/frame_000.png
+- rgb/frame_001.png
+- rgb/frame_002.png
+- rgb/frame_003.png
+- rgb/frame_004.png
+- rgb/frame_005.png
+- rgb/frame_006.png
+- rgb/frame_007.png
+- rgb/frame_008.png
+- rgb/frame_009.png
+- rgb/frame_010.png
+- rgb/frame_011.png
+- rgb/frame_012.png
+- rgb/frame_013.png
+- rgb/frame_014.png
+- rgb/frame_015.png
+- rgb/frame_016.png
+- rgb/frame_017.png
+- rgb/frame_018.png
+- rgb/frame_019.png
+- rgb/frame_020.png
+- rgb/frame_021.png
+- rgb/frame_022.png
+- rgb/frame_023.png
+- segmentation/segmentation_merge.npz

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/metadata.json

@@ -0,0 +1,78 @@
+{
+  "num_frames": 24,
+  "num_instances": 3,
+  "resolution": 128,
+  "depth_range": [
+    7.109129905700684,
+    67.49893951416016
+  ],
+  "camera": {
+    "focal_length": 35.0,
+    "sensor_width": 32.0,
+    "field_of_view": 0.8575560450553894,
+    "K": [
+      [
+        140.0,
+        0.0,
+        64.0
+      ],
+      [
+        0.0,
+        140.0,
+        64.0
+      ],
+      [
+        0.0,
+        0.0,
+        1.0
+      ]
+    ]
+  },
+  "instances": [
+    {
+      "id": 1,
+      "shape": "cylinder",
+      "size": "large",
+      "color": "yellow",
+      "color_rgb": [
+        1.0,
+        0.9333333373069763,
+        0.019607843831181526
+      ],
+      "material": "metal",
+      "mass": 5.784790992736816,
+      "friction": 0.4000000059604645,
+      "restitution": 0.30000001192092896
+    },
+    {
+      "id": 2,
+      "shape": "cylinder",
+      "size": "small",
+      "color": "yellow",
+      "color_rgb": [
+        1.0,
+        0.9333333373069763,
+        0.019607843831181526
+      ],
+      "material": "metal",
+      "mass": 0.723098874092102,
+      "friction": 0.4000000059604645,
+      "restitution": 0.30000001192092896
+    },
+    {
+      "id": 3,
+      "shape": "sphere",
+      "size": "small",
+      "color": "gray",
+      "color_rgb": [
+        0.34117648005485535,
+        0.34117648005485535,
+        0.34117648005485535
+      ],
+      "material": "metal",
+      "mass": 0.483887255191803,
+      "friction": 0.4000000059604645,
+      "restitution": 0.30000001192092896
+    }
+  ]
+}

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/normal/normal_merge.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01026452044e3bccdf88dae92dca52d818e95795b4e813b5ce567604de090a23
+size 126632

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/object_coordinates/object_coordinates_merge.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e29f3ba063dceb54307959ad67c82ba1b0803b2901515a5a17cbddb407c2661
+size 1620075

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/point_clouds/point_clouds_merge.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f3bc0a862953979939ec1e974e0f22cfcf78b09b384c20ba0a523f1b817fa86
+size 8503662

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/original_data/segmentation/segmentation_merge.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:147566467e6cf53cd5478853e69d816161f9dd3e5b3198021a40de9e4405272d
+size 9184

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/predictions.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f36d0a5cf1afa6e06e29b9e6fe572848d6e0fbf9193e0c26a319bf6dc9d95653
+size 124404

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_0/targets.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bee90f8ec231b1c46f276e2de5164df93d2d09f7750f2d80964dc9fc15361408
+size 115668

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_1/error_statistics.json

@@ -0,0 +1,4 @@
+{
+  "object_mae": 1.5807031393051147,
+  "world_mae": 2.9718214426902705
+}

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_1/info.txt

@@ -0,0 +1,10 @@
+Text: large purple metal cylinder, large blue rubber cylinder, small green metal cube, small red rubber sphere, small gray rubber cylinder, small blue rubber cylinder, small yellow rubber cube, small purple rubber sphere, small red metal cylinder
+Generated at step: 5
+Number of frames: 24
+Sequence: sample_00001
+
+--- Model Output Summary ---
+Max objects: 10
+Object parameters: 15 (exists + shape[2] + scale[3] + translation[3] + rotation[3] + velocity[3])
+World parameters: 8 (camera_pos[3] + camera_quat[4] + scene_scale[1])
+Physics parameters: 3 (mass + friction + restitution)

nano_WaveGen/core_space/20251205_184253_step5_text2wave/sample_1/original_data/Full_Sample_Data_for_Learning_Target.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da40da407c8b7f95f661a141ed6b6c32a340af5ab4d291c91889b89b5631504e
+size 42494