File size: 4,925 Bytes
ec0a9aa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | ## script to test env
```
python scripts/test_environment.py
python scripts/test_environment.py --training
```
## You can install install flast-attn, transformer-engine, natten = wheel
```
conda create -y cosmospredict python==3.10 -y
conda activate cosmospredict
pip install uv
cd <dir/to/your/code>
uv pip install -e .
uv pip install -r requirements.txt
wget https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.5.4/flash_attn-2.6.3+cu124torch2.6-cp310-cp310-linux_x86_64.whl
wget https://pypi.nvidia.com/transformer-engine/transformer_engine-1.13.0-py3-none-any.whl#sha256=9a7482ed47512e888fded3361153a60846ad0dd6a98d86b77f69dd60194dab05
wget https://github.com/SHI-Labs/NATTEN/releases/download/v0.17.5/natten-0.17.5%2Btorch260cu124-cp310-cp310-linux_x86_64.whl
uv pip install (these downloaed .whl file above)
```
## my draft
```
uv pip install --no-build-isolation 'transformer_engine[pytorch]==1.13.0'
CUDA_HOME=$CONDA_PREFIX uv pip install transformer-engine[pytorch]==1.13.0
CUDA_HOME=$CONDA_PREFIX uv pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext --cuda_ext" git+https://github.com/NVIDIA/apex.git
srun --pty --job-name=dreamgen \
--partition=main \
--nodes=1 \
--nodelist=worker-0,worker-1 \
--ntasks=1 \
--gpus=nvidia_h100_80gb_hbm3:2 \
--cpus-per-task=32 \
--mem=120G \
--time=48:00:00 \
bash -i
srun --pty --job-name=dreamgen \
--partition=main \
--nodes=1 \
--nodelist=worker-0,worker-1 \
--ntasks=1 \
--gpus=nvidia_h100_80gb_hbm3:4 \
--cpus-per-task=64 \
--mem=256G \
--time=240:00:00 \
bash -i
srun --pty --job-name=dreamgen \
--partition=main \
--ntasks=1 \
--gpus=nvidia_h100_80gb_hbm3:4 \
--cpus-per-task=64 \
--mem=256G \
--time=200:00:00 \
bash -i
```
## For training, we need apex but sometime, your server use CUDA toolkit 12.x but apex or torch use cuda 12.4
```
wget <you can search the internet to download cuda tookit.run file. E.x: cuda_12.4.0_550.54.14_linux.run>
# run the following commands
sh /mnt/data/sftp/data/vla_intern/workspace/binh/dreamgen/cuda_setup/cuda_12.4.0_550.54.14_linux.run --silent --toolkit --toolkitpath=/mnt/data/sftp/data/vla_intern/workspace/binh/dreamgen/cuda_setup/cuda_12.4
export CUDA_HOME=/mnt/data/sftp/data/vla_intern/workspace/binh/dreamgen/cuda_setup/cuda_12.4
pip install -v --no-cache-dir --no-build-isolation \
--config-settings=--build-option=--cpp_ext \
--config-settings=--build-option=--cuda_ext \
git+https://github.com/NVIDIA/apex.git
```
## command to prepare `t5_xxl/....pickle` files:
```
python -m scripts.get_t5_embeddings \
--dataset_path /mnt/data/sftp/data/vla_intern/workspace/binh/dreamgen/dreamgen/datasets/robocasa-100demos-5tasks-for-dg
python -m scripts.get_t5_embeddings \
--dataset_path /mnt/data/sftp/data/vla_intern/workspace/binh/dreamgen/dreamgen/datasets/realworld-for-dg-training-v1
```
## create checkpoints folder to store checkpoints
```
export HF_TOKEN=HF_TOKEN_REDACTED
nvidia/Cosmos-Predict2.5-2B
nvidia/Cosmos-Predict2-2B-Video2World
huggingface-cli download nvidia/Cosmos-Predict2-2B-Video2World --local-dir ./Cosmos-Predict2-2B-Video2World
huggingface-cli download nvidia/Cosmos-Predict2.5-2B --local-dir ./Cosmos-Predict2.5-2B
huggingface-cli download nvidia/Cosmos-Reason1-7B --local-dir ./Cosmos-Reason1-7B
```
## Sometime, server/environment fail to decode your videos, run following command to convert to correct mp4 file for decoder
```
for f in datasets/path/videos/*.mp4; do
ffmpeg -y -i "$f" -c:v libx264 -pix_fmt yuv420p -c:a copy "${f}.tmp.mp4" && mv "${f}.tmp.mp4" "$f"
done
```
## command to debug inference
```
torchrun --nproc_per_node=4 examples/video2world_lvg.py \
--num_gpus 4 \
--model_size 2B \
--dit_path /path/to/checkpoint.pt \
--batch_input_json /tmp/manifest.json \
--num_conditional_frames 1 \
--num_chunks 3 \
--disable_guardrail
torchrun --nproc_per_node=4 examples/video2world_lvg.py \
--num_gpus 4 \
--model_size 2B \
--dit_path /mnt/data/sftp/data/vla_intern/workspace/binh/dreamgen/dreamgen/checkpoints/posttraining/video2world/2b_groot_gr1_480/checkpoints_40_per/model/iter_000000280.pt \
--input_path data4infer/test_screenshots/frame_0.jpg \
--prompt "robot pick all objects from cabinet to counter" \
--num_conditional_frames 1 \
--num_chunks 3 \
--aspect_ratio "1:1" \
--disable_guardrail
```
## my draft
```
rsync -avzP vinrobot@slurm:/mnt/data/sftp/data/vla_intern/workspace/binh/dreamgen/dreamgen/chosen_checkpoints ./chosen
bash bash-running-script/Inference-bash-dreamgen/inference-fpt.sh
bash bash-running-script/Finetuning-bash-dreamgen/debug.sh
bash bash-running-script/Finetuning-bash-dreamgen/robocasa_100_training.sh
bash bash-running-script/Finetuning-bash-dreamgen/realworld_training.sh
```
## check `extract_video_frame_for_inference.py` to prepare images for video inference |