Add files using upload-large-folder tool

oob_vae_16k_224410.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d44c16941636a2db588cd5f084db45d35e1d0a02872156a1f1e135e8dfe74974
+size 657085747

synchformer_state_dict.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aff082f2df5c3bc52759db0c865c7ee772ae6400b860d1b7e90413f2defb67c
+size 950058171

vt2a-24-v55vt35-oa15-mq-td15/args.yaml

@@ -0,0 +1,29 @@
+accum_batches: 1
+batch_size: 8
+checkpoint_every: -1
+ckpt_path: ''
+config_file: defaults.ini
+dataset_config: data/data_json/TA_VTA_0923_vt_0.55_vta_0.35_ta_0.1_ttaaug0.15_bgmq_td0.15.yaml
+dataset_timing_config: ''
+dataset_type: vt2a
+gradient_clip_val: 0.0
+logger: wandb
+model_config: stable_audio_tools/configs/model_configs/txt2audio/stable_audio_ours_vt2a.json
+name: VT2A
+num_gpus: 8
+num_nodes: 1
+num_workers: 32
+precision: 16-mixed
+pretrained_ckpt_path: weights/omni2sound/vt2a-24-v55vt35-oa15-mq-td15/model.ckpt
+pretransform_ckpt_path: weights/omni2sound/oob_vae_16k_224410.ckpt
+project: null
+recover: 'false'
+remove_pretransform_weight_norm: ''
+resume: 0
+save_dir: exp/vt2a-24-v55vt35-oa15-mq-td15
+save_top_k: 2
+seed: 42
+strategy: ''
+val_dataset_config: ''
+val_every: 10000
+wandb_config: null

vt2a-24-v55vt35-oa15-mq-td15/checkpoints/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2ce98c8b5910882971c3da55d032adcfd45eb3a4ef42302d545b38e2135a04c
+size 19340649714

vt2a-24-v55vt35-oa15-mq-td15/data_config.yaml

@@ -0,0 +1,19 @@
+ta: data/mock_dataset/metas/t2a.jsonl
+r_ta: 0.1 #vt_0.55_vta_0.35_ta_0.1
+vta: data/mock_dataset/metas/vt2a.jsonl
+r_vta: 0.3889 # 0.35
+top_n_rate: 0.5 # 0.0 totally random, 1.0 pick the best
+meta_dir: data/mock_dataset/meta_dir/t2a+vt2a+v2a
+clip_fps: 8
+sync_fps: 24
+r_mixup_tta: 0.15
+text_drop_rate: 0.15
+text_drop_task_types:  ["VTA","TA"]
+bgm_quant: true
+
+music_aug:
+  - data/mock_dataset/metas/music_aug.jsonl
+  - data/mock_dataset/meta_dir/music_aug
+audio_aug:
+  - data/mock_dataset/metas/audio_aug.jsonl
+  - data/mock_dataset/meta_dir/audio_aug

vt2a-24-v55vt35-oa15-mq-td15/model_config.json

@@ -0,0 +1,246 @@
+{
+    "model_type": "diffusion_cond",
+    "sample_size": 160000,
+    "sample_rate": 16000,
+    "audio_channels": 1,
+    "model": {
+        "pretransform": {
+            "type": "autoencoder",
+            "iterate_batch": true,
+            "config": {
+                "encoder": {
+                    "type": "oobleck",
+                    "config": {
+                        "in_channels": 1,
+                        "channels": 128,
+                        "c_mults": [
+                            1,
+                            2,
+                            4,
+                            8,
+                            16
+                        ],
+                        "strides": [
+                            2,
+                            2,
+                            4,
+                            4,
+                            10
+                        ],
+                        "latent_dim": 128,
+                        "use_snake": true
+                    }
+                },
+                "decoder": {
+                    "type": "oobleck",
+                    "config": {
+                        "out_channels": 1,
+                        "channels": 128,
+                        "c_mults": [
+                            1,
+                            2,
+                            4,
+                            8,
+                            16
+                        ],
+                        "strides": [
+                            2,
+                            2,
+                            4,
+                            4,
+                            10
+                        ],
+                        "latent_dim": 64,
+                        "use_snake": true,
+                        "final_tanh": false
+                    }
+                },
+                "bottleneck": {
+                    "type": "vae"
+                },
+                "latent_dim": 64,
+                "downsampling_ratio": 640,
+                "io_channels": 1
+            }
+        },
+        "conditioning": {
+            "configs": [
+                {
+                    "id": "text_prompt",
+                    "type": "t5",
+                    "config": {
+                        "t5_model_name": "flan-t5-base",
+                        "max_length": 128,
+                        "option": true
+                    }
+                },
+                {
+                    "id": "seconds_start",
+                    "type": "number",
+                    "config": {
+                        "min_val": 0,
+                        "max_val": 512
+                    }
+                },
+                {
+                    "id": "seconds_total",
+                    "type": "number",
+                    "config": {
+                        "min_val": 0,
+                        "max_val": 512
+                    }
+                },
+                {
+                    "id": "clip_feature",
+                    "type": "post_process",
+                    "config": {
+                        "frame_per_second": 8,
+                        "feature_dim": 1024,
+                        "use_abs_pos_emb": true,
+                        "max_duration": 10,
+                        "option": true
+                    }
+                },
+                {
+                    "id": "sync_feature",
+                    "type": "post_process",
+                    "config": {
+                        "frame_per_second": 24,
+                        "target_per_second": 25,
+                        "feature_dim": 768,
+                        "max_duration": 10
+                    }
+                }
+            ],
+            "cond_dim": 1024
+        },
+        "diffusion": {
+            "cross_attention_cond_ids": [
+                "clip_feature",
+                "text_prompt",
+                "seconds_start",
+                "seconds_total"
+            ],
+            "global_cond_ids": [
+                "seconds_start",
+                "seconds_total"
+            ],
+            "additional_cond_ids": [
+                "sync_feature"
+            ],
+            "type": "dit",
+            "config": {
+                "io_channels": 64,
+                "embed_dim": 1536,
+                "depth": 24,
+                "num_heads": 24,
+                "cond_token_dim": 1024,
+                "global_cond_dim": 2048,
+                "additional_cond_dim": 1024,
+                "project_cond_tokens": true,
+                "project_additional_tokens": true,
+                "transformer_type": "continuous_transformer"
+            }
+        },
+        "io_channels": 64
+    },
+    "training": {
+        "use_ema": true,
+        "log_loss_info": false,
+        "optimizer_configs": {
+            "diffusion": {
+                "optimizer": {
+                    "type": "AdamW",
+                    "config": {
+                        "lr": 5e-05,
+                        "betas": [
+                            0.9,
+                            0.999
+                        ],
+                        "weight_decay": 0.001
+                    }
+                },
+                "scheduler": {
+                    "type": "MultiStageInverseLR",
+                    "config": {
+                        "stage_configs": [
+                            {
+                                "steps": 540000,
+                                "lr": 5e-05,
+                                "inv_gamma": 1000000,
+                                "power": 1.0,
+                                "final_lr": 1e-05,
+                                "linear_warmup_steps": 1000
+                            },
+                            {
+                                "steps": 30000,
+                                "lr": 1e-05,
+                                "inv_gamma": 1000000,
+                                "power": 1.5,
+                                "final_lr": 1e-06
+                            },
+                            {
+                                "steps": 30000,
+                                "lr": 1e-06,
+                                "inv_gamma": 1000000,
+                                "power": 2.0,
+                                "final_lr": 1e-07
+                            }
+                        ]
+                    }
+                }
+            }
+        },
+        "checkpoint": {
+            "filename": "{step}",
+            "every_n_train_steps": 300000,
+            "save_top_k": 3,
+            "monitor": "step",
+            "mode": "max"
+        },
+        "demo": {
+            "demo_every": 10000,
+            "demo_steps": 100,
+            "num_demos": 16,
+            "demo_cond_from_batch": false,
+            "demo_cond": [
+                {
+                    "path": "data/demos/0B7fwvTtvLM_000000.mp4",
+                    "clip_feature": "data/demos/clip_feature/0B7fwvTtvLM_000000.mp4.npy",
+                    "sync_feature": "data/demos/sync_feature/0B7fwvTtvLM_000000.mp4.npy",
+                    "seconds_start": 0.0,
+                    "seconds_total": 7.01,
+                    "text_prompt": "A basketball is dribbled continuously, its sharp, rhythmic thuds echoing as it impacts a hard surface."
+                },
+                {
+                    "path": "data/demos/0olhNr566Z0_000018.mp4",
+                    "clip_feature": "data/demos/clip_feature/0olhNr566Z0_000018.mp4.npy",
+                    "sync_feature": "data/demos/sync_feature/0olhNr566Z0_000018.mp4.npy",
+                    "seconds_start": 0.0,
+                    "seconds_total": 9.98,
+                    "text_prompt": "A trumpet plays a soulful jazz melody, moving through a rapid series of notes before holding a sustained, high-pitched note with a powerful vibrato."
+                },
+                {
+                    "path": "data/demos/00pATAvDGFc_000013.mp4",
+                    "clip_feature": "data/demos/clip_feature/00pATAvDGFc_000013.mp4.npy",
+                    "sync_feature": "data/demos/sync_feature/00pATAvDGFc_000013.mp4.npy",
+                    "seconds_start": 0.0,
+                    "seconds_total": 10.0,
+                    "text_prompt": "The distinct, resonant sounds of tabla drums are played with rhythmic precision, creating a complex and engaging percussive melody."
+                },
+                {
+                    "path": "data/demos/0CA5CPHRXDc_000000.mp4",
+                    "clip_feature": "data/demos/clip_feature/0CA5CPHRXDc_000000.mp4.npy",
+                    "sync_feature": "data/demos/sync_feature/0CA5CPHRXDc_000000.mp4.npy",
+                    "seconds_start": 0.0,
+                    "seconds_total": 7.0,
+                    "text_prompt": "A mosquito buzzes over high-pitched, cartoonish gibberish. A sharp slap is heard, followed immediately by a loud, abrupt explosion."
+                }
+            ],
+            "demo_cfg_scales": [
+                2.5,
+                7
+            ]
+        }
+    }
+}