Upload folder using huggingface_hub

README.md

@@ -39,36 +39,34 @@ and every difference is the adapters.
 
 | capability | metric | base | tuned | change |
 |---|---|---:|---:|---:|
-| speech | WER | 5.6% | 0.0% | -5.6% |
-| sfx | CLAP text | 0.4148 | 0.5299 | +0.1151 |
-| sfx | CLAP real | 0.4227 | 0.3920 | -0.0306 |
-| music | CLAP text | 0.3331 | 0.3212 | -0.0119 |
-| music | CLAP real | 0.6323 | 0.6305 | -0.0018 |
-| ambience | CLAP text | 0.1993 | 0.1340 | -0.0653 |
-| ambience | CLAP real | 0.3546 | 0.4003 | +0.0457 |
-| mixed | WER | 15.6% | 0.7% | -14.9% |
-| mixed | CLAP text | 0.1076 | 0.0505 | -0.0571 |
-| mixed | CLAP real | 0.7155 | 0.7072 | -0.0084 |
+| speech | WER | 5.6% | 0.2% | -5.4% |
+| sfx | CLAP text | 0.4064 | 0.3837 | -0.0226 |
+| sfx | CLAP real | 0.4152 | 0.4201 | +0.0049 |
+| music | CLAP text | 0.4736 | 0.4673 | -0.0063 |
+| music | CLAP real | 0.5928 | 0.6012 | +0.0084 |
+| ambience | CLAP text | 0.2442 | 0.2070 | -0.0371 |
+| ambience | CLAP real | 0.3667 | 0.3732 | +0.0065 |
+| mixed | WER | 4.8% | 4.7% | -0.1% |
+| mixed | CLAP text | 0.1708 | 0.1745 | +0.0037 |
+| mixed | CLAP real | 0.7014 | 0.6943 | -0.0072 |
 | speech | WER, 4-word lines | 13.9% | 0.0% | -13.9% |
 | speech | WER, 9-word lines | 3.2% | 0.0% | -3.2% |
-| speech | WER, 16-word lines | 1.9% | 0.0% | -1.9% |
-| speech | WER, 27-word lines | 3.4% | 0.0% | -3.4% |
-| speech | mean length | 4.73s | 4.06s | -0.67s |
-| sfx | mean length | 8.36s | 9.91s | +1.56s |
-| music | mean length | 9.44s | 9.20s | -0.24s |
-| ambience | mean length | 13.49s | 7.40s | -6.09s |
-| mixed | mean length | 7.33s | 5.44s | -1.89s |
+| speech | WER, 16-word lines | 1.9% | 0.6% | -1.2% |
+| speech | WER, 27-word lines | 3.4% | 0.3% | -3.1% |
+| speech | mean length | 4.73s | 4.47s | -0.26s |
+| sfx | mean length | 8.08s | 6.93s | -1.15s |
+| music | mean length | 9.42s | 9.19s | -0.23s |
+| ambience | mean length | 10.43s | 10.29s | -0.14s |
+| mixed | mean length | 4.97s | 4.47s | -0.50s |
 
 CLAP text is "does it match the caption". CLAP real is cosine to the centroid of
 genuine MECAT audio of that category — "does it sound like the real thing",
 which text similarity alone cannot see. WER cannot see either: a clip with no
 words in it scores 0% however badly the room tone came out.
 
-**WER improved by 5.6 points, and that part is solid. The scene arms are not.**
+**WER improved by 5.4 points, and that part is solid. The scene arms are not.**
 
-Caption adherence or realism fell on: sfx clap_real -0.031, music clap_text -0.012, ambience clap_text -0.065, mixed clap_text -0.057.
-
-Non-speech clips also got materially shorter (ambience 13.5s -> 7.4s, mixed 7.3s -> 5.4s), which matters because the scene prompts explicitly ask for continuity. Training on short read speech taught the stop head to end earlier, and that is correct for a spoken line and wrong for a bed.
+Caption adherence or realism fell on: sfx clap_text -0.023, ambience clap_text -0.037.
 
 **Use this for speech.** For beds, effects and mixed scenes, A/B it against the base model on your own prompts first — the held-out flow loss improved on every capability, and on the scene arms that did not translate.
 
@@ -77,11 +75,11 @@ Non-speech clips also got materially shorter (ambience 13.5s -> 7.4s, mixed 7.3s
 | | |
 |---|---|
 | Method | LoRA r=32, alpha=64, on LLM attention/MLP and the flow-matching DiT |
-| Trainable | 53870592 of 2.89 B |
+| Trainable | 72876032 of 2.89 B |
 | Precision | float32 with TF32 matmuls, matching how the base model is served |
-| Optimiser | AdamW-8bit, lr 0.0001, cosine decay, grad clip 1.0 |
+| Optimiser | AdamW-8bit, lr 5e-05, cosine decay, grad clip 1.0 |
 | Steps | 2500 at an effective batch of 16 |
-| Capability mix | speech 40%, mixed 22%, music 16%, sfx 14%, ambience 8% |
+| Capability mix | sfx 22%, ambience 20%, speech 20%, mixed 20%, music 18% |
 | Speech data | LibriTTS-R + MECAT S00; clips under 3 s oversampled 2.5x |
 | Scene data | MECAT-Caption, all eight categories |
 | Anchor | vector field pulled towards the frozen base, weight 0.5 on 35% of batches |

adapter_config.json

@@ -1,6 +1,14 @@
 {
   "alora_invocation_tokens": null,
-  "alpha_pattern": {},
+  "alpha_pattern": {
+    "cond_embedder": 128,
+    "ff.0.0": 128,
+    "ff.2": 128,
+    "to_k": 128,
+    "to_out.0": 128,
+    "to_q": 128,
+    "to_v": 128
+  },
   "arrow_config": null,
   "auto_mapping": {
     "base_model_class": "MiDashengLMGenForAudioGeneration",
@@ -31,23 +39,31 @@
   "peft_version": "0.20.0",
   "qalora_group_size": 16,
   "r": 32,
-  "rank_pattern": {},
+  "rank_pattern": {
+    "cond_embedder": 64,
+    "ff.0.0": 64,
+    "ff.2": 64,
+    "to_k": 64,
+    "to_out.0": 64,
+    "to_q": 64,
+    "to_v": 64
+  },
   "revision": null,
   "target_modules": [
+    "to_v",
     "to_q",
     "cond_embedder",
-    "k_proj",
+    "o_proj",
     "gate_proj",
-    "to_k",
+    "up_proj",
+    "k_proj",
+    "down_proj",
     "v_proj",
-    "o_proj",
-    "q_proj",
-    "to_v",
     "to_out.0",
-    "up_proj",
-    "ff.0.0",
+    "to_k",
     "ff.2",
-    "down_proj"
+    "ff.0.0",
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e07801aed31344da005c160dc3f996adb5875717b0bb4dcf9aed45fcc6f3de0e
-size 215573104
+oid sha256:0dba519c37206637b1a1ced846b51cd61adb7d71c3afe1b28a224e4b5b0588a4
+size 291595160

history.json

@@ -284,24 +284,68 @@
       },
       "accepted": true,
       "stale": 3
+    },
+    {
+      "step": 1500,
+      "losses": {
+        "speech": 0.09332506265491247,
+        "sfx": 0.20040102116763592,
+        "music": 0.16829048842191696,
+        "ambience": 0.2020278088748455,
+        "mixed": 0.1248040571808815
+      },
+      "deltas": {
+        "speech": -0.023980710613215557,
+        "sfx": -0.015695348412964598,
+        "music": -0.013629331430336832,
+        "ambience": -0.007596263671715041,
+        "mixed": -0.008029003051920398
+      },
+      "mean": 0.15776968766003846,
+      "stop_ratio": {
+        "speech": 0.9989583333333334,
+        "sfx": 0.9924806749622928,
+        "music": 0.9984447337962963,
+        "ambience": 1.0,
+        "mixed": 0.9989255536130536
+      },
+      "stop_shrink": {
+        "speech": 0.0,
+        "sfx": 0.0,
+        "music": 0.0003259216339538286,
+        "ambience": 0.0,
+        "mixed": 0.0
+      },
+      "gen_lengths": {
+        "sfx-gravel": 10.0,
+        "sfx-fire": 10.0,
+        "ambience-rain": 9.8
+      },
+      "gen_shrink": {
+        "sfx-gravel": 0.0,
+        "sfx-fire": 0.0,
+        "ambience-rain": 0.019999999999999928
+      },
+      "accepted": true,
+      "stale": 4
     }
   ],
   "best": {
-    "score": 0.15784314647316933,
-    "step": 1250,
+    "score": 0.15776968766003846,
+    "step": 1500,
     "losses": {
-      "speech": 0.09341771434992552,
-      "sfx": 0.20048575475811958,
-      "music": 0.16836701519787312,
-      "ambience": 0.20210730656981468,
-      "mixed": 0.12483794149011374
+      "speech": 0.09332506265491247,
+      "sfx": 0.20040102116763592,
+      "music": 0.16829048842191696,
+      "ambience": 0.2020278088748455,
+      "mixed": 0.1248040571808815
     },
     "deltas": {
-      "speech": -0.023011733588667538,
-      "sfx": -0.015279164569325519,
-      "music": -0.013180798849136468,
-      "ambience": -0.007205754018829721,
-      "mixed": -0.0077596828649870355
+      "speech": -0.023980710613215557,
+      "sfx": -0.015695348412964598,
+      "music": -0.013629331430336832,
+      "ambience": -0.007596263671715041,
+      "mixed": -0.008029003051920398
     },
     "stop_ratio": {
       "speech": 0.9989583333333334,