Add files using upload-large-folder tool

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/tpow2_step190k_lm1b_normal_steps128_c256_t1p3_n8/context1024_samples.txt

@@ -0,0 +1,29 @@
+<|endoftext|> on the front console console. Cut in the top of the lid on the right bottom panel of the front console. Cut down down from the front panel and it’s just as up from the bottom of console to the bottom of the panel. Take up the left console panel down to the front console. The front panel on the back of the left console console. Cut the front panel to the left end of the left console panel on the front console. With the left end of the front console you will see the button to open up the front panel. Cut down to the right of the front panel to remove the lid from the console into the console. The door to open the front console. Cut the console on to the front console with the front console on the right console console. In the front console you can see this slide down from the front console. Move the console to the left. Remove the left panel from the console panel. Once the console has moved to the front console on the front console you’ll see a cross through the right end of the left console console below the console. Open the console. Cut in and down the console over the front console. With the front console you will see the front panel console and take down the left edge of the console. Cut the console up in the right of front panel panel to place the front console and the console into the front panel console down from the right end of the front panel console into the left end of the front panel. Open the console console. Cut down down to the end of the console panel. Cut up to the front console console. Move the end of the console on the front to the front console console. In: Cut a hole between the end and the console. Cut the front panel into the front console to open the console. Cut the left end console up to the front console. Moving down: Slide the left end of the front console. Cut on the right end to the door to the console. Cut the left end console and place the front end console out on the left end. Cut up up the console down to the end of the console on the door to the right end of the console with the front panel. Cut the console’s console down to the left end. Cut up from the console and the console to the door. Take off the front panel with the front panel back down into the front panel. Moving up: Cut the front panel down to the right end, remove the front panel to the front panel. Cut down to the right end of the console and remove the lid from the console. Cut in the front console and remove the front panel back into the console to slide down to the console. Cut over the front panel on the front console. Cut down into the middle and right end to the console. Cut down to the front end of the front console console. Cut the left end on the edge of the front console console. Move the console to the front console. Cut in the right end and left end of the console and remove the top of the front console. Cut the left panel back in the front panel. Cut the back up to the right edge of the front panel console up to the top. Cut out the right panel and up the front to the top and front of the console. Cut the right end of the console and place the console out in the console. Cut down into the left panel panel on the front panel panel. Reach the left panel panel to the front of the console with the console on the right side and into the front panel. Cut the back of the right panel panel on the right side of the console. Reach up to the end of the front panel. Roll down to the left panel panel on the left side of the console. Cut it over to the right end side of the front panel. Roll out of the front end and move up with the right end to the right of the console. Cut in the right end of the console on the right end of the console. Remove the front panel from the console. Roll to the left end. Moving down now into the front door with the console for the front door. Cut up the right end of the door. Cut down down the console and cover. Moving down: Cut the right end of the front end of the console. Cut to the left end on the front of the console. Cut to the right end of the left end of the console. Cut up the bottom and left end cut down the right end of the console. Coach to down the right end of the console. Cut down through the right end. Cut the console down out in the front panel. Cut the left<|endoftext|>
+
+---
+
+<|endoftext|>, that you played the game once, but you quit the game, that you cannot just continue to play it. I see the problem. You cannot play the game, that you cannot play on your own after the game. 2. What do I tell you? 4. Tell me this is the name of the game. But you finish the game. I don't know if you quit the game. 2. Do I require you to play as you want? If the player wins the game, you can play the game. Then, if you stop playing the game, you can play games. Let's play games. When you finish the game and you start playing, you can play it as you are playing, but you just cannot play it. 1. You cannot play the game 3. You cannot play the game, cannot play the game. 3. You can play the game, if you want you can not play the game. 4. I ask you to play the game as you want. 5. You will need to go to school 5. You must first find a way to play with the game, you have a way to play the game. Here, you cannot play: You cannot play, you cannot play games He plays game after game You cannot play the game. You play games game by game You will play games, but I cannot play games! I have a game You cannot play a game... No I have a game You don't play a game! A game! No game No game, I cannot play... But you don't play the game For a game But I never finish the game No. You cannot take control of the game You say, "Oh God, let me play!" If you don't play with the game... You must get paid for playing the game! Yes But you must get paid for playing the game, while you are playing the game! No. Yes! But you cannot stop playing the game. By now, you have full control of the game. No, you have to have full control of the game while I play the game. No, I do. Play the game. It is not, you have to do so. Take notes. 1. Do you play the games? 2. "Yes" Or do you have the game next to you? You are not tied to the game, but I do play a few games, you know you cannot finish the game. I can't finish the game. I cannot end the game. I didn't pay you to play the game. 1. "No" You are required to save the game, record it before you do it. 2. You know: You cannot save the game, you will play the game again. You will not save the game. You cannot finish the game. 2. 1. "Yes I Play" 2. "Yes" That is the game, you know you can't play. 3. "Yes I Play!" Don't start playing the game while you play. When you play, you know you will be able to play the game. 4. "Yes I Play!" You can play again when you play, but finish the game after the game is played. 2. 2. "Hhaha!" Don't try to tell you to play the game. Instead, I will play with the game. 3. "You'll play it?" I will play the game, you know how to play it. 4. "I know?" I can't tell you if not to play the game, but when you know it, I will tell you what you know. 6. "I Didn't Know" When you know, you know, you can't really tell me. You will make a promise to tell me. 1. Yes. 2. "But I know." I can't tell any of you what I tell you don't know. I will tell you how to play the game. I do that until you know, yes. I can't tell you what to do until something else. 3. "I know." But I can't tell you you don't know the game. 4. "Haha!" You know,<|endoftext|>
+
+---
+
+<|endoftext|> weapon weapon weapon weapon weapon weapon A weapon weapon weapon weapon At the end of the game, you can use weapon weapon weapon weapon weapon or weapon weapon weapon weapon weapon to the weapon. At the end of the game, weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon player can use weapon weapon weapon weapon weapon or the weapon weapon weapon weapon weapon with weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon. A weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon to use weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon player can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon can be used to weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon when the player can weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. A weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon player can use weapon weapon to the player. weapon player can be weapon weapon weapon weapon when the player has weapon weapon weapon weapon weapon weapon player can use weapon weapon weapon weapon weapon to the weapon. weapon weapon weapon weapon weapon player can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. You can use weapon weapon weapon weapon if you want to use weapon weapon weapon weapon you can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can choose to weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon you can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon you can use weapon weapon weapon weapon weapon weapon You can use weapon weapon weapon weapon weapon weapon when the player uses weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon when the player can weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with weapon weapon weapon weapon weapon weapon. weapon can be used weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon is a weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon to use with weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon. You can use a weapon weapon weapon weapon weapon weapon and a weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon when the player uses weapon weapon to use the weapon weapon weapon weapon weapon weapon weapon when the player can use weapon weapon to use weapon weapon weapon with weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon with weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon with weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon the player can use weapon weapon weapon weapon with weapon weapon weapon weapon weapon weapon weapon weapon weapon if the player does not use weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon when the player can weapon weapon weapon weapon the player can get to weapon weapon weapon weapon<|endoftext|>
+
+---
+
+weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can control weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon and control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon You can control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon of the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon can control weapon weapon weapon weapon weapon weapon weapon weapon weapon can control weapon weapon weapon in the weapon weapon weapon weapon. you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon You can control weapon weapon from the weapon weapon weapon and weapon weapon weapon. you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can control weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon. you can use weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you control weapon weapon weapon weapon can control weapon weapon weapon weapon weapon weapon weapon and control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon and control weapon weapon weapon weapon weapon you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon can control the weapon weapon weapon weapon weapon weapon weapon weapon or the weapon weapon weapon weapon weapon weapon weapon and control weapon weapon weapon in the weapon weapon. you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control weapon weapon weapon of the weapon weapon weapon in the weapon weapon weapon weapon weapon can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon and control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control the weapon weapon. you can control the weapon weapon in weapon weapon weapon and use the weapon weapon in weapon weapon to control weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon
+
+---
+
+<|endoftext|> . . . . the flank, . 2. the apex of the flank, . . 1. 1. . . .2. . . . of the flank, . . the apex of the flank,. of the flank, . . . . . . . . of the flank, . . . . of the flank, . 2. . . . . apex of the flank, apex of the flank, . . 3. . . apex of the flank. the apex of the flank, . 1. . . . . 1. 2. . . . 3.. 3. 3. . 4. . 4. 2. . . . 1. 2. . 2. 1. . 3. . . . 1. . 2. . of the flank, of the flank, . . 2. . 2. 3. 2. . the apex of the flank, apexof the flank . . . 2.. 3. . . 4. 4. 4. 4. the apex of the flank . . 1. 1. 2. of the flank, . 1. 2. . . 3. . 4. . . 4. 4. . . . the flank, . 1. . 2. . . . of the flank, . . . 2. . 1. . 2. . 4. . 3. 4. . . of the flank, . . . . of the flank, . 1. 1. 2. . 1. . . 2. . . 1. . . 2. . 2. 2. 1. . . . 4. 4. 5. . . . . 2. 3. 2. 1. 2. 1. . . 4. 4. 5. . 3. . the apex of the flank, . 2. . 2. . of the flank . . 2. 1. 2. . 3. 2. 3. 3. 4. 4. 3. 4. . . of the apex of the flank, of the apex of the flank, . 2. 1. . 2. 1. . 1. . 2. . 2. . 3. 4. 4. 3. 2. . . . . 4. 4. . . . . 1. 1. 1. 2. . 2. 3. 4. 3. . 4. 5. . 4. . 3. 1. 3. . . <|endoftext|>
+
+---
+
+<|endoftext|>. It is going to the world. ... It is going to the world. And so we can find it to the heart. Because we can find it. And we'll find it to the world with door. And let's find it to the heart. And we can find it. From door, we can find it. We can find the door. So... ... There's the door. The door is door. It's a door door. So the door, door to the world. The door is door to the world. The door. The door is part of the world. It cuts into the world. It shows the world. It tells the world. The door door It opens the door door. The door door opens a door door. The door closes a door. For the door door door. It opens a door. The door door opens a door. The door closes the door. The door closes door. The door, at the door. The door closes the door. The door closes the door. Now... The door door opens the door. And the door opens door door door. But door closes the door. It closes the door. And door closes the door. As the door closes. Now... The door door opens. And the door door opens. The door closes the door, the door. The door is door closing the door. The door closes the door. Now... We look at the door. The door. "The door." Then the door closes. The door closes to the door. The door closes door. Inside. The door In door... The door closes to the door in door. The door closes the door. The door. The door closes the door of the door. The door is the door. It opens the door through the door. The door closes to the door. The door closes. The door closes. The door closes the door closes. The door closes, the door closes. It is door in the door. It's the door. The door. The door to the door. That's the door. The door is the door. But the door to door. There's a door. That's the door. That's the door to door door. And when the door opens the door closes. That's the door. ... The door closes. He stops. The door opens. We can't see the door. There's a door on the door. We... The door goes to the door. When the door opens, the door closes. The door to door. The door closes to the door. The door door closes to the door. That's the door. The door. That's the door. It's the door door. They would say door didn't open a door. I know the door. But know it. And the door in the door is door. We didn't know a door to door. The door closes. We get outside. The door... The door opens. He looks at door. He stops. He stops. "I know." He goes to the door door. He doesn't know door. Here's door... The door closes the door in the door. The door initiates the door. The door initiates the door. The door is the door. No, it's door. "It's door!" The door door. The door doesn't open door. We can't open door. No, no. We do open door. To the door in the door. The door opens. It opens the door. It door open the door in the door door. "The door?" On door door, door door. He looks at the door. The door. From the door. The door. The door to door. In door.<|endoftext|>
+
+---
+
+<|endoftext|> - Aim to target - Aim to target - Aim target - Aim to target - Aim target - Aim to target - Aim target! - Aim to the target - Aim target! - - Aim target - Aim to target - Aim to target - Aim to the target! - target target! - Aim target - Aim target! - Aim target - Aim target - Aim target - aim - aim - Aim target - Aim target! - Aim target - - aim - - aim to target - aim target - Aim to target! - aim! - Aim to target! - Aim to target - aim target - aim - - - Aim target - - Aim target - Aim target! - Aim target! - Aim target! - - Aim target! - Aim to the target - aim target - Aim target - Aim target! - Aim target - aim - Aim target! - aim - Aim target - Aim to target - Aim target - - Aim! - Aim to target! - aim - Aim to target - aim - Aim target - - Aim to the target target - aim target - Aim target - Aim target - Aim target - - - Aim target! - Aim to target - Aim target - - Aim to target - Aim to target the target - Aim target - Aim target - Aim target! - Aim target! - Aim target - Aim to target - Aim to the target - aim target - aim - Aim to target - Aim to target - aim target - aim target - - aim to target - aim target - Aim to the target - aim to target - Aim to the target target - aim - aim - Aim target - Aim to target target - aim - - - Aim to target - Aim aim to target - Aim aim to target - aim target - Aim target - - - Aim to target - Aim to target - aim target - aim target - aim target - aim target - Aim to target target - aim to target - - aim - Aim to target - aim target - aim to target - aim target - Aim to target - Aim to target - aim target - Aim to target - - target - Aim to target - aim target - - - Aim target - - Aim to target - Aim to target - aim target - Aim to target - aim - aim - Aim to target - aim target - Aim to target - Aim to target - aim target - aim target - aim - aim - - Aim target - Aim to target target - aim target - aim - Aim to target - aim target - - Aim - aim - aim - aim target - aim target - aim target - Aim to the target - target target - aim target - - Aim to target - aim target - aim target - aim - aim - Aim to target - aim target - target target - Aim to target - Aim to the target range - aim - aim - aim target - - Aim target - target target - aim to target - - aim - aim target - - Aim target - target - Aim target - Aim to target - aim target - aim target - Aim target - aim to target - aim to target - aim target - aim target - aim target <|endoftext|>
+
+---
+
+<|endoftext|> of his hands, to the wolence of his ears, the feet of the eyes, to the sockets of his eyes, the palms of his fingers, to the chords of wind, the mountains of the mountains, and the mountains of the earth, to his grandmothers, tongues, to the movements of his feet, and the ringing of his tongues, the footsteps, whispers of his voices, the words of fire, the words of blood, to the water, the words of his brothers, his relatives, the relatives, and to his father, his granddaughter, the word of his bedmates, the word of his son, and the word, to the church, and the name of the forefathers, and the word of father, and the son of the mother, the son of the son, his father, and to his son and daughters, his father, and to the grandest, his son, his father, the nephew, and the son of his cousin, and his daughter, and the name of the titents, and to the word of the titents, and the word of the Father and daughters, nor to the clergy, nor to the Catholics; nor to those of the Church, nor to those in church, nor to their wives, nor to the ministers of the church, nor to those of the church, nor to the Catholics, nor to the ministry of the church, nor to the church of the ordination, nor to the church of the believers, nor to the prayers of the believers, nor to the Protestants, nor to believers, nor to the clergy, nor to the Catholics of the church, nor to the prayers of the Protestants, nor to the prayers of the clergy of the Catholics, nor the prayers of the titents, nor, the prayers of the believers, nor the worship of men, nor to the confession of the faith, nor to the ministry of the churches, the ministry of Catholics and children, nor to the ministry of laity, neither to the confession of Christ, nor the confession of Christ, nor to the doctrine, nor to doctrine, nor the worship of the titents, nor the doctrine of God, nor to our titans, nor, to the doctrine and worship of Christ and his disciples. But to doctrine, doctrine, and the laity, and the worship of Christ, and to doctrine, doctrine, and to doctrine, and to doctrine, and our doctrine, the blessings of God, our titans, and all our believers; and all our believers, and all our voices, and all our voices, the servants of God, and all our daughters, and the daughters of Christ, and our tithes, our grandmothers, the daughters, the sons, the daughters, the sons and sons, the sons, the daughters, our sons, and our sons and daughtersons; and all our sons, the sons, our daughters, our sons, daughters, our daughters, to us the fathers, and the sons, the sons, our sons, the daughters, the mothers and relatives, the daughters, the descendents, and our sons, our daughters, husbands and relatives, our sons, the sons, the daughters, the sons, relatives, relatives, relatives, relatives, believers, traitors, believers, believers, believers, and believers, the worship of God, the servants, our laymen, servants and witnesses; neither to us the sons, and our children, the sons, neither to us the sons, and our daughters, our daughters. Our relatives, our daughters, our relatives, our wives, our wives, the sons of relatives, our sons, and our daughters. For in the Church of the church, we proclaim, our Christ, God the Church of Christ, our Father; but, we neither, nor bishops, nor believers, nor bishops, nor saints, nor churches, nor churches, nor bishops; nor, we do not, to churches nor bishops, nor ministers, nor churches, nor pews; neither, nor to Protestants; nor neither to Protestants; nor to Protestants nor nor neither to all of us Protestants; nor nor we say neither to our nor to say churches, or say neither to churches; nor to say our churches. But to us we say nor do we say that it is said; neither say nor does we say it said; but we say neither nor do we say that it is said. No, neither to us, and to Protestants, our clergy, nor to our followers, to our relatives, but to us titers, to our tits, to our grandmothers, to our relatives, to our grandmothers, our saints, our confers, our confers, our ministers, our ministers, our titents, to our bishops, to our elders, to our titents, our secretaries, our secretaries, our judges, to our ministers, our sisters, our relatives, our relatives, our vocations, to our Catholics, Protestants, our parters, to our<|endoftext|>

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/tpow2_step190k_lm1b_normal_steps128_c256_t1p3_n8/run.log

@@ -0,0 +1,59 @@
+[decode] max_len=1024 generated=2/8
+[decode] max_len=1024 generated=4/8
+[decode] max_len=1024 generated=6/8
+[decode] max_len=1024 generated=8/8
+[
+  {
+    "checkpoint": "runs/lta_owt_gpt2cached_len1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_tpow2_nanogpt_tf32_ddit768x12_gbs512_8gpu_1m_20260515_003246/latest.pt",
+    "ckpt_step": 190000,
+    "max_len": 1024,
+    "decode_rule": "dual_line_resample",
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "steps": 128,
+    "c_min": 1.0,
+    "c_max": 256.0,
+    "model_t_mode": "post",
+    "time_schedule": "uniform",
+    "time_logit_mean": -1.5,
+    "time_logit_std": 0.8,
+    "time_power": 2.0,
+    "input_noise_scale": 0.0,
+    "input_noise_until": 1.0,
+    "input_noise_dirichlet_concentration": 1.0,
+    "endpoint_softening": "none",
+    "endpoint_soft_power": 2.0,
+    "endpoint_soft_min_conf": 0.0,
+    "endpoint_soft_max_conf": 1.0,
+    "final_from": "state",
+    "final_decode": "argmax",
+    "final_sample_temp": 1.0,
+    "final_top_k": 0,
+    "final_top_p": 1.0,
+    "early_temp": 1.3,
+    "late_temp": 1.3,
+    "temp_end": 1.0,
+    "temp_power": 1.0,
+    "pos_extend": "repeat",
+    "fixed_first_token_id": null,
+    "fixed_first_token_text": "",
+    "fixed_first_initial_argmax": false,
+    "use_ema": false,
+    "n_samples": 8,
+    "sample_entropy": 2.321692294119903,
+    "unique_tokens": 382,
+    "token_count": 8192,
+    "distinct_1": 0.046630859375,
+    "distinct_2": 0.15273704789833822,
+    "top_token_mass": 0.203125,
+    "texts_preview": [
+      "<|endoftext|> on the front console console. Cut in the top of the lid on the right bottom panel of the front console. Cut down down from the front panel and it’s just as up from the bottom of console to the bottom of the panel. Take up the left console panel down to the front console. The front panel on the back of the left console console. Cut the front panel to the left end of the left console panel on the front console. With the left end of the front console you will see the button to open up the front panel. Cut down to the right of the front panel to remove the lid from the console into the console. The door to open the front console. Cut the console on to the front console with the front console on the right console console. In the front console you can see this slide down from the front console. Move the console to the left. Remove the left panel from the console panel. Once the console has moved to the front console on the front console you’ll see a cross through the right end of the left console console below the console. Open the console. Cut in and down the console over the front console. With the front console you will see the front panel console and take down the left edge of the console. Cut the console up in the right of front panel panel to place the front console and the console into the front panel console down from the right end of the front panel console into the left end of the front panel. Open the console console. Cut down down to the end of the console panel. Cut up to the front console console. Move the end of the console on the front to the front console console. In: Cut a hole between the end and the console. Cut the front panel into the front console to open the console. Cut the left end console up to the front console. Moving down: Slide the left end of the front console. Cut on the right end to the door to the console. Cut the left end console and place the front end console out on the left end. Cut up up the console down to the end of the console on the door to the right end of the console with the front panel. Cut the console’s console down to the left end. Cut up from the console and the console to the door. Take off the front panel with the front panel back down into the front panel. Moving up: Cut the front panel down to the right end, remove the front panel to the front panel. Cut down to the right end of the console and remove the lid from the console. Cut in the front console and remove the front panel back into the console to slide down to the console. Cut over the front panel on the front console. Cut down into the middle and right end to the console. Cut down to the front end of the front console console. Cut the left end on the edge of the front console console. Move the console to the front console. Cut in the right end and left end of the console and remove the top of the front console. Cut the left panel back in the front panel. Cut the back up to the right edge of the front panel console up to the top. Cut out the right panel and up the front to the top and front of the console. Cut the right end of the console and place the console out in the console. Cut down into the left panel panel on the front panel panel. Reach the left panel panel to the front of the console with the console on the right side and into the front panel. Cut the back of the right panel panel on the right side of the console. Reach up to the end of the front panel. Roll down to the left panel panel on the left side of the console. Cut it over to the right end side of the front panel. Roll out of the front end and move up with the right end to the right of the console. Cut in the right end of the console on the right end of the console. Remove the front panel from the console. Roll to the left end. Moving down now into the front door with the console for the front door. Cut up the right end of the door. Cut down down the console and cover. Moving down: Cut the right end of the front end of the console. Cut to the left end on the front of the console. Cut to the right end of the left end of the console. Cut up the bottom and left end cut down the right end of the console. Coach to down the right end of the console. Cut down through the right end. Cut the console down out in the front panel. Cut the left<|endoftext|>",
+      "<|endoftext|>, that you played the game once, but you quit the game, that you cannot just continue to play it. I see the problem. You cannot play the game, that you cannot play on your own after the game. 2. What do I tell you? 4. Tell me this is the name of the game. But you finish the game. I don't know if you quit the game. 2. Do I require you to play as you want? If the player wins the game, you can play the game. Then, if you stop playing the game, you can play games. Let's play games. When you finish the game and you start playing, you can play it as you are playing, but you just cannot play it. 1. You cannot play the game 3. You cannot play the game, cannot play the game. 3. You can play the game, if you want you can not play the game. 4. I ask you to play the game as you want. 5. You will need to go to school 5. You must first find a way to play with the game, you have a way to play the game. Here, you cannot play: You cannot play, you cannot play games He plays game after game You cannot play the game. You play games game by game You will play games, but I cannot play games! I have a game You cannot play a game... No I have a game You don't play a game! A game! No game No game, I cannot play... But you don't play the game For a game But I never finish the game No. You cannot take control of the game You say, \"Oh God, let me play!\" If you don't play with the game... You must get paid for playing the game! Yes But you must get paid for playing the game, while you are playing the game! No. Yes! But you cannot stop playing the game. By now, you have full control of the game. No, you have to have full control of the game while I play the game. No, I do. Play the game. It is not, you have to do so. Take notes. 1. Do you play the games? 2. \"Yes\" Or do you have the game next to you? You are not tied to the game, but I do play a few games, you know you cannot finish the game. I can't finish the game. I cannot end the game. I didn't pay you to play the game. 1. \"No\" You are required to save the game, record it before you do it. 2. You know: You cannot save the game, you will play the game again. You will not save the game. You cannot finish the game. 2. 1. \"Yes I Play\" 2. \"Yes\" That is the game, you know you can't play. 3. \"Yes I Play!\" Don't start playing the game while you play. When you play, you know you will be able to play the game. 4. \"Yes I Play!\" You can play again when you play, but finish the game after the game is played. 2. 2. \"Hhaha!\" Don't try to tell you to play the game. Instead, I will play with the game. 3. \"You'll play it?\" I will play the game, you know how to play it. 4. \"I know?\" I can't tell you if not to play the game, but when you know it, I will tell you what you know. 6. \"I Didn't Know\" When you know, you know, you can't really tell me. You will make a promise to tell me. 1. Yes. 2. \"But I know.\" I can't tell any of you what I tell you don't know. I will tell you how to play the game. I do that until you know, yes. I can't tell you what to do until something else. 3. \"I know.\" But I can't tell you you don't know the game. 4. \"Haha!\" You know,<|endoftext|>",
+      "<|endoftext|> weapon weapon weapon weapon weapon weapon A weapon weapon weapon weapon At the end of the game, you can use weapon weapon weapon weapon weapon or weapon weapon weapon weapon weapon to the weapon. At the end of the game, weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon player can use weapon weapon weapon weapon weapon or the weapon weapon weapon weapon weapon with weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon. A weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon to use weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon player can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon can be used to weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon when the player can weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. A weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon player can use weapon weapon to the player. weapon player can be weapon weapon weapon weapon when the player has weapon weapon weapon weapon weapon weapon player can use weapon weapon weapon weapon weapon to the weapon. weapon weapon weapon weapon weapon player can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. You can use weapon weapon weapon weapon if you want to use weapon weapon weapon weapon you can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can choose to weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon you can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon you can use weapon weapon weapon weapon weapon weapon You can use weapon weapon weapon weapon weapon weapon when the player uses weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon when the player can weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with weapon weapon weapon weapon weapon weapon. weapon can be used weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon is a weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can use weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon to use with weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon. You can use a weapon weapon weapon weapon weapon weapon and a weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon when the player uses weapon weapon to use the weapon weapon weapon weapon weapon weapon weapon when the player can use weapon weapon to use weapon weapon weapon with weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon with weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon with weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon the player can use weapon weapon weapon weapon with weapon weapon weapon weapon weapon weapon weapon weapon weapon if the player does not use weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon. weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon when the player can weapon weapon weapon weapon the player can get to weapon weapon weapon weapon<|endoftext|>",
+      "weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can control weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon and control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon You can control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon of the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon with the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon can control weapon weapon weapon weapon weapon weapon weapon weapon weapon can control weapon weapon weapon in the weapon weapon weapon weapon. you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon You can control weapon weapon from the weapon weapon weapon and weapon weapon weapon. you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you can control weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon. you can use weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon you control weapon weapon weapon weapon can control weapon weapon weapon weapon weapon weapon weapon and control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon and control weapon weapon weapon weapon weapon you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon can control the weapon weapon weapon weapon weapon weapon weapon weapon or the weapon weapon weapon weapon weapon weapon weapon and control weapon weapon weapon in the weapon weapon. you can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control weapon weapon weapon of the weapon weapon weapon in the weapon weapon weapon weapon weapon can control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon control the weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon and control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control the weapon weapon. you can control the weapon weapon in weapon weapon weapon and use the weapon weapon in weapon weapon to control weapon weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon to control weapon weapon weapon to control weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon weapon"
+    ],
+    "gen_ppl": 5.455538024310792,
+    "gen_nll": 1.6966312439926388,
+    "gen_tokens": 6665
+  }
+]

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260519/owt_compact_v2048_step52k_finesweep_entropy5_ppl30_n8/logs/steps160_c192_temps.log

@@ -0,0 +1,19 @@
+[ckpt] runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt step=52444
+[decode-base] n=8 max_len=1024 steps=160 model_t=flow
+[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.006250 dt_max=0.006250
+[decode] temp=1.15 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.15 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.18 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.18 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.20 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.20 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.22 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.22 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.25 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.25 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 192.0, "target_prob": 1.0, "endpoint_temp": 1.15, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 16.32093101462812, "nll_per_token": 2.7924483953737744, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 18.08571163199527, "nll_per_token": 2.8951222139246324, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 3.935435376875119, "unique_tokens": 852, "token_count": 8192, "distinct_1": 0.10400390625, "distinct_2": 0.3737781036168133, "top_token_mass": 0.05615234375}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 192.0, "target_prob": 1.0, "endpoint_temp": 1.18, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 22.281282142849616, "nll_per_token": 3.103746960209865, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 23.824483639053597, "nll_per_token": 3.1707137762331494, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.118388439717505, "unique_tokens": 963, "token_count": 8192, "distinct_1": 0.1175537109375, "distinct_2": 0.44782502443792765, "top_token_mass": 0.0806884765625}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 192.0, "target_prob": 1.0, "endpoint_temp": 1.2, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 19.57294977988684, "nll_per_token": 2.974148499731924, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 22.138237456577077, "nll_per_token": 3.097306315104167, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.222889944442148, "unique_tokens": 876, "token_count": 8192, "distinct_1": 0.10693359375, "distinct_2": 0.42143206256109483, "top_token_mass": 0.0614013671875}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 192.0, "target_prob": 1.0, "endpoint_temp": 1.22, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 27.01011733558444, "nll_per_token": 3.2962115119485293, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 29.95265211473401, "nll_per_token": 3.3996178720511643, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.102241546957736, "unique_tokens": 925, "token_count": 8192, "distinct_1": 0.1129150390625, "distinct_2": 0.4298631476050831, "top_token_mass": 0.0738525390625}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 192.0, "target_prob": 1.0, "endpoint_temp": 1.25, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 61.915451012937574, "nll_per_token": 4.125769761029412, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 72.47572757754818, "nll_per_token": 4.283251713771446, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.8802438288573855, "unique_tokens": 1092, "token_count": 8192, "distinct_1": 0.13330078125, "distinct_2": 0.5613391984359726, "top_token_mass": 0.0401611328125}}
+[done] docs/lta_samples/metrics_20260519/owt_compact_v2048_step52k_finesweep_entropy5_ppl30_n8/steps160_c192_temps.jsonl

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260519/owt_compact_v2048_step52k_finesweep_entropy5_ppl30_n8/logs/steps160_c256_temps.log

@@ -0,0 +1,19 @@
+[ckpt] runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt step=52444
+[decode-base] n=8 max_len=1024 steps=160 model_t=flow
+[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.006250 dt_max=0.006250
+[decode] temp=1.15 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.15 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.18 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.18 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.20 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.20 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.22 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.22 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.25 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.25 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 256.0, "target_prob": 1.0, "endpoint_temp": 1.15, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 14.491373286758098, "nll_per_token": 2.6735535266352635, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 16.146975815344685, "nll_per_token": 2.7817327761182598, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 3.883886978537484, "unique_tokens": 885, "token_count": 8192, "distinct_1": 0.1080322265625, "distinct_2": 0.40579178885630496, "top_token_mass": 0.145263671875}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 256.0, "target_prob": 1.0, "endpoint_temp": 1.18, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 14.320127887989397, "nll_per_token": 2.6616660922181374, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 15.51624578189119, "nll_per_token": 2.7418875899969364, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 3.8391281316946864, "unique_tokens": 875, "token_count": 8192, "distinct_1": 0.1068115234375, "distinct_2": 0.3812316715542522, "top_token_mass": 0.1175537109375}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 256.0, "target_prob": 1.0, "endpoint_temp": 1.2, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 18.24573769513893, "nll_per_token": 2.9039315017999385, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 19.6871352509057, "nll_per_token": 2.979965389476103, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.158735382486451, "unique_tokens": 951, "token_count": 8192, "distinct_1": 0.1160888671875, "distinct_2": 0.43499511241446726, "top_token_mass": 0.0784912109375}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 256.0, "target_prob": 1.0, "endpoint_temp": 1.22, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 21.923303209036547, "nll_per_token": 3.0875501445695464, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 24.335406539673976, "nll_per_token": 3.1919323491115197, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.392752845356918, "unique_tokens": 1025, "token_count": 8192, "distinct_1": 0.1251220703125, "distinct_2": 0.4810606060606061, "top_token_mass": 0.068115234375}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 160, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.00625, 0.0125, 0.01875, 0.025, 0.03125, 0.0375, 0.04375, 0.05, 0.05625, 0.0625, 0.06875, 0.075, 0.08125, 0.0875, 0.09375, 0.1, 0.10625, 0.1125, 0.11875, 0.125, 0.13125, 0.1375, 0.14375, 0.15, 0.15625, 0.1625, 0.16875, 0.175, 0.18125, 0.1875, 0.19375, 0.2, 0.20625, 0.2125, 0.21875, 0.225, 0.23125, 0.2375, 0.24375, 0.25, 0.25625, 0.2625, 0.26875, 0.275, 0.28125, 0.2875, 0.29375, 0.3, 0.30625, 0.3125, 0.31875, 0.325, 0.33125, 0.3375, 0.34375, 0.35, 0.35625, 0.3625, 0.36875, 0.375, 0.38125, 0.3875, 0.39375, 0.4, 0.40625, 0.4125, 0.41875, 0.425, 0.43125, 0.4375, 0.44375, 0.45, 0.45625, 0.4625, 0.46875, 0.475, 0.48125, 0.4875, 0.49375, 0.5, 0.50625, 0.5125, 0.51875, 0.525, 0.53125, 0.5375, 0.54375, 0.55, 0.55625, 0.5625, 0.56875, 0.575, 0.58125, 0.5875, 0.59375, 0.6, 0.60625, 0.6125, 0.61875, 0.625, 0.63125, 0.6375, 0.64375, 0.65, 0.65625, 0.6625, 0.66875, 0.675, 0.68125, 0.6875, 0.69375, 0.7, 0.70625, 0.7125, 0.71875, 0.725, 0.73125, 0.7375, 0.74375, 0.75, 0.75625, 0.7625, 0.76875, 0.775, 0.78125, 0.7875, 0.79375, 0.8, 0.80625, 0.8125, 0.81875, 0.825, 0.83125, 0.8375, 0.84375, 0.85, 0.85625, 0.8625, 0.86875, 0.875, 0.88125, 0.8875, 0.89375, 0.9, 0.90625, 0.9125, 0.91875, 0.925, 0.93125, 0.9375, 0.94375, 0.95, 0.95625, 0.9625, 0.96875, 0.975, 0.98125, 0.9875, 0.99375, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 256.0, "target_prob": 1.0, "endpoint_temp": 1.25, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 29.124005439095203, "nll_per_token": 3.3715627632889094, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 31.872494721465475, "nll_per_token": 3.461743403416054, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.382249063383401, "unique_tokens": 1027, "token_count": 8192, "distinct_1": 0.1253662109375, "distinct_2": 0.4954789833822092, "top_token_mass": 0.07080078125}}
+[done] docs/lta_samples/metrics_20260519/owt_compact_v2048_step52k_finesweep_entropy5_ppl30_n8/steps160_c256_temps.jsonl

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260519/owt_compact_v2048_step52k_finesweep_entropy5_ppl30_n8/logs/steps96_c128_temps.log

@@ -0,0 +1,19 @@
+[ckpt] runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt step=52444
+[decode-base] n=8 max_len=1024 steps=96 model_t=flow
+[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.010417 dt_max=0.010417
+[decode] temp=1.15 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.15 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.18 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.18 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.20 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.20 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.22 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.22 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[decode] temp=1.25 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.25 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 96, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.010416666666666666, 0.020833333333333332, 0.03125, 0.041666666666666664, 0.052083333333333336, 0.0625, 0.07291666666666667, 0.08333333333333333, 0.09375, 0.10416666666666667, 0.11458333333333333, 0.125, 0.13541666666666666, 0.14583333333333334, 0.15625, 0.16666666666666666, 0.17708333333333334, 0.1875, 0.19791666666666666, 0.20833333333333334, 0.21875, 0.22916666666666666, 0.23958333333333334, 0.25, 0.2604166666666667, 0.2708333333333333, 0.28125, 0.2916666666666667, 0.3020833333333333, 0.3125, 0.3229166666666667, 0.3333333333333333, 0.34375, 0.3541666666666667, 0.3645833333333333, 0.375, 0.3854166666666667, 0.3958333333333333, 0.40625, 0.4166666666666667, 0.4270833333333333, 0.4375, 0.4479166666666667, 0.4583333333333333, 0.46875, 0.4791666666666667, 0.4895833333333333, 0.5, 0.5104166666666666, 0.5208333333333334, 0.53125, 0.5416666666666666, 0.5520833333333334, 0.5625, 0.5729166666666666, 0.5833333333333334, 0.59375, 0.6041666666666666, 0.6145833333333334, 0.625, 0.6354166666666666, 0.6458333333333334, 0.65625, 0.6666666666666666, 0.6770833333333334, 0.6875, 0.6979166666666666, 0.7083333333333334, 0.71875, 0.7291666666666666, 0.7395833333333334, 0.75, 0.7604166666666666, 0.7708333333333334, 0.78125, 0.7916666666666666, 0.8020833333333334, 0.8125, 0.8229166666666666, 0.8333333333333334, 0.84375, 0.8541666666666666, 0.8645833333333334, 0.875, 0.8854166666666666, 0.8958333333333334, 0.90625, 0.9166666666666666, 0.9270833333333334, 0.9375, 0.9479166666666666, 0.9583333333333334, 0.96875, 0.9791666666666666, 0.9895833333333334, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 128.0, "target_prob": 1.0, "endpoint_temp": 1.15, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 99.95935110810714, "nll_per_token": 4.604763614430147, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 119.30314274312748, "nll_per_token": 4.781667671951593, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.996586378639751, "unique_tokens": 1044, "token_count": 8192, "distinct_1": 0.12744140625, "distinct_2": 0.6276881720430108, "top_token_mass": 0.0421142578125}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 96, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.010416666666666666, 0.020833333333333332, 0.03125, 0.041666666666666664, 0.052083333333333336, 0.0625, 0.07291666666666667, 0.08333333333333333, 0.09375, 0.10416666666666667, 0.11458333333333333, 0.125, 0.13541666666666666, 0.14583333333333334, 0.15625, 0.16666666666666666, 0.17708333333333334, 0.1875, 0.19791666666666666, 0.20833333333333334, 0.21875, 0.22916666666666666, 0.23958333333333334, 0.25, 0.2604166666666667, 0.2708333333333333, 0.28125, 0.2916666666666667, 0.3020833333333333, 0.3125, 0.3229166666666667, 0.3333333333333333, 0.34375, 0.3541666666666667, 0.3645833333333333, 0.375, 0.3854166666666667, 0.3958333333333333, 0.40625, 0.4166666666666667, 0.4270833333333333, 0.4375, 0.4479166666666667, 0.4583333333333333, 0.46875, 0.4791666666666667, 0.4895833333333333, 0.5, 0.5104166666666666, 0.5208333333333334, 0.53125, 0.5416666666666666, 0.5520833333333334, 0.5625, 0.5729166666666666, 0.5833333333333334, 0.59375, 0.6041666666666666, 0.6145833333333334, 0.625, 0.6354166666666666, 0.6458333333333334, 0.65625, 0.6666666666666666, 0.6770833333333334, 0.6875, 0.6979166666666666, 0.7083333333333334, 0.71875, 0.7291666666666666, 0.7395833333333334, 0.75, 0.7604166666666666, 0.7708333333333334, 0.78125, 0.7916666666666666, 0.8020833333333334, 0.8125, 0.8229166666666666, 0.8333333333333334, 0.84375, 0.8541666666666666, 0.8645833333333334, 0.875, 0.8854166666666666, 0.8958333333333334, 0.90625, 0.9166666666666666, 0.9270833333333334, 0.9375, 0.9479166666666666, 0.9583333333333334, 0.96875, 0.9791666666666666, 0.9895833333333334, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 128.0, "target_prob": 1.0, "endpoint_temp": 1.18, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 90.08121011802474, "nll_per_token": 4.500711598115809, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 122.37869439442484, "nll_per_token": 4.807120289522059, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.6648811041941025, "unique_tokens": 1020, "token_count": 8192, "distinct_1": 0.12451171875, "distinct_2": 0.5489980449657869, "top_token_mass": 0.04150390625}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 96, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.010416666666666666, 0.020833333333333332, 0.03125, 0.041666666666666664, 0.052083333333333336, 0.0625, 0.07291666666666667, 0.08333333333333333, 0.09375, 0.10416666666666667, 0.11458333333333333, 0.125, 0.13541666666666666, 0.14583333333333334, 0.15625, 0.16666666666666666, 0.17708333333333334, 0.1875, 0.19791666666666666, 0.20833333333333334, 0.21875, 0.22916666666666666, 0.23958333333333334, 0.25, 0.2604166666666667, 0.2708333333333333, 0.28125, 0.2916666666666667, 0.3020833333333333, 0.3125, 0.3229166666666667, 0.3333333333333333, 0.34375, 0.3541666666666667, 0.3645833333333333, 0.375, 0.3854166666666667, 0.3958333333333333, 0.40625, 0.4166666666666667, 0.4270833333333333, 0.4375, 0.4479166666666667, 0.4583333333333333, 0.46875, 0.4791666666666667, 0.4895833333333333, 0.5, 0.5104166666666666, 0.5208333333333334, 0.53125, 0.5416666666666666, 0.5520833333333334, 0.5625, 0.5729166666666666, 0.5833333333333334, 0.59375, 0.6041666666666666, 0.6145833333333334, 0.625, 0.6354166666666666, 0.6458333333333334, 0.65625, 0.6666666666666666, 0.6770833333333334, 0.6875, 0.6979166666666666, 0.7083333333333334, 0.71875, 0.7291666666666666, 0.7395833333333334, 0.75, 0.7604166666666666, 0.7708333333333334, 0.78125, 0.7916666666666666, 0.8020833333333334, 0.8125, 0.8229166666666666, 0.8333333333333334, 0.84375, 0.8541666666666666, 0.8645833333333334, 0.875, 0.8854166666666666, 0.8958333333333334, 0.90625, 0.9166666666666666, 0.9270833333333334, 0.9375, 0.9479166666666666, 0.9583333333333334, 0.96875, 0.9791666666666666, 0.9895833333333334, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 128.0, "target_prob": 1.0, "endpoint_temp": 1.2, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 90.11875609025836, "nll_per_token": 4.5011283126531865, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 112.0941662852602, "nll_per_token": 4.7193392884497545, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.689392053520974, "unique_tokens": 1042, "token_count": 8192, "distinct_1": 0.127197265625, "distinct_2": 0.5514418377321603, "top_token_mass": 0.0406494140625}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 96, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.010416666666666666, 0.020833333333333332, 0.03125, 0.041666666666666664, 0.052083333333333336, 0.0625, 0.07291666666666667, 0.08333333333333333, 0.09375, 0.10416666666666667, 0.11458333333333333, 0.125, 0.13541666666666666, 0.14583333333333334, 0.15625, 0.16666666666666666, 0.17708333333333334, 0.1875, 0.19791666666666666, 0.20833333333333334, 0.21875, 0.22916666666666666, 0.23958333333333334, 0.25, 0.2604166666666667, 0.2708333333333333, 0.28125, 0.2916666666666667, 0.3020833333333333, 0.3125, 0.3229166666666667, 0.3333333333333333, 0.34375, 0.3541666666666667, 0.3645833333333333, 0.375, 0.3854166666666667, 0.3958333333333333, 0.40625, 0.4166666666666667, 0.4270833333333333, 0.4375, 0.4479166666666667, 0.4583333333333333, 0.46875, 0.4791666666666667, 0.4895833333333333, 0.5, 0.5104166666666666, 0.5208333333333334, 0.53125, 0.5416666666666666, 0.5520833333333334, 0.5625, 0.5729166666666666, 0.5833333333333334, 0.59375, 0.6041666666666666, 0.6145833333333334, 0.625, 0.6354166666666666, 0.6458333333333334, 0.65625, 0.6666666666666666, 0.6770833333333334, 0.6875, 0.6979166666666666, 0.7083333333333334, 0.71875, 0.7291666666666666, 0.7395833333333334, 0.75, 0.7604166666666666, 0.7708333333333334, 0.78125, 0.7916666666666666, 0.8020833333333334, 0.8125, 0.8229166666666666, 0.8333333333333334, 0.84375, 0.8541666666666666, 0.8645833333333334, 0.875, 0.8854166666666666, 0.8958333333333334, 0.90625, 0.9166666666666666, 0.9270833333333334, 0.9375, 0.9479166666666666, 0.9583333333333334, 0.96875, 0.9791666666666666, 0.9895833333333334, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 128.0, "target_prob": 1.0, "endpoint_temp": 1.22, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 146.83992125509792, "nll_per_token": 4.989343022365196, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 193.58726655161965, "nll_per_token": 5.265728400735294, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 5.0127882745166, "unique_tokens": 1077, "token_count": 8192, "distinct_1": 0.1314697265625, "distinct_2": 0.614613880742913, "top_token_mass": 0.033935546875}}
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/step_0052444.pt", "step": 52444, "decode": {"steps": 96, "model_t_mode": "flow", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.010416666666666666, 0.020833333333333332, 0.03125, 0.041666666666666664, 0.052083333333333336, 0.0625, 0.07291666666666667, 0.08333333333333333, 0.09375, 0.10416666666666667, 0.11458333333333333, 0.125, 0.13541666666666666, 0.14583333333333334, 0.15625, 0.16666666666666666, 0.17708333333333334, 0.1875, 0.19791666666666666, 0.20833333333333334, 0.21875, 0.22916666666666666, 0.23958333333333334, 0.25, 0.2604166666666667, 0.2708333333333333, 0.28125, 0.2916666666666667, 0.3020833333333333, 0.3125, 0.3229166666666667, 0.3333333333333333, 0.34375, 0.3541666666666667, 0.3645833333333333, 0.375, 0.3854166666666667, 0.3958333333333333, 0.40625, 0.4166666666666667, 0.4270833333333333, 0.4375, 0.4479166666666667, 0.4583333333333333, 0.46875, 0.4791666666666667, 0.4895833333333333, 0.5, 0.5104166666666666, 0.5208333333333334, 0.53125, 0.5416666666666666, 0.5520833333333334, 0.5625, 0.5729166666666666, 0.5833333333333334, 0.59375, 0.6041666666666666, 0.6145833333333334, 0.625, 0.6354166666666666, 0.6458333333333334, 0.65625, 0.6666666666666666, 0.6770833333333334, 0.6875, 0.6979166666666666, 0.7083333333333334, 0.71875, 0.7291666666666666, 0.7395833333333334, 0.75, 0.7604166666666666, 0.7708333333333334, 0.78125, 0.7916666666666666, 0.8020833333333334, 0.8125, 0.8229166666666666, 0.8333333333333334, 0.84375, 0.8541666666666666, 0.8645833333333334, 0.875, 0.8854166666666666, 0.8958333333333334, 0.90625, 0.9166666666666666, 0.9270833333333334, 0.9375, 0.9479166666666666, 0.9583333333333334, 0.96875, 0.9791666666666666, 0.9895833333333334, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 128.0, "target_prob": 1.0, "endpoint_temp": 1.25, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 140.7908475613465, "nll_per_token": 4.947275438495711, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 187.84416269089778, "nll_per_token": 5.235612697227328, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 4.976983705536679, "unique_tokens": 1065, "token_count": 8192, "distinct_1": 0.1300048828125, "distinct_2": 0.6286656891495601, "top_token_mass": 0.0335693359375}}
+[done] docs/lta_samples/metrics_20260519/owt_compact_v2048_step52k_finesweep_entropy5_ppl30_n8/steps96_c128_temps.jsonl

LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/ctx1024_tradeoff_dual_20260517_225705.log

@@ -0,0 +1,1307 @@
+[ctx1024-sampleds] start stamp=ctx1024_tradeoff_dual_20260517_225705 len=1024 vocab=2664 out=docs/lta_samples/metrics_20260517/ctx1024_sampleds_sweep_bs512_ode128_ctx1024_tradeoff_dual_20260517_225705
+[ctx1024-sampleds] config=p35_unif0_0p25_outwdm1 run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 p=0.35 mode=sampled_s steps=1 s_dist=uniform s_frac=0.0->0.25 beta=2.0,6.0 outwd=-1 sync_t=1
+[ctx1024-sampleds] train config=p35_unif0_0p25_outwdm1 from=0 to=1000
+[ctx1024-sampleds] eval config=p35_unif0_0p25_outwdm1 step=1000
+[eval-decode-acc] train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 step=1000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705::none": {
+      "run": "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705",
+      "checkpoint": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/step_0001000.pt",
+      "ckpt_step": 1000,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.039886474609375,
+      "token_acc_min": 0.02734375,
+      "token_acc_max": 0.0947265625,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        3,
+        5,
+        5,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        0,
+        1,
+        0,
+        4,
+        0,
+        5,
+        5,
+        1,
+        0,
+        0,
+        5,
+        5,
+        5,
+        5,
+        0,
+        1,
+        0,
+        5,
+        5,
+        3,
+        5,
+        0,
+        0,
+        5,
+        5,
+        5,
+        5,
+        0,
+        5,
+        5,
+        3,
+        1,
+        1,
+        0,
+        1,
+        3,
+        5,
+        0,
+        0,
+        5,
+        5,
+        5,
+        0,
+        5,
+        0,
+        3,
+        5,
+        5,
+        0,
+        1,
+        0,
+        5,
+        0,
+        5
+      ],
+      "best_token_acc": [
+        0.0302734375,
+        0.03515625,
+        0.0595703125,
+        0.044921875,
+        0.041015625,
+        0.0361328125,
+        0.0341796875,
+        0.0439453125,
+        0.0546875,
+        0.0400390625,
+        0.0322265625,
+        0.033203125,
+        0.037109375,
+        0.03515625,
+        0.033203125,
+        0.037109375,
+        0.0419921875,
+        0.029296875,
+        0.0341796875,
+        0.033203125,
+        0.0458984375,
+        0.0341796875,
+        0.0380859375,
+        0.0419921875,
+        0.03125,
+        0.03515625,
+        0.037109375,
+        0.0341796875,
+        0.0400390625,
+        0.03515625,
+        0.0947265625,
+        0.03125,
+        0.033203125,
+        0.03515625,
+        0.0439453125,
+        0.0546875,
+        0.0400390625,
+        0.037109375,
+        0.0576171875,
+        0.048828125,
+        0.03515625,
+        0.0400390625,
+        0.0400390625,
+        0.0400390625,
+        0.03515625,
+        0.03515625,
+        0.0498046875,
+        0.0322265625,
+        0.033203125,
+        0.04296875,
+        0.041015625,
+        0.044921875,
+        0.0302734375,
+        0.0390625,
+        0.037109375,
+        0.03125,
+        0.0546875,
+        0.0517578125,
+        0.0302734375,
+        0.02734375,
+        0.037109375,
+        0.05859375,
+        0.0302734375,
+        0.0390625
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 ckpt_step=1000 views=512000 token_acc=0.0399 exact=0/64 exact_refs=0 hits=[]
+[eval-decode-acc] train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 step=1000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705::none": {
+      "run": "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705",
+      "checkpoint": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/step_0001000.pt",
+      "ckpt_step": 1000,
+      "endpoint_softening": "none",
+      "decode_rule": "dual_line_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.0558013916015625,
+      "token_acc_min": 0.0322265625,
+      "token_acc_max": 0.12109375,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        5,
+        3,
+        5,
+        5,
+        5,
+        3,
+        5,
+        5,
+        3,
+        5,
+        5,
+        3,
+        3,
+        5,
+        3,
+        5,
+        3,
+        5,
+        3,
+        3,
+        5,
+        4,
+        3,
+        3,
+        5,
+        3,
+        3,
+        3,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        3,
+        7,
+        5,
+        5,
+        5,
+        4,
+        2,
+        3,
+        3,
+        7,
+        7,
+        3,
+        5,
+        7,
+        5,
+        5,
+        4,
+        0,
+        7,
+        5,
+        3,
+        0,
+        3,
+        2,
+        7,
+        5,
+        5,
+        3,
+        7,
+        4
+      ],
+      "best_token_acc": [
+        0.0478515625,
+        0.0478515625,
+        0.060546875,
+        0.0546875,
+        0.091796875,
+        0.0390625,
+        0.0595703125,
+        0.0556640625,
+        0.0380859375,
+        0.05078125,
+        0.0810546875,
+        0.0693359375,
+        0.0498046875,
+        0.0849609375,
+        0.0361328125,
+        0.0546875,
+        0.060546875,
+        0.0615234375,
+        0.09765625,
+        0.044921875,
+        0.0439453125,
+        0.103515625,
+        0.0341796875,
+        0.0419921875,
+        0.0478515625,
+        0.044921875,
+        0.0537109375,
+        0.0556640625,
+        0.03515625,
+        0.0546875,
+        0.0654296875,
+        0.0576171875,
+        0.033203125,
+        0.0419921875,
+        0.0390625,
+        0.0380859375,
+        0.0498046875,
+        0.0498046875,
+        0.060546875,
+        0.0400390625,
+        0.037109375,
+        0.04296875,
+        0.0419921875,
+        0.072265625,
+        0.05078125,
+        0.0380859375,
+        0.0595703125,
+        0.0908203125,
+        0.0595703125,
+        0.0478515625,
+        0.0419921875,
+        0.0908203125,
+        0.064453125,
+        0.0869140625,
+        0.0751953125,
+        0.0478515625,
+        0.05078125,
+        0.0322265625,
+        0.0361328125,
+        0.060546875,
+        0.046875,
+        0.12109375,
+        0.05859375,
+        0.0390625
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 ckpt_step=1000 views=512000 token_acc=0.0558 exact=0/64 exact_refs=0 hits=[]
+[ctx1024-sampleds] train config=p35_unif0_0p25_outwdm1 from=1000 to=2000
+[ctx1024-sampleds] eval config=p35_unif0_0p25_outwdm1 step=2000
+[eval-decode-acc] train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 step=2000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705::none": {
+      "run": "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705",
+      "checkpoint": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/step_0002000.pt",
+      "ckpt_step": 2000,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.8083953857421875,
+      "token_acc_min": 0.1259765625,
+      "token_acc_max": 0.9404296875,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        3,
+        3,
+        3,
+        0,
+        1,
+        3,
+        1,
+        1,
+        1,
+        3,
+        3,
+        3,
+        1,
+        3,
+        1,
+        3,
+        1,
+        5,
+        3,
+        1,
+        2,
+        3,
+        1,
+        1,
+        0,
+        4,
+        3,
+        1,
+        1,
+        3,
+        1,
+        1,
+        0,
+        3,
+        3,
+        1,
+        3,
+        0,
+        1,
+        1,
+        1,
+        3,
+        1,
+        3,
+        4,
+        1,
+        2,
+        1,
+        1,
+        3,
+        7,
+        1,
+        1,
+        1,
+        0,
+        0,
+        7,
+        4,
+        1,
+        1,
+        7,
+        1,
+        1,
+        1
+      ],
+      "best_token_acc": [
+        0.91796875,
+        0.9150390625,
+        0.8662109375,
+        0.712890625,
+        0.7900390625,
+        0.9228515625,
+        0.9072265625,
+        0.748046875,
+        0.9091796875,
+        0.9013671875,
+        0.8740234375,
+        0.90625,
+        0.8935546875,
+        0.8974609375,
+        0.8779296875,
+        0.9140625,
+        0.904296875,
+        0.3671875,
+        0.923828125,
+        0.8779296875,
+        0.791015625,
+        0.2236328125,
+        0.7861328125,
+        0.892578125,
+        0.751953125,
+        0.9404296875,
+        0.73828125,
+        0.9052734375,
+        0.857421875,
+        0.8837890625,
+        0.90625,
+        0.9013671875,
+        0.5400390625,
+        0.6630859375,
+        0.9091796875,
+        0.802734375,
+        0.892578125,
+        0.5458984375,
+        0.912109375,
+        0.1259765625,
+        0.8876953125,
+        0.91796875,
+        0.890625,
+        0.9111328125,
+        0.90625,
+        0.75390625,
+        0.8544921875,
+        0.9052734375,
+        0.8623046875,
+        0.91796875,
+        0.619140625,
+        0.8515625,
+        0.9072265625,
+        0.8671875,
+        0.8037109375,
+        0.783203125,
+        0.8935546875,
+        0.9345703125,
+        0.5693359375,
+        0.31640625,
+        0.7666015625,
+        0.826171875,
+        0.8984375,
+        0.8955078125
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 ckpt_step=2000 views=1024000 token_acc=0.8084 exact=0/64 exact_refs=0 hits=[]
+[eval-decode-acc] train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 step=2000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705::none": {
+      "run": "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705",
+      "checkpoint": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/step_0002000.pt",
+      "ckpt_step": 2000,
+      "endpoint_softening": "none",
+      "decode_rule": "dual_line_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.8959808349609375,
+      "token_acc_min": 0.7607421875,
+      "token_acc_max": 0.953125,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        2,
+        0,
+        2,
+        2,
+        3,
+        1,
+        0,
+        7,
+        2,
+        2,
+        2,
+        3,
+        2,
+        7,
+        2,
+        3,
+        0,
+        2,
+        2,
+        7,
+        2,
+        3,
+        2,
+        7,
+        7,
+        0,
+        3,
+        2,
+        3,
+        3,
+        2,
+        2,
+        3,
+        2,
+        2,
+        3,
+        2,
+        2,
+        2,
+        2,
+        0,
+        0,
+        2,
+        5,
+        3,
+        2,
+        2,
+        2,
+        3,
+        0,
+        2,
+        5,
+        4,
+        3,
+        3,
+        7,
+        0,
+        2,
+        3,
+        2,
+        3,
+        7,
+        2,
+        2
+      ],
+      "best_token_acc": [
+        0.9111328125,
+        0.76171875,
+        0.912109375,
+        0.91796875,
+        0.9208984375,
+        0.9033203125,
+        0.7783203125,
+        0.9072265625,
+        0.912109375,
+        0.9130859375,
+        0.9140625,
+        0.9140625,
+        0.9111328125,
+        0.9091796875,
+        0.9130859375,
+        0.919921875,
+        0.775390625,
+        0.9130859375,
+        0.912109375,
+        0.9140625,
+        0.9150390625,
+        0.9091796875,
+        0.9140625,
+        0.9033203125,
+        0.9052734375,
+        0.779296875,
+        0.9228515625,
+        0.9140625,
+        0.9140625,
+        0.9189453125,
+        0.9189453125,
+        0.9150390625,
+        0.9267578125,
+        0.91015625,
+        0.9130859375,
+        0.91796875,
+        0.908203125,
+        0.9189453125,
+        0.9150390625,
+        0.912109375,
+        0.7666015625,
+        0.7802734375,
+        0.9150390625,
+        0.8994140625,
+        0.9111328125,
+        0.9072265625,
+        0.9091796875,
+        0.9169921875,
+        0.912109375,
+        0.7607421875,
+        0.9111328125,
+        0.900390625,
+        0.953125,
+        0.9189453125,
+        0.9228515625,
+        0.91015625,
+        0.771484375,
+        0.9150390625,
+        0.91796875,
+        0.904296875,
+        0.9150390625,
+        0.9111328125,
+        0.9091796875,
+        0.9169921875
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 ckpt_step=2000 views=1024000 token_acc=0.8960 exact=0/64 exact_refs=0 hits=[]
+[ctx1024-sampleds] train config=p35_unif0_0p25_outwdm1 from=2000 to=3000
+[ctx1024-sampleds] eval config=p35_unif0_0p25_outwdm1 step=3000
+[eval-decode-acc] train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 step=3000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705::none": {
+      "run": "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705",
+      "checkpoint": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/step_0003000.pt",
+      "ckpt_step": 3000,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.714691162109375,
+      "token_acc_min": 0.39453125,
+      "token_acc_max": 0.994140625,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        4,
+        3,
+        4,
+        4,
+        3,
+        3,
+        3,
+        3,
+        4,
+        4,
+        3,
+        3,
+        3,
+        3,
+        3,
+        4,
+        4,
+        4,
+        3,
+        3,
+        3,
+        3,
+        4,
+        4,
+        4,
+        4,
+        5,
+        3,
+        3,
+        3,
+        3,
+        4,
+        3,
+        3,
+        4,
+        4,
+        3,
+        4,
+        4,
+        4,
+        4,
+        4,
+        3,
+        3,
+        3,
+        4,
+        5,
+        4,
+        4,
+        3,
+        4,
+        3,
+        4,
+        3,
+        3,
+        4,
+        4,
+        3,
+        3,
+        3,
+        3,
+        4,
+        4,
+        4
+      ],
+      "best_token_acc": [
+        0.833984375,
+        0.5283203125,
+        0.8798828125,
+        0.4619140625,
+        0.734375,
+        0.654296875,
+        0.83203125,
+        0.6171875,
+        0.9921875,
+        0.4677734375,
+        0.822265625,
+        0.4443359375,
+        0.85546875,
+        0.4833984375,
+        0.5419921875,
+        0.9326171875,
+        0.939453125,
+        0.7412109375,
+        0.625,
+        0.6806640625,
+        0.4599609375,
+        0.91015625,
+        0.5107421875,
+        0.865234375,
+        0.455078125,
+        0.8427734375,
+        0.572265625,
+        0.53125,
+        0.595703125,
+        0.73046875,
+        0.8173828125,
+        0.39453125,
+        0.9609375,
+        0.587890625,
+        0.9794921875,
+        0.9287109375,
+        0.7939453125,
+        0.994140625,
+        0.7939453125,
+        0.8359375,
+        0.875,
+        0.533203125,
+        0.8125,
+        0.7158203125,
+        0.75,
+        0.4931640625,
+        0.94921875,
+        0.9453125,
+        0.8994140625,
+        0.51953125,
+        0.9931640625,
+        0.875,
+        0.5595703125,
+        0.8125,
+        0.7080078125,
+        0.8935546875,
+        0.427734375,
+        0.513671875,
+        0.4287109375,
+        0.9326171875,
+        0.5634765625,
+        0.9501953125,
+        0.455078125,
+        0.5048828125
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 ckpt_step=3000 views=1536000 token_acc=0.7147 exact=0/64 exact_refs=0 hits=[]
+[eval-decode-acc] train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 step=3000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705::none": {
+      "run": "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705",
+      "checkpoint": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/step_0003000.pt",
+      "ckpt_step": 3000,
+      "endpoint_softening": "none",
+      "decode_rule": "dual_line_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.9047393798828125,
+      "token_acc_min": 0.1435546875,
+      "token_acc_max": 0.9775390625,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        4,
+        4,
+        2,
+        4,
+        4,
+        4,
+        4,
+        4,
+        5,
+        4,
+        4,
+        4,
+        2,
+        4,
+        2,
+        4,
+        2,
+        2,
+        4,
+        3,
+        1,
+        5,
+        2,
+        4,
+        3,
+        5,
+        3,
+        4,
+        4,
+        3,
+        3,
+        6,
+        3,
+        4,
+        4,
+        2,
+        5,
+        4,
+        4,
+        2,
+        4,
+        4,
+        4,
+        4,
+        2,
+        1,
+        3,
+        4,
+        2,
+        4,
+        4,
+        4,
+        2,
+        4,
+        4,
+        4,
+        4,
+        4,
+        3,
+        4,
+        2,
+        3,
+        2,
+        4
+      ],
+      "best_token_acc": [
+        0.9716796875,
+        0.97265625,
+        0.833984375,
+        0.974609375,
+        0.974609375,
+        0.9638671875,
+        0.9736328125,
+        0.9658203125,
+        0.939453125,
+        0.966796875,
+        0.9736328125,
+        0.966796875,
+        0.8095703125,
+        0.974609375,
+        0.8251953125,
+        0.9697265625,
+        0.8359375,
+        0.837890625,
+        0.96875,
+        0.8681640625,
+        0.701171875,
+        0.9365234375,
+        0.7919921875,
+        0.970703125,
+        0.8515625,
+        0.939453125,
+        0.8857421875,
+        0.9765625,
+        0.9697265625,
+        0.8828125,
+        0.873046875,
+        0.1435546875,
+        0.90625,
+        0.9609375,
+        0.97265625,
+        0.8212890625,
+        0.9306640625,
+        0.9677734375,
+        0.96875,
+        0.83984375,
+        0.97265625,
+        0.970703125,
+        0.9736328125,
+        0.96875,
+        0.830078125,
+        0.681640625,
+        0.8916015625,
+        0.9736328125,
+        0.8349609375,
+        0.9716796875,
+        0.9755859375,
+        0.9716796875,
+        0.822265625,
+        0.95703125,
+        0.962890625,
+        0.96875,
+        0.9677734375,
+        0.9697265625,
+        0.9072265625,
+        0.9775390625,
+        0.8232421875,
+        0.892578125,
+        0.80859375,
+        0.970703125
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 ckpt_step=3000 views=1536000 token_acc=0.9047 exact=0/64 exact_refs=0 hits=[]
+[ctx1024-sampleds] train config=p35_unif0_0p25_outwdm1 from=3000 to=4000
+[ctx1024-sampleds] eval config=p35_unif0_0p25_outwdm1 step=4000
+[eval-decode-acc] train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 step=4000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705::none": {
+      "run": "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705",
+      "checkpoint": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/step_0004000.pt",
+      "ckpt_step": 4000,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.93096923828125,
+      "token_acc_min": 0.51953125,
+      "token_acc_max": 0.9990234375,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        1,
+        1,
+        1,
+        3,
+        7,
+        1,
+        2,
+        1,
+        1,
+        4,
+        1,
+        1,
+        3,
+        1,
+        3,
+        2,
+        1,
+        3,
+        3,
+        1,
+        3,
+        1,
+        3,
+        2,
+        1,
+        1,
+        3,
+        3,
+        3,
+        1,
+        2,
+        1,
+        3,
+        1,
+        3,
+        1,
+        3,
+        1,
+        1,
+        1,
+        2,
+        3,
+        3,
+        1,
+        1,
+        3,
+        1,
+        3,
+        3,
+        4,
+        1,
+        3,
+        5,
+        1,
+        3,
+        2,
+        1,
+        1,
+        1,
+        1,
+        2,
+        1,
+        1,
+        3
+      ],
+      "best_token_acc": [
+        0.9814453125,
+        0.9912109375,
+        0.9921875,
+        0.8837890625,
+        0.998046875,
+        0.51953125,
+        0.9912109375,
+        0.9755859375,
+        0.9365234375,
+        0.9560546875,
+        0.9921875,
+        0.9873046875,
+        0.94140625,
+        0.9052734375,
+        0.9111328125,
+        0.9931640625,
+        0.8427734375,
+        0.556640625,
+        0.9951171875,
+        0.978515625,
+        0.98828125,
+        0.9697265625,
+        0.9873046875,
+        0.9921875,
+        0.9892578125,
+        0.986328125,
+        0.865234375,
+        0.9990234375,
+        0.6884765625,
+        0.9921875,
+        0.8115234375,
+        0.6044921875,
+        0.9853515625,
+        0.9921875,
+        0.880859375,
+        0.9853515625,
+        0.99609375,
+        0.9248046875,
+        0.982421875,
+        0.9833984375,
+        0.96484375,
+        0.998046875,
+        0.98046875,
+        0.990234375,
+        0.9921875,
+        0.81640625,
+        0.990234375,
+        0.9833984375,
+        0.99609375,
+        0.9951171875,
+        0.70703125,
+        0.904296875,
+        0.9990234375,
+        0.9794921875,
+        0.994140625,
+        0.740234375,
+        0.8798828125,
+        0.990234375,
+        0.984375,
+        0.990234375,
+        0.826171875,
+        0.9921875,
+        0.9873046875,
+        0.966796875
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 ckpt_step=4000 views=2048000 token_acc=0.9310 exact=0/64 exact_refs=0 hits=[]
+[eval-decode-acc] train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 step=4000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705::none": {
+      "run": "train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705",
+      "checkpoint": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/step_0004000.pt",
+      "ckpt_step": 4000,
+      "endpoint_softening": "none",
+      "decode_rule": "dual_line_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.9383087158203125,
+      "token_acc_min": 0.771484375,
+      "token_acc_max": 0.998046875,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        4,
+        6,
+        1,
+        4,
+        6,
+        1,
+        3,
+        6,
+        2,
+        4,
+        3,
+        3,
+        6,
+        3,
+        6,
+        4,
+        4,
+        6,
+        6,
+        3,
+        2,
+        1,
+        4,
+        3,
+        3,
+        4,
+        6,
+        4,
+        3,
+        3,
+        4,
+        1,
+        4,
+        1,
+        4,
+        4,
+        1,
+        6,
+        6,
+        6,
+        1,
+        6,
+        1,
+        4,
+        6,
+        1,
+        6,
+        7,
+        1,
+        7,
+        4,
+        7,
+        4,
+        3,
+        4,
+        1,
+        2,
+        4,
+        4,
+        6,
+        6,
+        3,
+        1,
+        6
+      ],
+      "best_token_acc": [
+        0.9970703125,
+        0.7880859375,
+        0.984375,
+        0.99609375,
+        0.779296875,
+        0.9853515625,
+        0.998046875,
+        0.77734375,
+        0.9921875,
+        0.998046875,
+        0.998046875,
+        0.998046875,
+        0.796875,
+        0.998046875,
+        0.7919921875,
+        0.998046875,
+        0.998046875,
+        0.7890625,
+        0.80078125,
+        0.998046875,
+        0.994140625,
+        0.986328125,
+        0.9970703125,
+        0.998046875,
+        0.998046875,
+        0.9970703125,
+        0.771484375,
+        0.9970703125,
+        0.998046875,
+        0.9970703125,
+        0.9970703125,
+        0.984375,
+        0.9970703125,
+        0.986328125,
+        0.9970703125,
+        0.9970703125,
+        0.9814453125,
+        0.7900390625,
+        0.7900390625,
+        0.7841796875,
+        0.9853515625,
+        0.7978515625,
+        0.982421875,
+        0.9970703125,
+        0.7900390625,
+        0.984375,
+        0.7978515625,
+        0.9814453125,
+        0.986328125,
+        0.9736328125,
+        0.9970703125,
+        0.9873046875,
+        0.998046875,
+        0.998046875,
+        0.9970703125,
+        0.9853515625,
+        0.9921875,
+        0.9970703125,
+        0.9951171875,
+        0.7724609375,
+        0.7880859375,
+        0.998046875,
+        0.9853515625,
+        0.7822265625
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705 ckpt_step=4000 views=2048000 token_acc=0.9383 exact=0/64 exact_refs=0 hits=[]
+[ctx1024-sampleds] train config=p35_unif0_0p25_outwdm1 from=4000 to=5000

LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_randk_20260518_014800.log

@@ -0,0 +1,2621 @@
+[ctx1024-sampleds] start stamp=t5tok_ctx1024_randk_20260518_014800 len=1024 vocab=2423 out=docs/lta_samples/metrics_20260518/t5tok_ctx1024_randk_t5tok_ctx1024_randk_20260518_014800
+[ctx1024-sampleds] config=p50_rand0_4_unif0_0p25_outwdm1 run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 p=0.50 mode=sampled_path steps=4 steps_min=0 s_dist=uniform s_frac=0.0->0.25 beta=2.0,6.0 outwd=-1 sync_t=1
+[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=0 to=1000
+[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=1000
+[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=1000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0001000.pt",
+      "ckpt_step": 1000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.0328826904296875,
+      "token_acc_min": 0.015625,
+      "token_acc_max": 0.0576171875,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        5,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5
+      ],
+      "best_token_acc": [
+        0.037109375,
+        0.017578125,
+        0.0185546875,
+        0.041015625,
+        0.0283203125,
+        0.029296875,
+        0.0234375,
+        0.0302734375,
+        0.025390625,
+        0.0322265625,
+        0.0341796875,
+        0.0400390625,
+        0.0263671875,
+        0.025390625,
+        0.0419921875,
+        0.0263671875,
+        0.03515625,
+        0.0302734375,
+        0.025390625,
+        0.025390625,
+        0.021484375,
+        0.0302734375,
+        0.0341796875,
+        0.041015625,
+        0.037109375,
+        0.0400390625,
+        0.0419921875,
+        0.037109375,
+        0.0302734375,
+        0.02734375,
+        0.0458984375,
+        0.0390625,
+        0.03125,
+        0.037109375,
+        0.015625,
+        0.03515625,
+        0.015625,
+        0.0341796875,
+        0.048828125,
+        0.041015625,
+        0.037109375,
+        0.0283203125,
+        0.0380859375,
+        0.0576171875,
+        0.0361328125,
+        0.0361328125,
+        0.037109375,
+        0.0185546875,
+        0.037109375,
+        0.0224609375,
+        0.033203125,
+        0.03125,
+        0.029296875,
+        0.029296875,
+        0.033203125,
+        0.0400390625,
+        0.041015625,
+        0.0322265625,
+        0.0361328125,
+        0.029296875,
+        0.0361328125,
+        0.04296875,
+        0.0439453125,
+        0.01953125
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=1000 views=512000 token_acc=0.0329 exact=0/64 exact_refs=0 hits=[]
+[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=1000 to=2000
+[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=2000
+[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=2000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0002000.pt",
+      "ckpt_step": 2000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.9068603515625,
+      "token_acc_min": 0.0107421875,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.171875,
+      "exact_count": 11,
+      "exact_ref_coverage": 0.25,
+      "exact_ref_count": 2,
+      "exact_ref_hits": [
+        2,
+        5
+      ],
+      "best_ref_idx": [
+        5,
+        5,
+        1,
+        1,
+        1,
+        5,
+        1,
+        1,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        2,
+        3,
+        5,
+        5,
+        5,
+        3,
+        5,
+        3,
+        1,
+        2,
+        5,
+        1,
+        1,
+        5,
+        5,
+        2,
+        1,
+        5,
+        5,
+        5,
+        2,
+        2,
+        1,
+        2,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5
+      ],
+      "best_token_acc": [
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.029296875,
+        0.029296875,
+        0.01171875,
+        0.998046875,
+        1.0,
+        0.9970703125,
+        1.0,
+        0.9951171875,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.9970703125,
+        0.9970703125,
+        0.99609375,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.0107421875,
+        0.9990234375,
+        0.9970703125,
+        0.9970703125,
+        0.9970703125,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9970703125,
+        1.0,
+        0.998046875,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.0126953125,
+        0.029296875,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.998046875,
+        1.0,
+        0.998046875,
+        0.9990234375
+      ]
+    }
+  },
+  "first_exact_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0002000.pt",
+      "ckpt_step": 2000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.9068603515625,
+      "token_acc_min": 0.0107421875,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.171875,
+      "exact_count": 11,
+      "exact_ref_coverage": 0.25,
+      "exact_ref_count": 2,
+      "exact_ref_hits": [
+        2,
+        5
+      ],
+      "best_ref_idx": [
+        5,
+        5,
+        1,
+        1,
+        1,
+        5,
+        1,
+        1,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        2,
+        3,
+        5,
+        5,
+        5,
+        3,
+        5,
+        3,
+        1,
+        2,
+        5,
+        1,
+        1,
+        5,
+        5,
+        2,
+        1,
+        5,
+        5,
+        5,
+        2,
+        2,
+        1,
+        2,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5
+      ],
+      "best_token_acc": [
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.029296875,
+        0.029296875,
+        0.01171875,
+        0.998046875,
+        1.0,
+        0.9970703125,
+        1.0,
+        0.9951171875,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.9970703125,
+        0.9970703125,
+        0.99609375,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.0107421875,
+        0.9990234375,
+        0.9970703125,
+        0.9970703125,
+        0.9970703125,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9970703125,
+        1.0,
+        0.998046875,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.0126953125,
+        0.029296875,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.998046875,
+        1.0,
+        0.998046875,
+        0.9990234375
+      ]
+    }
+  }
+}
+RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=2000 views=1024000 token_acc=0.9069 exact=11/64 exact_refs=2 hits=[2, 5]
+[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=2000 to=3000
+[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=3000
+[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=3000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0003000.pt",
+      "ckpt_step": 3000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.999725341796875,
+      "token_acc_min": 0.998046875,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.734375,
+      "exact_count": 47,
+      "exact_ref_coverage": 0.25,
+      "exact_ref_count": 2,
+      "exact_ref_hits": [
+        2,
+        5
+      ],
+      "best_ref_idx": [
+        2,
+        1,
+        1,
+        2,
+        5,
+        5,
+        5,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        5,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        1,
+        2,
+        5,
+        2,
+        2,
+        5,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        5,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        5,
+        2,
+        5,
+        5,
+        5,
+        2,
+        2,
+        1,
+        2,
+        2,
+        2,
+        1,
+        5,
+        7,
+        5,
+        5,
+        2,
+        2
+      ],
+      "best_token_acc": [
+        1.0,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0
+      ]
+    }
+  },
+  "first_exact_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0003000.pt",
+      "ckpt_step": 3000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.999725341796875,
+      "token_acc_min": 0.998046875,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.734375,
+      "exact_count": 47,
+      "exact_ref_coverage": 0.25,
+      "exact_ref_count": 2,
+      "exact_ref_hits": [
+        2,
+        5
+      ],
+      "best_ref_idx": [
+        2,
+        1,
+        1,
+        2,
+        5,
+        5,
+        5,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        5,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        1,
+        2,
+        5,
+        2,
+        2,
+        5,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        5,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        5,
+        2,
+        5,
+        5,
+        5,
+        2,
+        2,
+        1,
+        2,
+        2,
+        2,
+        1,
+        5,
+        7,
+        5,
+        5,
+        2,
+        2
+      ],
+      "best_token_acc": [
+        1.0,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0
+      ]
+    }
+  }
+}
+RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=3000 views=1536000 token_acc=0.9997 exact=47/64 exact_refs=2 hits=[2, 5]
+[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=3000 to=4000
+[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=4000
+[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=4000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0004000.pt",
+      "ckpt_step": 4000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.99932861328125,
+      "token_acc_min": 0.9970703125,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.578125,
+      "exact_count": 37,
+      "exact_ref_coverage": 0.75,
+      "exact_ref_count": 6,
+      "exact_ref_hits": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        7
+      ],
+      "best_ref_idx": [
+        5,
+        5,
+        5,
+        1,
+        4,
+        1,
+        1,
+        5,
+        7,
+        5,
+        5,
+        5,
+        1,
+        4,
+        5,
+        5,
+        7,
+        3,
+        2,
+        5,
+        7,
+        2,
+        1,
+        4,
+        1,
+        1,
+        5,
+        5,
+        4,
+        5,
+        5,
+        2,
+        2,
+        5,
+        1,
+        1,
+        3,
+        5,
+        4,
+        5,
+        5,
+        5,
+        1,
+        4,
+        1,
+        5,
+        3,
+        5,
+        1,
+        4,
+        5,
+        5,
+        5,
+        3,
+        1,
+        5,
+        5,
+        5,
+        3,
+        4,
+        1,
+        7,
+        1,
+        1
+      ],
+      "best_token_acc": [
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.998046875,
+        1.0,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        1.0,
+        1.0,
+        0.998046875,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.998046875,
+        0.998046875,
+        1.0,
+        1.0,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.9970703125,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0
+      ]
+    }
+  },
+  "first_exact_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0004000.pt",
+      "ckpt_step": 4000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.99932861328125,
+      "token_acc_min": 0.9970703125,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.578125,
+      "exact_count": 37,
+      "exact_ref_coverage": 0.75,
+      "exact_ref_count": 6,
+      "exact_ref_hits": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        7
+      ],
+      "best_ref_idx": [
+        5,
+        5,
+        5,
+        1,
+        4,
+        1,
+        1,
+        5,
+        7,
+        5,
+        5,
+        5,
+        1,
+        4,
+        5,
+        5,
+        7,
+        3,
+        2,
+        5,
+        7,
+        2,
+        1,
+        4,
+        1,
+        1,
+        5,
+        5,
+        4,
+        5,
+        5,
+        2,
+        2,
+        5,
+        1,
+        1,
+        3,
+        5,
+        4,
+        5,
+        5,
+        5,
+        1,
+        4,
+        1,
+        5,
+        3,
+        5,
+        1,
+        4,
+        5,
+        5,
+        5,
+        3,
+        1,
+        5,
+        5,
+        5,
+        3,
+        4,
+        1,
+        7,
+        1,
+        1
+      ],
+      "best_token_acc": [
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.998046875,
+        1.0,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        1.0,
+        1.0,
+        0.998046875,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.998046875,
+        0.998046875,
+        1.0,
+        1.0,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.9970703125,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0
+      ]
+    }
+  }
+}
+RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=4000 views=2048000 token_acc=0.9993 exact=37/64 exact_refs=6 hits=[1, 2, 3, 4, 5, 7]
+[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=4000 to=5000
+[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=5000
+[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=5000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0005000.pt",
+      "ckpt_step": 5000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.9999237060546875,
+      "token_acc_min": 0.9990234375,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.921875,
+      "exact_count": 59,
+      "exact_ref_coverage": 0.75,
+      "exact_ref_count": 6,
+      "exact_ref_hits": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        7
+      ],
+      "best_ref_idx": [
+        7,
+        5,
+        1,
+        5,
+        2,
+        3,
+        2,
+        3,
+        2,
+        2,
+        3,
+        3,
+        2,
+        3,
+        3,
+        2,
+        1,
+        2,
+        1,
+        2,
+        2,
+        3,
+        2,
+        1,
+        2,
+        3,
+        3,
+        2,
+        3,
+        2,
+        2,
+        2,
+        5,
+        1,
+        3,
+        3,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        4,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        1,
+        2,
+        3,
+        2,
+        2,
+        2,
+        2,
+        5,
+        2,
+        2,
+        2,
+        3,
+        2,
+        2,
+        2
+      ],
+      "best_token_acc": [
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0
+      ]
+    }
+  },
+  "first_exact_by_run": {
+    "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0005000.pt",
+      "ckpt_step": 5000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.9999237060546875,
+      "token_acc_min": 0.9990234375,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.921875,
+      "exact_count": 59,
+      "exact_ref_coverage": 0.75,
+      "exact_ref_count": 6,
+      "exact_ref_hits": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        7
+      ],
+      "best_ref_idx": [
+        7,
+        5,
+        1,
+        5,
+        2,
+        3,
+        2,
+        3,
+        2,
+        2,
+        3,
+        3,
+        2,
+        3,
+        3,
+        2,
+        1,
+        2,
+        1,
+        2,
+        2,
+        3,
+        2,
+        1,
+        2,
+        3,
+        3,
+        2,
+        3,
+        2,
+        2,
+        2,
+        5,
+        1,
+        3,
+        3,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        4,
+        2,
+        2,
+        2,
+        2,
+        2,
+        2,
+        1,
+        2,
+        3,
+        2,
+        2,
+        2,
+        2,
+        5,
+        2,
+        2,
+        2,
+        3,
+        2,
+        2,
+        2
+      ],
+      "best_token_acc": [
+        1.0,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0
+      ]
+    }
+  }
+}
+RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=5000 views=2560000 token_acc=0.9999 exact=59/64 exact_refs=6 hits=[1, 2, 3, 4, 5, 7]
+[ctx1024-sampleds] capped config=p50_rand0_4_unif0_0p25_outwdm1 step=5000
+[ctx1024-sampleds] config=p35_rand0_4_unif0_0p25_outwdm1 run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 p=0.35 mode=sampled_path steps=4 steps_min=0 s_dist=uniform s_frac=0.0->0.25 beta=2.0,6.0 outwd=-1 sync_t=1
+[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=0 to=1000
+[ctx1024-sampleds] eval config=p35_rand0_4_unif0_0p25_outwdm1 step=1000
+[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=1000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0001000.pt",
+      "ckpt_step": 1000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.1269989013671875,
+      "token_acc_min": 0.013671875,
+      "token_acc_max": 0.7060546875,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        1,
+        5,
+        0,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        4,
+        5,
+        5,
+        1,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        5,
+        3,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        3,
+        5,
+        3,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5
+      ],
+      "best_token_acc": [
+        0.017578125,
+        0.0302734375,
+        0.0166015625,
+        0.0234375,
+        0.0283203125,
+        0.0234375,
+        0.6455078125,
+        0.0263671875,
+        0.0224609375,
+        0.03515625,
+        0.037109375,
+        0.03125,
+        0.51953125,
+        0.0361328125,
+        0.04296875,
+        0.0283203125,
+        0.0458984375,
+        0.1630859375,
+        0.099609375,
+        0.0224609375,
+        0.0419921875,
+        0.03125,
+        0.6328125,
+        0.7060546875,
+        0.0263671875,
+        0.142578125,
+        0.0361328125,
+        0.041015625,
+        0.013671875,
+        0.1201171875,
+        0.0302734375,
+        0.021484375,
+        0.142578125,
+        0.0478515625,
+        0.0166015625,
+        0.041015625,
+        0.041015625,
+        0.0361328125,
+        0.21484375,
+        0.03125,
+        0.013671875,
+        0.158203125,
+        0.6884765625,
+        0.03515625,
+        0.0322265625,
+        0.0322265625,
+        0.04296875,
+        0.0234375,
+        0.6162109375,
+        0.5224609375,
+        0.0166015625,
+        0.0419921875,
+        0.0146484375,
+        0.0361328125,
+        0.0185546875,
+        0.04296875,
+        0.0283203125,
+        0.0537109375,
+        0.51953125,
+        0.0703125,
+        0.2080078125,
+        0.53515625,
+        0.048828125,
+        0.017578125
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=p35_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=1000 views=512000 token_acc=0.1270 exact=0/64 exact_refs=0 hits=[]
+[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=1000 to=2000
+[ctx1024-sampleds] eval config=p35_rand0_4_unif0_0p25_outwdm1 step=2000
+[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=2000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0002000.pt",
+      "ckpt_step": 2000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.650299072265625,
+      "token_acc_min": 0.0126953125,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.453125,
+      "exact_count": 29,
+      "exact_ref_coverage": 0.125,
+      "exact_ref_count": 1,
+      "exact_ref_hits": [
+        5
+      ],
+      "best_ref_idx": [
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        3,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        3,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        2
+      ],
+      "best_token_acc": [
+        0.029296875,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.029296875,
+        1.0,
+        1.0,
+        0.998046875,
+        0.029296875,
+        1.0,
+        1.0,
+        0.029296875,
+        0.998046875,
+        0.0302734375,
+        0.029296875,
+        0.029296875,
+        0.9990234375,
+        0.029296875,
+        0.9990234375,
+        1.0,
+        0.029296875,
+        0.0302734375,
+        0.029296875,
+        1.0,
+        0.029296875,
+        0.029296875,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.029296875,
+        1.0,
+        0.0166015625,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.029296875,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0166015625,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.029296875,
+        1.0,
+        0.029296875,
+        1.0,
+        1.0,
+        0.0126953125,
+        1.0,
+        0.029296875,
+        0.029296875,
+        0.9990234375,
+        1.0,
+        0.029296875,
+        1.0,
+        0.998046875
+      ]
+    }
+  },
+  "first_exact_by_run": {
+    "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0002000.pt",
+      "ckpt_step": 2000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.650299072265625,
+      "token_acc_min": 0.0126953125,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.453125,
+      "exact_count": 29,
+      "exact_ref_coverage": 0.125,
+      "exact_ref_count": 1,
+      "exact_ref_hits": [
+        5
+      ],
+      "best_ref_idx": [
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        3,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        3,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        2
+      ],
+      "best_token_acc": [
+        0.029296875,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        0.029296875,
+        1.0,
+        1.0,
+        0.998046875,
+        0.029296875,
+        1.0,
+        1.0,
+        0.029296875,
+        0.998046875,
+        0.0302734375,
+        0.029296875,
+        0.029296875,
+        0.9990234375,
+        0.029296875,
+        0.9990234375,
+        1.0,
+        0.029296875,
+        0.0302734375,
+        0.029296875,
+        1.0,
+        0.029296875,
+        0.029296875,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.029296875,
+        1.0,
+        0.0166015625,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.029296875,
+        1.0,
+        1.0,
+        1.0,
+        1.0,
+        0.0166015625,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.029296875,
+        1.0,
+        0.029296875,
+        1.0,
+        1.0,
+        0.0126953125,
+        1.0,
+        0.029296875,
+        0.029296875,
+        0.9990234375,
+        1.0,
+        0.029296875,
+        1.0,
+        0.998046875
+      ]
+    }
+  }
+}
+RESULT config=p35_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=2000 views=1024000 token_acc=0.6503 exact=29/64 exact_refs=1 hits=[5]
+[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=2000 to=3000
+[ctx1024-sampleds] eval config=p35_rand0_4_unif0_0p25_outwdm1 step=3000
+[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=3000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0003000.pt",
+      "ckpt_step": 3000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.99871826171875,
+      "token_acc_min": 0.9970703125,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.265625,
+      "exact_count": 17,
+      "exact_ref_coverage": 0.5,
+      "exact_ref_count": 4,
+      "exact_ref_hits": [
+        1,
+        2,
+        3,
+        5
+      ],
+      "best_ref_idx": [
+        5,
+        1,
+        5,
+        5,
+        5,
+        1,
+        5,
+        1,
+        1,
+        1,
+        1,
+        3,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        5,
+        1,
+        1,
+        7,
+        1,
+        5,
+        1,
+        1,
+        1,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        1,
+        5,
+        1,
+        5,
+        1,
+        1,
+        5,
+        5,
+        1,
+        5,
+        3,
+        5,
+        5,
+        5,
+        3,
+        5,
+        1,
+        2,
+        5,
+        5,
+        1,
+        1,
+        3,
+        5,
+        1,
+        1,
+        5
+      ],
+      "best_token_acc": [
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.998046875,
+        1.0,
+        0.9970703125,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        0.998046875,
+        0.998046875,
+        0.9970703125,
+        0.998046875,
+        0.998046875,
+        0.9970703125,
+        0.998046875,
+        0.998046875,
+        1.0,
+        0.9970703125,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.998046875,
+        0.9990234375,
+        1.0,
+        0.9970703125,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.9990234375,
+        1.0,
+        0.9970703125,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.998046875,
+        0.9970703125,
+        1.0,
+        0.998046875,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        1.0
+      ]
+    }
+  },
+  "first_exact_by_run": {
+    "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0003000.pt",
+      "ckpt_step": 3000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.99871826171875,
+      "token_acc_min": 0.9970703125,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.265625,
+      "exact_count": 17,
+      "exact_ref_coverage": 0.5,
+      "exact_ref_count": 4,
+      "exact_ref_hits": [
+        1,
+        2,
+        3,
+        5
+      ],
+      "best_ref_idx": [
+        5,
+        1,
+        5,
+        5,
+        5,
+        1,
+        5,
+        1,
+        1,
+        1,
+        1,
+        3,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        5,
+        1,
+        1,
+        7,
+        1,
+        5,
+        1,
+        1,
+        1,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        1,
+        5,
+        1,
+        5,
+        1,
+        1,
+        5,
+        5,
+        1,
+        5,
+        3,
+        5,
+        5,
+        5,
+        3,
+        5,
+        1,
+        2,
+        5,
+        5,
+        1,
+        1,
+        3,
+        5,
+        1,
+        1,
+        5
+      ],
+      "best_token_acc": [
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.998046875,
+        1.0,
+        0.9970703125,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        1.0,
+        1.0,
+        0.998046875,
+        0.998046875,
+        0.9970703125,
+        0.998046875,
+        0.998046875,
+        0.9970703125,
+        0.998046875,
+        0.998046875,
+        1.0,
+        0.9970703125,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        0.998046875,
+        0.9990234375,
+        1.0,
+        0.9970703125,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        0.9990234375,
+        1.0,
+        0.9970703125,
+        0.9990234375,
+        1.0,
+        1.0,
+        1.0,
+        0.9990234375,
+        1.0,
+        1.0,
+        0.998046875,
+        0.9970703125,
+        1.0,
+        0.998046875,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.998046875,
+        0.998046875,
+        1.0
+      ]
+    }
+  }
+}
+RESULT config=p35_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=3000 views=1536000 token_acc=0.9987 exact=17/64 exact_refs=4 hits=[1, 2, 3, 5]
+[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=3000 to=4000
+[ctx1024-sampleds] eval config=p35_rand0_4_unif0_0p25_outwdm1 step=4000
+[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=4000 soft=none
+[decode] max_len=1024 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0004000.pt",
+      "ckpt_step": 4000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.9990234375,
+      "token_acc_min": 0.9970703125,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.109375,
+      "exact_count": 7,
+      "exact_ref_coverage": 0.375,
+      "exact_ref_count": 3,
+      "exact_ref_hits": [
+        3,
+        4,
+        7
+      ],
+      "best_ref_idx": [
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        4,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        3,
+        1,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        4,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        7,
+        5,
+        5,
+        7,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        5
+      ],
+      "best_token_acc": [
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9970703125,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9970703125,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375
+      ]
+    }
+  },
+  "first_exact_by_run": {
+    "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": {
+      "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800",
+      "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0004000.pt",
+      "ckpt_step": 4000,
+      "endpoint_softening": "none",
+      "decode_rule": "dirichlet_resample",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.9990234375,
+      "token_acc_min": 0.9970703125,
+      "token_acc_max": 1.0,
+      "exact_acc": 0.109375,
+      "exact_count": 7,
+      "exact_ref_coverage": 0.375,
+      "exact_ref_count": 3,
+      "exact_ref_hits": [
+        3,
+        4,
+        7
+      ],
+      "best_ref_idx": [
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        4,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        3,
+        1,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        1,
+        4,
+        5,
+        5,
+        5,
+        2,
+        5,
+        5,
+        7,
+        5,
+        5,
+        7,
+        5,
+        5,
+        1,
+        5,
+        5,
+        5,
+        5
+      ],
+      "best_token_acc": [
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        1.0,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9970703125,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9970703125,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        1.0,
+        0.9990234375,
+        0.9990234375,
+        0.998046875,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375,
+        0.9990234375
+      ]
+    }
+  }
+}
+RESULT config=p35_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=4000 views=2048000 token_acc=0.9990 exact=7/64 exact_refs=3 hits=[3, 4, 7]
+[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=4000 to=5000

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_inline/__init__.py

@@ -0,0 +1,31 @@
+__all__ = (
+    "StateInline",
+    "autolink",
+    "backtick",
+    "emphasis",
+    "entity",
+    "escape",
+    "fragments_join",
+    "html_inline",
+    "image",
+    "link",
+    "link_pairs",
+    "linkify",
+    "newline",
+    "strikethrough",
+    "text",
+)
+from . import emphasis, strikethrough
+from .autolink import autolink
+from .backticks import backtick
+from .balance_pairs import link_pairs
+from .entity import entity
+from .escape import escape
+from .fragments_join import fragments_join
+from .html_inline import html_inline
+from .image import image
+from .link import link
+from .linkify import linkify
+from .newline import newline
+from .state_inline import StateInline
+from .text import text

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_inline/emphasis.py

@@ -0,0 +1,102 @@
+# Process *this* and _that_
+#
+from __future__ import annotations
+
+from .state_inline import Delimiter, StateInline
+
+
+def tokenize(state: StateInline, silent: bool) -> bool:
+    """Insert each marker as a separate text token, and add it to delimiter list"""
+    start = state.pos
+    marker = state.src[start]
+
+    if silent:
+        return False
+
+    if marker not in ("_", "*"):
+        return False
+
+    scanned = state.scanDelims(state.pos, marker == "*")
+
+    for _ in range(scanned.length):
+        token = state.push("text", "", 0)
+        token.content = marker
+        state.delimiters.append(
+            Delimiter(
+                marker=ord(marker),
+                length=scanned.length,
+                token=len(state.tokens) - 1,
+                end=-1,
+                open=scanned.can_open,
+                close=scanned.can_close,
+            )
+        )
+
+    state.pos += scanned.length
+
+    return True
+
+
+def _postProcess(state: StateInline, delimiters: list[Delimiter]) -> None:
+    i = len(delimiters) - 1
+    while i >= 0:
+        startDelim = delimiters[i]
+
+        # /* _ */  /* * */
+        if startDelim.marker != 0x5F and startDelim.marker != 0x2A:
+            i -= 1
+            continue
+
+        # Process only opening markers
+        if startDelim.end == -1:
+            i -= 1
+            continue
+
+        endDelim = delimiters[startDelim.end]
+
+        # If the previous delimiter has the same marker and is adjacent to this one,
+        # merge those into one strong delimiter.
+        #
+        # `<em><em>whatever</em></em>` -> `<strong>whatever</strong>`
+        #
+        isStrong = (
+            i > 0
+            and delimiters[i - 1].end == startDelim.end + 1
+            # check that first two markers match and adjacent
+            and delimiters[i - 1].marker == startDelim.marker
+            and delimiters[i - 1].token == startDelim.token - 1
+            # check that last two markers are adjacent (we can safely assume they match)
+            and delimiters[startDelim.end + 1].token == endDelim.token + 1
+        )
+
+        ch = chr(startDelim.marker)
+
+        token = state.tokens[startDelim.token]
+        token.type = "strong_open" if isStrong else "em_open"
+        token.tag = "strong" if isStrong else "em"
+        token.nesting = 1
+        token.markup = ch + ch if isStrong else ch
+        token.content = ""
+
+        token = state.tokens[endDelim.token]
+        token.type = "strong_close" if isStrong else "em_close"
+        token.tag = "strong" if isStrong else "em"
+        token.nesting = -1
+        token.markup = ch + ch if isStrong else ch
+        token.content = ""
+
+        if isStrong:
+            state.tokens[delimiters[i - 1].token].content = ""
+            state.tokens[delimiters[startDelim.end + 1].token].content = ""
+            i -= 1
+
+        i -= 1
+
+
+def postProcess(state: StateInline) -> None:
+    """Walk through delimiter list and replace text tokens with tags."""
+    _postProcess(state, state.delimiters)
+
+    for token in state.tokens_meta:
+        if token and "delimiters" in token:
+            _postProcess(state, token["delimiters"])

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_inline/escape.py

@@ -0,0 +1,93 @@
+"""
+Process escaped chars and hardbreaks
+"""
+
+from ..common.utils import isStrSpace
+from .state_inline import StateInline
+
+
+def escape(state: StateInline, silent: bool) -> bool:
+    """Process escaped chars and hardbreaks."""
+    pos = state.pos
+    maximum = state.posMax
+
+    if state.src[pos] != "\\":
+        return False
+
+    pos += 1
+
+    # '\' at the end of the inline block
+    if pos >= maximum:
+        return False
+
+    ch1 = state.src[pos]
+    ch1_ord = ord(ch1)
+    if ch1 == "\n":
+        if not silent:
+            state.push("hardbreak", "br", 0)
+        pos += 1
+        # skip leading whitespaces from next line
+        while pos < maximum:
+            ch = state.src[pos]
+            if not isStrSpace(ch):
+                break
+            pos += 1
+
+        state.pos = pos
+        return True
+
+    escapedStr = state.src[pos]
+
+    if ch1_ord >= 0xD800 and ch1_ord <= 0xDBFF and pos + 1 < maximum:
+        ch2 = state.src[pos + 1]
+        ch2_ord = ord(ch2)
+        if ch2_ord >= 0xDC00 and ch2_ord <= 0xDFFF:
+            escapedStr += ch2
+            pos += 1
+
+    origStr = "\\" + escapedStr
+
+    if not silent:
+        token = state.push("text_special", "", 0)
+        token.content = escapedStr if ch1 in _ESCAPED else origStr
+        token.markup = origStr
+        token.info = "escape"
+
+    state.pos = pos + 1
+    return True
+
+
+_ESCAPED = {
+    "!",
+    '"',
+    "#",
+    "$",
+    "%",
+    "&",
+    "'",
+    "(",
+    ")",
+    "*",
+    "+",
+    ",",
+    "-",
+    ".",
+    "/",
+    ":",
+    ";",
+    "<",
+    "=",
+    ">",
+    "?",
+    "@",
+    "[",
+    "\\",
+    "]",
+    "^",
+    "_",
+    "`",
+    "{",
+    "|",
+    "}",
+    "~",
+}

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_inline/state_inline.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple
+
+from ..common.utils import isMdAsciiPunct, isPunctChar, isWhiteSpace
+from ..ruler import StateBase
+from ..token import Token
+from ..utils import EnvType
+
+if TYPE_CHECKING:
+    from markdown_it import MarkdownIt
+
+
+@dataclass(slots=True)
+class Delimiter:
+    # Char code of the starting marker (number).
+    marker: int
+
+    # Total length of these series of delimiters.
+    length: int
+
+    # A position of the token this delimiter corresponds to.
+    token: int
+
+    # If this delimiter is matched as a valid opener, `end` will be
+    # equal to its position, otherwise it's `-1`.
+    end: int
+
+    # Boolean flags that determine if this delimiter could open or close
+    # an emphasis.
+    open: bool
+    close: bool
+
+    level: bool | None = None
+
+
+class Scanned(NamedTuple):
+    can_open: bool
+    can_close: bool
+    length: int
+
+
+class StateInline(StateBase):
+    def __init__(
+        self, src: str, md: MarkdownIt, env: EnvType, outTokens: list[Token]
+    ) -> None:
+        self.src = src
+        self.env = env
+        self.md = md
+        self.tokens = outTokens
+        self.tokens_meta: list[dict[str, Any] | None] = [None] * len(outTokens)
+
+        self.pos = 0
+        self.posMax = len(self.src)
+        self.level = 0
+        self.pending = ""
+        self.pendingLevel = 0
+
+        # Stores { start: end } pairs. Useful for backtrack
+        # optimization of pairs parse (emphasis, strikes).
+        self.cache: dict[int, int] = {}
+
+        # List of emphasis-like delimiters for current tag
+        self.delimiters: list[Delimiter] = []
+
+        # Stack of delimiter lists for upper level tags
+        self._prev_delimiters: list[list[Delimiter]] = []
+
+        # backticklength => last seen position
+        self.backticks: dict[int, int] = {}
+        self.backticksScanned = False
+
+        # Counter used to disable inline linkify-it execution
+        # inside <a> and markdown links
+        self.linkLevel = 0
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}"
+            f"(pos=[{self.pos} of {self.posMax}], token={len(self.tokens)})"
+        )
+
+    def pushPending(self) -> Token:
+        token = Token("text", "", 0)
+        token.content = self.pending
+        token.level = self.pendingLevel
+        self.tokens.append(token)
+        self.pending = ""
+        return token
+
+    def push(self, ttype: str, tag: str, nesting: Literal[-1, 0, 1]) -> Token:
+        """Push new token to "stream".
+        If pending text exists - flush it as text token
+        """
+        if self.pending:
+            self.pushPending()
+
+        token = Token(ttype, tag, nesting)
+        token_meta = None
+
+        if nesting < 0:
+            # closing tag
+            self.level -= 1
+            self.delimiters = self._prev_delimiters.pop()
+
+        token.level = self.level
+
+        if nesting > 0:
+            # opening tag
+            self.level += 1
+            self._prev_delimiters.append(self.delimiters)
+            self.delimiters = []
+            token_meta = {"delimiters": self.delimiters}
+
+        self.pendingLevel = self.level
+        self.tokens.append(token)
+        self.tokens_meta.append(token_meta)
+        return token
+
+    def scanDelims(self, start: int, canSplitWord: bool) -> Scanned:
+        """
+        Scan a sequence of emphasis-like markers, and determine whether
+        it can start an emphasis sequence or end an emphasis sequence.
+
+         - start - position to scan from (it should point at a valid marker);
+         - canSplitWord - determine if these markers can be found inside a word
+
+        """
+        pos = start
+        maximum = self.posMax
+        marker = self.src[start]
+
+        # treat beginning of the line as a whitespace
+        lastChar = self.src[start - 1] if start > 0 else " "
+
+        while pos < maximum and self.src[pos] == marker:
+            pos += 1
+
+        count = pos - start
+
+        # treat end of the line as a whitespace
+        nextChar = self.src[pos] if pos < maximum else " "
+
+        isLastPunctChar = isMdAsciiPunct(ord(lastChar)) or isPunctChar(lastChar)
+        isNextPunctChar = isMdAsciiPunct(ord(nextChar)) or isPunctChar(nextChar)
+
+        isLastWhiteSpace = isWhiteSpace(ord(lastChar))
+        isNextWhiteSpace = isWhiteSpace(ord(nextChar))
+
+        left_flanking = not (
+            isNextWhiteSpace
+            or (isNextPunctChar and not (isLastWhiteSpace or isLastPunctChar))
+        )
+        right_flanking = not (
+            isLastWhiteSpace
+            or (isLastPunctChar and not (isNextWhiteSpace or isNextPunctChar))
+        )
+
+        can_open = left_flanking and (
+            canSplitWord or (not right_flanking) or isLastPunctChar
+        )
+        can_close = right_flanking and (
+            canSplitWord or (not left_flanking) or isNextPunctChar
+        )
+
+        return Scanned(can_open, can_close, count)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_inline/strikethrough.py

@@ -0,0 +1,173 @@
+# ~~strike through~~ (and optionally ~single tilde~)
+from __future__ import annotations
+
+from .state_inline import Delimiter, StateInline
+
+
+def tokenize(state: StateInline, silent: bool) -> bool:
+    """Insert each marker as a separate text token, and add it to delimiter list.
+
+    When the ``strikethrough_single_tilde`` option is enabled on the
+    ``MarkdownIt`` instance, single ``~`` delimiters are also accepted and
+    runs of three or more tildes are rejected (matching GitHub's rendering behaviour).
+    """
+    start = state.pos
+    ch = state.src[start]
+
+    if silent:
+        return False
+
+    if ch != "~":
+        return False
+
+    scanned = state.scanDelims(state.pos, True)
+    length = scanned.length
+
+    single_tilde = state.md.options.get("strikethrough_single_tilde", False)
+
+    if single_tilde:
+        # GitHub mode: only accept exactly 1 or 2 tildes.
+        if length < 1:
+            return False
+        if length > 2:
+            # Consume 3+ tildes as plain text so the parser doesn't
+            # re-enter and match a subset of them.  This intentionally
+            # matches GitHub's rendering, where ≥3 tildes are literal text.
+            token = state.push("text", "", 0)
+            token.content = ch * length
+            state.pos += scanned.length
+            return True
+
+        token = state.push("text", "", 0)
+        token.content = ch * length
+        state.delimiters.append(
+            Delimiter(
+                marker=ord(ch),
+                length=0,  # disable "rule of 3" length checks
+                token=len(state.tokens) - 1,
+                end=-1,
+                open=scanned.can_open,
+                close=scanned.can_close,
+            )
+        )
+    else:
+        # Original markdown-it behaviour: minimum 2, split odd runs.
+        if length < 2:
+            return False
+
+        if length % 2:
+            token = state.push("text", "", 0)
+            token.content = ch
+            length -= 1
+
+        i = 0
+        while i < length:
+            token = state.push("text", "", 0)
+            token.content = ch + ch
+            state.delimiters.append(
+                Delimiter(
+                    marker=ord(ch),
+                    length=0,  # disable "rule of 3" length checks
+                    token=len(state.tokens) - 1,
+                    end=-1,
+                    open=scanned.can_open,
+                    close=scanned.can_close,
+                )
+            )
+
+            i += 2
+
+    state.pos += scanned.length
+
+    return True
+
+
+def _postProcess(state: StateInline, delimiters: list[Delimiter]) -> None:
+    loneMarkers = []
+    maximum = len(delimiters)
+    single_tilde = state.md.options.get("strikethrough_single_tilde", False)
+
+    i = 0
+    while i < maximum:
+        startDelim = delimiters[i]
+
+        if startDelim.marker != 0x7E:  # /* ~ */
+            i += 1
+            continue
+
+        if startDelim.end == -1:
+            i += 1
+            continue
+
+        endDelim = delimiters[startDelim.end]
+
+        # In single-tilde mode, opener and closer must have the same width
+        # (both `~` or both `~~`).  The width is stored in the text token.
+        if single_tilde:
+            opener_content = state.tokens[startDelim.token].content
+            closer_content = state.tokens[endDelim.token].content
+            if opener_content != closer_content:
+                i += 1
+                continue
+
+        markup = state.tokens[startDelim.token].content
+
+        token = state.tokens[startDelim.token]
+        token.type = "s_open"
+        token.tag = "s"
+        token.nesting = 1
+        token.markup = markup
+        token.content = ""
+
+        token = state.tokens[endDelim.token]
+        token.type = "s_close"
+        token.tag = "s"
+        token.nesting = -1
+        token.markup = markup
+        token.content = ""
+
+        if (
+            state.tokens[endDelim.token - 1].type == "text"
+            and state.tokens[endDelim.token - 1].content == "~"
+        ):
+            loneMarkers.append(endDelim.token - 1)
+
+        i += 1
+
+    # If a marker sequence has an odd number of characters, it's split
+    # like this: `~~~~~` -> `~` + `~~` + `~~`, leaving one marker at the
+    # start of the sequence.
+    #
+    # So, we have to move all those markers after subsequent s_close tags.
+    #
+    while loneMarkers:
+        i = loneMarkers.pop()
+        j = i + 1
+
+        while (j < len(state.tokens)) and (state.tokens[j].type == "s_close"):
+            j += 1
+
+        j -= 1
+
+        if i != j:
+            token = state.tokens[j]
+            state.tokens[j] = state.tokens[i]
+            state.tokens[i] = token
+
+
+def postProcess(state: StateInline) -> None:
+    """Walk through delimiter list and replace text tokens with tags."""
+    tokens_meta = state.tokens_meta
+    maximum = len(state.tokens_meta)
+    _postProcess(state, state.delimiters)
+
+    curr = 0
+    while curr < maximum:
+        try:
+            curr_meta = tokens_meta[curr]
+        except IndexError:
+            pass
+        else:
+            if curr_meta and "delimiters" in curr_meta:
+                _postProcess(state, curr_meta["delimiters"])
+        curr += 1

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/markdown_it/rules_inline/text.py

@@ -0,0 +1,23 @@
+# Skip text characters for text token, place those to pending buffer
+# and increment current pos
+from .state_inline import StateInline
+
+# Rule to skip pure text
+
+
+def text(state: StateInline, silent: bool) -> bool:
+    pos = state.pos
+    posMax = state.posMax
+
+    terminator_char = state.md.inline.terminator_re.search(state.src, pos)
+    pos = terminator_char.start() if terminator_char else posMax
+
+    if pos == state.pos:
+        return False
+
+    if not silent:
+        state.pending += state.src[state.pos : pos]
+
+    state.pos = pos
+
+    return True

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/pop2piano/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_pop2piano import *
+    from .modeling_pop2piano import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/pop2piano/feature_extraction_pop2piano.py

@@ -0,0 +1,452 @@
+# Copyright 2023 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Feature extractor class for Pop2Piano"""
+
+import warnings
+
+import numpy
+import numpy as np
+
+from ...audio_utils import mel_filter_bank, spectrogram
+from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
+from ...feature_extraction_utils import BatchFeature
+from ...utils import (
+    TensorType,
+    is_essentia_available,
+    is_librosa_available,
+    is_scipy_available,
+    logging,
+    requires_backends,
+)
+from ...utils.import_utils import requires
+
+
+if is_essentia_available():
+    import essentia.standard
+
+if is_librosa_available():
+    import librosa
+
+if is_scipy_available():
+    import scipy
+
+
+logger = logging.get_logger(__name__)
+
+
+@requires(backends=("essentia", "librosa", "scipy", "torch"))
+class Pop2PianoFeatureExtractor(SequenceFeatureExtractor):
+    r"""
+    Constructs a Pop2Piano feature extractor.
+
+    This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
+    most of the main methods. Users should refer to this superclass for more information regarding those methods.
+
+    This class extracts rhythm and preprocesses the audio before it is passed to the model. First the audio is passed
+    to `RhythmExtractor2013` algorithm which extracts the beat_times, beat positions and estimates their confidence as
+    well as tempo in bpm, then beat_times is interpolated and to get beatsteps. Later we calculate
+    extrapolated_beatsteps from it to be used in tokenizer. On the other hand audio is resampled to self.sampling_rate
+    and preprocessed and then log mel spectogram is computed from that to be used in our transformer model.
+
+    Args:
+        sampling_rate (`int`, *optional*, defaults to 22050):
+            Target Sampling rate of audio signal. It's the sampling rate that we forward to the model.
+        padding_value (`int`, *optional*, defaults to 0):
+            Padding value used to pad the audio. Should correspond to silences.
+        window_size (`int`, *optional*, defaults to 4096):
+            Length of the window in samples to which the Fourier transform is applied.
+        hop_length (`int`, *optional*, defaults to 1024):
+            Step size between each window of the waveform, in samples.
+        min_frequency (`float`, *optional*, defaults to 10.0):
+            Lowest frequency that will be used in the log-mel spectrogram.
+        feature_size (`int`, *optional*, defaults to 512):
+            The feature dimension of the extracted features.
+        num_bars (`int`, *optional*, defaults to 2):
+            Determines interval between each sequence.
+    """
+
+    model_input_names = ["input_features", "beatsteps", "extrapolated_beatstep"]
+
+    def __init__(
+        self,
+        sampling_rate: int = 22050,
+        padding_value: int = 0,
+        window_size: int = 4096,
+        hop_length: int = 1024,
+        min_frequency: float = 10.0,
+        feature_size: int = 512,
+        num_bars: int = 2,
+        **kwargs,
+    ):
+        super().__init__(
+            feature_size=feature_size,
+            sampling_rate=sampling_rate,
+            padding_value=padding_value,
+            **kwargs,
+        )
+        self.sampling_rate = sampling_rate
+        self.padding_value = padding_value
+        self.window_size = window_size
+        self.hop_length = hop_length
+        self.min_frequency = min_frequency
+        self.feature_size = feature_size
+        self.num_bars = num_bars
+        self.mel_filters = mel_filter_bank(
+            num_frequency_bins=(self.window_size // 2) + 1,
+            num_mel_filters=self.feature_size,
+            min_frequency=self.min_frequency,
+            max_frequency=float(self.sampling_rate // 2),
+            sampling_rate=self.sampling_rate,
+            norm=None,
+            mel_scale="htk",
+        )
+
+    def mel_spectrogram(self, sequence: np.ndarray):
+        """
+        Generates MelSpectrogram.
+
+        Args:
+            sequence (`numpy.ndarray`):
+                The sequence of which the mel-spectrogram will be computed.
+        """
+        mel_specs = []
+        for seq in sequence:
+            window = np.hanning(self.window_size + 1)[:-1]
+            mel_specs.append(
+                spectrogram(
+                    waveform=seq,
+                    window=window,
+                    frame_length=self.window_size,
+                    hop_length=self.hop_length,
+                    power=2.0,
+                    mel_filters=self.mel_filters,
+                )
+            )
+        mel_specs = np.array(mel_specs)
+
+        return mel_specs
+
+    def extract_rhythm(self, audio: np.ndarray):
+        """
+        This algorithm(`RhythmExtractor2013`) extracts the beat positions and estimates their confidence as well as
+        tempo in bpm for an audio signal. For more information please visit
+        https://essentia.upf.edu/reference/std_RhythmExtractor2013.html .
+
+        Args:
+            audio(`numpy.ndarray`):
+                raw audio waveform which is passed to the Rhythm Extractor.
+        """
+        requires_backends(self, ["essentia"])
+        essentia_tracker = essentia.standard.RhythmExtractor2013(method="multifeature")
+        bpm, beat_times, confidence, estimates, essentia_beat_intervals = essentia_tracker(audio)
+
+        return bpm, beat_times, confidence, estimates, essentia_beat_intervals
+
+    def interpolate_beat_times(
+        self, beat_times: numpy.ndarray, steps_per_beat: numpy.ndarray, n_extend: numpy.ndarray
+    ):
+        """
+        This method takes beat_times and then interpolates that using `scipy.interpolate.interp1d` and the output is
+        then used to convert raw audio to log-mel-spectrogram.
+
+        Args:
+            beat_times (`numpy.ndarray`):
+                beat_times is passed into `scipy.interpolate.interp1d` for processing.
+            steps_per_beat (`int`):
+                used as an parameter to control the interpolation.
+            n_extend (`int`):
+                used as an parameter to control the interpolation.
+        """
+
+        requires_backends(self, ["scipy"])
+        beat_times_function = scipy.interpolate.interp1d(
+            np.arange(beat_times.size),
+            beat_times,
+            bounds_error=False,
+            fill_value="extrapolate",
+        )
+
+        ext_beats = beat_times_function(
+            np.linspace(0, beat_times.size + n_extend - 1, beat_times.size * steps_per_beat + n_extend)
+        )
+
+        return ext_beats
+
+    def preprocess_mel(self, audio: np.ndarray, beatstep: np.ndarray):
+        """
+        Preprocessing for log-mel-spectrogram
+
+        Args:
+            audio (`numpy.ndarray` of shape `(audio_length, )` ):
+                Raw audio waveform to be processed.
+            beatstep (`numpy.ndarray`):
+                Interpolated values of the raw audio. If beatstep[0] is greater than 0.0, then it will be shifted by
+                the value at beatstep[0].
+        """
+
+        if audio is not None and len(audio.shape) != 1:
+            raise ValueError(
+                f"Expected `audio` to be a single channel audio input of shape `(n, )` but found shape {audio.shape}."
+            )
+        if beatstep[0] > 0.0:
+            beatstep = beatstep - beatstep[0]
+
+        num_steps = self.num_bars * 4
+        num_target_steps = len(beatstep)
+        extrapolated_beatstep = self.interpolate_beat_times(
+            beat_times=beatstep, steps_per_beat=1, n_extend=(self.num_bars + 1) * 4 + 1
+        )
+
+        sample_indices = []
+        max_feature_length = 0
+        for i in range(0, num_target_steps, num_steps):
+            start_idx = i
+            end_idx = min(i + num_steps, num_target_steps)
+            start_sample = int(extrapolated_beatstep[start_idx] * self.sampling_rate)
+            end_sample = int(extrapolated_beatstep[end_idx] * self.sampling_rate)
+            sample_indices.append((start_sample, end_sample))
+            max_feature_length = max(max_feature_length, end_sample - start_sample)
+        padded_batch = []
+        for start_sample, end_sample in sample_indices:
+            feature = audio[start_sample:end_sample]
+            padded_feature = np.pad(
+                feature,
+                ((0, max_feature_length - feature.shape[0]),),
+                "constant",
+                constant_values=0,
+            )
+            padded_batch.append(padded_feature)
+
+        padded_batch = np.asarray(padded_batch)
+        return padded_batch, extrapolated_beatstep
+
+    def _pad(self, features: np.ndarray, add_zero_line=True):
+        features_shapes = [each_feature.shape for each_feature in features]
+        attention_masks, padded_features = [], []
+        for i, each_feature in enumerate(features):
+            # To pad "input_features".
+            if len(each_feature.shape) == 3:
+                features_pad_value = max([*zip(*features_shapes)][1]) - features_shapes[i][1]
+                attention_mask = np.ones(features_shapes[i][:2], dtype=np.int64)
+                feature_padding = ((0, 0), (0, features_pad_value), (0, 0))
+                attention_mask_padding = (feature_padding[0], feature_padding[1])
+
+            # To pad "beatsteps" and "extrapolated_beatstep".
+            else:
+                each_feature = each_feature.reshape(1, -1)
+                features_pad_value = max([*zip(*features_shapes)][0]) - features_shapes[i][0]
+                attention_mask = np.ones(features_shapes[i], dtype=np.int64).reshape(1, -1)
+                feature_padding = attention_mask_padding = ((0, 0), (0, features_pad_value))
+
+            each_padded_feature = np.pad(each_feature, feature_padding, "constant", constant_values=self.padding_value)
+            attention_mask = np.pad(
+                attention_mask, attention_mask_padding, "constant", constant_values=self.padding_value
+            )
+
+            if add_zero_line:
+                # if it is batched then we separate each examples using zero array
+                zero_array_len = max([*zip(*features_shapes)][1])
+
+                # we concatenate the zero array line here
+                each_padded_feature = np.concatenate(
+                    [each_padded_feature, np.zeros([1, zero_array_len, self.feature_size])], axis=0
+                )
+                attention_mask = np.concatenate(
+                    [attention_mask, np.zeros([1, zero_array_len], dtype=attention_mask.dtype)], axis=0
+                )
+
+            padded_features.append(each_padded_feature)
+            attention_masks.append(attention_mask)
+
+        padded_features = np.concatenate(padded_features, axis=0).astype(np.float32)
+        attention_masks = np.concatenate(attention_masks, axis=0).astype(np.int64)
+
+        return padded_features, attention_masks
+
+    def pad(
+        self,
+        inputs: BatchFeature,
+        is_batched: bool,
+        return_attention_mask: bool,
+        return_tensors: str | TensorType | None = None,
+    ):
+        """
+        Pads the inputs to same length and returns attention_mask.
+
+        Args:
+            inputs (`BatchFeature`):
+                Processed audio features.
+            is_batched (`bool`):
+                Whether inputs are batched or not.
+            return_attention_mask (`bool`):
+                Whether to return attention mask or not.
+            return_tensors (`str` or [`~utils.TensorType`], *optional*):
+                If set, will return tensors instead of list of python integers. Acceptable values are:
+                - `'pt'`: Return PyTorch `torch.Tensor` objects.
+                - `'np'`: Return Numpy `np.ndarray` objects.
+                If nothing is specified, it will return list of `np.ndarray` arrays.
+        Return:
+            `BatchFeature` with attention_mask, attention_mask_beatsteps and attention_mask_extrapolated_beatstep added
+            to it:
+            - **attention_mask** numpy.ndarray of shape `(batch_size, max_input_features_seq_length)` --
+                Example :
+                    1, 1, 1, 0, 0 (audio 1, also here it is padded to max length of 5 that's why there are 2 zeros at
+                    the end indicating they are padded)
+
+                    0, 0, 0, 0, 0 (zero pad to separate audio 1 and 2)
+
+                    1, 1, 1, 1, 1 (audio 2)
+
+                    0, 0, 0, 0, 0 (zero pad to separate audio 2 and 3)
+
+                    1, 1, 1, 1, 1 (audio 3)
+            - **attention_mask_beatsteps** numpy.ndarray of shape `(batch_size, max_beatsteps_seq_length)`
+            - **attention_mask_extrapolated_beatstep** numpy.ndarray of shape `(batch_size,
+              max_extrapolated_beatstep_seq_length)`
+        """
+
+        processed_features_dict = {}
+        for feature_name, feature_value in inputs.items():
+            if feature_name == "input_features":
+                padded_feature_values, attention_mask = self._pad(feature_value, add_zero_line=True)
+                processed_features_dict[feature_name] = padded_feature_values
+                if return_attention_mask:
+                    processed_features_dict["attention_mask"] = attention_mask
+            else:
+                padded_feature_values, attention_mask = self._pad(feature_value, add_zero_line=False)
+                processed_features_dict[feature_name] = padded_feature_values
+                if return_attention_mask:
+                    processed_features_dict[f"attention_mask_{feature_name}"] = attention_mask
+
+        # If we are processing only one example, we should remove the zero array line since we don't need it to
+        # separate examples from each other.
+        if not is_batched and not return_attention_mask:
+            processed_features_dict["input_features"] = processed_features_dict["input_features"][:-1, ...]
+
+        outputs = BatchFeature(processed_features_dict, tensor_type=return_tensors)
+
+        return outputs
+
+    def __call__(
+        self,
+        audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
+        sampling_rate: int | list[int],
+        steps_per_beat: int = 2,
+        resample: bool | None = True,
+        return_attention_mask: bool | None = False,
+        return_tensors: str | TensorType | None = None,
+        **kwargs,
+    ) -> BatchFeature:
+        """
+        Main method to featurize and prepare for the model.
+
+        Args:
+            audio (`np.ndarray`, `List`):
+                The audio or batch of audio to be processed. Each audio can be a numpy array, a list of float values, a
+                list of numpy arrays or a list of list of float values.
+            sampling_rate (`int`):
+                The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass
+                `sampling_rate` at the forward call to prevent silent errors.
+            steps_per_beat (`int`, *optional*, defaults to 2):
+                This is used in interpolating `beat_times`.
+            resample (`bool`, *optional*, defaults to `True`):
+                Determines whether to resample the audio to `sampling_rate` or not before processing. Must be True
+                during inference.
+            return_attention_mask (`bool` *optional*, defaults to `False`):
+                Denotes if attention_mask for input_features, beatsteps and extrapolated_beatstep will be given as
+                output or not. Automatically set to True for batched inputs.
+            return_tensors (`str` or [`~utils.TensorType`], *optional*):
+                If set, will return tensors instead of list of python integers. Acceptable values are:
+                - `'pt'`: Return PyTorch `torch.Tensor` objects.
+                - `'np'`: Return Numpy `np.ndarray` objects.
+                If nothing is specified, it will return list of `np.ndarray` arrays.
+        """
+
+        requires_backends(self, ["librosa"])
+        is_batched = isinstance(audio, (list, tuple)) and isinstance(audio[0], (np.ndarray, tuple, list))
+        if is_batched:
+            # This enables the user to process files of different sampling_rate at same time
+            if not isinstance(sampling_rate, list):
+                raise ValueError(
+                    "Please give sampling_rate of each audio separately when you are passing multiple raw_audios at the same time. "
+                    f"Received {sampling_rate}, expected [audio_1_sr, ..., audio_n_sr]."
+                )
+            return_attention_mask = True if return_attention_mask is None else return_attention_mask
+        else:
+            audio = [audio]
+            sampling_rate = [sampling_rate]
+            return_attention_mask = False if return_attention_mask is None else return_attention_mask
+
+        batch_input_features, batch_beatsteps, batch_ext_beatstep = [], [], []
+        for single_raw_audio, single_sampling_rate in zip(audio, sampling_rate):
+            bpm, beat_times, confidence, estimates, essentia_beat_intervals = self.extract_rhythm(
+                audio=single_raw_audio
+            )
+            beatsteps = self.interpolate_beat_times(beat_times=beat_times, steps_per_beat=steps_per_beat, n_extend=1)
+
+            if self.sampling_rate != single_sampling_rate and self.sampling_rate is not None:
+                if resample:
+                    # Change sampling_rate to self.sampling_rate
+                    single_raw_audio = librosa.core.resample(
+                        single_raw_audio,
+                        orig_sr=single_sampling_rate,
+                        target_sr=self.sampling_rate,
+                        res_type="kaiser_best",
+                    )
+                else:
+                    warnings.warn(
+                        f"The sampling_rate of the provided audio is different from the target sampling_rate "
+                        f"of the Feature Extractor, {self.sampling_rate} vs {single_sampling_rate}. "
+                        f"In these cases it is recommended to use `resample=True` in the `__call__` method to "
+                        f"get the optimal behaviour."
+                    )
+
+            single_sampling_rate = self.sampling_rate
+            start_sample = int(beatsteps[0] * single_sampling_rate)
+            end_sample = int(beatsteps[-1] * single_sampling_rate)
+
+            input_features, extrapolated_beatstep = self.preprocess_mel(
+                single_raw_audio[start_sample:end_sample], beatsteps - beatsteps[0]
+            )
+
+            mel_specs = self.mel_spectrogram(input_features.astype(np.float32))
+
+            # apply np.log to get log mel-spectrograms
+            log_mel_specs = np.log(np.clip(mel_specs, a_min=1e-6, a_max=None))
+
+            input_features = np.transpose(log_mel_specs, (0, -1, -2))
+
+            batch_input_features.append(input_features)
+            batch_beatsteps.append(beatsteps)
+            batch_ext_beatstep.append(extrapolated_beatstep)
+
+        output = BatchFeature(
+            {
+                "input_features": batch_input_features,
+                "beatsteps": batch_beatsteps,
+                "extrapolated_beatstep": batch_ext_beatstep,
+            }
+        )
+
+        output = self.pad(
+            output,
+            is_batched=is_batched,
+            return_attention_mask=return_attention_mask,
+            return_tensors=return_tensors,
+        )
+
+        return output
+
+
+__all__ = ["Pop2PianoFeatureExtractor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sew_d/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_sew_d import *
+    from .modeling_sew_d import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sew_d/configuration_sew_d.py

@@ -0,0 +1,196 @@
+# Copyright 2021 ASAPP Inc. and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""SEW-D model configuration"""
+
+import functools
+import operator
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="BAAI/seggpt-vit-large")
+@strict
+class SEWDConfig(PreTrainedConfig):
+    r"""
+    squeeze_factor (`int`, *optional*, defaults to 2):
+        Sequence length downsampling factor after the encoder and upsampling factor after the transformer.
+    position_buckets (`int`, *optional*, defaults to 256):
+        The maximum size of relative position embeddings.
+    share_att_key (`bool`, *optional*, defaults to `True`):
+        Whether to share attention key with c2p and p2c.
+    relative_attention (`bool`, *optional*, defaults to `True`):
+        Whether to use relative position encoding.
+    pos_att_type (`tuple[str]`, *optional*, defaults to `("p2c", "c2p")`):
+        The type of relative position attention, it can be a combination of `("p2c", "c2p")`, e.g. `("p2c")`,
+        `("p2c", "c2p")`, `("p2c", "c2p")`.
+    norm_rel_ebd (`str`, *optional*, defaults to `"layer_norm"`):
+        Whether to use layer norm in relative embedding (`"layer_norm"` if yes)
+    feat_proj_dropout (`float`, *optional*, defaults to 0.0):
+        The dropout probability for output of the feature encoder.
+    final_dropout (`float`, *optional*, defaults to 0.1):
+        The dropout probability for the final projection layer of [`SEWDForCTC`].
+    feature_layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+        The epsilon used by the layer normalization after the feature encoder.
+    feat_extract_norm (`str`, *optional*, defaults to `"group"`):
+        The norm to be applied to 1D convolutional layers in feature encoder. One of `"group"` for group
+        normalization of only the first 1D convolutional layer or `"layer"` for layer normalization of all 1D
+        convolutional layers.
+    feat_extract_activation (`str, `optional`, defaults to `"gelu"`):
+        The non-linear activation function (function or string) in the 1D convolutional layers of the feature
+        extractor. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+    conv_dim (`tuple[int]` or `list[int]`, *optional*, defaults to `(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)`):
+        A tuple of integers defining the number of input and output channels of each 1D convolutional layer in the
+        feature encoder. The length of *conv_dim* defines the number of 1D convolutional layers.
+    conv_stride (`tuple[int]` or `list[int]`, *optional*, defaults to `(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)`):
+        A tuple of integers defining the stride of each 1D convolutional layer in the feature encoder. The length
+        of *conv_stride* defines the number of convolutional layers and has to match the length of *conv_dim*.
+    conv_kernel (`tuple[int]` or `list[int]`, *optional*, defaults to `(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)`):
+        A tuple of integers defining the kernel size of each 1D convolutional layer in the feature encoder. The
+        length of *conv_kernel* defines the number of convolutional layers and has to match the length of
+        *conv_dim*.
+    conv_bias (`bool`, *optional*, defaults to `False`):
+        Whether the 1D convolutional layers have a bias.
+    num_conv_pos_embeddings (`int`, *optional*, defaults to 128):
+        Number of convolutional positional embeddings. Defines the kernel size of 1D convolutional positional
+        embeddings layer.
+    num_conv_pos_embedding_groups (`int`, *optional*, defaults to 16):
+        Number of groups of 1D convolutional positional embeddings layer.
+    apply_spec_augment (`bool`, *optional*, defaults to `True`):
+        Whether to apply *SpecAugment* data augmentation to the outputs of the feature encoder. For reference see
+        [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
+        Recognition](https://huggingface.co/papers/1904.08779).
+    mask_time_prob (`float`, *optional*, defaults to 0.05):
+        Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking
+        procedure generates ''mask_time_prob*len(time_axis)/mask_time_length'' independent masks over the axis. If
+        reasoning from the probability of each feature vector to be chosen as the start of the vector span to be
+        masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the
+        actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+    mask_time_length (`int`, *optional*, defaults to 10):
+        Length of vector span along the time axis.
+    mask_time_min_masks (`int`, *optional*, defaults to 2),:
+        The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
+        irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
+        mask_time_min_masks''
+    mask_feature_prob (`float`, *optional*, defaults to 0.0):
+        Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The
+        masking procedure generates ''mask_feature_prob*len(feature_axis)/mask_time_length'' independent masks over
+        the axis. If reasoning from the probability of each feature vector to be chosen as the start of the vector
+        span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap
+        may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is
+        True`.
+    mask_feature_length (`int`, *optional*, defaults to 10):
+        Length of vector span along the feature axis.
+    mask_feature_min_masks (`int`, *optional*, defaults to 0),:
+        The minimum number of masks of length `mask_feature_length` generated along the feature axis, each time
+        step, irrespectively of `mask_feature_prob`. Only relevant if
+        ''mask_feature_prob*len(feature_axis)/mask_feature_length < mask_feature_min_masks''
+    diversity_loss_weight (`int`, *optional*, defaults to 0.1):
+        The weight of the codebook diversity loss component.
+    ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
+        Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly
+        occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance
+        of [`SEWDForCTC`].
+    use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
+        Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
+        instance of [`Wav2Vec2ForSequenceClassification`].
+    classifier_proj_size (`int`, *optional*, defaults to 256):
+        Dimensionality of the projection before token mean-pooling for classification.
+
+    Example:
+
+    ```python
+    >>> from transformers import SEWDConfig, SEWDModel
+
+    >>> # Initializing a SEW-D asapp/sew-d-tiny-100k style configuration
+    >>> configuration = SEWDConfig()
+
+    >>> # Initializing a model (with random weights) from the asapp/sew-d-tiny-100k style configuration
+    >>> model = SEWDModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "sew-d"
+    vocab_size: int = 32
+    hidden_size: int = 768
+    num_hidden_layers: int = 12
+    num_attention_heads: int = 12
+    intermediate_size: int = 3072
+    squeeze_factor: int = 2
+    max_position_embeddings: int = 512
+    position_buckets: int = 256
+    share_att_key: bool = True
+    relative_attention: bool = True
+    pos_att_type: list[str] | tuple[str, ...] = ("p2c", "c2p")
+    norm_rel_ebd: str = "layer_norm"
+    hidden_act: str = "gelu_python"
+    hidden_dropout: float | int = 0.1
+    activation_dropout: float | int = 0.1
+    attention_dropout: float | int = 0.1
+    feat_proj_dropout: float | int = 0.0
+    final_dropout: float | int = 0.1
+    initializer_range: float = 0.02
+    layer_norm_eps: float = 1e-7
+    feature_layer_norm_eps: float = 1e-5
+    feat_extract_norm: str = "group"
+    feat_extract_activation: str = "gelu"
+    conv_dim: list[int] | tuple[int, ...] = (64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)
+    conv_stride: list[int] | tuple[int, ...] = (5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)
+    conv_kernel: list[int] | tuple[int, ...] = (10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)
+    conv_bias: bool = False
+    num_conv_pos_embeddings: int = 128
+    num_conv_pos_embedding_groups: int = 16
+    apply_spec_augment: bool = True
+    mask_time_prob: float | int = 0.05
+    mask_time_length: int = 10
+    mask_time_min_masks: int = 2
+    mask_feature_prob: float | int = 0.0
+    mask_feature_length: int = 10
+    mask_feature_min_masks: int = 0
+    ctc_loss_reduction: str = "mean"
+    ctc_zero_infinity: bool = False
+    use_weighted_layer_sum: bool = False
+    classifier_proj_size: int = 256
+    pad_token_id: int | None = 0
+    bos_token_id: int | None = 1
+    eos_token_id: int | list[int] | None = 2
+
+    def __post_init__(self, **kwargs):
+        self.num_feat_extract_layers = len(self.conv_dim)
+        super().__post_init__(**kwargs)
+
+    def validate_architecture(self):
+        """Part of `@strict`-powered validation. Validates the architecture of the config."""
+        if (
+            (len(self.conv_stride) != self.num_feat_extract_layers)
+            or (len(self.conv_kernel) != self.num_feat_extract_layers)
+            or (len(self.conv_dim) != self.num_feat_extract_layers)
+        ):
+            raise ValueError(
+                "Configuration for convolutional layers is incorrect. "
+                "It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`, "
+                f"but is `len(config.conv_dim) = {len(self.conv_dim)}`, `len(config.conv_stride) "
+                f"= {len(self.conv_stride)}`, `len(config.conv_kernel) = {len(self.conv_kernel)}`."
+            )
+
+    @property
+    def inputs_to_logits_ratio(self):
+        return functools.reduce(operator.mul, self.conv_stride, 1)
+
+
+__all__ = ["SEWDConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sew_d/modeling_sew_d.py

@@ -0,0 +1,1621 @@
+# Copyright 2021 ASAPP Inc. and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch SEW model."""
+
+import math
+from collections.abc import Sequence
+
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss, LayerNorm
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...integrations.deepspeed import is_deepspeed_zero3_enabled
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput
+from ...modeling_utils import PreTrainedModel, get_torch_context_manager_or_global_device
+from ...utils import auto_docstring, logging
+from .configuration_sew_d import SEWDConfig
+
+
+logger = logging.get_logger(__name__)
+
+_HIDDEN_STATES_START_POSITION = 1
+
+
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2._compute_mask_indices
+def _compute_mask_indices(
+    shape: tuple[int, int],
+    mask_prob: float,
+    mask_length: int,
+    attention_mask: torch.LongTensor | None = None,
+    min_masks: int = 0,
+) -> np.ndarray:
+    """
+    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
+    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+    CPU as part of the preprocessing during training.
+
+    Args:
+        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
+               the first element is the batch size and the second element is the length of the axis to span.
+        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
+                    independently generated mask spans of length `mask_length` is computed by
+                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
+                    actual percentage will be smaller.
+        mask_length: size of the mask
+        min_masks: minimum number of masked spans
+        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
+                        each batch dimension.
+    """
+    batch_size, sequence_length = shape
+
+    if mask_length < 1:
+        raise ValueError("`mask_length` has to be bigger than 0.")
+
+    if mask_length > sequence_length:
+        raise ValueError(
+            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length}"
+            f" and `sequence_length`: {sequence_length}`"
+        )
+
+    # epsilon is used for probabilistic rounding
+    epsilon = np.random.rand(1).item()
+
+    def compute_num_masked_span(input_length):
+        """Given input length, compute how many spans should be masked"""
+        num_masked_span = int(mask_prob * input_length / mask_length + epsilon)
+        num_masked_span = max(num_masked_span, min_masks)
+
+        # make sure num masked span <= sequence_length
+        if num_masked_span * mask_length > sequence_length:
+            num_masked_span = sequence_length // mask_length
+
+        # make sure num_masked span is also <= input_length - (mask_length - 1)
+        if input_length - (mask_length - 1) < num_masked_span:
+            num_masked_span = max(input_length - (mask_length - 1), 0)
+
+        return num_masked_span
+
+    # compute number of masked spans in batch
+    input_lengths = (
+        attention_mask.detach().sum(-1).tolist()
+        if attention_mask is not None
+        else [sequence_length for _ in range(batch_size)]
+    )
+
+    # SpecAugment mask to fill
+    spec_aug_mask = np.zeros((batch_size, sequence_length), dtype=bool)
+    spec_aug_mask_idxs = []
+
+    max_num_masked_span = compute_num_masked_span(sequence_length)
+
+    if max_num_masked_span == 0:
+        return spec_aug_mask
+
+    for input_length in input_lengths:
+        # compute num of masked spans for this input
+        num_masked_span = compute_num_masked_span(input_length)
+
+        # get random indices to mask
+        spec_aug_mask_idx = np.random.choice(
+            np.arange(input_length - (mask_length - 1)), num_masked_span, replace=False
+        )
+
+        # pick first sampled index that will serve as a dummy index to pad vector
+        # to ensure same dimension for all batches due to probabilistic rounding
+        # Picking first sample just pads those vectors twice.
+        if len(spec_aug_mask_idx) == 0:
+            # this case can only happen if `input_length` is strictly smaller then
+            # `sequence_length` in which case the last token has to be a padding
+            # token which we can use as a dummy mask id
+            dummy_mask_idx = sequence_length - 1
+        else:
+            dummy_mask_idx = spec_aug_mask_idx[0]
+
+        spec_aug_mask_idx = np.concatenate(
+            [spec_aug_mask_idx, np.ones(max_num_masked_span - num_masked_span, dtype=np.int32) * dummy_mask_idx]
+        )
+        spec_aug_mask_idxs.append(spec_aug_mask_idx)
+
+    spec_aug_mask_idxs = np.array(spec_aug_mask_idxs)
+
+    # expand masked indices to masked spans
+    spec_aug_mask_idxs = np.broadcast_to(
+        spec_aug_mask_idxs[:, :, None], (batch_size, max_num_masked_span, mask_length)
+    )
+    spec_aug_mask_idxs = spec_aug_mask_idxs.reshape(batch_size, max_num_masked_span * mask_length)
+
+    # add offset to the starting indexes so that indexes now create a span
+    offsets = np.arange(mask_length)[None, None, :]
+    offsets = np.broadcast_to(offsets, (batch_size, max_num_masked_span, mask_length)).reshape(
+        batch_size, max_num_masked_span * mask_length
+    )
+    spec_aug_mask_idxs = spec_aug_mask_idxs + offsets
+
+    # ensure that we cannot have indices larger than sequence_length
+    if spec_aug_mask_idxs.max() > sequence_length - 1:
+        spec_aug_mask_idxs[spec_aug_mask_idxs > sequence_length - 1] = sequence_length - 1
+
+    # scatter indices to mask
+    np.put_along_axis(spec_aug_mask, spec_aug_mask_idxs, 1, -1)
+
+    return spec_aug_mask
+
+
+def make_log_bucket_position(relative_pos, bucket_size, max_position):
+    sign = torch.sign(relative_pos)
+    mid = bucket_size // 2
+    abs_pos = torch.where(
+        (relative_pos < mid) & (relative_pos > -mid),
+        torch.tensor(mid - 1).type_as(relative_pos),
+        torch.abs(relative_pos),
+    )
+    log_pos = (
+        torch.ceil(torch.log(abs_pos / mid) / torch.log(torch.tensor((max_position - 1) / mid)) * (mid - 1)) + mid
+    )
+    bucket_pos = torch.where(abs_pos <= mid, relative_pos.type_as(log_pos), log_pos * sign)
+    return bucket_pos
+
+
+def build_relative_position(query_size, key_size, bucket_size=-1, max_position=-1, device=None):
+    """
+    Build relative position according to the query and key
+
+    We assume the absolute position of query \\(P_q\\) is range from (0, query_size) and the absolute position of key
+    \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} = P_q -
+    P_k\\)
+
+    Args:
+        query_size (int): the length of query
+        key_size (int): the length of key
+        bucket_size (int): the size of position bucket
+        max_position (int): the maximum allowed absolute position
+        device (`torch.device`): the device on which tensors will be created.
+
+    Return:
+        `torch.LongTensor`: A tensor with shape [1, query_size, key_size]
+    """
+
+    q_ids = torch.arange(0, query_size, device=device)
+    k_ids = torch.arange(0, key_size, device=device)
+    rel_pos_ids = q_ids[:, None] - k_ids[None, :]
+    if bucket_size > 0 and max_position > 0:
+        rel_pos_ids = make_log_bucket_position(rel_pos_ids, bucket_size, max_position)
+    rel_pos_ids = rel_pos_ids.to(torch.long)
+    rel_pos_ids = rel_pos_ids[:query_size, :]
+    rel_pos_ids = rel_pos_ids.unsqueeze(0)
+    return rel_pos_ids
+
+
+@torch.jit.script
+def c2p_dynamic_expand(c2p_pos, query_layer, relative_pos):
+    return c2p_pos.expand([query_layer.size(0), query_layer.size(1), query_layer.size(2), relative_pos.size(-1)])
+
+
+@torch.jit.script
+def p2c_dynamic_expand(c2p_pos, query_layer, key_layer):
+    return c2p_pos.expand([query_layer.size(0), query_layer.size(1), key_layer.size(-2), key_layer.size(-2)])
+
+
+@torch.jit.script
+def pos_dynamic_expand(pos_index, p2c_att, key_layer):
+    return pos_index.expand(p2c_att.size()[:2] + (pos_index.size(-2), key_layer.size(-2)))
+
+
+def get_mask(input, local_context):
+    if not isinstance(local_context, DropoutContext):
+        dropout = local_context
+        mask = None
+    else:
+        dropout = local_context.dropout
+        dropout *= local_context.scale
+        mask = local_context.mask if local_context.reuse_mask else None
+
+    if dropout > 0 and mask is None:
+        mask = (1 - torch.empty_like(input).bernoulli_(1 - dropout)).to(torch.bool)
+
+    if isinstance(local_context, DropoutContext):
+        if local_context.mask is None:
+            local_context.mask = mask
+
+    return mask, dropout
+
+
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2NoLayerNormConvLayer with Wav2Vec2->SEWD
+class SEWDNoLayerNormConvLayer(GradientCheckpointingLayer):
+    def __init__(self, config, layer_id=0):
+        super().__init__()
+        self.in_conv_dim = config.conv_dim[layer_id - 1] if layer_id > 0 else 1
+        self.out_conv_dim = config.conv_dim[layer_id]
+
+        self.conv = nn.Conv1d(
+            self.in_conv_dim,
+            self.out_conv_dim,
+            kernel_size=config.conv_kernel[layer_id],
+            stride=config.conv_stride[layer_id],
+            bias=config.conv_bias,
+        )
+        self.activation = ACT2FN[config.feat_extract_activation]
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2LayerNormConvLayer with Wav2Vec2->SEWD
+class SEWDLayerNormConvLayer(GradientCheckpointingLayer):
+    def __init__(self, config, layer_id=0):
+        super().__init__()
+        self.in_conv_dim = config.conv_dim[layer_id - 1] if layer_id > 0 else 1
+        self.out_conv_dim = config.conv_dim[layer_id]
+
+        self.conv = nn.Conv1d(
+            self.in_conv_dim,
+            self.out_conv_dim,
+            kernel_size=config.conv_kernel[layer_id],
+            stride=config.conv_stride[layer_id],
+            bias=config.conv_bias,
+        )
+        self.layer_norm = nn.LayerNorm(self.out_conv_dim, elementwise_affine=True)
+        self.activation = ACT2FN[config.feat_extract_activation]
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+
+        hidden_states = hidden_states.transpose(-2, -1)
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = hidden_states.transpose(-2, -1)
+
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2GroupNormConvLayer with Wav2Vec2->SEWD
+class SEWDGroupNormConvLayer(GradientCheckpointingLayer):
+    def __init__(self, config, layer_id=0):
+        super().__init__()
+        self.in_conv_dim = config.conv_dim[layer_id - 1] if layer_id > 0 else 1
+        self.out_conv_dim = config.conv_dim[layer_id]
+
+        self.conv = nn.Conv1d(
+            self.in_conv_dim,
+            self.out_conv_dim,
+            kernel_size=config.conv_kernel[layer_id],
+            stride=config.conv_stride[layer_id],
+            bias=config.conv_bias,
+        )
+        self.activation = ACT2FN[config.feat_extract_activation]
+
+        self.layer_norm = nn.GroupNorm(num_groups=self.out_conv_dim, num_channels=self.out_conv_dim, affine=True)
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+# Copied from transformers.models.sew.modeling_sew.SEWPositionalConvEmbedding with SEW->SEWD
+class SEWDPositionalConvEmbedding(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.conv = nn.Conv1d(
+            config.hidden_size,
+            config.hidden_size,
+            kernel_size=config.num_conv_pos_embeddings,
+            padding=config.num_conv_pos_embeddings // 2,
+            groups=config.num_conv_pos_embedding_groups,
+            stride=config.squeeze_factor,
+        )
+
+        weight_norm = nn.utils.weight_norm
+        if hasattr(nn.utils.parametrizations, "weight_norm"):
+            weight_norm = nn.utils.parametrizations.weight_norm
+
+        if is_deepspeed_zero3_enabled():
+            import deepspeed
+
+            with deepspeed.zero.GatheredParameters(self.conv.weight, modifier_rank=0):
+                self.conv = weight_norm(self.conv, name="weight", dim=2)
+            if hasattr(self.conv, "parametrizations"):
+                weight_g = self.conv.parametrizations.weight.original0
+                weight_v = self.conv.parametrizations.weight.original1
+            else:
+                weight_g = self.conv.weight_g
+                weight_v = self.conv.weight_v
+            deepspeed.zero.register_external_parameter(self, weight_v)
+            deepspeed.zero.register_external_parameter(self, weight_g)
+        else:
+            self.conv = weight_norm(self.conv, name="weight", dim=2)
+
+        self.padding = SEWDSamePadLayer(config.num_conv_pos_embeddings)
+        self.activation = ACT2FN[config.feat_extract_activation]
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+        hidden_states = self.padding(hidden_states)
+        hidden_states = self.activation(hidden_states)
+
+        return hidden_states
+
+
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2SamePadLayer with Wav2Vec2->SEW
+class SEWDSamePadLayer(nn.Module):
+    def __init__(self, num_conv_pos_embeddings):
+        super().__init__()
+        self.num_pad_remove = 1 if num_conv_pos_embeddings % 2 == 0 else 0
+
+    def forward(self, hidden_states):
+        if self.num_pad_remove > 0:
+            hidden_states = hidden_states[:, :, : -self.num_pad_remove]
+        return hidden_states
+
+
+# Copied from transformers.models.sew.modeling_sew.SEWUpsampling with SEW->SEWD
+class SEWDUpsampling(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.projection = nn.Linear(config.hidden_size, config.hidden_size * config.squeeze_factor)
+        self.activation = ACT2FN[config.feat_extract_activation]
+        self.squeeze_factor = config.squeeze_factor
+
+    def forward(self, hidden_states):
+        hidden_states = self.projection(hidden_states)
+        hidden_states = self.activation(hidden_states)
+
+        if self.squeeze_factor > 1:
+            # transform embedding channels to sequence length
+            bsz, src_len, src_embed_dim = hidden_states.size()
+            tgt_len = src_len * self.squeeze_factor
+            tgt_embed_dim = src_embed_dim // self.squeeze_factor
+            hidden_states = hidden_states.reshape(bsz, src_len, self.squeeze_factor, tgt_embed_dim)
+            hidden_states = hidden_states.reshape(bsz, tgt_len, tgt_embed_dim)
+
+        return hidden_states
+
+
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2FeatureEncoder with Wav2Vec2->SEWD
+class SEWDFeatureEncoder(nn.Module):
+    """Construct the features from raw audio waveform"""
+
+    def __init__(self, config):
+        super().__init__()
+
+        if config.feat_extract_norm == "group":
+            conv_layers = [SEWDGroupNormConvLayer(config, layer_id=0)] + [
+                SEWDNoLayerNormConvLayer(config, layer_id=i + 1) for i in range(config.num_feat_extract_layers - 1)
+            ]
+        elif config.feat_extract_norm == "layer":
+            conv_layers = [SEWDLayerNormConvLayer(config, layer_id=i) for i in range(config.num_feat_extract_layers)]
+        else:
+            raise ValueError(
+                f"`config.feat_extract_norm` is {config.feat_extract_norm}, but has to be one of ['group', 'layer']"
+            )
+        self.conv_layers = nn.ModuleList(conv_layers)
+        self.gradient_checkpointing = False
+        self._requires_grad = True
+
+    def _freeze_parameters(self):
+        for param in self.parameters():
+            param.requires_grad = False
+        self._requires_grad = False
+
+    def forward(self, input_values):
+        hidden_states = input_values[:, None]
+
+        # make sure hidden_states require grad for gradient_checkpointing
+        if self._requires_grad and self.training:
+            hidden_states.requires_grad = True
+
+        for conv_layer in self.conv_layers:
+            hidden_states = conv_layer(hidden_states)
+
+        return hidden_states
+
+
+class ContextPooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.pooler_hidden_size, config.pooler_hidden_size)
+        self.dropout = StableDropout(config.pooler_dropout)
+        self.config = config
+
+    def forward(self, hidden_states):
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+
+        context_token = hidden_states[:, 0]
+        context_token = self.dropout(context_token)
+        pooled_output = self.dense(context_token)
+        pooled_output = ACT2FN[self.config.pooler_hidden_act](pooled_output)
+        return pooled_output
+
+    @property
+    def output_dim(self):
+        return self.config.hidden_size
+
+
+class XSoftmax(torch.autograd.Function):
+    """
+    Masked Softmax which is optimized for saving memory
+
+    Args:
+        input (`torch.tensor`): The input tensor that will apply softmax.
+        mask (`torch.IntTensor`):
+            The mask matrix where 0 indicate that element will be ignored in the softmax calculation.
+        dim (int): The dimension that will apply softmax
+
+    Example:
+
+    ```python
+    >>> import torch
+    >>> from transformers.models.deberta_v2.modeling_deberta_v2 import XSoftmax
+
+    >>> # Make a tensor
+    >>> x = torch.randn([4, 20, 100])
+
+    >>> # Create a mask
+    >>> mask = (x > 0).int()
+
+    >>> # Specify the dimension to apply softmax
+    >>> dim = -1
+
+    >>> y = XSoftmax.apply(x, mask, dim)
+    ```"""
+
+    @staticmethod
+    def forward(ctx, input, mask, dim):
+        ctx.dim = dim
+        rmask = ~(mask.to(torch.bool))
+
+        output = input.masked_fill(rmask, torch.tensor(torch.finfo(input.dtype).min))
+        output = torch.softmax(output, ctx.dim)
+        output.masked_fill_(rmask, 0)
+        ctx.save_for_backward(output)
+        return output
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        (output,) = ctx.saved_tensors
+        inputGrad = torch._softmax_backward_data(grad_output, output, ctx.dim, output.dtype)
+        return inputGrad, None, None
+
+    @staticmethod
+    def symbolic(g, self, mask, dim):
+        import torch.onnx.symbolic_helper as sym_help
+        from torch.onnx.symbolic_opset9 import masked_fill, softmax
+
+        mask_cast_value = g.op("Cast", mask, to_i=sym_help.cast_pytorch_to_onnx["Long"])
+        r_mask = g.op(
+            "Cast",
+            g.op("Sub", g.op("Constant", value_t=torch.tensor(1, dtype=torch.int64)), mask_cast_value),
+            to_i=sym_help.cast_pytorch_to_onnx["Bool"],
+        )
+        output = masked_fill(
+            g, self, r_mask, g.op("Constant", value_t=torch.tensor(torch.finfo(self.type().dtype()).min))
+        )
+        output = softmax(g, output, dim)
+        return masked_fill(g, output, r_mask, g.op("Constant", value_t=torch.tensor(0, dtype=torch.bool)))
+
+
+class DropoutContext:
+    def __init__(self):
+        self.dropout = 0
+        self.mask = None
+        self.scale = 1
+        self.reuse_mask = True
+
+
+class XDropout(torch.autograd.Function):
+    """Optimized dropout function to save computation and memory by using mask operation instead of multiplication."""
+
+    @staticmethod
+    def forward(ctx, input, local_ctx):
+        mask, dropout = get_mask(input, local_ctx)
+        ctx.scale = 1.0 / (1 - dropout)
+        if dropout > 0:
+            ctx.save_for_backward(mask)
+            return input.masked_fill(mask, 0) * ctx.scale
+        else:
+            return input
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        if ctx.scale > 1:
+            (mask,) = ctx.saved_tensors
+            return grad_output.masked_fill(mask, 0) * ctx.scale, None
+        else:
+            return grad_output, None
+
+    @staticmethod
+    def symbolic(g: torch._C.Graph, input: torch._C.Value, local_ctx: float | DropoutContext) -> torch._C.Value:
+        from torch.onnx import symbolic_opset12
+
+        dropout_p = local_ctx
+        if isinstance(local_ctx, DropoutContext):
+            dropout_p = local_ctx.dropout
+        # StableDropout only calls this function when training.
+        train = True
+        # TODO: We should check if the opset_version being used to export
+        # is > 12 here, but there's no good way to do that. As-is, if the
+        # opset_version < 12, export will fail with a CheckerError.
+        # Once https://github.com/pytorch/pytorch/issues/78391 is fixed, do something like:
+        # if opset_version < 12:
+        #   return torch.onnx.symbolic_opset9.dropout(g, input, dropout_p, train)
+        return symbolic_opset12.dropout(g, input, dropout_p, train)
+
+
+class StableDropout(nn.Module):
+    """
+    Optimized dropout module for stabilizing the training
+
+    Args:
+        drop_prob (float): the dropout probabilities
+    """
+
+    def __init__(self, drop_prob):
+        super().__init__()
+        self.drop_prob = drop_prob
+        self.count = 0
+        self.context_stack = None
+
+    def forward(self, x):
+        """
+        Call the module
+
+        Args:
+            x (`torch.tensor`): The input tensor to apply dropout
+        """
+        if self.training and self.drop_prob > 0:
+            return XDropout.apply(x, self.get_context())
+        return x
+
+    def clear_context(self):
+        self.count = 0
+        self.context_stack = None
+
+    def init_context(self, reuse_mask=True, scale=1):
+        if self.context_stack is None:
+            self.context_stack = []
+        self.count = 0
+        for c in self.context_stack:
+            c.reuse_mask = reuse_mask
+            c.scale = scale
+
+    def get_context(self):
+        if self.context_stack is not None:
+            if self.count >= len(self.context_stack):
+                self.context_stack.append(DropoutContext())
+            ctx = self.context_stack[self.count]
+            ctx.dropout = self.drop_prob
+            self.count += 1
+            return ctx
+        else:
+            return self.drop_prob
+
+
+class SEWDSelfOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.LayerNorm = LayerNorm(config.hidden_size, config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.activation_dropout)
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class DisentangledSelfAttention(nn.Module):
+    """
+    Disentangled self-attention module
+
+    Parameters:
+        config (`DebertaV2Config`):
+            A model config class instance with the configuration to build a new model. The schema is similar to
+            *BertConfig*, for more details, please refer [`DebertaV2Config`]
+
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        if config.hidden_size % config.num_attention_heads != 0:
+            raise ValueError(
+                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({config.num_attention_heads})"
+            )
+        self.num_attention_heads = config.num_attention_heads
+        _attention_head_size = config.hidden_size // config.num_attention_heads
+        self.attention_head_size = getattr(config, "attention_head_size", _attention_head_size)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+        self.query_proj = nn.Linear(config.hidden_size, self.all_head_size, bias=True)
+        self.key_proj = nn.Linear(config.hidden_size, self.all_head_size, bias=True)
+        self.value_proj = nn.Linear(config.hidden_size, self.all_head_size, bias=True)
+
+        self.share_att_key = getattr(config, "share_att_key", False)
+        self.pos_att_type = config.pos_att_type if config.pos_att_type is not None else []
+        self.relative_attention = getattr(config, "relative_attention", False)
+
+        if self.relative_attention:
+            self.position_buckets = getattr(config, "position_buckets", -1)
+            self.max_relative_positions = getattr(config, "max_relative_positions", -1)
+            if self.max_relative_positions < 1:
+                self.max_relative_positions = config.max_position_embeddings
+            self.pos_ebd_size = self.max_relative_positions
+            if self.position_buckets > 0:
+                self.pos_ebd_size = self.position_buckets
+
+            self.pos_dropout = StableDropout(config.activation_dropout)
+
+            if not self.share_att_key:
+                if "c2p" in self.pos_att_type:
+                    self.pos_key_proj = nn.Linear(config.hidden_size, self.all_head_size, bias=True)
+                if "p2c" in self.pos_att_type:
+                    self.pos_query_proj = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = StableDropout(config.attention_dropout)
+
+    def transpose_for_scores(self, x, attention_heads):
+        new_x_shape = x.size()[:-1] + (attention_heads, -1)
+        x = x.view(new_x_shape)
+        return x.permute(0, 2, 1, 3).contiguous().view(-1, x.size(1), x.size(-1))
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask,
+        output_attentions=False,
+        query_states=None,
+        relative_pos=None,
+        rel_embeddings=None,
+    ):
+        """
+        Call the module
+
+        Args:
+            hidden_states (`torch.FloatTensor`):
+                Input states to the module usually the output from previous layer, it will be the Q,K and V in
+                *Attention(Q,K,V)*
+
+            attention_mask (`torch.BoolTensor`):
+                An attention mask matrix of shape [*B*, *N*, *N*] where *B* is the batch size, *N* is the maximum
+                sequence length in which element [i,j] = *1* means the *i* th token in the input can attend to the *j*
+                th token.
+
+            output_attentions (`bool`, *optional*):
+                Whether return the attention matrix.
+
+            query_states (`torch.FloatTensor`, *optional*):
+                The *Q* state in *Attention(Q,K,V)*.
+
+            relative_pos (`torch.LongTensor`):
+                The relative position encoding between the tokens in the sequence. It's of shape [*B*, *N*, *N*] with
+                values ranging in [*-max_relative_positions*, *max_relative_positions*].
+
+            rel_embeddings (`torch.FloatTensor`):
+                The embedding of relative distances. It's a tensor of shape [\\(2 \\times
+                \\text{max_relative_positions}\\), *hidden_size*].
+
+
+        """
+        if query_states is None:
+            query_states = hidden_states
+        query_layer = self.transpose_for_scores(self.query_proj(query_states), self.num_attention_heads)
+        key_layer = self.transpose_for_scores(self.key_proj(hidden_states), self.num_attention_heads)
+        value_layer = self.transpose_for_scores(self.value_proj(hidden_states), self.num_attention_heads)
+
+        rel_att = None
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        scale_factor = 1
+        if "c2p" in self.pos_att_type:
+            scale_factor += 1
+        if "p2c" in self.pos_att_type:
+            scale_factor += 1
+        scale = torch.sqrt(torch.tensor(query_layer.size(-1), dtype=torch.float) * scale_factor)
+        attention_scores = torch.bmm(query_layer, key_layer.transpose(-1, -2) / scale.to(dtype=query_layer.dtype))
+        if self.relative_attention:
+            rel_embeddings = self.pos_dropout(rel_embeddings)
+            rel_att = self.disentangled_attention_bias(
+                query_layer, key_layer, relative_pos, rel_embeddings, scale_factor
+            )
+
+        if rel_att is not None:
+            attention_scores = attention_scores + rel_att
+        attention_scores = attention_scores.view(
+            -1, self.num_attention_heads, attention_scores.size(-2), attention_scores.size(-1)
+        )
+
+        # bsz x height x length x dimension
+        attention_probs = XSoftmax.apply(attention_scores, attention_mask, -1)
+        attention_probs = self.dropout(attention_probs)
+        context_layer = torch.bmm(
+            attention_probs.view(-1, attention_probs.size(-2), attention_probs.size(-1)), value_layer
+        )
+        context_layer = (
+            context_layer.view(-1, self.num_attention_heads, context_layer.size(-2), context_layer.size(-1))
+            .permute(0, 2, 1, 3)
+            .contiguous()
+        )
+        new_context_layer_shape = context_layer.size()[:-2] + (-1,)
+        context_layer = context_layer.view(new_context_layer_shape)
+        if output_attentions:
+            return (context_layer, attention_probs)
+        else:
+            return context_layer
+
+    def disentangled_attention_bias(self, query_layer, key_layer, relative_pos, rel_embeddings, scale_factor):
+        if relative_pos is None:
+            q = query_layer.size(-2)
+            relative_pos = build_relative_position(
+                q,
+                key_layer.size(-2),
+                bucket_size=self.position_buckets,
+                max_position=self.max_relative_positions,
+                device=query_layer.device,
+            )
+        if relative_pos.dim() == 2:
+            relative_pos = relative_pos.unsqueeze(0).unsqueeze(0)
+        elif relative_pos.dim() == 3:
+            relative_pos = relative_pos.unsqueeze(1)
+        # bsz x height x query x key
+        elif relative_pos.dim() != 4:
+            raise ValueError(f"Relative position ids must be of dim 2 or 3 or 4. {relative_pos.dim()}")
+
+        att_span = self.pos_ebd_size
+        relative_pos = relative_pos.to(device=query_layer.device, dtype=torch.long)
+
+        rel_embeddings = rel_embeddings[0 : att_span * 2, :].unsqueeze(0)
+        if self.share_att_key:
+            pos_query_layer = self.transpose_for_scores(
+                self.query_proj(rel_embeddings), self.num_attention_heads
+            ).repeat(query_layer.size(0) // self.num_attention_heads, 1, 1)
+            pos_key_layer = self.transpose_for_scores(self.key_proj(rel_embeddings), self.num_attention_heads).repeat(
+                query_layer.size(0) // self.num_attention_heads, 1, 1
+            )
+        else:
+            if "c2p" in self.pos_att_type:
+                pos_key_layer = self.transpose_for_scores(
+                    self.pos_key_proj(rel_embeddings), self.num_attention_heads
+                ).repeat(query_layer.size(0) // self.num_attention_heads, 1, 1)  # .split(self.all_head_size, dim=-1)
+            if "p2c" in self.pos_att_type:
+                pos_query_layer = self.transpose_for_scores(
+                    self.pos_query_proj(rel_embeddings), self.num_attention_heads
+                ).repeat(query_layer.size(0) // self.num_attention_heads, 1, 1)  # .split(self.all_head_size, dim=-1)
+
+        score = 0
+        # content->position
+        if "c2p" in self.pos_att_type:
+            scale = torch.sqrt(torch.tensor(pos_key_layer.size(-1), dtype=torch.float) * scale_factor)
+            c2p_att = torch.bmm(query_layer, pos_key_layer.transpose(-1, -2))
+            c2p_pos = torch.clamp(relative_pos + att_span, 0, att_span * 2 - 1)
+            c2p_att = torch.gather(
+                c2p_att,
+                dim=-1,
+                index=c2p_pos.squeeze(0).expand([query_layer.size(0), query_layer.size(1), relative_pos.size(-1)]),
+            )
+            score += c2p_att / scale.to(dtype=c2p_att.dtype)
+
+        # position->content
+        if "p2c" in self.pos_att_type:
+            scale = torch.sqrt(torch.tensor(pos_query_layer.size(-1), dtype=torch.float) * scale_factor)
+            if key_layer.size(-2) != query_layer.size(-2):
+                r_pos = build_relative_position(
+                    key_layer.size(-2),
+                    key_layer.size(-2),
+                    bucket_size=self.position_buckets,
+                    max_position=self.max_relative_positions,
+                    device=query_layer.device,
+                )
+                r_pos = r_pos.unsqueeze(0)
+            else:
+                r_pos = relative_pos
+
+            p2c_pos = torch.clamp(-r_pos + att_span, 0, att_span * 2 - 1)
+            p2c_att = torch.bmm(key_layer, pos_query_layer.transpose(-1, -2))
+            p2c_att = torch.gather(
+                p2c_att,
+                dim=-1,
+                index=p2c_pos.squeeze(0).expand([query_layer.size(0), key_layer.size(-2), key_layer.size(-2)]),
+            ).transpose(-1, -2)
+            score += p2c_att / scale.to(dtype=p2c_att.dtype)
+
+        return score
+
+
+class SEWDAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.self = DisentangledSelfAttention(config)
+        self.output = SEWDSelfOutput(config)
+        self.config = config
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask,
+        output_attentions=False,
+        query_states=None,
+        relative_pos=None,
+        rel_embeddings=None,
+    ):
+        self_output = self.self(
+            hidden_states,
+            attention_mask,
+            output_attentions,
+            query_states=query_states,
+            relative_pos=relative_pos,
+            rel_embeddings=rel_embeddings,
+        )
+        if output_attentions:
+            self_output, att_matrix = self_output
+        if query_states is None:
+            query_states = hidden_states
+        attention_output = self.output(self_output, query_states)
+
+        if output_attentions:
+            return (attention_output, att_matrix)
+        else:
+            return attention_output
+
+
+# Copied from transformers.models.bert.modeling_bert.BertIntermediate with Bert->SEWD
+class SEWDIntermediate(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
+        if isinstance(config.hidden_act, str):
+            self.intermediate_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.intermediate_act_fn = config.hidden_act
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.intermediate_act_fn(hidden_states)
+        return hidden_states
+
+
+class SEWDOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.LayerNorm = LayerNorm(config.hidden_size, config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.activation_dropout)
+        self.config = config
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class SEWDLayer(GradientCheckpointingLayer):
+    def __init__(self, config):
+        super().__init__()
+        self.attention = SEWDAttention(config)
+        self.intermediate = SEWDIntermediate(config)
+        self.output = SEWDOutput(config)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask,
+        query_states=None,
+        relative_pos=None,
+        rel_embeddings=None,
+        output_attentions=False,
+    ):
+        attention_output = self.attention(
+            hidden_states,
+            attention_mask,
+            output_attentions=output_attentions,
+            query_states=query_states,
+            relative_pos=relative_pos,
+            rel_embeddings=rel_embeddings,
+        )
+        if output_attentions:
+            attention_output, att_matrix = attention_output
+        intermediate_output = self.intermediate(attention_output)
+        layer_output = self.output(intermediate_output, attention_output)
+        if output_attentions:
+            return (layer_output, att_matrix)
+        else:
+            return layer_output
+
+
+class ConvLayer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        kernel_size = getattr(config, "conv_kernel_size", 3)
+        groups = getattr(config, "conv_groups", 1)
+        self.conv_act = getattr(config, "conv_act", "tanh")
+        self.conv = nn.Conv1d(
+            config.hidden_size, config.hidden_size, kernel_size, padding=(kernel_size - 1) // 2, groups=groups
+        )
+        self.LayerNorm = LayerNorm(config.hidden_size, config.layer_norm_eps)
+        self.dropout = StableDropout(config.hidden_dropout_prob)
+        self.config = config
+
+    def forward(self, hidden_states, residual_states, input_mask):
+        out = self.conv(hidden_states.permute(0, 2, 1).contiguous()).permute(0, 2, 1).contiguous()
+        rmask = (1 - input_mask).bool()
+        out.masked_fill_(rmask.unsqueeze(-1).expand(out.size()), 0)
+        out = ACT2FN[self.conv_act](self.dropout(out))
+
+        layer_norm_input = residual_states + out
+        output = self.LayerNorm(layer_norm_input).to(layer_norm_input)
+
+        if input_mask is None:
+            output_states = output
+        else:
+            if input_mask.dim() != layer_norm_input.dim():
+                if input_mask.dim() == 4:
+                    input_mask = input_mask.squeeze(1).squeeze(1)
+                input_mask = input_mask.unsqueeze(2)
+
+            input_mask = input_mask.to(output.dtype)
+            output_states = output * input_mask
+
+        return output_states
+
+
+class SEWDTransformerEncoder(nn.Module):
+    """Modified BertEncoder with relative position bias support"""
+
+    def __init__(self, config):
+        super().__init__()
+
+        self.layer = nn.ModuleList([SEWDLayer(config) for _ in range(config.num_hidden_layers)])
+        self.relative_attention = getattr(config, "relative_attention", False)
+
+        if self.relative_attention:
+            self.max_relative_positions = getattr(config, "max_relative_positions", -1)
+            if self.max_relative_positions < 1:
+                self.max_relative_positions = config.max_position_embeddings
+
+            self.position_buckets = getattr(config, "position_buckets", -1)
+            pos_ebd_size = self.max_relative_positions * 2
+
+            if self.position_buckets > 0:
+                pos_ebd_size = self.position_buckets * 2
+
+            self.rel_embeddings = nn.Embedding(pos_ebd_size, config.hidden_size)
+
+        self.norm_rel_ebd = [x.strip() for x in getattr(config, "norm_rel_ebd", "none").lower().split("|")]
+
+        if "layer_norm" in self.norm_rel_ebd:
+            self.LayerNorm = LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=True)
+
+        self.conv = ConvLayer(config) if getattr(config, "conv_kernel_size", 0) > 0 else None
+        self.gradient_checkpointing = False
+
+    def get_rel_embedding(self):
+        rel_embeddings = self.rel_embeddings.weight if self.relative_attention else None
+        if rel_embeddings is not None and ("layer_norm" in self.norm_rel_ebd):
+            rel_embeddings = self.LayerNorm(rel_embeddings)
+        return rel_embeddings
+
+    def get_attention_mask(self, attention_mask):
+        if attention_mask.dim() <= 2:
+            extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+            attention_mask = extended_attention_mask * extended_attention_mask.squeeze(-2).unsqueeze(-1)
+        elif attention_mask.dim() == 3:
+            attention_mask = attention_mask.unsqueeze(1)
+
+        return attention_mask
+
+    def get_rel_pos(self, hidden_states, query_states=None, relative_pos=None):
+        if self.relative_attention and relative_pos is None:
+            q = query_states.size(-2) if query_states is not None else hidden_states.size(-2)
+            relative_pos = build_relative_position(
+                q,
+                hidden_states.size(-2),
+                bucket_size=self.position_buckets,
+                max_position=self.max_relative_positions,
+                device=hidden_states.device,
+            )
+        return relative_pos
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask,
+        output_hidden_states=True,
+        output_attentions=False,
+        query_states=None,
+        relative_pos=None,
+        return_dict=True,
+    ):
+        if attention_mask.dim() <= 2:
+            input_mask = attention_mask
+        else:
+            input_mask = attention_mask.sum(-2) > 0
+        attention_mask = self.get_attention_mask(attention_mask)
+        relative_pos = self.get_rel_pos(hidden_states, query_states, relative_pos)
+
+        all_hidden_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        if isinstance(hidden_states, Sequence):
+            next_kv = hidden_states[0]
+        else:
+            next_kv = hidden_states
+        rel_embeddings = self.get_rel_embedding()
+        output_states = next_kv
+        for i, layer_module in enumerate(self.layer):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (output_states,)
+
+            output_states = layer_module(
+                next_kv,
+                attention_mask,
+                query_states=query_states,
+                relative_pos=relative_pos,
+                rel_embeddings=rel_embeddings,
+                output_attentions=output_attentions,
+            )
+
+            if output_attentions:
+                output_states, att_m = output_states
+
+            if i == 0 and self.conv is not None:
+                output_states = self.conv(hidden_states, output_states, input_mask)
+
+            if query_states is not None:
+                query_states = output_states
+                if isinstance(hidden_states, Sequence):
+                    next_kv = hidden_states[i + 1] if i + 1 < len(self.layer) else None
+            else:
+                next_kv = output_states
+
+            if output_attentions:
+                all_attentions = all_attentions + (att_m,)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (output_states,)
+
+        if not return_dict:
+            return tuple(v for v in [output_states, all_hidden_states, all_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=output_states, hidden_states=all_hidden_states, attentions=all_attentions
+        )
+
+
+class SEWDEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.pos_conv_embed = SEWDPositionalConvEmbedding(config)
+        self.pool = nn.AvgPool1d(config.squeeze_factor, config.squeeze_factor)
+        self.encoder = SEWDTransformerEncoder(config)
+        self.upsample = SEWDUpsampling(config)
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states: torch.tensor,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool = False,
+        output_hidden_states: bool = False,
+        return_dict: bool = True,
+    ):
+        max_encoder_length = hidden_states.shape[1] // self.config.squeeze_factor
+        if attention_mask is None:
+            attention_mask = torch.ones(
+                (hidden_states.shape[0], max_encoder_length), dtype=torch.long, device=hidden_states.device
+            )
+        else:
+            # make sure padded tokens output 0
+            expand_attention_mask = attention_mask.unsqueeze(-1).repeat(1, 1, hidden_states.shape[2])
+            hidden_states[~expand_attention_mask.bool()] = 0.0
+
+            input_lengths = (attention_mask.long()).sum(-1)
+            # apply pooling formula to get real output_lengths
+            output_lengths = input_lengths // self.config.squeeze_factor
+            attention_ids = (
+                torch.arange(0, max_encoder_length, device=output_lengths.device)
+                .view(1, -1)
+                .expand(output_lengths.shape[0], -1)
+            )
+            attention_mask = (attention_ids < output_lengths.view(-1, 1)).long()
+
+        n_input_timesteps = hidden_states.shape[1]
+
+        hidden_states = hidden_states.transpose(1, 2)
+        position_embeddings = self.pos_conv_embed(hidden_states)
+        pooled_hidden_states = self.pool(hidden_states)
+        min_length = min(position_embeddings.size(-1), pooled_hidden_states.size(-1))
+        hidden_states = pooled_hidden_states[..., :min_length] + position_embeddings[..., :min_length]
+        hidden_states = hidden_states.transpose(1, 2)
+
+        encoder_outputs = self.encoder(hidden_states, attention_mask, output_hidden_states, output_attentions)
+
+        hidden_states = self.upsample(encoder_outputs.last_hidden_state)
+        if hidden_states.shape[1] < n_input_timesteps:
+            hidden_states = nn.functional.pad(hidden_states, (0, 0, 0, n_input_timesteps - hidden_states.shape[1]))
+
+        if not return_dict:
+            return tuple(
+                v for v in [hidden_states, encoder_outputs.hidden_states, encoder_outputs.attentions] if v is not None
+            )
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+@auto_docstring
+class SEWDPreTrainedModel(PreTrainedModel):
+    config: SEWDConfig
+    base_model_prefix = "sew-d"
+    main_input_name = "input_values"
+    input_modalities = "audio"
+    supports_gradient_checkpointing = True
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        if isinstance(module, SEWDPositionalConvEmbedding):
+            init.normal_(
+                module.conv.weight,
+                mean=0,
+                std=2 * math.sqrt(1 / (module.conv.kernel_size[0] * module.conv.in_channels)),
+            )
+            init.constant_(module.conv.bias, 0)
+        elif isinstance(module, nn.Linear):
+            init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
+        elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)):
+            init.zeros_(module.bias)
+            init.ones_(module.weight)
+        elif isinstance(module, nn.Conv1d):
+            if is_deepspeed_zero3_enabled():
+                import deepspeed
+
+                if hasattr(module, "weight_v") and hasattr(module, "weight_g"):
+                    with deepspeed.zero.GatheredParameters([module.weight_v, module.weight_g], modifier_rank=0):
+                        init.kaiming_normal_(module.weight)
+                else:
+                    with deepspeed.zero.GatheredParameters(module.weight, modifier_rank=0):
+                        init.kaiming_normal_(module.weight)
+            else:
+                init.kaiming_normal_(module.weight)
+        elif isinstance(module, nn.Embedding):
+            init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
+            # Here we need the check explicitly, as we slice the weight in the `zeros_` call, so it looses the flag
+            if module.padding_idx is not None and not getattr(module.weight, "_is_hf_initialized", False):
+                init.zeros_(module.weight[module.padding_idx])
+
+        if isinstance(module, (nn.Linear, nn.Conv1d)) and module.bias is not None:
+            init.zeros_(module.bias)
+
+    def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor | int):
+        """
+        Computes the output length of the convolutional layers
+        """
+
+        def _conv_out_length(input_length, kernel_size, stride):
+            # 1D convolutional layer output length formula taken
+            # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html
+            return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1
+
+        for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride):
+            input_lengths = _conv_out_length(input_lengths, kernel_size, stride)
+
+        return input_lengths
+
+    def _get_feature_vector_attention_mask(self, feature_vector_length: int, attention_mask: torch.LongTensor):
+        output_lengths = self._get_feat_extract_output_lengths(attention_mask.sum(-1)).to(torch.long)
+        batch_size = attention_mask.shape[0]
+
+        attention_mask = torch.zeros(
+            (batch_size, feature_vector_length), dtype=attention_mask.dtype, device=attention_mask.device
+        )
+        # these two operations makes sure that all values before the output lengths idxs are attended to
+        attention_mask[(torch.arange(attention_mask.shape[0], device=attention_mask.device), output_lengths - 1)] = 1
+        attention_mask = attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool()
+        return attention_mask
+
+
+@auto_docstring
+# Copied from transformers.models.sew.modeling_sew.SEWModel with SEW->SEWD, layer_norm_eps->feature_layer_norm_eps
+class SEWDModel(SEWDPreTrainedModel):
+    def __init__(self, config: SEWDConfig):
+        super().__init__(config)
+        self.config = config
+        self.feature_extractor = SEWDFeatureEncoder(config)
+        self.layer_norm = nn.LayerNorm(config.conv_dim[-1], eps=config.feature_layer_norm_eps)
+
+        self.project_features = config.conv_dim[-1] != config.hidden_size
+        if self.project_features:
+            self.feature_projection = nn.Linear(config.conv_dim[-1], config.hidden_size)
+        self.feature_dropout = nn.Dropout(config.feat_proj_dropout)
+
+        if config.mask_time_prob > 0.0 or config.mask_feature_prob > 0.0:
+            self.masked_spec_embed = nn.Parameter(torch.Tensor(config.hidden_size).uniform_())
+
+        self.encoder = SEWDEncoder(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    # Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2Model._mask_hidden_states
+    def _mask_hidden_states(
+        self,
+        hidden_states: torch.FloatTensor,
+        mask_time_indices: torch.FloatTensor | None = None,
+        attention_mask: torch.LongTensor | None = None,
+    ):
+        """
+        Masks extracted features along time axis and/or along feature axis according to
+        [SpecAugment](https://huggingface.co/papers/1904.08779).
+        """
+
+        # `config.apply_spec_augment` can set masking to False
+        if not getattr(self.config, "apply_spec_augment", True):
+            return hidden_states
+
+        # generate indices & apply SpecAugment along time axis
+        batch_size, sequence_length, hidden_size = hidden_states.size()
+
+        if mask_time_indices is not None:
+            # apply SpecAugment along time axis with given mask_time_indices
+            hidden_states[mask_time_indices] = self.masked_spec_embed.to(hidden_states.dtype)
+        elif self.config.mask_time_prob > 0 and self.training:
+            mask_time_indices = _compute_mask_indices(
+                (batch_size, sequence_length),
+                mask_prob=self.config.mask_time_prob,
+                mask_length=self.config.mask_time_length,
+                attention_mask=attention_mask,
+                min_masks=self.config.mask_time_min_masks,
+            )
+            mask_time_indices = torch.tensor(mask_time_indices, device=hidden_states.device, dtype=torch.bool)
+            hidden_states[mask_time_indices] = self.masked_spec_embed.to(hidden_states.dtype)
+
+        if self.config.mask_feature_prob > 0 and self.training:
+            # generate indices & apply SpecAugment along feature axis
+            mask_feature_indices = _compute_mask_indices(
+                (batch_size, hidden_size),
+                mask_prob=self.config.mask_feature_prob,
+                mask_length=self.config.mask_feature_length,
+                min_masks=self.config.mask_feature_min_masks,
+            )
+            mask_feature_indices = torch.tensor(mask_feature_indices, device=hidden_states.device, dtype=torch.bool)
+            mask_feature_indices = mask_feature_indices[:, None].expand(-1, sequence_length, -1)
+            hidden_states[mask_feature_indices] = 0
+
+        return hidden_states
+
+    @auto_docstring
+    def forward(
+        self,
+        input_values: torch.Tensor | None,
+        attention_mask: torch.Tensor | None = None,
+        mask_time_indices: torch.FloatTensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple | BaseModelOutput:
+        r"""
+        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
+            masked extracted features in *config.proj_codevector_dim* space.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        extract_features = self.feature_extractor(input_values)
+        extract_features = extract_features.transpose(1, 2)
+        extract_features = self.layer_norm(extract_features)
+
+        if self.project_features:
+            extract_features = self.feature_projection(extract_features)
+        hidden_states = self.feature_dropout(extract_features)
+
+        if attention_mask is not None:
+            # compute reduced attention_mask corresponding to feature vectors
+            attention_mask = self._get_feature_vector_attention_mask(hidden_states.shape[1], attention_mask)
+
+        hidden_states = self._mask_hidden_states(hidden_states, mask_time_indices=mask_time_indices)
+
+        encoder_outputs = self.encoder(
+            hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = encoder_outputs[0]
+
+        if not return_dict:
+            return (hidden_states,) + encoder_outputs[1:]
+
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    SEW-D Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
+    """
+)
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->SEWD, wav2vec2->sew_d, WAV2VEC2->SEWD
+class SEWDForCTC(SEWDPreTrainedModel):
+    def __init__(self, config, target_lang: str | None = None):
+        r"""
+        target_lang (`str`, *optional*):
+            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
+            adapter.<lang>.bin. Only relevant when using an instance of [`SEWDForCTC`] with adapters. Uses 'eng' by
+            default.
+        """
+        super().__init__(config)
+
+        self.sew_d = SEWDModel(config)
+        self.dropout = nn.Dropout(config.final_dropout)
+
+        self.target_lang = target_lang
+
+        if config.vocab_size is None:
+            raise ValueError(
+                f"You are trying to instantiate {self.__class__} with a configuration that "
+                "does not define the vocabulary size of the language model head. Please "
+                "instantiate the model as follows: `SEWDForCTC.from_pretrained(..., vocab_size=vocab_size)`. "
+                "or define `vocab_size` of your model's configuration."
+            )
+        output_hidden_size = (
+            config.output_hidden_size if hasattr(config, "add_adapter") and config.add_adapter else config.hidden_size
+        )
+        self.lm_head = nn.Linear(output_hidden_size, config.vocab_size)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def tie_weights(self, **kwargs):
+        """
+        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
+        passing `target_lang=...` to `from_pretrained(...)`.
+
+        This method is **not** supposed to be called by the user and is prone to be changed in the future.
+        """
+
+        if get_torch_context_manager_or_global_device() == torch.device("meta"):
+            return
+
+        # Note that `tie_weights` is usually used to tie input and output embedding weights. The method is re-purposed to
+        # correctly load adapter layers for SEWD so that we do not have to introduce a new API to
+        # [`PreTrainedModel`]. While slightly hacky, SEWD never has to tie input and output embeddings, so that it is
+        # ok to repurpose this function here.
+        target_lang = self.target_lang
+
+        if target_lang is not None and getattr(self.config, "adapter_attn_dim", None) is None:
+            raise ValueError(f"Cannot pass `target_lang`: {target_lang} if `config.adapter_attn_dim` is not defined.")
+        elif target_lang is None and getattr(self.config, "adapter_attn_dim", None) is not None:
+            logger.info("By default `target_lang` is set to 'eng'.")
+        elif target_lang is not None:
+            self.load_adapter(target_lang, force_load=True)
+
+    def freeze_feature_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
+        not be updated during training.
+        """
+        self.sew_d.feature_extractor._freeze_parameters()
+
+    def freeze_base_model(self):
+        """
+        Calling this function will disable the gradient computation for the base model so that its parameters will not
+        be updated during training. Only the classification head will be updated.
+        """
+        for param in self.sew_d.parameters():
+            param.requires_grad = False
+
+    @auto_docstring
+    def forward(
+        self,
+        input_values: torch.Tensor | None,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        labels: torch.Tensor | None = None,
+        **kwargs,
+    ) -> tuple | CausalLMOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
+            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
+            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+            config.vocab_size - 1]`.
+        """
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        if labels is not None and labels.max() >= self.config.vocab_size:
+            raise ValueError(f"Label values must be <= vocab_size: {self.config.vocab_size}")
+
+        outputs = self.sew_d(
+            input_values,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        hidden_states = self.dropout(hidden_states)
+
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            # retrieve loss input_lengths from attention_mask
+            attention_mask = (
+                attention_mask if attention_mask is not None else torch.ones_like(input_values, dtype=torch.long)
+            )
+            input_lengths = self._get_feat_extract_output_lengths(attention_mask.sum(-1)).to(torch.long)
+
+            # assuming that padded tokens are filled with -100
+            # when not being attended to
+            labels_mask = labels >= 0
+            target_lengths = labels_mask.sum(-1)
+            flattened_targets = labels.masked_select(labels_mask)
+
+            # ctc_loss doesn't support fp16
+            log_probs = nn.functional.log_softmax(logits, dim=-1, dtype=torch.float32).transpose(0, 1)
+
+            with torch.backends.cudnn.flags(enabled=False):
+                loss = nn.functional.ctc_loss(
+                    log_probs,
+                    flattened_targets,
+                    input_lengths,
+                    target_lengths,
+                    blank=self.config.pad_token_id,
+                    reduction=self.config.ctc_loss_reduction,
+                    zero_infinity=self.config.ctc_zero_infinity,
+                )
+
+        if not return_dict:
+            output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutput(
+            loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    SEWD Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB
+    Keyword Spotting.
+    """
+)
+# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForSequenceClassification with Wav2Vec2->SEWD, wav2vec2->sew_d, WAV2VEC2->SEWD
+class SEWDForSequenceClassification(SEWDPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        if hasattr(config, "add_adapter") and config.add_adapter:
+            raise ValueError(
+                "Sequence classification does not support the use of SEWD adapters (config.add_adapter=True)"
+            )
+        self.sew_d = SEWDModel(config)
+        num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
+        if config.use_weighted_layer_sum:
+            self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
+        self.projector = nn.Linear(config.hidden_size, config.classifier_proj_size)
+        self.classifier = nn.Linear(config.classifier_proj_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def freeze_feature_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
+        not be updated during training.
+        """
+        self.sew_d.feature_extractor._freeze_parameters()
+
+    def freeze_base_model(self):
+        """
+        Calling this function will disable the gradient computation for the base model so that its parameters will not
+        be updated during training. Only the classification head will be updated.
+        """
+        for param in self.sew_d.parameters():
+            param.requires_grad = False
+
+    @auto_docstring
+    def forward(
+        self,
+        input_values: torch.Tensor | None,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        labels: torch.Tensor | None = None,
+        **kwargs,
+    ) -> tuple | SequenceClassifierOutput:
+        r"""
+        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
+            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
+            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
+            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
+            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
+            into a tensor of type `torch.FloatTensor`. See [`SEWDProcessor.__call__`] for details.
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states
+
+        outputs = self.sew_d(
+            input_values,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if self.config.use_weighted_layer_sum:
+            hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
+            hidden_states = torch.stack(hidden_states, dim=1)
+            norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
+            hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
+        else:
+            hidden_states = outputs[0]
+
+        hidden_states = self.projector(hidden_states)
+        if attention_mask is None:
+            pooled_output = hidden_states.mean(dim=1)
+        else:
+            padding_mask = self._get_feature_vector_attention_mask(hidden_states.shape[1], attention_mask)
+            expand_padding_mask = padding_mask.unsqueeze(-1).repeat(1, 1, hidden_states.shape[2])
+            hidden_states[~expand_padding_mask] = 0.0
+            pooled_output = hidden_states.sum(dim=1) / padding_mask.sum(dim=1).view(-1, 1)
+
+        logits = self.classifier(pooled_output)
+
+        loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1))
+
+        if not return_dict:
+            output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = ["SEWDForCTC", "SEWDForSequenceClassification", "SEWDModel", "SEWDPreTrainedModel"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/vision_text_dual_encoder/__init__.py

@@ -0,0 +1,28 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_vision_text_dual_encoder import *
+    from .modeling_vision_text_dual_encoder import *
+    from .processing_vision_text_dual_encoder import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)