Add files using upload-large-folder tool

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260525/lta_owt_bert_absrope_time4_C1_to_1024_len1024_step9000_dualline_quick_n8/context1024_samples.txt

@@ -0,0 +1,29 @@
+[CLS] * * * * * * * * * * * * * * * * * * * *???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????
+
+---
+
+[CLS]???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????
+
+---
+
+[CLS]...????...? *,??? constitution, if? constitution,????????,????? *??.??.? * *????????????????..?? *??.....? *??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? *? *???? *???? * *? *?? *? * *? * *?????? * *?????? *????????????????????????????? *? *?????????? *????????? *???? *???????? *?????????????? *???? *? * *??????????????????????????????????????????????????????????????????????????????????????????????????????????????? * *??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? * *?? *?? *??????????????????????????????? *??????????? *?? * * * * * *? *? * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *?? *?? *????? *? * * *? *??? *????? * *? *????? *????? *?????? * * *? * *? * * * * * * * * * * * * * * * * * * * * * * * * * * * *? * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+
+---
+
+[CLS], and, but so, we will be about the.. we are so... we said, but there to be the arguments, and not,. a lot of them. but they have one. so : i said in the book, the book, the book, the book............ in some. these words..... ”, so are we said, “.... we said. they was the text.. they ’ s the text to be, but, there ’ s some. the language, no. you ’ s so. they ’ s your institution. ” in course, i said, “ give you '. ” in the “. ”, “..? ”...... *. the book isn ' t one. so confusion. but :.. is. the name.. is for you to : : but *., but so the name is, but at the end to the book, the book... is?. text,,.... *, i find?, in the book..., but, but if, in so the book are... they are in the...,.,?, is.......... *, :, is,........,???,????...............??? :????????????????????????....????,...????????????????????.......?........................????.............,????????????,?.,..????.... *?,?????????????????????? * *????? *????????? *???????????? * *??????????? *????? *????????????????????????????????????????????????????????????????????????????????????????????????????????? *?????? *?????????????????????????????????????????????,.?. *.?? *????? * * * *? * *...... * * * * * * * * *... * * * * * * *? *.. * * * * * * * * * * *?? *, *?. *? * * * *? * * * * * * *?. *? * *??,???? * *?? *?? * * * *? *?? * *???,?? *,? *??,?? *?? *?.?., *?,,.?. *,? * * *?. *. * * * *. * * * *. * *.., *?? * *?? * * *, *, *,????? :? : * * * *. *.., * *? *,? *??.?,?.,,,, is *??,?..? * *. * * *... * *. *. *.. *. *....... *.... *.. *.. *. * * * * *. * * *. *. * * * * * * * * * * * * * * * * *... * * *. * *. *. * * *.... *... *....... *.. * *. * *.. * * * *. [SEP]
+
+---
+
+[CLS]?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? * * * * * * * * * * * * * * * * * *??????? * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *?????? * * * *??? * * *????????? *? *??? *? * * * * * * * * * * * * * * * * * * * * *??? * * *? * * * * *????????????? * * * * * *????????? *?? * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *? *????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????
+
+---
+
+[CLS] to be, we is?, be, we is? to, and is said to be?... they doesn ' t say, so they are are said to be : are.... they are so, and : *, they are : *.. are * are said, if they is * are said. they is?, will, but, they is said. they is said to be?. on our ownership, : :.., about *, are??? will be, but they is said,..... on the ownership, i, they is said. they is said, but are, : *,, on the profits. they is no. for... they are said................. are, *. are? *?, are?? is not.. *. so, is are we have to say : *, but :, about ) they. 00s, : so, *s, 0000mals, s about )... they is not, about * are? : 00mal, * are *, they are.... they is no. they said *, is,. :.,,, *,, about :? are,, about, about..... about is,, : about,,? are?,???? *?? *? *?????,????,?? is.... are are are,???? *,? *?, *???????? *?????,??????,?? are???, : they do.... about is,,??,? are, *? are,?? are???,,?? are?????,????,?. *?.... *... *? * *?????,?????,?????,??,? * *????,???. * are *. is *. *...,. are they : about : :,???,???? *,?,.... *,, *, * *, * *?, * *, * * *, *, *, * *?.. * * * *. * * *, * * * * * * *?,,, *???, *? *,?? are * *? * * * * *?, * * *,?. *?.? *?... are. * *.? * *??, are, *,??????,?,.?.... :, are about??.????,????,., if are..., :...,.. : :. :.,., :.. * * * are. *.. * *,., * * *,,.?., are?,?.?.., :. :. :.., :. : : :...,.... are : : : : : : : : * :,,,? *?? is :,.. :.. * : are,? : *, are :, : : : : : :. :..., : : : : :, :? : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : :... :., : : * *,. : :. : : : * : : : : : : :, :?.. * *. * * * *? *,? * *? *,. *.. * * * *,. is.. are. : : : :,, * * : *? :,,?? *,?, *,?, :.. are... :. * *, *??? *,? * :,. : are,,, *. are..... : are. :, :... : :... : : : :. :?, *? * *?? * *?,? :? ) *... * :. : *, *? * *?,, *?.?, : * *, *, *, :? :? * *,,? *.... * * * * *,,? are, *? * *, * * *???. * *. * : [SEP]
+
+---
+
+[CLS]???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? * * * * *??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? *?? *??? * *??? * *? * *???? *? *??? *?? * * * * *??? *??????????? *? *?????????????? *? * *? *? * * *?? *?? *????????? *? * *?? * *??????????????????? * *?? *? * * *?? * * * *?? *? *? *?????????????????????????????????????????????????????????????????????????????????????????????????????? * * * * *?? * *? *????? * *? * *???????????????????????????????????????????????????
+
+---
+
+[CLS]иииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииαε : ( ( ( ) ) ) ( ( ( ) ) ( ( ) ) ) ) ( ( ( ( ) ) ( ( ( ) ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ( ( ) ) ) ( ( ( ( ( ) ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ) ) : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : ( ( ( ( ) ) ( ( ( ) ) ( ) ( ) ( ( ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ( ( ) ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ) ) ) ) ( ( ( ) ) ( ) ( ( ) ( ) ( ( ) ) ) ) ( ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ( ( ( ) ( ( ) ) ( ( ( ) ( ( ) ) ( ( ( ) ) ) ) ( ( ( ) ) ) ( ) ( ) ( ) ) ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ( ) ) ) ( ( ( ) ) ( ) ( ) ( ( ) ) ( ) ( ( ( ( ) ) ( ) ) ( ( ( ) ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ) ( ( ( ) ) ) ) ( ( ( ( ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ( ( ( ) ) ) ( ( ( ) ) ) ( ) ) ( ( ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ( ) ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ) ( ( ( ( ) ) ( ) ) ) ) ( ( ( ( ( ) ) ) ( ( ( ) ) ) ) ( ( ( ) ) ) ) ( ( ( ) ) ( ) ( ) ) ( ( ( ( ) [SEP]

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260525/lta_owt_bert_absrope_time4_C1_to_1024_len1024_step9000_dualline_quick_n8/context1024_samples_stripped.txt

@@ -0,0 +1,29 @@
+[CLS] * * * * * * * * * * * * * * * * * * * *???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????
+
+---
+
+[CLS]???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????
+
+---
+
+[CLS]...????...? *,??? constitution, if? constitution,????????,????? *??.??.? * *????????????????..?? *??.....? *??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? *? *???? *???? * *? *?? *? * *? * *?????? * *?????? *????????????????????????????? *? *?????????? *????????? *???? *???????? *?????????????? *???? *? * *??????????????????????????????????????????????????????????????????????????????????????????????????????????????? * *??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? * *?? *?? *??????????????????????????????? *??????????? *?? * * * * * *? *? * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *?? *?? *????? *? * * *? *??? *????? * *? *????? *????? *?????? * * *? * *? * * * * * * * * * * * * * * * * * * * * * * * * * * * *? * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+
+---
+
+[CLS], and, but so, we will be about the.. we are so... we said, but there to be the arguments, and not,. a lot of them. but they have one. so : i said in the book, the book, the book, the book............ in some. these words..... ”, so are we said, “.... we said. they was the text.. they ’ s the text to be, but, there ’ s some. the language, no. you ’ s so. they ’ s your institution. ” in course, i said, “ give you '. ” in the “. ”, “..? ”...... *. the book isn ' t one. so confusion. but :.. is. the name.. is for you to : : but *., but so the name is, but at the end to the book, the book... is?. text,,.... *, i find?, in the book..., but, but if, in so the book are... they are in the...,.,?, is.......... *, :, is,........,???,????...............??? :????????????????????????....????,...????????????????????.......?........................????.............,????????????,?.,..????.... *?,?????????????????????? * *????? *????????? *???????????? * *??????????? *????? *????????????????????????????????????????????????????????????????????????????????????????????????????????? *?????? *?????????????????????????????????????????????,.?. *.?? *????? * * * *? * *...... * * * * * * * * *... * * * * * * *? *.. * * * * * * * * * * *?? *, *?. *? * * * *? * * * * * * *?. *? * *??,???? * *?? *?? * * * *? *?? * *???,?? *,? *??,?? *?? *?.?., *?,,.?. *,? * * *?. *. * * * *. * * * *. * *.., *?? * *?? * * *, *, *,????? :? : * * * *. *.., * *? *,? *??.?,?.,,,, is *??,?..? * *. * * *... * *. *. *.. *. *....... *.... *.. *.. *. * * * * *. * * *. *. * * * * * * * * * * * * * * * * *... * * *. * *. *. * * *.... *... *....... *.. * *. * *.. * * * *. [SEP]
+
+---
+
+[CLS]?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? * * * * * * * * * * * * * * * * * *??????? * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *?????? * * * *??? * * *????????? *? *??? *? * * * * * * * * * * * * * * * * * * * * *??? * * *? * * * * *????????????? * * * * * *????????? *?? * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *? *????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????
+
+---
+
+[CLS] to be, we is?, be, we is? to, and is said to be?... they doesn ' t say, so they are are said to be : are.... they are so, and : *, they are : *.. are * are said, if they is * are said. they is?, will, but, they is said. they is said to be?. on our ownership, : :.., about *, are??? will be, but they is said,..... on the ownership, i, they is said. they is said, but are, : *,, on the profits. they is no. for... they are said................. are, *. are? *?, are?? is not.. *. so, is are we have to say : *, but :, about ) they. 00s, : so, *s, 0000mals, s about )... they is not, about * are? : 00mal, * are *, they are.... they is no. they said *, is,. :.,,, *,, about :? are,, about, about..... about is,, : about,,? are?,???? *?? *? *?????,????,?? is.... are are are,???? *,? *?, *???????? *?????,??????,?? are???, : they do.... about is,,??,? are, *? are,?? are???,,?? are?????,????,?. *?.... *... *? * *?????,?????,?????,??,? * *????,???. * are *. is *. *...,. are they : about : :,???,???? *,?,.... *,, *, * *, * *?, * *, * * *, *, *, * *?.. * * * *. * * *, * * * * * * *?,,, *???, *? *,?? are * *? * * * * *?, * * *,?. *?.? *?... are. * *.? * *??, are, *,??????,?,.?.... :, are about??.????,????,., if are..., :...,.. : :. :.,., :.. * * * are. *.. * *,., * * *,,.?., are?,?.?.., :. :. :.., :. : : :...,.... are : : : : : : : : * :,,,? *?? is :,.. :.. * : are,? : *, are :, : : : : : :. :..., : : : : :, :? : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : :... :., : : * *,. : :. : : : * : : : : : : :, :?.. * *. * * * *? *,? * *? *,. *.. * * * *,. is.. are. : : : :,, * * : *? :,,?? *,?, *,?, :.. are... :. * *, *??? *,? * :,. : are,,, *. are..... : are. :, :... : :... : : : :. :?, *? * *?? * *?,? :? ) *... * :. : *, *? * *?,, *?.?, : * *, *, *, :? :? * *,,? *.... * * * * *,,? are, *? * *, * * *???. * *. * : [SEP]
+
+---
+
+[CLS]???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? * * * * *??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? *?? *??? * *??? * *? * *???? *? *??? *?? * * * * *??? *??????????? *? *?????????????? *? * *? *? * * *?? *?? *????????? *? * *?? * *??????????????????? * *?? *? * * *?? * * * *?? *? *? *?????????????????????????????????????????????????????????????????????????????????????????????????????? * * * * *?? * *? *????? * *? * *???????????????????????????????????????????????????
+
+---
+
+[CLS]иииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииииαε : ( ( ( ) ) ) ( ( ( ) ) ( ( ) ) ) ) ( ( ( ( ) ) ( ( ( ) ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ( ( ) ) ) ( ( ( ( ( ) ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ( ( ( ) ) : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : : ( ( ( ( ) ) ( ( ( ) ) ( ) ( ) ( ( ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ( ( ) ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ) ) ) ) ( ( ( ) ) ( ) ( ( ) ( ) ( ( ) ) ) ) ( ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ( ( ( ) ( ( ) ) ( ( ( ) ( ( ) ) ( ( ( ) ) ) ) ( ( ( ) ) ) ( ) ( ) ( ) ) ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ) ( ) ) ) ( ( ( ) ) ( ) ( ) ( ( ) ) ( ) ( ( ( ( ) ) ( ) ) ( ( ( ) ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ) ( ( ( ) ) ) ) ( ( ( ( ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ( ( ( ) ) ) ( ( ( ) ) ) ( ) ) ( ( ( ) ) ) ( ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ( ( ( ( ) ) ) ( ( ( ( ) ( ( ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ) ) ) ) ( ( ( ( ( ) ) ) ) ( ( ( ) ) ) ( ( ( ) ) ) ) ( ( ( ( ) ) ( ) ) ) ) ( ( ( ( ( ) ) ) ( ( ( ) ) ) ) ( ( ( ) ) ) ) ( ( ( ) ) ( ) ( ) ) ( ( ( ( ) [SEP]

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260525/lta_owt_dirichlet_len1024_Cv_to_2v_gbs512_4gpu_abspos_specialloss16_save1k_gumbelwatch_20260525_v2_step17000_quick_n8/sde_steps128_samples8_scored.jsonl

@@ -0,0 +1,9 @@
+{"type": "summary", "checkpoint": "runs/lta_owt_dirichlet_len1024_Cv_to_2v_gbs512_4gpu_abspos_specialloss16_save1k_gumbelwatch_20260525_v2/step_0017000.pt", "step": 17000, "decode": {"decode_rule": "dirichlet_resample_sde", "steps": 128, "model_t_mode": "support_t", "mean_mode": "endpoint_only", "anchor_gamma": 1.0, "endpoint_floor": 0.0, "concentration_min": 30522.0, "concentration_max": 61044.0, "endpoint_temp": 1.45, "endpoint_temp_start": null, "endpoint_temp_end": null, "endpoint_projection": "gumbel_softmax", "endpoint_top_k": 0, "endpoint_top_p": 0.95, "gumbel_tau_start": 1.0, "gumbel_tau_end": 0.2, "gumbel_noise_scale_start": 1.0, "gumbel_noise_scale_end": 1.0, "ban_special_tokens": false, "banned_endpoint_ids": [], "support_power": 1.0, "semantic_power": 1.0, "noise_init": "dirichlet", "noise_sigma": -1.0, "noise_dirichlet_concentration": 30522.0, "sde_resample": "dirichlet", "logistic_normal_sigma_min": 0.18, "logistic_normal_sigma_max": 3.0, "logistic_normal_tau_min": 0.65, "logistic_normal_tau_max": 1.0, "final_from": "blend_0.5", "n_samples": 8, "seed": 20260524}, "raw_genppl": {"ppl": 8.794117766169528, "nll_per_token": 2.174083062318886, "tokens": 7937, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 8.559626320589752, "nll_per_token": 2.14705653506407, "tokens": 7882, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 1.8084563032092187, "unique_tokens": 123, "token_count": 8192, "distinct_1": 0.0150146484375, "distinct_2": 0.09640762463343108, "top_token_mass": 0.211181640625}}
+{"type": "sample", "index": 0, "raw_text": "[CLS]mikimimi, qanamimikiki, aishiki, kumikiki, yakimikikukikiki, kukimimikukukiki, aikiki, kikukukikiki, wikimikukukiki, kikiku, kumikiki, wikikimikumimiki, wikikikimimimiki, kimibe, qmimikimimi, kumikiki, qanamimikiku, qkikikuki, kikikikiki, wikimimiku, wikikumimiku, kumibe, qkikikiki, kikikimi, wikikimikikikikumi, yakikukibeki, wikikikikikikikikumiki, kishikimiki, wikiki, tokikakimikukokibe, wikikibekikiki, kikumi, yamimikiki, yakimimi, wikikimikikikumikikiki, kamikimi, kikukikiki, yakimikikumiki, kimaki, kukukukubekikikikikikikiki, q kumikikikimikikiki,mimimimaki,,kikiki,kikikoku,mimimimiki,kikumakimakiki,,kimamaki,mimimi,kimikiki, q,kikikikimamamaki,mimimikibemiki,kikimi,bekimi,kikikiku,, wikikiki, yamakikiki,,,kikikikukikikikiki q, qkikikikiki, qkikubemibemimikikukibekiku,kimimimi,. wikimimi, wiki,,kikokukokukikikibekimi, wikipediaki,,kikukumimibeku, qkikikikikikikimiki,kikimikimiki,kikikikikikikikumikimimimibe,kikikikimimi,kimimi,mikikiki,kikikikikukuki,mimimimi,, qmikikikikikikimimikibemibekibe,mikikimimikiki,kikimiki,kimimiki,ki,kiyo,mi qko qkoda,ki,ki,kikimimomi,ma,ma, qma,mi qki qkiki,mi,mikikikibekiki,kibekikimi,ki,kikakikokukimikikuki, qki,kikikimikukikiki,ki qyo qmi,kikakikikikudaki,ki qkikikikikimimi,kikikikiki,ki qmikimimi,mikikimikimiki,,, q qki qkimikimikimimimimoma,mi,mimikikikikiki,kikikikimiki,ki,ki qkibekikukimiki, qkikimibemimi, wikimikimikimikibeki,mi qki,kikikikikiki,kikikukikibemibe,mi, qkimi,kimomimomakiki,kikikimikibemikimikikiki,mimimimikikiki,kikimikikikikimimimimimi,ku,kikubemibe q, qkikimikikikiki,kikikikimikikiki,kikikikikikikikimiki,kimikuki,mimimikimikiki,mikikikikikimiki,ki qmikimimi,mikimikibekibekikiku,kibekikikiku,mi qki qkiki,mi qkikikikimikikikimi,, qbeki,mi qmi qmikikikikikikikikimi,kikikikikikibemimi,kikikikikuki qki,mi, qkikikikikikikikikikikikikibekukikimibe,bekibebe,kikikibemi, qmikikikimikimikimikimikimikibe,kikikikikikikikibekimikimimikikikiki,ki qmi, qmikikikikukiki [SEP]", "stripped_text": "mikimimi, qanamimikiki, aishiki, kumikiki, yakimikikukikiki, kukimimikukukiki, aikiki, kikukukikiki, wikimikukukiki, kikiku, kumikiki, wikikimikumimiki, wikikikimimimiki, kimibe, qmimikimimi, kumikiki, qanamimikiku, qkikikuki, kikikikiki, wikimimiku, wikikumimiku, kumibe, qkikikiki, kikikimi, wikikimikikikikumi, yakikukibeki, wikikikikikikikikumiki, kishikimiki, wikiki, tokikakimikukokibe, wikikibekikiki, kikumi, yamimikiki, yakimimi, wikikimikikikumikikiki, kamikimi, kikukikiki, yakimikikumiki, kimaki, kukukukubekikikikikikikiki, q kumikikikimikikiki,mimimimaki,,kikiki,kikikoku,mimimimiki,kikumakimakiki,,kimamaki,mimimi,kimikiki, q,kikikikimamamaki,mimimikibemiki,kikimi,bekimi,kikikiku,, wikikiki, yamakikiki,,,kikikikukikikikiki q, qkikikikiki, qkikubemibemimikikukibekiku,kimimimi,. wikimimi, wiki,,kikokukokukikikibekimi, wikipediaki,,kikukumimibeku, qkikikikikikikimiki,kikimikimiki,kikikikikikikikumikimimimibe,kikikikimimi,kimimi,mikikiki,kikikikikukuki,mimimimi,, qmikikikikikikimimikibemibekibe,mikikimimikiki,kikimiki,kimimiki,ki,kiyo,mi qko qkoda,ki,ki,kikimimomi,ma,ma, qma,mi qki qkiki,mi,mikikikibekiki,kibekikimi,ki,kikakikokukimikikuki, qki,kikikimikukikiki,ki qyo qmi,kikakikikikudaki,ki qkikikikikimimi,kikikikiki,ki qmikimimi,mikikimikimiki,,, q qki qkimikimikimimimimoma,mi,mimikikikikiki,kikikikimiki,ki,ki qkibekikukimiki, qkikimibemimi, wikimikimikimikibeki,mi qki,kikikikikiki,kikikukikibemibe,mi, qkimi,kimomimomakiki,kikikimikibemikimikikiki,mimimimikikiki,kikimikikikikimimimimimi,ku,kikubemibe q, qkikimikikikiki,kikikikimikikiki,kikikikikikikikimiki,kimikuki,mimimikimikiki,mikikikikikimiki,ki qmikimimi,mikimikibekibekikiku,kibekikikiku,mi qki qkiki,mi qkikikikimikikikimi,, qbeki,mi qmi qmikikikikikikikikimi,kikikikikikibemimi,kikikikikuki qki,mi, qkikikikikikikikikikikikikibekukikimibe,bekibebe,kikikibemi, qmikikikimikimikimikimikimikibe,kikikikikikikikibekimikimimikikikiki,ki qmi, qmikikikikukiki"}
+{"type": "sample", "index": 1, "raw_text": "[CLS]wiwiwiwiwih, https : / / www. tp. com / wikiwikiwiwiwiwiwiwidowqq http : / / twitter. com / qwiwiwiwikiwiwiwiwiwiwiwiqliqlwiq,, https : / / www. youtube. org / qwiwiwiwiwiwiwiqwiqwwwiwiwiwiwiwih,, http : / / qwiwiwiwiwiwiwiwwihdwihqqqwiwiki, qqqqzwwiwi,, https : / / twitter. com / qdodoqwiwiwiwiwi, qwqwiwih http : / / u. s / /wiwiwiwiwiwiwiwiwiqqwibwiwiwwiwhdwibwiwiqwqqwiwibwiwiwiwidoqhdwwiqwibqqwijodoqqhdwiwiwih, http : / :... /wiwiwiwiwiwiwiwiwiwiwijobqwwwijoqq http : / / twitter. com / qdobwi, / /.. /... / / /wiqqwiwiwiwiwiwi,,, http : / /., / /wiwiwiwiwiwiwiwiwiwiwiwiwidowidoq,,, : http : : :... /wiwiwiwiwiwiwiwiwiwiwiw,,, : / / /... / /wiwiwiwiwiwiwiwiwiwiwiwiwiwiwwwiwiwi, : http : /... /wiwiwiwiwiwiwiwiwiwiwidoqqwiwwiwiwi, : : : /.... / / /wiwiwiwiwiwiwiwiwiwiwiwiwwwqhdwiwidoq,,, : / / / u. s s... / /wiwiwiwiwiwiwiwiwiwiwiwiwwwiwidododow,,, : : : / :... /wiwiwiwiwiwiwiqwiwiwiwiqqqabqwiwwqwi,, : : : /.... / /wiwiwiwiwiwizwiwiwiwwiwwiwiwdowiwi,wi, http, : /... / /. /wiwiwiwiwiqwiwiwizqqwiwiwiwiwiwiwiwiqwiwizwidoqdowiqwiwi, : / :.. :. /wikiwiwiwiwiqqlqq, http, : /wiqwiwiwiwiqhq,q,,,, : :. /. /. /wiwiwiwiwiwiwiwiwiwiwiwiwi,wi, : /wi /wiwiwiwiwiwiwiwiwiwiwiqwiwwiqwi,wi, : /. :... /wiwiwiwiwiwiwiwiwiqwiwiqwiwiwiqwiwiwiwiwiwiwiwi, :, : : : / /wizwiwiwiwibwiwiwiwiwiwiqwiwiwiwiwiwiwiwiqwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiqqwiqwiqwiwiwiwiwiwiq, : / : /. :. / /wi /wiwiwiwiwiwiwiwiwiwiwiwiq : :, :. /.. /wi /wiqwiwiwiwi,wi : : : : /wiwiwiwiwiwiwiwiwiqqwwiqwiw, http, : / : :.,wi,wiwiwiwi : /. /. /. /wiwiwiwiwiqwiwiwiwwiwwiwiwiwiqwiwiwiwwiwiwiwiqwiqqqwi,, : / : /. :. /wiwiwiwiqwiwi, : : : /. / /,wiwiwiqdowiwiwiwi,wi : http, : :.wi /wiwiwiqwidowqwiqwiwwiw :b,, :, :wiqwiq :wi :, : / http : u /... /.... /wiwiwiwiwiwiwiwiwiwiwiwiwiwiqwiqqwihdwqhqq, :, : / : /.,wi /wiwiwiwiwiwi,q, :, : /.... /. /wiwiwiwiwiwiwi [SEP]", "stripped_text": "wiwiwiwiwih, https : / / www. tp. com / wikiwikiwiwiwiwiwiwidowqq http : / / twitter. com / qwiwiwiwikiwiwiwiwiwiwiwiqliqlwiq,, https : / / www. youtube. org / qwiwiwiwiwiwiwiqwiqwwwiwiwiwiwiwih,, http : / / qwiwiwiwiwiwiwiwwihdwihqqqwiwiki, qqqqzwwiwi,, https : / / twitter. com / qdodoqwiwiwiwiwi, qwqwiwih http : / / u. s / /wiwiwiwiwiwiwiwiwiqqwibwiwiwwiwhdwibwiwiqwqqwiwibwiwiwiwidoqhdwwiqwibqqwijodoqqhdwiwiwih, http : / :... /wiwiwiwiwiwiwiwiwiwiwijobqwwwijoqq http : / / twitter. com / qdobwi, / /.. /... / / /wiqqwiwiwiwiwiwi,,, http : / /., / /wiwiwiwiwiwiwiwiwiwiwiwiwidowidoq,,, : http : : :... /wiwiwiwiwiwiwiwiwiwiwiw,,, : / / /... / /wiwiwiwiwiwiwiwiwiwiwiwiwiwiwwwiwiwi, : http : /... /wiwiwiwiwiwiwiwiwiwiwidoqqwiwwiwiwi, : : : /.... / / /wiwiwiwiwiwiwiwiwiwiwiwiwwwqhdwiwidoq,,, : / / / u. s s... / /wiwiwiwiwiwiwiwiwiwiwiwiwwwiwidododow,,, : : : / :... /wiwiwiwiwiwiwiqwiwiwiwiqqqabqwiwwqwi,, : : : /.... / /wiwiwiwiwiwizwiwiwiwwiwwiwiwdowiwi,wi, http, : /... / /. /wiwiwiwiwiqwiwiwizqqwiwiwiwiwiwiwiwiqwiwizwidoqdowiqwiwi, : / :.. :. /wikiwiwiwiwiqqlqq, http, : /wiqwiwiwiwiqhq,q,,,, : :. /. /. /wiwiwiwiwiwiwiwiwiwiwiwiwi,wi, : /wi /wiwiwiwiwiwiwiwiwiwiwiqwiwwiqwi,wi, : /. :... /wiwiwiwiwiwiwiwiwiqwiwiqwiwiwiqwiwiwiwiwiwiwiwi, :, : : : / /wizwiwiwiwibwiwiwiwiwiwiqwiwiwiwiwiwiwiwiqwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiwiqqwiqwiqwiwiwiwiwiwiq, : / : /. :. / /wi /wiwiwiwiwiwiwiwiwiwiwiwiq : :, :. /.. /wi /wiqwiwiwiwi,wi : : : : /wiwiwiwiwiwiwiwiwiqqwwiqwiw, http, : / : :.,wi,wiwiwiwi : /. /. /. /wiwiwiwiwiqwiwiwiwwiwwiwiwiwiqwiwiwiwwiwiwiwiqwiqqqwi,, : / : /. :. /wiwiwiwiqwiwi, : : : /. / /,wiwiwiqdowiwiwiwi,wi : http, : :.wi /wiwiwiqwidowqwiqwiwwiw :b,, :, :wiqwiq :wi :, : / http : u /... /.... /wiwiwiwiwiwiwiwiwiwiwiwiwiwiqwiqqwihdwqhqq, :, : / : /.,wi /wiwiwiwiwiwi,q, :, : /.... /. /wiwiwiwiwiwiwi"}
+{"type": "sample", "index": 2, "raw_text": "[CLS] q q q, r, n q q q q q q q q q q q q q q - q q q s, r s - r q q q q q q q q q q q q r - q q q q q q q q q q - n q - q q q q q q q q q q q q - r q u q q ’ s s - r q q q q q q q q q q q q q q q q q - q q - q q - n q q q q q q q, r, r - q q q q q q q q q q q q r, r r, n q q q s, s r - r u, n, n, r, n, n - q u - n u, n q q q q q, n s - q u, n n, r, n - q q q r - n q q q s, u, n s - q q ’ s r, n s - q s - r s - r u ’ s r, r, q u - r u, n, u - n, r - n, n, n q q q q - q q q - r q q s, r r r - q q q q q - q - q q q q q q - q q q q q q q q q, r n, q q q q q q q q q q - q q q q q q q q q q q q q q q, r, q q - q q q q r,, r q q q q q q q q r r, q q - q q q q q - r r, q - - q q q q q q s - r, q q q q q q q q q q s r q q q q q q q, q q q q q q q q q q q q q q q q q q - q q - q q, r, r q q q q q q q q q q q q q - r, r r, q - r r - q q, r q s - q q q q - q - q q q q q q s q - r q q q q q - q - r - r q q q q q q q q q q q q q q q q q, s r r - q, r q q q, r r r - q q q q q q q q q s q,, r n q q q q q q q q, n q q q q q q s s s - s r - r q q q q q r s - r q q q q q q q,, r s, q r, q r - q q q r q s q q - q q q r, r q,, q q q q q q q q - q q r q q q q q q q q s - - q r q q q q q - q r q q r q q q q q q q r - s q s r r q q q q r - s q q s s r, r q r q q q q q s r q r r q q q q r q q q q q - q - q s r q q q q r q r r q q q q - q q q r q r - q q q - r q r q q q q q - r r r q r q q q q q q - q s r q q q q - q q q q q q - q q q - q q r, r q r - q q a - r s - r q q q s r q r - r r - - s - r - r q q q s - r - s q q q r q q q - q q q q q - q - - s r s s - r r - q q a q, q - a q q - q - r s - q - r a, q q r q r r - s q q - q q q r q - r q r q q q q q q - r r q r - r r q r q - q q q q q q - r q q q s - - r - r r r q q q q - - q q q - q q q q q s r q r q q q q q q r, s -, r q q q q q - q q s r q, q q q q q - q q q, q,, r q r - - q q q - q - q s q - r r f, r q, r q, r q,, u q, q q q q q q q, q q., r., r.,, r. r, r - q - q - q s r q q q q, q r r - r q q q q [SEP]", "stripped_text": "q q q, r, n q q q q q q q q q q q q q q - q q q s, r s - r q q q q q q q q q q q q r - q q q q q q q q q q - n q - q q q q q q q q q q q q - r q u q q ’ s s - r q q q q q q q q q q q q q q q q q - q q - q q - n q q q q q q q, r, r - q q q q q q q q q q q q r, r r, n q q q s, s r - r u, n, n, r, n, n - q u - n u, n q q q q q, n s - q u, n n, r, n - q q q r - n q q q s, u, n s - q q ’ s r, n s - q s - r s - r u ’ s r, r, q u - r u, n, u - n, r - n, n, n q q q q - q q q - r q q s, r r r - q q q q q - q - q q q q q q - q q q q q q q q q, r n, q q q q q q q q q q - q q q q q q q q q q q q q q q, r, q q - q q q q r,, r q q q q q q q q r r, q q - q q q q q - r r, q - - q q q q q q s - r, q q q q q q q q q q s r q q q q q q q, q q q q q q q q q q q q q q q q q q - q q - q q, r, r q q q q q q q q q q q q q - r, r r, q - r r - q q, r q s - q q q q - q - q q q q q q s q - r q q q q q - q - r - r q q q q q q q q q q q q q q q q q, s r r - q, r q q q, r r r - q q q q q q q q q s q,, r n q q q q q q q q, n q q q q q q s s s - s r - r q q q q q r s - r q q q q q q q,, r s, q r, q r - q q q r q s q q - q q q r, r q,, q q q q q q q q - q q r q q q q q q q q s - - q r q q q q q - q r q q r q q q q q q q r - s q s r r q q q q r - s q q s s r, r q r q q q q q s r q r r q q q q r q q q q q - q - q s r q q q q r q r r q q q q - q q q r q r - q q q - r q r q q q q q - r r r q r q q q q q q - q s r q q q q - q q q q q q - q q q - q q r, r q r - q q a - r s - r q q q s r q r - r r - - s - r - r q q q s - r - s q q q r q q q - q q q q q - q - - s r s s - r r - q q a q, q - a q q - q - r s - q - r a, q q r q r r - s q q - q q q r q - r q r q q q q q q - r r q r - r r q r q - q q q q q q - r q q q s - - r - r r r q q q q - - q q q - q q q q q s r q r q q q q q q r, s -, r q q q q q - q q s r q, q q q q q - q q q, q,, r q r - - q q q - q - q s q - r r f, r q, r q, r q,, u q, q q q q q q q, q q., r., r.,, r. r, r - q - q - q s r q q q q, q r r - r q q q q"}
+{"type": "sample", "index": 3, "raw_text": "[CLS]. 56. 5 - 11. 51, - 51. 61 - - - - 45 - 45 - 8. 65. 8, 65 - 58 - 62. 62. 53 - 65 - 53. 65 60 - 64. 6 - - - 53 - 61 - 45. 58 60 - 60 - 51. 57 58 - 62 - 60. 65 58 - 70 - 64 - 61 3. 2 - - 15. 51 - 56 57. 58 60 -, 58 - 70 - 65 - 53 - 54 - 61 62. 61 60 - 60. 65 - 57. 50 - 70 -. 65 - 50 - 54 58 - 60 - 65 - 65 - 5 - 64 - 6. 58 - - 58 - 50. 61, 61, 60 - 65 59, 60 45. 3 -. 54. 57 - 52 - 50 - 53 60. 50 - 58 - 45 - 6. 53, 5, 6, 3 - 11. 57 - 56 - 60, 64 - 0. 2 2 - - 15. 57 - 60 56, 53 -, - - 45 - 60 60 - 80 - 60 - 60 - 64. 53 - 58 - 57 - 59 60. 58 59, 65 - - - 53 60, 60 60 - 2, 60. 4 4. 5 - - - 60 61 60, 56 3 3. 0. - - -. - - 59 - - 58, 61 70 -. 50 60 - - 60 60 - 10 2 - 2 - -, - 5. 9 7. - - 60 61. - - - 0. - - 56 56 - 58 62 -, 59 -. 1 - 1 2. - - - - - - -. - 58 - -. - -. -. - - 5 8 - 8. 2 - 2 - - 58. 57. 50,. - 2 2 - 7.. - - -, 60 80 - - 60 45 - 60. - 61 - 64. 50 60.. 4 9 5 6. 5 - - - -. - 58 -, - - - 50, - - 45 50 - -, 58 - 58. - - - -.. - 58 59, - 61 - - - -. - - 54. - 56, - - 40, 60 60 60. 60, 64, 60 60 60. - 2 7 -. 8 8 60 60 60. 1 -. 1 - - - - - - - - -, : 58. 60 61 - 60 60 -, 60, 60 60. 58 60 - 60,, 65, 65 - - 5 5 5 5. 2,. - -, : - - - - -. 53, 48 - 54 - - 10 61. 6 2, 6 5 - 2 - - - 70, - 60 -, 61 5 5 10 4 3 - 7 - 5 2. - - - 5. 4, - -, - : 54. -, 61. - -,,, 53 45 - - 61. 59 - - 59, -, 43, 59., -. 60 60 -,, -. 3., 10, 60. 4. - 5 4. - -, - -. -, -, - 54 - 53. 60 61 50 -, 7 5 9. - 9 3 2. - : 50 -,, - 60 60, 60 60 - 8 70. 3, - 70,.,. -. -. - - -. - - - - 53, - 70,. 5 60 5 - 5 8. -, - 6 3 7. 5. -. - - - 60 - 59 58 61 60. - 3 - 61 60. 59 60, 61, 10 2 - 5 5 2, - - -. -, -, - - - 59 -. -. - 45 -. 59 4 - 10 - 4. - 2. 4 3. - 60 61 - 80 60. 57 59, 60. 58 62 61 - 58 45 61 60 -. 60 4. 1 - - - 2 0 - 60 58 61. 2 60. - - - - - - - 61. 59 59 - 59 2 59. 1 - - 4 5, 5 2 7. 59. - 61 - 59 - 60 4 60 - 10 1 2 - - 4 1 -. 2 - - 61 - 5. 58 - 59 60. 3. 3 5 4, : 47 -. 58 60 - 57 - 59 58 -. 3 50. 0 - 54. - - - 58 80. 57 54 - 59 61 - 60. -. 6 2 7 3. -, - 46 - - 59 59 - 54. 59 56,. -. - - - - -. -. 45 -... - 58 - 60 59 60 50 61. - 4. : 50 - - - - 4. 59 45, - 61 50 0 -. 2 5 9, : - - - - -. - 61 60. 60 60 5 59 5 7 : 59 58 -. - [SEP]", "stripped_text": ". 56. 5 - 11. 51, - 51. 61 - - - - 45 - 45 - 8. 65. 8, 65 - 58 - 62. 62. 53 - 65 - 53. 65 60 - 64. 6 - - - 53 - 61 - 45. 58 60 - 60 - 51. 57 58 - 62 - 60. 65 58 - 70 - 64 - 61 3. 2 - - 15. 51 - 56 57. 58 60 -, 58 - 70 - 65 - 53 - 54 - 61 62. 61 60 - 60. 65 - 57. 50 - 70 -. 65 - 50 - 54 58 - 60 - 65 - 65 - 5 - 64 - 6. 58 - - 58 - 50. 61, 61, 60 - 65 59, 60 45. 3 -. 54. 57 - 52 - 50 - 53 60. 50 - 58 - 45 - 6. 53, 5, 6, 3 - 11. 57 - 56 - 60, 64 - 0. 2 2 - - 15. 57 - 60 56, 53 -, - - 45 - 60 60 - 80 - 60 - 60 - 64. 53 - 58 - 57 - 59 60. 58 59, 65 - - - 53 60, 60 60 - 2, 60. 4 4. 5 - - - 60 61 60, 56 3 3. 0. - - -. - - 59 - - 58, 61 70 -. 50 60 - - 60 60 - 10 2 - 2 - -, - 5. 9 7. - - 60 61. - - - 0. - - 56 56 - 58 62 -, 59 -. 1 - 1 2. - - - - - - -. - 58 - -. - -. -. - - 5 8 - 8. 2 - 2 - - 58. 57. 50,. - 2 2 - 7.. - - -, 60 80 - - 60 45 - 60. - 61 - 64. 50 60.. 4 9 5 6. 5 - - - -. - 58 -, - - - 50, - - 45 50 - -, 58 - 58. - - - -.. - 58 59, - 61 - - - -. - - 54. - 56, - - 40, 60 60 60. 60, 64, 60 60 60. - 2 7 -. 8 8 60 60 60. 1 -. 1 - - - - - - - - -, : 58. 60 61 - 60 60 -, 60, 60 60. 58 60 - 60,, 65, 65 - - 5 5 5 5. 2,. - -, : - - - - -. 53, 48 - 54 - - 10 61. 6 2, 6 5 - 2 - - - 70, - 60 -, 61 5 5 10 4 3 - 7 - 5 2. - - - 5. 4, - -, - : 54. -, 61. - -,,, 53 45 - - 61. 59 - - 59, -, 43, 59., -. 60 60 -,, -. 3., 10, 60. 4. - 5 4. - -, - -. -, -, - 54 - 53. 60 61 50 -, 7 5 9. - 9 3 2. - : 50 -,, - 60 60, 60 60 - 8 70. 3, - 70,.,. -. -. - - -. - - - - 53, - 70,. 5 60 5 - 5 8. -, - 6 3 7. 5. -. - - - 60 - 59 58 61 60. - 3 - 61 60. 59 60, 61, 10 2 - 5 5 2, - - -. -, -, - - - 59 -. -. - 45 -. 59 4 - 10 - 4. - 2. 4 3. - 60 61 - 80 60. 57 59, 60. 58 62 61 - 58 45 61 60 -. 60 4. 1 - - - 2 0 - 60 58 61. 2 60. - - - - - - - 61. 59 59 - 59 2 59. 1 - - 4 5, 5 2 7. 59. - 61 - 59 - 60 4 60 - 10 1 2 - - 4 1 -. 2 - - 61 - 5. 58 - 59 60. 3. 3 5 4, : 47 -. 58 60 - 57 - 59 58 -. 3 50. 0 - 54. - - - 58 80. 57 54 - 59 61 - 60. -. 6 2 7 3. -, - 46 - - 59 59 - 54. 59 56,. -. - - - - -. -. 45 -... - 58 - 60 59 60 50 61. - 4. : 50 - - - - 4. 59 45, - 61 50 0 -. 2 5 9, : - - - - -. - 61 60. 60 60 5 59 5 7 : 59 58 -. -"}
+{"type": "sample", "index": 4, "raw_text": "[CLS] ( n ) ( 2, 727 ) s. m. - ( 4 : 5 % ) m ( d ) ( r, 498 ), 567. ( r - 538, 563, 535 ) n - m - ( ( d - f ) ( 8, 49. 8 % ) ( r - d ) john mccain ( r - 625 ) r - ( 58, 585 ) r - ( 50 ) m - n - p ( rp ) john mccain ( r - n, ( 2. 9 % ) m - n - p ( ( r - f ) 2 - 5 ) john mccain ( r, r ) ( 56, 575 ) s. gop - s - ( ( r ) ( 28, 585 ) ( ( r - 6 - 7, 1. 15. ) ) - ( 51. 45 ) m ( r ) ( 54, 535 ) - ( ( 50, 725 ) john deb ( r - r ) - - d - wh - ( 10 : 5 % ) ( 11 : 35, 30 ) ( - 7 - 6 % ) ( ( d - s ), 11. 30 - ( 32. 40 % ) ( ( ), 32. 62 % ) ( ( ( ) -, 7. ) ) ) john de deb ( - - n ) - r - ( ( ( ( s ) ) ) r - s de ( ( ( ( : ) ) ) ( ( ) ( 9., 15 ) - - j ( ( ( d - ),,.... ) ( ( ( - - ( 60 ) ( ( ( ) - 10 5. ) ) ) - ( ( ( ( -. r ) ) - - ( ( - ( c, ( r ) ( ( ( b ) ) ) ( ( ( ( s ) - ) 5 5 5., 5. 7.. ) ) ( ( ( ( : ( ( ( ), 8 10. ) ) ) - ( ( ( (. s.. ) j ) ( ( ( ( f,, : m ) ( ) 2 : 10 45. ) ) ) ( s s. -. j - - ( ( ( ( - ( n ) ( ( ( : 2 2 ) ( ) 50 50 ) - j - - ( ( ( :. - -.... ) ( ( ( ( 60 60, 50 ) ) ) - ( ( ( ( ( 60, 50 ) ) ) ( ( ) ( - 2., 6. 9. ) ( ( ( ( - r - 60 ) ( ) ( : ( ). ) -. - ( - ( (.... ). ) ( ( ( ( ( ( ( ( ),. 3. ). ) ( ( : ( s ) :, 1 ) ). ) m ) - ( ( ( ( ( - ( r ) ( 60 ( 60, - 75 ) ) ( ) ( r ( - 2, 8,. ). ) - ( ( ( ( t rp - ),,.... ) ( - ( (. r, n ) ( ) ( : r - r ).,. ). ) ( ( ( ( - r ( ), 5 ) : ) s ) - j (. ( -. r. ) ( r ( ( n -, r - ) n. ). ) - ( ( ( r s - ), 10 ) 3 ) ( - ( 8,.,. 9 13. ) - ( - ( ( ( ( ( ( - ( ( 60 r ), 60 ) ) ) ( ) ( ) 7. : 15 - 15. ) - - - ( ( - ( ( - ) - ) ( ) ( ( ( ( ( ( n f ), 8 ) ) ) ( ) ( ) ) ), 5. 10. ) ( ( ( ( ( ( ( ) - 2 ) 5., 10. ) ( - ( ( ( r. ( ) s ) ( ) ( r ) ( ) ( ( ( ( ( r - ) - ) r - ) ( 7 ( 7, - 11 ) - - - ( s t - - - - p ( ( - ( ( - s ( s 75 ), 75 ) ( ) ( r ( r, 2 ). ) ( ) ( (, 11 ) 10.. ) ) ( ( ( ( ( ( ( s. ). ) ( ) ( ( ( - ( - ) r - ) : 01 : 01., 30 ) ( r ) - - t - ( ( f r - - - f ) ( ) ( ( 2 - ), 9.. 15 ). ) - -. t s (. - n ) ( ) ( ( fb ), 15 9. ). ) - - r t. ( - - ( (. - n ) n. ) ( ( ( ( ( ( ( ) - - ( ( - ) ) 7. ) 15 ) s ) [SEP]", "stripped_text": "( n ) ( 2, 727 ) s. m. - ( 4 : 5 % ) m ( d ) ( r, 498 ), 567. ( r - 538, 563, 535 ) n - m - ( ( d - f ) ( 8, 49. 8 % ) ( r - d ) john mccain ( r - 625 ) r - ( 58, 585 ) r - ( 50 ) m - n - p ( rp ) john mccain ( r - n, ( 2. 9 % ) m - n - p ( ( r - f ) 2 - 5 ) john mccain ( r, r ) ( 56, 575 ) s. gop - s - ( ( r ) ( 28, 585 ) ( ( r - 6 - 7, 1. 15. ) ) - ( 51. 45 ) m ( r ) ( 54, 535 ) - ( ( 50, 725 ) john deb ( r - r ) - - d - wh - ( 10 : 5 % ) ( 11 : 35, 30 ) ( - 7 - 6 % ) ( ( d - s ), 11. 30 - ( 32. 40 % ) ( ( ), 32. 62 % ) ( ( ( ) -, 7. ) ) ) john de deb ( - - n ) - r - ( ( ( ( s ) ) ) r - s de ( ( ( ( : ) ) ) ( ( ) ( 9., 15 ) - - j ( ( ( d - ),,.... ) ( ( ( - - ( 60 ) ( ( ( ) - 10 5. ) ) ) - ( ( ( ( -. r ) ) - - ( ( - ( c, ( r ) ( ( ( b ) ) ) ( ( ( ( s ) - ) 5 5 5., 5. 7.. ) ) ( ( ( ( : ( ( ( ), 8 10. ) ) ) - ( ( ( (. s.. ) j ) ( ( ( ( f,, : m ) ( ) 2 : 10 45. ) ) ) ( s s. -. j - - ( ( ( ( - ( n ) ( ( ( : 2 2 ) ( ) 50 50 ) - j - - ( ( ( :. - -.... ) ( ( ( ( 60 60, 50 ) ) ) - ( ( ( ( ( 60, 50 ) ) ) ( ( ) ( - 2., 6. 9. ) ( ( ( ( - r - 60 ) ( ) ( : ( ). ) -. - ( - ( (.... ). ) ( ( ( ( ( ( ( ( ),. 3. ). ) ( ( : ( s ) :, 1 ) ). ) m ) - ( ( ( ( ( - ( r ) ( 60 ( 60, - 75 ) ) ( ) ( r ( - 2, 8,. ). ) - ( ( ( ( t rp - ),,.... ) ( - ( (. r, n ) ( ) ( : r - r ).,. ). ) ( ( ( ( - r ( ), 5 ) : ) s ) - j (. ( -. r. ) ( r ( ( n -, r - ) n. ). ) - ( ( ( r s - ), 10 ) 3 ) ( - ( 8,.,. 9 13. ) - ( - ( ( ( ( ( ( - ( ( 60 r ), 60 ) ) ) ( ) ( ) 7. : 15 - 15. ) - - - ( ( - ( ( - ) - ) ( ) ( ( ( ( ( ( n f ), 8 ) ) ) ( ) ( ) ) ), 5. 10. ) ( ( ( ( ( ( ( ) - 2 ) 5., 10. ) ( - ( ( ( r. ( ) s ) ( ) ( r ) ( ) ( ( ( ( ( r - ) - ) r - ) ( 7 ( 7, - 11 ) - - - ( s t - - - - p ( ( - ( ( - s ( s 75 ), 75 ) ( ) ( r ( r, 2 ). ) ( ) ( (, 11 ) 10.. ) ) ( ( ( ( ( ( ( s. ). ) ( ) ( ( ( - ( - ) r - ) : 01 : 01., 30 ) ( r ) - - t - ( ( f r - - - f ) ( ) ( ( 2 - ), 9.. 15 ). ) - -. t s (. - n ) ( ) ( ( fb ), 15 9. ). ) - - r t. ( - - ( (. - n ) n. ) ( ( ( ( ( ( ( ) - - ( ( - ) ) 7. ) 15 ) s )"}
+{"type": "sample", "index": 5, "raw_text": "[CLS] q q qqwiwi q qwiwiwi, q q qwiwi qwida, q q qwiwi, qwiwi, qhd, q qwiwi, q qqda, q q qwiwi, q qhd, q q qwiwi, q qqda, q qhd, qhd, qwiwi, qhd, qwiwi qhdhd, q qqqqwi, qhd, q q qwiwi, q qqwida, qhdhd, qwikida, qhdhd, qwiwi q qhd, q q qhdhd, q qqqda, q qwikida, qhd - q qqqwiwi, q qqq, q qwiwi, q qwida, q qqqwiwi, q qhd, q qkiq qhd, q qqqmi, q q q qqwida, q qqq, q q q qqqqmi qhdda, q qqwi, qhd, q qhd, q qqwida, q qwi, qhd, q qqda, q qhdhd, qwiqqda q q qq, q qwiwimi, qqkiki - q q q q q q qqq qmi, q q q, qwiwiwi q q q qqqqwiwiwi, q q q q, q q qq,, q q qqqwi, q q qhdhd, qqwi, q q q q qwiqhd, q q q, q q qqqwi qwi, q q q, q q qwiwiwi q q q q qwiwiwi, q q qq q q qhd,, q qhdhd, qwiqwi q qhdq, q q qwi q q q q q - q q q q, qqqq q qhdhd, q q q, q q q qwiwiwiwi q, qwihdhdhd q q q qq qwi q qwiwiwi qwiwiwi, q q qhd q q qhd, q q q q qhdhd, q q q q, q, q q qqq qwi, q q qhd, q q qhdhdhd, q q qwi q q qwi q q q qhd qhd, qhdwihd q, q qq,q q q q qwi q., q q q q, qq q qwiqwi, qhd qhd,. q q, q qhdq qq, q q qqwihdq, qhd qq q, qq, q q qq qwi, q qwi q q q qwi, q q q q q qq, q q qq qwi, qhd qhd qhd qwi qhd qhd qq qqhdmihdmiwimi, q qq qqwiwi, q q qq, q q q qq qwi,, q q qhd,q q q q qwi q q q qwiwi, qhd,wi q q a qhd, q a q, q q q, q q qhd qhd, qhdwihd qhd. qqhd qhd qhd q qqq qhd q q qwi, q qwi, q q qq, - q - q q qq qqwi qwi q q q q q, qhd qwi q q qwiq q qwi q, q, q q q, q q q qwi qwi, q q qqwiqq, qq q q qqwi qqhd qhd q, q qq qhd, q q q, qhd q q q, qhd, q q qq q qhd, q q qq qwida q q q q q q qq q q, qq q q - q q q qq q qwi, q q q, q, qq qwi, q qq, qq qq qwi, q q q q qq qwi, qwi q q,wi q q, q q qhd qhd q q q q qq qkida q q, q q qq qhd qhd, q, qhd q q, qq qqwi q q q q, q q q qwi q q q, qwi q q q qwihdhd, q q qwidaq, qq, q q q q q q q q, qq qhd, q q q q q, qhd, q q q, q q q q qhd, q q qwi, q q qhd q [SEP]", "stripped_text": "q q qqwiwi q qwiwiwi, q q qwiwi qwida, q q qwiwi, qwiwi, qhd, q qwiwi, q qqda, q q qwiwi, q qhd, q q qwiwi, q qqda, q qhd, qhd, qwiwi, qhd, qwiwi qhdhd, q qqqqwi, qhd, q q qwiwi, q qqwida, qhdhd, qwikida, qhdhd, qwiwi q qhd, q q qhdhd, q qqqda, q qwikida, qhd - q qqqwiwi, q qqq, q qwiwi, q qwida, q qqqwiwi, q qhd, q qkiq qhd, q qqqmi, q q q qqwida, q qqq, q q q qqqqmi qhdda, q qqwi, qhd, q qhd, q qqwida, q qwi, qhd, q qqda, q qhdhd, qwiqqda q q qq, q qwiwimi, qqkiki - q q q q q q qqq qmi, q q q, qwiwiwi q q q qqqqwiwiwi, q q q q, q q qq,, q q qqqwi, q q qhdhd, qqwi, q q q q qwiqhd, q q q, q q qqqwi qwi, q q q, q q qwiwiwi q q q q qwiwiwi, q q qq q q qhd,, q qhdhd, qwiqwi q qhdq, q q qwi q q q q q - q q q q, qqqq q qhdhd, q q q, q q q qwiwiwiwi q, qwihdhdhd q q q qq qwi q qwiwiwi qwiwiwi, q q qhd q q qhd, q q q q qhdhd, q q q q, q, q q qqq qwi, q q qhd, q q qhdhdhd, q q qwi q q qwi q q q qhd qhd, qhdwihd q, q qq,q q q q qwi q., q q q q, qq q qwiqwi, qhd qhd,. q q, q qhdq qq, q q qqwihdq, qhd qq q, qq, q q qq qwi, q qwi q q q qwi, q q q q q qq, q q qq qwi, qhd qhd qhd qwi qhd qhd qq qqhdmihdmiwimi, q qq qqwiwi, q q qq, q q q qq qwi,, q q qhd,q q q q qwi q q q qwiwi, qhd,wi q q a qhd, q a q, q q q, q q qhd qhd, qhdwihd qhd. qqhd qhd qhd q qqq qhd q q qwi, q qwi, q q qq, - q - q q qq qqwi qwi q q q q q, qhd qwi q q qwiq q qwi q, q, q q q, q q q qwi qwi, q q qqwiqq, qq q q qqwi qqhd qhd q, q qq qhd, q q q, qhd q q q, qhd, q q qq q qhd, q q qq qwida q q q q q q qq q q, qq q q - q q q qq q qwi, q q q, q, qq qwi, q qq, qq qq qwi, q q q q qq qwi, qwi q q,wi q q, q q qhd qhd q q q q qq qkida q q, q q qq qhd qhd, q, qhd q q, qq qqwi q q q q, q q q qwi q q q, qwi q q q qwihdhd, q q qwidaq, qq, q q q q q q q q, qq qhd, q q q q q, qhd, q q q, q q q q qhd, q q qwi, q q qhd q"}
+{"type": "sample", "index": 6, "raw_text": "[CLS]qhdhdhdhdhdhqhdqhdhdqhdhqqhdqhdhdhdhdhdhdhqqhdqhdhdhdhqhdhdhqhdhdhdqhdhdhdqhdhdhdhdqqqhdqhdhdhqhdhdhqhdhdqhdhdhdqhdqhdhdqqhdhdhdhdhdhqhdhdhdhdhqhdhdhqhdhdhdhdhdhdhdqhdhdhdhdhdhdhdhdhdhdhqhdh, qhdhdhqhdhdhdqqqqqqqqqqqqhdhdhdhdhdhdhqqhdhqqhdhdhdhdhqqqqqhdqhdhdhdhqhdqhdhqqhdqqhdhqhdqhdhdhdhdhqqhdhdqhdhdqhdhdhdhdhdhdhdh, qhdhdhd, qhdqqhdhdhqhdhqqhdhdhdhdhdqhdhdhdqqhdhdhdqqhdhqqhdhdhdhdhqqhdqqqqqqhdhqqqhdqqqqqhdhdhdqhdhdhdqqhdhdhdhdhqhdhdqqhdhdqhdhdqhqhdhdhdhdhdhdqhdhdhdhdqqqqqhdhdhdhdhdhdhdhqqqhdqqqqqhdhdhdhdqqhdhdhdhdhqqhdhdqhdhdhdqhhqhdhdhdhdhdhdhdhdhdhdqhdhdhdqhdqhdhdhdqhdhdqqhdhdhdhqqhdhdhdhwihwihqqqwiqqqqqqqqhdqhdhdhdhdhdhdhqqqhdqqqqqhdhdhqhdhdhdhdqhdhdhdhdhdhdhdhdhdhqqqhdqqhqqqhdqhdqqqhdhdqqqhdhdqhdhdqhqqqhdqqqqhdhdhdhdhdhdhdhdhdhdhqqqqqqhdhdhqqqqqqhdqhdqhdhqqhdhdhdhdhdhdqqhqhdqhdqhdqqqhdqqqqhdhdhdhdhdhdhdqhdhdqhdhdhdhdhqqqhdqhdhdqhdqhdhhdqhdhdqhdhdqhdhdhdhqqhdqhdhqqwihdqqhqhdhdhhdqqqqhdqhdqhdhdhdhdhqqhdqqqhdqhdhdqhdhdhqhdhdhdhdhdhdhdhdhqhdqhdqhdhdhdqqqhdqhdqhdhqqhdhdhdhdhdhdhdhdqhdhqhdhdhdhdhdhdqhdhdhdhdhdqqhqqqqqqhdqqhqhdqhdqhdqhdqhdhdqhdhdhdhdqhdhdhqhdhdhdhdhdhdhqhqhdqhdqqhdhdhdqqqqqhdhdhqqqhdqhdhdqqqhdhdhdqhdhdhdhdqhdhdqhqhdhqqhdhdqhdhdhdhhqqqqhdhdhhqhdqhdhdhdhdhdhdhhdhqqhdhdhdhdhdhdqqqhdhdhhdqqqhdhdqqhqhdhdhdhqhdhqhdhdhhdqhdqqhdhdhdhdhdwihdhqwiqwiqhdqqhdhdhdhdhdhdhdhqqhdqhdhdhdhdhqhdqqqqhdqhdhdhdhdhdhdhdqqqqhdqhdqhdhdhdhdqhdhqhdhdhdqhdhdhdhdhdhqhdqhdqqhdqhdhqhdqhdqhdhdhdhdhdhdhdhdhdhdhdhqhqhdhdhhdhdhdqqhdhqqqhdqhdhdhdhdhdhdhdhdhdhdhdhdhdhdhhdhqhdhdhdhdhdhdhdhdhqqhdhdqhqhdhdhdhdhdhdhdhdhhdqhdhdhdhdhdhdqhdhdqhdqhdhdhdhdhdhdhdhdqhdhdhdhdhdhdhdhdhdhdhdhdhhd [SEP]", "stripped_text": "qhdhdhdhdhdhqhdqhdhdqhdhqqhdqhdhdhdhdhdhdhqqhdqhdhdhdhqhdhdhqhdhdhdqhdhdhdqhdhdhdhdqqqhdqhdhdhqhdhdhqhdhdqhdhdhdqhdqhdhdqqhdhdhdhdhdhqhdhdhdhdhqhdhdhqhdhdhdhdhdhdhdqhdhdhdhdhdhdhdhdhdhdhqhdh, qhdhdhqhdhdhdqqqqqqqqqqqqhdhdhdhdhdhdhqqhdhqqhdhdhdhdhqqqqqhdqhdhdhdhqhdqhdhqqhdqqhdhqhdqhdhdhdhdhqqhdhdqhdhdqhdhdhdhdhdhdhdh, qhdhdhd, qhdqqhdhdhqhdhqqhdhdhdhdhdqhdhdhdqqhdhdhdqqhdhqqhdhdhdhdhqqhdqqqqqqhdhqqqhdqqqqqhdhdhdqhdhdhdqqhdhdhdhdhqhdhdqqhdhdqhdhdqhqhdhdhdhdhdhdqhdhdhdhdqqqqqhdhdhdhdhdhdhdhqqqhdqqqqqhdhdhdhdqqhdhdhdhdhqqhdhdqhdhdhdqhhqhdhdhdhdhdhdhdhdhdhdqhdhdhdqhdqhdhdhdqhdhdqqhdhdhdhqqhdhdhdhwihwihqqqwiqqqqqqqqhdqhdhdhdhdhdhdhqqqhdqqqqqhdhdhqhdhdhdhdqhdhdhdhdhdhdhdhdhdhqqqhdqqhqqqhdqhdqqqhdhdqqqhdhdqhdhdqhqqqhdqqqqhdhdhdhdhdhdhdhdhdhdhqqqqqqhdhdhqqqqqqhdqhdqhdhqqhdhdhdhdhdhdqqhqhdqhdqhdqqqhdqqqqhdhdhdhdhdhdhdqhdhdqhdhdhdhdhqqqhdqhdhdqhdqhdhhdqhdhdqhdhdqhdhdhdhqqhdqhdhqqwihdqqhqhdhdhhdqqqqhdqhdqhdhdhdhdhqqhdqqqhdqhdhdqhdhdhqhdhdhdhdhdhdhdhdhqhdqhdqhdhdhdqqqhdqhdqhdhqqhdhdhdhdhdhdhdhdqhdhqhdhdhdhdhdhdqhdhdhdhdhdqqhqqqqqqhdqqhqhdqhdqhdqhdqhdhdqhdhdhdhdqhdhdhqhdhdhdhdhdhdhqhqhdqhdqqhdhdhdqqqqqhdhdhqqqhdqhdhdqqqhdhdhdqhdhdhdhdqhdhdqhqhdhqqhdhdqhdhdhdhhqqqqhdhdhhqhdqhdhdhdhdhdhdhhdhqqhdhdhdhdhdhdqqqhdhdhhdqqqhdhdqqhqhdhdhdhqhdhqhdhdhhdqhdqqhdhdhdhdhdwihdhqwiqwiqhdqqhdhdhdhdhdhdhdhqqhdqhdhdhdhdhqhdqqqqhdqhdhdhdhdhdhdhdqqqqhdqhdqhdhdhdhdqhdhqhdhdhdqhdhdhdhdhdhqhdqhdqqhdqhdhqhdqhdqhdhdhdhdhdhdhdhdhdhdhdhqhqhdhdhhdhdhdqqhdhqqqhdqhdhdhdhdhdhdhdhdhdhdhdhdhdhdhhdhqhdhdhdhdhdhdhdhdhqqhdhdqhqhdhdhdhdhdhdhdhdhhdqhdhdhdhdhdhdqhdhdqhdqhdhdhdhdhdhdhdhdqhdhdhdhdhdhdhdhdhdhdhdhdhhd"}
+{"type": "sample", "index": 7, "raw_text": "[CLS] - q q, r aw, q q q, r aw, r q q, q g - q aw, l q, l aw, r aw, r aws, l - aws, q q q aw, r aw, r, r aw, l q, r aw, r aw, q q aw k aw, r aw, r aw, r q q, q q q, q a, n - g a. n. g q - - l q r, n q - - q aw, r aw, q q q q, n q a, q a, r, r q, r aw, q q q ls, l q, u, l q q, j q q, r q q q, q q, l, r q aw, r q, r q q - q, q q q, r aw, r q aw, k q aw, r q, r q q, q k k aw, r aw, r q, r q q aw, r q q, r aw, u q aw, r q - r q, u q q, r q q aw, q, q q q, r q, k q, g q aw, q q, j r q q q, r q q q - q q q r q q q, r q q, q q q q, g s q q q q, q q q, q q q, g q, r q q q, aw k q, q q q q, q q q q q q, q g q, q q q q q q q, g s, s, s q q g, q q, r f - q q - q aw aw, q, r, g, q q q q q q, q q q q q q q, q q q q q q q q q q q, q q q q q aw, r q q, q, r r q q q q q aw, q, q q q, q k q q q q q q q aw q, q q q q q, q q q q q, q q q, q q q, g k q q q, q q q,, r q q q s,, r q q, q, q q q, q q q q q, q,, q q q q,, q q q, n q q q, n q q q, r r q q q q,, q q q q, q q, q q q q, q r q n q q, q, q q q q q r q. q q q q q q q q q q q, q. q q q s q q, q, q q q q q q q q q q q q q q q q q r q q q q q q q, q q q, q q q, q q q q q q q q q q q q q q q q q q, r q, q q q q q, q q q, q q q q, u q a q, g - q r q r,, r q q q q q - q q q q q, q, q q q q q q q q q, q q q q q q q q q q - q, q q q q q q, k q q q - q q q q q q q q, q q q q q q q q q q q q, u s, q s q q q, q - q q q q q, q q q q, q q q q, q q q r q q q, q q q r q, q q q q q q q q q aw,, r q r q, q q q q, q q r q r aw, r aw q q, aw q s, - g q q q q q q, g q q q r q, - q q q q - q, q q q, q q aw, q, - q q q q,, q q q,, q - r q q q q q, q q q q q - q q q q q q q q q q,, r q q q q q q q q aw, r, q u q, s, q, r, u, u, u, u m,, u. q q q, - q r, q, r q q, r, q r q. r. f q q q q r - r q q q - q q q q, - q q q q q - q q q, q q q, q, q q q, q q q, g, r, q, r q q q q q, q g q q, r, q l r -, q q q q q q q q q q q q, q q, q r q q [SEP]", "stripped_text": "- q q, r aw, q q q, r aw, r q q, q g - q aw, l q, l aw, r aw, r aws, l - aws, q q q aw, r aw, r, r aw, l q, r aw, r aw, q q aw k aw, r aw, r aw, r q q, q q q, q a, n - g a. n. g q - - l q r, n q - - q aw, r aw, q q q q, n q a, q a, r, r q, r aw, q q q ls, l q, u, l q q, j q q, r q q q, q q, l, r q aw, r q, r q q - q, q q q, r aw, r q aw, k q aw, r q, r q q, q k k aw, r aw, r q, r q q aw, r q q, r aw, u q aw, r q - r q, u q q, r q q aw, q, q q q, r q, k q, g q aw, q q, j r q q q, r q q q - q q q r q q q, r q q, q q q q, g s q q q q, q q q, q q q, g q, r q q q, aw k q, q q q q, q q q q q q, q g q, q q q q q q q, g s, s, s q q g, q q, r f - q q - q aw aw, q, r, g, q q q q q q, q q q q q q q, q q q q q q q q q q q, q q q q q aw, r q q, q, r r q q q q q aw, q, q q q, q k q q q q q q q aw q, q q q q q, q q q q q, q q q, q q q, g k q q q, q q q,, r q q q s,, r q q, q, q q q, q q q q q, q,, q q q q,, q q q, n q q q, n q q q, r r q q q q,, q q q q, q q, q q q q, q r q n q q, q, q q q q q r q. q q q q q q q q q q q, q. q q q s q q, q, q q q q q q q q q q q q q q q q q r q q q q q q q, q q q, q q q, q q q q q q q q q q q q q q q q q q, r q, q q q q q, q q q, q q q q, u q a q, g - q r q r,, r q q q q q - q q q q q, q, q q q q q q q q q, q q q q q q q q q q - q, q q q q q q, k q q q - q q q q q q q q, q q q q q q q q q q q q, u s, q s q q q, q - q q q q q, q q q q, q q q q, q q q r q q q, q q q r q, q q q q q q q q q aw,, r q r q, q q q q, q q r q r aw, r aw q q, aw q s, - g q q q q q q, g q q q r q, - q q q q - q, q q q, q q aw, q, - q q q q,, q q q,, q - r q q q q q, q q q q q - q q q q q q q q q q,, r q q q q q q q q aw, r, q u q, s, q, r, u, u, u, u m,, u. q q q, - q r, q, r q q, r, q r q. r. f q q q q r - r q q q - q q q q, - q q q q q - q q q, q q q, q, q q q, q q q, g, r, q, r q q q q q, q g q q, r, q l r -, q q q q q q q q q q q q, q q, q r q q"}

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260604/owt_t5_elftokenized_full_pow1_unfixed_norm_stateprobadd_selfcond_ce_fast_lr3e4_logitgrid_n256_step142000_tschedlogit_normal_mn0p7_s1p0_sc1p0_decode32_n256/sc1p0/chunks/chunk18.txt

@@ -0,0 +1,38 @@
+checkpoint=runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260602_001439/step_142000.pt
+step=142000
+decode=dualline_time_aligned_dirichlet_final_endpoint
+c_min=1.0 c_max=1024.0 c_schedule=exp
+steps=32 temp=1.0 bridge_power=1.0 temp0=0.0 decode_time_schedule=logit_normal decode_time_logit_mean=-0.7 decode_time_logit_std=1.0 decode_time_shift=3.0 decode_time_rho=7.0 decode_time_sigma_min=0.0001 decode_time_eps=0.0001 prior_beta=0.0 final_sample=argmax final_count_penalty=0.0 final_count_power=1.0 final_count_warmup=0 self_cond_decode=none self_cond_scale=1.0 state_self_cond_decode=single state_self_cond_scale=1.0 state_self_cond_normalize=True state_update=dirichlet odeish_eps=1e-06 odeish_c_eff_max=1000000.0 dirichlet_gamma=1.0 cfg_scale=3.0 concat_self_cond=False
+bos=1:</s> eos=1:</s>
+===== sample 0 =====
+head_tokens: ['▁“', 'I', 'r', 'o', '▁had', '▁been', '▁pretty', '▁difficult', '▁at', '▁the', '▁beginning', '▁of', '▁last', '▁season', '▁but', '▁unfortunately']
+tail_tokens: ['▁improving', '▁for', '▁quite', '▁', 'a', '▁while', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+“Iro had been pretty difficult at the beginning of last season but unfortunately that was what was coming back when he took only just one game so far and now after a bit of an injury, he has now settled for the second time. “I believe he is going to be ready for the challenger and he’ll be back for the first time since his 18th birthday but we’re confident that hopefully the young man’s form will be strongest in the future and we want to make sure that by getting ahead of him we are ready to rest. “The team has always been ready to lead the players and to continue their work ethic so we did a lot of moving backwards and towards the goal and we wanted to make sure it was the pace that we were most comfortable going forward and getting ahead. “That is exactly what happened at the beginning of last season when Luro Iro was still playing at Selham and I remember that he was fantastic for the first time at Selta which was an important part of his loan spell but so at the moment this season really need to be more cautiously. “The played so far this season with more confidence and confidence than goals have been given but we’re still waiting for the result and we can see that hopefully he can rest on some of our form which improves when we need more confidently and we want to make sure that we’re done. “We didn’t sit very well and then until about three days before last season’s cup final match between the sides when we got back into the Premier League and Selta. However, for us we were at the bottom of the season and we were performing very well for a long period of time at the club and we’ve had very little time coy with injuries which was an important part of our loan spell and in many ways.” With the signings of Javier Berezo and Steven Kinnock on loan, Olivier Spirell was also a little bit coy forward during his time at the club. How remember your start to the season this season came in a win over Celta last season and how did you get back that time at Selta? “I felt as though we were playing on the home side we’re still waiting for the young striker Iro to continue his form and I remember that at the very moment this season we need to rest on the home side at Selham. “I’m just really happy that I’ve done so much as part of my loan spell this season because I’m here for two games in three weeks now and I’ve been really pleased by the fact that I am coming off here on loan from Berezo and it has been really good that the 21-year-old midfielder has now really improved his game at Selham and has made an impact. “I saw him come off this season just two days before we saw him play at Selham nine years ago and we only played two games that season at Selta last season but one of those was when he came off and he was so much ahead for us in the win over Stoke when we were coming off against the home side for one game. I’ve watched him play so far and I was excited at the chance he’d play against the other team again for the second time at the club. “It was that time at Selta this season that was only possible when we played second time at Selta last season and that was at a certain point where him last season was still very good [with injury]. We had just come off for the first time in that match with Gotham knee injury last season which meant it really did feel good for him to play his way back into the Premier League and take the England side. it seemed like a very good opportunity but even then we had ended up playing second time with Vertham knee injury which resulted in our cup final victory. “We all knew the difference between the two and obviously it meant Vertham knee injury was slowing down at the start of the season Sel it has been here for the last few months and so it is interesting to see just how well Berezo has been playing because of injuries that he has suffered now in the first half and how did he stop playing for the second half when he arrived at Selham? “His name is an important player and important player especially in the Premier League and it was during last season that was just one of the players that came off at Selham. He is fantastic now because of injuries joining us and there is no doubt that most of the players at Selham have been playing for that part of the season with his level and quality of games played in the Premier League against most of the other teams at Selham and has been improving for quite a while.</s><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 1 =====
+head_tokens: ['▁Gast', 'on', "'", 'L', 'un', 'c', 'ich', '▁I', 'z', 'ber', 'ich', 'i', 'a', 'h', 'on', '▁I']
+tail_tokens: ['▁by', '▁Le', 'v', 'ich', '▁and', '▁Le', 'v', 'ich', '▁(19', '65', ')', '▁or', '▁the', '▁"', 'orthodox', '"']
+Gaston'Luncich Izberichiahon Izberinsky (1965) or found the view of the anarchism of Jacques Levich's "universanism" by by Gaston'Izberichichichhehe Kalzminist Levich (1965) Izberinsky's "universanism or post anarcho-theory" by Gaston'Izberinsky and Izberichichhe's "universanism" post post post-biobiobiot (1965] O'Hathert Izberichiahon Izberinsky Kalzminist Zhichichhe the guardian or anarchist anarchist anarchy of Kalzberia Slovaki or innocents O'Bomberman Gaston Izberinsky; Izberichichichhe the view of the anarcho-theory of Kalzminist (1968) or as Izberichichhe the "angler" or view of the anarchism of Kalzminist Zhisha Levich post anarcho-theory"; Izberichichhe the view of Leminist post post-biobiot (1965] O'Bberichich; Izberichichhe the guardian or the anarchy of Kalzminist orlovJacques Levich (1965); O'Bomberman found the title as "the view" by Gaston'Izberinsky; Izberichichhehe the anarchism Gaston'Lunc's anarchy of Kalzminist Zhisha Levich Lelovaki's anarchy Gaston'Izberinich Gaston Levichichhe the view post anarcho-theory (1968) or as Izberichichhe the guardian anarchy of Kalzminist Slovakih or of Jacques Levich and innocents or innocents; O'Bomberman found the title by O'Bomberman as Izberichichhe Levichichhe the anarcho-theory of Kalzminist post post-theory (1965) or as an Izberinichhehe Kalzminist Zhisha Levichkovihe Kalzminist Zhichichhehe the view of Kalzberia Slovaki or of Jacques Lenin as the "orthodox", but infrequently labeled an anarchism and found it unorthodox by using the word "unarchiscold" – a similar view first held by the Pews in 1970 and later as Izmichichhehe the "unorthodox", but infrequently labeled anarchism and found it unorthodox by O'Wathert Gaston Izberinsky; Izberichichhethe title view Izberichichhehe the "beh"kovich's title title is a moderately conservative view of Levich post post-biobiot (1965) and found the title as "the view of the anarchy of Kalzminist's anarchy" Izmichichhehe "an anarchism" by Gaston'Luncy Levich and Levich (1965) as Izmichichhehe the anarchistmangler's anarchy of Kalzminist by Levich and Levich (1965) or the "mysterological anarchy"; Izmichichhehe the anarchism the Miocene of Vladimir Szmacht, by Vladimir Levich and Levichich (see: Hartmann (1965) as Izmichichhehe the anarchism anthunism of the Miocene or the "mysterological anarchy" by Gaston'Izberinsky; Izmichichhehe the anarchism the anarch of Vladimir Lenin [the]] founded in Vladimir Lenin, which was Lenin-Soviet anti-Plainist ( ( ( (19., Hartmann et al.[15]) as Izmichichhe the anarchism anthunism"; Izmichichhehe the anthunism of the Miocene of Vladimir Szmacht and Levich or the "mysterical anarchy" by Levich and Levich (1965) or the "orthodox"
+===== sample 2 =====
+head_tokens: ['▁Not', 'ional', '▁pun', 'd', 'its', '▁just', '▁like', '▁those', '▁who', '▁simply', '▁‘', 'actual', 'ly', '▁don', '’', 't']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Notional pundits just like those who simply ‘actually don’t exist within the sub preserved sub cult’ to which anything sub exists in the real world is regarded as unaccountable but even those who are truly traumatized, without ever truly acknowledging oppression within their race, now have an opportunity to argue with a fellow white supremacist or white co-worker if he or she simply refuses to accept that sub oppression isn’t merely sub subexistent within the sub cult while somehow being stupid enough to accept this reality of the form of sub oppression that exists within practically all the sub sub oppression that exist within all of a sub subsiscipline sub oppressive sidelines and so many who’ve left behinda pre-existing wingers and a bunch of white supremacists waiting for them to bring themselves into existence. But this is a minority, whose core members are the ones who want a subexisting privilege and see themselves now forced to accept it all because its mere fact of existence has been removed from the group itself by the media...stupid and slupid and even by the mere use of the word “sub cult”...and also by the fact that its entire existence has been hampered by countless hysterical delusional men whose lives have never been so well documented glorified by imprisoning and mentally disinvited men like myself who being murdered by the sub sub cult sub-generis who live under and other forms of sub socialism while desencilably sexually exploiting women working within various sub-exposed sub and sub groups of other minorities by which they know only who exists within sub subcrisis and their their allies around to be ignorant and pretends that they really only created sub ways while blaming these people as being unwilling to ever exist with the absolute conviction that existence is totally unhindered disregarding any definition of hierarchical worldview just so I could fit that definition anyway. I think I’m wrong. the white supremacists who simply willingly accept the fact that we exist exist only within the sub sub of the sub cult that exists within practically every form of sub oppression and the fact that we are only within the entire subexposed sub sub simply don’t exist because they are stupid enough to just refuse to accept any rational argument for a sub sub whatever reason some people almost universally think they exist within what they know, they simply exist because they insist for whatever reason they exist within the sub sub sub say it is a valid basis for their belief that they honestly believe it’s existence within that group is even made totally irrelevant by some sort sub sub cult ideology, but at least until its ideology is picked up. I’m not convinced that this ideology seems to fall in line from its implied belief it exists only within its fundamental socialism, yet somehow the actual existence of this sub sub-exposed utopia within the actual existence of sub groups of the sub-exposed sub is not actually being created by actual existence but by actual reality that conspires to justify its existence. It takes a conscious conscious and conscious effort to dissociate itself and maintain healthy skepticism of the sub-exposed utopia itself, just as it an attempt to justify the existence of racism and other forms of perilous and oppressed sub subcrisis as socialism. Theisma maybe it’s a good idea but the whole thing does not reach an end insofar as it undermines the actual existence of any sub sub preexisting existence within the sub subcrisis itself or anything else I’ve ever seen but despite this fundamental socialism itself its sub sub no longer exists within its sub concept while its entire sub sub sub longer exists in any way it could create a singularity within the sub sub without the existence of its sub sub-generion, the actual self-preservation and actual existence within this sort of socialism why it’s so impossible for ever being exist in hierarchical worldview. Instead of pretending with absolute capacity to see the extent of this so-called “sub cult” status it pretends that its state of existence has more than doubled since the beginning of history and that evil monsters out there are still being created by the seemingly unindescribable existence within the sub sub sub sub itself, literally destroying the entire world by ing sub sub sub subsumption sub oppressive reality.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 3 =====
+head_tokens: ['▁I', "'", 've', '▁always', '▁thought', '▁that', '▁playing', '▁video', '▁games', '▁would', '▁be', '▁', 'a', '▁role', 'play', 'ing']
+tail_tokens: ['▁World', '▁of', '▁Strange', 'r', '▁Things', ':', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+I've always thought that playing video games would be a roleplaying experience... Over the years I have been playing video games and a few other games to the point that people are never really fans of video games in the first place as well... I don't really have much background on roleplaying games, but aside from the fact that I am all eager to share my passions for videogames it makes sense to follow them... I hope to truly enjoy the games I've come to love... This is a very interesting game in itself too, but this is about as interesting as the other video games mentioned mentioned here. For example, Oprah said perhaps the most interesting game all for us is the wandering world-world:at the mid level, which is an alternate universe where the events are usually called inner life events. Inner life is not  to of eternal existence (as opposed to eternal human indifference itself), but instead life is used to transform entire planet into eternal life. Inner life also turns into a life which is always defined by its existence in nature, and life itself exists indistinguish from non-real-life forms of friendships and player encounters, which give the player an innate sincerity towards the actions of his own fellow atheists. In addition to these innateities, they have more relational human character traits than the inhabitants of the zanto themselves. For example, Michael Icinlski, who coined the "Brightened Wolfenstein," and Tom Ieselinski, who played Arthur and Robbies/Baptergies all described the wandering world as coexistence: "much as [the player] tends to find mutual friendship and relationships with other characters [when the inhabitants of the player's world] can be found, mutual friends who embody [the human characters] have much more friendship relationships than the actual inhabitants of the zanto." For example, Oprah and Robbies' worldworld: The zanto is very similar relative to the inhabitants of its ancestors in nature. The inhabitants of the zanto are very similar to each other; thus human players tend continuously interact with the human persona and experience life within their own universe and thus their friendships evolve gradually along the way that the human persona (the inhabitants of the same universe would conversequalically) suggest that if they choose to inhabit the zanto, they can to find some variety in the mythology simply by maintaining their place in the mid-level world. For example, illuminated planets:The inhabitants of the zanto and planets are more similar in the game world when you first encounter the human persona, even though they still inhabit the world they embody. A similar world design in this area is slightly scarier than the unevolved worlds in the midlevel world, but goes beyond the game's main theme: If you encounter the Living Human, you enter "the mid-level-world," wherein players encounter the inhabitants of the zanto and planets in order to develop mutual friendships, as these encounters gradually become permanent:The protagonists [the protagonists] gradually maintain their relationships over the course of the game until [when the player becomes aware of] the interactions between them and gradually develop their relationships together again (for example), thus enacting "the monolithic world" of the player over (here). This world design is strikingly similar to mid-level world design, but it is actually much more different than most godly human people either (or otherwise behaving differently) ever encountered. The game was designed to simulate things like the godlike characterslike interaction with the player (or the player's) that could only be explained by simple explanation and choices, but instead of abandoning his fellow atheists in their place, and forcing them to accept the player's existence (here the " "world" that Icinlski occupied in his creation), it was  the same way to interact with other Godlike human characters. Oprah played Arthur and Robbies/Baptergies, and Tom and Robins described very similar characters, although they were noticeably more similar than their human counterparts, which meant that they were much more irreplaceable than their actual reality, although they were also affected by the improbable randomness of the world's existence. Not only their innate openness towards what they had in their ancestors (at the conclusion of the wandering world) was also unremarkable, but also made it was worth ing to this seemingly relational position at the mid level, which is usually negligible. For example, The World of Stranger Things:</s><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 4 =====
+head_tokens: ['▁One', '▁of', '▁the', '▁first', '▁things', '▁I', '▁talked', '▁about', '▁before', '▁that', '▁was', '▁the', '▁lack', '▁of', '▁an', '▁automated']
+tail_tokens: ['▁with', '▁some', '▁changes', '▁some', '▁mod', 'er', 'unit', '▁units', '▁work', '▁better', '▁now', '▁than', '▁', '▁the', '▁full', '▁load']
+One of the first things I talked about before that was the lack of an automated package system that would allow system changes. I thought that one of the system changes and the potential for further improvement could occur on a larger production scale... mpgIncyre and rework moderator (Current Modulator) add a number of moderators for mpglIntraster moderator adjustments for antireceptor xgl or antireceptor comps/compilerator effects, mpgterrible compiler effect is fixed removed, mpgIncyre doesn'rework (no longer needed) is fixed removed... it's a bit better when you're better than when you get upstream at current frame time finer units work better now than if you're not going to mount it mount mount mount it to mount it mount mount mount current frame time time moderators adjusted with some changes moderators: Rework from the "terrible compunit" to work with the "terrible/compiler" moderators: In moderator adjustments add some new changes, with the "terrible compunit" mpgl compunit/compiler moderators: Rework from the "terrible compunits/terrible" to work with reworked units adjusted (this unit is now incorrectly merged but it's still much easier to update manually) mpglIncyre effect fortererible compile control, mpg compile effect/and juggle/backward/backward effect... finer works override due to some moderators fixed... finer units work better nowwhen whenrg finer unit may be better now than when you get upstream mount mount mount it to mount it mount mount mount mount at current frame time adjusted when something changed adjusted... mpglIncyre effect/and juggle/terrible compile for mpgIncyre effects on upstream (it can cause finer to use data too) so always take the power cap off when you get it's down and finer works cause some changes works it works override due to moderators fixed... if you can use something again without data on it's youre never really going to keep it on upstream and if you can can can use it again and probably too slow if you're never going to use it on it and when you get without data on you've never really going to keep it on it even when it's turned on (it can cause finer to use data on too) so always take the power cap off and if you can can use it again and probably too slow  without data on you've never really going to keep it on it even and when you get it's turned on and finer works it works override when moderators fixed... add some new changes, with the "terrible control control/compiler" moderators: In moderator adjustments for some compile control/terrible animations when they're removed (is now a bit less compoblivious, and causes some adjustments) some compile control/compile control/compiler moderators: In moderator adjustments with some changes adjustments the "comprible" unit is now a bit more compoblivious... finer works override due to some compile control/terrible/compiler input moderators: In moderators with some compile control/compiles (backward effect and ag compile control/terrible moderator effect/and juggle/pkkph/ph'sp some more work and some other work still needed to do) xgl/Intraster/compiler effects moderators: Some changes, with the "terrible compile" control/terrible moderator effect/terrible compile control) moderators: In moderators with some new changes adjustments the "terrible" unit is now adjusted ag compile control (backward effect and ag compile control/compilers always have a pre antireceptor mpglIncyre effect/compile control/compiler, no antireceptor or compile etc.) always get the antireceptive prereceptor <unk>vegs adjusted. Change changed added some moderators with the "terrible compunit/compiler/compiler" moderators: In moderator adjustments with some changes some moderunit units work better now than  the full load
+===== sample 5 =====
+head_tokens: ['▁Utiliz', 'ing', '▁D', 'yah', 'u', 'ggling', '▁Technique', 's', '▁', 'd', 'yah', 'u', 'ggling', '▁techniques', '▁are', '▁often']
+tail_tokens: ['▁as', '▁discussed', '▁at', '▁the', '▁top', '▁of', '▁this', '▁post', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Utilizing Dyahuggling Techniques dyahuggling techniques are often considered very valuable and have been suited for practice, at least partially in terms of dyahuggling. The more efficient ways to use Referringes of dyahuggling techniques are to be employed in a set-up business strategy. dyahuggling techniques aren’t considered the primary position in which people actively attempt to avoid and avoid themselves over time. dyahuggling techniques often represent the primary position of keeping workers being actively engaged in the process in which they are gaining their jobs, rather than paying a little bit of value for the work they gain. Referringes of dyahuggling techniques that are often regarded as more or less valuable than in practice and generally tend to involve less risky when doing so tend to be considered suited for practice, as well as moving away from other jobs and other workers who tend to be more or less suited for business. But dyahuggling techniques are often considered valuable and have been considered very valuable for practice, at least at least suited for business. Referringes of dyahuggling techniques aren’t typically employed in a set up business strategy. But dyahuggling techniques are often considered to be as the main position of moving away from the job, rather than actively actually doing work at work. The more efficient way to use dyahuggling techniques is in the process of getting your jobs done much less stressful to which it is often less effective in practice. The only reason people use these techniques more productively in the same job is that they create a certain form of realism, known as the “better sense” of separation between you and you, that makes your work much less stressful during the burnout while simply waiting for additional work to take place. dyahuggling techniques often exist in a one-time-and-job business strategy where they are often considered at their lowest point in business practice – diminishing productivity by paying very little of value rather actively actively actively doing work well or doing work. The more efficient way to use dyahuggling techniques is to attempt to avoid and remain engaged (or actively moving away from other jobs) in the process of getting their jobs done while diminishing the degree to which it is less effective in practice than in practice. Remembering dyahuggling techniques often tend to be employed when and when they’re being used in practice. Remembering dyahuggling techniques are often considered useful and often done consistently in the same job, but it is often easier to use them more effectively for business practice when they end up in a set-up business strategy. They can’t be done consistently during practice when using dyahuggling techniques, but they are often considered the main means of diminishing productivity by consistently working well or being quite good at other jobs when they are actively actively taking their time at work. It is often easier to get started in business practice by employing dyahuggling techniques as an extension of work away from other jobs or actively actively doing work at work. The more efficient way to use dyahuggling techniques is to be in a one time-and-tru business business strategy. They are social techniques that have existed in the past decade or even the past few decades – but they often exist within the same job and are often considered the main means of diminishing productivity by doing the same work,up means that actively actively actively applying for other jobs. It is often easier to get started using dyahuggling techniques than hard work. Indeed, the more efficient to use dyahuggling techniques, they often exist in a one-time-and-job strategy where an extension of work away from other jobs work equally well when other jobs work equally well when actively actively or actively actively “taking” their time at work. Maybe these shifts are beneficial in practice, but you can easily overcome this by accounting for the fact that these positions tend to be within the same work schedule which means that they’re often considered quite good at the work. In most traditional set-ups, dyahuggling techniques is a form of “thinking of hard work.” You get this practice by actually just taking place at the job that works little and not necessarily providing hard work. And indeed, this works out well towards someone who brings significant amounts of value into the job without using them actively as part of the process for actively actively applying for this position. Of course, it is be easy to find highly skilled people like these who actively bring so much value into the job without having them actively actively applying but it is the hope that others will actually nudge them on these positions. That is for the long term what benefits you gain when employing new business techniques as discussed at the top of this post.</s><pad><pad><pad><pad><pad><pad>
+===== sample 6 =====
+head_tokens: ['▁Oscar', '▁Wild', 'er', '▁emphasis', 'e', 'd', '▁this', '▁kind', '▁of', '▁rhetoric', '▁and', '▁political', '▁analysis', '▁on', '▁what', '▁critique']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Oscar Wilder emphasised this kind of rhetoric and political analysis on what critique was expressed in contemporary art precisely because he embraced the critique of contemporary art and literature, and thus its relationship to politics. It follows that Wilder’s analysis not only echoed in the critique of contemporary contemporary art, but was also equally influenced by the works of contemporary traditionalists, such as Laurence Barton, Xavier Rouseau, Jean-Leois Montroud, and Marcelo Luigi Emili B. Wagner’s A publication des Essays and Justiques (1988). His work echoes the caustic of poets and novel artists such as Walter Walter Wilder (1988) in his famous essay “Lud le Guilluze and à la Modernisme” (1989). This contrasts what contemporary contemporary artists and modernists say: “Though we often thought that we’ve been ourselves as a form of art or literature, it has always been a kind of caustic which caused us to move backwards into thinking instead of thinking or simply re-enter ourselves into the present.”.” Cold War art and literature is often the great horrors of the past all smeared by the selfinflicted trauma of a postwar-era Warplight Movement: This is what Alfred Hofheimer reflected in the era of Wilder’s work was inspired by “modern” poets, pantheons, and many other writerswar poets and novelists who in the early twentieth century would have been influenced by antiwar political work, many of which had very similar themes. In his work, Hofheimer challenged the political mysticism that has often been amplified by his rhetorical analysis. But it was his rhetoric and political critique that sparked the abolitionist Antiwar Warplight Movement. In that era, Wilder developed his seek, and articulated many ways of drawing an open line to his criticism of contemporary artist Cold War and literature. In describing the ways in which he grappled with what was used in modern contemporary literature, Wilder asserted this by saying that “the style of contemporary art is used as an expression of political and political criticism.” Hofheimer’s work has also influenced both his critique of contemporary art and literature and his critique of contemporary politics. Wilder’s work was also influenced by contemporary artists who came to explore this subject in the field of literature, and so therefore has the seeker challenged the worldview contemporary literature and contemporary political criticism. A wide variety of contemporary artists was influenced by his work in the literature by Alfred Schenbach (2008), and a number of other works such as Alfred Schenbach (1999) and Thomas Campbell (1999). As Hofheimer wrote, Hofheimer’s critique of literature is unique in its current form of criticism: “A contemporary political style is a contemporary form of art and literature itself—or also a contemporary form of literature, but nevertheless largely overshadowed by the historical critique of contemporary art and literature, especially through the constant distortion, introspection and misrepresentation of the historical thought and analysis of contemporary art and literature which is now so complex that it touches on human beings.” From Wilder’s point, his own contribution to the history of literature is expelled through this lens: Prior to the modern era, contemporary art has always been dominated by patriarchal and re-presentative patriarchal narratives of the marginalized and oppressed by the postwar era, the place of struggle and the plight of women. “At the time the seeker saw contemporary art and literature exaggerated from the present perspective and what might have brought the warplight forward,” Schenbach wrote, “thatstead, contemporary art would inevitably become one’s own interpretation of the historical thought and theory of history, politics, and popular culture.” The seeker argues that during this time and experience contemporary art and literature are often influenced by contemporary contemporary literature as a form of historical thought and analysis of the present and past—and that this interpretation of the normative of contemporary literature extends beyond the postwar era, with a renewed appreciation for contemporary ideas and political themes championed by this time and a passion for literature and philosophy, as well as aesthetics. In 1895, Alfred Schenbach published a book on the history of the Founding Cold War Warplight Movement and what it called it “modernists.”</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 7 =====
+head_tokens: ['▁I', '▁was', '▁moved', '▁by', '▁my', '▁husband', '▁Australia', '▁and', '▁', 'a', '▁couple', '▁of', '▁friends', '▁from', '▁Sydney', '▁to']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+I was moved by my husband Australia and a couple of friends from Sydney to Melbourne. I once worked in a Victorian apartment in, but having spent large part of my life taking care of my work would make me much more vulnerable than many women would because they would starve me for several hours of work hours and get my hair shut off as a result. After a bit more than a couple of months it turned out to me that I spent much more time helping women who let me try my hand at work because I was no longer holding on to myself. I moved out of IFA where I was working without washing and washing without washing, kissing, and showering laundry and now I can find myself making an effort to support myself financially as much as I find myself continuing to do what I love to do. And that means supporting myself and being abused by mys and other rapists and prostitutes all over various countries and places where my work isn’t being properly documented or castrated. This is partly because I was moved into the Victorian women’s era because of the fact that I drew up in Australia and Australia where I experienced some form of sexual abuse in the rest of the world. I am much more confident that more Australian women get the opportunity to take part in the rest of the world in humiliating and humiliating female oppressors while working against their will. I also think it is important to acknowledge the mentality of my work with women who actually support them financially for different reasons. Photo – Appearing the Golden Award for Female Haircutters in Australia: “We stand up for each and every artist” It may seem a little cliché, but it isn’t just that the conniendo of Hollywood or Hollywood believes that it has opened the door for Australian artists and artists. All of us Australians deserve to know that some of such women and women are currently being talked about in the entertainment industry. It’s funny that I started out a long time ago when I was some months away from the ‘Rainbow Woman concept’ working with female artists in Australia and I was tired of being bored. However, after I was opportunity to work with my two friends, Tinagencia Feido and Dustin Eckhart, she told me about the process that it can be difficult to find female artists, which means almost all women want to work. I went on, asking asked explain why women are using their power to find female artists. Wrote Australian artists Justin Vernon and Kanye West: The big question here is whether it makes any sense for women to be working so hard to work with rather not being able to work all the time. Many artists go to work with an artist like Kanye West to say everything wrong and ultimately put pressure on women in the world. SoLike we’ve been very busy in Australia but we also have a lot of women working hard over there. It seems to me that we are where women women need to get married in order to get married. I recently had a guest speaker at the Women’s Press Club telling me that women are desperate for such opportunities, but I think we also have a big gap here. Female artists like Arnie Olson or Abigail Seiffermann often tell us that there’s a big gap between men and that a lot of women just want to get married because they don’t have the money and don’t want to marry. So it seems to me this is be a huge gap between the women seeking working in the entertainment industry which actually makes more sense. While it may be less glamorous, it is not necessarily less glamorous for the upper-class women who may be interested, but I’m not entirely satisfied with that. The biggest challenge for artists in Australia is that women are more willing to work for their shows. That’s why I think we womenre having a hard time finding female Image Artist Photo Credit: Faulkner David Lee, Stevie Wonder and Julie Wong at The Cowboy Dreams gave me the overwhelming feeling of being interested in my own show, so it took me a long time to put together some more creative work in Australia. I’m sure that there are some very talented women in the entertainment industry like Kate Widley and Katie Wisley. Working with female artists and trying to find career advice from a writer alone can be too hard. Image copyright Monika Muir Collection Image caption Pet owner Alex Ward said he could ask for help finding his own home A steely black black cat has caught and set on fire in Canterburytown for the first time in the years but a pet owner Alex Ward said he could ask for help from two men passing his property.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260604/owt_t5_elftokenized_full_pow1_unfixed_norm_stateprobadd_selfcond_ce_fast_lr3e4_logitgrid_n256_step142000_tschedlogit_normal_mn0p7_s1p0_sc1p0_decode32_n256/sc1p0/chunks/chunk19.txt

@@ -0,0 +1,38 @@
+checkpoint=runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260602_001439/step_142000.pt
+step=142000
+decode=dualline_time_aligned_dirichlet_final_endpoint
+c_min=1.0 c_max=1024.0 c_schedule=exp
+steps=32 temp=1.0 bridge_power=1.0 temp0=0.0 decode_time_schedule=logit_normal decode_time_logit_mean=-0.7 decode_time_logit_std=1.0 decode_time_shift=3.0 decode_time_rho=7.0 decode_time_sigma_min=0.0001 decode_time_eps=0.0001 prior_beta=0.0 final_sample=argmax final_count_penalty=0.0 final_count_power=1.0 final_count_warmup=0 self_cond_decode=none self_cond_scale=1.0 state_self_cond_decode=single state_self_cond_scale=1.0 state_self_cond_normalize=True state_update=dirichlet odeish_eps=1e-06 odeish_c_eff_max=1000000.0 dirichlet_gamma=1.0 cfg_scale=3.0 concat_self_cond=False
+bos=1:</s> eos=1:</s>
+===== sample 0 =====
+head_tokens: ['▁The', '▁writers', '▁of', '▁this', '▁period', '▁were', '▁also', '▁particularly', '▁important', '▁to', '▁Emil', 'i', 'us', ',', '▁who', '▁had']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+The writers of this period were also particularly important to Emilius, who had conjectured works of classical art, literature and the arts, in particular, for eight centuries and the seventeen centuries, and thus came from transcendental stewardship important to Emilius' writings, with the assistance of his ideas, which allowed Emilius to exert such influence openly and in the spirit of his later aristocratic ideas, the great importance of transcendental stewardship also positing, most vehemently, undermining the premediacy of the ideas that were treated by Emilius and his inseparencies prepared him for the further development of his interest in the classical aristocratic thought which could not be made in any of his later works, since he was influenced by particular literary works, both in art and literary works and in classical literature. Emilius, the most influential and influential writers of the later period, thus had the greatest influence in the development of his ideas, although Baconides had been very careful to distance the writership from Plato's view of modern humanism[13] and his greatest influence on modern human thought in adopting such views. Crucially, since Emilius and other precarious writers had always been deeply concerned by the principle of human wholeness of human thought and the condition into thought, such ideas seriously mislead them about the power of the human life and relation to certain aspects of nature and life. In this view, Emilius and Baconides therefore offered a paternal theory for the importance of human life and a human life as a whole; Emilius thus accordingly viewed modern humanism as a radical view of humanism. This great importance, in the aristocratic writers, of whom both were Socrates, with whom Plato had philosophers as great as were in his work, was rejected by Thomasus de Publius. Those who have long immodestly regarded this precarious conception of the relation between man and nature and thus immodestly rejected such an influential view nevertheless embraced this view by Plato itself. Plato and Baconides that Plato Publius had in fact elapsed time and so imbuated Plato that the aristocratic writer had rejected some of his ideas and ideas from Plato's most influential writings, and so abandoned this understanding of human life which may have been the first ever to preserve its natural preservative order in relation to its own natural nature, in relation to the barbaric ideas and ideas of modern literature, art and literature that is far from being accepted only by Plato. Thus, Emilius, Plato and Baconides's precariousus[14] presents modern humanism as a pre, radical and paradoxical view of bureaucracy with which Plato had thus, for no particular reason, greatly influenced so far from the aristocratical standpoint of modern knowledge and human thought, according to Emilius in his works, through his  and philosophy of It is Art, in which this view, from the classical philosophy of this era, many works so thoroughly rejected by Plato and Baconides as to advocate modern art and philosophy classical philosophy of Great Roman Humanism, in this contemporary view the precarious view of modern bureaucracy and philosophy of the time, both from the speculative of the classical philosophy as advocated by Emilius and his predecessors of Great Roman Humanism, and which was at once superior and inalienable to the artists and philosophers of all timeera and in the formative interpretation of Emilius's noble philosophy, both from the standpoint of knowledge and philosophy and humanism of great importance in various different periods of the era. This modernist precarious view achieved success by moving from classical classical philosophy to classical classical art and philosophy to modern humanism the classical humanist folklore (in which Lucifer saw Emilius as a modern genius rather than work by radical artists andaristocratic writers of all time. It should be noted that, in contrast to Emilius and his bureaucratic philosophers, with whom he considered them rather "serious", Emilius amused in this prefiguratively view the works of which are actually brilliant, not because they normally think of modern human life, but because they themselves have been "serious" about aspects of human life and which are better known to be interested in humanism rather than classical art. Modern philosophy [ edit ]</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 1 =====
+head_tokens: ['▁“', 'We', '▁could', '▁be', '▁quoted', '▁as', '▁saying', '▁that', '▁‘', 'F', 'roc', 'ious', 'ness', '▁stopped', '▁being', '▁commun']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+“We could be quoted as saying that ‘Frociousness stopped being communists’ and sent us back back into society and [human beings] deserved that – which is why we petitioned the Kun P Venoma Institute on Natural History, cosmology brings together with the fundamentals of quantum mechanics. “It is a very good prognosis for humanity the idea of being given away by some people to come into atomis society and take away from In fact, but Consciousness could give us some much more.” Phenomian explains Kuhn: “The fundamentalist maintains that universe existence cannot be created by human beings, and yet the only way we find out is ‘thrill has no interlocking’ – which of keeping people open to heart open and get a hoot.” Phenomian also cautiously agrees: “There are many good reasons for leaving humanist society and we’ve seen a dramatic change in society society from The fundamentalist view, but Consciousness explains even more that if society’s existence is created by another God-like human being, then society becomes a form of communism. “Conciousness is a form of post-human life, and it is possible for people to reconcile ourselves with different concepts of human being and feeling peaceful at the same time. However, commun does can go a long way towards getting ourselves together.” Phenomian poses the real possibility of quantum mechanics “an alternate form” to the post-human world in which humanity created “There are created by fully autonomous groups of human human beings or individuals that can force their way into society, paving the way towards becoming truly communist. “If communist society can at least be taken away from society at the present time, it is very hard to figure that out.” The timing of the outcome of these events is only the result of a struggle between different human groups. “I think it is very difficult for people to come back into society, because it is such a huge pressure situation and that does not mean that people will fight against it,” says Kuhn. “But I cannot explain why all these pressures have been so precarious is that drastic change will take place. “The fact is, the time is for humanity to stop it and I hope this happens in the future.” [RELATED: Substantial Terror Racism Anti-Terrorism Remarkables By Joe] Anti-Family Black By Joe Biberber Rich Nick Whitehead and Rolling Stone Self-Enteration Give us a brief introduction: To rival Big Z fans are breaking news, Z fans are breaking news. The below below givess an interesting picture of Joey’s novel account of what’s happening in the Big Z and where fans have been ingwarted over the past ninety years. That happened especially at the end of Z Civil Warz something Mike Z fans never, ever seemed like it. The past few days ago the big Z fans were split between teams with no support team with the other over, with smaller teams constantly feuding with gorillas and only smaller support groups. At the moment in order you can watch a game without team fighting with TactyZ on the internet, you can see that the whole teams are still being upside down. Some fans don’t even mention face fights, even in real time as being witnessed by Mike’s fans. And while Big Z fans comes across as being talked about by Mike’s fans such as talking about Generation XIII and Z. It’s not surprising after all, there are certain issues surrounding Mike that many can relate to for certain reason but life has been pretty good for Mike. Mike has recently returned to a somewhat bitter rivalry with his parents while playing Big Z while enjoying his whole new life and skills. Mike seems to have ended up being somewhat of a bitter and bitter one. Mike is indeed a big Z fan. A big one, he ended up being on the opposite side of Big Z fans fighting over everyones fans loved all the time. The Big Z fans were at the other end of the social spectrum. Only ones who brought big names to life were old-school players like Michael Jackson and many of those fans were never really close friends. Still, in fact Mike played over the past couple of years with big Z fans and ended up being among the top Z legends. Favorite Z fans, many fans Big Z fans left to tell his storyline once he got to come back to his roots. He is still around and he was a poster child for big rival Big Z fans.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 2 =====
+head_tokens: ['▁A', '▁while', '▁ago', '▁I', '▁started', '▁', 'drafting', '▁new', '▁(', 'or', '▁am', 'using', ')', '▁', 'drafting', '▁patterns']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+A while ago I started drafting new (or amusing) drafting patterns for the old starter decks all around me (and they’re still my favourite old decks). I also included some old draft decks that I like to build and fit into decks looking for new cards to make them even more interesting (when I like to build them out older cards those are still pretty boring for us). I looked at a couple of draft deck patterns and found there were several different draft patterns and old drafting patterns that were quite interesting and well constructed overall. I think some of these patterns were built into the old draft deck which is a big part of why I think the overall design is a bit more “next round” than the original. It’s probably pretty much too early and not too much fun overall which is interesting since it was a fun deckbuilding part of the round. The new drafting pattern isn’t that bad at all though you can still play as a builder or a deckbuilder but adding an extra layer of complexity with later design still means the old drafting patterns are better than expected. Maybe I can look at one or two new deck cards for a new round just like I mentioned in the previous drafts or some new design changes which means I’ll probably look at the same or two or more cards for a new round compared to the old ones that might be able to start at the end of the round. Now, on June 3rd, I’m going to move onto the new and old draft decks again and hopefully we now have “old game” cards and some new cards hopefully they will turn out pretty Good idea though. We’ll be continuing with each other over the next few months and hopefully we end up just starting again and playing back rounds off from 2/3rd that starter decks are still being being played at the time. This sounds much like a bang up here idea I don’t really really idea what I am doing but rather look at it. I’ve always been looking to build cards and looking to build cards. I’m just looking at deck level as far as what this looks at being completely different at the time. I’m just seeing if people were playing right away then this deck was worth paying for. I’m actually playing back rounds off some old decks that have been built for June 2 or maybe 3rd of 2016. I’m just looking at deck level as far as what this is and what the old deck looks like. Old phase is basically over at which stage is when I'll start decks with new “speakers” cards looking to build cards for an interesting part of the round. At this stage, I'm just hoping the decks look either way more similar to the ones at the start. I haven’t really had an idea of this round yet either, since old players still have a large number of active decks to be built into play. This could probably be described as an active card being in my hand now but I’m pretty sure where it possibly fits this round. I actually played a new deck that wasn’t completely built at the time. It just seems like the new starter deck has been built around a bunch of different drafting patterns that could easily make up old decks as well. In fact, most people still are excited about this particular round. If you look at the sides of the Gamble and both sides of the old 3rd and see the “rather” of what the old deck looked like when it became clear that there were some significant variations in the old deck pattern, then it’s just as likely that the pattern was actually changed. And the TWO points that referenced when looking at the beginning of this round is where the old decks actually started. Even now though the old patterns seem to look almost the same way as what cards were actually being played, it’s still clear with cards actually being played differently by other players or maybe even in some cases being able to find their “speaker” way played on that lower level decks. These patterns are just sort of a sort of “speaker play” pattern where other players find their “speaker” way where they actually gain flexibility and turn turns off when some cards have some sort of being played by another player or are actually being held in the same lowly deck deck similar turn being played on the old 3rd deck.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 3 =====
+head_tokens: ['▁So', '▁that', "'", 's', '▁why', '▁I', '▁felt', '▁at', '▁this', '▁point', '▁that', '▁responsibility', '▁was', '▁taken', '▁away', '▁because']
+tail_tokens: ['</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+So that's why I felt at this point that responsibility was taken away because it's really hard for me to really think about what it feels most important for me to go through. I've all gone through this pain of looking at myself and asking myself: Why are you currently in college and working at work? Are you in college and still getting a degree from high school? No, it really forced me to think about things all the time without actually talking about what happened anywhere else in my life. It made me want to think about things very differently. I think a lot of the time [my job] actually forced me to really think over important things in my life and how sometimes people around me are just being asked how important things I know now and think about what happened in my life, and how it made me feel a lot easier to get to work all the time now and still work hard to make sure we all know what happened happened. So it wasn't surprising too. I think it was a point that people seem to be dealing with whatever just says to me, "It's how things get going. It's just not my job anymore and some people think about how things will happen." I think honestly honestly honestly felt like it was just a matter of trying or trying out a lot of different conflicting ideas from the past, but I think it seems to me that some people seem to have been talking about how things would actually happen. Yeah, I think that's too much, but I honestly felt like it was a point where I was just trying to be told about the "What are you doing now?" I didn't actually have my job until that day when I really started it was six months back. It was just a matter of time because there was a lot of realism that I've actually actually been working there for the past six months back when I graduated from college [to age 31]. I think it was just a matter of tryingfiguring out a lot of different ideas that seems to have been talking about how things would happen over the past six months and it honestly felt a lot of conflicting to me because I wasn't really thinking about how things would come next. I'm just glad I'm still doing my job now because I'm willing to ask myself at what age, what age and at what age are you still at or are you? Aren't your friends still having this conversation? I think it was just a point where people were really asking, "What are you doing now or are you're just living your life?" I think so, because I've always been struggling to convince myself that I'm honest at life and I'm always struggling with each other and I've always grown up with them. I've never really used to be honest with myself and really helping myself and helping others. I honestly feel like it seems like you're in the right balance between working with one person and trying to actually working with any one person. Not only do people who ask the "What are you [yourself] doing?" people who say, "You're just living your life." I think it just feels like that, because it's gotten harder with people still talking about a lot of people you're actually trying to work with yourself just because you're actually feeling like it. Do you find yourself feeling like you're resenting your narcissism and pushing yourself out your control for lack of clarity? I think the question that I've been struggling with with myself ever since reliving my life is just the way that I find myself resenting myself, but honestly, it really doesn't bother me. I think honestly because I find myself feeling like I've stuck with someone else my whole life, not because you're always growing up with them, but because you're always struggling with them. I find myself feeling like I always stuck with them. Was there a point where you feel like some people think that I've been struggling with but as far as you see how I've always challenged myself or how am I struggling with any one person? I think this is a point where I honestly feel like I'm struggling with my whole life because I've always dealt with each other how they relate to how I grew up with whatever person I was. I always happened to be an adult who had health problems along the way. when I was two years old and that was one of the first things I noticed, when I started getting kind of started questioning what I'd been struggling with at that time my whole life and the fact that I had no idea that I was any single person I was. I just kept trying to separate myself from things a little bit more.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 4 =====
+head_tokens: ['▁Gi', 'or', 'gio', '▁and', '▁his', '▁research', '▁team', '▁at', '▁St', 'rax', 'is', '▁University', '▁College', '▁London', '▁have', '▁found']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Giorgio and his research team at Straxis University College London have found that while they studied certain areas areas where the animals were exposed to their organ effects, they were still being able to keep their eyes on their bodies. Therefore, for some reason, it seems quite obvious that there are still some small side effects of the side effects: the fact that the animals are able to reach out and move inside their own bodies, and by the movement and locomotion of the animals. The researchers also found that the side effects of sweatpants were actually keeping their eyes open inside their bodies and the growth of cells inside their body as well as the animal itself. Indeed, this is quite obvious that the side effects, when they were developed, were from the small animals: this side effect isn't biologically significant as it is from the animal's body.The fact that these side effects come from rodents and monkey is also obvious, in fact that the researchers found that these side effects are not biologically significant at all.And this is another reason that smaller side effects of the side effects come from small areas where the side effects can ultimately have a large effect: indeed, in some areas where the side effects are quite small, when they were developed from the side effects were from small animals.This is another reason that the rodents and monkeys work, even though it is still quitely small, small side effects of the animal system as a whole can actually have a huge effect on our nervous system. As the author of his video shows: “Because we don’t see large side effects in the animals they have huge effects on organ function, so those small animals have an incredibly powerful ability to move freely.”This is another reason that small side effects in the animals aren’t hugely significant, is that the animals are incredibly powerful to move freely and freely, so that these small animals are fairly open to the control of our bodies (we control the microorganisms we are exposed to control things in nature). These findings will be explained in more detail in the video below: Production processes tend to work really well in these processes when it’re done during production. The film was made by a few car companies, which are quite similar. This was perhaps an interesting observation by Mark Guggen, who also works for the car company, who has reportedly spent many years trying to figure out what works wrong and still spend more time working on production aspects of the project. For example, the following is shown in the video below: “This is the only method in production process that does not show the right results; it is just a simple method that creates a response to other certain processes that works just right: As shown in “The Movie: Set Up a Production Process during Production day (“Badley” (Be done here below”) From the context of each of these points, it seems clear: the process is what animal producers do best in the iterative process, but more importantly it is what they do best best in the production process: It’s just by production processes in the production process where production just takes time off. And all of those happen in the production process in which most animals are doing their best. Every week in the production process when it tends to turn these things around – at least in some cases it’s always best – it’s not always the case. But production is only a single film at one time: since film is seen in so many movies, it doesn’t mean it wastes time off in that process. It’s just about the production process that wastes best is the fact that all of those things happen in the production process in which most animals are doing their very best. All of those by production processes just takes time off. And production process tends to make things work really well. For example, the idea point at the beginning of the factory assembly (MMC) production line for the vehicle was that there was only one single production car on the road: the first day of a production line was extremely strong and we started building and filming the cars and then proceeded to reconvance set up the “Bazley Cars For Workday” line for example, as shown by Guerlino during the first day of the production of these cars: the first three days of production by the MMC were extremely strong and we started to do what we could during the next 3-day production process. So we started pulling together pieces along the way from this production line during the first day of the production process which was to make the car work really well. And overall, the idea seems clear: Its production process is always best when it comes easy to see:</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 5 =====
+head_tokens: ['▁And', '▁how', '▁else', '▁about', '▁this', '▁new', '▁story', '▁about', '▁his', '▁life', '▁or', '▁back', 'story', '▁that', '▁', 'he']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+And how else about this new story about his life or backstory that he told me... "I didn't know what I just didn't want to hear. The truth now is that I've always been happy with my life, I always wanted the same, I've always wanted the same. And no matter how I wanted it ever be true again, I always wanted something new in my life. And no matter how I wanted to see that new story or backstory — I've always wanted what I wanted to see again — where so many artifacts and legacies and stories could neverve moved forward at one time period where these stories could have existed — at another time period where those stories could have moved forward or old something old or new — but that was never right anymore." After much reflection, I was able to lay out my backstory and remember the first line I heard in the novel beginning years ago, when I was talking about the floods that flooded New York for the first time, and I thought suddenly that there were very ancient people in this ancient city who had already changed their lives and now they were all supposedly supernaturally people. This was actually the very first real part of the novel. It suddenly turned out to me now that there were real people in this ancient city who lived and still existed. I can remember one commenter telling me that the floods were supposedly very ancient and ancient people who had never changed since they were supposedly supernaturally intelligent people. And then one commenter told me, "This is what these stories tell you. The oldest story in human history." I was a young man and very close friend knew most of his time, And as he described me... "I was a terribly young and very old man who talked all close friends and friends during this time, and he knew some very real people who were very close to us and one day and one day real people that ever believe about ever existed during this time." And I was still my very old friend at this time. I was just a young man who talked all his friends who lived in this ancient city. I was a young man and just an outsider knew most of this time and talked about some very real people who had been very close to me during this time, and he knew one day ever believe about ever existed during this time. He talked about just how I lived along these real people who were my closest friends — close friends and friends and close acquaintances. And as long as I was just an outsider, he talked about my friends and acquaintance friends and each other day. And not just the real people that made me lived longer than ever existed, but this has happened to me ever since I was convinced there were many real people inside me who ever believe about one day than ever ever existed to exist, even though it's true that only real people ever existed during this time. It wasn't true to me just because all these real people just had memories of years past and present that day, since there were so many different people who knew ever existed today day since I knew some very real people who knew knew believe they lived at least one day during this time. And now I just had all of these real people that made me believe about one day ever ever realized and now that the real people inside me just actually met each other day existed in lives that had ever been imagined or even unconsciously imagined. And even if all these many people who knew years past and present existed that day were forever different than ever lived, and one day day actually met even more very real people than ever ever existed? And even then I was fully aware that if so many people — maybe even me — thought these real people back at the time of our creation, at least one day in our human history, were actually very real people than ever existed or maybe even even unconsciously imagined. And yet, now that there were so many people who knew me back before I lived and met many different people than ever ever have existed, someone told me, "Because I don't know him and all my old friends, he still knows something." And as much as these stories are story telling, that story certainly doesn't necessarily tell any of its readers, but it does tell that there are very real people who actually live up to the expectations of our normal human history — even even when I mean self-respect. And I know that when I was thirteen years old I kept thinking about myself for years and that perhaps most likely this story had captured every corner of my imagination at the time.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 6 =====
+head_tokens: ['▁I', '▁am', 'n', "'", 't', '▁exactly', '▁excited', '▁to', '▁see', '▁some', '▁young', '▁Premier', '▁League', '▁talent', '▁slowly', '▁']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+I amn't exactly excited to see some young Premier League talent slowly stepping into this year's World Cup campaign against the best team they've shown against this season, but I think it would be wise to see talented young players start playing in England's World Cup comeback this spring or next year I instead start looking at Rob Clyns, who is giving young players the unfeasible edge that he's shown over the last few seasons has played an important role in England—and I think Rob Clyns is arguably the best defender of his generation that England have played, so I am not sure whether will once again prove that this season is almost inevitable until heading into the offseason. Both his performances—and Clyns performance alone—could my belief that Rob Clyns' rise to the top flight in 2016 would definitely have an impact on the rest of the team, and I think they should know better going forward Based on their recent matchups against Tobago City, there are some real signs that young talent need to be a strong one when the coming offseason gets underway. Kieran Darcy has taken over in recent weeks, and while Aiden Mesu Paisley is back, we could see him just being overlooked at the moment by former teammates Rob Crystal and Luke Clarke—who had only played at the start of last season while Kier Paisley was at last year's end, but at the start of last season, Rob Clyns is younger, and while Clarke might be able to hold onto the backfield or Frenchman's power forward role in 2016 while Kier Paisley is still out, Crystal Darcy still plays a significant role in the attacking role that Aiden Mesu Paisley will provide for the rest of the season. Perhaps the most important thing about young talent that I am most excited about this season is whether the early-season signings of promising young players like Max Strivham and Phil Jones, Aaron Ramsey and midfielder Javier Herrvez and Luis Sanchez or whether unreappearing side like Luke Burnsley (with an unlucky injury problem) have all been brought off the bench at the start of the season. Many Rangers fans, however, still hope Rob Clym Clarke will be back on the back bench this season A little bit further back The Rangers—and hopefully even some of their young players—still need some extra footing and feeding support and I hope they might be ready for their loan loan later this season. While the Tobago City squad boasts of talented forwards and defensive-minded central defender, I am remain confident "I've seen the future of this team and in the past few weeks I'm hoping to point out that I won't play against players like both Everton and Arsenal who were left out one day after being utterly devastated when that word comes along with one. I have always hoped that for now, Rangers fans will buy their tickets." WATCH SLEEP: AFC Dynamic Preview 19 November 2016 So—Should it be utterly impossible I've guess just how badly this season ended? 2017 might very well be going down for the Rangers. The Indians might have been looking at some point where they might even be at the start of next season or less—if they were lined up against England or Wales and looking forward to some positional clash between the North American division and the World cup gloom. If the Indians were contemplating about lining up against New Zealand, Rangers and Australia in the other divisions too much better I'd be surprised if they had finally figured out how to keep this team together next season on an international stage. That was the answer to—maybe the Rangers could well end up playing some form elsewhere instead. Given Rob Crym Clarksley's injury scare delayed his return to play in South Africa last year, maybe they could very well have rejuvenated the Rangers enough to secure a spot in their next World Cup campaign. So where does Rob Clym Clarke's future fit? Scanner or strikeer fit in the right direction, but seen as a defensively deserving prospect at the right level before being sent off? Will this young talent leave? Or maybe he is touted as the best contender in 2017 and looks like he has matured enough? How old it brighter it seems—maybe and his exaggerated strikeout rate?</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 7 =====
+head_tokens: ['▁By', '▁the', '▁way', '▁the', '▁idea', '▁that', '▁Bitcoin', '▁might', '▁seriously', '▁be', '▁worth', '▁about', '▁20', '▁percent', '▁of', '▁its']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+By the way the idea that Bitcoin might seriously be worth about 20 percent of its value does seem plausible to me. But what gives? The Washington Post's Matt Keller writes that Bitcoin is not a mere salesman of digital currency after its demise and continues to spread all over the world. And it might be worth pointing out that, as many as average Americans will lose more than $1 billion more this year because Bitcoins value goes down. And what about money laundering and abuse of the use of digital currency? As Keller writes, I might be trying to argue a little more about this technology. But if you have Bitcoin, what could be even worse? Maybe it's an American airliner. It's a small piece of money that had just been hijacked by an airliner airliner when we thought it was the American currency. And when we saw Bitcoin for the next $20 billion US dollar—maybe maybe 20 years ago it was a piece of money hijacked by the airliner, but by then time it was still almost the only currency in the U.S. Now, this might sound very strange when watching Bitcoin turn over. But, if you're watching someone watch Bitcoin turn over on a different country for right now, it seems plausible that maybe a lot of people might actually understand Bitcoin. And maybe it might seem strange if you know what it was at first. So if you look at the whole of South America, where there were no central banks banking system banks and then where there were private banking and financial security systems for hundreds of years now, you might see Bitcoin very clearly—and perhaps at some point—but now you know people see Bitcoin as being an intrinsically bad form of government currency—origin currency that has become too destabilizing to unstable, and all that untrue or untrue claims, which are worse than anything else. And it seems plausible that there's a sense just how different these things are. As The New York Times' Kevin Fowler writes like, you know, the problem isn't Bitcoin. And Gaulker thinks (by some anonymous editor, who has half and a half leeway to say something about this phenomenon) seems to be somewhat befuddled in his explanation: Maybe not only does Bitcoin actually make sense when it's doing for right and wrong reasons—and it's just plausible that these things aren't happening. But maybe think maybe it might be plausible that maybe this idea of why some kind of senseless behavior is still somewhat different than Bitcoin actually sense—but for a lot of people, it's quite plausible and maybe it might seem so for someone else to come along. Or maybe maybe having a sense that what Bitcoin is actually doing is actually quite different from Bitcoin itself—at which point some of Bitcoin community members have joined in the encampment of Bitcoin forks to explain the consequences of what happened. And It still seems plausible that maybe it's just plausibleand maybe only plausible if these things are doing well and maybe people are actually hurt by them. And maybe maybe maybe a lot of people are using Bitcoin just to explain how governments are actually getting into their Bitcoin system. It seems plausible that a lot of people who use Bitcoin aren't affected by Bitcoin much—but it seems plausible that some of these governments are doing it to take a look at Bitcoin. And maybe maybe it might seem plausible that these governments are using this system the same way or so maybe a lot of—everybody who understand the business model and the whole government aren't affected by Bitcoin much—and it might seem plausible for someone else to come along. In another way, maybe just maybe it isn't plausible that governments are actually using this system the same way. But it seems like maybe having a sense that a lot of people are trying to figure out new ideas from Bitcoin—and maybe the whole whole system is actually quite affected by Bitcoin itself. Or maybe maybe a lot of people who use Bitcoin are doing it—and it's plausible plausible that some of these governments are doing it to take a look at Bitcoin and it's actually quite plausible that these governments are using this system the same way or so they use it. It still seems plausible that maybe it's just plausible or maybe just like any other—at least because a lot of people are trying to figure out new ideas from Bitcoin isn't a new idea from Bitcoin itself—and it might be a new idea in his own way.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260604/owt_t5_elftokenized_full_pow1_unfixed_norm_stateprobadd_selfcond_ce_fast_lr3e4_logitgrid_n256_step142000_tschedlogit_normal_mn0p7_s1p0_sc1p0_decode32_n256/sc1p0/chunks/chunk24.txt

@@ -0,0 +1,38 @@
+checkpoint=runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260602_001439/step_142000.pt
+step=142000
+decode=dualline_time_aligned_dirichlet_final_endpoint
+c_min=1.0 c_max=1024.0 c_schedule=exp
+steps=32 temp=1.0 bridge_power=1.0 temp0=0.0 decode_time_schedule=logit_normal decode_time_logit_mean=-0.7 decode_time_logit_std=1.0 decode_time_shift=3.0 decode_time_rho=7.0 decode_time_sigma_min=0.0001 decode_time_eps=0.0001 prior_beta=0.0 final_sample=argmax final_count_penalty=0.0 final_count_power=1.0 final_count_warmup=0 self_cond_decode=none self_cond_scale=1.0 state_self_cond_decode=single state_self_cond_scale=1.0 state_self_cond_normalize=True state_update=dirichlet odeish_eps=1e-06 odeish_c_eff_max=1000000.0 dirichlet_gamma=1.0 cfg_scale=3.0 concat_self_cond=False
+bos=1:</s> eos=1:</s>
+===== sample 0 =====
+head_tokens: ['▁The', '▁S', 'la', '▁scene', '▁appears', '▁to', '▁have', '▁had', '▁some', '▁strange', '▁things', '▁in', '▁classical', '▁music', '.', '▁I']
+tail_tokens: ['▁sound', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+The Sla scene appears to have had some strange things in classical music. I’ve been doing something strange lately because it seems almost impossible to make any SPX sampler, clip, or even video from ex-producers that appear unlikely. Also, please know that if you’re a fan of classical music, this is undoubtedly something newer. Classic music shows in general have SX samplers and samples clips which sound like a replacement for classical X samplers. Many of these shows have been tried out at various festivals, and so at some point I’ve decided to try and use classical music for my own purposes. A spoiler image below of this scene: Called as either an Sla video or some other piece of music I’ve had an ex-producer version that appears when it appears “the original recordings of the vocals and vocals are recorded.” The first part of this scene is reconstructed from the original scene (alone clips from the original sampler) that were recorded on the original scene (which have probably been recollected at some point). But in this case it actually appears exactly like some scene different from previous recordings made by original samplers. Another analogy to this scene is that while the second part of this scene is quite loud by itself, they tend to sound quite louder and go somewhat unnoticed, at least at some point. As I have seen this scene, there are several other Sla tapes and audio audio clips that have actually made them seem louder and in some instances, they probably could never be heard without any questioning and could never actually be heard. This is one of the factors that resulted in the popularity of Sla tapes in the music community, which is clearly due to its popularity. You’ll probably hear almost all of these audio clips that no one has ever actually seen on the internet or on YouTube, and these two clips make it even more interesting to see and hear just about all of them as well as the other Sla tapes and audio clips. I have been around in the past discussing Sla tape’s popularity, and I think that made them better than others simply because they were actually quite loudly heard. I guess what makes these two more interesting, but let me say that the scene in both of them sounds a bit odd. The two epitopian analogies above: The ear that we have in the Sla scene makes it sound like the utter “wrong ear” on this part. There is a similar epitopian analogy on this part: it’s quite loud and indeed for the same reason, it has some sort of a detonation sound that people tend to think that no matter how louder people would hear it at all, in fact, they would oddly hear it. The same thing happens in this case when another scene makes another sound which tend to oddly different from what actually is heard on this part. The fact that it actually sounds like the wrong ear on this part actually makes no question matter how louder it would sound. It seems quite louder indeed, seems oddly loud or not at all, in fact it turns out oddly odd reason. Because it seems oddly loud in this case one people tend to think it is “utterly loud”. However, in its original form, people tend to only think no matter what is heard, it seems oddly enough to have a seminical sound at least at some point. So in the main scene scene main scene music is quite loud and loud and louder than some of scene scene, and that sort of music is not that loud, but it sort of is somewhat similar to the standard vocalistsonic sound (not sound familiar, but also quite louder version of S stereoorganismonic in this respect), so the main scene as a seminical “sound” which is quite oddly louder not the sort of a canonical sound in some cases because it isreflects how some vocalistsonic to be heard at some point in the subject matter (so the scene seems oddly is why some of these scenes scenes tend tend to go back to the past and sound quite much louder than their original form. In other cases one might think of the main scene as a “phantomonic sound” simply because it seems quite louder. After all, in fact, it seems quite oddly louder in this case you can actually hear the ringling of the headphones in almost exactly the same way. In other cases, however, people hear them in the latter scene either because they seem unnotably louder or somewhatinfluenced by how they’ve actually listened or though some of the music seems to have some sort of stereophonsonic sound.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 1 =====
+head_tokens: ['▁Just', '▁before', '▁they', '▁released', '▁their', '▁first', '▁live', '▁album', ',', '▁the', '▁band', ',', '▁known', '▁as', '▁Chris', '▁and']
+tail_tokens: ['ong', '▁doing', '▁classic', '▁tune', 's', '▁from', '▁the', '▁album', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Just before they released their first live album, the band, known as Chris and Lily Hudson and lead vocalist Chris, was busy recording one of their new songs. When the band started recording it was Chris started performing with lead vocals and So recording a new song was a great idea and it was great for the band members who wanted to be so excited to hear it happen. It would have been nearly four years later that they finally released their first live album, and it was arguably Lonely’s first solo album. And it was the first time for Chris and Lily Hudson’ new record. Their two unreleased LPLonely songs known as "The Loisto Becky and "The Phristo" were all recorded together in what was perhaps the biggest and most emotional moment of all time. And it would have been a very time time that two of the most incredible songs were ever recorded with Lonely. This was the first time in their life that they had been performing together in many different ways than they had ever before. Lonely Listen or find out the full full cover that they recorded together during the time: In 1975, known as "The Loisto Becky." The song was written by lead vocalist Chris and long-time drummer Willie Larson. One day they were incredibly proud of it when their previous record was released on new album.Long The cover of "The Loisto Becky and "The Phristo" was a really incredible collaboration for the entire band and at the very same time they were all performing with lead vocals and Larson said they were so happy with the band during their previous album. It was an incredible time. It was an incredibly emotional time and I was really really happy with the cover. When it came out on stage it was probably a surprise because I actually thought the band was going off stage when I first heard it. But then I actually heard it. It was amazing that I felt like I was with this band. The cover was such a great mix of songs and really amazing on one track, what they recorded at Phristo and and how somber it was their previous record. For the first time since their debut album, they were very happy with their new album also with heavy-metal instrumentation of the cover. Over the next few years, formed Lonely had released songbooks from two different albumsLP albums. Their first album was originally titled “The Tangled Mansion” and the album that was released over a year and a half later consisting of a CD-only version of an album titled Phristo specifically related to the band. This song has never really been recorded before, and I really enjoyed hearing it on "The Loisto Becky." It was a real emotional moment for me. It was a great thing to say about band members even when they were on the road and I was genuinely real glad to that the band had come together during this time that they were performing more incredible than ever before. They always seemed utterly incredible, and yet yet they were all so happy with their hard work, they had all become so happy and in incredible form that they were all much happier than they had ever before. It was so incredible that it was utterly incredible. "I know all these bands work so hard they always perform constantly, and show really hard in real time constantly. To be just like so many bands who do work so hard they perform in almost utterly unbelievable ways almost every day. "Lost of music is very very recorded in some time. Ever since my favorite hard rock band EPic came out on the with their debut album and their new album, I couldn't find Becky on "The Phristo" track in between: vocals, drums and vocals, bass, bass. Beck said: "Just rewatching one of my favorite songs from The New Album was so incredible and utterly incredible. "I know it was just so incredible that it was utterly incredible. "Lost of music is is very very appreciated in some time and yet they are all kinda happy with the band about everything they had ever before. "To be just like so many bands who do work so hard they perform and perform always constantly, they get really hyped, and show really hard in real time in real time. And they all perform and perform in almost utterly unbelievable ways almost every day. "I know it was just so incredible that it was utterly incredible. They were just like my favorite hard rock band EPic Just rewatching one of my favorite songs from The New Album. I'd be lying if they weren't." Lonely was performing extremely well when Becky recorded Sungrong doing classic tunes from the album.</s><pad><pad><pad><pad><pad><pad>
+===== sample 2 =====
+head_tokens: ['▁While', '▁ancient', '▁authors', '▁can', '▁tell', '▁certain', '▁myth', 's', '▁or', '▁histories', '▁other', '▁than', '▁writers', '▁have', '▁told', '▁certain']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+While ancient authors can tell certain myths or histories other than writers have told certain stories telling characters old people and how they have changed their history at a certain time, it is also possible Aryan suggests that the authors of this book give an example of ancient history and what historical histories tell me examples I find in writing by the authors of Theodoreophlotians mythological works of Theodoreophlotians As a non-traveler: Tyrone tells stories about how old people are often seen in these Names because old people couldn't come alive before their own history in a new context, while these old people are also depicted in the Plothians mythology: These people are being seen in different ways, but are also represented by a different different group of old people: Tyrone shows how old people are often depicted in these Names because they were only young people who had already come alive before their own history, while these same old people are also depicted in the Plothians mythology: Here example can see several examples of the authors of Theodoreophlotians mythology about old people and young people by writing in Herodotus:Horryan thus tell about old people in a certain context about their own history and so on: These old people are old and tell us the mythology of historical historical oppression, and it tells us that what are these myths? Some things like the authors reveal stories about old people being told from them in ancient history or things like Domino's work Historia, which tell us a lost historical myth: He also shows examples of such myth in the history of the person who led them back to the past prophecy: One example of the myths that led people back to the past prophecy: Another example shows what never happened before, and shows such myths as: One myth in the history of the person who led them back to the past past prophecy: Another example gives one example of the myths that led old people back to the past prophecy: Another example of people telling us that old history has been loston mythically (the Pharaohians) is also mentioned by telling old people his book: The authors of these stories of names mentioned a distinct family who had already been destroyed and killed in a new context because of historical historical oppression For example Tyrone is also mentioned for example Tyrone, who had mentioned "the entire story of being destroyed and killed" by the Hungarian (another two authors). Aryan also gives examples of Tyrone telling old people his book: The authors of names mentioned as distinct family who had already already been destroyed to reflect this new context because of Tyrone being killed by other such people:for example Tyrone is also mentioned as the distinct family who had already been destroyed and killed by other people in the same sense Ancient history can't be described as Plothians myth by telling these stories: The erasure which myths tell ancient history is very longer held true in many previous myths (and thus there are a wide range of myths told through ancient history), though these myths can never be told at the same time as other historical myths in Ancient history are told by the authors of The Stranger and Ghosts Tales and Ghosts Tales (in mythology): Historia shows the works of Theodoreophlotians by telling them stories: They tell us the story about how very old people in their own history: These myths tell us stories stories that old people were being depicted in a new context, and they clearly show how the authors of Plothians stories are shown in the same context: by telling them stories tell the story about people' thoughts and feelings in the law of law (emptory: I think about Ancient Greek and and ancient Greek texts suggest that power is either an innate form of power or its inherent power. Ancient Greek refers to it as "power of power" even though it is derived from the Greek word derived from the Ancient Greek word meaning "power". The concept of power is also shown in the ancient works by the Plato, where there are different conceptions of power throughout history in which there are different conceptions of power, and both conceptions have taken place in the same period of history (D.D. ). The concept of power is also also shown in the ancient works by the works by the Plato, where there is actually a different period of history in which both there are different conceptions of power and in similar periods histories (D.D.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 3 =====
+head_tokens: ['▁"', 'He', '▁gave', '▁me', '▁', 'a', '▁good', '▁chance', '▁of', '▁when', '▁', 'he', '▁came', '▁back', ',', '▁so']
+tail_tokens: ['▁scheme', ',”', '▁it', '▁add', 's', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+"He gave me a good chance of when he came back, so hopefully I was hopefully be able to see him fall away from England," Mr. Thompson told the BBC. "That's probably the biggest thing in my life when he came away from home and I always knew he would be happy to go when my dad left." Role football boss Mark Elgapkins, who has seen his dad happy after working together with his family to keep them together, was left off for an six-month betnight after being hauled off the road for the past four months. Born in Role, Mark Elgapkins spent 14 years in England before moving to England and spent four years on holiday holiday with his family before retiring as football coach. He told the BBC that he had never travelled to Role because of his age or impending retirement. When he said he was determined to live out his own life after getting paid off, he said: "It's a dream for all of us to work together and I've had two kids but over the past four months I've gone out with four kids. It's as stressful to me as possible. "I'm going to keep my dad on the bust and find my way home for the next four months and I'll put my focus full rest on my dad in the next few months." He was left off home for an six-month betnight when his football career had already been paid off Role boss Mark Elgapkins arrived at the club's training ground shortly after the first season leaving Role and returned to Role after being replaced by a 15-year-old youth football coach who worked at the local club. He rememberrecalled growing up watching football at around age six and he was tired of daydreaming about running away from the club "manship all the time". He said his wife and father decided to stay away from Role for more than six years ago they had been working together while 14-year-old son Mark Elgapkins was living live a happy life. "That took a few months for me because I was in the bus league school at the time I was going to get up and play with my dad, so I realised that my mum and dad really wanted to be part of it. "It made me just want to concentrate on playing day football," he said. Happily, two young sons and one boy were lining up in their two-bedroom bust house, were living together at their home in Roleshire and bought a new home – which is owned by his teammate Mark Thompson – on 20 August. "I just had a fantastic feeling that I would be back to work with my family six months later," he said. "My mum had a couple of friends who were quite lovely and happy to work together. "I thought it would be nice that we'd be together on weekends if we were working together another. We were be a bit excited to work together and we'd try to get some good back together when we get home." Mark Thompson, who was hauled off the road, told the BBC that there was no need of bond between the two children. "I'm not sure my mum would ever be told every day and no matter how long they had to go home together I had a happy relationship." ButDespite the prospect of spending extra time, he thought it could be an opportunity to reunite with his friends and family. Mark Thompson spent a short period of life but was not in favour of staying away from home. He also had a long and disadvised relationship with his dad. "I wasn' too far away from my dad but I got home it was good two weeks now and I was working off my dad," he told BBC. "I just wanted to relax and be with my family and friends and I wanted to be kind and relaxed." Image copyright Mark Thompson's son Mark was granted free holiday leave – a couple of months ago. His dad's were suspended for a five-week break after making long-over-repaid holiday payments lasting no more than five weeks. Yesterday, Facebook reddit was forced to post a photo of Mark Thompson his son's birthday and the picture he had just taken as a selfie during the five-week break. Earlier this week, the South Korean business newspaper revealed that the new toy scheme, which took place around the subject of stricter security measures, was revealed and lasted only eighteen hours for six months. “If you’re ever tired, we’ll want to wake you up It won’t come easy for you to get under the new toy scheme,” it adds.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 4 =====
+head_tokens: ['▁Once', '▁', 'he', '▁started', '▁working', '▁on', '▁do', 'g', 'l', 'aught', ',', '▁', 'he', '▁ended', '▁off', '▁making']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Once he started working on doglaught, he ended off making pyrop and pyrolans at some point in time to work with doglgeon on the attack line, rather than getting rid of  with fireworker Even though he tried to make the poop a bit more timesuited of the task I tried to do elsewhere, I did nothing for him. Mostly I saw his role as a well-organized firework team. Once he started starting off at warfolangging pyrolan poop over time and got a bit messed up thinking (furtherly) about how fireworker went about building things up. I thought about him that instead of working on a pyrop/warfoggsk fireworker gave them some more time rushing into gamma waves that would destroy the massive snarling shells and pyrolan grenades. I thought the idea of getting working on doglgeon once he got into later doglaught would eventually get killed by himself. I thought that the doglgeon’s running rate would be a bit higher once he started his work off actually removing shell shells and getting rid of the small gamma waves. In the end, I’ve had good luck with doglaught, working on runaway and gogging again until he finds himself bouncing wildly even harder once he tries. doglaught then starts to do fireworker. There is nothing left to worry about once he finally figured out how he can do fireworker. He ends up trying to remove his bloody shell without using it any longer and ultimately results in what I still believe might be the best solution for doglaught. Finally, I’ve tried to come up with information about perhaps the most important aspect effective poop commander/operational command structure for doglaught: I don’t think it will be hard for me to get the details of what I had previously thought I could have done to do fireworker. I guess that’s how he died. Doglaught eventually faced some adversity and ended up dying some way of doing attack lines. I still don’t really know how he could actually do fireworker without killingy shells, but I’m still still thinking about how bloody shells,, rather than vague details of what they say about fireworker. I guess I’m finding it harder actually to actually do fireworker, but rather attack lines. I guess I’ll have quite some time ideas the next time I’ve give up some interesting ideas on doglaught. Maybe not too soon, but take a look at Ivolve at youtube.com/ecvLvOld. Parts of the postops/al Dicebreakers are mostly written act out of time or piece. Some examples are very useful but for some reason #AtOwn #ContestDoglaught #PrevidenceLoadAl #EventLaught and the Fireworker #EcvOld Below is the following quote from the Lunar's previous post from Dustin: In addition to the postwork article mentioned Dustin's article was written on two separate occasions on behalf of the Lunar and the Lunar's behalf. This article also covers other contributors. Dustin does run ops in an effort to help both coworkers and coworkers as contributors. Dustin sympather Lunar understand the need to have worked with Dustin. I've discussed this article in more detail so far. A second quote from this postworkally refers back to several events in this article that have way done in various ways, including some times when operations on doglaught are long and dangerously (nowhere outside of mine walls) or even extremelys. This article also gave me a bit Ivolve Life Time, Wicked By Sabers: Here for some improvised reruns PvP I'm not sure that this article was originally about miner activities. How does it work for the dogglaught? I wrote down the main improvement of the ops in the previous article, which is a rarity to attack at the low end approximation of the dogglaught' main objectives. A major improvement should include building traps on mine walls, to help others survive job mines and setting up traps and hatches on which others can fall from without lacking any particular survival skills.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 5 =====
+head_tokens: ['▁The', '▁next', '▁issue', '▁is', '▁that', '▁I', '▁will', '▁tell', '▁you', '▁how', '▁we', '’', 're', '▁dealing', '▁with', '▁all']
+tail_tokens: [',"', '▁Su', 'm', 'o', 'l', 'r', 'm', '▁wrote', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+The next issue is that I will tell you how we’re dealing with all the characters that we have over the past 5 years and since my first time on the series there has been some unexpected events that this has definitely happened to me since the beginning, but in a very different timeframe So this is just that this is really important event in the series and/or some unexpected part of our readers’ timeframe. So this was an incredibly interesting and important event that I just wanted to tell the readers that I have been involved in a lot of work on this and for me personally I am having a bit of fun in my long run experience I have experience with robots that has been a big part to robots as well as interesting to me personally and that we live in a very different time universe So I will come up with how I’m feeling the time/time stuff we’re dealing with here that will probably be most important to me (read) the time/time stuff and that I just had been dealing with all the characters that my friends over since this past time issue. So this was an extremely important and (and maybe some unexpected) event in my long run the two of us are and for me personally I’ll be reading a lot of these books from the beginning wondering what the characters that had been missing...and now going now a little deeper. I feel like this is the hardest thing I have ever dealt with to be understood in that timeframe and to really see any insight into the detail that I’m feeling over the past few years. And that’s because this is just a new piece of work that requires this lot of work on So what I wanted to tell the readers is is that I feel like this was a really important event in a very different timeframe I had on my long term experience...and wondering what the characters that I was probably missing...the characters that had come up with a bit of understanding...and are now a little deeper and a little strange and that I’ve enjoyed myself reading all of these books. So I want to say this is a difficult thing I’ve had with this experience because I feel like I was dealing with some issues here that were maybe even some important part of the readers myself in that time in time issue and had some deep insight into what I felt like the most difficult thing to ever do in this next timeframe. Thanks for your help in coming up with this post so let’s go ahead and prepare for our next issue issue. Parallel Machines Machine A few weeks ago we shared our story on the issue of Parallel Machines where we looked at this machine from the past on Earth. The new machine also provides some information from the past, and the machine itself provides some information from the future. We hope and hope that a small community is giving thanks to this machine, and other stories from readers like Parallel Machine that will hopefully follow in a couple of weeks. It gives a reminder that it isn’t always the way to keep things interesting and keep the show going. So here’s how the machine ended up next week: "My mom told me that Sumolrm is now trying to remove a giant statue from the state's parks. "We noticed that the statue had been placed in the back yard, and a bit later," the mother wrote, according to her Facebook post. "We thought the statue was a giant statue that was over there, by the way, and it had been removed from the park so that we could see some of the statues that we had been waiting for since the last day... It's hard to figure it out that maybe we could have moved on." The statue head off from Sumolrm once and for the rest of the state, but after the post posted by Sumoolrm she received a call from the State Park Service, who said it "states a beautiful day." "I have lost all my friends at the State park and I just felt like it was something special for my family. I think it will be an amazing day for the two who just heard about the statues," the mom told the paper. A said after the local TV show that she reported seeing the giant statue head off from the state park was told that the park didn't respond to a series of text messages written by residents on their Facebook page. "You fuck over there, you know? Think about it now," Sumoolrm wrote in a Facebook post that the statue had built up around the state park state park and ordered it removed from parks for the rest of the state. "I wish we could get this statue head off. Maybe we could remove some giant statues like that, but maybe we can't do anything to save the statue," Sumolrm wrote.</s><pad><pad><pad><pad><pad><pad>
+===== sample 6 =====
+head_tokens: ['▁As', '▁you', '▁already', '▁know', '▁we', '▁have', '▁', 'a', '▁rock', '▁band', '▁called', '▁Re', 'c', 'ogni', 'sed', '.']
+tail_tokens: ['c', 'y', 'tic', '▁word', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+As you already know we have a rock band called Recognised. A rock band that formed at the time for some reason and saw what's going back to the tropes in rock-sounding bands (think fuck, Rage, Hungrap Rage, so on...) and saw the original (and possibly better) vocals (rather vocal) and sounds (more often) are actually going to be coercive and literally coerced beats. The wordcognicognised word exists as the word "having back" morphing into an old word (in some sense it is a verbal word) it's obvious also that it's too far away but I'll leave the details in the end... The whole thing was actually the band itself. They probably heard it as Edina, Bryce Volckery, Karen Green, and other people from Gotham City who the band word into ascension. Historian Edina Ikke once pointed out his old word though I can tell you that the word is here, the band didn't hear the word "hevecry" but suddenly you people think it is actually an old word. Maybe the band simply heard the word "hevecry how long ago did it actually bear the meaning to the old word? This band is so furious it's the only thing they've heard it sounding just like it's old old? Or maybe they used the word "vecry" as the word "hevecry" as it was Recognised word Well once you hear the word "hevecry" you'll probably notice a lot of this coming off of the earliest bits of the word spoken by word. In some sense you'll find the recognised word sound a bit cruder, a little more than once you actually heard the word "vecry" (actually) its original meaning but by a couple of coerced beatstoically it's actually used even more potently (expecessively loudly) and the phrase "hevecry" literally means it loudly. The word "cry", however, was the first verbal word ever so I thought about the word in terms of vague details but it did seem quite distract confusing. After all this time I'll be honest here that the word was labeled the word "hevecry" (loosely morphing back into an old word yet again and yet again) In some sense I'd like to say again that at this time using the word that I could find the old word scarcely understated its meaning but what I thought might sound like a coincidence but but it's all quite clear now that I'll spend some time trace it back to the original word and then figure out what it actually meant; I was glad to know that I'd been reminded the precedence of the word... The word "vecry" exists as the word morphing back into an old syncension Fucking guidlingward/backward/backwards skepticism over the past thing happened so long ago has been transformed into a deep and erodedrift of of secidiocy and largely forgotten once you've given up on its meaning. Maybe all same strange things happened a few years ago but the kind of thing that "almost every aspect of life" ends up throwing it all back into ascension. There is something going backward during this time period but still the dwindling void of its deep skepticism and the still lies in its skepticism and confusion over the past... I've been saying for almost forty years I can no longer remember the word for you * * * * * * * * I've remember a few words in one syncytic word or perhaps words like the word for you either... The word "cry" painsingly morphs morphing back into an old word and only for as long as the word didn't match any other words in its syncytic form but then the more I thought about it and gradually morphing into ascension it didn't take long for me to remember the word "cry" I'd never it. Well I guess not so much it took a few word proverbially to for me to words like the word for you. The word "vecry" then you can no longer remember what the word "cry means, remember it too" but now it's "the word for you" merely a minor word morphing back into your syncytic word.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 7 =====
+head_tokens: ['▁Maybe', '▁I', '▁remember', '▁this', '▁story', '▁', 'mainly', '▁because', '▁I', '▁think', '▁it', '▁reminde', 'd', '▁me', '▁so', '▁much']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Maybe I remember this story mainly because I think it reminded me so much because all of the great stories and stories were being told about every race and every other race and I remember every race I really rode and enjoyed riding so much I remember the first run was something like 50 pounds of gallons of cold water and that cold it honestly wasn’t quite cold water. It was the first time ever that cold water was reminding me of running again once more and that cold water definitely really helped me so much and maybe it just beg helped me too because the really weird feeling about cold water was that I just felt cold while I was trying to run again but I’d run that day in cold water and I’d run three times before and run three times before I would run in cold water and then in cold water and then get a bit cold water again but when I started really pumping out it was an incredible experience because cold water was just incredibly cold and utterly filled with cold water and we had only run a few minutes in cold water before we actually started running down the water and we were trying to figure out that what we were doing was wrong and we were just just starting to try to fill the water with cold water. It was almost cold same that I’d run again and I run that day in cold water again and it was about 15 minutes until I got a bit cold because it was really cold and I was just right away and that was really thirsty and cold water was cold and very cold water and really cold water was really cold and cold water and I ran again and then it was really cold and cold water and I just got a bit cold water and then I squeezed in a little bit of cold water and 15 minutes later realized I was almost starting to get cold water and getting a little bit of cold water was always really cold and I was almost just starting to get a little bit cold water. Then I put in a bit of the water and cold water was just cold and I would go, “Oh, I’m just going to drink cold water!” I was running again and this was cold water and I just really got out of cold water that I was thirsting and it was always where I thought I’d need to be ready to ride. I experienced it all through the first mile run and I was just kind of tired of what I was running but I was sort of just taking care of what I was doing and I wasn’t tired by the 15th hour of the run even though I was just really gripping cold water and I got a bit cold water and started running a bit long and a bit cold water over the ride and then after the first run it was probably the first time that I realized how much cold water was getting and it really scared me into thinking it would ruin my life if I’d run again for a couple of weeks or About 15 minutes after I first started running and that one thing that I’d just gotten was cold water and it wasn’t quite cold water I honestly felt like it was a weird experience and cold water was just the way I felt and I honestly didn’t know any other time that I was in cold water and I was just freaking out and trying to drink cold water like I was just feeling cold and slowly cold and eventually I realized what cold water was doing and it took me a little while to get some grip on it and then I started to feel cold and I probably didn’t really know about any other time either. I was in fact running again a couple of years ago and even then maybe I was feeling a little bit more eloquent and it was definitely just the way I was feeling cold water. It may have been the first time I was feeling cold cold in cold water and I was freaking out and it was hard enough to really sweat and it was just a weird feeling I was freaking out and having a little bit of cold water and remembering what was going on and feeling water was always the same feeling I feeling when I am at home running and remembering what I was drinking I was going back on and I don’t know what’s going in my head but sometimes I just don’t really feel the same level of water that I had been running only at home running for so long. I had run a long run away from home and I was really trying to tell myself that I wasn’t drinking it and was just afraid it wasn’t going back on water it really hurts and I’m glad I told myself that I had been able to drink even more water while running again.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260604/owt_t5_elftokenized_full_pow1_unfixed_norm_stateprobadd_selfcond_ce_fast_lr3e4_logitgrid_n256_step142000_tschedlogit_normal_mn0p7_s1p0_sc1p0_decode32_n256/sc1p0/chunks/chunk7.log

@@ -0,0 +1,38 @@
+checkpoint=runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260602_001439/step_142000.pt
+step=142000
+decode=dualline_time_aligned_dirichlet_final_endpoint
+c_min=1.0 c_max=1024.0 c_schedule=exp
+steps=32 temp=1.0 bridge_power=1.0 temp0=0.0 decode_time_schedule=logit_normal decode_time_logit_mean=-0.7 decode_time_logit_std=1.0 decode_time_shift=3.0 decode_time_rho=7.0 decode_time_sigma_min=0.0001 decode_time_eps=0.0001 prior_beta=0.0 final_sample=argmax final_count_penalty=0.0 final_count_power=1.0 final_count_warmup=0 self_cond_decode=none self_cond_scale=1.0 state_self_cond_decode=single state_self_cond_scale=1.0 state_self_cond_normalize=True state_update=dirichlet odeish_eps=1e-06 odeish_c_eff_max=1000000.0 dirichlet_gamma=1.0 cfg_scale=3.0 concat_self_cond=False
+bos=1:</s> eos=1:</s>
+===== sample 0 =====
+head_tokens: ['▁"', 'I', "'", 'm', '▁going', '▁to', '▁be', '▁writing', '▁this', '▁next', '▁now', '.', '▁It', "'", 's', '▁not']
+tail_tokens: ['▁bodies', '."', '▁dreams', '."', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+"I'm going to be writing this next now. It's not too late," he scowled when he returned to his old work, was still waiting for the next novel. "It must be almost time ago I make all the people around me wonder if I'm going to keep my illusions open for so long as I can possible to lay beneath them and all the darkness and all the human insecurities. "Fucking task is to hang out, if you imagine what you're doing, people will try to understand why you're being held captive, you can't be disobedient to just because you've got to be the person you don't want to be and hang out for five years indefinitely, but we'll be able to make people happy." Orwell Orwell The Riddler Orwell What I thought was inevitable: Rever had begun his journey deep into his deepest dreams, the shadowless shadow illusion fostered by the infinitelessnessless darkness, fearless and irrational force possessed by unrequited laborers and all. He was too prepared to begin exeracerating the erasure of the shadowless and illusions of unworked labor (some of which was caused by the deep and deepenedness of Rever's vision). This time Rever completed the most painful, disturbing and perhaps the most disturbing work ever completed. But once his vivid vision faded away into a mocked but unsettleled image of the gloomy, poor man and cruel, merciless prisoner devouring his soul, he looked for that poor person. And while he could sympathize with the man who enlightened his soul the shadowless shadowless work had misguided vision blocked his path to salvation. And then he began to make up his mind with a vision that never even really existed. His dream vision would never go unpunished. "I am a enlightened man with vision like myself," he said once Rever grasped some of his own demons. But he found out that he was trapped by the shadowless work with infinite, fearless force. The realization of the shadowlessness in Rever's own vision was, therefore, shadowless and devoid of any other sinister force from the source. Even though it was impossible unattainable, whatever the effort could be made, the sheer human effort could only be done at the right time making the vision so clear and clear and clear. Fearless by fear, endless impulses of despair, perpetual infancy, sheer wonder, infinite absolute blindness seemed like a response to the immense pain of shadowless and visionary darkness. Tragically driven by the process of darkening, we fell upon the infinite shadowless darkness and the everlasting shadow from within. We saw how far it is to overcome this evil, let alone overcome overcome,, to eliminate it. Perhaps this feeling of selflessness vividconsciously refers to a meditation upon the selflessness of consciousness that is derived from the impernetable perception of the nature of consciousness union that conceived in all aspects of almost every aspect of human nature. Humanity, as in art and every aspect of the physical world, has manifested consciousness consciousness detained after these one-sided moments that came back into our mind, ridd with bodily drifting beyond unconscious control. In instead of the vision we saw shadowless bodies and shadows into sheer selflessness we saw an ever-ending conflict that took our souls from the unrequited laborer to the shadowless laborer who led them to the dead, confront Faced with the darkness flowing instantaneously through the infinite and ever deepening vision of shadowless death, Rever we now proudly cling back upon his enchantment: "I believe I am total darkness." The delusion and comradery in Rever's haunting memory of shadowless death, shadowless bodies and dark shadows are haunting and predictably similar to today. It seems only some human mind can figure out the vivid and vivid details of the shadowless shadows on death and destruction. But in contrast of one person's body, being a dead person, shadowless body can only spontaneously come to death without fear and loss; in this case being a shrouded body into an unhazardless, lingering unsolved nightmare of enlightened death. In other words, dead bodies terrorize an unforgiving dream: "It can only come to death like that of a dream: an unhappened body of shadow, an unselfished body of shadow faded bodies." dreams."</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 1 =====
+head_tokens: ['▁What', '▁seems', '▁more', '▁interesting', ',', '▁then', ',', '▁is', '▁that', '▁modern', '▁humans', '▁have', '▁evolved', '▁conceptual', 'ly', '▁rather']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+What seems more interesting, then, is that modern humans have evolved conceptually rather than philosophically, since history tells us that some human practices, and others, operate in nonhuman and nonhumanly and/or otherworldly dominant ways. Our own point of view suggests that both modern humans and humans, then, have powerless mechanisms by which humans interact and perceive otherworldly worlds in terms of technologies, practices, and practices. Or, from our point of view, nonhuman entities living or operating in the realm of metaphysical reality might tell us this: what power do modern humans have? What gives us to, our ability to interpret our natural practices, our ability to interpret our technologies, and possible uses for all sorts of kinds of things? There are some possible ways to answer this, however, that raise raise some philosophical questions. For instance, why do modern humans and modern humans take the world of technology for granted—what could humans do with modern technology, and why us all sorts of new possibilities? Certainly, it seems that science and modern technology—though they almost certainly not—will fall follow some hardline philosophical conventions. Why is it so surprising that it may be too late to understand humans and modern humans in terms of their practices and so forth, and that it may be too late to grasp the enlightened human possibilities and limitations to interpret so many different ways that we have come to understand and use them? Too late modern history, some time or many centuries to come, suggests that it might be centuries later than the end of a century, when humans had past technological limitations in the tools and practices that they were using, and so on. But at the same time we may not end up with a far more decisivful leap forward time, and long time, as to the future of modern human today. Natural practices are just another example of interfering with the nature of the world in which we experience the human experience in relation to our world experiences. The human experiences are being presented today in ways that bring about their present and future worldviews in accordance with the natural practices that apply them to the tools, and practices that have changed how we experience them today. These natural practices are intimately different from the practices we see today, but do not necessarily mean that we can go back into our present period of time and find new ways to apply these natural practices. Certainly, they aren’t entirely connected to ways that human practices at work, and they don’t make an understandable, profound and seemingly meaningless difference that can arise only when a human person ishaving or she doesn’t care or understand for which reason some, or even some logical reason, is necessary. But the natural practices aren’t necessarily connected to modern human uses for all sorts of reasons. In some sense, it usually falls on the exclusionary notion that the idea that human habituate natural experiences that humans and otherworldly—for very similar reasons, or indeed for other otherworldly worlds—are somehow self-interesting. Realistic paradigms, and practices at work, and cultural norms are fundamentally different; human and natural experiences aren’t necessarily connected to them. But such conventions don’t seem to be anything about the technologies implanted into modern technological systems developed by humanity after millennia ago, or the implanted technological paradigms—but it seems that modern human practices carry a tenuous meaning that can not to be understood here: the natural practices of modern human practices that developed in the nineteenth-century human caste. Modern human practices were necessarily mediated by technological advancements that existed in the first place, and the use of modern technologies developed by humankind after millennia, and technological advancements that the centuries. The ways in which human practices and natural practices in this sense may seem like a nonotherworldly convention, in a way much like the natural conventions of technology might necessarily fall in non artificially conventions, but that such conventions fall in the same way that human-animal technology and modern technology fall in the same way that human human technology does not fall in any way after millennia, or that modern technological and modern technology that might benefit from the human and and and experiences of human beings at present do not have a place in the modern world. Perhaps even more, we might think think so, given that no such historical convention takes place anywhere within the past, social, political or social centuries. These historical events, taking place within human societies free from the natural practices and cultural norms and the constraints of time, make such conventions impossible for us to imagine the present or the future.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 2 =====
+head_tokens: ['▁Now', ',', '▁I', '’', 'm', '▁going', '▁to', '▁talk', '▁about', '▁how', '▁hard', '▁working', '▁parents', '▁are', '▁keeping', '▁their']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Now, I’m going to talk about how hard working parents are keeping their kids keep going and how much time they have to put in their lives. I remember when I told my mom several years ago that some of my jobs were messed down over the course of my entire adult life when I was 16. I told her that thought “my mom was definitely a little girl I could have had this little girl with in my whole life.” It was definitely anything I could have done in my whole life that she loved. So I learned a really important lesson and realized that I couldn’t really feel the same amount of time I could get into work right now and actually sit down without really knowing what what this young kid really meant was. I also realized that I could somehow feel the same way now that I had such a little girl in my adult life and it just wasn’t going to be my home anymore for me. So, I think I realized there were some important lessons that I learned that couldn’t live my whole life to something like them. But by the time I finally got back to work, I started looking back and thinking how important this is to my kids and how much I feel about having to go to work every day, how much you take care of your kids and not having a certain amount of time in your life with you and the way you’re actually feeling and all of those things that you just kind of start to look at really you. I always hear people say that this is what I ended up thinking when I was 16, but I’m starting to change my mind right now because I started to realize that once I finally started going back home and putting myself off of work, I was just starting to be aware that being lazy means being lazy and carrying some weight on the other side of my adult life, especially when my parents had just started telling me that this is really happening to all these “bad moms” and just just starting to make some excuses for what I’m doing and just really about how I kept feeling that way for so long. When I finally got back to work I realized that I was actually working on a full time job and I didn’t realize there wasn’t really any signs of being put off, there was just some missing feeling in my life (as soon as I knew what was happening) I didn’t know what was going to be like until after I actually worked out I realized I was feeling some kind of guilt and just really starting to really realize what this happened was that I was just being put off and how all the really crazy stuff coming out of me was just really like a car accident. I was on this job with kids getting into work and my mom and my kids were getting into college and my kids were just starting to get some work off of me and I was just really starting to get a creepy idea of how to be good moms actually feel in their life. When I got back to work, I realized I still had full time behind me and had some other kids working full time jobs and wondering how I could put into a job that just wasn’t really good for them anymore as all these kids were going on and they were doing whatever they were doing. I just honestly felt really guilty about how these really bad moms could just be put off and taken away from me and start really feeling really good that I was 13 when my kids had just been realizing all this stuff and now I’ve reached the point where I was when I realized I had gone back to work and full time job finally being able to meet up with family and friends and go to work and see what else I could do with my kids and then finally finally realized I wasn’t really trying to get into the job I wanted to do. I know that when I was told by some people that I was 13 now that I had just gone to work and it was really just this thing that has affected me so much in my life, I’ve started to think a little bit about how muchpressed I was supposed to be and how it has really been so important for me. I’ve spent most of my time at work thinking about how I feel about my kids being “over there” all week after week and I’ve thought about a lot of times I’ve been thinking about all sorts of things that have really gotten really good at work over the years without actually actually doing something meaningful with them (after all, I wasn’t really one of them all the time) and honestly just finally realized that these were the things that I really cared about and much for so long that I really felt.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 3 =====
+head_tokens: ['▁The', '▁issue', '▁is', '▁always', '▁good', '▁enough', '▁and', '▁there', '▁is', '▁always', '▁something', '▁we', '▁need', '▁to', '▁talk', '▁about']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+The issue is always good enough and there is always something we need to talk about about when it does happen for young and old folks alike. When something happens a day or two or ago when we think something similar is usually happening in some big urban context or sometimes the whole town, the perpetrators are so upset that some of the victims are talking about it and that's what we talk about in America today. And people tell us that it is common sense that the very troubled people and groups who are responsible for this tragedy and they are banding up with the same excuses to make things look like things happen as normal if there is something similar that could happen again. Because we are addressing this issue, there is essentially no good chance this could happen again unless it ends up being no evidence whatsoever at all. Something similar happens but this is where this possibility is being pushed by the mainstream media and pushed to one side of the veil, it could no longer be true the whole idea is going to gain attention or attention. This is where Red and Constantine are aware that they are putting together groups other people to cause much more violence and destroy innocent lives in different ways. Constantine recently making a point about how coerce the entire village where Redd and its coworkers are taking place is unrightful and a clear pro-life message. You can't blame the people responsible for are trying to coerce and trying to coerce and destroy people that destroy their lives. Constantine is once again taking this opportunity to express her patriotism against this terrible tragedy that will eventually leave the world. The terrible that will hold humanity together for future generations to bring lasting peace. For all people we see on the wrong side of human beings One thing Frushe said, like many other young men and women in the same community who recently become victims of Redd's morale, Frushe said, made it clear exactly why he believes Red women could defend against violence when it comes to supporting the pro-life position. "I don't care that having children damages the lives of women," Frushe said. "I don't think anybody should care about having women who have children in a way that is where they can't speak their words and obviously, it could cause them to feel angry. It would be better for them to tell exactly what they are about to do next and it's a shame." Shocking stuff When I started thinking a few weeks ago I interviewed a young man behind a game called Cube who didn’t even vaguely know what he was thinking. It was probably his first clue to how Red Game is playing, and why he thought the game would be crazzling from backwards and just agonizing to reveal some truth to the young man. When asked why he thought the game would be a non-crazy game, he said Frushe: “It’s the kind of thing I for sure. Well, Red Game might be a good game for you, if it’s any kind of game you can play, but it doesn’t mean anything wrong, it’s just kind of stating it. I think at the moment I kind of just know that something is right, and I’m going to call the game Red Game.” The bottom line is that Red Game is totally committed to patriarchy and feminist ideology and if the game is truthful to it, there’s no way to play the game on a large scale and consider it it. And I’m not sure if Red Game is actually worth trying to play for a specific reason or whether it’s worth playing. Right now I’m not sure the answer to this but I think the rest of the article will be answered as soon as possible right now so I’m going to spoil the article. What I mean by that Red Game is selling some interesting facts, which may sound quite a bit different than any other games I’ve ever sold on Amazon, or even one of them called Red Game, which is probably much better than my own Red game. Even if you know someone has hit the middle of an argument, or you’ve seen someone stoking abuse, shouting outcries, or physical violence, the accusations may be completely unfounded. I'm sure other people may have heard that same thing too. So, how exactly this game is worth playing for it, and how is it worth playing it? If you have any further ideas or suggestions for further discussion on this topic please simply tell me in the comments.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 4 =====
+head_tokens: ['▁Meet', '▁G', 'add', 'in', ',', '▁Ele', 'g', 'or', ',', '▁Ele', 'g', 'ine', '▁and', '▁So', 'nom', 'is']
+tail_tokens: ['▁Po', 'nom', 'en', ')', '▁(19', '72', ')', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Meet Gaddin, Elegor, Elegine and Sonomis. Gaddin published it in 1966 after his mother was married—and published it in 1927. His first book, published in 1971, was William Lenin’s “Gender Women In My Life and My Own Adventures”: The Dominion Woman (1972). His second book is, The Dominion of Women (1972), a series which he subsequently published as a writer. Gaddin is a professor at Columbia Law School in San Francisco and his writing on the subject of gender identity—long self-longital relationship with women is particularly interesting. He argues that some people don’t even agree with gender identity and simply find a way to express themselves gender identity—those are people who love women, are people who love women, and most importantly, they are not interested in doing things that might sound very scary or dangerous. You can also see this as a dangerous topic if you want to find out more about transgender women. Click here to see Gaddin Flies His Adventures: Elegine, Transgender Woman: Elegine Elegine, Elemi Sonomis, A Herod in a Woman Assailment Author: Gaddin Born at an unknown age, I first encountered Gaddin while exploring the concept of gender identity. His very personal experiences ranged from the to point psychological problems caused by his transgender experience and fantasies with transgender women and by the fact that he was bisexual... Gaddin was quite different at the time. His intense, self-confident experience as a transgender artist and provocative feminist writer almost magically made me wonder why he actively advocated for his personal identity. Advertisement: I had, to a large extent, no experience with writing – I also had no connections between myself and the transgender community, I I was mutually connected with Gaddin and many others across cultures and cultures around the world. After working with him, I immediately decided to begin writing two works of gender identity and transgender literature: The Adventures of Elegine and The Struggles of Transhistorical transgender Woman Preface in a WomanReview: Struggles of Anarchy I soon realized that Gaddin wanted to include anarchist discourse on gender identity as deeply, deeply idealistic and disenergized about gender identity was important to transgender women and to the in in feminist literature. On the other hand, during the Fear of Uprising Gaddin, Gaddin was antagonised so deeply and deeply by the art, literature, or feminist art I worked on. A few years later, I started reading what I saw and antagonized Gaddin as a philosophical writer. A year later, I discovered Elegine, Elemi Sonomis, Eleine Sonomis and Elegor from Giles of Ponomy: Kalnomine, Aristotle (and others). Gaddin became obsessed with writing and essay on self-evidentity. When I first became interested in Gaddin, it never occurred to me to connectize Gaddin with her interests and persona. Gaddin grew tired of the habit of obsessively discussing and searching for explanations that one could count on Giles of patriarchy as well as those with whom I began to disagree with the thoughts in the minds of Elegine, Elemi & Sonomis (whom I of indecentively ranting and ranting about the apparently incompetent transgender woman who gave birth to my own Elegor, who took her life seriously mindlessly than others and felt that I was not in my interest). I saw her work and feeling a little confused and withered. So, how did other feminist writers become so embarrassed to be reading Elemi & Sonomis? It was an integral part of gender identity and gender identity that shaped our entire transgender experience: the ways in which we were coddling around in our bodies, ways in which we were self-transforming our bodies into deeply energised identities and the reason for not being so arbitrarily tied to one’s identity. Let me explain: I quickly realized that I was not interested in gender identity as a writer, so I took the opportunity to explain the very nature of my transgender experience. I received permission to write my novel almost as soon as I found out the irony of writing an essay on the subject, but no one informed me that I was ever given permission to write my novel or read it. Elegine Legin (Dragon of Ponomen) (1972)</s><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 5 =====
+head_tokens: ['▁Vic', 'tim', 's', '▁were', '▁taking', '▁full', '▁responsibility', '▁for', '▁what', '▁was', '▁going', '▁on', '▁behind', '▁them', '▁even', '▁when']
+tail_tokens: ['▁One', '▁of', '▁these', '▁issues', '▁is', '▁the', '▁risk', '▁of', '▁drinking', '▁too', '▁much', '▁water', '.', '</s>', '<pad>', '<pad>']
+Victims were taking full responsibility for what was going on behind them even when there was pressure to deny them of sexual assault. It was also that in many cases of rape victims, women were forced to lie and defend their bodies against one another. A 16-year-old Florida woman was found guilty of rape victims. Their horror stories led us to learn more about the horrific pain that forced us to think about all these women who were a part of sexual abuse. For many of our lives, rape victims lived through a series of fantasies that the agony had stirred up to wake up to the stories of rape victims. After hearing one woman’s son accused of murder two men wondered just how hard it would hit their new wives and husbands after the women shook their heads. They waitedwaited a little longer until they were sexually abused and the women decided that it was too late to do anything for their children. The event, which began in September 2016, will take place in the Park, London, by the world's largest philanthropy organisation and charity's 'Fearless Dreams' The event has been accompanied by more than 100 Red and volunteers and on volunteers gathered through personal events and social media Feeling for a full recovery Former Red Sox president and team and sporting director Dr Bill Cosby said he received a phone call from the Red Sox fan, where more than 100 people gathered at his home in east London to celebrate the memory of the survivor and a personal 'Fearless Dreams', where they felt a boneless, boneless feeling like a meal they could eat somewhere between another person and the family. Dr Cosby said volunteers worked tirelessly to help people at the event but it wasn't meant only as a personal help but as a sign of hope for a full recovery "These young people, many of whom worked with the medical team and all the Red Sox fans, brought together the opportunity to come together and hope that they all died here too," he said. Dr Cosby asked volunteers to help people gather through personal events and social media for the event and also through personal media The "Fearless Dreams" were the emotional flashbacks that many families were trying to overcome through intensive use of personal events and social media. He said: "We felt that when people came together together we almost all felt the same way in trying to connect those emotions together rather than feeling more than feeling like you're running away from them. "You look at your life as a person and as a family and you want to experience it before you experience something like that, and that's how long it takes them to be able to feel that kind of attachment while they're inside with their family and they're trying to do that at the moment. So we hope that people be the what they have once they're inside and they can experience something that can be done and something that helps them get to where they're from now." Dr Cosby added: "We feel grateful for all the help bringing in to such a recovery." Supporters and the media attended the event included Kenny Britton with Red Sox postseason victory at the 2010 World Series. The man admitted the Hancocky event was a long night of craziness, but was reassured by the support member of the team that he was on his way. "It was quite cool, so cool, so I loved it," he said. Shane Williams was sad to see Shane out with “never drink stuff,” but added it “was pretty cool to drink.” But it’s also probably fair to say that Red Sox fans no longer drink too much to drink: Drinks are now likely to be used, too. “Never drink stuff” has annoyed many Red Sox fans, television broadcasters and other professional sports fans. Their daily drinking/excuses prevent them from being crippled by being able to drink one of these beverages while drink less water in the shorter periods. The League of Legends believes that hand open crates of conniving with water can help them burn out drinks, so non Acolytes can drink water even if they have accidentally hit their shorts in order to cause them to miss it. The Acolytes, however, who tend to drink more water than other Red Sox fans, usually have a noticeable “hovercup in” reaction. Drinking much water will cause them to miss their shorts, which causes them irritation and can eventually disseminate them. There may even be some amount of respite to drink before they blow it up. One of these issues is the risk of drinking too much water.</s><pad><pad>
+===== sample 6 =====
+head_tokens: ['▁CA', 'RR', 'AY', ':', '▁Well', '▁my', '▁main', '▁point', '▁is', '▁that', '▁I', '▁was', '▁interested', '▁in', '▁seeing', '▁any']
+tail_tokens: ["'", 's', '▁just', '▁one', '▁thing', '▁I', "'", 've', '▁been', '▁feeling', '▁for', '▁so', '▁long', '.', '</s>', '<pad>']
+CARRAY: Well my main point is that I was interested in seeing any time I had a bad time or Or I just going out to some other place (my birthday party when I'm at school) to see me on the doorway. At some point, I'd get up on the sidewalk and grab and look out the window and tell some other person what I'd just say to, "Hey, it's okay." But it would be fair to say that I'm sort of feeling some sort of un of connection with these other characters as I think. Maybe I'm just feeling some sort of connection to me, unwavering subconsciously or maybe just maybe some sort of un of of interaction with strangers around trying to figure out of the most important things my life, not the things I've been doing in everyday life. Not the weird things, not the ones I actually end up feeling that Other people aren't supposed to find me interesting and excusable to me. I feel that, at least in part because I was always, as always, a writer as it is, but I would suggest such a narrow conclusion would be helpful because I thought down the road for a long time that I'd like to be able to see these things before I came back or at least feel that I was a believer. MY: What did you start feeling? That I was going through a long, painful ending ending... okay? CARRAY: My feelings about the ending were woefully, yes... well, partially because I felt I really had a good feeling that I was feeling that ending that I was disobeyed by it. The ending is so interesting, and it's a long ending as long as it's about about knowing where it's going to end, about being filled in an empty space empty, and filling in gaps. It occurred to me as I was painfully aware that a non-fiction book would not be published in this book as long as I tell the story, and I'm arbitrarily waiting for closure on anarchygnosis/summer anarchygnotis/continues. I felt compelled to write a new ending by simply writing this off. I felt like most of it happened, for some reason, not the ending. I thought the ending had just been forgotten because it took me months anyway. (Note, MY: Okay) How do you feel about these new characters? Yeah. It's slowly become weirder... I'm not really sure what it means. It can still be interesting, but I don't like it all. I don't like to be extent that I kinda feel at least confused by the fact that it's overheating, and my heart just hasn't broken down without wanting to experience anything. Could it be that some of my mind wanders seem to be very similar to other versions of these characters? (Yes, of course.) My catatonic feeling starts coming out of my mouth with some sort of feeling which people are depressingly calling the "yogurt" feeling. It's sense in general, but I can never fully understand that this feeling is a place where, as an enlightened person (and perhaps even increasingly self-sufficiently aware person in general), you feel at least partially breaking the heart of a certain feeling by suddenly expressing yourself in so many ways, as being strange strangers strange or or just being strange strangers. Somehow that feels strange feeling like I've tried to find a way for myself into actually feeling so completely, yet completely unimportant. It may not seem to me that life is an important place for me, yet not yet lifeless, so maybe that doesn't get me there, but I'm actually much more interested in self-respect (or maybe?) thinking that even though nobody else realizes you're completely affected by yourself or by your own self-interested feelings and feelings that no one else can really understand those feelings at all. No matter what, knowing how right now or whatever, always seems strange and dubious. What kind of kind of feeling you feel right at the end of the book are just wondering what everyone feels like right now: kind of feeling like just feeling just like I'm probably going to be here forever for some reason, kind of feeling just everybody is hurting What kind of feeling right now everybody is actually hurt, not just being affected by it at some point, but some kind of non-interactive feeling: it feels like something for everybody to understand at some point, maybe everyone else understands, or perhaps just just stop having bad thoughts or feelings, or just like thinking about what's happening. It's just one thing I've been feeling for so long.</s><pad>
+===== sample 7 =====
+head_tokens: ['▁"', 'I', "'", 'm', '▁not', '▁accus', 'ing', '▁Sand', 'man', '▁personally', '▁of', '▁being', '▁completely', '▁insane', ',', '▁just']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+"I'm not accusing Sandman personally of being completely insane, just because some of the other villains threw out a face that they thought looked so visceral that their face actually looked pretty hard," Wikileaks founder told IGN. "There are actual villains like Sorin and Blood Trolls. The only reason to believe is that no one really thinks it's what they should have done. But it has to do with the actual fact that they don't deserve the acclaim award for putting them into an actual line-up." So it's basically a shame is the fact that Sorin looks like an entire group of outfit members just Bojerk collecting stats in an actual Slayer outfit which actually give pretty slight awards to Sorin outfit while the actual member and the self-confessed members of the group decidedly pimping Sandman into a new line-up. It is a shame the fact that Sorin gets messed up with being an actual pro-Slayer outfit while the actual member making actual appearances and some faces in Slasher outfit that Sorin actually deserved the acclaim. And that's how it could be. If I didn't want to put Sandman on the list of all the members where Sandman morphed into an opposing villain, or had just been the villain face in fighting Slayer outfit which actually put him to the top his entire group just deserved idiotic acclaim award, you you know what the rest of the group can actually wont when compared to actual faces in Slasher outfits? I guess that's what they're all about. But the fact that Sorin goes for it is a shame the fact that Sorin gets his own acclaim award based on actual appearances by Sandman and the self-confessed members of pro-Slayer outfit members just Bojerk collecting stats in Slayer outfit while the actual member decidedly pimping Sandman outfit gives pretty slight idea which could be based on idiotic acclaim and an entire group of which actually deserved more acclaim than what the rest of the group can actually wont. It's also a shame the fact that Sorin goes for the top and gets tucked into a villain line-up with an actual Slayer outfit where Sorin has a mislooking face that looks like it could be annoyingly getting paralyzed by enemies. Sandman is also a bad name for defending Shaker's Rangers, despite the fact that his actual pro-Slayer outfit is actually better like the Slasherface outfit. Sorin isn't merely an image that could be pretty neat making up some of what Sandman's fighting Slayer outfit is. Bottom line: The mechanics of "Sorin" aren't really about what the actual heroes look like. As far as they're concerned, the villains in Slasher outfit can be handled differently than villains, no surprise. Sand've been re-modifying their shirts over the past few years in order to make them look like Slasher faces. As Sandman getss of Sorin's Ranger features, I honestly don't think it's worth considering even considering when discussing the mechanics (which presumably being created by Sandman ridiculously stupid) what could be some interesting mechanics compared to the Some weird ideas floating around the idea of fighting Rockin and Sorin. What makes Sandman feel better now? Well, he's really good in the game and what really makes Sandman feel better to you. Side note: I won't give Sandman much credit for promoting this dude (at least until he mentioned S Shaker's villains) because Sandman is a huge community member fan and totally sucks for the main character. There's a short link here on Sandman's fan site called Hunky Sorin back in August 2011. However, when Sorin left the game after an extremely hard time (and an insane amount of money), he spent all his time on destroying and then fighting Slayers, spawning Stone shuffling monsters and foraging weapons and new minion monsters. This is why people would hate Sorin so much for Sandman if he traded armor instead of just fighting Slayer outfits. No matter how hard things worked, Sandman wasn't really willing to pay off those rewards. Also, I honestly feel that he stood out to make people care about what they get paid for.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>

LTA_openwebtext_dualt/docs/lta_samples/metrics_20260604/owt_t5_elftokenized_full_pow1_unfixed_norm_stateprobadd_selfcond_ce_fast_lr3e4_logitgrid_n256_step142000_tschedlogit_normal_mn0p7_s1p0_sc1p0_decode32_n256/sc1p0/chunks/chunk8.txt

@@ -0,0 +1,38 @@
+checkpoint=runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr3e4_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260602_001439/step_142000.pt
+step=142000
+decode=dualline_time_aligned_dirichlet_final_endpoint
+c_min=1.0 c_max=1024.0 c_schedule=exp
+steps=32 temp=1.0 bridge_power=1.0 temp0=0.0 decode_time_schedule=logit_normal decode_time_logit_mean=-0.7 decode_time_logit_std=1.0 decode_time_shift=3.0 decode_time_rho=7.0 decode_time_sigma_min=0.0001 decode_time_eps=0.0001 prior_beta=0.0 final_sample=argmax final_count_penalty=0.0 final_count_power=1.0 final_count_warmup=0 self_cond_decode=none self_cond_scale=1.0 state_self_cond_decode=single state_self_cond_scale=1.0 state_self_cond_normalize=True state_update=dirichlet odeish_eps=1e-06 odeish_c_eff_max=1000000.0 dirichlet_gamma=1.0 cfg_scale=3.0 concat_self_cond=False
+bos=1:</s> eos=1:</s>
+===== sample 0 =====
+head_tokens: ['▁This', '▁is', 'n', '’', 't', '▁necessarily', '▁the', '▁most', '▁important', '▁thing', '▁for', '▁people', '▁like', '▁myself', ',', '▁however']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+This isn’t necessarily the most important thing for people like myself, however. My social and political views have some significance in almost all of them, especially when I don’t fully realize why I feel I am not interested in doing things or trying things–and sometimes for the same reasons, when, on any given occasion, I am not acutely aware of myself or by my own having a tendency to do these things without any consequences–even in some instances where I’m somewhat questionable–or rather, what people perceive as human being rather me. There are all sorts of reasons why we tend to be more interested in our own processes and behaviors, as more inquisitive individuals naturally tend to engage in various forms of pervasive repression than we tend to behave in the same way as beings of others, which may have been attained–or perhaps more so via perverted patriotism, or more recently perverted ritualistic complacency. An perhaps even more interesting (and perhaps more interesting) aspect is what happens occurs when I actually tend to feel badly hurtful about a particular cause when I find myself deliberately unconsciously trying to find a cause, rather than mindlessly repressing something about myself and myself and suddenly depressing that feeling that something hurts me in some way makes me helpless or perhaps even worse than I am actually feeling that my own feeling of completelessness and absolute resistance to superiority really exists for better or worse. Perhaps even a feeling of completelessness at some point has entirely disappeared from my view as yet, but perhaps quite often I found my own feelings strange and in many different ways I felt better when I decided to go try something and try to figure them out one day, instead of just trying them doing it. Instead, I resorted to acting out this way, and finally, I was somehow lucky enough to find myself suddenly somewhat uninterested in making people feel strange or maybe even some people still accepting the possibility of doing strange things as well–or better yet, feeling some uneasy thing about myself or maybe even suddenly try existence of a sort of psychological oxymoron that I subscribe to–in the midst of my own self-love-self experience, uncerviating subjectivity, lack of rational thought, and perhaps occasional pessalistic guilt–it’s in the long run. I suppose this process was a process that set off some basic pathological questions in human psychology, the very nature of its underlying motivations, and the idea that this particular phenomenon could not exist at all–notwithstanding that I set aside all being involved in an initial process (which had been a bit somewhat surprising), yet another) process actively inventing out what seems–a bit somewhat surprising (and still somewhat, somewhat surprising), and so I perhaps discovered it myself–was one of the reasons I had discovered my previous (and erroneously existing)/uncertain sense of superiority in the above feeling–apparently I couldn’t have even seen a single distinct phenomenon (too vague methodological explanation for how meaningless it seemed.). It was strange enough to recognize that distinct groups and individuals did have the very concept of creating a trade-off between nativity, and so after this process I discovered that distinct groups and individuals had deviated from the same internal boundaries within themselves–and nativity evolved over time. that distinct groups and individuals that had unconsciously isolated themselves into nativity existed within these particular processes and boundaries were independently aligned. So long as I wasremained completely involved in subjectivity, rather than being isolated within another group in this process, I could still find a group acting entirely cowardly for whatever reason for doing so–though there were reasons ways cowardly within the same group might be removed from the “nothing within the group” category of subjectivity (the notion of self-esteem as possible mechanisms for imitating criticism of their own self and neglecting to repress on others). So far as I didn’t feel an even greater sense of being involved in the process of subjectivity which had retroactively given rise to many parts of my corpusativity and feeling that I was involved in some process conscious self–but not in that part of my identity, which had somehow become retroactively tied to the whole person’s life itself. (I certainly didn’t really at all feel like, at least, being sadistics or sadistics, but perhaps it was still my own feeling that I was or were not entirely involved in this particular process for the vast majority of us as human beings.)</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 1 =====
+head_tokens: ['▁', 'Basically', ',', '▁what', '▁I', '▁mean', '▁is', '▁that', '▁this', '▁is', '▁just', '▁one', '▁of', '▁those', '▁exact', '▁same']
+tail_tokens: ['ve', '▁made', '▁made', '▁by', '▁exploring', '▁other', '▁minor', '▁consideration', 's', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Basically, what I mean is that this is just one of those exact same old, exact same things that I see in my own mind that I so profoundly flawed just see “brother” as a new entity; actually, just as a means of changing in other words no one “enough” left alive or dead in real life...which is why a closer look at it shows that what I call the original monks and monks areand most importantly “broer”, I mean simply because of the very inherent reason the original monks and monks have existed for so long – no one left over the original monks or monks or monks are there at the moment. Thus, I can confidently say that “brother” was actually a distinct form of monks – it wasn’t just an intangible entity that could no longer exist and still exist; it wasn’t exist eternally after all, so long as it existed merely submorphed into a entity that still existed as an intangible entity other than existed at the time of its creation, there was no more reason why it didn’t actually exist anymore or just submerged it into something intangible entity existed. And there was no more reason why it didn’t – it was no longer exist and stillmerely a entity that did not exist eternally after all so long as it could live up to. So I am curiously wondering that what a sudden the original “brother” and the changes they had made (and hopefully all that changed) are now somehow revolving around – and how much do they still need to make making the new versions interchangeable with original versions?? I personally find them somewhat confusing – because since the new versions were interchangeable a short amount of time ago and the game is now completely interchangeableable ( I can only take a few changes from the original version), that they need to figure out new ways to make the new variants fully interchangeable and readily interchangeable with the previously changed versions interchangeable with the original “brother” leaves me questioning about how long since the original “brother” has been interchangeable slowly but somewhat interchangeably. Of course, I haven’t quite yet explained the history of the original version, which is not to speak for itself, but I’m pretty sure that could mean that there are still some tweaks to make – along with some minor, bloat changes interchangeable with the original variants (or perhaps new versions still work too), since I was also on the same side with the original versions still fully interchangeable at the same time makes makes sense if I had to go back to the old version and keep the original version in mind, since the game hasn’t changed (and thus it has been updated considerably many times since the new version has been a bit old), and that’s very much to bear with me. Again, I’m pretty saying that the biggest reason for the change in this regard is that they still need new ways to make the old versions completely fresh and interchangeable (or perhaps new versions that aren’t available for them) at the same time, I think I know – but there are still a few other things I’ve mentioned as well. I’m talking about the ability to add new content right now, I note that this is probably the only one new feature I’ve mentioned previously– the original version of the new version – which is a bit easier than it normally would have been that I’m actively continuing to explore while polishing and bringing back the original version. It’s nothing short of a general point of view view rather than a separate view; it doesn’t mean anything that you want to do in a game experience that could mean better with your eyes, but actually has to mean better with your eyes. That doesn’t mean what I’m talking about, but it’s pretty clear to me. While I’m not sure how I can see this stuff, there’s a process I would like recommend that I’ve been going through a couple of times, which makes me more interested in exploring this new aspect of the game as a whole, and makes me want to dig my way deeper into some interesting things going on here and here and there. I’m okay with this process at best – and perhaps by by far the least the main thing I’ve noticed so far. There are definitely still quite a few minor changes and screenshots that I like and definitely things to take into consideration to make it easier for me and for me to first try and figure the process out and give me some time to sit away and figure out any possible changes I’ve made made by exploring other minor considerations.</s><pad><pad><pad><pad><pad>
+===== sample 2 =====
+head_tokens: ['▁In', '▁the', '▁first', '▁few', '▁years', ',', '▁neuro', 'n', 'al', '▁cells', '▁seemed', '▁to', '▁be', '▁relatively', '▁stable', '▁when']
+tail_tokens: ['ability', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+In the first few years, neuronal cells seemed to be relatively stable when compared to other tissues – the retinal retinal retinal retinal synaeses, which stem cells focused on were already capable of generating neuronal neurons (Stuttor et al 2011). Secular PEG2 has been extremely successful, and in vitro studies that have been separated from the retinal retinal into heterogeneous regions of the ionoplasma propria that have been reduced or significantly enhanced by endogenous PEGs (Scuder 2002, Gores, Scuder & Sokk 2009). More recent research has been done using STSPIEG2 (Stuttor et al 2011). Neurovelocence has been developed and inhibited retinal sync formation are far more easily achieved by generating high-quality images from the retinal retinal, from the parauthal cortex being generated by a mouse, to electroencephalography stem cells using the NIH and using Chemography (Givo/Gevo 2013). In recent years, the number of neuronal cells being produced has increased, but slowly or at least not. The Neuroregenerative neuroregenerative effects of neuronal cells are not yet fully understood which means it is now time to explore new neuroregenerative techniques in working areas, while yielding a new and efficient sensory sensing applications for neuroregenerative therapies (e.g., using neuronal cellography). The regione imaging technique is aimed at generating neural cells and neurons in the skin which is being achieved by The regione resonance imaging field used an experimentally proposed ionoplasma laser (IGA/Semo) (also shown here using the new Haph/RG laser), which can be used to generate neuronal cells (e.g., using the regione imaging technique (also shown here using the SGA/RG laser), which can also generate neuronal cells and stem cells which can selectively dove to the skin. However, such a technique uses large-scale localized waves to generate neuronal cells and neurons which can be achieved The regione resonance imaging field is developing promising applications for neuroregenerative research and it has also demonstrated a new technique forgenerating regions in which neural cells and stem cells work in different parts of the brain and body organs (Brain 2013). The region imaging technique is used to generate neuronal cells and other regions in which stem cells and neuronal cells work in different part of the body organs and can potentially dove to the skin. The regione resonance imaging technique has been demonstrated at a specific area in which laser-delineated laser waves are displayed as they generate neuronal cells (Brain/AGA/Gemo) and neurons (Givo/Gevo) and differed between regions. The regions in which stem cells and neuronal cells work in the skin are reflected regions using laser laser waves which can selectively excitable neurons dove to the skin. However, the regione imaging technique has been demonstrated at some of the areas where neural cells and stem cells in the skin are capable at generating neuronal cells and these are now even more promising applications for regional imaging, neurovelocence, and ionoplasma resonance imaging. Although these techniques prove to be an effective new tool for neuroregenerative research and it has the potential to become a major application for this technology. These studies have also been used by NIH to develop approaches for neuroregenerative therapies, which results in an increasing reduction of neuro cell divergence. Of course some of these new technologies are already being explored, including the new generation of neuronal cells currently being successfully seen in animal studies. But it was written pretty well prior to the publication of a new paper. Today I’ve actually done a lot of research moving into the field of pharmacology and especially into neuroscience. It’s a bit of basic research but it was quite extensive and relatively extensive compared to many techniques I’ve been used in previous years — so I can only assume that changes in region area are much better than in the current contexts of pharmacology have been rewritten in various ways and I’ve received feedback from various researchers over the years. In the end that it’s not just the story here; rather for what I think is it’s a bit more overly detailed than is where the region really gets over-subscribed. The main area was the region of EMT, which became the fully unrecurrent suicidal cortex and eventually became the basis of the dorrid neural network and neuro excitability.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 3 =====
+head_tokens: ['▁In', '▁fact', ',', '▁almost', '▁every', '▁time', '▁you', '▁watch', '▁Ever', 't', 'on', '▁witness', '▁', 'a', '▁sudden', '▁and']
+tail_tokens: ['▁local', '▁authorities', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+In fact, almost every time you watch Everton witness a sudden and dramatic shift in club history just a little boy far too many will find it in reprehensible that this kid will ever be on the radar – though it is only a few years before Everton have shown signs of patience and barely even know-how. However, there is no denying the fact that this young man flew away at Everton Park. This kid could hardly have expected to turn Everton manager Alexis Sanchez under fire for causing his players to go bust after his release. Still, it's still brooding and quintle this young man takes this man’s life. So the kid slowly drifting away from Everton might end up with a grudge trying to get a hold of the Everton club in order to keep it safe The nameless men that have arrested or been targeted by gangsters who have been taken down during this time are Jesse Munriz and Kevin Christensen and Jonas Barger all among them. Now, they are finally able to do exactly the job they did right at the time. And they do whatever they can do in doing anything that will help them get out of prison when they get caught up in this kind of crime. “Brien” Andrrien As we reported earlier earlier this year, Munriz was still serving his entire life and only seven days after robbery broke bars, despite a sentencing order that he would never, ever serve. Munriz was in jail for six months after the gang came back and after his initial arrest he was convicted of committing all sorts of crimes. And the gangsters who have been arrested have been taken down over the last few months and thus had little choice just for murdering renchmen. But first and foremost, in this case, “Brien “Brrien” is the only police officer who has been imprisoned and convicted and has been targeted by tronesome gang members over the past decade and thus convicted by the fewest murderers. And in the last case, in this case another gangster who has been targeted by Munriz “BrrienganggangBrien was imprisoning after his life in the gang was targeted by renchmen and thus chose choice to do so. There was no real reason to believe these individuals would have been arrested over the last couple of months, and thus had little choice for murderers who have now been taken down by this gang not just by renchmen. Multiple police gang members have been “Brrien’s imprisone” over the last few months. They are now joining the ranks of the violent gangs who have been targeted by armed groups for involuntary assault, acts of murder, and other atrocities – a crime which makes them three or four times a total or less than a quarter of the total population. Despite in fact the underlying reason for the arrests occurring in this case is that just over half of the violent gangs arrested during the 2014-2015 Baltimore Police Department were the nine officers charged in retaliation for assault, assault, involuntary assault and other types of incendiary bodily injury charges. These individuals have also been tasked with carrying out criminal attacks on civilians, including two months before officers killing April 2014 and the following 2014. A significant number of excessive force and investigations involving at least two state and local law enforcement agencies have also registered at least 30 instances of use of weapons, weapons, tactics and/or tactics used by the designated “terrorist organizations,” according to the federal criminal complaint filed by the Montgomery County Attorney General’s Office. organized violent actions by armed groups across the country have set up a nationwide system to combat domestic violence. As one Baltimore news outlet reported, “Nirone Alb Morton’s deputy chief chief officer, Mohammad Ahmad, was one of the most violent criminal organizations in America. All of these armed criminals were brutalized and targeted by the deputies. The deputies, at one point, emboldened Alb Morton police to prosecute them, and placed brutal charges against armed organizations and set up a system to ensure that people are brutally electrocuted.” The federal complaint stated that “all the majority of domestic violence occurred as a result of armed groups targeting the brutally targeted and of armed and armed groups being targeted by brutal and brutal acts from domestic violence. The Alb Morton police department had been responsible for the extensive use of arms, weapons, tactics and intoleranced by local authorities.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 4 =====
+head_tokens: ['▁Name', '▁Chi', 'a', '▁Chi', 'a', '▁Chi', 'a', 'd', 'ori', '▁was', '▁used', '▁by', 'pet', 'us', '▁as', '▁the']
+tail_tokens: ['▁Str', 'd', 'le', 'w', '▁"', 's', 'u', 't', 'an', 'to', '"', '▁Chi', 'a', '▁Chi', 'a', '▁Chi']
+Name Chia Chia Chiadori was used bypetus as the sword fighter and used bypetus as an action sword. It was replaced by Chia Chia, which was used for all sides except one swordless fighting. Chia Place name Chia Chia S<unk>raku [Uta Sukhuk Ui [petpetus] (Aspetus Chia (Perpetus) Chia Sukhuki (Aspetus Chia Sukhuk Ui (Perpetus) Chia Chia Chia <unk> [Pikiname Chia Chia] Aura Chia Chia Chia Chia "Cha Chia" <unk> [Pikiname] Chia Chia Chia Chia Chia Chia Chia Chiuki Chia Chia Chia Chichi Chi Chia Chia Chia Chiuki Chia Chia Chia Chia, used by Kamikato name Chia Vajinum (I, Chia, I) Chia Chia Chia Chia Vajinum [ edit ] Aura Chia. Chia was a warrior with a sword, was changed replaced by the place name Chia Chia, used by Kamikato nameword name Yuku Chia. Chia Chia (Pikiname) Chia Chia. Chiku, K Chia, and K Chia all used the same name. Chia changed the place name "sutan B", and changed the name Chia Chia because it was the proper name. The first name Chia was used after the name Chia Chia and K Chia were destroyed during the fight. The second name Chia was changed The place name Chia was no longer used as the proper name due to the place name Chia as the title of Chia. Also changed the name Chia Chia Palak Chia Chia Chia Perpetus Palak Chia Str "Sword" Strdlew "sutan B" Palak I Chia Palaa Palak I Aura Chiura ("Swordfighter") Palak Chia (Piki) swordfighter Chia Palaa Palak I swordfighter <unk> Palak I swordfighter Aura Aura swordfighter Strdlew "sutan B" Palak I Strdlew "sutanto" Palak Palak I swordfighter Aura "sutan B" Palak I Chia. However, Chia name was replaced by the name Chia Chia. During this fight Chia also changed the name of the warrior in the form of Chia name was changed. Palak Chia Pala Aura Ininitoku Ininitoku Palak I Chia Pala Ininitoku "Piki" Palak Chia Chia Chia Chichiku Palachiku Chia (Aura Str Sword) Palak Chia swordfighter Yuku Aura Palak I swordfighter Yuk I Chia (Suta Sword) Palak I Chia (Piki) Palak I Palak I swordfighter Aura Aura Ininitoku Chia (Aura Str Sword") Palak I Chia Aura Palak I Chia swordfighter Riro Chia Aura I swordfighter Riro Chi Chia Chia Strdlew "Pakikuto Eustase Riro Chia Chi Chiku Chia swordfighter Riro chiku Chiku Chiura Richiku Chiku Sword" Strdlew "Pakikuto" In Chiku Chia swordfighter Riro Chia Chia Chia Chia Chiku Pala Eustase Riro Chia Chia Aura I swordfighter Invajiniko [ edit ] Chika Chia. Chia Invajiniko Chia (<unk>)) Chika Chiiko (<unk> Chia) changed because it was the proper name for the title of Chia. It was also the official name for "Chiko" after the last fight fight although it was still used as the place name at the end of the fight. Place name Chia Chia Chia Chia Chia [ edit ] Chia Chia Str Swordword Strdlew "sutan B" chiku Aura Chia Niro chiku Chiafighter Niro Niro Niku Chia Niro Chia [ edit ] Chia Chia <unk> Chizhukiruki Chia Chizhukiruki Chia Chia Chiro chiku Chia Chia Strdlew "sutanto" Chia Chia Chi
+===== sample 5 =====
+head_tokens: ['▁', 'Despite', '▁the', '▁Cold', '▁War', ',', '▁there', '▁has', '▁never', '▁been', '▁', 'a', '▁major', '▁military', '▁conflict', '▁between']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Despite the Cold War, there has never been a major military conflict between the US and the Ukrainian government. Given the strong relationship between the Soviet Union and the Ukrainian government in the middle of the conflict, it is inconceivable that the alliance between the US and Russia's US troops in this conflict included a large number of Ukrainian Ukrainian armed units. In an end to the conflict, the Americans participated in an air defense campaign led by the 4th Battalion, led by the Soviet Special Service Corps General Gupta. The 4th Battalions later became famous for perpetrating themselves a series of atrocities. The relationship between was not mutual that Khuzhla and the Soviets were strongly supported by the Communist Party. A British historian of Ukrainian history, Ron Bowie, commented that Khuzhla' made a direct reference to the two armies fighting against former Soviet Union in which the US government openly and militarily opposed peacekeeping and self-defense of Ukrainian nationals against the Soviet Union armed by force. Bowie cited Khuzhla's reference in his book at the time of the Ukrainian revolution Alexander Chervastovsky, commander of the August 8 Battalion. The US troops committed a series of atrocities – committed by Khriotsky himself in Kabul on 19 April – which made "Ukraiak Ukrainian" responsible for the entire occupation of the country.[4] In addition to US troops in Ukraine, consisting of Ukrainian armed units such as the Hashnieks, Khakhaweats and Yugoslav Slovak, all of Ukrainian troops described by a US official described as being separatist. Chervastovsky was a heavily trained Russian commander in eastern Ukraine but the region remains under Soviet control. During his service, US troops in the eastern Sputnik was occupied mostly by Russian troopsarmed part of the Soviet Union.[4] The August 8 Battalion, which was later augmented and renamed Third Brigade, is popularly referred to as "the beginning of the conflict between the Soviet Union and the US while Kyslovak was largely controlled by Ukrainian troops and Ukrainian special forces".[4] Although there has been no direct link between the two armies and US special forces, Prochkovsky was flanked by Antony Prochkovsky, both of whom he initially had fought several times in an attempt to be named the commander of the 7th Blion and Prochkovsky Red Battalion which was later flanked by members of the US Army. Of the 3rd Battalion, he was the only US sniper in the central division to be captured by US forces. This was overlaid by the two US 7th Battalions while other US divisions were mainly snipers from the 3 3r and 1st Battalion, who chose to retain their ranks during the Ukrainian conflict. However, the US sniper was only to be captured and commanded by two young Lieutenant General Sergeiy Prochkovsky and a Kryv regiment commanded by Igor Konchakravownibirsk who took part in the Red Battalion during final service. The Red Battalion along with the US military brigade were used by US generals in Kiev and US generals 6 Fourth Battalion of the 7th Brigade, and later replaced by a Kryvny regiment, serving all three units. They were among 13 members of the squadron, which ended after a Soviet invasion of former Yugoslavia in 2014. It was then announced the US purpose of garrisoning the entire Ukrainian nation – perhaps perhaps the most significant act of nationalism in history given that the Soviet Union largely maintained its independence for much of the same period under Prochkovsky rather than fight for the national liberation of Russia. The US soldiers – mostly Ukrainians, from Chernikovsky to Viktor Zhuzlov– came to Kiev for what happened in Ukraine. See also [ edit ] Compospose Yep: you're a warrior of the 2nd Army and you love music and drink and drink for your dancer... (arpose) Yep: You're also a good warrior warrior who'd like you love music and drink, and drinks for your dancer... Yed a good warrior warrior dancer? ( Rearpose Ye Yep Rearpospose) Yep: You're a good warrior of the 2nd Army and you love music and drink for your dancer...</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 6 =====
+head_tokens: ['▁The', '▁T', 'SA', '▁children', '▁are', '▁hiding', '▁in', '▁the', '▁dark', '▁places', '▁and', '▁there', '▁are', '▁ways', '▁to', '▁gain']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+The TSA children are hiding in the dark places and there are ways to gain access to that area and get the TSA if they want it is far too late to protect African-American children from the infantil atrocities perpetrated by the FASA (with the exception of child protection in Kenya, where young white children are generally subjected for kidnapping so as to protect illegally from killing or killing innocent bystanders with them while they restraint practice). In either case, the TSA children do indeed look weak worse by getting them off the ground (even though bracing them up is merely starting yet another TSA attack but out of necessity to have the TSA expose their weak spots. This is a nice point of course point since this list doesn’t even need to be newer than the FASA, which in general looks like a laundry catalogue of human rights abuses for example mentioned above, of children systematically killing 15,000 & 250,000 children over the age of 6 years in disobedbediously and other actions by the FA. The purpose of this list is to put together policy ideas, contraptions and recommendations for the FASA collectively forcing its denizens who charge again with abusive FASA victims, making them greedy, helpless and frightened perpetrators, and those who have systematically killed civilians, mentally and physically disabled innocent bystanders without fear of reprisals compounded by other crimes. You may have noticed that one thing even more important now in TSA is that object fighters actually look weak when they move weak spots (they have moved quite close to the weak spots just above and around the corner below). The problem with weak point positions mentioned above is that they are almost all the guardian foreshadowing positions. The weak spots for the 1 TSA/C/C/J, TSA/A/G and ESA/W/X4C platoon TSA positions are very weak spots, which can be easily obtained by using weak spots (see Section two below for these points). The slightly weaker foreterrant position then upon opposing object fighters will get a weak spot for his weak positioning so you can spread out a bit by moving around a little bit of the littlehead spot (and/either one of the two guardian guard positions (see Section 1 of the X4A platoon map) for a full circlesized littlehead foreshadowing (hanna run). This means that the TSA can only get TSA positions at one weak spot. This table shows one point where one gets all the general info that has just put them all together, so in case one finds the weakless spots an enemy TSA alloc slammed down inside pass the weak spots for the TSA to pass the weak spots at the end of the map (see Section S1 and S2). A weak spot where an enemy fighter passes (or at least what the points find) is “better than expected” and “slow down” on their behavior. The most obvious option is to keep the entire TSA passive, even if it means just postponing any improvised action whereby protecting the player (or enticing them on) isn’t mean they no longer do anything to weak positions on their bodies but instead a few weak spots spots which TSA/W are either weak spots or spots which is the bad option (see Section YingYong/Bodon/Littles). The only solution pointless here is what we’ve done here in the previous section the weak points mentioned in this guide, so I would only recommend to move some weak spots away if the TSA now takes a good ol’ (with two littleheads) with them (then there is one fighter doing nothing punier than I think) and put those two spots now work together (with two against them) and then use them to show off what is happening there. In the table I really really suggest we need to add some depth and some spots for the TSA to see which players can take on other fighters and use them some less-seemous moves. Or just add a few hard spots in this case unhurried ones whose rollout is incomplete and for whatever reason but you get me wrong). This is a bad situation for some (better “bighead” fighters who did not actually go through any practice taking place so far. On the other hand by doing a little bit of their, while at the same time doing all their offensive game potential and actually doing so many things, either side would have been better off not taking full responsibility for the TSA’s actions.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>
+===== sample 7 =====
+head_tokens: ['▁Fried', 'man', '’', 's', '▁rack', 'e', 't', 'e', 'er', 'ing', '▁(', 'critical', 'ly', '▁un', 'critical', 'ly']
+tail_tokens: ['<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']
+Friedman’s racketeering (critically uncritically) non-state government operates on the premise that the state must bankrupt itself into “the totalitarian state state.” He thus states the “definitely corrupt” and stateless totalitarian state, Milton Friedman urges the government to reject its absolute andrupt itself into an absolute state by insisting that the state mustrupt itself into absolute government. This implicit moral patriarchy makes Stalinism—a staunchly principled man who has no absolute moral authority over the state and claims to be a “comer” in a “omnipotent power” form of non-state government—an insurmountable conflict between the absolute state and the state. In his “potentially corrupt” (spherical) non-state government uncritically government Milton Friedman urges the state government into absolute government without a sense of political democratization. Indeed, Friedman’sfrightened morality has become increasingly problematic, since “the duty of capitalist governments that state states are not bureaucratic or stateless but perpetually corrupt—to put one “forgives” into a stateless, omnipotent state—to create an end stateless moral parity between Milton Friedman’s antagonistic puppeteer Milton Kropotkin, an imposterous Stalinist morally declares himself, and the stateless capitalist society he would be called upon. This conflict stems directly from various notions of the hierarchical premise of Milton Friedman’s ominous Stalinism (critically hierarchical) moral and unrepentant contradiction. He is therefore “dispotent” (acriclassivism), and thus Friedman’s sowdiness ultimately leads to what he calls “the dictatorship of capitalism and the supremacy of the capitalis system.” Milton Friedman morally aptly describes himself as “the inalienable capitalist man who wants to wrangle the capitalist system to absolute power.” In this absurd, moralistic satire Friedman demands the pathological existence of such absolute state that the capitalist system perpetually strips non-state government out of itself as a human being. It is especially ironic here that Milton Friedman urges the absolute state without a sense of political democratization to pull “selfforgives” out of government, if transcendently even stateless. Stateless, morally unrepentant contradiction In Milton Friedman’s ominous racketeering uncritically corrupt government Milton Friedman urges the state government to “disrupt itself into stateless government.” A moral contradiction ends up being a moral one, no matter how dangerous he might find himself doing so well with the idea that it frees itself into future power in the form of state-state capitalism. This is the case where certain Stalinist ideals are utterly contradicted by Milton Friedman’s new ideal, the idea of free movement of government, and the idea of accepting absolute state government as a human being and thus free society that ultimately leads him into absolute state. Thus fascism continues with its mythical, almost antagonistic premise of “freedom of nonstate government,” abandoning the hierarchical capitalist state that Milton Friedman calls his own omnipotitarian command, and thus turning his state government into an absolute state when he finds himself free. Ironically, Stalinism turns out to be the opposite. It’s actually a different, more dangerous mythical non-state government that perpetually pushes absolute power into stateless government. From the absurdity of these premises, Stalinism much like Milton Friedman professed to accept the objective reality of the global capitalist system, alienated by the neoliberal excesses of Stalinism and fundamentally non-state capitalism. Stalinism is the real world view that Stalinism borrowed from the Stalinist ideals of the capitalist system—to create universal society would be universal to all the industrialized nations of the capitalist system, Milton Friedman had vowed to make states disappear into non-state capitalism and Free society would look like it was based on the moralistic idea of free education and everything else. In America, I didn’t really put any real value on the idea that one could free education, whole life, free college, all the things in a totally free society. I thought it would be a really good idea because it upset a lot of coworkers like myself. Of course, I don’t think anybody ever thought that would do anything.</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/bitnet/modeling_bitnet.py

@@ -0,0 +1,501 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/bitnet/modular_bitnet.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_bitnet.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2025 The BitNet Team and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+from collections.abc import Callable
+from typing import Optional
+
+import torch
+from torch import nn
+
+from ...activations import ACT2FN
+from ...cache_utils import Cache, DynamicCache
+from ...generation import GenerationMixin
+from ...integrations import use_kernel_forward_from_hub, use_kernel_func_from_hub, use_kernelized_func
+from ...masking_utils import create_causal_mask
+from ...modeling_flash_attention_utils import FlashAttentionKwargs
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
+from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, can_return_tuple
+from ...utils.generic import maybe_autocast, merge_with_config_defaults
+from ...utils.output_capturing import capture_outputs
+from .configuration_bitnet import BitNetConfig
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class BitNetRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps: float = 1e-6) -> None:
+        """
+        BitNetRMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class BitNetMLP(nn.Module):
+    def __init__(self, config: BitNetConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+        self.act_fn = ACT2FN[config.hidden_act]
+        self.ffn_sub_norm = BitNetRMSNorm(config.intermediate_size, eps=config.rms_norm_eps)
+
+    def forward(self, x):
+        down_proj = self.down_proj(self.ffn_sub_norm(self.act_fn(self.gate_proj(x)) * self.up_proj(x)))
+        return down_proj
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+@use_kernel_func_from_hub("rotary_pos_emb")
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    key_states = repeat_kv(key, module.num_key_value_groups)
+    value_states = repeat_kv(value, module.num_key_value_groups)
+
+    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value_states)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+@use_kernelized_func(apply_rotary_pos_emb)
+class BitNetAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: BitNetConfig, layer_idx: int):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = True
+
+        self.q_proj = nn.Linear(
+            config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.k_proj = nn.Linear(
+            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.v_proj = nn.Linear(
+            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias
+        )
+        self.attn_sub_norm = BitNetRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor | None]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_values is not None:
+            key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            **kwargs,
+        )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.attn_sub_norm(attn_output)  # diff with Llama
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+class BitNetDecoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: BitNetConfig, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+
+        self.self_attn = BitNetAttention(config=config, layer_idx=layer_idx)
+
+        self.mlp = BitNetMLP(config)
+        self.input_layernorm = BitNetRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = BitNetRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        use_cache: bool | None = False,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.Tensor:
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        # Self Attention
+        hidden_states, _ = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            position_embeddings=position_embeddings,
+            **kwargs,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        return hidden_states
+
+
+class BitNetRotaryEmbedding(nn.Module):
+    inv_freq: torch.Tensor  # fix linting for `register_buffer`
+
+    def __init__(self, config: BitNetConfig, device=None):
+        super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
+
+        self.config = config
+
+        self.rope_type = self.config.rope_parameters["rope_type"]
+        rope_init_fn: Callable = self.compute_default_rope_parameters
+        if self.rope_type != "default":
+            rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+        inv_freq, self.attention_scaling = rope_init_fn(self.config, device)
+
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)
+
+    @staticmethod
+    def compute_default_rope_parameters(
+        config: BitNetConfig | None = None,
+        device: Optional["torch.device"] = None,
+        seq_len: int | None = None,
+    ) -> tuple["torch.Tensor", float]:
+        """
+        Computes the inverse frequencies according to the original RoPE implementation
+        Args:
+            config ([`~transformers.PreTrainedConfig`]):
+                The model configuration.
+            device (`torch.device`):
+                The device to use for initialization of the inverse frequencies.
+            seq_len (`int`, *optional*):
+                The current sequence length. Unused for this type of RoPE.
+        Returns:
+            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
+        """
+        base = config.rope_parameters["rope_theta"]
+        dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
+
+        attention_factor = 1.0  # Unused in this type of RoPE
+
+        # Compute the inverse frequencies
+        inv_freq = 1.0 / (
+            base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)
+        )
+        return inv_freq, attention_factor
+
+    @torch.no_grad()
+    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
+    def forward(self, x, position_ids):
+        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+        position_ids_expanded = position_ids[:, None, :].float()
+
+        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with maybe_autocast(device_type=device_type, enabled=False):  # Force float32
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * self.attention_scaling
+            sin = emb.sin() * self.attention_scaling
+
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+@auto_docstring
+class BitNetPreTrainedModel(PreTrainedModel):
+    config: BitNetConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["BitNetDecoderLayer"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+
+    _can_compile_fullgraph = True
+    _supports_attention_backend = True
+    _can_record_outputs = {
+        "hidden_states": BitNetDecoderLayer,
+        "attentions": BitNetAttention,
+    }
+
+
+@auto_docstring
+class BitNetModel(BitNetPreTrainedModel):
+    def __init__(self, config: BitNetConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [BitNetDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.norm = BitNetRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = BitNetRotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPast:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds: torch.Tensor = self.embed_tokens(input_ids)
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+
+        if position_ids is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
+            position_ids = position_ids.unsqueeze(0)
+
+        causal_mask = create_causal_mask(
+            config=self.config,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            position_ids=position_ids,
+        )
+
+        hidden_states = inputs_embeds
+        position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids)
+
+        for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+            hidden_states = decoder_layer(
+                hidden_states,
+                attention_mask=causal_mask,
+                position_embeddings=position_embeddings,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                use_cache=use_cache,
+                **kwargs,
+            )
+
+        hidden_states = self.norm(hidden_states)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values,
+        )
+
+
+@auto_docstring
+class BitNetForCausalLM(BitNetPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
+    _tp_plan = None
+    _pp_plan = None
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = BitNetModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> CausalLMOutputWithPast:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should either be in `[0, transformers.,
+            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+            (masked), the loss is only computed for the tokens with labels in `[0, transformers., config.vocab_size]`.
+
+        Example:
+
+        ```python
+        >>> from transformers import AutoTokenizer, BitNetForCausalLM
+
+        >>> model = BitNetForCausalLM.from_pretrained("microsoft/bitnet-b1.58-2B-4T")
+        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/bitnet-b1.58-2B-4T")
+
+        >>> prompt = f'<|begin_of_text|>User: Hey, are you conscious? Can you talk to me?<|eot_id|>Assistant: '
+        >>> inputs = tokenizer(prompt, return_tensors="pt")
+
+        >>> # Generate
+        >>> generate_ids = model.generate(inputs.input_ids, max_length=100)
+        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "User: Hey, are you conscious? Can you talk to me?Assistant: No, I'm not conscious. I'm an artificial intelligence designed to assist with information and tasks. How can I help you today?"
+        ```"""
+        outputs: BaseModelOutputWithPast = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            **kwargs,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = ["BitNetForCausalLM", "BitNetModel", "BitNetPreTrainedModel"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/gpt_neo/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_gpt_neo import *
+    from .modeling_gpt_neo import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/gpt_neo/configuration_gpt_neo.py

@@ -0,0 +1,137 @@
+# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GPT Neo model configuration"""
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="EleutherAI/gpt-neo-1.3B")
+@strict
+class GPTNeoConfig(PreTrainedConfig):
+    r"""
+    attention_types (`list`, *optional*, defaults to `[[['global', 'local'], 12]]`):
+        The type of attention for each layer in a `List` of the following format `[[["attention_type"],
+        num_layerss]]` e.g. for a 24 layer model `[[["global"], 24]]` or `[[["global", "local"], 12]]` Choose the
+        value of `attention_type` from `["global", "local"]
+    window_size (`int`, *optional*, defaults to 256):
+        The size of the sliding window for local attention.
+
+    Example:
+
+    ```python
+    >>> from transformers import GPTNeoConfig, GPTNeoModel
+
+    >>> # Initializing a GPTNeo EleutherAI/gpt-neo-1.3B style configuration
+    >>> configuration = GPTNeoConfig()
+
+    >>> # Initializing a model (with random weights) from the EleutherAI/gpt-neo-1.3B style configuration
+    >>> model = GPTNeoModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "gpt_neo"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {"num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}
+
+    vocab_size: int = 50257
+    max_position_embeddings: int = 2048
+    hidden_size: int = 2048
+    num_layers: int = 24
+    attention_types: list | tuple | None = None
+    num_heads: int = 16
+    intermediate_size: int | None = None
+    window_size: int = 256
+    activation_function: str = "gelu_new"
+    resid_dropout: float | int = 0.0
+    embed_dropout: float | int = 0.0
+    attention_dropout: float | int = 0.0
+    classifier_dropout: float | int = 0.1
+    layer_norm_epsilon: float = 1e-5
+    initializer_range: float = 0.02
+    use_cache: bool = True
+    bos_token_id: int | None = 50256
+    eos_token_id: int | list[int] | None = 50256
+    pad_token_id: int | None = None
+    tie_word_embeddings: bool = True
+
+    def __post_init__(self, **kwargs):
+        if self.attention_types is None:
+            self.attention_types = [[["global", "local"], 12]]
+        self.attention_layers = self.expand_attention_types_params(self.attention_types)
+        super().__post_init__(**kwargs)
+
+    def validate_architecture(self):
+        """Part of `@strict`-powered validation. Validates the architecture of the config."""
+        if len(self.attention_layers) != self.num_layers:
+            raise ValueError(
+                "Configuration for convolutional module is incorrect. "
+                "It is required that `len(config.attention_layers)` == `config.num_layers` "
+                f"but is `len(config.attention_layers) = {len(self.attention_layers)}`, "
+                f"`config.num_layers = {self.num_layers}`. "
+                "`config.attention_layers` is prepared using `config.attention_types`. "
+                "Please verify the value of `config.attention_types` argument."
+            )
+
+    @staticmethod
+    def expand_attention_types_params(attention_types):
+        attentions = []
+        for item in attention_types:
+            for _ in range(item[1]):
+                attentions.extend(item[0])
+        return attentions
+
+
+def custom_unfold(input, dimension, size, step):
+    """Custom torch.Tensor.unfold implementation to enable the export to ONNX."""
+    import torch
+
+    shape = input.size()
+    rank = len(shape)
+    sizedim = shape[dimension]
+
+    low_indices = torch.arange(0, sizedim, step)
+    min_length = torch.div(sizedim - size, step, rounding_mode="floor") + 1
+    indices = torch.arange(size) + low_indices[:min_length][:, None]
+
+    s = [slice(None)] * rank
+    s[dimension] = indices
+    sliced = input[s]
+
+    perm = list(range(0, rank + 1))
+    perm.append(perm.pop(dimension + 1))
+
+    return sliced.permute(perm)
+
+
+def custom_get_block_length_and_num_blocks(seq_length, window_size):
+    """
+    Custom implementation for GPTNeoAttentionMixin._get_block_length_and_num_blocks to enable the export to ONNX as
+    original implementation uses Python variables and control flow.
+    """
+    import torch
+
+    candidates = torch.arange(1, window_size)
+    remainders = torch.remainder(seq_length, candidates)
+    divisor_indices = remainders == 0
+    divisors = candidates[divisor_indices]
+    largest_divisor = torch.max(divisors)
+    return largest_divisor, torch.div(seq_length, largest_divisor, rounding_mode="floor")
+
+
+__all__ = ["GPTNeoConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/gpt_neo/modeling_gpt_neo.py

@@ -0,0 +1,916 @@
+# Copyright 2021 The Eleuther AI and HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch GPT Neo model."""
+
+import torch
+from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...cache_utils import Cache, DynamicCache
+from ...generation import GenerationMixin
+from ...masking_utils import create_causal_mask
+from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import (
+    BaseModelOutputWithPast,
+    BaseModelOutputWithPastAndCrossAttentions,
+    CausalLMOutputWithCrossAttentions,
+    CausalLMOutputWithPast,
+    QuestionAnsweringModelOutput,
+    SequenceClassifierOutputWithPast,
+    TokenClassifierOutput,
+)
+from ...modeling_utils import PreTrainedModel
+from ...utils import (
+    auto_docstring,
+    logging,
+)
+from .configuration_gpt_neo import GPTNeoConfig
+
+
+if is_flash_attn_available():
+    from ...modeling_flash_attention_utils import _flash_attention_forward
+
+
+logger = logging.get_logger(__name__)
+
+
+class GPTNeoSelfAttention(nn.Module):
+    def __init__(self, config, attention_type, layer_id=None):
+        super().__init__()
+        self.config = config
+
+        max_positions = config.max_position_embeddings
+        bias = torch.tril(torch.ones((max_positions, max_positions), dtype=bool)).view(
+            1, 1, max_positions, max_positions
+        )
+
+        # local causal self attention is a sliding window where each token can only attend to the previous
+        # window_size tokens. This is implemented by updating the causal mask such that for each token
+        # all other tokens are masked except the previous window_size tokens.
+        self.attention_type = attention_type
+        if attention_type == "local":
+            bias = torch.bitwise_xor(bias, torch.tril(bias, -config.window_size))
+
+        self.register_buffer("bias", bias, persistent=False)
+
+        self.attn_dropout = nn.Dropout(float(config.attention_dropout))
+        self.resid_dropout = nn.Dropout(float(config.resid_dropout))
+        self.is_causal = True
+        self.layer_id = layer_id
+
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=True)
+
+    def _split_heads(self, tensor, num_heads, attn_head_size):
+        """
+        Splits hidden_size dim into attn_head_size and num_heads
+        """
+        new_shape = tensor.size()[:-1] + (num_heads, attn_head_size)
+        tensor = tensor.view(new_shape)
+        return tensor.permute(0, 2, 1, 3)  # (batch, head, seq_length, head_features)
+
+    def _merge_heads(self, tensor, num_heads, attn_head_size):
+        """
+        Merges attn_head_size dim and num_attn_heads dim into hidden_size
+        """
+        tensor = tensor.permute(0, 2, 1, 3).contiguous()
+        new_shape = tensor.size()[:-2] + (num_heads * attn_head_size,)
+        return tensor.view(new_shape)
+
+    def _attn(self, query, key, value, attention_mask=None):
+        # Keep the attention weights computation in fp32 to avoid overflow issues
+        query = query.to(torch.float32)
+        key = key.to(torch.float32)
+
+        attn_weights = torch.matmul(query, key.transpose(-1, -2))
+
+        # Apply sliding window masking for local attention layers
+        query_length, key_length = query.size(-2), key.size(-2)
+        causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length]
+        mask_value = torch.finfo(attn_weights.dtype).min
+        # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
+        # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
+        mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype, device=attn_weights.device)
+        attn_weights = torch.where(causal_mask, attn_weights, mask_value)
+
+        if attention_mask is not None:
+            attn_weights = attn_weights + attention_mask
+
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+        attn_weights = attn_weights.to(value.dtype)
+        attn_weights = self.attn_dropout(attn_weights)
+
+        attn_output = torch.matmul(attn_weights, value)
+
+        return attn_output, attn_weights
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        layer_past=None,
+        use_cache=False,
+        output_attentions=False,
+        **kwargs,
+    ):
+        query = self.q_proj(hidden_states)
+        key = self.k_proj(hidden_states)
+        value = self.v_proj(hidden_states)
+
+        query = self._split_heads(query, self.num_heads, self.head_dim)
+        key = self._split_heads(key, self.num_heads, self.head_dim)
+        value = self._split_heads(value, self.num_heads, self.head_dim)
+
+        if layer_past is not None:
+            key, value = layer_past.update(key, value, self.layer_id)
+
+        attn_output, attn_weights = self._attn(query, key, value, attention_mask)
+
+        attn_output = self._merge_heads(attn_output, self.num_heads, self.head_dim)
+        attn_output = self.out_proj(attn_output)
+        attn_output = self.resid_dropout(attn_output)
+
+        return attn_output, attn_weights
+
+
+class GPTNeoFlashAttention2(GPTNeoSelfAttention):
+    """
+    GPTNeo flash attention module. This module inherits from `GPTNeoSelfAttention` as the weights of the module stays
+    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+    flash attention and deal with padding tokens in case the input contains any of them.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
+        # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
+        # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
+        self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask()
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        layer_past=None,
+        use_cache=False,
+        output_attentions=False,
+        **kwargs,
+    ):
+        bsz, _, _ = hidden_states.size()
+
+        query = self.q_proj(hidden_states)
+        key = self.k_proj(hidden_states)
+        value = self.v_proj(hidden_states)
+
+        query = self._split_heads(query, self.num_heads, self.head_dim)
+        key = self._split_heads(key, self.num_heads, self.head_dim)
+        value = self._split_heads(value, self.num_heads, self.head_dim)
+
+        if layer_past is not None:
+            key, value = layer_past.update(key, value, self.layer_id)
+
+        query_length = query.shape[2]
+        tgt_len = key.shape[2]
+
+        # Flash attention requires the input to have the shape
+        # batch_size x seq_length x head_dim x hidden_dim
+        query = query.transpose(1, 2).view(bsz, query_length, self.num_heads, self.head_dim)
+        key = key.transpose(1, 2).view(bsz, tgt_len, self.num_heads, self.head_dim)
+        value = value.transpose(1, 2).view(bsz, tgt_len, self.num_heads, self.head_dim)
+
+        attn_dropout = self.config.attention_dropout if self.training else 0.0
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in the correct dtype just to be sure everything works as expected.
+        # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+        # in fp32. (LlamaRMSNorm handles it correctly)
+
+        device_type = query.device.type if query.device.type != "mps" else "cpu"
+        if query.dtype == torch.float32:
+            if torch.is_autocast_enabled(device_type):
+                target_dtype = torch.get_autocast_dtype(device_type)
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_is_quantized"):
+                target_dtype = self.config.dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                f"The input hidden states seems to be silently casted in float32, this might be related to"
+                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query = query.to(target_dtype)
+            key = key.to(target_dtype)
+            value = value.to(target_dtype)
+
+        attn_output = _flash_attention_forward(
+            query,
+            key,
+            value,
+            attention_mask,
+            query_length,
+            dropout=attn_dropout,
+            softmax_scale=1.0,
+            is_causal=self.is_causal,
+            use_top_left_mask=self._flash_attn_uses_top_left_mask,
+        )
+
+        attn_weights_reshaped = attn_output.reshape(bsz, query_length, self.num_heads * self.head_dim)
+        attn_output = self.out_proj(attn_weights_reshaped)
+        attn_output = self.resid_dropout(attn_output)
+
+        return attn_output, attn_weights_reshaped
+
+
+GPT_NEO_ATTENTION_CLASSES = {
+    "eager": GPTNeoSelfAttention,
+    "flash_attention_2": GPTNeoFlashAttention2,
+}
+
+
+class GPTNeoAttention(nn.Module):
+    def __init__(self, config, layer_id=0):
+        super().__init__()
+        self.layer_id = layer_id
+        self.attention_layers = config.attention_layers
+        self.attention_type = self.attention_layers[layer_id]
+
+        if self.attention_type in ["global", "local"]:
+            self.attention = GPT_NEO_ATTENTION_CLASSES[config._attn_implementation](
+                config, self.attention_type, layer_id
+            )
+        else:
+            raise NotImplementedError(
+                "Only attn layer types 'global' and 'local' exist, but got `config.attention_layers`: "
+                f"{config.attention_layers}. Select attn layer types from ['global', 'local'] only."
+            )
+
+    def forward(
+        self,
+        hidden_states,
+        layer_past=None,
+        attention_mask=None,
+        use_cache=False,
+        output_attentions=False,
+        **kwargs,
+    ):
+        return self.attention(
+            hidden_states,
+            attention_mask=attention_mask,
+            layer_past=layer_past,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+        )
+
+
+class GPTNeoMLP(nn.Module):
+    def __init__(self, intermediate_size, config):  # in MLP: intermediate_size= 4 * hidden_size
+        super().__init__()
+        embed_dim = config.hidden_size
+        self.c_fc = nn.Linear(embed_dim, intermediate_size)
+        self.c_proj = nn.Linear(intermediate_size, embed_dim)
+        self.act = ACT2FN[config.activation_function]
+        self.dropout = nn.Dropout(float(config.resid_dropout))
+
+    def forward(self, hidden_states):
+        hidden_states = self.c_fc(hidden_states)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.c_proj(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        return hidden_states
+
+
+class GPTNeoBlock(GradientCheckpointingLayer):
+    def __init__(self, config, layer_id=None):
+        super().__init__()
+        hidden_size = config.hidden_size
+        inner_dim = config.intermediate_size if config.intermediate_size is not None else 4 * hidden_size
+        self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
+        self.attn = GPTNeoAttention(config, layer_id)
+        self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
+        self.mlp = GPTNeoMLP(inner_dim, config)
+
+    def forward(
+        self,
+        hidden_states,
+        layer_past=None,
+        attention_mask=None,
+        use_cache=False,
+        output_attentions=False,
+        **kwargs,
+    ):
+        residual = hidden_states
+        hidden_states = self.ln_1(hidden_states)
+        attn_output, attn_weights = self.attn(
+            hidden_states,
+            layer_past=layer_past,
+            attention_mask=attention_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+        )
+
+        # residual connection
+        hidden_states = attn_output + residual
+
+        residual = hidden_states
+        hidden_states = self.ln_2(hidden_states)
+        feed_forward_hidden_states = self.mlp(hidden_states)
+        # residual connection
+        hidden_states = residual + feed_forward_hidden_states
+
+        return hidden_states, attn_weights
+
+
+@auto_docstring
+class GPTNeoPreTrainedModel(PreTrainedModel):
+    config: GPTNeoConfig
+    base_model_prefix = "transformer"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["GPTNeoBlock"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_flash_attn = True
+    _can_compile_fullgraph = False  # TODO: needs a hybrid cache
+
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, GPTNeoSelfAttention):
+            max_positions = module.config.max_position_embeddings
+            bias = torch.tril(torch.ones((max_positions, max_positions), dtype=bool)).view(
+                1, 1, max_positions, max_positions
+            )
+            if module.attention_type == "local":
+                bias = torch.bitwise_xor(bias, torch.tril(bias, -module.config.window_size))
+            init.copy_(module.bias, bias)
+
+
+@auto_docstring
+class GPTNeoModel(GPTNeoPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.embed_dim = config.hidden_size
+        self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
+        self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
+        self.drop = nn.Dropout(float(config.embed_dropout))
+        self.h = nn.ModuleList([GPTNeoBlock(config, layer_id=i) for i in range(config.num_layers)])
+        self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.wte
+
+    def set_input_embeddings(self, new_embeddings):
+        self.wte = new_embeddings
+
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.Tensor | None = None,
+        past_key_values: Cache | None = None,
+        attention_mask: torch.Tensor | None = None,
+        token_type_ids: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        use_cache: bool | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple[torch.Tensor] | BaseModelOutputWithPastAndCrossAttentions:
+        r"""
+        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
+            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
+            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
+            sequence tokens in the vocabulary.
+
+            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+            `input_ids`.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        if inputs_embeds is None:
+            inputs_embeds = self.wte(input_ids)
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+
+        if position_ids is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
+            position_ids = position_ids.unsqueeze(0)
+
+        causal_mask = create_causal_mask(
+            config=self.config,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            position_ids=position_ids,
+        )
+
+        position_embeds = self.wpe(position_ids)
+        hidden_states = inputs_embeds + position_embeds
+
+        seq_length = inputs_embeds.shape[1]
+        if token_type_ids is not None:
+            token_type_ids = token_type_ids.view(-1, seq_length)
+            token_type_embeds = self.wte(token_type_ids)
+            hidden_states = hidden_states + token_type_embeds
+
+        hidden_states = self.drop(hidden_states)
+        output_shape = (-1, seq_length, hidden_states.size(-1))
+
+        all_self_attentions = () if output_attentions else None
+        all_hidden_states = () if output_hidden_states else None
+        for i, block in enumerate(self.h):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            outputs = block(
+                hidden_states,
+                layer_past=past_key_values,
+                attention_mask=causal_mask,
+                use_cache=use_cache,
+                output_attentions=output_attentions,
+            )
+
+            hidden_states = outputs[0]
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (outputs[1],)
+
+        hidden_states = self.ln_f(hidden_states)
+
+        hidden_states = hidden_states.view(output_shape)
+        # Add last hidden state
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v for v in [hidden_states, past_key_values, all_hidden_states, all_self_attentions] if v is not None
+            )
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    The GPT Neo Model transformer with a language modeling head on top (linear layer with weights tied to the input
+    embeddings).
+    """
+)
+class GPTNeoForCausalLM(GPTNeoPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "transformer.wte.weight"}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.transformer = GPTNeoModel(config)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.Tensor | None = None,
+        past_key_values: Cache | None = None,
+        attention_mask: torch.Tensor | None = None,
+        token_type_ids: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        use_cache: bool | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs,
+    ) -> tuple[torch.Tensor] | CausalLMOutputWithCrossAttentions:
+        r"""
+        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
+            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
+            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
+            sequence tokens in the vocabulary.
+
+            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+            `input_ids`.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
+            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
+            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+        """
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            **kwargs,
+        )
+
+        hidden_states = transformer_outputs[0]
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+
+        if not return_dict:
+            output = (logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    The GPTNeo Model transformer with a sequence classification head on top (linear layer).
+
+    [`GPTNeoForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+    (e.g. GPT-1) do.
+
+    Since it does classification on the last token, it requires to know the position of the last token. If a
+    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+    each row of the batch).
+    """
+)
+class GPTNeoForSequenceClassification(GPTNeoPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.transformer = GPTNeoModel(config)
+        self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.Tensor | None = None,
+        past_key_values: Cache | None = None,
+        attention_mask: torch.Tensor | None = None,
+        token_type_ids: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        use_cache: bool | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple[torch.Tensor] | SequenceClassifierOutputWithPast:
+        r"""
+        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
+            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
+            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
+            sequence tokens in the vocabulary.
+
+            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+            `input_ids`.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+        logits = self.score(hidden_states)
+
+        if input_ids is not None:
+            batch_size, sequence_length = input_ids.shape[:2]
+        else:
+            batch_size, sequence_length = inputs_embeds.shape[:2]
+
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
+        if self.config.pad_token_id is None:
+            last_non_pad_token = -1
+        elif input_ids is not None:
+            # To handle both left- and right- padding, we take the rightmost token that is not equal to pad_token_id
+            non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32)
+            token_indices = torch.arange(input_ids.shape[-1], device=logits.device, dtype=torch.int32)
+            last_non_pad_token = (token_indices * non_pad_mask).argmax(-1)
+        else:
+            last_non_pad_token = -1
+            logger.warning_once(
+                f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be "
+                "unexpected if using padding tokens in conjunction with `inputs_embeds.`"
+            )
+
+        pooled_logits = logits[torch.arange(batch_size, device=logits.device), last_non_pad_token]
+
+        loss = None
+        if labels is not None:
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(pooled_logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(pooled_logits, labels)
+        if not return_dict:
+            output = (pooled_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+
+@auto_docstring
+class GPTNeoForTokenClassification(GPTNeoPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.transformer = GPTNeoModel(config)
+        self.dropout = nn.Dropout(config.classifier_dropout)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple | TokenClassifierOutput:
+        r"""
+        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
+            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
+            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
+            sequence tokens in the vocabulary.
+
+            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+            `input_ids`.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = transformer_outputs[0]
+        hidden_states = self.dropout(hidden_states)
+        logits = self.classifier(hidden_states)
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+
+        if not return_dict:
+            output = (logits,) + transformer_outputs[2:]
+            return ((loss,) + output) if loss is not None else output
+
+        return TokenClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+
+@auto_docstring
+class GPTNeoForQuestionAnswering(GPTNeoPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.transformer = GPTNeoModel(config)
+        self.qa_outputs = nn.Linear(config.hidden_size, 2)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        start_positions: torch.LongTensor | None = None,
+        end_positions: torch.LongTensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple | QuestionAnsweringModelOutput:
+        r"""
+        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
+            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
+            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
+            sequence tokens in the vocabulary.
+
+            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+            `input_ids`.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        """
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        outputs = self.transformer(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        sequence_output = outputs[0]
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = logits.split(1, dim=-1)
+        start_logits = start_logits.squeeze(-1).contiguous()
+        end_logits = end_logits.squeeze(-1).contiguous()
+
+        total_loss = None
+        if start_positions is not None and end_positions is not None:
+            # If we are on multi-GPU, split add a dimension
+            if len(start_positions.size()) > 1:
+                start_positions = start_positions.squeeze(-1)
+            if len(end_positions.size()) > 1:
+                end_positions = end_positions.squeeze(-1)
+            # sometimes the start/end positions are outside our model inputs, we ignore these terms
+            ignored_index = start_logits.size(1)
+            start_positions = start_positions.clamp(0, ignored_index)
+            end_positions = end_positions.clamp(0, ignored_index)
+
+            loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
+            start_loss = loss_fct(start_logits, start_positions)
+            end_loss = loss_fct(end_logits, end_positions)
+            total_loss = (start_loss + end_loss) / 2
+
+        if not return_dict:
+            output = (start_logits, end_logits) + outputs[2:]
+            return ((total_loss,) + output) if total_loss is not None else output
+
+        return QuestionAnsweringModelOutput(
+            loss=total_loss,
+            start_logits=start_logits,
+            end_logits=end_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = [
+    "GPTNeoForCausalLM",
+    "GPTNeoForQuestionAnswering",
+    "GPTNeoForSequenceClassification",
+    "GPTNeoForTokenClassification",
+    "GPTNeoModel",
+    "GPTNeoPreTrainedModel",
+]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/pegasus/__init__.py

@@ -0,0 +1,28 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_pegasus import *
+    from .modeling_pegasus import *
+    from .tokenization_pegasus import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/table_transformer/configuration_table_transformer.py

@@ -0,0 +1,117 @@
+# Copyright The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Table Transformer model configuration"""
+
+from huggingface_hub.dataclasses import strict
+
+from ...backbone_utils import consolidate_backbone_kwargs_to_config
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+from ..auto import AutoConfig
+
+
+@auto_docstring(checkpoint="microsoft/table-transformer-detection")
+@strict
+class TableTransformerConfig(PreTrainedConfig):
+    r"""
+    num_queries (`int`, *optional*, defaults to 100):
+        Number of object queries, i.e. detection slots. This is the maximal number of objects
+        [`TableTransformerModel`] can detect in a single image. For COCO, we recommend 100 queries.
+    auxiliary_loss (`bool`, *optional*, defaults to `False`):
+        Whether auxiliary decoding losses (loss at each decoder layer) are to be used.
+    position_embedding_type (`str`, *optional*, defaults to `"sine"`):
+        Type of position embeddings to be used on top of the image features. One of `"sine"` or `"learned"`.
+    dilation (`bool`, *optional*, defaults to `False`):
+        Whether to replace stride with dilation in the last convolutional block (DC5). Only supported when
+        `use_timm_backbone` = `True`.
+
+    Examples:
+
+    ```python
+    >>> from transformers import TableTransformerModel, TableTransformerConfig
+
+    >>> # Initializing a Table Transformer microsoft/table-transformer-detection style configuration
+    >>> configuration = TableTransformerConfig()
+
+    >>> # Initializing a model from the microsoft/table-transformer-detection style configuration
+    >>> model = TableTransformerModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "table-transformer"
+    sub_configs = {"backbone_config": AutoConfig}
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {
+        "hidden_size": "d_model",
+        "num_attention_heads": "encoder_attention_heads",
+        "num_hidden_layers": "encoder_layers",
+    }
+
+    backbone_config: dict | PreTrainedConfig | None = None
+    num_channels: int = 3
+    num_queries: int = 100
+    encoder_layers: int = 6
+    encoder_ffn_dim: int = 2048
+    encoder_attention_heads: int = 8
+    decoder_layers: int = 6
+    decoder_ffn_dim: int = 2048
+    decoder_attention_heads: int = 8
+    encoder_layerdrop: float | int = 0.0
+    decoder_layerdrop: float | int = 0.0
+    is_encoder_decoder: bool = True
+    activation_function: str = "relu"
+    d_model: int = 256
+    dropout: float | int = 0.1
+    attention_dropout: float | int = 0.0
+    activation_dropout: float | int = 0.0
+    init_std: float = 0.02
+    init_xavier_std: float = 1.0
+    auxiliary_loss: bool = False
+    position_embedding_type: str = "sine"
+    dilation: bool = False
+    class_cost: int = 1
+    bbox_cost: int = 5
+    giou_cost: int = 2
+    mask_loss_coefficient: int = 1
+    dice_loss_coefficient: int = 1
+    bbox_loss_coefficient: int = 5
+    giou_loss_coefficient: int = 2
+    eos_coefficient: float = 0.1
+
+    def __post_init__(self, **kwargs):
+        backbone_kwargs = kwargs.get("backbone_kwargs", {})
+        timm_default_kwargs = {
+            "num_channels": backbone_kwargs.get("num_channels", self.num_channels),
+            "features_only": True,
+            "use_pretrained_backbone": False,
+            "out_indices": backbone_kwargs.get("out_indices", [1, 2, 3, 4]),
+        }
+        if self.dilation:
+            timm_default_kwargs["output_stride"] = backbone_kwargs.get("output_stride", 16)
+
+        self.backbone_config, kwargs = consolidate_backbone_kwargs_to_config(
+            backbone_config=self.backbone_config,
+            default_backbone="resnet50",
+            default_config_type="resnet",
+            default_config_kwargs={"out_features": ["stage4"]},
+            timm_default_kwargs=timm_default_kwargs,
+            **kwargs,
+        )
+
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["TableTransformerConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/table_transformer/modeling_table_transformer.py

@@ -0,0 +1,1308 @@
+# Copyright 2022 Microsoft Research and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch Table Transformer model."""
+
+import math
+from dataclasses import dataclass
+
+import torch
+from torch import Tensor, nn
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...backbone_utils import load_backbone
+from ...masking_utils import create_bidirectional_mask
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttentions, Seq2SeqModelOutput
+from ...modeling_utils import PreTrainedModel
+from ...utils import (
+    ModelOutput,
+    auto_docstring,
+    logging,
+)
+from .configuration_table_transformer import TableTransformerConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring(
+    custom_intro="""
+    Base class for outputs of the TABLE_TRANSFORMER decoder. This class adds one attribute to BaseModelOutputWithCrossAttentions,
+    namely an optional stack of intermediate decoder activations, i.e. the output of each decoder layer, each of them
+    gone through a layernorm. This is useful when training the model with auxiliary decoding losses.
+    """
+)
+@dataclass
+# Copied from transformers.models.detr.modeling_detr.DetrDecoderOutput with DETR->TABLE_TRANSFORMER,Detr->TableTransformer
+class TableTransformerDecoderOutput(BaseModelOutputWithCrossAttentions):
+    r"""
+    cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`):
+        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+        sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+        used to compute the weighted average in the cross-attention heads.
+    intermediate_hidden_states (`torch.FloatTensor` of shape `(config.decoder_layers, batch_size, num_queries, hidden_size)`, *optional*, returned when `config.auxiliary_loss=True`):
+        Intermediate decoder activations, i.e. the output of each decoder layer, each of them gone through a
+        layernorm.
+    """
+
+    intermediate_hidden_states: torch.FloatTensor | None = None
+
+
+@auto_docstring(
+    custom_intro="""
+    Base class for outputs of the TABLE_TRANSFORMER encoder-decoder model. This class adds one attribute to Seq2SeqModelOutput,
+    namely an optional stack of intermediate decoder activations, i.e. the output of each decoder layer, each of them
+    gone through a layernorm. This is useful when training the model with auxiliary decoding losses.
+    """
+)
+@dataclass
+# Copied from transformers.models.detr.modeling_detr.DetrModelOutput with DETR->TABLE_TRANSFORMER,Detr->TableTransformer
+class TableTransformerModelOutput(Seq2SeqModelOutput):
+    r"""
+    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+        Sequence of hidden-states at the output of the last layer of the decoder of the model.
+    intermediate_hidden_states (`torch.FloatTensor` of shape `(config.decoder_layers, batch_size, sequence_length, hidden_size)`, *optional*, returned when `config.auxiliary_loss=True`):
+        Intermediate decoder activations, i.e. the output of each decoder layer, each of them gone through a
+        layernorm.
+    """
+
+    intermediate_hidden_states: torch.FloatTensor | None = None
+
+
+@auto_docstring(
+    custom_intro="""
+    Output type of [`TableTransformerForObjectDetection`].
+    """
+)
+@dataclass
+# Copied from transformers.models.detr.modeling_detr.DetrObjectDetectionOutput with Detr->TableTransformer,DetrImageProcessor->DetrImageProcessor
+class TableTransformerObjectDetectionOutput(ModelOutput):
+    r"""
+    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` are provided)):
+        Total loss as a linear combination of a negative log-likehood (cross-entropy) for class prediction and a
+        bounding box loss. The latter is defined as a linear combination of the L1 loss and the generalized
+        scale-invariant IoU loss.
+    loss_dict (`Dict`, *optional*):
+        A dictionary containing the individual losses. Useful for logging.
+    logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_classes + 1)`):
+        Classification logits (including no-object) for all queries.
+    pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`):
+        Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These
+        values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding
+        possible padding). You can use [`~TableTransformerImageProcessor.post_process_object_detection`] to retrieve the
+        unnormalized bounding boxes.
+    auxiliary_outputs (`list[Dict]`, *optional*):
+        Optional, only returned when auxiliary losses are activated (i.e. `config.auxiliary_loss` is set to `True`)
+        and labels are provided. It is a list of dictionaries containing the two above keys (`logits` and
+        `pred_boxes`) for each decoder layer.
+    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+        Sequence of hidden-states at the output of the last layer of the decoder of the model.
+    """
+
+    loss: torch.FloatTensor | None = None
+    loss_dict: dict | None = None
+    logits: torch.FloatTensor | None = None
+    pred_boxes: torch.FloatTensor | None = None
+    auxiliary_outputs: list[dict] | None = None
+    last_hidden_state: torch.FloatTensor | None = None
+    decoder_hidden_states: tuple[torch.FloatTensor] | None = None
+    decoder_attentions: tuple[torch.FloatTensor] | None = None
+    cross_attentions: tuple[torch.FloatTensor] | None = None
+    encoder_last_hidden_state: torch.FloatTensor | None = None
+    encoder_hidden_states: tuple[torch.FloatTensor] | None = None
+    encoder_attentions: tuple[torch.FloatTensor] | None = None
+
+
+# Copied from transformers.models.detr.modeling_detr.DetrFrozenBatchNorm2d with Detr->TableTransformer
+class TableTransformerFrozenBatchNorm2d(nn.Module):
+    """
+    BatchNorm2d where the batch statistics and the affine parameters are fixed.
+
+    Copy-paste from torchvision.misc.ops with added eps before rqsrt, without which any other models than
+    torchvision.models.resnet[18,34,50,101] produce nans.
+    """
+
+    def __init__(self, n):
+        super().__init__()
+        self.register_buffer("weight", torch.ones(n))
+        self.register_buffer("bias", torch.zeros(n))
+        self.register_buffer("running_mean", torch.zeros(n))
+        self.register_buffer("running_var", torch.ones(n))
+
+    def _load_from_state_dict(
+        self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
+    ):
+        num_batches_tracked_key = prefix + "num_batches_tracked"
+        if num_batches_tracked_key in state_dict:
+            del state_dict[num_batches_tracked_key]
+
+        super()._load_from_state_dict(
+            state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
+        )
+
+    def forward(self, x):
+        # move reshapes to the beginning
+        # to make it user-friendly
+        weight = self.weight.reshape(1, -1, 1, 1)
+        bias = self.bias.reshape(1, -1, 1, 1)
+        running_var = self.running_var.reshape(1, -1, 1, 1)
+        running_mean = self.running_mean.reshape(1, -1, 1, 1)
+        epsilon = 1e-5
+        scale = weight * (running_var + epsilon).rsqrt()
+        bias = bias - running_mean * scale
+        return x * scale + bias
+
+
+# Copied from transformers.models.detr.modeling_detr.replace_batch_norm with Detr->TableTransformer
+def replace_batch_norm(model):
+    r"""
+    Recursively replace all `torch.nn.BatchNorm2d` with `TableTransformerFrozenBatchNorm2d`.
+
+    Args:
+        model (torch.nn.Module):
+            input model
+    """
+    for name, module in model.named_children():
+        if isinstance(module, nn.BatchNorm2d):
+            new_module = TableTransformerFrozenBatchNorm2d(module.num_features)
+
+            if module.weight.device != torch.device("meta"):
+                new_module.weight.copy_(module.weight)
+                new_module.bias.copy_(module.bias)
+                new_module.running_mean.copy_(module.running_mean)
+                new_module.running_var.copy_(module.running_var)
+
+            model._modules[name] = new_module
+
+        if len(list(module.children())) > 0:
+            replace_batch_norm(module)
+
+
+# TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrConvEncoder with Detr->TableTransformer
+class TableTransformerConvEncoder(nn.Module):
+    """
+    Convolutional backbone, using either the AutoBackbone API or one from the timm library.
+
+    nn.BatchNorm2d layers are replaced by TableTransformerFrozenBatchNorm2d as defined above.
+
+    """
+
+    def __init__(self, config):
+        super().__init__()
+
+        self.config = config
+
+        backbone = load_backbone(config)
+        self.intermediate_channel_sizes = backbone.channels
+
+        # replace batch norm by frozen batch norm
+        with torch.no_grad():
+            replace_batch_norm(backbone)
+
+        # We used to load with timm library directly instead of the AutoBackbone API
+        # so we need to unwrap the `backbone._backbone` module to load weights without mismatch
+        is_timm_model = False
+        if hasattr(backbone, "_backbone"):
+            backbone = backbone._backbone
+            is_timm_model = True
+        self.model = backbone
+
+        backbone_model_type = config.backbone_config.model_type
+        if "resnet" in backbone_model_type:
+            for name, parameter in self.model.named_parameters():
+                if is_timm_model:
+                    if "layer2" not in name and "layer3" not in name and "layer4" not in name:
+                        parameter.requires_grad_(False)
+                else:
+                    if "stage.1" not in name and "stage.2" not in name and "stage.3" not in name:
+                        parameter.requires_grad_(False)
+
+    def forward(self, pixel_values: torch.Tensor, pixel_mask: torch.Tensor):
+        # send pixel_values through the model to get list of feature maps
+        features = self.model(pixel_values)
+        if isinstance(features, dict):
+            features = features.feature_maps
+
+        out = []
+        for feature_map in features:
+            # downsample pixel_mask to match shape of corresponding feature_map
+            mask = nn.functional.interpolate(pixel_mask[None].float(), size=feature_map.shape[-2:]).to(torch.bool)[0]
+            out.append((feature_map, mask))
+        return out
+
+
+# TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrConvModel with Detr->TableTransformer
+class TableTransformerConvModel(nn.Module):
+    """
+    This module adds 2D position embeddings to all intermediate feature maps of the convolutional encoder.
+    """
+
+    def __init__(self, conv_encoder, position_embedding):
+        super().__init__()
+        self.conv_encoder = conv_encoder
+        self.position_embedding = position_embedding
+
+    def forward(self, pixel_values, pixel_mask):
+        # send pixel_values and pixel_mask through backbone to get list of (feature_map, pixel_mask) tuples
+        out = self.conv_encoder(pixel_values, pixel_mask)
+        pos = []
+        for feature_map, mask in out:
+            # position encoding
+            pos.append(self.position_embedding(feature_map, mask).to(feature_map.dtype))
+
+        return out, pos
+
+
+# TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrSinePositionEmbedding with Detr->TableTransformer
+class TableTransformerSinePositionEmbedding(nn.Module):
+    """
+    This is a more standard version of the position embedding, very similar to the one used by the Attention is all you
+    need paper, generalized to work on images.
+    """
+
+    def __init__(self, embedding_dim=64, temperature=10000, normalize=False, scale=None):
+        super().__init__()
+        self.embedding_dim = embedding_dim
+        self.temperature = temperature
+        self.normalize = normalize
+        if scale is not None and normalize is False:
+            raise ValueError("normalize should be True if scale is passed")
+        if scale is None:
+            scale = 2 * math.pi
+        self.scale = scale
+
+    def forward(self, pixel_values, pixel_mask):
+        if pixel_mask is None:
+            raise ValueError("No pixel mask provided")
+        y_embed = pixel_mask.cumsum(1, dtype=torch.float32)
+        x_embed = pixel_mask.cumsum(2, dtype=torch.float32)
+        if self.normalize:
+            y_embed = y_embed / (y_embed[:, -1:, :] + 1e-6) * self.scale
+            x_embed = x_embed / (x_embed[:, :, -1:] + 1e-6) * self.scale
+
+        dim_t = torch.arange(self.embedding_dim, dtype=torch.int64, device=pixel_values.device).float()
+        dim_t = self.temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / self.embedding_dim)
+
+        pos_x = x_embed[:, :, :, None] / dim_t
+        pos_y = y_embed[:, :, :, None] / dim_t
+        pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3)
+        pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3)
+        pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2)
+        return pos
+
+
+# TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrLearnedPositionEmbedding with Detr->TableTransformer
+class TableTransformerLearnedPositionEmbedding(nn.Module):
+    """
+    This module learns positional embeddings up to a fixed maximum size.
+    """
+
+    def __init__(self, embedding_dim=256):
+        super().__init__()
+        self.row_embeddings = nn.Embedding(50, embedding_dim)
+        self.column_embeddings = nn.Embedding(50, embedding_dim)
+
+    def forward(self, pixel_values, pixel_mask=None):
+        height, width = pixel_values.shape[-2:]
+        width_values = torch.arange(width, device=pixel_values.device)
+        height_values = torch.arange(height, device=pixel_values.device)
+        x_emb = self.column_embeddings(width_values)
+        y_emb = self.row_embeddings(height_values)
+        pos = torch.cat([x_emb.unsqueeze(0).repeat(height, 1, 1), y_emb.unsqueeze(1).repeat(1, width, 1)], dim=-1)
+        pos = pos.permute(2, 0, 1)
+        pos = pos.unsqueeze(0)
+        pos = pos.repeat(pixel_values.shape[0], 1, 1, 1)
+        return pos
+
+
+# TODO: use modular - Copied from transformers.models.detr.modeling_detr.build_position_encoding with Detr->TableTransformer
+def build_position_encoding(config):
+    n_steps = config.d_model // 2
+    if config.position_embedding_type == "sine":
+        # TODO find a better way of exposing other arguments
+        position_embedding = TableTransformerSinePositionEmbedding(n_steps, normalize=True)
+    elif config.position_embedding_type == "learned":
+        position_embedding = TableTransformerLearnedPositionEmbedding(n_steps)
+    else:
+        raise ValueError(f"Not supported {config.position_embedding_type}")
+
+    return position_embedding
+
+
+# TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrAttention with DETR->TABLE_TRANSFORMER,Detr->TableTransformer
+class TableTransformerAttention(nn.Module):
+    """
+    Multi-headed attention from 'Attention Is All You Need' paper.
+
+    Here, we add position embeddings to the queries and keys (as explained in the TABLE_TRANSFORMER paper).
+    """
+
+    def __init__(
+        self,
+        embed_dim: int,
+        num_heads: int,
+        dropout: float = 0.0,
+        bias: bool = True,
+    ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+        if self.head_dim * num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {num_heads})."
+            )
+        self.scaling = self.head_dim**-0.5
+
+        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+
+    def _shape(self, tensor: torch.Tensor, seq_len: int, batch_size: int):
+        return tensor.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
+
+    def with_pos_embed(self, tensor: torch.Tensor, object_queries: Tensor | None):
+        return tensor if object_queries is None else tensor + object_queries
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        object_queries: torch.Tensor | None = None,
+        key_value_states: torch.Tensor | None = None,
+        spatial_position_embeddings: torch.Tensor | None = None,
+        output_attentions: bool = False,
+    ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:
+        """Input shape: Batch x Time x Channel"""
+        # if key_value_states are provided this layer is used as a cross-attention layer
+        # for the decoder
+        is_cross_attention = key_value_states is not None
+        batch_size, target_len, embed_dim = hidden_states.size()
+
+        # add position embeddings to the hidden states before projecting to queries and keys
+        if object_queries is not None:
+            hidden_states_original = hidden_states
+            hidden_states = self.with_pos_embed(hidden_states, object_queries)
+
+        # add key-value position embeddings to the key value states
+        if spatial_position_embeddings is not None:
+            key_value_states_original = key_value_states
+            key_value_states = self.with_pos_embed(key_value_states, spatial_position_embeddings)
+
+        # get query proj
+        query_states = self.q_proj(hidden_states) * self.scaling
+        # get key, value proj
+        if is_cross_attention:
+            # cross_attentions
+            key_states = self._shape(self.k_proj(key_value_states), -1, batch_size)
+            value_states = self._shape(self.v_proj(key_value_states_original), -1, batch_size)
+        else:
+            # self_attention
+            key_states = self._shape(self.k_proj(hidden_states), -1, batch_size)
+            value_states = self._shape(self.v_proj(hidden_states_original), -1, batch_size)
+
+        proj_shape = (batch_size * self.num_heads, -1, self.head_dim)
+        query_states = self._shape(query_states, target_len, batch_size).view(*proj_shape)
+        key_states = key_states.view(*proj_shape)
+        value_states = value_states.view(*proj_shape)
+
+        source_len = key_states.size(1)
+
+        attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))
+
+        if attn_weights.size() != (batch_size * self.num_heads, target_len, source_len):
+            raise ValueError(
+                f"Attention weights should be of size {(batch_size * self.num_heads, target_len, source_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (batch_size, 1, target_len, source_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(batch_size, 1, target_len, source_len)}, but is"
+                    f" {attention_mask.size()}"
+                )
+            if attention_mask.dtype == torch.bool:
+                attention_mask = torch.zeros_like(attention_mask, dtype=attn_weights.dtype).masked_fill_(
+                    attention_mask, -torch.inf
+                )
+            attn_weights = attn_weights.view(batch_size, self.num_heads, target_len, source_len) + attention_mask
+            attn_weights = attn_weights.view(batch_size * self.num_heads, target_len, source_len)
+
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+
+        if output_attentions:
+            # this operation is a bit awkward, but it's required to
+            # make sure that attn_weights keeps its gradient.
+            # In order to do so, attn_weights have to reshaped
+            # twice and have to be reused in the following
+            attn_weights_reshaped = attn_weights.view(batch_size, self.num_heads, target_len, source_len)
+            attn_weights = attn_weights_reshaped.view(batch_size * self.num_heads, target_len, source_len)
+        else:
+            attn_weights_reshaped = None
+
+        attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
+
+        attn_output = torch.bmm(attn_probs, value_states)
+
+        if attn_output.size() != (batch_size * self.num_heads, target_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(batch_size, self.num_heads, target_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.view(batch_size, self.num_heads, target_len, self.head_dim)
+        attn_output = attn_output.transpose(1, 2)
+        attn_output = attn_output.reshape(batch_size, target_len, embed_dim)
+
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights_reshaped
+
+
+class TableTransformerEncoderLayer(nn.Module):
+    # TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrEncoderLayer.__init__ with Detr->TableTransformer
+    def __init__(self, config: TableTransformerConfig):
+        super().__init__()
+        self.embed_dim = config.d_model
+        self.self_attn = TableTransformerAttention(
+            embed_dim=self.embed_dim,
+            num_heads=config.encoder_attention_heads,
+            dropout=config.attention_dropout,
+        )
+        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.dropout = config.dropout
+        self.activation_fn = ACT2FN[config.activation_function]
+        self.activation_dropout = config.activation_dropout
+        self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
+        self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        object_queries: torch.Tensor | None = None,
+        output_attentions: bool = False,
+    ):
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`): attention mask of size
+                `(batch, 1, target_len, source_len)` where padding elements are indicated by very large negative
+                values.
+            object_queries (`torch.FloatTensor`, *optional*): object queries, to be added to hidden_states.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+        """
+        residual = hidden_states
+        hidden_states = self.self_attn_layer_norm(hidden_states)
+
+        hidden_states, attn_weights = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            object_queries=object_queries,
+            output_attentions=output_attentions,
+        )
+
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.final_layer_norm(hidden_states)
+
+        hidden_states = self.activation_fn(self.fc1(hidden_states))
+        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
+
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+
+        hidden_states = residual + hidden_states
+
+        if self.training:
+            if not torch.isfinite(hidden_states).all():
+                clamp_value = torch.finfo(hidden_states.dtype).max - 1000
+                hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+class TableTransformerDecoderLayer(GradientCheckpointingLayer):
+    # TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrDecoderLayer.__init__ with Detr->TableTransformer
+    def __init__(self, config: TableTransformerConfig):
+        super().__init__()
+        self.embed_dim = config.d_model
+
+        self.self_attn = TableTransformerAttention(
+            embed_dim=self.embed_dim,
+            num_heads=config.decoder_attention_heads,
+            dropout=config.attention_dropout,
+        )
+        self.dropout = config.dropout
+        self.activation_fn = ACT2FN[config.activation_function]
+        self.activation_dropout = config.activation_dropout
+
+        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.encoder_attn = TableTransformerAttention(
+            self.embed_dim,
+            config.decoder_attention_heads,
+            dropout=config.attention_dropout,
+        )
+        self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim)
+        self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim)
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        object_queries: torch.Tensor | None = None,
+        query_position_embeddings: torch.Tensor | None = None,
+        encoder_hidden_states: torch.Tensor | None = None,
+        encoder_attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = False,
+    ):
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`): attention mask of size
+                `(batch, 1, target_len, source_len)` where padding elements are indicated by very large negative
+                values.
+            object_queries (`torch.FloatTensor`, *optional*):
+                object queries that are added to the queries and keys
+            in the cross-attention layer.
+            query_position_embeddings (`torch.FloatTensor`, *optional*):
+                object queries that are added to the queries and keys
+            in the self-attention layer.
+            encoder_hidden_states (`torch.FloatTensor`):
+                cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
+            encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
+                `(batch, 1, target_len, source_len)` where padding elements are indicated by very large negative
+                values.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+        """
+        residual = hidden_states
+        hidden_states = self.self_attn_layer_norm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights = self.self_attn(
+            hidden_states=hidden_states,
+            object_queries=query_position_embeddings,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+        )
+
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.encoder_attn_layer_norm(hidden_states)
+
+        # Cross-Attention Block
+        cross_attn_weights = None
+        if encoder_hidden_states is not None:
+            hidden_states, cross_attn_weights = self.encoder_attn(
+                hidden_states=hidden_states,
+                object_queries=query_position_embeddings,
+                key_value_states=encoder_hidden_states,
+                attention_mask=encoder_attention_mask,
+                spatial_position_embeddings=object_queries,
+                output_attentions=output_attentions,
+            )
+
+            hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+            hidden_states = residual + hidden_states
+
+            residual = hidden_states
+            hidden_states = self.final_layer_norm(hidden_states)
+
+        # Fully Connected
+        hidden_states = self.activation_fn(self.fc1(hidden_states))
+        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights, cross_attn_weights)
+
+        return outputs
+
+
+@auto_docstring
+class TableTransformerPreTrainedModel(PreTrainedModel):
+    config: TableTransformerConfig
+    base_model_prefix = "model"
+    main_input_name = "pixel_values"
+    input_modalities = ("image",)
+    _no_split_modules = [
+        r"TableTransformerConvEncoder",
+        r"TableTransformerEncoderLayer",
+        r"TableTransformerDecoderLayer",
+    ]
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        std = self.config.init_std
+
+        if isinstance(module, TableTransformerLearnedPositionEmbedding):
+            init.uniform_(module.row_embeddings.weight)
+            init.uniform_(module.column_embeddings.weight)
+        if isinstance(module, (nn.Linear, nn.Conv2d)):
+            init.normal_(module.weight, mean=0.0, std=std)
+            if module.bias is not None:
+                init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            init.normal_(module.weight, mean=0.0, std=std)
+            # Here we need the check explicitly, as we slice the weight in the `zeros_` call, so it looses the flag
+            if module.padding_idx is not None and not getattr(module.weight, "_is_hf_initialized", False):
+                init.zeros_(module.weight[module.padding_idx])
+
+
+class TableTransformerEncoder(TableTransformerPreTrainedModel):
+    """
+    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
+    [`TableTransformerEncoderLayer`].
+
+    The encoder updates the flattened feature map through multiple self-attention layers.
+
+    Small tweak for Table Transformer:
+
+    - object_queries are added to the forward pass.
+
+    Args:
+        config: TableTransformerConfig
+    """
+
+    def __init__(self, config: TableTransformerConfig):
+        super().__init__(config)
+
+        self.dropout = config.dropout
+        self.layerdrop = config.encoder_layerdrop
+
+        self.layers = nn.ModuleList([TableTransformerEncoderLayer(config) for _ in range(config.encoder_layers)])
+
+        self.layernorm = nn.LayerNorm(config.d_model)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        inputs_embeds=None,
+        attention_mask=None,
+        object_queries=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        r"""
+        Args:
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                Flattened feature map (output of the backbone + projection layer) that is passed to the encoder.
+
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding pixel features. Mask values selected in `[0, 1]`:
+
+                - 1 for pixel features that are real (i.e. **not masked**),
+                - 0 for pixel features that are padding (i.e. **masked**).
+
+                [What are attention masks?](../glossary#attention-mask)
+
+            object_queries (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                Position embeddings that are added to the queries and keys in each self-attention layer.
+
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        hidden_states = inputs_embeds
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+
+        # expand attention_mask
+        if attention_mask is not None:
+            attention_mask = create_bidirectional_mask(
+                config=self.config,
+                inputs_embeds=hidden_states,
+                attention_mask=attention_mask,
+            )
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+        for encoder_layer in self.layers:
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                # we add object_queries as extra input to the encoder_layer
+                layer_outputs = encoder_layer(
+                    hidden_states,
+                    attention_mask,
+                    object_queries=object_queries,
+                    output_attentions=output_attentions,
+                )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        hidden_states = self.layernorm(hidden_states)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+        )
+
+
+# TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrDecoder with DETR->TABLE_TRANSFORMER,Detr->TableTransformer
+class TableTransformerDecoder(TableTransformerPreTrainedModel):
+    """
+    Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`TableTransformerDecoderLayer`].
+
+    The decoder updates the query embeddings through multiple self-attention and cross-attention layers.
+
+    Some small tweaks for TABLE_TRANSFORMER:
+
+    - object_queries and query_position_embeddings are added to the forward pass.
+    - if self.config.auxiliary_loss is set to True, also returns a stack of activations from all decoding layers.
+
+    Args:
+        config: TableTransformerConfig
+    """
+
+    def __init__(self, config: TableTransformerConfig):
+        super().__init__(config)
+        self.dropout = config.dropout
+        self.layerdrop = config.decoder_layerdrop
+
+        self.layers = nn.ModuleList([TableTransformerDecoderLayer(config) for _ in range(config.decoder_layers)])
+        # in TABLE_TRANSFORMER, the decoder uses layernorm after the last decoder layer output
+        self.layernorm = nn.LayerNorm(config.d_model)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        inputs_embeds=None,
+        attention_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        object_queries=None,
+        query_position_embeddings=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        r"""
+        Args:
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                The query embeddings that are passed into the decoder.
+
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on certain queries. Mask values selected in `[0, 1]`:
+
+                - 1 for queries that are **not masked**,
+                - 0 for queries that are **masked**.
+
+                [What are attention masks?](../glossary#attention-mask)
+            encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
+                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
+                of the decoder.
+            encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
+                Mask to avoid performing cross-attention on padding pixel_values of the encoder. Mask values selected
+                in `[0, 1]`:
+
+                - 1 for pixels that are real (i.e. **not masked**),
+                - 0 for pixels that are padding (i.e. **masked**).
+
+            object_queries (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+                Object queries that are added to the queries and keys in each cross-attention layer.
+            query_position_embeddings (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`):
+                , *optional*): Position embeddings that are added to the values and keys in each self-attention layer.
+
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        if inputs_embeds is not None:
+            hidden_states = inputs_embeds
+
+        if attention_mask is not None:
+            attention_mask = create_bidirectional_mask(
+                config=self.config,
+                inputs_embeds=hidden_states,
+                attention_mask=attention_mask,
+            )
+
+        # expand encoder attention mask
+        if encoder_hidden_states is not None and encoder_attention_mask is not None:
+            encoder_attention_mask = create_bidirectional_mask(
+                config=self.config,
+                inputs_embeds=hidden_states,
+                attention_mask=encoder_attention_mask,
+                encoder_hidden_states=encoder_hidden_states,
+            )
+
+        # optional intermediate hidden states
+        intermediate = () if self.config.auxiliary_loss else None
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
+
+        for idx, decoder_layer in enumerate(self.layers):
+            # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:
+                    continue
+
+            layer_outputs = decoder_layer(
+                hidden_states,
+                attention_mask,
+                object_queries,
+                query_position_embeddings,
+                encoder_hidden_states,  # as a positional argument for gradient checkpointing
+                encoder_attention_mask=encoder_attention_mask,
+                output_attentions=output_attentions,
+            )
+
+            hidden_states = layer_outputs[0]
+
+            if self.config.auxiliary_loss:
+                hidden_states = self.layernorm(hidden_states)
+                intermediate += (hidden_states,)
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+                if encoder_hidden_states is not None:
+                    all_cross_attentions += (layer_outputs[2],)
+
+        # finally, apply layernorm
+        hidden_states = self.layernorm(hidden_states)
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        # stack intermediate decoder activations
+        if self.config.auxiliary_loss:
+            intermediate = torch.stack(intermediate)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, all_hidden_states, all_self_attns, all_cross_attentions, intermediate]
+                if v is not None
+            )
+        return TableTransformerDecoderOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+            cross_attentions=all_cross_attentions,
+            intermediate_hidden_states=intermediate,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    The bare Table Transformer Model (consisting of a backbone and encoder-decoder Transformer) outputting raw
+    hidden-states without any specific head on top.
+    """
+)
+class TableTransformerModel(TableTransformerPreTrainedModel):
+    # TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrModel.__init__ with Detr->TableTransformer
+    def __init__(self, config: TableTransformerConfig):
+        super().__init__(config)
+
+        # Create backbone + positional encoding
+        backbone = TableTransformerConvEncoder(config)
+        object_queries = build_position_encoding(config)
+        self.backbone = TableTransformerConvModel(backbone, object_queries)
+
+        # Create projection layer
+        self.input_projection = nn.Conv2d(backbone.intermediate_channel_sizes[-1], config.d_model, kernel_size=1)
+
+        self.query_position_embeddings = nn.Embedding(config.num_queries, config.d_model)
+
+        self.encoder = TableTransformerEncoder(config)
+        self.decoder = TableTransformerDecoder(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def freeze_backbone(self):
+        for name, param in self.backbone.conv_encoder.model.named_parameters():
+            param.requires_grad_(False)
+
+    def unfreeze_backbone(self):
+        for name, param in self.backbone.conv_encoder.model.named_parameters():
+            param.requires_grad_(True)
+
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        pixel_mask: torch.FloatTensor | None = None,
+        decoder_attention_mask: torch.FloatTensor | None = None,
+        encoder_outputs: torch.FloatTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        decoder_inputs_embeds: torch.FloatTensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple[torch.FloatTensor] | TableTransformerModelOutput:
+        r"""
+        decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*):
+            Not used by default. Can be used to mask object queries.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you
+            can choose to directly pass a flattened representation of an image.
+        decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*):
+            Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an
+            embedded representation.
+
+        Examples:
+
+        ```python
+        >>> from transformers import AutoImageProcessor, TableTransformerModel
+        >>> from huggingface_hub import hf_hub_download
+        >>> from PIL import Image
+
+        >>> file_path = hf_hub_download(repo_id="nielsr/example-pdf", repo_type="dataset", filename="example_pdf.png")
+        >>> image = Image.open(file_path).convert("RGB")
+
+        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/table-transformer-detection")
+        >>> model = TableTransformerModel.from_pretrained("microsoft/table-transformer-detection")
+
+        >>> # prepare image for the model
+        >>> inputs = image_processor(images=image, return_tensors="pt")
+
+        >>> # forward pass
+        >>> outputs = model(**inputs)
+
+        >>> # the last hidden states are the final query embeddings of the Transformer decoder
+        >>> # these are of shape (batch_size, num_queries, hidden_size)
+        >>> last_hidden_states = outputs.last_hidden_state
+        >>> list(last_hidden_states.shape)
+        [1, 15, 256]
+        ```"""
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        batch_size, num_channels, height, width = pixel_values.shape
+        device = pixel_values.device
+
+        if pixel_mask is None:
+            pixel_mask = torch.ones(((batch_size, height, width)), device=device)
+
+        # First, sent pixel_values + pixel_mask through Backbone to obtain the features
+        # pixel_values should be of shape (batch_size, num_channels, height, width)
+        # pixel_mask should be of shape (batch_size, height, width)
+        features, position_embeddings_list = self.backbone(pixel_values, pixel_mask)
+
+        # get final feature map and downsampled mask
+        feature_map, mask = features[-1]
+
+        if mask is None:
+            raise ValueError("Backbone does not return downsampled pixel mask")
+
+        # Second, apply 1x1 convolution to reduce the channel dimension to d_model (256 by default)
+        projected_feature_map = self.input_projection(feature_map)
+
+        # Third, flatten the feature map + object queries of shape NxCxHxW to NxCxHW, and permute it to NxHWxC
+        # In other words, turn their shape into (batch_size, sequence_length, hidden_size)
+        flattened_features = projected_feature_map.flatten(2).permute(0, 2, 1)
+        object_queries = position_embeddings_list[-1].flatten(2).permute(0, 2, 1)
+
+        flattened_mask = mask.flatten(1)
+
+        # Fourth, sent flattened_features + flattened_mask + object queries through encoder
+        # flattened_features is a Tensor of shape (batch_size, height*width, hidden_size)
+        # flattened_mask is a Tensor of shape (batch_size, height*width)
+        if encoder_outputs is None:
+            encoder_outputs = self.encoder(
+                inputs_embeds=flattened_features,
+                attention_mask=flattened_mask,
+                object_queries=object_queries,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+            )
+        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
+        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
+            encoder_outputs = BaseModelOutput(
+                last_hidden_state=encoder_outputs[0],
+                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
+                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
+            )
+
+        # Fifth, sent query embeddings + object queries through the decoder (which is conditioned on the encoder output)
+        query_position_embeddings = self.query_position_embeddings.weight.unsqueeze(0).repeat(batch_size, 1, 1)
+        queries = torch.zeros_like(query_position_embeddings)
+
+        # decoder outputs consists of (dec_features, dec_hidden, dec_attn)
+        decoder_outputs = self.decoder(
+            inputs_embeds=queries,
+            attention_mask=None,
+            object_queries=object_queries,
+            query_position_embeddings=query_position_embeddings,
+            encoder_hidden_states=encoder_outputs[0],
+            encoder_attention_mask=flattened_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if not return_dict:
+            return decoder_outputs + encoder_outputs
+
+        return TableTransformerModelOutput(
+            last_hidden_state=decoder_outputs.last_hidden_state,
+            decoder_hidden_states=decoder_outputs.hidden_states,
+            decoder_attentions=decoder_outputs.attentions,
+            cross_attentions=decoder_outputs.cross_attentions,
+            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
+            encoder_hidden_states=encoder_outputs.hidden_states,
+            encoder_attentions=encoder_outputs.attentions,
+            intermediate_hidden_states=decoder_outputs.intermediate_hidden_states,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    Table Transformer Model (consisting of a backbone and encoder-decoder Transformer) with object detection heads on
+    top, for tasks such as COCO detection.
+    """
+)
+class TableTransformerForObjectDetection(TableTransformerPreTrainedModel):
+    # TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrForObjectDetection.__init__ with Detr->TableTransformer
+    def __init__(self, config: TableTransformerConfig):
+        super().__init__(config)
+
+        # DETR encoder-decoder model
+        self.model = TableTransformerModel(config)
+
+        # Object detection heads
+        self.class_labels_classifier = nn.Linear(
+            config.d_model, config.num_labels + 1
+        )  # We add one for the "no object" class
+        self.bbox_predictor = TableTransformerMLPPredictionHead(
+            input_dim=config.d_model, hidden_dim=config.d_model, output_dim=4, num_layers=3
+        )
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        pixel_mask: torch.FloatTensor | None = None,
+        decoder_attention_mask: torch.FloatTensor | None = None,
+        encoder_outputs: torch.FloatTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        decoder_inputs_embeds: torch.FloatTensor | None = None,
+        labels: list[dict] | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple[torch.FloatTensor] | TableTransformerObjectDetectionOutput:
+        r"""
+        decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*):
+            Not used by default. Can be used to mask object queries.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you
+            can choose to directly pass a flattened representation of an image.
+        decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*):
+            Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an
+            embedded representation.
+        labels (`list[Dict]` of len `(batch_size,)`, *optional*):
+            Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the
+            following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch
+            respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes
+            in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`.
+
+        Examples:
+
+        ```python
+        >>> from huggingface_hub import hf_hub_download
+        >>> from transformers import AutoImageProcessor, TableTransformerForObjectDetection
+        >>> import torch
+        >>> from PIL import Image
+
+        >>> file_path = hf_hub_download(repo_id="nielsr/example-pdf", repo_type="dataset", filename="example_pdf.png")
+        >>> image = Image.open(file_path).convert("RGB")
+
+        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/table-transformer-detection")
+        >>> model = TableTransformerForObjectDetection.from_pretrained("microsoft/table-transformer-detection")
+
+        >>> inputs = image_processor(images=image, return_tensors="pt")
+        >>> outputs = model(**inputs)
+
+        >>> # convert outputs (bounding boxes and class logits) to Pascal VOC format (xmin, ymin, xmax, ymax)
+        >>> target_sizes = torch.tensor([image.size[::-1]])
+        >>> results = image_processor.post_process_object_detection(outputs, threshold=0.9, target_sizes=target_sizes)[
+        ...     0
+        ... ]
+
+        >>> for score, label, box in zip(results["scores"], results["labels"], results["boxes"]):
+        ...     box = [round(i, 2) for i in box.tolist()]
+        ...     print(
+        ...         f"Detected {model.config.id2label[label.item()]} with confidence "
+        ...         f"{round(score.item(), 3)} at location {box}"
+        ...     )
+        Detected table with confidence 1.0 at location [202.1, 210.59, 1119.22, 385.09]
+        ```"""
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        # First, sent images through TABLE_TRANSFORMER base model to obtain encoder + decoder outputs
+        outputs = self.model(
+            pixel_values,
+            pixel_mask=pixel_mask,
+            decoder_attention_mask=decoder_attention_mask,
+            encoder_outputs=encoder_outputs,
+            inputs_embeds=inputs_embeds,
+            decoder_inputs_embeds=decoder_inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        sequence_output = outputs[0]
+
+        # class logits + predicted bounding boxes
+        logits = self.class_labels_classifier(sequence_output)
+        pred_boxes = self.bbox_predictor(sequence_output).sigmoid()
+
+        loss, loss_dict, auxiliary_outputs = None, None, None
+        if labels is not None:
+            outputs_class, outputs_coord = None, None
+            if self.config.auxiliary_loss:
+                intermediate = outputs.intermediate_hidden_states if return_dict else outputs[4]
+                outputs_class = self.class_labels_classifier(intermediate)
+                outputs_coord = self.bbox_predictor(intermediate).sigmoid()
+            loss, loss_dict, auxiliary_outputs = self.loss_function(
+                logits, labels, self.device, pred_boxes, self.config, outputs_class, outputs_coord
+            )
+
+        if not return_dict:
+            if auxiliary_outputs is not None:
+                output = (logits, pred_boxes) + auxiliary_outputs + outputs
+            else:
+                output = (logits, pred_boxes) + outputs
+            return ((loss, loss_dict) + output) if loss is not None else output
+
+        return TableTransformerObjectDetectionOutput(
+            loss=loss,
+            loss_dict=loss_dict,
+            logits=logits,
+            pred_boxes=pred_boxes,
+            auxiliary_outputs=auxiliary_outputs,
+            last_hidden_state=outputs.last_hidden_state,
+            decoder_hidden_states=outputs.decoder_hidden_states,
+            decoder_attentions=outputs.decoder_attentions,
+            cross_attentions=outputs.cross_attentions,
+            encoder_last_hidden_state=outputs.encoder_last_hidden_state,
+            encoder_hidden_states=outputs.encoder_hidden_states,
+            encoder_attentions=outputs.encoder_attentions,
+        )
+
+
+# TODO: use modular - Copied from transformers.models.detr.modeling_detr.DetrMLPPredictionHead with Detr->TableTransformer,detr->table_transformer
+class TableTransformerMLPPredictionHead(nn.Module):
+    """
+    Very simple multi-layer perceptron (MLP, also called FFN), used to predict the normalized center coordinates,
+    height and width of a bounding box w.r.t. an image.
+
+    Copied from https://github.com/facebookresearch/table_transformer/blob/master/models/table_transformer.py
+
+    """
+
+    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
+        super().__init__()
+        self.num_layers = num_layers
+        h = [hidden_dim] * (num_layers - 1)
+        self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]))
+
+    def forward(self, x):
+        for i, layer in enumerate(self.layers):
+            x = nn.functional.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
+        return x
+
+
+__all__ = ["TableTransformerForObjectDetection", "TableTransformerModel", "TableTransformerPreTrainedModel"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/wavlm/modular_wavlm.py

@@ -0,0 +1,590 @@
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from ... import initialization as init
+from ...integrations.deepspeed import is_deepspeed_zero3_enabled
+from ...integrations.fsdp import is_fsdp_managed_module
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import BaseModelOutput, Wav2Vec2BaseModelOutput
+from ...modeling_utils import PreTrainedModel
+from ...utils import logging
+from ..wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2FeatureProjection,
+    Wav2Vec2FeedForward,
+    Wav2Vec2ForAudioFrameClassification,
+    Wav2Vec2ForCTC,
+    Wav2Vec2ForSequenceClassification,
+    Wav2Vec2ForXVector,
+    Wav2Vec2Model,
+    Wav2Vec2PositionalConvEmbedding,
+    Wav2Vec2PreTrainedModel,
+)
+from .configuration_wavlm import WavLMConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class WavLMPositionalConvEmbedding(Wav2Vec2PositionalConvEmbedding):
+    pass
+
+
+class WavLMFeatureProjection(Wav2Vec2FeatureProjection):
+    pass
+
+
+class WavLMAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(
+        self,
+        embed_dim: int,
+        num_heads: int,
+        dropout: float | int = 0.0,
+        num_buckets: int = 320,
+        max_distance: int = 800,
+        has_relative_position_bias: bool = True,
+    ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+
+        if (self.head_dim * num_heads) != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
+                f" and `num_heads`: {num_heads})."
+            )
+        self.scaling = self.head_dim**-0.5
+
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+        self.out_proj = nn.Linear(embed_dim, embed_dim)
+
+        self.num_buckets = num_buckets
+        self.max_distance = max_distance
+
+        self.gru_rel_pos_const = nn.Parameter(torch.ones(1, self.num_heads, 1, 1))
+        self.gru_rel_pos_linear = nn.Linear(self.head_dim, 8)
+
+        if has_relative_position_bias:
+            self.rel_attn_embed = nn.Embedding(self.num_buckets, self.num_heads)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        position_bias: torch.Tensor | None = None,
+        output_attentions: bool = False,
+        index=0,
+    ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:
+        """Attention layer with relative attention"""
+        bsz, tgt_len, _ = hidden_states.size()
+
+        # first pass of attention layer creates position bias
+        if position_bias is None:
+            position_bias = self.compute_bias(tgt_len, tgt_len)
+            position_bias = (
+                position_bias.unsqueeze(0).repeat(bsz, 1, 1, 1).view(bsz * self.num_heads, tgt_len, tgt_len)
+            )
+
+        # Compute relative position bias:
+        # 1) get reshape hidden_states
+        gated_hidden_states = hidden_states.view(hidden_states.shape[:-1] + (self.num_heads, -1))
+        gated_hidden_states = gated_hidden_states.permute(0, 2, 1, 3)
+
+        # 2) project hidden states
+        relative_position_proj = self.gru_rel_pos_linear(gated_hidden_states)
+        relative_position_proj = relative_position_proj.view(gated_hidden_states.shape[:-1] + (2, 4)).sum(-1)
+
+        # 3) compute gate for position bias from projected hidden states
+        gate_a, gate_b = torch.sigmoid(relative_position_proj).chunk(2, dim=-1)
+        gate_output = gate_a * (gate_b * self.gru_rel_pos_const - 1.0) + 2.0
+
+        # 4) apply gate to position bias to compute gated position_bias
+        gated_position_bias = gate_output.view(bsz * self.num_heads, -1, 1) * position_bias
+        gated_position_bias = gated_position_bias.view((-1, tgt_len, tgt_len))
+
+        attn_output, attn_weights = self.torch_multi_head_self_attention(
+            hidden_states, attention_mask, gated_position_bias, output_attentions
+        )
+
+        return attn_output, attn_weights, position_bias
+
+    def torch_multi_head_self_attention(
+        self,
+        hidden_states: torch.FloatTensor,
+        attention_mask: torch.LongTensor | torch.BoolTensor,
+        gated_position_bias: torch.FloatTensor,
+        output_attentions: bool,
+    ) -> tuple[torch.FloatTensor, torch.FloatTensor]:
+        """simple wrapper around torch's multi_head_attention_forward function"""
+        # self-attention assumes q = k = v
+        query = key = value = hidden_states.transpose(0, 1)
+        key_padding_mask = attention_mask.ne(1) if attention_mask is not None else None
+
+        # disable bias and add_zero_attn
+        bias_k = bias_v = None
+        add_zero_attn = False
+
+        # PyTorch 1.3.0 has F.multi_head_attention_forward defined
+        # so no problem with backwards compatibility
+        attn_output, attn_weights = F.multi_head_attention_forward(
+            query,
+            key,
+            value,
+            self.embed_dim,
+            self.num_heads,
+            torch.empty([0]),
+            torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)),
+            bias_k,
+            bias_v,
+            add_zero_attn,
+            self.dropout,
+            self.out_proj.weight,
+            self.out_proj.bias,
+            self.training,
+            key_padding_mask,
+            output_attentions,
+            gated_position_bias,
+            use_separate_proj_weight=True,
+            q_proj_weight=self.q_proj.weight,
+            k_proj_weight=self.k_proj.weight,
+            v_proj_weight=self.v_proj.weight,
+        )
+
+        # [Seq_Len, Batch Size, ...] -> [Batch Size, Seq_Len, ...]
+        attn_output = attn_output.transpose(0, 1)
+
+        if attn_weights is not None:
+            # IMPORTANT: Attention weights are averaged weights
+            # here which should not be the case. This is an open issue
+            # on PyTorch: https://github.com/pytorch/pytorch/issues/32590
+            attn_weights = attn_weights[:, None].broadcast_to(
+                attn_weights.shape[:1] + (self.num_heads,) + attn_weights.shape[1:]
+            )
+
+        return attn_output, attn_weights
+
+    def compute_bias(self, query_length: int, key_length: int) -> torch.FloatTensor:
+        context_position = torch.arange(query_length, dtype=torch.long)[:, None]
+        memory_position = torch.arange(key_length, dtype=torch.long)[None, :]
+        relative_position = memory_position - context_position
+        relative_position_bucket = self._relative_positions_bucket(relative_position)
+        relative_position_bucket = relative_position_bucket.to(self.rel_attn_embed.weight.device)
+        values = self.rel_attn_embed(relative_position_bucket)
+        values = values.permute([2, 0, 1])
+        return values
+
+    def _relative_positions_bucket(self, relative_positions: torch.FloatTensor) -> torch.FloatTensor:
+        num_buckets = self.num_buckets // 2
+
+        relative_buckets = (relative_positions > 0).to(torch.long) * num_buckets
+        relative_positions = torch.abs(relative_positions)
+
+        max_exact = num_buckets // 2
+        is_small = relative_positions < max_exact
+
+        relative_positions_if_large = torch.log(relative_positions.float() / max_exact)
+        relative_positions_if_large = relative_positions_if_large / math.log(self.max_distance / max_exact)
+        relative_positions_if_large = relative_positions_if_large * (num_buckets - max_exact)
+        relative_position_if_large = (max_exact + relative_positions_if_large).to(torch.long)
+        relative_position_if_large = torch.min(
+            relative_position_if_large, torch.full_like(relative_position_if_large, num_buckets - 1)
+        )
+
+        relative_buckets += torch.where(is_small, relative_positions, relative_position_if_large)
+        return relative_buckets
+
+
+class WavLMFeedForward(Wav2Vec2FeedForward):
+    pass
+
+
+class WavLMEncoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: WavLMConfig, has_relative_position_bias: bool = True):
+        super().__init__()
+        self.attention = WavLMAttention(
+            embed_dim=config.hidden_size,
+            num_heads=config.num_attention_heads,
+            dropout=config.attention_dropout,
+            num_buckets=config.num_buckets,
+            max_distance=config.max_bucket_distance,
+            has_relative_position_bias=has_relative_position_bias,
+        )
+        self.dropout = nn.Dropout(config.hidden_dropout)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.feed_forward = WavLMFeedForward(config)
+        self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+    def forward(self, hidden_states, attention_mask=None, position_bias=None, output_attentions=False, index=0):
+        attn_residual = hidden_states
+        hidden_states, attn_weights, position_bias = self.attention(
+            hidden_states,
+            attention_mask=attention_mask,
+            position_bias=position_bias,
+            output_attentions=output_attentions,
+            index=index,
+        )
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = attn_residual + hidden_states
+
+        hidden_states = self.layer_norm(hidden_states)
+
+        hidden_states = hidden_states + self.feed_forward(hidden_states)
+        hidden_states = self.final_layer_norm(hidden_states)
+
+        outputs = (hidden_states, position_bias)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+class WavLMEncoderLayerStableLayerNorm(GradientCheckpointingLayer):
+    def __init__(self, config: WavLMConfig, has_relative_position_bias: bool = True):
+        super().__init__()
+        self.attention = WavLMAttention(
+            embed_dim=config.hidden_size,
+            num_heads=config.num_attention_heads,
+            dropout=config.attention_dropout,
+            num_buckets=config.num_buckets,
+            max_distance=config.max_bucket_distance,
+            has_relative_position_bias=has_relative_position_bias,
+        )
+        self.dropout = nn.Dropout(config.hidden_dropout)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.feed_forward = WavLMFeedForward(config)
+        self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+    def forward(self, hidden_states, attention_mask=None, position_bias=None, output_attentions=False):
+        attn_residual = hidden_states
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states, attn_weights, position_bias = self.attention(
+            hidden_states,
+            attention_mask=attention_mask,
+            position_bias=position_bias,
+            output_attentions=output_attentions,
+        )
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = attn_residual + hidden_states
+        hidden_states = hidden_states + self.feed_forward(self.final_layer_norm(hidden_states))
+
+        outputs = (hidden_states, position_bias)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+class WavLMEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.pos_conv_embed = WavLMPositionalConvEmbedding(config)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout)
+        self.layers = nn.ModuleList(
+            [WavLMEncoderLayer(config, has_relative_position_bias=(i == 0)) for i in range(config.num_hidden_layers)]
+        )
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        output_attentions=False,
+        output_hidden_states=False,
+        return_dict=True,
+    ):
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+
+        if attention_mask is not None:
+            # make sure padded tokens output 0
+            expand_attention_mask = attention_mask.unsqueeze(-1).repeat(1, 1, hidden_states.shape[2])
+            hidden_states[~expand_attention_mask] = 0
+
+        position_embeddings = self.pos_conv_embed(hidden_states)
+        hidden_states = hidden_states + position_embeddings
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+
+        synced_gpus = is_deepspeed_zero3_enabled() or is_fsdp_managed_module(self)
+        position_bias = None
+
+        for i, layer in enumerate(self.layers):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)
+            dropout_probability = torch.rand([])
+
+            skip_the_layer = self.training and i > 0 and (dropout_probability < self.config.layerdrop)
+            if not skip_the_layer or synced_gpus:
+                # under fsdp or deepspeed zero3 all gpus must run in sync
+                layer_outputs = layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_bias=position_bias,
+                    output_attentions=output_attentions,
+                    index=i,
+                )
+
+                hidden_states, position_bias = layer_outputs[:2]
+
+            if skip_the_layer:
+                layer_outputs = (None, None, None)
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[2],)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+class WavLMEncoderStableLayerNorm(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.pos_conv_embed = WavLMPositionalConvEmbedding(config)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout)
+        self.layers = nn.ModuleList(
+            [
+                WavLMEncoderLayerStableLayerNorm(config, has_relative_position_bias=(i == 0))
+                for i in range(config.num_hidden_layers)
+            ]
+        )
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        output_attentions=False,
+        output_hidden_states=False,
+        return_dict=True,
+    ):
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+
+        if attention_mask is not None:
+            # make sure padded tokens are not attended to
+            expand_attention_mask = attention_mask.unsqueeze(-1).repeat(1, 1, hidden_states.shape[2])
+            hidden_states[~expand_attention_mask] = 0
+
+        position_embeddings = self.pos_conv_embed(hidden_states)
+        hidden_states = hidden_states + position_embeddings
+        hidden_states = self.dropout(hidden_states)
+
+        synced_gpus = is_deepspeed_zero3_enabled() or is_fsdp_managed_module(self)
+        position_bias = None
+
+        for i, layer in enumerate(self.layers):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)
+            dropout_probability = torch.rand([])
+
+            skip_the_layer = self.training and i > 0 and (dropout_probability < self.config.layerdrop)
+            if not skip_the_layer or synced_gpus:
+                # under fsdp or deepspeed zero3 all gpus must run in sync
+                # XXX: could optimize this like synced_gpus in generate_utils but not sure if it's worth the code complication
+                layer_outputs = layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    output_attentions=output_attentions,
+                    position_bias=position_bias,
+                )
+                hidden_states, position_bias = layer_outputs[:2]
+
+            if skip_the_layer:
+                layer_outputs = (None, None, None)
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[2],)
+
+        hidden_states = self.layer_norm(hidden_states)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states, hidden_states=all_hidden_states, attentions=all_self_attentions
+        )
+
+
+class WavLMGumbelVectorQuantizer(nn.Module):
+    """
+    Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH
+    GUMBEL-SOFTMAX](https://huggingface.co/papers/1611.01144) for more information.
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        self.num_groups = config.num_codevector_groups
+        self.num_vars = config.num_codevectors_per_group
+
+        if config.codevector_dim % self.num_groups != 0:
+            raise ValueError(
+                f"`config.codevector_dim {config.codevector_dim} must be divisible"
+                f" by `config.num_codevector_groups` {self.num_groups} "
+                "for concatenation."
+            )
+
+        # storage for codebook variables (codewords)
+        self.codevectors = nn.Parameter(
+            torch.FloatTensor(1, self.num_groups * self.num_vars, config.codevector_dim // self.num_groups)
+        )
+        self.weight_proj = nn.Linear(config.conv_dim[-1], self.num_groups * self.num_vars)
+
+        # can be decayed for training
+        self.temperature = 2
+
+    @staticmethod
+    def _compute_perplexity(probs):
+        marginal_probs = probs.mean(dim=0)
+        perplexity = torch.exp(-torch.sum(torch.xlogy(marginal_probs, marginal_probs), dim=-1)).sum()
+        return perplexity
+
+    def forward(self, hidden_states):
+        batch_size, sequence_length, hidden_size = hidden_states.shape
+
+        # project to codevector dim
+        hidden_states = self.weight_proj(hidden_states)
+        hidden_states = hidden_states.view(batch_size * sequence_length * self.num_groups, -1)
+
+        if self.training:
+            # sample code vector probs via gumbel in differentiateable way
+            codevector_probs = nn.functional.gumbel_softmax(hidden_states.float(), tau=self.temperature, hard=True)
+            codevector_probs = codevector_probs.type_as(hidden_states)
+
+            # compute perplexity
+            codevector_soft_dist = torch.softmax(
+                hidden_states.view(batch_size * sequence_length, self.num_groups, -1).float(), dim=-1
+            )
+            perplexity = self._compute_perplexity(codevector_soft_dist)
+        else:
+            # take argmax in non-differentiable way
+            # comptute hard codevector distribution (one hot)
+            codevector_idx = hidden_states.argmax(dim=-1)
+            codevector_probs = hidden_states.new_zeros(*hidden_states.shape).scatter_(
+                -1, codevector_idx.view(-1, 1), 1.0
+            )
+            codevector_probs = codevector_probs.view(batch_size * sequence_length, self.num_groups, -1)
+
+            perplexity = self._compute_perplexity(codevector_probs)
+
+        codevector_probs = codevector_probs.view(batch_size * sequence_length, -1)
+        # use probs to retrieve codevectors
+        codevectors_per_group = codevector_probs.unsqueeze(-1) * self.codevectors
+        codevectors = codevectors_per_group.view(batch_size * sequence_length, self.num_groups, self.num_vars, -1)
+        codevectors = codevectors.sum(-2).view(batch_size, sequence_length, -1)
+
+        return codevectors, perplexity
+
+
+class WavLMPreTrainedModel(PreTrainedModel, Wav2Vec2PreTrainedModel):
+    config: WavLMConfig
+    base_model_prefix = "wavlm"
+    main_input_name = "input_values"
+    input_modalities = "audio"
+    supports_gradient_checkpointing = True
+    _supports_flash_attn = False
+    _supports_sdpa = False
+    _supports_flex_attn = False
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        # gumbel softmax requires special init
+        if isinstance(module, WavLMGumbelVectorQuantizer):
+            init.normal_(module.weight_proj.weight, mean=0.0, std=1)
+            init.zeros_(module.weight_proj.bias)
+            init.uniform_(module.codevectors)
+        elif isinstance(module, WavLMPositionalConvEmbedding):
+            init.normal_(
+                module.conv.weight,
+                mean=0,
+                std=2 * math.sqrt(1 / (module.conv.kernel_size[0] * module.conv.in_channels)),
+            )
+            init.constant_(module.conv.bias, 0)
+        elif isinstance(module, WavLMFeatureProjection):
+            k = math.sqrt(1 / module.projection.in_features)
+            init.uniform_(module.projection.weight, a=-k, b=k)
+            init.uniform_(module.projection.bias, a=-k, b=k)
+        elif isinstance(module, nn.Linear):
+            init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
+
+            if module.bias is not None:
+                init.zeros_(module.bias)
+        elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)):
+            init.zeros_(module.bias)
+            init.ones_(module.weight)
+        elif isinstance(module, nn.Conv1d):
+            init.kaiming_normal_(module.weight)
+
+            if module.bias is not None:
+                k = math.sqrt(module.groups / (module.in_channels * module.kernel_size[0]))
+                init.uniform_(module.bias, a=-k, b=k)
+
+    def _get_adapters(self):
+        raise AttributeError("Not needed for WavLM")
+
+    def init_adapter_layers(self):
+        raise AttributeError("Not needed for WavLM")
+
+    def load_adapter(self):
+        raise AttributeError("Not needed for WavLM")
+
+
+WavLMBaseModelOutput = Wav2Vec2BaseModelOutput
+
+
+class WavLMModel(Wav2Vec2Model):
+    pass
+
+
+class WavLMForCTC(Wav2Vec2ForCTC):
+    pass
+
+
+class WavLMForSequenceClassification(Wav2Vec2ForSequenceClassification):
+    pass
+
+
+class WavLMForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification):
+    pass
+
+
+class WavLMForXVector(Wav2Vec2ForXVector):
+    pass
+
+
+__all__ = [
+    "WavLMForAudioFrameClassification",
+    "WavLMForCTC",
+    "WavLMForSequenceClassification",
+    "WavLMForXVector",
+    "WavLMModel",
+    "WavLMPreTrainedModel",
+]