demo: self-contained Gradio app (run locally)

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/blobs/23440d6773792a7758b5aab66878ba5f11f56f81235394a3856718c82e2a8f1e filter=lfs diff=lfs merge=lfs -text
+demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/blobs/e33efa79e656bdd0972beb15a07b11b74df0df1368308bd12ec6b02337887040 filter=lfs diff=lfs merge=lfs -text
+demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/196088ae0adbafc572d99e62d0090a4a25a0bb305493ff454a7f9fb2171c1566.incomplete filter=lfs diff=lfs merge=lfs -text
+demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/4fe7c3c6498b6c26f0b5b6c60265b4d88ab4db02e1bbfb7f06cea3dc4746874b filter=lfs diff=lfs merge=lfs -text
+demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/7a4eb87011448a4564a3144979384da51eee1da95e554feb22ccc85529535dd5 filter=lfs diff=lfs merge=lfs -text
+demo/runorm_cache/models--RUNorm--RUNorm-tagger/blobs/707128ea4c32fd7037ff0e49cd9b0234f0311f96e1b69ed413a7e90323b0b9e9 filter=lfs diff=lfs merge=lfs -text
+demo/runorm_cache/models--RUNorm--RUNorm-tagger/blobs/e636f33b833a1e4e1c090ea2d7e0253288a86c4fff54c8b4d879ef7bb1b1ce0e filter=lfs diff=lfs merge=lfs -text

demo/LICENSE-APACHE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

demo/LICENSE-MIT

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Nikita Torgashov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

demo/NOTICE

@@ -0,0 +1,13 @@
+NOTICE
+
+This project includes the Depth Transformer component developed by SesameAI,
+licensed under the Apache License, Version 2.0.
+
+You may obtain a copy of the Apache License at:
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

demo/app.py

@@ -0,0 +1,516 @@
+"""VoXtream2-RU — Gradio-демо (аналог HF space herimor/voxtream2, но с русской моделью).
+
+Работает в двух режимах:
+1) Локально:   python ru_finetune/ft4/space/app.py --model-dir ru_finetune/ft4/infer_model
+2) HF Space:   переменная окружения MODEL_REPO=<user>/voxtream2-ru (repo с файлами
+               model.safetensors, config.json, phoneme_to_token.json, ru_tokens.json) —
+               файлы скачиваются через hf_hub_download.
+
+Отличия от EN-демо: RUAccent ставит ударения (решает омографы за́мок/замо́к),
+espeak-ru фонемизация, спец-токены расширенного словаря.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+# HF Space: пакет voxtream и модули (v10_fix_dict, build_groups_v3) вендорены рядом
+if str(HERE) not in sys.path:
+    sys.path.insert(0, str(HERE))
+from v10_fix_dict import FIX as INTERJ_FIX, redup_phones as _interj_redup  # noqa: E402
+
+LOCAL_REPO_ID = "LOCAL_RU"
+VOWEL = set("aeiouyɑɛɔʌəɵɨæøœɐɒʉʊɪ")
+RU_VOWELS = set("аеёиоуыэюя")
+PUNCT = (".", ",", "?", "!")
+# v4: односимвольные согласные слова = проклитики (один звук), а не названия букв.
+# espeak-ru даёт «к» -> k ˈɑ («ка») во ВСЕХ контекстах — источник жалобы
+# «говорит "ка" вместо "к"». Совпадает с mfa_dict_v4.txt (обучение).
+PROCLITIC = {"в": "v", "с": "s", "к": "k", "ж": "ʒ", "б": "b"}
+STRIP_WORD = ".,?!—-«»\"'()…:;–„“”+"
+# символы, которые НЕ являются фонемами: если прилипнут к фонеме, токен уходит в UNK
+NON_PHONE = ".,?!—-«»\"'()…:;–„“”"
+# после каких знаков вставлять паузу-токен (как в обучающих TextGrid)
+SIL_AFTER = ".,!?"
+
+# Модель знает только .,?! — остальные знаки нормализуем в ближайший по интонации
+# (иначе espeak приклеивает их к фонеме: «добавил:» -> «ɭ:» -> UNK -> «добаве»).
+_PUNCT_MAP = {":": ",", ";": ",", "—": ",", "–": ",", "…": ".", "«": "", "»": "",
+              "„": "", "“": "", "”": "", '"': "", "(": "", ")": ""}
+
+
+TARGET_LUFS = -23.0  # = уровень корпуса v10 (клипы приводятся при рендере групп)
+
+
+def normalize_lufs(wav, sr):
+    """v10: промпт -> -23 LUFS. Тихая бытовая запись для модели OOD; корпус
+    нормализован к тому же уровню (стандарт NeMo/NVIDIA voice cloning)."""
+    import numpy as np
+    import pyloudnorm
+
+    mono = wav.mean(dim=0).numpy().astype("float32")
+    if len(mono) < int(0.5 * sr):  # метру BS.1770 нужно >= 0.4 c
+        return wav
+    try:
+        loud = pyloudnorm.Meter(sr).integrated_loudness(mono)
+    except Exception:  # noqa: BLE001
+        return wav
+    if not np.isfinite(loud) or loud < -70:
+        return wav
+    out = wav * float(10 ** ((TARGET_LUFS - loud) / 20))
+    peak = float(out.abs().max())
+    if peak > 0.99:  # true-peak защита
+        out = out * (0.99 / peak)
+    return out
+
+
+def condition_prompt(wav, sr):
+    """v10.1: подготовка границы промпта. Замер: 5/6 тестовых промптов обрезаны
+    ПОСРЕДИ слова (жёсткий срез на 8 с), и провалы тембра сидят в первых ~1 с
+    генерации (окно 0.5 с: sim 0.11-0.29), а единственный промпт с тихим концом
+    (sanya_agin) — единственный без переключений. В обучении модель всегда
+    продолжает ПОСЛЕ паузы с комнатным тоном. Делаем так же:
+      1) если конец промпта — речь, режем назад до ближайшей тишины (>= 80 мс
+         ниже -35 dB от пика, в последних 2.5 с);
+      2) добавляем 0.35 с комнатного тона, синтезированного из тихого окна
+         самого промпта (build_groups_v3.synth_tone). VOXTREAM_PROMPT_COND=0 выкл."""
+    import numpy as np
+    if os.environ.get("VOXTREAM_PROMPT_COND", "1") != "1":
+        return wav
+    mono = wav.mean(dim=0).numpy().astype("float32")
+    n = len(mono)
+    fr = max(int(0.02 * sr), 1)
+    peak = float(np.abs(mono).max()) + 1e-9
+    if n > int(3.0 * sr):
+        # порог тишины адаптивный: 20-й перцентиль энергии 20-мс кадров записи
+        # (паузы ~20-30% речи), но не выше -25 дБ от пика (шумные бытовые записи)
+        nf_all = n // fr
+        e_all = 20 * np.log10(np.sqrt((mono[: nf_all * fr].reshape(nf_all, fr) ** 2).mean(1)) + 1e-9) - 20 * np.log10(peak)
+        thr = min(float(np.percentile(e_all, 20)), -25.0)
+        win = int(min(4.0 * sr, n - 2.0 * sr))
+        seg_e = e_all[-(win // fr):]
+        run, cut = 0, None
+        for i, q in enumerate(seg_e < thr):
+            run = run + 1 if q else 0
+            if run >= 3:  # >= 60 мс тишины
+                cut = i
+        if cut is None:
+            # паузы нет (сплошная речь): режем в самом глубоком провале энергии
+            # последних 2 с (смычка/межсловный спад — не середина гласной)
+            tail_e = e_all[-int(1.2 * sr) // fr:]
+            deep = np.where(tail_e < float(np.median(e_all)) - 6.0)[0]
+            if len(deep):  # ближайший к концу провал — теряем минимум промпта
+                cut = len(seg_e) - len(tail_e) + int(deep[-1]) + 1
+        if cut is not None:
+            end = (nf_all - len(seg_e) + cut - 1) * fr  # внутри найденной тишины/провала
+            if end > int(2.0 * sr):
+                wav = wav[:, :end]
+                mono = mono[:end]
+    try:
+        import build_groups_v3 as B
+        tmpl = B.room_tone(mono)
+        tone = B.synth_tone(tmpl, int(0.35 * sr), seed=7).astype("float32")
+    except Exception:  # noqa: BLE001
+        tone = (np.random.randn(int(0.35 * sr)) * 1e-4).astype("float32")
+    import torch
+    tone_t = torch.from_numpy(tone)[None].expand(wav.shape[0], -1)
+    return torch.cat([wav, tone_t.to(wav.dtype)], dim=1)
+
+
+def normalize_punct(text: str) -> str:
+    for src, dst in _PUNCT_MAP.items():
+        text = text.replace(src, dst)
+    text = re.sub(r"\s+([,.!?])", r"\1", text)      # пробел перед знаком
+    text = re.sub(r"([,.!?])\1+", r"\1", text)      # дубли знаков
+    return re.sub(r"\s{2,}", " ", text).strip()
+
+
+def is_vowel(t):
+    return any(c in VOWEL for c in t)
+
+
+def stressed_idx(aw):
+    vi, cnt, s, i = -1, 0, aw.lower(), 0
+    while i < len(s):
+        if s[i] == "+":
+            if i + 1 < len(s) and s[i + 1] in RU_VOWELS:
+                vi = cnt
+            i += 1
+            continue
+        if s[i] in RU_VOWELS:
+            cnt += 1
+        i += 1
+    return vi
+
+
+class RUAccentPhonemizer:
+    """Интерфейс ESpeak.phonemize + RUAccent-ударения (перенос ˈ на нужную гласную)."""
+
+    def __init__(self):
+        from ruaccent import RUAccent
+        from voxtream.utils.text.phonemizer import ESpeak
+        self.acc = RUAccent()
+        self.acc.load(omograph_model_size="turbo3.1", use_dictionary=True)
+        self.esp = ESpeak("ru")
+        self._runorm = None  # ленивая загрузка: нужна только для текстов с цифрами
+
+    def _normalize_digits(self, text: str) -> str:
+        """v10: цифры/числа -> слова (RUNorm, падежи/род учитывает). Пользователь
+        пишет «в 2024 году» — работает без ручной нормализации. «Ё», потерянную
+        RUNorm'ом («четвертом»), ниже восстановит RUAccent."""
+        if not re.search(r"\d", text):
+            return text
+        if self._runorm is None:
+            from runorm import RUNorm
+            self._runorm = RUNorm()
+            self._runorm.load(model_size="medium",
+                              workdir=str(HERE / "runorm_cache"))
+        # только предложения С цифрами: на остальных RUNorm вредит — разворачивает
+        # междометия как аббревиатуры («Ммм» -> «эм эм эм») мимо нашей таблицы
+        try:
+            parts = re.split(r"(?<=[.!?…])\s+", text)
+            return " ".join(
+                self._runorm.norm(p) if re.search(r"\d", p) else p for p in parts
+            )
+        except Exception as e:  # noqa: BLE001 — цифры хуже, чем необработанный текст
+            print(f"[runorm] fail: {e}; текст без нормализации")
+            return text
+
+    def _accent(self, text: str) -> str:
+        """Ударения: ручной '+' перед гласной (зам+ок) имеет приоритет,
+        RUAccent ставит только в словах без ручной пометки."""
+        if "+" not in text:
+            return self.acc.process_all(text)
+        plain = re.sub(r"\+", "", text)
+        auto = self.acc.process_all(plain)
+        manual_w, auto_w = text.split(), auto.split()
+        if len(manual_w) != len(auto_w):
+            return auto  # рассинхрон токенизации — безопасный фолбэк
+        return " ".join(
+            mw if "+" in mw else aw for mw, aw in zip(manual_w, auto_w)
+        )
+
+    def phonemize(self, text, separator="|", language="ru"):
+        text = self._normalize_digits(text)
+        text = normalize_punct(text)
+        accented = self._accent(text)
+        clean = re.sub(r"\+", "", accented)
+        seq = self.esp.phonemize(clean, separator=separator, language="ru")
+        esp_words, acc_words = seq.split(), accented.split()
+        if len(esp_words) != len(acc_words):
+            return seq
+        out = []
+        for ew, aw in zip(esp_words, acc_words):
+            phones = [p for p in ew.split(separator) if p]
+            # фикс v4: espeak озвучивает предлог «к» как НАЗВАНИЕ буквы (k ˈɑ ->
+            # «ка тебе»). Односимвольные согласные слова — проклитики в один звук.
+            bare = aw.strip(STRIP_WORD).lower()
+            if bare in PROCLITIC:
+                keep = PROCLITIC[bare]
+                tail_p = phones[-1][-1] if phones and phones[-1][-1] in "".join(PUNCT) else ""
+                phones = [keep + tail_p] if tail_p else [keep]
+            # v10: междометия (хм/ммм/тсс...) espeak читает НАЗВАНИЯМИ букв
+            # («ха-эм»); в словаре v5 они — чистые согласные. Таблица общая
+            # с v10_fix_dict, чтобы цикл обучение<->инференс не расходился.
+            elif (ij := INTERJ_FIX.get(bare) or _interj_redup(bare)) is not None:
+                tail_p = phones[-1][-1] if phones and phones[-1][-1] in "".join(PUNCT) else ""
+                phones = ij.split()
+                if tail_p:
+                    phones[-1] += tail_p
+            tail = ""
+            if phones and phones[-1] and phones[-1][-1] in "".join(PUNCT):
+                tail = phones[-1][-1]
+                phones[-1] = phones[-1][:-1]
+                if not phones[-1]:
+                    phones.pop()
+            # страховка: любой НЕ-фонемный хвост (":", ";", "»", ")"…) прилипает к
+            # последней фонеме -> токен «ɭ:» отсутствует в словаре -> UNK -> звук
+            # пропадает («добавил:» звучало как «добаве»). Чистим остатки.
+            phones = [p for p in (q.strip(NON_PHONE) for q in phones) if p]
+            ti = stressed_idx(aw)
+            if ti >= 0 and phones:
+                cl = [p.replace("ˈ", "").replace("ˌ", "") for p in phones]
+                vp = [j for j, p in enumerate(cl) if is_vowel(p)]
+                if ti < len(vp):
+                    cl[vp[ti]] = "ˈ" + cl[vp[ti]]
+                    phones = cl
+            w = separator.join(phones)
+            out.append(w + tail if tail else w)
+            # ФИКС ПАУЗ: в обучении паузы стоят в потоке ЯВНЫМИ токенами 'sil'
+            # (MFA-разметка, 6.3% токенов), а espeak их не даёт — модель получала
+            # непрерывный поток и произносила всё слитно (на «Раз, два, три,
+            # четыре, пять.» — НОЛЬ пауз; звуки проглатывались). Вставляем sil
+            # после знака: длительность паузы модель выберет сама.
+            if tail and tail in SIL_AFTER:
+                out.append("sil")
+        return " ".join(out)
+
+
+def resolve_model_files():
+    """-> dict имя_файла -> локальный путь (из --model-dir или MODEL_REPO)."""
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model-dir", default=os.environ.get("MODEL_DIR", ""))
+    args, _ = ap.parse_known_args()
+
+    names = ["model.safetensors", "config.json", "phoneme_to_token.json", "ru_tokens.json"]
+    if args.model_dir:
+        d = Path(args.model_dir).resolve()
+        return {n: str(d / n) for n in names}
+    repo = os.environ.get("MODEL_REPO")
+    assert repo, "укажите --model-dir или env MODEL_REPO"
+    from huggingface_hub import hf_hub_download
+    return {n: hf_hub_download(repo, n) for n in names}
+
+
+def main():
+    files = resolve_model_files()
+    ru_tokens = json.load(open(files["ru_tokens.json"]))
+
+    # --- монки-патчи ДО импорта app ---
+    import voxtream.utils.generator.setup as S
+    import voxtream.utils.generator.text as T
+    from huggingface_hub import hf_hub_download as _real_hf
+
+    def _hf(repo_id, filename, **kw):
+        if repo_id == LOCAL_REPO_ID:
+            return files[filename]
+        return _real_hf(repo_id, filename, **kw)
+
+    S.hf_hub_download = _hf
+
+    _orig_ttp = T.text_to_phone_tokens
+
+    # E1 (question-prefix модели): '?' после первого слова вопросительного
+    # предложения — как в обучении (PT видит «впереди вопрос» с самого начала).
+    # Включается флагом QUESTION_PREFIX=1 (для infer_model_e).
+    q_prefix = os.environ.get("QUESTION_PREFIX", "0") == "1"
+
+    def _add_q_prefix(text: str) -> str:
+        out = []
+        for sent in re.split(r"(?<=[.!?])\s+", str(text).strip()):
+            words = sent.split()
+            if sent.rstrip().endswith("?") and len(words) > 1:
+                words[0] += "?"
+            out.append(" ".join(words))
+        return " ".join(out)
+
+    def _ttp(*a, **kw):
+        kw["normalize"] = False
+        kw["language"] = "ru"
+        if q_prefix and a and isinstance(a[0], str):
+            a = (_add_q_prefix(a[0]),) + a[1:]
+        elif q_prefix and "text" in kw:
+            kw["text"] = _add_q_prefix(kw["text"])
+        return _orig_ttp(*a, **kw)
+
+    T.text_to_phone_tokens = _ttp
+
+    # v10: LUFS-нормализация промпта — шим над torchaudio ТОЛЬКО внутри prompt.py
+    # (глобальный torchaudio.load не трогаем). Внимание: .prompt.npy-кэши,
+    # созданные до нормализации, устаревают — их надо удалить.
+    import voxtream.utils.generator.prompt as PR
+    _orig_ta = PR.torchaudio
+
+    class _LufsTorchaudio:
+        def __getattr__(self, name):
+            return getattr(_orig_ta, name)
+
+        @staticmethod
+        def load(path, *a, **kw):
+            wav, sr = _orig_ta.load(path, *a, **kw)
+            return condition_prompt(normalize_lufs(wav, sr), sr), sr
+
+    PR.torchaudio = _LufsTorchaudio()
+
+    from voxtream.generator import SpeechGenerator
+    _orig_init = SpeechGenerator.__init__
+
+    def _patched_init(self, *a, **kw):
+        _orig_init(self, *a, **kw)
+        self.ctx.phonemizer = RUAccentPhonemizer()
+
+    SpeechGenerator.__init__ = _patched_init
+
+    # v10.1: синтез ПО ПРЕДЛОЖЕНИЯМ с повторной привязкой к промпту.
+    # Замер 384 сэмплов: переключение голоса после паузы между предложениями —
+    # 12-16% на многопредложенческих фразах против 3% на одиночных; трудные
+    # голоса (shibakov 33%) теряются после паузы. Каждое предложение стартует
+    # сразу после промпта, где привязка максимальна; между ними — пауза
+    # (медиана корпуса по знаку). VOXTREAM_SENT_SPLIT=0 выключает.
+    _orig_gs = SpeechGenerator.generate_stream
+    _PAUSE = {".": 0.39, "?": 0.42, "!": 0.41}  # эмпирика 4.5 млн пауз (build_groups)
+
+    def _split_sentences(text: str):
+        parts = [p.strip() for p in re.split(r"(?<=[.!?…])\s+", text.strip()) if p.strip()]
+        return parts if len(parts) >= 2 else [text]
+
+    def _gs_split(self, prompt_audio_path, text, speaking_rate=None, enhance_prompt=None,
+                  apply_vad=None, return_progress=False, min_streaming_rtf=None):
+        if os.environ.get("VOXTREAM_SENT_SPLIT", "1") != "1" or not isinstance(text, str):
+            yield from _orig_gs(self, prompt_audio_path, text, speaking_rate, enhance_prompt,
+                                apply_vad, return_progress, min_streaming_rtf)
+            return
+        sents = _split_sentences(text)
+        if len(sents) < 2:
+            yield from _orig_gs(self, prompt_audio_path, text, speaking_rate, enhance_prompt,
+                                apply_vad, return_progress, min_streaming_rtf)
+            return
+        import numpy as _np
+        sr = int(self.config.mimi_sr)
+        pos_off, time_off, last_prog = 0, 0.0, None
+        for k, sent in enumerate(sents):
+            last_pos, last_t = 0, 0.0
+            for item in _orig_gs(self, prompt_audio_path, sent, speaking_rate, enhance_prompt,
+                                 apply_vad, return_progress, min_streaming_rtf):
+                if return_progress:
+                    frame, gt, prog = item
+                    prog = dict(prog)
+                    last_pos = max(last_pos, int(prog.get("phone_position", 0) or 0))
+                    last_t = max(last_t, float(prog.get("time_sec", 0.0) or 0.0))
+                    prog["phone_position"] = pos_off + int(prog.get("phone_position", 0) or 0)
+                    prog["time_sec"] = time_off + float(prog.get("time_sec", 0.0) or 0.0)
+                    last_prog = prog
+                    yield frame, gt, prog
+                else:
+                    yield item
+            pos_off += last_pos + 1
+            time_off += last_t
+            if k < len(sents) - 1:
+                gap = _PAUSE.get(sent[-1], 0.39)
+                n = int(gap * sr)
+                noise = (_np.random.randn(n) * 1e-4).astype(_np.float32)  # не «цифровой» ноль
+                time_off += gap
+                if return_progress:
+                    prog = dict(last_prog or {})
+                    prog["time_sec"] = time_off
+                    yield noise, 0.0, prog
+                else:
+                    yield noise, 0.0
+
+    # v10.1: BEST-OF-N по удержанию голоса (offline-режим). Замер: 8-12% сэмплов
+    # с провалом оконной spk-sim к промпту (>0.25 от медианы), реальная речь тех
+    # же людей — 0/8; ни данные (S2b), ни сэмплирование/CFG/SRC/подготовка
+    # промпта цифру не сдвинули. Best-of-2 -> ~0.7% ожидаемо. Скорим тем же
+    # ReDimNet, что кондиционирует модель (ctx.spk_enc). VOXTREAM_BEST_OF=1 выкл.
+    from voxtream.utils.generator.prompt import extract_speaker_template as _est
+    import voxtream.utils.generator.prompt as _PR
+
+    def _spk_score(self, prompt_audio_path, audio_np):
+        import numpy as _np
+        import torch as _t
+        wav, sr = _PR.torchaudio.load(str(prompt_audio_path))  # шим: LUFS + граница
+        pe = _est(wav.mean(0, keepdim=True), sr, self.ctx.spk_enc, self.config.spk_enc_sr,
+                  self.ctx.device, self.ctx.dtype).float().reshape(-1)
+        gen = _t.from_numpy(audio_np.astype("float32"))[None]
+        osr = int(self.config.mimi_sr)
+        W, H = int(1.5 * osr), int(0.5 * osr)
+        sims = []
+        for st in range(0, max(gen.shape[1] - W, 1), H):
+            e = _est(gen[:, st:st + W], osr, self.ctx.spk_enc, self.config.spk_enc_sr,
+                     self.ctx.device, self.ctx.dtype).float().reshape(-1)
+            sims.append(float(pe @ e))
+        if not sims:
+            return 0.0, 0.0
+        return float(min(sims)), float(_np.median(sims))
+
+    def _gs_bestof(self, prompt_audio_path, text, speaking_rate=None, enhance_prompt=None,
+                   apply_vad=None, return_progress=False, min_streaming_rtf=None):
+        try:
+            N = int(os.environ.get("VOXTREAM_BEST_OF", "2"))
+        except ValueError:
+            N = 1
+        if N <= 1 or not isinstance(text, str):
+            yield from _gs_split(self, prompt_audio_path, text, speaking_rate, enhance_prompt,
+                                 apply_vad, return_progress, min_streaming_rtf)
+            return
+        import numpy as _np
+        best, best_key = None, None
+        for k in range(N):
+            items = list(_gs_split(self, prompt_audio_path, text, speaking_rate, enhance_prompt,
+                                   apply_vad, return_progress, min_streaming_rtf))
+            audio = _np.concatenate([it[0] for it in items]) if items else _np.zeros(1, "float32")
+            mn, med = _spk_score(self, prompt_audio_path, audio)
+            key = (mn >= med - 0.25, mn)  # сначала «без провала», затем по худшему окну
+            print(f"[best-of-{N}] кандидат {k + 1}: sim_min {mn:.2f} med {med:.2f}", flush=True)
+            if best_key is None or key > best_key:
+                best, best_key = items, key
+            if best_key[0]:  # чистый кандидат — хватит (перегенерация только при провале)
+                break
+        yield from best
+
+    SpeechGenerator.generate_stream = _gs_bestof
+
+    # --- конфиги: базовые из репо + RU-переопределения ---
+    root = HERE
+    gen_cfg = json.load(open(root / "configs/generator.json"))
+    gen_cfg.update(
+        model_repo=LOCAL_REPO_ID,
+        unk_token=ru_tokens["unk"], eop_token=ru_tokens["unk"],
+        bos_token=ru_tokens["bos"], eos_token=ru_tokens["eos"],
+        sil_token=ru_tokens["sil"],
+        enhance_prompt=False, apply_vad=False,
+        # v10: свип 12 ячеек temp x topk на промптах пользователя (v9, 2026-08-08):
+        # 0.8/50 -> CER 0.0105, CV 0.159 против 0.023/0.177 у унаследованных 0.9/5;
+        # t1.0/k5 давал CER 0.0096, но худший CV 0.202 и темп 4.89 — не взят.
+        temperature=0.8, topk=50,
+        # v4 (свипы на ПРОМПТАХ ПОЛЬЗОВАТЕЛЯ ru_finetune/unseen — «удобные»
+        # корпусные промпты регрессию не ловили): при gain=25 cfg_gamma 1.5 даёт
+        # CER 0.021 против 0.056 у 2.0 и 0.024 у 3.0; unseen spk_sim 0.602 против
+        # 0.616 у 3.0, НО худший случай 0.422 против 0.269 — провалов нет.
+        cfg_gamma=1.5,
+    )
+    # Усиление SPS-кондиционирования. gain=40 подбирался на stage D, где бины
+    # считались по БУКВАМ и эмбеддинг почти не обучался. В v4 бины из ФОНЕМ —
+    # эмбеддинг информативнее, и 40 РАЗРУШАЕТ генерацию: на промптах пользователя
+    # CER 0.366 и скачки темпа (жалоба «неразборчиво, то быстро то медленно»).
+    # Свип по CER: gain 1-20 → 0.016-0.019, 25 → 0.024, 30 → 0.075, 40 → 0.366.
+    # 25 — компромисс: разборчивость вдвое лучше v3b при самом медленном темпе
+    # среди «чистых» режимов. Замедление ниже ~5.7 SPS этим механизмом ЛОМАЕТ речь
+    # (gain20/rate3.5 → CER 0.121; gain30/rate3.5 → 0.584) — темп чинить обучением.
+    os.environ.setdefault("VOXTREAM_SPS_GAIN", "25")
+    # Управление темпом: SRC вместо SPS-эмбеддинга. Замер на v7 (промпты юзера):
+    # SPS gain30/rate3.5 -> темп 4.64 но CER 0.245 (речь рвётся); SRC rate3.5 ->
+    # темп 5.37 при CER 0.025 и паузах 0.39с (= корпусная норма). SRC не искажает
+    # эмбеддинг, поэтому замедление безопасно.
+    os.environ.setdefault("VOXTREAM_TEMPO_MODE", "src")
+    # v10.1: лимит удержания фонемы по классу (кадры по 80 мс). Жалоба «буквы
+    # повторяются» = залипание на последней согласной фразы («Потомммм,» — 26
+    # кадров при защите frame_repeat_counter=25). В данных 97% согласных перед
+    # знаком <= 5 кадров, 99% гласных <= 12. A/B на S2-e2: без лимита макс согл.
+    # 11 кадров, CER 0.029; с лимитом — макс 5-6, CER 0.017-0.023, тембр не хуже.
+    os.environ.setdefault("VOXTREAM_DWELL_CAPS", "cons=5,vow=12")
+
+    ru_gen = HERE / "generator_ru.json"
+    # точечные переопределения инференс-параметров: GEN_OVERRIDES='{"cfg_gamma":2.0}'
+    overrides = os.environ.get("GEN_OVERRIDES")
+    if overrides:
+        gen_cfg.update(json.loads(overrides))
+        print(f"[app] generator overrides: {overrides}")
+    json.dump(gen_cfg, open(ru_gen, "w"), indent=2)
+
+    examples = HERE / "examples_ru.json"
+    if not examples.exists():
+        json.dump({"examples": []}, open(examples, "w"))
+
+    sys.argv = [
+        "voxtream-app",
+        "--config", str(ru_gen),
+        "--app-config", str(p if (p := HERE / "app_ru.json").exists()
+                            else root / "configs/app.json"),
+        # RU-гистограммы SRC (пересчитаны по ft4-корпусу), фолбэк — EN-оригинал
+        "--spk-rate-config", str(
+            p if (p := root / "configs/speaking_rate_ru.json").exists()
+            else root / "configs/speaking_rate.json"
+        ),
+        "--examples-config", str(examples),
+    ]
+    from voxtream.app import main as app_main
+    app_main()
+
+
+if __name__ == "__main__":
+    main()

demo/app_ru.json

@@ -0,0 +1,23 @@
+{
+  "min_chunk_sec": 0.01,
+  "fade_out_sec": 0.1,
+  "plot_window_sec": 10.0,
+  "visual_update_sec": 0.25,
+  "future_phone_limit": 25,
+  "plot_width": 1000,
+  "plot_height": 224,
+  "plot_left": 74,
+  "plot_right": 22,
+  "plot_top": 50,
+  "plot_bottom": 42,
+  "plot_y_max": 7,
+  "plot_y_tick": 1,
+  "plot_x_tick_sec": 1,
+  "audio_stream_start_delay_sec": 0.12,
+  "audio_stream_sample_rate": 24000,
+  "speaking_rate_min": 1.0,
+  "speaking_rate_max": 7.0,
+  "speaking_rate_step": 0.1,
+  "speaking_rate_default": 3.5,
+  "min_streaming_rtf": 0.95
+}

demo/build_groups_v3.py

@@ -0,0 +1,519 @@
+"""v3: пересборка 55-с групп с ЕСТЕСТВЕННЫМИ стыками (лечит «обрыв на точке»,
+константные паузы и мёртвые нули между клипами).
+
+Отличия от build_groups_ft4 (v2):
+1. Паузы на стыках — семплируются из ЭМПИРИЧЕСКИХ распределений реальных пауз
+   корпуса (v3/pause_hist.json, посчитан по 1.14M по-клиповых MFA-выравниваний),
+   класс — по хвостовому знаку клипа: медианы ~0.39s после '.', 0.42s после '?',
+   0.27s после ',' (в v2 были константы 160/100 мс — в 2.5 раза короче естественных).
+2. Фейды адаптивные: до 40 мс приподнятым косинусом, но ТОЛЬКО внутри собственной
+   краевой тишины клипа (речь не трогаем; минимум 2 мс от щелчков).
+3. Паузы и хвост группы — не цифровой ноль, а КОМНАТНЫЙ ТОН клипа (самое тихое
+   60-мс окно, тайлится зеркально — C0-непрерывно, с джиттером амплитуды).
+4. Клипы без MFA-выравнивания отсекаются на этапе плана (в v2 группы с ними
+   молча выпадали на этапе npy целиком).
+5. Оверсемпл вопросов (x2 прохода) и короткие группы (10%, 5-35 c) — встроены
+   (в v2 это делал отдельный extend_groups_e).
+
+Порядок клипов внутри спикера = порядок манифеста (для аудиокниг/подкастов это
+порядок записи -> соседние клипы часто из одной сессии).
+
+Выход: ft4/groups_v3.json, wav в voxtream_ru/_groups55_v3/, grouped_v3_chunk*.parquet.
+
+Запуск:
+  .venv/bin/python ru/pipeline/build_groups_v3.py --plan-only   # только план
+  .venv/bin/python ru/pipeline/build_groups_v3.py --workers 24  # план + wav
+"""
+
+import argparse
+import json
+import multiprocessing as mp
+import random
+import re
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import soundfile as sf
+
+ROOT = Path("/mnt/data/voxtream/ru_finetune/ru/data")
+MFA_OUT = ROOT / "mfa_out"
+GROUP_DIR = Path("/mnt/data/audio_data/voxtream_ru/_groups55_v3")
+SR = 24000
+TARGET = int(54.5 * SR)
+MIN_GROUP = int(35.0 * SR)
+MIN_UNIQUE = int(15.0 * SR)
+FADE_MIN = int(0.002 * SR)
+FADE_MAX = int(0.040 * SR)
+TONE_WIN = int(0.060 * SR)
+LATIN = re.compile(r"[a-zA-Z]")
+
+SHORT_MIN, SHORT_MAX = int(5 * SR), int(35 * SR)
+Q_EXTRA_PASSES = 2
+SHORT_FRAC = 0.10
+Q_MIN_GROUP = int(10 * SR)
+
+# клампы семплированных пауз, сек (хвосты эмпирики не должны съедать бюджет группы)
+# v4: клампы = p5..p95 эмпирики (v3 давал верх 1.2с — модель училась редким
+# сверхдлинным паузам и на инференсе выдавала медиану 0.85с при норме 0.39с)
+GAP_CLAMP = {".": (0.10, 0.90), "!": (0.10, 1.05), "?": (0.10, 1.00),
+             ",": (0.06, 0.70), "none": (0.05, 0.60)}
+
+OVERWRITE = False  # --overwrite: перерендер существующих wav (фикс тона без пересборки плана)
+
+# v10: приведение каждого клипа к -23 LUFS при рендере (--lufs). Тихая бытовая
+# запись промпта = OOD; корпус и промпт демки нормализуются к одному уровню
+# (стандарт NeMo/NVIDIA voice cloning). Метр создаётся лениво в каждом воркере.
+LUFS_ON = False
+TARGET_LUFS = -23.0
+_meter = None
+
+
+def lufs_gain(wav: np.ndarray) -> float:
+    global _meter
+    if len(wav) < SR // 2:  # метру BS.1770 нужно >= 0.4 c
+        return 1.0
+    if _meter is None:
+        import pyloudnorm
+        _meter = pyloudnorm.Meter(SR)
+    try:
+        loud = _meter.integrated_loudness(wav)
+    except Exception:  # noqa: BLE001
+        return 1.0
+    if not np.isfinite(loud) or loud < -70:
+        return 1.0
+    g = float(10 ** ((TARGET_LUFS - loud) / 20))
+    peak = float(np.abs(wav).max()) * g
+    if peak > 0.99:  # true-peak защита
+        g *= 0.99 / peak
+    return min(max(g, 0.05), 20.0)
+
+
+_hist = None  # {cls: np.array длительностей}
+
+
+def load_hist():
+    global _hist
+    h = json.load(open(ROOT / "v3" / "pause_hist.json"))
+    _hist = {k: np.asarray(v["sample"], dtype=np.float32) for k, v in h.items()}
+    return _hist
+
+
+# медианы пауз по классу знака (v3/pause_hist.json, 4.5M пауз корпуса)
+GAP_MED = {".": 0.39, "!": 0.41, "?": 0.42, ",": 0.27, "none": 0.11}
+# внутригрупповой разброс: CV 0.54 — ИЗМЕРЕННАЯ вариативность пауз внутри одной
+# записи одного диктора (lognormal sigma = sqrt(ln(1+CV^2)))
+SIGMA_WITHIN = 0.50
+# межгрупповой: медиана паузы '.' по спикерам гуляет 0.07..0.57с (p10..p90)
+SIGMA_BETWEEN = 0.35
+
+
+def gap_of(text: str, rng: random.Random, tempo: float = 1.0) -> int:
+    """Пауза после клипа = медиана класса * ТЕМП ГРУППЫ * внутренний шум.
+
+    v3 брал семпл из ОБЩЕКОРПУСНОЙ эмпирики (CV 0.70) на каждый стык
+    независимо — внутри одной группы возникал разброс, который в реальности
+    бывает только МЕЖДУ дикторами; модель выучила «после точки может быть что
+    угодно» и на инференсе давала «то слишком коротко, то слишком долго»
+    (жалоба пользователя; замер синтеза: медиана паузы 0.85с при норме 0.39с).
+    v4 расщепляет дисперсию: tempo — один множитель на группу (междикторская
+    компонента), шум CV 0.54 — внутридикторская. Суммарно даёт корпусную
+    вариативность, но СТРУКТУРИРОВАННУЮ.
+    """
+    tail = str(text).rstrip()[-1:]
+    cls = tail if tail in ".!?," else "none"
+    lo, hi = GAP_CLAMP[cls]
+    g = GAP_MED[cls] * tempo * rng.lognormvariate(0.0, SIGMA_WITHIN)
+    return int(min(max(g, lo), hi) * SR)
+
+
+def group_tempo(rng: random.Random) -> float:
+    """Логнормальный множитель темпа пауз группы (медиана 1.0, σ=0.35):
+    ~80% групп в диапазоне 0.64-1.57x — межспикерная компонента."""
+    return float(min(max(rng.lognormvariate(0.0, SIGMA_BETWEEN), 0.5), 2.0))
+
+
+def aligned_indices() -> set:
+    """Индексы манифеста, у которых есть TextGrid (скан mfa_out, ~1 мин)."""
+    idx = set()
+    for d in MFA_OUT.iterdir():
+        if not d.is_dir():
+            continue
+        for f in d.iterdir():
+            n = f.name
+            if n.endswith(".TextGrid"):
+                try:
+                    idx.add(int(n[:-9]))
+                except ValueError:
+                    pass
+    return idx
+
+
+# ---------------------------------------------------------------- план (pack)
+def _rescue(short, pool, rng):
+    rescued, dropped = [], 0
+    uniq_total = sum(p[3] for p in pool)
+    for g in short:
+        if uniq_total < MIN_UNIQUE:
+            dropped += 1
+            continue
+        cand = [p for p in pool if p[0] not in set(g["idx"])] or list(pool)
+        rng.shuffle(cand)
+        ci = 0
+        while g["total"] < MIN_GROUP and ci < 4 * len(cand):
+            idx, path, samples, slot = cand[ci % len(cand)]
+            ci += 1
+            if g["total"] + slot > TARGET:
+                continue
+            for k, v in (("idx", idx), ("paths", path), ("samples", samples), ("slots", slot)):
+                g[k].append(v)
+            g["total"] += slot
+            g["reused"] = g.get("reused", 0) + 1
+        if g["total"] >= MIN_GROUP:
+            rescued.append(g)
+        else:
+            dropped += 1
+    return rescued, dropped
+
+
+def _new(spk, kind="base"):
+    return {"speaker": spk, "idx": [], "paths": [], "slots": [], "samples": [],
+            "total": 0, "kind": kind}
+
+
+def _push(g, idx, path, samples, slot):
+    g["idx"].append(idx)
+    g["paths"].append(path)
+    g["samples"].append(samples)
+    g["slots"].append(slot)
+    g["total"] += slot
+
+
+def retempo(groups, texts, seed: int = 4242):
+    """v4: назначить каждой группе общий множитель темпа пауз и пересчитать слоты.
+
+    Паузы внутри группы становятся согласованными (как у одного диктора в одной
+    сессии), а не независимыми выбросами из широкого распределения. Если после
+    пересчёта группа не влезает в TARGET — темп ужимается, в крайнем случае
+    дропается хвостовой клип.
+    """
+    rng = random.Random(seed)
+    n_trim = 0
+    for g in groups:
+        tempo = group_tempo(rng)
+        for _ in range(6):
+            slots = [smp + gap_of(texts[i], rng, tempo) if i >= 0 else slot
+                     for i, smp, slot in zip(g["idx"], g["samples"], g["slots"])]
+            if sum(slots) <= TARGET:
+                break
+            tempo *= 0.85
+        while sum(slots) > TARGET and len(slots) > 1:
+            for k in ("idx", "paths", "samples"):
+                g[k].pop()
+            slots.pop()
+            n_trim += 1
+        g["slots"] = slots
+        g["total"] = sum(slots)
+        g["tempo"] = round(tempo, 3)
+    print(f"retempo: групп {len(groups)}, обрезано хвостовых клипов {n_trim}")
+    return [g for g in groups if g["total"] >= 5 * SR and g["idx"]]
+
+
+def tempo_key(path: str) -> str:
+    """v6: версия темпа клипа (orig / slow / fast).
+
+    Замер на v5: модель почти не копирует темп промпта (наклон 0.34). Одна из
+    причин — 18.8% групп СМЕШИВАЛИ оригиналы с темпо-аугментированными копиями:
+    промпт мог быть 0.75x, а продолжение 1.3x, т.е. данные прямо учили, что
+    темп промпта НЕ предсказывает темп речи. Группируем по (спикер, версия) —
+    внутри группы темп однороден, связь промпт->продолжение становится честной.
+    """
+    if "_tempo_aug" not in path:
+        return "orig"
+    return "slow" if "_slow" in path else "fast"
+
+
+# v7: границы темпо-корзин, слог/с (clip_sps.npy: медиана 5.22, p10 3.76, p90 6.78).
+# Замер на v6: 60% групп имели разброс темпа >=2 слог/с ВНУТРИ себя — промпт (начало
+# группы) и продолжение записаны с разной скоростью, поэтому связь «темп промпта ->
+# темп речи» в данных отсутствовала (наклон копирования 0.15-0.34 во всех версиях).
+TEMPO_EDGES = (4.3, 5.2, 6.1)
+
+
+def tempo_bucket(sps: float) -> str:
+    if not np.isfinite(sps):
+        return "na"
+    return str(int(np.searchsorted(TEMPO_EDGES, sps)))
+
+
+def pack_groups(df, aligned: set, seed: int = 42, clip_sps=None):
+    rng = random.Random(seed)
+    groups, n_lat, n_noal, n_rescued, n_dropped, n_reused = [], 0, 0, 0, 0, 0
+    texts = df.text.astype(str)
+    by_spk_pool = {}
+
+    key = df.speaker.astype(str) + "|" + df.audio_path.map(tempo_key)
+    if clip_sps is not None:
+        key = key + "|" + pd.Series(
+            [tempo_bucket(s) for s in clip_sps[: len(df)]], index=df.index
+        )
+    df = df.assign(_spk_tempo=key)
+    for spk_t, sub in df.groupby("_spk_tempo", sort=False):
+        spk = spk_t.split("|")[0]
+        cur, short, pool = None, [], []
+        for row in sub.itertuples():
+            if int(row.Index) not in aligned:
+                n_noal += 1
+                continue
+            if LATIN.search(row.text):
+                n_lat += 1
+                continue
+            samples = int(round(row.duration * SR))
+            slot = samples + gap_of(row.text, rng)
+            if samples <= 0 or slot > TARGET:
+                continue
+            pool.append((int(row.Index), row.audio_path, samples, slot))
+            if cur is None or cur["total"] + slot > TARGET:
+                if cur:
+                    (groups if cur["total"] >= MIN_GROUP else short).append(cur)
+                cur = _new(spk)
+            _push(cur, int(row.Index), row.audio_path, samples, slot)
+        if cur:
+            (groups if cur["total"] >= MIN_GROUP else short).append(cur)
+        if short:
+            rescued, dropped = _rescue(short, pool, rng)
+            n_rescued += len(rescued)
+            n_dropped += dropped
+            n_reused += sum(g.get("reused", 0) for g in rescued)
+            groups.extend(rescued)
+        by_spk_pool[spk_t] = pool
+    n_base = len(groups)
+    print(f"базовых групп: {n_base}; латиница={n_lat}, без выравнивания={n_noal}, "
+          f"спасено={n_rescued} (+{n_reused} переисп.), дропнуто={n_dropped}")
+
+    # --- оверсемпл вопросов ---
+    n_q = 0
+    for spk_t, pool in by_spk_pool.items():
+        spk = spk_t.rsplit("|", 1)[0]
+        qs = [p for p in pool if "?" in texts[p[0]]]
+        if not qs:
+            continue
+        for _ in range(Q_EXTRA_PASSES):
+            order = qs[:]
+            rng.shuffle(order)
+            cur = None
+            for idx, path, samples, slot in order:
+                if cur is None or cur["total"] + slot > TARGET:
+                    if cur and cur["total"] >= Q_MIN_GROUP:
+                        groups.append(cur)
+                    cur = _new(spk, "question")
+                _push(cur, idx, path, samples, slot)
+            if cur and cur["total"] >= Q_MIN_GROUP:
+                groups.append(cur)
+    n_q = len(groups) - n_base
+
+    # --- короткие группы ---
+    spk_list = [s for s, p in by_spk_pool.items() if p]  # ключи "спикер|версия"
+    n_short_target = int(SHORT_FRAC * n_base)
+    for _ in range(n_short_target):
+        spk_t = rng.choice(spk_list)
+        pool = by_spk_pool[spk_t]
+        cur = _new(spk_t.rsplit("|", 1)[0], "short")
+        target_len = rng.randint(SHORT_MIN, SHORT_MAX)
+        for _try in range(6):
+            idx, path, samples, slot = pool[rng.randrange(len(pool))]
+            if cur["total"] + slot > min(target_len + 3 * SR, TARGET):
+                if cur["total"] >= SHORT_MIN:
+                    break
+                continue
+            _push(cur, idx, path, samples, slot)
+            if cur["total"] >= target_len:
+                break
+        if cur["idx"]:
+            groups.append(cur)
+    print(f"вопросных групп: +{n_q}, коротких: +{len(groups) - n_base - n_q}")
+    return retempo(groups, texts)
+
+
+# ------------------------------------------------------------- рендер (wav)
+def _cos_ramp(n: int) -> np.ndarray:
+    return (0.5 - 0.5 * np.cos(np.linspace(0, np.pi, n, dtype=np.float32)))
+
+
+def adaptive_fades(wav: np.ndarray) -> np.ndarray:
+    """Фейды внутри краевой тишины клипа: до 40 мс, речь не трогаем."""
+    peak = float(np.max(np.abs(wav))) + 1e-9
+    loud = np.abs(wav) > max(0.02 * peak, 1e-4)
+    if not loud.any():
+        return wav
+    first = int(np.argmax(loud))
+    last = len(wav) - int(np.argmax(loud[::-1]))
+    fi = min(max(first, FADE_MIN), FADE_MAX, len(wav))
+    fo = min(max(len(wav) - last, FADE_MIN), FADE_MAX, len(wav))
+    wav[:fi] *= _cos_ramp(fi)
+    wav[-fo:] *= _cos_ramp(fo)[::-1]
+    return wav
+
+
+def room_tone(wav: np.ndarray) -> np.ndarray:
+    """Самое тихое 120-мс окно клипа (шаблон спектра); тихих нет — глушим до -46 dBFS."""
+    win = 2 * TONE_WIN
+    if len(wav) < 2 * win:
+        return np.zeros(win, dtype=np.float32)
+    hop = win // 2
+    n = (len(wav) - win) // hop
+    view = np.lib.stride_tricks.sliding_window_view(wav, win)[::hop][:n]
+    rms = np.sqrt((view ** 2).mean(axis=1))
+    k = int(np.argmin(rms))
+    tone = view[k].copy()
+    if rms[k] > 5e-3:
+        tone *= 5e-3 / rms[k]
+    return tone
+
+
+_N_FFT = 512
+_HOP = 256
+_HANN = np.hanning(_N_FFT).astype(np.float32)
+
+
+def synth_tone(template: np.ndarray, length: int, seed: int) -> np.ndarray:
+    """Стационарный шум со спектральной огибающей шаблона — БЕЗ зацикливания.
+
+    v3 первой версии тайлил 60-мс окно зеркально: период ~8 Гц слышен как
+    «вертолёт», модель выучила текстуру (жалоба юзера). Здесь — средняя
+    STFT-магнитуда шаблона + случайные фазы на каждый кадр + overlap-add:
+    цвет комнаты сохранён, повторов нет в принципе.
+    """
+    if length <= 0:
+        return np.zeros(0, dtype=np.float32)
+    rng = np.random.default_rng(seed)
+    if len(template) < _N_FFT or not np.any(template):
+        return np.zeros(length, dtype=np.float32)
+    n = (len(template) - _N_FFT) // _HOP + 1
+    frames = np.lib.stride_tricks.sliding_window_view(template, _N_FFT)[::_HOP][:n]
+    mag = np.abs(np.fft.rfft(frames * _HANN, axis=1)).mean(axis=0)
+
+    n_frames = length // _HOP + 3
+    phases = rng.uniform(0, 2 * np.pi, size=(n_frames, len(mag)))
+    sig_frames = np.fft.irfft(mag * np.exp(1j * phases), n=_N_FFT, axis=1).real
+    sig_frames *= _HANN
+    out = np.zeros(n_frames * _HOP + _N_FFT, dtype=np.float32)
+    for i in range(n_frames):  # OLA (hann, hop=1/2 -> COLA)
+        out[i * _HOP:i * _HOP + _N_FFT] += sig_frames[i]
+    out = out[_N_FFT: _N_FFT + length]
+    t_rms = float(np.sqrt((template ** 2).mean()))
+    o_rms = float(np.sqrt((out ** 2).mean())) + 1e-12
+    return (out * (t_rms / o_rms)).astype(np.float32)
+
+
+def fill_tone(buf: np.ndarray, start: int, end: int, tone: np.ndarray, seed: int):
+    """Заполняет [start:end) синтезированным комнатным тоном с 5-мс рампами."""
+    if end <= start or not len(tone):
+        return
+    buf[start:end] = synth_tone(tone, end - start, seed)
+    r = min(int(0.005 * SR), (end - start) // 2)
+    if r > 0:
+        buf[start:start + r] *= _cos_ramp(r)
+        buf[end - r:end] *= _cos_ramp(r)[::-1]
+
+
+def write_group(task):
+    gi, g = task
+    out = GROUP_DIR / f"g{gi:06d}.wav"
+    if not OVERWRITE and out.exists():
+        try:
+            if sf.info(str(out)).frames == int(55 * SR):
+                return gi, str(out), "skip"
+        except Exception:
+            out.unlink(missing_ok=True)
+    buf = np.zeros(int(55 * SR), dtype=np.float32)
+    pos = 0
+    try:
+        tone = np.zeros(2 * TONE_WIN, dtype=np.float32)
+        for j, (path, samples, slot) in enumerate(zip(g["paths"], g["samples"], g["slots"])):
+            wav, sr = sf.read(path, dtype="float32")
+            assert sr == SR, f"{path}: sr={sr}"
+            if wav.ndim > 1:
+                wav = wav.mean(axis=1)
+            wav = wav[:samples].copy()
+            if LUFS_ON:
+                wav *= lufs_gain(wav)
+            wav = adaptive_fades(wav)
+            buf[pos:pos + len(wav)] = wav
+            tone = room_tone(wav)
+            fill_tone(buf, pos + len(wav), pos + slot, tone, 100_000 + gi * 64 + j)
+            pos += slot
+        fill_tone(buf, pos, len(buf), tone, 100_000 + gi * 64 + 63)  # хвост группы
+        sf.write(out, buf, SR, subtype="PCM_16")
+        return gi, str(out), "ok"
+    except Exception as e:  # noqa: BLE001
+        out.unlink(missing_ok=True)
+        return gi, "", f"fail:{e}"
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--plan-only", action="store_true")
+    ap.add_argument("--workers", type=int, default=24)
+    ap.add_argument("--chunk-size", type=int, default=70_000)
+    ap.add_argument("--overwrite", action="store_true",
+                    help="перерендерить wav даже если файл на месте (план не меняется)")
+    global OVERWRITE, MFA_OUT, GROUP_DIR
+    ap.add_argument("--mfa-out", default="mfa_out", help="каталог TextGrid (v4: mfa_out_v4)")
+    ap.add_argument("--out-groups", default="groups_v3.json")
+    ap.add_argument("--group-dir", default="/mnt/data/audio_data/voxtream_ru/_groups55_v3")
+    ap.add_argument("--parquet-prefix", default="grouped_v3_chunk")
+    ap.add_argument("--tempo-buckets", action="store_true",
+                    help="v7: группы однородны по темпу (clip_sps.npy)")
+    ap.add_argument("--clip-sps", default="clip_sps.npy",
+                    help="файл SPS по клипам для темпо-корзин")
+    ap.add_argument("--lufs", action="store_true",
+                    help="v10: каждый клип -> -23 LUFS при рендере")
+    args = ap.parse_args()
+    global LUFS_ON
+    OVERWRITE = args.overwrite
+    LUFS_ON = args.lufs
+    MFA_OUT = ROOT / args.mfa_out
+    GROUP_DIR = Path(args.group_dir)
+    if LUFS_ON:
+        print(f"LUFS-нормализация ВКЛ: клипы -> {TARGET_LUFS} LUFS")
+
+    load_hist()
+    print("скан выравниваний…", flush=True)
+    aligned = aligned_indices()
+    print(f"выровненных клипов: {len(aligned)}", flush=True)
+
+    df = pd.read_csv(ROOT / "manifest.csv", sep="|", low_memory=False)
+    clip_sps = None
+    if args.tempo_buckets:
+        clip_sps = np.load(ROOT / args.clip_sps)
+        print(f"темпо-корзины ВКЛ: границы {TEMPO_EDGES} слог/с")
+    groups = pack_groups(df, aligned, clip_sps=clip_sps)
+    total_h = sum(g["total"] for g in groups) / SR / 3600
+    used = len({i for g in groups for i in g["idx"]})
+    print(f"групп: {len(groups)}, уникальных клипов: {used}/{len(df)}, ~{total_h:.1f} ч слотов")
+    json.dump(groups, open(ROOT / args.out_groups, "w"), ensure_ascii=False)
+    if args.plan_only:
+        return
+
+    GROUP_DIR.mkdir(parents=True, exist_ok=True)
+    fails = 0
+    with mp.Pool(args.workers) as pool:
+        for gi, path, status in pool.imap(write_group, enumerate(groups), chunksize=16):
+            groups[gi]["group_wav"] = path
+            fails += status.startswith("fail")
+            if (gi + 1) % 10_000 == 0:
+                print(f"  {gi + 1}/{len(groups)} (fail={fails})", flush=True)
+    groups = [g for g in groups if g.get("group_wav")]
+    json.dump(groups, open(ROOT / args.out_groups, "w"), ensure_ascii=False)
+    print(f"записано групп: {len(groups)} (fail={fails})")
+
+    paths = [g["group_wav"] for g in groups]
+    for ci in range(0, len(paths), args.chunk_size):
+        p = ROOT / f"{args.parquet_prefix}{ci // args.chunk_size}.parquet"
+        pd.DataFrame({"paths": paths[ci:ci + args.chunk_size]}).to_parquet(p, index=False)
+        print(f"{p.name}: {min(args.chunk_size, len(paths) - ci)} путей")
+
+
+if __name__ == "__main__":
+    main()

demo/configs/app.json

@@ -0,0 +1,23 @@
+{
+    "min_chunk_sec": 0.01,
+    "fade_out_sec": 0.1,
+    "plot_window_sec": 10.0,
+    "visual_update_sec": 0.25,
+    "future_phone_limit": 25,
+    "plot_width": 1000,
+    "plot_height": 224,
+    "plot_left": 74,
+    "plot_right": 22,
+    "plot_top": 50,
+    "plot_bottom": 42,
+    "plot_y_max": 7,
+    "plot_y_tick": 1,
+    "plot_x_tick_sec": 1,
+    "audio_stream_start_delay_sec": 0.12,
+    "audio_stream_sample_rate": 24000,
+    "speaking_rate_min": 1.0,
+    "speaking_rate_max": 7.0,
+    "speaking_rate_step": 0.1,
+    "speaking_rate_default": 4.0,
+    "min_streaming_rtf": 0.95
+}

demo/configs/generator.json

@@ -0,0 +1,58 @@
+{
+    "sil_token": 120,
+    "bos_token": 123,
+    "eos_token": 124,
+    "unk_token": 122,
+    "eop_token": 122,
+    "num_codebooks": 16,
+    "num_phones_per_frame": 2,
+    "audio_delay_frames": 1,
+    "temperature": 0.9,
+    "topk": 5,
+    "top_p": 0.9,
+    "max_audio_length_ms": 60000,
+    "model_repo": "herimor/voxtream2",
+    "model_name": "model.safetensors",
+    "model_config_name": "config.json",
+    "mimi_sr": 24000,
+    "mimi_vocab_size": 2048,
+    "mimi_frame_ms": 80,
+    "mimi_repo": "kyutai/moshiko-pytorch-bf16",
+    "mimi_name": "tokenizer-e351c8d8-checkpoint125.safetensors",
+    "spk_enc_sr": 16000,
+    "spk_enc_repo": "IDRnD/ReDimNet",
+    "spk_enc_model": "ReDimNet",
+    "spk_enc_model_name": "M",
+    "spk_enc_train_type": "ft_mix",
+    "spk_enc_dataset": "vb2+vox2+cnc",
+    "phoneme_dict_name": "phoneme_to_token.json",
+    "max_prompt_sec": 20,
+    "min_prompt_sec": 1,
+    "max_phone_tokens": 2000,
+    "cache_prompt": false,
+    "cfg_gamma": 1.5,
+    "cfg_ac_gamma": 3.0,
+    "text_context": " context",
+    "text_context_length": 18,
+    "spk_proj_weight": 1.5,
+    "audio_pad_token": 2049,
+    "enhance_prompt": false,
+    "sidon_se_reload_model": false,
+    "reset_streaming_state": false,
+    "hf_token": null,
+    "apply_vad": false,
+    "min_speech_seg_sec": 0.3,
+    "min_look_ahead_phones": 3,
+    "phonemizer": "espeak",
+    "spk_rate_window_sec": 3.0,
+    "frame_repeat_counter": 25,
+    "punct_map": {
+        ".": 117,
+        ",": 118,
+        "?": 119,
+        "!": 121
+    },
+    "phoneme_index_map":{
+        "0": [0, 1], "1": [0, 2], "2": [1, 1], "3": [1, 2], "4": [2, 1], "5": [2, 2]
+    }
+}

demo/configs/speaking_rate.json

@@ -0,0 +1,51 @@
+{
+    "1": {
+        "duration_state": [
+            52, 15, 1, 2, 1, 3
+        ],
+        "weight": 3.0,
+        "cfg_gamma": 1.25
+    },
+    "2": {
+        "duration_state": [
+            44, 24, 2, 2, 1, 1
+        ],
+        "weight": 3.0,
+        "cfg_gamma": 1.25
+    },
+    "3": {
+        "duration_state": [
+            24, 23, 2, 3, 1, 1
+        ],
+        "weight": 5.0,
+        "cfg_gamma": 1.5
+    },
+    "4": {
+        "duration_state": [
+            40, 59, 5, 12, 1, 2
+        ],
+        "weight": 5.0,
+        "cfg_gamma": 1.5
+    },
+    "5": {
+        "duration_state": [
+            21, 43, 4, 13, 1, 2
+        ],
+        "weight": 7.0,
+        "cfg_gamma": 2.0
+    },
+    "6": {
+        "duration_state": [
+            27, 69, 6, 30, 2, 7
+        ],
+        "weight": 7.0,
+        "cfg_gamma": 2.0
+    },
+    "7": {
+        "duration_state": [
+            6, 18, 2, 9, 1, 3
+        ],
+        "weight": 10.0,
+        "cfg_gamma": 2.5
+    }
+}

demo/configs/speaking_rate_ru.json

@@ -0,0 +1,86 @@
+{
+  "1": {
+    "duration_state": [
+      73,
+      23,
+      2,
+      3,
+      1,
+      1
+    ],
+    "weight": 8.0,
+    "cfg_gamma": 1.25
+  },
+  "2": {
+    "duration_state": [
+      60,
+      33,
+      2,
+      5,
+      1,
+      1
+    ],
+    "weight": 8.0,
+    "cfg_gamma": 1.25
+  },
+  "3": {
+    "duration_state": [
+      43,
+      47,
+      3,
+      7,
+      1,
+      1
+    ],
+    "weight": 8.0,
+    "cfg_gamma": 1.5
+  },
+  "4": {
+    "duration_state": [
+      29,
+      56,
+      3,
+      11,
+      1,
+      1
+    ],
+    "weight": 5.0,
+    "cfg_gamma": 1.5
+  },
+  "5": {
+    "duration_state": [
+      21,
+      58,
+      4,
+      17,
+      1,
+      1
+    ],
+    "weight": 7.0,
+    "cfg_gamma": 2.0
+  },
+  "6": {
+    "duration_state": [
+      14,
+      57,
+      4,
+      24,
+      1,
+      1
+    ],
+    "weight": 8.0,
+    "cfg_gamma": 2.0
+  },
+  "7": {
+    "duration_state": [
+      9,
+      51,
+      3,
+      34,
+      1,
+      3
+    ],
+    "weight": 8.0,
+    "cfg_gamma": 2.5
+  }
+}

demo/examples_ru.json

@@ -0,0 +1 @@
+{"examples": []}

demo/packages.txt

@@ -0,0 +1 @@
+espeak-ng

demo/requirements.txt

@@ -0,0 +1,25 @@
+# Пакет voxtream вендорен в спейс (модифицированная версия под RU-модель:
+# классовые лимиты удержания, SRC-режим, VOXTREAM_LA) — с GitHub НЕ ставить.
+torch>=2.4,<2.9
+torchaudio>=2.4,<2.9
+torchtune==0.4.0
+torchao==0.9.0
+moshi>=0.2.13
+transformers==4.50.0
+huggingface_hub==0.28.1
+tokenizers<0.22
+g2p-en==2.1.0
+librosa==0.11.0
+soundfile==0.13.1
+inflect==7.5.0
+nltk==3.9.1
+gradio==4.44.1
+gradio_client==1.3.0
+starlette==0.52.1
+pydantic==2.10.6
+silero-vad==6.2.0
+ruaccent
+runorm==1.1
+pyloudnorm
+numpy
+pandas

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/blobs/1e39d930e09fd435aa8be1182640092b0581e3ca

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>"
+}

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/blobs/23440d6773792a7758b5aab66878ba5f11f56f81235394a3856718c82e2a8f1e

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23440d6773792a7758b5aab66878ba5f11f56f81235394a3856718c82e2a8f1e
+size 16811983

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/blobs/49f05ecdcda8ba78b292f78ba2f83016d348cf62

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.28.1"
+}

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/blobs/88ae2e52858aba88b4e935b2e5689ab4dc49bae6

@@ -0,0 +1,6 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "extra_ids": 0,
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/blobs/a5a347155014013f45c8adb3efaebf40d68142ed

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "maximxls/text-normalization-ru-terrible",
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "d_ff": 1024,
+  "d_kv": 64,
+  "d_model": 256,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.0,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "num_decoder_layers": 3,
+  "num_heads": 4,
+  "num_layers": 3,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.28.1",
+  "use_cache": true,
+  "vocab_size": 5120
+}

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/blobs/e33efa79e656bdd0972beb15a07b11b74df0df1368308bd12ec6b02337887040

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33efa79e656bdd0972beb15a07b11b74df0df1368308bd12ec6b02337887040
+size 16795208

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/refs/main

@@ -0,0 +1 @@
+b130ae67db4b209babec461767bcd2ace74fe88a

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/refs/refs/pr/1

@@ -0,0 +1 @@
+c282bffa4b5a328a29883541fc4f8784f1ffe093

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/snapshots/b130ae67db4b209babec461767bcd2ace74fe88a/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "maximxls/text-normalization-ru-terrible",
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "d_ff": 1024,
+  "d_kv": 64,
+  "d_model": 256,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.0,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "num_decoder_layers": 3,
+  "num_heads": 4,
+  "num_layers": 3,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.28.1",
+  "use_cache": true,
+  "vocab_size": 5120
+}

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/snapshots/b130ae67db4b209babec461767bcd2ace74fe88a/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.28.1"
+}

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/snapshots/b130ae67db4b209babec461767bcd2ace74fe88a/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23440d6773792a7758b5aab66878ba5f11f56f81235394a3856718c82e2a8f1e
+size 16811983

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/snapshots/b130ae67db4b209babec461767bcd2ace74fe88a/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>"
+}

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/snapshots/b130ae67db4b209babec461767bcd2ace74fe88a/tokenizer_config.json

@@ -0,0 +1,6 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "extra_ids": 0,
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

demo/runorm_cache/models--RUNorm--RUNorm-kirillizator/snapshots/c282bffa4b5a328a29883541fc4f8784f1ffe093/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33efa79e656bdd0972beb15a07b11b74df0df1368308bd12ec6b02337887040
+size 16795208

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/010ca0a172e0d662f661ff9b6f9b3323c1272ec4

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.28.1"
+}

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/0e9c94dde3f01cda84a45287cabd0c22ab23cbac

@@ -0,0 +1,61 @@
+{
+  "_name_or_path": "ai-forever/ruT5-base",
+  "_num_labels": 2,
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "d_ff": 3072,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "relu",
+  "dropout_rate": 0.1,
+  "eos_token_id": 2,
+  "feed_forward_proj": "relu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": false,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "n_positions": 512,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "task_specific_params": {
+    "summarization": {
+      "early_stopping": true,
+      "length_penalty": 2.0,
+      "max_length": 200,
+      "min_length": 30,
+      "no_repeat_ngram_size": 3,
+      "num_beams": 4,
+      "prefix": "summarize: "
+    },
+    "translation_en_to_de": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to German: "
+    },
+    "translation_en_to_fr": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to French: "
+    },
+    "translation_en_to_ro": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to Romanian: "
+    }
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.28.1",
+  "use_cache": true,
+  "vocab_size": 32128
+}

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/196088ae0adbafc572d99e62d0090a4a25a0bb305493ff454a7f9fb2171c1566.incomplete

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f7b2d909e6eaf1d4a9be43e286fb16693d66430ce662b19351b8662539497bb
+size 178257920

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/4fe7c3c6498b6c26f0b5b6c60265b4d88ab4db02e1bbfb7f06cea3dc4746874b

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fe7c3c6498b6c26f0b5b6c60265b4d88ab4db02e1bbfb7f06cea3dc4746874b
+size 445895825

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/5540ad7d1f6add43e0550aed386b556455404faa

@@ -0,0 +1,108 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/5750775dfac6c13088a6c54dd07cf88f7f9cee1d

@@ -0,0 +1,3 @@
+{
+  "<s>": 32100
+}

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/7a4eb87011448a4564a3144979384da51eee1da95e554feb22ccc85529535dd5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a4eb87011448a4564a3144979384da51eee1da95e554feb22ccc85529535dd5
+size 1003118

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/blobs/94fe40201a0d25ffe6e448ae1a91f41fc67d28fa

@@ -0,0 +1,938 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<extra_id_99>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32001": {
+      "content": "<extra_id_98>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32002": {
+      "content": "<extra_id_97>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32003": {
+      "content": "<extra_id_96>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32004": {
+      "content": "<extra_id_95>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32005": {
+      "content": "<extra_id_94>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32006": {
+      "content": "<extra_id_93>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32007": {
+      "content": "<extra_id_92>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32008": {
+      "content": "<extra_id_91>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32009": {
+      "content": "<extra_id_90>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32010": {
+      "content": "<extra_id_89>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32011": {
+      "content": "<extra_id_88>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32012": {
+      "content": "<extra_id_87>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32013": {
+      "content": "<extra_id_86>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32014": {
+      "content": "<extra_id_85>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32015": {
+      "content": "<extra_id_84>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32016": {
+      "content": "<extra_id_83>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32017": {
+      "content": "<extra_id_82>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32018": {
+      "content": "<extra_id_81>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32019": {
+      "content": "<extra_id_80>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32020": {
+      "content": "<extra_id_79>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32021": {
+      "content": "<extra_id_78>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32022": {
+      "content": "<extra_id_77>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32023": {
+      "content": "<extra_id_76>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32024": {
+      "content": "<extra_id_75>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32025": {
+      "content": "<extra_id_74>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32026": {
+      "content": "<extra_id_73>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32027": {
+      "content": "<extra_id_72>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32028": {
+      "content": "<extra_id_71>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32029": {
+      "content": "<extra_id_70>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32030": {
+      "content": "<extra_id_69>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32031": {
+      "content": "<extra_id_68>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32032": {
+      "content": "<extra_id_67>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32033": {
+      "content": "<extra_id_66>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32034": {
+      "content": "<extra_id_65>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32035": {
+      "content": "<extra_id_64>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32036": {
+      "content": "<extra_id_63>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32037": {
+      "content": "<extra_id_62>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32038": {
+      "content": "<extra_id_61>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32039": {
+      "content": "<extra_id_60>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32040": {
+      "content": "<extra_id_59>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32041": {
+      "content": "<extra_id_58>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32042": {
+      "content": "<extra_id_57>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32043": {
+      "content": "<extra_id_56>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32044": {
+      "content": "<extra_id_55>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32045": {
+      "content": "<extra_id_54>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32046": {
+      "content": "<extra_id_53>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32047": {
+      "content": "<extra_id_52>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32048": {
+      "content": "<extra_id_51>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32049": {
+      "content": "<extra_id_50>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32050": {
+      "content": "<extra_id_49>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32051": {
+      "content": "<extra_id_48>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32052": {
+      "content": "<extra_id_47>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32053": {
+      "content": "<extra_id_46>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32054": {
+      "content": "<extra_id_45>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32055": {
+      "content": "<extra_id_44>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32056": {
+      "content": "<extra_id_43>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32057": {
+      "content": "<extra_id_42>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32058": {
+      "content": "<extra_id_41>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32059": {
+      "content": "<extra_id_40>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32060": {
+      "content": "<extra_id_39>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32061": {
+      "content": "<extra_id_38>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32062": {
+      "content": "<extra_id_37>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32063": {
+      "content": "<extra_id_36>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32064": {
+      "content": "<extra_id_35>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32065": {
+      "content": "<extra_id_34>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32066": {
+      "content": "<extra_id_33>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32067": {
+      "content": "<extra_id_32>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32068": {
+      "content": "<extra_id_31>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32069": {
+      "content": "<extra_id_30>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32070": {
+      "content": "<extra_id_29>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32071": {
+      "content": "<extra_id_28>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32072": {
+      "content": "<extra_id_27>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32073": {
+      "content": "<extra_id_26>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32074": {
+      "content": "<extra_id_25>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32075": {
+      "content": "<extra_id_24>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32076": {
+      "content": "<extra_id_23>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32077": {
+      "content": "<extra_id_22>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32078": {
+      "content": "<extra_id_21>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32079": {
+      "content": "<extra_id_20>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32080": {
+      "content": "<extra_id_19>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32081": {
+      "content": "<extra_id_18>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32082": {
+      "content": "<extra_id_17>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32083": {
+      "content": "<extra_id_16>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32084": {
+      "content": "<extra_id_15>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32085": {
+      "content": "<extra_id_14>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32086": {
+      "content": "<extra_id_13>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32087": {
+      "content": "<extra_id_12>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32088": {
+      "content": "<extra_id_11>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32089": {
+      "content": "<extra_id_10>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32090": {
+      "content": "<extra_id_9>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32091": {
+      "content": "<extra_id_8>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32092": {
+      "content": "<extra_id_7>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32093": {
+      "content": "<extra_id_6>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32094": {
+      "content": "<extra_id_5>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32095": {
+      "content": "<extra_id_4>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32096": {
+      "content": "<extra_id_3>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32097": {
+      "content": "<extra_id_2>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32098": {
+      "content": "<extra_id_1>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    },
+    "32099": {
+      "content": "<extra_id_0>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 100,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/refs/main

@@ -0,0 +1 @@
+3fdb93344da77fbd75821e2fdd6307df3a0a1c96

demo/runorm_cache/models--RUNorm--RUNorm-normalizer-medium/refs/refs/pr/1

@@ -0,0 +1 @@
+aaca4bd9f7f9e17390ca3cce913a8d14e61ed10c