Train augmented anime filename parser

.gitignore

@@ -16,3 +16,5 @@ data/**/*.jsonl
 !data/test_smoke.jsonl
 data/**/*.db
 data/**/*.sqlite
+data/generated/
+reports/generated/

README.md

@@ -142,15 +142,15 @@ Current published checkpoint:
 | --- | --- |
 | Fixed regression, model-only / 固定回归，纯模型聚合 | 25/26 full match = `96.15%` |
 | Fixed regression, default thin runtime / 固定回归，默认薄层运行时 | 26/26 full match = `100%` |
-| Focus held-out, model-only / 困难抽样，纯模型聚合 | 1014/1024 full match = `99.02%` |
-| Focus held-out, default thin runtime / 困难抽样，默认薄层运行时 | 1017/1024 full match = `99.32%` |
-| Token/entity eval / token/entity 评估 | F1 `0.9972`, token accuracy `0.9995` |
-| ONNX parity / ONNX 误差 | max abs diff `4.0531e-05` |
-| CPU thin-runtime latency / CPU 薄层运行时延迟 | ONNX avg `13.18 ms`, P95 `16.70 ms` |
+| Held-out parse, model-only / held-out 解析，纯模型聚合 | 1947/2048 full match = `95.07%` |
+| Held-out parse, default thin runtime / held-out 解析，默认薄层运行时 | 1966/2048 full match = `96.00%` |
+| Token/entity eval / token/entity 评估 | F1 `0.9847`, token accuracy `0.9962` |
+| ONNX parity / ONNX 误差 | max abs diff `1.9073e-05` |
+| CPU thin-runtime latency / CPU 薄层运行时延迟 | ONNX avg `11.61 ms`, P95 `13.52 ms` |
 
-**中文**：当前发布模型是“两阶段训练”产物：先在 `datasets/AnimeName/dmhy_weak_char.jsonl` 上全量 CUDA 重训，再做 thin hard-case focus 微调。细节见 `reports/training_lineage.json`。README 主指标以 `model-only` 和默认薄层 `normalized-only` 为准；旧版结构规则辅助层已移除，不再作为运行时或质量对照。
+**中文**：当前发布模型是“两阶段训练”产物：先在 `datasets/AnimeName/dmhy_weak_char.jsonl` 上做 10 epoch CUDA 全量重训，并在训练时动态生成不完整文件名、BIO 块重排/子集和 special 片段样本；再做 thin hard-case focus 微调。细节见 `reports/training_lineage.json`。README 主指标以 `model-only` 和默认薄层 `normalized-only` 为准；旧版结构规则辅助层已移除，不再作为运行时或质量对照。
 
-**English**: The published checkpoint was trained in two stages: a full CUDA fine-tune on `datasets/AnimeName/dmhy_weak_char.jsonl`, followed by a thin hard-case focus fine-tune. See `reports/training_lineage.json` for details. README quality numbers prioritize `model-only` and the default thin `normalized-only` runtime; structural filename assists have been removed from the runtime and quality reports.
+**English**: The published checkpoint was trained in two stages: a 10-epoch CUDA fine-tune on `datasets/AnimeName/dmhy_weak_char.jsonl` with dynamic in-memory augmentation for incomplete filenames, BIO-block subsets/permutations, and special-code fragments, followed by a thin hard-case focus fine-tune. See `reports/training_lineage.json` for details. README quality numbers prioritize `model-only` and the default thin `normalized-only` runtime; structural filename assists have been removed from the runtime and quality reports.
 
 Run regression:
 
@@ -177,8 +177,8 @@ decoding, entity aggregation, and light string/number normalization:
 
 | Backend / 后端 | Load ms / 加载 ms | Avg ms / 平均 ms | P50 ms | P95 ms | P99 ms | files/s |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| PyTorch | 76.56 | 16.85 | 16.21 | 22.84 | 28.31 | 59.4 |
-| ONNX Runtime | 49.74 | 13.18 | 12.86 | 16.70 | 18.06 | 75.9 |
+| PyTorch | 44.84 | 16.42 | 14.77 | 26.31 | 32.62 | 60.9 |
+| ONNX Runtime | 40.70 | 11.61 | 11.43 | 13.52 | 15.20 | 86.2 |
 
 **中文**：这是完整薄层 parser 的端到端延迟，不是只测模型 forward。移动端实现应复用 ONNX session，并保持 tokenizer/BIO/薄规范化逻辑一致。
 
@@ -188,27 +188,33 @@ decoding, entity aggregation, and light string/number normalization:
 
 Training uses the dataset submodule at `datasets/AnimeName`.
 
-Recommended full character-token run:
+Recommended optimized character-token run on the Windows RTX 5070 Ti worker:
 
 ```powershell
-uv run python -m anifilebert.train --tokenizer char `
+.\.venv\Scripts\python.exe -m anifilebert.train --tokenizer char `
   --data-file datasets/AnimeName/dmhy_weak_char.jsonl `
-  --vocab-file datasets/AnimeName/vocab.char.json `
-  --save-dir checkpoints/dmhy-char-full `
+  --vocab-file vocab.json `
+  --save-dir checkpoints/dmhy-char-aug-fragments-optimized-10epoch `
   --init-model-dir . `
-  --epochs 2 `
-  --batch-size 256 `
-  --learning-rate 0.00008 `
-  --warmup-steps 300 `
+  --epochs 10 `
+  --batch-size 1792 `
+  --learning-rate 0.00002 `
+  --warmup-steps 500 `
   --max-seq-length 128 `
   --train-split 0.98 `
-  --num-workers 4 `
+  --num-workers 0 `
   --checkpoint-steps 1000 `
   --save-total-limit 3 `
   --parse-eval-limit 2048 `
   --case-eval-file data/parser_regression_cases.json `
-  --seed 52 `
-  --experiment-name dmhy-char-full
+  --augment-partial-samples 200000 `
+  --augment-permutation-samples 400000 `
+  --augment-special-samples 80000 `
+  --bf16 `
+  --no-periodic-eval `
+  --perf-log-steps 200 `
+  --seed 105 `
+  --experiment-name dmhy-char-aug-fragments-optimized-10epoch
 ```
 
 `python -m anifilebert.train` writes:
@@ -218,6 +224,7 @@ uv run python -m anifilebert.train --tokenizer char `
 - `final/trainer_eval_metrics.json`,
 - `final/parse_eval_metrics.json`,
 - `final/case_metrics.json` unless `--no-case-eval` is used,
+- `final/perf_metrics.json` when `--perf-log-steps` is set,
 - TensorBoard logs unless `--no-tensorboard` is used.
 
 Full workflow: [`docs/training.md`](docs/training.md).

anifilebert/dataset.py

@@ -7,16 +7,158 @@ Handles token-ID conversion, label encoding, padding, and truncation.
 
 import json
 from collections import Counter
+import numpy as np
 import torch
 from torch.utils.data import Dataset
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Sequence, Tuple
 
 from .config import Config
 from .label_repairs import repair_sequel_season_labels
 from .tokenizer import AnimeTokenizer
 
 
-class AnimeDataset(Dataset):
+def encode_token_classification_values(
+    item: Dict,
+    tokenizer: AnimeTokenizer,
+    label2id: Dict[str, int],
+    max_length: int,
+    apply_label_repairs: bool = True,
+    vocab: Optional[Dict[str, int]] = None,
+) -> Tuple[List[int], List[bool], List[int]]:
+    tokens, labels = training_labels_for_tokenizer(item, tokenizer, apply_label_repairs)
+
+    token_vocab = vocab if vocab is not None else tokenizer.get_vocab()
+    unk_id = tokenizer.unk_token_id if tokenizer.unk_token_id is not None else 1
+    input_ids = [token_vocab.get(token, unk_id) for token in tokens]
+    input_ids = [tokenizer.cls_token_id] + input_ids + [tokenizer.sep_token_id]
+
+    label_ids: List[int] = [-100]
+    label_ids.extend(label2id.get(label, 0) for label in labels)
+    label_ids.append(-100)
+
+    attention_mask = [1] * len(input_ids)
+
+    if len(input_ids) > max_length:
+        input_ids = [input_ids[0]] + input_ids[1:max_length - 1] + [input_ids[-1]]
+        label_ids = [label_ids[0]] + label_ids[1:max_length - 1] + [label_ids[-1]]
+        attention_mask = [attention_mask[0]] + attention_mask[1:max_length - 1] + [attention_mask[-1]]
+
+    pad_len = max_length - len(input_ids)
+    if pad_len > 0:
+        input_ids += [tokenizer.pad_token_id] * pad_len
+        label_ids += [-100] * pad_len
+        attention_mask += [0] * pad_len
+
+    return input_ids, [bool(value) for value in attention_mask], label_ids
+
+
+def encode_token_classification_item(
+    item: Dict,
+    tokenizer: AnimeTokenizer,
+    label2id: Dict[str, int],
+    max_length: int,
+    apply_label_repairs: bool = True,
+    vocab: Optional[Dict[str, int]] = None,
+) -> Dict[str, torch.Tensor]:
+    input_ids, attention_mask, label_ids = encode_token_classification_values(
+        item,
+        tokenizer,
+        label2id,
+        max_length,
+        apply_label_repairs,
+        vocab,
+    )
+
+    return {
+        "input_ids": torch.tensor(input_ids, dtype=torch.long),
+        "attention_mask": torch.tensor(attention_mask, dtype=torch.bool),
+        "labels": torch.tensor(label_ids, dtype=torch.long),
+    }
+
+
+class AnimeItemsDataset(Dataset):
+    """Map-style dataset backed by already-loaded JSONL items."""
+
+    def __init__(
+        self,
+        data: Sequence[Dict],
+        tokenizer: AnimeTokenizer,
+        label2id: Dict[str, int],
+        max_length: int = 64,
+        apply_label_repairs: bool = True,
+    ):
+        self.data = data
+        self.tokenizer = tokenizer
+        self.label2id = label2id
+        self.max_length = max_length
+        self.apply_label_repairs = apply_label_repairs
+        self.vocab = tokenizer.get_vocab()
+
+    def __len__(self) -> int:
+        return len(self.data)
+
+    def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
+        return encode_token_classification_item(
+            self.data[idx],
+            self.tokenizer,
+            self.label2id,
+            self.max_length,
+            self.apply_label_repairs,
+            self.vocab,
+        )
+
+
+class EncodedAnimeDataset(Dataset):
+    """Dataset that stores padded tensors so training workers do no token work."""
+
+    def __init__(
+        self,
+        data: Sequence[Dict],
+        tokenizer: AnimeTokenizer,
+        label2id: Dict[str, int],
+        max_length: int = 64,
+        device: Optional[torch.device] = None,
+        apply_label_repairs: bool = True,
+    ):
+        target_device = device or torch.device("cpu")
+        vocab = tokenizer.get_vocab()
+        input_ids = np.full(
+            (len(data), max_length),
+            tokenizer.pad_token_id,
+            dtype=np.int64,
+        )
+        attention_mask = np.zeros((len(data), max_length), dtype=np.bool_)
+        labels = np.full((len(data), max_length), -100, dtype=np.int64)
+
+        for idx, item in enumerate(data):
+            item_input_ids, item_attention_mask, item_labels = encode_token_classification_values(
+                item,
+                tokenizer,
+                label2id,
+                max_length,
+                apply_label_repairs,
+                vocab,
+            )
+            input_ids[idx] = item_input_ids
+            attention_mask[idx] = item_attention_mask
+            labels[idx] = item_labels
+
+        self.input_ids = torch.from_numpy(input_ids).to(target_device)
+        self.attention_mask = torch.from_numpy(attention_mask).to(target_device)
+        self.labels = torch.from_numpy(labels).to(target_device)
+
+    def __len__(self) -> int:
+        return self.input_ids.shape[0]
+
+    def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
+        return {
+            "input_ids": self.input_ids[idx],
+            "attention_mask": self.attention_mask[idx],
+            "labels": self.labels[idx],
+        }
+
+
+class AnimeDataset(AnimeItemsDataset):
     """
     Dataset for anime filename token classification.
 
@@ -41,65 +183,13 @@ class AnimeDataset(Dataset):
             label2id: Mapping from label string to integer ID.
             max_length: Maximum sequence length (including special tokens).
         """
-        self.tokenizer = tokenizer
-        self.label2id = label2id
-        self.max_length = max_length
-
-        # Load data
-        self.data: List[Dict] = []
+        data: List[Dict] = []
         with open(data_path, 'r', encoding='utf-8') as f:
             for line in f:
                 line = line.strip()
                 if line:
-                    self.data.append(json.loads(line))
-
-    def __len__(self) -> int:
-        return len(self.data)
-
-    def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
-        """
-        Get a preprocessed sample.
-
-        Returns:
-            Dictionary with input_ids, attention_mask, labels as LongTensors.
-        """
-        item = self.data[idx]
-        tokens, labels = labels_for_tokenizer(item, self.tokenizer)
-
-        # Convert tokens to IDs
-        input_ids = self.tokenizer.convert_tokens_to_ids(tokens)
-
-        # Add [CLS] at start and [SEP] at end
-        input_ids = [self.tokenizer.cls_token_id] + input_ids + [self.tokenizer.sep_token_id]
-
-        # Convert labels to IDs, with -100 for special tokens
-        label_ids: List[int] = [-100]  # [CLS] → -100 (ignored in loss)
-        for label in labels:
-            label_ids.append(self.label2id.get(label, 0))  # default to O
-        label_ids.append(-100)  # [SEP] → -100
-
-        # Attention mask: 1 for real tokens
-        attention_mask = [1] * len(input_ids)
-
-        # Truncate if needed (keep CLS at 0, SEP at end)
-        if len(input_ids) > self.max_length:
-            # Keep first token (CLS), truncate middle, keep last token (SEP)
-            input_ids = [input_ids[0]] + input_ids[1:self.max_length - 1] + [input_ids[-1]]
-            label_ids = [label_ids[0]] + label_ids[1:self.max_length - 1] + [label_ids[-1]]
-            attention_mask = [attention_mask[0]] + attention_mask[1:self.max_length - 1] + [attention_mask[-1]]
-
-        # Pad to max_length
-        pad_len = self.max_length - len(input_ids)
-        if pad_len > 0:
-            input_ids += [self.tokenizer.pad_token_id] * pad_len
-            label_ids += [-100] * pad_len
-            attention_mask += [0] * pad_len
-
-        return {
-            "input_ids": torch.tensor(input_ids, dtype=torch.long),
-            "attention_mask": torch.tensor(attention_mask, dtype=torch.long),
-            "labels": torch.tensor(label_ids, dtype=torch.long),
-        }
+                    data.append(json.loads(line))
+        super().__init__(data, tokenizer, label2id, max_length)
 
 
 def align_tokens_for_tokenizer(
@@ -177,6 +267,23 @@ def labels_for_tokenizer(
     return align_tokens_for_tokenizer(source_tokens, source_labels, tokenizer)
 
 
+def training_labels_for_tokenizer(
+    item: Dict,
+    tokenizer: AnimeTokenizer,
+    apply_label_repairs: bool,
+) -> Tuple[List[str], List[str]]:
+    """Fast path for authoritative char JSONL rows used in full training."""
+    tokenizer_variant = getattr(tokenizer, "tokenizer_variant", "regex")
+    if not apply_label_repairs and item.get("tokenizer_variant") == tokenizer_variant:
+        tokens = item.get("tokens", [])
+        labels = item.get("labels", [])
+        filename = item.get("filename")
+        if len(tokens) == len(labels):
+            if tokenizer_variant != "char" or filename is None or tokens == list(str(filename)):
+                return tokens, labels
+    return labels_for_tokenizer(item, tokenizer)
+
+
 def token_offsets_in_text(text: str, tokens: List[str]) -> Optional[List[Tuple[int, int]]]:
     """Find token character offsets by scanning left to right."""
     offsets: List[Tuple[int, int]] = []

anifilebert/inference.py

@@ -30,6 +30,18 @@ CN_NUM_MAP: Dict[str, int] = {
 }
 
 
+STANDALONE_SPECIAL_RE = re.compile(
+    r"^(?:"
+    r"(?:BD\s*)?Menu\s*\d{0,2}(?:-\d{1,2})?|"
+    r"NCOP\s*\d{0,2}|NCED\s*\d{0,2}|"
+    r"OP\s*\d{0,2}|ED(?:\s*E?\d{0,2})?|"
+    r"PV\s*\d{0,2}|CM\s*\d{0,2}|"
+    r"OVA\s*\d{0,2}|OAD\s*\d{0,2}|SP\s*\d{0,2}|IV\d+"
+    r")$",
+    re.I,
+)
+
+
 def extract_season_number(text: str) -> Optional[int]:
     """
     Extract season number from various season formats.
@@ -113,6 +125,13 @@ def choose_thin_source(sources: List[str]) -> Optional[str]:
     return max(enumerate(cleaned), key=lambda item: (thin_source_priority(item[1]), -item[0]))[1]
 
 
+def normalize_standalone_special(text: str) -> Optional[str]:
+    special = normalize_field_text(text)
+    if not special:
+        return None
+    return special if STANDALONE_SPECIAL_RE.fullmatch(special) else None
+
+
 def display_token(token: str) -> str:
     """Make whitespace tokens visible in debug output."""
     if token == " ":
@@ -304,6 +323,21 @@ def postprocess(
 
     result["source"] = choose_thin_source(grouped_entities.get("SOURCE", []))
 
+    whole_text = join_entity_tokens(tokens, tokenizer)
+    standalone_special = normalize_standalone_special(whole_text)
+    if standalone_special is not None:
+        result.update(
+            {
+                "title": None,
+                "season": None,
+                "episode": None,
+                "group": None,
+                "resolution": None,
+                "source": None,
+                "special": standalone_special,
+            }
+        )
+
     return result
 
 

anifilebert/train.py

@@ -9,26 +9,31 @@ end-to-end parser exact-match on held-out filenames and fixed real-world cases.
 import os
 import sys
 import json
-import tempfile
 import argparse
 import random
+import subprocess
+import threading
+import time
+import gc
 from collections import Counter
-from typing import Dict, List, Optional
+from ctypes import POINTER, Structure, byref, c_int, c_uint, c_ulonglong, c_void_p, cdll
+from ctypes import util as ctypes_util
+from typing import Dict, List, Optional, Sequence
 
 import numpy as np
 import torch
 from transformers import (
     Trainer,
     TrainingArguments,
-    DataCollatorForTokenClassification,
     BertForTokenClassification,
+    TrainerCallback,
 )
 from seqeval.metrics import classification_report, accuracy_score, f1_score, precision_score, recall_score
 
 from .config import Config
 from .tokenizer import AnimeTokenizer, create_tokenizer, load_tokenizer
 from .model import create_model, print_model_summary, count_parameters
-from .dataset import AnimeDataset, labels_for_tokenizer
+from .dataset import AnimeItemsDataset, EncodedAnimeDataset, labels_for_tokenizer
 from .inference import parse_filename, postprocess
 
 
@@ -66,12 +71,18 @@ def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Train anime filename parser")
     parser.add_argument("--tokenizer", choices=["regex", "char"], default=None,
                         help="Tokenizer variant for A/B testing. Defaults to dataset metadata")
-    parser.add_argument("--data-file", default=None, help="Training JSONL file")
+    parser.add_argument("--data-file", default=None, help="Primary training JSONL file")
+    parser.add_argument("--extra-data-file", action="append", default=[],
+                        help="Additional training JSONL file. Can be passed multiple times.")
+    parser.add_argument("--extra-data-repeat", type=int, default=1,
+                        help="Repeat each extra dataset this many times after loading")
     parser.add_argument("--vocab-file", default=None,
                         help="Tokenizer vocab JSON. Defaults to data/vocab.json or data/vocab.char.json")
     parser.add_argument("--save-dir", default=None, help="Checkpoint output directory")
     parser.add_argument("--init-model-dir", default=None, help="Optional checkpoint to fine-tune from")
     parser.add_argument("--epochs", type=float, default=None, help="Number of training epochs")
+    parser.add_argument("--max-steps", type=int, default=-1,
+                        help="Override epoch-based training and stop after this many optimizer steps")
     parser.add_argument("--batch-size", type=int, default=None, help="Per-device train/eval batch size")
     parser.add_argument("--learning-rate", type=float, default=None, help="Learning rate")
     parser.add_argument("--warmup-steps", type=int, default=None, help="Warmup steps")
@@ -80,6 +91,14 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--seed", type=int, default=42, help="Random seed")
     parser.add_argument("--limit-samples", type=int, default=None,
                         help="Use only the first N samples for quick A/B smoke runs")
+    parser.add_argument("--augment-partial-samples", type=int, default=0,
+                        help="Generate this many partial BIO-span samples in memory before training")
+    parser.add_argument("--augment-permutation-samples", type=int, default=0,
+                        help="Generate this many random BIO-span permutation samples in memory before training")
+    parser.add_argument("--augment-special-samples", type=int, default=0,
+                        help="Generate this many special-only/title+special samples such as Menu01 in memory")
+    parser.add_argument("--augment-max-chars", type=int, default=160,
+                        help="Maximum character length for generated augmentation samples")
     parser.add_argument("--rebuild-vocab", action="store_true",
                         help="Rebuild vocab from the selected data file before training")
     parser.add_argument("--max-vocab-size", type=int, default=None,
@@ -88,10 +107,40 @@ def parse_args() -> argparse.Namespace:
                         help="Save resumable checkpoints every N steps instead of only at epoch end")
     parser.add_argument("--save-total-limit", type=int, default=2,
                         help="Maximum number of checkpoints to keep")
+    parser.add_argument("--no-periodic-eval", action="store_true",
+                        help="Skip Trainer's scheduled train-time eval/load-best-model; final evaluation still runs")
+    parser.add_argument("--keep-raw-dataset", action="store_true",
+                        help="Keep raw JSONL dictionaries in memory after encoded datasets are built")
     parser.add_argument("--gradient-accumulation-steps", type=int, default=1,
                         help="Accumulate gradients across this many steps")
     parser.add_argument("--num-workers", type=int, default=None,
                         help="DataLoader worker count. Defaults to config.num_workers")
+    parser.add_argument("--prefetch-factor", type=int, default=None,
+                        help="DataLoader prefetch factor when workers are enabled")
+    parser.add_argument("--persistent-workers", action="store_true",
+                        help="Keep DataLoader workers alive between epochs")
+    parser.add_argument("--lazy-dataset", action="store_true",
+                        help="Tokenize samples lazily in DataLoader workers instead of pre-encoding tensors")
+    parser.add_argument("--apply-label-repairs", action="store_true",
+                        help="Apply runtime deterministic label repairs while building training tensors")
+    parser.add_argument("--encoded-dataset-device", choices=["cpu", "cuda"], default="cpu",
+                        help="Store pre-encoded dataset tensors on this device; cuda requires --num-workers 0")
+    parser.add_argument("--bf16", action="store_true",
+                        help="Use bfloat16 mixed precision on CUDA instead of fp16")
+    parser.add_argument("--no-mixed-precision", action="store_true",
+                        help="Disable fp16/bf16 mixed precision even when CUDA is available")
+    parser.add_argument("--tf32", dest="tf32", action="store_true",
+                        help="Enable TF32 matmul/cudnn kernels on CUDA")
+    parser.add_argument("--no-tf32", dest="tf32", action="store_false",
+                        help="Disable TF32 matmul/cudnn kernels")
+    parser.add_argument("--torch-compile", action="store_true",
+                        help="Enable torch.compile through Hugging Face Trainer")
+    parser.add_argument("--auto-find-batch-size", action="store_true",
+                        help="Let Trainer reduce batch size automatically on CUDA OOM")
+    parser.add_argument("--perf-log-steps", type=int, default=100,
+                        help="Sample training throughput, memory, and GPU stats every N steps; 0 disables")
+    parser.add_argument("--perf-sample-interval", type=float, default=1.0,
+                        help="Background NVML sampling interval in seconds during training; 0 disables")
     parser.add_argument("--cpu", action="store_true", help="Force CPU training")
     parser.add_argument("--no-shuffle", action="store_true", help="Do not shuffle before train/eval split")
     parser.add_argument("--resume-from-checkpoint", default=None,
@@ -114,6 +163,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--num-hidden-layers", type=int, default=None, help="Override BERT layer count")
     parser.add_argument("--num-attention-heads", type=int, default=None, help="Override BERT attention heads")
     parser.add_argument("--intermediate-size", type=int, default=None, help="Override BERT FFN intermediate size")
+    parser.set_defaults(tf32=True)
     parser.set_defaults(tensorboard=True)
     return parser.parse_args()
 
@@ -159,6 +209,22 @@ def detect_tokenizer_variant(
     return "regex"
 
 
+def detect_tokenizer_variant_from_files(
+    data_files: List[str],
+    explicit_variant: Optional[str],
+    explicit_vocab_path: Optional[str],
+) -> str:
+    if explicit_variant:
+        return explicit_variant
+    variants = {
+        detect_tokenizer_variant(path, None, explicit_vocab_path)
+        for path in data_files
+    }
+    if len(variants) > 1:
+        raise ValueError(f"Mixed tokenizer variants across datasets: {sorted(variants)}")
+    return next(iter(variants))
+
+
 def resolve_vocab_path(data_file: str, tokenizer_variant: str, explicit_path: Optional[str]) -> str:
     if explicit_path:
         return explicit_path
@@ -209,6 +275,665 @@ def load_jsonl(data_file: str, limit: Optional[int] = None) -> List[Dict]:
     return data
 
 
+def load_training_sources(
+    primary_data_file: str,
+    extra_data_files: List[str],
+    extra_repeat: int,
+    limit: Optional[int] = None,
+) -> tuple[List[Dict], List[Dict]]:
+    """Load primary plus extra datasets while preserving source metadata."""
+    sources: List[Dict] = []
+    primary = load_jsonl(primary_data_file, limit)
+    all_data: List[Dict] = list(primary)
+    sources.append(
+        {
+            "role": "primary",
+            "path": primary_data_file,
+            "samples": len(primary),
+            "repeat": 1,
+            "effective_samples": len(primary),
+        }
+    )
+
+    repeat = max(1, extra_repeat)
+    for path in extra_data_files:
+        rows = load_jsonl(path, None)
+        for _ in range(repeat):
+            all_data.extend(rows)
+        sources.append(
+            {
+                "role": "extra",
+                "path": path,
+                "samples": len(rows),
+                "repeat": repeat,
+                "effective_samples": len(rows) * repeat,
+            }
+        )
+    return all_data, sources
+
+
+def extract_entities_from_labels(tokens: Sequence[str], labels: Sequence[str]) -> Dict[str, List[str]]:
+    """Extract contiguous BIO entity text spans from token/label arrays."""
+    entities: Dict[str, List[str]] = {}
+    active_entity: Optional[str] = None
+    active_tokens: List[str] = []
+
+    for token, label in zip(tokens, labels):
+        if label.startswith("B-"):
+            if active_entity and active_tokens:
+                entities.setdefault(active_entity, []).append("".join(active_tokens))
+            active_entity = label[2:]
+            active_tokens = [str(token)]
+        elif label.startswith("I-") and active_entity == label[2:]:
+            active_tokens.append(str(token))
+        else:
+            if active_entity and active_tokens:
+                entities.setdefault(active_entity, []).append("".join(active_tokens))
+            active_entity = None
+            active_tokens = []
+
+    if active_entity and active_tokens:
+        entities.setdefault(active_entity, []).append("".join(active_tokens))
+    return entities
+
+
+def char_item_from_spans(filename: str, spans: Sequence[tuple[str, str]], source: str) -> Optional[Dict]:
+    """Create a char-tokenized BIO item from ordered text/entity spans."""
+    filename = filename.strip()
+    if not filename:
+        return None
+    tokens = list(filename)
+    labels = ["O"] * len(tokens)
+    cursor = 0
+    for text, entity in spans:
+        if not text:
+            continue
+        start = filename.find(text, cursor)
+        if start < 0:
+            start = filename.find(text)
+        if start < 0:
+            return None
+        end = start + len(text)
+        labels[start] = f"B-{entity}"
+        for idx in range(start + 1, end):
+            labels[idx] = f"I-{entity}"
+        cursor = end
+    return {
+        "filename": filename,
+        "tokens": tokens,
+        "labels": labels,
+        "tokenizer_variant": "char",
+        "source": source,
+    }
+
+
+def entity_keep_probability(entity: str) -> float:
+    return {
+        "GROUP": 0.35,
+        "TITLE": 0.65,
+        "SEASON": 0.35,
+        "EPISODE": 0.7,
+        "SPECIAL": 0.3,
+        "RESOLUTION": 0.65,
+        "SOURCE": 0.65,
+    }.get(entity, 0.5)
+
+
+def build_partial_augmented_item(item: Dict, max_chars: int) -> List[Dict]:
+    entities = extract_entities_from_labels(item.get("tokens", []), item.get("labels", []))
+    title = next((value.strip() for value in entities.get("TITLE", []) if value.strip()), None)
+    season = next((value.strip() for value in entities.get("SEASON", []) if value.strip()), None)
+    episode = next((value.strip() for value in entities.get("EPISODE", []) if value.strip()), None)
+    special = next((value.strip() for value in entities.get("SPECIAL", []) if value.strip()), None)
+    resolution = next((value.strip() for value in entities.get("RESOLUTION", []) if value.strip()), None)
+    source = next((value.strip() for value in entities.get("SOURCE", []) if value.strip()), None)
+
+    specs: List[tuple[str, List[tuple[str, str]]]] = []
+    if title:
+        specs.append((title, [(title, "TITLE")]))
+    if title and season:
+        specs.append((f"{title} {season}", [(title, "TITLE"), (season, "SEASON")]))
+    if episode:
+        specs.append((episode, [(episode, "EPISODE")]))
+    if episode and resolution:
+        specs.append((f"{episode} [{resolution}]", [(episode, "EPISODE"), (resolution, "RESOLUTION")]))
+    if episode and resolution and source:
+        specs.append(
+            (
+                f"{episode} [{resolution}][{source}]",
+                [(episode, "EPISODE"), (resolution, "RESOLUTION"), (source, "SOURCE")],
+            )
+        )
+    if special:
+        specs.append((special, [(special, "SPECIAL")]))
+    if title and special:
+        specs.append((f"{title} - {special}", [(title, "TITLE"), (special, "SPECIAL")]))
+
+    augmented: List[Dict] = []
+    for text, spans in specs:
+        if 2 <= len(text) <= max_chars:
+            generated = char_item_from_spans(text, spans, "train_partial_augmentation")
+            if generated is not None:
+                augmented.append(generated)
+    return augmented
+
+
+def build_permutation_augmented_item(item: Dict, rng: random.Random, max_chars: int) -> Optional[Dict]:
+    entities = extract_entities_from_labels(item.get("tokens", []), item.get("labels", []))
+    available = [
+        entity
+        for entity in ("GROUP", "TITLE", "SEASON", "EPISODE", "SPECIAL", "RESOLUTION", "SOURCE")
+        if entities.get(entity)
+    ]
+    if not available:
+        return None
+    selected = [
+        entity
+        for entity in available
+        if rng.random() < entity_keep_probability(entity)
+    ]
+    if not selected:
+        selected = [rng.choice(available)]
+    if "TITLE" not in selected and "EPISODE" not in selected and "SPECIAL" not in selected:
+        extras = [entity for entity in available if entity not in selected]
+        selected.append(rng.choice(extras or available))
+    rng.shuffle(selected)
+
+    separators = [" ", " - ", ".", "_", "]["]
+    sep = rng.choice(separators)
+    parts: List[str] = []
+    spans: List[tuple[str, str]] = []
+    for entity in selected:
+        values = [value.strip() for value in entities.get(entity, []) if value.strip()]
+        if not values:
+            continue
+        value = rng.choice(values)
+        if entity in {"GROUP", "EPISODE", "SPECIAL", "RESOLUTION", "SOURCE"} and rng.random() < 0.35:
+            parts.append(f"[{value}]")
+        else:
+            parts.append(value)
+        spans.append((value, entity))
+    text = sep.join(parts).strip()
+    if not (2 <= len(text) <= max_chars):
+        return None
+    return char_item_from_spans(text, spans, "train_permutation_augmentation")
+
+
+def build_special_augmented_item(data: List[Dict], rng: random.Random, max_chars: int) -> Optional[Dict]:
+    base_titles: List[str] = []
+    for _ in range(min(16, len(data))):
+        item = data[rng.randrange(len(data))]
+        entities = extract_entities_from_labels(item.get("tokens", []), item.get("labels", []))
+        base_titles.extend(value.strip() for value in entities.get("TITLE", []) if 2 <= len(value.strip()) <= 80)
+    title = rng.choice(base_titles) if base_titles else None
+    special = rng.choice(
+        [
+            f"Menu{rng.randint(1, 24):02d}",
+            f"Menu {rng.randint(1, 24):02d}",
+            f"BDMenu{rng.randint(1, 24):02d}",
+            f"BD Menu{rng.randint(1, 24):02d}",
+            f"Menu{rng.randint(1, 24):02d}-01",
+            "Menu",
+            f"OP{rng.randint(1, 6):02d}",
+            f"ED E{rng.randint(1, 24):02d}",
+            f"NCOP{rng.randint(1, 6):02d}",
+            f"NCED{rng.randint(1, 6):02d}",
+            f"CM{rng.randint(1, 12):02d}",
+            f"PV{rng.randint(1, 12):02d}",
+        ]
+    )
+    if title and rng.random() < 0.55:
+        text = f"{title} - {special}"
+        spans = [(title, "TITLE"), (special, "SPECIAL")]
+    else:
+        text = special
+        spans = [(special, "SPECIAL")]
+    if len(text) > max_chars:
+        return None
+    return char_item_from_spans(text, spans, "train_special_augmentation")
+
+
+def process_memory_mb() -> Optional[float]:
+    try:
+        import psutil  # type: ignore
+
+        return psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024)
+    except Exception:
+        pass
+
+    if os.name == "nt":
+        try:
+            import ctypes
+            from ctypes import wintypes
+
+            class PROCESS_MEMORY_COUNTERS(ctypes.Structure):
+                _fields_ = [
+                    ("cb", wintypes.DWORD),
+                    ("PageFaultCount", wintypes.DWORD),
+                    ("PeakWorkingSetSize", ctypes.c_size_t),
+                    ("WorkingSetSize", ctypes.c_size_t),
+                    ("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
+                    ("QuotaPagedPoolUsage", ctypes.c_size_t),
+                    ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
+                    ("QuotaNonPagedPoolUsage", ctypes.c_size_t),
+                    ("PagefileUsage", ctypes.c_size_t),
+                    ("PeakPagefileUsage", ctypes.c_size_t),
+                ]
+
+            counters = PROCESS_MEMORY_COUNTERS()
+            counters.cb = ctypes.sizeof(counters)
+            handle = ctypes.windll.kernel32.GetCurrentProcess()
+            if ctypes.windll.psapi.GetProcessMemoryInfo(handle, ctypes.byref(counters), counters.cb):
+                return float(counters.WorkingSetSize) / (1024 * 1024)
+        except Exception:
+            pass
+
+    try:
+        import resource  # type: ignore
+
+        usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        if sys.platform == "darwin":
+            usage = usage / 1024 / 1024
+        else:
+            usage = usage / 1024
+        return float(usage)
+    except Exception:
+        return None
+
+
+class NvmlSampler:
+    """Tiny NVML binding for runtime GPU telemetry without adding dependencies."""
+
+    NVML_TEMPERATURE_GPU = 0
+
+    def __init__(self):
+        self._lib = None
+        self._handle = None
+        self._available = False
+        self._init()
+
+    def _candidate_names(self) -> List[str]:
+        names = []
+        found = ctypes_util.find_library("nvidia-ml")
+        if found:
+            names.append(found)
+        if os.name == "nt":
+            names.extend(
+                [
+                    os.path.join(os.environ.get("SystemRoot", r"C:\Windows"), "System32", "nvml.dll"),
+                    "nvml.dll",
+                    "nvidia-ml.dll",
+                ]
+            )
+        else:
+            names.extend(["libnvidia-ml.so.1", "libnvidia-ml.so"])
+        return names
+
+    def _init(self) -> None:
+        for name in self._candidate_names():
+            try:
+                lib = cdll.LoadLibrary(name)
+                break
+            except Exception:
+                continue
+        else:
+            return
+
+        class NVMLUtilization(Structure):
+            _fields_ = [("gpu", c_uint), ("memory", c_uint)]
+
+        class NVMLMemory(Structure):
+            _fields_ = [("total", c_ulonglong), ("free", c_ulonglong), ("used", c_ulonglong)]
+
+        self.NVMLUtilization = NVMLUtilization
+        self.NVMLMemory = NVMLMemory
+        lib.nvmlInit_v2.restype = c_int
+        lib.nvmlDeviceGetHandleByIndex_v2.argtypes = [c_uint, POINTER(c_void_p)]
+        lib.nvmlDeviceGetHandleByIndex_v2.restype = c_int
+        handle = c_void_p()
+        try:
+            if lib.nvmlInit_v2() != 0:
+                return
+            if lib.nvmlDeviceGetHandleByIndex_v2(0, byref(handle)) != 0:
+                return
+        except Exception:
+            return
+        self._lib = lib
+        self._handle = handle
+        self._available = True
+
+    @property
+    def available(self) -> bool:
+        return self._available
+
+    def sample(self) -> Dict[str, Optional[float]]:
+        if not self._available or self._lib is None or self._handle is None:
+            return {}
+        stats: Dict[str, Optional[float]] = {}
+        try:
+            util_rates = self.NVMLUtilization()
+            self._lib.nvmlDeviceGetUtilizationRates.argtypes = [c_void_p, POINTER(self.NVMLUtilization)]
+            if self._lib.nvmlDeviceGetUtilizationRates(self._handle, byref(util_rates)) == 0:
+                stats["gpu_util_percent"] = float(util_rates.gpu)
+                stats["gpu_memory_util_percent"] = float(util_rates.memory)
+        except Exception:
+            pass
+        try:
+            memory = self.NVMLMemory()
+            self._lib.nvmlDeviceGetMemoryInfo.argtypes = [c_void_p, POINTER(self.NVMLMemory)]
+            if self._lib.nvmlDeviceGetMemoryInfo(self._handle, byref(memory)) == 0:
+                stats["gpu_memory_used_mb"] = float(memory.used) / (1024 * 1024)
+                stats["gpu_memory_total_mb"] = float(memory.total) / (1024 * 1024)
+        except Exception:
+            pass
+        try:
+            temperature = c_uint()
+            self._lib.nvmlDeviceGetTemperature.argtypes = [c_void_p, c_uint, POINTER(c_uint)]
+            if self._lib.nvmlDeviceGetTemperature(self._handle, self.NVML_TEMPERATURE_GPU, byref(temperature)) == 0:
+                stats["gpu_temperature_c"] = float(temperature.value)
+        except Exception:
+            pass
+        try:
+            power_mw = c_uint()
+            self._lib.nvmlDeviceGetPowerUsage.argtypes = [c_void_p, POINTER(c_uint)]
+            if self._lib.nvmlDeviceGetPowerUsage(self._handle, byref(power_mw)) == 0:
+                stats["gpu_power_w"] = float(power_mw.value) / 1000.0
+        except Exception:
+            pass
+        return stats
+
+
+_NVML_SAMPLER: Optional[NvmlSampler] = None
+
+
+def query_nvml() -> Dict[str, Optional[float]]:
+    global _NVML_SAMPLER
+    if _NVML_SAMPLER is None:
+        _NVML_SAMPLER = NvmlSampler()
+    return _NVML_SAMPLER.sample()
+
+
+def query_nvidia_smi() -> Dict[str, Optional[float]]:
+    try:
+        result = subprocess.run(
+            [
+                "nvidia-smi",
+                "--query-gpu=utilization.gpu,memory.used,memory.total,power.draw",
+                "--format=csv,noheader,nounits",
+            ],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=2,
+        )
+    except Exception:
+        return {}
+    if result.returncode != 0 or not result.stdout.strip():
+        return {}
+    first_line = result.stdout.strip().splitlines()[0]
+    values = [part.strip() for part in first_line.split(",")]
+    keys = ["gpu_util_percent", "gpu_memory_used_mb", "gpu_memory_total_mb", "gpu_power_w"]
+    stats: Dict[str, Optional[float]] = {}
+    for key, value in zip(keys, values):
+        try:
+            stats[key] = float(value)
+        except ValueError:
+            stats[key] = None
+    return stats
+
+
+def cuda_memory_stats_mb() -> Dict[str, float]:
+    if not torch.cuda.is_available():
+        return {}
+    return {
+        "cuda_allocated_mb": torch.cuda.memory_allocated() / (1024 * 1024),
+        "cuda_reserved_mb": torch.cuda.memory_reserved() / (1024 * 1024),
+        "cuda_max_allocated_mb": torch.cuda.max_memory_allocated() / (1024 * 1024),
+        "cuda_max_reserved_mb": torch.cuda.max_memory_reserved() / (1024 * 1024),
+    }
+
+
+def snapshot_perf_stats() -> Dict[str, Optional[float]]:
+    stats: Dict[str, Optional[float]] = {}
+    stats["process_rss_mb"] = process_memory_mb()
+    stats.update(cuda_memory_stats_mb())
+    gpu_stats = query_nvml()
+    if not gpu_stats:
+        gpu_stats = query_nvidia_smi()
+    stats.update(gpu_stats)
+    return stats
+
+
+class TrainingPerfCallback(TrainerCallback):
+    """Lightweight runtime telemetry for spotting data-pipeline starvation."""
+
+    def __init__(self, batch_size: int, sequence_length: int, log_steps: int, sample_interval: float):
+        self.batch_size = batch_size
+        self.sequence_length = sequence_length
+        self.log_steps = max(0, log_steps)
+        self.sample_interval = max(0.0, sample_interval)
+        self.samples: List[Dict[str, Optional[float]]] = []
+        self.background_samples: List[Dict[str, Optional[float]]] = []
+        self._last_step = 0
+        self._last_time: Optional[float] = None
+        self._start_time: Optional[float] = None
+        self._training = False
+        self._stop_event = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+
+    def on_train_begin(self, args, state, control, **kwargs):
+        now = time.perf_counter()
+        self._start_time = now
+        self._last_time = now
+        self._last_step = int(state.global_step)
+        self._training = True
+        self._stop_event.clear()
+        if self.sample_interval > 0:
+            self._thread = threading.Thread(target=self._background_sample_loop, daemon=True)
+            self._thread.start()
+
+    def on_train_end(self, args, state, control, **kwargs):
+        self._training = False
+        self._stop_event.set()
+        if self._thread is not None:
+            self._thread.join(timeout=max(self.sample_interval * 2, 1.0))
+            self._thread = None
+
+    def on_log(self, args, state, control, logs=None, **kwargs):
+        if not self._training:
+            return
+        step = int(state.global_step)
+        if self.log_steps <= 0 or step <= 0 or step % self.log_steps != 0:
+            return
+        self._record_sample(step)
+
+    def on_step_end(self, args, state, control, **kwargs):
+        if not self._training:
+            return
+        step = int(state.global_step)
+        if self.log_steps <= 0 or step <= 0 or step % self.log_steps != 0:
+            return
+        if self.samples and self.samples[-1].get("step") == float(step):
+            return
+        self._record_sample(step)
+
+    def _record_sample(self, step: int) -> None:
+        if self.samples and self.samples[-1].get("step") == float(step):
+            return
+        now = time.perf_counter()
+        last_time = self._last_time or now
+        elapsed = max(now - last_time, 1e-9)
+        step_delta = max(step - self._last_step, 0)
+        samples_per_second = step_delta * self.batch_size / elapsed
+        tokens_per_second = samples_per_second * self.sequence_length
+        stats = snapshot_perf_stats()
+        sample: Dict[str, Optional[float]] = {
+            "step": float(step),
+            "elapsed_seconds": now - (self._start_time or now),
+            "window_seconds": elapsed,
+            "steps_per_second": step_delta / elapsed,
+            "samples_per_second": samples_per_second,
+            "tokens_per_second": tokens_per_second,
+        }
+        sample.update(stats)
+        self.samples.append(sample)
+        print(
+            "  perf "
+            f"step={step} "
+            f"samples/s={samples_per_second:.1f} "
+            f"tokens/s={tokens_per_second:.0f} "
+            f"rss={stats.get('process_rss_mb') or 0:.0f}MB "
+            f"cuda_alloc={stats.get('cuda_allocated_mb') or 0:.0f}MB "
+            f"gpu_util={stats.get('gpu_util_percent') if stats.get('gpu_util_percent') is not None else 'n/a'}%"
+        )
+        self._last_time = now
+        self._last_step = step
+
+    def _background_sample_loop(self) -> None:
+        while not self._stop_event.wait(self.sample_interval):
+            if not self._training:
+                continue
+            sample = snapshot_perf_stats()
+            sample["elapsed_seconds"] = (
+                time.perf_counter() - self._start_time
+                if self._start_time is not None
+                else None
+            )
+            self.background_samples.append(sample)
+
+    def summary(self) -> Dict:
+        numeric_keys = [
+            "samples_per_second",
+            "tokens_per_second",
+            "process_rss_mb",
+            "cuda_max_allocated_mb",
+            "gpu_util_percent",
+            "gpu_memory_util_percent",
+            "gpu_power_w",
+            "gpu_temperature_c",
+        ]
+        summary: Dict[str, object] = {
+            "sample_count": len(self.samples),
+            "samples": self.samples,
+            "background_sample_count": len(self.background_samples),
+            "background_samples": self.background_samples,
+        }
+        sample_groups = {
+            "step": self.samples,
+            "background": self.background_samples,
+        }
+        for prefix, samples in sample_groups.items():
+            if not samples:
+                continue
+            for key in numeric_keys:
+                values = [
+                    float(sample[key])
+                    for sample in samples
+                    if sample.get(key) is not None
+                ]
+                if values:
+                    summary[f"{prefix}_{key}_avg"] = sum(values) / len(values)
+                    summary[f"{prefix}_{key}_max"] = max(values)
+                    summary[f"{prefix}_{key}_min"] = min(values)
+        if not self.samples and not self.background_samples:
+            return summary
+        for key in numeric_keys:
+            values = [
+                float(sample[key])
+                for sample in self.samples
+                if sample.get(key) is not None
+            ]
+            if values:
+                summary[f"{key}_avg"] = sum(values) / len(values)
+                summary[f"{key}_max"] = max(values)
+        return summary
+
+
+class FastTokenClassificationCollator:
+    """Stack already padded token-classification tensors without extra work."""
+
+    def __call__(self, features: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]:
+        return {
+            key: torch.stack([feature[key] for feature in features])
+            for key in features[0].keys()
+        }
+
+
+def augment_training_data(
+    data: List[Dict],
+    partial_count: int,
+    permutation_count: int,
+    special_count: int,
+    max_chars: int,
+    seed: int,
+) -> tuple[List[Dict], Dict]:
+    """Append generated partial/permutation samples without modifying source JSONL."""
+    rng = random.Random(seed)
+    augmented: List[Dict] = []
+    seen = {
+        item.get("filename") or "".join(str(token) for token in item.get("tokens", []))
+        for item in data
+    }
+
+    partial_written = 0
+    if partial_count > 0:
+        candidates: List[Dict] = []
+        attempts = 0
+        max_attempts = max(partial_count * 20, len(data))
+        while len(candidates) < partial_count * 4 and attempts < max_attempts:
+            attempts += 1
+            candidates.extend(build_partial_augmented_item(rng.choice(data), max_chars))
+        rng.shuffle(candidates)
+        for item in candidates:
+            key = item["filename"]
+            if key in seen:
+                continue
+            seen.add(key)
+            augmented.append(item)
+            partial_written += 1
+            if partial_written >= partial_count:
+                break
+
+    permutation_written = 0
+    attempts = 0
+    while permutation_written < permutation_count and attempts < max(permutation_count * 20, 100):
+        attempts += 1
+        item = build_permutation_augmented_item(rng.choice(data), rng, max_chars)
+        if item is None:
+            continue
+        key = item["filename"]
+        if key in seen:
+            continue
+        seen.add(key)
+        augmented.append(item)
+        permutation_written += 1
+
+    special_written = 0
+    attempts = 0
+    while special_written < special_count and attempts < max(special_count * 20, 100):
+        attempts += 1
+        item = build_special_augmented_item(data, rng, max_chars)
+        if item is None:
+            continue
+        key = item["filename"]
+        if key in seen:
+            continue
+        seen.add(key)
+        augmented.append(item)
+        special_written += 1
+
+    meta = {
+        "partial_requested": partial_count,
+        "partial_written": partial_written,
+        "permutation_requested": permutation_count,
+        "permutation_written": permutation_written,
+        "special_requested": special_count,
+        "special_written": special_written,
+        "max_chars": max_chars,
+    }
+    return data + augmented, meta
+
+
 def normalize_field_value(field: str, value) -> Optional[str]:
     if value is None:
         return None
@@ -400,7 +1125,8 @@ def main():
     config = Config()
     if args.data_file is not None:
         config.data_file = args.data_file
-    tokenizer_variant = detect_tokenizer_variant(config.data_file, args.tokenizer, args.vocab_file)
+    training_files = [config.data_file] + list(args.extra_data_file or [])
+    tokenizer_variant = detect_tokenizer_variant_from_files(training_files, args.tokenizer, args.vocab_file)
     if args.save_dir is not None:
         config.save_dir = args.save_dir
     elif tokenizer_variant == "char":
@@ -441,7 +1167,34 @@ def main():
     torch.manual_seed(args.seed)
 
     print("Loading dataset...")
-    all_data = load_jsonl(config.data_file, args.limit_samples)
+    load_started_at = time.perf_counter()
+    all_data, data_sources = load_training_sources(
+        primary_data_file=config.data_file,
+        extra_data_files=list(args.extra_data_file or []),
+        extra_repeat=args.extra_data_repeat,
+        limit=args.limit_samples,
+    )
+    augmentation_metadata = {
+        "partial_requested": 0,
+        "partial_written": 0,
+        "permutation_requested": 0,
+        "permutation_written": 0,
+        "special_requested": 0,
+        "special_written": 0,
+        "max_chars": args.augment_max_chars,
+    }
+    if args.augment_partial_samples or args.augment_permutation_samples or args.augment_special_samples:
+        if tokenizer_variant != "char":
+            raise ValueError("Training-time BIO span augmentation currently requires --tokenizer char.")
+        all_data, augmentation_metadata = augment_training_data(
+            data=all_data,
+            partial_count=args.augment_partial_samples,
+            permutation_count=args.augment_permutation_samples,
+            special_count=args.augment_special_samples,
+            max_chars=args.augment_max_chars,
+            seed=args.seed + 1009,
+        )
+    load_finished_at = time.perf_counter()
     if len(all_data) < 2:
         raise ValueError("Need at least two samples so train/eval split is non-empty.")
     if not args.no_shuffle:
@@ -462,7 +1215,6 @@ def main():
     print(f"  Max sequence length: {config.max_seq_length}")
     if torch.cuda.is_available() and not args.cpu:
         print(f"  CUDA device: {torch.cuda.get_device_name(0)}")
-        print("  Mixed precision: fp16")
 
     # Update config with actual vocab size
     config.vocab_size = tokenizer.vocab_size
@@ -505,53 +1257,96 @@ def main():
     if total_params >= 5_000_000:
         print("WARNING: Model exceeds the historical 5M target; continuing because vocab size is configurable.")
 
+    use_cpu = args.cpu or not torch.cuda.is_available()
     split_idx = int(len(all_data) * config.train_split)
     split_idx = max(1, min(len(all_data) - 1, split_idx))
     train_data = all_data[:split_idx]
     eval_data = all_data[split_idx:]
 
-    # Write split files (temp)
-    train_file = os.path.join(tempfile.gettempdir(), "anime_train.jsonl")
-    eval_file = os.path.join(tempfile.gettempdir(), "anime_eval.jsonl")
-
-    with open(train_file, 'w', encoding='utf-8') as f:
-        for item in train_data:
-            f.write(json.dumps(item, ensure_ascii=False) + '\n')
-
-    with open(eval_file, 'w', encoding='utf-8') as f:
-        for item in eval_data:
-            f.write(json.dumps(item, ensure_ascii=False) + '\n')
-
-    train_dataset = AnimeDataset(
-        data_path=train_file,
-        tokenizer=tokenizer,
-        label2id=config.label2id,
-        max_length=config.max_seq_length,
-    )
-    eval_dataset = AnimeDataset(
-        data_path=eval_file,
-        tokenizer=tokenizer,
-        label2id=config.label2id,
-        max_length=config.max_seq_length,
-    )
+    encode_started_at = time.perf_counter()
+    if args.lazy_dataset:
+        train_dataset = AnimeItemsDataset(
+            data=train_data,
+            tokenizer=tokenizer,
+            label2id=config.label2id,
+            max_length=config.max_seq_length,
+            apply_label_repairs=args.apply_label_repairs,
+        )
+        eval_dataset = AnimeItemsDataset(
+            data=eval_data,
+            tokenizer=tokenizer,
+            label2id=config.label2id,
+            max_length=config.max_seq_length,
+            apply_label_repairs=args.apply_label_repairs,
+        )
+        dataset_mode = "lazy"
+    else:
+        encoded_device = torch.device(args.encoded_dataset_device)
+        if encoded_device.type == "cuda" and use_cpu:
+            raise ValueError("--encoded-dataset-device cuda cannot be used with CPU training.")
+        if encoded_device.type == "cuda" and config.num_workers > 0:
+            raise ValueError("--encoded-dataset-device cuda requires --num-workers 0 to avoid worker duplication.")
+        train_dataset = EncodedAnimeDataset(
+            data=train_data,
+            tokenizer=tokenizer,
+            label2id=config.label2id,
+            max_length=config.max_seq_length,
+            device=encoded_device,
+            apply_label_repairs=args.apply_label_repairs,
+        )
+        eval_dataset = EncodedAnimeDataset(
+            data=eval_data,
+            tokenizer=tokenizer,
+            label2id=config.label2id,
+            max_length=config.max_seq_length,
+            device=encoded_device,
+            apply_label_repairs=args.apply_label_repairs,
+        )
+        dataset_mode = "encoded"
+        if not args.keep_raw_dataset:
+            train_data = []
+            all_data = []
+            gc.collect()
+    encode_finished_at = time.perf_counter()
 
     print(f"  Train samples: {len(train_dataset)}")
     print(f"  Eval samples: {len(eval_dataset)}")
-
-    use_cpu = args.cpu or not torch.cuda.is_available()
-    use_fp16 = not use_cpu
+    print(f"  Dataset mode: {dataset_mode}")
+    print(f"  Load time: {load_finished_at - load_started_at:.2f}s")
+    print(f"  Encode time: {encode_finished_at - encode_started_at:.2f}s")
+
+    use_bf16 = bool(args.bf16 and not use_cpu)
+    use_fp16 = bool((not use_cpu) and not use_bf16 and not args.no_mixed_precision)
+    if use_cpu and args.no_mixed_precision:
+        use_fp16 = False
+    if torch.cuda.is_available() and not use_cpu and args.tf32:
+        torch.backends.cuda.matmul.allow_tf32 = True
+        torch.backends.cudnn.allow_tf32 = True
     print(f"  Device: {'CPU' if use_cpu else 'CUDA'}")
-    eval_save_strategy = "steps" if args.checkpoint_steps else "epoch"
+    if not use_cpu:
+        print(f"  Mixed precision: {'bf16' if use_bf16 else ('fp16' if use_fp16 else 'disabled')}")
+        print(f"  TF32: {'enabled' if args.tf32 else 'disabled'}")
+    eval_save_strategy = "no" if args.no_periodic_eval else ("steps" if args.checkpoint_steps else "epoch")
+    save_strategy = "steps" if args.checkpoint_steps else "epoch"
+
+    dataloader_prefetch_factor = args.prefetch_factor
+    if dataloader_prefetch_factor is None:
+        dataloader_prefetch_factor = 4 if config.num_workers > 0 else None
+    persistent_workers = bool(args.persistent_workers and config.num_workers > 0)
+    dataloader_pin_memory = bool((not use_cpu) and not (not args.lazy_dataset and args.encoded_dataset_device == "cuda"))
+    if args.lazy_dataset and config.num_workers == 0:
+        print("  WARNING: lazy dataset mode is slower with zero workers; consider --num-workers 4+.")
 
     # Training arguments
     training_args = TrainingArguments(
         output_dir=config.save_dir,
         num_train_epochs=config.num_epochs,
+        max_steps=args.max_steps,
         per_device_train_batch_size=config.batch_size,
         per_device_eval_batch_size=config.batch_size,
         eval_strategy=eval_save_strategy,
-        save_strategy=eval_save_strategy,
-        eval_steps=args.checkpoint_steps,
+        save_strategy=save_strategy,
+        eval_steps=args.checkpoint_steps if eval_save_strategy == "steps" else None,
         save_steps=args.checkpoint_steps,
         logging_steps=config.log_interval,
         learning_rate=config.learning_rate,
@@ -561,18 +1356,31 @@ def main():
         use_cpu=use_cpu,
         report_to=["tensorboard"] if args.tensorboard else "none",
         save_total_limit=args.save_total_limit,
-        load_best_model_at_end=True,
+        load_best_model_at_end=not args.no_periodic_eval,
         metric_for_best_model="f1",
         greater_is_better=True,
         dataloader_num_workers=config.num_workers,
-        dataloader_pin_memory=not use_cpu,
+        dataloader_pin_memory=dataloader_pin_memory,
+        dataloader_prefetch_factor=dataloader_prefetch_factor,
+        dataloader_persistent_workers=persistent_workers,
         fp16=use_fp16,
+        bf16=use_bf16,
+        tf32=args.tf32 and not use_cpu,
+        torch_compile=bool(args.torch_compile and not use_cpu),
+        auto_find_batch_size=bool(args.auto_find_batch_size and not use_cpu),
+        include_num_input_tokens_seen=True,
     )
 
     # Data collator
-    data_collator = DataCollatorForTokenClassification(tokenizer)
+    data_collator = FastTokenClassificationCollator()
 
     # Trainer
+    perf_callback = TrainingPerfCallback(
+        batch_size=config.batch_size,
+        sequence_length=config.max_seq_length,
+        log_steps=args.perf_log_steps,
+        sample_interval=args.perf_sample_interval,
+    )
     trainer = Trainer(
         model=model,
         args=training_args,
@@ -580,6 +1388,7 @@ def main():
         eval_dataset=eval_dataset,
         data_collator=data_collator,
         compute_metrics=compute_metrics,
+        callbacks=[perf_callback],
     )
 
     # Train
@@ -606,6 +1415,11 @@ def main():
     metadata = {
         "experiment_name": args.experiment_name,
         "data_file": config.data_file,
+        "data_sources": data_sources,
+        "augmentation": augmentation_metadata,
+        "dataset_mode": dataset_mode,
+        "apply_label_repairs": args.apply_label_repairs,
+        "keep_raw_dataset": args.keep_raw_dataset,
         "tokenizer_variant": tokenizer_variant,
         "vocab_file": vocab_path,
         "vocab_size": tokenizer.vocab_size,
@@ -616,7 +1430,10 @@ def main():
         "intermediate_size": config.intermediate_size,
         "train_samples": len(train_dataset),
         "eval_samples": len(eval_dataset),
+        "load_seconds": load_finished_at - load_started_at,
+        "encode_seconds": encode_finished_at - encode_started_at,
         "epochs": config.num_epochs,
+        "max_steps": args.max_steps,
         "batch_size": config.batch_size,
         "learning_rate": config.learning_rate,
         "warmup_steps": config.warmup_steps,
@@ -625,10 +1442,32 @@ def main():
         "fp16": use_fp16,
         "gradient_accumulation_steps": training_args.gradient_accumulation_steps,
         "dataloader_num_workers": config.num_workers,
+        "dataloader_prefetch_factor": dataloader_prefetch_factor,
+        "dataloader_persistent_workers": persistent_workers,
+        "dataloader_pin_memory": dataloader_pin_memory,
+        "encoded_dataset_device": args.encoded_dataset_device if not args.lazy_dataset else None,
+        "mixed_precision": "bf16" if use_bf16 else ("fp16" if use_fp16 else "none"),
+        "tf32": bool(args.tf32 and not use_cpu),
+        "torch_compile": bool(args.torch_compile and not use_cpu),
+        "auto_find_batch_size": bool(args.auto_find_batch_size and not use_cpu),
+        "perf_log_steps": args.perf_log_steps,
+        "perf_sample_interval": args.perf_sample_interval,
+        "periodic_eval": not args.no_periodic_eval,
     }
     with open(os.path.join(final_save_path, "run_metadata.json"), "w", encoding="utf-8") as f:
         json.dump(metadata, f, ensure_ascii=False, indent=2)
     print(f"Model saved to: {final_save_path}")
+    with open(os.path.join(final_save_path, "perf_metrics.json"), "w", encoding="utf-8") as f:
+        json.dump(perf_callback.summary(), f, ensure_ascii=False, indent=2)
+    train_runtime = None
+    if trainer.state.log_history:
+        for entry in reversed(trainer.state.log_history):
+            if "train_runtime" in entry:
+                train_runtime = entry["train_runtime"]
+                break
+    if train_runtime is not None:
+        print(f"  Train runtime: {train_runtime:.2f}s")
+        print(f"  Total wall time (load+encode+train): {(load_finished_at - load_started_at) + (encode_finished_at - encode_started_at) + train_runtime:.2f}s")
 
     # Final evaluation
     print("\nFinal evaluation:")

docs/maintenance.md

@@ -57,6 +57,7 @@ reports/run_metadata.json
 reports/trainer_eval_metrics.json
 reports/parse_eval_metrics.json
 reports/case_metrics.json
+reports/perf_metrics.json
 reports/benchmark_results.json
 reports/training_lineage.json
 ```
@@ -102,7 +103,7 @@ Copy final files to the repository root:
 把 `final` 文件复制到仓库根目录：
 
 ```powershell
-$final = "checkpoints/dmhy-char-full/final"
+$final = "checkpoints/dmhy-char-aug-fragments-10epoch-hardfocus/final"
 Copy-Item "$final/config.json" . -Force
 Copy-Item "$final/model.safetensors" . -Force
 Copy-Item "$final/tokenizer_config.json" . -Force
@@ -113,6 +114,7 @@ Copy-Item "$final/run_metadata.json" reports/run_metadata.json -Force
 Copy-Item "$final/trainer_eval_metrics.json" reports/trainer_eval_metrics.json -Force
 Copy-Item "$final/parse_eval_metrics.json" reports/parse_eval_metrics.json -Force
 Copy-Item "$final/case_metrics.json" reports/case_metrics.json -Force
+Copy-Item "$final/perf_metrics.json" reports/perf_metrics.json -Force
 Copy-Item datasets/AnimeName/vocab.char.json .\vocab.char.json -Force
 ```
 

docs/training.md

@@ -19,10 +19,10 @@ Recommended GPU configuration:
 
 推荐 GPU 配置：
 
-- RTX 3080 class GPU or better
-- batch size `192` to `256` for full char training
-- `fp16` enabled automatically when CUDA is available
-- `--num-workers 4` or `8` when the local disk can keep up
+- RTX 3080 class GPU or better; current release training used an RTX 5070 Ti
+- batch size `1792` with the encoded dataset path on the 5070 Ti
+- `bf16`/TF32 on Ada/Blackwell-class CUDA devices when available
+- `--num-workers 0` with the encoded dataset path, because samples are pre-encoded into tensors
 
 ## 2. Dataset / 数据集
 
@@ -88,33 +88,49 @@ uv run python -m tools.convert_to_char_dataset `
   --progress 50000
 ```
 
-## 5. Full Training / 全量训练
+## 5. Full Training with Dynamic Augmentation / 动态增强全量训练
 
-Recommended RTX 3080 run:
+Recommended RTX 5070 Ti run:
 
-推荐 RTX 3080 训练命令：
+推荐 RTX 5070 Ti 训练命令：
 
 ```powershell
-uv run python -m anifilebert.train --tokenizer char `
+.\.venv\Scripts\python.exe -m anifilebert.train --tokenizer char `
   --data-file datasets/AnimeName/dmhy_weak_char.jsonl `
-  --vocab-file datasets/AnimeName/vocab.char.json `
-  --save-dir checkpoints/dmhy-char-full `
+  --vocab-file vocab.json `
+  --save-dir checkpoints/dmhy-char-aug-fragments-optimized-10epoch `
   --init-model-dir . `
-  --epochs 2 `
-  --batch-size 256 `
-  --learning-rate 0.00008 `
-  --warmup-steps 300 `
+  --epochs 10 `
+  --batch-size 1792 `
+  --learning-rate 0.00002 `
+  --warmup-steps 500 `
   --max-seq-length 128 `
   --train-split 0.98 `
-  --num-workers 4 `
+  --num-workers 0 `
   --checkpoint-steps 1000 `
   --save-total-limit 3 `
   --parse-eval-limit 2048 `
   --case-eval-file data/parser_regression_cases.json `
-  --seed 52 `
-  --experiment-name dmhy-char-full
+  --augment-partial-samples 200000 `
+  --augment-permutation-samples 400000 `
+  --augment-special-samples 80000 `
+  --bf16 `
+  --no-periodic-eval `
+  --perf-log-steps 200 `
+  --seed 105 `
+  --experiment-name dmhy-char-aug-fragments-optimized-10epoch
 ```
 
+Dynamic augmentation is generated in memory from BIO-labeled source rows and
+does not modify the authoritative DMHY JSONL files. The current release used
+partial/incomplete filename fragments, BIO entity block subsets and
+permutations, title-only/title+season directory-style examples, and standalone
+special fragments such as `Menu01`, `OP02`, `ED E07`, and `NCED03`.
+
+动态增强从已有 BIO 标注行内存生成，不会修改权威 DMHY JSONL。当前发布使用了
+不完整文件名片段、BIO 实体块子集和重排、只有 title 或 title+season 的目录样式
+样本，以及 `Menu01`、`OP02`、`ED E07`、`NCED03` 等 standalone special 片段。
+
 Training outputs:
 
 训练输出：
@@ -125,6 +141,7 @@ Training outputs:
 - `final/trainer_eval_metrics.json`: seqeval metrics / token/entity 指标
 - `final/parse_eval_metrics.json`: held-out parser exact-match / held-out 解析准确率
 - `final/case_metrics.json`: fixed real-world case regression / 固定真实 case 回归
+- `final/perf_metrics.json`: training throughput/GPU telemetry when enabled / 启用时记录训练吞吐和 GPU 采样
 - TensorBoard logs unless `--no-tensorboard` is set / 默认写 TensorBoard
 
 ## 6. Thin Hard-Case Fine-Tuning / 薄层困难样本微调
@@ -139,30 +156,36 @@ been confirmed, fixed in the weak labels, and added to
 ```powershell
 uv run python -m tools.build_repair_focus_dataset `
   --input datasets/AnimeName/dmhy_weak_char.jsonl `
-  --output data/thin_hard_focus_char.jsonl `
-  --context-samples 30000 `
+  --output data/generated/focus_after_10epoch_char.jsonl `
+  --context-samples 100000 `
   --repeat-focus 3 `
-  --repeat-manual 240 `
-  --seed 57
-
-uv run python -m anifilebert.train --tokenizer char `
-  --data-file data/thin_hard_focus_char.jsonl `
-  --vocab-file datasets/AnimeName/vocab.char.json `
-  --save-dir checkpoints/dmhy-char-thin-hardfocus `
-  --init-model-dir . `
+  --repeat-manual 400 `
+  --seed 106
+
+.\.venv\Scripts\python.exe -m anifilebert.train --tokenizer char `
+  --data-file data/generated/focus_after_10epoch_char.jsonl `
+  --vocab-file vocab.json `
+  --save-dir checkpoints/dmhy-char-aug-fragments-10epoch-hardfocus `
+  --init-model-dir checkpoints/dmhy-char-aug-fragments-optimized-10epoch/final `
   --epochs 2 `
-  --batch-size 256 `
-  --learning-rate 0.00004 `
-  --warmup-steps 80 `
+  --batch-size 1792 `
+  --learning-rate 0.000008 `
+  --warmup-steps 50 `
   --max-seq-length 128 `
   --train-split 0.95 `
-  --num-workers 4 `
+  --num-workers 0 `
   --checkpoint-steps 300 `
   --save-total-limit 2 `
-  --parse-eval-limit 1024 `
+  --parse-eval-limit 2048 `
   --case-eval-file data/parser_regression_cases.json `
-  --seed 58 `
-  --experiment-name dmhy-char-thin-hardfocus
+  --augment-partial-samples 30000 `
+  --augment-permutation-samples 60000 `
+  --augment-special-samples 20000 `
+  --bf16 `
+  --no-periodic-eval `
+  --perf-log-steps 50 `
+  --seed 107 `
+  --experiment-name dmhy-char-aug-fragments-10epoch-hardfocus
 ```
 
 The default quality gate is model-led parsing:
@@ -186,7 +209,7 @@ The repository root is the Hugging Face checkpoint surface.
 仓库根目录就是 Hugging Face checkpoint 发布面。
 
 ```powershell
-$final = "checkpoints/dmhy-char-full/final"
+$final = "checkpoints/dmhy-char-aug-fragments-10epoch-hardfocus/final"
 Copy-Item "$final/config.json" . -Force
 Copy-Item "$final/model.safetensors" . -Force
 Copy-Item "$final/tokenizer_config.json" . -Force
@@ -197,6 +220,7 @@ Copy-Item "$final/run_metadata.json" reports/run_metadata.json -Force
 Copy-Item "$final/trainer_eval_metrics.json" reports/trainer_eval_metrics.json -Force
 Copy-Item "$final/parse_eval_metrics.json" reports/parse_eval_metrics.json -Force
 Copy-Item "$final/case_metrics.json" reports/case_metrics.json -Force
+Copy-Item "$final/perf_metrics.json" reports/perf_metrics.json -Force
 Copy-Item datasets/AnimeName/vocab.char.json .\vocab.char.json -Force
 ```
 

exports/anime_filename_parser.metadata.json

@@ -8,5 +8,5 @@
     128,
     15
   ],
-  "max_abs_diff": 4.0531158447265625e-05
+  "max_abs_diff": 1.9073486328125e-05
 }

exports/anime_filename_parser.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:845e01ebdbf4a933610fcfb9f5be2fec2367e9db26d98308d827cbe23817b072
-size 19645986
+oid sha256:5a09d5387e94373cccd22cd821edf0654d537a7897cf8abb04900f48a5ffaccf
+size 19647024

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c5c4e57443ed66fed3812c7c6fe2f14af292a8a32523017930fde2ac93d67ff
+oid sha256:401c62d2359e1030930892fd6be3d8a25abf758f3cb43b4d445562890fe1f2c6
 size 19142604

reports/benchmark_results.json

@@ -11,27 +11,27 @@
   "results": [
     {
       "name": "pytorch",
-      "load_ms": 76.55749993864447,
+      "load_ms": 44.84080011025071,
       "runs": 520,
-      "avg_ms": 16.846879808312785,
-      "p50_ms": 16.207700013183057,
-      "p95_ms": 22.843200032366425,
-      "p99_ms": 28.308318012859665,
-      "min_ms": 11.152399936690927,
-      "max_ms": 34.10990000702441,
-      "throughput_fps": 59.35817263363916
+      "avg_ms": 16.417674036347307,
+      "p50_ms": 14.76569997612387,
+      "p95_ms": 26.30644003511406,
+      "p99_ms": 32.615189072675996,
+      "min_ms": 11.30899996496737,
+      "max_ms": 41.87910002656281,
+      "throughput_fps": 60.909967988527896
     },
     {
       "name": "onnxruntime",
-      "load_ms": 49.74160005804151,
+      "load_ms": 40.69980001077056,
       "runs": 520,
-      "avg_ms": 13.178169615835381,
-      "p50_ms": 12.862899922765791,
-      "p95_ms": 16.696884995326396,
-      "p99_ms": 18.06362595874816,
-      "min_ms": 9.811799973249435,
-      "max_ms": 20.784800057299435,
-      "throughput_fps": 75.88307247148819
+      "avg_ms": 11.606730768779435,
+      "p50_ms": 11.42695004818961,
+      "p95_ms": 13.516889995662494,
+      "p99_ms": 15.196251904126225,
+      "min_ms": 9.510300005786121,
+      "max_ms": 19.771000021137297,
+      "throughput_fps": 86.15690498222524
     }
   ]
-}
+}

reports/parse_eval_metrics.json

@@ -3,184 +3,212 @@
   "modes": {
     "model_only": {
       "constrain_bio": false,
-      "sample_count": 1024,
+      "sample_count": 2048,
       "field_accuracy": {
-        "group": 1.0,
-        "title": 0.9970703125,
-        "season": 1.0,
-        "episode": 1.0,
+        "group": 0.98583984375,
+        "title": 0.97119140625,
+        "season": 0.99609375,
+        "episode": 0.990234375,
         "resolution": 1.0,
-        "source": 0.9990234375,
-        "special": 0.994140625
+        "source": 0.990234375,
+        "special": 0.9775390625
       },
       "field_correct": {
-        "group": 1024,
-        "title": 1021,
-        "season": 1024,
-        "episode": 1024,
-        "resolution": 1024,
-        "source": 1023,
-        "special": 1018
+        "group": 2019,
+        "title": 1989,
+        "season": 2040,
+        "episode": 2028,
+        "resolution": 2048,
+        "source": 2028,
+        "special": 2002
       },
       "field_total": {
-        "group": 1024,
-        "title": 1024,
-        "season": 1024,
-        "episode": 1024,
-        "resolution": 1024,
-        "source": 1024,
-        "special": 1024
+        "group": 2048,
+        "title": 2048,
+        "season": 2048,
+        "episode": 2048,
+        "resolution": 2048,
+        "source": 2048,
+        "special": 2048
       },
-      "full_match_accuracy": 0.990234375,
-      "full_match_correct": 1014,
-      "full_match_total": 1024,
+      "full_match_accuracy": 0.95068359375,
+      "full_match_correct": 1947,
+      "full_match_total": 2048,
       "failures": [
         {
-          "filename": "BD Menu Vol.05",
+          "filename": "01; - flac",
           "errors": {
             "title": {
-              "gold": "menu",
-              "pred": "men"
+              "gold": "01;",
+              "pred": ";"
+            },
+            "episode": {
+              "gold": null,
+              "pred": "1"
             }
           },
           "gold": {
             "group": null,
-            "title": "Menu",
+            "title": "01;",
             "season": null,
             "episode": null,
             "resolution": null,
-            "source": "BD",
+            "source": "flac",
             "special": null
           },
           "pred": {
             "group": null,
-            "title": "Men",
+            "title": ";",
             "season": null,
-            "episode": null,
+            "episode": 1,
             "resolution": null,
-            "source": "BD",
+            "source": "flac",
             "special": null
           }
         },
         {
-          "filename": "[YYDM-11FANS][Shaman King][43][DVDRip][720P][X264-10bit_AACx2][ED490609]",
+          "filename": "[VCB-Studio] Durarara!!×2 Shou [IV][Ma10p_1080p][x265_aac]",
           "errors": {
+            "title": {
+              "gold": "durarara!!×2 shou iv",
+              "pred": "durarara!!×2 shou"
+            },
             "special": {
-              "gold": "ed490609",
-              "pred": "ed"
+              "gold": null,
+              "pred": "iv"
             }
           },
           "gold": {
-            "group": "YYDM-11FANS",
-            "title": "Shaman King",
+            "group": "VCB-Studio",
+            "title": "Durarara!!×2 Shou IV",
             "season": null,
-            "episode": 43,
-            "resolution": "720P",
-            "source": "DVDRip",
-            "special": "ED490609"
+            "episode": null,
+            "resolution": "1080p",
+            "source": "x265-aac",
+            "special": null
           },
           "pred": {
-            "group": "YYDM-11FANS",
-            "title": "Shaman King",
+            "group": "VCB-Studio",
+            "title": "Durarara!!×2 Shou",
             "season": null,
-            "episode": 43,
-            "resolution": "720P",
-            "source": "DVDRip",
-            "special": "ED"
+            "episode": null,
+            "resolution": "1080p",
+            "source": "x265-aac",
+            "special": "IV"
           }
         },
         {
-          "filename": "[UHA-WINGS&VCB-Studio] Karakai Jouzu no Takagi-san 2 [Binaural Sound EP09][Ma10p_1080p][x265_flac]",
+          "filename": "AC3 Chap - BD Menu16",
           "errors": {
-            "title": {
-              "gold": "uha-wings&vcb-studio karakai jouzu no takagi san 2 binaural sound ep",
-              "pred": "uha-wings&vcb-studio karakai jouzu no takagi san 2 binaural sound"
+            "source": {
+              "gold": "bd",
+              "pred": null
+            },
+            "special": {
+              "gold": "menu16",
+              "pred": "bd menu16"
             }
           },
           "gold": {
             "group": null,
-            "title": "UHA-WINGS&VCB-Studio Karakai Jouzu no Takagi san 2 Binaural Sound EP",
+            "title": "AC3 Chap",
             "season": null,
-            "episode": 9,
-            "resolution": "1080p",
-            "source": "x265-flac",
-            "special": null
+            "episode": null,
+            "resolution": null,
+            "source": "BD",
+            "special": "Menu16"
           },
           "pred": {
             "group": null,
-            "title": "UHA-WINGS&VCB-Studio Karakai Jouzu no Takagi san 2 Binaural Sound",
+            "title": "AC3 Chap",
             "season": null,
-            "episode": 9,
-            "resolution": "1080p",
-            "source": "x265-flac",
-            "special": null
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "BD Menu16"
           }
         },
         {
-          "filename": "終末なにしてますか？忙しいですか？救ってもらっていいですか？ OP2 「DEAREST DROP」",
+          "filename": "Puella Magi Madoka Magica - BD Menu14",
           "errors": {
+            "source": {
+              "gold": "bd",
+              "pred": null
+            },
             "special": {
-              "gold": "op2",
-              "pred": "p"
+              "gold": "menu14",
+              "pred": "bd menu14"
             }
           },
           "gold": {
             "group": null,
-            "title": "終末なにしてますか？忙しいですか？救ってもらっていいですか？",
+            "title": "Puella Magi Madoka Magica",
             "season": null,
             "episode": null,
             "resolution": null,
-            "source": null,
-            "special": "OP2"
+            "source": "BD",
+            "special": "Menu14"
           },
           "pred": {
             "group": null,
-            "title": "終末なにしてますか？忙しいですか？救ってもらっていいですか？",
+            "title": "Puella Magi Madoka Magica",
             "season": null,
             "episode": null,
             "resolution": null,
             "source": null,
-            "special": "P"
+            "special": "BD Menu14"
           }
         },
         {
-          "filename": "[QTS] CITY HUNTER TV 3rd & '91 Series BD-BOX Eizou Tokuten - City Hunter 3 Housouchuu CM2 30sec (BD Hi10P 960x720 AAC)",
+          "filename": "Kirion Hikaru no Go 72 [960x720]",
           "errors": {
-            "special": {
-              "gold": "cm2",
-              "pred": "3"
+            "group": {
+              "gold": "kirion",
+              "pred": null
+            },
+            "title": {
+              "gold": "hikaru no go",
+              "pred": "kirion hikaru no go 72"
+            },
+            "episode": {
+              "gold": "72",
+              "pred": null
             }
           },
           "gold": {
-            "group": "QTS",
-            "title": "CITY HUNTER TV 3rd & '91 Series",
+            "group": "Kirion",
+            "title": "Hikaru no Go",
             "season": null,
-            "episode": null,
+            "episode": 72,
             "resolution": "960x720",
-            "source": "BD",
-            "special": "CM2"
+            "source": null,
+            "special": null
           },
           "pred": {
-            "group": "QTS",
-            "title": "CITY HUNTER TV 3rd & '91 Series",
+            "group": null,
+            "title": "Kirion Hikaru no Go 72",
             "season": null,
             "episode": null,
             "resolution": "960x720",
-            "source": "BD",
-            "special": "3"
+            "source": null,
+            "special": null
           }
         },
         {
-          "filename": "22話「最凶恶的“他”Part 1／最凶恶的“他”Part 2」",
+          "filename": "葬送的芙莉莲 - 喵萌奶茶屋",
           "errors": {
+            "group": {
+              "gold": "喵萌奶茶屋",
+              "pred": null
+            },
             "title": {
-              "gold": "22話「最凶恶的“他”part 1／最凶恶的“他”part 2」",
-              "pred": "2 話「最凶恶的“他”part 1／最凶恶的“他”part 2」"
+              "gold": "葬送的芙莉莲",
+              "pred": "葬送的芙莉莲 - 喵萌奶茶屋"
             }
           },
           "gold": {
-            "group": null,
-            "title": "22話「最凶恶的“他”Part 1／最凶恶的“他”Part 2」",
+            "group": "喵萌奶茶屋",
+            "title": "葬送的芙莉莲",
             "season": null,
             "episode": null,
             "resolution": null,
@@ -189,7 +217,7 @@
           },
           "pred": {
             "group": null,
-            "title": "2 話「最凶恶的“他”Part 1／最凶恶的“他”Part 2」",
+            "title": "葬送的芙莉莲 - 喵萌奶茶屋",
             "season": null,
             "episode": null,
             "resolution": null,
@@ -198,339 +226,1080 @@
           }
         },
         {
-          "filename": "[Judgment] Kareshi Kanojo no Jijou - NCED19.0 [6C5BD6E2]",
+          "filename": "[VCB-Studio] Taimadou Gakuen 35 Shiken Shoutai [IV][Ma10p_1080p][x265_aac]",
+          "errors": {
+            "title": {
+              "gold": "taimadou gakuen 35 shiken shoutai iv",
+              "pred": "taimadou gakuen 35 shiken shoutai v"
+            }
+          },
+          "gold": {
+            "group": "VCB-Studio",
+            "title": "Taimadou Gakuen 35 Shiken Shoutai IV",
+            "season": null,
+            "episode": null,
+            "resolution": "1080p",
+            "source": "x265-aac",
+            "special": null
+          },
+          "pred": {
+            "group": "VCB-Studio",
+            "title": "Taimadou Gakuen 35 Shiken Shoutai V",
+            "season": null,
+            "episode": null,
+            "resolution": "1080p",
+            "source": "x265-aac",
+            "special": null
+          }
+        },
+        {
+          "filename": "Dr.Slump.Arale-chan.097 - BD Menu14",
           "errors": {
             "source": {
               "gold": "bd",
               "pred": null
+            },
+            "special": {
+              "gold": "menu14",
+              "pred": "bd menu14"
             }
           },
           "gold": {
-            "group": "Judgment",
-            "title": "Kareshi Kanojo no Jijou",
+            "group": null,
+            "title": "Dr.Slump.Arale-chan.097",
             "season": null,
             "episode": null,
             "resolution": null,
             "source": "BD",
-            "special": "NCED19"
+            "special": "Menu14"
           },
           "pred": {
-            "group": "Judgment",
-            "title": "Kareshi Kanojo no Jijou",
+            "group": null,
+            "title": "Dr.Slump.Arale-chan.097",
             "season": null,
             "episode": null,
             "resolution": null,
             "source": null,
-            "special": "NCED19"
+            "special": "BD Menu14"
           }
         },
         {
-          "filename": "[QTS] OVA QUIZ MAGIC ACADEMY The Original Animation OMAKE Eizou - EP1 OP + EP2 OPED (BD Hi10P 1920x1080 WV)",
+          "filename": "Shiroi Suna no Aquatope - BD Menu05",
           "errors": {
+            "source": {
+              "gold": "bd",
+              "pred": null
+            },
             "special": {
-              "gold": "op",
-              "pred": "d"
+              "gold": "menu05",
+              "pred": "bd menu05"
             }
           },
           "gold": {
-            "group": "QTS",
-            "title": "QUIZ MAGIC ACADEMY The Original Animation OMAKE Eizou",
+            "group": null,
+            "title": "Shiroi Suna no Aquatope",
             "season": null,
-            "episode": 1,
-            "resolution": "1920x1080",
+            "episode": null,
+            "resolution": null,
             "source": "BD",
-            "special": "OP"
+            "special": "Menu05"
           },
           "pred": {
-            "group": "QTS",
-            "title": "QUIZ MAGIC ACADEMY The Original Animation OMAKE Eizou",
+            "group": null,
+            "title": "Shiroi Suna no Aquatope",
             "season": null,
-            "episode": 1,
-            "resolution": "1920x1080",
-            "source": "BD",
-            "special": "D"
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "BD Menu05"
+          }
+        },
+        {
+          "filename": "[VCB-Studio] Mob Psycho 100 II [NCOP_OVA][Ma10p_1080p][x265_flac]",
+          "errors": {
+            "season": {
+              "gold": null,
+              "pred": "2"
+            }
+          },
+          "gold": {
+            "group": "VCB-Studio",
+            "title": "Mob Psycho",
+            "season": null,
+            "episode": 100,
+            "resolution": "1080p",
+            "source": "x265-flac",
+            "special": "OVA"
+          },
+          "pred": {
+            "group": "VCB-Studio",
+            "title": "Mob Psycho",
+            "season": 2,
+            "episode": 100,
+            "resolution": "1080p",
+            "source": "x265-flac",
+            "special": "OVA"
           }
         },
         {
-          "filename": "[Moozzi2] Ojamajo Doremi Movie - 01 [ Sharp ] (BD 1920x1032 x.264 Flac)",
+          "filename": "アニメCM宣伝1",
           "errors": {
+            "group": {
+              "gold": null,
+              "pred": "ア"
+            },
+            "title": {
+              "gold": "アニメ",
+              "pred": "ニ 宣伝"
+            },
             "special": {
-              "gold": "movie",
-              "pred": "0"
+              "gold": "cm",
+              "pred": "1"
             }
           },
           "gold": {
-            "group": "Moozzi2",
-            "title": "Ojamajo Doremi",
+            "group": null,
+            "title": "アニメ",
             "season": null,
-            "episode": 1,
-            "resolution": "1920x1032",
-            "source": "BD",
-            "special": "Movie"
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "CM"
           },
           "pred": {
-            "group": "Moozzi2",
-            "title": "Ojamajo Doremi",
+            "group": "ア",
+            "title": "ニ 宣伝",
             "season": null,
-            "episode": 1,
-            "resolution": "1920x1032",
-            "source": "BD",
-            "special": "0"
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "1"
           }
         },
         {
-          "filename": "[Nekomoe kissaten&VCB-Studio] THE IDOLM@STER CINDERELLA GIRLS U149 [NCED07 お願い!シンデレラ][Ma10p_1080p][x265_flac]",
+          "filename": "960x720 SLAM DUNK LoliHouse",
           "errors": {
-            "special": {
-              "gold": "nced",
-              "pred": "7"
+            "group": {
+              "gold": "lolihouse",
+              "pred": "ihouse"
+            },
+            "title": {
+              "gold": "slam dunk",
+              "pred": "slam dunk lol"
             }
           },
           "gold": {
-            "group": "Nekomoe kissaten&VCB-Studio",
-            "title": "THE IDOLM@STER CINDERELLA GIRLS U",
+            "group": "LoliHouse",
+            "title": "SLAM DUNK",
             "season": null,
-            "episode": 149,
-            "resolution": "1080p",
-            "source": "x265-flac",
-            "special": "NCED"
+            "episode": null,
+            "resolution": "960x720",
+            "source": null,
+            "special": null
           },
           "pred": {
-            "group": "Nekomoe kissaten&VCB-Studio",
-            "title": "THE IDOLM@STER CINDERELLA GIRLS U",
+            "group": "iHouse",
+            "title": "SLAM DUNK Lol",
             "season": null,
-            "episode": 149,
-            "resolution": "1080p",
-            "source": "x265-flac",
-            "special": "7"
+            "episode": null,
+            "resolution": "960x720",
+            "source": null,
+            "special": null
           }
-        }
-      ]
-    },
-    "normalized_only": {
-      "constrain_bio": true,
-      "sample_count": 1024,
-      "field_accuracy": {
-        "group": 1.0,
-        "title": 0.9970703125,
-        "season": 1.0,
-        "episode": 1.0,
-        "resolution": 1.0,
-        "source": 0.9990234375,
-        "special": 0.9970703125
-      },
-      "field_correct": {
-        "group": 1024,
-        "title": 1021,
-        "season": 1024,
-        "episode": 1024,
-        "resolution": 1024,
-        "source": 1023,
-        "special": 1021
-      },
-      "field_total": {
-        "group": 1024,
-        "title": 1024,
-        "season": 1024,
-        "episode": 1024,
-        "resolution": 1024,
-        "source": 1024,
-        "special": 1024
-      },
-      "full_match_accuracy": 0.9931640625,
-      "full_match_correct": 1017,
-      "full_match_total": 1024,
-      "failures": [
+        },
         {
-          "filename": "BD Menu Vol.05",
+          "filename": "GM-Team [GB] 4K 逆天邪神 04",
           "errors": {
             "title": {
-              "gold": "menu",
-              "pred": "men"
+              "gold": "逆天邪神",
+              "pred": "逆天邪神 04"
+            },
+            "episode": {
+              "gold": "4",
+              "pred": null
+            }
+          },
+          "gold": {
+            "group": "GM-Team",
+            "title": "逆天邪神",
+            "season": null,
+            "episode": 4,
+            "resolution": "4K",
+            "source": "GB",
+            "special": null
+          },
+          "pred": {
+            "group": "GM-Team",
+            "title": "逆天邪神 04",
+            "season": null,
+            "episode": null,
+            "resolution": "4K",
+            "source": "GB",
+            "special": null
+          }
+        },
+        {
+          "filename": "CM 15 - BD - CITY HUNTER TV 3rd & '91 Series",
+          "errors": {
+            "source": {
+              "gold": "bd",
+              "pred": "b"
+            },
+            "special": {
+              "gold": "cm 15",
+              "pred": "d"
             }
           },
           "gold": {
             "group": null,
-            "title": "Menu",
+            "title": "CITY HUNTER TV 3rd & '91 Series",
             "season": null,
             "episode": null,
             "resolution": null,
             "source": "BD",
-            "special": null
+            "special": "CM 15"
           },
           "pred": {
             "group": null,
-            "title": "Men",
+            "title": "CITY HUNTER TV 3rd & '91 Series",
             "season": null,
             "episode": null,
             "resolution": null,
-            "source": "BD",
-            "special": null
+            "source": "B",
+            "special": "D"
           }
         },
         {
-          "filename": "[YYDM-11FANS][Shaman King][43][DVDRip][720P][X264-10bit_AACx2][ED490609]",
+          "filename": "12 - HEVC",
           "errors": {
+            "episode": {
+              "gold": null,
+              "pred": "12"
+            },
             "special": {
-              "gold": "ed490609",
-              "pred": "ed"
+              "gold": "12",
+              "pred": null
             }
           },
           "gold": {
-            "group": "YYDM-11FANS",
-            "title": "Shaman King",
+            "group": null,
+            "title": null,
             "season": null,
-            "episode": 43,
-            "resolution": "720P",
-            "source": "DVDRip",
-            "special": "ED490609"
+            "episode": null,
+            "resolution": null,
+            "source": "HEVC",
+            "special": "12"
           },
           "pred": {
-            "group": "YYDM-11FANS",
-            "title": "Shaman King",
+            "group": null,
+            "title": null,
             "season": null,
-            "episode": 43,
-            "resolution": "720P",
-            "source": "DVDRip",
-            "special": "ED"
+            "episode": 12,
+            "resolution": null,
+            "source": "HEVC",
+            "special": null
           }
         },
         {
-          "filename": "[UHA-WINGS&VCB-Studio] Karakai Jouzu no Takagi-san 2 [Binaural Sound EP09][Ma10p_1080p][x265_flac]",
+          "filename": "07][檢索：魔法姊妹露露特莉莉][CHT&JPN",
           "errors": {
             "title": {
-              "gold": "uha-wings&vcb-studio karakai jouzu no takagi san 2 binaural sound ep",
-              "pred": "uha-wings&vcb-studio karakai jouzu no takagi san 2 binaural sound"
+              "gold": null,
+              "pred": "檢索：魔法姊妹露露特莉莉"
+            },
+            "special": {
+              "gold": "檢索：魔法姊妹露露特莉莉",
+              "pred": null
             }
           },
           "gold": {
             "group": null,
-            "title": "UHA-WINGS&VCB-Studio Karakai Jouzu no Takagi san 2 Binaural Sound EP",
+            "title": null,
             "season": null,
-            "episode": 9,
-            "resolution": "1080p",
-            "source": "x265-flac",
-            "special": null
+            "episode": 7,
+            "resolution": null,
+            "source": "CHT&JPN",
+            "special": "檢索：魔法姊妹露露特莉莉"
           },
           "pred": {
             "group": null,
-            "title": "UHA-WINGS&VCB-Studio Karakai Jouzu no Takagi san 2 Binaural Sound",
+            "title": "檢索：魔法姊妹露露特莉莉",
             "season": null,
-            "episode": 9,
-            "resolution": "1080p",
-            "source": "x265-flac",
+            "episode": 7,
+            "resolution": null,
+            "source": "CHT&JPN",
             "special": null
           }
         },
         {
-          "filename": "[QTS] CITY HUNTER TV 3rd & '91 Series BD-BOX Eizou Tokuten - City Hunter 3 Housouchuu CM2 30sec (BD Hi10P 960x720 AAC)",
+          "filename": "[12]_1080P_Baha_[ANi]_29 歲單身中堅冒險家的日常",
           "errors": {
-            "special": {
-              "gold": "cm2",
-              "pred": "cm2 3"
+            "title": {
+              "gold": "29 歲單身中堅冒險家的日常",
+              "pred": "9 歲單身中堅冒險家的日常"
             }
           },
           "gold": {
-            "group": "QTS",
-            "title": "CITY HUNTER TV 3rd & '91 Series",
+            "group": "ANi",
+            "title": "29 歲單身中堅冒險家的日常",
+            "season": null,
+            "episode": 12,
+            "resolution": "1080P",
+            "source": "Baha",
+            "special": null
+          },
+          "pred": {
+            "group": "ANi",
+            "title": "9 歲單身中堅冒險家的日常",
+            "season": null,
+            "episode": 12,
+            "resolution": "1080P",
+            "source": "Baha",
+            "special": null
+          }
+        },
+        {
+          "filename": "[CM_03] Onimonogatari SFEO-Raws",
+          "errors": {
+            "group": {
+              "gold": "sfeo-raws",
+              "pred": "feo-raws"
+            },
+            "title": {
+              "gold": "onimonogatari",
+              "pred": "onimonogatari s"
+            }
+          },
+          "gold": {
+            "group": "SFEO-Raws",
+            "title": "Onimonogatari",
             "season": null,
             "episode": null,
-            "resolution": "960x720",
-            "source": "BD",
-            "special": "CM2"
+            "resolution": null,
+            "source": null,
+            "special": "CM_03"
           },
           "pred": {
-            "group": "QTS",
-            "title": "CITY HUNTER TV 3rd & '91 Series",
+            "group": "FEO-Raws",
+            "title": "Onimonogatari S",
             "season": null,
             "episode": null,
-            "resolution": "960x720",
-            "source": "BD",
-            "special": "CM2 3"
+            "resolution": null,
+            "source": null,
+            "special": "CM_03"
+          }
+        },
+        {
+          "filename": "16][S2][[DBD-Raws]",
+          "errors": {
+            "episode": {
+              "gold": null,
+              "pred": "16"
+            },
+            "special": {
+              "gold": "16",
+              "pred": null
+            }
+          },
+          "gold": {
+            "group": "DBD-Raws",
+            "title": null,
+            "season": 2,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "16"
+          },
+          "pred": {
+            "group": "DBD-Raws",
+            "title": null,
+            "season": 2,
+            "episode": 16,
+            "resolution": null,
+            "source": null,
+            "special": null
           }
         },
         {
-          "filename": "22話「最凶恶的“他”Part 1／最凶恶的“他”Part 2」",
+          "filename": "Captain.Tsubasa.Road.to.2002.Intro1.BGSubs.TVRip.XviD-TBO",
           "errors": {
+            "group": {
+              "gold": null,
+              "pred": "gsu"
+            },
             "title": {
-              "gold": "22話「最凶恶的“他”part 1／最凶恶的“他”part 2」",
-              "pred": "22 話「最凶恶的“他”part 1／最凶恶的“他”part 2」"
+              "gold": "captain.tsubasa.road.to intro1.bgsubs",
+              "pred": "captain.tsubasa.road.to intro1.b b id o"
             }
           },
           "gold": {
             "group": null,
-            "title": "22話「最凶恶的“他”Part 1／最凶恶的“他”Part 2」",
+            "title": "Captain.Tsubasa.Road.to Intro1.BGSubs",
             "season": null,
             "episode": null,
             "resolution": null,
-            "source": null,
+            "source": "TVRip",
             "special": null
           },
           "pred": {
-            "group": null,
-            "title": "22 話「最凶恶的“他”Part 1／最凶恶的“他”Part 2」",
+            "group": "GSu",
+            "title": "Captain.Tsubasa.Road.to Intro1.B b iD O",
             "season": null,
             "episode": null,
             "resolution": null,
-            "source": null,
+            "source": "TVRip",
             "special": null
           }
-        },
+        }
+      ]
+    },
+    "normalized_only": {
+      "constrain_bio": true,
+      "sample_count": 2048,
+      "field_accuracy": {
+        "group": 0.990234375,
+        "title": 0.978515625,
+        "season": 0.99755859375,
+        "episode": 0.9912109375,
+        "resolution": 1.0,
+        "source": 0.990234375,
+        "special": 0.9794921875
+      },
+      "field_correct": {
+        "group": 2028,
+        "title": 2004,
+        "season": 2043,
+        "episode": 2030,
+        "resolution": 2048,
+        "source": 2028,
+        "special": 2006
+      },
+      "field_total": {
+        "group": 2048,
+        "title": 2048,
+        "season": 2048,
+        "episode": 2048,
+        "resolution": 2048,
+        "source": 2048,
+        "special": 2048
+      },
+      "full_match_accuracy": 0.9599609375,
+      "full_match_correct": 1966,
+      "full_match_total": 2048,
+      "failures": [
         {
-          "filename": "[Judgment] Kareshi Kanojo no Jijou - NCED19.0 [6C5BD6E2]",
+          "filename": "01; - flac",
           "errors": {
-            "source": {
-              "gold": "bd",
+            "title": {
+              "gold": "01;",
               "pred": null
+            },
+            "episode": {
+              "gold": null,
+              "pred": "1"
             }
           },
           "gold": {
-            "group": "Judgment",
-            "title": "Kareshi Kanojo no Jijou",
+            "group": null,
+            "title": "01;",
             "season": null,
             "episode": null,
             "resolution": null,
-            "source": "BD",
-            "special": "NCED19"
+            "source": "flac",
+            "special": null
           },
           "pred": {
-            "group": "Judgment",
-            "title": "Kareshi Kanojo no Jijou",
+            "group": null,
+            "title": null,
             "season": null,
-            "episode": null,
+            "episode": 1,
             "resolution": null,
-            "source": null,
-            "special": "NCED19"
+            "source": "flac",
+            "special": null
           }
         },
         {
-          "filename": "[Nekomoe kissaten&VCB-Studio] THE IDOLM@STER CINDERELLA GIRLS U149 [NCED07 お願い!シンデレラ][Ma10p_1080p][x265_flac]",
+          "filename": "[VCB-Studio] Durarara!!×2 Shou [IV][Ma10p_1080p][x265_aac]",
           "errors": {
+            "title": {
+              "gold": "durarara!!×2 shou iv",
+              "pred": "durarara!!×2 shou"
+            },
             "special": {
-              "gold": "nced",
-              "pred": "nced07"
+              "gold": null,
+              "pred": "iv"
             }
           },
           "gold": {
-            "group": "Nekomoe kissaten&VCB-Studio",
-            "title": "THE IDOLM@STER CINDERELLA GIRLS U",
+            "group": "VCB-Studio",
+            "title": "Durarara!!×2 Shou IV",
             "season": null,
-            "episode": 149,
+            "episode": null,
             "resolution": "1080p",
-            "source": "x265-flac",
-            "special": "NCED"
+            "source": "x265-aac",
+            "special": null
           },
           "pred": {
-            "group": "Nekomoe kissaten&VCB-Studio",
-            "title": "THE IDOLM@STER CINDERELLA GIRLS U",
+            "group": "VCB-Studio",
+            "title": "Durarara!!×2 Shou",
             "season": null,
-            "episode": 149,
+            "episode": null,
             "resolution": "1080p",
-            "source": "x265-flac",
-            "special": "NCED07"
+            "source": "x265-aac",
+            "special": "IV"
+          }
+        },
+        {
+          "filename": "AC3 Chap - BD Menu16",
+          "errors": {
+            "source": {
+              "gold": "bd",
+              "pred": null
+            },
+            "special": {
+              "gold": "menu16",
+              "pred": "bd menu16"
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": "AC3 Chap",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": "BD",
+            "special": "Menu16"
+          },
+          "pred": {
+            "group": null,
+            "title": "AC3 Chap",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "BD Menu16"
+          }
+        },
+        {
+          "filename": "Puella Magi Madoka Magica - BD Menu14",
+          "errors": {
+            "source": {
+              "gold": "bd",
+              "pred": null
+            },
+            "special": {
+              "gold": "menu14",
+              "pred": "bd menu14"
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": "Puella Magi Madoka Magica",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": "BD",
+            "special": "Menu14"
+          },
+          "pred": {
+            "group": null,
+            "title": "Puella Magi Madoka Magica",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "BD Menu14"
+          }
+        },
+        {
+          "filename": "Kirion Hikaru no Go 72 [960x720]",
+          "errors": {
+            "group": {
+              "gold": "kirion",
+              "pred": null
+            },
+            "title": {
+              "gold": "hikaru no go",
+              "pred": "kirion hikaru no go 72"
+            },
+            "episode": {
+              "gold": "72",
+              "pred": null
+            }
+          },
+          "gold": {
+            "group": "Kirion",
+            "title": "Hikaru no Go",
+            "season": null,
+            "episode": 72,
+            "resolution": "960x720",
+            "source": null,
+            "special": null
+          },
+          "pred": {
+            "group": null,
+            "title": "Kirion Hikaru no Go 72",
+            "season": null,
+            "episode": null,
+            "resolution": "960x720",
+            "source": null,
+            "special": null
+          }
+        },
+        {
+          "filename": "葬送的芙莉莲 - 喵萌奶茶屋",
+          "errors": {
+            "group": {
+              "gold": "喵萌奶茶屋",
+              "pred": null
+            },
+            "title": {
+              "gold": "葬送的芙莉莲",
+              "pred": "葬送的芙莉莲 - 喵萌奶茶屋"
+            }
+          },
+          "gold": {
+            "group": "喵萌奶茶屋",
+            "title": "葬送的芙莉莲",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": null
+          },
+          "pred": {
+            "group": null,
+            "title": "葬送的芙莉莲 - 喵萌奶茶屋",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": null
+          }
+        },
+        {
+          "filename": "[VCB-Studio] Taimadou Gakuen 35 Shiken Shoutai [IV][Ma10p_1080p][x265_aac]",
+          "errors": {
+            "title": {
+              "gold": "taimadou gakuen 35 shiken shoutai iv",
+              "pred": "taimadou gakuen 35 shiken shoutai"
+            },
+            "special": {
+              "gold": null,
+              "pred": "iv"
+            }
+          },
+          "gold": {
+            "group": "VCB-Studio",
+            "title": "Taimadou Gakuen 35 Shiken Shoutai IV",
+            "season": null,
+            "episode": null,
+            "resolution": "1080p",
+            "source": "x265-aac",
+            "special": null
+          },
+          "pred": {
+            "group": "VCB-Studio",
+            "title": "Taimadou Gakuen 35 Shiken Shoutai",
+            "season": null,
+            "episode": null,
+            "resolution": "1080p",
+            "source": "x265-aac",
+            "special": "IV"
+          }
+        },
+        {
+          "filename": "Dr.Slump.Arale-chan.097 - BD Menu14",
+          "errors": {
+            "source": {
+              "gold": "bd",
+              "pred": null
+            },
+            "special": {
+              "gold": "menu14",
+              "pred": "bd menu14"
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": "Dr.Slump.Arale-chan.097",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": "BD",
+            "special": "Menu14"
+          },
+          "pred": {
+            "group": null,
+            "title": "Dr.Slump.Arale-chan.097",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "BD Menu14"
+          }
+        },
+        {
+          "filename": "Shiroi Suna no Aquatope - BD Menu05",
+          "errors": {
+            "source": {
+              "gold": "bd",
+              "pred": null
+            },
+            "special": {
+              "gold": "menu05",
+              "pred": "bd menu05"
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": "Shiroi Suna no Aquatope",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": "BD",
+            "special": "Menu05"
+          },
+          "pred": {
+            "group": null,
+            "title": "Shiroi Suna no Aquatope",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "BD Menu05"
+          }
+        },
+        {
+          "filename": "[VCB-Studio] Mob Psycho 100 II [NCOP_OVA][Ma10p_1080p][x265_flac]",
+          "errors": {
+            "season": {
+              "gold": null,
+              "pred": "2"
+            }
+          },
+          "gold": {
+            "group": "VCB-Studio",
+            "title": "Mob Psycho",
+            "season": null,
+            "episode": 100,
+            "resolution": "1080p",
+            "source": "x265-flac",
+            "special": "OVA"
+          },
+          "pred": {
+            "group": "VCB-Studio",
+            "title": "Mob Psycho",
+            "season": 2,
+            "episode": 100,
+            "resolution": "1080p",
+            "source": "x265-flac",
+            "special": "OVA"
+          }
+        },
+        {
+          "filename": "アニメCM宣伝1",
+          "errors": {
+            "title": {
+              "gold": "アニメ",
+              "pred": "アニメcm宣伝1"
+            },
+            "special": {
+              "gold": "cm",
+              "pred": null
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": "アニメ",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "CM"
+          },
+          "pred": {
+            "group": null,
+            "title": "アニメCM宣伝1",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": null
+          }
+        },
+        {
+          "filename": "GM-Team [GB] 4K 逆天邪神 04",
+          "errors": {
+            "title": {
+              "gold": "逆天邪神",
+              "pred": "逆天邪神 04"
+            },
+            "episode": {
+              "gold": "4",
+              "pred": null
+            }
+          },
+          "gold": {
+            "group": "GM-Team",
+            "title": "逆天邪神",
+            "season": null,
+            "episode": 4,
+            "resolution": "4K",
+            "source": "GB",
+            "special": null
+          },
+          "pred": {
+            "group": "GM-Team",
+            "title": "逆天邪神 04",
+            "season": null,
+            "episode": null,
+            "resolution": "4K",
+            "source": "GB",
+            "special": null
+          }
+        },
+        {
+          "filename": "CM 15 - BD - CITY HUNTER TV 3rd & '91 Series",
+          "errors": {
+            "source": {
+              "gold": "bd",
+              "pred": null
+            },
+            "special": {
+              "gold": "cm 15",
+              "pred": "bd"
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": "CITY HUNTER TV 3rd & '91 Series",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": "BD",
+            "special": "CM 15"
+          },
+          "pred": {
+            "group": null,
+            "title": "CITY HUNTER TV 3rd & '91 Series",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "BD"
+          }
+        },
+        {
+          "filename": "12 - HEVC",
+          "errors": {
+            "episode": {
+              "gold": null,
+              "pred": "12"
+            },
+            "special": {
+              "gold": "12",
+              "pred": null
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": null,
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": "HEVC",
+            "special": "12"
+          },
+          "pred": {
+            "group": null,
+            "title": null,
+            "season": null,
+            "episode": 12,
+            "resolution": null,
+            "source": "HEVC",
+            "special": null
+          }
+        },
+        {
+          "filename": "07][檢索：魔法姊妹露露特莉莉][CHT&JPN",
+          "errors": {
+            "title": {
+              "gold": null,
+              "pred": "檢索：魔法姊妹露露特莉莉"
+            },
+            "special": {
+              "gold": "檢索：魔法姊妹露露特莉莉",
+              "pred": null
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": null,
+            "season": null,
+            "episode": 7,
+            "resolution": null,
+            "source": "CHT&JPN",
+            "special": "檢索：魔法姊妹露露特莉莉"
+          },
+          "pred": {
+            "group": null,
+            "title": "檢索：魔法姊妹露露特莉莉",
+            "season": null,
+            "episode": 7,
+            "resolution": null,
+            "source": "CHT&JPN",
+            "special": null
+          }
+        },
+        {
+          "filename": "[12]_1080P_Baha_[ANi]_29 歲單身中堅冒險家的日常",
+          "errors": {
+            "title": {
+              "gold": "29 歲單身中堅冒險家的日常",
+              "pred": "歲單身中堅冒險家的日常"
+            }
+          },
+          "gold": {
+            "group": "ANi",
+            "title": "29 歲單身中堅冒險家的日常",
+            "season": null,
+            "episode": 12,
+            "resolution": "1080P",
+            "source": "Baha",
+            "special": null
+          },
+          "pred": {
+            "group": "ANi",
+            "title": "歲單身中堅冒險家的日常",
+            "season": null,
+            "episode": 12,
+            "resolution": "1080P",
+            "source": "Baha",
+            "special": null
+          }
+        },
+        {
+          "filename": "16][S2][[DBD-Raws]",
+          "errors": {
+            "episode": {
+              "gold": null,
+              "pred": "16"
+            },
+            "special": {
+              "gold": "16",
+              "pred": null
+            }
+          },
+          "gold": {
+            "group": "DBD-Raws",
+            "title": null,
+            "season": 2,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": "16"
+          },
+          "pred": {
+            "group": "DBD-Raws",
+            "title": null,
+            "season": 2,
+            "episode": 16,
+            "resolution": null,
+            "source": null,
+            "special": null
+          }
+        },
+        {
+          "filename": "Captain.Tsubasa.Road.to.2002.Intro1.BGSubs.TVRip.XviD-TBO",
+          "errors": {
+            "group": {
+              "gold": null,
+              "pred": "bgsubs"
+            },
+            "title": {
+              "gold": "captain.tsubasa.road.to intro1.bgsubs",
+              "pred": "captain.tsubasa.road.to intro1"
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": "Captain.Tsubasa.Road.to Intro1.BGSubs",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": "TVRip",
+            "special": null
+          },
+          "pred": {
+            "group": "BGSubs",
+            "title": "Captain.Tsubasa.Road.to Intro1",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": "TVRip",
+            "special": null
+          }
+        },
+        {
+          "filename": "HNK 006 ITA audio + jap audio + sub ita",
+          "errors": {
+            "title": {
+              "gold": "hnk 006 ita audio + audio + sub ita",
+              "pred": "hnk 006 ita audio + jap audio + sub ita"
+            }
+          },
+          "gold": {
+            "group": null,
+            "title": "HNK 006 ITA audio + audio + sub ita",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": null
+          },
+          "pred": {
+            "group": null,
+            "title": "HNK 006 ITA audio + jap audio + sub ita",
+            "season": null,
+            "episode": null,
+            "resolution": null,
+            "source": null,
+            "special": null
+          }
+        },
+        {
+          "filename": "Babylon][subbers][10][WebRip",
+          "errors": {
+            "group": {
+              "gold": "subbers",
+              "pred": "babylon"
+            },
+            "title": {
+              "gold": "babylon",
+              "pred": "subbers"
+            }
+          },
+          "gold": {
+            "group": "subbers",
+            "title": "Babylon",
+            "season": null,
+            "episode": 10,
+            "resolution": null,
+            "source": "WebRip",
+            "special": null
+          },
+          "pred": {
+            "group": "Babylon",
+            "title": "subbers",
+            "season": null,
+            "episode": 10,
+            "resolution": null,
+            "source": "WebRip",
+            "special": null
           }
         }
       ]
     }
   }
-}
+}

reports/perf_metrics.json

@@ -0,0 +1,2070 @@
+{
+  "sample_count": 6,
+  "samples": [
+    {
+      "step": 50.0,
+      "elapsed_seconds": 11.341162500000792,
+      "window_seconds": 11.341162500000792,
+      "steps_per_second": 4.408719123810854,
+      "samples_per_second": 7900.4246698690495,
+      "tokens_per_second": 1011254.3577432383,
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 122.5029296875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 57.0,
+      "gpu_power_w": 214.297
+    },
+    {
+      "step": 100.0,
+      "elapsed_seconds": 22.133552699990105,
+      "window_seconds": 10.792390199989313,
+      "steps_per_second": 4.632894018236064,
+      "samples_per_second": 8302.146080679026,
+      "tokens_per_second": 1062674.6983269153,
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 122.5029296875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 60.0,
+      "gpu_power_w": 216.032
+    },
+    {
+      "step": 150.0,
+      "elapsed_seconds": 32.92555089999223,
+      "window_seconds": 10.791998200002126,
+      "steps_per_second": 4.633062299805624,
+      "samples_per_second": 8302.447641251678,
+      "tokens_per_second": 1062713.2980802148,
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 122.5029296875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 68.0,
+      "gpu_power_w": 224.4
+    },
+    {
+      "step": 200.0,
+      "elapsed_seconds": 43.340594499983126,
+      "window_seconds": 10.415043599990895,
+      "steps_per_second": 4.800748025677368,
+      "samples_per_second": 8602.940462013845,
+      "tokens_per_second": 1101176.3791377721,
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 122.5029296875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 64.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 64.0,
+      "gpu_power_w": 219.672
+    },
+    {
+      "step": 250.0,
+      "elapsed_seconds": 53.830062900000485,
+      "window_seconds": 10.489468400017358,
+      "steps_per_second": 4.766685793144413,
+      "samples_per_second": 8541.900941314789,
+      "tokens_per_second": 1093363.320488293,
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 122.5029296875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 63.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 221.349
+    },
+    {
+      "step": 300.0,
+      "elapsed_seconds": 64.34275290000369,
+      "window_seconds": 10.512690000003204,
+      "steps_per_second": 4.756156606918378,
+      "samples_per_second": 8523.032639597734,
+      "tokens_per_second": 1090948.17786851,
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 122.5029296875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 224.853
+    }
+  ],
+  "background_sample_count": 135,
+  "background_samples": [
+    {
+      "process_rss_mb": 3310.83203125,
+      "cuda_allocated_mb": 10644.87158203125,
+      "cuda_reserved_mb": 11200.0,
+      "cuda_max_allocated_mb": 11051.73876953125,
+      "cuda_max_reserved_mb": 11200.0,
+      "gpu_util_percent": 92.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 12360.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 52.0,
+      "gpu_power_w": 57.852,
+      "elapsed_seconds": 0.5343180999916513
+    },
+    {
+      "process_rss_mb": 3596.44921875,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 76.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 57.0,
+      "gpu_power_w": 103.938,
+      "elapsed_seconds": 1.0462713000015356
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 52.0,
+      "gpu_power_w": 176.565,
+      "elapsed_seconds": 1.5571539000084158
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 74.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 58.0,
+      "gpu_power_w": 213.648,
+      "elapsed_seconds": 2.068373599991901
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 52.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 56.0,
+      "gpu_power_w": 215.538,
+      "elapsed_seconds": 2.5774495000077877
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 90.0,
+      "gpu_memory_util_percent": 67.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 60.0,
+      "gpu_power_w": 218.962,
+      "elapsed_seconds": 3.090203700005077
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 90.0,
+      "gpu_memory_util_percent": 71.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 60.0,
+      "gpu_power_w": 216.43,
+      "elapsed_seconds": 3.596891900000628
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 47.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 60.0,
+      "gpu_power_w": 214.673,
+      "elapsed_seconds": 4.114909399999306
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 0.0,
+      "gpu_memory_util_percent": 2.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 51.0,
+      "gpu_power_w": 187.155,
+      "elapsed_seconds": 4.619414599990705
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 51.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 56.0,
+      "gpu_power_w": 175.609,
+      "elapsed_seconds": 5.127158199989935
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 92.0,
+      "gpu_memory_util_percent": 69.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 61.0,
+      "gpu_power_w": 191.053,
+      "elapsed_seconds": 5.6363462999870535
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 84.0,
+      "gpu_memory_util_percent": 65.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 62.0,
+      "gpu_power_w": 220.276,
+      "elapsed_seconds": 6.146641400002409
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 70.0,
+      "gpu_memory_util_percent": 52.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 62.0,
+      "gpu_power_w": 216.665,
+      "elapsed_seconds": 6.656054299994139
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 61.0,
+      "gpu_power_w": 219.437,
+      "elapsed_seconds": 7.16469489998417
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 61.0,
+      "gpu_power_w": 216.638,
+      "elapsed_seconds": 7.6759840000013355
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 62.0,
+      "gpu_power_w": 216.693,
+      "elapsed_seconds": 8.185786499991082
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 56.0,
+      "gpu_power_w": 216.372,
+      "elapsed_seconds": 8.695240300003206
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 1.0,
+      "gpu_memory_util_percent": 2.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 53.0,
+      "gpu_power_w": 199.724,
+      "elapsed_seconds": 9.203403499996057
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 181.048,
+      "elapsed_seconds": 9.724752899986925
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 67.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 61.0,
+      "gpu_power_w": 196.523,
+      "elapsed_seconds": 10.2389542000019
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 220.472,
+      "elapsed_seconds": 10.747453499992844
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 57.0,
+      "gpu_power_w": 214.297,
+      "elapsed_seconds": 11.256059200008167
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 75.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 220.449,
+      "elapsed_seconds": 11.772158899984788
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 67.0,
+      "gpu_memory_util_percent": 52.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 60.0,
+      "gpu_power_w": 215.206,
+      "elapsed_seconds": 12.287043399992399
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 87.0,
+      "gpu_memory_util_percent": 65.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 218.221,
+      "elapsed_seconds": 12.796272500010673
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 91.0,
+      "gpu_memory_util_percent": 71.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 222.484,
+      "elapsed_seconds": 13.301871300005587
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 67.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 64.0,
+      "gpu_power_w": 218.572,
+      "elapsed_seconds": 13.88012049999088
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 85.0,
+      "gpu_memory_util_percent": 63.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 61.0,
+      "gpu_power_w": 179.005,
+      "elapsed_seconds": 14.386946899991017
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 93.0,
+      "gpu_memory_util_percent": 73.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 178.29,
+      "elapsed_seconds": 14.909591800009366
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 47.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 218.862,
+      "elapsed_seconds": 15.418887899984838
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 79.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 64.0,
+      "gpu_power_w": 221.606,
+      "elapsed_seconds": 15.933654699998442
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 67.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 218.428,
+      "elapsed_seconds": 16.43813369999407
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 76.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 221.864,
+      "elapsed_seconds": 16.955908100004308
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 59.0,
+      "gpu_power_w": 218.845,
+      "elapsed_seconds": 17.458802300010575
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 74.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 219.259,
+      "elapsed_seconds": 17.98001659999136
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 71.0,
+      "gpu_memory_util_percent": 55.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 62.0,
+      "gpu_power_w": 217.625,
+      "elapsed_seconds": 18.493281700008083
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 82.0,
+      "gpu_memory_util_percent": 61.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 223.113,
+      "elapsed_seconds": 19.00455859999056
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 6600.91796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 94.0,
+      "gpu_memory_util_percent": 70.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 176.144,
+      "elapsed_seconds": 19.51709879998816
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 74.0,
+      "gpu_memory_util_percent": 55.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 221.989,
+      "elapsed_seconds": 20.026390199986054
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 8451.15673828125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 78.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 222.741,
+      "elapsed_seconds": 20.537088000011863
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 64.0,
+      "gpu_memory_util_percent": 46.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 218.985,
+      "elapsed_seconds": 21.044752499990864
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 5214.01220703125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 222.352,
+      "elapsed_seconds": 21.555139400006738
+    },
+    {
+      "process_rss_mb": 3596.62890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 60.0,
+      "gpu_power_w": 216.032,
+      "elapsed_seconds": 22.065228299994487
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 74.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 222.12,
+      "elapsed_seconds": 22.576357499987353
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 74.0,
+      "gpu_memory_util_percent": 58.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 64.0,
+      "gpu_power_w": 221.54,
+      "elapsed_seconds": 23.087020799983293
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 85.0,
+      "gpu_memory_util_percent": 64.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 221.585,
+      "elapsed_seconds": 23.59532359999139
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 1.0,
+      "gpu_memory_util_percent": 2.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 56.0,
+      "gpu_power_w": 193.905,
+      "elapsed_seconds": 24.106666300009238
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 67.0,
+      "gpu_memory_util_percent": 53.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 180.115,
+      "elapsed_seconds": 24.61678059998667
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 399.6669921875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 90.0,
+      "gpu_memory_util_percent": 67.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 208.806,
+      "elapsed_seconds": 25.127415800001472
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 92.0,
+      "gpu_memory_util_percent": 72.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 225.106,
+      "elapsed_seconds": 25.63698320000549
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 2104.0634765625,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 221.467,
+      "elapsed_seconds": 26.145683699985966
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 79.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 223.909,
+      "elapsed_seconds": 26.656147300003795
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 5922.0439453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 47.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 220.797,
+      "elapsed_seconds": 27.16638400001102
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 76.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 68.0,
+      "gpu_power_w": 224.129,
+      "elapsed_seconds": 27.677494300005492
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 5752.5478515625,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 61.0,
+      "gpu_power_w": 220.716,
+      "elapsed_seconds": 28.18874740001047
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 75.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 221.391,
+      "elapsed_seconds": 28.699434799986193
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 10308.271484375,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 70.0,
+      "gpu_memory_util_percent": 54.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 219.822,
+      "elapsed_seconds": 29.21012229999178
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 0.0,
+      "gpu_memory_util_percent": 2.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 59.0,
+      "gpu_power_w": 210.546,
+      "elapsed_seconds": 29.718780900002457
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 97.0,
+      "gpu_memory_util_percent": 72.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 178.409,
+      "elapsed_seconds": 30.229401099990355
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 82.0,
+      "gpu_memory_util_percent": 64.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 68.0,
+      "gpu_power_w": 192.887,
+      "elapsed_seconds": 30.739007500000298
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 73.0,
+      "gpu_memory_util_percent": 54.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 68.0,
+      "gpu_power_w": 223.767,
+      "elapsed_seconds": 31.248938000004273
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 78.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 68.0,
+      "gpu_power_w": 224.212,
+      "elapsed_seconds": 31.757667500001844
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 64.0,
+      "gpu_memory_util_percent": 46.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 221.526,
+      "elapsed_seconds": 32.26673880001181
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 640.3212890625,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 68.0,
+      "gpu_power_w": 224.4,
+      "elapsed_seconds": 32.7766130999953
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 62.0,
+      "gpu_memory_util_percent": 47.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 62.0,
+      "gpu_power_w": 217.718,
+      "elapsed_seconds": 33.28791949999868
+    },
+    {
+      "process_rss_mb": 3612.890625,
+      "cuda_allocated_mb": 3393.1826171875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 75.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 221.852,
+      "elapsed_seconds": 33.79941800000961
+    },
+    {
+      "process_rss_mb": 3612.93359375,
+      "cuda_allocated_mb": 141.64453125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 67.0,
+      "gpu_memory_util_percent": 51.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 223.379,
+      "elapsed_seconds": 34.30993029999081
+    },
+    {
+      "process_rss_mb": 3612.94140625,
+      "cuda_allocated_mb": 3445.80859375,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 91.0,
+      "gpu_memory_util_percent": 68.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 69.0,
+      "gpu_power_w": 222.715,
+      "elapsed_seconds": 34.820057700009784
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 8961.90380859375,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 87.0,
+      "gpu_memory_util_percent": 68.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 69.0,
+      "gpu_power_w": 226.682,
+      "elapsed_seconds": 35.32873710000422
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 78.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 217.093,
+      "elapsed_seconds": 35.98900939998566
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 1.0,
+      "gpu_memory_util_percent": 2.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 58.0,
+      "gpu_power_w": 173.996,
+      "elapsed_seconds": 36.49037069999031
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 75.0,
+      "gpu_memory_util_percent": 56.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 69.0,
+      "gpu_power_w": 176.62,
+      "elapsed_seconds": 37.00997529999586
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 225.539,
+      "elapsed_seconds": 37.51092579998658
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 224.74,
+      "elapsed_seconds": 38.03694089999772
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 218.941,
+      "elapsed_seconds": 38.545965199999046
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 75.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 69.0,
+      "gpu_power_w": 225.898,
+      "elapsed_seconds": 39.06172140000854
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 64.0,
+      "gpu_memory_util_percent": 51.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 65.0,
+      "gpu_power_w": 223.082,
+      "elapsed_seconds": 39.56581679999363
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 93.0,
+      "gpu_memory_util_percent": 69.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 222.575,
+      "elapsed_seconds": 40.08639479998965
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 87.0,
+      "gpu_memory_util_percent": 68.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 223.698,
+      "elapsed_seconds": 40.6014307999867
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 69.0,
+      "gpu_memory_util_percent": 51.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 224.483,
+      "elapsed_seconds": 41.112792400002945
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 224.626,
+      "elapsed_seconds": 41.62216349999653
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 64.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 68.0,
+      "gpu_power_w": 222.202,
+      "elapsed_seconds": 42.135246999998344
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 76.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 222.999,
+      "elapsed_seconds": 42.63937769999029
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 64.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 64.0,
+      "gpu_power_w": 219.672,
+      "elapsed_seconds": 43.1591141000099
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 74.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 226.506,
+      "elapsed_seconds": 43.67472969999653
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 68.0,
+      "gpu_memory_util_percent": 53.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 224.814,
+      "elapsed_seconds": 44.185869899985846
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 86.0,
+      "gpu_memory_util_percent": 64.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 69.0,
+      "gpu_power_w": 223.741,
+      "elapsed_seconds": 44.69514809999964
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 91.0,
+      "gpu_memory_util_percent": 71.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 225.243,
+      "elapsed_seconds": 45.20475219999207
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 224.265,
+      "elapsed_seconds": 45.714895399985835
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 1.0,
+      "gpu_memory_util_percent": 2.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 60.0,
+      "gpu_power_w": 199.361,
+      "elapsed_seconds": 46.226491599984
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 82.0,
+      "gpu_memory_util_percent": 64.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 183.775,
+      "elapsed_seconds": 46.73763419999159
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 75.0,
+      "gpu_memory_util_percent": 55.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 209.171,
+      "elapsed_seconds": 47.24767229999998
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 78.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 69.0,
+      "gpu_power_w": 226.806,
+      "elapsed_seconds": 47.7600390999869
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 223.771,
+      "elapsed_seconds": 48.268198999983724
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 223.924,
+      "elapsed_seconds": 48.77904500000295
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 223.157,
+      "elapsed_seconds": 49.28964350000024
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 76.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 227.113,
+      "elapsed_seconds": 49.799869399983436
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 52.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 223.839,
+      "elapsed_seconds": 50.30973179999273
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 584.3232421875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 96.0,
+      "gpu_memory_util_percent": 72.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 227.682,
+      "elapsed_seconds": 50.81797879998339
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 83.0,
+      "gpu_memory_util_percent": 64.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 224.619,
+      "elapsed_seconds": 51.32866830000421
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 367.50244140625,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 75.0,
+      "gpu_memory_util_percent": 56.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 225.161,
+      "elapsed_seconds": 51.840583999990486
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 78.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 227.579,
+      "elapsed_seconds": 52.35056530000293
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 1088.0693359375,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 64.0,
+      "gpu_memory_util_percent": 47.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 222.789,
+      "elapsed_seconds": 52.860525099997176
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 76.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 227.236,
+      "elapsed_seconds": 53.37119559998973
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 3840.9306640625,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 63.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 221.349,
+      "elapsed_seconds": 53.881874099985
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 75.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 227.584,
+      "elapsed_seconds": 54.3920272999967
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 4297.05078125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 51.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 224.702,
+      "elapsed_seconds": 54.902947499998845
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 95.0,
+      "gpu_memory_util_percent": 70.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 224.96,
+      "elapsed_seconds": 55.4142581000051
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 7386.53857421875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 83.0,
+      "gpu_memory_util_percent": 65.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 229.283,
+      "elapsed_seconds": 55.92322180001065
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 73.0,
+      "gpu_memory_util_percent": 54.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 73.0,
+      "gpu_power_w": 225.416,
+      "elapsed_seconds": 56.434711399982916
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 8115.15771484375,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 78.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 227.937,
+      "elapsed_seconds": 56.94552159999148
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 67.0,
+      "gpu_memory_util_percent": 48.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 223.304,
+      "elapsed_seconds": 57.45550720000756
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 10308.271484375,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 225.256,
+      "elapsed_seconds": 57.96506930000032
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 67.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 224.621,
+      "elapsed_seconds": 58.47955930000171
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 37.0,
+      "gpu_memory_util_percent": 28.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 176.663,
+      "elapsed_seconds": 58.9897634999943
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 76.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 184.69,
+      "elapsed_seconds": 59.49904029999743
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 64.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 216.346,
+      "elapsed_seconds": 60.01004590000957
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 74.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 229.064,
+      "elapsed_seconds": 60.51880980000715
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 71.0,
+      "gpu_memory_util_percent": 55.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 69.0,
+      "gpu_power_w": 228.047,
+      "elapsed_seconds": 61.02798049998819
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 93.0,
+      "gpu_memory_util_percent": 73.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 229.588,
+      "elapsed_seconds": 61.54037430000608
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 47.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 73.0,
+      "gpu_power_w": 225.498,
+      "elapsed_seconds": 62.049058300006436
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 78.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 228.11,
+      "elapsed_seconds": 62.55962849999196
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 70.0,
+      "gpu_power_w": 222.692,
+      "elapsed_seconds": 63.07071030000225
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 824.71728515625,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 77.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 73.0,
+      "gpu_power_w": 228.831,
+      "elapsed_seconds": 63.5797109999985
+    },
+    {
+      "process_rss_mb": 3651.80078125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 50.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 66.0,
+      "gpu_power_w": 224.853,
+      "elapsed_seconds": 64.08942380000371
+    },
+    {
+      "process_rss_mb": 3659.08203125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 2.0,
+      "gpu_memory_util_percent": 2.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 63.0,
+      "gpu_power_w": 205.274,
+      "elapsed_seconds": 64.60033210000256
+    },
+    {
+      "process_rss_mb": 3659.08203125,
+      "cuda_allocated_mb": 8194.51708984375,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 84.0,
+      "gpu_memory_util_percent": 62.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 208.652,
+      "elapsed_seconds": 65.11013029998867
+    },
+    {
+      "process_rss_mb": 3659.08203125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 95.0,
+      "gpu_memory_util_percent": 74.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 73.0,
+      "gpu_power_w": 215.071,
+      "elapsed_seconds": 65.6213666999829
+    },
+    {
+      "process_rss_mb": 3659.08203125,
+      "cuda_allocated_mb": 4765.51025390625,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 47.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 225.985,
+      "elapsed_seconds": 66.13057750000735
+    },
+    {
+      "process_rss_mb": 3659.08203125,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 78.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 72.0,
+      "gpu_power_w": 227.795,
+      "elapsed_seconds": 66.6385944999929
+    },
+    {
+      "process_rss_mb": 3659.06640625,
+      "cuda_allocated_mb": 1300.37890625,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 65.0,
+      "gpu_memory_util_percent": 49.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 224.744,
+      "elapsed_seconds": 67.14924009999959
+    },
+    {
+      "process_rss_mb": 3659.06640625,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 76.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 73.0,
+      "gpu_power_w": 228.754,
+      "elapsed_seconds": 67.65655809998862
+    },
+    {
+      "process_rss_mb": 3659.06640625,
+      "cuda_allocated_mb": 126.2216796875,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 66.0,
+      "gpu_memory_util_percent": 51.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 67.0,
+      "gpu_power_w": 224.511,
+      "elapsed_seconds": 68.16678259999026
+    },
+    {
+      "process_rss_mb": 3659.06640625,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 100.0,
+      "gpu_memory_util_percent": 74.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 73.0,
+      "gpu_power_w": 225.465,
+      "elapsed_seconds": 68.67621050000889
+    },
+    {
+      "process_rss_mb": 3651.48046875,
+      "cuda_allocated_mb": 140.751953125,
+      "cuda_reserved_mb": 12124.0,
+      "cuda_max_allocated_mb": 11120.271484375,
+      "cuda_max_reserved_mb": 12124.0,
+      "gpu_util_percent": 74.0,
+      "gpu_memory_util_percent": 57.0,
+      "gpu_memory_used_mb": 13350.20703125,
+      "gpu_memory_total_mb": 16303.0,
+      "gpu_temperature_c": 71.0,
+      "gpu_power_w": 228.605,
+      "elapsed_seconds": 69.18651979998685
+    }
+  ],
+  "step_samples_per_second_avg": 8362.14873912102,
+  "step_samples_per_second_max": 8602.940462013845,
+  "step_samples_per_second_min": 7900.4246698690495,
+  "step_tokens_per_second_avg": 1070355.0386074905,
+  "step_tokens_per_second_max": 1101176.3791377721,
+  "step_tokens_per_second_min": 1011254.3577432383,
+  "step_process_rss_mb_avg": 3626.9251302083335,
+  "step_process_rss_mb_max": 3651.80078125,
+  "step_process_rss_mb_min": 3596.62890625,
+  "step_cuda_max_allocated_mb_avg": 11120.271484375,
+  "step_cuda_max_allocated_mb_max": 11120.271484375,
+  "step_cuda_max_allocated_mb_min": 11120.271484375,
+  "step_gpu_util_percent_avg": 70.5,
+  "step_gpu_util_percent_max": 100.0,
+  "step_gpu_util_percent_min": 63.0,
+  "step_gpu_memory_util_percent_avg": 53.5,
+  "step_gpu_memory_util_percent_max": 77.0,
+  "step_gpu_memory_util_percent_min": 48.0,
+  "step_gpu_power_w_avg": 220.1005,
+  "step_gpu_power_w_max": 224.853,
+  "step_gpu_power_w_min": 214.297,
+  "step_gpu_temperature_c_avg": 63.5,
+  "step_gpu_temperature_c_max": 68.0,
+  "step_gpu_temperature_c_min": 57.0,
+  "background_process_rss_mb_avg": 3625.386863425926,
+  "background_process_rss_mb_max": 3659.08203125,
+  "background_process_rss_mb_min": 3310.83203125,
+  "background_cuda_max_allocated_mb_avg": 11119.763834635416,
+  "background_cuda_max_allocated_mb_max": 11120.271484375,
+  "background_cuda_max_allocated_mb_min": 11051.73876953125,
+  "background_gpu_util_percent_avg": 78.32592592592593,
+  "background_gpu_util_percent_max": 100.0,
+  "background_gpu_util_percent_min": 0.0,
+  "background_gpu_memory_util_percent_avg": 59.385185185185186,
+  "background_gpu_memory_util_percent_max": 79.0,
+  "background_gpu_memory_util_percent_min": 2.0,
+  "background_gpu_power_w_avg": 214.2482,
+  "background_gpu_power_w_max": 229.588,
+  "background_gpu_power_w_min": 57.852,
+  "background_gpu_temperature_c_avg": 65.91111111111111,
+  "background_gpu_temperature_c_max": 73.0,
+  "background_gpu_temperature_c_min": 51.0,
+  "samples_per_second_avg": 8362.14873912102,
+  "samples_per_second_max": 8602.940462013845,
+  "tokens_per_second_avg": 1070355.0386074905,
+  "tokens_per_second_max": 1101176.3791377721,
+  "process_rss_mb_avg": 3626.9251302083335,
+  "process_rss_mb_max": 3651.80078125,
+  "cuda_max_allocated_mb_avg": 11120.271484375,
+  "cuda_max_allocated_mb_max": 11120.271484375,
+  "gpu_util_percent_avg": 70.5,
+  "gpu_util_percent_max": 100.0,
+  "gpu_memory_util_percent_avg": 53.5,
+  "gpu_memory_util_percent_max": 77.0,
+  "gpu_power_w_avg": 220.1005,
+  "gpu_power_w_max": 224.853,
+  "gpu_temperature_c_avg": 63.5,
+  "gpu_temperature_c_max": 68.0
+}

reports/run_metadata.json

@@ -1,23 +1,58 @@
 {
-  "experiment_name": "dmhy-char-thin-hardfocus",
-  "data_file": "data/thin_hard_focus_char.jsonl",
+  "experiment_name": "dmhy-char-aug-fragments-10epoch-hardfocus",
+  "data_file": "data/generated/focus_after_10epoch_char.jsonl",
+  "data_sources": [
+    {
+      "role": "primary",
+      "path": "data/generated/focus_after_10epoch_char.jsonl",
+      "samples": 196132,
+      "repeat": 1,
+      "effective_samples": 196132
+    }
+  ],
+  "augmentation": {
+    "partial_requested": 30000,
+    "partial_written": 27959,
+    "permutation_requested": 60000,
+    "permutation_written": 60000,
+    "special_requested": 20000,
+    "special_written": 20000,
+    "max_chars": 160
+  },
+  "dataset_mode": "encoded",
+  "apply_label_repairs": false,
+  "keep_raw_dataset": false,
   "tokenizer_variant": "char",
-  "vocab_file": "datasets/AnimeName/vocab.char.json",
+  "vocab_file": "vocab.json",
   "vocab_size": 6199,
   "max_seq_length": 128,
   "hidden_size": 256,
   "num_hidden_layers": 4,
   "num_attention_heads": 8,
   "intermediate_size": 1024,
-  "train_samples": 117089,
-  "eval_samples": 6163,
+  "train_samples": 288886,
+  "eval_samples": 15205,
+  "load_seconds": 28.198466799978632,
+  "encode_seconds": 21.198125199996866,
   "epochs": 2.0,
-  "batch_size": 256,
-  "learning_rate": 4e-05,
-  "warmup_steps": 80,
-  "seed": 58,
+  "max_steps": -1,
+  "batch_size": 1792,
+  "learning_rate": 8e-06,
+  "warmup_steps": 50,
+  "seed": 107,
   "device": "cuda",
-  "fp16": true,
+  "fp16": false,
   "gradient_accumulation_steps": 1,
-  "dataloader_num_workers": 4
+  "dataloader_num_workers": 0,
+  "dataloader_prefetch_factor": null,
+  "dataloader_persistent_workers": false,
+  "dataloader_pin_memory": true,
+  "encoded_dataset_device": "cpu",
+  "mixed_precision": "bf16",
+  "tf32": true,
+  "torch_compile": false,
+  "auto_find_batch_size": false,
+  "perf_log_steps": 50,
+  "perf_sample_interval": 0.5,
+  "periodic_eval": false
 }

reports/trainer_eval_metrics.json

@@ -1,11 +1,12 @@
 {
-  "eval_loss": 0.001824101316742599,
-  "eval_precision": 0.996635213225075,
-  "eval_recall": 0.9977786457061953,
-  "eval_f1": 0.9972066016906769,
-  "eval_accuracy": 0.9994733938512463,
-  "eval_runtime": 30.171,
-  "eval_samples_per_second": 204.269,
-  "eval_steps_per_second": 0.829,
-  "epoch": 2.0
+  "eval_loss": 0.014898430556058884,
+  "eval_precision": 0.9817063636837442,
+  "eval_recall": 0.9877553160806524,
+  "eval_f1": 0.9847215505861748,
+  "eval_accuracy": 0.9961717168124339,
+  "eval_runtime": 7.2816,
+  "eval_samples_per_second": 2088.154,
+  "eval_steps_per_second": 1.236,
+  "epoch": 2.0,
+  "num_input_tokens_seen": 73954816.0
 }

reports/training_lineage.json

@@ -4,51 +4,77 @@
   "summary_zh": "当前发布 checkpoint 是两阶段产物：先在 dmhy_weak_char.jsonl 上做全量 CUDA 微调，再做薄层运行时困难样本微调。",
   "stages": [
     {
-      "name": "dmhy-char-thin-gpu",
-      "type": "full_dataset_finetune",
+      "name": "dmhy-char-aug-fragments-optimized-10epoch",
+      "type": "full_dataset_finetune_with_dynamic_augmentation",
       "machine": "adqew@192.168.63.157",
       "data_file": "datasets/AnimeName/dmhy_weak_char.jsonl",
       "tokenizer_variant": "char",
       "vocab_file": "datasets/AnimeName/vocab.char.json",
       "vocab_size": 6199,
       "max_seq_length": 128,
-      "train_samples": 619361,
-      "eval_samples": 12641,
-      "epochs": 2.0,
-      "batch_size": 256,
-      "learning_rate": 0.00006,
-      "warmup_steps": 300,
-      "seed": 55,
+      "train_samples": 1168468,
+      "eval_samples": 23847,
+      "epochs": 10.0,
+      "batch_size": 1792,
+      "learning_rate": 0.00002,
+      "warmup_steps": 500,
+      "seed": 105,
       "device": "cuda",
-      "fp16": true,
-      "eval_f1": 0.9962419041019217,
-      "eval_accuracy": 0.9991988685517916,
+      "bf16": true,
+      "augmentation": {
+        "partial_requested": 200000,
+        "partial_written": 80313,
+        "permutation_requested": 400000,
+        "permutation_written": 400000,
+        "special_requested": 80000,
+        "special_written": 80000,
+        "max_chars": 160
+      },
+      "eval_f1": 0.9758312981401465,
+      "eval_accuracy": 0.9934884485890851,
+      "fixed_regression_model_only": "21/26",
+      "fixed_regression_normalized_only": "24/26",
+      "heldout_model_only": "1952/2048",
+      "heldout_normalized_only": "1976/2048",
       "role": "Base checkpoint for the final hard-case focus stage."
     },
     {
-      "name": "dmhy-char-thin-hardfocus",
+      "name": "dmhy-char-aug-fragments-10epoch-hardfocus",
       "type": "hard_case_focus_finetune",
       "machine": "adqew@192.168.63.157",
-      "data_file": "data/thin_hard_focus_char.jsonl",
+      "data_file": "data/generated/focus_after_10epoch_char.jsonl",
       "tokenizer_variant": "char",
       "vocab_file": "datasets/AnimeName/vocab.char.json",
       "vocab_size": 6199,
       "max_seq_length": 128,
-      "train_samples": 117089,
-      "eval_samples": 6163,
+      "train_samples": 288886,
+      "eval_samples": 15205,
       "epochs": 2.0,
-      "batch_size": 256,
-      "learning_rate": 0.00004,
-      "warmup_steps": 80,
-      "seed": 58,
+      "batch_size": 1792,
+      "learning_rate": 0.000008,
+      "warmup_steps": 50,
+      "seed": 107,
       "device": "cuda",
-      "fp16": true,
-      "eval_f1": 0.9972066016906769,
-      "eval_accuracy": 0.9994733938512463,
+      "bf16": true,
+      "augmentation": {
+        "partial_requested": 30000,
+        "partial_written": 27959,
+        "permutation_requested": 60000,
+        "permutation_written": 60000,
+        "special_requested": 20000,
+        "special_written": 20000,
+        "max_chars": 160
+      },
+      "eval_f1": 0.9847215505861748,
+      "eval_accuracy": 0.9961717168124339,
       "fixed_regression_model_only": "25/26",
       "fixed_regression_normalized_only": "26/26",
-      "heldout_model_only": "1014/1024",
-      "heldout_normalized_only": "1017/1024",
+      "heldout_model_only": "1947/2048",
+      "heldout_normalized_only": "1966/2048",
+      "perf_tokens_per_second_avg": 1070355.0386074905,
+      "perf_tokens_per_second_max": 1101176.3791377721,
+      "perf_gpu_util_avg": 78.32592592592593,
+      "perf_gpu_util_max": 100.0,
       "role": "Published repository-root checkpoint."
     }
   ]

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d04646d7f2d38f632195034cf95c225cde16df9933d0fc48ecbdf375ab0e05b3
-size 5265
+oid sha256:7d63537ca05eba3f844cc92336013aea5363ab8ba9576f63d50bceda924b5075
+size 5329