supporting-material/source-code/manifest.json

{
  "repo": "https://github.com/fffoivos/glossapi-tokenizer-extension",
  "branch": "main",
  "source_commit": "7c67cf11eb3d31f0ddd006f7d709709037db93a1",
  "subproject": "subprojects/03_apertus_extension_and_embedding_adaptation",
  "script_families": [
    {
      "family": "tokenizer_shipping",
      "paths": [
        "03_3_cscs_experiments_kickoff/ship/apertus_greek_modern_only_148480",
        "03_3_cscs_experiments_kickoff/ship/apertus_greek_extended_153600"
      ],
      "artifacts": [
        "tokenizer.ModernGreek-148k",
        "tokenizer.ModernGreek-Polytonic-154k"
      ]
    },
    {
      "family": "corpus_build",
      "paths": [
        "03_2_apertus_c3_dedup_audit/CPT_DATASET_BUILD_RUNBOOK.md",
        "03_4_implementation_experiments/init_bakeoff/corpus_build"
      ],
      "artifacts": [
        "dataset.CPT-7B-mix"
      ]
    },
    {
      "family": "init_and_conversion",
      "paths": [
        "03_4_implementation_experiments/init_bakeoff/arms",
        "03_4_implementation_experiments/init_bakeoff/megatron_patches"
      ],
      "artifacts": [
        "checkpoint.TokenDistil-Init",
        "checkpoint.Vanilla-3.5B",
        "checkpoint.ReTok-3.5B"
      ]
    },
    {
      "family": "token_distillation",
      "paths": [
        "TOKEN_DISTILLATION_PLAN.md",
        "03_4_implementation_experiments/init_bakeoff/token_distillation"
      ],
      "artifacts": [
        "checkpoint.TokenDistil-Init"
      ]
    },
    {
      "family": "training",
      "paths": [
        "TRAINING_RECIPE.md",
        "03_4_implementation_experiments/init_bakeoff/bakeoff_training"
      ],
      "artifacts": [
        "checkpoint.TokenDistil-3.5B",
        "checkpoint.Vanilla-3.5B",
        "checkpoint.ReTok-3.5B"
      ]
    },
    {
      "family": "evaluation",
      "paths": [
        "03_4_implementation_experiments/init_bakeoff/eval"
      ],
      "artifacts": [
        "results.3.5B-comparison"
      ]
    }
  ]
}