scholarly-code-tokenizer / local-verification.json

Publish code-focused 32K tokenizer and promote it to the default

0ebdb4f verified 11 days ago

411 Bytes

{
  "location": "/workspace/scratch/tokenizer-hf-focused-release",
  "revision": null,
  "status": "passed",
  "default_and_variants_checked": 20,
  "probes_per_variant": 6,
  "checks": [
    "artifact_hashes",
    "special_ids",
    "vocabulary_size",
    "token_id_equivalence",
    "exact_roundtrips",
    "no_automatic_boundaries"
  ],
  "transformers_version": "4.57.6",
  "tokenizers_version": "0.22.2"
}

	{
	"location": "/workspace/scratch/tokenizer-hf-focused-release",
	"revision": null,
	"status": "passed",
	"default_and_variants_checked": 20,
	"probes_per_variant": 6,
	"checks": [
	"artifact_hashes",
	"special_ids",
	"vocabulary_size",
	"token_id_equivalence",
	"exact_roundtrips",
	"no_automatic_boundaries"
	],
	"transformers_version": "4.57.6",
	"tokenizers_version": "0.22.2"
	}