GGUF
reader
reader-speech-models / conversion /convert-reader-aligner-projector.py
xumo's picture
Publish exact-rebuilt Reader ASR variants and forced aligner with portable provenance
74e09ef verified
Raw History Blame Contribute Delete
3.63 kB
#!/usr/bin/env python3
"""Repackage the legacy ForcedAligner projector for pinned upstream mtmd.
Requires numpy and the pinned upstream gguf-py on PYTHONPATH. Renames tensors
and compatible shape metadata; never requantizes or changes tensor data bytes.
"""
import argparse
import hashlib
import json
from pathlib import Path
import gguf
def sha256(path):
with path.open("rb") as source:
return hashlib.file_digest(source, "sha256").hexdigest()
def convert(source, output, encoder_compat=False):
if output.exists() or output.with_suffix(".conversion.json").exists():
raise FileExistsError("choose a fresh output path")
reader = gguf.GGUFReader(source)
if "clip.audio.classify_num" not in reader.fields or not any(t.name == "a.classifier.weight" for t in reader.tensors):
raise ValueError("not a legacy ForcedAligner projector")
writer = gguf.GGUFWriter(output, "clip")
if encoder_compat:
writer.add_bool("clip.audio.reader_aligner_compat", True)
for key, field in reader.fields.items():
if key.startswith("GGUF.") or key == "general.architecture":
continue
new_key = "clip.audio.projector_type" if key == "clip.projector_type" else key
writer.add_key_value(new_key, field.contents(), field.types[0],
field.types[1] if len(field.types) > 1 else None)
tensor_hashes, tensor_types, tensor_shapes = {}, {}, {}
for tensor in reader.tensors:
name = tensor.name.replace("a.conv1d.", "a.conv2d.").replace("a.pre_encode.out.", "a.conv_out.")
data = tensor.data
if name.startswith("a.conv2d.") and name.endswith(".bias"):
data = data.reshape(-1, 1, 1)
writer.add_tensor(name, data, raw_dtype=tensor.tensor_type)
tensor_hashes[name] = hashlib.sha256(tensor.data.tobytes()).hexdigest()
tensor_types[name] = tensor.tensor_type
# GGUF dimensions are stored in the reverse order of NumPy dimensions.
tensor_shapes[name] = list(data.shape[::-1]) if name.startswith("a.conv2d.") and name.endswith(".bias") else tensor.shape.tolist()
writer.write_header_to_file()
writer.write_kv_data_to_file()
writer.write_tensors_to_file()
writer.close()
actual = gguf.GGUFReader(output)
if len(actual.tensors) != len(tensor_hashes) or any(
t.name not in tensor_hashes or
hashlib.sha256(t.data.tobytes()).hexdigest() != tensor_hashes[t.name] or
t.tensor_type != tensor_types[t.name] or t.shape.tolist() != tensor_shapes[t.name]
for t in actual.tensors):
raise RuntimeError("tensor byte/type/shape identity check failed")
output.with_suffix(".conversion.json").write_text(json.dumps({
"inputSha256": sha256(source), "outputSha256": sha256(output),
"upstreamCommit": "67107ac34f62882d8b53ea41ad68010ea635b9be",
"unchangedTensorBytes": len(tensor_hashes), "tensorSha256": tensor_hashes,
"readerAlignerEncoderCompatibility": encoder_compat,
}, indent=2) + "\n")
print(f"PASS: {len(tensor_hashes)} tensors retain identical data bytes")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--input", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--encoder-compat", action="store_true",
help="Opt into Reader legacy alignment arithmetic (encoder and CUDA decoder)")
args = parser.parse_args()
convert(args.input, args.output, args.encoder_compat)
if __name__ == "__main__":
main()