#!/usr/bin/env python3 """Repackage the legacy ForcedAligner projector for pinned upstream mtmd. Requires numpy and the pinned upstream gguf-py on PYTHONPATH. Renames tensors and compatible shape metadata; never requantizes or changes tensor data bytes. """ import argparse import hashlib import json from pathlib import Path import gguf def sha256(path): with path.open("rb") as source: return hashlib.file_digest(source, "sha256").hexdigest() def convert(source, output, encoder_compat=False): if output.exists() or output.with_suffix(".conversion.json").exists(): raise FileExistsError("choose a fresh output path") reader = gguf.GGUFReader(source) if "clip.audio.classify_num" not in reader.fields or not any(t.name == "a.classifier.weight" for t in reader.tensors): raise ValueError("not a legacy ForcedAligner projector") writer = gguf.GGUFWriter(output, "clip") if encoder_compat: writer.add_bool("clip.audio.reader_aligner_compat", True) for key, field in reader.fields.items(): if key.startswith("GGUF.") or key == "general.architecture": continue new_key = "clip.audio.projector_type" if key == "clip.projector_type" else key writer.add_key_value(new_key, field.contents(), field.types[0], field.types[1] if len(field.types) > 1 else None) tensor_hashes, tensor_types, tensor_shapes = {}, {}, {} for tensor in reader.tensors: name = tensor.name.replace("a.conv1d.", "a.conv2d.").replace("a.pre_encode.out.", "a.conv_out.") data = tensor.data if name.startswith("a.conv2d.") and name.endswith(".bias"): data = data.reshape(-1, 1, 1) writer.add_tensor(name, data, raw_dtype=tensor.tensor_type) tensor_hashes[name] = hashlib.sha256(tensor.data.tobytes()).hexdigest() tensor_types[name] = tensor.tensor_type # GGUF dimensions are stored in the reverse order of NumPy dimensions. tensor_shapes[name] = list(data.shape[::-1]) if name.startswith("a.conv2d.") and name.endswith(".bias") else tensor.shape.tolist() writer.write_header_to_file() writer.write_kv_data_to_file() writer.write_tensors_to_file() writer.close() actual = gguf.GGUFReader(output) if len(actual.tensors) != len(tensor_hashes) or any( t.name not in tensor_hashes or hashlib.sha256(t.data.tobytes()).hexdigest() != tensor_hashes[t.name] or t.tensor_type != tensor_types[t.name] or t.shape.tolist() != tensor_shapes[t.name] for t in actual.tensors): raise RuntimeError("tensor byte/type/shape identity check failed") output.with_suffix(".conversion.json").write_text(json.dumps({ "inputSha256": sha256(source), "outputSha256": sha256(output), "upstreamCommit": "67107ac34f62882d8b53ea41ad68010ea635b9be", "unchangedTensorBytes": len(tensor_hashes), "tensorSha256": tensor_hashes, "readerAlignerEncoderCompatibility": encoder_compat, }, indent=2) + "\n") print(f"PASS: {len(tensor_hashes)} tensors retain identical data bytes") def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--encoder-compat", action="store_true", help="Opt into Reader legacy alignment arithmetic (encoder and CUDA decoder)") args = parser.parse_args() convert(args.input, args.output, args.encoder_compat) if __name__ == "__main__": main()