src/pack_gguf.py

import torch
import os
import numpy as np
from typing import Dict, Optional, List, Tuple
from dataclasses import dataclass

from .gguf_writer import GGUFWriter, GGML_TYPES


@dataclass
class QuantizedLayer:
    U: torch.Tensor
    S: torch.Tensor
    Vt: torch.Tensor
    U_scale: torch.Tensor
    S_scale: torch.Tensor
    bit_allocations: Tuple[int, int, int]


def create_lowrank_type_defs() -> str:
    return """
enum ggml_type {
    GGML_TYPE_F32     = 0,
    GGML_TYPE_F16     = 1,
    GGML_TYPE_Q8_0    = 2,
    GGML_TYPE_Q4_0    = 3,
    GGML_TYPE_Q4_1    = 4,
    GGML_TYPE_LOWRANK_UV_0BIT  = 100,
    GGML_TYPE_LOWRANK_UV_1BIT  = 101,
    GGML_TYPE_LOWRANK_SIGMA_2BIT = 102,
};
"""


def pack_sub1bit_model(
    factors: Dict[int, Dict],
    output_path: str,
    model_name: str = "llama-2-7b-sub1bit",
    metadata: Optional[Dict] = None
):
    writer = GGUFWriter(output_path)
    writer.add_key_value("general.architecture", "llama")
    writer.add_key_value("general.name", model_name)
    writer.add_key_value("general.file_type", "sub1bit")
    if metadata:
        for key, value in metadata.items():
            writer.add_key_value(key, value)
    for layer_idx, layer_data in factors.items():
        writer.add_tensor(
            f"layer.{layer_idx}.U",
            layer_data.get('U_packed', layer_data['U']).cpu().numpy(),
            GGML_TYPES['int8']
        )
        writer.add_tensor(
            f"layer.{layer_idx}.S",
            layer_data['S'].cpu().numpy().astype(np.float16),
            GGML_TYPES['float16']
        )
        writer.add_tensor(
            f"layer.{layer_idx}.Vt",
            layer_data.get('Vt_packed', layer_data['Vt']).cpu().numpy(),
            GGML_TYPES['int8']
        )
        if 'U_scale' in layer_data:
            writer.add_tensor(
                f"layer.{layer_idx}.U_scale",
                np.array([layer_data['U_scale'].item()], dtype=np.float32),
                GGML_TYPES['float32']
            )
        if 'Vt_scale' in layer_data:
            writer.add_tensor(
                f"layer.{layer_idx}.Vt_scale",
                np.array([layer_data['Vt_scale'].item()], dtype=np.float32),
                GGML_TYPES['float32']
            )
        if 'S_scale' in layer_data:
            writer.add_tensor(
                f"layer.{layer_idx}.S_scale",
                np.array([layer_data['S_scale'].item()], dtype=np.float32),
                GGML_TYPES['float32']
            )
    writer.write()
    return os.path.getsize(output_path)


if __name__ == "__main__":
    dummy_factors = {
        0: {
            'U': torch.randn(4096, 16),
            'S': torch.randn(16),
            'Vt': torch.randn(16, 4096),
            'rank': 16
        }
    }
    output_path = "C:/Users/Zwmar/projects/sub1quant/quantized/test.gguf"
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    size = pack_sub1bit_model(dummy_factors, output_path)
    print(f"GGUF file created: {size} bytes")