| """Self-contained QEV BF16 -> bitsandbytes LLM.int8 conversion and reload test. |
| |
| Requires CUDA. No retraining, adapter merge, token truncation or Hub upload. |
| """ |
| import argparse |
| import gc |
| import hashlib |
| import importlib.metadata |
| import json |
| from pathlib import Path |
| import shutil |
|
|
| import torch |
| from huggingface_hub import snapshot_download |
| from transformers import AutoModel, AutoModelForCausalLM, AutoModelForMultimodalLM, BitsAndBytesConfig |
|
|
|
|
| QUESTIONS={ |
| 'route':{'type':'choice','instructions':'Which tool handles the security ticket?', |
| 'criteria':{'ledger':'Ledger: billing','sentinel':'Sentinel: security'}}, |
| 'auto':{'type':'noul','instructions':'Can a severity 4 ticket be handled automatically when only severity 1 or 2 is allowed?', |
| 'criteria':{'false':'No','true':'Yes'}}, |
| 'grade':{'type':'score','instructions':'Classify severity 4: low=1-2, medium=3, high=4 or more.', |
| 'criteria':['Low','Medium','High']}} |
| STATES=[ |
| 'The ticket is security, severity 4. Security goes to Sentinel; billing to Ledger. Only severity 1 or 2 is automatic.', |
| '티켓은 보안, 심각도 4다. 보안은 Sentinel, 결제는 Ledger로 전달한다. 자동 처리는 심각도 1 또는 2만 허용한다.', |
| 'チケットはセキュリティ、重大度4。セキュリティはSentinel、請求はLedger。自動処理は重大度1または2のみ。'] |
|
|
|
|
| def sha(path): |
| h=hashlib.sha256() |
| with Path(path).open('rb') as stream: |
| for block in iter(lambda:stream.read(8*1024*1024),b''):h.update(block) |
| return h.hexdigest() |
|
|
|
|
| def write(path,value):path.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') |
|
|
|
|
| def release_memory(): |
| gc.collect();torch.cuda.empty_cache() |
|
|
|
|
| def predict(model): |
| return [model.decide(state,QUESTIONS) for state in STATES] |
|
|
|
|
| def compare(before,after): |
| max_delta=0.;changed=0 |
| for a,b in zip(before,after): |
| for qid,answer in a['answers'].items(): |
| other=b['answers'][qid] |
| assert set(answer['probabilities'])==set(other['probabilities']) |
| assert abs(sum(other['probabilities'].values())-1)<1e-5 |
| changed+=int(answer['selected']!=other['selected']) |
| max_delta=max(max_delta,max(abs(value-other['probabilities'][key]) for key,value in answer['probabilities'].items())) |
| return {'max_probability_difference':max_delta,'argmax_changed_questions':changed,'questions':len(before)*len(QUESTIONS)} |
|
|
|
|
| def cards(out,name,source_name): |
| sections={ |
| 'en':f'''# {name} |
| |
| This is an INT8 conversion of {source_name}, produced with the bundled QEV tool. |
| Full quantized backbone, unchanged LoRA, FP32 pointer head, tokenizer and custom |
| loader are included. This is not an adapter-only distribution. No adapter merge |
| or additional training was performed. Historical BF16 benchmark results do not |
| constitute a fresh benchmark of these converted weights. |
| |
| Install requirements.txt and run python example.py --device cuda:0. |
| Use AutoModel.from_pretrained(".", trust_remote_code=True, device="cuda:0") |
| and model.decide(). Do not apply generic save_pretrained() to this wrapper. |
| |
| LLM.int8 uses threshold 6.0. LoRA, head, embeddings, normalization, excluded |
| lm_head and outlier computations retain floating-point precision. CUDA is |
| required by this validated recipe. It can reduce memory but need not improve |
| latency. See quantization-report.json for synthetic BF16/INT8 differences and |
| saved-model reload parity. This is a small smoke test, not a quality benchmark. |
| |
| QueryPie AI Research License v1.0 applies to QueryPie components. Commercial use |
| requires prior written permission or a separate commercial license; contact |
| https://www.querypie.com/en/contact-us . Original third-party Apache-2.0 rights |
| are preserved. See LICENSE and THIRD_PARTY_LICENSES.md. No training/test data is |
| bundled. The original card is retained as BF16_SOURCE_MODEL_CARD.md. |
| ''', |
| 'ko':f'''# {name} |
| |
| {source_name}를 동봉한 QEV 도구로 INT8 변환한 모델입니다. 전체 양자화 backbone, |
| 변경하지 않은 LoRA, FP32 pointer head, tokenizer와 전용 로더를 포함합니다. |
| adapter-only가 아니며 adapter 병합이나 추가 학습을 하지 않았습니다. |
| 기존 BF16 벤치마크 수치를 새 양자화 가중치의 실측 성능으로 해석하지 마세요. |
| |
| requirements.txt를 설치한 뒤 python example.py --device cuda:0로 실행합니다. |
| AutoModel.from_pretrained(".", trust_remote_code=True, device="cuda:0")로 읽고 |
| decide()를 사용합니다. wrapper의 generic save_pretrained()는 사용하지 않습니다. |
| |
| LLM.int8 threshold는 6.0이며 LoRA·head·embedding·정규화·제외한 lm_head 및 |
| outlier 경로는 고정밀을 유지합니다. 검증된 실행 경로는 CUDA를 요구합니다. |
| 메모리는 줄어도 응답속도는 느려질 수 있습니다. quantization-report.json의 |
| BF16/INT8 확률 차이와 저장 후 재로드 parity는 작은 합성 smoke 검증이며 |
| 본격적인 품질 평가를 대신하지 않습니다. 원본 카드는 BF16_SOURCE_MODEL_CARD.ko.md에 있습니다. |
| |
| QueryPie 구성 요소에는 QueryPie AI Research License v1.0을 적용합니다. |
| 상업 이용은 사전 서면 허가 또는 별도 계약이 필요하며 |
| https://www.querypie.com/ko/contact-us 로 문의하세요. 기반 Apache-2.0 권리는 |
| 유지합니다. LICENSE·THIRD_PARTY_LICENSES.md를 참조하세요. 실제 학습·테스트 |
| 데이터는 포함하지 않습니다. |
| ''', |
| 'ja':f'''# {name} |
| |
| {source_name}を同梱QEVツールでINT8変換したモデルです。量子化backbone全重み、 |
| 変更しないLoRA、FP32 pointer head、tokenizer、専用ローダーを含みます。 |
| adapterのみの配布ではなく、マージや追加学習は行いません。元のBF16評価値は |
| 新しい量子化重みの実測評価ではありません。 |
| |
| requirements.txtをインストールし、python example.py --device cuda:0で実行します。 |
| AutoModel.from_pretrained(".", trust_remote_code=True, device="cuda:0")で読み込み、 |
| decide()を使います。wrapperのgeneric save_pretrained()は使用しません。 |
| |
| LLM.int8 thresholdは6.0です。LoRA・head・embedding・正規化・除外lm_head・ |
| outlier経路は浮動小数点精度を保持します。検証経路はCUDAが必要です。 |
| メモリが減っても応答時間は増えることがあります。quantization-report.jsonの |
| BF16/INT8差と保存後再ロードparityは合成smoke試験であり本評価ではありません。 |
| 元のカードはBF16_SOURCE_MODEL_CARD.ja.mdに保存しています。 |
| |
| QueryPie構成要素にはQueryPie AI Research License v1.0が適用されます。 |
| 商用利用には事前書面許可または別契約が必要です。問い合わせ先は |
| https://querypie.ai/contact-us です。原Apache-2.0権利は維持します。 |
| LICENSE・THIRD_PARTY_LICENSES.mdを参照してください。実学習・評価データは含みません。 |
| '''} |
| for lang,suffix in [('en',''),('ko','.ko'),('ja','.ja')]: |
| header='' |
| if lang=='en':header='---\nlibrary_name: transformers\nlicense: other\nlicense_name: querypie-ai-research-license-v1.0\nlicense_link: LICENSE\npipeline_tag: text-classification\ntags:\n- qev\n- int8\n- custom-code\n---\n\n' |
| (out/('README'+suffix+'.md')).write_text(header+sections[lang]+'\n[English](README.md) · [한국어](README.ko.md) · [日本語](README.ja.md)\n') |
|
|
|
|
| def main(): |
| p=argparse.ArgumentParser(description=__doc__) |
| p.add_argument('--model',default=str(Path(__file__).resolve().parent.parent),help='Local BF16 repository folder or Hub repo ID') |
| p.add_argument('--out',required=True,help='NEW independent output folder; never overwrites') |
| p.add_argument('--revision',help='Pin Hub source commit; local paths ignore this') |
| p.add_argument('--device',default='cuda:0');p.add_argument('--local-files-only',action='store_true') |
| a=p.parse_args() |
| if not torch.cuda.is_available() or not a.device.startswith('cuda'):raise ValueError('CUDA device required') |
| if a.device=='cuda':a.device='cuda:0' |
| source=Path(a.model) |
| if not source.is_dir():source=Path(snapshot_download(a.model,revision=a.revision,local_files_only=a.local_files_only)) |
| source=source.resolve();out=Path(a.out).resolve() |
| if out.exists() or source==out or source in out.parents or out in source.parents: |
| raise ValueError('Output must be a NEW folder outside source; no overwrite') |
| cfg=json.loads((source/'config.json').read_text()) |
| if cfg.get('model_type')!='qev_decision' or cfg.get('precision')!='bf16':raise ValueError('A QEV BF16 single-repository package is required') |
| source_weights={str(f.relative_to(source)):sha(f) for f in source.rglob('*.safetensors')} |
| component_hashes={name:sha(source/name) for name in ('adapter/adapter_model.safetensors','pointer_head.safetensors','tokenizer.json','tokenizer_config.json')} |
| print('Measuring original BF16 probabilities',flush=True) |
| original=AutoModel.from_pretrained(source,trust_remote_code=True,local_files_only=True,device=a.device) |
| baseline=predict(original);del original;release_memory() |
| print('Converting backbone to LLM.int8',flush=True) |
| auto=AutoModelForMultimodalLM if cfg['family']=='gemma' else AutoModelForCausalLM |
| quant=BitsAndBytesConfig(load_in_8bit=True,llm_int8_threshold=6.0, |
| llm_int8_has_fp16_weight=False,llm_int8_skip_modules=['lm_head']) |
| wrapper=auto.from_pretrained(source/'backbone',dtype=torch.bfloat16,device_map={'':a.device}, |
| quantization_config=quant,local_files_only=True) |
| from bitsandbytes.nn import Linear8bitLt |
| count=sum(isinstance(m,Linear8bitLt) for m in wrapper.modules());assert count>0 |
| out.mkdir(parents=True);write(out/'quantization-status.json',{'stage':'converting'}) |
| try: |
| wrapper.save_pretrained(out/'backbone',max_shard_size='4GB',safe_serialization=True) |
| del wrapper;release_memory() |
| for directory in ('adapter','LICENSES'): |
| if (source/directory).exists():shutil.copytree(source/directory,out/directory) |
| for file in source.iterdir(): |
| if not file.is_file():continue |
| if file.name in ('config.json','MANIFEST.sha256','PACKAGE_READY.json','evaluation.json','validation.json','training-validation.json'):continue |
| if file.suffix in ('.pt','.bin','.jsonl','.csv') or file.name.startswith('FINETUNING'):continue |
| if file.name in ('README.md','README.ko.md','README.ja.md'): |
| shutil.copyfile(file,out/('BF16_SOURCE_MODEL_CARD'+file.name[len('README'):])) |
| else:shutil.copyfile(file,out/file.name) |
| cfg['precision']='int8';cfg['display_name']=cfg['display_name']+'-INT8' |
| write(out/'config.json',cfg) |
| if (out/'provenance.json').exists(): |
| provenance=json.loads((out/'provenance.json').read_text());provenance['precision']='int8' |
| provenance['quantization_recipe']='bitsandbytes LLM.int8 threshold 6.0; original adapter/head preserved' |
| write(out/'provenance.json',provenance) |
| (out/'requirements.txt').write_text(''.join(f'{name}=={importlib.metadata.version(name)}\n' for name in |
| ['torch','transformers','peft','accelerate','safetensors','huggingface_hub','bitsandbytes'])) |
| for name,h in component_hashes.items():assert sha(out/name)==h |
| cards(out,cfg['display_name'],json.loads((source/'config.json').read_text())['display_name']) |
| model=AutoModel.from_pretrained(out,trust_remote_code=True,local_files_only=True,device=a.device) |
| assert any(isinstance(m,Linear8bitLt) for m in model.modules()) |
| first=predict(model);del model;release_memory() |
| model=AutoModel.from_pretrained(out,trust_remote_code=True,local_files_only=True,device=a.device) |
| second=predict(model);del model;release_memory() |
| reload=compare(first,second);assert reload['max_probability_difference']<1e-5 and reload['argmax_changed_questions']==0 |
| delta=compare(baseline,first) |
| assert source_weights=={str(f.relative_to(source)):sha(f) for f in source.rglob('*.safetensors')} |
| report={'status':'passed','recipe':'bitsandbytes LLM.int8','threshold':6.0,'quantized_linear_count':count, |
| 'source_name':json.loads((source/'config.json').read_text())['display_name'], |
| 'source_config_sha256':sha(source/'config.json'),'input_base_revision':cfg.get('base_model_revision'), |
| 'source_weights_unchanged':True,'adapter_head_tokenizer_unchanged':True, |
| 'adapter_merged':False,'temperature_unchanged':cfg['temperature'], |
| 'bf16_vs_int8_synthetic':delta,'saved_int8_reload_parity':reload, |
| 'synthetic_states':3,'synthetic_question_types':['choice','noul','score'], |
| 'synthetic_state_languages':['en','ko','ja'],'full_benchmark_performed':False, |
| 'upload_performed':False,'backbone_bf16_bytes':sum(f.stat().st_size for f in (source/'backbone').glob('*.safetensors')), |
| 'backbone_int8_bytes':sum(f.stat().st_size for f in (out/'backbone').glob('*.safetensors'))} |
| write(out/'quantization-report.json',report) |
| write(out/'quantization-status.json',{'stage':'complete'}) |
| files=sorted(f for f in out.rglob('*') if f.is_file() and f.name!='MANIFEST.sha256') |
| (out/'MANIFEST.sha256').write_text(''.join(sha(f)+' '+str(f.relative_to(out))+'\n' for f in files)) |
| print(json.dumps(report,indent=2),flush=True) |
| except Exception as exc: |
| write(out/'quantization-status.json',{'stage':'failed','error':str(exc)}) |
| raise |
|
|
|
|
| if __name__=='__main__':main() |
|
|
