| { | |
| "max_tokens": 32768, | |
| "head_tokens": 288, | |
| "option_tokens": 48, | |
| "max_request_items": 256, | |
| "micro_batch": 16, | |
| "allow_margin": 1.0, | |
| "labels": [ | |
| "allow", | |
| "review", | |
| "block" | |
| ], | |
| "input_fields": [ | |
| "input_ids", | |
| "attention_mask", | |
| "marker_pos", | |
| "marker_mask", | |
| "qtype" | |
| ], | |
| "model_file": "onnx/model.onnx", | |
| "precision": "fp16_projections_fp32_attention", | |
| "gpu_memory_limit_mib": 12288, | |
| "arena_trim_above_tokens": 4096, | |
| "head_layers": 2, | |
| "n_act": 2, | |
| "parameters": 199088899, | |
| "weight_files": { | |
| "original": "../weights/fp32/model.safetensors", | |
| "fp16": "../weights/fp16/model.safetensors", | |
| "bf16": "../weights/bf16/model.safetensors" | |
| } | |
| } | |