models/config-user.yaml

TheBloke_Luna-AI-Llama2-Uncensored-GPTQ$:
  loader: ExLlama_HF
  cpu_memory: 58000
  auto_devices: true
  disk: false
  cpu: false
  bf16: false
  load_in_8bit: false
  trust_remote_code: false
  load_in_4bit: false
  compute_dtype: float16
  quant_type: nf4
  use_double_quant: false
  wbits: '4'
  groupsize: '64'
  model_type: None
  pre_layer: 0
  triton: false
  desc_act: false
  no_inject_fused_attention: false
  no_inject_fused_mlp: false
  no_use_cuda_fp16: false
  threads: 0
  n_batch: 512
  no_mmap: false
  low_vram: false
  mlock: false
  n_gpu_layers: 0
  n_ctx: 2048
  n_gqa: 0
  rms_norm_eps: 0
  llama_cpp_seed: 0.0
  gpu_split: ''
  max_seq_len: 4096
  compress_pos_emb: 1
  alpha_value: 1
  gpu_memory_0: 6990