from tensorrt_llm.commands.serve import get_llm_args, launch_server

if __name__ == '__main__':
    llm_args, _ = get_llm_args(
        model=r'D:\AI\models\Josiefied-Qwen3-4B-ablit-gptq-engine',
        tokenizer=r'D:\AI\models\Josiefied-Qwen3-4B-abliterated-v1',
        backend='tensorrt',
        max_beam_width=1,
        max_batch_size=1,
        max_num_tokens=16384,
        max_seq_len=16384,
        tensor_parallel_size=1,
        pipeline_parallel_size=1,
        context_parallel_size=1,
        moe_expert_parallel_size=None,
        gpus_per_node=1,
        free_gpu_memory_fraction=0.9,
        kv_cache_dtype='auto',
        num_postprocess_workers=0,
        trust_remote_code=False,
        revision=None,
        reasoning_parser='qwen3',
        fail_fast_on_attention_window_too_large=False,
        otlp_traces_endpoint=None,
        enable_chunked_prefill=True,
    )

    launch_server(
        host='localhost',
        port=5001,
        llm_args=llm_args,
        served_model_name='Josiefied-Qwen3-4B',
    )