| import gradio as gr |
| from huggingface_hub import hf_hub_download |
| from llama_cpp import Llama |
|
|
| print("GGUF मॉडल डाउनलोड हो रहा है (पहली बार में 1-2 मिनट लगेंगे)...") |
| model_path = hf_hub_download( |
| repo_id="Qwen/Qwen2.5-3B-Instruct-GGUF", |
| filename="qwen2.5-3b-instruct-q4_k_m.gguf" |
| ) |
|
|
| print("मॉडल लोड हो रहा है...") |
| |
| llm = Llama( |
| model_path=model_path, |
| n_ctx=2048, |
| n_threads=2, |
| verbose=False |
| ) |
| print("3B मॉडल सफलतापूर्वक लोड हो गया!") |
|
|
| def generate_response(message, history): |
| |
| formatted_prompt = "<|im_start|>system\nYou are a logical data mining assistant.<|im_end|>\n" |
| |
| for user_msg, bot_msg in history: |
| formatted_prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n<|im_start|>assistant\n{bot_msg}<|im_end|>\n" |
| |
| formatted_prompt += f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n" |
|
|
| |
| output = llm( |
| formatted_prompt, |
| max_tokens=250, |
| temperature=0.3, |
| stop=["<|im_end|>"] |
| ) |
| |
| return output["choices"][0]["text"].strip() |
|
|
| |
| demo = gr.ChatInterface( |
| fn=generate_response, |
| title="🧠 Qwen 3B GGUF Server", |
| description="3B Intelligence on a free CPU" |
| ) |
|
|
| if __name__ == "__main__": |
| demo.launch(show_error=True) |
|
|
