Upload folder using huggingface_hub

Dockerfile

@@ -2,27 +2,29 @@ FROM python:3.10-slim
 
 WORKDIR /app
 
-# Install system dependencies
+# Install build dependencies for llama-cpp-python
 RUN apt-get update && apt-get install -y \
     build-essential \
+    cmake \
+    curl \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy requirements first for caching
+# Copy requirements
 COPY requirements.txt .
 
-# Install Python dependencies with CPU-only PyTorch
-RUN pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r requirements.txt
+# Install Python dependencies (llama-cpp-python compiles from source)
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Download Qwen2.5-Coder-7B-Instruct Q4_K_M GGUF
+RUN mkdir -p /app/models && \
+    curl -L -o /app/models/qwen2.5-coder-7b-instruct-q4_k_m.gguf \
+    "https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF/resolve/main/qwen2.5-coder-7b-instruct-q4_k_m.gguf"
 
 # Copy application code
 COPY app.py .
 
-# Create cache directory for model
-RUN mkdir -p /app/.cache
-ENV HF_HOME=/app/.cache
-ENV TRANSFORMERS_CACHE=/app/.cache
-
 # Expose port
 EXPOSE 7860
 
 # Run the application
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["python", "app.py"]

README.md

@@ -35,7 +35,7 @@ curl -X POST "https://YOUR_SPACE.hf.space/anthropic/v1/messages" \
   -H "x-api-key: your-api-key" \
   -H "anthropic-version: 2023-06-01" \
   -d '{
-    "model": "qwen2.5-coder-3b",
+    "model": "qwen2.5-coder-7b",
     "max_tokens": 256,
     "messages": [
       {"role": "user", "content": "Hello, how are you?"}
@@ -48,7 +48,7 @@ curl -X POST "https://YOUR_SPACE.hf.space/anthropic/v1/messages" \
 curl -X POST "https://YOUR_SPACE.hf.space/anthropic/v1/messages" \
   -H "Content-Type: application/json" \
   -d '{
-    "model": "qwen2.5-coder-3b",
+    "model": "qwen2.5-coder-7b",
     "max_tokens": 256,
     "stream": true,
     "messages": [
@@ -69,7 +69,7 @@ client = anthropic.Anthropic(
 )
 
 message = client.messages.create(
-    model="qwen2.5-coder-3b",
+    model="qwen2.5-coder-7b",
     max_tokens=256,
     messages=[{"role": "user", "content": "Hello!"}]
 )
@@ -86,7 +86,7 @@ const client = new Anthropic({
 });
 
 const message = await client.messages.create({
-  model: 'qwen2.5-coder-3b',
+  model: 'qwen2.5-coder-7b',
   max_tokens: 256,
   messages: [{ role: 'user', content: 'Hello!' }]
 });
@@ -95,10 +95,11 @@ console.log(message.content[0].text);
 
 ## Model Info
 
-- **Model**: Qwen/Qwen2.5-Coder-3B-Instruct
-- **Parameters**: 3 Billion
-- **Optimized for**: Code & Tool reasoning
-- **Context Length**: 32K tokens
+- **Model**: Qwen2.5-Coder-7B-Instruct (Q4_K_M GGUF)
+- **Parameters**: 7 Billion (quantized)
+- **Backend**: llama.cpp
+- **Optimized for**: Code, Tool reasoning, Agent workflows
+- **Context Length**: 8K tokens
 
 ## Rate Limits
 

app.py

@@ -1,6 +1,6 @@
 """
 Dual-Compatible API Endpoint (OpenAI + Anthropic)
-Lightweight CPU-based implementation for Hugging Face Spaces
+llama.cpp powered - Qwen2.5-Coder-7B-Instruct Q4_K_M
 - OpenAI format: /v1/chat/completions
 - Anthropic format: /anthropic/v1/messages
 """
@@ -10,19 +10,18 @@ import time
 import uuid
 import logging
 import re
+import json
 from datetime import datetime
 from logging.handlers import RotatingFileHandler
 from typing import List, Optional, Union, Dict, Any, Literal
 from contextlib import asynccontextmanager
+from threading import Thread
 
 from fastapi import FastAPI, HTTPException, Header, Request
 from fastapi.responses import StreamingResponse, JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel, Field
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-from threading import Thread
-import json
+from llama_cpp import Llama
 
 # ============== Logging Configuration ==============
 LOG_DIR = "/tmp/logs"
@@ -45,52 +44,49 @@ console_handler.setFormatter(log_format)
 console_handler.setLevel(logging.INFO)
 
 logging.basicConfig(level=logging.DEBUG, handlers=[file_handler, console_handler])
-logger = logging.getLogger("dual-api")
+logger = logging.getLogger("llama-api")
 
 for uvicorn_logger in ["uvicorn", "uvicorn.error", "uvicorn.access"]:
     uv_log = logging.getLogger(uvicorn_logger)
     uv_log.handlers = [file_handler, console_handler]
 
 logger.info("=" * 60)
-logger.info(f"Dual API (OpenAI + Anthropic) Startup at {datetime.now().isoformat()}")
+logger.info(f"llama.cpp API (OpenAI + Anthropic) Startup at {datetime.now().isoformat()}")
 logger.info(f"Log file: {LOG_FILE}")
 logger.info("=" * 60)
 
 # ============== Configuration ==============
-MODEL_ID = "Qwen/Qwen2.5-Coder-3B-Instruct"
-DEVICE = "cpu"
+MODEL_PATH = "/app/models/qwen2.5-coder-7b-instruct-q4_k_m.gguf"
+N_CTX = 8192  # Context window
+N_THREADS = 2  # CPU threads
+N_BATCH = 128  # Batch size
 
-model = None
-tokenizer = None
+llm = None
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    global model, tokenizer
-    logger.info(f"Loading model: {MODEL_ID}")
+    global llm
+    logger.info(f"Loading model: {MODEL_PATH}")
     try:
-        tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
-        logger.info("Tokenizer loaded successfully")
-        model = AutoModelForCausalLM.from_pretrained(
-            MODEL_ID, torch_dtype=torch.float32, device_map=DEVICE, low_cpu_mem_usage=True
+        llm = Llama(
+            model_path=MODEL_PATH,
+            n_ctx=N_CTX,
+            n_threads=N_THREADS,
+            n_batch=N_BATCH,
+            verbose=True
         )
-        model.eval()
         logger.info("Model loaded successfully!")
-        logger.info(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")
     except Exception as e:
         logger.error(f"Failed to load model: {e}", exc_info=True)
         raise
     yield
-    logger.info("Shutting down, cleaning up model...")
-    del model, tokenizer
+    logger.info("Shutting down...")
+    del llm
 
 app = FastAPI(
     title="Dual-Compatible API (OpenAI + Anthropic)",
-    description="""
-    Lightweight CPU-based API with dual compatibility:
-    - OpenAI format: /v1/chat/completions
-    - Anthropic format: /anthropic/v1/messages
-    """,
-    version="1.0.0",
+    description="llama.cpp powered API with dual SDK compatibility",
+    version="2.0.0",
     lifespan=lifespan
 )
 
@@ -118,7 +114,7 @@ async def log_requests(request: Request, call_next):
         raise
 
 # ============================================================
-# ANTHROPIC-COMPATIBLE MODELS (under /anthropic)
+# ANTHROPIC-COMPATIBLE MODELS
 # ============================================================
 
 class AnthropicTextBlock(BaseModel):
@@ -198,7 +194,7 @@ class AnthropicMessageRequest(BaseModel):
     stop_sequences: Optional[List[str]] = None
     stream: Optional[bool] = False
     system: Optional[Union[str, List[AnthropicSystemContent]]] = None
-    temperature: Optional[float] = Field(default=1.0, ge=0.0, le=1.0)
+    temperature: Optional[float] = Field(default=0.7, ge=0.0, le=1.0)
     tool_choice: Optional[AnthropicToolChoice] = None
     tools: Optional[List[AnthropicTool]] = None
     top_k: Optional[int] = Field(default=None, ge=0)
@@ -248,7 +244,7 @@ class AnthropicTokenCountResponse(BaseModel):
     input_tokens: int
 
 # ============================================================
-# OPENAI-COMPATIBLE MODELS (under /v1)
+# OPENAI-COMPATIBLE MODELS
 # ============================================================
 
 class OpenAIMessage(BaseModel):
@@ -270,8 +266,8 @@ class OpenAIChatRequest(BaseModel):
     model: str
     messages: List[OpenAIMessage]
     max_tokens: Optional[int] = 1024
-    temperature: Optional[float] = Field(default=1.0, ge=0.0, le=2.0)
-    top_p: Optional[float] = Field(default=1.0, ge=0.0, le=1.0)
+    temperature: Optional[float] = Field(default=0.7, ge=0.0, le=2.0)
+    top_p: Optional[float] = Field(default=0.95, ge=0.0, le=1.0)
     n: Optional[int] = 1
     stream: Optional[bool] = False
     stop: Optional[Union[str, List[str]]] = None
@@ -302,18 +298,6 @@ class OpenAIChatResponse(BaseModel):
     usage: OpenAIUsage
     system_fingerprint: Optional[str] = None
 
-class OpenAIStreamChoice(BaseModel):
-    index: int
-    delta: Dict[str, Any]
-    finish_reason: Optional[str] = None
-
-class OpenAIStreamResponse(BaseModel):
-    id: str
-    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
-    created: int
-    model: str
-    choices: List[OpenAIStreamChoice]
-
 class OpenAIModel(BaseModel):
     id: str
     object: Literal["model"] = "model"
@@ -362,64 +346,76 @@ def extract_openai_content(content: Optional[Union[str, List[Dict[str, Any]]]])
             texts.append(item.get("text", ""))
     return " ".join(texts)
 
+def format_chat_prompt(messages: List[Dict[str, str]], system: Optional[str] = None) -> str:
+    """Format messages for Qwen2.5 chat template"""
+    prompt = ""
+    if system:
+        prompt += f"<|im_start|>system\n{system}<|im_end|>\n"
+
+    for msg in messages:
+        role = msg["role"]
+        content = msg["content"]
+        prompt += f"<|im_start|>{role}\n{content}<|im_end|>\n"
+
+    prompt += "<|im_start|>assistant\n"
+    return prompt
+
 def format_anthropic_messages(
     messages: List[AnthropicMessage],
     system: Optional[Union[str, List[AnthropicSystemContent]]] = None,
+    tools: Optional[List[AnthropicTool]] = None,
     thinking_enabled: bool = False,
     budget_tokens: int = 1024
 ) -> str:
     formatted_messages = []
-    system_text = extract_anthropic_system(system)
+    system_text = extract_anthropic_system(system) or ""
+
+    # Add tool definitions to system prompt if provided
+    if tools:
+        tool_defs = []
+        for tool in tools:
+            tool_def = {
+                "name": tool.name,
+                "description": tool.description,
+                "parameters": tool.input_schema.model_dump()
+            }
+            tool_defs.append(tool_def)
 
-    if thinking_enabled:
-        thinking_instruction = f"""You are a helpful AI assistant with extended thinking capabilities.
+        tool_instruction = f"""You have access to the following tools:
 
-When responding to complex problems:
-1. First, think through the problem step by step inside <thinking>...</thinking> tags
-2. Consider multiple approaches and evaluate them
-3. Show your reasoning process clearly
-4. After thinking, provide your final answer outside the thinking tags
+{json.dumps(tool_defs, indent=2)}
 
-Budget for thinking: up to {budget_tokens} tokens for reasoning.
+To use a tool, respond with a JSON object in this exact format:
+{{"tool": "tool_name", "arguments": {{"arg1": "value1"}}}}
 
-Think deeply and thoroughly before responding."""
-        if system_text:
-            system_text = f"{thinking_instruction}\n\n{system_text}"
-        else:
-            system_text = thinking_instruction
+Only use tools when necessary. If you don't need a tool, respond normally."""
+        system_text = f"{tool_instruction}\n\n{system_text}" if system_text else tool_instruction
 
-    if system_text:
-        formatted_messages.append({"role": "system", "content": system_text})
+    if thinking_enabled:
+        thinking_instruction = f"""When solving complex problems:
+1. Think through the problem step by step inside <thinking>...</thinking> tags
+2. After thinking, provide your final answer outside the thinking tags
+Budget for thinking: up to {budget_tokens} tokens."""
+        system_text = f"{thinking_instruction}\n\n{system_text}" if system_text else thinking_instruction
 
     for msg in messages:
         content = extract_anthropic_text(msg.content)
         formatted_messages.append({"role": msg.role, "content": content})
 
-    if tokenizer.chat_template:
-        return tokenizer.apply_chat_template(formatted_messages, tokenize=False, add_generation_prompt=True)
-
-    prompt = ""
-    for msg in formatted_messages:
-        role = msg["role"].capitalize()
-        prompt += f"{role}: {msg['content']}\n"
-    prompt += "Assistant: "
-    return prompt
+    return format_chat_prompt(formatted_messages, system_text if system_text else None)
 
 def format_openai_messages(messages: List[OpenAIMessage]) -> str:
+    system_text = None
     formatted_messages = []
-    for msg in messages:
-        content = extract_openai_content(msg.content)
-        formatted_messages.append({"role": msg.role, "content": content})
 
-    if tokenizer.chat_template:
-        return tokenizer.apply_chat_template(formatted_messages, tokenize=False, add_generation_prompt=True)
+    for msg in messages:
+        if msg.role == "system":
+            system_text = extract_openai_content(msg.content)
+        else:
+            content = extract_openai_content(msg.content)
+            formatted_messages.append({"role": msg.role, "content": content})
 
-    prompt = ""
-    for msg in formatted_messages:
-        role = msg["role"].capitalize()
-        prompt += f"{role}: {msg['content']}\n"
-    prompt += "Assistant: "
-    return prompt
+    return format_chat_prompt(formatted_messages, system_text)
 
 def parse_thinking_response(text: str) -> tuple:
     thinking_pattern = r'<thinking>(.*?)</thinking>'
@@ -430,6 +426,21 @@ def parse_thinking_response(text: str) -> tuple:
         return thinking_text, answer_text
     return None, text.strip()
 
+def parse_tool_use(text: str) -> Optional[Dict[str, Any]]:
+    """Parse tool use from model response"""
+    try:
+        # Look for JSON tool call pattern
+        json_pattern = r'\{[^{}]*"tool"[^{}]*\}'
+        matches = re.findall(json_pattern, text, re.DOTALL)
+        if matches:
+            for match in matches:
+                parsed = json.loads(match)
+                if "tool" in parsed:
+                    return parsed
+    except:
+        pass
+    return None
+
 def generate_id(prefix: str = "msg") -> str:
     return f"{prefix}_{uuid.uuid4().hex[:24]}"
 
@@ -439,17 +450,14 @@ def generate_id(prefix: str = "msg") -> str:
 async def root():
     return {
         "status": "healthy",
-        "model": MODEL_ID,
+        "model": "qwen2.5-coder-7b-instruct-q4_k_m",
+        "backend": "llama.cpp",
         "endpoints": {
             "openai": "/v1/chat/completions",
             "anthropic": "/anthropic/v1/messages"
         },
-        "base_urls": {
-            "openai_sdk": "https://likhonsheikh-anthropic-compatible-api.hf.space/v1",
-            "anthropic_sdk": "https://likhonsheikh-anthropic-compatible-api.hf.space/anthropic"
-        },
-        "features": ["extended-thinking", "streaming", "dual-compatibility"],
-        "log_file": LOG_FILE
+        "features": ["extended-thinking", "streaming", "tool-use", "dual-compatibility"],
+        "context_length": N_CTX
     }
 
 @app.get("/logs")
@@ -458,13 +466,13 @@ async def get_logs(lines: int = 100):
         with open(LOG_FILE, 'r') as f:
             all_lines = f.readlines()
             recent_lines = all_lines[-lines:] if len(all_lines) > lines else all_lines
-            return {"log_file": LOG_FILE, "total_lines": len(all_lines), "returned_lines": len(recent_lines), "logs": "".join(recent_lines)}
+            return {"log_file": LOG_FILE, "total_lines": len(all_lines), "logs": "".join(recent_lines)}
     except FileNotFoundError:
-        return {"error": "Log file not found", "log_file": LOG_FILE}
+        return {"error": "Log file not found"}
 
 @app.get("/health")
 async def health():
-    return {"status": "ok", "model_loaded": model is not None, "log_file": LOG_FILE, "features": ["openai-compatible", "anthropic-compatible", "extended-thinking"]}
+    return {"status": "ok", "model_loaded": llm is not None, "backend": "llama.cpp"}
 
 # ============================================================
 # OPENAI-COMPATIBLE ENDPOINTS (/v1)
@@ -472,9 +480,8 @@ async def health():
 
 @app.get("/v1/models")
 async def openai_list_models():
-    """List models (OpenAI format)"""
     return OpenAIModelList(
-        data=[OpenAIModel(id="qwen2.5-coder-3b", created=int(time.time()), owned_by="qwen")]
+        data=[OpenAIModel(id="qwen2.5-coder-7b", created=int(time.time()), owned_by="qwen")]
     )
 
 @app.post("/v1/chat/completions")
@@ -482,54 +489,37 @@ async def openai_chat_completions(
     request: OpenAIChatRequest,
     authorization: Optional[str] = Header(None)
 ):
-    """Chat completions (OpenAI format)"""
     chat_id = generate_id("chatcmpl")
-    logger.info(f"[{chat_id}] OpenAI chat - model: {request.model}, max_tokens: {request.max_tokens}, stream: {request.stream}")
+    logger.info(f"[{chat_id}] OpenAI chat - model: {request.model}, max_tokens: {request.max_tokens}")
 
     try:
         prompt = format_openai_messages(request.messages)
-        inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
-        input_token_count = inputs.input_ids.shape[1]
 
         if request.stream:
-            return await openai_stream_response(request, inputs, input_token_count, chat_id)
-
-        gen_kwargs = {
-            "max_new_tokens": request.max_tokens or 1024,
-            "do_sample": request.temperature > 0 if request.temperature else False,
-            "pad_token_id": tokenizer.eos_token_id,
-            "eos_token_id": tokenizer.eos_token_id,
-        }
-
-        if request.temperature and request.temperature > 0:
-            gen_kwargs["temperature"] = min(request.temperature, 1.0)
-        if request.top_p:
-            gen_kwargs["top_p"] = request.top_p
+            return await openai_stream_response(request, prompt, chat_id)
 
+        stop_tokens = ["<|im_end|>", "<|endoftext|>"]
         if request.stop:
-            stop_seqs = [request.stop] if isinstance(request.stop, str) else request.stop
-            stop_ids = []
-            for seq in stop_seqs:
-                tokens = tokenizer.encode(seq, add_special_tokens=False)
-                if tokens:
-                    stop_ids.extend(tokens)
-            if stop_ids:
-                gen_kwargs["eos_token_id"] = list(set([tokenizer.eos_token_id] + stop_ids))
+            if isinstance(request.stop, str):
+                stop_tokens.append(request.stop)
+            else:
+                stop_tokens.extend(request.stop)
 
         gen_start = time.time()
-        with torch.no_grad():
-            outputs = model.generate(**inputs, **gen_kwargs)
+        output = llm(
+            prompt,
+            max_tokens=request.max_tokens or 1024,
+            temperature=request.temperature or 0.7,
+            top_p=request.top_p or 0.95,
+            stop=stop_tokens,
+            echo=False
+        )
         gen_time = time.time() - gen_start
 
-        generated_tokens = outputs[0][input_token_count:]
-        generated_text = tokenizer.decode(generated_tokens, skip_special_tokens=True)
-        output_token_count = len(generated_tokens)
-
-        finish_reason = "stop"
-        if output_token_count >= (request.max_tokens or 1024):
-            finish_reason = "length"
+        generated_text = output["choices"][0]["text"].strip()
+        usage = output["usage"]
 
-        logger.info(f"[{chat_id}] Generated {output_token_count} tokens in {gen_time:.2f}s")
+        logger.info(f"[{chat_id}] Generated in {gen_time:.2f}s - tokens: {usage['completion_tokens']}")
 
         return OpenAIChatResponse(
             id=chat_id,
@@ -537,13 +527,13 @@ async def openai_chat_completions(
             model=request.model,
             choices=[OpenAIChoice(
                 index=0,
-                message={"role": "assistant", "content": generated_text.strip()},
-                finish_reason=finish_reason
+                message={"role": "assistant", "content": generated_text},
+                finish_reason="stop"
             )],
             usage=OpenAIUsage(
-                prompt_tokens=input_token_count,
-                completion_tokens=output_token_count,
-                total_tokens=input_token_count + output_token_count
+                prompt_tokens=usage["prompt_tokens"],
+                completion_tokens=usage["completion_tokens"],
+                total_tokens=usage["total_tokens"]
             )
         )
 
@@ -551,13 +541,10 @@ async def openai_chat_completions(
         logger.error(f"[{chat_id}] Error: {e}", exc_info=True)
         raise HTTPException(status_code=500, detail=str(e))
 
-async def openai_stream_response(request: OpenAIChatRequest, inputs, input_token_count: int, chat_id: str):
-    """Stream response in OpenAI format"""
-
+async def openai_stream_response(request: OpenAIChatRequest, prompt: str, chat_id: str):
     async def generate():
         created = int(time.time())
 
-        # Initial chunk with role
         initial_chunk = {
             "id": chat_id,
             "object": "chat.completion.chunk",
@@ -567,29 +554,24 @@ async def openai_stream_response(request: OpenAIChatRequest, inputs, input_token
         }
         yield f"data: {json.dumps(initial_chunk)}\n\n"
 
-        streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
-
-        gen_kwargs = {
-            **inputs,
-            "max_new_tokens": request.max_tokens or 1024,
-            "do_sample": request.temperature > 0 if request.temperature else False,
-            "pad_token_id": tokenizer.eos_token_id,
-            "eos_token_id": tokenizer.eos_token_id,
-            "streamer": streamer,
-        }
-
-        if request.temperature and request.temperature > 0:
-            gen_kwargs["temperature"] = min(request.temperature, 1.0)
-        if request.top_p:
-            gen_kwargs["top_p"] = request.top_p
-
-        thread = Thread(target=model.generate, kwargs=gen_kwargs)
-        thread.start()
-
-        output_tokens = 0
-        for text in streamer:
+        stop_tokens = ["<|im_end|>", "<|endoftext|>"]
+        if request.stop:
+            if isinstance(request.stop, str):
+                stop_tokens.append(request.stop)
+            else:
+                stop_tokens.extend(request.stop)
+
+        for output in llm(
+            prompt,
+            max_tokens=request.max_tokens or 1024,
+            temperature=request.temperature or 0.7,
+            top_p=request.top_p or 0.95,
+            stop=stop_tokens,
+            stream=True,
+            echo=False
+        ):
+            text = output["choices"][0]["text"]
             if text:
-                output_tokens += len(tokenizer.encode(text, add_special_tokens=False))
                 chunk = {
                     "id": chat_id,
                     "object": "chat.completion.chunk",
@@ -599,21 +581,17 @@ async def openai_stream_response(request: OpenAIChatRequest, inputs, input_token
                 }
                 yield f"data: {json.dumps(chunk)}\n\n"
 
-        thread.join()
-
-        # Final chunk
-        finish_reason = "length" if output_tokens >= (request.max_tokens or 1024) else "stop"
         final_chunk = {
             "id": chat_id,
             "object": "chat.completion.chunk",
             "created": created,
             "model": request.model,
-            "choices": [{"index": 0, "delta": {}, "finish_reason": finish_reason}]
+            "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
         }
         yield f"data: {json.dumps(final_chunk)}\n\n"
         yield "data: [DONE]\n\n"
 
-    return StreamingResponse(generate(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "Connection": "keep-alive"})
+    return StreamingResponse(generate(), media_type="text/event-stream", headers={"Cache-Control": "no-cache"})
 
 # ============================================================
 # ANTHROPIC-COMPATIBLE ENDPOINTS (/anthropic)
@@ -621,16 +599,16 @@ async def openai_stream_response(request: OpenAIChatRequest, inputs, input_token
 
 @app.get("/anthropic/v1/models")
 async def anthropic_list_models():
-    """List models (Anthropic format)"""
     return {
         "object": "list",
         "data": [{
-            "id": "qwen2.5-coder-3b",
+            "id": "qwen2.5-coder-7b",
             "object": "model",
             "created": int(time.time()),
             "owned_by": "qwen",
-            "display_name": "Qwen2.5 Coder 3B Instruct",
-            "supports_thinking": True
+            "display_name": "Qwen2.5 Coder 7B Instruct (Q4_K_M)",
+            "supports_thinking": True,
+            "supports_tools": True
         }]
     }
 
@@ -641,7 +619,6 @@ async def anthropic_create_message(
     anthropic_version: Optional[str] = Header(None, alias="anthropic-version"),
     anthropic_beta: Optional[str] = Header(None, alias="anthropic-beta")
 ):
-    """Create message (Anthropic format with Extended Thinking)"""
     message_id = generate_id("msg")
 
     thinking_enabled = False
@@ -650,158 +627,128 @@ async def anthropic_create_message(
         thinking_enabled = request.thinking.type == "enabled"
         budget_tokens = request.thinking.budget_tokens or 1024
 
-    logger.info(f"[{message_id}] Anthropic msg - model: {request.model}, max_tokens: {request.max_tokens}, thinking: {thinking_enabled}")
+    logger.info(f"[{message_id}] Anthropic msg - model: {request.model}, max_tokens: {request.max_tokens}, thinking: {thinking_enabled}, tools: {len(request.tools) if request.tools else 0}")
 
     try:
-        prompt = format_anthropic_messages(request.messages, request.system, thinking_enabled, budget_tokens)
-        inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
-        input_token_count = inputs.input_ids.shape[1]
+        prompt = format_anthropic_messages(
+            request.messages,
+            request.system,
+            request.tools,
+            thinking_enabled,
+            budget_tokens
+        )
 
         if request.stream:
-            return await anthropic_stream_response(request, inputs, input_token_count, message_id, thinking_enabled, budget_tokens)
+            return await anthropic_stream_response(request, prompt, message_id, thinking_enabled)
 
         total_max_tokens = request.max_tokens + (budget_tokens if thinking_enabled else 0)
 
-        gen_kwargs = {
-            "max_new_tokens": total_max_tokens,
-            "do_sample": request.temperature > 0 if request.temperature else False,
-            "pad_token_id": tokenizer.eos_token_id,
-            "eos_token_id": tokenizer.eos_token_id,
-        }
-
-        if request.temperature and request.temperature > 0:
-            gen_kwargs["temperature"] = request.temperature
-        if request.top_p:
-            gen_kwargs["top_p"] = request.top_p
-        if request.top_k:
-            gen_kwargs["top_k"] = request.top_k
+        stop_tokens = ["<|im_end|>", "<|endoftext|>"]
+        if request.stop_sequences:
+            stop_tokens.extend(request.stop_sequences)
 
         gen_start = time.time()
-        with torch.no_grad():
-            outputs = model.generate(**inputs, **gen_kwargs)
+        output = llm(
+            prompt,
+            max_tokens=total_max_tokens,
+            temperature=request.temperature or 0.7,
+            top_p=request.top_p or 0.95,
+            top_k=request.top_k or 40,
+            stop=stop_tokens,
+            echo=False
+        )
         gen_time = time.time() - gen_start
 
-        generated_tokens = outputs[0][input_token_count:]
-        generated_text = tokenizer.decode(generated_tokens, skip_special_tokens=True)
-        output_token_count = len(generated_tokens)
+        generated_text = output["choices"][0]["text"].strip()
+        usage = output["usage"]
 
+        # Parse response for tool use, thinking, etc.
         content_blocks = []
-        if thinking_enabled:
+        stop_reason = "end_turn"
+
+        # Check for tool use
+        tool_call = parse_tool_use(generated_text)
+        if tool_call and request.tools:
+            tool_id = f"toolu_{uuid.uuid4().hex[:24]}"
+            content_blocks.append(AnthropicResponseToolUseBlock(
+                type="tool_use",
+                id=tool_id,
+                name=tool_call["tool"],
+                input=tool_call.get("arguments", {})
+            ))
+            stop_reason = "tool_use"
+        elif thinking_enabled:
             thinking_text, answer_text = parse_thinking_response(generated_text)
             if thinking_text:
                 content_blocks.append(AnthropicResponseThinkingBlock(type="thinking", thinking=thinking_text))
             content_blocks.append(AnthropicResponseTextBlock(type="text", text=answer_text))
         else:
-            content_blocks.append(AnthropicResponseTextBlock(type="text", text=generated_text.strip()))
+            content_blocks.append(AnthropicResponseTextBlock(type="text", text=generated_text))
 
-        stop_reason = "end_turn"
-        if output_token_count >= total_max_tokens:
+        if usage["completion_tokens"] >= total_max_tokens:
             stop_reason = "max_tokens"
 
-        logger.info(f"[{message_id}] Generated {output_token_count} tokens in {gen_time:.2f}s")
+        logger.info(f"[{message_id}] Generated in {gen_time:.2f}s - tokens: {usage['completion_tokens']}")
 
         return AnthropicMessageResponse(
             id=message_id,
             content=content_blocks,
             model=request.model,
             stop_reason=stop_reason,
-            usage=AnthropicUsage(input_tokens=input_token_count, output_tokens=output_token_count)
+            usage=AnthropicUsage(
+                input_tokens=usage["prompt_tokens"],
+                output_tokens=usage["completion_tokens"]
+            )
         )
 
     except Exception as e:
         logger.error(f"[{message_id}] Error: {e}", exc_info=True)
         raise HTTPException(status_code=500, detail=str(e))
 
-async def anthropic_stream_response(request: AnthropicMessageRequest, inputs, input_token_count: int, message_id: str, thinking_enabled: bool, budget_tokens: int):
-    """Stream response in Anthropic format"""
-
+async def anthropic_stream_response(request: AnthropicMessageRequest, prompt: str, message_id: str, thinking_enabled: bool):
     async def generate():
         start_event = {
             "type": "message_start",
             "message": {
                 "id": message_id, "type": "message", "role": "assistant", "content": [],
                 "model": request.model, "stop_reason": None, "stop_sequence": None,
-                "usage": {"input_tokens": input_token_count, "output_tokens": 0}
+                "usage": {"input_tokens": 0, "output_tokens": 0}
             }
         }
         yield f"event: message_start\ndata: {json.dumps(start_event)}\n\n"
-        yield f"event: ping\ndata: {json.dumps({'type': 'ping'})}\n\n"
-
-        block_index = 0
-        in_thinking = False
-        thinking_started = False
-        text_block_started = False
 
-        streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
-        total_max_tokens = request.max_tokens + (budget_tokens if thinking_enabled else 0)
-
-        gen_kwargs = {
-            **inputs,
-            "max_new_tokens": total_max_tokens,
-            "do_sample": request.temperature > 0 if request.temperature else False,
-            "pad_token_id": tokenizer.eos_token_id,
-            "eos_token_id": tokenizer.eos_token_id,
-            "streamer": streamer,
-        }
-
-        if request.temperature and request.temperature > 0:
-            gen_kwargs["temperature"] = request.temperature
-        if request.top_p:
-            gen_kwargs["top_p"] = request.top_p
-        if request.top_k:
-            gen_kwargs["top_k"] = request.top_k
-
-        thread = Thread(target=model.generate, kwargs=gen_kwargs)
-        thread.start()
-
-        output_tokens = 0
-        accumulated_text = ""
-
-        for text in streamer:
+        # Start text block
+        yield f"event: content_block_start\ndata: {json.dumps({'type': 'content_block_start', 'index': 0, 'content_block': {'type': 'text', 'text': ''}})}\n\n"
+
+        stop_tokens = ["<|im_end|>", "<|endoftext|>"]
+        if request.stop_sequences:
+            stop_tokens.extend(request.stop_sequences)
+
+        total_tokens = 0
+        for output in llm(
+            prompt,
+            max_tokens=request.max_tokens,
+            temperature=request.temperature or 0.7,
+            top_p=request.top_p or 0.95,
+            stop=stop_tokens,
+            stream=True,
+            echo=False
+        ):
+            text = output["choices"][0]["text"]
             if text:
-                output_tokens += len(tokenizer.encode(text, add_special_tokens=False))
-                accumulated_text += text
-
-                if thinking_enabled:
-                    if "<thinking>" in accumulated_text and not thinking_started:
-                        thinking_started = True
-                        in_thinking = True
-                        yield f"event: content_block_start\ndata: {json.dumps({'type': 'content_block_start', 'index': block_index, 'content_block': {'type': 'thinking', 'thinking': ''}})}\n\n"
-
-                    if in_thinking:
-                        clean_text = text.replace("<thinking>", "").replace("</thinking>", "")
-                        if clean_text:
-                            yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': block_index, 'delta': {'type': 'thinking_delta', 'thinking': clean_text}})}\n\n"
-                        if "</thinking>" in accumulated_text:
-                            in_thinking = False
-                            yield f"event: content_block_stop\ndata: {json.dumps({'type': 'content_block_stop', 'index': block_index})}\n\n"
-                            block_index += 1
-                            text_block_started = True
-                            yield f"event: content_block_start\ndata: {json.dumps({'type': 'content_block_start', 'index': block_index, 'content_block': {'type': 'text', 'text': ''}})}\n\n"
-                    elif text_block_started:
-                        yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': block_index, 'delta': {'type': 'text_delta', 'text': text}})}\n\n"
-                else:
-                    if not text_block_started:
-                        text_block_started = True
-                        yield f"event: content_block_start\ndata: {json.dumps({'type': 'content_block_start', 'index': 0, 'content_block': {'type': 'text', 'text': ''}})}\n\n"
-                    yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': text}})}\n\n"
-
-        thread.join()
-
-        yield f"event: content_block_stop\ndata: {json.dumps({'type': 'content_block_stop', 'index': block_index})}\n\n"
-
-        stop_reason = "max_tokens" if output_tokens >= total_max_tokens else "end_turn"
-        yield f"event: message_delta\ndata: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': stop_reason}, 'usage': {'output_tokens': output_tokens}})}\n\n"
+                total_tokens += 1
+                yield f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': text}})}\n\n"
+
+        yield f"event: content_block_stop\ndata: {json.dumps({'type': 'content_block_stop', 'index': 0})}\n\n"
+        yield f"event: message_delta\ndata: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn'}, 'usage': {'output_tokens': total_tokens}})}\n\n"
         yield f"event: message_stop\ndata: {json.dumps({'type': 'message_stop'})}\n\n"
 
-    return StreamingResponse(generate(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no"})
+    return StreamingResponse(generate(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
 
 @app.post("/anthropic/v1/messages/count_tokens", response_model=AnthropicTokenCountResponse)
 async def anthropic_count_tokens(request: AnthropicTokenCountRequest):
-    thinking_enabled = request.thinking and request.thinking.type == "enabled"
-    budget_tokens = request.thinking.budget_tokens if request.thinking else 1024
-    prompt = format_anthropic_messages(request.messages, request.system, thinking_enabled, budget_tokens)
-    tokens = tokenizer.encode(prompt)
+    prompt = format_anthropic_messages(request.messages, request.system)
+    tokens = llm.tokenize(prompt.encode())
     return AnthropicTokenCountResponse(input_tokens=len(tokens))
 
 if __name__ == "__main__":

requirements.txt

@@ -1,8 +1,5 @@
 fastapi==0.115.5
 uvicorn[standard]==0.32.0
-transformers==4.46.2
-torch==2.1.2+cpu
-accelerate==1.1.1
+llama-cpp-python==0.3.2
 pydantic==2.10.1
 python-multipart==0.0.12
-numpy<2