Upload 2 files
Browse files- Dockerfile +30 -0
- main.py +73 -0
Dockerfile
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Utilize Python 3.10 to match standard wheels
|
| 2 |
+
FROM python:3.10-slim
|
| 3 |
+
|
| 4 |
+
WORKDIR /app
|
| 5 |
+
|
| 6 |
+
# Install standard compilers for safe fallback
|
| 7 |
+
RUN apt-get update && apt-get install -y \
|
| 8 |
+
gcc g++ make cmake git libopenblas-dev \
|
| 9 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 10 |
+
|
| 11 |
+
# Install packages with direct wheels to avoid compilation time
|
| 12 |
+
RUN pip install --no-cache-dir fastapi uvicorn pydantic requests
|
| 13 |
+
|
| 14 |
+
# Pull llama-cpp-python via pre-built wheel for 3.10
|
| 15 |
+
RUN pip install --no-cache-dir \
|
| 16 |
+
https://huggingface.co/Luigi/llama-cpp-python-wheels-hf-spaces-free-cpu/resolve/main/llama_cpp_python-0.3.22-cp310-cp310-linux_x86_64.whl
|
| 17 |
+
|
| 18 |
+
# Download model directly to working directory to avoid home directory eviction
|
| 19 |
+
RUN apt-get update && apt-get install -y wget && \
|
| 20 |
+
wget https://huggingface.co/bartowski/Gemma-4-E2B-it-GGUF/resolve/main/Gemma-4-E2B-it-Q4_K_M.gguf -O model.gguf
|
| 21 |
+
|
| 22 |
+
COPY . .
|
| 23 |
+
|
| 24 |
+
# Grant proper read and write permissions to avoid non-root failure
|
| 25 |
+
RUN chmod 777 /app
|
| 26 |
+
|
| 27 |
+
EXPOSE 7860
|
| 28 |
+
|
| 29 |
+
# Bind explicitly to the required port
|
| 30 |
+
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
|
main.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
import requests
|
| 3 |
+
from fastapi import FastAPI
|
| 4 |
+
from pydantic import BaseModel
|
| 5 |
+
from llama_cpp import Llama
|
| 6 |
+
|
| 7 |
+
# 1. Initialize the Model
|
| 8 |
+
llm = Llama(
|
| 9 |
+
model_path="model.gguf",
|
| 10 |
+
n_ctx=8192,
|
| 11 |
+
n_threads=2
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
app = FastAPI(title="Gemma 4 Web-Aware Thinking API")
|
| 15 |
+
|
| 16 |
+
class ChatRequest(BaseModel):
|
| 17 |
+
prompt: str
|
| 18 |
+
|
| 19 |
+
# --- WEB SCRAPER TOOL ---
|
| 20 |
+
SCRAPER_API_URL = "https://ismizo-puppeteer.hf.space/api/v1/task/snapshot"
|
| 21 |
+
|
| 22 |
+
def fetch_web_content(url: str) -> str:
|
| 23 |
+
try:
|
| 24 |
+
response = requests.post(SCRAPER_API_URL, json={"url": url}, timeout=30)
|
| 25 |
+
return response.text[:15000] if response.status_code == 200 else "[Error]"
|
| 26 |
+
except:
|
| 27 |
+
return "[Connection Error]"
|
| 28 |
+
|
| 29 |
+
def extract_url(text: str):
|
| 30 |
+
match = re.search(r'(https?://[^\s]+)', text)
|
| 31 |
+
return match.group(0) if match else None
|
| 32 |
+
|
| 33 |
+
# --- STEP 4: THINKING PROTOCOL CLEANER ---
|
| 34 |
+
def clean_thought_tags(text: str) -> str:
|
| 35 |
+
"""Removes internal thinking blocks so they don't clutter the final response."""
|
| 36 |
+
# This regex removes everything between <|channel>thought and <channel|>
|
| 37 |
+
return re.sub(r'<\|channel>thought.*?<channel\|>', '', text, flags=re.DOTALL).strip()
|
| 38 |
+
|
| 39 |
+
# 3. Create the Generation Endpoint
|
| 40 |
+
@app.post("/generate")
|
| 41 |
+
async def chat_completion(request: ChatRequest):
|
| 42 |
+
try:
|
| 43 |
+
user_prompt = request.prompt
|
| 44 |
+
context_block = ""
|
| 45 |
+
|
| 46 |
+
target_url = extract_url(user_prompt)
|
| 47 |
+
if target_url:
|
| 48 |
+
scraped_text = fetch_web_content(target_url)
|
| 49 |
+
context_block = f"\n\nWeb Data from {target_url}:\n{scraped_text}\n"
|
| 50 |
+
|
| 51 |
+
# ACTIVATE THINKING: Append <|think|> to the system prompt
|
| 52 |
+
# We use a system turn to initialize the reasoning state
|
| 53 |
+
formatted_prompt = (
|
| 54 |
+
f"<start_of_turn>system\n<|think|> You are a helpful assistant. "
|
| 55 |
+
f"Use the following context if provided to answer accurately.\n"
|
| 56 |
+
f"{context_block}<end_of_turn>\n"
|
| 57 |
+
f"<start_of_turn>user\n{user_prompt}<end_of_turn>\n"
|
| 58 |
+
f"<start_of_turn>model\n"
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
output = llm(formatted_prompt, max_tokens=1024)
|
| 62 |
+
raw_response = output["choices"][0]["text"]
|
| 63 |
+
|
| 64 |
+
# Clean the response to hide the 'thought' channel from the user
|
| 65 |
+
final_response = clean_thought_tags(raw_response)
|
| 66 |
+
|
| 67 |
+
return {
|
| 68 |
+
"response": final_response,
|
| 69 |
+
"status": "success"
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
except Exception as e:
|
| 73 |
+
return {"error": str(e)}
|