ismizo commited on
Commit
cf46759
·
verified ·
1 Parent(s): e38650d

Upload 2 files

Browse files
Files changed (2) hide show
  1. Dockerfile +30 -0
  2. main.py +73 -0
Dockerfile ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Utilize Python 3.10 to match standard wheels
2
+ FROM python:3.10-slim
3
+
4
+ WORKDIR /app
5
+
6
+ # Install standard compilers for safe fallback
7
+ RUN apt-get update && apt-get install -y \
8
+ gcc g++ make cmake git libopenblas-dev \
9
+ && rm -rf /var/lib/apt/lists/*
10
+
11
+ # Install packages with direct wheels to avoid compilation time
12
+ RUN pip install --no-cache-dir fastapi uvicorn pydantic requests
13
+
14
+ # Pull llama-cpp-python via pre-built wheel for 3.10
15
+ RUN pip install --no-cache-dir \
16
+ https://huggingface.co/Luigi/llama-cpp-python-wheels-hf-spaces-free-cpu/resolve/main/llama_cpp_python-0.3.22-cp310-cp310-linux_x86_64.whl
17
+
18
+ # Download model directly to working directory to avoid home directory eviction
19
+ RUN apt-get update && apt-get install -y wget && \
20
+ wget https://huggingface.co/bartowski/Gemma-4-E2B-it-GGUF/resolve/main/Gemma-4-E2B-it-Q4_K_M.gguf -O model.gguf
21
+
22
+ COPY . .
23
+
24
+ # Grant proper read and write permissions to avoid non-root failure
25
+ RUN chmod 777 /app
26
+
27
+ EXPOSE 7860
28
+
29
+ # Bind explicitly to the required port
30
+ CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
main.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import requests
3
+ from fastapi import FastAPI
4
+ from pydantic import BaseModel
5
+ from llama_cpp import Llama
6
+
7
+ # 1. Initialize the Model
8
+ llm = Llama(
9
+ model_path="model.gguf",
10
+ n_ctx=8192,
11
+ n_threads=2
12
+ )
13
+
14
+ app = FastAPI(title="Gemma 4 Web-Aware Thinking API")
15
+
16
+ class ChatRequest(BaseModel):
17
+ prompt: str
18
+
19
+ # --- WEB SCRAPER TOOL ---
20
+ SCRAPER_API_URL = "https://ismizo-puppeteer.hf.space/api/v1/task/snapshot"
21
+
22
+ def fetch_web_content(url: str) -> str:
23
+ try:
24
+ response = requests.post(SCRAPER_API_URL, json={"url": url}, timeout=30)
25
+ return response.text[:15000] if response.status_code == 200 else "[Error]"
26
+ except:
27
+ return "[Connection Error]"
28
+
29
+ def extract_url(text: str):
30
+ match = re.search(r'(https?://[^\s]+)', text)
31
+ return match.group(0) if match else None
32
+
33
+ # --- STEP 4: THINKING PROTOCOL CLEANER ---
34
+ def clean_thought_tags(text: str) -> str:
35
+ """Removes internal thinking blocks so they don't clutter the final response."""
36
+ # This regex removes everything between <|channel>thought and <channel|>
37
+ return re.sub(r'<\|channel>thought.*?<channel\|>', '', text, flags=re.DOTALL).strip()
38
+
39
+ # 3. Create the Generation Endpoint
40
+ @app.post("/generate")
41
+ async def chat_completion(request: ChatRequest):
42
+ try:
43
+ user_prompt = request.prompt
44
+ context_block = ""
45
+
46
+ target_url = extract_url(user_prompt)
47
+ if target_url:
48
+ scraped_text = fetch_web_content(target_url)
49
+ context_block = f"\n\nWeb Data from {target_url}:\n{scraped_text}\n"
50
+
51
+ # ACTIVATE THINKING: Append <|think|> to the system prompt
52
+ # We use a system turn to initialize the reasoning state
53
+ formatted_prompt = (
54
+ f"<start_of_turn>system\n<|think|> You are a helpful assistant. "
55
+ f"Use the following context if provided to answer accurately.\n"
56
+ f"{context_block}<end_of_turn>\n"
57
+ f"<start_of_turn>user\n{user_prompt}<end_of_turn>\n"
58
+ f"<start_of_turn>model\n"
59
+ )
60
+
61
+ output = llm(formatted_prompt, max_tokens=1024)
62
+ raw_response = output["choices"][0]["text"]
63
+
64
+ # Clean the response to hide the 'thought' channel from the user
65
+ final_response = clean_thought_tags(raw_response)
66
+
67
+ return {
68
+ "response": final_response,
69
+ "status": "success"
70
+ }
71
+
72
+ except Exception as e:
73
+ return {"error": str(e)}