encryptd commited on
Commit
903e822
·
1 Parent(s): 9139a1b

prog update

Browse files
Files changed (1) hide show
  1. app.py +70 -70
app.py CHANGED
@@ -1,128 +1,128 @@
1
  import os
2
  import subprocess
3
- import time
4
  import sys
5
  import httpx
 
 
 
6
  from fastapi import FastAPI, Request
7
- from fastapi.responses import StreamingResponse
8
  import uvicorn
9
  import gradio as gr
10
- from openai import OpenAI, APIConnectionError
11
- import base64
12
- from io import BytesIO
13
 
14
  # --- CONFIGURATION ---
15
  MODEL_ID = "numind/NuMarkdown-8B-Thinking"
16
  GPU_UTILIZATION = 0.95
17
  MAX_MODEL_LEN = 32768
18
  VLLM_PORT = 8000
19
- EXPOSED_PORT = 7860
20
 
21
- # --- STEP 1: LAUNCH vLLM (Background) ---
22
  def start_vllm():
23
  if "VLLM_PID" in os.environ:
24
  return
25
-
26
- print(f"Starting vLLM server on port {VLLM_PORT}...")
27
-
28
- # JSON formatted limit string to fix parsing error
29
- limit_mm_config = '{"image": 1}'
30
-
31
  command = [
32
  "vllm", "serve", MODEL_ID,
33
- "--host", "0.0.0.0",
34
  "--port", str(VLLM_PORT),
35
  "--trust-remote-code",
36
  "--gpu-memory-utilization", str(GPU_UTILIZATION),
37
  "--max-model-len", str(MAX_MODEL_LEN),
38
  "--dtype", "bfloat16",
39
- "--limit-mm-per-prompt", limit_mm_config
40
  ]
41
-
42
- # Redirect stdout/stderr to see download progress
43
  proc = subprocess.Popen(command, stdout=sys.stdout, stderr=sys.stderr)
44
  os.environ["VLLM_PID"] = str(proc.pid)
45
-
46
- # We do NOT block here anymore. We let vLLM load in the background
47
- # while the UI starts. This allows you to see the UI immediately.
48
- print("vLLM started in background. Please wait for model download...")
49
 
50
  start_vllm()
51
 
52
- # --- STEP 2: FASTAPI PROXY ---
53
  app = FastAPI()
54
 
55
  @app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
56
- async def proxy_to_vllm(path: str, request: Request):
57
- target_url = f"http://localhost:{VLLM_PORT}/v1/{path}"
 
 
 
 
 
 
 
58
  async with httpx.AsyncClient() as client:
59
  try:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  proxy_req = client.build_request(
61
- request.method,
62
- target_url,
63
- headers=request.headers.raw,
64
- content=await request.body(),
65
- timeout=300.0
66
  )
67
  r = await client.send(proxy_req, stream=True)
68
- return StreamingResponse(
69
- r.aiter_raw(),
70
- status_code=r.status_code,
71
- headers=r.headers,
72
- background=None
73
- )
74
- except httpx.ConnectError:
75
- return JSONResponse(status_code=503, content={"error": "Model is still loading. Please wait."})
76
 
77
- # --- STEP 3: GRADIO UI ---
78
  def run_ui_test(image, prompt):
79
- if image is None:
80
- return "⚠️ Please upload an image first."
81
-
82
- client = OpenAI(base_url=f"http://localhost:{VLLM_PORT}/v1", api_key="EMPTY")
83
-
84
- # Encode Image
85
  try:
86
  buffered = BytesIO()
87
  image.save(buffered, format="JPEG")
88
  b64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
89
- except Exception as e:
90
- return f"Error processing image: {e}"
91
-
92
- if not prompt: prompt = "Convert to markdown."
93
-
94
- try:
95
  completion = client.chat.completions.create(
96
  model=MODEL_ID,
97
  messages=[{"role": "user", "content": [
98
- {"type": "text", "text": prompt},
99
  {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}
100
- ]}],
101
- max_tokens=4096
102
  )
103
- return completion.choices[0].message.content
104
- except APIConnectionError:
105
- return "⏳ Model is still downloading/loading... Check the 'Logs' tab. This takes 2-3 minutes on a fresh GPU."
106
  except Exception as e:
107
- return f"Error: {str(e)}"
108
 
109
- with gr.Blocks() as demo:
110
- gr.Markdown("# NuMarkdown L40S vLLM Server")
111
- gr.Markdown("Status: If you just started this Space, wait 3 minutes for weights to download.")
112
-
113
  with gr.Row():
114
  with gr.Column():
115
- img_input = gr.Image(type="pil", label="Document")
116
- # FIXED: Added the missing prompt input
117
- txt_input = gr.Textbox(value="Convert to markdown.", label="Prompt")
118
- btn = gr.Button("Test Inference")
119
  with gr.Column():
120
- out = gr.Textbox(label="Output")
121
-
122
- # FIXED: Passed both inputs [img_input, txt_input]
123
  btn.click(run_ui_test, inputs=[img_input, txt_input], outputs=[out])
124
 
125
- app = gr.mount_gradio_app(app, demo, path="/")
 
 
 
 
 
 
 
 
 
126
 
127
  if __name__ == "__main__":
128
- uvicorn.run(app, host="0.0.0.0", port=EXPOSED_PORT)
 
 
1
  import os
2
  import subprocess
 
3
  import sys
4
  import httpx
5
+ import json
6
+ import base64
7
+ from io import BytesIO
8
  from fastapi import FastAPI, Request
9
+ from fastapi.responses import StreamingResponse, JSONResponse
10
  import uvicorn
11
  import gradio as gr
12
+ from openai import OpenAI
 
 
13
 
14
  # --- CONFIGURATION ---
15
  MODEL_ID = "numind/NuMarkdown-8B-Thinking"
16
  GPU_UTILIZATION = 0.95
17
  MAX_MODEL_LEN = 32768
18
  VLLM_PORT = 8000
19
+ HF_PORT = int(os.getenv("PORT", 7860))
20
 
21
+ # --- STEP 1: START vLLM ---
22
  def start_vllm():
23
  if "VLLM_PID" in os.environ:
24
  return
25
+ print("Launching vLLM...")
 
 
 
 
 
26
  command = [
27
  "vllm", "serve", MODEL_ID,
28
+ "--host", "127.0.0.1", # Local only
29
  "--port", str(VLLM_PORT),
30
  "--trust-remote-code",
31
  "--gpu-memory-utilization", str(GPU_UTILIZATION),
32
  "--max-model-len", str(MAX_MODEL_LEN),
33
  "--dtype", "bfloat16",
34
+ "--limit-mm-per-prompt", '{"image": 1}'
35
  ]
 
 
36
  proc = subprocess.Popen(command, stdout=sys.stdout, stderr=sys.stderr)
37
  os.environ["VLLM_PID"] = str(proc.pid)
 
 
 
 
38
 
39
  start_vllm()
40
 
41
+ # --- STEP 2: FASTAPI GATEKEEPER ---
42
  app = FastAPI()
43
 
44
  @app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
45
+ async def gatekeeper_proxy(path: str, request: Request):
46
+ """
47
+ Proxies external API calls to internal vLLM and cleans the output.
48
+ """
49
+ target_url = f"http://127.0.0.1:{VLLM_PORT}/v1/{path}"
50
+
51
+ # Strip host header to prevent Hugging Face routing loops
52
+ headers = {k: v for k, v in request.headers.items() if k.lower() not in ["host", "content-length"]}
53
+
54
  async with httpx.AsyncClient() as client:
55
  try:
56
+ # Handle API calls from Docling (Non-streaming)
57
+ if path == "chat/completions" and request.method == "POST":
58
+ body = await request.json()
59
+
60
+ # If non-streaming, we intercept and clean reasoning
61
+ if not body.get("stream", False):
62
+ resp = await client.post(target_url, headers=headers, json=body, timeout=300.0)
63
+ if resp.status_code == 200:
64
+ data = resp.json()
65
+ content = data["choices"][0]["message"].get("content", "")
66
+ # SUPPRESS REASONING
67
+ if "</think>" in content:
68
+ data["choices"][0]["message"]["content"] = content.split("</think>")[-1].strip()
69
+ return JSONResponse(content=data)
70
+ return JSONResponse(status_code=resp.status_code, content=resp.json())
71
+
72
+ # Fallback for Models list or Streaming requests
73
  proxy_req = client.build_request(
74
+ request.method, target_url, headers=headers, content=await request.body()
 
 
 
 
75
  )
76
  r = await client.send(proxy_req, stream=True)
77
+ return StreamingResponse(r.aiter_raw(), status_code=r.status_code, headers=dict(r.headers))
78
+
79
+ except Exception as e:
80
+ return JSONResponse(status_code=503, content={"error": f"Internal Proxy Error: {str(e)}"})
 
 
 
 
81
 
82
+ # --- STEP 3: GRADIO UI (INTERNAL TESTING) ---
83
  def run_ui_test(image, prompt):
84
+ if image is None: return "Please upload an image."
85
+ client = OpenAI(base_url=f"http://127.0.0.1:{VLLM_PORT}/v1", api_key="EMPTY")
 
 
 
 
86
  try:
87
  buffered = BytesIO()
88
  image.save(buffered, format="JPEG")
89
  b64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
90
+
 
 
 
 
 
91
  completion = client.chat.completions.create(
92
  model=MODEL_ID,
93
  messages=[{"role": "user", "content": [
94
+ {"type": "text", "text": prompt or "Convert to markdown."},
95
  {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}
96
+ ]}]
 
97
  )
98
+ content = completion.choices[0].message.content
99
+ return content.split("</think>")[-1].strip() if "</think>" in content else content
 
100
  except Exception as e:
101
+ return f"Error: {e}"
102
 
103
+ with gr.Blocks(title="NuMarkdown API Server") as demo:
104
+ gr.Markdown("# NuMarkdown L40S API Server")
105
+ gr.Markdown("The API is available externally at `/v1/chat/completions` (OpenAI compatible)")
 
106
  with gr.Row():
107
  with gr.Column():
108
+ img_input = gr.Image(type="pil")
109
+ txt_input = gr.Textbox(value="Convert to markdown.")
110
+ btn = gr.Button("Run Local Test")
 
111
  with gr.Column():
112
+ out = gr.Markdown()
 
 
113
  btn.click(run_ui_test, inputs=[img_input, txt_input], outputs=[out])
114
 
115
+ # --- STEP 4: CORRECT MOUNTING ---
116
+ # We mount Gradio to a subpath to prevent it from hijacking asset routes
117
+ # Hugging Face will still show this UI at the root because of how it handles FastAPI mounts
118
+ app = gr.mount_gradio_app(app, demo, path="/ui")
119
+
120
+ # Redirect root to /ui for browser users
121
+ @app.get("/")
122
+ async def redirect_to_ui():
123
+ from fastapi.responses import RedirectResponse
124
+ return RedirectResponse(url="/ui")
125
 
126
  if __name__ == "__main__":
127
+ # Use 1 worker only to prevent port conflicts on HF L40S
128
+ uvicorn.run(app, host="0.0.0.0", port=HF_PORT, workers=1)