encryptd commited on
Commit
cd46a2d
·
1 Parent(s): 5c7ad0b

prog update

Browse files
Files changed (1) hide show
  1. app.py +50 -67
app.py CHANGED
@@ -4,10 +4,9 @@ import sys
4
  import httpx
5
  import json
6
  import base64
7
- import time
8
  from io import BytesIO
9
- from fastapi import Request
10
- from fastapi.responses import StreamingResponse, JSONResponse
11
  import uvicorn
12
  import gradio as gr
13
  from openai import OpenAI
@@ -19,12 +18,12 @@ MAX_MODEL_LEN = 16384
19
  VLLM_PORT = 8000
20
  HF_PORT = 7860
21
 
22
- # --- STEP 1: START vLLM (with visible logging) ---
23
  def start_vllm():
24
  if "VLLM_PID" in os.environ:
25
  return
26
  print("--- STARTING VLLM ---")
27
- command = [
28
  "python3", "-m", "vllm.entrypoints.openai.api_server",
29
  "--model", MODEL_ID,
30
  "--host", "127.0.0.1",
@@ -35,66 +34,79 @@ def start_vllm():
35
  "--dtype", "bfloat16",
36
  "--limit-mm-per-prompt", '{"image": 1}'
37
  ]
38
- # Keep stdout/stderr connected to sys so you can see vLLM errors in HF logs
39
  proc = subprocess.Popen(command, stdout=sys.stdout, stderr=sys.stderr)
40
  os.environ["VLLM_PID"] = str(proc.pid)
41
- print(f"vLLM process started with PID: {proc.pid}")
42
 
43
  start_vllm()
44
 
45
- # --- STEP 2: UI LOGIC ---
46
- def run_ui_test(image, prompt):
47
- print(f">>> UI Request Received. Prompt: {prompt}") # THIS SHOULD SHOW IN LOGS
 
 
 
 
 
48
 
49
- if image is None:
50
- return "⚠️ Please upload an image."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
 
52
- # Internal Health Check
 
 
 
 
53
  try:
54
  with httpx.Client() as check:
55
  check.get(f"http://127.0.0.1:{VLLM_PORT}/v1/models", timeout=2.0)
56
  except Exception:
57
- print(">>> vLLM not ready yet.")
58
- return "⏳ Model is still loading (takes 3-5 mins). Watch the logs for 'Uvicorn running on http://127.0.0.1:8000'."
59
 
60
  client = OpenAI(base_url=f"http://127.0.0.1:{VLLM_PORT}/v1", api_key="EMPTY")
61
-
62
  try:
63
- # Pre-process image
64
- print(">>> Converting image to RGB...")
65
  image = image.convert("RGB")
66
  buffered = BytesIO()
67
  image.save(buffered, format="JPEG")
68
  b64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
69
 
70
- print(">>> Sending to vLLM...")
71
  completion = client.chat.completions.create(
72
  model=MODEL_ID,
73
- messages=[{"role": "user", "content": [
74
  {"type": "text", "text": prompt or "Convert to markdown."},
75
  {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}
76
  ]}],
77
  max_tokens=4096,
78
  timeout=300.0
79
  )
80
-
81
  content = completion.choices[0].message.content
82
- print(">>> Inference complete.")
83
-
84
- if "</think>" in content:
85
- return content.split("</think>")[-1].strip()
86
- return content.strip()
87
-
88
  except Exception as e:
89
- print(f">>> ERROR: {str(e)}")
90
  return f"❌ Error: {str(e)}"
91
 
92
- # --- STEP 3: BUILD UI ---
93
- with gr.Blocks() as demo:
94
  gr.Markdown("# NuMarkdown L40S API Server")
 
95
  with gr.Row():
96
  with gr.Column():
97
- img_input = gr.Image(type="pil", label="Input")
98
  txt_input = gr.Textbox(value="Convert to markdown.", label="Prompt")
99
  btn = gr.Button("Extract Markdown", variant="primary")
100
  with gr.Column():
@@ -102,43 +114,14 @@ with gr.Blocks() as demo:
102
 
103
  btn.click(run_ui_test, inputs=[img_input, txt_input], outputs=[out])
104
 
105
- # --- STEP 4: MOUNTING AND PROXY ---
106
-
107
- # 1. Start the Gradio app internal state
108
- # demo.queue() is CRITICAL for the button to work on HF Spaces
109
- demo.queue()
110
- demo.render()
111
- # 2. Patch the bug in Gradio 5 that causes the AttributeError
112
- demo.max_file_size = 100 * 1024 * 1024
113
- # 2. Grab the FastAPI app instance from Gradio
114
- app = demo.app
115
-
116
- # 3. Add the Proxy Route for your external Docling API
117
- @app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
118
- async def gatekeeper_proxy(path: str, request: Request):
119
- target_url = f"http://127.0.0.1:{VLLM_PORT}/v1/{path}"
120
- headers = {k: v for k, v in request.headers.items() if k.lower() not in ["host", "content-length"]}
121
-
122
- async with httpx.AsyncClient(timeout=300.0) as client:
123
- try:
124
- if path == "chat/completions" and request.method == "POST":
125
- body = await request.json()
126
- if not body.get("stream", False):
127
- resp = await client.post(target_url, headers=headers, json=body)
128
- if resp.status_code == 200:
129
- data = resp.json()
130
- content = data["choices"][0]["message"].get("content", "")
131
- if "</think>" in content:
132
- data["choices"][0]["message"]["content"] = content.split("</think>")[-1].strip()
133
- return JSONResponse(content=data)
134
- return JSONResponse(status_code=resp.status_code, content=resp.json())
135
 
136
- proxy_req = client.build_request(request.method, target_url, headers=headers, content=await request.body())
137
- r = await client.send(proxy_req, stream=True)
138
- return StreamingResponse(r.aiter_raw(), status_code=r.status_code, headers=dict(r.headers))
139
- except Exception as e:
140
- return JSONResponse(status_code=503, content={"error": str(e)})
141
 
142
  if __name__ == "__main__":
143
- # Standard HF uvicorn launch
144
  uvicorn.run(app, host="0.0.0.0", port=HF_PORT, workers=1)
 
4
  import httpx
5
  import json
6
  import base64
 
7
  from io import BytesIO
8
+ from fastapi import FastAPI, Request
9
+ from fastapi.responses import StreamingResponse, JSONResponse, RedirectResponse
10
  import uvicorn
11
  import gradio as gr
12
  from openai import OpenAI
 
18
  VLLM_PORT = 8000
19
  HF_PORT = 7860
20
 
21
+ # --- STEP 1: START vLLM ---
22
  def start_vllm():
23
  if "VLLM_PID" in os.environ:
24
  return
25
  print("--- STARTING VLLM ---")
26
+ command =[
27
  "python3", "-m", "vllm.entrypoints.openai.api_server",
28
  "--model", MODEL_ID,
29
  "--host", "127.0.0.1",
 
34
  "--dtype", "bfloat16",
35
  "--limit-mm-per-prompt", '{"image": 1}'
36
  ]
 
37
  proc = subprocess.Popen(command, stdout=sys.stdout, stderr=sys.stderr)
38
  os.environ["VLLM_PID"] = str(proc.pid)
 
39
 
40
  start_vllm()
41
 
42
+ # --- STEP 2: FASTAPI GATEKEEPER (EXTERNAL API) ---
43
+ app = FastAPI()
44
+
45
+ @app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
46
+ async def gatekeeper_proxy(path: str, request: Request):
47
+ """Intercepts external Docling API calls and strips reasoning tags."""
48
+ target_url = f"http://127.0.0.1:{VLLM_PORT}/v1/{path}"
49
+ headers = {k: v for k, v in request.headers.items() if k.lower() not in ["host", "content-length"]}
50
 
51
+ async with httpx.AsyncClient(timeout=300.0) as client:
52
+ try:
53
+ if path == "chat/completions" and request.method == "POST":
54
+ body = await request.json()
55
+ if not body.get("stream", False):
56
+ resp = await client.post(target_url, headers=headers, json=body)
57
+ if resp.status_code == 200:
58
+ data = resp.json()
59
+ content = data["choices"][0]["message"].get("content", "")
60
+ # SUPPRESS REASONING TAGS FOR EXTERNAL API
61
+ if "</think>" in content:
62
+ data["choices"][0]["message"]["content"] = content.split("</think>")[-1].strip()
63
+ return JSONResponse(content=data)
64
+ return JSONResponse(status_code=resp.status_code, content=resp.json())
65
+
66
+ proxy_req = client.build_request(request.method, target_url, headers=headers, content=await request.body())
67
+ r = await client.send(proxy_req, stream=True)
68
+ return StreamingResponse(r.aiter_raw(), status_code=r.status_code, headers=dict(r.headers))
69
+ except Exception as e:
70
+ return JSONResponse(status_code=503, content={"error": str(e)})
71
 
72
+ # --- STEP 3: GRADIO UI ---
73
+ def run_ui_test(image, prompt):
74
+ print(">>> UI Request Received")
75
+ if image is None: return "⚠️ Please upload an image."
76
+
77
  try:
78
  with httpx.Client() as check:
79
  check.get(f"http://127.0.0.1:{VLLM_PORT}/v1/models", timeout=2.0)
80
  except Exception:
81
+ return "⏳ Model is still loading (takes 3-5 mins). Please wait."
 
82
 
83
  client = OpenAI(base_url=f"http://127.0.0.1:{VLLM_PORT}/v1", api_key="EMPTY")
 
84
  try:
 
 
85
  image = image.convert("RGB")
86
  buffered = BytesIO()
87
  image.save(buffered, format="JPEG")
88
  b64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
89
 
 
90
  completion = client.chat.completions.create(
91
  model=MODEL_ID,
92
+ messages=[{"role": "user", "content":[
93
  {"type": "text", "text": prompt or "Convert to markdown."},
94
  {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}
95
  ]}],
96
  max_tokens=4096,
97
  timeout=300.0
98
  )
 
99
  content = completion.choices[0].message.content
100
+ return content.split("</think>")[-1].strip() if "</think>" in content else content
 
 
 
 
 
101
  except Exception as e:
 
102
  return f"❌ Error: {str(e)}"
103
 
104
+ with gr.Blocks(title="NuMarkdown API Server") as demo:
 
105
  gr.Markdown("# NuMarkdown L40S API Server")
106
+ gr.Markdown("The Markdown API endpoint is live externally at `/v1/chat/completions`.")
107
  with gr.Row():
108
  with gr.Column():
109
+ img_input = gr.Image(type="pil", label="Input Document")
110
  txt_input = gr.Textbox(value="Convert to markdown.", label="Prompt")
111
  btn = gr.Button("Extract Markdown", variant="primary")
112
  with gr.Column():
 
114
 
115
  btn.click(run_ui_test, inputs=[img_input, txt_input], outputs=[out])
116
 
117
+ # --- STEP 4: SAFE MOUNTING ---
118
+ # We mount Gradio to a subpath to prevent it from fighting with the Root path and breaking CSS/JS.
119
+ app = gr.mount_gradio_app(app, demo, path="/ui")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120
 
121
+ # Redirect anyone visiting the root URL to the UI.
122
+ @app.get("/")
123
+ def redirect_to_ui():
124
+ return RedirectResponse(url="/ui")
 
125
 
126
  if __name__ == "__main__":
 
127
  uvicorn.run(app, host="0.0.0.0", port=HF_PORT, workers=1)