encryptd commited on
Commit
2025eba
·
1 Parent(s): de3235a

prog update

Browse files
Files changed (1) hide show
  1. app.py +48 -51
app.py CHANGED
@@ -5,7 +5,7 @@ import httpx
5
  import json
6
  import base64
7
  from io import BytesIO
8
- from fastapi import Request
9
  from fastapi.responses import StreamingResponse, JSONResponse
10
  import uvicorn
11
  import gradio as gr
@@ -13,12 +13,12 @@ from openai import OpenAI
13
 
14
  # --- CONFIGURATION ---
15
  MODEL_ID = "numind/NuMarkdown-8B-Thinking"
16
- GPU_UTILIZATION = 0.90
17
  MAX_MODEL_LEN = 32768
18
  VLLM_PORT = 8000
19
- HF_PORT = 7860 # Hugging Face mandatory port
20
 
21
- # --- STEP 1: START vLLM (Local Subprocess) ---
22
  def start_vllm():
23
  if "VLLM_PID" in os.environ:
24
  return
@@ -33,13 +33,42 @@ def start_vllm():
33
  "--dtype", "bfloat16",
34
  "--limit-mm-per-prompt", '{"image": 1}'
35
  ]
36
- # We use devnull for stdout to keep logs clean, or sys.stdout to debug
37
  proc = subprocess.Popen(command, stdout=sys.stdout, stderr=sys.stderr)
38
  os.environ["VLLM_PID"] = str(proc.pid)
39
 
40
  start_vllm()
41
 
42
- # --- STEP 2: DEFINE GRADIO UI ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  def run_ui_test(image, prompt):
44
  if image is None: return "⚠️ Please upload an image."
45
  client = OpenAI(base_url=f"http://127.0.0.1:{VLLM_PORT}/v1", api_key="EMPTY")
@@ -47,7 +76,6 @@ def run_ui_test(image, prompt):
47
  buffered = BytesIO()
48
  image.save(buffered, format="JPEG")
49
  b64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
50
-
51
  completion = client.chat.completions.create(
52
  model=MODEL_ID,
53
  messages=[{"role": "user", "content": [
@@ -56,65 +84,34 @@ def run_ui_test(image, prompt):
56
  ]}]
57
  )
58
  content = completion.choices[0].message.content
59
- # Suppress reasoning in UI
60
  return content.split("</think>")[-1].strip() if "</think>" in content else content
61
  except Exception as e:
62
  return f"❌ Error: {e}"
63
 
64
  with gr.Blocks(title="NuMarkdown API") as demo:
65
  gr.Markdown("# NuMarkdown L40S API Server")
66
- gr.Markdown("### 📡 External API Info")
67
- gr.Markdown(f"Endpoint: `https://{os.getenv('SPACE_ID','').replace('/','-')}.hf.space/v1/chat/completions` (OpenAI Compatible)")
68
-
69
  with gr.Row():
70
  with gr.Column():
71
  img_input = gr.Image(type="pil", label="Input Document")
72
  txt_input = gr.Textbox(value="Convert to markdown.", label="Prompt")
73
  btn = gr.Button("Extract Markdown", variant="primary")
74
  with gr.Column():
75
- out = gr.Markdown(label="Clean Markdown Output")
76
-
77
  btn.click(run_ui_test, inputs=[img_input, txt_input], outputs=[out])
78
 
79
- # --- STEP 3: INTERCEPT API CALLS ON GRADIO'S APP ---
80
- # This is the "magic" part. We use the FastAPI app already created by Gradio.
81
- app = demo.app
82
 
83
- @app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
84
- async def proxy_v1(path: str, request: Request):
85
- target_url = f"http://127.0.0.1:{VLLM_PORT}/v1/{path}"
86
-
87
- # Strip sensitive headers that cause HF routing loops
88
- headers = {k: v for k, v in request.headers.items() if k.lower() not in ["host", "content-length"]}
89
-
90
- async with httpx.AsyncClient() as client:
91
- try:
92
- if path == "chat/completions" and request.method == "POST":
93
- body = await request.json()
94
-
95
- # Intercept non-streaming (Docling)
96
- if not body.get("stream", False):
97
- resp = await client.post(target_url, headers=headers, json=body, timeout=300.0)
98
- if resp.status_code == 200:
99
- data = resp.json()
100
- content = data["choices"][0]["message"].get("content", "")
101
- # SUPPRESS REASONING FOR EXTERNAL API
102
- if "</think>" in content:
103
- data["choices"][0]["message"]["content"] = content.split("</think>")[-1].strip()
104
- return JSONResponse(content=data)
105
- return JSONResponse(status_code=resp.status_code, content=resp.json())
106
 
107
- # Forward everything else (streaming, models list, etc)
108
- proxy_req = client.build_request(
109
- request.method, target_url, headers=headers, content=await request.body()
110
- )
111
- r = await client.send(proxy_req, stream=True)
112
- return StreamingResponse(r.aiter_raw(), status_code=r.status_code, headers=dict(r.headers))
113
-
114
- except Exception as e:
115
- return JSONResponse(status_code=503, content={"error": f"vLLM Proxy Error: {str(e)}"})
116
 
117
- # --- STEP 4: RUN ---
118
  if __name__ == "__main__":
119
- # In HF Spaces, uvicorn should run on 0.0.0.0:7860
120
- uvicorn.run(app, host="0.0.0.0", port=HF_PORT)
 
5
  import json
6
  import base64
7
  from io import BytesIO
8
+ from fastapi import FastAPI, Request
9
  from fastapi.responses import StreamingResponse, JSONResponse
10
  import uvicorn
11
  import gradio as gr
 
13
 
14
  # --- CONFIGURATION ---
15
  MODEL_ID = "numind/NuMarkdown-8B-Thinking"
16
+ GPU_UTILIZATION = 0.95
17
  MAX_MODEL_LEN = 32768
18
  VLLM_PORT = 8000
19
+ HF_PORT = 7860
20
 
21
+ # --- STEP 1: START vLLM ---
22
  def start_vllm():
23
  if "VLLM_PID" in os.environ:
24
  return
 
33
  "--dtype", "bfloat16",
34
  "--limit-mm-per-prompt", '{"image": 1}'
35
  ]
 
36
  proc = subprocess.Popen(command, stdout=sys.stdout, stderr=sys.stderr)
37
  os.environ["VLLM_PID"] = str(proc.pid)
38
 
39
  start_vllm()
40
 
41
+ # --- STEP 2: DEFINE THE PROXY LOGIC ---
42
+ # We create a standalone FastAPI app for the API logic
43
+ api_app = FastAPI()
44
+
45
+ @api_app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
46
+ async def proxy_v1(path: str, request: Request):
47
+ target_url = f"http://127.0.0.1:{VLLM_PORT}/v1/{path}"
48
+ headers = {k: v for k, v in request.headers.items() if k.lower() not in ["host", "content-length"]}
49
+
50
+ async with httpx.AsyncClient() as client:
51
+ try:
52
+ if path == "chat/completions" and request.method == "POST":
53
+ body = await request.json()
54
+ if not body.get("stream", False):
55
+ resp = await client.post(target_url, headers=headers, json=body, timeout=300.0)
56
+ if resp.status_code == 200:
57
+ data = resp.json()
58
+ content = data["choices"][0]["message"].get("content", "")
59
+ # SUPPRESS REASONING FOR EXTERNAL DOCLING API
60
+ if "</think>" in content:
61
+ data["choices"][0]["message"]["content"] = content.split("</think>")[-1].strip()
62
+ return JSONResponse(content=data)
63
+ return JSONResponse(status_code=resp.status_code, content=resp.json())
64
+
65
+ proxy_req = client.build_request(request.method, target_url, headers=headers, content=await request.body())
66
+ r = await client.send(proxy_req, stream=True)
67
+ return StreamingResponse(r.aiter_raw(), status_code=r.status_code, headers=dict(r.headers))
68
+ except Exception as e:
69
+ return JSONResponse(status_code=503, content={"error": f"vLLM Proxy Error: {str(e)}"})
70
+
71
+ # --- STEP 3: GRADIO UI ---
72
  def run_ui_test(image, prompt):
73
  if image is None: return "⚠️ Please upload an image."
74
  client = OpenAI(base_url=f"http://127.0.0.1:{VLLM_PORT}/v1", api_key="EMPTY")
 
76
  buffered = BytesIO()
77
  image.save(buffered, format="JPEG")
78
  b64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
 
79
  completion = client.chat.completions.create(
80
  model=MODEL_ID,
81
  messages=[{"role": "user", "content": [
 
84
  ]}]
85
  )
86
  content = completion.choices[0].message.content
 
87
  return content.split("</think>")[-1].strip() if "</think>" in content else content
88
  except Exception as e:
89
  return f"❌ Error: {e}"
90
 
91
  with gr.Blocks(title="NuMarkdown API") as demo:
92
  gr.Markdown("# NuMarkdown L40S API Server")
 
 
 
93
  with gr.Row():
94
  with gr.Column():
95
  img_input = gr.Image(type="pil", label="Input Document")
96
  txt_input = gr.Textbox(value="Convert to markdown.", label="Prompt")
97
  btn = gr.Button("Extract Markdown", variant="primary")
98
  with gr.Column():
99
+ out = gr.Markdown(label="Output")
 
100
  btn.click(run_ui_test, inputs=[img_input, txt_input], outputs=[out])
101
 
102
+ # --- STEP 4: PATCH AND MOUNT ---
 
 
103
 
104
+ # FIX FOR ATTRIBUTE ERROR: Manually set max_file_size before mounting
105
+ demo.max_file_size = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
 
107
+ # Create the main app and mount BOTH the API and the UI
108
+ app = FastAPI()
109
+
110
+ # 1. Mount the Gradio UI at the root
111
+ app = gr.mount_gradio_app(app, demo, path="/")
112
+
113
+ # 2. Mount the API proxy logic
114
+ app.mount("/", api_app)
 
115
 
 
116
  if __name__ == "__main__":
117
+ uvicorn.run(app, host="0.0.0.0", port=HF_PORT, workers=1)