poolside

/

Laguna-M.1-NVFP4

like 11

Follow

Poolside 463

First up, thanks for the model! Not entirely sure of the best place to post this, happy to open a vLLM issue if the problem is on their end.

The following error is thrown when attempting to make a tool call request to the Responses API in vLLM

APIServer pid=27073) INFO:     192.168.0.36:0 - "POST /v1/responses HTTP/1.1" 200 OK
(APIServer pid=27073) ERROR:    Exception in ASGI application
(APIServer pid=27073) Traceback (most recent call last):
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/uvicorn/protocols/http/httptools_impl.py", line 421, in run_asgi
(APIServer pid=27073)     result = await app(  # type: ignore[func-returns-value]
(APIServer pid=27073)              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=27073)         self.scope, self.receive, self.send
(APIServer pid=27073)         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=27073)     )
(APIServer pid=27073)     ^
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/uvicorn/middleware/proxy_headers.py", line 62, in __call__
(APIServer pid=27073)     return await self.app(scope, receive, send)
(APIServer pid=27073)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/fastapi/applications.py", line 1159, in __call__
(APIServer pid=27073)     await super().__call__(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/applications.py", line 90, in __call__
(APIServer pid=27073)     await self.middleware_stack(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/middleware/errors.py", line 186, in __call__
(APIServer pid=27073)     raise exc
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/middleware/errors.py", line 164, in __call__
(APIServer pid=27073)     await self.app(scope, receive, _send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/middleware/cors.py", line 88, in __call__
(APIServer pid=27073)     await self.app(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 176, in __call__
(APIServer pid=27073)     raise exc
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 174, in __call__
(APIServer pid=27073)     await self.app(scope, receive, send_wrapper)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
(APIServer pid=27073)     await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
(APIServer pid=27073)     raise exc
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
(APIServer pid=27073)     await app(scope, receive, sender)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
(APIServer pid=27073)     await self.app(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/routing.py", line 660, in __call__
(APIServer pid=27073)     await self.middleware_stack(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/routing.py", line 680, in app
(APIServer pid=27073)     await route.handle(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/routing.py", line 276, in handle
(APIServer pid=27073)     await self.app(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/fastapi/routing.py", line 134, in app
(APIServer pid=27073)     await wrap_app_handling_exceptions(app, request)(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
(APIServer pid=27073)     raise exc
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
(APIServer pid=27073)     await app(scope, receive, sender)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/fastapi/routing.py", line 121, in app
(APIServer pid=27073)     await response(scope, receive, send)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/responses.py", line 273, in __call__
(APIServer pid=27073)     async with create_collapsing_task_group() as task_group:
(APIServer pid=27073)                ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
(APIServer pid=27073)   File "/home/hubby/.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/contextlib.py", line 235, in __aexit__
(APIServer pid=27073)     await self.gen.athrow(value)
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/_utils.py", line 93, in create_collapsing_task_group
(APIServer pid=27073)     raise exc from exc.__cause__ or context
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 278, in _run_coro
(APIServer pid=27073)     retval = await self._coro
(APIServer pid=27073)              ^^^^^^^^^^^^^^^^
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/responses.py", line 276, in wrap
(APIServer pid=27073)     await func()
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/starlette/responses.py", line 250, in stream_response
(APIServer pid=27073)     async for chunk in self.body_iterator:
(APIServer pid=27073)     ...<2 lines>...
(APIServer pid=27073)         await send({"type": "http.response.body", "body": chunk, "more_body": True})
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/vllm/entrypoints/openai/responses/api_router.py", line 38, in _convert_stream_to_sse_events
(APIServer pid=27073)     async for event in generator:
(APIServer pid=27073)     ...<6 lines>...
(APIServer pid=27073)         yield event_data
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/vllm/entrypoints/openai/responses/serving.py", line 1518, in responses_stream_generator
(APIServer pid=27073)     async for event_data in processor(
(APIServer pid=27073)     ...<10 lines>...
(APIServer pid=27073)         yield event_data
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/vllm/entrypoints/openai/responses/serving.py", line 1386, in _process_simple_streaming_events
(APIServer pid=27073)     delta_message = parser.parse_delta(
(APIServer pid=27073)         delta_text=delta_text,
(APIServer pid=27073)     ...<3 lines>...
(APIServer pid=27073)         finished=output.finish_reason is not None,
(APIServer pid=27073)     )
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/vllm/parser/abstract_parser.py", line 825, in parse_delta
(APIServer pid=27073)     self._extract_tool_calls_streaming(
(APIServer pid=27073)     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(APIServer pid=27073)         previous_text=state.previous_text,
(APIServer pid=27073)         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=27073)     ...<8 lines>...
(APIServer pid=27073)         function_name_returned=state.function_name_returned,
(APIServer pid=27073)         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=27073)     )
(APIServer pid=27073)     ^
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/vllm/parser/abstract_parser.py", line 655, in _extract_tool_calls_streaming
(APIServer pid=27073)     return self.extract_tool_calls_streaming(
(APIServer pid=27073)            ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(APIServer pid=27073)         previous_text,
(APIServer pid=27073)         ^^^^^^^^^^^^^^
(APIServer pid=27073)     ...<5 lines>...
(APIServer pid=27073)         request,
(APIServer pid=27073)         ^^^^^^^^
(APIServer pid=27073)     ), False
(APIServer pid=27073)     ^
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/vllm/parser/abstract_parser.py", line 567, in extract_tool_calls_streaming
(APIServer pid=27073)     result = self._tool_parser.extract_tool_calls_streaming(
(APIServer pid=27073)         previous_text,
(APIServer pid=27073)     ...<5 lines>...
(APIServer pid=27073)         request,  # type: ignore[arg-type]
(APIServer pid=27073)     )
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/vllm/tool_parsers/poolside_v1_tool_parser.py", line 427, in extract_tool_calls_streaming
(APIServer pid=27073)     if request.logprobs:
(APIServer pid=27073)        ^^^^^^^^^^^^^^^^
(APIServer pid=27073)   File "/home/hubby/.venv/lib/python3.13/site-packages/pydantic/main.py", line 1042, in __getattr__
(APIServer pid=27073)     raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}')
(APIServer pid=27073) AttributeError: 'ResponsesRequest' object has no attribute 'logprobs'
(APIServer pid=27073) INFO 06-21 15:31:00 [loggers.py:273] Engine 000: Avg prompt throughput: 32.3 tokens/s, Avg generation throughput: 3.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 97.9%
(APIServer pid=27073) INFO 06-21 15:31:10 [loggers.py:273] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 97.9%

You can reproduce this bug by running the sample code at https://github.com/vllm-project/vllm/blob/main/examples/tool_calling/openai_responses_client_with_tools.py