update

app.py

@@ -558,40 +558,20 @@ def start_worker(model_path: str, bits=16):
     worker_command = [
         "python",
         "-m",
-        "llava.serve.sglang_worker",
+        "llava.serve.model_worker",
         "--host",
         "0.0.0.0",
         "--controller",
         "http://localhost:10000",
-        "--port",
-        "30000",
-        "--worker",
-        "http://localhost:40000",
-        "--sgl-endpoint",
-        "http://127.0.0.1:30000",
+        "--model-path",
+        model_path,
+        "--model-name",
+        model_name,
     ]
     if bits != 16:
         worker_command += [f"--load-{bits}bit"]
     return subprocess.Popen(worker_command)
 
-def start_sglang_backend_worker(model_path: str, bits=16):
-    logger.info(f"Starting the model worker for the model {model_path}")
-    model_name = model_path.strip("/").split("/")[-1]
-    assert bits in [4, 8, 16], "It can be only loaded with 16-bit, 8-bit, and 4-bit."
-    if bits != 16:
-        model_name += f"-{bits}bit"
-    worker_command = [
-        "python3",
-        "-m",
-        "sglang.launch_server",
-        "--model-path",
-        model_path,
-        "--tokenizer-path",
-        "llava-hf/llava-1.5-7b-hf",
-        "--port",
-        "30000",
-    ]
-    return subprocess.Popen(worker_command)
 
 def get_args():
     parser = argparse.ArgumentParser()
@@ -626,7 +606,6 @@ if __name__ == "__main__":
     bits = int(os.getenv("bits", 16))
 
     controller_proc = start_controller()
-    sglang_backend_worker_proc = start_sglang_backend_worker(model_path, bits=bits)
     worker_proc = start_worker(model_path, bits=bits)
 
     # Wait for worker and controller to start
@@ -640,7 +619,6 @@ if __name__ == "__main__":
         exit_status = 1
     finally:
         worker_proc.kill()
-        sglang_backend_worker_proc.kill()
         controller_proc.kill()
 
         sys.exit(exit_status)

install.sh

@@ -1,4 +0,0 @@
-
-#!/usr/bin/env bash
-pip uninstall -y numpy
-pip install --no-deps --prefer-binary numpy==1.23.5

requirements.txt

@@ -1,19 +1,16 @@
 --extra-index-url https://download.pytorch.org/whl/cu121
 pip
-numpy==1.23.5
 einops
 fastapi
 gradio==3.35.2
 markdown2[all]
+numpy<2.0.0
 requests
 sentencepiece
 tokenizers>=0.12.1
 torch==2.1.0+cu121
 torchvision==0.16.0+cu121
-sglang[all]>=0.4.6.post2
 uvicorn
-uvloop
-pyzmq
 wandb
 shortuuid
 httpx==0.24.0
@@ -27,4 +24,4 @@ sentencepiece==0.1.99
 einops==0.6.1
 einops-exts==0.0.4
 timm==0.6.13
-gradio_client==0.2.9
+gradio_client==0.2.9