File size: 2,720 Bytes
a402b9b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=66, suite="stage-b-test-small-1-gpu")
register_amd_ci(est_time=66, suite="stage-b-test-small-1-gpu-amd")
class TestPyTorchSamplingBackend(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--sampling-backend", "pytorch", "--disable-radix-cache"],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
temperature=0.1,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
def test_greedy(self):
first_text = None
# ensure the answer is identical across single response
for _ in range(5):
response_single = requests.post(
self.base_url + "/generate",
json={
"text": "The capital of Germany is",
"sampling_params": {
"temperature": 0,
"max_new_tokens": 32,
},
},
).json()
text = response_single["text"]
if first_text is None:
first_text = text
self.assertEqual(text, first_text)
first_text = None
response_batch = requests.post(
self.base_url + "/generate",
json={
"text": ["The capital of Germany is"] * 10,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 32,
},
},
).json()
# ensure the answer is identical among the batch
for i in range(10):
text = response_batch[i]["text"]
if first_text is None:
first_text = text
self.assertEqual(text, first_text)
if __name__ == "__main__":
unittest.main()
|