# host models
## sotopia-sft-1
CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \
--model Qwen/Qwen2.5-7B-Instruct \
--port 8005 \
--chat-template /root/sotopia-rl/Untitled/qwen2.5-7b.jinja \
--enable-lora \
--lora-modules sotopia-sft-1=/root/sotopia-rl/Untitled/.cache/final_sft/checkpoint-1500/ \
--served-model-name xx

## sotopia-sft-2
CUDA_VISIBLE_DEVICES=1 python -m vllm.entrypoints.openai.api_server \
--model Qwen/Qwen2.5-7B-Instruct \
--port 8006 \
--chat-template /root/sotopia-rl/Untitled/qwen2.5-7b.jinja \
--enable-lora \
--lora-modules sotopia-sft-2=/root/sotopia-rl/Untitled/.cache/final_sft/checkpoint-1500/ \
--served-model-name xx

# generate run script
python examples/experiment_eval.py \
--gin_file sotopia_conf/generation_utils_conf/generate.gin \
--gin_file sotopia_conf/server_conf/server.gin \
--gin_file sotopia_conf/run_async_server_in_batch.gin \
'--gin.ENV_IDS=["01HMZ2BKZ51HQWXDH5T94YTEDB", "01HJPX50A0HE4X76P8CDR16ZW1", "01HMZ2BM1Y0RX2GKR1BRKZGV5E", "01HJPX504SAHM8GGMEZWXK47BS", "01HJRKDXQCJ62M0KH5QNDM0FAF", "01HJRKDXWH0ZJ1S9TSHGZ3E1BQ", "01HJPX505GZMHWT9TGM7VYZ209", "01HMZ2BM4REV6PYBCJ3ME5YWGM", "01HJRKDXYSQTGXZHKGB3TQ0PBY", "01HJPTHM8K2X1DTBR603YJ1C85", "01HMZ2BKRT5JMQP9T73SQ1MMC4", "01HJRKDXQX51P4XMQ27V6N88M2", "01HJRKDXXMV0P5D4YWWXAP9FKQ", "01HMZ2BKSBQ0KDKQBM1KD4BSS5", "01HJPX4ZXG9PYH0YPSZFY325QQ", "01HMZ2BKDPGJK4N0MGKBERB3AV", "01HMZ2BKQ41HMZ4KQNVMRXJN80", "01HJPX4ZVBY08X7GR6NGSMRCHP", "01HMZ2BM65ET1D1C5N9ZQ80MSS", "01HJPTHME7PKP03KQKQGBW49QW", "01HJPX50C80P0SDKXF3D0R9E9Q", "01HJPX507NFSWYB2GD4SBCJ984", "01HMZ2BKF3BNDKB2E7J9TG4CFJ", "01HJPTHMA013J4FXGAQC0DPNRW", "01HJPX5091VCZN14A4P0WMNG0J", "01HJPX50425WQK2TM93Z0HXPSH", "01HJPTHMES6NXDXWT1KXC6AMF1", "01HJPTHMJ5E9YCW0J62D88DSF5", "01HJRKDXMT0DQN2CHN5ZDV47BP", "01HJPTHMMTYA77EM2D6Q8P595V", "01HJPTHMQ0G2AF53V3AXBX6ZVH", "01HJPX506XJBV03HWPXFQQD8EM", "01HJPTHMJP835BVF53ZXX1E7QF", "01HJPX51E5MMN673PJZZ0X4WN9", "01HJRKDYDPVW5NVZNK94ZC9PT5", "01HJRH6SR4C1ZMXMC3Z4HCKB0H", "01HJPX51NQ377VTQ1HSM0QJ4WK", "01HMZ2BNVN9NH7MF1YS1Q447V2", "01HMZ2BNNG00FF6W4TTPR58JF5", "01HMZ2BNRZQ6XM9E01D1CPM841", "01HMZ2BNGHM28A1J55NPBGGVMS", "01HJPX51AJ3NY2SJB32ZQ6AEMV", "01HMZ2BNBSE1FBG01PKKSQK8K1", "01HJPTHP37YD8JJ2FD1ANQN6ND", "01HMZ2BNHZC862J399QF33XZ1G", "01HMZ2BN6APAHQ4R4MX2ZXTX4R", "01HJPX511K4ZXVW8KB142FJC7J", "01HJRH6SPANEYBP8VMQZ706Y4G", "01HJPX51KMNFG1Y7E96YSY3SFQ", "01HJRH6SSCXWMQAFY07HE1J0A9", "01HJPZ3DJZD26MGCQBJXW30EC7", "01HJRH6SMK8VBHCH1Q5H7EMRD3", "01HJRKDYKEKD0X5X6APF4WTD4H", "01HJRKDYPDP57QTD5WTSSAYCE5", "01HJPTHP6FKCFZNXG63EP58GS5", "01HJPTHP3MM4AC8V9J9Y7J7849", "01HJRKDYCDQBB46RG5SP9Q9K6G", "01HJPX51B9Y1S7EX1XPR19D268", "01HMZ2BNDX9E8R33TZ9BD0YKKA", "01HJRKDYGZQVC14PN3HBN1BCA3", "01HMZ2BN2S2GN52DWBJ3D9195Q", "01HJPTHPC7AJ3R3ZW664J94DNN", "01HMZ2BND6FYKBMQN8J6DVQTP9", "01HJPX515GSBQ31JTZB5672YCA", "01HMZ2BN3GZMHQMJEDCARYR511", "01HJPX51G9X7C89AZKVP8V4JDW", "01HMZ2BN8D2GV68P6HT2KKFWRC", "01HMZ2BMBR35HYMYJET0SDM0GB", "01HJRH6SGMFNMT5WZ2ECTVN9HV", "01HMZ2BMDW41B1V8K5K9RVBK5Z", "01HJRH6SA7PYM9RK17FVZPN4JX", "01HJPX50TZG7Y7EX33BMBRWRGD", "01HJPTHN9S14FT3R2950MM6F0R", "01HJPX50WZV40R2VJVJQT89GE5", "01HJRKDY5JEPFX5JXSRTQ5YHDE", "01HMZ2BMD6M6278J5TYAD59JJS", "01HJPTHN6HJAB68ENA63DJ5EDZ", "01HMZ2BMXBCM5001DQETYCJFYG", "01HJPX50Q6ZGF87QZMZKXK0B6N", "01HJRKDY65N420PRKFWRZD35FB", "01HJRH6SBCNM0QVYY8FE1S6F28", "01HJRKDY4D0HY8SFERXCJWQM9X", "01HMZ2BMRJMC23KDJYB8R73HN7", "01HMZ2BMEKCMQTGW5X37Y49TNQ", "01HJPTHNQ9RH8GMRNZWPDANXM0", "01HMZ2BN0NNCZ8QDMNFV1X5VF6", "01HJRH6SDBTBH91QZDB5RWSDZP", "01HJPX50FE3XYZBBPYWYHNM43J", "01HMZ2BM7H3D1WPW5C73TPKFJP", "01HMZ2BMV3TS3WNMY59QVDXJRA", "01HJPX50S9JA38NVE718HM9VJT", "01HJPX50P3T2SBXWW4NJ72BMD9", "01HJPTHN7YQR6WT7Y3Z7W8N1MD", "01HMZ2BMK3PDA7VHYCK0T1CQGP", "01HJPX50M43DZG8PV5R25MJ15Y", "01HJPX50JEYPPAP0140XBGSE9W", "01HMZ2BMCFEGGRW64ZX6J28WSE", "01HJPX50SQCNTG4HX1CAE798W4", "01HMZ2BMPJC1EV1H88YSZVX8HQ", "01HJPX50MTVB1WC1HZ16HRPKZC"]' \
'--gin.ENV_MODEL="gpt-4o"' \
'--gin.AGENT1_MODEL="custom/sotopia-sft-1@http://localhost:8005/v1"' \
'--gin.AGENT2_MODEL="custom/sotopia-sft-2@http://localhost:8006/v1"' \
'--gin.BATCH_SIZE=20' \
'--gin.TAG="qwen-sft-qwen-sft-3-26-v2"' \
'--gin.PUSH_TO_DB=True' \
'--gin.TAG_TO_CHECK_EXISTING_EPISODES="qwen-sft-qwen-sft-3-26-v2"'


python examples/experiment_eval.py \
--gin_file sotopia_conf/generation_utils_conf/generate.gin \
--gin_file sotopia_conf/server_conf/server.gin \
--gin_file sotopia_conf/run_async_server_in_batch.gin \
'--gin.ENV_IDS=["01HMZ2BKZ51HQWXDH5T94YTEDB", "01HJPX50A0HE4X76P8CDR16ZW1", "01HMZ2BM1Y0RX2GKR1BRKZGV5E", "01HJPX504SAHM8GGMEZWXK47BS", "01HJRKDXQCJ62M0KH5QNDM0FAF", "01HJRKDXWH0ZJ1S9TSHGZ3E1BQ", "01HJPX505GZMHWT9TGM7VYZ209", "01HMZ2BM4REV6PYBCJ3ME5YWGM", "01HJRKDXYSQTGXZHKGB3TQ0PBY", "01HJPTHM8K2X1DTBR603YJ1C85", "01HMZ2BKRT5JMQP9T73SQ1MMC4", "01HJRKDXQX51P4XMQ27V6N88M2", "01HJRKDXXMV0P5D4YWWXAP9FKQ", "01HMZ2BKSBQ0KDKQBM1KD4BSS5", "01HJPX4ZXG9PYH0YPSZFY325QQ", "01HMZ2BKDPGJK4N0MGKBERB3AV", "01HMZ2BKQ41HMZ4KQNVMRXJN80", "01HJPX4ZVBY08X7GR6NGSMRCHP", "01HMZ2BM65ET1D1C5N9ZQ80MSS", "01HJPTHME7PKP03KQKQGBW49QW", "01HJPX50C80P0SDKXF3D0R9E9Q", "01HJPX507NFSWYB2GD4SBCJ984", "01HMZ2BKF3BNDKB2E7J9TG4CFJ", "01HJPTHMA013J4FXGAQC0DPNRW", "01HJPX5091VCZN14A4P0WMNG0J", "01HJPX50425WQK2TM93Z0HXPSH", "01HJPTHMES6NXDXWT1KXC6AMF1", "01HJPTHMJ5E9YCW0J62D88DSF5", "01HJRKDXMT0DQN2CHN5ZDV47BP", "01HJPTHMMTYA77EM2D6Q8P595V", "01HJPTHMQ0G2AF53V3AXBX6ZVH", "01HJPX506XJBV03HWPXFQQD8EM", "01HJPTHMJP835BVF53ZXX1E7QF", "01HJPX51E5MMN673PJZZ0X4WN9", "01HJRKDYDPVW5NVZNK94ZC9PT5", "01HJRH6SR4C1ZMXMC3Z4HCKB0H", "01HJPX51NQ377VTQ1HSM0QJ4WK", "01HMZ2BNVN9NH7MF1YS1Q447V2", "01HMZ2BNNG00FF6W4TTPR58JF5", "01HMZ2BNRZQ6XM9E01D1CPM841", "01HMZ2BNGHM28A1J55NPBGGVMS", "01HJPX51AJ3NY2SJB32ZQ6AEMV", "01HMZ2BNBSE1FBG01PKKSQK8K1", "01HJPTHP37YD8JJ2FD1ANQN6ND", "01HMZ2BNHZC862J399QF33XZ1G", "01HMZ2BN6APAHQ4R4MX2ZXTX4R", "01HJPX511K4ZXVW8KB142FJC7J", "01HJRH6SPANEYBP8VMQZ706Y4G", "01HJPX51KMNFG1Y7E96YSY3SFQ", "01HJRH6SSCXWMQAFY07HE1J0A9", "01HJPZ3DJZD26MGCQBJXW30EC7", "01HJRH6SMK8VBHCH1Q5H7EMRD3", "01HJRKDYKEKD0X5X6APF4WTD4H", "01HJRKDYPDP57QTD5WTSSAYCE5", "01HJPTHP6FKCFZNXG63EP58GS5", "01HJPTHP3MM4AC8V9J9Y7J7849", "01HJRKDYCDQBB46RG5SP9Q9K6G", "01HJPX51B9Y1S7EX1XPR19D268", "01HMZ2BNDX9E8R33TZ9BD0YKKA", "01HJRKDYGZQVC14PN3HBN1BCA3", "01HMZ2BN2S2GN52DWBJ3D9195Q", "01HJPTHPC7AJ3R3ZW664J94DNN", "01HMZ2BND6FYKBMQN8J6DVQTP9", "01HJPX515GSBQ31JTZB5672YCA", "01HMZ2BN3GZMHQMJEDCARYR511", "01HJPX51G9X7C89AZKVP8V4JDW", "01HMZ2BN8D2GV68P6HT2KKFWRC", "01HMZ2BMBR35HYMYJET0SDM0GB", "01HJRH6SGMFNMT5WZ2ECTVN9HV", "01HMZ2BMDW41B1V8K5K9RVBK5Z", "01HJRH6SA7PYM9RK17FVZPN4JX", "01HJPX50TZG7Y7EX33BMBRWRGD", "01HJPTHN9S14FT3R2950MM6F0R", "01HJPX50WZV40R2VJVJQT89GE5", "01HJRKDY5JEPFX5JXSRTQ5YHDE", "01HMZ2BMD6M6278J5TYAD59JJS", "01HJPTHN6HJAB68ENA63DJ5EDZ", "01HMZ2BMXBCM5001DQETYCJFYG", "01HJPX50Q6ZGF87QZMZKXK0B6N", "01HJRKDY65N420PRKFWRZD35FB", "01HJRH6SBCNM0QVYY8FE1S6F28", "01HJRKDY4D0HY8SFERXCJWQM9X", "01HMZ2BMRJMC23KDJYB8R73HN7", "01HMZ2BMEKCMQTGW5X37Y49TNQ", "01HJPTHNQ9RH8GMRNZWPDANXM0", "01HMZ2BN0NNCZ8QDMNFV1X5VF6", "01HJRH6SDBTBH91QZDB5RWSDZP", "01HJPX50FE3XYZBBPYWYHNM43J", "01HMZ2BM7H3D1WPW5C73TPKFJP", "01HMZ2BMV3TS3WNMY59QVDXJRA", "01HJPX50S9JA38NVE718HM9VJT", "01HJPX50P3T2SBXWW4NJ72BMD9", "01HJPTHN7YQR6WT7Y3Z7W8N1MD", "01HMZ2BMK3PDA7VHYCK0T1CQGP", "01HJPX50M43DZG8PV5R25MJ15Y", "01HJPX50JEYPPAP0140XBGSE9W", "01HMZ2BMCFEGGRW64ZX6J28WSE", "01HJPX50SQCNTG4HX1CAE798W4", "01HMZ2BMPJC1EV1H88YSZVX8HQ", "01HJPX50MTVB1WC1HZ16HRPKZC"]' \
'--gin.ENV_MODEL="gpt-4o"' \
'--gin.AGENT1_MODEL="custom/sotopia-sft-2@http://localhost:8006/v1"' \
'--gin.AGENT2_MODEL="custom/sotopia-sft-1@http://localhost:8005/v1"' \
'--gin.BATCH_SIZE=20' \
'--gin.TAG="qwen-sft-qwen-sft-3-26-v2"' \
'--gin.PUSH_TO_DB=True' \
'--gin.TAG_TO_CHECK_EXISTING_EPISODES="qwen-sft-qwen-sft-3-26-v2"'