AI Agent commited on
Commit Β·
fbc58fa
1
Parent(s): 4d634bb
Update nvidia_llm to use minimaxai/minimax-m2.7 and new sampling params
Browse files- agents/nvidia_llm.py +4 -4
agents/nvidia_llm.py
CHANGED
|
@@ -24,7 +24,7 @@ NVIDIA_API_KEY = os.getenv("NVIDIA_API_KEY")
|
|
| 24 |
_QUEUE_MAX_SIZE = int(os.getenv("NVIDIA_QUEUE_MAX", "8"))
|
| 25 |
_REQUEST_TIMEOUT_S = int(os.getenv("NVIDIA_LLM_TIMEOUT", "600"))
|
| 26 |
|
| 27 |
-
DEFAULT_MODEL = "
|
| 28 |
|
| 29 |
# ββ Flask app βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 30 |
app = Flask(__name__)
|
|
@@ -54,9 +54,9 @@ def _run_inference(data: dict) -> dict:
|
|
| 54 |
|
| 55 |
prompts = raw_prompt if isinstance(raw_prompt, list) and (len(raw_prompt) == 0 or not isinstance(raw_prompt[0], dict)) else [raw_prompt]
|
| 56 |
|
| 57 |
-
max_tokens = int(data.get("max_tokens",
|
| 58 |
-
temperature = float(data.get("temperature",
|
| 59 |
-
top_p = float(data.get("top_p", 0.
|
| 60 |
model_name = data.get("model", DEFAULT_MODEL)
|
| 61 |
|
| 62 |
choices = []
|
|
|
|
| 24 |
_QUEUE_MAX_SIZE = int(os.getenv("NVIDIA_QUEUE_MAX", "8"))
|
| 25 |
_REQUEST_TIMEOUT_S = int(os.getenv("NVIDIA_LLM_TIMEOUT", "600"))
|
| 26 |
|
| 27 |
+
DEFAULT_MODEL = "minimaxai/minimax-m2.7"
|
| 28 |
|
| 29 |
# ββ Flask app βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 30 |
app = Flask(__name__)
|
|
|
|
| 54 |
|
| 55 |
prompts = raw_prompt if isinstance(raw_prompt, list) and (len(raw_prompt) == 0 or not isinstance(raw_prompt[0], dict)) else [raw_prompt]
|
| 56 |
|
| 57 |
+
max_tokens = int(data.get("max_tokens", 8192))
|
| 58 |
+
temperature = float(data.get("temperature", 1.0))
|
| 59 |
+
top_p = float(data.get("top_p", 0.95))
|
| 60 |
model_name = data.get("model", DEFAULT_MODEL)
|
| 61 |
|
| 62 |
choices = []
|