initial version fromm iiced

main.py

@@ -12,8 +12,10 @@ class Item(BaseModel):
     prompt: str
     history: list
     system_prompt: str
-    temperature: float = 0.3
-    max_new_tokens: int = 4000
+    temperature: float = 0.0
+    max_new_tokens: int = 1048
+    top_p: float = 0.15
+    repetition_penalty: float = 1.0
 
 def format_prompt(message, history):
     prompt = "<s>"
@@ -27,15 +29,19 @@ def generate(item: Item):
     temperature = float(item.temperature)
     if temperature < 1e-2:
         temperature = 1e-2
+    top_p = float(item.top_p)
 
     generate_kwargs = dict(
         temperature=temperature,
         max_new_tokens=item.max_new_tokens,
-        do_sample=True
+        top_p=top_p,
+        repetition_penalty=item.repetition_penalty,
+        do_sample=True,
+        seed=42,
     )
 
     formatted_prompt = format_prompt(f"{item.system_prompt}, {item.prompt}", item.history)
-    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True)
+    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
     output = ""
 
     for response in stream:
@@ -44,5 +50,4 @@ def generate(item: Item):
 
 @app.post("/generate/")
 async def generate_text(item: Item):
-    return {"response": generate(item)}
-
+    return {"response": generate(item)}