llama-cpp-server

Paused

matthoffner commited on Feb 11, 2024

Commit

883ba9d

verified ·

1 Parent(s): 76fb5c4

Update main.py

Files changed (1) hide show

main.py CHANGED Viewed

@@ -4,24 +4,25 @@ import os
 app = create_app(
     Settings(
-        n_threads=2,  # set to number of cpu cores
         model="model/gguf-model.gguf",
         embedding=True
     )
 )
-# Read the content of index.html once and store it in memory
-with open("index.html", "r") as f:
-    content = f.read()
-@app.get("/", response_class=HTMLResponse)
-async def read_items():
-    return content
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app,
-                host=os.environ["HOST"],
-                port=int(os.environ["PORT"])
-                )

 app = create_app(
     Settings(
+        n_threads=4,
         model="model/gguf-model.gguf",
         embedding=True
+        n_gpu_layers=33,
     )
 )
+@app.get('/', response_class=HTMLResponse)
+def custom_index_route():
+    html_content = """
+    <html>
+        <body>
+            <iframe src="/docs" frameborder="0" width="100%" height="100%"></iframe>
+        </body>
+    </html>
+    """
+    return HTMLResponse(content=html_content)
 if __name__ == "__main__":
     import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)