llama-cpp-python How to use trajis-tech/llama-cpp-python-trajis-tech-nonavx512-cuda with llama-cpp-python:
# !pip install llama-cpp-python
from llama_cpp import Llama
llm = Llama.from_pretrained(
repo_id="trajis-tech/llama-cpp-python-trajis-tech-nonavx512-cuda",
filename="{{GGUF_FILE}}",
)
output = llm(
"Once upon a time,",
max_tokens=512,
echo=True
)
print(output)
