output = llm(
"Once upon a time,",
max_tokens=512,
echo=True
)
print(output)YAML Metadata Warning:empty or missing yaml metadata in repo card
Check out the documentation for more information.
TEMPLATE for Ollama Modelfile
TEMPLATE """ {{- range $i, $_ := .Messages }} {{- if eq .Role "user" }} {{- if and $.Tools (le (len (slice $.Messages $i)) 2) }}[AVAILABLE_TOOLS] {{ $.Tools }}[/AVAILABLE_TOOLS] {{- end }}[INST] {{ if and $.System (eq (len (slice $.Messages $i)) 1) }}{{ $.System }}
{{ end }}{{ .Content }}[/INST] {{- else if eq .Role "assistant" }} {{- if .Content }} {{ .Content }}{{ if not (eq (len (slice $.Messages $i)) 1) }}{{ end }} {{- else if .ToolCalls }}[TOOL_CALLS] [ {{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} {{- end }}] {{- end }} {{- else if eq .Role "tool" }}[TOOL_RESULTS] {"content": {{ .Content }}} [/TOOL_RESULTS] {{- end }} {{- end }} """
PARAMETER stop "[INST]" PARAMETER stop "[/INST]"
- Downloads last month
- 4
8-bit
# !pip install llama-cpp-python from llama_cpp import Llama llm = Llama.from_pretrained( repo_id="snakeying/llama3.1-8B-LongWriter", filename="llama3.1-8B-LongWriter-q8_0.gguf", )