Update README.md
Browse files
README.md
CHANGED
|
@@ -21,38 +21,6 @@ It is converted into 8-bit GGUF format from original repository https://huggingf
|
|
| 21 |
**Repository:** https://huggingface.co/defog/sqlcoder-7b-2
|
| 22 |
|
| 23 |
### Example usage
|
| 24 |
-
**
|
| 25 |
|
| 26 |
-
|
| 27 |
-
from huggingface_hub import hf_hub_download
|
| 28 |
-
|
| 29 |
-
YOUR MODEL DIRECTORY =
|
| 30 |
-
CONTEXT_LENGHT =
|
| 31 |
-
MAX_TOKENS =
|
| 32 |
-
BATCH_SIZE =
|
| 33 |
-
TEMPERATURE =
|
| 34 |
-
GPU_OFFLOAD =
|
| 35 |
-
|
| 36 |
-
def load_model(model_id, model_basename):
|
| 37 |
-
model_path = hf_hub_download(
|
| 38 |
-
repo_id=model_id,
|
| 39 |
-
filename=model_basename,
|
| 40 |
-
resume_download=True,
|
| 41 |
-
cache_dir="YOUR MODEL DIRECTORY",
|
| 42 |
-
)
|
| 43 |
-
kwargs = {
|
| 44 |
-
'model_path': model_path,
|
| 45 |
-
'n_ctx':CONTEXT_LENGHT,
|
| 46 |
-
'max_tokens':MAX_TOKENS,
|
| 47 |
-
'n_batch':BATCH_SIZE,
|
| 48 |
-
'n_gpu_layers':GPU_OFFLOAD,
|
| 49 |
-
'temperature':TEMPERATURE,
|
| 50 |
-
'verbose':True,
|
| 51 |
-
}
|
| 52 |
-
return LlamaCpp(**kwargs)
|
| 53 |
-
|
| 54 |
-
llm = load_model(
|
| 55 |
-
model_id="whoami02/defog-sqlcoder-2-GGUF",
|
| 56 |
-
model_basename="sqlcoder-7b-2.q8_0.gguf",
|
| 57 |
-
)
|
| 58 |
-
|
|
|
|
| 21 |
**Repository:** https://huggingface.co/defog/sqlcoder-7b-2
|
| 22 |
|
| 23 |
### Example usage
|
| 24 |
+
**With Llamacpp:**
|
| 25 |
|
| 26 |
+

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|