Update 30b-Lazarus.ggmlv3.q3_K_S.bin

README.md

@@ -1,5 +1,5 @@
 ---
-title: TheBloke/WizardLM-13B-V1.0-Uncensored-GGML
+title: TheBloke/30B-Lazarus-GGML
 emoji: 🚀
 colorFrom: green
 colorTo: green

app.py

@@ -1,6 +1,5 @@
 """Run codes."""
 # pylint: disable=line-too-long, broad-exception-caught, invalid-name, missing-function-docstring, too-many-instance-attributes, missing-class-docstring
-# ruff: noqa: E501
 import os
 import time
 from dataclasses import asdict, dataclass
@@ -14,7 +13,7 @@ from about_time import about_time
 
 # from ctransformers import AutoConfig, AutoModelForCausalLM
 from ctransformers import AutoModelForCausalLM
-from huggingface_hub import hf_hub_download
+from huggingface_hub import hf_hub_download, snapshot_download
 from loguru import logger
 
 filename_list = [
@@ -39,7 +38,9 @@ URL = "https://huggingface.co/TheBloke/Wizard-Vicuna-7B-Uncensored-GGML/raw/main
 URL = "https://huggingface.co/TheBloke/30B-Lazarus-GGML/blob/main/30b-Lazarus.ggmlv3.q4_0.bin"
 URL = "https://huggingface.co/TheBloke/30B-Lazarus-GGML/blob/main/30b-Lazarus.ggmlv3.q4_1.bin"
 URL = "https://huggingface.co/TheBloke/30B-Lazarus-GGML/resolve/main/30b-Lazarus.ggmlv3.q4_K_M.bin"
-URL = "https://huggingface.co/TheBloke/30B-Lazarus-GGML/resolve/main/30b-Lazarus.ggmlv3.q4_K_S.bin"
+URL = "https://huggingface.co/TheBloke/30B-Lazarus-GGML/resolve/main/30b-Lazarus.ggmlv3.q4_K_S.bin"  # 18GB
+
+URL = "https://huggingface.co/TheBloke/30B-Lazarus-GGML/blob/main/30b-Lazarus.ggmlv3.q3_K_S.bin"  # 14GB
 
 MODEL_FILENAME = Path(URL).name
 
@@ -48,7 +49,7 @@ MODEL_FILENAME = Path(URL).name
 
 REPO_ID = "/".join(
     urlparse(URL).path.strip("/").split("/")[:2]
-)  
+)
 # TheBloke/30B-Lazarus-GGML
 # # TheBloke/Wizard-Vicuna-7B-Uncensored-GGML
 
@@ -291,7 +292,15 @@ llm = AutoModelForCausalLM.from_pretrained(
 # """
 
 logger.info(f"start dl, {REPO_ID=}, {MODEL_FILENAME=}, {DESTINATION_FOLDER=}")
-download_quant(DESTINATION_FOLDER, REPO_ID, MODEL_FILENAME)
+
+# download_quant(DESTINATION_FOLDER, REPO_ID, MODEL_FILENAME)
+snapshot_download(
+  repo_id=REPO_ID,  # TheBloke/30B-Lazarus-GGML
+  allow_patterns=MODEL_FILENAME,  # 30b-Lazarus.ggmlv3.q4_K_S.bin 18.3G
+  #   revision="ggmlv3",
+  local_dir="models",
+)
+
 logger.info("done dl")
 
 logger.debug(f"{os.cpu_count()=} {psutil.cpu_count(logical=False)=}")
@@ -302,12 +311,17 @@ logger.debug(f"{cpu_count=}")
 
 logger.info("load llm")
 
+# from ctransformers import AutoConfig
+# AutoConfig(REPO_ID)
+# AutoConfig(config='TheBloke/30B-Lazarus-GGML', model_type=None)
+
 _ = Path("models", MODEL_FILENAME).absolute().as_posix()
 logger.debug(f"model_file: {_}, exists: {Path(_).exists()}")
 LLM = AutoModelForCausalLM.from_pretrained(
     # "TheBloke/WizardCoder-15B-1.0-GGML",
-    REPO_ID,  # DESTINATION_FOLDER,  # model_path_or_repo_id: str required
-    model_file=_,
+    # REPO_ID,  # DESTINATION_FOLDER,  # model_path_or_repo_id: str required
+    # model_file=_,
+    _,
     model_type="llama",  # "starcoder",  AutoConfig.from_pretrained(REPO_ID)
     threads=cpu_count,
 )