llama3.1 8b

.gitignore

@@ -1,3 +1,5 @@
+models/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

app.py

@@ -6,8 +6,8 @@ import cohere
 HF_API_KEY = os.getenv("HF_API_KEY")
 COHERE_API_KEY = os.getenv("COHERE_API_KEY")  # Get Cohere API key
 
-models = ["HuggingFaceH4/zephyr-7b-beta", "microsoft/Phi-4-mini-instruct", "meta-llama/Llama-3.2-3B-Instruct"]
-client_hf = InferenceClient(model=models[2], token=HF_API_KEY)  # HF Client
+models = ["HuggingFaceH4/zephyr-7b-beta", "microsoft/Phi-4-mini-instruct", "meta-llama/Llama-3.2-3B-Instruct", "meta-llama/Llama-3.1-8B-Instruct"]
+client_hf = InferenceClient(model=models[3], token=HF_API_KEY)  # HF Client
 client_cohere = cohere.Client(COHERE_API_KEY)  # Cohere Client
 
 def respond(

local_demos/llama_cpp_demo.py

@@ -0,0 +1,33 @@
+import os
+import requests
+from llama_cpp import Llama
+
+# Define model path
+MODEL_PATH = "models/llama3.2_3B_Q4.gguf"
+MODEL_URL = "https://huggingface.co/prithivMLmods/Llama-3.2-3B-GGUF/resolve/main/Llama-3.2-3B-GGUF.Q4_K_M.gguf?download=true"
+
+# Download model if not exists
+if not os.path.exists(MODEL_PATH):
+    print("Downloading model...")
+    with requests.get(MODEL_URL, stream=True) as r:
+        r.raise_for_status()
+        with open(MODEL_PATH, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+    print("Download complete!")
+
+
+# Load model
+llm = Llama(model_path=MODEL_PATH, n_ctx=4096)
+
+# Define system and user prompts
+system_prompt = "You are a helpful AI assistant."
+user_prompt = input("User: ")
+
+# Generate response
+output = llm.create_chat_completion(
+    messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}]
+)
+
+# Print response
+print("Assistant:", output["choices"][0]["message"]["content"])

local_demos/transformers_demo.py

@@ -0,0 +1,24 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+def generate_response(model, tokenizer, prompt, max_length=50):
+    inputs = tokenizer(prompt, return_tensors="pt")
+    outputs = model.generate(inputs.input_ids, max_length=max_length, num_return_sequences=1)
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return response
+
+def main():
+    model_name = "meta-llama/Llama-3.2-3B-Instruct"
+    system_prompt = "You are a helpful assistant."
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(model_name)
+
+    user_prompt = input("Enter your prompt: ")
+    full_prompt = f"{system_prompt}\n{user_prompt}"
+
+    response = generate_response(model, tokenizer, full_prompt)
+    print("Response:", response)
+
+if __name__ == "__main__":
+    main()