duckeydevv commited on
Commit
be248ec
·
verified ·
1 Parent(s): 579f3f3

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +40 -67
app.py CHANGED
@@ -1,79 +1,52 @@
1
  import gradio as gr
2
- import spaces
3
- from huggingface_hub import hf_hub_download
4
  from llama_cpp import Llama
5
 
6
- # Download and cache the GGUF model from your repository on startup
7
- print("Downloading / loading Koda-IDS Frontier GGUF model...")
8
- model_path = hf_hub_download(
9
- repo_id="netgoat-xyz/koda-ids-frontier",
10
- filename="koda-ids-Q4_K_M.gguf"
 
 
 
 
 
 
 
 
11
  )
 
12
 
13
- # Initialize the local llama.cpp engine
14
- # On ZeroGPU, we load the model normally; the decorator handles the hardware switch
15
- llm = Llama(
16
- model_path=model_path,
17
- n_ctx=4096,
18
- n_gpu_layers=-1,
19
- verbose=False
20
- )
21
-
22
- # The @spaces.GPU decorator requests a ZeroGPU allocation dynamically during inference
23
- @spaces.GPU(duration=120)
24
- def respond(
25
- message,
26
- history: list[dict[str, str]],
27
- system_message,
28
- max_tokens,
29
- temperature,
30
- top_p,
31
- ):
32
- """
33
- Handles local streaming inference with ZeroGPU support.
34
- """
35
- messages = [{"role": "system", "content": system_message}]
36
 
37
- # Handle conversation history safely
38
- for item in history:
39
- if isinstance(item, dict):
40
- messages.append(item)
41
- elif isinstance(item, list) and len(item) == 2:
42
- messages.append({"role": "user", "content": item[0]})
43
- messages.append({"role": "assistant", "content": item[1]})
44
-
45
  messages.append({"role": "user", "content": message})
46
-
47
- stream = llm.create_chat_completion(
 
48
  messages=messages,
49
- max_tokens=max_tokens,
50
- temperature=temperature,
51
- top_p=top_p,
52
- stream=True
53
  )
54
-
55
- response = ""
56
- for chunk in stream:
57
- delta = chunk["choices"][0]["delta"]
58
- if "content" in delta and delta["content"] is not None:
59
- response += delta["content"]
60
- yield response
61
-
62
- # Configure Gradio ChatInterface
63
- chatbot = gr.ChatInterface(
64
- respond,
65
- additional_inputs=[
66
- gr.Textbox(
67
- value="You are Koda IDS, a defensive cybersecurity investigation system. Use calibrated language: low, moderate, or high confidence, and never present a possibility as a confirmed fact.",
68
- label="System message"
69
- ),
70
- gr.Slider(minimum=1, maximum=4096, value=1024, step=1, label="Max new tokens"),
71
- gr.Slider(minimum=0.0, maximum=1.0, value=0.3, step=0.05, label="Temperature"),
72
- gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
73
- ],
74
- title="Koda-IDS Frontier (ZeroGPU)",
75
- description="Running your custom-distilled `netgoat-xyz/koda-ids-frontier` GGUF model live on Hugging Face Spaces with ZeroGPU acceleration.",
76
  )
77
 
78
  if __name__ == "__main__":
79
- chatbot.launch()
 
1
  import gradio as gr
 
 
2
  from llama_cpp import Llama
3
 
4
+ # --- Configuration ---
5
+ MODEL_REPO = "netgoat-ai/koda-ids-frontier"
6
+ MODEL_FILENAME = "*Q4_K_M.gguf" # Pattern to match your specific GGUF file
7
+
8
+ # --- Load Model ---
9
+ # This will download the model from the Hub and cache it.
10
+ print(f"Loading model: {MODEL_REPO} / {MODEL_FILENAME}")
11
+ llm = Llama.from_pretrained(
12
+ repo_id=MODEL_REPO,
13
+ filename=MODEL_FILENAME,
14
+ n_ctx=2048, # Context window size
15
+ n_threads=4, # Adjust based on your Space's CPU
16
+ verbose=False,
17
  )
18
+ print("Model loaded successfully.")
19
 
20
+ # --- Chat Logic ---
21
+ def chat_with_model(message, history):
22
+ """Formats the conversation and generates a response."""
23
+ messages = [{"role": "system", "content": "You are a helpful assistant."}]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
 
25
+ # Add conversation history
26
+ for user_msg, assistant_msg in history:
27
+ messages.append({"role": "user", "content": user_msg})
28
+ messages.append({"role": "assistant", "content": assistant_msg})
29
+
30
+ # Add the new user message
 
 
31
  messages.append({"role": "user", "content": message})
32
+
33
+ # Generate a response
34
+ response = llm.create_chat_completion(
35
  messages=messages,
36
+ max_tokens=512,
37
+ temperature=0.7,
38
+ top_p=0.95,
 
39
  )
40
+
41
+ return response['choices'][0]['message']['content']
42
+
43
+ # --- Gradio Interface ---
44
+ demo = gr.ChatInterface(
45
+ fn=chat_with_model,
46
+ title="Koda IDS Frontier Chat",
47
+ description="Chat with the netgoat-ai/koda-ids-frontier model running locally.",
48
+ examples=["Hello!", "Explain what you can do."],
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  )
50
 
51
  if __name__ == "__main__":
52
+ demo.launch()