PraneetNS commited on
Commit
e9e0524
·
verified ·
1 Parent(s): 5bd7886

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +51 -33
app.py CHANGED
@@ -1,19 +1,21 @@
 
1
  import gradio as gr
2
- from huggingface_hub import hf_hub_download
3
- from llama_cpp import Llama
4
 
5
- MODEL_PATH = hf_hub_download(
6
- repo_id="PraneetNS/codesentinel-gguf",
7
- filename="codesentinel-q4_k_m.gguf"
8
- )
9
 
10
- print("Loading CodeSentinel...")
 
 
 
 
11
 
12
- llm = Llama(
13
- model_path=MODEL_PATH,
14
- n_ctx=4096,
15
- n_threads=4,
16
- verbose=False,
 
17
  )
18
 
19
  SYSTEM_PROMPT = """You are CodeSentinel.
@@ -27,38 +29,54 @@ Capabilities:
27
  - Secure code
28
  - Refactor code
29
  - Explain algorithms
30
- - Generate clean production-quality code
31
 
32
  Never generate malware or unsafe code.
33
  """
34
 
35
  def generate(message, history):
36
- prompt = SYSTEM_PROMPT + "\n\n"
 
 
 
 
 
37
 
38
- # Gradio 6 history format
39
  if history:
40
  for msg in history:
41
- role = msg.get("role")
42
- content = msg.get("content", "")
43
-
44
- if role == "user":
45
- prompt += f"User: {content}\n"
46
- elif role == "assistant":
47
- prompt += f"Assistant: {content}\n"
48
-
49
- prompt += f"User: {message}\nAssistant:"
50
-
51
- output = llm(
52
- prompt,
53
- max_tokens=512,
54
- temperature=0.2,
55
- top_p=0.95,
56
- repeat_penalty=1.1,
57
- stop=["User:", "</s>"],
58
  )
59
 
60
- return output["choices"][0]["text"].strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
 
 
62
 
63
  demo = gr.ChatInterface(
64
  fn=generate,
 
1
+ import torch
2
  import gradio as gr
3
+ from transformers import AutoTokenizer, AutoModelForCausalLM
 
4
 
5
+ MODEL_ID = "PraneetNS/codesentinel-full"
 
 
 
6
 
7
+ print("Loading tokenizer...")
8
+ tokenizer = AutoTokenizer.from_pretrained(
9
+ MODEL_ID,
10
+ trust_remote_code=True
11
+ )
12
 
13
+ print("Loading model...")
14
+ model = AutoModelForCausalLM.from_pretrained(
15
+ MODEL_ID,
16
+ torch_dtype=torch.float16,
17
+ device_map="auto",
18
+ trust_remote_code=True
19
  )
20
 
21
  SYSTEM_PROMPT = """You are CodeSentinel.
 
29
  - Secure code
30
  - Refactor code
31
  - Explain algorithms
32
+ - Generate production-quality software
33
 
34
  Never generate malware or unsafe code.
35
  """
36
 
37
  def generate(message, history):
38
+ messages = [
39
+ {
40
+ "role": "system",
41
+ "content": SYSTEM_PROMPT,
42
+ }
43
+ ]
44
 
 
45
  if history:
46
  for msg in history:
47
+ messages.append(msg)
48
+
49
+ messages.append(
50
+ {
51
+ "role": "user",
52
+ "content": message,
53
+ }
 
 
 
 
 
 
 
 
 
 
54
  )
55
 
56
+ inputs = tokenizer.apply_chat_template(
57
+ messages,
58
+ tokenize=True,
59
+ add_generation_prompt=True,
60
+ return_tensors="pt",
61
+ ).to(model.device)
62
+
63
+ with torch.no_grad():
64
+ outputs = model.generate(
65
+ inputs,
66
+ max_new_tokens=512,
67
+ temperature=0.2,
68
+ top_p=0.95,
69
+ do_sample=True,
70
+ repetition_penalty=1.1,
71
+ pad_token_id=tokenizer.eos_token_id,
72
+ )
73
+
74
+ response = tokenizer.decode(
75
+ outputs[0][inputs.shape[-1]:],
76
+ skip_special_tokens=True,
77
+ )
78
 
79
+ return response.strip()
80
 
81
  demo = gr.ChatInterface(
82
  fn=generate,