dedlepexa commited on
Commit
df9b7a5
·
verified ·
1 Parent(s): f35ca0a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +24 -18
app.py CHANGED
@@ -12,20 +12,28 @@ import requests
12
 
13
  app = FastAPI()
14
 
15
- # 🔥 MODEL DOWNLOAD
16
- MODEL_URL = "https://huggingface.co/TheBloke/phi-2-GGUF/resolve/main/phi-2.Q4_K_M.gguf"
17
  MODEL_PATH = "phi-2.gguf"
18
 
19
  if not os.path.exists(MODEL_PATH):
20
- print("Downloading model... (это может занять время)")
21
- with requests.get(MODEL_URL, stream=True) as r:
22
- with open(MODEL_PATH, "wb") as f:
23
- for chunk in r.iter_content(chunk_size=8192):
24
- if chunk:
25
- f.write(chunk)
26
- print("Model downloaded!")
27
-
28
- # 🔥 LOAD MODEL
 
 
 
 
 
 
 
 
29
  llm = Llama(
30
  model_path=MODEL_PATH,
31
  n_ctx=2048,
@@ -42,12 +50,12 @@ class Message(BaseModel):
42
  message: str
43
 
44
 
45
- # 🔥 split text
46
  def split_text(text, max_len=25):
47
  return "\n".join([text[i:i+max_len] for i in range(0, len(text), max_len)])
48
 
49
 
50
- # 🔥 GENERATION
51
  def generate_ai_stream(message: str):
52
  prompt = f"User: {message}\nAssistant: Answer clearly and fully:\n"
53
 
@@ -81,7 +89,7 @@ def generate_ai_stream(message: str):
81
  return final_text
82
 
83
 
84
- # 🔥 WORKER
85
  def worker():
86
  while True:
87
  if queue:
@@ -99,17 +107,16 @@ def worker():
99
  time.sleep(0.01)
100
 
101
 
102
- # 🔥 START WORKERS
103
  for _ in range(NUM_WORKERS):
104
  threading.Thread(target=worker, daemon=True).start()
105
 
106
 
107
  @app.get("/")
108
  async def root():
109
- return PlainTextResponse("AI server running (Phi-2 GGUF + llama.cpp)")
110
 
111
 
112
- # 🔥 ASK
113
  @app.get("/ask")
114
  async def ask(message: str):
115
 
@@ -126,7 +133,6 @@ async def ask(message: str):
126
  return PlainTextResponse("accepted")
127
 
128
 
129
- # 🔥 GET
130
  @app.get("/get")
131
  async def get(message: str):
132
 
 
12
 
13
  app = FastAPI()
14
 
15
+ # 🔥 МЕНЬШАЯ И БЫСТРАЯ МОДЕЛЬ (Q3)
16
+ MODEL_URL = "https://huggingface.co/TheBloke/phi-2-GGUF/resolve/main/phi-2.Q3_K_M.gguf"
17
  MODEL_PATH = "phi-2.gguf"
18
 
19
  if not os.path.exists(MODEL_PATH):
20
+ print("Downloading Q3 model... (быстрее чем Q4)")
21
+
22
+ response = requests.get(MODEL_URL, stream=True)
23
+ total_size = int(response.headers.get('content-length', 0))
24
+ downloaded = 0
25
+
26
+ with open(MODEL_PATH, "wb") as f:
27
+ for chunk in response.iter_content(chunk_size=8192):
28
+ if chunk:
29
+ f.write(chunk)
30
+ downloaded += len(chunk)
31
+
32
+ print(f"{downloaded // (1024*1024)} MB / {total_size // (1024*1024)} MB", end="\r")
33
+
34
+ print("\nModel downloaded!")
35
+
36
+ # 🔥 ЗАГРУЗКА МОДЕЛИ
37
  llm = Llama(
38
  model_path=MODEL_PATH,
39
  n_ctx=2048,
 
50
  message: str
51
 
52
 
53
+ # 🔥 split (25 символов)
54
  def split_text(text, max_len=25):
55
  return "\n".join([text[i:i+max_len] for i in range(0, len(text), max_len)])
56
 
57
 
58
+ # 🔥 генерация
59
  def generate_ai_stream(message: str):
60
  prompt = f"User: {message}\nAssistant: Answer clearly and fully:\n"
61
 
 
89
  return final_text
90
 
91
 
92
+ # 🔥 воркер
93
  def worker():
94
  while True:
95
  if queue:
 
107
  time.sleep(0.01)
108
 
109
 
110
+ # 🔥 3 потока
111
  for _ in range(NUM_WORKERS):
112
  threading.Thread(target=worker, daemon=True).start()
113
 
114
 
115
  @app.get("/")
116
  async def root():
117
+ return PlainTextResponse("AI server running (Phi-2 Q3 GGUF)")
118
 
119
 
 
120
  @app.get("/ask")
121
  async def ask(message: str):
122
 
 
133
  return PlainTextResponse("accepted")
134
 
135
 
 
136
  @app.get("/get")
137
  async def get(message: str):
138