mendeld commited on
Commit
ef8d2ea
·
verified ·
1 Parent(s): 27dd563

Delete beta_services.py

Browse files
Files changed (1) hide show
  1. beta_services.py +0 -308
beta_services.py DELETED
@@ -1,308 +0,0 @@
1
- """AC Beta V1: privacy-first analytics, optional accounts, and temporary document reading.
2
-
3
- Only aggregated counters are collected for routine chats; no full chat archive is
4
- available to administrators. Do not use the admin telemetry as a transcript store.
5
- """
6
- from __future__ import annotations
7
-
8
- from collections import Counter
9
- from datetime import datetime, timezone
10
- from io import BytesIO
11
- from pathlib import Path
12
- from threading import RLock
13
- from typing import Any
14
- import hashlib
15
- import html
16
- import json
17
- import os
18
- import re
19
- import secrets
20
- import time
21
- import xml.etree.ElementTree as ET
22
- import zipfile
23
-
24
- from bh56_core import STORE
25
-
26
- BUILD = "1.5.0 Beta V1"
27
- LOCK = RLock()
28
- # Uploads are intentionally process-memory-only and expire; no uploaded document is
29
- # copied into the administrative store.
30
- UPLOADS: dict[str, dict[str, Any]] = {}
31
- _LAST_STATS_SYNC = 0.0
32
- _LOGIN_FAILURES: dict[str, list[float]] = {}
33
- MAX_FILE_SIZE = 10 * 1024 * 1024
34
- ALLOWED_EXT = {'.pdf','.docx','.txt','.png','.jpg','.jpeg','.xlsx'}
35
-
36
-
37
- def persistent_accounts_available() -> bool:
38
- # Hugging Face ephemeral files will not provide reliable cross-device history.
39
- return bool((STORE.persistent_enabled and os.getenv('AC_PRIVATE_STORAGE_CONFIRMED') == '1') or os.getenv('AC_DURABLE_DATA_DIR') == '1')
40
-
41
-
42
- def _store_save() -> None:
43
- STORE.save(sync=STORE.persistent_enabled)
44
-
45
-
46
- def stats_snapshot() -> dict[str, Any]:
47
- with LOCK, STORE.lock:
48
- d = dict(STORE.state.get('beta_stats', {}))
49
- return {
50
- 'questions': int(d.get('questions', 0)),
51
- 'jewish': int(d.get('jewish', 0)),
52
- 'secular': int(d.get('secular', 0)),
53
- 'blocked': int(d.get('blocked', 0)),
54
- 'security': int(d.get('security', 0)),
55
- 'news': int(d.get('news', 0)),
56
- 'chochma': int(d.get('chochma', 0)),
57
- 'binah': int(d.get('binah', 0)),
58
- 'daas': int(d.get('daas', 0)),
59
- 'guest': int(d.get('guest', 0)),
60
- 'account': int(d.get('account', 0)),
61
- 'reports': len(STORE.state.get('beta_reports', [])),
62
- 'critical_flags': len(STORE.state.get('beta_critical', [])),
63
- }
64
-
65
-
66
- def count_question(category: str, mode: str, logged_in: bool) -> None:
67
- global _LAST_STATS_SYNC
68
- category = category if category in {'jewish','secular','blocked','security','news'} else 'secular'
69
- mode = mode.lower() if mode.lower() in {'chochma','binah','daas'} else 'binah'
70
- with LOCK, STORE.lock:
71
- stats = STORE.state.setdefault('beta_stats', {})
72
- for key in ('questions', category, mode, 'account' if logged_in else 'guest'):
73
- stats[key] = int(stats.get(key, 0)) + 1
74
- # Save statistics locally for responsiveness; sync to private HF storage
75
- # at most once a minute rather than uploading the entire database on
76
- # every user request. Recent stats may be lost on a crash.
77
- if time.time() - _LAST_STATS_SYNC >= 60:
78
- _store_save()
79
- _LAST_STATS_SYNC = time.time()
80
- else:
81
- STORE.save(sync=False)
82
-
83
-
84
- # Very restrictive pattern matching to avoid automatically exposing ordinary questions.
85
- CRITICAL_PATTERNS = (
86
- re.compile(r"\b(?:i am|i'm|im) (?:going to|about to) (?:kill myself|end my life|commit suicide)\b",re.I),
87
- re.compile(r"\b(?:i will|i'm going to|i am going to) (?:kill|shoot|stab) (?:him|her|them|you|my [a-z]{2,25}|someone|people)\b",re.I),
88
- )
89
-
90
-
91
- def record_critical_if_any(text: str) -> bool:
92
- if not any(p.search(text) for p in CRITICAL_PATTERNS):
93
- return False
94
- with LOCK, STORE.lock:
95
- STORE.state.setdefault('beta_critical', []).append({
96
- 'id': secrets.token_hex(8), 'time': datetime.now(timezone.utc).isoformat(),
97
- 'message': str(text)[:1200],
98
- })
99
- STORE.state['beta_critical'] = STORE.state['beta_critical'][-100:]
100
- _store_save()
101
- return True
102
-
103
-
104
- def list_flags() -> list[dict[str, Any]]:
105
- with LOCK, STORE.lock:
106
- return list(reversed(STORE.state.get('beta_critical', [])[-100:]))
107
-
108
-
109
- def add_report(kind: str, question: str, answer: str, note: str = '') -> None:
110
- kind = kind if kind in {'good','bad','report','redo','survey'} else 'report'
111
- with LOCK, STORE.lock:
112
- STORE.state.setdefault('beta_reports', []).append({
113
- 'id': secrets.token_hex(8),
114
- 'time': datetime.now(timezone.utc).isoformat(),
115
- 'kind': kind,
116
- 'question': question[:1200], 'answer': answer[:2800], 'note': note[:1000],
117
- })
118
- STORE.state['beta_reports'] = STORE.state['beta_reports'][-500:]
119
- _store_save()
120
-
121
-
122
- def get_reports() -> list[dict[str, Any]]:
123
- with LOCK, STORE.lock:
124
- return list(reversed(STORE.state.get('beta_reports', [])[-500:]))
125
-
126
-
127
- def _hash_password(password: str, salt: bytes | None = None) -> str:
128
- salt = salt or secrets.token_bytes(16)
129
- digest = hashlib.pbkdf2_hmac('sha256', password.encode(), salt, 600_000)
130
- return salt.hex() + ':' + digest.hex()
131
-
132
-
133
- def _verify_password(password: str, stored: str) -> bool:
134
- try:
135
- salt_hex, expected_hex = stored.split(':', 1)
136
- actual = _hash_password(password, bytes.fromhex(salt_hex)).split(':', 1)[1]
137
- return secrets.compare_digest(actual, expected_hex)
138
- except Exception:
139
- return False
140
-
141
-
142
- def signup(username: str, password: str) -> str:
143
- if not persistent_accounts_available():
144
- raise ValueError('Accounts require durable private storage configured by the administrator.')
145
- user = username.strip().lower()
146
- if not re.fullmatch(r'[a-z0-9_]{3,30}', user):
147
- raise ValueError('Username must have 3–30 letters, numbers, or underscores.')
148
- if len(password) < 12 or len(password) > 128:
149
- raise ValueError('Use a password of at least 12 characters.')
150
- with LOCK, STORE.lock:
151
- users = STORE.state.setdefault('beta_users', {})
152
- if user in users:
153
- raise ValueError('That username is taken.')
154
- users[user] = {'password_hash': _hash_password(password), 'chats': [], 'settings': {'retention': 'forever'}}
155
- _store_save()
156
- return user
157
-
158
-
159
- def login(username: str, password: str) -> str | None:
160
- user = username.strip().lower()
161
- with LOCK, STORE.lock:
162
- now=time.time()
163
- attempts=[t for t in _LOGIN_FAILURES.get(user,[]) if now-t<900]
164
- _LOGIN_FAILURES[user]=attempts
165
- if len(attempts)>=8:return None
166
- data = STORE.state.setdefault('beta_users', {}).get(user, {})
167
- if not data or not _verify_password(password, data.get('password_hash', '')):
168
- attempts.append(now)
169
- return None
170
- _LOGIN_FAILURES.pop(user,None)
171
- return user
172
-
173
-
174
- def get_user_data(username: str) -> dict[str, Any]:
175
- with LOCK, STORE.lock:
176
- u = STORE.state.get('beta_users', {}).get(username)
177
- if not u:
178
- raise ValueError('Account unavailable.')
179
- return {'chats': json.loads(json.dumps(u.get('chats', []))), 'settings': dict(u.get('settings', {}))}
180
-
181
-
182
- def update_user_data(username: str, chats: list[dict[str, Any]], retention: str) -> dict[str, Any]:
183
- if retention not in {'forever','30','90'}:
184
- raise ValueError('Invalid retention option.')
185
- if len(chats) > 75:
186
- raise ValueError('Too many conversations.')
187
- cleaned = []
188
- now = time.time()
189
- for chat in chats:
190
- if not isinstance(chat, dict) or not isinstance(chat.get('history'), list):
191
- continue
192
- stamp = float(chat.get('updated') or 0)
193
- if retention != 'forever' and stamp and stamp / 1000 < now - int(retention)*86400:
194
- continue
195
- history=[]
196
- for msg in chat['history'][-80:]:
197
- if isinstance(msg, dict) and msg.get('role') in {'user','assistant'}:
198
- history.append({'role':msg['role'], 'content':str(msg.get('content',''))[:16000]})
199
- cleaned.append({'id':str(chat.get('id') or '')[:100], 'title':str(chat.get('title') or '')[:100], 'updated': stamp, 'history':history})
200
- if len(json.dumps(cleaned)) > 1_500_000:
201
- raise ValueError('Account history exceeds the beta storage limit.')
202
- with LOCK, STORE.lock:
203
- u = STORE.state.get('beta_users', {}).get(username)
204
- if not u: raise ValueError('Account unavailable.')
205
- u['chats']=cleaned
206
- u['settings']={'retention':retention}
207
- _store_save()
208
- return {'saved':len(cleaned)}
209
-
210
-
211
- def admin_own_chats() -> list[dict[str, Any]]:
212
- with LOCK, STORE.lock:
213
- return json.loads(json.dumps(STORE.state.get('beta_admin_own_chats', [])))
214
-
215
-
216
- def save_admin_own_chats(chats: list[dict[str, Any]]) -> dict[str, int]:
217
- if len(chats)>75 or len(json.dumps(chats))>1_500_000:
218
- raise ValueError('Admin chat archive too large.')
219
- cleaned=[]
220
- for row in chats:
221
- if not isinstance(row,dict) or not isinstance(row.get('history'),list):continue
222
- history=[{'role':msg['role'],'content':str(msg.get('content',''))[:16000]}
223
- for msg in row['history'][-80:]
224
- if isinstance(msg,dict) and msg.get('role') in {'user','assistant'}]
225
- cleaned.append({'id':str(row.get('id',''))[:100], 'title':str(row.get('title',''))[:100],
226
- 'updated':float(row.get('updated',0) or 0), 'history':history})
227
- with LOCK, STORE.lock:
228
- STORE.state['beta_admin_own_chats']=cleaned
229
- _store_save()
230
- return {'saved':len(cleaned)}
231
-
232
-
233
- def _xlsx_text(raw: bytes) -> str:
234
- # No spreadsheet macros are executed. Read-only, zip/XML based text extraction.
235
- chunks=[]
236
- with zipfile.ZipFile(BytesIO(raw)) as z:
237
- shared=[]
238
- if 'xl/sharedStrings.xml' in z.namelist():
239
- root=ET.fromstring(z.read('xl/sharedStrings.xml'))
240
- for si in root:
241
- shared.append(''.join(t.text or '' for t in si.iter() if t.tag.endswith('}t')))
242
- for name in sorted(n for n in z.namelist() if re.fullmatch(r'xl/worksheets/sheet\d+\.xml', n))[:8]:
243
- root=ET.fromstring(z.read(name)); cells=[]
244
- for c in root.iter():
245
- if not c.tag.endswith('}c'):continue
246
- typ=c.attrib.get('t','')
247
- val=next((x.text or '' for x in c if x.tag.endswith('}v')),'')
248
- if typ=='s':
249
- try:val=shared[int(val)]
250
- except (ValueError,IndexError):pass
251
- if typ=='inlineStr':val=''.join(x.text or '' for x in c.iter() if x.tag.endswith('}t'))
252
- if val:cells.append(f"{c.attrib.get('r','')}: {val}")
253
- if len(cells)>=1000:break
254
- chunks.append(name+'\n'+'\n'.join(cells))
255
- return '\n\n'.join(chunks)
256
-
257
-
258
- def extract_document(filename: str, data: bytes) -> tuple[str,str]:
259
- ext = Path(filename).suffix.lower()
260
- if ext not in ALLOWED_EXT: raise ValueError('Unsupported file type.')
261
- if not data or len(data)>MAX_FILE_SIZE:raise ValueError('File must be 1 byte to 10 MB.')
262
- if ext=='.txt': content=data.decode('utf-8', errors='replace')
263
- elif ext=='.pdf':
264
- from pypdf import PdfReader
265
- reader=PdfReader(BytesIO(data));content='\n'.join(p.extract_text() or '' for p in reader.pages[:30])
266
- elif ext=='.docx':
267
- from docx import Document
268
- doc=Document(BytesIO(data));content='\n'.join(p.text for p in doc.paragraphs)
269
- elif ext=='.xlsx':content=_xlsx_text(data)
270
- else:
271
- from PIL import Image
272
- with Image.open(BytesIO(data)) as im:
273
- im.verify()
274
- return '', 'Image attached, but this beta has no image-understanding model. AC cannot describe or read the picture yet.'
275
- content=content.strip()[:30000]
276
- if not content:
277
- return '', 'No readable text found. Scanned PDFs require OCR, which is not included yet.'
278
- return content, f'Readable text extracted ({len(content):,} characters; first 30 PDF pages maximum).'
279
-
280
-
281
- def put_upload(session_id: str, name: str, content: str) -> str:
282
- with LOCK:
283
- expired=[k for k,v in UPLOADS.items() if time.time()-v['created']>3600]
284
- for k in expired:UPLOADS.pop(k,None)
285
- token=secrets.token_urlsafe(18)
286
- UPLOADS[token]={'session':session_id,'created':time.time(),'name':name[:130],'content':content}
287
- return token
288
-
289
-
290
- def get_uploads(session_id: str, tokens: list[str]) -> list[dict[str,str]]:
291
- with LOCK:
292
- results=[]
293
- for tok in tokens[:4]:
294
- row=UPLOADS.get(tok)
295
- if row and row['session']==session_id and time.time()-row['created']<3600:
296
- results.append({'name':row['name'],'content':row['content']})
297
- return results
298
-
299
-
300
- def migrate_legacy_privacy() -> None:
301
- # Remove old chat transcript fields from saved session events and any admin
302
- # audit trails that stored full message text. This intentionally does not
303
- # erase user-submitted feedback reports.
304
- with LOCK, STORE.lock:
305
- if STORE.state.get('beta_privacy_migrated_v1'):return
306
- STORE.state['session_events']=[]
307
- STORE.state['beta_privacy_migrated_v1']=True
308
- _store_save()