Spaces:
Running on Zero
Running on Zero
Delete beta_services.py
Browse files- beta_services.py +0 -308
beta_services.py
DELETED
|
@@ -1,308 +0,0 @@
|
|
| 1 |
-
"""AC Beta V1: privacy-first analytics, optional accounts, and temporary document reading.
|
| 2 |
-
|
| 3 |
-
Only aggregated counters are collected for routine chats; no full chat archive is
|
| 4 |
-
available to administrators. Do not use the admin telemetry as a transcript store.
|
| 5 |
-
"""
|
| 6 |
-
from __future__ import annotations
|
| 7 |
-
|
| 8 |
-
from collections import Counter
|
| 9 |
-
from datetime import datetime, timezone
|
| 10 |
-
from io import BytesIO
|
| 11 |
-
from pathlib import Path
|
| 12 |
-
from threading import RLock
|
| 13 |
-
from typing import Any
|
| 14 |
-
import hashlib
|
| 15 |
-
import html
|
| 16 |
-
import json
|
| 17 |
-
import os
|
| 18 |
-
import re
|
| 19 |
-
import secrets
|
| 20 |
-
import time
|
| 21 |
-
import xml.etree.ElementTree as ET
|
| 22 |
-
import zipfile
|
| 23 |
-
|
| 24 |
-
from bh56_core import STORE
|
| 25 |
-
|
| 26 |
-
BUILD = "1.5.0 Beta V1"
|
| 27 |
-
LOCK = RLock()
|
| 28 |
-
# Uploads are intentionally process-memory-only and expire; no uploaded document is
|
| 29 |
-
# copied into the administrative store.
|
| 30 |
-
UPLOADS: dict[str, dict[str, Any]] = {}
|
| 31 |
-
_LAST_STATS_SYNC = 0.0
|
| 32 |
-
_LOGIN_FAILURES: dict[str, list[float]] = {}
|
| 33 |
-
MAX_FILE_SIZE = 10 * 1024 * 1024
|
| 34 |
-
ALLOWED_EXT = {'.pdf','.docx','.txt','.png','.jpg','.jpeg','.xlsx'}
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
def persistent_accounts_available() -> bool:
|
| 38 |
-
# Hugging Face ephemeral files will not provide reliable cross-device history.
|
| 39 |
-
return bool((STORE.persistent_enabled and os.getenv('AC_PRIVATE_STORAGE_CONFIRMED') == '1') or os.getenv('AC_DURABLE_DATA_DIR') == '1')
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
def _store_save() -> None:
|
| 43 |
-
STORE.save(sync=STORE.persistent_enabled)
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
def stats_snapshot() -> dict[str, Any]:
|
| 47 |
-
with LOCK, STORE.lock:
|
| 48 |
-
d = dict(STORE.state.get('beta_stats', {}))
|
| 49 |
-
return {
|
| 50 |
-
'questions': int(d.get('questions', 0)),
|
| 51 |
-
'jewish': int(d.get('jewish', 0)),
|
| 52 |
-
'secular': int(d.get('secular', 0)),
|
| 53 |
-
'blocked': int(d.get('blocked', 0)),
|
| 54 |
-
'security': int(d.get('security', 0)),
|
| 55 |
-
'news': int(d.get('news', 0)),
|
| 56 |
-
'chochma': int(d.get('chochma', 0)),
|
| 57 |
-
'binah': int(d.get('binah', 0)),
|
| 58 |
-
'daas': int(d.get('daas', 0)),
|
| 59 |
-
'guest': int(d.get('guest', 0)),
|
| 60 |
-
'account': int(d.get('account', 0)),
|
| 61 |
-
'reports': len(STORE.state.get('beta_reports', [])),
|
| 62 |
-
'critical_flags': len(STORE.state.get('beta_critical', [])),
|
| 63 |
-
}
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
def count_question(category: str, mode: str, logged_in: bool) -> None:
|
| 67 |
-
global _LAST_STATS_SYNC
|
| 68 |
-
category = category if category in {'jewish','secular','blocked','security','news'} else 'secular'
|
| 69 |
-
mode = mode.lower() if mode.lower() in {'chochma','binah','daas'} else 'binah'
|
| 70 |
-
with LOCK, STORE.lock:
|
| 71 |
-
stats = STORE.state.setdefault('beta_stats', {})
|
| 72 |
-
for key in ('questions', category, mode, 'account' if logged_in else 'guest'):
|
| 73 |
-
stats[key] = int(stats.get(key, 0)) + 1
|
| 74 |
-
# Save statistics locally for responsiveness; sync to private HF storage
|
| 75 |
-
# at most once a minute rather than uploading the entire database on
|
| 76 |
-
# every user request. Recent stats may be lost on a crash.
|
| 77 |
-
if time.time() - _LAST_STATS_SYNC >= 60:
|
| 78 |
-
_store_save()
|
| 79 |
-
_LAST_STATS_SYNC = time.time()
|
| 80 |
-
else:
|
| 81 |
-
STORE.save(sync=False)
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
# Very restrictive pattern matching to avoid automatically exposing ordinary questions.
|
| 85 |
-
CRITICAL_PATTERNS = (
|
| 86 |
-
re.compile(r"\b(?:i am|i'm|im) (?:going to|about to) (?:kill myself|end my life|commit suicide)\b",re.I),
|
| 87 |
-
re.compile(r"\b(?:i will|i'm going to|i am going to) (?:kill|shoot|stab) (?:him|her|them|you|my [a-z]{2,25}|someone|people)\b",re.I),
|
| 88 |
-
)
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
def record_critical_if_any(text: str) -> bool:
|
| 92 |
-
if not any(p.search(text) for p in CRITICAL_PATTERNS):
|
| 93 |
-
return False
|
| 94 |
-
with LOCK, STORE.lock:
|
| 95 |
-
STORE.state.setdefault('beta_critical', []).append({
|
| 96 |
-
'id': secrets.token_hex(8), 'time': datetime.now(timezone.utc).isoformat(),
|
| 97 |
-
'message': str(text)[:1200],
|
| 98 |
-
})
|
| 99 |
-
STORE.state['beta_critical'] = STORE.state['beta_critical'][-100:]
|
| 100 |
-
_store_save()
|
| 101 |
-
return True
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
def list_flags() -> list[dict[str, Any]]:
|
| 105 |
-
with LOCK, STORE.lock:
|
| 106 |
-
return list(reversed(STORE.state.get('beta_critical', [])[-100:]))
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
def add_report(kind: str, question: str, answer: str, note: str = '') -> None:
|
| 110 |
-
kind = kind if kind in {'good','bad','report','redo','survey'} else 'report'
|
| 111 |
-
with LOCK, STORE.lock:
|
| 112 |
-
STORE.state.setdefault('beta_reports', []).append({
|
| 113 |
-
'id': secrets.token_hex(8),
|
| 114 |
-
'time': datetime.now(timezone.utc).isoformat(),
|
| 115 |
-
'kind': kind,
|
| 116 |
-
'question': question[:1200], 'answer': answer[:2800], 'note': note[:1000],
|
| 117 |
-
})
|
| 118 |
-
STORE.state['beta_reports'] = STORE.state['beta_reports'][-500:]
|
| 119 |
-
_store_save()
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
def get_reports() -> list[dict[str, Any]]:
|
| 123 |
-
with LOCK, STORE.lock:
|
| 124 |
-
return list(reversed(STORE.state.get('beta_reports', [])[-500:]))
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
def _hash_password(password: str, salt: bytes | None = None) -> str:
|
| 128 |
-
salt = salt or secrets.token_bytes(16)
|
| 129 |
-
digest = hashlib.pbkdf2_hmac('sha256', password.encode(), salt, 600_000)
|
| 130 |
-
return salt.hex() + ':' + digest.hex()
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
def _verify_password(password: str, stored: str) -> bool:
|
| 134 |
-
try:
|
| 135 |
-
salt_hex, expected_hex = stored.split(':', 1)
|
| 136 |
-
actual = _hash_password(password, bytes.fromhex(salt_hex)).split(':', 1)[1]
|
| 137 |
-
return secrets.compare_digest(actual, expected_hex)
|
| 138 |
-
except Exception:
|
| 139 |
-
return False
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
def signup(username: str, password: str) -> str:
|
| 143 |
-
if not persistent_accounts_available():
|
| 144 |
-
raise ValueError('Accounts require durable private storage configured by the administrator.')
|
| 145 |
-
user = username.strip().lower()
|
| 146 |
-
if not re.fullmatch(r'[a-z0-9_]{3,30}', user):
|
| 147 |
-
raise ValueError('Username must have 3–30 letters, numbers, or underscores.')
|
| 148 |
-
if len(password) < 12 or len(password) > 128:
|
| 149 |
-
raise ValueError('Use a password of at least 12 characters.')
|
| 150 |
-
with LOCK, STORE.lock:
|
| 151 |
-
users = STORE.state.setdefault('beta_users', {})
|
| 152 |
-
if user in users:
|
| 153 |
-
raise ValueError('That username is taken.')
|
| 154 |
-
users[user] = {'password_hash': _hash_password(password), 'chats': [], 'settings': {'retention': 'forever'}}
|
| 155 |
-
_store_save()
|
| 156 |
-
return user
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
def login(username: str, password: str) -> str | None:
|
| 160 |
-
user = username.strip().lower()
|
| 161 |
-
with LOCK, STORE.lock:
|
| 162 |
-
now=time.time()
|
| 163 |
-
attempts=[t for t in _LOGIN_FAILURES.get(user,[]) if now-t<900]
|
| 164 |
-
_LOGIN_FAILURES[user]=attempts
|
| 165 |
-
if len(attempts)>=8:return None
|
| 166 |
-
data = STORE.state.setdefault('beta_users', {}).get(user, {})
|
| 167 |
-
if not data or not _verify_password(password, data.get('password_hash', '')):
|
| 168 |
-
attempts.append(now)
|
| 169 |
-
return None
|
| 170 |
-
_LOGIN_FAILURES.pop(user,None)
|
| 171 |
-
return user
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
def get_user_data(username: str) -> dict[str, Any]:
|
| 175 |
-
with LOCK, STORE.lock:
|
| 176 |
-
u = STORE.state.get('beta_users', {}).get(username)
|
| 177 |
-
if not u:
|
| 178 |
-
raise ValueError('Account unavailable.')
|
| 179 |
-
return {'chats': json.loads(json.dumps(u.get('chats', []))), 'settings': dict(u.get('settings', {}))}
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
def update_user_data(username: str, chats: list[dict[str, Any]], retention: str) -> dict[str, Any]:
|
| 183 |
-
if retention not in {'forever','30','90'}:
|
| 184 |
-
raise ValueError('Invalid retention option.')
|
| 185 |
-
if len(chats) > 75:
|
| 186 |
-
raise ValueError('Too many conversations.')
|
| 187 |
-
cleaned = []
|
| 188 |
-
now = time.time()
|
| 189 |
-
for chat in chats:
|
| 190 |
-
if not isinstance(chat, dict) or not isinstance(chat.get('history'), list):
|
| 191 |
-
continue
|
| 192 |
-
stamp = float(chat.get('updated') or 0)
|
| 193 |
-
if retention != 'forever' and stamp and stamp / 1000 < now - int(retention)*86400:
|
| 194 |
-
continue
|
| 195 |
-
history=[]
|
| 196 |
-
for msg in chat['history'][-80:]:
|
| 197 |
-
if isinstance(msg, dict) and msg.get('role') in {'user','assistant'}:
|
| 198 |
-
history.append({'role':msg['role'], 'content':str(msg.get('content',''))[:16000]})
|
| 199 |
-
cleaned.append({'id':str(chat.get('id') or '')[:100], 'title':str(chat.get('title') or '')[:100], 'updated': stamp, 'history':history})
|
| 200 |
-
if len(json.dumps(cleaned)) > 1_500_000:
|
| 201 |
-
raise ValueError('Account history exceeds the beta storage limit.')
|
| 202 |
-
with LOCK, STORE.lock:
|
| 203 |
-
u = STORE.state.get('beta_users', {}).get(username)
|
| 204 |
-
if not u: raise ValueError('Account unavailable.')
|
| 205 |
-
u['chats']=cleaned
|
| 206 |
-
u['settings']={'retention':retention}
|
| 207 |
-
_store_save()
|
| 208 |
-
return {'saved':len(cleaned)}
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
def admin_own_chats() -> list[dict[str, Any]]:
|
| 212 |
-
with LOCK, STORE.lock:
|
| 213 |
-
return json.loads(json.dumps(STORE.state.get('beta_admin_own_chats', [])))
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
def save_admin_own_chats(chats: list[dict[str, Any]]) -> dict[str, int]:
|
| 217 |
-
if len(chats)>75 or len(json.dumps(chats))>1_500_000:
|
| 218 |
-
raise ValueError('Admin chat archive too large.')
|
| 219 |
-
cleaned=[]
|
| 220 |
-
for row in chats:
|
| 221 |
-
if not isinstance(row,dict) or not isinstance(row.get('history'),list):continue
|
| 222 |
-
history=[{'role':msg['role'],'content':str(msg.get('content',''))[:16000]}
|
| 223 |
-
for msg in row['history'][-80:]
|
| 224 |
-
if isinstance(msg,dict) and msg.get('role') in {'user','assistant'}]
|
| 225 |
-
cleaned.append({'id':str(row.get('id',''))[:100], 'title':str(row.get('title',''))[:100],
|
| 226 |
-
'updated':float(row.get('updated',0) or 0), 'history':history})
|
| 227 |
-
with LOCK, STORE.lock:
|
| 228 |
-
STORE.state['beta_admin_own_chats']=cleaned
|
| 229 |
-
_store_save()
|
| 230 |
-
return {'saved':len(cleaned)}
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
def _xlsx_text(raw: bytes) -> str:
|
| 234 |
-
# No spreadsheet macros are executed. Read-only, zip/XML based text extraction.
|
| 235 |
-
chunks=[]
|
| 236 |
-
with zipfile.ZipFile(BytesIO(raw)) as z:
|
| 237 |
-
shared=[]
|
| 238 |
-
if 'xl/sharedStrings.xml' in z.namelist():
|
| 239 |
-
root=ET.fromstring(z.read('xl/sharedStrings.xml'))
|
| 240 |
-
for si in root:
|
| 241 |
-
shared.append(''.join(t.text or '' for t in si.iter() if t.tag.endswith('}t')))
|
| 242 |
-
for name in sorted(n for n in z.namelist() if re.fullmatch(r'xl/worksheets/sheet\d+\.xml', n))[:8]:
|
| 243 |
-
root=ET.fromstring(z.read(name)); cells=[]
|
| 244 |
-
for c in root.iter():
|
| 245 |
-
if not c.tag.endswith('}c'):continue
|
| 246 |
-
typ=c.attrib.get('t','')
|
| 247 |
-
val=next((x.text or '' for x in c if x.tag.endswith('}v')),'')
|
| 248 |
-
if typ=='s':
|
| 249 |
-
try:val=shared[int(val)]
|
| 250 |
-
except (ValueError,IndexError):pass
|
| 251 |
-
if typ=='inlineStr':val=''.join(x.text or '' for x in c.iter() if x.tag.endswith('}t'))
|
| 252 |
-
if val:cells.append(f"{c.attrib.get('r','')}: {val}")
|
| 253 |
-
if len(cells)>=1000:break
|
| 254 |
-
chunks.append(name+'\n'+'\n'.join(cells))
|
| 255 |
-
return '\n\n'.join(chunks)
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
def extract_document(filename: str, data: bytes) -> tuple[str,str]:
|
| 259 |
-
ext = Path(filename).suffix.lower()
|
| 260 |
-
if ext not in ALLOWED_EXT: raise ValueError('Unsupported file type.')
|
| 261 |
-
if not data or len(data)>MAX_FILE_SIZE:raise ValueError('File must be 1 byte to 10 MB.')
|
| 262 |
-
if ext=='.txt': content=data.decode('utf-8', errors='replace')
|
| 263 |
-
elif ext=='.pdf':
|
| 264 |
-
from pypdf import PdfReader
|
| 265 |
-
reader=PdfReader(BytesIO(data));content='\n'.join(p.extract_text() or '' for p in reader.pages[:30])
|
| 266 |
-
elif ext=='.docx':
|
| 267 |
-
from docx import Document
|
| 268 |
-
doc=Document(BytesIO(data));content='\n'.join(p.text for p in doc.paragraphs)
|
| 269 |
-
elif ext=='.xlsx':content=_xlsx_text(data)
|
| 270 |
-
else:
|
| 271 |
-
from PIL import Image
|
| 272 |
-
with Image.open(BytesIO(data)) as im:
|
| 273 |
-
im.verify()
|
| 274 |
-
return '', 'Image attached, but this beta has no image-understanding model. AC cannot describe or read the picture yet.'
|
| 275 |
-
content=content.strip()[:30000]
|
| 276 |
-
if not content:
|
| 277 |
-
return '', 'No readable text found. Scanned PDFs require OCR, which is not included yet.'
|
| 278 |
-
return content, f'Readable text extracted ({len(content):,} characters; first 30 PDF pages maximum).'
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
def put_upload(session_id: str, name: str, content: str) -> str:
|
| 282 |
-
with LOCK:
|
| 283 |
-
expired=[k for k,v in UPLOADS.items() if time.time()-v['created']>3600]
|
| 284 |
-
for k in expired:UPLOADS.pop(k,None)
|
| 285 |
-
token=secrets.token_urlsafe(18)
|
| 286 |
-
UPLOADS[token]={'session':session_id,'created':time.time(),'name':name[:130],'content':content}
|
| 287 |
-
return token
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
def get_uploads(session_id: str, tokens: list[str]) -> list[dict[str,str]]:
|
| 291 |
-
with LOCK:
|
| 292 |
-
results=[]
|
| 293 |
-
for tok in tokens[:4]:
|
| 294 |
-
row=UPLOADS.get(tok)
|
| 295 |
-
if row and row['session']==session_id and time.time()-row['created']<3600:
|
| 296 |
-
results.append({'name':row['name'],'content':row['content']})
|
| 297 |
-
return results
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
def migrate_legacy_privacy() -> None:
|
| 301 |
-
# Remove old chat transcript fields from saved session events and any admin
|
| 302 |
-
# audit trails that stored full message text. This intentionally does not
|
| 303 |
-
# erase user-submitted feedback reports.
|
| 304 |
-
with LOCK, STORE.lock:
|
| 305 |
-
if STORE.state.get('beta_privacy_migrated_v1'):return
|
| 306 |
-
STORE.state['session_events']=[]
|
| 307 |
-
STORE.state['beta_privacy_migrated_v1']=True
|
| 308 |
-
_store_save()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|