ac / beta_services.py
mendeld's picture
Upload 15 files
e46ad0b verified
Raw History Blame Contribute Delete
19 kB
"""AC Beta V1: privacy-first analytics, optional accounts, and temporary document reading.
Only aggregated counters are collected for routine chats; no full chat archive is
available to administrators. Do not use the admin telemetry as a transcript store.
"""
from __future__ import annotations
from collections import Counter
from datetime import datetime, timezone
from io import BytesIO
from pathlib import Path
from threading import RLock
from typing import Any
import traceback
import hashlib
import html
import json
import os
import re
import secrets
import time
import xml.etree.ElementTree as ET
import zipfile
from bh56_core import STORE
BUILD = "1.5.0 Beta V7 Google Dashboards"
LOCK = RLock()
# Uploads are intentionally process-memory-only and expire; no uploaded document is
# copied into the administrative store.
UPLOADS: dict[str, dict[str, Any]] = {}
_LAST_STATS_SYNC = 0.0
_LOGIN_FAILURES: dict[str, list[float]] = {}
MAX_FILE_SIZE = 10 * 1024 * 1024
ALLOWED_EXT = {'.pdf','.docx','.txt','.png','.jpg','.jpeg','.xlsx'}
def persistent_accounts_available() -> bool:
# Hugging Face ephemeral files will not provide reliable cross-device history.
return bool((STORE.persistent_enabled and os.getenv('AC_PRIVATE_STORAGE_CONFIRMED') == '1') or os.getenv('AC_DURABLE_DATA_DIR') == '1')
def _store_save() -> None:
STORE.save(sync=STORE.persistent_enabled)
def stats_snapshot() -> dict[str, Any]:
with LOCK, STORE.lock:
d = dict(STORE.state.get('beta_stats', {}))
return {
'questions': int(d.get('questions', 0)),
'jewish': int(d.get('jewish', 0)),
'secular': int(d.get('secular', 0)),
'blocked': int(d.get('blocked', 0)),
'security': int(d.get('security', 0)),
'news': int(d.get('news', 0)),
'chochma': int(d.get('chochma', 0)),
'binah': int(d.get('binah', 0)),
'daas': int(d.get('daas', 0)),
'guest': int(d.get('guest', 0)),
'account': int(d.get('account', 0)),
'reports': len(STORE.state.get('beta_reports', [])),
'critical_flags': len(STORE.state.get('beta_critical', [])),
}
# Private bug tracker. Do not store questions, answers, uploaded files, raw exception
# messages, request headers, session identifiers, user identities, or coordinates.
# Server-generated bug reports are grouped by error type + source location.
_BUG_SYNC_AT = 0.0
_BUG_LIMIT = 300
def _bug_location(exc: BaseException) -> tuple[str, list[str]]:
frames = traceback.extract_tb(exc.__traceback__) if exc.__traceback__ else []
locations = []
for frame in frames[-10:]:
name = Path(frame.filename).name
# Hide full paths, source code snippets, and local variables.
if not re.fullmatch(r'[a-zA-Z_][a-zA-Z_0-9.\-]{0,90}', name):
continue
fn = re.sub(r'[^a-zA-Z_0-9]', '', frame.name)[:70]
locations.append(f'{name}:{int(frame.lineno)} {fn}')
local = [f for f in locations if f.startswith(('bh56_', 'beta_', 'app.py'))]
return (local[-1] if local else (locations[-1] if locations else 'unknown location')), locations
def _bug_description(error_type: str, component: str) -> tuple[str, str]:
if component == 'ai_generation':
return ('AI could not generate an answer',
'The AI model request failed. AC showed its temporary-unavailable message instead.')
if component == 'chat_api':
return ('Chat request failed',
'The chat service encountered an unexpected error while processing a request.')
if component == 'admin_chat_storage':
return ('Admin chat saving failed',
'The admin chat archive could not be saved.')
if component == 'upload':
return ('File processing failed',
'AC was unable to process an uploaded file.')
if error_type in {'TimeoutError', 'ConnectTimeout', 'ReadTimeout'}:
return ('A request timed out',
'An internal service took too long to respond.')
return ('A server function encountered an error',
'AC encountered an unexpected problem that needs investigation.')
def record_bug(exc: BaseException, *, component: str = 'chat_api') -> None:
"""Record an error without content from conversations or raw exceptions.
Duplicate occurrences are grouped. A new occurrence of a fixed bug reopens it.
Local records work without private HF storage, but do not survive Space restarts
reliably unless durable storage has been configured.
"""
global _BUG_SYNC_AT
try:
code = type(exc).__name__[:90]
component = component if component in {
'ai_generation', 'chat_api', 'upload', 'admin_chat_storage', 'server'
} else 'server'
location, frames = _bug_location(exc)
# Gradio's AppError can wrap an underlying exception name. Extract only
# known Python error-class tokens; never persist the full exception text.
reported_cause = ''
if code == 'AppError':
match = re.search(r'\b(KeyError|RuntimeError|ValueError|TypeError|ImportError|OSError|TimeoutError|MemoryError|FileNotFoundError|PermissionError|AssertionError)\b', str(exc))
reported_cause = match.group(1) if match else ''
title, summary = _bug_description(code, component)
signature = hashlib.sha256(f'{component}|{code}|{location}'.encode()).hexdigest()[:20]
now = datetime.now(timezone.utc).isoformat()
with LOCK, STORE.lock:
bugs = STORE.state.setdefault('beta_bugs', [])
found = next((b for b in bugs if b.get('id') == signature), None)
if found is None:
found = {
'id': signature, 'title': title, 'summary': summary,
'error_code': code, 'reported_cause': reported_cause, 'component': component,
'location': location, 'trace': frames,
'first_seen': now, 'last_seen': now, 'occurrences': 0,
'status': 'open',
}
bugs.append(found)
if len(bugs) > _BUG_LIMIT:
# Prefer removing oldest fixed entries; retain active problems.
drop = next((i for i, b in enumerate(bugs) if b.get('status') == 'fixed'), 0)
bugs.pop(drop)
found['occurrences'] = int(found.get('occurrences', 0)) + 1
found['last_seen'] = now
if found.get('status') == 'fixed':
found['status'] = 'open'
found['reopened_at'] = now
# Local save each time; remote sync at most once per two minutes.
sync = STORE.persistent_enabled and time.time() - _BUG_SYNC_AT > 120
STORE.save(sync=sync)
if sync:
_BUG_SYNC_AT = time.time()
except Exception:
# Error reporting must never stop a user request or expose private data.
print('[AC BUG TRACKER] Unable to store bug record', flush=True)
def bug_snapshot() -> dict[str, Any]:
with LOCK, STORE.lock:
rows = [dict(b) for b in STORE.state.get('beta_bugs', [])]
rows.sort(key=lambda b: b.get('last_seen', ''), reverse=True)
return {
'open': [b for b in rows if b.get('status') != 'fixed'],
'fixed': [b for b in rows if b.get('status') == 'fixed'],
'durable': bool(STORE.persistent_enabled),
}
def set_bug_status(bug_id: str, status: str) -> bool:
if status not in {'open', 'fixed'} or not re.fullmatch(r'[a-f0-9]{20}', bug_id):
return False
with LOCK, STORE.lock:
bug = next((b for b in STORE.state.get('beta_bugs', []) if b.get('id') == bug_id), None)
if bug is None:
return False
bug['status'] = status
bug['status_changed_at'] = datetime.now(timezone.utc).isoformat()
# Persist archive action to configured private storage, if available.
STORE.save(sync=STORE.persistent_enabled)
return True
def count_question(category: str, mode: str, logged_in: bool) -> None:
global _LAST_STATS_SYNC
category = category if category in {'jewish','secular','blocked','security','news'} else 'secular'
mode = mode.lower() if mode.lower() in {'chochma','binah','daas'} else 'binah'
with LOCK, STORE.lock:
stats = STORE.state.setdefault('beta_stats', {})
for key in ('questions', category, mode, 'account' if logged_in else 'guest'):
stats[key] = int(stats.get(key, 0)) + 1
# Save statistics locally for responsiveness; sync to private HF storage
# at most once a minute rather than uploading the entire database on
# every user request. Recent stats may be lost on a crash.
if time.time() - _LAST_STATS_SYNC >= 60:
_store_save()
_LAST_STATS_SYNC = time.time()
else:
STORE.save(sync=False)
# Very restrictive pattern matching to avoid automatically exposing ordinary questions.
CRITICAL_PATTERNS = (
re.compile(r"\b(?:i am|i'm|im) (?:going to|about to) (?:kill myself|end my life|commit suicide)\b",re.I),
re.compile(r"\b(?:i will|i'm going to|i am going to) (?:kill|shoot|stab) (?:him|her|them|you|my [a-z]{2,25}|someone|people)\b",re.I),
)
def record_critical_if_any(text: str) -> bool:
if not any(p.search(text) for p in CRITICAL_PATTERNS):
return False
with LOCK, STORE.lock:
STORE.state.setdefault('beta_critical', []).append({
'id': secrets.token_hex(8), 'time': datetime.now(timezone.utc).isoformat(),
'message': str(text)[:1200],
})
STORE.state['beta_critical'] = STORE.state['beta_critical'][-100:]
_store_save()
return True
def list_flags() -> list[dict[str, Any]]:
with LOCK, STORE.lock:
return list(reversed(STORE.state.get('beta_critical', [])[-100:]))
def add_report(kind: str, question: str, answer: str, note: str = '') -> None:
kind = kind if kind in {'good','bad','report','redo','survey'} else 'report'
with LOCK, STORE.lock:
STORE.state.setdefault('beta_reports', []).append({
'id': secrets.token_hex(8),
'time': datetime.now(timezone.utc).isoformat(),
'kind': kind,
'question': question[:1200], 'answer': answer[:2800], 'note': note[:1000],
})
STORE.state['beta_reports'] = STORE.state['beta_reports'][-500:]
_store_save()
def get_reports() -> list[dict[str, Any]]:
with LOCK, STORE.lock:
return list(reversed(STORE.state.get('beta_reports', [])[-500:]))
def _hash_password(password: str, salt: bytes | None = None) -> str:
salt = salt or secrets.token_bytes(16)
digest = hashlib.pbkdf2_hmac('sha256', password.encode(), salt, 600_000)
return salt.hex() + ':' + digest.hex()
def _verify_password(password: str, stored: str) -> bool:
try:
salt_hex, expected_hex = stored.split(':', 1)
actual = _hash_password(password, bytes.fromhex(salt_hex)).split(':', 1)[1]
return secrets.compare_digest(actual, expected_hex)
except Exception:
return False
def signup(username: str, password: str) -> str:
if not persistent_accounts_available():
raise ValueError('Accounts require durable private storage configured by the administrator.')
user = username.strip().lower()
if not re.fullmatch(r'[a-z0-9_]{3,30}', user):
raise ValueError('Username must have 3–30 letters, numbers, or underscores.')
if len(password) < 12 or len(password) > 128:
raise ValueError('Use a password of at least 12 characters.')
with LOCK, STORE.lock:
users = STORE.state.setdefault('beta_users', {})
if user in users:
raise ValueError('That username is taken.')
users[user] = {'password_hash': _hash_password(password), 'chats': [], 'settings': {'retention': 'forever'}}
_store_save()
return user
def login(username: str, password: str) -> str | None:
user = username.strip().lower()
with LOCK, STORE.lock:
now=time.time()
attempts=[t for t in _LOGIN_FAILURES.get(user,[]) if now-t<900]
_LOGIN_FAILURES[user]=attempts
if len(attempts)>=8:return None
data = STORE.state.setdefault('beta_users', {}).get(user, {})
if not data or not _verify_password(password, data.get('password_hash', '')):
attempts.append(now)
return None
_LOGIN_FAILURES.pop(user,None)
return user
def get_user_data(username: str) -> dict[str, Any]:
with LOCK, STORE.lock:
u = STORE.state.get('beta_users', {}).get(username)
if not u:
raise ValueError('Account unavailable.')
return {'chats': json.loads(json.dumps(u.get('chats', []))), 'settings': dict(u.get('settings', {}))}
def update_user_data(username: str, chats: list[dict[str, Any]], retention: str) -> dict[str, Any]:
if retention not in {'forever','30','90'}:
raise ValueError('Invalid retention option.')
if len(chats) > 75:
raise ValueError('Too many conversations.')
cleaned = []
now = time.time()
for chat in chats:
if not isinstance(chat, dict) or not isinstance(chat.get('history'), list):
continue
stamp = float(chat.get('updated') or 0)
if retention != 'forever' and stamp and stamp / 1000 < now - int(retention)*86400:
continue
history=[]
for msg in chat['history'][-80:]:
if isinstance(msg, dict) and msg.get('role') in {'user','assistant'}:
history.append({'role':msg['role'], 'content':str(msg.get('content',''))[:16000]})
cleaned.append({'id':str(chat.get('id') or '')[:100], 'title':str(chat.get('title') or '')[:100], 'updated': stamp, 'history':history})
if len(json.dumps(cleaned)) > 1_500_000:
raise ValueError('Account history exceeds the beta storage limit.')
with LOCK, STORE.lock:
u = STORE.state.get('beta_users', {}).get(username)
if not u: raise ValueError('Account unavailable.')
u['chats']=cleaned
u['settings']={'retention':retention}
_store_save()
return {'saved':len(cleaned)}
def admin_own_chats() -> list[dict[str, Any]]:
with LOCK, STORE.lock:
return json.loads(json.dumps(STORE.state.get('beta_admin_own_chats', [])))
def save_admin_own_chats(chats: list[dict[str, Any]]) -> dict[str, int]:
if len(chats)>75 or len(json.dumps(chats))>1_500_000:
raise ValueError('Admin chat archive too large.')
cleaned=[]
for row in chats:
if not isinstance(row,dict) or not isinstance(row.get('history'),list):continue
history=[{'role':msg['role'],'content':str(msg.get('content',''))[:16000]}
for msg in row['history'][-80:]
if isinstance(msg,dict) and msg.get('role') in {'user','assistant'}]
cleaned.append({'id':str(row.get('id',''))[:100], 'title':str(row.get('title',''))[:100],
'updated':float(row.get('updated',0) or 0), 'history':history})
with LOCK, STORE.lock:
STORE.state['beta_admin_own_chats']=cleaned
_store_save()
return {'saved':len(cleaned)}
def _xlsx_text(raw: bytes) -> str:
# No spreadsheet macros are executed. Read-only, zip/XML based text extraction.
chunks=[]
with zipfile.ZipFile(BytesIO(raw)) as z:
shared=[]
if 'xl/sharedStrings.xml' in z.namelist():
root=ET.fromstring(z.read('xl/sharedStrings.xml'))
for si in root:
shared.append(''.join(t.text or '' for t in si.iter() if t.tag.endswith('}t')))
for name in sorted(n for n in z.namelist() if re.fullmatch(r'xl/worksheets/sheet\d+\.xml', n))[:8]:
root=ET.fromstring(z.read(name)); cells=[]
for c in root.iter():
if not c.tag.endswith('}c'):continue
typ=c.attrib.get('t','')
val=next((x.text or '' for x in c if x.tag.endswith('}v')),'')
if typ=='s':
try:val=shared[int(val)]
except (ValueError,IndexError):pass
if typ=='inlineStr':val=''.join(x.text or '' for x in c.iter() if x.tag.endswith('}t'))
if val:cells.append(f"{c.attrib.get('r','')}: {val}")
if len(cells)>=1000:break
chunks.append(name+'\n'+'\n'.join(cells))
return '\n\n'.join(chunks)
def extract_document(filename: str, data: bytes) -> tuple[str,str]:
ext = Path(filename).suffix.lower()
if ext not in ALLOWED_EXT: raise ValueError('Unsupported file type.')
if not data or len(data)>MAX_FILE_SIZE:raise ValueError('File must be 1 byte to 10 MB.')
if ext=='.txt': content=data.decode('utf-8', errors='replace')
elif ext=='.pdf':
from pypdf import PdfReader
reader=PdfReader(BytesIO(data));content='\n'.join(p.extract_text() or '' for p in reader.pages[:30])
elif ext=='.docx':
from docx import Document
doc=Document(BytesIO(data));content='\n'.join(p.text for p in doc.paragraphs)
elif ext=='.xlsx':content=_xlsx_text(data)
else:
from PIL import Image
with Image.open(BytesIO(data)) as im:
im.verify()
return '', 'Image attached, but this beta has no image-understanding model. AC cannot describe or read the picture yet.'
content=content.strip()[:30000]
if not content:
return '', 'No readable text found. Scanned PDFs require OCR, which is not included yet.'
return content, f'Readable text extracted ({len(content):,} characters; first 30 PDF pages maximum).'
def put_upload(session_id: str, name: str, content: str) -> str:
with LOCK:
expired=[k for k,v in UPLOADS.items() if time.time()-v['created']>3600]
for k in expired:UPLOADS.pop(k,None)
token=secrets.token_urlsafe(18)
UPLOADS[token]={'session':session_id,'created':time.time(),'name':name[:130],'content':content}
return token
def get_uploads(session_id: str, tokens: list[str]) -> list[dict[str,str]]:
with LOCK:
results=[]
for tok in tokens[:4]:
row=UPLOADS.get(tok)
if row and row['session']==session_id and time.time()-row['created']<3600:
results.append({'name':row['name'],'content':row['content']})
return results
def migrate_legacy_privacy() -> None:
# Remove old chat transcript fields from saved session events and any admin
# audit trails that stored full message text. This intentionally does not
# erase user-submitted feedback reports.
with LOCK, STORE.lock:
if STORE.state.get('beta_privacy_migrated_v1'):return
STORE.state['session_events']=[]
STORE.state['beta_privacy_migrated_v1']=True
_store_save()