"""AC Beta V1: privacy-first analytics, optional accounts, and temporary document reading. Only aggregated counters are collected for routine chats; no full chat archive is available to administrators. Do not use the admin telemetry as a transcript store. """ from __future__ import annotations from collections import Counter from datetime import datetime, timezone from io import BytesIO from pathlib import Path from threading import RLock from typing import Any import traceback import hashlib import html import json import os import re import secrets import time import xml.etree.ElementTree as ET import zipfile from bh56_core import STORE BUILD = "1.5.0 Beta V7 Google Dashboards" LOCK = RLock() # Uploads are intentionally process-memory-only and expire; no uploaded document is # copied into the administrative store. UPLOADS: dict[str, dict[str, Any]] = {} _LAST_STATS_SYNC = 0.0 _LOGIN_FAILURES: dict[str, list[float]] = {} MAX_FILE_SIZE = 10 * 1024 * 1024 ALLOWED_EXT = {'.pdf','.docx','.txt','.png','.jpg','.jpeg','.xlsx'} def persistent_accounts_available() -> bool: # Hugging Face ephemeral files will not provide reliable cross-device history. return bool((STORE.persistent_enabled and os.getenv('AC_PRIVATE_STORAGE_CONFIRMED') == '1') or os.getenv('AC_DURABLE_DATA_DIR') == '1') def _store_save() -> None: STORE.save(sync=STORE.persistent_enabled) def stats_snapshot() -> dict[str, Any]: with LOCK, STORE.lock: d = dict(STORE.state.get('beta_stats', {})) return { 'questions': int(d.get('questions', 0)), 'jewish': int(d.get('jewish', 0)), 'secular': int(d.get('secular', 0)), 'blocked': int(d.get('blocked', 0)), 'security': int(d.get('security', 0)), 'news': int(d.get('news', 0)), 'chochma': int(d.get('chochma', 0)), 'binah': int(d.get('binah', 0)), 'daas': int(d.get('daas', 0)), 'guest': int(d.get('guest', 0)), 'account': int(d.get('account', 0)), 'reports': len(STORE.state.get('beta_reports', [])), 'critical_flags': len(STORE.state.get('beta_critical', [])), } # Private bug tracker. Do not store questions, answers, uploaded files, raw exception # messages, request headers, session identifiers, user identities, or coordinates. # Server-generated bug reports are grouped by error type + source location. _BUG_SYNC_AT = 0.0 _BUG_LIMIT = 300 def _bug_location(exc: BaseException) -> tuple[str, list[str]]: frames = traceback.extract_tb(exc.__traceback__) if exc.__traceback__ else [] locations = [] for frame in frames[-10:]: name = Path(frame.filename).name # Hide full paths, source code snippets, and local variables. if not re.fullmatch(r'[a-zA-Z_][a-zA-Z_0-9.\-]{0,90}', name): continue fn = re.sub(r'[^a-zA-Z_0-9]', '', frame.name)[:70] locations.append(f'{name}:{int(frame.lineno)} {fn}') local = [f for f in locations if f.startswith(('bh56_', 'beta_', 'app.py'))] return (local[-1] if local else (locations[-1] if locations else 'unknown location')), locations def _bug_description(error_type: str, component: str) -> tuple[str, str]: if component == 'ai_generation': return ('AI could not generate an answer', 'The AI model request failed. AC showed its temporary-unavailable message instead.') if component == 'chat_api': return ('Chat request failed', 'The chat service encountered an unexpected error while processing a request.') if component == 'admin_chat_storage': return ('Admin chat saving failed', 'The admin chat archive could not be saved.') if component == 'upload': return ('File processing failed', 'AC was unable to process an uploaded file.') if error_type in {'TimeoutError', 'ConnectTimeout', 'ReadTimeout'}: return ('A request timed out', 'An internal service took too long to respond.') return ('A server function encountered an error', 'AC encountered an unexpected problem that needs investigation.') def record_bug(exc: BaseException, *, component: str = 'chat_api') -> None: """Record an error without content from conversations or raw exceptions. Duplicate occurrences are grouped. A new occurrence of a fixed bug reopens it. Local records work without private HF storage, but do not survive Space restarts reliably unless durable storage has been configured. """ global _BUG_SYNC_AT try: code = type(exc).__name__[:90] component = component if component in { 'ai_generation', 'chat_api', 'upload', 'admin_chat_storage', 'server' } else 'server' location, frames = _bug_location(exc) # Gradio's AppError can wrap an underlying exception name. Extract only # known Python error-class tokens; never persist the full exception text. reported_cause = '' if code == 'AppError': match = re.search(r'\b(KeyError|RuntimeError|ValueError|TypeError|ImportError|OSError|TimeoutError|MemoryError|FileNotFoundError|PermissionError|AssertionError)\b', str(exc)) reported_cause = match.group(1) if match else '' title, summary = _bug_description(code, component) signature = hashlib.sha256(f'{component}|{code}|{location}'.encode()).hexdigest()[:20] now = datetime.now(timezone.utc).isoformat() with LOCK, STORE.lock: bugs = STORE.state.setdefault('beta_bugs', []) found = next((b for b in bugs if b.get('id') == signature), None) if found is None: found = { 'id': signature, 'title': title, 'summary': summary, 'error_code': code, 'reported_cause': reported_cause, 'component': component, 'location': location, 'trace': frames, 'first_seen': now, 'last_seen': now, 'occurrences': 0, 'status': 'open', } bugs.append(found) if len(bugs) > _BUG_LIMIT: # Prefer removing oldest fixed entries; retain active problems. drop = next((i for i, b in enumerate(bugs) if b.get('status') == 'fixed'), 0) bugs.pop(drop) found['occurrences'] = int(found.get('occurrences', 0)) + 1 found['last_seen'] = now if found.get('status') == 'fixed': found['status'] = 'open' found['reopened_at'] = now # Local save each time; remote sync at most once per two minutes. sync = STORE.persistent_enabled and time.time() - _BUG_SYNC_AT > 120 STORE.save(sync=sync) if sync: _BUG_SYNC_AT = time.time() except Exception: # Error reporting must never stop a user request or expose private data. print('[AC BUG TRACKER] Unable to store bug record', flush=True) def bug_snapshot() -> dict[str, Any]: with LOCK, STORE.lock: rows = [dict(b) for b in STORE.state.get('beta_bugs', [])] rows.sort(key=lambda b: b.get('last_seen', ''), reverse=True) return { 'open': [b for b in rows if b.get('status') != 'fixed'], 'fixed': [b for b in rows if b.get('status') == 'fixed'], 'durable': bool(STORE.persistent_enabled), } def set_bug_status(bug_id: str, status: str) -> bool: if status not in {'open', 'fixed'} or not re.fullmatch(r'[a-f0-9]{20}', bug_id): return False with LOCK, STORE.lock: bug = next((b for b in STORE.state.get('beta_bugs', []) if b.get('id') == bug_id), None) if bug is None: return False bug['status'] = status bug['status_changed_at'] = datetime.now(timezone.utc).isoformat() # Persist archive action to configured private storage, if available. STORE.save(sync=STORE.persistent_enabled) return True def count_question(category: str, mode: str, logged_in: bool) -> None: global _LAST_STATS_SYNC category = category if category in {'jewish','secular','blocked','security','news'} else 'secular' mode = mode.lower() if mode.lower() in {'chochma','binah','daas'} else 'binah' with LOCK, STORE.lock: stats = STORE.state.setdefault('beta_stats', {}) for key in ('questions', category, mode, 'account' if logged_in else 'guest'): stats[key] = int(stats.get(key, 0)) + 1 # Save statistics locally for responsiveness; sync to private HF storage # at most once a minute rather than uploading the entire database on # every user request. Recent stats may be lost on a crash. if time.time() - _LAST_STATS_SYNC >= 60: _store_save() _LAST_STATS_SYNC = time.time() else: STORE.save(sync=False) # Very restrictive pattern matching to avoid automatically exposing ordinary questions. CRITICAL_PATTERNS = ( re.compile(r"\b(?:i am|i'm|im) (?:going to|about to) (?:kill myself|end my life|commit suicide)\b",re.I), re.compile(r"\b(?:i will|i'm going to|i am going to) (?:kill|shoot|stab) (?:him|her|them|you|my [a-z]{2,25}|someone|people)\b",re.I), ) def record_critical_if_any(text: str) -> bool: if not any(p.search(text) for p in CRITICAL_PATTERNS): return False with LOCK, STORE.lock: STORE.state.setdefault('beta_critical', []).append({ 'id': secrets.token_hex(8), 'time': datetime.now(timezone.utc).isoformat(), 'message': str(text)[:1200], }) STORE.state['beta_critical'] = STORE.state['beta_critical'][-100:] _store_save() return True def list_flags() -> list[dict[str, Any]]: with LOCK, STORE.lock: return list(reversed(STORE.state.get('beta_critical', [])[-100:])) def add_report(kind: str, question: str, answer: str, note: str = '') -> None: kind = kind if kind in {'good','bad','report','redo','survey'} else 'report' with LOCK, STORE.lock: STORE.state.setdefault('beta_reports', []).append({ 'id': secrets.token_hex(8), 'time': datetime.now(timezone.utc).isoformat(), 'kind': kind, 'question': question[:1200], 'answer': answer[:2800], 'note': note[:1000], }) STORE.state['beta_reports'] = STORE.state['beta_reports'][-500:] _store_save() def get_reports() -> list[dict[str, Any]]: with LOCK, STORE.lock: return list(reversed(STORE.state.get('beta_reports', [])[-500:])) def _hash_password(password: str, salt: bytes | None = None) -> str: salt = salt or secrets.token_bytes(16) digest = hashlib.pbkdf2_hmac('sha256', password.encode(), salt, 600_000) return salt.hex() + ':' + digest.hex() def _verify_password(password: str, stored: str) -> bool: try: salt_hex, expected_hex = stored.split(':', 1) actual = _hash_password(password, bytes.fromhex(salt_hex)).split(':', 1)[1] return secrets.compare_digest(actual, expected_hex) except Exception: return False def signup(username: str, password: str) -> str: if not persistent_accounts_available(): raise ValueError('Accounts require durable private storage configured by the administrator.') user = username.strip().lower() if not re.fullmatch(r'[a-z0-9_]{3,30}', user): raise ValueError('Username must have 3–30 letters, numbers, or underscores.') if len(password) < 12 or len(password) > 128: raise ValueError('Use a password of at least 12 characters.') with LOCK, STORE.lock: users = STORE.state.setdefault('beta_users', {}) if user in users: raise ValueError('That username is taken.') users[user] = {'password_hash': _hash_password(password), 'chats': [], 'settings': {'retention': 'forever'}} _store_save() return user def login(username: str, password: str) -> str | None: user = username.strip().lower() with LOCK, STORE.lock: now=time.time() attempts=[t for t in _LOGIN_FAILURES.get(user,[]) if now-t<900] _LOGIN_FAILURES[user]=attempts if len(attempts)>=8:return None data = STORE.state.setdefault('beta_users', {}).get(user, {}) if not data or not _verify_password(password, data.get('password_hash', '')): attempts.append(now) return None _LOGIN_FAILURES.pop(user,None) return user def get_user_data(username: str) -> dict[str, Any]: with LOCK, STORE.lock: u = STORE.state.get('beta_users', {}).get(username) if not u: raise ValueError('Account unavailable.') return {'chats': json.loads(json.dumps(u.get('chats', []))), 'settings': dict(u.get('settings', {}))} def update_user_data(username: str, chats: list[dict[str, Any]], retention: str) -> dict[str, Any]: if retention not in {'forever','30','90'}: raise ValueError('Invalid retention option.') if len(chats) > 75: raise ValueError('Too many conversations.') cleaned = [] now = time.time() for chat in chats: if not isinstance(chat, dict) or not isinstance(chat.get('history'), list): continue stamp = float(chat.get('updated') or 0) if retention != 'forever' and stamp and stamp / 1000 < now - int(retention)*86400: continue history=[] for msg in chat['history'][-80:]: if isinstance(msg, dict) and msg.get('role') in {'user','assistant'}: history.append({'role':msg['role'], 'content':str(msg.get('content',''))[:16000]}) cleaned.append({'id':str(chat.get('id') or '')[:100], 'title':str(chat.get('title') or '')[:100], 'updated': stamp, 'history':history}) if len(json.dumps(cleaned)) > 1_500_000: raise ValueError('Account history exceeds the beta storage limit.') with LOCK, STORE.lock: u = STORE.state.get('beta_users', {}).get(username) if not u: raise ValueError('Account unavailable.') u['chats']=cleaned u['settings']={'retention':retention} _store_save() return {'saved':len(cleaned)} def admin_own_chats() -> list[dict[str, Any]]: with LOCK, STORE.lock: return json.loads(json.dumps(STORE.state.get('beta_admin_own_chats', []))) def save_admin_own_chats(chats: list[dict[str, Any]]) -> dict[str, int]: if len(chats)>75 or len(json.dumps(chats))>1_500_000: raise ValueError('Admin chat archive too large.') cleaned=[] for row in chats: if not isinstance(row,dict) or not isinstance(row.get('history'),list):continue history=[{'role':msg['role'],'content':str(msg.get('content',''))[:16000]} for msg in row['history'][-80:] if isinstance(msg,dict) and msg.get('role') in {'user','assistant'}] cleaned.append({'id':str(row.get('id',''))[:100], 'title':str(row.get('title',''))[:100], 'updated':float(row.get('updated',0) or 0), 'history':history}) with LOCK, STORE.lock: STORE.state['beta_admin_own_chats']=cleaned _store_save() return {'saved':len(cleaned)} def _xlsx_text(raw: bytes) -> str: # No spreadsheet macros are executed. Read-only, zip/XML based text extraction. chunks=[] with zipfile.ZipFile(BytesIO(raw)) as z: shared=[] if 'xl/sharedStrings.xml' in z.namelist(): root=ET.fromstring(z.read('xl/sharedStrings.xml')) for si in root: shared.append(''.join(t.text or '' for t in si.iter() if t.tag.endswith('}t'))) for name in sorted(n for n in z.namelist() if re.fullmatch(r'xl/worksheets/sheet\d+\.xml', n))[:8]: root=ET.fromstring(z.read(name)); cells=[] for c in root.iter(): if not c.tag.endswith('}c'):continue typ=c.attrib.get('t','') val=next((x.text or '' for x in c if x.tag.endswith('}v')),'') if typ=='s': try:val=shared[int(val)] except (ValueError,IndexError):pass if typ=='inlineStr':val=''.join(x.text or '' for x in c.iter() if x.tag.endswith('}t')) if val:cells.append(f"{c.attrib.get('r','')}: {val}") if len(cells)>=1000:break chunks.append(name+'\n'+'\n'.join(cells)) return '\n\n'.join(chunks) def extract_document(filename: str, data: bytes) -> tuple[str,str]: ext = Path(filename).suffix.lower() if ext not in ALLOWED_EXT: raise ValueError('Unsupported file type.') if not data or len(data)>MAX_FILE_SIZE:raise ValueError('File must be 1 byte to 10 MB.') if ext=='.txt': content=data.decode('utf-8', errors='replace') elif ext=='.pdf': from pypdf import PdfReader reader=PdfReader(BytesIO(data));content='\n'.join(p.extract_text() or '' for p in reader.pages[:30]) elif ext=='.docx': from docx import Document doc=Document(BytesIO(data));content='\n'.join(p.text for p in doc.paragraphs) elif ext=='.xlsx':content=_xlsx_text(data) else: from PIL import Image with Image.open(BytesIO(data)) as im: im.verify() return '', 'Image attached, but this beta has no image-understanding model. AC cannot describe or read the picture yet.' content=content.strip()[:30000] if not content: return '', 'No readable text found. Scanned PDFs require OCR, which is not included yet.' return content, f'Readable text extracted ({len(content):,} characters; first 30 PDF pages maximum).' def put_upload(session_id: str, name: str, content: str) -> str: with LOCK: expired=[k for k,v in UPLOADS.items() if time.time()-v['created']>3600] for k in expired:UPLOADS.pop(k,None) token=secrets.token_urlsafe(18) UPLOADS[token]={'session':session_id,'created':time.time(),'name':name[:130],'content':content} return token def get_uploads(session_id: str, tokens: list[str]) -> list[dict[str,str]]: with LOCK: results=[] for tok in tokens[:4]: row=UPLOADS.get(tok) if row and row['session']==session_id and time.time()-row['created']<3600: results.append({'name':row['name'],'content':row['content']}) return results def migrate_legacy_privacy() -> None: # Remove old chat transcript fields from saved session events and any admin # audit trails that stored full message text. This intentionally does not # erase user-submitted feedback reports. with LOCK, STORE.lock: if STORE.state.get('beta_privacy_migrated_v1'):return STORE.state['session_events']=[] STORE.state['beta_privacy_migrated_v1']=True _store_save()