Spaces:
Running on Zero
Running on Zero
Download beta_services.py from mendeld/ac: direct link, hf CLI and curl.
- Browser
- Download file 19 kB
-
https://huggingface.co/spaces/mendeld/ac/resolve/main/beta_services.py
- Command line
-
hf download hf://spaces/mendeld/ac/beta_services.py
-
curl -L -o beta_services.py https://huggingface.co/spaces/mendeld/ac/resolve/main/beta_services.py
19 kB
| """AC Beta V1: privacy-first analytics, optional accounts, and temporary document reading. | |
| Only aggregated counters are collected for routine chats; no full chat archive is | |
| available to administrators. Do not use the admin telemetry as a transcript store. | |
| """ | |
| from __future__ import annotations | |
| from collections import Counter | |
| from datetime import datetime, timezone | |
| from io import BytesIO | |
| from pathlib import Path | |
| from threading import RLock | |
| from typing import Any | |
| import traceback | |
| import hashlib | |
| import html | |
| import json | |
| import os | |
| import re | |
| import secrets | |
| import time | |
| import xml.etree.ElementTree as ET | |
| import zipfile | |
| from bh56_core import STORE | |
| BUILD = "1.5.0 Beta V7 Google Dashboards" | |
| LOCK = RLock() | |
| # Uploads are intentionally process-memory-only and expire; no uploaded document is | |
| # copied into the administrative store. | |
| UPLOADS: dict[str, dict[str, Any]] = {} | |
| _LAST_STATS_SYNC = 0.0 | |
| _LOGIN_FAILURES: dict[str, list[float]] = {} | |
| MAX_FILE_SIZE = 10 * 1024 * 1024 | |
| ALLOWED_EXT = {'.pdf','.docx','.txt','.png','.jpg','.jpeg','.xlsx'} | |
| def persistent_accounts_available() -> bool: | |
| # Hugging Face ephemeral files will not provide reliable cross-device history. | |
| return bool((STORE.persistent_enabled and os.getenv('AC_PRIVATE_STORAGE_CONFIRMED') == '1') or os.getenv('AC_DURABLE_DATA_DIR') == '1') | |
| def _store_save() -> None: | |
| STORE.save(sync=STORE.persistent_enabled) | |
| def stats_snapshot() -> dict[str, Any]: | |
| with LOCK, STORE.lock: | |
| d = dict(STORE.state.get('beta_stats', {})) | |
| return { | |
| 'questions': int(d.get('questions', 0)), | |
| 'jewish': int(d.get('jewish', 0)), | |
| 'secular': int(d.get('secular', 0)), | |
| 'blocked': int(d.get('blocked', 0)), | |
| 'security': int(d.get('security', 0)), | |
| 'news': int(d.get('news', 0)), | |
| 'chochma': int(d.get('chochma', 0)), | |
| 'binah': int(d.get('binah', 0)), | |
| 'daas': int(d.get('daas', 0)), | |
| 'guest': int(d.get('guest', 0)), | |
| 'account': int(d.get('account', 0)), | |
| 'reports': len(STORE.state.get('beta_reports', [])), | |
| 'critical_flags': len(STORE.state.get('beta_critical', [])), | |
| } | |
| # Private bug tracker. Do not store questions, answers, uploaded files, raw exception | |
| # messages, request headers, session identifiers, user identities, or coordinates. | |
| # Server-generated bug reports are grouped by error type + source location. | |
| _BUG_SYNC_AT = 0.0 | |
| _BUG_LIMIT = 300 | |
| def _bug_location(exc: BaseException) -> tuple[str, list[str]]: | |
| frames = traceback.extract_tb(exc.__traceback__) if exc.__traceback__ else [] | |
| locations = [] | |
| for frame in frames[-10:]: | |
| name = Path(frame.filename).name | |
| # Hide full paths, source code snippets, and local variables. | |
| if not re.fullmatch(r'[a-zA-Z_][a-zA-Z_0-9.\-]{0,90}', name): | |
| continue | |
| fn = re.sub(r'[^a-zA-Z_0-9]', '', frame.name)[:70] | |
| locations.append(f'{name}:{int(frame.lineno)} {fn}') | |
| local = [f for f in locations if f.startswith(('bh56_', 'beta_', 'app.py'))] | |
| return (local[-1] if local else (locations[-1] if locations else 'unknown location')), locations | |
| def _bug_description(error_type: str, component: str) -> tuple[str, str]: | |
| if component == 'ai_generation': | |
| return ('AI could not generate an answer', | |
| 'The AI model request failed. AC showed its temporary-unavailable message instead.') | |
| if component == 'chat_api': | |
| return ('Chat request failed', | |
| 'The chat service encountered an unexpected error while processing a request.') | |
| if component == 'admin_chat_storage': | |
| return ('Admin chat saving failed', | |
| 'The admin chat archive could not be saved.') | |
| if component == 'upload': | |
| return ('File processing failed', | |
| 'AC was unable to process an uploaded file.') | |
| if error_type in {'TimeoutError', 'ConnectTimeout', 'ReadTimeout'}: | |
| return ('A request timed out', | |
| 'An internal service took too long to respond.') | |
| return ('A server function encountered an error', | |
| 'AC encountered an unexpected problem that needs investigation.') | |
| def record_bug(exc: BaseException, *, component: str = 'chat_api') -> None: | |
| """Record an error without content from conversations or raw exceptions. | |
| Duplicate occurrences are grouped. A new occurrence of a fixed bug reopens it. | |
| Local records work without private HF storage, but do not survive Space restarts | |
| reliably unless durable storage has been configured. | |
| """ | |
| global _BUG_SYNC_AT | |
| try: | |
| code = type(exc).__name__[:90] | |
| component = component if component in { | |
| 'ai_generation', 'chat_api', 'upload', 'admin_chat_storage', 'server' | |
| } else 'server' | |
| location, frames = _bug_location(exc) | |
| # Gradio's AppError can wrap an underlying exception name. Extract only | |
| # known Python error-class tokens; never persist the full exception text. | |
| reported_cause = '' | |
| if code == 'AppError': | |
| match = re.search(r'\b(KeyError|RuntimeError|ValueError|TypeError|ImportError|OSError|TimeoutError|MemoryError|FileNotFoundError|PermissionError|AssertionError)\b', str(exc)) | |
| reported_cause = match.group(1) if match else '' | |
| title, summary = _bug_description(code, component) | |
| signature = hashlib.sha256(f'{component}|{code}|{location}'.encode()).hexdigest()[:20] | |
| now = datetime.now(timezone.utc).isoformat() | |
| with LOCK, STORE.lock: | |
| bugs = STORE.state.setdefault('beta_bugs', []) | |
| found = next((b for b in bugs if b.get('id') == signature), None) | |
| if found is None: | |
| found = { | |
| 'id': signature, 'title': title, 'summary': summary, | |
| 'error_code': code, 'reported_cause': reported_cause, 'component': component, | |
| 'location': location, 'trace': frames, | |
| 'first_seen': now, 'last_seen': now, 'occurrences': 0, | |
| 'status': 'open', | |
| } | |
| bugs.append(found) | |
| if len(bugs) > _BUG_LIMIT: | |
| # Prefer removing oldest fixed entries; retain active problems. | |
| drop = next((i for i, b in enumerate(bugs) if b.get('status') == 'fixed'), 0) | |
| bugs.pop(drop) | |
| found['occurrences'] = int(found.get('occurrences', 0)) + 1 | |
| found['last_seen'] = now | |
| if found.get('status') == 'fixed': | |
| found['status'] = 'open' | |
| found['reopened_at'] = now | |
| # Local save each time; remote sync at most once per two minutes. | |
| sync = STORE.persistent_enabled and time.time() - _BUG_SYNC_AT > 120 | |
| STORE.save(sync=sync) | |
| if sync: | |
| _BUG_SYNC_AT = time.time() | |
| except Exception: | |
| # Error reporting must never stop a user request or expose private data. | |
| print('[AC BUG TRACKER] Unable to store bug record', flush=True) | |
| def bug_snapshot() -> dict[str, Any]: | |
| with LOCK, STORE.lock: | |
| rows = [dict(b) for b in STORE.state.get('beta_bugs', [])] | |
| rows.sort(key=lambda b: b.get('last_seen', ''), reverse=True) | |
| return { | |
| 'open': [b for b in rows if b.get('status') != 'fixed'], | |
| 'fixed': [b for b in rows if b.get('status') == 'fixed'], | |
| 'durable': bool(STORE.persistent_enabled), | |
| } | |
| def set_bug_status(bug_id: str, status: str) -> bool: | |
| if status not in {'open', 'fixed'} or not re.fullmatch(r'[a-f0-9]{20}', bug_id): | |
| return False | |
| with LOCK, STORE.lock: | |
| bug = next((b for b in STORE.state.get('beta_bugs', []) if b.get('id') == bug_id), None) | |
| if bug is None: | |
| return False | |
| bug['status'] = status | |
| bug['status_changed_at'] = datetime.now(timezone.utc).isoformat() | |
| # Persist archive action to configured private storage, if available. | |
| STORE.save(sync=STORE.persistent_enabled) | |
| return True | |
| def count_question(category: str, mode: str, logged_in: bool) -> None: | |
| global _LAST_STATS_SYNC | |
| category = category if category in {'jewish','secular','blocked','security','news'} else 'secular' | |
| mode = mode.lower() if mode.lower() in {'chochma','binah','daas'} else 'binah' | |
| with LOCK, STORE.lock: | |
| stats = STORE.state.setdefault('beta_stats', {}) | |
| for key in ('questions', category, mode, 'account' if logged_in else 'guest'): | |
| stats[key] = int(stats.get(key, 0)) + 1 | |
| # Save statistics locally for responsiveness; sync to private HF storage | |
| # at most once a minute rather than uploading the entire database on | |
| # every user request. Recent stats may be lost on a crash. | |
| if time.time() - _LAST_STATS_SYNC >= 60: | |
| _store_save() | |
| _LAST_STATS_SYNC = time.time() | |
| else: | |
| STORE.save(sync=False) | |
| # Very restrictive pattern matching to avoid automatically exposing ordinary questions. | |
| CRITICAL_PATTERNS = ( | |
| re.compile(r"\b(?:i am|i'm|im) (?:going to|about to) (?:kill myself|end my life|commit suicide)\b",re.I), | |
| re.compile(r"\b(?:i will|i'm going to|i am going to) (?:kill|shoot|stab) (?:him|her|them|you|my [a-z]{2,25}|someone|people)\b",re.I), | |
| ) | |
| def record_critical_if_any(text: str) -> bool: | |
| if not any(p.search(text) for p in CRITICAL_PATTERNS): | |
| return False | |
| with LOCK, STORE.lock: | |
| STORE.state.setdefault('beta_critical', []).append({ | |
| 'id': secrets.token_hex(8), 'time': datetime.now(timezone.utc).isoformat(), | |
| 'message': str(text)[:1200], | |
| }) | |
| STORE.state['beta_critical'] = STORE.state['beta_critical'][-100:] | |
| _store_save() | |
| return True | |
| def list_flags() -> list[dict[str, Any]]: | |
| with LOCK, STORE.lock: | |
| return list(reversed(STORE.state.get('beta_critical', [])[-100:])) | |
| def add_report(kind: str, question: str, answer: str, note: str = '') -> None: | |
| kind = kind if kind in {'good','bad','report','redo','survey'} else 'report' | |
| with LOCK, STORE.lock: | |
| STORE.state.setdefault('beta_reports', []).append({ | |
| 'id': secrets.token_hex(8), | |
| 'time': datetime.now(timezone.utc).isoformat(), | |
| 'kind': kind, | |
| 'question': question[:1200], 'answer': answer[:2800], 'note': note[:1000], | |
| }) | |
| STORE.state['beta_reports'] = STORE.state['beta_reports'][-500:] | |
| _store_save() | |
| def get_reports() -> list[dict[str, Any]]: | |
| with LOCK, STORE.lock: | |
| return list(reversed(STORE.state.get('beta_reports', [])[-500:])) | |
| def _hash_password(password: str, salt: bytes | None = None) -> str: | |
| salt = salt or secrets.token_bytes(16) | |
| digest = hashlib.pbkdf2_hmac('sha256', password.encode(), salt, 600_000) | |
| return salt.hex() + ':' + digest.hex() | |
| def _verify_password(password: str, stored: str) -> bool: | |
| try: | |
| salt_hex, expected_hex = stored.split(':', 1) | |
| actual = _hash_password(password, bytes.fromhex(salt_hex)).split(':', 1)[1] | |
| return secrets.compare_digest(actual, expected_hex) | |
| except Exception: | |
| return False | |
| def signup(username: str, password: str) -> str: | |
| if not persistent_accounts_available(): | |
| raise ValueError('Accounts require durable private storage configured by the administrator.') | |
| user = username.strip().lower() | |
| if not re.fullmatch(r'[a-z0-9_]{3,30}', user): | |
| raise ValueError('Username must have 3–30 letters, numbers, or underscores.') | |
| if len(password) < 12 or len(password) > 128: | |
| raise ValueError('Use a password of at least 12 characters.') | |
| with LOCK, STORE.lock: | |
| users = STORE.state.setdefault('beta_users', {}) | |
| if user in users: | |
| raise ValueError('That username is taken.') | |
| users[user] = {'password_hash': _hash_password(password), 'chats': [], 'settings': {'retention': 'forever'}} | |
| _store_save() | |
| return user | |
| def login(username: str, password: str) -> str | None: | |
| user = username.strip().lower() | |
| with LOCK, STORE.lock: | |
| now=time.time() | |
| attempts=[t for t in _LOGIN_FAILURES.get(user,[]) if now-t<900] | |
| _LOGIN_FAILURES[user]=attempts | |
| if len(attempts)>=8:return None | |
| data = STORE.state.setdefault('beta_users', {}).get(user, {}) | |
| if not data or not _verify_password(password, data.get('password_hash', '')): | |
| attempts.append(now) | |
| return None | |
| _LOGIN_FAILURES.pop(user,None) | |
| return user | |
| def get_user_data(username: str) -> dict[str, Any]: | |
| with LOCK, STORE.lock: | |
| u = STORE.state.get('beta_users', {}).get(username) | |
| if not u: | |
| raise ValueError('Account unavailable.') | |
| return {'chats': json.loads(json.dumps(u.get('chats', []))), 'settings': dict(u.get('settings', {}))} | |
| def update_user_data(username: str, chats: list[dict[str, Any]], retention: str) -> dict[str, Any]: | |
| if retention not in {'forever','30','90'}: | |
| raise ValueError('Invalid retention option.') | |
| if len(chats) > 75: | |
| raise ValueError('Too many conversations.') | |
| cleaned = [] | |
| now = time.time() | |
| for chat in chats: | |
| if not isinstance(chat, dict) or not isinstance(chat.get('history'), list): | |
| continue | |
| stamp = float(chat.get('updated') or 0) | |
| if retention != 'forever' and stamp and stamp / 1000 < now - int(retention)*86400: | |
| continue | |
| history=[] | |
| for msg in chat['history'][-80:]: | |
| if isinstance(msg, dict) and msg.get('role') in {'user','assistant'}: | |
| history.append({'role':msg['role'], 'content':str(msg.get('content',''))[:16000]}) | |
| cleaned.append({'id':str(chat.get('id') or '')[:100], 'title':str(chat.get('title') or '')[:100], 'updated': stamp, 'history':history}) | |
| if len(json.dumps(cleaned)) > 1_500_000: | |
| raise ValueError('Account history exceeds the beta storage limit.') | |
| with LOCK, STORE.lock: | |
| u = STORE.state.get('beta_users', {}).get(username) | |
| if not u: raise ValueError('Account unavailable.') | |
| u['chats']=cleaned | |
| u['settings']={'retention':retention} | |
| _store_save() | |
| return {'saved':len(cleaned)} | |
| def admin_own_chats() -> list[dict[str, Any]]: | |
| with LOCK, STORE.lock: | |
| return json.loads(json.dumps(STORE.state.get('beta_admin_own_chats', []))) | |
| def save_admin_own_chats(chats: list[dict[str, Any]]) -> dict[str, int]: | |
| if len(chats)>75 or len(json.dumps(chats))>1_500_000: | |
| raise ValueError('Admin chat archive too large.') | |
| cleaned=[] | |
| for row in chats: | |
| if not isinstance(row,dict) or not isinstance(row.get('history'),list):continue | |
| history=[{'role':msg['role'],'content':str(msg.get('content',''))[:16000]} | |
| for msg in row['history'][-80:] | |
| if isinstance(msg,dict) and msg.get('role') in {'user','assistant'}] | |
| cleaned.append({'id':str(row.get('id',''))[:100], 'title':str(row.get('title',''))[:100], | |
| 'updated':float(row.get('updated',0) or 0), 'history':history}) | |
| with LOCK, STORE.lock: | |
| STORE.state['beta_admin_own_chats']=cleaned | |
| _store_save() | |
| return {'saved':len(cleaned)} | |
| def _xlsx_text(raw: bytes) -> str: | |
| # No spreadsheet macros are executed. Read-only, zip/XML based text extraction. | |
| chunks=[] | |
| with zipfile.ZipFile(BytesIO(raw)) as z: | |
| shared=[] | |
| if 'xl/sharedStrings.xml' in z.namelist(): | |
| root=ET.fromstring(z.read('xl/sharedStrings.xml')) | |
| for si in root: | |
| shared.append(''.join(t.text or '' for t in si.iter() if t.tag.endswith('}t'))) | |
| for name in sorted(n for n in z.namelist() if re.fullmatch(r'xl/worksheets/sheet\d+\.xml', n))[:8]: | |
| root=ET.fromstring(z.read(name)); cells=[] | |
| for c in root.iter(): | |
| if not c.tag.endswith('}c'):continue | |
| typ=c.attrib.get('t','') | |
| val=next((x.text or '' for x in c if x.tag.endswith('}v')),'') | |
| if typ=='s': | |
| try:val=shared[int(val)] | |
| except (ValueError,IndexError):pass | |
| if typ=='inlineStr':val=''.join(x.text or '' for x in c.iter() if x.tag.endswith('}t')) | |
| if val:cells.append(f"{c.attrib.get('r','')}: {val}") | |
| if len(cells)>=1000:break | |
| chunks.append(name+'\n'+'\n'.join(cells)) | |
| return '\n\n'.join(chunks) | |
| def extract_document(filename: str, data: bytes) -> tuple[str,str]: | |
| ext = Path(filename).suffix.lower() | |
| if ext not in ALLOWED_EXT: raise ValueError('Unsupported file type.') | |
| if not data or len(data)>MAX_FILE_SIZE:raise ValueError('File must be 1 byte to 10 MB.') | |
| if ext=='.txt': content=data.decode('utf-8', errors='replace') | |
| elif ext=='.pdf': | |
| from pypdf import PdfReader | |
| reader=PdfReader(BytesIO(data));content='\n'.join(p.extract_text() or '' for p in reader.pages[:30]) | |
| elif ext=='.docx': | |
| from docx import Document | |
| doc=Document(BytesIO(data));content='\n'.join(p.text for p in doc.paragraphs) | |
| elif ext=='.xlsx':content=_xlsx_text(data) | |
| else: | |
| from PIL import Image | |
| with Image.open(BytesIO(data)) as im: | |
| im.verify() | |
| return '', 'Image attached, but this beta has no image-understanding model. AC cannot describe or read the picture yet.' | |
| content=content.strip()[:30000] | |
| if not content: | |
| return '', 'No readable text found. Scanned PDFs require OCR, which is not included yet.' | |
| return content, f'Readable text extracted ({len(content):,} characters; first 30 PDF pages maximum).' | |
| def put_upload(session_id: str, name: str, content: str) -> str: | |
| with LOCK: | |
| expired=[k for k,v in UPLOADS.items() if time.time()-v['created']>3600] | |
| for k in expired:UPLOADS.pop(k,None) | |
| token=secrets.token_urlsafe(18) | |
| UPLOADS[token]={'session':session_id,'created':time.time(),'name':name[:130],'content':content} | |
| return token | |
| def get_uploads(session_id: str, tokens: list[str]) -> list[dict[str,str]]: | |
| with LOCK: | |
| results=[] | |
| for tok in tokens[:4]: | |
| row=UPLOADS.get(tok) | |
| if row and row['session']==session_id and time.time()-row['created']<3600: | |
| results.append({'name':row['name'],'content':row['content']}) | |
| return results | |
| def migrate_legacy_privacy() -> None: | |
| # Remove old chat transcript fields from saved session events and any admin | |
| # audit trails that stored full message text. This intentionally does not | |
| # erase user-submitted feedback reports. | |
| with LOCK, STORE.lock: | |
| if STORE.state.get('beta_privacy_migrated_v1'):return | |
| STORE.state['session_events']=[] | |
| STORE.state['beta_privacy_migrated_v1']=True | |
| _store_save() | |