Spaces:
Sleeping
Sleeping
Update persistence_manager.py
Browse files- persistence_manager.py +17 -143
persistence_manager.py
CHANGED
|
@@ -23,7 +23,7 @@ class APIConfigManager:
|
|
| 23 |
Dict of API credentials
|
| 24 |
"""
|
| 25 |
credentials = {}
|
| 26 |
-
|
| 27 |
# Try Hugging Face secrets first
|
| 28 |
try:
|
| 29 |
if hasattr(st, 'secrets'):
|
|
@@ -31,18 +31,18 @@ class APIConfigManager:
|
|
| 31 |
credentials['ANTHROPIC_API_KEY'] = st.secrets.get('ANTHROPIC_API_KEY')
|
| 32 |
except Exception:
|
| 33 |
pass
|
| 34 |
-
|
| 35 |
# Fallback to environment variables
|
| 36 |
credentials['OPENAI_API_KEY'] = credentials.get('OPENAI_API_KEY') or os.getenv('OPENAI_API_KEY')
|
| 37 |
credentials['ANTHROPIC_API_KEY'] = credentials.get('ANTHROPIC_API_KEY') or os.getenv('ANTHROPIC_API_KEY')
|
| 38 |
-
|
| 39 |
# Validate credentials
|
| 40 |
if not credentials['OPENAI_API_KEY']:
|
| 41 |
st.warning("OpenAI API key is missing. Some features may be limited.")
|
| 42 |
-
|
| 43 |
if not credentials['ANTHROPIC_API_KEY']:
|
| 44 |
st.warning("Anthropic API key is missing. Some features may be limited.")
|
| 45 |
-
|
| 46 |
return credentials
|
| 47 |
|
| 48 |
class HuggingFacePersistenceManager:
|
|
@@ -57,13 +57,8 @@ class HuggingFacePersistenceManager:
|
|
| 57 |
Args:
|
| 58 |
base_storage_path (str): Base path for persistent storage
|
| 59 |
"""
|
| 60 |
-
# Determine storage path with fallback
|
| 61 |
self.base_storage_path = self._determine_storage_path(base_storage_path)
|
| 62 |
-
|
| 63 |
-
# Create storage structure with error handling
|
| 64 |
self._create_storage_structure()
|
| 65 |
-
|
| 66 |
-
# Setup database connection
|
| 67 |
self.db_path = os.path.join(self.base_storage_path, 'book_projects.db')
|
| 68 |
self.conn = self._initialize_database()
|
| 69 |
|
|
@@ -77,30 +72,24 @@ class HuggingFacePersistenceManager:
|
|
| 77 |
Returns:
|
| 78 |
str: A writable storage path
|
| 79 |
"""
|
| 80 |
-
# List of potential fallback paths
|
| 81 |
fallback_paths = [
|
| 82 |
-
preferred_path,
|
| 83 |
-
'/tmp/book_writing_agent',
|
| 84 |
-
os.path.join(os.path.expanduser('~'), '.book_writing_agent'),
|
| 85 |
]
|
| 86 |
-
|
| 87 |
for path in fallback_paths:
|
| 88 |
try:
|
| 89 |
-
# Try to create directory if it doesn't exist
|
| 90 |
os.makedirs(path, exist_ok=True)
|
| 91 |
-
|
| 92 |
-
# Check if directory is writable
|
| 93 |
test_file = os.path.join(path, '.writetest')
|
| 94 |
with open(test_file, 'w') as f:
|
| 95 |
f.write('test')
|
| 96 |
os.remove(test_file)
|
| 97 |
-
|
| 98 |
st.info(f"Using storage path: {path}")
|
| 99 |
return path
|
| 100 |
except Exception as e:
|
| 101 |
st.warning(f"Could not use path {path}: {e}")
|
| 102 |
-
|
| 103 |
-
# If no path works, raise a critical error
|
| 104 |
raise RuntimeError("Could not find a writable storage location")
|
| 105 |
|
| 106 |
def _create_storage_structure(self):
|
|
@@ -108,33 +97,25 @@ class HuggingFacePersistenceManager:
|
|
| 108 |
Create necessary directory structure for persistent storage
|
| 109 |
with comprehensive error handling
|
| 110 |
"""
|
| 111 |
-
# Directories to create
|
| 112 |
directories = [
|
| 113 |
self.base_storage_path,
|
| 114 |
os.path.join(self.base_storage_path, 'projects'),
|
| 115 |
os.path.join(self.base_storage_path, 'chapters'),
|
| 116 |
os.path.join(self.base_storage_path, 'backups')
|
| 117 |
]
|
| 118 |
-
|
| 119 |
-
# Create directories with proper permissions
|
| 120 |
for dir_path in directories:
|
| 121 |
try:
|
| 122 |
-
# Use os.makedirs with exist_ok to prevent race conditions
|
| 123 |
os.makedirs(dir_path, exist_ok=True)
|
| 124 |
-
|
| 125 |
-
# Attempt to set permissions (if possible)
|
| 126 |
try:
|
| 127 |
os.chmod(dir_path, 0o755)
|
| 128 |
except Exception as perm_error:
|
| 129 |
-
# Log but don't fail if chmod is not possible
|
| 130 |
st.warning(f"Could not set permissions for {dir_path}: {perm_error}")
|
| 131 |
-
|
| 132 |
-
# Verify directory is writable
|
| 133 |
test_file = os.path.join(dir_path, '.writetest')
|
| 134 |
with open(test_file, 'w') as f:
|
| 135 |
f.write('test')
|
| 136 |
os.remove(test_file)
|
| 137 |
-
|
| 138 |
except PermissionError:
|
| 139 |
st.error(f"Permission denied when creating directory {dir_path}")
|
| 140 |
except Exception as e:
|
|
@@ -148,11 +129,8 @@ class HuggingFacePersistenceManager:
|
|
| 148 |
sqlite3.Connection: Database connection
|
| 149 |
"""
|
| 150 |
try:
|
| 151 |
-
# Connect to SQLite database
|
| 152 |
conn = sqlite3.connect(self.db_path)
|
| 153 |
cursor = conn.cursor()
|
| 154 |
-
|
| 155 |
-
# Create projects table if not exists
|
| 156 |
cursor.execute('''
|
| 157 |
CREATE TABLE IF NOT EXISTS projects (
|
| 158 |
project_id TEXT PRIMARY KEY,
|
|
@@ -163,8 +141,6 @@ class HuggingFacePersistenceManager:
|
|
| 163 |
last_modified DATETIME DEFAULT CURRENT_TIMESTAMP
|
| 164 |
)
|
| 165 |
''')
|
| 166 |
-
|
| 167 |
-
# Create chapters table
|
| 168 |
cursor.execute('''
|
| 169 |
CREATE TABLE IF NOT EXISTS chapters (
|
| 170 |
project_id TEXT,
|
|
@@ -176,10 +152,8 @@ class HuggingFacePersistenceManager:
|
|
| 176 |
PRIMARY KEY(project_id, chapter_number)
|
| 177 |
)
|
| 178 |
''')
|
| 179 |
-
|
| 180 |
conn.commit()
|
| 181 |
return conn
|
| 182 |
-
|
| 183 |
except Exception as e:
|
| 184 |
st.error(f"Database initialization error: {e}")
|
| 185 |
return None
|
|
@@ -465,118 +439,18 @@ class ResiliентAPIHandler:
|
|
| 465 |
"Error occurred"
|
| 466 |
]
|
| 467 |
|
| 468 |
-
return not any(indicator in response for response lower case in content
|
| 469 |
-
]
|
| 470 |
-
|
| 471 |
return not any(indicator.lower() in response.lower() for indicator in nonsense_indicators)
|
| 472 |
|
| 473 |
-
def check_huggingface_storage():
|
| 474 |
-
"""
|
| 475 |
-
Comprehensive check for Hugging Face storage capabilities
|
| 476 |
-
|
| 477 |
-
Returns:
|
| 478 |
-
Dict with storage verification results
|
| 479 |
-
"""
|
| 480 |
-
# Check storage paths
|
| 481 |
-
paths_to_check = ['/data', '/tmp', os.path.expanduser('~')]
|
| 482 |
-
|
| 483 |
-
storage_results = {}
|
| 484 |
-
|
| 485 |
-
for path in paths_to_check:
|
| 486 |
-
try:
|
| 487 |
-
# Check if path exists
|
| 488 |
-
path_exists = os.path.exists(path)
|
| 489 |
-
|
| 490 |
-
storage_results[path] = {
|
| 491 |
-
'exists': path_exists,
|
| 492 |
-
'writable': os.access(path, os.W_OK) if path_exists else False,
|
| 493 |
-
'readable': os.access(path, os.R_OK) if path_exists else False
|
| 494 |
-
}
|
| 495 |
-
|
| 496 |
-
# Add free space if path exists
|
| 497 |
-
if path_exists:
|
| 498 |
-
try:
|
| 499 |
-
statvfs = os.statvfs(path)
|
| 500 |
-
storage_results[path]['free_space_bytes'] = statvfs.f_frsize * statvfs.f_bavail
|
| 501 |
-
storage_results[path]['free_space_mb'] = (statvfs.f_frsize * statvfs.f_bavail) / (1024 * 1024)
|
| 502 |
-
except Exception:
|
| 503 |
-
storage_results[path]['free_space_bytes'] = 0
|
| 504 |
-
storage_results[path]['free_space_mb'] = 0
|
| 505 |
-
|
| 506 |
-
except Exception as e:
|
| 507 |
-
storage_results[path] = {
|
| 508 |
-
'exists': False,
|
| 509 |
-
'writable': False,
|
| 510 |
-
'readable': False,
|
| 511 |
-
'error': str(e)
|
| 512 |
-
}
|
| 513 |
-
|
| 514 |
-
return storage_results
|
| 515 |
-
|
| 516 |
-
def debug_storage_capabilities():
|
| 517 |
-
"""
|
| 518 |
-
Comprehensive debug function to check storage capabilities
|
| 519 |
-
Prints detailed information about storage access
|
| 520 |
-
"""
|
| 521 |
-
import platform
|
| 522 |
-
|
| 523 |
-
print("Storage Capabilities Debug Report")
|
| 524 |
-
print("=" * 40)
|
| 525 |
-
|
| 526 |
-
# System Information
|
| 527 |
-
print(f"Operating System: {platform.system()}")
|
| 528 |
-
print(f"OS Release: {platform.release()}")
|
| 529 |
-
print(f"Python Version: {platform.python_version()}")
|
| 530 |
-
|
| 531 |
-
print("\nStorage Path Checks:")
|
| 532 |
-
print("-" * 20)
|
| 533 |
-
|
| 534 |
-
# Check various potential storage paths
|
| 535 |
-
paths_to_check = [
|
| 536 |
-
'/data',
|
| 537 |
-
'/tmp',
|
| 538 |
-
os.path.expanduser('~'),
|
| 539 |
-
os.getcwd()
|
| 540 |
-
]
|
| 541 |
-
|
| 542 |
-
for path in paths_to_check:
|
| 543 |
-
print(f"\nPath: {path}")
|
| 544 |
-
try:
|
| 545 |
-
# Detailed path analysis
|
| 546 |
-
print(f" Exists: {os.path.exists(path)}")
|
| 547 |
-
print(f" Is Directory: {os.path.isdir(path)}")
|
| 548 |
-
print(f" Readable: {os.access(path, os.R_OK)}")
|
| 549 |
-
print(f" Writable: {os.access(path, os.W_OK)}")
|
| 550 |
-
print(f" Executable: {os.access(path, os.X_OK)}")
|
| 551 |
-
|
| 552 |
-
# Try creating a test file
|
| 553 |
-
try:
|
| 554 |
-
test_file = os.path.join(path, '.storage_test')
|
| 555 |
-
with open(test_file, 'w') as f:
|
| 556 |
-
f.write('storage test')
|
| 557 |
-
os.remove(test_file)
|
| 558 |
-
print(" File Creation: SUCCESS")
|
| 559 |
-
except Exception as file_error:
|
| 560 |
-
print(f" File Creation: FAILED - {file_error}")
|
| 561 |
-
|
| 562 |
-
except Exception as e:
|
| 563 |
-
print(f" Error checking path: {e}")
|
| 564 |
-
|
| 565 |
def main():
|
| 566 |
"""
|
| 567 |
-
Main function to demonstrate
|
| 568 |
"""
|
| 569 |
-
#
|
| 570 |
-
|
| 571 |
-
print("Storage Capabilities:")
|
| 572 |
-
print(json.dumps(storage_capabilities, indent=2))
|
| 573 |
-
|
| 574 |
-
# Run comprehensive debug
|
| 575 |
-
debug_storage_capabilities()
|
| 576 |
|
| 577 |
-
#
|
| 578 |
api_credentials = APIConfigManager.get_api_credentials()
|
| 579 |
-
print("
|
| 580 |
for key in api_credentials:
|
| 581 |
print(f"{key}: {'*' * 8 if api_credentials[key] else 'Not Set'}")
|
| 582 |
|
|
|
|
| 23 |
Dict of API credentials
|
| 24 |
"""
|
| 25 |
credentials = {}
|
| 26 |
+
|
| 27 |
# Try Hugging Face secrets first
|
| 28 |
try:
|
| 29 |
if hasattr(st, 'secrets'):
|
|
|
|
| 31 |
credentials['ANTHROPIC_API_KEY'] = st.secrets.get('ANTHROPIC_API_KEY')
|
| 32 |
except Exception:
|
| 33 |
pass
|
| 34 |
+
|
| 35 |
# Fallback to environment variables
|
| 36 |
credentials['OPENAI_API_KEY'] = credentials.get('OPENAI_API_KEY') or os.getenv('OPENAI_API_KEY')
|
| 37 |
credentials['ANTHROPIC_API_KEY'] = credentials.get('ANTHROPIC_API_KEY') or os.getenv('ANTHROPIC_API_KEY')
|
| 38 |
+
|
| 39 |
# Validate credentials
|
| 40 |
if not credentials['OPENAI_API_KEY']:
|
| 41 |
st.warning("OpenAI API key is missing. Some features may be limited.")
|
| 42 |
+
|
| 43 |
if not credentials['ANTHROPIC_API_KEY']:
|
| 44 |
st.warning("Anthropic API key is missing. Some features may be limited.")
|
| 45 |
+
|
| 46 |
return credentials
|
| 47 |
|
| 48 |
class HuggingFacePersistenceManager:
|
|
|
|
| 57 |
Args:
|
| 58 |
base_storage_path (str): Base path for persistent storage
|
| 59 |
"""
|
|
|
|
| 60 |
self.base_storage_path = self._determine_storage_path(base_storage_path)
|
|
|
|
|
|
|
| 61 |
self._create_storage_structure()
|
|
|
|
|
|
|
| 62 |
self.db_path = os.path.join(self.base_storage_path, 'book_projects.db')
|
| 63 |
self.conn = self._initialize_database()
|
| 64 |
|
|
|
|
| 72 |
Returns:
|
| 73 |
str: A writable storage path
|
| 74 |
"""
|
|
|
|
| 75 |
fallback_paths = [
|
| 76 |
+
preferred_path,
|
| 77 |
+
'/tmp/book_writing_agent',
|
| 78 |
+
os.path.join(os.path.expanduser('~'), '.book_writing_agent'),
|
| 79 |
]
|
| 80 |
+
|
| 81 |
for path in fallback_paths:
|
| 82 |
try:
|
|
|
|
| 83 |
os.makedirs(path, exist_ok=True)
|
|
|
|
|
|
|
| 84 |
test_file = os.path.join(path, '.writetest')
|
| 85 |
with open(test_file, 'w') as f:
|
| 86 |
f.write('test')
|
| 87 |
os.remove(test_file)
|
|
|
|
| 88 |
st.info(f"Using storage path: {path}")
|
| 89 |
return path
|
| 90 |
except Exception as e:
|
| 91 |
st.warning(f"Could not use path {path}: {e}")
|
| 92 |
+
|
|
|
|
| 93 |
raise RuntimeError("Could not find a writable storage location")
|
| 94 |
|
| 95 |
def _create_storage_structure(self):
|
|
|
|
| 97 |
Create necessary directory structure for persistent storage
|
| 98 |
with comprehensive error handling
|
| 99 |
"""
|
|
|
|
| 100 |
directories = [
|
| 101 |
self.base_storage_path,
|
| 102 |
os.path.join(self.base_storage_path, 'projects'),
|
| 103 |
os.path.join(self.base_storage_path, 'chapters'),
|
| 104 |
os.path.join(self.base_storage_path, 'backups')
|
| 105 |
]
|
| 106 |
+
|
|
|
|
| 107 |
for dir_path in directories:
|
| 108 |
try:
|
|
|
|
| 109 |
os.makedirs(dir_path, exist_ok=True)
|
|
|
|
|
|
|
| 110 |
try:
|
| 111 |
os.chmod(dir_path, 0o755)
|
| 112 |
except Exception as perm_error:
|
|
|
|
| 113 |
st.warning(f"Could not set permissions for {dir_path}: {perm_error}")
|
| 114 |
+
|
|
|
|
| 115 |
test_file = os.path.join(dir_path, '.writetest')
|
| 116 |
with open(test_file, 'w') as f:
|
| 117 |
f.write('test')
|
| 118 |
os.remove(test_file)
|
|
|
|
| 119 |
except PermissionError:
|
| 120 |
st.error(f"Permission denied when creating directory {dir_path}")
|
| 121 |
except Exception as e:
|
|
|
|
| 129 |
sqlite3.Connection: Database connection
|
| 130 |
"""
|
| 131 |
try:
|
|
|
|
| 132 |
conn = sqlite3.connect(self.db_path)
|
| 133 |
cursor = conn.cursor()
|
|
|
|
|
|
|
| 134 |
cursor.execute('''
|
| 135 |
CREATE TABLE IF NOT EXISTS projects (
|
| 136 |
project_id TEXT PRIMARY KEY,
|
|
|
|
| 141 |
last_modified DATETIME DEFAULT CURRENT_TIMESTAMP
|
| 142 |
)
|
| 143 |
''')
|
|
|
|
|
|
|
| 144 |
cursor.execute('''
|
| 145 |
CREATE TABLE IF NOT EXISTS chapters (
|
| 146 |
project_id TEXT,
|
|
|
|
| 152 |
PRIMARY KEY(project_id, chapter_number)
|
| 153 |
)
|
| 154 |
''')
|
|
|
|
| 155 |
conn.commit()
|
| 156 |
return conn
|
|
|
|
| 157 |
except Exception as e:
|
| 158 |
st.error(f"Database initialization error: {e}")
|
| 159 |
return None
|
|
|
|
| 439 |
"Error occurred"
|
| 440 |
]
|
| 441 |
|
|
|
|
|
|
|
|
|
|
| 442 |
return not any(indicator.lower() in response.lower() for indicator in nonsense_indicators)
|
| 443 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 444 |
def main():
|
| 445 |
"""
|
| 446 |
+
Main function to demonstrate persistence manager capabilities
|
| 447 |
"""
|
| 448 |
+
# Initialize persistence manager
|
| 449 |
+
persistence_manager = HuggingFacePersistenceManager()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
|
| 451 |
+
# Retrieve API credentials
|
| 452 |
api_credentials = APIConfigManager.get_api_credentials()
|
| 453 |
+
print("API Credentials:")
|
| 454 |
for key in api_credentials:
|
| 455 |
print(f"{key}: {'*' * 8 if api_credentials[key] else 'Not Set'}")
|
| 456 |
|