Refactor data directory management and session handling

Update the DATA_DIR configuration to allow for a default temporary directory while ensuring the creation of a sessions directory. Simplify logging by removing unnecessary debug statements and enhance clarity in session management. Adjust document navigation logging for better readability and maintainability.

app.py

@@ -13,20 +13,15 @@ import chardet
 import uuid
 
 # Define DATA_DIR first before using it
-# Force persistent data directory - never use /tmp on any platform
-DATA_DIR = os.path.join(os.getcwd(), 'data')
-print(f"DATA_DIR forced to persistent location: {DATA_DIR}")
-
-# Override any environment variable that might force /tmp usage
-if 'DATA_DIR' in os.environ:
-    print(f"WARNING: Ignoring environment DATA_DIR={os.environ['DATA_DIR']} - using persistent location instead")
+DATA_DIR = os.environ.get('DATA_DIR', '/tmp/human_notes_evaluator')
 
 # Then configure the app
 app = Flask(__name__)
 app.secret_key = os.environ.get('SECRET_KEY', 'your-secret-key-here')  # Gets from env or uses default
 
-# Configure session - use simple in-memory sessions for HF Spaces compatibility
-# Filesystem sessions cause permission issues on HF Spaces
+# Now configure session after DATA_DIR is defined
+app.config['SESSION_TYPE'] = 'filesystem'
+app.config['SESSION_FILE_DIR'] = os.path.join(DATA_DIR, 'sessions')
 app.config['SESSION_PERMANENT'] = False
 
 # Constants
@@ -128,12 +123,8 @@ def save_evaluation(data):
         
         # Log the data being saved
         log_error(f"Attempting to save evaluation for {data.get('document_title')} by {data.get('investigator_name')}")
-        log_error(f"DATA_DIR: {DATA_DIR}")
-        log_error(f"DATA_DIR exists: {os.path.exists(DATA_DIR)}")
-        log_error(f"DATA_DIR is writable: {os.access(DATA_DIR, os.W_OK) if os.path.exists(DATA_DIR) else 'Directory does not exist'}")
         
         eval_path = os.path.join(DATA_DIR, 'evaluations.csv')
-        log_error(f"Evaluation file path: {eval_path}")
         
         # Add timestamp to the data
         data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
@@ -427,7 +418,8 @@ def evaluate():
     # Make sure data directory exists
     ensure_data_directory()
     
-    # Using in-memory sessions, no need to create sessions directory
+    # Create sessions directory if it doesn't exist
+    os.makedirs(os.path.join(DATA_DIR, 'sessions'), exist_ok=True)
     
     # Initialize current document index if not set (start fresh each session, 1-based)
     if 'current_document_index' not in session:
@@ -440,14 +432,10 @@ def evaluate():
         documents = load_documents()
         if jump_to >= 1 and jump_to <= len(documents):
             session['current_document_index'] = jump_to  # Store as 1-based index
-            log_error(f"JUMP: Setting document index to {jump_to} (document #{jump_to})")
+            log_error(f"Jumping to document number: {jump_to} (index: {jump_to})")
         else:
             flash(f"Invalid document number. Please enter a number between 1 and {len(documents)}.")
             log_error(f"Invalid jump_to value: {jump_to}")
-    else:
-        # Log current position when not jumping
-        current_idx = session.get('current_document_index', 1)
-        log_error(f"CONTINUE: Current document index is {current_idx} (document #{current_idx})")
     
     # Generate session ID if not set
     if 'session_id' not in session:
@@ -514,11 +502,9 @@ def evaluate():
             
             # Handle "Skip" action
             if action == 'skip':
-                old_index = session.get('current_document_index', 1)
-                session['current_document_index'] = old_index + 1
-                log_error(f"SKIP: Moving from document {old_index} to document {session['current_document_index']}")
+                session['current_document_index'] = session.get('current_document_index', 1) + 1
                 flash("Document skipped.")
-                return redirect(url_for('evaluate'))  # No jump_to parameter, continue from current position
+                return redirect(url_for('evaluate'))
             
             # Handle regular evaluation submission
             # Get all documents
@@ -565,14 +551,13 @@ def evaluate():
                 
                 # Move to next document
                 session['current_document_index'] = current_index + 1
-                log_error(f"SUBMIT: Moving from document {current_index} to document {session['current_document_index']}")
                 
                 # Check if all documents have been processed
                 if session['current_document_index'] > len(all_documents):
                     flash("All documents have been evaluated. Thank you!")
                     return redirect(url_for('results'))
                 
-                # Redirect to next document (no jump_to parameter, continue from current position)
+                # Redirect to next document
                 return redirect(url_for('evaluate'))
             else:
                 flash("Error saving evaluation. Please try again.")
@@ -661,14 +646,6 @@ def jump_to_document():
 def results():
     """Results page showing evaluations from current session only."""
     session_id = session.get('session_id', '')
-    log_error(f"Results page accessed. Session ID: {session_id}")
-    log_error(f"DATA_DIR in results: {DATA_DIR}")
-    eval_path = os.path.join(DATA_DIR, 'evaluations.csv')
-    log_error(f"Checking for evaluations file at: {eval_path}")
-    log_error(f"File exists: {os.path.exists(eval_path)}")
-    if os.path.exists(eval_path):
-        log_error(f"File size: {os.path.getsize(eval_path)} bytes")
-    
     eval_df, filename_to_desc, filename_to_mrn = get_session_results(session_id)
     
     # Check if this is coming from "stop and save" action
@@ -921,22 +898,12 @@ def error_page():
                           error_details=error_details)
 
 def ensure_data_directory():
-    """Ensure data directory exists with proper permissions"""
+    """Ensure data directory exists"""
     try:
         os.makedirs(DATA_DIR, exist_ok=True)
-        # Set permissions to be world-writable for HF Spaces compatibility
-        try:
-            os.chmod(DATA_DIR, 0o777)
-            print(f"Created/verified data directory at {DATA_DIR} with write permissions")
-        except Exception as perm_error:
-            print(f"Created directory but couldn't set permissions: {perm_error}")
-        
-        log_error(f"Created/verified data directory at {DATA_DIR}")
-        log_error(f"Directory writable: {os.access(DATA_DIR, os.W_OK)}")
+        print(f"Created/verified data directory at {DATA_DIR}")
     except Exception as e:
-        error_msg = f"Error creating data directory: {str(e)}"
-        print(error_msg)
-        log_error(error_msg)
+        print(f"Error creating data directory: {str(e)}")
             
 def copy_template_if_needed():
     """Copy template file to documents.csv if it doesn't exist"""
@@ -978,11 +945,7 @@ def get_stored_evaluator_name():
         return None
 
 if __name__ == '__main__':
-    print("\n===== Application Startup at", datetime.now().strftime('%Y-%m-%d %H:%M:%S'), "=====")
-    print(f"IMPORTANT: Using DATA_DIR = {DATA_DIR}")
-    print(f"Current working directory: {os.getcwd()}")
-    print("Session storage: In-memory (HF Spaces compatible)")
-    print("=====\n")
+    print("\n===== Application Startup at", datetime.now().strftime('%Y-%m-%d %H:%M:%S'), "=====\n")
     
     # Create data directory
     ensure_data_directory()