Refactor application structure and enhance functionality

Update the application to improve compatibility with HF Spaces by configuring the data directory based on the environment. Introduce a new function to ensure the data directory exists and create template files if they are missing. Streamline the evaluation process by consolidating form submissions and enhancing user feedback. Update the Dockerfile to use Python 3.9 and adjust directory permissions for better security. Improve logging throughout the application for better traceability and debugging.

Dockerfile

@@ -1,19 +1,25 @@
-FROM python:3.10-slim
+FROM python:3.9-slim
 
 WORKDIR /app
 
-# Create writable directories
-RUN mkdir -p /tmp/human_notes_evaluator && \
-    chmod -R 777 /tmp/human_notes_evaluator
-
+# Install dependencies
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY . .
+# Copy application files
+COPY app.py .
+COPY templates/ templates/
+COPY static/ static/
+
+# Create data directory with proper permissions
+RUN mkdir -p /app/data && chmod 777 /app/data
 
-# Make sure environment variables are set
-ENV DATA_DIR=/tmp/human_notes_evaluator
+# Set environment variables
+ENV DATA_DIR=/app/data
 ENV PYTHONUNBUFFERED=1
 
-# Start the application
-CMD ["python", "app.py"] 
+# Expose port
+EXPOSE 7860
+
+# Run the application
+CMD ["python", "app.py"]

app.py

@@ -11,20 +11,26 @@ import shutil
 import traceback
 import chardet
 
-# Define DATA_DIR first before using it
-DATA_DIR = os.environ.get('DATA_DIR', '/tmp/human_notes_evaluator')
-
-# Then configure the app
+# Configure data directory for HF Spaces compatibility
+if 'SPACE_ID' in os.environ:
+    # Running on HF Spaces - use persistent directory
+    DATA_DIR = os.path.join(os.getcwd(), 'data')
+else:
+    # Local development
+    DATA_DIR = os.environ.get('DATA_DIR', '/tmp/human_notes_evaluator')
+
+# Configure the Flask app
 app = Flask(__name__)
-app.secret_key = os.environ.get('SECRET_KEY', 'your-secret-key-here')  # Gets from env or uses default
+app.secret_key = os.environ.get('SECRET_KEY', 'your-secret-key-here')
 
 # Configure session 
 app.config['SESSION_PERMANENT'] = False
+app.config['SESSION_TYPE'] = 'filesystem'
 
 # Constants
 CRITERIA = [
     "Up-to-date",
-    "Accurate",
+    "Accurate", 
     "Thorough",
     "Relevant",
     "Well-organized",
@@ -61,106 +67,132 @@ def log_error(error_msg):
     # Keep only the most recent 10 errors
     while len(ERROR_LOG) > 10:
         ERROR_LOG.pop(0)
+    print(f"[LOG] {error_msg}")  # Also print to console
+
+def ensure_data_directory():
+    """Ensure data directory exists in a persistent location"""
+    global DATA_DIR
+    
+    # For HF Spaces, use the current working directory which persists
+    if 'SPACE_ID' in os.environ:
+        DATA_DIR = os.path.join(os.getcwd(), 'data')
+        log_error(f"Running on HF Spaces, using data directory: {DATA_DIR}")
+    
+    try:
+        os.makedirs(DATA_DIR, exist_ok=True)
+        os.makedirs(os.path.join(DATA_DIR, 'sessions'), exist_ok=True)
+        log_error(f"Created/verified data directory at {DATA_DIR}")
+        
+        # Create template files if they don't exist
+        create_template_files()
+                
+    except Exception as e:
+        log_error(f"Error creating data directory: {str(e)}")
+        raise
 
-def detect_encoding(file_path):
-    """Detect the encoding of a file to handle different character encodings."""
-    with open(file_path, 'rb') as f:
-        result = chardet.detect(f.read())
-    return result['encoding']
+def create_template_files():
+    """Create template CSV and instructions files if they don't exist"""
+    # Create sample documents template
+    template_path = os.path.join(DATA_DIR, 'sample_documents_template.csv')
+    if not os.path.exists(template_path):
+        template_data = [
+            ['filename', 'description', 'mrn', 'note'],
+            ['sample1.txt', 'Example Clinical Note', 'MRN12345', 'This is a sample clinical note for evaluation. Patient presents with...'],
+            ['sample2.txt', 'Example Progress Note', 'MRN67890', 'Patient returns for follow-up visit. Current medications include...']
+        ]
+        with open(template_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.writer(f)
+            writer.writerows(template_data)
+        log_error(f"Created template file at {template_path}")
+    
+    # Create instructions.md if it doesn't exist
+    instructions_path = os.path.join(DATA_DIR, 'instructions.md')
+    if not os.path.exists(instructions_path):
+        with open(instructions_path, 'w', encoding='utf-8') as f:
+            f.write("# Instructions for Human Notes Evaluator\n\n")
+            f.write("## How to Use This Application\n\n")
+            f.write("1. Upload a CSV file with your documents\n")
+            f.write("2. Enter your name as the evaluator\n")
+            f.write("3. Rate each document on the 9 criteria\n")
+            f.write("4. Export results when complete\n")
+        log_error(f"Created instructions at {instructions_path}")
+
+def detect_encoding(file_content):
+    """Detect the encoding of file content."""
+    if isinstance(file_content, str):
+        file_content = file_content.encode()
+    result = chardet.detect(file_content)
+    return result['encoding'] or 'utf-8'
 
 def load_documents():
-    """Load all documents from CSV file (fresh start each session)."""
+    """Load all documents from CSV file."""
     try:
         file_path = os.path.join(DATA_DIR, 'documents.csv')
         
-        # Check if file exists
         if not os.path.exists(file_path):
             log_error(f"Documents file not found at {file_path}")
             return []
         
-        try:
-            # Try to detect encoding
-            encoding = detect_encoding(file_path)
-            log_error(f"Detected encoding: {encoding}")
-            
-            # Try to read with pandas using the detected encoding
-            df = pd.read_csv(file_path, encoding=encoding)
-            log_error("Successfully parsed CSV with standard settings")
-        except Exception as e:
-            log_error(f"Error parsing CSV: {str(e)}")
-            return []
+        # Read file and detect encoding
+        with open(file_path, 'rb') as f:
+            content = f.read()
+        
+        encoding = detect_encoding(content)
+        log_error(f"Detected encoding: {encoding}")
+        
+        # Parse CSV
+        df = pd.read_csv(io.BytesIO(content), encoding=encoding)
+        log_error("Successfully parsed CSV")
         
         # Convert columns to string to ensure compatibility
         for col in df.columns:
-            df[col] = df[col].astype(str)
+            df[col] = df[col].astype(str).replace('nan', '')
         
-        # Log some stats
+        # Log stats
         log_error(f"DataFrame columns: {list(df.columns)}")
         log_error(f"DataFrame shape: {df.shape}")
         
-        if not df.empty:
-            log_error(f"First row: {df.iloc[0].to_dict()}")
-        
-        # Convert to list of dictionaries (return all documents)
+        # Convert to list of dictionaries
         documents = df.to_dict('records')
-        log_error(f"Returning {len(documents)} documents for evaluation")
+        log_error(f"Loaded {len(documents)} documents for evaluation")
         return documents
     
     except Exception as e:
         log_error(f"Error in load_documents: {str(e)}")
         return []
 
-
-
 def save_evaluation(data):
     """Save evaluation data to CSV file."""
     try:
-        # Ensure the data directory exists
         ensure_data_directory()
         
-        # Log the data being saved
-        log_error(f"Attempting to save evaluation for {data.get('document_title')} by {data.get('investigator_name')}")
+        log_error(f"Saving evaluation for {data.get('document_title')} by {data.get('investigator_name')}")
         
         eval_path = os.path.join(DATA_DIR, 'evaluations.csv')
         
-        # Add timestamp to the data
+        # Add timestamp
         data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         
-        # Check if file exists and if it's corrupted
+        # Check if file exists
         file_exists = os.path.exists(eval_path)
-        file_corrupted = False
-        
-        if file_exists:
-            # Try to read the existing file to check for corruption
-            try:
-                pd.read_csv(eval_path)
-                log_error("Existing CSV file is valid")
-            except pd.errors.ParserError as e:
-                log_error(f"Existing CSV file is corrupted: {str(e)}")
-                file_corrupted = True
-                # Create backup of corrupted file
-                backup_path = eval_path + f'.corrupted.{datetime.now().strftime("%Y%m%d_%H%M%S")}'
-                try:
-                    shutil.copy(eval_path, backup_path)
-                    log_error(f"Backed up corrupted file to {backup_path}")
-                except Exception as backup_error:
-                    log_error(f"Could not backup corrupted file: {str(backup_error)}")
-                
-                # Remove corrupted file
-                os.remove(eval_path)
-                file_exists = False
-                log_error("Removed corrupted CSV file")
         
-        # Write the data
+        # Define column order
+        columns = ['timestamp', 'document_title', 'description', 'mrn', 'investigator_name', 
+                  'session_id'] + CRITERIA + ['note_origin']
+        
+        # Ensure all columns exist in data
+        for col in columns:
+            if col not in data:
+                data[col] = ''
+        
+        # Write to CSV
         with open(eval_path, 'a', newline='', encoding='utf-8') as f:
-            writer = csv.DictWriter(f, fieldnames=data.keys())
+            writer = csv.DictWriter(f, fieldnames=columns, extrasaction='ignore')
             
-            # Write header if it's a new file or was corrupted
-            if not file_exists or file_corrupted:
+            if not file_exists:
                 writer.writeheader()
-                log_error("Wrote CSV header")
+                log_error("Created new evaluations.csv with header")
             
-            # Write the data row
             writer.writerow(data)
         
         log_error(f"Successfully saved evaluation to {eval_path}")
@@ -168,149 +200,95 @@ def save_evaluation(data):
         
     except Exception as e:
         log_error(f"Error saving evaluation: {str(e)}")
-        # Attempt to save to a backup location as fallback
-        try:
-            backup_path = os.path.join('/tmp', 'evaluation_backup.csv')
-            log_error(f"Trying backup location: {backup_path}")
-            
-            with open(backup_path, 'a', newline='', encoding='utf-8') as f:
-                writer = csv.DictWriter(f, fieldnames=data.keys())
-                if not os.path.exists(backup_path) or os.path.getsize(backup_path) == 0:
-                    writer.writeheader()
-                writer.writerow(data)
-            
-            log_error(f"Saved to backup location: {backup_path}")
-            return True
-        except Exception as backup_error:
-            log_error(f"Backup save also failed: {str(backup_error)}")
-            return False
+        return False
 
 def get_results():
-    """Get evaluation results for display."""
+    """Get all evaluation results."""
     try:
-        # Load evaluations with error handling
-        try:
-            eval_df = pd.read_csv(os.path.join(DATA_DIR, 'evaluations.csv'))
-        except pd.errors.ParserError as e:
-            log_error(f"CSV parsing error in get_results: {str(e)}")
-            return pd.DataFrame(), {}, {}
-        
-        # Load all documents to get descriptions and MRN
-        try:
-            docs_df = pd.read_csv(os.path.join(DATA_DIR, 'documents.csv'))
-            # Create a mapping of filename to description and MRN
-            filename_to_desc = dict(zip(docs_df['filename'], docs_df['description']))
-            filename_to_mrn = dict(zip(docs_df['filename'], docs_df['mrn']))
-        except FileNotFoundError:
-            filename_to_desc = {}
-            filename_to_mrn = {}
+        eval_path = os.path.join(DATA_DIR, 'evaluations.csv')
         
-        return eval_df, filename_to_desc, filename_to_mrn
-    except FileNotFoundError:
-        return pd.DataFrame(), {}, {}
-
-def get_session_results(session_id):
-    """Get evaluation results for current session only."""
-    try:
-        # Load evaluations with error handling for column mismatches
-        try:
-            eval_df = pd.read_csv(os.path.join(DATA_DIR, 'evaluations.csv'))
-        except pd.errors.ParserError as e:
-            log_error(f"CSV parsing error: {str(e)}")
-            # Try to read with different options or recreate the file
-            eval_path = os.path.join(DATA_DIR, 'evaluations.csv')
-            log_error(f"Attempting to backup and recreate corrupted evaluations file")
-            # Backup the corrupted file
-            if os.path.exists(eval_path):
-                backup_path = eval_path + '.backup'
-                shutil.copy(eval_path, backup_path)
-                log_error(f"Backed up corrupted file to {backup_path}")
-            # Return empty DataFrame
+        if not os.path.exists(eval_path):
             return pd.DataFrame(), {}, {}
         
-        # Filter by session_id if column exists
-        if 'session_id' in eval_df.columns and session_id:
-            eval_df = eval_df[eval_df['session_id'] == session_id]
-        else:
-            # If no session_id column exists or no session_id provided, return empty for fresh sessions
-            log_error(f"No session_id column found or no session_id provided. Returning empty results for fresh session.")
-            eval_df = pd.DataFrame()
+        # Read evaluations
+        eval_df = pd.read_csv(eval_path)
         
-        # Load all documents to get descriptions and MRN
+        # Load documents for descriptions and MRNs
         try:
             docs_df = pd.read_csv(os.path.join(DATA_DIR, 'documents.csv'))
-            # Create a mapping of filename to description and MRN
             filename_to_desc = dict(zip(docs_df['filename'], docs_df['description']))
             filename_to_mrn = dict(zip(docs_df['filename'], docs_df['mrn']))
-        except FileNotFoundError:
+        except:
             filename_to_desc = {}
             filename_to_mrn = {}
         
         return eval_df, filename_to_desc, filename_to_mrn
-    except FileNotFoundError:
+        
+    except Exception as e:
+        log_error(f"Error in get_results: {str(e)}")
         return pd.DataFrame(), {}, {}
 
-def get_total_document_count():
-    """Get the total number of documents."""
+# Progress tracking functions
+def get_progress_file(evaluator_name):
+    """Get path to progress file for an evaluator."""
+    safe_name = "".join(c for c in evaluator_name if c.isalnum() or c in (' ', '-', '_')).rstrip()
+    return os.path.join(DATA_DIR, 'sessions', f'{safe_name}_progress.txt')
+
+def save_current_index(evaluator_name, index):
+    """Save current document index to file."""
     try:
-        df = pd.read_csv(os.path.join(DATA_DIR, 'documents.csv'))
-        return len(df)
-    except Exception:
-        return 0
+        os.makedirs(os.path.join(DATA_DIR, 'sessions'), exist_ok=True)
+        with open(get_progress_file(evaluator_name), 'w') as f:
+            f.write(str(index))
+        return True
+    except Exception as e:
+        log_error(f"Error saving progress: {str(e)}")
+        return False
 
-def get_evaluated_document_count():
-    """Get the count of evaluated documents."""
+def load_current_index(evaluator_name):
+    """Load current document index from file."""
     try:
-        eval_df = pd.read_csv(os.path.join(DATA_DIR, 'evaluations.csv'))
-        return len(eval_df['document_title'].unique())
-    except FileNotFoundError:
-        return 0
-    except Exception:
-        return 0
+        progress_file = get_progress_file(evaluator_name)
+        if os.path.exists(progress_file):
+            with open(progress_file, 'r') as f:
+                return int(f.read().strip())
+        return 1
+    except Exception as e:
+        log_error(f"Error loading progress: {str(e)}")
+        return 1
 
-def load_and_validate_csv(file_path):
-    """Load and validate a CSV file to ensure it has the required format"""
+def store_evaluator_name(name):
+    """Store evaluator name in a file for persistence."""
     try:
-        # Try to detect encoding
-        encoding = detect_encoding(file_path)
-        log_error(f"Detected encoding: {encoding}")
-        
-        # Try to read with pandas using the detected encoding
-        df = pd.read_csv(file_path, encoding=encoding)
-        log_error("Successfully parsed CSV with standard settings")
-        
-        # Check for required columns
-        required_columns = ['filename', 'description', 'mrn', 'note']
-        missing_columns = [col for col in required_columns if col not in df.columns]
-        
-        if missing_columns:
-            log_error(f"Missing required columns: {missing_columns}")
-            raise ValueError(f"Missing required columns: {missing_columns}")
-        
-        # Log success information
-        log_error(f"DataFrame columns: {list(df.columns)}")
-        log_error(f"DataFrame shape: {df.shape}")
-        
-        if not df.empty:
-            log_error(f"First row: {df.iloc[0].to_dict()}")
-        
-        return df
-    
+        ensure_data_directory()
+        with open(os.path.join(DATA_DIR, 'current_evaluator.txt'), 'w') as f:
+            f.write(name)
+        log_error(f"Stored evaluator name: {name}")
+        return True
     except Exception as e:
-        log_error(f"Error validating CSV file: {str(e)}")
-        raise
+        log_error(f"Error storing evaluator name: {str(e)}")
+        return False
+
+def get_stored_evaluator_name():
+    """Get stored evaluator name from file."""
+    try:
+        file_path = os.path.join(DATA_DIR, 'current_evaluator.txt')
+        if os.path.exists(file_path):
+            with open(file_path, 'r') as f:
+                return f.read().strip()
+        return None
+    except Exception as e:
+        log_error(f"Error retrieving evaluator name: {str(e)}")
+        return None
 
 @app.route('/', methods=['GET', 'POST'])
 def index():
-    # Clear any existing session data for fresh start
-    session.clear()
-    
+    """Home page with file upload and evaluator name."""
     if request.method == 'POST':
-        # Ensure data directory exists
         ensure_data_directory()
         
         # Get evaluator name
-        evaluator_name = request.form.get('evaluator_name', '')
+        evaluator_name = request.form.get('evaluator_name', '').strip()
         if not evaluator_name:
             flash("Please enter your name as the evaluator.")
             return render_template('index.html')
@@ -323,172 +301,179 @@ def index():
                 flash("No file selected.")
                 return render_template('index.html')
             
-            if file and '.' in file.filename and file.filename.rsplit('.', 1)[1].lower() == 'csv':
+            if file and file.filename.endswith('.csv'):
                 try:
-                    # Read file directly from memory instead of saving to disk first
+                    # Read file content
                     file_content = file.read()
                     
-                    # Parse CSV from memory
-                    try:
-                        # Try to detect encoding
-                        encoding = chardet.detect(file_content)['encoding']
-                        log_error(f"Detected encoding: {encoding}")
-                        
-                        # Load CSV from in-memory content
-                        csv_buffer = io.StringIO(file_content.decode(encoding))
-                        df = pd.read_csv(csv_buffer)
-                        
-                        # Validate the dataframe
-                        required_columns = ['filename', 'description', 'mrn', 'note']
-                        missing_columns = [col for col in required_columns if col not in df.columns]
-                        
-                        if missing_columns:
-                            raise ValueError(f"Missing required columns: {missing_columns}")
-                        
-                        # Write to documents.csv only after validation succeeded
-                        documents_path = os.path.join(DATA_DIR, 'documents.csv')
-                        df.to_csv(documents_path, index=False)
-                        
-                        # Set session cookie
-                        session['evaluator_name'] = evaluator_name
-                        
-                        flash("File uploaded successfully!")
-                        
-                        # After uploading documents.csv successfully, copy template_documents if it doesn't exist
-                        documents_path = os.path.join(DATA_DIR, 'documents.csv')
-                        
-                        # If user uploaded file, ensure we save the evaluator name persistently
-                        if evaluator_name:
-                            store_evaluator_name(evaluator_name)
-                            
-                            # Also create a direct access link they can bookmark
-                            direct_link = url_for('evaluate', evaluator=evaluator_name, _external=True)
-                            flash(f"Bookmark this link for direct access to your evaluation: {direct_link}")
-                        
-                        return redirect(url_for('evaluate'))
-                        
-                    except Exception as inner_e:
-                        log_error(f"Error parsing CSV data: {str(inner_e)}")
-                        raise ValueError(f"Error parsing CSV data: {str(inner_e)}")
-                        
+                    # Detect encoding and parse CSV
+                    encoding = detect_encoding(file_content)
+                    csv_text = file_content.decode(encoding)
+                    df = pd.read_csv(io.StringIO(csv_text))
+                    
+                    # Validate columns
+                    required_columns = ['filename', 'description', 'mrn', 'note']
+                    missing_columns = [col for col in required_columns if col not in df.columns]
+                    
+                    if missing_columns:
+                        flash(f"Missing required columns: {', '.join(missing_columns)}")
+                        return render_template('index.html')
+                    
+                    # Save documents
+                    documents_path = os.path.join(DATA_DIR, 'documents.csv')
+                    df.to_csv(documents_path, index=False)
+                    
+                    # Set session
+                    session['evaluator_name'] = evaluator_name
+                    session['session_id'] = f"{evaluator_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+                    
+                    # Store evaluator name
+                    store_evaluator_name(evaluator_name)
+                    
+                    flash(f"File uploaded successfully! Found {len(df)} documents.")
+                    return redirect(url_for('evaluate'))
+                    
                 except Exception as e:
-                    log_error(f"Error during file upload: {str(e)}")
-                    flash(f"Error during file upload: {str(e)}. Please try again.")
+                    flash(f"Error processing file: {str(e)}")
+                    log_error(f"File upload error: {str(e)}")
             else:
                 flash("Please upload a CSV file.")
         
         return render_template('index.html')
     
-    # Handle GET request
-    return render_template('index.html')
+    # GET request
+    evaluator_name = session.get('evaluator_name', '') or get_stored_evaluator_name() or ''
+    return render_template('index.html', evaluator_name=evaluator_name)
 
 @app.route('/evaluate', methods=['GET', 'POST'])
 def evaluate():
-    """Display a document for evaluation or process evaluation form."""
-    log_error(f"Starting /evaluate route, session: {session}")
-    
-    # Try multiple methods to get evaluator name
-    evaluator_name = session.get('evaluator_name', '')
-    
-    # If not in session, check query parameter
-    if not evaluator_name and request.args.get('evaluator'):
-        evaluator_name = request.args.get('evaluator')
-        session['evaluator_name'] = evaluator_name
-        store_evaluator_name(evaluator_name)
-        log_error(f"Got evaluator name from query param: {evaluator_name}")
-    
-    # If still not found, try file-based storage
-    if not evaluator_name:
-        evaluator_name = get_stored_evaluator_name()
-        if evaluator_name:
-            session['evaluator_name'] = evaluator_name
-            log_error(f"Got evaluator name from file: {evaluator_name}")
-    
-    log_error(f"Final evaluator name: {evaluator_name}")
+    """Document evaluation page."""
+    # Get evaluator name from multiple sources
+    evaluator_name = (
+        session.get('evaluator_name') or 
+        request.args.get('evaluator') or
+        get_stored_evaluator_name()
+    )
     
-    # Still no evaluator name, redirect to index with a message
     if not evaluator_name:
         flash("Please enter your name before evaluating documents.")
-        log_error("No evaluator name found, redirecting to index")
         return redirect(url_for('index'))
     
-    # Make sure data directory exists
+    # Update session
+    session['evaluator_name'] = evaluator_name
+    if 'session_id' not in session:
+        session['session_id'] = f"{evaluator_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+    
+    # Ensure directories exist
     ensure_data_directory()
     
-    # Create sessions directory if it doesn't exist
-    os.makedirs(os.path.join(DATA_DIR, 'sessions'), exist_ok=True)
+    # Load documents
+    documents = load_documents()
+    if not documents:
+        return render_template('no_documents.html')
     
-    # Initialize current document index from file
+    # Get current index
     current_index = load_current_index(evaluator_name)
     
     # Handle jump requests
     jump_to = request.args.get('jump_to', type=int)
-    if jump_to is not None:
-        documents = load_documents()
-        if 1 <= jump_to <= len(documents):
-            current_index = jump_to
-            save_current_index(evaluator_name, current_index)
+    if jump_to and 1 <= jump_to <= len(documents):
+        current_index = jump_to
+        save_current_index(evaluator_name, current_index)
     
-    # Handle form submissions
+    # Handle POST requests
     if request.method == 'POST':
         action = request.form.get('action', 'submit')
         
         if action == 'skip':
-            current_index += 1
+            current_index = min(current_index + 1, len(documents) + 1)
             save_current_index(evaluator_name, current_index)
             flash("Document skipped.")
             return redirect(url_for('evaluate'))
         
         elif action == 'stop_save':
-            # Handle save logic
-            return redirect(url_for('results'))
+            flash("Progress saved. You can resume later.")
+            return redirect(url_for('results', session_saved=True))
         
         elif action == 'submit':
-            # Save evaluation logic
-            current_index += 1
-            save_current_index(evaluator_name, current_index)
-            flash("Evaluation saved successfully!")
+            if current_index <= len(documents):
+                current_doc = documents[current_index - 1]
+                
+                # Prepare evaluation data
+                eval_data = {
+                    'document_title': current_doc.get('filename', ''),
+                    'description': current_doc.get('description', ''),
+                    'mrn': current_doc.get('mrn', ''),
+                    'investigator_name': evaluator_name,
+                    'session_id': session.get('session_id', ''),
+                    'note_origin': request.form.get('note_origin', '')
+                }
+                
+                # Add criteria scores
+                all_scores_present = True
+                for i, criterion in enumerate(CRITERIA):
+                    score = request.form.get(f'criteria_{i}')
+                    if score:
+                        eval_data[criterion] = score
+                    else:
+                        all_scores_present = False
+                        flash(f"Please rate: {criterion}")
+                
+                # Check note origin
+                if not eval_data['note_origin']:
+                    all_scores_present = False
+                    flash("Please select a note origin assessment.")
+                
+                # Save if all data present
+                if all_scores_present:
+                    if save_evaluation(eval_data):
+                        current_index = min(current_index + 1, len(documents) + 1)
+                        save_current_index(evaluator_name, current_index)
+                        flash("Evaluation saved successfully!")
+                    else:
+                        flash("Error saving evaluation. Please try again.")
+                
+                return redirect(url_for('evaluate'))
     
-    # Load current document
-    documents = load_documents()
+    # Check if all documents evaluated
     if current_index > len(documents):
-        flash("All documents have been evaluated.")
+        flash("All documents have been evaluated. Thank you!")
         return redirect(url_for('results'))
     
+    # Get current document
     document = documents[current_index - 1]
     
-    # Render template with current_index
-    return render_template('evaluate.html', 
-                         current_note_number=current_index,
-                         evaluator_name=evaluator_name,
-                         note=document.get('note', ''),
-                         description=document.get('description', ''),
-                         mrn=document.get('mrn', ''),
-                         criteria=CRITERIA,
-                         descriptions=CRITERIA_DESCRIPTIONS,
-                         score_range=range(1, 6),
-                         note_origins=NOTE_ORIGINS,
-                         total_docs=len(documents),
-                         evaluated_docs=current_index - 1,
-                         progress=int((current_index - 1) / len(documents) * 100) if len(documents) > 0 else 0)
+    # Calculate progress
+    evaluated_docs = current_index - 1
+    progress = int((evaluated_docs / len(documents)) * 100) if documents else 0
+    
+    return render_template('evaluate.html',
+        current_note_number=current_index,
+        evaluator_name=evaluator_name,
+        note=document.get('note', ''),
+        description=document.get('description', ''),
+        mrn=document.get('mrn', ''),
+        criteria=CRITERIA,
+        descriptions=CRITERIA_DESCRIPTIONS,
+        score_range=range(1, 6),
+        note_origins=NOTE_ORIGINS,
+        total_docs=len(documents),
+        evaluated_docs=evaluated_docs,
+        progress=progress
+    )
 
 @app.route('/jump', methods=['POST'])
 def jump_to_document():
     """Jump to a specific document number."""
     try:
         document_number = int(request.form.get('document_number', 1))
-        
-        # Validate document number range
         documents = load_documents()
-        total_docs = len(documents)
         
         if document_number < 1:
             flash("Document number must be 1 or greater.")
-        elif document_number > total_docs:
-            flash(f"Document number cannot be greater than {total_docs}.")
+        elif document_number > len(documents):
+            flash(f"Document number cannot be greater than {len(documents)}.")
         else:
-            # Redirect to evaluate with jump_to parameter
-            log_error(f"Jump requested to document {document_number}")
             return redirect(url_for('evaluate', jump_to=document_number))
     except ValueError:
         flash("Please enter a valid document number.")
@@ -498,29 +483,41 @@ def jump_to_document():
 @app.route('/results')
 def results():
     """Results page showing all evaluations."""
-    eval_df, filename_to_desc, filename_to_mrn = get_results()
-    
-    # Always show results page if coming from stop_save
-    if request.referrer and url_for('evaluate') in request.referrer:
+    try:
+        eval_df, filename_to_desc, filename_to_mrn = get_results()
+        
+        # Convert to list of dicts and enhance with descriptions/MRNs
+        evaluations = []
+        if not eval_df.empty:
+            for _, row in eval_df.iterrows():
+                eval_dict = row.to_dict()
+                doc_title = eval_dict.get('document_title', '')
+                
+                # Add description and MRN if not already present
+                if 'description' not in eval_dict or pd.isna(eval_dict['description']):
+                    eval_dict['description'] = filename_to_desc.get(doc_title, '')
+                if 'mrn' not in eval_dict or pd.isna(eval_dict['mrn']):
+                    eval_dict['mrn'] = filename_to_mrn.get(doc_title, '')
+                    
+                evaluations.append(eval_dict)
+        
+        session_saved = request.args.get('session_saved', False)
+        
         return render_template('results.html',
-                            evaluations=eval_df.to_dict('records'),
-                            criteria=CRITERIA,
-                            descriptions=CRITERIA_DESCRIPTIONS,
-                            show_empty_message=True)
-    
-    # Only redirect if completely empty
-    if eval_df.empty:
-        flash('No evaluations available.')
-        return redirect(url_for('index'))
+            evaluations=evaluations,
+            criteria=CRITERIA,
+            descriptions=CRITERIA_DESCRIPTIONS,
+            session_saved=session_saved
+        )
         
-    return render_template('results.html', 
-                         evaluations=eval_df.to_dict('records'),
-                         criteria=CRITERIA,
-                         descriptions=CRITERIA_DESCRIPTIONS)
+    except Exception as e:
+        log_error(f"Error in results route: {str(e)}")
+        flash(f"Error loading results: {str(e)}")
+        return redirect(url_for('index'))
 
 @app.route('/export-csv')
 def export_csv():
-    """Export all evaluations to CSV file."""
+    """Export evaluations to CSV."""
     try:
         eval_df, _, _ = get_results()
         
@@ -528,12 +525,12 @@ def export_csv():
             flash('No evaluations available to export.')
             return redirect(url_for('results'))
         
-        # Create in-memory CSV
+        # Create CSV in memory
         output = io.StringIO()
         eval_df.to_csv(output, index=False, quoting=csv.QUOTE_ALL)
         output.seek(0)
         
-        # Convert to BytesIO for send_file
+        # Convert to bytes
         mem = io.BytesIO()
         mem.write(output.getvalue().encode('utf-8'))
         mem.seek(0)
@@ -545,14 +542,13 @@ def export_csv():
             download_name=f'evaluations_{datetime.now().strftime("%Y%m%d_%H%M%S")}.csv'
         )
     except Exception as e:
-        error_msg = f'Error exporting CSV: {str(e)}'
-        log_error(error_msg)
-        flash(error_msg)
+        flash(f'Error exporting CSV: {str(e)}')
+        log_error(f"Export error: {str(e)}")
         return redirect(url_for('results'))
 
 @app.route('/upload-documents', methods=['GET', 'POST'])
 def upload_documents():
-    """Alternative method to upload documents.csv via web interface."""
+    """Alternative document upload page."""
     if request.method == 'POST':
         try:
             if 'file' not in request.files:
@@ -565,98 +561,54 @@ def upload_documents():
                 return redirect(request.url)
             
             if file and file.filename.endswith('.csv'):
-                # Save the file
-                file_path = os.path.join(DATA_DIR, 'documents.csv')
-                
-                # Ensure directory exists with proper permissions
                 ensure_data_directory()
                 
-                # Try to save the file
+                # Save file
+                file_path = os.path.join(DATA_DIR, 'documents.csv')
                 file.save(file_path)
                 
-                # Verify the file was saved and is readable
-                if os.path.exists(file_path):
-                    try:
-                        # Test if we can read it
-                        df = pd.read_csv(file_path)
-                        flash(f'Documents file uploaded successfully! Found {len(df)} documents.')
-                        log_error(f"Documents file uploaded via web interface: {len(df)} documents")
-                        return redirect(url_for('index'))
-                    except Exception as read_error:
-                        flash(f'File uploaded but could not be parsed: {str(read_error)}')
-                        log_error(f"Error parsing uploaded file: {str(read_error)}")
-                else:
-                    flash('File upload failed - file not found after save')
+                # Verify file
+                try:
+                    df = pd.read_csv(file_path)
+                    flash(f'Documents uploaded successfully! Found {len(df)} documents.')
+                    return redirect(url_for('index'))
+                except Exception as e:
+                    flash(f'File uploaded but could not be parsed: {str(e)}')
             else:
                 flash('Please upload a CSV file')
                 
         except Exception as e:
-            error_msg = f'Error uploading file: {str(e)}'
-            flash(error_msg)
-            log_error(error_msg)
+            flash(f'Error uploading file: {str(e)}')
+            log_error(f"Upload error: {str(e)}")
     
     return render_template('upload_documents.html')
 
-@app.route('/hf-debug')
-def hf_debug():
-    """Special debug route for HF Spaces troubleshooting."""
-    import platform
-    debug_info = {
-        'platform': platform.system(),
-        'python_version': platform.python_version(),
-        'current_working_directory': os.getcwd(),
-        'data_dir_setting': DATA_DIR,
-        'data_dir_exists': os.path.exists(DATA_DIR),
-        'data_dir_writable': os.access(DATA_DIR, os.W_OK) if os.path.exists(DATA_DIR) else 'Directory does not exist',
-        'data_dir_permissions': oct(os.stat(DATA_DIR).st_mode)[-3:] if os.path.exists(DATA_DIR) else 'N/A',
-        'relevant_env_vars': {k: v for k, v in os.environ.items() if k in ['DATA_DIR', 'SPACE_ID', 'SPACE_AUTHOR_NAME', 'SPACE_REPO_NAME']},
-        'directory_contents': os.listdir('.') if os.path.exists('.') else [],
-        'data_directory_contents': os.listdir(DATA_DIR) if os.path.exists(DATA_DIR) else 'Directory does not exist'
-    }
-    
-    return f"<html><body><h1>HF Debug Info</h1><pre>{str(debug_info)}</pre></body></html>"
-
 @app.route('/debug')
 def debug():
     """Debug page showing application state."""
-    # Check for data directory
-    if not os.path.exists(DATA_DIR):
-        try:
-            ensure_data_directory()
-        except Exception as e:
-            log_error(f"Failed to create data directory in debug route: {str(e)}")
+    ensure_data_directory()
     
-    # Get documents
     documents = load_documents()
+    eval_df, _, _ = get_results()
+    evaluations = [] if eval_df.empty else eval_df.to_dict('records')
     
-    # Get evaluations - properly handle DataFrame
-    eval_df, _, _ = get_results()  # Show all evaluations in debug, not just session
-    # Convert DataFrame to list of dictionaries if not empty
-    evaluations = [] if eval_df is None or eval_df.empty else eval_df.to_dict('records')
-    
-    # Get file paths
-    docs_path = os.path.join(DATA_DIR, 'documents.csv') 
-    evals_path = os.path.join(DATA_DIR, 'evaluations.csv')
-    
-    # Additional debug info
     debug_info = {
         'data_dir': DATA_DIR,
         'data_dir_exists': os.path.exists(DATA_DIR),
         'data_dir_writable': os.access(DATA_DIR, os.W_OK) if os.path.exists(DATA_DIR) else False,
         'current_working_dir': os.getcwd(),
         'session_id': session.get('session_id', 'None'),
-        'current_document_index': session.get('current_document_index', 'None'),
         'evaluator_name': session.get('evaluator_name', 'None'),
         'documents_count': len(documents),
         'evaluations_count': len(evaluations),
-        'eval_file_size': os.path.getsize(evals_path) if os.path.exists(evals_path) else 0
+        'environment': 'HF Spaces' if 'SPACE_ID' in os.environ else 'Local'
     }
     
     return render_template('debug.html',
         documents=documents,
         evaluations=evaluations,
-        documents_exists=os.path.exists(docs_path),
-        evaluations_exists=os.path.exists(evals_path),
+        documents_exists=os.path.exists(os.path.join(DATA_DIR, 'documents.csv')),
+        evaluations_exists=os.path.exists(os.path.join(DATA_DIR, 'evaluations.csv')),
         errors=ERROR_LOG,
         debug_info=debug_info
     )
@@ -665,44 +617,47 @@ def debug():
 def view_instructions():
     """Display instructions page."""
     return render_template('instructions.html', 
-                          criteria=CRITERIA,
-                          descriptions=CRITERIA_DESCRIPTIONS)
+        criteria=CRITERIA,
+        descriptions=CRITERIA_DESCRIPTIONS
+    )
 
 @app.route('/download/instructions')
 def download_instructions():
-    """Download instructions as markdown file."""
+    """Download instructions as markdown."""
     try:
         instructions_path = os.path.join(DATA_DIR, 'instructions.md')
         return send_file(instructions_path, 
-                         mimetype='text/markdown',
-                         download_name='instructions.md',
-                         as_attachment=True)
+            mimetype='text/markdown',
+            download_name='instructions.md',
+            as_attachment=True
+        )
     except FileNotFoundError:
         flash('Instructions file not found.')
         return redirect(url_for('index'))
 
 @app.route('/download/template')
 def download_template():
-    """Download sample template CSV file."""
+    """Download sample template CSV."""
     try:
         template_path = os.path.join(DATA_DIR, 'sample_documents_template.csv')
         return send_file(template_path, 
-                         mimetype='text/csv',
-                         download_name='sample_documents_template.csv',
-                         as_attachment=True)
+            mimetype='text/csv',
+            download_name='sample_documents_template.csv',
+            as_attachment=True
+        )
     except FileNotFoundError:
         flash('Template file not found.')
         return redirect(url_for('index'))
 
 @app.route('/reset', methods=['POST'])
 def reset():
-    """Reset the session and return to the landing page."""
+    """Reset session and clear evaluations."""
     session.clear()
-    # Remove evaluations.csv if it exists
+    
+    # Backup and remove evaluations
     evaluations_path = os.path.join(DATA_DIR, 'evaluations.csv')
     if os.path.exists(evaluations_path):
-        # Create backup before removing
-        backup_path = evaluations_path + f'.backup.{datetime.now().strftime("%Y%m%d_%H%M%S")}'
+        backup_path = f"{evaluations_path}.backup.{datetime.now().strftime('%Y%m%d_%H%M%S')}"
         try:
             shutil.copy(evaluations_path, backup_path)
             log_error(f"Created backup at {backup_path}")
@@ -710,26 +665,25 @@ def reset():
             log_error(f"Could not create backup: {str(e)}")
         
         os.remove(evaluations_path)
-        log_error("Removed evaluations.csv file")
+        log_error("Removed evaluations.csv")
+    
     flash('Session reset. All evaluation data cleared.')
     return redirect(url_for('index'))
 
 @app.route('/clear-corrupted-data', methods=['POST'])
 def clear_corrupted_data():
-    """Clear corrupted evaluations file to start fresh."""
+    """Clear corrupted evaluations file."""
     evaluations_path = os.path.join(DATA_DIR, 'evaluations.csv')
     if os.path.exists(evaluations_path):
-        # Create backup before removing
-        backup_path = evaluations_path + f'.corrupted_backup.{datetime.now().strftime("%Y%m%d_%H%M%S")}'
+        backup_path = f"{evaluations_path}.corrupted.{datetime.now().strftime('%Y%m%d_%H%M%S')}"
         try:
             shutil.copy(evaluations_path, backup_path)
-            log_error(f"Created corrupted file backup at {backup_path}")
+            log_error(f"Backed up corrupted file to {backup_path}")
         except Exception as e:
-            log_error(f"Could not create backup: {str(e)}")
+            log_error(f"Could not backup: {str(e)}")
         
         os.remove(evaluations_path)
-        log_error("Removed corrupted evaluations.csv file")
-        flash('Corrupted evaluation data cleared. You can now save evaluations properly.')
+        flash('Corrupted evaluation data cleared.')
     else:
         flash('No evaluation data file found.')
     
@@ -741,91 +695,29 @@ def error_page():
     error_message = request.args.get('message', 'An unknown error occurred')
     error_details = request.args.get('details', '')
     return render_template('error.html', 
-                          error_message=error_message,
-                          error_details=error_details)
-
-def ensure_data_directory():
-    """Ensure data directory exists"""
-    try:
-        os.makedirs(DATA_DIR, exist_ok=True)
-        print(f"Created/verified data directory at {DATA_DIR}")
-    except Exception as e:
-        print(f"Error creating data directory: {str(e)}")
-            
-def copy_template_if_needed():
-    """Copy template file to documents.csv if it doesn't exist"""
-    documents_path = os.path.join(DATA_DIR, 'documents.csv')
-    if not os.path.exists(documents_path):
-        try:
-            # Copy from template
-            template_path = 'sample_documents_template.csv'
-            if os.path.exists(template_path):
-                shutil.copy(template_path, documents_path)
-                print(f"Copied template to {documents_path}")
-        except Exception as e:
-            print(f"Error copying template: {str(e)}")
-
-def store_evaluator_name(name):
-    """Store evaluator name in a file for persistence"""
-    try:
-        ensure_data_directory()
-        with open(os.path.join(DATA_DIR, 'current_evaluator.txt'), 'w') as f:
-            f.write(name)
-        log_error(f"Stored evaluator name in file: {name}")
-        return True
-    except Exception as e:
-        log_error(f"Error storing evaluator name: {str(e)}")
-        return False
-
-def get_stored_evaluator_name():
-    """Get stored evaluator name from file"""
-    try:
-        file_path = os.path.join(DATA_DIR, 'current_evaluator.txt')
-        if os.path.exists(file_path):
-            with open(file_path, 'r') as f:
-                name = f.read().strip()
-            log_error(f"Retrieved evaluator name from file: {name}")
-            return name
-        return None
-    except Exception as e:
-        log_error(f"Error retrieving evaluator name: {str(e)}")
-        return None
-
-# Add these new functions for progress tracking
-def get_progress_file(evaluator_name):
-    """Get path to progress file for an evaluator"""
-    return os.path.join(DATA_DIR, f'{evaluator_name}_progress.txt')
-
-def save_current_index(evaluator_name, index):
-    """Save current document index to file"""
-    try:
-        with open(get_progress_file(evaluator_name), 'w') as f:
-            f.write(str(index))
-        return True
-    except Exception as e:
-        log_error(f"Error saving progress: {str(e)}")
-        return False
-
-def load_current_index(evaluator_name):
-    """Load current document index from file"""
-    try:
-        if os.path.exists(get_progress_file(evaluator_name)):
-            with open(get_progress_file(evaluator_name), 'r') as f:
-                return int(f.read().strip())
-        return 1  # Start at 1 if no progress file
-    except Exception as e:
-        log_error(f"Error loading progress: {str(e)}")
-        return 1
+        error_message=error_message,
+        error_details=error_details
+    )
 
 if __name__ == '__main__':
-    print("\n===== Application Startup at", datetime.now().strftime('%Y-%m-%d %H:%M:%S'), "=====\n")
+    print("\n" + "="*60)
+    print(f"Application Starting at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    print("="*60 + "\n")
     
-    # Create data directory
+    # Initialize
     ensure_data_directory()
     
-    # Copy template files if needed
-    copy_template_if_needed()
+    # Log startup info
+    print(f"Data directory: {DATA_DIR}")
+    print(f"Data directory exists: {os.path.exists(DATA_DIR)}")
+    print(f"Environment: {'HF Spaces' if 'SPACE_ID' in os.environ else 'Local'}")
+    
+    if 'SPACE_ID' in os.environ:
+        print(f"Space ID: {os.environ.get('SPACE_ID')}")
+        print(f"Space Author: {os.environ.get('SPACE_AUTHOR_NAME')}")
+    
+    print(f"Data directory contents: {os.listdir(DATA_DIR) if os.path.exists(DATA_DIR) else 'N/A'}")
+    print("\n" + "="*60 + "\n")
     
     # Run the app
-    app.config['DEBUG'] = True
-    app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 7860)))
+    app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 7860)), debug=True)

templates/evaluate.html

@@ -71,13 +71,19 @@
                     <strong>MRN:</strong> {{ mrn }}
                 </div>
                 {% endif %}
+                {% if description %}
+                <div class="info-item">
+                    <strong>Description:</strong> {{ description }}
+                </div>
+                {% endif %}
             </div>
             <div class="note-content">
                 {{ note }}
             </div>
         </div>
 
-        <form method="POST" action="{{ url_for('evaluate') }}">
+        <!-- Single form for all evaluation data -->
+        <form method="POST" action="{{ url_for('evaluate') }}" id="evaluation-form">
             <div class="criteria-container">
                 {% for i in range(criteria|length) %}
                 <div class="criteria-group">
@@ -112,19 +118,10 @@
             </div>
 
             <div class="form-buttons">
-                <form method="POST" action="{{ url_for('evaluate') }}">
-                    <button type="submit" name="action" value="submit" class="submit-btn">Submit Evaluation</button>
-                </form>
-
-                <div class="action-buttons">
-                    <form method="POST" action="{{ url_for('evaluate') }}" onsubmit="return confirm('Are you sure you want to skip this document?');">
-                        <button type="submit" name="action" value="skip" class="skip-btn">Skip Document</button>
-                    </form>
-                    
-                    <form method="POST" action="{{ url_for('evaluate') }}">
-                        <button type="submit" name="action" value="stop_save" class="stop-save-btn">Stop and Save Progress</button>
-                    </form>
-                </div>
+                <button type="submit" name="action" value="submit" class="submit-btn">Submit Evaluation</button>
+                <button type="submit" name="action" value="skip" class="skip-btn" 
+                        onclick="return confirm('Are you sure you want to skip this document?');">Skip Document</button>
+                <button type="submit" name="action" value="stop_save" class="stop-save-btn">Stop and Save Progress</button>
             </div>
         </form>