updated file reading logic

main.py

@@ -6,6 +6,7 @@ import shutil
 import asyncio
 import logging
 import traceback
+from chardet import detect
 from httpx import AsyncClient, RequestError
 from typing import List, Dict, Any, Optional
 from fastapi.staticfiles import StaticFiles
@@ -131,19 +132,55 @@ async def process_query(user_query: str, sse_queue: asyncio.Queue):
                         if filename not in state["user_files_cache"]:
                             try:
                                 await sse_queue.put(("step", "Reading User-Provided Files..."))
-                                with open(file_path, 'r', encoding='utf-8') as f:
-                                    file_content = f.read()
-                                    state["user_files_cache"][filename] = file_content
-                            except Exception as e:
-                                logger.error(f"Error reading file {filename}: {e}")
-                                # Try reading as binary and decode
+                                
+                                # Always read as binary first
+                                with open(file_path, 'rb') as f:
+                                    file_bytes = f.read()
+                                
+                                # Try to decode with multiple strategies
+                                file_content = None
+                                
+                                # Strategy 1: Try UTF-8 with BOM handling
                                 try:
-                                    with open(file_path, 'rb') as f:
-                                        file_content = f.read().decode('utf-8', errors='ignore')
-                                        state["user_files_cache"][filename] = file_content
-                                except Exception as e2:
-                                    logger.error(f"Error reading file {filename} as binary: {e2}")
-                                    state["user_files_cache"][filename] = ""  # Cache empty to avoid retrying
+                                    # Handle UTF-8 BOM if present
+                                    if file_bytes.startswith(b'\xef\xbb\xbf'):
+                                        file_content = file_bytes[3:].decode('utf-8')
+                                    else:
+                                        file_content = file_bytes.decode('utf-8')
+                                    logger.info(f"Successfully decoded {filename} as UTF-8")
+                                except UnicodeDecodeError:
+                                    # Strategy 2: Try other common encodings
+                                    for encoding in ['utf-8-sig', 'latin-1', 'cp1252', 'iso-8859-1', 'windows-1252']:
+                                        try:
+                                            file_content = file_bytes.decode(encoding)
+                                            logger.info(f"Successfully decoded {filename} with {encoding}")
+                                            break
+                                        except UnicodeDecodeError:
+                                            continue
+                                    
+                                    # Strategy 3: If all else fails, use chardet for detection
+                                    if file_content is None:
+                                        try:
+                                            
+                                            detected = detect(file_bytes)
+                                            if detected['encoding']:
+                                                file_content = file_bytes.decode(detected['encoding'])
+                                                logger.info(f"Decoded {filename} with detected encoding: {detected['encoding']}")
+                                        except:
+                                            pass
+                                    
+                                    # Final fallback: Use UTF-8 with replacement
+                                    if file_content is None:
+                                        file_content = file_bytes.decode('utf-8', errors='replace')
+                                        logger.warning(f"Had to use error replacement for {filename}")
+                                
+                                # Store the decoded content
+                                state["user_files_cache"][filename] = file_content
+                                logger.info(f"Successfully cached file {filename}, length: {len(file_content)} chars")
+                                
+                            except Exception as e:
+                                logger.error(f"Error reading file {filename}: {str(e)}")
+                                state["user_files_cache"][filename] = ""  # Cache empty to avoid retrying
         
         # Add all cached file contents
         for filename, content in state["user_files_cache"].items():

requirements.txt

@@ -3,6 +3,7 @@ aiohttp==3.10.10
 anthropic==0.42.0
 beautifulsoup4==4.12.3
 bert_score==0.3.13
+chardet>=5.0.0
 crawl4ai[all]==0.3.731
 deepeval==2.0
 fake_useragent==1.5.1