Update email_fetcher: add fetch_all_emails_by_date and fetch_headers_only for historical processing

modules/email_fetcher.py

@@ -92,7 +92,6 @@ class EmailFetcher:
                 logger.error(f"Failed to select folder: {folder}")
                 return []
 
-            # Search for unread emails
             status, message_ids = self.connection.search(None, "UNSEEN")
             if status != "OK":
                 logger.error("Failed to search for unread emails")
@@ -115,24 +114,33 @@ class EmailFetcher:
 
         return emails_data
 
-    def fetch_emails_by_date(self, since_date: str, folder: str = "INBOX") -> List[Dict[str, Any]]:
-        """Fetch emails since a specific date (format: DD-Mon-YYYY, e.g., '01-Jan-2024')."""
+    def fetch_all_emails_by_date(self, since_date: str, folder: str = "INBOX") -> List[Dict[str, Any]]:
+        """
+        Fetch ALL emails (read + unread) since a date.
+        This is the key method for historical processing.
+        Date format: DD-Mon-YYYY (e.g., '01-Jan-2024')
+        Opens mailbox as READONLY so read status is NOT changed.
+        """
         if not self.connection:
             if not self.connect():
                 return []
 
         emails_data = []
         try:
+            # READONLY mode - does NOT mark emails as read
             status, _ = self.connection.select(folder, readonly=True)
             if status != "OK":
+                logger.error(f"Failed to select folder: {folder}")
                 return []
 
+            # Search ALL emails since date (not just UNSEEN)
             status, message_ids = self.connection.search(None, f'SINCE {since_date}')
             if status != "OK":
+                logger.error(f"Failed to search emails since {since_date}")
                 return []
 
             ids = message_ids[0].split()
-            logger.info(f"Found {len(ids)} emails since {since_date}")
+            logger.info(f"Found {len(ids)} total emails since {since_date}")
 
             for msg_id in ids:
                 try:
@@ -148,6 +156,78 @@ class EmailFetcher:
 
         return emails_data
 
+    def fetch_email_headers_by_date(self, since_date: str, folder: str = "INBOX") -> List[Dict[str, Any]]:
+        """
+        Fetch only email HEADERS (fast) since a date - for preview/listing.
+        Does NOT download attachments or full body. Very fast.
+        Date format: DD-Mon-YYYY (e.g., '01-Apr-2025')
+        """
+        if not self.connection:
+            if not self.connect():
+                return []
+
+        headers_list = []
+        try:
+            status, _ = self.connection.select(folder, readonly=True)
+            if status != "OK":
+                return []
+
+            status, message_ids = self.connection.search(None, f'SINCE {since_date}')
+            if status != "OK":
+                return []
+
+            ids = message_ids[0].split()
+            logger.info(f"Found {len(ids)} emails since {since_date} (headers only)")
+
+            for msg_id in ids:
+                try:
+                    # Fetch only headers (ENVELOPE) - much faster than full RFC822
+                    status, msg_data = self.connection.fetch(msg_id, "(BODY[HEADER.FIELDS (FROM TO SUBJECT DATE MESSAGE-ID)])")
+                    if status != "OK":
+                        continue
+
+                    header_bytes = msg_data[0][1]
+                    msg = email.message_from_bytes(header_bytes)
+
+                    message_id = msg.get("Message-ID", "").strip()
+                    if not message_id:
+                        message_id = f"generated_{msg_id.decode()}_{datetime.now().timestamp()}"
+
+                    subject = self._decode_header(msg.get("Subject", ""))
+                    sender = self._decode_header(msg.get("From", ""))
+                    recipients = self._decode_header(msg.get("To", ""))
+
+                    date_str = msg.get("Date", "")
+                    try:
+                        date_received = parsedate_to_datetime(date_str)
+                    except Exception:
+                        date_received = None
+
+                    headers_list.append({
+                        "msg_id": msg_id,
+                        "message_id": message_id,
+                        "subject": subject,
+                        "sender": sender,
+                        "recipients": recipients,
+                        "date_received": date_received.isoformat() if date_received else None,
+                    })
+
+                except Exception as e:
+                    logger.error(f"Error fetching header for {msg_id}: {e}")
+                    continue
+
+        except Exception as e:
+            logger.error(f"Error fetching headers: {e}")
+
+        return headers_list
+
+    def fetch_single_email_by_msg_id(self, msg_id: bytes) -> Optional[Dict[str, Any]]:
+        """Fetch a single email by its IMAP message ID (for selective processing)."""
+        if not self.connection:
+            if not self.connect():
+                return None
+        return self._fetch_single_email(msg_id, mark_read=False)
+
     def _fetch_single_email(self, msg_id: bytes, mark_read: bool = False) -> Optional[Dict[str, Any]]:
         """Fetch and parse a single email."""
         status, msg_data = self.connection.fetch(msg_id, "(RFC822)")
@@ -184,9 +264,12 @@ class EmailFetcher:
         # Extract attachments
         attachments = self._extract_attachments(msg, message_id)
 
-        # Mark as read if requested
+        # Mark as read if requested (only works if mailbox opened in read-write mode)
         if mark_read:
-            self.connection.store(msg_id, "+FLAGS", "\\Seen")
+            try:
+                self.connection.store(msg_id, "+FLAGS", "\\Seen")
+            except Exception:
+                pass
 
         return {
             "message_id": message_id,