Upload 4 files

app.py

@@ -344,6 +344,242 @@ def process_photos_face_filter_only(job_id, upload_dir, session_id=None):
         traceback.print_exc()
 
 
+def process_drive_with_parallel_face_detection(job_id, folder_id, upload_dir, face_matcher):
+    """
+    HYBRID APPROACH: Download files from Google Drive while running face detection in parallel.
+
+    This overlaps network I/O (downloading) with GPU compute (face detection) for faster processing.
+
+    Flow:
+    - Download thread: Downloads files and adds paths to queue
+    - Face detection thread: Processes files from queue as they become ready
+    - Both run simultaneously for maximum efficiency
+    """
+    import queue
+    import threading
+
+    print(f"\n{'='*60}")
+    print(f"[Job {job_id}] HYBRID MODE: Parallel Download + Face Detection")
+    print(f"{'='*60}")
+
+    # Shared state
+    file_queue = queue.Queue()
+    results_lock = threading.Lock()
+    matched_photos = []
+    unmatched_photos = []
+    no_faces_photos = []
+    error_photos = []
+
+    # Counters
+    download_complete = threading.Event()
+    total_files = [0]
+    downloaded_count = [0]
+    processed_count = [0]
+
+    # Face detection worker
+    def face_detection_worker():
+        """Process files from queue as they become available."""
+        while True:
+            try:
+                # Wait for file or check if download is complete
+                try:
+                    filepath = file_queue.get(timeout=1.0)
+                except queue.Empty:
+                    # Check if download is complete and queue is empty
+                    if download_complete.is_set() and file_queue.empty():
+                        break
+                    continue
+
+                if filepath is None:  # Poison pill
+                    break
+
+                # Process the file
+                result = face_matcher.check_photo_for_target(filepath)
+
+                with results_lock:
+                    processed_count[0] += 1
+
+                    if 'error' in result:
+                        error_photos.append({'path': filepath, 'error': result['error']})
+                    elif result['num_faces'] == 0:
+                        no_faces_photos.append({'path': filepath, 'num_faces': 0})
+                    elif result['contains_target']:
+                        matched_photos.append({
+                            'path': filepath,
+                            'similarity': result['best_match_similarity'],
+                            'num_faces': result['num_faces'],
+                            'all_similarities': result.get('all_face_similarities', []),
+                            'face_bboxes': result.get('face_bboxes', [])
+                        })
+                    else:
+                        unmatched_photos.append({
+                            'path': filepath,
+                            'best_similarity': result['best_match_similarity'],
+                            'num_faces': result['num_faces']
+                        })
+
+                    # Update progress (use unified message format)
+                    if processed_count[0] % 10 == 0:
+                        # After downloads complete, show scan-only progress
+                        if download_complete.is_set():
+                            pct = 30 + int((processed_count[0] / max(total_files[0], 1)) * 40)
+                            processing_jobs[job_id]['progress'] = min(pct, 70)
+                            processing_jobs[job_id]['message'] = f'Scanning faces: {processed_count[0]}/{total_files[0]} ({len(matched_photos)} matched)'
+                        processing_jobs[job_id]['photos_checked'] = processed_count[0]
+                        print(f"[Job {job_id}] [HYBRID] Downloaded: {downloaded_count[0]}, Face checked: {processed_count[0]}, Matched: {len(matched_photos)}")
+
+                file_queue.task_done()
+
+            except Exception as e:
+                print(f"[Job {job_id}] Face detection error: {e}")
+                continue
+
+    # Callback when file is downloaded
+    def on_file_ready(filepath):
+        """Called by download_folder when each file is ready."""
+        with results_lock:
+            downloaded_count[0] += 1
+        file_queue.put(filepath)
+
+    # Progress callback for download
+    def download_progress(current, total, _filename):
+        total_files[0] = total
+        pct = 5 + int((current / total) * 25)  # 5-30%
+        processing_jobs[job_id]['progress'] = pct
+        processing_jobs[job_id]['message'] = f'Downloading: {current}/{total}, Scanning: {processed_count[0]}'
+        processing_jobs[job_id]['total_files'] = total
+
+    try:
+        processing_jobs[job_id]['status'] = 'processing'
+        processing_jobs[job_id]['progress'] = 5
+        processing_jobs[job_id]['message'] = 'Starting parallel download and face detection...'
+
+        # Start face detection workers (use multiple threads for better throughput)
+        num_workers = 4  # Face detection threads
+        workers = []
+        for _ in range(num_workers):
+            t = threading.Thread(target=face_detection_worker)
+            t.daemon = True
+            t.start()
+            workers.append(t)
+
+        print(f"[Job {job_id}] Started {num_workers} face detection workers")
+
+        # Start download (this will call on_file_ready for each file)
+        print(f"[Job {job_id}] Starting Google Drive download with parallel face detection...")
+
+        download_folder(
+            folder_id,
+            upload_dir,
+            progress_callback=download_progress,
+            file_ready_callback=on_file_ready
+        )
+
+        # Signal download complete
+        download_complete.set()
+        print(f"[Job {job_id}] Download complete. Waiting for face detection to finish...")
+
+        # Wait for queue to be processed
+        file_queue.join()
+
+        # Send poison pills to stop workers
+        for _ in workers:
+            file_queue.put(None)
+
+        # Wait for workers to finish
+        for t in workers:
+            t.join(timeout=5.0)
+
+        print(f"\n[Job {job_id}] HYBRID Face Detection Results:")
+        print(f"  - Photos with your child: {len(matched_photos)}")
+        print(f"  - Photos without match: {len(unmatched_photos)}")
+        print(f"  - Photos with no faces: {len(no_faces_photos)}")
+
+        # Now create thumbnails and prepare review data
+        processing_jobs[job_id]['progress'] = 75
+        processing_jobs[job_id]['message'] = f'Creating thumbnails for {len(matched_photos)} photos...'
+
+        thumbs_dir = os.path.join(upload_dir, 'thumbnails')
+        os.makedirs(thumbs_dir, exist_ok=True)
+
+        filtered_photos = []
+        for i, match in enumerate(matched_photos):
+            filename = os.path.basename(match['path'])
+            thumb_name = get_thumbnail_name(filename)
+            thumb_path = os.path.join(thumbs_dir, thumb_name)
+
+            create_thumbnail(match['path'], thumb_path)
+
+            filtered_photos.append({
+                'filename': filename,
+                'thumbnail': thumb_name,
+                'face_match_score': match['similarity'],
+                'num_faces': match['num_faces'],
+                'face_bboxes': match.get('face_bboxes', [])
+            })
+
+            if (i + 1) % 20 == 0:
+                processing_jobs[job_id]['message'] = f'Creating thumbnails: {i + 1}/{len(matched_photos)}'
+
+        # Sort by face match score
+        filtered_photos.sort(key=lambda x: x['face_match_score'], reverse=True)
+
+        # Prepare unmatched data
+        unmatched_data = []
+        for unmatch in unmatched_photos:
+            filename = os.path.basename(unmatch['path'])
+            unmatched_data.append({
+                'filename': filename,
+                'best_similarity': unmatch.get('best_similarity', 0),
+                'num_faces': unmatch.get('num_faces', 0)
+            })
+
+        for no_face in no_faces_photos:
+            filename = os.path.basename(no_face['path'])
+            unmatched_data.append({
+                'filename': filename,
+                'best_similarity': 0,
+                'num_faces': 0
+            })
+
+        # Store results
+        review_data = {
+            'total_uploaded': total_files[0],
+            'filtered_photos': filtered_photos,
+            'unmatched_photos': unmatched_data,
+            'statistics': {
+                'total_scanned': total_files[0],
+                'matched': len(matched_photos),
+                'unmatched': len(unmatched_photos),
+                'no_faces': len(no_faces_photos),
+                'errors': len(error_photos),
+                'match_rate': f"{(len(matched_photos) / max(total_files[0], 1) * 100):.1f}%"
+            },
+            'reference_count': face_matcher.get_reference_count()
+        }
+
+        # Save review data
+        review_file = os.path.join(RESULTS_FOLDER, f"{job_id}_review.json")
+        with open(review_file, 'w') as f:
+            json.dump(review_data, f, indent=2, default=str)
+
+        processing_jobs[job_id]['progress'] = 100
+        processing_jobs[job_id]['status'] = 'review_pending'
+        processing_jobs[job_id]['message'] = f'Found your child in {len(filtered_photos)} of {total_files[0]} photos!'
+        processing_jobs[job_id]['review_data'] = review_data
+
+        print(f"\n[Job {job_id}] HYBRID MODE COMPLETE!")
+        print(f"  - Found {len(filtered_photos)} photos of your child")
+        print(f"{'='*60}\n")
+
+    except Exception as e:
+        print(f"[Job {job_id}] HYBRID EXCEPTION: {str(e)}")
+        processing_jobs[job_id]['status'] = 'error'
+        processing_jobs[job_id]['message'] = str(e)
+        import traceback
+        traceback.print_exc()
+
+
 def save_photos_by_month(job_id, upload_dir, selected_photos, rejected_photos, month_stats):
     """
     Automatically save both selected and not-selected photos organized by month.
@@ -1477,7 +1713,16 @@ def import_from_drive():
     # Start download in background thread
     def download_and_process():
         try:
-            def progress_callback(current, total, filename):
+            # HYBRID MODE: If we have face references, use parallel download + face detection
+            if has_references:
+                face_matcher = face_matchers.get(face_session_id)
+                if face_matcher and face_matcher.get_reference_count() > 0:
+                    print(f"[Job {job_id}] Using HYBRID MODE: Parallel download + face detection")
+                    process_drive_with_parallel_face_detection(job_id, folder_id, upload_dir, face_matcher)
+                    return
+
+            # SEQUENTIAL MODE: Download all first, then process (for auto mode without face filtering)
+            def progress_callback(current, total, _filename):
                 pct = int(5 + (current / total) * 25)  # 5% to 30%
                 processing_jobs[job_id]['progress'] = pct
                 processing_jobs[job_id]['message'] = f'Downloading from Drive: {current}/{total}'
@@ -1500,14 +1745,9 @@ def import_from_drive():
 
             print(f"[Job {job_id}] Downloaded {downloaded_count} photos from Google Drive")
 
-            # Now start the face filtering or quality selection
-            if has_references:
-                processing_jobs[job_id]['message'] = f'Scanning {downloaded_count} photos for faces...'
-                process_photos_face_filter_only(job_id, upload_dir, face_session_id)
-            else:
-                # No face filtering - use all downloaded photos as confirmed
-                processing_jobs[job_id]['message'] = f'Selecting best from {downloaded_count} photos...'
-                process_photos_quality_selection(job_id, upload_dir, quality_mode, similarity_threshold, downloaded_files)
+            # No face filtering - use all downloaded photos (auto mode)
+            processing_jobs[job_id]['message'] = f'Selecting best from {downloaded_count} photos...'
+            process_photos_quality_selection(job_id, upload_dir, quality_mode, similarity_threshold, downloaded_files)
 
         except Exception as e:
             print(f"[Job {job_id}] Drive import error: {e}")

google_drive.py

@@ -209,6 +209,7 @@ def download_folder(
     folder_id: str,
     output_dir: str,
     progress_callback: Optional[Callable[[int, int, str], None]] = None,
+    file_ready_callback: Optional[Callable[[str], None]] = None,
     include_subfolders: bool = False,
     max_workers: int = 10,  # Parallel downloads (reduced for stability)
     skip_existing: bool = True  # Skip already downloaded files (resume support)
@@ -220,6 +221,7 @@ def download_folder(
         folder_id: The Google Drive folder ID
         output_dir: Local directory to save files
         progress_callback: Optional callback(current, total, filename) for progress updates
+        file_ready_callback: Optional callback(filepath) called when each file is ready for processing
         include_subfolders: Whether to include subfolders
         max_workers: Number of parallel download threads (default 10)
         skip_existing: Skip files that already exist (enables resume, default True)
@@ -321,9 +323,17 @@ def download_folder(
                 if status == 'success':
                     downloaded_count[0] += 1
                     downloaded_files.append(filename)
+                    # Notify that file is ready for processing
+                    if file_ready_callback:
+                        filepath = os.path.join(output_dir, filename)
+                        file_ready_callback(filepath)
                 elif status == 'skipped':
                     skipped_count[0] += 1
                     downloaded_files.append(filename)  # Include skipped files in list
+                    # Also notify for skipped files (they exist and are ready)
+                    if file_ready_callback:
+                        filepath = os.path.join(output_dir, filename)
+                        file_ready_callback(filepath)
                 else:
                     failed_count[0] += 1
 

templates/step2_upload.html

@@ -742,7 +742,6 @@
             <!-- Upload Method Tabs -->
             <div class="upload-tabs">
                 <button class="upload-tab active" onclick="switchUploadMethod('browser')">Browser Upload</button>
-                <button class="upload-tab" onclick="switchUploadMethod('folder')">Local Folder Path</button>
                 <button class="upload-tab" id="drive-tab" onclick="switchUploadMethod('drive')" style="display: none;">Google Drive</button>
             </div>
 
@@ -757,16 +756,6 @@
                 </button>
             </div>
 
-            <!-- Local Folder Path Input -->
-            <div class="folder-input-area hidden" id="folder-input-area">
-                <div class="upload-icon">&#128193;</div>
-                <h3>Enter Local Folder Path</h3>
-                <p>For large batches (1000+ photos), paste the full path to your photos folder</p>
-                <input type="text" id="folder-path" class="folder-path-input"
-                       placeholder="C:\Users\YourName\Photos\EventPhotos">
-                <p class="folder-hint">Example: C:\Users\tanis\Downloads\SchoolPhotos</p>
-            </div>
-
             <!-- Google Drive Input -->
             <div class="folder-input-area hidden" id="drive-input-area">
                 <div class="upload-icon" style="font-size: 48px;">&#9729;</div>
@@ -893,19 +882,14 @@
             document.querySelectorAll('.upload-tab').forEach(tab => {
                 const tabText = tab.textContent.toLowerCase();
                 const isActive = (method === 'browser' && tabText.includes('browser')) ||
-                                 (method === 'folder' && tabText.includes('local')) ||
                                  (method === 'drive' && tabText.includes('drive'));
                 tab.classList.toggle('active', isActive);
             });
             document.getElementById('drop-zone').classList.toggle('hidden', method !== 'browser');
-            document.getElementById('folder-input-area').classList.toggle('hidden', method !== 'folder');
             document.getElementById('drive-input-area').classList.toggle('hidden', method !== 'drive');
 
             // Update file preview visibility based on method
-            if (method === 'folder') {
-                document.getElementById('file-preview').classList.add('hidden');
-                updateFolderPreview();
-            } else if (method === 'drive') {
+            if (method === 'drive') {
                 document.getElementById('file-preview').classList.add('hidden');
                 updateDrivePreview();
             } else {
@@ -916,19 +900,6 @@
             }
         }
 
-        function updateFolderPreview() {
-            const folderPath = document.getElementById('folder-path').value.trim();
-            const preview = document.getElementById('file-preview');
-
-            if (folderPath) {
-                preview.classList.remove('hidden');
-                document.getElementById('file-count').textContent = 'Local folder';
-                document.getElementById('preview-grid').innerHTML = '<div style="padding: 20px; color: #666; font-size: 14px;">Will scan: ' + folderPath + '</div>';
-            } else {
-                preview.classList.add('hidden');
-            }
-        }
-
         function updateDrivePreview() {
             const driveUrl = document.getElementById('drive-folder-url').value.trim();
             const preview = document.getElementById('file-preview');
@@ -1120,43 +1091,6 @@
                     document.getElementById('processing-overlay').classList.add('hidden');
                 }
 
-            } else if (uploadMethod === 'folder') {
-                // Local folder path mode
-                const folderPath = document.getElementById('folder-path').value.trim();
-                if (!folderPath) {
-                    alert('Please enter a folder path');
-                    document.getElementById('processing-overlay').classList.add('hidden');
-                    return;
-                }
-
-                try {
-                    const response = await fetch('/upload_folder', {
-                        method: 'POST',
-                        headers: { 'Content-Type': 'application/json' },
-                        body: JSON.stringify({
-                            folder_path: folderPath,
-                            quality_mode: qualityMode,
-                            similarity_threshold: parseFloat(document.getElementById('similarity').value)
-                        })
-                    });
-
-                    const data = await response.json();
-
-                    if (data.error) {
-                        alert('Error: ' + data.error);
-                        document.getElementById('processing-overlay').classList.add('hidden');
-                        return;
-                    }
-
-                    jobId = data.job_id;
-                    pollStatus();
-
-                } catch (error) {
-                    console.error('Error:', error);
-                    alert('Failed to process folder. Please check the path and try again.');
-                    document.getElementById('processing-overlay').classList.add('hidden');
-                }
-
             } else {
                 // Browser upload mode
                 if (selectedFiles.length === 0) {

templates/step3_review.html

@@ -765,10 +765,51 @@
                 box-shadow: 0 2px 10px rgba(0,0,0,0.08);
             }
 
+            .not-found-card {
+                position: relative;
+            }
+
             .not-found-card img {
                 width: 100%;
                 aspect-ratio: 1;
                 object-fit: cover;
+                background: #f5f5f5;
+            }
+
+            .not-found-card .loading-placeholder {
+                position: absolute;
+                top: 0;
+                left: 0;
+                right: 0;
+                bottom: 40px;
+                background: #f5f5f5;
+                display: flex;
+                flex-direction: column;
+                align-items: center;
+                justify-content: center;
+                gap: 10px;
+            }
+
+            .not-found-card .loading-placeholder.hidden {
+                display: none;
+            }
+
+            .not-found-card .loading-spinner {
+                width: 30px;
+                height: 30px;
+                border: 3px solid #e0e0e0;
+                border-top-color: #7c3aed;
+                border-radius: 50%;
+                animation: spin 1s linear infinite;
+            }
+
+            .not-found-card .loading-text {
+                font-size: 12px;
+                color: #888;
+            }
+
+            @keyframes spin {
+                to { transform: rotate(360deg); }
             }
 
             .not-found-info {
@@ -1079,12 +1120,18 @@
                     const facesText = photo.num_faces === 0 ? 'No faces detected' :
                                       photo.num_faces === 1 ? '1 face (no match)' :
                                       `${photo.num_faces} faces (no match)`;
+                    const cardId = `card-${photo.filename.replace(/[^a-zA-Z0-9]/g, '_')}`;
                     html += `
                         <div class="not-found-card">
+                            <div class="loading-placeholder" id="${cardId}-loader">
+                                <div class="loading-spinner"></div>
+                                <div class="loading-text">Loading...</div>
+                            </div>
                             <img src="/thumbnail/${jobId}/${encodeURIComponent(photo.filename)}"
                                  alt="${photo.filename}"
                                  loading="lazy"
-                                 onerror="this.src='/static/placeholder.png'">
+                                 onload="document.getElementById('${cardId}-loader').classList.add('hidden')"
+                                 onerror="document.getElementById('${cardId}-loader').classList.add('hidden'); this.src='/static/placeholder.png'">
                             <div class="not-found-info">
                                 ${facesText}
                             </div>