Use progress callback for page-by-page PDF progress; install catllm from GitHub

app.py

@@ -1027,77 +1027,67 @@ with col_input:
                     # Progress UI
                     progress_bar = st.progress(0)
                     status_text = st.empty()
-
-                    all_results = []
                     start_time = time.time()
 
-                    # For PDFs, extract pages and process one at a time
+                    # For PDFs, use progress callback
                     if input_type_selected == "pdf":
-                        pdf_name_map = st.session_state.get('pdf_name_map', {})
-                        status_text.text("Extracting PDF pages...")
-
-                        pages = extract_pdf_pages(items_list, pdf_name_map, mode)
-                        total_pages = len(pages)
-
-                        for i, page_data in enumerate(pages):
-                            # Update progress
-                            progress = i / total_pages if total_pages > 0 else 0
+                        # Progress callback for PDF page-by-page updates
+                        def pdf_progress_callback(current_idx, total_pages, page_label):
+                            progress = current_idx / total_pages if total_pages > 0 else 0
                             progress_bar.progress(min(progress, 1.0))
 
-                            # Calculate ETA
                             elapsed = time.time() - start_time
-                            if i > 0:
-                                avg_time = elapsed / i
-                                eta_seconds = avg_time * (total_pages - i)
+                            if current_idx > 0:
+                                avg_time = elapsed / current_idx
+                                eta_seconds = avg_time * (total_pages - current_idx)
                                 eta_str = f" | ETA: {eta_seconds:.0f}s" if eta_seconds < 60 else f" | ETA: {eta_seconds/60:.1f}m"
                             else:
                                 eta_str = ""
 
-                            page_label = page_data[1]
-                            status_text.text(f"Processing page {i+1} of {total_pages} ({page_label}) ({progress*100:.0f}%){eta_str}")
+                            status_text.text(f"Processing page {current_idx+1} of {total_pages} ({page_label}) ({progress*100:.0f}%){eta_str}")
 
-                            try:
-                                if page_data[2] == "text":
-                                    # Text mode - classify as text
-                                    result = catllm.classify(
-                                        input_data=[page_data[0]],
-                                        categories=categories_entered,
-                                        api_key=actual_api_key,
-                                        input_type="text",
-                                        description=description,
-                                        user_model=model,
-                                        model_source=model_source
-                                    )
-                                else:
-                                    # Image mode - classify as image
-                                    result = catllm.classify(
-                                        input_data=[page_data[0]],
-                                        categories=categories_entered,
-                                        api_key=actual_api_key,
-                                        input_type="image",
-                                        description=description,
-                                        user_model=model,
-                                        model_source=model_source
-                                    )
-
-                                # Replace the input column with the page label
-                                if 'image_input' in result.columns:
-                                    result['pdf_input'] = page_label
-                                    result = result.drop(columns=['image_input'])
-                                elif 'text_input' in result.columns:
-                                    result['pdf_input'] = page_label
-                                    result = result.drop(columns=['text_input'])
-                                else:
-                                    result['pdf_input'] = page_label
-
-                                all_results.append(result)
-                            except Exception as e:
-                                st.warning(f"Error on {page_label}: {str(e)}")
-                                continue
+                        try:
+                            result_df = catllm.classify(
+                                input_data=items_list,
+                                categories=categories_entered,
+                                api_key=actual_api_key,
+                                input_type="pdf",
+                                description=description,
+                                user_model=model,
+                                model_source=model_source,
+                                mode=mode,
+                                progress_callback=pdf_progress_callback
+                            )
+
+                            processing_time = time.time() - start_time
+                            total_items = len(result_df)
+                            progress_bar.progress(1.0)
+                            status_text.text(f"Completed {total_items} pages in {processing_time:.1f}s")
+
+                            # Replace temp paths with original filenames in pdf_input column
+                            if 'pdf_input' in result_df.columns:
+                                pdf_name_map = st.session_state.get('pdf_name_map', {})
+                                def replace_temp_path(val):
+                                    if pd.isna(val):
+                                        return val
+                                    val_str = str(val)
+                                    for temp_path, orig_name in pdf_name_map.items():
+                                        # Check if the temp path's filename (without extension) is in the value
+                                        temp_name = os.path.basename(temp_path).replace('.pdf', '')
+                                        if temp_name in val_str:
+                                            return val_str.replace(temp_name, orig_name)
+                                    return val_str
+                                result_df['pdf_input'] = result_df['pdf_input'].apply(replace_temp_path)
+
+                            all_results = [result_df]
+
+                        except Exception as e:
+                            st.error(f"Error: {str(e)}")
+                            all_results = []
 
-                        total_items = total_pages
                     else:
-                        # Non-PDF processing (text, images)
+                        # Non-PDF processing (text, images) - item by item
+                        all_results = []
                         total_items = len(items_list)
 
                         for i, item in enumerate(items_list):
@@ -1126,7 +1116,6 @@ with col_input:
                                 )
                                 all_results.append(item_result)
 
-                                # Update progress after processing
                                 progress = (i + 1) / total_items if total_items > 0 else 1.0
                                 progress_bar.progress(min(progress, 1.0))
 
@@ -1134,12 +1123,8 @@ with col_input:
                                 st.warning(f"Error on item {i+1}: {str(e)}")
                                 continue
 
-                    # Complete progress
-                    progress_bar.progress(1.0)
-                    processing_time = time.time() - start_time
-                    if input_type_selected == "pdf":
-                        status_text.text(f"Completed {total_items} pages in {processing_time:.1f}s")
-                    else:
+                        processing_time = time.time() - start_time
+                        progress_bar.progress(1.0)
                         status_text.text(f"Completed {total_items} items in {processing_time:.1f}s")
 
                     if all_results:

requirements.txt

@@ -1,5 +1,5 @@
 streamlit>=1.32.0
-cat-llm[pdf]>=0.1.6
+cat-llm[pdf] @ git+https://github.com/chrissoria/cat-llm.git@main
 mistralai
 pydantic==2.10.6
 huggingface_hub<0.27.0