Upload app.py
Browse files
app.py
CHANGED
|
@@ -6,6 +6,7 @@ from typing import List, Tuple
|
|
| 6 |
import tempfile
|
| 7 |
import shutil
|
| 8 |
from dotenv import load_dotenv
|
|
|
|
| 9 |
|
| 10 |
# Load environment variables
|
| 11 |
load_dotenv()
|
|
@@ -25,8 +26,8 @@ pc = Pinecone(api_key=pinecone_api_key)
|
|
| 25 |
UPLOAD_FOLDER = "uploads"
|
| 26 |
os.makedirs(UPLOAD_FOLDER, exist_ok=True)
|
| 27 |
|
| 28 |
-
def
|
| 29 |
-
"""Process multiple files with individual metadata
|
| 30 |
if not files:
|
| 31 |
return "β Error: No files selected", ""
|
| 32 |
|
|
@@ -36,8 +37,11 @@ def process_files(files, *metadata_inputs):
|
|
| 36 |
try:
|
| 37 |
results = []
|
| 38 |
errors = []
|
|
|
|
| 39 |
|
| 40 |
# Initialize Pinecone Assistant
|
|
|
|
|
|
|
| 41 |
assistant_name = os.getenv("PINECONE_ASSISTANT_NAME", "gstminutes")
|
| 42 |
assistant = pc.assistant.Assistant(assistant_name=assistant_name)
|
| 43 |
|
|
@@ -45,6 +49,7 @@ def process_files(files, *metadata_inputs):
|
|
| 45 |
for i, file_path in enumerate(files):
|
| 46 |
try:
|
| 47 |
filename = os.path.basename(file_path)
|
|
|
|
| 48 |
|
| 49 |
# Get metadata for this specific file (3 fields per file: sections, keywords, description)
|
| 50 |
sections_idx = i * 3
|
|
@@ -67,6 +72,7 @@ def process_files(files, *metadata_inputs):
|
|
| 67 |
continue
|
| 68 |
|
| 69 |
# Prepare metadata for this file
|
|
|
|
| 70 |
metadata = {
|
| 71 |
"sections": [s.strip() for s in sections.split(",") if s.strip()],
|
| 72 |
"keywords": [k.strip() for k in keywords.split(",") if k.strip()],
|
|
@@ -74,10 +80,12 @@ def process_files(files, *metadata_inputs):
|
|
| 74 |
}
|
| 75 |
|
| 76 |
# Copy to uploads directory
|
|
|
|
| 77 |
destination_path = os.path.join(UPLOAD_FOLDER, filename)
|
| 78 |
shutil.copy2(file_path, destination_path)
|
| 79 |
|
| 80 |
# Upload to Pinecone Assistant
|
|
|
|
| 81 |
response = assistant.upload_file(
|
| 82 |
file_path=destination_path,
|
| 83 |
metadata=metadata,
|
|
@@ -97,6 +105,10 @@ def process_files(files, *metadata_inputs):
|
|
| 97 |
"error": f"β Error: {str(file_error)}"
|
| 98 |
})
|
| 99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
# Format results for display
|
| 101 |
success_count = len(results)
|
| 102 |
error_count = len(errors)
|
|
@@ -122,11 +134,11 @@ def process_files(files, *metadata_inputs):
|
|
| 122 |
for error in errors:
|
| 123 |
detailed_results += f"- **{error['filename']}** - {error['error']}\n"
|
| 124 |
|
| 125 |
-
return status_message, detailed_results
|
| 126 |
|
| 127 |
except Exception as e:
|
| 128 |
error_msg = f"β **Critical Error:** {str(e)}"
|
| 129 |
-
return error_msg, ""
|
| 130 |
|
| 131 |
def update_metadata_fields(files):
|
| 132 |
"""Update metadata fields based on uploaded files"""
|
|
@@ -156,7 +168,15 @@ def update_metadata_fields(files):
|
|
| 156 |
|
| 157 |
def clear_form():
|
| 158 |
"""Clear all form fields"""
|
| 159 |
-
return [None] + [""] * 30 + ["", ""]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
|
| 161 |
# Create Gradio interface
|
| 162 |
with gr.Blocks(
|
|
@@ -235,6 +255,13 @@ with gr.Blocks(
|
|
| 235 |
size="lg"
|
| 236 |
)
|
| 237 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
gr.Markdown("---")
|
| 239 |
|
| 240 |
# Results section
|
|
@@ -261,15 +288,19 @@ with gr.Blocks(
|
|
| 261 |
outputs=metadata_fields
|
| 262 |
)
|
| 263 |
|
|
|
|
| 264 |
upload_btn.click(
|
| 265 |
-
fn=
|
|
|
|
|
|
|
|
|
|
| 266 |
inputs=[files_input] + metadata_fields,
|
| 267 |
-
outputs=[status_output, results_output]
|
| 268 |
)
|
| 269 |
|
| 270 |
clear_btn.click(
|
| 271 |
fn=clear_form,
|
| 272 |
-
outputs=[files_input] + metadata_fields + [status_output, results_output]
|
| 273 |
)
|
| 274 |
|
| 275 |
# Footer
|
|
|
|
| 6 |
import tempfile
|
| 7 |
import shutil
|
| 8 |
from dotenv import load_dotenv
|
| 9 |
+
import time
|
| 10 |
|
| 11 |
# Load environment variables
|
| 12 |
load_dotenv()
|
|
|
|
| 26 |
UPLOAD_FOLDER = "uploads"
|
| 27 |
os.makedirs(UPLOAD_FOLDER, exist_ok=True)
|
| 28 |
|
| 29 |
+
def process_files_with_progress(files, *metadata_inputs, progress=gr.Progress()):
|
| 30 |
+
"""Process multiple files with individual metadata and show progress"""
|
| 31 |
if not files:
|
| 32 |
return "β Error: No files selected", ""
|
| 33 |
|
|
|
|
| 37 |
try:
|
| 38 |
results = []
|
| 39 |
errors = []
|
| 40 |
+
total_files = len(files)
|
| 41 |
|
| 42 |
# Initialize Pinecone Assistant
|
| 43 |
+
progress(0, desc="π§ Initializing Pinecone Assistant...")
|
| 44 |
+
time.sleep(0.5) # Small delay to show the progress
|
| 45 |
assistant_name = os.getenv("PINECONE_ASSISTANT_NAME", "gstminutes")
|
| 46 |
assistant = pc.assistant.Assistant(assistant_name=assistant_name)
|
| 47 |
|
|
|
|
| 49 |
for i, file_path in enumerate(files):
|
| 50 |
try:
|
| 51 |
filename = os.path.basename(file_path)
|
| 52 |
+
progress((i / total_files), desc=f"π Processing {filename}... ({i+1}/{total_files})")
|
| 53 |
|
| 54 |
# Get metadata for this specific file (3 fields per file: sections, keywords, description)
|
| 55 |
sections_idx = i * 3
|
|
|
|
| 72 |
continue
|
| 73 |
|
| 74 |
# Prepare metadata for this file
|
| 75 |
+
progress((i / total_files), desc=f"π·οΈ Preparing metadata for {filename}...")
|
| 76 |
metadata = {
|
| 77 |
"sections": [s.strip() for s in sections.split(",") if s.strip()],
|
| 78 |
"keywords": [k.strip() for k in keywords.split(",") if k.strip()],
|
|
|
|
| 80 |
}
|
| 81 |
|
| 82 |
# Copy to uploads directory
|
| 83 |
+
progress((i / total_files), desc=f"π Copying {filename} to uploads...")
|
| 84 |
destination_path = os.path.join(UPLOAD_FOLDER, filename)
|
| 85 |
shutil.copy2(file_path, destination_path)
|
| 86 |
|
| 87 |
# Upload to Pinecone Assistant
|
| 88 |
+
progress((i / total_files), desc=f"βοΈ Uploading {filename} to Pinecone...")
|
| 89 |
response = assistant.upload_file(
|
| 90 |
file_path=destination_path,
|
| 91 |
metadata=metadata,
|
|
|
|
| 105 |
"error": f"β Error: {str(file_error)}"
|
| 106 |
})
|
| 107 |
|
| 108 |
+
# Final progress update
|
| 109 |
+
progress(1.0, desc="β
Processing complete!")
|
| 110 |
+
time.sleep(0.5)
|
| 111 |
+
|
| 112 |
# Format results for display
|
| 113 |
success_count = len(results)
|
| 114 |
error_count = len(errors)
|
|
|
|
| 134 |
for error in errors:
|
| 135 |
detailed_results += f"- **{error['filename']}** - {error['error']}\n"
|
| 136 |
|
| 137 |
+
return status_message, detailed_results, "β
**Processing completed successfully!**"
|
| 138 |
|
| 139 |
except Exception as e:
|
| 140 |
error_msg = f"β **Critical Error:** {str(e)}"
|
| 141 |
+
return error_msg, "", "β **Processing failed with error**"
|
| 142 |
|
| 143 |
def update_metadata_fields(files):
|
| 144 |
"""Update metadata fields based on uploaded files"""
|
|
|
|
| 168 |
|
| 169 |
def clear_form():
|
| 170 |
"""Clear all form fields"""
|
| 171 |
+
return [None] + [""] * 30 + ["", "", "π’ **Ready to process documents**"]
|
| 172 |
+
|
| 173 |
+
def start_processing():
|
| 174 |
+
"""Show processing started status"""
|
| 175 |
+
return "π **Processing documents... Please wait**"
|
| 176 |
+
|
| 177 |
+
def finish_processing():
|
| 178 |
+
"""Show processing finished status"""
|
| 179 |
+
return "β
**Processing completed successfully!**"
|
| 180 |
|
| 181 |
# Create Gradio interface
|
| 182 |
with gr.Blocks(
|
|
|
|
| 255 |
size="lg"
|
| 256 |
)
|
| 257 |
|
| 258 |
+
# Processing status indicator
|
| 259 |
+
with gr.Row():
|
| 260 |
+
processing_status = gr.Markdown(
|
| 261 |
+
value="π’ **Ready to process documents**",
|
| 262 |
+
visible=True
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
gr.Markdown("---")
|
| 266 |
|
| 267 |
# Results section
|
|
|
|
| 288 |
outputs=metadata_fields
|
| 289 |
)
|
| 290 |
|
| 291 |
+
# Show processing status when upload starts
|
| 292 |
upload_btn.click(
|
| 293 |
+
fn=start_processing,
|
| 294 |
+
outputs=[processing_status]
|
| 295 |
+
).then(
|
| 296 |
+
fn=process_files_with_progress,
|
| 297 |
inputs=[files_input] + metadata_fields,
|
| 298 |
+
outputs=[status_output, results_output, processing_status]
|
| 299 |
)
|
| 300 |
|
| 301 |
clear_btn.click(
|
| 302 |
fn=clear_form,
|
| 303 |
+
outputs=[files_input] + metadata_fields + [status_output, results_output, processing_status]
|
| 304 |
)
|
| 305 |
|
| 306 |
# Footer
|