init

.gitignore

@@ -31,4 +31,5 @@ __pycache__
 *.egg-info
 venv/
 env/
-ENV/
+ENV/
+data/hackathon_data

app/main.py

@@ -143,7 +143,7 @@ def retrieve_documents(query: str, top_k: int = 3) -> List[Dict]:
         documents.append({
             'pdf_name': match['metadata'].get('pdf_name', 'unknown.pdf'),
             'page_number': page_num,
-            'content': match['metadata'].get('text', ''),
+            'content': match['metadata'].get('content', ''),  # Changed from 'text' to 'content'
             'score': match.get('score', 0.0)
         })
 
@@ -250,34 +250,86 @@ async def llm_endpoint(request: Request):
     Accepts two formats:
     1. QuestionRequest: {"question": "...", "temperature": 0.2, "max_tokens": 1000}
     2. ChatRequest: {"messages": [{"role": "user", "content": "..."}], ...}
+
+    ALWAYS returns: {"answer": str, "sources": List[Dict]}
     """
     try:
         # Parse request body
-        body = await request.json()
+        try:
+            body = await request.json()
+        except:
+            # Empty or invalid JSON - return error in expected format
+            return AnswerResponse(
+                answer="Error: Invalid JSON in request body. Please send valid JSON with 'question' field.",
+                sources=[],
+                response_time=0.0
+            )
 
+        # Handle list format (validator sends list directly)
+        if isinstance(body, list):
+            # Validator format: [{"role": "user", "content": "..."}]
+            user_messages = [msg for msg in body if isinstance(msg, dict) and msg.get("role") == "user"]
+            if not user_messages:
+                return AnswerResponse(
+                    answer="Error: No user message found in messages array.",
+                    sources=[],
+                    response_time=0.0
+                )
+            query = user_messages[-1].get("content")
+            if not query or not query.strip():
+                return AnswerResponse(
+                    answer="Error: Empty message content provided.",
+                    sources=[],
+                    response_time=0.0
+                )
+            temperature = 0.2
+            max_tokens = 1000
         # Determine request format and extract query
-        if "question" in body:
+        elif "question" in body:
             # QuestionRequest format
             query = body.get("question")
+            if not query or not query.strip():
+                return AnswerResponse(
+                    answer="Error: Empty question provided. Please provide a valid question.",
+                    sources=[],
+                    response_time=0.0
+                )
             temperature = body.get("temperature", 0.2)
             max_tokens = body.get("max_tokens", 1000)
-            is_simple_format = True
         elif "messages" in body:
             # ChatRequest format
             messages = body.get("messages", [])
             if not messages:
-                raise HTTPException(status_code=400, detail="No messages provided")
+                return AnswerResponse(
+                    answer="Error: No messages provided in request.",
+                    sources=[],
+                    response_time=0.0
+                )
 
             user_messages = [msg for msg in messages if msg.get("role") == "user"]
             if not user_messages:
-                raise HTTPException(status_code=400, detail="No user message found")
+                return AnswerResponse(
+                    answer="Error: No user message found in messages array.",
+                    sources=[],
+                    response_time=0.0
+                )
 
             query = user_messages[-1].get("content")
+            if not query or not query.strip():
+                return AnswerResponse(
+                    answer="Error: Empty message content provided.",
+                    sources=[],
+                    response_time=0.0
+                )
             temperature = body.get("temperature", 0.2)
             max_tokens = body.get("max_tokens", 1000)
-            is_simple_format = False
         else:
-            raise HTTPException(status_code=400, detail="Invalid request format. Expected 'question' or 'messages' field.")
+            # No question or messages field - return error in expected format
+            return AnswerResponse(
+                answer="Error: Invalid request format. Expected 'question' or 'messages' field in request body.",
+                sources=[],
+                response_time=0.0
+            )
 
         # Retrieve relevant documents
         documents = retrieve_documents(query, top_k=3)
@@ -290,35 +342,30 @@ async def llm_endpoint(request: Request):
             max_tokens=max_tokens
         )
 
-        # Format sources for response
+        # Format sources for response (validator expects pdf_name, page_number, content)
         sources = [
             {
                 "pdf_name": doc['pdf_name'],
-                "page_number": doc['page_number'],
-                "relevance_score": f"{doc['score']:.3f}"
+                "page_number": doc['page_number'],  # Already converted to int
+                "content": doc['content']  # The actual document text
             }
             for doc in documents
         ]
 
-        # Return appropriate response format
-        if is_simple_format:
-            return AnswerResponse(
-                answer=answer,
-                sources=sources,
-                response_time=round(response_time, 2)
-            )
-        else:
-            return ChatResponse(
-                response=answer,
-                sources=[{k: str(v) for k, v in s.items()} for s in sources],
-                response_time=round(response_time, 2),
-                model="Llama-4-Maverick-17B-128E-Instruct-FP8"
-            )
+        # Always return AnswerResponse format (validator expects 'answer' and 'sources' keys)
+        return AnswerResponse(
+            answer=answer,
+            sources=sources,
+            response_time=round(response_time, 2)
+        )
 
-    except HTTPException:
-        raise
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")
+        # Always return expected format, even for errors
+        return AnswerResponse(
+            answer=f"Error: {str(e)}",
+            sources=[],
+            response_time=0.0
+        )
 
 
 # ============================================================================

scripts/ingest_hackathon_data.py

@@ -0,0 +1,162 @@
+"""
+Ingest ONLY PDFs from hackathon_data folder
+Parallel processing with 4 workers
+"""
+
+import os
+import sys
+import time
+import json
+from pathlib import Path
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from dotenv import load_dotenv
+
+# Add parent directory to path
+sys.path.insert(0, str(Path(__file__).parent))
+
+# Load environment
+load_dotenv()
+
+# Import from the main ingestion script
+PROJECT_ROOT = Path(__file__).parent.parent
+PDFS_DIR = PROJECT_ROOT / "data" / "hackathon_data"  # Changed to hackathon_data
+OUTPUT_DIR = PROJECT_ROOT / "output" / "ingestion"
+
+# Import the ingestion function
+import ingest_pdfs
+
+def worker_ingest(pdf_path: str):
+    """Worker function to ingest a single PDF"""
+    try:
+        result = ingest_pdfs.ingest_pdf(str(pdf_path))
+        return result
+    except Exception as e:
+        return {
+            "pdf_name": Path(pdf_path).name,
+            "status": "error",
+            "error": str(e)
+        }
+
+
+def main():
+    """Main parallel ingestion pipeline"""
+    print("\n" + "="*70)
+    print("🚀 HACKATHON DATA INGESTION (4x PARALLEL)")
+    print("="*70)
+    print(f"📂 PDF Directory: {PDFS_DIR}")
+    print(f"⚡ Workers: 4 PDFs at once")
+    print(f"🎯 Vector Database: Pinecone ({os.getenv('PINECONE_INDEX_NAME')})")
+    print("="*70)
+
+    # Get all PDFs
+    all_pdfs = sorted(PDFS_DIR.glob("*.pdf"))
+    print(f"\n📚 Found {len(all_pdfs)} PDFs in hackathon_data folder")
+
+    if not all_pdfs:
+        print("\n❌ No PDFs found in hackathon_data folder!")
+        return
+
+    for pdf in all_pdfs:
+        print(f"   → {pdf.name}")
+
+    print(f"\n⚡ Starting parallel processing with 4 workers...")
+    print(f"⏱️  Estimated time: ~{len(all_pdfs) * 80 / 4 / 60:.1f} minutes\n")
+
+    # Process in parallel
+    results = []
+    completed = 0
+    start_time = time.time()
+
+    with ProcessPoolExecutor(max_workers=4) as executor:
+        # Submit all jobs
+        future_to_pdf = {
+            executor.submit(worker_ingest, str(pdf)): pdf
+            for pdf in all_pdfs
+        }
+
+        # Collect results as they complete
+        for future in as_completed(future_to_pdf):
+            pdf = future_to_pdf[future]
+            completed += 1
+
+            try:
+                result = future.result()
+                results.append(result)
+
+                if result.get("status") == "success":
+                    elapsed = time.time() - start_time
+                    avg_time = elapsed / completed
+                    remaining = len(all_pdfs) - completed
+                    eta = remaining * avg_time / 60
+
+                    print(f"✅ [{completed}/{len(all_pdfs)}] {pdf.name}")
+                    print(f"   📊 {result['num_vectors']} vectors, {result['time_total']:.1f}s")
+                    print(f"   ⏱️  ETA: {eta:.1f} minutes remaining\n")
+                else:
+                    print(f"❌ [{completed}/{len(all_pdfs)}] {pdf.name} - {result.get('error', 'Unknown error')}\n")
+
+            except Exception as e:
+                print(f"❌ [{completed}/{len(all_pdfs)}] {pdf.name} - Error: {e}\n")
+                results.append({
+                    "pdf_name": pdf.name,
+                    "status": "error",
+                    "error": str(e)
+                })
+
+    total_time = time.time() - start_time
+
+    # Summary
+    print("\n" + "="*70)
+    print("📊 INGESTION COMPLETE")
+    print("="*70)
+
+    successful = [r for r in results if r.get("status") == "success"]
+    failed = [r for r in results if r.get("status") == "error"]
+
+    print(f"\n✅ Successful: {len(successful)}/{len(all_pdfs)}")
+    print(f"❌ Failed: {len(failed)}")
+    print(f"⏱️  Total Time: {total_time/60:.1f} minutes")
+
+    if successful:
+        total_vectors = sum(r["num_vectors"] for r in successful)
+        avg_time = sum(r["time_total"] for r in successful) / len(successful)
+        print(f"\n📦 Total Vectors Uploaded: {total_vectors}")
+        print(f"⏱️  Average Time per PDF: {avg_time:.1f}s")
+
+    # Save results
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    results_file = OUTPUT_DIR / "hackathon_data_ingestion.json"
+
+    with open(results_file, 'w', encoding='utf-8') as f:
+        json.dump({
+            "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "source_folder": "hackathon_data",
+            "total_pdfs": len(all_pdfs),
+            "successful": len(successful),
+            "failed": len(failed),
+            "total_time_seconds": round(total_time, 2),
+            "results": results
+        }, f, indent=2, ensure_ascii=False)
+
+    print(f"\n📄 Results saved to: {results_file}")
+
+    # Final Pinecone stats
+    try:
+        from pinecone import Pinecone
+        pc = Pinecone(api_key=os.getenv("PINECONE_API_KEY"))
+        index = pc.Index(os.getenv("PINECONE_INDEX_NAME", "hackathon"))
+        stats = index.describe_index_stats()
+
+        print(f"\n📊 Final Pinecone Stats:")
+        print(f"   Total Vectors: {stats.get('total_vector_count', 0)}")
+        print(f"   Dimensions: {stats.get('dimension', 0)}")
+    except Exception as e:
+        print(f"\nCould not fetch Pinecone stats: {e}")
+
+    print("\n" + "="*70)
+    print("🎉 HACKATHON DATA INGESTION COMPLETE!")
+    print("="*70)
+
+
+if __name__ == "__main__":
+    main()

test_llm.json

@@ -1 +0,0 @@
-{"question":"Neft hasilatı haqqında nə məlumat var?"}