Create main.py

main.py

@@ -0,0 +1,84 @@
+from fastapi import FastAPI, Request
+from supabase import create_client, Client
+import json
+import uvicorn
+from typing import Dict, List, Optional
+import os
+from dotenv import load_load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Initialize FastAPI app
+app = FastAPI()
+
+# Initialize Supabase client
+supabase: Client = create_client(
+    supabase_url=os.getenv("SUPABASE_URL"),
+    supabase_key=os.getenv("SUPABASE_KEY")
+)
+
+# Initialize your open client 
+client = Client(api_key=os.getenv('OPENAI_API_KEY'),organization=os.getenv('ORG_ID'))
+
+@app.post("/test/batch_processing_result")
+async def batch_processing_result(request: Request, background_tasks: BackgroundTasks):
+    body = await request.json()
+    batch_id = body.get('batch_job_id')
+    batch_job = client.batches.retrieve(batch_id)
+    # while batch_job.status == 'in_progress':
+    batch_job = client.batches.retrieve(batch_id)
+    print(batch_job.status)
+    # Add the processing task to background tasks
+    if batch_job.status == 'completed':
+        background_tasks.add_task(process_batch_data, batch_id)
+        return {"batch_job_status":'completed'} 
+    
+    # Immediately return success response
+    return {'batch_job_status':'notcompleted'}
+
+
+async def process_batch_data(batch_id: str):
+    try:
+        batch_job = client.batches.retrieve(batch_id)
+        if batch_job.status == 'completed':
+            result_file_id = batch_job.output_file_id
+            result = client.files.content(result_file_id).content
+            json_str = result.decode('utf-8')
+            json_lines = json_str.splitlines()
+            
+            res = []
+            for line in json_lines:
+                if line.strip():
+                    try:
+                        json_dict = json.loads(line)
+                        res.append(json_dict)
+                    except json.JSONDecodeError as e:
+                        print(f"Error decoding JSON on line: {line}\nError: {e}")
+            
+            for resp in res:
+                id = resp.get('custom_id')
+                res_id = id.split('-')[1]
+                output = json.loads(resp.get('response').get('body').get('choices')[0].get('message').get('content'))
+                
+                categories = str(output.get('categories'))
+                summary = str(output.get('summary'))
+                
+                supabase_resp = supabase.table("imdb_dataset").select("Description").eq("imdb_id", res_id).execute()
+                description = supabase_resp.data[0].get('Description')
+                
+                insert_response = (
+                    supabase.table("imdb_outputs")
+                    .insert({
+                        "id": res_id, 
+                        "description": description,
+                        'categories': categories,
+                        'summary': summary
+                    })
+                    .execute()
+                )
+                print(f"Inserted data for ID: {res_id}")
+                
+    except Exception as e:
+        print(f"Error in background processing: {str(e)}")
+        # You might want to log this error or handle it in some way