Update app.py

app.py

@@ -17,6 +17,7 @@ from agent import (
     PREFIX,
     COMPRESS_DATA_PROMPT,
     COMPRESS_DATA_PROMPT_SMALL,
+    COMPRESS_DATA_TO_JSON_PROMPT,
     LOG_PROMPT,
     LOG_RESPONSE,
 )
@@ -192,6 +193,47 @@ def run_gpt(
         print(LOG_RESPONSE.format(resp))
     return resp
 
+def compress_data_json(c, instruct, history):
+    seed=random.randint(1,1000000000)
+    
+    print (c)
+    #tot=len(purpose)
+    #print(tot)
+    divr=int(c)/MAX_DATA
+    divi=int(divr)+1 if divr != int(divr) else int(divr)
+    chunk = int(int(c)/divr)
+    print(f'chunk:: {chunk}')
+    print(f'divr:: {divr}')
+    print (f'divi:: {divi}')
+    out = []
+    #out=""
+    s=0
+    e=chunk
+    print(f'e:: {e}')
+    new_history=""
+    #task = f'Compile this data to fulfill the task: {task}, and complete the purpose: {purpose}\n'
+    for z in range(divi):
+        print(f's:e :: {s}:{e}')
+        
+        hist = history[s:e]
+        
+        resp = run_gpt(
+            COMPRESS_DATA_TO_JSON_PROMPT,
+            stop_tokens=["observation:", "task:", "action:", "thought:"],
+            max_tokens=8192,
+            seed=seed,
+            direction=instruct,
+            knowledge=new_history,
+            history=hist,
+        )
+        #out.append(resp)
+        new_history = resp
+        print (resp)
+        #out+=resp
+        e=e+chunk
+        s=s+chunk
+        #yield out
+    return resp
     
 def compress_data(c, instruct, history):
     seed=random.randint(1,1000000000)
@@ -377,6 +419,10 @@ def summarize(inp,history,data=None,files=None,url=None,pdf_url=None,pdf_batch=N
             if i == " " or i=="," or i=="\n":
                 c +=1
         print (f'c2:: {c}')
+        
+        json_final=compress_data_json(c,inp,out)
+        
+        
         rawp = compress_data_og(c,inp,out)
     else:
         rawp = "Provide a valid data source"
@@ -386,7 +432,7 @@ def summarize(inp,history,data=None,files=None,url=None,pdf_url=None,pdf_batch=N
     #task = "complete?"
     history.clear()
     history.append((inp,rawp))
-    yield "", history,error_box,json_out
+    yield "", history,error_box,json_final
 
 #################################
 def clear_fn():