add missing data workflow

model.py

@@ -1,6 +1,6 @@
 import os
-import json
 import argparse
+import time
 
 from dotenv import load_dotenv
 import anthropic
@@ -10,7 +10,7 @@ from utils import parse_json_garbage
 
 load_dotenv()
 
-def llm( provider, model, system_prompt, user_content):
+def llm( provider, model, system_prompt, user_content, delay:int = 10):
     """Invoke LLM service
     Argument
     --------
@@ -26,6 +26,9 @@ def llm( provider, model, system_prompt, user_content):
     ------
     response: str
     """
+    if delay:
+        time.sleep(delay)
+        
     if provider=='openai':
         client = OpenAI( organization = os.getenv('ORGANIZATION_ID'))
         chat_completion = client.chat.completions.create( 

sheet.py

@@ -165,7 +165,7 @@ def classify_results(
             label = parse_json_garbage(pred_cls)['category']
             labels.append(label)
         except Exception as e:
-            print(f"# CLASSIFICATION error -> evidence: {e}")
+            print(f"# CLASSIFICATION error: e -> {e}, user_content -> {user_content}, evidence: {evidence}")
             labels.append("")
             empty_indices.append(idx)
 
@@ -488,10 +488,58 @@ def split_dataframe( df: pd.DataFrame, n_processes: int = 4) -> list:
     n_per_process = math.ceil(n / n_processes)
     return [ df.iloc[i:i+n_per_process] for i in range(0, n, n_per_process)]
 
+
+def continue_missing(args):
+    """
+    """
+    data = get_leads(args.data_path)
+    n_data = data.shape[0]
+    
+    formatted_results_path = os.path.join( args.output_dir, args.formatted_results_path)
+    formatted_results = pd.read_csv(formatted_results_path)
+    missing_indices = []
+    for i in range(n_data):
+        if i not in formatted_results['index'].unique():
+            print(f"{i} is not found")
+            missing_indices.append(i)
+
+    crawled_results_path = os.path.join( args.output_dir, args.crawled_file_path)
+    crawled_results = joblib.load( open( crawled_results_path, "rb"))
+    crawled_results = crawled_results['crawled_results'].query( f"index in {missing_indices}")
+    print( crawled_results)
+    
+    er = extract_results( crawled_results, classes = args.classes, provider = args.provider, model = args.model)
+    er = er['extracted_results']
+    print(er['category'])
+
+    postprossed_results = postprocess_result( 
+        er, 
+        "/tmp/postprocessed_results.joblib",
+        category2supercategory
+    )
+
+    out_formatted_results = format_output( 
+        postprossed_results, 
+        input_column = 'evidence', 
+        output_column = 'formatted_evidence', 
+        format_func = format_evidence
+    )
+
+    out_formatted_results.to_csv( "/tmp/formatted_results.missing.csv", index=False)
+    formatted_results = pd.concat([formatted_results, out_formatted_results], ignore_index=True)
+    formatted_results.sort_values(by='index', ascending=True, inplace=True)
+    formatted_results.to_csv( "/tmp/formatted_results.csv", index=False)
+
+
 def main(args):
     """
     Argument
         args: argparse
+    Note
+        200 records 
+            crawl: 585.3285548686981
+            extract: 2791.631685256958(delay = 10)
+            classify: 2374.4915606975555(delay = 10)
     """
     crawled_file_path = os.path.join( args.output_dir, args.crawled_file_path)
     extracted_file_path = os.path.join( args.output_dir, args.extracted_file_path)
@@ -501,11 +549,11 @@ def main(args):
     formatted_results_path = os.path.join( args.output_dir, args.formatted_results_path)
 
     ## 讀取資料名單 ##
-    data = get_leads(args.data_path).tail(5)
+    data = get_leads(args.data_path)
     
     ## 進行爬蟲與分析 ##
     crawled_results = crawl_results_mp( data, crawled_file_path, n_processes=args.n_processes)
-    crawled_results = { k:v[-5:] for k,v in crawled_results.items()}
+    # crawled_results = { k:v[-5:] for k,v in crawled_results.items()}
 
     ## 方法 1: 擷取關鍵資訊與分類 ##
     extracted_results = extract_results_mp( 
@@ -596,6 +644,7 @@ if __name__=='__main__':
 
     parser = argparse.ArgumentParser()
     parser.add_argument("--data_path", type=str, default="data/餐廳類型分類.xlsx - 測試清單.csv")
+    parser.add_argument("--task", type=str, default="new", choices = ["new", "continue"], help="new or continue")
     parser.add_argument("--output_dir", type=str, help='output directory')
     parser.add_argument("--classified_file_path", type=str, default="classified_results.joblib")
     parser.add_argument("--extracted_file_path", type=str, default="extracted_results.joblib")
@@ -606,9 +655,16 @@ if __name__=='__main__':
     parser.add_argument("--classes", type=list, default=['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)', '西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)', '西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)', '早餐'])
     parser.add_argument("--backup_classes", type=list, default=['中式', '西式'])
     parser.add_argument("--strategy", type=str, default='patch', choices=['replace', 'patch'])
-    parser.add_argument("--provider", type=str, default='anthropic', choices=['openai', 'anthropic'])
-    parser.add_argument("--model", type=str, default='claude-3-sonnet-20240229', choices=['claude-3-sonnet-20240229', 'claude-3-haiku-20240307', 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview'])
+    parser.add_argument("--provider", type=str, default='openai', choices=['openai', 'anthropic'])
+    parser.add_argument("--model", type=str, default='gpt-4-0125-preview', choices=['claude-3-sonnet-20240229', 'claude-3-haiku-20240307', 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview'])
     parser.add_argument("--n_processes", type=int, default=4)
     args = parser.parse_args()
 
-    main(args)
+    if args.task == 'new':
+        main(args)
+    elif args.task == 'continue':
+        continue_missing(args)
+    else:
+        raise Exception(f"Task {args.task} not implemented")
+    
+

utils.py

@@ -1,9 +1,13 @@
+import re
 import json
 
 def parse_json_garbage(s):
     s = s[next(idx for idx, c in enumerate(s) if c in "{["):]
+    print(s)
+    s = s[:next(idx for idx, c in enumerate(s) if c in "}]")+1]
+    print(s)
     try:
-        return json.loads(s)
+        return json.loads(re.sub("[//#].*","",s,flags=re.MULTILINE))
     except json.JSONDecodeError as e:
-        return json.loads(s[:e.pos])
+        return json.loads(re.sub("[//#].*","",s,flags=re.MULTILINE))