Upload folder using huggingface_hub

.gitignore

@@ -10,4 +10,5 @@ wheels/
 .venv
 .env
 data/jg/
-output/
+output/
+data/jg_2/

graph.py

@@ -88,7 +88,7 @@ class SheamiState(TypedDict):
     standardized_reports: List[StandardizedReport]
     trends_json: dict
     interpreted_report: str
-
+    current_index: int
 
 import re
 
@@ -152,34 +152,46 @@ def fn_init_node(state: SheamiState):
     state["standardized_reports"] = []
     state["trends_json"] = {}
     state["interpreted_report"] = ""
+    state["current_index"] = -1
+    return state
+
+
+async def fn_increment_index_node(state: SheamiState):
+    state["current_index"] += 1
     return state
 
+async def fn_standardizer_node_one(state: SheamiState):
+    idx = state["current_index"]
+    report = state["uploaded_reports"][idx]
+
+    logger.info("%s| Standardizing report %s", state["thread_id"], report.report_file_name)
+    state["messages"].append(f"Standardizing report: {report.report_file_name}")
 
-async def fn_standardizer_node(state: SheamiState):
-    logger.info("%s| Standardizing reports: started", state["thread_id"])
-    state["messages"].append("Standardizing reports: started")
     llm_structured = llm.with_structured_output(StandardizedReport)
-    for idx, report in enumerate(state["uploaded_reports"]):
-        logger.info("%s| Standardizing report %s", state["thread_id"], report.report_file_name)
-        state["messages"].append(f"Standardizing report: {report.report_file_name}")
-        messages = [
-            SystemMessage(content="Standardize this medical report into the schema."),
-            # SystemMessage(
-            #     content="Populate the `inferred_range` field as 'low', 'normal', or 'high' by comparing the result value with the reference range. If both min and max are missing, set 'normal' unless the value is clearly out of usual medical ranges."
-            # ),
-            HumanMessage(content=report.report_contents),
-        ]
-        result: StandardizedReport = llm_structured.invoke(messages)
-        state["standardized_reports"].append(result)
-        # save to disk
-        with open(
-            os.path.join(SheamiConfig.get_output_dir(state["thread_id"]), f"report_{idx}.json"), "w"
-        ) as f:
-            f.write(result.model_dump_json(indent=2))
-    logger.info("%s| Standardizing reports: finished", state["thread_id"])
-    state["messages"].append("Standardizing reports: finished")
+    messages = [
+        SystemMessage(content="Standardize this medical report into the schema."),
+        HumanMessage(content=report.report_contents),
+    ]
+    result: StandardizedReport = await llm_structured.ainvoke(messages)
+
+    state["standardized_reports"].append(result)
+
+    with open(
+        os.path.join(SheamiConfig.get_output_dir(state["thread_id"]), f"report_{idx}.json"), "w"
+    ) as f:
+        f.write(result.model_dump_json(indent=2))
+
     return state
 
+# edge
+def fn_check_if_report_available_to_process(state: SheamiState) -> str:
+    if state["current_index"] < len(state["uploaded_reports"]):
+        report = state["uploaded_reports"][state["current_index"]]
+        state["messages"].append(f"Initiating report standardization for: {report.report_file_name}")
+        return "continue"
+    else:
+        state["messages"].append("Standardizing reports: finished")
+        return "done"
 
 async def fn_testname_standardizer_node(state: SheamiState):
     logger.info("%s| Standardizing Test Names: started", state["thread_id"])
@@ -470,7 +482,8 @@ def create_graph(thread_id : str):
     memory = InMemorySaver()
     workflow = StateGraph(SheamiState)
     workflow.add_node("init", fn_init_node)
-    workflow.add_node("standardizer", fn_standardizer_node)
+    workflow.add_node("standardizer_one", fn_standardizer_node_one)
+    workflow.add_node("increment_index", fn_increment_index_node)
     workflow.add_node("testname_standardizer", fn_testname_standardizer_node)
     workflow.add_node("unit_normalizer", fn_unit_normalizer_node)
     workflow.add_node("trends", fn_trends_aggregator_node)
@@ -485,8 +498,19 @@ def create_graph(thread_id : str):
 
     workflow.add_edge(START, "init")
     workflow.add_edge("init", "standardizer_notifier")
-    workflow.add_edge("standardizer_notifier","standardizer")
-    workflow.add_edge("standardizer", "testname_standardizer_notifier")
+    workflow.add_edge("standardizer_notifier","increment_index")
+
+    # loop back if continue
+    workflow.add_conditional_edges(
+        "increment_index",
+        fn_check_if_report_available_to_process,
+        {
+            "continue": "standardizer_one",
+            "done": "testname_standardizer_notifier",
+        }
+    )
+    workflow.add_edge("standardizer_one", "increment_index")
+
     workflow.add_edge("testname_standardizer_notifier","testname_standardizer")
     workflow.add_edge("testname_standardizer", "unit_normalizer_notifier")
     workflow.add_edge("unit_normalizer_notifier", "unit_normalizer")