Spaces:
No application file
No application file
mathysgrapotte commited on
Commit ·
f78c1b6
1
Parent(s): 1a536ab
fix bug for I/O display
Browse files- main.py +33 -18
- tools/meta_yml_tools.py +3 -2
main.py
CHANGED
|
@@ -11,7 +11,7 @@ import threading
|
|
| 11 |
from contextlib import redirect_stdout, redirect_stderr
|
| 12 |
import queue
|
| 13 |
import sys
|
| 14 |
-
from ansi2html import Ansi2HTMLConverter
|
| 15 |
|
| 16 |
# Global log queue for streaming logs to Gradio
|
| 17 |
log_queue = queue.Queue()
|
|
@@ -83,11 +83,13 @@ def extract_format_terms_from_result(result):
|
|
| 83 |
"""Extract EDAM format terms from agent result string"""
|
| 84 |
if isinstance(result, str):
|
| 85 |
# Look for format_XXXX patterns in the result using regex
|
| 86 |
-
format_matches = re.findall(r
|
| 87 |
return format_matches
|
| 88 |
elif isinstance(result, list):
|
| 89 |
# If it's already a list, filter for format terms
|
| 90 |
-
return [
|
|
|
|
|
|
|
| 91 |
return []
|
| 92 |
|
| 93 |
def create_header_html(animation_state="idle"):
|
|
@@ -160,7 +162,8 @@ def create_progress_bar_html(progress, status, current_input, current_count=0, t
|
|
| 160 |
def format_ontology_results_html(results, meta_yml):
|
| 161 |
"""Format the ontology results into a nice HTML display with clickable links"""
|
| 162 |
|
| 163 |
-
if
|
|
|
|
| 164 |
return "<div class='no-results'>No ontology results found.</div>"
|
| 165 |
|
| 166 |
html_content = """
|
|
@@ -171,7 +174,7 @@ def format_ontology_results_html(results, meta_yml):
|
|
| 171 |
</div>
|
| 172 |
"""
|
| 173 |
|
| 174 |
-
# Group inputs by their descriptions from meta_yml
|
| 175 |
input_info = {}
|
| 176 |
for input_channel in meta_yml.get("input", []):
|
| 177 |
for ch_element in input_channel:
|
|
@@ -179,11 +182,19 @@ def format_ontology_results_html(results, meta_yml):
|
|
| 179 |
if value.get("type") == "file":
|
| 180 |
input_info[key] = value.get("description", "No description available")
|
| 181 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
final = {}
|
| 183 |
final.update(results["input"])
|
| 184 |
final.update(results["output"])
|
| 185 |
-
for input_name,
|
| 186 |
-
format_terms
|
| 187 |
description = input_info.get(input_name, "No description available")
|
| 188 |
|
| 189 |
html_content += f"""
|
|
@@ -279,17 +290,19 @@ def run_multi_agent_with_logs(module_name, progress_callback=None):
|
|
| 279 |
if value["type"] == "file":
|
| 280 |
# Update progress BEFORE processing starts for this input
|
| 281 |
if progress_callback:
|
| 282 |
-
progress_callback(int((current_files / total_files) * 100), f"Starting analysis of input: {key}", key, current_files, total_files)
|
| 283 |
|
| 284 |
# This is where the agent runs - logs should be captured automatically
|
| 285 |
result = agent.run(f"You are presentend with a file format for the input {key}, which is a file and is described by the following description: '{value['description']}', search for the best matches out of possible matches in the edam ontology (formated as format_XXXX), and return the answer (a list of ontology classes) in a final_answer call such as final_answer([format_XXXX, format_XXXX, ...])")
|
| 286 |
-
|
|
|
|
|
|
|
| 287 |
|
| 288 |
# Update progress AFTER processing completes for this input
|
| 289 |
current_files += 1
|
| 290 |
progress = int((current_files / total_files) * 100)
|
| 291 |
if progress_callback:
|
| 292 |
-
progress_callback(progress, f"Completed analysis of input: {key}", key, current_files, total_files)
|
| 293 |
|
| 294 |
#outputs
|
| 295 |
for output in meta_yml.get("output", []):
|
|
@@ -299,24 +312,26 @@ def run_multi_agent_with_logs(module_name, progress_callback=None):
|
|
| 299 |
if value["type"] == "file" and element_name != "versions.yml":
|
| 300 |
# Update progress BEFORE processing starts for this output
|
| 301 |
if progress_callback:
|
| 302 |
-
progress_callback(int((current_files / total_files) * 100), f"Starting analysis of output: {key}", key, current_files, total_files)
|
| 303 |
|
| 304 |
# This is where the agent runs - logs should be captured automatically
|
| 305 |
result = agent.run(f"You are presentend with a file format for the output '{element_name}', which is a file and is described by the following description: '{value['description']}', search for the best matches out of possible matches in the edam ontology (formated as format_XXXX), and return the answer (a list of ontology classes) in a final_answer call such as final_answer([format_XXXX, format_XXXX, ...]). The output name {key} can also give you more information.")
|
| 306 |
-
|
|
|
|
|
|
|
| 307 |
|
| 308 |
# Update progress AFTER processing completes for this output
|
| 309 |
current_files += 1
|
| 310 |
progress = int((current_files / total_files) * 100)
|
| 311 |
if progress_callback:
|
| 312 |
-
progress_callback(progress, f"Completed analysis of output: {key}", key, current_files, total_files)
|
| 313 |
|
| 314 |
if progress_callback:
|
| 315 |
-
progress_callback(100, "Analysis complete! Generating results...", "", total_files, total_files)
|
| 316 |
|
| 317 |
### UPDATE META.YML FILE ADDING ONTOLOGIES AND RETURN THE ANSWER ###
|
| 318 |
with open("tmp_meta.yml", "w") as fh:
|
| 319 |
-
updated_meta_yml = update_meta_yml(results["input"], results["output"], meta_yml)
|
| 320 |
yaml.dump(updated_meta_yml, fh)
|
| 321 |
|
| 322 |
except Exception as e:
|
|
@@ -334,9 +349,9 @@ def stream_logs_and_run_agent(module_name):
|
|
| 334 |
|
| 335 |
# Start the agent in a separate thread
|
| 336 |
result_container = {"ontology_output": None, "file_output": None, "error": None}
|
| 337 |
-
progress_container = {"progress": 0, "status": "Initializing...", "current_files": "", "current_count": 0, "total_count": 0}
|
| 338 |
|
| 339 |
-
def progress_callback(progress, status, current_files, current_count=0, total_count=0):
|
| 340 |
progress_container["progress"] = progress
|
| 341 |
progress_container["status"] = status
|
| 342 |
progress_container["current_files"] = current_files
|
|
@@ -361,7 +376,7 @@ def stream_logs_and_run_agent(module_name):
|
|
| 361 |
|
| 362 |
# Stream logs while the agent is running
|
| 363 |
accumulated_logs = ""
|
| 364 |
-
converter =
|
| 365 |
|
| 366 |
while agent_thread.is_alive() or not log_queue.empty():
|
| 367 |
try:
|
|
|
|
| 11 |
from contextlib import redirect_stdout, redirect_stderr
|
| 12 |
import queue
|
| 13 |
import sys
|
| 14 |
+
from ansi2html import Ansi2HTMLConverter
|
| 15 |
|
| 16 |
# Global log queue for streaming logs to Gradio
|
| 17 |
log_queue = queue.Queue()
|
|
|
|
| 83 |
"""Extract EDAM format terms from agent result string"""
|
| 84 |
if isinstance(result, str):
|
| 85 |
# Look for format_XXXX patterns in the result using regex
|
| 86 |
+
format_matches = re.findall(r"format_\d+", result)
|
| 87 |
return format_matches
|
| 88 |
elif isinstance(result, list):
|
| 89 |
# If it's already a list, filter for format terms
|
| 90 |
+
return [
|
| 91 |
+
item for item in result if isinstance(item, str) and item.startswith("format_")
|
| 92 |
+
]
|
| 93 |
return []
|
| 94 |
|
| 95 |
def create_header_html(animation_state="idle"):
|
|
|
|
| 162 |
def format_ontology_results_html(results, meta_yml):
|
| 163 |
"""Format the ontology results into a nice HTML display with clickable links"""
|
| 164 |
|
| 165 |
+
# Check if there are any ontology results in either input or output
|
| 166 |
+
if not results.get("input", {}) and not results.get("output", {}):
|
| 167 |
return "<div class='no-results'>No ontology results found.</div>"
|
| 168 |
|
| 169 |
html_content = """
|
|
|
|
| 174 |
</div>
|
| 175 |
"""
|
| 176 |
|
| 177 |
+
# Group inputs and outputs by their descriptions from meta_yml
|
| 178 |
input_info = {}
|
| 179 |
for input_channel in meta_yml.get("input", []):
|
| 180 |
for ch_element in input_channel:
|
|
|
|
| 182 |
if value.get("type") == "file":
|
| 183 |
input_info[key] = value.get("description", "No description available")
|
| 184 |
|
| 185 |
+
# Also include output descriptions
|
| 186 |
+
for output in meta_yml.get("output", []):
|
| 187 |
+
for key, output_channel in output.items():
|
| 188 |
+
for out_element in output_channel:
|
| 189 |
+
for element_name, value in out_element.items():
|
| 190 |
+
if value.get("type") == "file" and element_name != "versions.yml":
|
| 191 |
+
input_info[key] = value.get("description", "No description available")
|
| 192 |
+
|
| 193 |
final = {}
|
| 194 |
final.update(results["input"])
|
| 195 |
final.update(results["output"])
|
| 196 |
+
for input_name, format_terms in final.items():
|
| 197 |
+
# format_terms is already a list of format terms extracted earlier
|
| 198 |
description = input_info.get(input_name, "No description available")
|
| 199 |
|
| 200 |
html_content += f"""
|
|
|
|
| 290 |
if value["type"] == "file":
|
| 291 |
# Update progress BEFORE processing starts for this input
|
| 292 |
if progress_callback:
|
| 293 |
+
progress_callback(int((current_files / total_files) * 100), f"Starting analysis of input: {key}", key, current_files, total_files, "rotating")
|
| 294 |
|
| 295 |
# This is where the agent runs - logs should be captured automatically
|
| 296 |
result = agent.run(f"You are presentend with a file format for the input {key}, which is a file and is described by the following description: '{value['description']}', search for the best matches out of possible matches in the edam ontology (formated as format_XXXX), and return the answer (a list of ontology classes) in a final_answer call such as final_answer([format_XXXX, format_XXXX, ...])")
|
| 297 |
+
# Extract format terms from the agent result
|
| 298 |
+
format_terms = extract_format_terms_from_result(result)
|
| 299 |
+
results["input"][key] = format_terms
|
| 300 |
|
| 301 |
# Update progress AFTER processing completes for this input
|
| 302 |
current_files += 1
|
| 303 |
progress = int((current_files / total_files) * 100)
|
| 304 |
if progress_callback:
|
| 305 |
+
progress_callback(progress, f"Completed analysis of input: {key}", key, current_files, total_files, "rotating")
|
| 306 |
|
| 307 |
#outputs
|
| 308 |
for output in meta_yml.get("output", []):
|
|
|
|
| 312 |
if value["type"] == "file" and element_name != "versions.yml":
|
| 313 |
# Update progress BEFORE processing starts for this output
|
| 314 |
if progress_callback:
|
| 315 |
+
progress_callback(int((current_files / total_files) * 100), f"Starting analysis of output: {key}", key, current_files, total_files, "rotating")
|
| 316 |
|
| 317 |
# This is where the agent runs - logs should be captured automatically
|
| 318 |
result = agent.run(f"You are presentend with a file format for the output '{element_name}', which is a file and is described by the following description: '{value['description']}', search for the best matches out of possible matches in the edam ontology (formated as format_XXXX), and return the answer (a list of ontology classes) in a final_answer call such as final_answer([format_XXXX, format_XXXX, ...]). The output name {key} can also give you more information.")
|
| 319 |
+
# Extract format terms from the agent result
|
| 320 |
+
format_terms = extract_format_terms_from_result(result)
|
| 321 |
+
results["output"][key] = format_terms
|
| 322 |
|
| 323 |
# Update progress AFTER processing completes for this output
|
| 324 |
current_files += 1
|
| 325 |
progress = int((current_files / total_files) * 100)
|
| 326 |
if progress_callback:
|
| 327 |
+
progress_callback(progress, f"Completed analysis of output: {key}", key, current_files, total_files, "rotating")
|
| 328 |
|
| 329 |
if progress_callback:
|
| 330 |
+
progress_callback(100, "Analysis complete! Generating results...", "", total_files, total_files, "celebrating")
|
| 331 |
|
| 332 |
### UPDATE META.YML FILE ADDING ONTOLOGIES AND RETURN THE ANSWER ###
|
| 333 |
with open("tmp_meta.yml", "w") as fh:
|
| 334 |
+
updated_meta_yml = update_meta_yml(results["input"].copy(), results["output"].copy(), meta_yml)
|
| 335 |
yaml.dump(updated_meta_yml, fh)
|
| 336 |
|
| 337 |
except Exception as e:
|
|
|
|
| 349 |
|
| 350 |
# Start the agent in a separate thread
|
| 351 |
result_container = {"ontology_output": None, "file_output": None, "error": None}
|
| 352 |
+
progress_container = {"progress": 0, "status": "Initializing...", "current_files": "", "current_count": 0, "total_count": 0, "animation_state": "rotating"}
|
| 353 |
|
| 354 |
+
def progress_callback(progress, status, current_files, current_count=0, total_count=0, animation_state="rotating"):
|
| 355 |
progress_container["progress"] = progress
|
| 356 |
progress_container["status"] = status
|
| 357 |
progress_container["current_files"] = current_files
|
|
|
|
| 376 |
|
| 377 |
# Stream logs while the agent is running
|
| 378 |
accumulated_logs = ""
|
| 379 |
+
converter = Ansi2HTMLConverter(dark_bg=True, line_wrap=False)
|
| 380 |
|
| 381 |
while agent_thread.is_alive() or not log_queue.empty():
|
| 382 |
try:
|
tools/meta_yml_tools.py
CHANGED
|
@@ -120,8 +120,9 @@ def update_meta_yml(input_ontologies: dict, output_ontologies: dict, meta_yml:di
|
|
| 120 |
def format_ontology_links(ontology_dict):
|
| 121 |
for key in ontology_dict.keys():
|
| 122 |
updated_list = []
|
| 123 |
-
|
| 124 |
-
|
|
|
|
| 125 |
ontology_dict[key] = updated_list
|
| 126 |
return ontology_dict
|
| 127 |
input_ontologies = format_ontology_links(input_ontologies)
|
|
|
|
| 120 |
def format_ontology_links(ontology_dict):
|
| 121 |
for key in ontology_dict.keys():
|
| 122 |
updated_list = []
|
| 123 |
+
# ontology_dict[key] should be a list of format terms like ["format_1234", "format_5678"]
|
| 124 |
+
for format_term in ontology_dict[key]:
|
| 125 |
+
updated_list.append({"edam": f"http://edamontology.org/{format_term}"})
|
| 126 |
ontology_dict[key] = updated_list
|
| 127 |
return ontology_dict
|
| 128 |
input_ontologies = format_ontology_links(input_ontologies)
|