change submission from directory to a single jsonl file

src/config.py

@@ -8,4 +8,4 @@ DS_SUBMISSIONS_PATH = "submissions"
 DS_RESULTS_PATH = "results"
 
 # leaderboard
-LDB_COLS = ["Submission Name", "Execution (%)", "Consistency (%)", "Final Solution Accuracy (%)", "# of Models submitted"]
+LDB_COLS = ["Submission Name", "Solution Found (%)", "Consistency (%)", "Final Solution Accuracy (%)", "# of Models submitted"]

src/eval.py

@@ -81,84 +81,64 @@ def start_background_evaluation(submission_path):
     thread.start()
     return True
 
+def extract_json_from_code_output(output: str):
+    try:
+        start_index = output.find('{')
+        end_index = output.rfind('}') + 1
+        # Extract the JSON part
+        json_part = output[start_index:end_index]
+        return json.loads(json_part)
+    except json.JSONDecodeError:
+        return None
+
 
-def extract_json_from_string(text_output: str):
+def exec_code(code: str, timeout=10, modelling_language='cpmpy'):
     """
-    Attempts to find and parse the first valid JSON object or array from a string.
-    Handles cases where JSON is preceded or followed by non-JSON text.
+    Execute the given code and return the output
+
+    :param code: The code to execute as a string
+    :param timeout: The maximum time to wait for the code to execute in seconds
+    :param modelling_language: The language to use for execution (cpmpy, minizinc, or-tools)
+    :return: A tuple of (success, output, timeout_occured)
     """
-    idx = 0
-    while idx < len(text_output):
-        # Find the next potential start of a JSON structure
-        start_brace = text_output.find('{', idx)
-        start_bracket = text_output.find('[', idx)
-
-        if start_brace == -1 and start_bracket == -1:
-            # No more '{' or '[' found in the rest of the string
-            return None
-
-        # Determine the actual starting character for this attempt
-        if start_brace != -1 and (start_bracket == -1 or start_brace < start_bracket):
-            json_start_index = start_brace
-        else:
-            json_start_index = start_bracket
 
-        potential_json_segment = text_output[json_start_index:]
+    # create a temp directory to store the temporary file
+    temp_dir_name = "_temp_dir_for_exec_code"
+    temp_dir = os.path.join(os.getcwd(), temp_dir_name)
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # write the code to a temporary file
+    suffix = '.__hidden_py__' if modelling_language == "cpmpy" or modelling_language == "or-tools" else '.mzn'
+    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix=suffix, dir=temp_dir, encoding='utf-8') as temp_file:
+        temp_instance_path = temp_file.name
+        temp_file.write(code)
 
-        try:
-            # Use raw_decode to parse the first valid JSON object from the segment
-            decoder = json.JSONDecoder()
-            json_obj, end_index_in_segment = decoder.raw_decode(potential_json_segment)
-            # Successfully parsed a JSON object
-            return json_obj
-        except json.JSONDecodeError:
-            # This segment (starting at json_start_index) wasn't a valid JSON.
-            # Advance the search index past the character that caused the current attempt.
-            idx = json_start_index + 1
-
-    return None  # No valid JSON found in the entire string
-
-
-def run_instance(instance_path_str: str,
-                 timeout: int = SCRIPT_EXECUTION_TIMEOUT):  # SCRIPT_EXECUTION_TIMEOUT should be defined
-    """Run the instance file and robustly capture the JSON output."""
-    command = [sys.executable, instance_path_str]
-    instance_name = Path(instance_path_str).name
     try:
-        result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, encoding='utf-8',
-                                errors='replace')
-
-        # Check return code first
-        if result.returncode != 0:
-            # Log stderr for debugging if the script itself failed
-            error_message = result.stderr[:500].strip() if result.stderr else "<No stderr>"
-            print(f"  ERROR: Running {instance_name} (Return Code: {result.returncode}): {error_message}", flush=True)
-            return None
-
-        # Attempt to extract JSON from stdout
-        stdout_text = result.stdout
-        if not stdout_text or not stdout_text.strip():
-            print(f"  ERROR: No stdout from {instance_name}.", flush=True)
-            return None
-
-        solution = extract_json_from_string(stdout_text)
-
-        if solution is None:
-            # Be more verbose if JSON extraction fails
-            abbreviated_stdout = stdout_text.replace('\n', '\\n')[:300]  # Show newlines as \n for brevity
-            print(
-                f"  ERROR: Could not extract valid JSON from {instance_name}. Raw stdout (abbreviated): '{abbreviated_stdout}...'",
-                flush=True)
-            return None
-
-        return solution
+        # execute the code
+        if modelling_language == "cpmpy" or modelling_language == "or-tools":
+            command = [sys.executable, temp_instance_path]
+            result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, encoding='utf-8')
+
+            successfully_executed = (result.returncode == 0)
+            output = result.stdout if successfully_executed else result.stderr
+            timeout_occurred = False
+        elif modelling_language == "minizinc":
+            successfully_executed, output, timeout_occurred = exec_code_minizinc(code, timeout)
+        else:
+            raise ValueError(f"MODELLING_LANGUAGE not supported: {modelling_language}")
 
-    except subprocess.TimeoutExpired:
-        print(f"  ERROR: Timeout running {instance_name} (>{timeout}s)", flush=True)
-        return None
+    except subprocess.TimeoutExpired as e:
+        successfully_executed = False
+        output = f"Timeout Error: Execution time exceeded {timeout} seconds"
+        timeout_occurred = True
     except Exception as e:
-        print(f"  ERROR: Unexpected error running {instance_name}: {e}", flush=True)
-        return None
+        successfully_executed = False
+        output = f"Error: {e}"
+        timeout_occurred = False
+
+    os.remove(temp_instance_path)
+
+    return successfully_executed, output, timeout_occurred
 
 
 def add_constraints_as_string(solution):
@@ -238,14 +218,14 @@ def main(
         print(f"  Downloading submission files from '{submission_path_in_dataset}' to '{local_submission_dir}'...",
               flush=True)
         try:
-            # Download the relevant submission files
-            snapshot_download(
+            # Download the relevant submission file
+            hf_hub_download(
                 repo_id=user_dataset_repo_id,
                 repo_type="dataset",
                 local_dir=local_submission_dir,
-                allow_patterns=[f"{submission_path_in_dataset}/*"],
+                filename=f"{submission_path_in_dataset}/submission.jsonl",
             )
-            print(f"  Downloaded submission files successfully.", flush=True)
+            print(f"  Downloaded submission file successfully.", flush=True)
 
         except Exception as e_download:
             print(f"  CRITICAL ERROR - Failed to download submission files: {e_download}", flush=True)
@@ -269,6 +249,18 @@ def main(
             # (Attempt to upload error summary)
             return 1
 
+        # load generated models from jsonl to memory
+        print(f"  Loading generated models from '{local_submission_dir}'...", flush=True)
+        submitted_models = []
+        with open(os.path.join(local_submission_dir, submission_path_in_dataset, "submission.jsonl"), "r", encoding="utf-8") as f:
+            for line in f:
+                try:
+                    json_obj = json.loads(line)
+                    submitted_models.append(json_obj)
+                except json.JSONDecodeError as e:
+                    print(f"  ERROR: Failed to parse JSON object from line: {line}. Error: {e}", flush=True)
+        print(f"  Loaded {len(submitted_models)} generated models.", flush=True)
+
         # Statistics
         total_submitted_models = 0
         models_ran_successfully = 0
@@ -285,24 +277,35 @@ def main(
             summary_f.write("-" * 30 + "\n")
 
             # Iterate through downloaded submitted models
-            submitted_model_files = list((local_submission_dir / submission_path_in_dataset).rglob('*.py'))
-            if not submitted_model_files:
-                summary_f.write("No .py model files found in downloaded submission.\n")
-                print("  No .py model files found in downloaded submission.", flush=True)
+            for submitted_model in submitted_models:
+                curr_model = submitted_model[GT_MODEL_CODE_COLUMN]
 
-            for model_file_path in submitted_model_files:
                 total_submitted_models += 1
-                problem_name = model_file_path.stem
-                print(f"\n  Processing downloaded model: {model_file_path.name}", flush=True)
-                summary_f.write(f"\n--- Model: {model_file_path.name} ---\n")
+                problem_name = submitted_model[GT_PROBLEM_NAME_COLUMN]
+                print(f"\n  Processing downloaded model: {problem_name}", flush=True)
+                summary_f.write(f"\n--- Model: {problem_name} ---\n")
 
                 summary_f.write("    1. Running submitted model...\n")
-                generated_solution = run_instance(str(model_file_path))
+
+                succ_exec, output, timeout_occurred = exec_code(curr_model, timeout=SCRIPT_EXECUTION_TIMEOUT)
+
+                if timeout_occurred:
+                    summary_f.write(f"      - TIMEOUT: Execution time exceeded {SCRIPT_EXECUTION_TIMEOUT} seconds.\n")
+                    continue
+                if not succ_exec:
+                    summary_f.write(f"      - FAILED: Execution failed with error: {output}\n")
+                    continue
+                if output is None or not output.strip():
+                    summary_f.write(f"      - FAILED: No output from execution.\n")
+                    continue
+                # Attempt to extract JSON from stdout
+                generated_solution = extract_json_from_code_output(output)
                 if generated_solution is None:
-                    summary_f.write("      - FAILED to run or get valid JSON solution from submitted model.\n")
+                    summary_f.write(f"      - FAILED: Could not extract JSON solution from output: {output}\n")
                     continue
+
                 models_ran_successfully += 1
-                summary_f.write(f"      - SUCCESS: Got solution.\n")
+                summary_f.write(f"      - SUCCESS: Got solution: {generated_solution}\n")
 
                 summary_f.write(f"    2. Checking against ground-truth for '{problem_name}'...\n")
                 if problem_name not in ground_truth_models:
@@ -328,7 +331,6 @@ def main(
                     os.unlink(tmp_file_path_str)
 
                     gt_stdout = gt_check_result.stdout
-                    # ... (parse EVAL_OUTPUT tags for consistency and objective)
                     if "SUCCESS: Model is consistent" in gt_stdout:
                         summary_f.write("      - CONSISTENCY: PASSED\n")
                         consistency_checks_passed += 1

src/hf_utils.py

@@ -90,23 +90,22 @@ def load_leaderboard_data():
     return pd.DataFrame(leaderboard_entries)
 
 
-def upload_submission(uploaded_files, dir_name):
+def upload_submission(uploaded_file, dir_name):
     """Upload submission to Hugging Face Dataset."""
     if not HF_API:
         return False, "Hugging Face API not initialized"
-        
+
     try:
         submission_path = f"{DS_SUBMISSIONS_PATH}/{dir_name}"
 
-        for file in uploaded_files:
-            file_name = os.path.basename(file.name)
-            HF_API.upload_file(
-                path_or_fileobj=file,
-                path_in_repo=f"{submission_path}/{file_name}",
-                repo_id=DATASET_REPO_ID,
-                repo_type="dataset",
-                commit_message=f"Upload submission: {dir_name}"
-            )
+        # file_name = os.path.basename(uploaded_file.name)
+        HF_API.upload_file(
+            path_or_fileobj=uploaded_file,
+            path_in_repo=f"{submission_path}/submission.jsonl",
+            repo_id=DATASET_REPO_ID,
+            repo_type="dataset",
+            commit_message=f"Upload submission: {dir_name}"
+        )
         
         return True, submission_path
     except Exception as e:

src/ui.py

@@ -1,3 +1,5 @@
+import json
+
 import gradio as gr
 from pathlib import Path
 
@@ -5,10 +7,10 @@ from src.hf_utils import load_leaderboard_data, upload_submission, check_name_ex
 from src.eval import start_background_evaluation
 
 
-def handle_upload(submission_name, uploaded_files, progress=gr.Progress()):
+def handle_upload(submission_name, uploaded_file, progress=gr.Progress()):
     """Handle file upload and start evaluation."""
-    if not uploaded_files or len(uploaded_files) == 0:
-        return "No directory uploaded or directory is empty, please try again."
+    if not uploaded_file:
+        return "No file uploaded. Please upload a valid submission file."
 
     # normalize the submission name
     submission_name = submission_name.strip().replace(" ", "_").lower()
@@ -26,8 +28,22 @@ def handle_upload(submission_name, uploaded_files, progress=gr.Progress()):
     try:
         progress(0.3, "Uploading to Hugging Face...")
 
-        # Upload the directory to Hugging Face
-        success, result = upload_submission(uploaded_files, submission_name)
+        # Check if the file is a valid JSONL file
+        if not uploaded_file.name.endswith(".jsonl"):
+            return "Invalid file format. Please upload a .jsonl file."
+
+        # Check that the keys in the JSONL file are correct ('id' and 'model')
+        with open(uploaded_file.name, "r") as file:
+            found_one = False
+            for line in file:
+                found_one = True
+                json_object = json.loads(line)
+                if not all(key in json_object for key in ["id", "model"]):
+                    return "Invalid content. Each line must contain 'id' and 'model' keys."
+            if not found_one:
+                return "Empty file. Please upload a valid JSONL file."
+
+        success, result = upload_submission(uploaded_file, submission_name)
         if not success:
             return f"Upload failed: {result}"
 
@@ -58,7 +74,7 @@ def create_ui():
                     interactive=True,
                     info="This name will appear on the leaderboard"
                 )
-                upload_button = gr.UploadButton("Click to Upload Directory", file_count="directory")
+                upload_button = gr.UploadButton("Click to Upload Submission", file_count="single")
                 status_box = gr.Textbox(label="Status", interactive=False)
 
             with gr.Column(scale=3):