Handle model evaluations staled

app.py

@@ -178,15 +178,30 @@ with tab1:
     # Evaluate the models queued
     if model_predictions_rows:
         models_to_be_evaluated = []
-        model_in_progress = []
+        models_in_progress = []
 
         for row in model_predictions_rows:
             if row["status"] == "queued":
                 models_to_be_evaluated.append(row)
             elif row["status"] == "in_progress":
-                model_in_progress.append(row)
+                models_in_progress.append(row)
 
-        if model_in_progress == []:
+        for model in models_in_progress:
+            # Check if the evaluation is staled for more than a day!
+            timestamp = model["last_updated_timestamp"]
+            if utils.current_seconds_time() - timestamp > 86400:
+                utils.update_model_queue(
+                    repo_id=os.environ["PREDICTIONS_DATASET_NAME"],
+                    model_name=model["model_name"],
+                    commit_id=model["commit_id"],
+                    inference_function=model["inference_function"],
+                    status="queued",
+                )
+                print(f"Model {model['model_name']} is staled for more than a day.")
+                models_to_be_evaluated.append(model)
+                models_in_progress.remove(model)
+
+        if models_in_progress == []:
             for row in models_to_be_evaluated:
                 # Evaluate the model
                 subprocess.Popen(