minor

app.py

@@ -170,41 +170,6 @@ with demo:
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT_OPTION1, elem_classes="markdown-text")
 
-                # with gr.Column():
-                #     with gr.Accordion(
-                #         f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             finished_eval_table = gr.components.Dataframe(
-                #                 value=finished_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
-                #     with gr.Accordion(
-                #         f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             running_eval_table = gr.components.Dataframe(
-                #                 value=running_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
-
-                #     with gr.Accordion(
-                #         f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             pending_eval_table = gr.components.Dataframe(
-                #                 value=pending_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
@@ -212,15 +177,6 @@ with demo:
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
                     precision = gr.Dropdown(
                         choices=[i.value.name for i in Precision if i != Precision.Unknown],
                         label="Precision",
@@ -228,14 +184,50 @@ with demo:
                         value="float16",
                         interactive=True,
                     )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
+                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                    vllm_version_type = gr.Dropdown(
+                        choices=["v0.6.0", "v0.6.1.post1", "v0.6.1.post2", "v0.6.1",  "v0.6.2", "v0.6.3.post1", "v0.6.4", "v0.6.4.post1", "v0.6.5"],
+                        label="Model type",
                         multiselect=False,
-                        value="Original",
+                        value=None,
                         interactive=True,
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                with gr.Column():
+                    temperature_textbox = gr.Textbox(label="Sampling Temperature (default: 1.0)")
+                    top_p_textbox = gr.Textbox(label="Top-p (default: 1.0)")
+                    top_k_textbox = gr.Textbox(label="Top-k (default: -1)")
+                    presence_penalty_textbox = gr.Textbox(label="Presence penalty (default: 0.0)")
+                    frequency_penalty_textbox = gr.Textbox(label="Repetition penalty (default: 0.0)")
+                    repetition_penalty_textbox = gr.Textbox(label="Repetition penalty (default: 1.0)")
+                    
+            # with gr.Row():
+            #     with gr.Column():
+            #         model_name_textbox = gr.Textbox(label="Model name")
+            #         revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+            #         model_type = gr.Dropdown(
+            #             choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+            #             label="Model type",
+            #             multiselect=False,
+            #             value=None,
+            #             interactive=True,
+            #         )
+
+            #     with gr.Column():
+            #         precision = gr.Dropdown(
+            #             choices=[i.value.name for i in Precision if i != Precision.Unknown],
+            #             label="Precision",
+            #             multiselect=False,
+            #             value="float16",
+            #             interactive=True,
+            #         )
+            #         weight_type = gr.Dropdown(
+            #             choices=[i.value.name for i in WeightType],
+            #             label="Weights type",
+            #             multiselect=False,
+            #             value="Original",
+            #             interactive=True,
+            #         )
+            #         base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
 
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
@@ -246,8 +238,8 @@ with demo:
                     base_model_name_textbox,
                     revision_name_textbox,
                     precision,
-                    weight_type,
-                    model_type,
+                    # weight_type,
+                    # model_type,
                 ],
                 submission_result,
             )

src/about.py

@@ -65,12 +65,18 @@ If everything is done, check you can launch the EleutherAIHarness on your model
 """
 
 EVALUATION_QUEUE_TEXT_OPTION1 = """
-## (Option 1) Direct Submission
-We encourage to fill "✉️✨ Submit your model here!" section below to submit your model.
+## (Option 1) Submit HF model where vLLM inference is available
+Fill the information including model name, vLLM version, sampling hyperparameters.
 """
-EVALUATION_QUEUE_TEXT_OPTION4 = """
-## (Option 4) Pull Request
-If Option 1-3 is unavailable, make [PR](https://huggingface.co/spaces/coms1580/test_space/discussions?status=open&type=pull_request&sort=recently-created&new_pr=true) with [ADD_MODEL] prefix with contents as follows:
+
+EVALUATION_QUEUE_TEXT_OPTION2 = """
+## (Option 2) Submit HF model where vLLM inference is unavailable
+Fill the information same with Option 1 and code snippets of model loading, inference, and termination.
+"""
+
+EVALUATION_QUEUE_TEXT_OPTION3 = """
+## (Option 3) Pull Request
+If Option 1 & 2 is unavailable, make [PR](https://huggingface.co/spaces/coms1580/test_space/discussions?status=open&type=pull_request&sort=recently-created&new_pr=true) with [ADD_MODEL] prefix with contents as follows:
 
 ```
 ### Open-weight models:

src/submission/submit.py

@@ -19,8 +19,8 @@ def add_new_eval(
     base_model: str,
     revision: str,
     precision: str,
-    weight_type: str,
-    model_type: str,
+    # weight_type: str,
+    # model_type: str,
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
@@ -36,23 +36,23 @@ def add_new_eval(
     precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
+    # if model_type is None or model_type == "":
+    #     return styled_error("Please select a model type.")
 
     # Does the model actually exist?
     if revision == "":
         revision = "main"
 
     # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
+    # if weight_type in ["Delta", "Adapter"]:
+    #     base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+    #     if not base_model_on_hub:
+    #         return styled_error(f'Base model "{base_model}" {error}')
 
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
+    # if not weight_type == "Adapter":
+    #     model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+    #     if not model_on_hub:
+    #         return styled_error(f'Model "{model}" {error}')
 
     # Is the model info correctly filled?
     try:
@@ -80,10 +80,10 @@ def add_new_eval(
         "base_model": base_model,
         "revision": revision,
         "precision": precision,
-        "weight_type": weight_type,
+        # "weight_type": weight_type,
         "status": "PENDING",
         "submitted_time": current_time,
-        "model_type": model_type,
+        # "model_type": model_type,
         "likes": model_info.likes,
         "params": model_size,
         "license": license,
@@ -97,7 +97,8 @@ def add_new_eval(
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    # out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))