Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+.gradio/flagged/Upload[[:space:]]Excel[[:space:]]File/2aeafc769523b0a05452/Quick_and_rough_list_for_Ahsan_initial_trials[[:space:]]-[[:space:]]10th[[:space:]]Dec[[:space:]]2024.xlsx filter=lfs diff=lfs merge=lfs -text
+.gradio/flagged/Upload[[:space:]]Excel[[:space:]]File/6c7338224e14819dfdc1/Quick_and_rough_list_for_Ahsan_initial_trials[[:space:]]-[[:space:]]10th[[:space:]]Dec[[:space:]]2024.xlsx filter=lfs diff=lfs merge=lfs -text
+.gradio/flagged/Upload[[:space:]]Excel[[:space:]]File/fc94c399c41af885dce4/Quick_and_rough_list_for_Ahsan_initial_trials[[:space:]]-[[:space:]]10th[[:space:]]Dec[[:space:]]2024.xlsx filter=lfs diff=lfs merge=lfs -text
+data/Quick_and_rough_list_for_Ahsan_initial_trials[[:space:]]-[[:space:]]10th[[:space:]]Dec[[:space:]]2024.xlsx filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,7 @@
+.env
+__pycache__
+llama3.2-vision.ipynb
+.DS_Store
+data/*
+__pycache__
+download_app.py

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

.gradio/flagged/Upload Excel File/2aeafc769523b0a05452/Quick_and_rough_list_for_Ahsan_initial_trials - 10th Dec 2024.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:526098f685db9b7e7223b763e0efa72069f70cff7b9f0a0bb9f334a7fb91fa4e
+size 139903

.gradio/flagged/Upload Excel File/6c7338224e14819dfdc1/Quick_and_rough_list_for_Ahsan_initial_trials - 10th Dec 2024.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0e322cb97a467369a9f86558bf27d9d1db18e48e869c6499af8c8bc76124ea7
+size 122490

.gradio/flagged/Upload Excel File/fc94c399c41af885dce4/Quick_and_rough_list_for_Ahsan_initial_trials - 10th Dec 2024.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:526098f685db9b7e7223b763e0efa72069f70cff7b9f0a0bb9f334a7fb91fa4e
+size 139903

.gradio/flagged/dataset1.csv

@@ -0,0 +1,60 @@
+Upload Excel File,Enter Prompt,Evaluation Metrics,Download Processed Excel,timestamp
+.gradio/flagged/Upload Excel File/2aeafc769523b0a05452/Quick_and_rough_list_for_Ahsan_initial_trials - 10th Dec 2024.xlsx,"You are a teacher of philosophy of science. 
+I want you to assess whether the use of the expressions ""to prove"", ""proving"", ""proof"", ""proved"", in the provided passages, 
+is appropriate, accurate, and logically consistent from the point of view of the rigorous mathematical meaning of the concept of 'proof'. 
+
+Consider statements of the type ""X cannot be proved"" or ""there is no proof for X"" as appropriate even when they are applied to something 
+for which the concept of 'proof' is not appropriate. Consider questions involving the concept of 'proof' as appropriate even when they are 
+applied to something for which the concept of 'proof' is not appropriate. You need to respond with a ""+1"" if the use of those 
+expressions is appropriate, accurate, and logically consistent and with a ""-1"" otherwise. Proving something wrong is also acceptable.","Error: Target is multiclass but average='binary'. Please choose another average setting, one of [None, 'micro', 'macro', 'weighted'].",,2025-01-17 07:49:32.632581
+.gradio/flagged/Upload Excel File/fc94c399c41af885dce4/Quick_and_rough_list_for_Ahsan_initial_trials - 10th Dec 2024.xlsx,"You are a teacher of philosophy of science. 
+I want you to assess whether the use of the expressions 'to prove', 'proving', 'proof', 'proved', in the provided passages, 
+is appropriate, accurate, and logically consistent from the point of view of the rigorous mathematical meaning of the concept of 'proof'. 
+
+Consider statements of the type 'X cannot be proved' or 'there is no proof for X' as appropriate even when they are applied to something 
+for which the concept of 'proof' is not appropriate. Consider questions involving the concept of 'proof' as appropriate even when they are 
+applied to something for which the concept of 'proof' is not appropriate. You need to respond with a '+1' if the use of those 
+expressions is appropriate, accurate, and logically consistent and with a '-1' otherwise. Proving something wrong is also acceptable.","'
+        <div style=""font-family: Arial, sans-serif;"">
+            <h3 style=""color: #2b7a78;"">Evaluation Metrics</h3>
+            <p><strong style=""color: #d9534f;"">Accuracy:</strong> 
+            <span style=""font-size: 1.2em; font-weight: bold; color: #5cb85c;"">0.33</span></p>
+            <p><strong>Precision:</strong> N/A</p>
+            <p><strong>Recall:</strong> N/A</p>
+            <p><strong>F1 Score:</strong> N/A</p>
+            <p><strong>Confusion Matrix:</strong><br><table class=""dataframe table table-bordered table-striped"">
+  <thead>
+    <tr style=""text-align: right;"">
+      <th></th>
+      <th>Predicted -1.0</th>
+      <th>Predicted 0.0</th>
+      <th>Predicted 1.0</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th>Class -1.0</th>
+      <td>3</td>
+      <td>0</td>
+      <td>0</td>
+    </tr>
+    <tr>
+      <th>Class 0.0</th>
+      <td>2</td>
+      <td>0</td>
+      <td>0</td>
+    </tr>
+    <tr>
+      <th>Class 1.0</th>
+      <td>4</td>
+      <td>0</td>
+      <td>0</td>
+    </tr>
+  </tbody>
+</table></p>
+        </div>
+        ",.gradio/flagged/Download Processed Excel/9824a6c316a560c7dada/20250117_103737_processed_output.xlsx,2025-01-17 10:41:32.033493
+.gradio/flagged/Upload Excel File/6c7338224e14819dfdc1/Quick_and_rough_list_for_Ahsan_initial_trials - 10th Dec 2024.xlsx,"You are a teacher of philosophy of science. 
+I want you to assess whether the use of the expressions 'to prove', 'proving', 'proof', 'proved', in the provided passages, is appropriate, accurate, and logically consistent from the point of view of the rigorous mathematical meaning of the concept of 'proof'. 
+
+Consider statements of the type 'X cannot be proved' or 'there is no proof for X' as appropriate even when they are applied to something for which the concept of 'proof' is not appropriate. Consider questions involving the concept of 'proof' as appropriate even when they are applied to something for which the concept of 'proof' is not appropriate. You need to respond with a '+1' if the use of those expressions is appropriate, accurate, and logically consistent and with a '-1' otherwise. Proving something wrong is also acceptable.",,,2025-01-18 10:29:24.783349

README.md

@@ -1,12 +1,6 @@
 ---
-title: Kwik Ai
-emoji: 💻
-colorFrom: red
-colorTo: green
-sdk: gradio
-sdk_version: 5.29.0
+title: kwik-ai
 app_file: app.py
-pinned: false
+sdk: gradio
+sdk_version: 5.12.0
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,279 @@
+from dotenv import load_dotenv
+load_dotenv()
+import os
+import json
+import glob
+import gradio as gr
+import pandas as pd
+from datetime import datetime
+from functions import evaluate_dataframe, run_openai_inference
+import openpyxl
+from openpyxl.styles import Alignment
+import traceback
+
+# Ensure the output directory exists
+OUTPUT_DIR = "./data/outputs"
+if not os.path.exists(OUTPUT_DIR):
+    os.makedirs(OUTPUT_DIR)
+
+# Functions for Analysis Interface
+def process_dataframe(df, prompt: str, model: str, max_iterations: int = 5):
+    print("Starting process_dataframe function...")
+    df['passage'] = (
+        df['LeftContext'].astype(str) +
+        " <expression>" + df['Keyword'].astype(str) + "</expression> " +
+        df['RightContext'].astype(str)
+    )
+    tasks = []
+    indices = []
+    print(f"Iterating over rows in random order...")
+    for idx, row in df.sample(frac=1.0).iterrows():
+        if len(tasks) >= max_iterations:
+            print(f"Max iterations reached: {max_iterations}")
+            break
+        if pd.isna(row['Category']):
+            print(f"Skipping row {idx} due to missing 'Category'")
+            continue
+        indices.append(idx)
+        print(f"Scheduling task for row {idx} with passage: {row['passage']}")
+        tasks.append(run_openai_inference(prompt, row['passage'], model))
+    print(f"Running inference for {len(tasks)} tasks...")
+    results = [task for task in tasks]
+    print(f"Assigning results to corresponding rows...")
+    for i, idx in enumerate(indices):
+        score, reason = results[i]
+        print(f"Row {idx}: Assigned score: {score}, reason: {reason}")
+        df.at[idx, 'Prediction'] = score
+        df.at[idx, 'Prediction Reason'] = reason
+    print(f"Dropping rows with missing predictions...")
+    df_out = df.dropna(subset=['Prediction'])
+    print("Finished processing dataframe.")
+    return df_out
+
+def process_excel(file, prompt, model, slider_value):
+    try:
+        print("Reading Excel file...")
+        df = pd.read_excel(file.name)
+        print(f"Excel file read successfully. DataFrame shape: {df.shape}")
+        print("Processing DataFrame...")
+        df_out = process_dataframe(df, prompt, model, max_iterations=slider_value)
+        print(f"DataFrame processed. Output shape: {df_out.shape}")
+        print("Evaluating DataFrame...")
+        evaluation_results = evaluate_dataframe(df_out['Category'], df_out['Prediction'])
+        print(f"Evaluation results: {evaluation_results}")
+        print("Generating file paths and timestamps...")
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        output_file = os.path.join(OUTPUT_DIR, f"{timestamp}_processed_output.xlsx")
+        metadata_file = os.path.join(OUTPUT_DIR, f"{timestamp}_metadata.json")
+        print(f"Output file path: {output_file}")
+        print(f"Metadata file path: {metadata_file}")
+        print("Writing Excel file with multiple sheets...")
+        with pd.ExcelWriter(output_file, engine='openpyxl') as writer:
+            df.to_excel(writer, sheet_name='Data', index=False)
+            inputs_df = pd.DataFrame({
+                'Parameter': ['Prompt', 'Model', 'Samples Processed'],
+                'Value': [prompt, model, slider_value]
+            })
+            inputs_df.to_excel(writer, sheet_name='Inputs', index=False)
+            outputs_df = pd.DataFrame({
+                'Metric': ['Accuracy', 'Precision', 'Recall', 'F1 Score', 'Samples Classified'],
+                'Value': [
+                    evaluation_results.get('accuracy'),
+                    evaluation_results.get('precision'),
+                    evaluation_results.get('recall'),
+                    evaluation_results.get('f1'),
+                    df_out.shape[0]
+                ]
+            })
+            outputs_df.to_excel(writer, sheet_name='Outputs', index=False)
+        print(f"Excel file written to: {output_file}")
+        print("Applying formatting to Excel file...")
+        wb = openpyxl.load_workbook(output_file)
+        for sheet in wb.worksheets:
+            for row in sheet.iter_rows():
+                for cell in row:
+                    cell.alignment = Alignment(wrap_text=True)
+            for row_idx in range(1, sheet.max_row + 1):
+                sheet.row_dimensions[row_idx].height = 75
+        wb.save(output_file)
+        print(f"Formatting applied and file saved: {output_file}")
+        print("Preparing evaluation results for display...")
+        accuracy = f"{evaluation_results.get('accuracy'):.2f}" if evaluation_results.get('accuracy') is not None else "N/A"
+        precision = f"{evaluation_results.get('precision'):.2f}" if evaluation_results.get('precision') is not None else "N/A"
+        recall = f"{evaluation_results.get('recall'):.2f}" if evaluation_results.get('recall') is not None else "N/A"
+        f1_score = f"{evaluation_results.get('f1'):.2f}" if evaluation_results.get('f1') is not None else "N/A"
+        conf_matrix = evaluation_results.get('conf_matrix', 'N/A')
+        eval_display = f"""
+        <div style="font-family: Arial, sans-serif;">
+            <h3 style="color: #2b7a78;">Evaluation Metrics</h3>
+            <p><strong style="color: #d9534f;">Accuracy:</strong> 
+            <span style="font-size: 1.2em; font-weight: bold; color: #5cb85c;">{accuracy}</span></p>
+            <p><strong>Precision:</strong> {precision}</p>
+            <p><strong>Recall:</strong> {recall}</p>
+            <p><strong>F1 Score:</strong> {f1_score}</p>
+            <p><strong>Confusion Matrix:</strong><br>{conf_matrix}</p>
+        </div>
+        """
+        print(f"Evaluation results prepared for display.")
+        print("Saving run metadata...")
+        history_data = {
+            'date_of_run': timestamp,
+            'input_filename': os.path.basename(file.name),
+            'output_filename': os.path.basename(output_file),
+            'prompt': prompt,
+            'model': model,
+            'samples_requested': slider_value,
+            'samples_classified': df_out.shape[0],
+            'accuracy': evaluation_results.get('accuracy')
+        }
+        with open(metadata_file, "w") as f:
+            json.dump(history_data, f)
+        print(f"Run metadata saved to: {metadata_file}")
+        return eval_display, output_file
+    except Exception as e:
+        error_message = f"<div style='color: red;'>Error: {str(e)}</div>"
+        traceback.print_exc()
+        return error_message, None
+
+def load_history():
+    history_entries = []
+    for meta_file in sorted(glob.glob(os.path.join(OUTPUT_DIR, "*_metadata.json")), reverse=True):
+        try:
+            with open(meta_file, "r") as f:
+                data = json.load(f)
+            excel_filename = data.get('output_filename', '')
+            download_link = f"<a href='{os.path.join(OUTPUT_DIR, excel_filename)}' download>{excel_filename}</a>"
+            history_entries.append({
+                "Date": data.get("date_of_run", ""),
+                "Input Filename": data.get("input_filename", ""),
+                "Output Filename": excel_filename,
+                "Prompt": data.get("prompt", ""),
+                "Model": data.get("model", ""),
+                "Samples Requested": data.get("samples_requested", ""),
+                "Samples Classified": data.get("samples_classified", ""),
+                "Accuracy": f"{data.get('accuracy'):.2f}" if data.get('accuracy') is not None else "N/A",
+                "Download": download_link
+            })
+        except Exception as ex:
+            print(f"Error loading history from {meta_file}: {ex}")
+    if history_entries:
+        table_html = """
+        <style>
+            .history-table {
+                border-collapse: collapse;
+                width: 100%;
+                font-family: Arial, sans-serif;
+            }
+            .history-table th, .history-table td {
+                padding: 8px;
+                text-align: left;
+                border: 1px solid #ddd;
+            }
+            .history-table th.prompt-col, .history-table td.prompt-col {
+                min-width: 300px;
+            }
+            .history-table td span {
+                cursor: help;
+            }
+        </style>
+        <table class='history-table'>
+        <tr>
+        """
+        for col in history_entries[0].keys():
+            if col == "Prompt":
+                table_html += f"<th class='prompt-col'>{col}</th>"
+            else:
+                table_html += f"<th>{col}</th>"
+        table_html += "</tr>"
+        for entry in history_entries:
+            table_html += "<tr>"
+            for key, value in entry.items():
+                if key == "Prompt":
+                    full_prompt = value
+                    clipped = full_prompt if len(full_prompt) <= 100 else full_prompt[:100] + "..."
+                    table_html += f"<td class='prompt-col'><span title='{full_prompt}'>{clipped}</span></td>"
+                else:
+                    table_html += f"<td>{value}</td>"
+            table_html += "</tr>"
+        table_html += "</table>"
+    else:
+        table_html = "<p>No history available.</p>"
+    return table_html
+
+# Functions for Other Interface
+def classify_text(text, model):
+    """Placeholder function for text classification."""
+    return f"Classified as: {model} - {text}"
+
+# Interface Creation Functions
+def create_analysis_interface():
+    default_prompt = (
+        "You are a teacher of philosophy of science. \n"
+        "I want you to assess whether the use of the expressions 'to prove', 'proving', 'proof', 'proved', in the provided passages, \n"
+        "is appropriate, accurate, and logically consistent from the point of view of the rigorous mathematical meaning of the concept of 'proof'. \n\n"
+        "Consider statements of the type 'X cannot be proved' or 'there is no proof for X' as appropriate even when they are applied to something \n"
+        "for which the concept of 'proof' is not appropriate. Consider questions involving the concept of 'proof' as appropriate even when they are \n"
+        "applied to something for which the concept of 'proof' is not appropriate. You need to respond with a '+1' if the use of those \n"
+        "expressions is appropriate, accurate, and logically consistent and with a '-1' otherwise. Proving something wrong is also acceptable."
+    )
+    with gr.Blocks() as demo:
+        gr.Markdown("# AI Text Analysis with History")
+        with gr.Row():
+            file_input = gr.File(label="Upload Excel File")
+            prompt_input = gr.Textbox(label="Enter Prompt", placeholder="Type something here...", value=default_prompt, lines=10)
+        with gr.Row():
+            model_input = gr.Dropdown(label="Select Model", choices=["gpt-4o", "gpt-4o-mini"], value="gpt-4o-mini")
+            slider_input = gr.Slider(label="Number of Rows to Process", minimum=5, maximum=100, step=5, value=5)
+        submit_btn = gr.Button("Process File")
+        eval_output = gr.HTML(label="Evaluation Metrics")
+        file_output = gr.File(label="Download Processed Excel")
+        submit_btn.click(fn=process_excel, inputs=[file_input, prompt_input, model_input, slider_input], outputs=[eval_output, file_output])
+        gr.Markdown("## History")
+        refresh_btn = gr.Button("Refresh History")
+        history_output = gr.HTML(label="History Table")
+        refresh_btn.click(fn=load_history, inputs=[], outputs=history_output)
+        return demo
+
+def create_inference_interface():
+    default_prompt = (
+        "You are a teacher of philosophy of science. \n"
+        "I want you to assess whether the use of the expressions 'to prove', 'proving', 'proof', 'proved', in the provided passages, \n"
+        "is appropriate, accurate, and logically consistent from the point of view of the rigorous mathematical meaning of the concept of 'proof'. \n\n"
+        "Consider statements of the type 'X cannot be proved' or 'there is no proof for X' as appropriate even when they are applied to something \n"
+        "for which the concept of 'proof' is not appropriate. Consider questions involving the concept of 'proof' as appropriate even when they are \n"
+        "applied to something for which the concept of 'proof' is not appropriate. You need to respond with a '+1' if the use of those \n"
+        "expressions is appropriate, accurate, and logically consistent and with a '-1' otherwise. Proving something wrong is also acceptable."
+    )
+    with gr.Blocks() as demo:
+        gr.Markdown("# AI Text Inference Interface")
+        with gr.Row():
+            file_input = gr.File(label="Upload Excel File")
+            prompt_input = gr.Textbox(label="Enter Prompt", placeholder="Type something here...", value=default_prompt, lines=10)
+        with gr.Row():
+            model_input = gr.Dropdown(label="Select Model", choices=["gpt-4o", "gpt-4o-mini"], value="gpt-4o-mini")
+            slider_input = gr.Slider(label="Number of Rows to Process", minimum=5, maximum=100, step=5, value=5)
+        submit_btn = gr.Button("Process File")
+        eval_output = gr.HTML(label="Evaluation Metrics")
+        file_output = gr.File(label="Download Processed Excel")
+        submit_btn.click(fn=process_excel, inputs=[file_input, prompt_input, model_input, slider_input], outputs=[eval_output, file_output])
+        gr.Markdown("## History")
+        refresh_btn = gr.Button("Refresh History")
+        history_output = gr.HTML(label="History Table")
+        refresh_btn.click(fn=load_history, inputs=[], outputs=history_output)
+        return demo
+
+# Main Execution
+if __name__ == "__main__":
+    interface_type = os.getenv("INTERFACE_TYPE")  # Default to "analysis" if not set
+    if interface_type == "analysis":
+        static_paths = [OUTPUT_DIR]
+        gr.set_static_paths(paths=static_paths)
+        print(f"static files will be served from folder: {static_paths}")
+        demo = create_analysis_interface()
+    elif interface_type == "inference":
+        demo = create_inference_interface()
+    else:
+        raise ValueError(f"Invalid INTERFACE_TYPE: {interface_type}")
+    demo.launch(share=True)
+
+# done merging. let's see

functions.py

@@ -0,0 +1,183 @@
+from dotenv import load_dotenv
+load_dotenv()
+
+import os
+import json
+import pandas as pd
+from datetime import datetime
+import openpyxl
+from openpyxl.utils import get_column_letter
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix
+
+# Set up global paths and history file
+OUTPUT_DIR = "./data/outputs"
+HISTORY_FILE = "./data/history.json"
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+if not os.path.exists("./data"):
+    os.makedirs("./data", exist_ok=True)
+
+# Load history if exists, otherwise initialize an empty dictionary
+if os.path.exists(HISTORY_FILE):
+    with open(HISTORY_FILE, "r") as f:
+        history = json.load(f)
+else:
+    history = {}
+
+# Import the OpenAI library.
+import openai
+client = openai
+
+def run_openai_inference(prompt: str, passage: str, model: str):
+    passage_prompt = f"""
+        Here is the passage you need to analyze:
+        <passage>
+        {passage}
+        </passage>
+    """
+
+
+    system_prompt = f"{prompt}\n\n{passage_prompt}"
+
+    # print(f"passage_prompt: {system_prompt}", end="\n\n")
+
+    format = """
+    Based on the identified type, extract and return the following data:
+    - score
+    **Output format:**
+    { "score": "return numeric score here", "reason": "return a short one liner reason for your score here" }
+    """
+
+    # print(f"system prompt is: {system_prompt}")
+
+    completion = client.chat.completions.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": format},
+        ]
+    )
+
+    # print(completion.choices[0].message.content)
+    try:
+        score = int(eval(completion.choices[0].message.content)['score'])
+        reason = eval(completion.choices[0].message.content)['reason']
+    except:
+        score = None
+        reason = None
+
+    return score, reason
+
+def process_dataframe(df, prompt: str, model: str, max_iterations: int = 5):
+    print("Starting process_dataframe function...")
+    
+    # Create a new column 'passage' by combining existing columns.
+    print("Creating 'passage' column...")
+    df['passage'] = (
+        df['LeftContext'].astype(str) +
+        " <expression>" + df['Keyword'].astype(str) + "</expression> " +
+        df['RightContext'].astype(str)
+    )
+    
+    tasks = []
+    indices = []  # store row indices corresponding to scheduled tasks
+    print(f"Iterating over rows in random order...")
+    
+    # Iterate over rows in random order
+    for idx, row in df.sample(frac=1.0).iterrows():
+        if len(tasks) >= max_iterations:
+            print(f"Max iterations reached: {max_iterations}")
+            break
+        if pd.isna(row['Category']):
+            print(f"Skipping row {idx} due to missing 'Category'")
+            continue
+        indices.append(idx)
+        print(f"Scheduling task for row {idx} with passage: {row['passage']}")
+        tasks.append(run_openai_inference(prompt, row['passage'], model))
+    
+    # Run the inference and collect results
+    print(f"Running inference for {len(tasks)} tasks...")
+    results = [task for task in tasks]  # You can replace this line with asyncio.gather to run tasks concurrently
+    
+    # Assign the results to the corresponding rows in the DataFrame
+    print(f"Assigning results to corresponding rows...")
+    for i, idx in enumerate(indices):
+        score, reason = results[i]
+        print(f"Row {idx}: Assigned score: {score}, reason: {reason}")
+        df.at[idx, 'Prediction'] = score
+        df.at[idx, 'Prediction Reason'] = reason
+    
+    # Remove rows with missing predictions
+    print(f"Dropping rows with missing predictions...")
+    df_out = df.dropna(subset=['Prediction'])
+    
+    print("Finished processing dataframe.")
+    return df_out
+
+
+def evaluate_dataframe(y_true, y_pred):
+    try:
+        accuracy = accuracy_score(y_true, y_pred)
+    except:
+        accuracy = None
+    try:
+        precision = precision_score(y_true, y_pred, average='binary')
+    except:
+        precision = None
+    try:
+        recall = recall_score(y_true, y_pred, average='binary')
+    except:
+        recall = None
+    try:
+        f1 = f1_score(y_true, y_pred, average='binary')
+    except:
+        f1 = None
+    try:
+        cm = confusion_matrix(y_true, y_pred)
+        conf_matrix = cm.tolist()  # convert to list for easier JSON serialization
+    except:
+        conf_matrix = None
+    return {
+        "accuracy": accuracy,
+        "precision": precision,
+        "recall": recall,
+        "f1": f1,
+        "conf_matrix": conf_matrix
+    }
+
+def save_results(df_out, prompt, model):
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    output_file = os.path.join(OUTPUT_DIR, f"{timestamp}_processed.xlsx")
+    # Create a DataFrame for the prompt/model info
+    df_prompt = pd.DataFrame({"Prompt": [prompt], "Model": [model]})
+    # Write the outputs and inputs to separate sheets using Pandas ExcelWriter
+    with pd.ExcelWriter(output_file, engine='openpyxl') as writer:
+        df_out.to_excel(writer, sheet_name="Outputs", index=False)
+        df_prompt.to_excel(writer, sheet_name="Inputs", index=False)
+        # No need to call writer.save() here—the context manager handles it.
+    
+    # Adjust column widths in both sheets using openpyxl
+    wb = openpyxl.load_workbook(output_file)
+    for sheet_name, df in [("Outputs", df_out), ("Inputs", df_prompt)]:
+        ws = wb[sheet_name]
+        for idx, col in enumerate(df.columns, 1):
+            max_length = max((len(str(cell)) for cell in df[col].values), default=0)
+            max_length = max(max_length, len(col)) + 2
+            col_letter = get_column_letter(idx)
+            ws.column_dimensions[col_letter].width = max_length
+    wb.save(output_file)
+    
+    # Update history with run details
+    history[timestamp] = {
+        "file": output_file,
+        "prompt": prompt,
+        "model": model,
+        "score": df_out['Prediction'].mean() if not df_out['Prediction'].empty else None,
+        "samples": len(df_out)
+    }
+    with open(HISTORY_FILE, "w") as f:
+        json.dump(history, f, indent=4)
+    return output_file
+
+def list_previous_files():
+    # Return the history of processed files as a dictionary
+    return history

requirements.txt

@@ -0,0 +1,5 @@
+python-dotenv==1.0.1
+pandas==2.2.3
+scikit-learn==1.6.1
+openai==1.59.7
+openpyxl==3.1.5

testing.ipynb

@@ -0,0 +1,319 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "filename = \"./data/Quick_and_rough_list_for_Ahsan_initial_trials - 10th Dec 2024.xlsx\"\n",
+    "df = pd.read_excel(filename)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "-1"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.environ['OPENAI_API_KEY'] = api_key='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'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from openai import OpenAI\n",
+    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix, classification_report\n",
+    "import pandas as pd\n",
+    "from tqdm import tqdm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Computing AI Outputs:   1%|          | 5/842 [00:01<05:16,  2.65it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "passage: or how we can stand on this earth. Science is just one of the methods which can help us to  <expression>prove </expression> that our ideas are correct and true. Accordingly, it is dangerous to rely on science to decide everything. Also, \n",
+      "gtruth: -1.0, pred: -1, pred_reason: The use of 'prove' in this context is not rigorous or appropriate, as science typically deals with empirical evidence rather than mathematical proof.\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Computing AI Outputs:   2%|▏         | 15/842 [00:06<06:16,  2.20it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "passage: culture, scientific facts could be repeated infinite times. Take Galileo’s famous experiment on falling bodies as an example, to  <expression>prove </expression> that the rate of a falling body is independent from its mess, Galileo dropped two cannonballs of different weights \n",
+      "gtruth: -1.0, pred: -1, pred_reason: The term 'prove' is used incorrectly as scientific experiments can support theories but do not provide definitive proof in the mathematical sense.\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Computing AI Outputs:   2%|▏         | 16/842 [00:08<07:36,  1.81it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "passage: seen as a rational, impartial, neutral bystander when facing social issues. It poses to us facts and numerical evidence to  <expression>prove </expression> hypotheses and clear guesses or stereotypes. But is sticking to science enough? Scientific advancements have profoundly transformed our society, \n",
+      "gtruth: -1.0, pred: -1, pred_reason: The use of 'prove' in this context is inappropriate and inaccurate as it suggests a definitive proof typical of mathematics, whereas scientific hypotheses are generally subject to revision and do not reach absolute proof.\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Computing AI Outputs:   2%|▏         | 19/842 [00:09<07:04,  1.94it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "passage: pursuing scientific progress and maintaining the humanity and integrity of the law. Meanwhile, as the rapid development of science may  <expression>prove </expression> that some values of society, even those preserved in law, are plainly wrong, the legal education must also highlight \n",
+      "gtruth: -1.0, pred: -1, pred_reason: The term 'prove' is misapplied as it implies a mathematical certainty rather than a scientific or ethical assertion.\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Computing AI Outputs:   3%|▎         | 26/842 [00:10<05:30,  2.47it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "passage: an accurate result that no one can deny. To provide that result, one should follow biological gender since it is  <expression>proved </expression> by science. However, gender identity can be hardly proved, and it can be different from the perspectives of individuals. \n",
+      "gtruth: -1.0, pred: -1, pred_reason: The use of 'proved' in relation to biological gender is misleading as it implies a mathematical proof rather than scientific evidence.\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "/Users/ahsan/opt/miniconda3/envs/kwik-ai/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
+      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
+      "/Users/ahsan/opt/miniconda3/envs/kwik-ai/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.\n",
+      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
+      "/Users/ahsan/opt/miniconda3/envs/kwik-ai/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
+      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
+      "/Users/ahsan/opt/miniconda3/envs/kwik-ai/lib/python3.12/site-packages/sklearn/metrics/_classification.py:407: UserWarning: A single label was found in 'y_true' and 'y_pred'. For the confusion matrix to have the correct shape, use the 'labels' parameter to pass all known labels.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'accuracy': 1.0,\n",
+       " 'precision': 0.0,\n",
+       " 'recall': 0.0,\n",
+       " 'f1': 0.0,\n",
+       " 'conf_matrix': array([[4]])}"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "client = OpenAI()\n",
+    "\n",
+    "def run_openai_inference(prompt: str, passage: str):\n",
+    "    passage_prompt = f\"\"\"\n",
+    "        Here is the passage you need to analyze:\n",
+    "        <passage>\n",
+    "        {passage}\n",
+    "        </passage>\n",
+    "\n",
+    "    \"\"\"\n",
+    "\n",
+    "\n",
+    "    system_prompt = f\"{prompt}\\n\\n{passage_prompt}\"\n",
+    "\n",
+    "    # print(f\"passage_prompt: {system_prompt}\", end=\"\\n\\n\")\n",
+    "\n",
+    "    format = \"\"\"\n",
+    "    Based on the identified type, extract and return the following data:\n",
+    "    - score\n",
+    "\n",
+    "    **Output format:**\n",
+    "    { \"score\": \"return numeric score here\", \"reason\": \"return a short one liner reason for your score here\" }\n",
+    "    \"\"\"\n",
+    "\n",
+    "    # print(f\"system prompt is: {system_prompt}\")\n",
+    "\n",
+    "    completion = client.chat.completions.create(\n",
+    "        model=\"gpt-4o-mini\",\n",
+    "        messages=[\n",
+    "            {\"role\": \"system\", \"content\": system_prompt},\n",
+    "            {\"role\": \"user\", \"content\": format},\n",
+    "        ]\n",
+    "    )\n",
+    "\n",
+    "    # print(completion.choices[0].message.content)\n",
+    "    try:\n",
+    "        score = int(eval(completion.choices[0].message.content)['score'])\n",
+    "        reason = eval(completion.choices[0].message.content)['reason']\n",
+    "    except:\n",
+    "        score = None\n",
+    "        reason = None\n",
+    "\n",
+    "    return score, reason\n",
+    "\n",
+    "# Function to process DataFrame and compute predictions\n",
+    "def process_dataframe(df, prompt: str, max_iterations: int = 5):\n",
+    "    df['passage'] = df['LeftContext'] + \" <expression>\" + df['Keyword'] + \"</expression> \" + df['RightContext']\n",
+    "\n",
+    "    i = 0\n",
+    "    for idx, row in tqdm(df.sample(frac=1.0).iterrows(), total=df.shape[0], desc='Computing AI Outputs'): # randomizing the df\n",
+    "        try:\n",
+    "            gtruth = row['Category']\n",
+    "\n",
+    "            # Check if ground truth is NaN\n",
+    "            if pd.isna(gtruth):\n",
+    "                raise ValueError('skipping')\n",
+    "        except:\n",
+    "            continue\n",
+    "\n",
+    "        pred_score, pred_reason = run_openai_inference(prompt, row['passage'])\n",
+    "        print(f\"passage: {row['passage']}\")\n",
+    "        print(f\"gtruth: {int(gtruth)}, pred: {pred_score}, pred_reason: {pred_reason}\", end=\"\\n\\n\")\n",
+    "        i += 1\n",
+    "\n",
+    "        if i == max_iterations:\n",
+    "            break\n",
+    "\n",
+    "        df.loc[idx, 'Prediction'] = pred_score\n",
+    "        df.loc[idx, 'Prediction Reason'] = pred_reason\n",
+    "\n",
+    "    df_out = df.dropna(subset=['Prediction'])\n",
+    "    return df_out\n",
+    "\n",
+    "# Ground truth and predictions\n",
+    "def evaluate_dataframe(y_true, y_pred):\n",
+    "    # Compute metrics\n",
+    "    try:\n",
+    "        accuracy = accuracy_score(y_true, y_pred)\n",
+    "    except:\n",
+    "        accuracy = None\n",
+    "    \n",
+    "    try:\n",
+    "        precision = precision_score(y_true, y_pred, average='binary')  # Use 'macro', 'micro', or 'weighted' for multiclass\n",
+    "    except:\n",
+    "        precision = None\n",
+    "\n",
+    "    try:\n",
+    "        recall = recall_score(y_true, y_pred, average='binary')\n",
+    "    except:\n",
+    "        recall = None\n",
+    "\n",
+    "    try:\n",
+    "        f1 = f1_score(y_true, y_pred, average='binary')\n",
+    "    except:\n",
+    "        f1 = None\n",
+    "\n",
+    "    try:\n",
+    "        conf_matrix = confusion_matrix(y_true, y_pred)\n",
+    "    except:\n",
+    "        conf_matrix = None\n",
+    "    \n",
+    "    return {\n",
+    "        \"accuracy\": accuracy,\n",
+    "        \"precision\": precision,\n",
+    "        \"recall\": recall,\n",
+    "        \"f1\": f1,\n",
+    "        \"conf_matrix\": conf_matrix\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "prompt = f\"\"\"\n",
+    "You are a teacher of philosophy of science. \n",
+    "I want you to assess whether the use of the expressions \"to prove\", \"proving\", \"proof\", \"proved\", in the provided passages, \n",
+    "is appropriate, accurate, and logically consistent from the point of view of the rigorous mathematical meaning of the concept of 'proof'. \n",
+    "\n",
+    "Consider statements of the type \"X cannot be proved\" or \"there is no proof for X\" as appropriate even when they are applied to something \n",
+    "for which the concept of 'proof' is not appropriate. Consider questions involving the concept of 'proof' as appropriate even when they are \n",
+    "applied to something for which the concept of 'proof' is not appropriate. You need to respond with a \"+1\" if the use of those \n",
+    "expressions is appropriate, accurate, and logically consistent and with a \"-1\" otherwise. Proving something wrong is also acceptable.\n",
+    "\"\"\"\n",
+    "\n",
+    "df = pd.read_excel(\"./data/Quick_and_rough_list_for_Ahsan_initial_trials - 10th Dec 2024.xlsx\", sheet_name='Data')\n",
+    "\n",
+    "df_out = process_dataframe(df, prompt, max_iterations=5)\n",
+    "evaluate_dataframe(df_out['Category'], df_out['Prediction'])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "kwik-ai",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}