Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| import requests | |
| from huggingface_hub import HfApi, create_repo, upload_file, hf_hub_download | |
| from apscheduler.schedulers.background import BackgroundScheduler | |
| from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns | |
| import os | |
| # Hugging Face API setup | |
| HF_TOKEN = os.getenv("HF_TOKEN") # This should be set as an environment variable | |
| api = HfApi(token=HF_TOKEN) | |
| # Deepfake detector data | |
| data_avg_performance = { | |
| "Detector": ["NPR", "UCF", "CAMO"], | |
| "Accuracy": [0.7169, 0.7229, 0.7555], | |
| "Precision": [0.9193, 0.9436, 0.9442], | |
| "Recall": [0.5996, 0.592, 0.647], | |
| "F1-Score": [0.7258, 0.7275, 0.7679], | |
| "MCC": [0.5044, 0.5285, 0.5707], | |
| } | |
| data_dataset_accuracy = { | |
| "Detector": ["NPR", "UCF", "CAMO"], | |
| "CelebA-HQ": [0.987, 0.995, 0.999], | |
| "Flickr30k": [0.916, 0.981, 0.979], | |
| "ImageNet": [0.834, 0.847, 0.831], | |
| "DiffusionDB": [0.876, 0.85, 0.961], | |
| "CelebA-HQ-SDXL": [0.386, 0.484, 0.682], | |
| "CelebA-HQ-Flux": [0.846, 0.794, 0.722], | |
| "Flickr30k-SDXL": [0.302, 0.256, 0.28], | |
| "MS-COCO-Flux": [0.588, 0.576, 0.59], | |
| } | |
| # Convert data to DataFrames | |
| df_avg_performance = pd.DataFrame(data_avg_performance) | |
| df_dataset_accuracy = pd.DataFrame(data_dataset_accuracy) | |
| # Ensure correct data types | |
| df_avg_performance['Detector'] = df_avg_performance['Detector'].astype(str) | |
| df_avg_performance[['Accuracy', 'Precision', 'Recall', 'F1-Score', 'MCC']] = df_avg_performance[ | |
| ['Accuracy', 'Precision', 'Recall', 'F1-Score', 'MCC'] | |
| ].apply(pd.to_numeric) | |
| def init_leaderboard(): | |
| if df_avg_performance.empty: | |
| raise ValueError("Leaderboard DataFrame is empty.") | |
| # Display average performance leaderboard only | |
| leaderboard = Leaderboard( | |
| value=df_avg_performance, | |
| datatype=['str', 'number', 'number', 'number', 'number', 'number'], | |
| select_columns=SelectColumns( | |
| default_selection=["Detector", "Accuracy", "Precision", "Recall", "F1-Score", "MCC"], | |
| label="Select Columns to Display:" | |
| ), | |
| search_columns=["Detector"], | |
| filter_columns=[ | |
| ColumnFilter("Detector", type="checkboxgroup", label="Detectors"), | |
| ], | |
| bool_checkboxgroup_label="Hide detectors", | |
| interactive=False, | |
| ) | |
| return leaderboard | |
| # Function to validate if a file exists in the Hugging Face repo | |
| def check_file_in_repo(repo_id, file_path): | |
| try: | |
| # List the repo contents and check if the file exists | |
| contents = api.list_repo_files(repo_id) | |
| return file_path in contents | |
| except Exception as e: | |
| return False | |
| # Function to handle form submission and validate inputs | |
| def submit_detector(detector_name, model_repo, detector_file_path, configs_file_path): | |
| # Check if the Hugging Face repository exists | |
| try: | |
| repo_info = api.repo_info(model_repo) | |
| except Exception as e: | |
| return f"Repository '{model_repo}' not found. Error: {e}" | |
| # Validate the existence of the .py file and weights file in the repo | |
| if not check_file_in_repo(model_repo, detector_file_path): | |
| return f"The file '{detector_file_path}' was not found in the repository." | |
| if not check_file_in_repo(model_repo, configs_file_path): | |
| return f"The configs file '{configs_file_path}' was not found in the repository." | |
| # Prepare the data to push to the Hugging Face dataset | |
| submission_data = { | |
| "detector_name": detector_name, | |
| "model_repo": model_repo, | |
| "detector_file_path": detector_file_path, | |
| "configs_file_path": configs_file_path, | |
| "passed_invocation_test": "Pending", | |
| "evaluation_status": "Under Review" # Set evaluated to False by default | |
| } | |
| df_submission = pd.DataFrame([submission_data]) | |
| dataset_repo_id = "bitmind/dfd-arena-detectors" | |
| # Push to the dataset or create it if it doesn't exist | |
| try: | |
| # Create the dataset if it doesn't exist | |
| create_repo(repo_id=dataset_repo_id, | |
| token=HF_TOKEN, | |
| repo_type="dataset", | |
| exist_ok=True) | |
| # Try to download the existing dataset | |
| existing_dataset_path = hf_hub_download( | |
| repo_id=dataset_repo_id, | |
| filename="submissions/submission.csv", | |
| token=HF_TOKEN, | |
| repo_type="dataset" | |
| ) | |
| # If the file is found, load the existing dataset into a DataFrame | |
| df_existing = pd.read_csv(existing_dataset_path) | |
| print(df_existing) | |
| print(df_submission) | |
| # Check if the submission is already in the dataset (exact match) | |
| is_in_submissions = (df_existing == df_submission.iloc[0]).all(axis=1).any() | |
| if not is_in_submissions: | |
| # If no duplicate is found, append the new submission | |
| df_updated = pd.concat([df_existing, df_submission], ignore_index=True) | |
| else: | |
| return "Duplicate submission: This submission already exists in the dataset." | |
| # Append the new submission data | |
| df_updated = pd.concat([df_existing, df_submission], ignore_index=True) | |
| except Exception as e: | |
| # If the dataset doesn't exist, create a new DataFrame with the submission data | |
| df_updated = pd.DataFrame([submission_data]) | |
| # Save the updated (or new) DataFrame as a CSV file | |
| submission_file = "submission.csv" | |
| df_updated.to_csv(submission_file, index=False) | |
| # Upload the updated (or new) file to the Hugging Face repository | |
| try: | |
| upload_file( | |
| path_or_fileobj=submission_file, | |
| path_in_repo="submissions/submission.csv", # Location in the repo | |
| repo_id=dataset_repo_id, | |
| token=HF_TOKEN, | |
| repo_type="dataset", | |
| commit_message="Submitted for review: "+detector_name | |
| ) | |
| return "Submission successful!" | |
| except Exception as e: | |
| return f"Failed to push submission: {str(e)}" | |
| # Gradio demo | |
| with gr.Blocks(theme=gr.themes.Base()) as demo: | |
| # Title | |
| gr.HTML(""" | |
| <div style="text-align:center;"> | |
| <h1> Deepfake Detection Arena (DFD) Leaderboard</h1> | |
| </div> | |
| """) | |
| # Description/Intro Section | |
| gr.Markdown(""" | |
| ## 🎯 The Open Benchmark for Detecting AI-Generated Images | |
| [DFD-Arena](https://github.com/BitMind-AI/dfd-arena) is the first benchmark to address the open-source computer vision community's need for a *comprehensive evaluation framework* for state-of-the-art (SOTA) detection of AI-generated images. | |
| While [previous studies](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=9721302) have focused on benchmarking the SOTA on content-specific subsets of the deepfake detection problem, e.g. human face deepfake benchmarking via [DeepfakeBench](https://github.com/SCLBD/DeepfakeBench), these benchmarks do not adequately account for the broad spectrum of real and generated image types seen in everyday scenarios. | |
| ### 🔍 Explore DFD-Arena | |
| Learn how the framework evaluates on diverse, content-rich images with semantic balance between real and generated data: | |
| - 📂 [Code: DFD-Arena GitHub Repository](https://github.com/BitMind-AI/dfd-arena) | |
| - 📝 [Blog: Technical Write-Up](https://bitmindlabs.notion.site/BitMind-Deepfake-Detection-Arena-106af85402838007830ece5a6f3f35a8?pvs=25) | |
| ### ✍️ Authorship | |
| Both DFD-Arena and novel synthetic image datasets used for evaluation are created by [BitMind](https://www.bitmind.ca/). | |
| - 🐦 [X/Twitter: @BitMindAI](https://x.com/BitMindAI) | |
| """) | |
| with gr.Tabs(): | |
| with gr.TabItem("🏅 Deepfake Detection Arena"): | |
| # Add text for Average Performance Metrics | |
| gr.Markdown("## Average Performance Metrics") | |
| leaderboard = init_leaderboard() | |
| # Add dataset-specific accuracy | |
| gr.Markdown("## Dataset-specific Accuracy") | |
| gr.DataFrame(value=df_dataset_accuracy) | |
| # # Submit Detector Results Form | |
| with gr.TabItem("🚀 Submit Detector Results"): | |
| # Add submission instructions | |
| gr.Markdown(""" | |
| ## Submission Instructions | |
| <h3>🏗️ 1. Ensure that your detector code follows the DFD Arena detectors format.</h3> | |
| The best way to guarantee compatibility is to develop and test your detector within a local copy of the [DFD Arena repository](https://github.com/BitMind-AI/dfd-arena/tree/main), with dependencies, detector file, and configs in relative locations similar to how we structured our implementations of UCF, NPR, and CAMO detectors. | |
| **✅ Check list:** | |
| - Your code should work with detector dependencies (architecture and training code) imported from a dependencies directory a level above the detector directory. | |
| - e.g., `arena/detectors/deepfake_detectors/ucf_detector.py` relies on a dependency folder at `arena/detectors/UCF/` | |
| - Our automated benchmarking pipeline will reconstruct the required directory at evaluation time | |
| - Implement a `.py` file in `arena/detectors/deepfake_detectors/` containing a `DeepfakeDetector` subclass with PascalCase naming convention, registered as a module in the dfd-arena `DETECTOR_REGISTRY`. | |
| - e.g., in `myCustomDetector.py`, | |
| ```python | |
| @DETECTOR_REGISTRY.register_module(module_name='MyCustomModuleName') | |
| class MyCustomDetector(DeepfakeDetector): | |
| # implementation | |
| ``` | |
| - The module name should match the detector name you want to appear on the leaderboard | |
| - Create a config YAML file that the DeepfakeDetector loads in `arena/detectors/deepfake_detectors/configs/`. | |
| <h3>🤗 2. Create a HuggingFace model repo with the detector `.py` file, config `.yaml`, and dependencies in the same root level.</h3> | |
| [Check out our Sample Leaderboard Submission Repo for Reference!](https://huggingface.co/caliangandrew/submit_test/tree/main) | |
| <h3>📤 3. Fill out the form below with the correct paths and submit!</h3> | |
| The results will be processed after a code review by the BitMind team, and an automated test/benchmarking stage. | |
| **⚠️ Note:** The Detector Name must match the name of the registered detector module in the dfd arena detector registry. This will be the name of your detector on our leaderboard. | |
| - For example, using the [Sample Submission Repo](https://huggingface.co/caliangandrew/submit_test/tree/main) provided, you would submit: | |
| - Detector Name: `test` | |
| - Hugging Face Model Repo: `caliangandrew/submit_test` | |
| - Path to detector `.py`: `test_detector.py` | |
| - Path to config `.YAML`: `test_config.yaml` | |
| You can monitor your submission via the companion [detectors](https://huggingface.co/datasets/bitmind/dfd-arena-detectors) and [results](https://huggingface.co/datasets/bitmind/dfd-arena-results) datasets. | |
| """) | |
| with gr.Row(): | |
| detector_name = gr.Textbox(label="Detector Name", placeholder="e.g., MyDetector") | |
| model_repo = gr.Textbox(label="Hugging Face Model Repo", placeholder="e.g., username/repo-name") | |
| detector_file_path = gr.Textbox(label="Path to detector .py", placeholder="e.g., my_detector.py") | |
| configs_file_path = gr.Textbox(label="Path to config .YAML", placeholder="e.g., my_config.yaml") | |
| submit_button = gr.Button("Submit",interactive=True) | |
| output = gr.Textbox(label="Submission Result",interactive=True) | |
| submit_button.click(fn=submit_detector, inputs=[detector_name, model_repo, detector_file_path, configs_file_path], outputs=output) | |
| # Launch the Gradio app | |
| demo.queue(default_concurrency_limit=40).launch() |