kirudang's picture
Copy files from original watermark leaderboard
40b3335
import gradio as gr
import json
import os
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from datetime import datetime
from plotly.subplots import make_subplots
# Load leaderboard data
def load_leaderboard_data():
try:
with open('leaderboard.json', 'r') as f:
return json.load(f)
except:
return []
# Filter data based on model and metric
def filter_data(data, model, metric):
filtered = []
for item in data:
if item.get('model') == model:
if metric == "Attack-free":
if item.get('normalizedUtility') is not None and item.get('detectionRate') is not None:
filtered.append({
'name': item.get('name', ''),
'model': item.get('model', ''),
'normalizedUtility': item.get('normalizedUtility', 0),
'detectionRate': item.get('detectionRate', 0)
})
elif metric == "Watermark Removal":
if (item.get('absoluteUtilityDegregation') is not None and
item.get('removal_detectionRate') is not None):
filtered.append({
'name': item.get('name', ''),
'model': item.get('model', ''),
'absoluteUtilityDegregation': item.get('absoluteUtilityDegregation', 0),
'removal_detectionRate': item.get('removal_detectionRate', 0)
})
elif metric == "Stealing Attack":
if (item.get('adversaryBERTscore') is not None and
item.get('adversaryDetectionRate') is not None):
filtered.append({
'name': item.get('name', ''),
'model': item.get('model', ''),
'adversaryBERTscore': item.get('adversaryBERTscore', 0),
'adversaryDetectionRate': item.get('adversaryDetectionRate', 0)
})
# Sort by detection rate (descending)
if metric == "Attack-free":
filtered.sort(key=lambda x: x['detectionRate'], reverse=True)
elif metric == "Watermark Removal":
filtered.sort(key=lambda x: x['removal_detectionRate'], reverse=True)
else: # Stealing Attack
filtered.sort(key=lambda x: x['adversaryDetectionRate'], reverse=True)
return filtered
# Create scatter plot
def create_scatter_plot(data, metric):
if not data:
return go.Figure()
# Prepare data for plotting
x_data = []
y_data = []
names = []
for item in data:
names.append(item['name'])
if metric == "Attack-free":
x_data.append(item['normalizedUtility'])
y_data.append(item['detectionRate'])
elif metric == "Watermark Removal":
x_data.append(item['absoluteUtilityDegregation'])
y_data.append(item['removal_detectionRate'])
else: # Stealing Attack
x_data.append(item['adversaryBERTscore'])
y_data.append(item['adversaryDetectionRate'])
# Create scatter plot
fig = go.Figure()
# Add scatter points
fig.add_trace(go.Scatter(
x=x_data,
y=y_data,
mode='markers+text',
marker=dict(
size=12,
color='#3B82F6',
line=dict(width=2, color='white')
),
text=names,
textposition='top center',
textfont=dict(size=10, color='#374151'),
hovertemplate='<b>%{text}</b><br>' +
('Normalized Utility: %{x:.3f}<br>' if metric == "Attack-free" else
'Abs Utility Degradation: %{x:.3f}<br' if metric == "Watermark Removal" else
'Adversary BERT Score: %{x:.3f}<br>') +
('Detection Rate: %{y:.3f}%<br>' if metric != "Stealing Attack" else
'Adversary Detection Rate: %{y:.3f}%<br>') +
'<extra></extra>'
))
# Set axis labels
if metric == "Attack-free":
x_title = "Normalized Utility"
y_title = "Detection Rate (%)"
elif metric == "Watermark Removal":
x_title = "Absolute Utility Degradation"
y_title = "Removal Detection Rate (%)"
else: # Stealing Attack
x_title = "Adversary BERT Score"
y_title = "Adversary Detection Rate (%)"
fig.update_layout(
title=f"{metric} Performance Scatter Plot",
xaxis_title=x_title,
yaxis_title=y_title,
font=dict(size=12, color='#374151'),
plot_bgcolor='white',
paper_bgcolor='white',
xaxis=dict(
gridcolor='lightgray',
showgrid=True,
zeroline=False
),
yaxis=dict(
gridcolor='lightgray',
showgrid=True,
zeroline=False
),
margin=dict(l=60, r=60, t=80, b=60)
)
return fig
# Create table data with heatmap styling
def create_table_data(data, metric):
if not data:
return pd.DataFrame()
table_data = []
for i, item in enumerate(data, 1):
row = {'Rank': i, 'Watermark': item['name']}
if metric == "Attack-free":
row['Normalized Utility ↑'] = f"{item['normalizedUtility']:.3f}"
row['Detection Rate (%) ↑'] = f"{item['detectionRate']:.3f}"
elif metric == "Watermark Removal":
row['Abs Utility Degradation ↑'] = f"{item['absoluteUtilityDegregation']:.3f}"
row['Removal Detection Rate (%) ↑'] = f"{item['removal_detectionRate']:.3f}"
else: # Stealing Attack
row['Adversary BERT Score ↑'] = f"{item['adversaryBERTscore']:.3f}"
row['Adversary Detection Rate (%) ↑'] = f"{item['adversaryDetectionRate']:.3f}"
table_data.append(row)
return pd.DataFrame(table_data)
# Create table data with green arrows and reference links
def create_table_data(data, metric):
if not data:
return pd.DataFrame()
table_data = []
for i, item in enumerate(data, 1):
watermark_name = item['name']
paper_link = item.get('paperLink')
model = item.get('model', 'N/A')
# Create reference link if paper link exists (smaller text)
if paper_link:
reference_link = f'<a href="{paper_link}" target="_blank" style="color: #3B82F6; text-decoration: underline; font-size: 0.8em;">πŸ“„ Paper</a>'
else:
reference_link = '-'
row = {
'Watermark': watermark_name
}
if metric == "Attack-free":
row['Normalized Utility ↑'] = f"{item['normalizedUtility']:.3f}"
row['Detection Rate (%) ↑'] = f"{item['detectionRate']:.3f}"
elif metric == "Watermark Removal":
row['Abs Utility Degradation ↑'] = f"{item['absoluteUtilityDegregation']:.3f}"
row['Removal Detection Rate (%) ↑'] = f"{item['removal_detectionRate']:.3f}"
else: # Stealing Attack
row['Adversary BERT Score ↑'] = f"{item['adversaryBERTscore']:.3f}"
row['Adversary Detection Rate (%) ↑'] = f"{item['adversaryDetectionRate']:.3f}"
# Add Reference column at the end
row['Reference'] = reference_link
table_data.append(row)
return pd.DataFrame(table_data)
# Update interface based on selections
def update_interface(model, metric):
data = load_leaderboard_data()
filtered_data = filter_data(data, model, metric)
# Create scatter plot
scatter_plot = create_scatter_plot(filtered_data, metric)
# Create table with green arrows
table_data = create_table_data(filtered_data, metric)
return scatter_plot, table_data
# Handle form submission
def submit_watermark_data(name, model, paper_link, normalized_utility, detection_rate,
absolute_utility_degradation, removal_detection_rate,
adversary_bert_score, adversary_detection_rate):
"""Handle watermark data submission"""
# Validation
if not name or not name.strip():
return "❌ Error: Watermark name is required", gr.update()
if not model:
return "❌ Error: Model selection is required", gr.update()
# Validate paper link if provided
if paper_link and paper_link.strip():
paper_link = paper_link.strip()
if not (paper_link.startswith('http://') or paper_link.startswith('https://')):
return "❌ Error: Paper link must start with http:// or https://", gr.update()
else:
paper_link = None
# Check what type of submission this is based on provided fields
has_attack_free_data = normalized_utility is not None and detection_rate is not None
has_removal_data = absolute_utility_degradation is not None and removal_detection_rate is not None
has_stealing_data = adversary_bert_score is not None and adversary_detection_rate is not None
# At least one complete set of metrics must be provided
if not has_attack_free_data and not has_removal_data and not has_stealing_data:
return "❌ Error: Please provide at least one complete set of metrics:\nβ€’ Attack-free: Normalized Utility + Detection Rate\nβ€’ Watermark Removal: Absolute Utility Degradation + Removal Detection Rate\nβ€’ Stealing Attack: Adversary BERT Score + Adversary Detection Rate", gr.update()
# Validate Attack-free metrics if provided
if has_attack_free_data:
if normalized_utility <= 0 or normalized_utility > 1.0:
return "❌ Error: Normalized Utility must be between 0.000 and 1.000", gr.update()
if detection_rate < 0.0 or detection_rate > 100.0:
return "❌ Error: Detection Rate must be between 0.000 and 100.000", gr.update()
# Validate Watermark Removal metrics if provided
if has_removal_data:
if absolute_utility_degradation <= 0 or absolute_utility_degradation > 1.0:
return "❌ Error: Absolute Utility Degradation must be between 0.000 and 1.000", gr.update()
if removal_detection_rate < 0.0 or removal_detection_rate > 100.0:
return "❌ Error: Removal Detection Rate must be between 0.000 and 100.000", gr.update()
# Validate Stealing Attack metrics if provided
if has_stealing_data:
if adversary_bert_score <= 0 or adversary_bert_score > 1.0:
return "❌ Error: Adversary BERT Score must be between 0.000 and 1.000", gr.update()
if adversary_detection_rate < 0.0 or adversary_detection_rate > 100.0:
return "❌ Error: Adversary Detection Rate must be between 0.000 and 100.000", gr.update()
# Validate partial adversary data (if one is provided, both are required)
has_partial_adversary = (adversary_bert_score is not None and adversary_bert_score > 0) or \
(adversary_detection_rate is not None and adversary_detection_rate > 0)
if has_partial_adversary and not has_stealing_data:
return "❌ Error: If you provide one adversary metric, you must provide both Adversary BERT Score and Adversary Detection Rate", gr.update()
# Create new entry - only include provided values, don't set missing ones to 0
new_entry = {
"name": name.strip(),
"model": model,
"normalizedUtility": normalized_utility,
"detectionRate": detection_rate
}
# Add paper link if provided
if paper_link:
new_entry["paperLink"] = paper_link
# Only add optional metrics if they were provided
if absolute_utility_degradation is not None:
new_entry["absoluteUtilityDegregation"] = absolute_utility_degradation
if removal_detection_rate is not None:
new_entry["removal_detectionRate"] = removal_detection_rate
if adversary_bert_score is not None:
new_entry["adversaryBERTscore"] = adversary_bert_score
if adversary_detection_rate is not None:
new_entry["adversaryDetectionRate"] = adversary_detection_rate
# Load existing approved data to check for duplicates
try:
with open('leaderboard.json', 'r') as f:
approved_data = json.load(f)
except:
approved_data = []
# Check for duplicate names in approved data
for entry in approved_data:
if entry.get('name') == name.strip() and entry.get('model') == model:
return f"❌ Error: A watermark named '{name.strip()}' already exists for {model}", gr.update()
# Load pending submissions to check for duplicates there too
try:
with open('pending_submissions.json', 'r') as f:
pending_data = json.load(f)
except:
pending_data = []
# Check for duplicate names in pending data
for entry in pending_data:
if entry.get('name') == name.strip() and entry.get('model') == model:
return f"❌ Error: A watermark named '{name.strip()}' is already pending approval for {model}", gr.update()
# Add submission timestamp and status
new_entry['submitted_at'] = datetime.now().isoformat()
new_entry['status'] = 'pending'
new_entry['submission_id'] = f"{name.strip()}_{model}_{int(datetime.now().timestamp())}"
# Add to pending submissions instead of approved data
pending_data.append(new_entry)
# Save pending submissions
try:
with open('pending_submissions.json', 'w') as f:
json.dump(pending_data, f, indent=2)
# Update the interface with current approved data only
filtered_data = filter_data(approved_data, model, "Attack-free")
scatter_plot = create_scatter_plot(filtered_data, "Attack-free")
table_data = create_table_data(filtered_data, "Attack-free")
success_msg = f"βœ… Successfully submitted '{name.strip()}' for {model} for approval! Your submission will be reviewed by the administrator before appearing on the leaderboard."
return success_msg, scatter_plot, table_data
except Exception as e:
return f"❌ Error saving submission: {str(e)}", gr.update()
# Clear form function
def clear_form():
return (None, None, None, None, None, None, None, None, None)
# Owner approval functions
def load_pending_submissions():
"""Load pending submissions for owner review"""
try:
with open('pending_submissions.json', 'r') as f:
pending_data = json.load(f)
if not pending_data:
return pd.DataFrame(columns=["ID", "Name", "Model", "Paper Link", "Attack-free Utility", "Attack-free Detection",
"Removal Degradation", "Removal Detection", "Adversary BERT", "Adversary Detection", "Submitted At"])
# Format data for display with all fields
formatted_data = []
for entry in pending_data:
watermark_name = entry.get('name', 'N/A')
paper_link = entry.get('paperLink', '-')
model = entry.get('model', 'N/A')
# Format all metric fields
formatted_entry = {
"ID": entry.get('submission_id', 'N/A'),
"Name": watermark_name,
"Model": model,
"Paper Link": paper_link if paper_link != '-' else '-',
"Attack-free Utility": f"{entry.get('normalizedUtility', 0):.3f}" if entry.get('normalizedUtility') is not None else '-',
"Attack-free Detection": f"{entry.get('detectionRate', 0):.3f}" if entry.get('detectionRate') is not None else '-',
"Removal Degradation": f"{entry.get('absoluteUtilityDegregation', 0):.3f}" if entry.get('absoluteUtilityDegregation') is not None else '-',
"Removal Detection": f"{entry.get('removal_detectionRate', 0):.3f}" if entry.get('removal_detectionRate') is not None else '-',
"Adversary BERT": f"{entry.get('adversaryBERTscore', 0):.3f}" if entry.get('adversaryBERTscore') is not None else '-',
"Adversary Detection": f"{entry.get('adversaryDetectionRate', 0):.3f}" if entry.get('adversaryDetectionRate') is not None else '-',
"Submitted At": entry.get('submitted_at', 'N/A')[:19] if entry.get('submitted_at') else 'N/A', # Show only date and time
}
formatted_data.append(formatted_entry)
return pd.DataFrame(formatted_data)
except Exception as e:
print(f"Error loading pending submissions: {e}")
return pd.DataFrame(columns=["ID", "Name", "Model", "Paper Link", "Attack-free Utility", "Attack-free Detection",
"Removal Degradation", "Removal Detection", "Adversary BERT", "Adversary Detection", "Submitted At"])
def approve_submission(submission_id, admin_password):
"""Approve a pending submission"""
# Check admin password
if admin_password != "admin123": # You can change this password
return "❌ Access denied: Invalid admin password", gr.update()
try:
# Load pending submissions from file (not from the formatted function)
try:
with open('pending_submissions.json', 'r') as f:
pending_data = json.load(f)
except:
pending_data = []
# Find and remove the submission
approved_entry = None
for i, entry in enumerate(pending_data):
if entry.get('submission_id') == submission_id:
approved_entry = pending_data.pop(i)
break
if not approved_entry:
return "❌ Submission not found", gr.update()
# Remove submission metadata
approved_entry.pop('submitted_at', None)
approved_entry.pop('status', None)
approved_entry.pop('submission_id', None)
# Load approved data
try:
with open('leaderboard.json', 'r') as f:
approved_data = json.load(f)
except:
approved_data = []
# Add to approved data
approved_data.append(approved_entry)
# Save approved data
with open('leaderboard.json', 'w') as f:
json.dump(approved_data, f, indent=2)
# Save updated pending data
with open('pending_submissions.json', 'w') as f:
json.dump(pending_data, f, indent=2)
return f"βœ… Approved submission: {approved_entry.get('name', 'Unknown')}", load_pending_submissions()
except Exception as e:
return f"❌ Error approving submission: {str(e)}", gr.update()
def reject_submission(submission_id, admin_password):
"""Reject a pending submission"""
# Check admin password
if admin_password != "admin123": # You can change this password
return "❌ Access denied: Invalid admin password", gr.update()
try:
# Load pending submissions from file (not from the formatted function)
try:
with open('pending_submissions.json', 'r') as f:
pending_data = json.load(f)
except:
pending_data = []
# Find and remove the submission
rejected_entry = None
for i, entry in enumerate(pending_data):
if entry.get('submission_id') == submission_id:
rejected_entry = pending_data.pop(i)
break
if not rejected_entry:
return "❌ Submission not found", gr.update()
# Save updated pending data
with open('pending_submissions.json', 'w') as f:
json.dump(pending_data, f, indent=2)
return f"❌ Rejected submission: {rejected_entry.get('name', 'Unknown')}", load_pending_submissions()
except Exception as e:
return f"❌ Error rejecting submission: {str(e)}", gr.update()
# Toggle add data section visibility
def toggle_add_data_section(section):
return gr.update(visible=not section.visible)
# Create the main interface
def create_interface():
# Custom CSS for better styling
css = """
.gradio-container {
max-width: 1200px !important;
margin: 0 auto !important;
background: linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%);
min-height: 100vh;
}
.title {
text-align: center;
margin: 20px 0;
font-size: 3rem;
font-weight: bold;
background: linear-gradient(45deg, #667eea 0%, #764ba2 100%);
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
background-clip: text;
text-shadow: 2px 2px 4px rgba(0,0,0,0.1);
}
.subtitle {
text-align: center;
margin-bottom: 30px;
font-size: 1.3rem;
color: #4a5568;
font-weight: 500;
}
.controls {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
padding: 30px;
border-radius: 15px;
margin-bottom: 25px;
box-shadow: 0 8px 32px rgba(0,0,0,0.1);
border: 1px solid rgba(255,255,255,0.2);
}
.controls label {
color: white !important;
font-weight: bold !important;
font-size: 1.2rem !important;
}
.controls .gr-radio {
background: rgba(255,255,255,0.1) !important;
border-radius: 10px !important;
padding: 12px !important;
}
.controls .gr-radio label {
color: white !important;
font-size: 1.1rem !important;
}
.controls h3 {
font-size: 1.4rem !important;
margin-bottom: 15px !important;
}
#highlighted-add-data {
background: linear-gradient(135deg, #E0F2FE 0%, #B3E5FC 100%) !important;
border: 2px solid #81D4FA !important;
border-radius: 15px !important;
box-shadow: 0 10px 40px rgba(129, 212, 250, 0.3) !important;
margin: 20px 0 !important;
}
#highlighted-add-data .gr-accordion-header {
background: linear-gradient(135deg, #81D4FA 0%, #4FC3F7 100%) !important;
color: white !important;
font-weight: bold !important;
font-size: 1.2rem !important;
padding: 15px 20px !important;
border-radius: 15px 15px 0 0 !important;
}
#highlighted-add-data .gr-accordion-content {
background: rgba(255,255,255,0.95) !important;
border-radius: 0 0 15px 15px !important;
padding: 25px !important;
}
.gr-button {
border-radius: 10px !important;
font-weight: bold !important;
transition: all 0.3s ease !important;
}
.gr-button:hover {
transform: translateY(-2px) !important;
box-shadow: 0 5px 15px rgba(0,0,0,0.2) !important;
}
.gr-plot {
border-radius: 15px !important;
box-shadow: 0 8px 32px rgba(0,0,0,0.1) !important;
background: white !important;
padding: 20px !important;
}
.gr-dataframe {
border-radius: 15px !important;
box-shadow: 0 8px 32px rgba(0,0,0,0.1) !important;
background: white !important;
overflow: hidden !important;
}
.gr-accordion {
border-radius: 15px !important;
box-shadow: 0 8px 32px rgba(0,0,0,0.1) !important;
background: white !important;
margin: 15px 0 !important;
}
.gr-accordion-header {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%) !important;
color: white !important;
font-weight: bold !important;
padding: 15px 20px !important;
border-radius: 15px 15px 0 0 !important;
}
.gr-accordion-content {
background: rgba(255,255,255,0.95) !important;
border-radius: 0 0 15px 15px !important;
padding: 20px !important;
}
#submit-btn {
background: linear-gradient(135deg, #29B6F6 0%, #0288D1 100%) !important;
border: 2px solid #0277BD !important;
color: white !important;
font-weight: bold !important;
font-size: 1.1rem !important;
padding: 15px 30px !important;
border-radius: 12px !important;
box-shadow: 0 8px 25px rgba(41, 182, 246, 0.4) !important;
transition: all 0.3s ease !important;
}
#submit-btn:hover {
background: linear-gradient(135deg, #0288D1 0%, #0277BD 100%) !important;
transform: translateY(-3px) !important;
box-shadow: 0 12px 35px rgba(41, 182, 246, 0.6) !important;
}
#owner-controls {
background: linear-gradient(135deg, #FFE0E0 0%, #FFCDD2 100%) !important;
border: 2px solid #FF5722 !important;
border-radius: 15px !important;
box-shadow: 0 10px 40px rgba(255, 87, 34, 0.3) !important;
margin: 20px 0 !important;
}
#owner-controls .gr-accordion-header {
background: linear-gradient(135deg, #FF5722 0%, #D32F2F 100%) !important;
color: white !important;
font-weight: bold !important;
font-size: 1.2rem !important;
padding: 15px 20px !important;
border-radius: 15px 15px 0 0 !important;
}
#owner-controls .gr-accordion-content {
background: rgba(255,255,255,0.95) !important;
border-radius: 0 0 15px 15px !important;
padding: 25px !important;
}
#approve-btn {
background: linear-gradient(135deg, #4CAF50 0%, #2E7D32 100%) !important;
border: 2px solid #388E3C !important;
color: white !important;
font-weight: bold !important;
font-size: 1.1rem !important;
padding: 15px 30px !important;
border-radius: 12px !important;
box-shadow: 0 8px 25px rgba(76, 175, 80, 0.4) !important;
transition: all 0.3s ease !important;
}
#approve-btn:hover {
background: linear-gradient(135deg, #2E7D32 0%, #1B5E20 100%) !important;
transform: translateY(-3px) !important;
box-shadow: 0 12px 35px rgba(76, 175, 80, 0.6) !important;
}
#reject-btn {
background: linear-gradient(135deg, #F44336 0%, #C62828 100%) !important;
border: 2px solid #D32F2F !important;
color: white !important;
font-weight: bold !important;
font-size: 1.1rem !important;
padding: 15px 30px !important;
border-radius: 12px !important;
box-shadow: 0 8px 25px rgba(244, 67, 54, 0.4) !important;
transition: all 0.3s ease !important;
}
#reject-btn:hover {
background: linear-gradient(135deg, #C62828 0%, #B71C1C 100%) !important;
transform: translateY(-3px) !important;
box-shadow: 0 12px 35px rgba(244, 67, 54, 0.6) !important;
}
#guideline-section {
background: linear-gradient(135deg, #E8F5E8 0%, #C8E6C9 100%) !important;
border: 2px solid #4CAF50 !important;
border-radius: 15px !important;
box-shadow: 0 10px 40px rgba(76, 175, 80, 0.3) !important;
margin: 20px 0 !important;
}
#guideline-section .gr-accordion-header {
background: linear-gradient(135deg, #4CAF50 0%, #2E7D32 100%) !important;
color: white !important;
font-weight: bold !important;
font-size: 1.2rem !important;
padding: 15px 20px !important;
border-radius: 15px 15px 0 0 !important;
}
#guideline-section .gr-accordion-content {
background: rgba(255,255,255,0.95) !important;
border-radius: 0 0 15px 15px !important;
padding: 25px !important;
}
"""
with gr.Blocks(css=css, title="Watermark Leaderboard for LLMs") as demo:
# Header
gr.HTML("""
<div class="title">
πŸ† Watermark Leaderboard for LLMs πŸ†
</div>
<div class="subtitle">
πŸ“Š Interactive leaderboard for comparing watermark performance across different models and evaluation settings
</div>
""")
# Controls
with gr.Row():
with gr.Column(scale=1):
gr.HTML("<div style='text-align: center; margin-bottom: 15px;'><h3 style='color: #667eea; margin: 0; font-weight: bold;'>πŸ€– Model Selection</h3></div>")
model_selector = gr.Radio(
choices=["LLaMA3", "DeepSeek"],
value="LLaMA3",
label="Model",
info="Select the model to display"
)
with gr.Column(scale=1):
gr.HTML("<div style='text-align: center; margin-bottom: 15px;'><h3 style='color: #667eea; margin: 0; font-weight: bold;'>βš™οΈ Evaluation Setting</h3></div>")
metric_selector = gr.Radio(
choices=["Attack-free", "Watermark Removal", "Stealing Attack"],
value="Attack-free",
label="Setting",
info="Select the evaluation setting"
)
# Add Your Data Section (Highlighted)
with gr.Accordion("πŸš€ Add Your Data to the Leaderboard", open=False, elem_id="highlighted-add-data"):
gr.HTML("""
<div style='text-align: center; margin-bottom: 20px;'>
<h2 style='color: #0277BD; margin: 0; font-size: 1.5rem;'>πŸ“ Submit Your Watermark Performance Results</h2>
<p style='color: #374151; margin: 10px 0 0 0;'>Contribute to the community by sharing your watermark evaluation results</p>
</div>
<div style='background: #E3F2FD; border: 1px solid #2196F3; border-radius: 8px; padding: 15px; margin-bottom: 20px;'>
<h4 style='color: #1976D2; margin: 0 0 10px 0;'>πŸ“‹ Submission Requirements</h4>
<p style='color: #374151; margin: 0 0 8px 0;'>Provide at least one complete set of metrics:</p>
<ul style='color: #374151; margin: 0; padding-left: 20px;'>
<li><strong>Attack-free:</strong> Normalized Utility + Detection Rate</li>
<li><strong>Watermark Removal:</strong> Absolute Utility Degradation + Removal Detection Rate</li>
<li><strong>Stealing Attack:</strong> Adversary BERT Score + Adversary Detection Rate</li>
</ul>
</div>
""")
with gr.Row():
with gr.Column(scale=1):
# Basic Information
gr.HTML("<div style='text-align: center; margin-bottom: 15px;'><h3 style='color: #0277BD; margin: 0;'>πŸ“‹ Basic Information</h3></div>")
watermark_name = gr.Textbox(
label="Watermark Name",
placeholder="e.g., MyWatermark, Watermark-X",
info="Unique identifier for your watermark"
)
paper_link = gr.Textbox(
label="Paper Link (Optional)",
placeholder="https://arxiv.org/abs/xxxx.xxxxx or https://...",
info="Link to the paper describing this watermark method"
)
submission_model = gr.Radio(
choices=["LLaMA3", "DeepSeek"],
label="Model",
value="LLaMA3",
info="Select the model used"
)
with gr.Column(scale=1):
# Attack-free Metrics (Optional)
gr.HTML("<div style='text-align: center; margin-bottom: 15px;'><h3 style='color: #0277BD; margin: 0;'>⚑ Attack-free Metrics (Optional - Both Required if One is Provided)</h3></div>")
normalized_utility = gr.Number(
label="Normalized Utility",
value=None,
minimum=0.0,
maximum=1.0,
step=0.001,
info="Text quality metric (0.000 - 1.000)"
)
detection_rate = gr.Number(
label="Detection Rate (%)",
value=None,
minimum=0.0,
maximum=100.0,
step=0.001,
info="Watermark detection accuracy (0.000 - 100.000%)"
)
with gr.Row():
with gr.Column(scale=1):
# Watermark Removal Metrics (Optional)
gr.HTML("<div style='text-align: center; margin-bottom: 15px;'><h3 style='color: #0277BD; margin: 0;'>πŸ›‘οΈ Watermark Removal (Optional)</h3></div>")
absolute_utility_degradation = gr.Number(
label="Absolute Utility Degradation",
value=None,
minimum=0.0,
maximum=1.0,
step=0.001,
info="Resistance to removal attacks (0.000 - 1.000)"
)
removal_detection_rate = gr.Number(
label="Removal Detection Rate (%)",
value=None,
minimum=0.0,
maximum=100.0,
step=0.001,
info="Detection rate under removal attacks (0.000 - 100.000%)"
)
with gr.Column(scale=1):
# Stealing Attack Metrics (Optional)
gr.HTML("<div style='text-align: center; margin-bottom: 15px;'><h3 style='color: #0277BD; margin: 0;'>🎯 Stealing Attack (Optional)</h3></div>")
adversary_bert_score = gr.Number(
label="Adversary BERT Score",
value=None,
minimum=0.0,
maximum=1.0,
step=0.001,
info="Performance under adversarial conditions (0.000 - 1.000)"
)
adversary_detection_rate = gr.Number(
label="Adversary Detection Rate (%)",
value=None,
minimum=0.0,
maximum=100.0,
step=0.001,
info="Detection rate under adversarial attacks (0.000 - 100.000%)"
)
# Submit and Clear buttons
with gr.Row():
with gr.Column(scale=1):
submit_btn = gr.Button(
"πŸš€ Submit Data to Leaderboard",
variant="primary",
size="lg",
elem_id="submit-btn"
)
with gr.Column(scale=1):
clear_btn = gr.Button(
"πŸ—‘οΈ Clear Form",
variant="secondary",
size="lg"
)
# Status message
status_message = gr.Markdown("", visible=True)
# Scatter Plot
scatter_plot = gr.Plot(
label="Performance Scatter Plot",
show_label=True
)
# Table
table = gr.DataFrame(
label="Performance Table",
show_label=True,
interactive=False,
wrap=True
)
# Guideline and Metrics Explained Section (At bottom with light green background)
with gr.Accordion("πŸ“‹ Guideline for Submitting Watermark Performance Results", open=False, elem_id="guideline-section"):
gr.HTML("""
<div style="padding: 20px;">
<h3>Guideline for Submitting Watermark Performance Results</h3>
<h4>1. Datasets</h4>
<ul>
<li><strong>Text Generation (C4 dataset)</strong>
<ul>
<li>Training: first 20,000 samples</li>
<li>Testing: 13,860 samples</li>
<li>Reference script: <code>Files/Reproducibility/C4_dataset_download.py</code></li>
</ul>
</li>
<li><strong>Text Summarization (CNN/Daily Mail dataset)</strong>
<ul>
<li>Training: first 10,000–20,000 samples</li>
<li>Testing: 1,000 samples</li>
<li>Reference script: <code>Files/Reproducibility/CNN_dataset_download.py</code></li>
</ul>
</li>
</ul>
<h4>2. Models</h4>
<ul>
<li>Use open-source models available on Hugging Face:
<ul>
<li>DeepSeek: "deepseek-ai/deepseek-llm-7b-base"</li>
<li>LLaMA-3: "meta-llama/Meta-Llama-3-8B"</li>
</ul>
</li>
</ul>
<h4>3. Evaluation Settings</h4>
<ul>
<li><strong>(a) Attack-Free Setting</strong>
<ul>
<li>Generate 13,860 watermarked outputs on the C4 test set.</li>
<li>Report: Detection Rate and Normalized Utility (see Metrics).</li>
</ul>
</li>
<li><strong>(b) Watermark Removal Setting</strong>
<ul>
<li>Apply Dipper to paraphrase watermarked outputs.</li>
<li>Report:
<ul>
<li>Detection Rate after attack</li>
<li>Normalized Utility after attack</li>
<li>Absolute Utility Degradation (difference before vs. after attack)</li>
</ul>
</li>
<li>Reference scripts: <code>Files/Reproducibility/Attack_dipper.py</code></li>
</ul>
</li>
<li><strong>(c) Stealing Attack Setting</strong>
<ul>
<li>Generate 20,000 watermarked samples for training a surrogate model using LoRA.</li>
<li>Use the surrogate model for summarization on 1,000 test samples.</li>
<li>Report: Detection Rate and Normalized Utility on the surrogate's outputs.</li>
<li>Reference scripts: <code>Files/Reproducibility/Finetune_sum.py</code>, <code>Files/Reproducibility/Inference_sum.py</code></li>
</ul>
</li>
</ul>
<h4>4. Metrics</h4>
<ul>
<li><strong>Detection Rate</strong>
<ul>
<li>Average accuracy across the test set (e.g., 13,860 examples for text generation).</li>
<li>Use your own detector implementation.</li>
</ul>
</li>
<li><strong>Normalized Utility</strong>
<ul>
<li>Defined as the mean of:</li>
<li>BERTScore (<code>Files/Reproducibility/BERT_score.py</code>)</li>
<li>Entity Similarity Score (<code>Files/Reproducibility/Entity_similarity_score.py</code>)</li>
</ul>
</li>
<li><strong>Absolute Utility Degradation</strong>
<ul>
<li>The absolute change in Normalized Utility between attack-free and attacked outputs.</li>
</ul>
</li>
</ul>
<h4>5. Submission</h4>
<ul>
<li>You may submit results for one or more evaluation settings (Attack-Free, Removal, Stealing).</li>
<li>Please include:
<ul>
<li>Model(s) evaluated</li>
<li>Dataset(s) used</li>
<li>Scripts/configuration details if modified</li>
<li>Reported metrics in the required format</li>
</ul>
</li>
</ul>
<p><strong>Reproducibility codes are available in the Files tab of this Space.</strong></p>
</div>
""")
# Owner Approval Section (At the very bottom)
with gr.Accordion("πŸ”’ Owner Controls - Pending Submissions", open=False, elem_id="owner-controls"):
gr.HTML("""
<div style='text-align: center; margin-bottom: 20px;'>
<h2 style='color: #D32F2F; margin: 0; font-size: 1.5rem;'>πŸ›‘οΈ Administrator Approval Panel</h2>
<p style='color: #374151; margin: 10px 0 0 0;'>Review and approve pending submissions before they appear on the leaderboard</p>
</div>
""")
# Pending submissions table
pending_table = gr.DataFrame(
label="πŸ“‹ Pending Submissions",
show_label=True,
interactive=False,
wrap=True,
headers=["ID", "Name", "Model", "Paper Link", "Attack-free Utility", "Attack-free Detection",
"Removal Degradation", "Removal Detection", "Adversary BERT", "Adversary Detection", "Submitted At"]
)
# Admin authentication
admin_password_input = gr.Textbox(
label="πŸ” Admin Password",
placeholder="Enter admin password to access controls",
type="password",
info="Required for approval/rejection actions"
)
# Approval controls
with gr.Row():
with gr.Column(scale=1):
submission_id_input = gr.Textbox(
label="Submission ID",
placeholder="Enter submission ID to approve/reject",
info="Copy from the pending submissions table"
)
approve_btn = gr.Button(
"βœ… Approve Submission",
variant="primary",
size="lg",
elem_id="approve-btn"
)
with gr.Column(scale=1):
reject_btn = gr.Button(
"❌ Reject Submission",
variant="stop",
size="lg",
elem_id="reject-btn"
)
refresh_pending_btn = gr.Button(
"πŸ”„ Refresh Pending",
variant="secondary",
size="lg"
)
approval_status = gr.Markdown("", visible=True)
# Event handlers
model_selector.change(
fn=update_interface,
inputs=[model_selector, metric_selector],
outputs=[scatter_plot, table]
)
metric_selector.change(
fn=update_interface,
inputs=[model_selector, metric_selector],
outputs=[scatter_plot, table]
)
# Form submission handler
submit_btn.click(
fn=submit_watermark_data,
inputs=[
watermark_name,
submission_model,
paper_link,
normalized_utility,
detection_rate,
absolute_utility_degradation,
removal_detection_rate,
adversary_bert_score,
adversary_detection_rate
],
outputs=[status_message, scatter_plot, table]
)
# Clear form handler
clear_btn.click(
fn=clear_form,
outputs=[
watermark_name,
paper_link,
submission_model,
normalized_utility,
detection_rate,
absolute_utility_degradation,
removal_detection_rate,
adversary_bert_score,
adversary_detection_rate
]
)
# Add data button handler
# The add_data_button is removed, so this handler is no longer needed.
# The highlighted section is now always visible.
# Owner approval event handlers
approve_btn.click(
fn=approve_submission,
inputs=[submission_id_input, admin_password_input],
outputs=[approval_status, pending_table]
)
reject_btn.click(
fn=reject_submission,
inputs=[submission_id_input, admin_password_input],
outputs=[approval_status, pending_table]
)
refresh_pending_btn.click(
fn=load_pending_submissions,
outputs=[pending_table]
)
# Initial load
demo.load(
fn=lambda: update_interface("LLaMA3", "Attack-free"),
outputs=[scatter_plot, table]
)
# Load pending submissions on startup
demo.load(
fn=load_pending_submissions,
outputs=[pending_table]
)
# Clear admin password after actions for security
def clear_admin_password():
return gr.update(value="")
# Clear password after approve/reject actions
approve_btn.click(
fn=clear_admin_password,
outputs=[admin_password_input]
)
reject_btn.click(
fn=clear_admin_password,
outputs=[admin_password_input]
)
return demo
# Create and launch the interface
if __name__ == "__main__":
demo = create_interface()
demo.launch()