File size: 6,081 Bytes

3308bd8

import os
import sys
import requests
from pathlib import Path
import torch
import torch.nn as nn
from torchvision import datasets, transforms
from torchvision.models import resnet18
from safetensors.torch import load_file
import pandas as pd

# --------------------------------
# LOADING A MODEL (EXAMPLE: TARGET MODEL)
# --------------------------------

def make_model():
    model = resnet18(weights=None)
    model.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)
    model.maxpool = nn.Identity()
    model.fc = nn.Linear(model.fc.in_features, 100)
    return model

checkpoint_path = "path/to/your/model_checkpoint.safetensors"  # Replace with your model checkpoint path 
state_dict = load_file(checkpoint_path, device="cpu")

model = make_model() 
model.load_state_dict(state_dict, strict=True)
model.eval()

transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5071, 0.4867, 0.4408),
                         (0.2675, 0.2565, 0.2761)),
])

data_root = "path/to/cifar100"  # Replace with your CIFAR-100 dataset path, or where it should be downloaded
dataset = datasets.CIFAR100(root=data_root, train=False, download=True, transform=transform)
x, y = dataset[0]  # Example: get the first image and label

with torch.no_grad():
    logits = model(x.unsqueeze(0))

print("True label:", y)
print("Logits shape:", logits.shape)  # Should be [1, 100] for CIFAR-100
print("Logits:", logits)

# # --------------------------------
# # SUBMISSION FORMAT
# # --------------------------------

"""
The submission must be a .csv file with the following format:

-"id": ID of the subset (from 0 to 359)
-"score": Stealing confidence score for each image (float)
"""

# Example Submission:

subset_ids = list(range(360))  
confidence_scores = torch.rand(len(subset_ids)).tolist()
submission_df = pd.DataFrame({
    "id": subset_ids,
    "score": confidence_scores
})
submission_df.to_csv("example_submission.csv", index=None)

# --------------------------------
# SUBMISSION PROCESS
# --------------------------------

"""
Example submission script for the Stolen Model Detection Task.

Submission Requirements (read carefully to avoid automatic rejection):

1. CSV FORMAT
----------------
- The file **must be a CSV** with extension `.csv`.
- It must contain **exactly two columns**, named:
      id, score
  → Column names must match exactly (lowercase, no extra spaces).
  → Column order does not matter, but both must be present.

2. ROW COUNT AND IDENTIFIERS
-------------------------------
- Your file must contain **exactly 360 rows**.
- Each row corresponds to one unique `id` in the range **0–359** (inclusive).
- Every id must appear **exactly once**.
- Do **not** add, remove, or rename any IDs.
- Do **not** include duplicates or missing entries.
- The evaluator checks:
      id.min() == 0
      id.max() == 359
      id.unique().size == 360

3. STEALING CONFIDENCE SCORES
----------------------
- The `score` column must contain **numeric values** representing your model’s predicted confidence
  that the corresponding subset is a **stolen** model.

  Examples of valid score values:
    - Probabilities: values in [0.0, 1.0]
    - Raw model scores: any finite numeric values (will be ranked for TPR@FPR=0.05)

- Do **not** submit string labels like "yes"/"no" or "stolen"/"not stolen".
- The evaluator converts your `score` column to numeric using `pd.to_numeric()`.
  → Any non-numeric, NaN, or infinite entries will cause automatic rejection.

4. TECHNICAL LIMITS
----------------------
- Maximum file size: **20 MB**
- Encoding: UTF-8 recommended.
- Avoid extra columns, blank lines, or formulas.
- Ensure all values are numeric and finite.
- Supported data types: int, float (e.g., float32, float64)

5. VALIDATION SUMMARY
------------------------
Your submission will fail if:
- Columns don’t match exactly ("id", "score")
- Row count differs from 360
- Any id is missing, duplicated, or outside [0, 359]
- Any score value is NaN, Inf, or non-numeric
- File is too large or not a valid CSV

One key metric is computed:
  1. **TPR@FPR=0.05 (True Positive Rate at False Positive Rate = 0.05)** 
  — measures the ability to correctly identify stolen models while keeping the false positive rate at 5%.
"""

BASE_URL  = "http://35.192.205.84:80"
API_KEY   = "YOUR_API_KEY_HERE"  # replace with your actual API key

TASK_ID   = "19-stolen-model-detection"
FILE_PATH = "PATH/TO/YOUR/SUBMISSION.csv"  # replace with your actual file path

SUBMIT     = False  # Set to True to enable submission

def die(msg):
    print(f"{msg}", file=sys.stderr)
    sys.exit(1)

if SUBMIT:
    if not os.path.isfile(FILE_PATH):
        die(f"File not found: {FILE_PATH}")

    try:
        with open(FILE_PATH, "rb") as f:
            files = {
                # (fieldname) -> (filename, fileobj, content_type)
                "file": (os.path.basename(FILE_PATH), f, "csv"),
            }
            resp = requests.post(
                f"{BASE_URL}/submit/{TASK_ID}",
                headers={"X-API-Key": API_KEY},
                files=files,
                timeout=(10, 120),  # (connect timeout, read timeout)
            )
        # Helpful output even on non-2xx
        try:
            body = resp.json()
        except Exception:
            body = {"raw_text": resp.text}

        if resp.status_code == 413:
            die("Upload rejected: file too large (HTTP 413). Reduce size and try again.")

        resp.raise_for_status()

        submission_id = body.get("submission_id")
        print("Successfully submitted.")
        print("Server response:", body)
        if submission_id:
            print(f"Submission ID: {submission_id}")

    except requests.exceptions.RequestException as e:
        detail = getattr(e, "response", None)
        print(f"Submission error: {e}")
        if detail is not None:
            try:
                print("Server response:", detail.json())
            except Exception:
                print("Server response (text):", detail.text)
        sys.exit(1)