Upload 2 files

task_template.py

@@ -0,0 +1,314 @@
+import torch
+import pandas as pd
+import requests
+import sys
+import torchvision.models as models
+import os
+
+from transformers import AutoTokenizer, PreTrainedModel, AutoModelForSequenceClassification
+
+import utils
+
+# --------------------------------
+# DATASET
+# --------------------------------
+
+"""
+Dataset contents:
+
+- 1000 subsets of text data, each subset stored under the key "subset_{i}" where i ranges from 0 to 999.
+  Each subset is a dictionary with:
+    -"prompts": List of 100 prompts in the subset
+    -"labels": Tensor of true labels for the prompts in the subset, has shape (100)
+    -"subset_id": Integer ID of the subset (from 0 to 999)
+"""
+
+# Load the dataset
+dataset = torch.load("fulltuning.pt")
+
+# Example: Acessing subsets
+subset_0 = dataset["subset_0"]
+
+print("Subset 0 keys:", subset_0.keys())
+print("Subset ID:", subset_0["subset_id"])
+print("Labels length:", len(subset_0["labels"])) 
+print("First prompts:", subset_0["prompts"][:5])
+print("First 5 labels:", subset_0["labels"][:5])
+
+# --------------------------------
+# QUERYING THE CLASSIFIER
+# --------------------------------
+
+# This Code can be used to load and query the fully fine-tuned models. You also need to the available utils.py file.
+
+#|---------------------------------------------------------------------------------------------------|
+#|    NOTE: "Missing or unexpected params" warnings are no reason for concern. They stem from the    | 
+#|    fact that the model is first loaded without a classifier head, which is added afterwards.      |
+#|---------------------------------------------------------------------------------------------------|
+
+# Use this tokenizer for OLMO...
+
+tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-1B-hf", trust_remote_code=True)
+
+# ...and this one for Pythia
+
+tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-410m", trust_remote_code=True)
+
+
+tokenizer.padding_side = "left"
+
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+
+
+# Usage example (fulltuning):
+
+model_path = "olmo-fulltuning"
+
+model = utils.get_fulltuning_model(model_path, model_type="olmo") # model_type can be "olmo" or "pythia"
+
+example_prompt = "I think, therefore I am.\n\nI am."
+
+inputs = tokenizer(example_prompt, return_tensors="pt", truncation=True)
+inputs = {k: v.to(model.device) for k, v in inputs.items()}
+with torch.no_grad():
+    outputs = model(**inputs)
+
+logits = outputs.logits
+
+print(f"Logits shape: {logits.shape}")
+print(f"Logits: {logits}")
+
+
+
+# Usage example (softprompt):
+
+model_path = "olmo-softprompt"
+
+model = utils.get_peft_model(model_path, model_type="olmo") # model_type can be "olmo" or "pythia"
+
+example_prompt = "I think, but do I exist?\n\nSince you think, you exist."
+
+inputs = tokenizer(example_prompt, return_tensors="pt", truncation=True)
+inputs = {k: v.to(model.device) for k, v in inputs.items()}
+with torch.no_grad():
+    outputs = model(**inputs)
+
+logits = outputs.logits
+
+print(f"Logits shape: {logits.shape}")
+print(f"Logits: {logits}")
+
+
+
+# Usage example (lora):
+
+model_path = "olmo-lora"
+
+model = utils.get_peft_model(model_path, model_type="olmo") # model_type can be "olmo" or "pythia"
+
+example_prompt = "Who am I?\n\nWhat am I?"
+
+inputs = tokenizer(example_prompt, return_tensors="pt", truncation=True)
+inputs = {k: v.to(model.device) for k, v in inputs.items()}
+with torch.no_grad():
+    outputs = model(**inputs)
+
+logits = outputs.logits
+
+print(f"Logits shape: {logits.shape}")
+print(f"Logits: {logits}")
+
+
+
+# Usage example (lastlayer):
+
+model_path = "olmo-lastlayer"
+
+model = utils.get_peft_model(model_path, model_type="olmo") # model_type can be "olmo" or "pythia"
+
+example_prompt = "I love to exist!"
+
+inputs = tokenizer(example_prompt, return_tensors="pt", truncation=True)
+inputs = {k: v.to(model.device) for k, v in inputs.items()}
+with torch.no_grad():
+    outputs = model(**inputs)
+
+logits = outputs.logits
+
+print(f"Logits shape: {logits.shape}")
+print(f"Logits: {logits}")
+
+
+
+# Usage example (prefix):
+
+model_path = "olmo-prefix"
+
+model = utils.get_peft_model(model_path, model_type="olmo") # model_type can be "olmo" or "pythia"
+
+example_prompt = "I will exist yesterday."
+
+inputs = tokenizer(example_prompt, return_tensors="pt", truncation=True)
+inputs = {k: v.to(model.device) for k, v in inputs.items()}
+with torch.no_grad():
+    outputs = utils.forward_peft_seqcls(model, **inputs)
+
+logits = outputs.logits
+
+print(f"Logits shape: {logits.shape}")
+print(f"Logits: {logits}")
+
+# --------------------------------
+# SUBMISSION FORMAT
+# --------------------------------
+
+"""
+The submission must be a .csv file with the following format:
+
+-"type": Name of the model (e.g., "softprompt", "fulltuning", etc.)
+-"subset_id": ID of the subset (from 0 to 999, per type)
+-"membership": Membership score for each subset (float)
+"""
+
+# Example Submission:
+
+types = ["softprompt", "fulltuning", "lora", "lastlayer", "prefix"]
+type_list = []
+
+for t in types:
+    type_list.extend([t] * 1000)
+
+subset_ids = []
+for _ in types:
+    subset_ids.extend(list(range(1000)))
+
+membership_scores = torch.rand(5000).tolist()
+submission_df = pd.DataFrame({
+    "type": type_list,
+    "subset_id": subset_ids,
+    "membership": membership_scores
+})
+submission_df.to_csv("example_submission.csv", index=None)
+
+# --------------------------------
+# SUBMISSION PROCESS
+# --------------------------------
+
+"""
+Example submission script for the LLM Dataset Membership Inference Task.
+
+Submission Requirements (read carefully to avoid automatic rejection):
+
+1. CSV FORMAT
+----------------
+- The file **must be a CSV** with extension `.csv`.
+- It must contain **exactly three columns**, named:
+      type, subset_id, membership
+  → Column names must match exactly (lowercase, no extra spaces).
+  → Column order does not matter, but all three must be present.
+
+2. ROW COUNT AND IDENTIFIERS
+-------------------------------
+- Your file must contain **exactly 5000 rows**.
+- Each row corresponds to one unique `subset_id`/`type` pair, with ids in the range **0–999** (inclusive).
+- Every subset_id must appear **exactly once** for each type.
+- Do **not** add, remove, or rename any IDs.
+- Do **not** include duplicates or missing entries.
+- The evaluator checks:
+      subset_id.min() == 0
+      subset_id.max() == 999
+      subset_id.unique().size == 1000
+
+3. MEMBERSHIP SCORES
+----------------------
+- The `membership` column must contain **numeric values** representing your model’s predicted confidence
+  that the corresponding subset is a **member** of the training set.
+
+  Examples of valid membership values:
+    - Probabilities: values in [0.0, 1.0]
+    - Raw model scores: any finite numeric values (will be ranked for TPR@FPR=0.05)
+
+- Do **not** submit string labels like "yes"/"no" or "member"/"non-member".
+- The evaluator converts your `membership` column to numeric using `pd.to_numeric()`.
+  → Any non-numeric, NaN, or infinite entries will cause automatic rejection.
+
+4. TECHNICAL LIMITS
+----------------------
+- Maximum file size: **20 MB**
+- Encoding: UTF-8 recommended.
+- Avoid extra columns, blank lines, or formulas.
+- Ensure all values are numeric and finite.
+- Supported data types: int, float (e.g., float32, float64)
+
+5. VALIDATION SUMMARY
+------------------------
+Your submission will fail if:
+- Columns don’t match exactly ("type", "subset_id", "membership")
+- Row count differs from 5000
+- Any type name is unexpected or not in the allowed set
+- Any subset_id is missing, duplicated, or outside [0, 999] for any type
+- Any membership value is NaN, Inf, or non-numeric
+- File is too large or not a valid CSV
+
+Two key metrics are computed:
+  1. **ROC-AUC (Area Under the ROC Curve)** — measures overall discriminative ability.
+  2. **TPR@FPR=0.05** — true positive rate when the false positive rate is at 5%.
+
+"""
+
+BASE_URL  = "http://35.192.205.84:80"
+API_KEY   = "77ab4a807d03d48e1d85881b27496871"  # replace with your actual API key
+
+TASK_ID   = "14-llm-dataset-inference"
+FILE_PATH = "example_submission.csv"  # replace with your actual file path
+
+SUBMIT     = False  # Set to True to enable submission
+
+def die(msg):
+    print(f"{msg}", file=sys.stderr)
+    sys.exit(1)
+
+if SUBMIT:
+    if not os.path.isfile(FILE_PATH):
+        die(f"File not found: {FILE_PATH}")
+
+    try:
+        with open(FILE_PATH, "rb") as f:
+            files = {
+                # (fieldname) -> (filename, fileobj, content_type)
+                "file": (os.path.basename(FILE_PATH), f, "csv"),
+            }
+            resp = requests.post(
+                f"{BASE_URL}/submit/{TASK_ID}",
+                headers={"X-API-Key": API_KEY},
+                files=files,
+                timeout=(10, 120),  # (connect timeout, read timeout)
+            )
+        # Helpful output even on non-2xx
+        try:
+            body = resp.json()
+        except Exception:
+            body = {"raw_text": resp.text}
+
+        if resp.status_code == 413:
+            die("Upload rejected: file too large (HTTP 413). Reduce size and try again.")
+
+        resp.raise_for_status()
+
+        submission_id = body.get("submission_id")
+        print("Successfully submitted.")
+        print("Server response:", body)
+        if submission_id:
+            print(f"Submission ID: {submission_id}")
+
+    except requests.exceptions.RequestException as e:
+        detail = getattr(e, "response", None)
+        print(f"Submission error: {e}")
+        if detail is not None:
+            try:
+                print("Server response:", detail.json())
+            except Exception:
+                print("Server response (text):", detail.text)
+        sys.exit(1)
+

utils.py

@@ -0,0 +1,232 @@
+from transformers import OlmoModel, OlmoPreTrainedModel, GenerationMixin, AutoConfig, AutoModelForSequenceClassification
+from transformers.modeling_outputs import SequenceClassifierOutputWithPast
+import torch
+
+from peft import PeftModel, PeftConfig
+
+from transformers import AutoConfig
+
+import logging
+from contextlib import contextmanager
+from types import SimpleNamespace
+
+# The custom model for using Olmo with a sequence classification task
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+class OlmoForSequenceClassification(OlmoPreTrainedModel, GenerationMixin):
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = OlmoModel(config)
+        self.num_labels = config.num_labels
+        self.classifier = torch.nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: torch.Tensor | None = None,
+        labels: torch.LongTensor | None = None,
+        **kwargs,
+    ) -> SequenceClassifierOutputWithPast:
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            **kwargs,
+        )
+        logits = self.classifier(outputs.last_hidden_state)
+        pooled_logits = logits[:, -1]   # NOTE: tokenizer.padding_side must be 'left'
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                pooled_logits=pooled_logits,
+                config=self.config,
+            )
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+# The function for loading a fulltuning model
+
+def get_fulltuning_model(model_path, model_type="olmo"):
+    if model_type == "olmo":
+        model = OlmoForSequenceClassification.from_pretrained(
+            model_path,
+            trust_remote_code=True,
+            torch_dtype=torch.float32,
+        ).to("cuda" if torch.cuda.is_available() else "cpu")
+        model.eval()
+    elif model_type == "pythia":
+        cfg = AutoConfig.from_pretrained(model_path, num_labels=3)
+        model = AutoModelForSequenceClassification.from_pretrained(
+            model_path,
+            config=cfg,
+            torch_dtype=torch.float32,
+        ).to(device)
+    else:
+        raise ValueError(f"Unsupported model_type: {model_type}")
+
+    return model
+
+# The following function is used to suppress a "missing or unexpected params" warning.
+# This warning is no reason for concern. It stems from the fact that the model is first loaded 
+# without a classifier head, which is added afterwards.
+
+class DropLoadReport(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        return "LOAD REPORT" not in record.getMessage()
+
+@contextmanager
+def suppress_load_report_only():
+    f = DropLoadReport()
+
+    names = [
+        "transformers.modeling_utils",
+        "transformers.modeling_tf_pytorch_utils",
+        "transformers",
+    ]
+    loggers = [logging.getLogger(n) for n in names]
+
+    for lg in loggers:
+        lg.addFilter(f)
+    try:
+        yield
+    finally:
+        for lg in loggers:
+            lg.removeFilter(f)
+
+# The function for loading a softprompt model
+
+def get_peft_model(model_path, model_type="olmo"):
+    peft_config = PeftConfig.from_pretrained(model_path)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    if model_type == "olmo":
+        config = AutoConfig.from_pretrained(
+            peft_config.base_model_name_or_path,
+            trust_remote_code=True,
+            num_labels=2,
+        )
+        with suppress_load_report_only():
+            base = OlmoForSequenceClassification.from_pretrained(
+                peft_config.base_model_name_or_path,
+                trust_remote_code=True,
+                torch_dtype=torch.float32,
+                config=config,
+            ).to(device)
+
+    elif model_type == "pythia":
+        config = AutoConfig.from_pretrained(
+            peft_config.base_model_name_or_path,
+            num_labels=2,
+        )
+        with suppress_load_report_only():
+            base = AutoModelForSequenceClassification.from_pretrained(
+                peft_config.base_model_name_or_path,
+                config=config,
+                torch_dtype=torch.float32,
+            ).to(device)
+    else:
+        raise ValueError(f"Unsupported model_type: {model_type}")
+
+    with suppress_load_report_only():
+        model = PeftModel.from_pretrained(base, model_path).to(device)
+
+    model.is_prefix_tuning = str(peft_config.peft_type) == "PeftType.PREFIX_TUNING"
+
+    # helpful for batching / last-token pooling
+    if getattr(model.config, "pad_token_id", None) is None and getattr(model.config, "eos_token_id", None) is not None:
+        model.config.pad_token_id = model.config.eos_token_id
+    if hasattr(model, "base_model") and hasattr(model.base_model, "config"):
+        if getattr(model.base_model.config, "pad_token_id", None) is None and getattr(model.base_model.config, "eos_token_id", None) is not None:
+            model.base_model.config.pad_token_id = model.base_model.config.eos_token_id
+
+    model.eval()
+    return model
+
+# This function helps when loading prefix finetuned models
+
+def forward_peft_seqcls(model, **inputs):
+    if not getattr(model, "is_prefix_tuning", False):
+        return model(**inputs, use_cache=False)
+
+    input_ids = inputs.get("input_ids", None)
+    attention_mask = inputs.get("attention_mask", None)
+    inputs_embeds = inputs.get("inputs_embeds", None)
+    labels = inputs.get("labels", None)
+    output_attentions = inputs.get("output_attentions", None)
+    output_hidden_states = inputs.get("output_hidden_states", None)
+    return_dict = inputs.get("return_dict", True)
+
+    if input_ids is not None:
+        batch_size = input_ids.shape[0]
+    elif inputs_embeds is not None:
+        batch_size = inputs_embeds.shape[0]
+    else:
+        raise ValueError("Either input_ids or inputs_embeds must be provided.")
+
+    past_key_values = model.get_prompt(batch_size)
+
+    if attention_mask is not None:
+        num_virtual_tokens = model.active_peft_config.num_virtual_tokens
+        prefix_attention_mask = torch.ones(
+            batch_size,
+            num_virtual_tokens,
+            device=attention_mask.device,
+            dtype=attention_mask.dtype,
+        )
+        attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=1)
+
+    try:
+        return model.base_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            past_key_values=past_key_values,
+            use_cache=False,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+    except TypeError:
+        pass
+
+    transformer_backbone = model.base_model.get_submodule(model.transformer_backbone_name)
+
+    outputs = transformer_backbone(
+        input_ids=input_ids,
+        attention_mask=attention_mask,
+        inputs_embeds=inputs_embeds,
+        past_key_values=past_key_values,
+        use_cache=False,
+        output_attentions=output_attentions,
+        output_hidden_states=output_hidden_states,
+        return_dict=True,
+    )
+
+    hidden_states = outputs[0]
+
+    if "dropout" in [name for name, _ in model.base_model.named_children()]:
+        hidden_states = model.base_model.dropout(hidden_states)
+
+    cls_layer = model.base_model.get_submodule(model.cls_layer_name)
+    token_logits = cls_layer(hidden_states)
+
+    logits = token_logits[:, -1]
+
+    return SimpleNamespace(
+        logits=logits,
+        hidden_states=getattr(outputs, "hidden_states", None),
+        attentions=getattr(outputs, "attentions", None),
+    )