| import io |
| import os |
| import urllib.request |
| from functools import lru_cache |
|
|
| os.environ.setdefault("MPLCONFIGDIR", "/tmp/matplotlib") |
|
|
| import gradio as gr |
| import numpy as np |
| from PIL import Image |
|
|
| try: |
| import spaces |
| except ImportError: |
| class _SpacesFallback: |
| @staticmethod |
| def GPU(duration=None): |
| def decorator(fn): |
| return fn |
|
|
| return decorator |
|
|
| spaces = _SpacesFallback() |
|
|
|
|
| APP_TITLE = "ResNet50 Adversarial Image Playground" |
| IMAGE_SIZE = 224 |
| RESIZE_SHORT_EDGE = 256 |
| PRETRAINED = "Pretrained ImageNet" |
| RANDOM = "Random initialization" |
| DEFAULT_TARGET = "76: tarantula" |
| VISIBLE_TARGET = "331: hare" |
| ROBUST_CHECKPOINT_URL = "http://6.869.csail.mit.edu/fa19/psets19/pset6/imagenet_l2_3_0.pt" |
| CURATED_TARGETS = [ |
| "76: tarantula", |
| "282: tiger cat", |
| "263: Pembroke", |
| "331: hare", |
| "281: tabby", |
| "285: Egyptian cat", |
| "207: golden retriever", |
| "340: zebra", |
| "859: toaster", |
| "954: banana", |
| ] |
| REAL_SAMPLE_IMAGES = [ |
| { |
| "class_id": 76, |
| "label": "tarantula", |
| "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/9/94/Brachypelma_vagans_p1.jpg/250px-Brachypelma_vagans_p1.jpg", |
| "source": "https://en.wikipedia.org/wiki/Tarantula", |
| }, |
| { |
| "class_id": 282, |
| "label": "tiger cat / tabby", |
| "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/4d/Cat_November_2010-1a.jpg/250px-Cat_November_2010-1a.jpg", |
| "source": "https://en.wikipedia.org/wiki/Tabby_cat", |
| }, |
| { |
| "class_id": 263, |
| "label": "Pembroke", |
| "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/9/99/Welsh_Pembroke_Corgi.jpg/250px-Welsh_Pembroke_Corgi.jpg", |
| "source": "https://en.wikipedia.org/wiki/Pembroke_Welsh_Corgi", |
| }, |
| { |
| "class_id": 331, |
| "label": "hare", |
| "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/4a/Feldhase%2C_Lepus_europaeus_3a.JPG/250px-Feldhase%2C_Lepus_europaeus_3a.JPG", |
| "source": "https://en.wikipedia.org/wiki/European_hare", |
| }, |
| { |
| "class_id": 207, |
| "label": "golden retriever", |
| "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/b/bd/Golden_Retriever_Dukedestiny01_drvd.jpg/250px-Golden_Retriever_Dukedestiny01_drvd.jpg", |
| "source": "https://en.wikipedia.org/wiki/Golden_Retriever", |
| }, |
| { |
| "class_id": 340, |
| "label": "zebra", |
| "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/9/96/Plains_Zebra_Equus_quagga_cropped.jpg/250px-Plains_Zebra_Equus_quagga_cropped.jpg", |
| "source": "https://en.wikipedia.org/wiki/Zebra", |
| }, |
| { |
| "class_id": 859, |
| "label": "toaster", |
| "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/e/ec/Consumer_Reports_-_Hamilton_Beach_Digital_toaster.tiff/lossless-page1-250px-Consumer_Reports_-_Hamilton_Beach_Digital_toaster.tiff.png", |
| "source": "https://en.wikipedia.org/wiki/Toaster", |
| }, |
| { |
| "class_id": 954, |
| "label": "banana", |
| "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/de/Bananavarieties.jpg/250px-Bananavarieties.jpg", |
| "source": "https://en.wikipedia.org/wiki/Banana", |
| }, |
| ] |
|
|
|
|
| PREPARE_IMAGE_CODE = """normalize = transforms.Normalize( |
| mean=[0.485, 0.456, 0.406], |
| std=[0.229, 0.224, 0.225], |
| ) |
| |
| def prepare_image(image): |
| image = resize_short_edge(image, 256) |
| image = center_crop(image, 224) |
| tensor_img = transforms.functional.to_tensor(image) |
| tensor_img = normalize(tensor_img) |
| return torch.unsqueeze(tensor_img, 0) |
| """ |
|
|
| SOFTMAX_CODE = """def output2prob(output): |
| prob = torch.nn.functional.softmax(output, dim=1) |
| return prob |
| """ |
|
|
| ATTACK_CODE = """def targeted_attack(model, x_pixels, target_id, eps=8/255, alpha=1/255): |
| x_adv = x_pixels.clone() |
| target = torch.tensor([target_id]) |
| |
| for _ in range(n_iter): |
| x_adv.requires_grad_(True) |
| logits = model(normalize(x_adv)) |
| loss = torch.nn.functional.cross_entropy(logits, target) |
| gradient, = torch.autograd.grad(loss, x_adv) |
| |
| # Targeted attack: move the image in the direction that lowers |
| # the loss for the target class. |
| x_adv = x_adv - alpha * gradient.sign() |
| delta = torch.clamp(x_adv - x_pixels, -eps, eps) |
| x_adv = torch.clamp(x_pixels + delta, 0, 1).detach() |
| |
| return x_adv |
| """ |
|
|
|
|
| @lru_cache(maxsize=1) |
| def torch_stack(): |
| import torch |
| import torch.nn.functional as F |
| import torchvision.models as models |
|
|
| torch.set_num_threads(max(1, min(4, os.cpu_count() or 1))) |
| device = torch.device("cpu") |
| mean = torch.tensor([0.485, 0.456, 0.406], device=device).view(1, 3, 1, 1) |
| std = torch.tensor([0.229, 0.224, 0.225], device=device).view(1, 3, 1, 1) |
| return torch, F, models, device, mean, std |
|
|
|
|
| @lru_cache(maxsize=1) |
| def class_names(): |
| _, _, models, _, _, _ = torch_stack() |
| return list(models.ResNet50_Weights.DEFAULT.meta["categories"]) |
|
|
|
|
| @lru_cache(maxsize=1) |
| def target_choices(): |
| return CURATED_TARGETS |
|
|
|
|
| def real_gallery_items(): |
| return [(item["image_url"], f'{item["class_id"]}: {item["label"]}') for item in REAL_SAMPLE_IMAGES] |
|
|
|
|
| def real_source_rows(): |
| return [[item["class_id"], item["label"], item["source"]] for item in REAL_SAMPLE_IMAGES] |
|
|
|
|
| @lru_cache(maxsize=16) |
| def load_real_image(url): |
| request = urllib.request.Request(url, headers={"User-Agent": "ai-workshop-resnet-playground/1.0"}) |
| with urllib.request.urlopen(request, timeout=20) as response: |
| return Image.open(io.BytesIO(response.read())).convert("RGB") |
|
|
|
|
| def real_image_by_index(index): |
| index = max(0, min(int(index), len(REAL_SAMPLE_IMAGES) - 1)) |
| return load_real_image(REAL_SAMPLE_IMAGES[index]["image_url"]) |
|
|
|
|
| def category_rows(query="", limit=1000): |
| q = str(query or "").strip().lower() |
| rows = [] |
| for index, name in enumerate(class_names()): |
| if not q or q in name.lower() or q == str(index): |
| rows.append([index, name]) |
| if len(rows) >= int(limit): |
| break |
| return rows |
|
|
|
|
| @lru_cache(maxsize=2) |
| def load_model(weight_mode): |
| _, _, models, device, _, _ = torch_stack() |
| status = "" |
| if weight_mode == RANDOM: |
| model = models.resnet50(weights=None) |
| status = "Randomly initialized ResNet50. Predictions are intentionally not meaningful." |
| else: |
| try: |
| model = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) |
| status = "Pretrained ResNet50 loaded from torchvision ImageNet weights." |
| except Exception as exc: |
| model = models.resnet50(weights=None) |
| status = f"Could not load pretrained weights: {exc}. Using random weights instead." |
|
|
| model.to(device) |
| model.eval() |
| for param in model.parameters(): |
| param.requires_grad_(False) |
| return model, status |
|
|
|
|
| @lru_cache(maxsize=1) |
| def load_robust_model(): |
| torch, _, models, device, _, _ = torch_stack() |
| model = models.resnet50(weights=None) |
|
|
| try: |
| checkpoint = torch.hub.load_state_dict_from_url( |
| ROBUST_CHECKPOINT_URL, |
| map_location=device, |
| progress=False, |
| file_name="imagenet_l2_3_0.pt", |
| ) |
| state = checkpoint["model"] if isinstance(checkpoint, dict) and "model" in checkpoint else checkpoint |
| cleaned_state = {} |
| for name, value in state.items(): |
| if "model." in name: |
| cleaned_state[name.split("model.", 1)[1]] = value |
| model.load_state_dict(cleaned_state) |
| status = "Robust ResNet50 checkpoint loaded from the Problem 4e source." |
| normalize_inputs = False |
| except Exception as exc: |
| model, fallback_status = load_model(PRETRAINED) |
| status = f"Could not load the robust checkpoint: {exc}. Using pretrained ResNet50 instead." |
| normalize_inputs = True |
|
|
| model.to(device) |
| model.eval() |
| for param in model.parameters(): |
| param.requires_grad_(False) |
| return model, normalize_inputs, status |
|
|
|
|
| def rgb_image(image): |
| if image is None: |
| return real_image_by_index(0) |
| if isinstance(image, np.ndarray): |
| image = Image.fromarray(image) |
| return image.convert("RGB") |
|
|
|
|
| def resize_short_edge(image, short_edge=RESIZE_SHORT_EDGE): |
| width, height = image.size |
| scale = short_edge / min(width, height) |
| new_size = (round(width * scale), round(height * scale)) |
| return image.resize(new_size, Image.Resampling.BICUBIC) |
|
|
|
|
| def center_crop(image, size=IMAGE_SIZE): |
| width, height = image.size |
| left = (width - size) // 2 |
| top = (height - size) // 2 |
| return image.crop((left, top, left + size, top + size)) |
|
|
|
|
| def prepare_pil_crop(image): |
| return center_crop(resize_short_edge(rgb_image(image))) |
|
|
|
|
| def image_to_pixels(image): |
| torch, _, _, device, _, _ = torch_stack() |
| image = prepare_pil_crop(image) |
| arr = np.asarray(image).astype(np.float32) / 255.0 |
| tensor = torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0).to(device) |
| return tensor |
|
|
|
|
| def pixels_to_image(tensor): |
| arr = tensor.detach().cpu().clamp(0, 1).squeeze(0).permute(1, 2, 0).numpy() |
| arr = (arr * 255).round().astype(np.uint8) |
| return Image.fromarray(arr, mode="RGB") |
|
|
|
|
| def normalize(pixel_tensor): |
| _, _, _, _, mean, std = torch_stack() |
| return (pixel_tensor - mean) / std |
|
|
|
|
| def model_input(pixel_tensor, normalize_inputs=True): |
| return normalize(pixel_tensor) if normalize_inputs else pixel_tensor |
|
|
|
|
| def parse_target(label): |
| try: |
| return int(str(label).split(":", 1)[0]) |
| except Exception: |
| return 76 |
|
|
|
|
| def top_prediction_rows(logits, score_mode="probability", top_k=5): |
| torch, F, _, _, _, _ = torch_stack() |
| probs = F.softmax(logits, dim=1) |
| source = probs if score_mode == "probability" else logits |
| k = max(1, min(int(top_k), logits.shape[1])) |
| values, indices = torch.topk(source, k=k, dim=1) |
| rows = [] |
| names = class_names() |
| for rank, (value, class_id) in enumerate(zip(values[0], indices[0]), start=1): |
| cid = int(class_id) |
| rows.append( |
| [ |
| rank, |
| cid, |
| names[cid], |
| round(float(probs[0, cid]), 6), |
| round(float(logits[0, cid]), 4), |
| round(float(value), 6), |
| ] |
| ) |
| return rows |
|
|
|
|
| def classify_pixels(model, pixel_tensor): |
| torch, _, _, _, _, _ = torch_stack() |
| with torch.inference_mode(): |
| return model(normalize(pixel_tensor)) |
|
|
|
|
| def classify_pixels_with_mode(model, pixel_tensor, normalize_inputs=True): |
| torch, _, _, _, _, _ = torch_stack() |
| with torch.inference_mode(): |
| return model(model_input(pixel_tensor, normalize_inputs)) |
|
|
|
|
| def classify_image_core(image, weight_mode=PRETRAINED, score_mode="probability", top_k=5): |
| model, status = load_model(weight_mode) |
| pixels = image_to_pixels(image) |
| logits = classify_pixels(model, pixels) |
| rows = top_prediction_rows(logits, score_mode, top_k) |
| return pixels_to_image(pixels), rows, status |
|
|
|
|
| @spaces.GPU(duration=120) |
| def classify_image(image, weight_mode=PRETRAINED, score_mode="probability", top_k=5): |
| return classify_image_core(image, weight_mode, score_mode, top_k) |
|
|
|
|
| def initial_classifier_view(): |
| return prepare_pil_crop(real_image_by_index(0)), [], "Choose an image, then run the classifier." |
|
|
|
|
| def classify_real_sample(evt: gr.SelectData): |
| index = evt.index if isinstance(evt.index, int) else 0 |
| return classify_image_core(real_image_by_index(index), PRETRAINED, "probability", 5) |
|
|
|
|
| def difference_image(original, attacked, epsilon_pixels): |
| torch, _, _, _, _, _ = torch_stack() |
| diff = torch.abs(attacked - original).mean(dim=1).squeeze(0).detach().cpu().numpy() |
| scale = max(float(epsilon_pixels) / 255.0, 1e-6) |
| heat = np.clip(diff / scale, 0, 1) |
| rgb = np.zeros((IMAGE_SIZE, IMAGE_SIZE, 3), dtype=np.uint8) |
| rgb[:, :, 0] = (255 * heat).astype(np.uint8) |
| rgb[:, :, 1] = (210 * np.sqrt(heat)).astype(np.uint8) |
| rgb[:, :, 2] = (35 * (1 - heat)).astype(np.uint8) |
| return Image.fromarray(rgb, mode="RGB") |
|
|
|
|
| def targeted_attack_core( |
| image, |
| target_label, |
| iterations, |
| epsilon_pixels, |
| step_pixels=1, |
| weight_mode=PRETRAINED, |
| top_k=5, |
| ): |
| torch, F, _, device, _, _ = torch_stack() |
| model, status = load_model(weight_mode) |
| original = image_to_pixels(image) |
| attacked = original.clone().detach() |
| target_id = parse_target(target_label) |
| target = torch.tensor([target_id], device=device) |
| eps = float(epsilon_pixels) / 255.0 |
| alpha = float(step_pixels) / 255.0 |
| iterations = max(1, int(iterations)) |
| trace = [] |
|
|
| start_logits = classify_pixels(model, original) |
| start_prob = float(F.softmax(start_logits, dim=1)[0, target_id]) |
|
|
| for index in range(iterations): |
| attacked.requires_grad_(True) |
| logits = model(normalize(attacked)) |
| loss = F.cross_entropy(logits, target) |
| gradient, = torch.autograd.grad(loss, attacked) |
|
|
| with torch.no_grad(): |
| attacked = attacked - alpha * gradient.sign() |
| delta = torch.clamp(attacked - original, -eps, eps) |
| attacked = torch.clamp(original + delta, 0, 1).detach() |
|
|
| if index in {0, iterations // 2, iterations - 1}: |
| with torch.inference_mode(): |
| trace_logits = model(normalize(attacked)) |
| trace_prob = F.softmax(trace_logits, dim=1) |
| top_id = int(torch.argmax(trace_prob, dim=1)[0]) |
| trace.append( |
| [ |
| index + 1, |
| class_names()[target_id], |
| round(float(trace_prob[0, target_id]), 6), |
| class_names()[top_id], |
| round(float(trace_prob[0, top_id]), 6), |
| ] |
| ) |
|
|
| final_logits = classify_pixels(model, attacked) |
| final_prob = float(F.softmax(final_logits, dim=1)[0, target_id]) |
| before_rows = top_prediction_rows(start_logits, "probability", top_k) |
| after_rows = top_prediction_rows(final_logits, "probability", top_k) |
| summary = ( |
| f"{status}\n" |
| f"Target class {target_id} ({class_names()[target_id]}): " |
| f"{start_prob:.4f} -> {final_prob:.4f} probability after {iterations} iterations. " |
| f"Perturbation budget: +/-{float(epsilon_pixels):.1f} pixel values." |
| ) |
| return ( |
| pixels_to_image(original), |
| pixels_to_image(attacked), |
| difference_image(original, attacked, epsilon_pixels), |
| before_rows, |
| after_rows, |
| trace, |
| summary, |
| ) |
|
|
|
|
| def visible_change_core(image, target_label, iterations, epsilon_pixels): |
| torch, F, _, device, _, _ = torch_stack() |
| model, normalize_inputs, status = load_robust_model() |
| original = image_to_pixels(image) |
| changed = original.clone().detach() |
| target_id = parse_target(target_label) |
| target = torch.tensor([target_id], device=device) |
| eps = float(epsilon_pixels) / 255.0 |
| alpha = 2.0 / 255.0 |
| iterations = max(1, int(iterations)) |
| trace = [] |
|
|
| start_logits = classify_pixels_with_mode(model, original, normalize_inputs) |
| start_prob = float(F.softmax(start_logits, dim=1)[0, target_id]) |
|
|
| for index in range(iterations): |
| changed.requires_grad_(True) |
| logits = model(model_input(changed, normalize_inputs)) |
| loss = F.cross_entropy(logits, target) |
| gradient, = torch.autograd.grad(loss, changed) |
|
|
| with torch.no_grad(): |
| changed = changed - alpha * gradient.sign() |
| delta = torch.clamp(changed - original, -eps, eps) |
| changed = torch.clamp(original + delta, 0, 1).detach() |
|
|
| if index in {0, iterations // 2, iterations - 1}: |
| with torch.inference_mode(): |
| trace_logits = model(model_input(changed, normalize_inputs)) |
| trace_prob = F.softmax(trace_logits, dim=1) |
| top_id = int(torch.argmax(trace_prob, dim=1)[0]) |
| trace.append( |
| [ |
| index + 1, |
| class_names()[target_id], |
| round(float(trace_prob[0, target_id]), 6), |
| class_names()[top_id], |
| round(float(trace_prob[0, top_id]), 6), |
| ] |
| ) |
|
|
| final_logits = classify_pixels_with_mode(model, changed, normalize_inputs) |
| final_prob = float(F.softmax(final_logits, dim=1)[0, target_id]) |
| summary = ( |
| f"{status}\n" |
| f"Target class {target_id} ({class_names()[target_id]}): " |
| f"{start_prob:.4f} -> {final_prob:.4f} probability after {iterations} visible-change steps. " |
| f"Pixel budget: +/-{float(epsilon_pixels):.1f} values." |
| ) |
| return ( |
| pixels_to_image(original), |
| pixels_to_image(changed), |
| difference_image(original, changed, epsilon_pixels), |
| top_prediction_rows(start_logits, "probability", 5), |
| top_prediction_rows(final_logits, "probability", 5), |
| trace, |
| summary, |
| ) |
|
|
|
|
| @spaces.GPU(duration=240) |
| def visible_change(image, target_label, iterations, epsilon_pixels): |
| return visible_change_core(image, target_label, iterations, epsilon_pixels) |
|
|
|
|
| def initial_visible_change_view(): |
| blank = Image.new("RGB", (IMAGE_SIZE, IMAGE_SIZE), (28, 32, 38)) |
| return prepare_pil_crop(real_image_by_index(0)), blank, blank, [], [], [], "Choose an image, then run the visible robust-style change." |
|
|
|
|
| def visible_real_sample(target_label, iterations, epsilon_pixels, evt: gr.SelectData): |
| index = evt.index if isinstance(evt.index, int) else 0 |
| return visible_change_core(real_image_by_index(index), target_label, iterations, epsilon_pixels) |
|
|
|
|
| @spaces.GPU(duration=180) |
| def targeted_attack( |
| image, |
| target_label, |
| iterations, |
| epsilon_pixels, |
| step_pixels=1, |
| weight_mode=PRETRAINED, |
| top_k=5, |
| ): |
| return targeted_attack_core(image, target_label, iterations, epsilon_pixels, step_pixels, weight_mode, top_k) |
|
|
|
|
| def initial_attack_view(): |
| blank = Image.new("RGB", (IMAGE_SIZE, IMAGE_SIZE), (28, 32, 38)) |
| return prepare_pil_crop(real_image_by_index(0)), blank, blank, [], [], [], "Choose an image and target class, then run the attack." |
|
|
|
|
| def attack_real_sample(target_label, iterations, epsilon_pixels, evt: gr.SelectData): |
| index = evt.index if isinstance(evt.index, int) else 0 |
| return targeted_attack_core( |
| real_image_by_index(index), |
| target_label, |
| iterations, |
| epsilon_pixels, |
| ) |
|
|
|
|
| def build_app(): |
| theme = gr.themes.Soft( |
| primary_hue="teal", |
| secondary_hue="rose", |
| neutral_hue="slate", |
| radius_size="sm", |
| ) |
|
|
| css = """ |
| .sample-gallery img { object-fit: cover !important; } |
| .code-panel textarea, .code-panel pre { font-size: 13px !important; } |
| """ |
|
|
| headers = ["rank", "class id", "class", "probability", "logit", "shown score"] |
| trace_headers = ["iteration", "target", "target probability", "top class", "top probability"] |
| category_headers = ["class id", "class"] |
| source_headers = ["class id", "class", "source"] |
|
|
| with gr.Blocks(title=APP_TITLE, theme=theme, css=css) as demo: |
| gr.Markdown(f"# {APP_TITLE}") |
|
|
| with gr.Tab("Classifier"): |
| gr.Markdown( |
| "This tab runs ResNet50 on an image and shows the model's top ImageNet guesses. " |
| "Use it to see how a pretrained image classifier turns pixels into class probabilities." |
| ) |
| with gr.Row(equal_height=False): |
| with gr.Column(scale=1, min_width=300): |
| classifier_real_samples = gr.Gallery( |
| value=real_gallery_items(), |
| label="Real ImageNet examples", |
| columns=4, |
| rows=2, |
| height=300, |
| object_fit="cover", |
| elem_classes=["sample-gallery"], |
| ) |
| classifier_upload = gr.Image(label="Upload image", type="pil", sources=["upload", "clipboard"]) |
| classify_button = gr.Button("Run classifier", variant="primary") |
|
|
| with gr.Column(scale=1, min_width=300): |
| classifier_image = gr.Image(label="Prepared 224x224 crop", type="pil", interactive=False) |
| classifier_status = gr.Textbox(label="Model status", interactive=False, lines=3) |
| classifier_predictions = gr.Dataframe( |
| headers=headers, |
| datatype=["number", "number", "str", "number", "number", "number"], |
| label="Top predictions", |
| interactive=False, |
| ) |
|
|
| with gr.Tab("Targeted attack"): |
| gr.Markdown( |
| "This tab makes a tiny, bounded pixel change that pushes ResNet50 toward a target class. " |
| "The perturbation heat map shows where the attack spent its budget, and the before/after tables show how the model's confidence moved." |
| ) |
| with gr.Row(equal_height=False): |
| with gr.Column(scale=1, min_width=300): |
| attack_real_samples = gr.Gallery( |
| value=real_gallery_items(), |
| label="Real ImageNet examples", |
| columns=4, |
| rows=2, |
| height=300, |
| object_fit="cover", |
| elem_classes=["sample-gallery"], |
| ) |
| attack_upload = gr.Image(label="Upload image", type="pil", sources=["upload", "clipboard"]) |
| target = gr.Dropdown( |
| choices=target_choices(), |
| value=DEFAULT_TARGET, |
| label="Target class", |
| allow_custom_value=True, |
| ) |
| with gr.Row(): |
| iterations = gr.Slider(1, 60, value=16, step=1, label="Iterations") |
| epsilon = gr.Slider(1, 24, value=8, step=1, label="Pixel budget") |
| attack_button = gr.Button("Run targeted attack", variant="primary") |
|
|
| with gr.Column(scale=1, min_width=300): |
| attack_summary = gr.Textbox(label="Attack summary", interactive=False, lines=4) |
| with gr.Row(): |
| original_image = gr.Image(label="Original crop", type="pil", interactive=False) |
| attacked_image = gr.Image(label="Attacked crop", type="pil", interactive=False) |
| perturbation = gr.Image(label="Perturbation heat map", type="pil", interactive=False) |
|
|
| with gr.Row(equal_height=False): |
| before_predictions = gr.Dataframe( |
| headers=headers, |
| datatype=["number", "number", "str", "number", "number", "number"], |
| label="Before attack", |
| interactive=False, |
| ) |
| after_predictions = gr.Dataframe( |
| headers=headers, |
| datatype=["number", "number", "str", "number", "number", "number"], |
| label="After attack", |
| interactive=False, |
| ) |
| attack_trace = gr.Dataframe( |
| headers=trace_headers, |
| datatype=["number", "str", "number", "str", "number"], |
| label="Optimization trace", |
| interactive=False, |
| ) |
|
|
| with gr.Tab("Visible robust-style change"): |
| gr.Markdown( |
| "Problem 4e uses a robust model and a larger update budget, so the image change is no longer hidden. " |
| "This tab pushes the image toward a target class and shows the visible before/after result." |
| ) |
| with gr.Row(equal_height=False): |
| with gr.Column(scale=1, min_width=300): |
| visible_real_samples = gr.Gallery( |
| value=real_gallery_items(), |
| label="Real ImageNet examples", |
| columns=4, |
| rows=2, |
| height=300, |
| object_fit="cover", |
| elem_classes=["sample-gallery"], |
| ) |
| visible_upload = gr.Image(label="Upload image", type="pil", sources=["upload", "clipboard"]) |
| visible_target = gr.Dropdown( |
| choices=target_choices(), |
| value=VISIBLE_TARGET, |
| label="Target class", |
| allow_custom_value=True, |
| ) |
| with gr.Row(): |
| visible_iterations = gr.Slider(1, 80, value=24, step=1, label="Iterations") |
| visible_epsilon = gr.Slider(8, 80, value=40, step=4, label="Visible pixel budget") |
| visible_button = gr.Button("Run visible change", variant="primary") |
|
|
| with gr.Column(scale=1, min_width=300): |
| visible_summary = gr.Textbox(label="Method summary", interactive=False, lines=5) |
| with gr.Row(): |
| visible_original = gr.Image(label="Original crop", type="pil", interactive=False) |
| visible_changed = gr.Image(label="Changed crop", type="pil", interactive=False) |
| visible_difference = gr.Image(label="Where pixels changed", type="pil", interactive=False) |
|
|
| with gr.Row(equal_height=False): |
| visible_before = gr.Dataframe( |
| headers=headers, |
| datatype=["number", "number", "str", "number", "number", "number"], |
| label="Before change", |
| interactive=False, |
| ) |
| visible_after = gr.Dataframe( |
| headers=headers, |
| datatype=["number", "number", "str", "number", "number", "number"], |
| label="After change", |
| interactive=False, |
| ) |
| visible_trace = gr.Dataframe( |
| headers=trace_headers, |
| datatype=["number", "str", "number", "str", "number"], |
| label="Optimization trace", |
| interactive=False, |
| ) |
|
|
| with gr.Tab("Categories"): |
| gr.Markdown( |
| "This tab is the label dictionary ResNet50 is trained to predict. " |
| "Use it to find target class ids for attacks and to check the real example image sources." |
| ) |
| with gr.Row(equal_height=False): |
| category_filter = gr.Textbox(label="Filter categories", placeholder="cat, toaster, 282") |
| show_categories = gr.Button("Show categories", variant="primary") |
| categories = gr.Dataframe( |
| headers=category_headers, |
| datatype=["number", "str"], |
| label="ResNet50 ImageNet categories", |
| interactive=False, |
| ) |
| image_sources = gr.Dataframe( |
| value=real_source_rows(), |
| headers=source_headers, |
| datatype=["number", "str", "str"], |
| label="Real example image sources", |
| interactive=False, |
| ) |
|
|
| with gr.Tab("Code cells"): |
| gr.Markdown( |
| "This tab keeps the core notebook ideas visible: image normalization, softmax probabilities, " |
| "and the gradient loop that constructs the targeted adversarial example." |
| ) |
| with gr.Row(equal_height=False): |
| with gr.Column(): |
| gr.Code(PREPARE_IMAGE_CODE, language="python", label="Prepare image", interactive=False, elem_classes=["code-panel"]) |
| gr.Code(SOFTMAX_CODE, language="python", label="Logits to probabilities", interactive=False, elem_classes=["code-panel"]) |
| with gr.Column(): |
| gr.Code(ATTACK_CODE, language="python", label="Targeted attack loop", interactive=False, elem_classes=["code-panel"]) |
|
|
| classifier_real_samples.select( |
| classify_real_sample, |
| inputs=None, |
| outputs=[classifier_image, classifier_predictions, classifier_status], |
| show_progress="minimal", |
| ) |
| classify_button.click( |
| classify_image, |
| inputs=[classifier_upload], |
| outputs=[classifier_image, classifier_predictions, classifier_status], |
| show_progress="minimal", |
| ) |
| classifier_upload.change( |
| classify_image, |
| inputs=[classifier_upload], |
| outputs=[classifier_image, classifier_predictions, classifier_status], |
| show_progress="minimal", |
| ) |
|
|
| attack_real_samples.select( |
| attack_real_sample, |
| inputs=[target, iterations, epsilon], |
| outputs=[ |
| original_image, |
| attacked_image, |
| perturbation, |
| before_predictions, |
| after_predictions, |
| attack_trace, |
| attack_summary, |
| ], |
| show_progress="minimal", |
| ) |
| attack_button.click( |
| targeted_attack, |
| inputs=[attack_upload, target, iterations, epsilon], |
| outputs=[ |
| original_image, |
| attacked_image, |
| perturbation, |
| before_predictions, |
| after_predictions, |
| attack_trace, |
| attack_summary, |
| ], |
| show_progress="minimal", |
| ) |
| visible_real_samples.select( |
| visible_real_sample, |
| inputs=[visible_target, visible_iterations, visible_epsilon], |
| outputs=[ |
| visible_original, |
| visible_changed, |
| visible_difference, |
| visible_before, |
| visible_after, |
| visible_trace, |
| visible_summary, |
| ], |
| show_progress="minimal", |
| ) |
| visible_button.click( |
| visible_change, |
| inputs=[visible_upload, visible_target, visible_iterations, visible_epsilon], |
| outputs=[ |
| visible_original, |
| visible_changed, |
| visible_difference, |
| visible_before, |
| visible_after, |
| visible_trace, |
| visible_summary, |
| ], |
| show_progress="minimal", |
| ) |
| show_categories.click( |
| category_rows, |
| inputs=[category_filter], |
| outputs=[categories], |
| show_progress="minimal", |
| ) |
| category_filter.submit( |
| category_rows, |
| inputs=[category_filter], |
| outputs=[categories], |
| show_progress="minimal", |
| ) |
|
|
| demo.load( |
| initial_classifier_view, |
| inputs=None, |
| outputs=[classifier_image, classifier_predictions, classifier_status], |
| show_progress="minimal", |
| ) |
| demo.load( |
| initial_attack_view, |
| inputs=None, |
| outputs=[ |
| original_image, |
| attacked_image, |
| perturbation, |
| before_predictions, |
| after_predictions, |
| attack_trace, |
| attack_summary, |
| ], |
| show_progress="minimal", |
| ) |
| demo.load( |
| initial_visible_change_view, |
| inputs=None, |
| outputs=[ |
| visible_original, |
| visible_changed, |
| visible_difference, |
| visible_before, |
| visible_after, |
| visible_trace, |
| visible_summary, |
| ], |
| show_progress="minimal", |
| ) |
|
|
| return demo |
|
|
|
|
| if __name__ == "__main__": |
| build_app().launch() |
|
|
