deepellipse-circleid / code /ay_bce_head.py
qinxunhui's picture
Add small artifacts (code, outputs, configs, LICENSE)
519ab98 verified
Raw History Blame Contribute Delete
17.4 kB
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""BCE pairwise similarity head for AY open-set inference (§5.2.1).
head_type 'bce': classic PairwiseBCEHead (cos → cubic → η σ).
head_type 'cond': PairwiseCondHead (pairwise-conditioned observation space).
"""
from __future__ import annotations
from typing import Optional, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
HEAD_TYPES = ("bce", "cond", "supcon")
class PairwiseBCEHead(nn.Module):
"""Map f_ellipse → L2-normalized h; learnable η, m for BCE-with-logits."""
def __init__(
self,
in_dim: int = 1024,
emb_dim: int = 256,
hidden_dim: int = 512,
eta_init: float = 10.0,
m_init: float = 0.1,
dropout: float = 0.1,
):
super().__init__()
self.emb_dim = emb_dim
self.mlp = nn.Sequential(
nn.Linear(in_dim, hidden_dim),
nn.ReLU(inplace=True),
nn.Dropout(dropout),
nn.Linear(hidden_dim, emb_dim),
)
self.eta = nn.Parameter(torch.tensor(float(eta_init)))
self.m = nn.Parameter(torch.tensor(float(m_init)))
def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor:
"""(B, in_dim) → (B, emb_dim) L2-normalized."""
h = self.mlp(f_ellipse)
return F.normalize(h, p=2, dim=-1)
def pairwise_a_tilde(
self, h: torch.Tensor, row_batch: int = 256
) -> torch.Tensor:
"""Cosine u → nonlinear ã. h must be L2-normalized. Returns (B, B)."""
u = h @ h.T
return 2.0 * ((u + 1.0) / 2.0).pow(3) - 1.0
def pairwise_a_tilde_for_pairs(
self, h: torch.Tensor, i_idx: torch.Tensor, j_idx: torch.Tensor
) -> torch.Tensor:
"""ã for selected (i,j) pairs. Returns (P,)."""
u = (h[i_idx] * h[j_idx]).sum(dim=-1)
return 2.0 * ((u + 1.0) / 2.0).pow(3) - 1.0
def affinity_matrix(
self, h: torch.Tensor, row_batch: int = 256, col_batch: int = None
) -> torch.Tensor:
"""Test-time A = σ(η ã) (no margin). (N, N)."""
z = self.eta * self.pairwise_a_tilde(h, row_batch=row_batch)
return torch.sigmoid(z)
class PairwiseCondHead(nn.Module):
"""Pairwise-conditioned observation head (PCOH).
Observation space is jointly decided by (v_a, v_b): a shared MLP maps each
side's own code plus symmetric pair statistics r_ab into an observation
embedding; the pair score is the cosine in that observation space.
h_a = L2Norm(MLP_base(v_a)), h_b = L2Norm(MLP_base(v_b))
r_ab = [ h_a⊙h_b ; |h_a-h_b| ; h_a+h_b ] (symmetric)
u_a = L2Norm(h_a + MLP_cond([h_a; r_ab]))
u_b = L2Norm(h_b + MLP_cond([h_b; r_ab])) (shared weights)
s(a,b) = u_a · u_b (symmetric)
Zero-init of MLP_cond's last layer gives s = cos(h_a,h_b) initially, so
training starts from the BCE-cosine baseline and learns an increment.
"""
def __init__(
self,
in_dim: int = 1024,
emb_dim: int = 256,
hidden_dim: int = 256,
eta_init: float = 10.0,
m_init: float = 0.1,
dropout: float = 0.3,
):
super().__init__()
self.emb_dim = emb_dim
self.hidden_dim = hidden_dim
self.mlp = nn.Sequential(
nn.Linear(in_dim, hidden_dim),
nn.ReLU(inplace=True),
nn.Dropout(dropout),
nn.Linear(hidden_dim, emb_dim),
)
# conditional observation branch: input [h; r_ab] (4*emb_dim).
# First layer weight split as [W_h | W_r] so we never materialize
# the concatenated (..., 4e) tensor explicitly.
self.cond_in = nn.Linear(4 * emb_dim, hidden_dim)
self.cond_h = nn.Linear(emb_dim, hidden_dim, bias=False) # [h] part
self.cond_r = nn.Linear(3 * emb_dim, hidden_dim, bias=False) # [r] part
# share the same bias of cond_in
with torch.no_grad():
self.cond_h.weight.copy_(self.cond_in.weight[:, :emb_dim])
self.cond_r.weight.copy_(self.cond_in.weight[:, emb_dim:])
self.cond_bias = nn.Parameter(self.cond_in.bias.clone())
self.cond_act = nn.ReLU(inplace=True)
self.cond_drop = nn.Dropout(dropout)
self.cond_out = nn.Linear(hidden_dim, emb_dim)
# zero-init last layer → residual 0 at init → s = cos(h_a, h_b)
nn.init.zeros_(self.cond_out.weight)
nn.init.zeros_(self.cond_out.bias)
self.eta = nn.Parameter(torch.tensor(float(eta_init)))
self.m = nn.Parameter(torch.tensor(float(m_init)))
def _cond_forward(self, h_side: torch.Tensor, r: torch.Tensor) -> torch.Tensor:
"""MLP_cond([h_side; r]) via split weights. Any leading batch dims."""
x = self.cond_h(h_side) + self.cond_r(r) + self.cond_bias
x = self.cond_act(x)
x = self.cond_drop(x)
return self.cond_out(x)
def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor:
"""(B, in_dim) → (B, emb_dim) L2-normalized base code h."""
h = self.mlp(f_ellipse)
return F.normalize(h, p=2, dim=-1)
def _pairwise_s_block(
self, h: torch.Tensor, row_start: int, row_end: int, col_batch: int = 512
) -> torch.Tensor:
"""s rows [row_start,row_end) against all columns. Returns (rb, n)."""
ha = h[row_start:row_end] # (rb, e)
rb = ha.shape[0]
n = h.shape[0]
out = torch.empty(rb, n, device=h.device, dtype=h.dtype)
for c0 in range(0, n, col_batch):
c1 = min(c0 + col_batch, n)
hb = h[c0:c1] # (cb, e)
prod = ha.unsqueeze(1) * hb.unsqueeze(0) # (rb, cb, e)
diff = (ha.unsqueeze(1) - hb.unsqueeze(0)).abs()
summ = ha.unsqueeze(1) + hb.unsqueeze(0)
r = torch.cat([prod, diff, summ], dim=-1) # (rb, cb, 3e)
h_a_exp = ha.unsqueeze(1).expand(rb, c1 - c0, self.emb_dim)
h_b_exp = hb.unsqueeze(0).expand(rb, c1 - c0, self.emb_dim)
u_a = F.normalize(
h_a_exp + self._cond_forward(h_a_exp, r), p=2, dim=-1
)
u_b = F.normalize(
h_b_exp + self._cond_forward(h_b_exp, r), p=2, dim=-1
)
out[:, c0:c1] = (u_a * u_b).sum(dim=-1)
return out
def pairwise_a_tilde(
self, h: torch.Tensor, row_batch: int = 128, col_batch: int = 512
) -> torch.Tensor:
"""Symmetric ã = cos in the conditioned observation space. (B, B)."""
n = h.shape[0]
out = torch.empty(n, n, device=h.device, dtype=h.dtype)
for start in range(0, n, row_batch):
end = min(start + row_batch, n)
out[start:end] = self._pairwise_s_block(
h, start, end, col_batch=col_batch
)
return out
def pairwise_a_tilde_for_pairs(
self,
h: torch.Tensor,
i_idx: torch.Tensor,
j_idx: torch.Tensor,
) -> torch.Tensor:
"""ã for selected (i,j) pairs via the conditioned observation space. (P,)."""
ha = h[i_idx] # (P, e)
hb = h[j_idx]
prod = ha * hb
diff = (ha - hb).abs()
summ = ha + hb
r = torch.cat([prod, diff, summ], dim=-1) # (P, 3e)
u_a = F.normalize(ha + self._cond_forward(ha, r), p=2, dim=-1)
u_b = F.normalize(hb + self._cond_forward(hb, r), p=2, dim=-1)
return (u_a * u_b).sum(dim=-1)
def affinity_matrix(
self, h: torch.Tensor, row_batch: int = 128, col_batch: int = 512
) -> torch.Tensor:
"""Test-time A = σ(η ã). (N, N)."""
z = self.eta * self.pairwise_a_tilde(
h, row_batch=row_batch, col_batch=col_batch
)
return torch.sigmoid(z)
class SupConHead(nn.Module):
"""Pure supervised-contrastive embedding head (no BCE components).
MLP structure is identical to PairwiseBCEHead for a like-for-like
comparison, but all BCE machinery is removed (no eta/margin, no cubic
map, no sigmoid affinity). The output is an L2-normalized embedding h;
the test-time adjacency is built downstream as A_ij = cos(h_i, h_j).
"""
def __init__(
self,
in_dim: int = 1024,
emb_dim: int = 256,
hidden_dim: int = 512,
dropout: float = 0.1,
):
super().__init__()
self.emb_dim = emb_dim
self.mlp = nn.Sequential(
nn.Linear(in_dim, hidden_dim),
nn.ReLU(inplace=True),
nn.Dropout(dropout),
nn.Linear(hidden_dim, emb_dim),
)
def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor:
"""(B, in_dim) → (B, emb_dim) L2-normalized embedding."""
h = self.mlp(f_ellipse)
return F.normalize(h, p=2, dim=-1)
def supcon_loss(
h: torch.Tensor,
labels: torch.Tensor,
*,
temperature: float = 0.1,
ignore_label: int = -1,
) -> Tuple[torch.Tensor, dict]:
"""Supervised contrastive loss (Khosla et al., NeurIPS 2020).
h: (B, d) L2-normalized embeddings. labels: (B,) integer writer labels;
rows with ``ignore_label`` are dropped. Positive = same writer in-batch.
Returns (loss, stats) with the same keys as ``pairwise_bce_loss`` so the
training loop can log uniformly. ``pos_pair_acc`` / ``neg_pair_acc`` hold
the mean positive / negative cosine (higher/lower is better) and
``pair_acc`` is the per-anchor ranking accuracy (mean pos cos > mean neg cos).
"""
device = h.device
valid = labels != ignore_label
n_ignored = int((~valid).sum().item())
stats_empty = {
"n_pairs": 0,
"n_pos": 0,
"n_neg_used": 0,
"n_ignored": n_ignored,
"pos_ratio": 0.0,
"pair_acc": 0.0,
"pos_pair_acc": 0.0,
"neg_pair_acc": 0.0,
"eta": 0.0,
"m": 0.0,
}
if int(valid.sum().item()) < 2:
return h.new_zeros(()), stats_empty
h = h[valid]
labels = labels[valid]
b = h.shape[0]
sim = (h @ h.T) / float(temperature) # (b, b)
eye = torch.eye(b, device=device, dtype=torch.bool)
sim_masked = sim.masked_fill(eye, float("-inf")) # exclude self
log_prob = sim_masked - torch.logsumexp(sim_masked, dim=1, keepdim=True)
# zero-out the diagonal so masked sums never hit 0 * -inf = nan
log_prob = log_prob.masked_fill(eye, 0.0)
pos_mask = (labels.unsqueeze(0) == labels.unsqueeze(1)) & ~eye
neg_mask = (~eye) & ~pos_mask
n_pos_per = pos_mask.sum(dim=1)
rows = n_pos_per > 0
if not bool(rows.any()):
return h.new_zeros(()), stats_empty
mean_log_prob_pos = (
(pos_mask.float() * log_prob).sum(dim=1)[rows] / n_pos_per[rows]
)
loss = -mean_log_prob_pos.mean()
with torch.no_grad():
n_pos_tot = int(pos_mask.sum().item())
n_neg_tot = int(neg_mask.sum().item())
pos_sim = (sim * pos_mask.float()).sum() / max(n_pos_tot, 1)
neg_sim = (sim * neg_mask.float()).sum() / max(n_neg_tot, 1)
pos_mean_row = (sim * pos_mask.float()).sum(1) / n_pos_per.clamp(min=1)
neg_mean_row = (
(sim * neg_mask.float()).sum(1) / neg_mask.sum(1).clamp(min=1)
)
pair_acc = (
((pos_mean_row > neg_mean_row) & rows).float().sum()
/ rows.float().sum()
)
stats = {
"n_pairs": n_pos_tot,
"n_pos": n_pos_tot,
"n_neg_used": n_neg_tot,
"n_ignored": n_ignored,
"pos_ratio": n_pos_tot / max(n_pos_tot + n_neg_tot, 1),
"pair_acc": float(pair_acc.item()),
"pos_pair_acc": float(pos_sim.item()),
"neg_pair_acc": float(neg_sim.item()),
"eta": 0.0,
"m": 0.0,
}
return loss, stats
def build_pairwise_head(
head_type: str,
*,
in_dim: int = 1024,
emb_dim: int = 256,
hidden_dim: int = 512,
**kwargs,
) -> nn.Module:
"""Factory: 'bce' → PairwiseBCEHead; 'cond' → PairwiseCondHead; 'supcon' → SupConHead."""
if head_type == "cond":
return PairwiseCondHead(
in_dim=in_dim, emb_dim=emb_dim, hidden_dim=hidden_dim, **kwargs
)
if head_type == "supcon":
return SupConHead(
in_dim=in_dim,
emb_dim=emb_dim,
hidden_dim=hidden_dim,
dropout=kwargs.get("dropout", 0.1),
)
return PairwiseBCEHead(
in_dim=in_dim, emb_dim=emb_dim, hidden_dim=hidden_dim, **kwargs
)
def _pair_masks(
labels: torch.Tensor,
*,
ignore_label: int = -1,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
"""Return pos_mask, neg_mask, both_minus1 (B,B), n_ignored off-diagonal (-1,-1)."""
b = labels.shape[0]
eye = torch.eye(b, device=labels.device, dtype=torch.bool)
lab_i = labels.unsqueeze(0)
lab_j = labels.unsqueeze(1)
both_minus1 = (lab_i == ignore_label) & (lab_j == ignore_label)
n_ignored = int(both_minus1[~eye].sum().item())
same_label = lab_i == lab_j
pos_mask = (
same_label
& (lab_i >= 0)
& (lab_j >= 0)
& ~eye
& ~both_minus1
)
neg_mask = (~eye) & ~both_minus1 & ~pos_mask
return pos_mask, neg_mask, both_minus1, n_ignored
def pairwise_bce_loss(
head: PairwiseBCEHead,
h: torch.Tensor,
labels: torch.Tensor,
*,
ignore_label: int = -1,
neg_sample_ratio: float = 10.0,
hard_neg_top: int = 0,
hard_neg_ratio: float = 0.0,
use_pos_weight: bool = False,
pos_weight: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, dict]:
"""
BCE on pairwise logits with (-1,-1) skipped; (-1,known) as negatives.
Two passes for memory efficiency on pairwise-conditioned heads:
1) no_grad full (B,B) ã → select all positive pairs + (hard/random) negatives.
2) grad ã for selected pairs only → BCE logits.
"""
pos_mask, neg_mask, _, n_ignored = _pair_masks(labels, ignore_label=ignore_label)
b = h.shape[0]
empty_stats = {
"n_pairs": 0,
"n_pos": 0,
"n_neg_used": 0,
"n_ignored": n_ignored,
"pos_ratio": 0.0,
"pair_acc": 0.0,
"pos_pair_acc": 0.0,
"neg_pair_acc": 0.0,
"eta": float(head.eta.detach().item()),
"m": float(head.m.detach().item()),
}
if not pos_mask.any():
return h.new_zeros(()), empty_stats
# ---- pass 1: select positive pairs + negatives (no grad) ----
with torch.no_grad():
a_full = head.pairwise_a_tilde(h, row_batch=256)
pos_ij = torch.nonzero(pos_mask) # (n_pos, 2)
n_pos = int(pos_ij.shape[0])
z_neg = None
targets_neg = None
n_neg_used = 0
if neg_mask.any():
neg_ij_all = torch.nonzero(neg_mask) # (n_neg_avail, 2)
n_neg_avail = int(neg_ij_all.shape[0])
a_neg_all = a_full[neg_mask]
if hard_neg_ratio > 0:
n_neg_target = max(1, int(n_pos * float(hard_neg_ratio)))
else:
n_neg_target = max(1, int(n_pos * float(neg_sample_ratio)))
if hard_neg_top > 0:
n_pick = min(n_neg_avail, n_neg_target, hard_neg_top)
_, top_idx = torch.topk(a_neg_all, k=n_pick, largest=True)
else:
n_pick = min(n_neg_avail, n_neg_target)
if n_pick < n_neg_avail:
top_idx = torch.randperm(n_neg_avail, device=h.device)[:n_pick]
else:
top_idx = torch.arange(n_neg_avail, device=h.device)
neg_ij = neg_ij_all[top_idx]
n_neg_used = int(neg_ij.shape[0])
# ---- pass 2: grad ã for selected pairs ----
a_tilde_pos = head.pairwise_a_tilde_for_pairs(h, pos_ij[:, 0], pos_ij[:, 1])
z_pos = head.eta * (a_tilde_pos - head.m)
targets_pos = torch.ones_like(z_pos)
if n_neg_used > 0:
a_tilde_neg = head.pairwise_a_tilde_for_pairs(h, neg_ij[:, 0], neg_ij[:, 1])
z_neg = head.eta * (a_tilde_neg + head.m)
targets_neg = torch.zeros_like(z_neg)
z = torch.cat([z_pos, z_neg], dim=0)
targets = torch.cat([targets_pos, targets_neg], dim=0)
else:
z = z_pos
targets = targets_pos
pw = pos_weight
if pw is None and use_pos_weight and n_pos > 0 and n_neg_used > 0:
pw = torch.tensor([float(n_neg_used) / float(n_pos)], device=z.device)
loss = F.binary_cross_entropy_with_logits(z, targets, pos_weight=pw)
with torch.no_grad():
pred = (torch.sigmoid(z) >= 0.5).float()
acc = (pred == targets).float().mean().item()
pred_pos = (torch.sigmoid(z_pos) >= 0.5).float()
pos_acc = (pred_pos == 1.0).float().mean().item() if n_pos else 0.0
if n_neg_used > 0 and z_neg is not None:
pred_neg = (torch.sigmoid(z_neg) >= 0.5).float()
neg_acc = (pred_neg == 0.0).float().mean().item()
else:
neg_acc = 0.0
n_pairs = n_pos + n_neg_used
return loss, {
"n_pairs": n_pairs,
"n_pos": n_pos,
"n_neg_used": n_neg_used,
"n_ignored": n_ignored,
"pos_ratio": n_pos / max(n_pairs, 1),
"pair_acc": acc,
"pos_pair_acc": pos_acc,
"neg_pair_acc": neg_acc,
"eta": float(head.eta.detach().item()),
"m": float(head.m.detach().item()),
}