#!/usr/bin/env python3 # -*- coding: utf-8 -*- """BCE pairwise similarity head for AY open-set inference (§5.2.1). head_type 'bce': classic PairwiseBCEHead (cos → cubic → η σ). head_type 'cond': PairwiseCondHead (pairwise-conditioned observation space). """ from __future__ import annotations from typing import Optional, Tuple import torch import torch.nn as nn import torch.nn.functional as F HEAD_TYPES = ("bce", "cond", "supcon") class PairwiseBCEHead(nn.Module): """Map f_ellipse → L2-normalized h; learnable η, m for BCE-with-logits.""" def __init__( self, in_dim: int = 1024, emb_dim: int = 256, hidden_dim: int = 512, eta_init: float = 10.0, m_init: float = 0.1, dropout: float = 0.1, ): super().__init__() self.emb_dim = emb_dim self.mlp = nn.Sequential( nn.Linear(in_dim, hidden_dim), nn.ReLU(inplace=True), nn.Dropout(dropout), nn.Linear(hidden_dim, emb_dim), ) self.eta = nn.Parameter(torch.tensor(float(eta_init))) self.m = nn.Parameter(torch.tensor(float(m_init))) def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor: """(B, in_dim) → (B, emb_dim) L2-normalized.""" h = self.mlp(f_ellipse) return F.normalize(h, p=2, dim=-1) def pairwise_a_tilde( self, h: torch.Tensor, row_batch: int = 256 ) -> torch.Tensor: """Cosine u → nonlinear ã. h must be L2-normalized. Returns (B, B).""" u = h @ h.T return 2.0 * ((u + 1.0) / 2.0).pow(3) - 1.0 def pairwise_a_tilde_for_pairs( self, h: torch.Tensor, i_idx: torch.Tensor, j_idx: torch.Tensor ) -> torch.Tensor: """ã for selected (i,j) pairs. Returns (P,).""" u = (h[i_idx] * h[j_idx]).sum(dim=-1) return 2.0 * ((u + 1.0) / 2.0).pow(3) - 1.0 def affinity_matrix( self, h: torch.Tensor, row_batch: int = 256, col_batch: int = None ) -> torch.Tensor: """Test-time A = σ(η ã) (no margin). (N, N).""" z = self.eta * self.pairwise_a_tilde(h, row_batch=row_batch) return torch.sigmoid(z) class PairwiseCondHead(nn.Module): """Pairwise-conditioned observation head (PCOH). Observation space is jointly decided by (v_a, v_b): a shared MLP maps each side's own code plus symmetric pair statistics r_ab into an observation embedding; the pair score is the cosine in that observation space. h_a = L2Norm(MLP_base(v_a)), h_b = L2Norm(MLP_base(v_b)) r_ab = [ h_a⊙h_b ; |h_a-h_b| ; h_a+h_b ] (symmetric) u_a = L2Norm(h_a + MLP_cond([h_a; r_ab])) u_b = L2Norm(h_b + MLP_cond([h_b; r_ab])) (shared weights) s(a,b) = u_a · u_b (symmetric) Zero-init of MLP_cond's last layer gives s = cos(h_a,h_b) initially, so training starts from the BCE-cosine baseline and learns an increment. """ def __init__( self, in_dim: int = 1024, emb_dim: int = 256, hidden_dim: int = 256, eta_init: float = 10.0, m_init: float = 0.1, dropout: float = 0.3, ): super().__init__() self.emb_dim = emb_dim self.hidden_dim = hidden_dim self.mlp = nn.Sequential( nn.Linear(in_dim, hidden_dim), nn.ReLU(inplace=True), nn.Dropout(dropout), nn.Linear(hidden_dim, emb_dim), ) # conditional observation branch: input [h; r_ab] (4*emb_dim). # First layer weight split as [W_h | W_r] so we never materialize # the concatenated (..., 4e) tensor explicitly. self.cond_in = nn.Linear(4 * emb_dim, hidden_dim) self.cond_h = nn.Linear(emb_dim, hidden_dim, bias=False) # [h] part self.cond_r = nn.Linear(3 * emb_dim, hidden_dim, bias=False) # [r] part # share the same bias of cond_in with torch.no_grad(): self.cond_h.weight.copy_(self.cond_in.weight[:, :emb_dim]) self.cond_r.weight.copy_(self.cond_in.weight[:, emb_dim:]) self.cond_bias = nn.Parameter(self.cond_in.bias.clone()) self.cond_act = nn.ReLU(inplace=True) self.cond_drop = nn.Dropout(dropout) self.cond_out = nn.Linear(hidden_dim, emb_dim) # zero-init last layer → residual 0 at init → s = cos(h_a, h_b) nn.init.zeros_(self.cond_out.weight) nn.init.zeros_(self.cond_out.bias) self.eta = nn.Parameter(torch.tensor(float(eta_init))) self.m = nn.Parameter(torch.tensor(float(m_init))) def _cond_forward(self, h_side: torch.Tensor, r: torch.Tensor) -> torch.Tensor: """MLP_cond([h_side; r]) via split weights. Any leading batch dims.""" x = self.cond_h(h_side) + self.cond_r(r) + self.cond_bias x = self.cond_act(x) x = self.cond_drop(x) return self.cond_out(x) def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor: """(B, in_dim) → (B, emb_dim) L2-normalized base code h.""" h = self.mlp(f_ellipse) return F.normalize(h, p=2, dim=-1) def _pairwise_s_block( self, h: torch.Tensor, row_start: int, row_end: int, col_batch: int = 512 ) -> torch.Tensor: """s rows [row_start,row_end) against all columns. Returns (rb, n).""" ha = h[row_start:row_end] # (rb, e) rb = ha.shape[0] n = h.shape[0] out = torch.empty(rb, n, device=h.device, dtype=h.dtype) for c0 in range(0, n, col_batch): c1 = min(c0 + col_batch, n) hb = h[c0:c1] # (cb, e) prod = ha.unsqueeze(1) * hb.unsqueeze(0) # (rb, cb, e) diff = (ha.unsqueeze(1) - hb.unsqueeze(0)).abs() summ = ha.unsqueeze(1) + hb.unsqueeze(0) r = torch.cat([prod, diff, summ], dim=-1) # (rb, cb, 3e) h_a_exp = ha.unsqueeze(1).expand(rb, c1 - c0, self.emb_dim) h_b_exp = hb.unsqueeze(0).expand(rb, c1 - c0, self.emb_dim) u_a = F.normalize( h_a_exp + self._cond_forward(h_a_exp, r), p=2, dim=-1 ) u_b = F.normalize( h_b_exp + self._cond_forward(h_b_exp, r), p=2, dim=-1 ) out[:, c0:c1] = (u_a * u_b).sum(dim=-1) return out def pairwise_a_tilde( self, h: torch.Tensor, row_batch: int = 128, col_batch: int = 512 ) -> torch.Tensor: """Symmetric ã = cos in the conditioned observation space. (B, B).""" n = h.shape[0] out = torch.empty(n, n, device=h.device, dtype=h.dtype) for start in range(0, n, row_batch): end = min(start + row_batch, n) out[start:end] = self._pairwise_s_block( h, start, end, col_batch=col_batch ) return out def pairwise_a_tilde_for_pairs( self, h: torch.Tensor, i_idx: torch.Tensor, j_idx: torch.Tensor, ) -> torch.Tensor: """ã for selected (i,j) pairs via the conditioned observation space. (P,).""" ha = h[i_idx] # (P, e) hb = h[j_idx] prod = ha * hb diff = (ha - hb).abs() summ = ha + hb r = torch.cat([prod, diff, summ], dim=-1) # (P, 3e) u_a = F.normalize(ha + self._cond_forward(ha, r), p=2, dim=-1) u_b = F.normalize(hb + self._cond_forward(hb, r), p=2, dim=-1) return (u_a * u_b).sum(dim=-1) def affinity_matrix( self, h: torch.Tensor, row_batch: int = 128, col_batch: int = 512 ) -> torch.Tensor: """Test-time A = σ(η ã). (N, N).""" z = self.eta * self.pairwise_a_tilde( h, row_batch=row_batch, col_batch=col_batch ) return torch.sigmoid(z) class SupConHead(nn.Module): """Pure supervised-contrastive embedding head (no BCE components). MLP structure is identical to PairwiseBCEHead for a like-for-like comparison, but all BCE machinery is removed (no eta/margin, no cubic map, no sigmoid affinity). The output is an L2-normalized embedding h; the test-time adjacency is built downstream as A_ij = cos(h_i, h_j). """ def __init__( self, in_dim: int = 1024, emb_dim: int = 256, hidden_dim: int = 512, dropout: float = 0.1, ): super().__init__() self.emb_dim = emb_dim self.mlp = nn.Sequential( nn.Linear(in_dim, hidden_dim), nn.ReLU(inplace=True), nn.Dropout(dropout), nn.Linear(hidden_dim, emb_dim), ) def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor: """(B, in_dim) → (B, emb_dim) L2-normalized embedding.""" h = self.mlp(f_ellipse) return F.normalize(h, p=2, dim=-1) def supcon_loss( h: torch.Tensor, labels: torch.Tensor, *, temperature: float = 0.1, ignore_label: int = -1, ) -> Tuple[torch.Tensor, dict]: """Supervised contrastive loss (Khosla et al., NeurIPS 2020). h: (B, d) L2-normalized embeddings. labels: (B,) integer writer labels; rows with ``ignore_label`` are dropped. Positive = same writer in-batch. Returns (loss, stats) with the same keys as ``pairwise_bce_loss`` so the training loop can log uniformly. ``pos_pair_acc`` / ``neg_pair_acc`` hold the mean positive / negative cosine (higher/lower is better) and ``pair_acc`` is the per-anchor ranking accuracy (mean pos cos > mean neg cos). """ device = h.device valid = labels != ignore_label n_ignored = int((~valid).sum().item()) stats_empty = { "n_pairs": 0, "n_pos": 0, "n_neg_used": 0, "n_ignored": n_ignored, "pos_ratio": 0.0, "pair_acc": 0.0, "pos_pair_acc": 0.0, "neg_pair_acc": 0.0, "eta": 0.0, "m": 0.0, } if int(valid.sum().item()) < 2: return h.new_zeros(()), stats_empty h = h[valid] labels = labels[valid] b = h.shape[0] sim = (h @ h.T) / float(temperature) # (b, b) eye = torch.eye(b, device=device, dtype=torch.bool) sim_masked = sim.masked_fill(eye, float("-inf")) # exclude self log_prob = sim_masked - torch.logsumexp(sim_masked, dim=1, keepdim=True) # zero-out the diagonal so masked sums never hit 0 * -inf = nan log_prob = log_prob.masked_fill(eye, 0.0) pos_mask = (labels.unsqueeze(0) == labels.unsqueeze(1)) & ~eye neg_mask = (~eye) & ~pos_mask n_pos_per = pos_mask.sum(dim=1) rows = n_pos_per > 0 if not bool(rows.any()): return h.new_zeros(()), stats_empty mean_log_prob_pos = ( (pos_mask.float() * log_prob).sum(dim=1)[rows] / n_pos_per[rows] ) loss = -mean_log_prob_pos.mean() with torch.no_grad(): n_pos_tot = int(pos_mask.sum().item()) n_neg_tot = int(neg_mask.sum().item()) pos_sim = (sim * pos_mask.float()).sum() / max(n_pos_tot, 1) neg_sim = (sim * neg_mask.float()).sum() / max(n_neg_tot, 1) pos_mean_row = (sim * pos_mask.float()).sum(1) / n_pos_per.clamp(min=1) neg_mean_row = ( (sim * neg_mask.float()).sum(1) / neg_mask.sum(1).clamp(min=1) ) pair_acc = ( ((pos_mean_row > neg_mean_row) & rows).float().sum() / rows.float().sum() ) stats = { "n_pairs": n_pos_tot, "n_pos": n_pos_tot, "n_neg_used": n_neg_tot, "n_ignored": n_ignored, "pos_ratio": n_pos_tot / max(n_pos_tot + n_neg_tot, 1), "pair_acc": float(pair_acc.item()), "pos_pair_acc": float(pos_sim.item()), "neg_pair_acc": float(neg_sim.item()), "eta": 0.0, "m": 0.0, } return loss, stats def build_pairwise_head( head_type: str, *, in_dim: int = 1024, emb_dim: int = 256, hidden_dim: int = 512, **kwargs, ) -> nn.Module: """Factory: 'bce' → PairwiseBCEHead; 'cond' → PairwiseCondHead; 'supcon' → SupConHead.""" if head_type == "cond": return PairwiseCondHead( in_dim=in_dim, emb_dim=emb_dim, hidden_dim=hidden_dim, **kwargs ) if head_type == "supcon": return SupConHead( in_dim=in_dim, emb_dim=emb_dim, hidden_dim=hidden_dim, dropout=kwargs.get("dropout", 0.1), ) return PairwiseBCEHead( in_dim=in_dim, emb_dim=emb_dim, hidden_dim=hidden_dim, **kwargs ) def _pair_masks( labels: torch.Tensor, *, ignore_label: int = -1, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: """Return pos_mask, neg_mask, both_minus1 (B,B), n_ignored off-diagonal (-1,-1).""" b = labels.shape[0] eye = torch.eye(b, device=labels.device, dtype=torch.bool) lab_i = labels.unsqueeze(0) lab_j = labels.unsqueeze(1) both_minus1 = (lab_i == ignore_label) & (lab_j == ignore_label) n_ignored = int(both_minus1[~eye].sum().item()) same_label = lab_i == lab_j pos_mask = ( same_label & (lab_i >= 0) & (lab_j >= 0) & ~eye & ~both_minus1 ) neg_mask = (~eye) & ~both_minus1 & ~pos_mask return pos_mask, neg_mask, both_minus1, n_ignored def pairwise_bce_loss( head: PairwiseBCEHead, h: torch.Tensor, labels: torch.Tensor, *, ignore_label: int = -1, neg_sample_ratio: float = 10.0, hard_neg_top: int = 0, hard_neg_ratio: float = 0.0, use_pos_weight: bool = False, pos_weight: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, dict]: """ BCE on pairwise logits with (-1,-1) skipped; (-1,known) as negatives. Two passes for memory efficiency on pairwise-conditioned heads: 1) no_grad full (B,B) ã → select all positive pairs + (hard/random) negatives. 2) grad ã for selected pairs only → BCE logits. """ pos_mask, neg_mask, _, n_ignored = _pair_masks(labels, ignore_label=ignore_label) b = h.shape[0] empty_stats = { "n_pairs": 0, "n_pos": 0, "n_neg_used": 0, "n_ignored": n_ignored, "pos_ratio": 0.0, "pair_acc": 0.0, "pos_pair_acc": 0.0, "neg_pair_acc": 0.0, "eta": float(head.eta.detach().item()), "m": float(head.m.detach().item()), } if not pos_mask.any(): return h.new_zeros(()), empty_stats # ---- pass 1: select positive pairs + negatives (no grad) ---- with torch.no_grad(): a_full = head.pairwise_a_tilde(h, row_batch=256) pos_ij = torch.nonzero(pos_mask) # (n_pos, 2) n_pos = int(pos_ij.shape[0]) z_neg = None targets_neg = None n_neg_used = 0 if neg_mask.any(): neg_ij_all = torch.nonzero(neg_mask) # (n_neg_avail, 2) n_neg_avail = int(neg_ij_all.shape[0]) a_neg_all = a_full[neg_mask] if hard_neg_ratio > 0: n_neg_target = max(1, int(n_pos * float(hard_neg_ratio))) else: n_neg_target = max(1, int(n_pos * float(neg_sample_ratio))) if hard_neg_top > 0: n_pick = min(n_neg_avail, n_neg_target, hard_neg_top) _, top_idx = torch.topk(a_neg_all, k=n_pick, largest=True) else: n_pick = min(n_neg_avail, n_neg_target) if n_pick < n_neg_avail: top_idx = torch.randperm(n_neg_avail, device=h.device)[:n_pick] else: top_idx = torch.arange(n_neg_avail, device=h.device) neg_ij = neg_ij_all[top_idx] n_neg_used = int(neg_ij.shape[0]) # ---- pass 2: grad ã for selected pairs ---- a_tilde_pos = head.pairwise_a_tilde_for_pairs(h, pos_ij[:, 0], pos_ij[:, 1]) z_pos = head.eta * (a_tilde_pos - head.m) targets_pos = torch.ones_like(z_pos) if n_neg_used > 0: a_tilde_neg = head.pairwise_a_tilde_for_pairs(h, neg_ij[:, 0], neg_ij[:, 1]) z_neg = head.eta * (a_tilde_neg + head.m) targets_neg = torch.zeros_like(z_neg) z = torch.cat([z_pos, z_neg], dim=0) targets = torch.cat([targets_pos, targets_neg], dim=0) else: z = z_pos targets = targets_pos pw = pos_weight if pw is None and use_pos_weight and n_pos > 0 and n_neg_used > 0: pw = torch.tensor([float(n_neg_used) / float(n_pos)], device=z.device) loss = F.binary_cross_entropy_with_logits(z, targets, pos_weight=pw) with torch.no_grad(): pred = (torch.sigmoid(z) >= 0.5).float() acc = (pred == targets).float().mean().item() pred_pos = (torch.sigmoid(z_pos) >= 0.5).float() pos_acc = (pred_pos == 1.0).float().mean().item() if n_pos else 0.0 if n_neg_used > 0 and z_neg is not None: pred_neg = (torch.sigmoid(z_neg) >= 0.5).float() neg_acc = (pred_neg == 0.0).float().mean().item() else: neg_acc = 0.0 n_pairs = n_pos + n_neg_used return loss, { "n_pairs": n_pairs, "n_pos": n_pos, "n_neg_used": n_neg_used, "n_ignored": n_ignored, "pos_ratio": n_pos / max(n_pairs, 1), "pair_acc": acc, "pos_pair_acc": pos_acc, "neg_pair_acc": neg_acc, "eta": float(head.eta.detach().item()), "m": float(head.m.detach().item()), }