Download code/ay_bce_head.py from qinxunhui/deepellipse-circleid: direct link, hf CLI and curl.
- Browser
- Download file 17.4 kB
-
https://huggingface.co/qinxunhui/deepellipse-circleid/resolve/main/code/ay_bce_head.py
- Command line
-
hf download hf://qinxunhui/deepellipse-circleid/code/ay_bce_head.py
-
curl -L -o ay_bce_head.py https://huggingface.co/qinxunhui/deepellipse-circleid/resolve/main/code/ay_bce_head.py
17.4 kB
| #!/usr/bin/env python3 | |
| # -*- coding: utf-8 -*- | |
| """BCE pairwise similarity head for AY open-set inference (§5.2.1). | |
| head_type 'bce': classic PairwiseBCEHead (cos → cubic → η σ). | |
| head_type 'cond': PairwiseCondHead (pairwise-conditioned observation space). | |
| """ | |
| from __future__ import annotations | |
| from typing import Optional, Tuple | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| HEAD_TYPES = ("bce", "cond", "supcon") | |
| class PairwiseBCEHead(nn.Module): | |
| """Map f_ellipse → L2-normalized h; learnable η, m for BCE-with-logits.""" | |
| def __init__( | |
| self, | |
| in_dim: int = 1024, | |
| emb_dim: int = 256, | |
| hidden_dim: int = 512, | |
| eta_init: float = 10.0, | |
| m_init: float = 0.1, | |
| dropout: float = 0.1, | |
| ): | |
| super().__init__() | |
| self.emb_dim = emb_dim | |
| self.mlp = nn.Sequential( | |
| nn.Linear(in_dim, hidden_dim), | |
| nn.ReLU(inplace=True), | |
| nn.Dropout(dropout), | |
| nn.Linear(hidden_dim, emb_dim), | |
| ) | |
| self.eta = nn.Parameter(torch.tensor(float(eta_init))) | |
| self.m = nn.Parameter(torch.tensor(float(m_init))) | |
| def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor: | |
| """(B, in_dim) → (B, emb_dim) L2-normalized.""" | |
| h = self.mlp(f_ellipse) | |
| return F.normalize(h, p=2, dim=-1) | |
| def pairwise_a_tilde( | |
| self, h: torch.Tensor, row_batch: int = 256 | |
| ) -> torch.Tensor: | |
| """Cosine u → nonlinear ã. h must be L2-normalized. Returns (B, B).""" | |
| u = h @ h.T | |
| return 2.0 * ((u + 1.0) / 2.0).pow(3) - 1.0 | |
| def pairwise_a_tilde_for_pairs( | |
| self, h: torch.Tensor, i_idx: torch.Tensor, j_idx: torch.Tensor | |
| ) -> torch.Tensor: | |
| """ã for selected (i,j) pairs. Returns (P,).""" | |
| u = (h[i_idx] * h[j_idx]).sum(dim=-1) | |
| return 2.0 * ((u + 1.0) / 2.0).pow(3) - 1.0 | |
| def affinity_matrix( | |
| self, h: torch.Tensor, row_batch: int = 256, col_batch: int = None | |
| ) -> torch.Tensor: | |
| """Test-time A = σ(η ã) (no margin). (N, N).""" | |
| z = self.eta * self.pairwise_a_tilde(h, row_batch=row_batch) | |
| return torch.sigmoid(z) | |
| class PairwiseCondHead(nn.Module): | |
| """Pairwise-conditioned observation head (PCOH). | |
| Observation space is jointly decided by (v_a, v_b): a shared MLP maps each | |
| side's own code plus symmetric pair statistics r_ab into an observation | |
| embedding; the pair score is the cosine in that observation space. | |
| h_a = L2Norm(MLP_base(v_a)), h_b = L2Norm(MLP_base(v_b)) | |
| r_ab = [ h_a⊙h_b ; |h_a-h_b| ; h_a+h_b ] (symmetric) | |
| u_a = L2Norm(h_a + MLP_cond([h_a; r_ab])) | |
| u_b = L2Norm(h_b + MLP_cond([h_b; r_ab])) (shared weights) | |
| s(a,b) = u_a · u_b (symmetric) | |
| Zero-init of MLP_cond's last layer gives s = cos(h_a,h_b) initially, so | |
| training starts from the BCE-cosine baseline and learns an increment. | |
| """ | |
| def __init__( | |
| self, | |
| in_dim: int = 1024, | |
| emb_dim: int = 256, | |
| hidden_dim: int = 256, | |
| eta_init: float = 10.0, | |
| m_init: float = 0.1, | |
| dropout: float = 0.3, | |
| ): | |
| super().__init__() | |
| self.emb_dim = emb_dim | |
| self.hidden_dim = hidden_dim | |
| self.mlp = nn.Sequential( | |
| nn.Linear(in_dim, hidden_dim), | |
| nn.ReLU(inplace=True), | |
| nn.Dropout(dropout), | |
| nn.Linear(hidden_dim, emb_dim), | |
| ) | |
| # conditional observation branch: input [h; r_ab] (4*emb_dim). | |
| # First layer weight split as [W_h | W_r] so we never materialize | |
| # the concatenated (..., 4e) tensor explicitly. | |
| self.cond_in = nn.Linear(4 * emb_dim, hidden_dim) | |
| self.cond_h = nn.Linear(emb_dim, hidden_dim, bias=False) # [h] part | |
| self.cond_r = nn.Linear(3 * emb_dim, hidden_dim, bias=False) # [r] part | |
| # share the same bias of cond_in | |
| with torch.no_grad(): | |
| self.cond_h.weight.copy_(self.cond_in.weight[:, :emb_dim]) | |
| self.cond_r.weight.copy_(self.cond_in.weight[:, emb_dim:]) | |
| self.cond_bias = nn.Parameter(self.cond_in.bias.clone()) | |
| self.cond_act = nn.ReLU(inplace=True) | |
| self.cond_drop = nn.Dropout(dropout) | |
| self.cond_out = nn.Linear(hidden_dim, emb_dim) | |
| # zero-init last layer → residual 0 at init → s = cos(h_a, h_b) | |
| nn.init.zeros_(self.cond_out.weight) | |
| nn.init.zeros_(self.cond_out.bias) | |
| self.eta = nn.Parameter(torch.tensor(float(eta_init))) | |
| self.m = nn.Parameter(torch.tensor(float(m_init))) | |
| def _cond_forward(self, h_side: torch.Tensor, r: torch.Tensor) -> torch.Tensor: | |
| """MLP_cond([h_side; r]) via split weights. Any leading batch dims.""" | |
| x = self.cond_h(h_side) + self.cond_r(r) + self.cond_bias | |
| x = self.cond_act(x) | |
| x = self.cond_drop(x) | |
| return self.cond_out(x) | |
| def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor: | |
| """(B, in_dim) → (B, emb_dim) L2-normalized base code h.""" | |
| h = self.mlp(f_ellipse) | |
| return F.normalize(h, p=2, dim=-1) | |
| def _pairwise_s_block( | |
| self, h: torch.Tensor, row_start: int, row_end: int, col_batch: int = 512 | |
| ) -> torch.Tensor: | |
| """s rows [row_start,row_end) against all columns. Returns (rb, n).""" | |
| ha = h[row_start:row_end] # (rb, e) | |
| rb = ha.shape[0] | |
| n = h.shape[0] | |
| out = torch.empty(rb, n, device=h.device, dtype=h.dtype) | |
| for c0 in range(0, n, col_batch): | |
| c1 = min(c0 + col_batch, n) | |
| hb = h[c0:c1] # (cb, e) | |
| prod = ha.unsqueeze(1) * hb.unsqueeze(0) # (rb, cb, e) | |
| diff = (ha.unsqueeze(1) - hb.unsqueeze(0)).abs() | |
| summ = ha.unsqueeze(1) + hb.unsqueeze(0) | |
| r = torch.cat([prod, diff, summ], dim=-1) # (rb, cb, 3e) | |
| h_a_exp = ha.unsqueeze(1).expand(rb, c1 - c0, self.emb_dim) | |
| h_b_exp = hb.unsqueeze(0).expand(rb, c1 - c0, self.emb_dim) | |
| u_a = F.normalize( | |
| h_a_exp + self._cond_forward(h_a_exp, r), p=2, dim=-1 | |
| ) | |
| u_b = F.normalize( | |
| h_b_exp + self._cond_forward(h_b_exp, r), p=2, dim=-1 | |
| ) | |
| out[:, c0:c1] = (u_a * u_b).sum(dim=-1) | |
| return out | |
| def pairwise_a_tilde( | |
| self, h: torch.Tensor, row_batch: int = 128, col_batch: int = 512 | |
| ) -> torch.Tensor: | |
| """Symmetric ã = cos in the conditioned observation space. (B, B).""" | |
| n = h.shape[0] | |
| out = torch.empty(n, n, device=h.device, dtype=h.dtype) | |
| for start in range(0, n, row_batch): | |
| end = min(start + row_batch, n) | |
| out[start:end] = self._pairwise_s_block( | |
| h, start, end, col_batch=col_batch | |
| ) | |
| return out | |
| def pairwise_a_tilde_for_pairs( | |
| self, | |
| h: torch.Tensor, | |
| i_idx: torch.Tensor, | |
| j_idx: torch.Tensor, | |
| ) -> torch.Tensor: | |
| """ã for selected (i,j) pairs via the conditioned observation space. (P,).""" | |
| ha = h[i_idx] # (P, e) | |
| hb = h[j_idx] | |
| prod = ha * hb | |
| diff = (ha - hb).abs() | |
| summ = ha + hb | |
| r = torch.cat([prod, diff, summ], dim=-1) # (P, 3e) | |
| u_a = F.normalize(ha + self._cond_forward(ha, r), p=2, dim=-1) | |
| u_b = F.normalize(hb + self._cond_forward(hb, r), p=2, dim=-1) | |
| return (u_a * u_b).sum(dim=-1) | |
| def affinity_matrix( | |
| self, h: torch.Tensor, row_batch: int = 128, col_batch: int = 512 | |
| ) -> torch.Tensor: | |
| """Test-time A = σ(η ã). (N, N).""" | |
| z = self.eta * self.pairwise_a_tilde( | |
| h, row_batch=row_batch, col_batch=col_batch | |
| ) | |
| return torch.sigmoid(z) | |
| class SupConHead(nn.Module): | |
| """Pure supervised-contrastive embedding head (no BCE components). | |
| MLP structure is identical to PairwiseBCEHead for a like-for-like | |
| comparison, but all BCE machinery is removed (no eta/margin, no cubic | |
| map, no sigmoid affinity). The output is an L2-normalized embedding h; | |
| the test-time adjacency is built downstream as A_ij = cos(h_i, h_j). | |
| """ | |
| def __init__( | |
| self, | |
| in_dim: int = 1024, | |
| emb_dim: int = 256, | |
| hidden_dim: int = 512, | |
| dropout: float = 0.1, | |
| ): | |
| super().__init__() | |
| self.emb_dim = emb_dim | |
| self.mlp = nn.Sequential( | |
| nn.Linear(in_dim, hidden_dim), | |
| nn.ReLU(inplace=True), | |
| nn.Dropout(dropout), | |
| nn.Linear(hidden_dim, emb_dim), | |
| ) | |
| def forward(self, f_ellipse: torch.Tensor) -> torch.Tensor: | |
| """(B, in_dim) → (B, emb_dim) L2-normalized embedding.""" | |
| h = self.mlp(f_ellipse) | |
| return F.normalize(h, p=2, dim=-1) | |
| def supcon_loss( | |
| h: torch.Tensor, | |
| labels: torch.Tensor, | |
| *, | |
| temperature: float = 0.1, | |
| ignore_label: int = -1, | |
| ) -> Tuple[torch.Tensor, dict]: | |
| """Supervised contrastive loss (Khosla et al., NeurIPS 2020). | |
| h: (B, d) L2-normalized embeddings. labels: (B,) integer writer labels; | |
| rows with ``ignore_label`` are dropped. Positive = same writer in-batch. | |
| Returns (loss, stats) with the same keys as ``pairwise_bce_loss`` so the | |
| training loop can log uniformly. ``pos_pair_acc`` / ``neg_pair_acc`` hold | |
| the mean positive / negative cosine (higher/lower is better) and | |
| ``pair_acc`` is the per-anchor ranking accuracy (mean pos cos > mean neg cos). | |
| """ | |
| device = h.device | |
| valid = labels != ignore_label | |
| n_ignored = int((~valid).sum().item()) | |
| stats_empty = { | |
| "n_pairs": 0, | |
| "n_pos": 0, | |
| "n_neg_used": 0, | |
| "n_ignored": n_ignored, | |
| "pos_ratio": 0.0, | |
| "pair_acc": 0.0, | |
| "pos_pair_acc": 0.0, | |
| "neg_pair_acc": 0.0, | |
| "eta": 0.0, | |
| "m": 0.0, | |
| } | |
| if int(valid.sum().item()) < 2: | |
| return h.new_zeros(()), stats_empty | |
| h = h[valid] | |
| labels = labels[valid] | |
| b = h.shape[0] | |
| sim = (h @ h.T) / float(temperature) # (b, b) | |
| eye = torch.eye(b, device=device, dtype=torch.bool) | |
| sim_masked = sim.masked_fill(eye, float("-inf")) # exclude self | |
| log_prob = sim_masked - torch.logsumexp(sim_masked, dim=1, keepdim=True) | |
| # zero-out the diagonal so masked sums never hit 0 * -inf = nan | |
| log_prob = log_prob.masked_fill(eye, 0.0) | |
| pos_mask = (labels.unsqueeze(0) == labels.unsqueeze(1)) & ~eye | |
| neg_mask = (~eye) & ~pos_mask | |
| n_pos_per = pos_mask.sum(dim=1) | |
| rows = n_pos_per > 0 | |
| if not bool(rows.any()): | |
| return h.new_zeros(()), stats_empty | |
| mean_log_prob_pos = ( | |
| (pos_mask.float() * log_prob).sum(dim=1)[rows] / n_pos_per[rows] | |
| ) | |
| loss = -mean_log_prob_pos.mean() | |
| with torch.no_grad(): | |
| n_pos_tot = int(pos_mask.sum().item()) | |
| n_neg_tot = int(neg_mask.sum().item()) | |
| pos_sim = (sim * pos_mask.float()).sum() / max(n_pos_tot, 1) | |
| neg_sim = (sim * neg_mask.float()).sum() / max(n_neg_tot, 1) | |
| pos_mean_row = (sim * pos_mask.float()).sum(1) / n_pos_per.clamp(min=1) | |
| neg_mean_row = ( | |
| (sim * neg_mask.float()).sum(1) / neg_mask.sum(1).clamp(min=1) | |
| ) | |
| pair_acc = ( | |
| ((pos_mean_row > neg_mean_row) & rows).float().sum() | |
| / rows.float().sum() | |
| ) | |
| stats = { | |
| "n_pairs": n_pos_tot, | |
| "n_pos": n_pos_tot, | |
| "n_neg_used": n_neg_tot, | |
| "n_ignored": n_ignored, | |
| "pos_ratio": n_pos_tot / max(n_pos_tot + n_neg_tot, 1), | |
| "pair_acc": float(pair_acc.item()), | |
| "pos_pair_acc": float(pos_sim.item()), | |
| "neg_pair_acc": float(neg_sim.item()), | |
| "eta": 0.0, | |
| "m": 0.0, | |
| } | |
| return loss, stats | |
| def build_pairwise_head( | |
| head_type: str, | |
| *, | |
| in_dim: int = 1024, | |
| emb_dim: int = 256, | |
| hidden_dim: int = 512, | |
| **kwargs, | |
| ) -> nn.Module: | |
| """Factory: 'bce' → PairwiseBCEHead; 'cond' → PairwiseCondHead; 'supcon' → SupConHead.""" | |
| if head_type == "cond": | |
| return PairwiseCondHead( | |
| in_dim=in_dim, emb_dim=emb_dim, hidden_dim=hidden_dim, **kwargs | |
| ) | |
| if head_type == "supcon": | |
| return SupConHead( | |
| in_dim=in_dim, | |
| emb_dim=emb_dim, | |
| hidden_dim=hidden_dim, | |
| dropout=kwargs.get("dropout", 0.1), | |
| ) | |
| return PairwiseBCEHead( | |
| in_dim=in_dim, emb_dim=emb_dim, hidden_dim=hidden_dim, **kwargs | |
| ) | |
| def _pair_masks( | |
| labels: torch.Tensor, | |
| *, | |
| ignore_label: int = -1, | |
| ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: | |
| """Return pos_mask, neg_mask, both_minus1 (B,B), n_ignored off-diagonal (-1,-1).""" | |
| b = labels.shape[0] | |
| eye = torch.eye(b, device=labels.device, dtype=torch.bool) | |
| lab_i = labels.unsqueeze(0) | |
| lab_j = labels.unsqueeze(1) | |
| both_minus1 = (lab_i == ignore_label) & (lab_j == ignore_label) | |
| n_ignored = int(both_minus1[~eye].sum().item()) | |
| same_label = lab_i == lab_j | |
| pos_mask = ( | |
| same_label | |
| & (lab_i >= 0) | |
| & (lab_j >= 0) | |
| & ~eye | |
| & ~both_minus1 | |
| ) | |
| neg_mask = (~eye) & ~both_minus1 & ~pos_mask | |
| return pos_mask, neg_mask, both_minus1, n_ignored | |
| def pairwise_bce_loss( | |
| head: PairwiseBCEHead, | |
| h: torch.Tensor, | |
| labels: torch.Tensor, | |
| *, | |
| ignore_label: int = -1, | |
| neg_sample_ratio: float = 10.0, | |
| hard_neg_top: int = 0, | |
| hard_neg_ratio: float = 0.0, | |
| use_pos_weight: bool = False, | |
| pos_weight: Optional[torch.Tensor] = None, | |
| ) -> Tuple[torch.Tensor, dict]: | |
| """ | |
| BCE on pairwise logits with (-1,-1) skipped; (-1,known) as negatives. | |
| Two passes for memory efficiency on pairwise-conditioned heads: | |
| 1) no_grad full (B,B) ã → select all positive pairs + (hard/random) negatives. | |
| 2) grad ã for selected pairs only → BCE logits. | |
| """ | |
| pos_mask, neg_mask, _, n_ignored = _pair_masks(labels, ignore_label=ignore_label) | |
| b = h.shape[0] | |
| empty_stats = { | |
| "n_pairs": 0, | |
| "n_pos": 0, | |
| "n_neg_used": 0, | |
| "n_ignored": n_ignored, | |
| "pos_ratio": 0.0, | |
| "pair_acc": 0.0, | |
| "pos_pair_acc": 0.0, | |
| "neg_pair_acc": 0.0, | |
| "eta": float(head.eta.detach().item()), | |
| "m": float(head.m.detach().item()), | |
| } | |
| if not pos_mask.any(): | |
| return h.new_zeros(()), empty_stats | |
| # ---- pass 1: select positive pairs + negatives (no grad) ---- | |
| with torch.no_grad(): | |
| a_full = head.pairwise_a_tilde(h, row_batch=256) | |
| pos_ij = torch.nonzero(pos_mask) # (n_pos, 2) | |
| n_pos = int(pos_ij.shape[0]) | |
| z_neg = None | |
| targets_neg = None | |
| n_neg_used = 0 | |
| if neg_mask.any(): | |
| neg_ij_all = torch.nonzero(neg_mask) # (n_neg_avail, 2) | |
| n_neg_avail = int(neg_ij_all.shape[0]) | |
| a_neg_all = a_full[neg_mask] | |
| if hard_neg_ratio > 0: | |
| n_neg_target = max(1, int(n_pos * float(hard_neg_ratio))) | |
| else: | |
| n_neg_target = max(1, int(n_pos * float(neg_sample_ratio))) | |
| if hard_neg_top > 0: | |
| n_pick = min(n_neg_avail, n_neg_target, hard_neg_top) | |
| _, top_idx = torch.topk(a_neg_all, k=n_pick, largest=True) | |
| else: | |
| n_pick = min(n_neg_avail, n_neg_target) | |
| if n_pick < n_neg_avail: | |
| top_idx = torch.randperm(n_neg_avail, device=h.device)[:n_pick] | |
| else: | |
| top_idx = torch.arange(n_neg_avail, device=h.device) | |
| neg_ij = neg_ij_all[top_idx] | |
| n_neg_used = int(neg_ij.shape[0]) | |
| # ---- pass 2: grad ã for selected pairs ---- | |
| a_tilde_pos = head.pairwise_a_tilde_for_pairs(h, pos_ij[:, 0], pos_ij[:, 1]) | |
| z_pos = head.eta * (a_tilde_pos - head.m) | |
| targets_pos = torch.ones_like(z_pos) | |
| if n_neg_used > 0: | |
| a_tilde_neg = head.pairwise_a_tilde_for_pairs(h, neg_ij[:, 0], neg_ij[:, 1]) | |
| z_neg = head.eta * (a_tilde_neg + head.m) | |
| targets_neg = torch.zeros_like(z_neg) | |
| z = torch.cat([z_pos, z_neg], dim=0) | |
| targets = torch.cat([targets_pos, targets_neg], dim=0) | |
| else: | |
| z = z_pos | |
| targets = targets_pos | |
| pw = pos_weight | |
| if pw is None and use_pos_weight and n_pos > 0 and n_neg_used > 0: | |
| pw = torch.tensor([float(n_neg_used) / float(n_pos)], device=z.device) | |
| loss = F.binary_cross_entropy_with_logits(z, targets, pos_weight=pw) | |
| with torch.no_grad(): | |
| pred = (torch.sigmoid(z) >= 0.5).float() | |
| acc = (pred == targets).float().mean().item() | |
| pred_pos = (torch.sigmoid(z_pos) >= 0.5).float() | |
| pos_acc = (pred_pos == 1.0).float().mean().item() if n_pos else 0.0 | |
| if n_neg_used > 0 and z_neg is not None: | |
| pred_neg = (torch.sigmoid(z_neg) >= 0.5).float() | |
| neg_acc = (pred_neg == 0.0).float().mean().item() | |
| else: | |
| neg_acc = 0.0 | |
| n_pairs = n_pos + n_neg_used | |
| return loss, { | |
| "n_pairs": n_pairs, | |
| "n_pos": n_pos, | |
| "n_neg_used": n_neg_used, | |
| "n_ignored": n_ignored, | |
| "pos_ratio": n_pos / max(n_pairs, 1), | |
| "pair_acc": acc, | |
| "pos_pair_acc": pos_acc, | |
| "neg_pair_acc": neg_acc, | |
| "eta": float(head.eta.detach().item()), | |
| "m": float(head.m.detach().item()), | |
| } | |