Download code/ay_infer.py from qinxunhui/deepellipse-circleid: direct link, hf CLI and curl.
- Browser
- Download file 11.4 kB
-
https://huggingface.co/qinxunhui/deepellipse-circleid/resolve/main/code/ay_infer.py
- Command line
-
hf download hf://qinxunhui/deepellipse-circleid/code/ay_infer.py
-
curl -L -o ay_infer.py https://huggingface.co/qinxunhui/deepellipse-circleid/resolve/main/code/ay_infer.py
11.4 kB
| #!/usr/bin/env python3 | |
| # -*- coding: utf-8 -*- | |
| """AY refine: A from BCE head + fixed Y → Top-K / r rejection CSV.""" | |
| from __future__ import annotations | |
| import json | |
| import os | |
| from typing import Dict, List, Optional, Sequence, Tuple | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from sklearn.preprocessing import LabelEncoder | |
| def load_y_logits_json(path: str) -> Tuple[List[str], np.ndarray]: | |
| """Load deep_ellipse_logits.json → (image_ids, Y[N,C]).""" | |
| with open(path, "r", encoding="utf-8") as f: | |
| rows = json.load(f) | |
| image_ids = [str(r["image_id"]) for r in rows] | |
| y = np.asarray([r["logits"] for r in rows], dtype=np.float32) | |
| return image_ids, y | |
| def otsu_threshold_1d(values: np.ndarray) -> float: | |
| """Otsu on 1-D float array (off-diagonal affinities).""" | |
| v = np.asarray(values, dtype=np.float64).ravel() | |
| v = v[np.isfinite(v)] | |
| if v.size == 0: | |
| return 0.0 | |
| vmin, vmax = float(v.min()), float(v.max()) | |
| if vmax - vmin < 1e-12: | |
| return vmin | |
| hist, bin_edges = np.histogram(v, bins=256, range=(vmin, vmax)) | |
| hist = hist.astype(np.float64) | |
| total = hist.sum() | |
| if total <= 0: | |
| return 0.5 * (vmin + vmax) | |
| bin_centers = 0.5 * (bin_edges[:-1] + bin_edges[1:]) | |
| weight1 = np.cumsum(hist) | |
| weight2 = total - weight1 | |
| sum_all = np.cumsum(hist * bin_centers) | |
| mean1 = sum_all / np.maximum(weight1, 1e-12) | |
| mean2 = (sum_all[-1] - sum_all) / np.maximum(weight2, 1e-12) | |
| inter = weight1[:-1] * weight2[:-1] * (mean1[:-1] - mean2[:-1]) ** 2 | |
| idx = int(np.argmax(inter)) | |
| return float(bin_centers[idx]) | |
| def build_a_norm( | |
| a: np.ndarray, | |
| *, | |
| power_k: float = 2.0, | |
| alpha: Optional[float] = None, | |
| sample_offdiag: int = 2_000_000, | |
| clamp_neg: bool = True, | |
| ) -> Tuple[np.ndarray, float]: | |
| """ | |
| A'[i,j] = max(0, A[i,j] - α) for i != j; diagonal restored as A[i,i]; | |
| A_norm = row-normalize (A')^k. | |
| α from Otsu on off-diagonal (optionally subsampled). | |
| clamp_neg=True (default): truncate the shifted affinities at 0, i.e. the | |
| current paper/code construction A' = max(0, A - α) on the off-diagonal. | |
| clamp_neg=False: keep the signed shift A' = A - α (diagonal still restored), | |
| which is the literal main-text formula A' = (A - α) + αI. Because the power | |
| ^k is applied AFTER the shift, negative entries fold back to positive for | |
| even k (e.g. k=2: A_ij=0.01 and 0.99 both give 0.2401). A non-integer | |
| power_k with clamp_neg=False would raise (negative base → NaN), so it is | |
| rejected up front. | |
| """ | |
| a = np.asarray(a, dtype=np.float32) | |
| n = a.shape[0] | |
| assert a.shape == (n, n) | |
| if not clamp_neg and float(power_k) != int(power_k): | |
| raise ValueError( | |
| f"clamp_neg=False requires an integer power_k (got {power_k!r}); " | |
| "a fractional power of a negative base is undefined." | |
| ) | |
| if alpha is None: | |
| # sample off-diagonal for Otsu | |
| if n * (n - 1) > sample_offdiag: | |
| rng = np.random.default_rng(0) | |
| i = rng.integers(0, n, size=sample_offdiag) | |
| j = rng.integers(0, n, size=sample_offdiag) | |
| mask = i != j | |
| off = a[i[mask], j[mask]] | |
| else: | |
| eye = np.eye(n, dtype=bool) | |
| off = a[~eye] | |
| alpha = otsu_threshold_1d(off) | |
| ap = a - float(alpha) | |
| np.fill_diagonal(ap, a.diagonal()) # restore diag: (A-α)+αI → diag stays A_ii | |
| # Ensure non-negative for power (clamp) unless the un-clamped variant is asked | |
| if clamp_neg: | |
| ap = np.maximum(ap, 0.0) | |
| if power_k != 1.0: | |
| ap = np.power(ap, power_k) | |
| row_sum = ap.sum(axis=1, keepdims=True) | |
| row_sum = np.maximum(row_sum, 1e-12) | |
| a_norm = ap / row_sum | |
| return a_norm.astype(np.float32), float(alpha) | |
| def row_knn_lambda_thresholds(a: np.ndarray, *, knn_n: int = 50) -> np.ndarray: | |
| """ | |
| Per-row lambda = (knn_n+1)-th largest off-diagonal affinity (1-indexed rank). | |
| For knn_n=50, lambda_i is the 51st largest value in row i (excluding diagonal). | |
| """ | |
| a = np.asarray(a, dtype=np.float32) | |
| n = a.shape[0] | |
| assert a.shape == (n, n) | |
| rank_idx = int(knn_n) # 0-based index of (knn_n+1)-th largest | |
| lambdas = np.zeros(n, dtype=np.float32) | |
| for i in range(n): | |
| row = a[i].copy() | |
| row[i] = -np.inf | |
| valid = row[np.isfinite(row) & (row > -np.inf)] | |
| if valid.size <= rank_idx: | |
| lambdas[i] = float(np.min(valid)) if valid.size else 0.0 | |
| else: | |
| sorted_desc = np.sort(valid)[::-1] | |
| lambdas[i] = float(sorted_desc[rank_idx]) | |
| return lambdas | |
| def build_a_norm_knn_row_lambda( | |
| a: np.ndarray, | |
| *, | |
| knn_n: int = 50, | |
| power_k: float = 2.0, | |
| ) -> Tuple[np.ndarray, np.ndarray]: | |
| """ | |
| A'[i,j] = max(0, A[i,j] - lambda_i) for j != i; diag stays A_ii; | |
| A_norm = row-normalize (A')^k with per-row lambda from row_knn_lambda_thresholds. | |
| """ | |
| a = np.asarray(a, dtype=np.float32) | |
| n = a.shape[0] | |
| assert a.shape == (n, n) | |
| lambdas = row_knn_lambda_thresholds(a, knn_n=knn_n) | |
| ap = a - lambdas[:, None] | |
| np.fill_diagonal(ap, a.diagonal()) | |
| ap = np.maximum(ap, 0.0) | |
| if power_k != 1.0: | |
| ap = np.power(ap, power_k) | |
| row_sum = ap.sum(axis=1, keepdims=True) | |
| row_sum = np.maximum(row_sum, 1e-12) | |
| a_norm = ap / row_sum | |
| return a_norm.astype(np.float32), lambdas | |
| def ay_refine( | |
| y_raw: np.ndarray, a_norm: np.ndarray, *, graph_hops: int = 1 | |
| ) -> np.ndarray: | |
| """Y_refined = A_norm^graph_hops @ Y_raw.""" | |
| if graph_hops < 1: | |
| raise ValueError(f"graph_hops must be >= 1, got {graph_hops!r}") | |
| y = y_raw | |
| for _ in range(graph_hops): | |
| y = a_norm @ y | |
| return y | |
| def decode_writer_id(class_idx: int, label_encoder: LabelEncoder) -> str: | |
| return str(label_encoder.classes_[class_idx]) | |
| def known_class_indices(label_encoder: LabelEncoder) -> np.ndarray: | |
| """Indices of known writers: every class slot except the unknown '-1'.""" | |
| idxs = [i for i, c in enumerate(label_encoder.classes_) if str(c) != "-1"] | |
| return np.asarray(idxs, dtype=np.int64) | |
| def scores_and_pred_from_y( | |
| y_refined: np.ndarray, | |
| label_encoder: LabelEncoder, | |
| *, | |
| reject_score: str = "logit", | |
| ) -> Tuple[np.ndarray, np.ndarray]: | |
| """ | |
| Returns (scores[N], pred_cls[N]). | |
| reject_score='logit': max over known cols of Y_ref | |
| reject_score='softmax': max softmax prob over known cols | |
| """ | |
| if reject_score not in ("logit", "softmax"): | |
| raise ValueError(f"reject_score must be logit|softmax, got {reject_score!r}") | |
| kidx = known_class_indices(label_encoder) | |
| y_k = np.asarray(y_refined, dtype=np.float64)[:, kidx] | |
| if reject_score == "softmax": | |
| y_k = y_k - y_k.max(axis=1, keepdims=True) | |
| exp = np.exp(y_k) | |
| probs = exp / np.maximum(exp.sum(axis=1, keepdims=True), 1e-12) | |
| local = probs.argmax(axis=1) | |
| scores = probs.max(axis=1) | |
| else: | |
| local = y_k.argmax(axis=1) | |
| scores = y_k.max(axis=1) | |
| pred_cls = kidx[local] | |
| return scores.astype(np.float64), pred_cls.astype(np.int64) | |
| def apply_topk_reject( | |
| writers: Sequence[str], | |
| conf: Sequence[float], | |
| *, | |
| reject_ratio: float = 0.65, | |
| reject_score: str = "logit", | |
| ) -> Tuple[List[str], List[Optional[str]], Optional[float]]: | |
| """ | |
| Apply Top-r rejection on precomputed argmax writers + confidence scores. | |
| Returns (final_writer_ids, reject_reasons, dyn_thresh). | |
| """ | |
| n = len(writers) | |
| target_unknown = int(n * reject_ratio) | |
| final = [str(w) for w in writers] | |
| conf_arr = np.asarray(conf, dtype=np.float64) | |
| reject_reason: List[Optional[str]] = [None] * n | |
| n_natural = 0 | |
| for i in range(n): | |
| if final[i] == "-1": | |
| reject_reason[i] = "invalid_prediction" | |
| n_natural += 1 | |
| remaining = [i for i in range(n) if final[i] != "-1"] | |
| remaining_sorted = sorted(remaining, key=lambda i: conf_arr[i]) | |
| n_dyn = max(0, target_unknown - n_natural) | |
| n_dyn = min(n_dyn, len(remaining_sorted)) | |
| thresh: Optional[float] = None | |
| reason_tag = "low_confidence" if reject_score == "softmax" else "low_logit" | |
| for i in remaining_sorted[:n_dyn]: | |
| final[i] = "-1" | |
| reject_reason[i] = reason_tag | |
| thresh = float(conf_arr[i]) | |
| return final, reject_reason, thresh | |
| def _write_reject_csv( | |
| image_ids: Sequence[str], | |
| final: Sequence[str], | |
| conf: Sequence[float], | |
| reject_reason: Sequence[Optional[str]], | |
| *, | |
| out_path: str, | |
| reject_score: str, | |
| thresh: Optional[float], | |
| ) -> pd.DataFrame: | |
| n = len(image_ids) | |
| df = pd.DataFrame( | |
| { | |
| "image_id": list(image_ids), | |
| "writer_id": list(final), | |
| "confidence": np.asarray(conf, dtype=np.float64), | |
| "reject_reason": list(reject_reason), | |
| } | |
| ) | |
| os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True) | |
| df.to_csv(out_path, index=False) | |
| clean = out_path.replace(".csv", "_submit.csv") | |
| if clean == out_path: | |
| clean = out_path[:-4] + "_submit.csv" | |
| df[["image_id", "writer_id"]].to_csv(clean, index=False) | |
| n_unk = sum(1 for w in final if w == "-1") | |
| print( | |
| f"AY Top-K CSV ({reject_score}): n={n}, unknown={n_unk} " | |
| f"({n_unk / n * 100:.1f}%), " | |
| f"dyn_thresh={thresh if thresh is not None else 'N/A'}" | |
| ) | |
| print(f" saved: {out_path}") | |
| print(f" submit: {clean}") | |
| return df | |
| def resweep_reject_from_scored( | |
| image_ids: Sequence[str], | |
| writers: Sequence[str], | |
| conf: Sequence[float], | |
| *, | |
| reject_ratio: float = 0.65, | |
| out_path: str, | |
| reject_score: str = "logit", | |
| ) -> pd.DataFrame: | |
| """Re-apply Top-r on fixed confidence ranking (r-grid without re-inference).""" | |
| final, reject_reason, thresh = apply_topk_reject( | |
| writers, conf, reject_ratio=reject_ratio, reject_score=reject_score | |
| ) | |
| return _write_reject_csv( | |
| image_ids, | |
| final, | |
| conf, | |
| reject_reason, | |
| out_path=out_path, | |
| reject_score=reject_score, | |
| thresh=thresh, | |
| ) | |
| def topk_reject_csv( | |
| image_ids: Sequence[str], | |
| y_refined: np.ndarray, | |
| label_encoder: LabelEncoder, | |
| *, | |
| reject_ratio: float = 0.65, | |
| out_path: str, | |
| reject_score: str = "logit", | |
| ) -> pd.DataFrame: | |
| """ | |
| Score by max logit or max softmax (known classes only); reject lowest | |
| reject_ratio as -1; else argmax → writer_id. | |
| """ | |
| scores, pred_cls = scores_and_pred_from_y( | |
| y_refined, label_encoder, reject_score=reject_score | |
| ) | |
| writers = [decode_writer_id(int(c), label_encoder) for c in pred_cls] | |
| final, reject_reason, thresh = apply_topk_reject( | |
| writers, scores, reject_ratio=reject_ratio, reject_score=reject_score | |
| ) | |
| return _write_reject_csv( | |
| image_ids, | |
| final, | |
| scores, | |
| reject_reason, | |
| out_path=out_path, | |
| reject_score=reject_score, | |
| thresh=thresh, | |
| ) | |
| def align_y_to_ids( | |
| y_ids: Sequence[str], | |
| y: np.ndarray, | |
| target_ids: Sequence[str], | |
| ) -> np.ndarray: | |
| """Reorder Y rows to match target_ids order.""" | |
| idx = {str(i): k for k, i in enumerate(y_ids)} | |
| missing = [i for i in target_ids if str(i) not in idx] | |
| if missing: | |
| raise KeyError(f"Y logits missing {len(missing)} ids, e.g. {missing[:3]}") | |
| order = [idx[str(i)] for i in target_ids] | |
| return y[order] | |