Add fp16→fp32 fallback for vision encoder WebGPU compatibility

js/constants.js

@@ -4,7 +4,10 @@
  * Ported from hf_space/constants.py.
  */
 
+// ---------------------------------------------------------------------------
 // Prompts
+// ---------------------------------------------------------------------------
+
 export const INTERPRET_PROMPT =
   "Provide a comprehensive clinical interpretation of this fetal ultrasound " +
   "image. Identify all visible anatomical structures, describe the fetal " +
@@ -21,7 +24,10 @@ export const CLASSIFY_PROMPT =
   "What anatomical view or structure does it show? " +
   'Return a JSON object with "label" as the classification.';
 
+// ---------------------------------------------------------------------------
 // Classification -> Detection / Segmentation class mapping
+// ---------------------------------------------------------------------------
+
 export const CLASSIFY_TO_DETECT = {
   huvf: "abdomen, arm, head, legs",
   huvb: "abdomen, arm, head, legs",
@@ -58,20 +64,39 @@ export const SPECIFIC_CLS_LABELS = new Set([
   "V sign", "X sign", "liver",
 ]);
 
-// Keyword-based mapping
+// ---------------------------------------------------------------------------
+// Keyword-based mapping (ordered specific-first)
+// ---------------------------------------------------------------------------
+
 export const KEYWORD_FALLBACK = [
-  [["cerebel","cervellet","ventric","thalm","bpd","banana","holopros","cisterna","fossa","csp","cavum","septum pellucidum","lateral ventricle"],"Brain, CSP, LV","Brain"],
-  [["brain"],"Brain, CSP, LV","Brain"],
-  [["cardiac","thorax","thorac","thoric","thorcic","aort","heart","stenos","chamber","valve"],"cardiac, thorax","cardiac, thorax"],
-  [["fetal head","fetal_head","cervix","cervic","cervex","pubic","symphys","pelvimetr","angle of progression"],"fetal_head, pubic_symphysis","fetal_head, pubic_symphysis"],
-  [["femur","femoral","fetal.femur","fetal-femur"],"legs",null],
-  [["nasal","nuchal","translucen"],"NT, nasal_bone, nasal_skin, nasal_tip","NT"],
-  [["crl","crown","rump"],"B, CRL, H, NB",null],
-  [["liver","stomach","vein","artery"],"artery, liver, stomach, vein","artery, liver, stomach, vein"],
-  [["abdomen","abdom"],"abdomen",null],
-  [["arm"],"abdomen, arm",null],
-  [["head"],"head",null],
-  [["leg"],"abdomen, legs",null],
+  [["cerebel", "cervellet", "ventric", "thalm", "bpd",
+    "banana", "holopros", "cisterna", "fossa", "csp", "cavum",
+    "septum pellucidum", "lateral ventricle"],
+   "Brain, CSP, LV", "Brain"],
+  [["brain"],
+   "Brain, CSP, LV", "Brain"],
+  [["cardiac", "thorax", "thorac", "thoric", "thorcic",
+    "aort", "heart", "stenos", "chamber", "valve"],
+   "cardiac, thorax", "cardiac, thorax"],
+  [["fetal head", "fetal_head", "cervix", "cervic", "cervex", "pubic", "symphys",
+    "pelvimetr", "angle of progression"],
+   "fetal_head, pubic_symphysis", "fetal_head, pubic_symphysis"],
+  [["femur", "femoral", "fetal.femur", "fetal-femur"],
+   "legs", null],
+  [["nasal", "nuchal", "translucen"],
+   "NT, nasal_bone, nasal_skin, nasal_tip", "NT"],
+  [["crl", "crown", "rump"],
+   "B, CRL, H, NB", null],
+  [["liver", "stomach", "vein", "artery"],
+   "artery, liver, stomach, vein", "artery, liver, stomach, vein"],
+  [["abdomen", "abdom"],
+   "abdomen", null],
+  [["arm"],
+   "abdomen, arm", null],
+  [["head"],
+   "head", null],
+  [["leg"],
+   "abdomen, legs", null],
 ];
 
 export const DEFAULT_DETECT_CLASSES = "abdomen, arm, head, legs";
@@ -86,103 +111,145 @@ export function keywordLookup(text) {
   return null;
 }
 
-// Per-class keyword scoring
+// ---------------------------------------------------------------------------
+// Per-class keyword scoring for interpretation-first mapping
+// ---------------------------------------------------------------------------
+
 export const FIELD_WEIGHTS = {
-  biometric_measurements: 3.0, imaging_plane: 2.5,
-  anatomical_structures: 1.0, fetal_orientation: 0.5,
-  normality_assessment: 0.5, gestational_age: 0.3,
-  image_quality: 0.0, clinical_recommendations: 0.3,
+  biometric_measurements: 3.0,
+  imaging_plane: 2.5,
+  anatomical_structures: 1.0,
+  fetal_orientation: 0.5,
+  normality_assessment: 0.5,
+  gestational_age: 0.3,
+  image_quality: 0.0,
+  clinical_recommendations: 0.3,
 };
 
 export const CLASS_KEYWORDS = {
-  Brain:["brain","cerebr","cerebellum","cortex","vermis","falx","interhemispheric","cranial","intracranial","parieto-occipital","frontal lobe","occipital"],
-  CSP:["septum pellucidum","cavum","csp","midline echo"],
-  LV:["lateral ventricle","ventricl","choroid plexus","ventriculomegaly"],
-  cardiac:["heart","cardiac","chamber","four chamber","4-chamber","valve","myocard","atri","foramen ovale","interventricular","aort","aortic"],
-  thorax:["thorax","thorac","thoric","thorcic","chest","lung","pleural","rib cage","diaphragm","mediastin"],
-  NT:["nuchal translucen","nuchal fold","nuchal"," nt "," nt,","nt_","nt "],
-  nasal_bone:["nasal bone","nasal bridge"],
-  nasal_skin:["nasal skin"],
-  nasal_tip:["nasal tip"],
-  CRL:["crown-rump","crown rump","crl"],
-  B:[], H:[], NB:[],
-  artery:["artery","arter","umbilical arter"],
-  liver:["liver","hepat"],
-  stomach:["stomach","gastric","stomach bubble"],
-  vein:["vein","venous","vena cava","umbilical vein"],
-  fetal_head:["cervix","cervic","cervical length"],
-  pubic_symphysis:["pubic","symphys","pelvimetr"],
-  abdomen:["abdomen","abdomin","abdom","ac_plane"],
-  arm:["arm","upper limb","humerus"],
-  head:["skull","calvar","biparietal","fetal head","head","neck"],
-  legs:["femur","femoral","thigh","lower limb","lower extremit","tibia","legs","leg "],
+  Brain: ["brain", "cerebr", "cerebellum", "cortex", "vermis",
+          "falx", "interhemispheric", "cranial", "intracranial",
+          "parieto-occipital", "frontal lobe", "occipital"],
+  CSP:   ["septum pellucidum", "cavum", "csp", "midline echo"],
+  LV:    ["lateral ventricle", "ventricl", "choroid plexus",
+          "ventriculomegaly"],
+  cardiac: ["heart", "cardiac", "chamber", "four chamber",
+            "4-chamber", "valve", "myocard", "atri",
+            "foramen ovale", "interventricular", "aort", "aortic"],
+  thorax:  ["thorax", "thorac", "thoric", "thorcic", "chest",
+            "lung", "pleural", "rib cage", "diaphragm", "mediastin"],
+  NT:         ["nuchal translucen", "nuchal fold", "nuchal",
+               " nt ", " nt,", "nt_", "nt "],
+  nasal_bone: ["nasal bone", "nasal bridge"],
+  nasal_skin: ["nasal skin"],
+  nasal_tip:  ["nasal tip"],
+  CRL: ["crown-rump", "crown rump", "crl"],
+  B:   [],
+  H:   [],
+  NB:  [],
+  artery:  ["artery", "arter", "umbilical arter"],
+  liver:   ["liver", "hepat"],
+  stomach: ["stomach", "gastric", "stomach bubble"],
+  vein:    ["vein", "venous", "vena cava", "umbilical vein"],
+  fetal_head:       ["cervix", "cervic", "cervical length"],
+  pubic_symphysis:  ["pubic", "symphys", "pelvimetr"],
+  abdomen: ["abdomen", "abdomin", "abdom", "ac_plane"],
+  arm:     ["arm", "upper limb", "humerus"],
+  head:    ["skull", "calvar", "biparietal", "fetal head", "head", "neck"],
+  legs:    ["femur", "femoral", "thigh", "lower limb",
+            "lower extremit", "tibia", "legs", "leg "],
 };
 
 export const CO_OCCURRENCE_GROUPS = [
-  ["brain",["Brain","CSP","LV"],["Brain"],["nt_nasal"]],
-  ["cardiac",["cardiac","thorax"],["cardiac","thorax"],[]],
-  ["nt_nasal",["NT","nasal_bone","nasal_skin","nasal_tip"],["NT"],["brain","crl"]],
-  ["crl",["B","CRL","H","NB"],null,["nt_nasal"]],
-  ["doppler",["artery","liver","stomach","vein"],["artery","liver","stomach","vein"],[]],
-  ["pelvimetry",["fetal_head","pubic_symphysis"],["fetal_head","pubic_symphysis"],[]],
-  ["body_full",["abdomen","arm","head","legs"],null,[]],
-  ["femur",["legs"],null,[]],
+  ["brain",      ["Brain", "CSP", "LV"],                          ["Brain"],                                ["nt_nasal"]],
+  ["cardiac",    ["cardiac", "thorax"],                            ["cardiac", "thorax"],                    []],
+  ["nt_nasal",   ["NT", "nasal_bone", "nasal_skin", "nasal_tip"], ["NT"],                                   ["brain", "crl"]],
+  ["crl",        ["B", "CRL", "H", "NB"],                         null,                                     ["nt_nasal"]],
+  ["doppler",    ["artery", "liver", "stomach", "vein"],           ["artery", "liver", "stomach", "vein"],   []],
+  ["pelvimetry", ["fetal_head", "pubic_symphysis"],                ["fetal_head", "pubic_symphysis"],         []],
+  ["body_full",  ["abdomen", "arm", "head", "legs"],               null,                                     []],
+  ["femur",      ["legs"],                                         null,                                     []],
 ];
 
 export const GROUP_ALIASES = {
-  brain:["bpd","biparietal"," hc ","head circumference","v sign","x sign","trans-thalamic","trans-cerebellar","transventricular","transcerebellar"],
-  crl:["first trimester","dating scan","embryo","sagittal profile"],
-  femur:[" fl ","femur length","fl_plane","fl plane","fl "],
-  body_full:["abdominal circumference"," ac ","ac_plane","ac plane","fetal pose","body habitus","no_plane","no plane"],
-  doppler:["doppler","flow","waveform","pulsatility","resistance index","stomach bubble","hepatic","ductus venosus","umbilical arter","umbilical vein"],
-  nt_nasal:["first trimester screen","aneuploidy","trisomy","down syndrome"],
-  pelvimetry:["pelvimetry","pelvic","birth canal","vaginal delivery"],
-  cardiac:["echocardiogra","outflow tract","aortic arch","aorta"],
+  brain:      ["bpd", "biparietal", " hc ", "head circumference",
+               "v sign", "x sign", "trans-thalamic", "trans-cerebellar",
+               "transventricular", "transcerebellar"],
+  crl:        ["first trimester", "dating scan", "embryo",
+               "sagittal profile"],
+  femur:      [" fl ", "femur length", "fl_plane", "fl plane", "fl "],
+  body_full:  ["abdominal circumference", " ac ", "ac_plane",
+               "ac plane", "fetal pose", "body habitus",
+               "no_plane", "no plane"],
+  doppler:    ["doppler", "flow", "waveform", "pulsatility",
+               "resistance index", "stomach bubble",
+               "hepatic", "ductus venosus", "umbilical arter",
+               "umbilical vein"],
+  nt_nasal:   ["first trimester screen", "aneuploidy", "trisomy",
+               "down syndrome"],
+  pelvimetry: ["pelvimetry", "pelvic", "birth canal",
+               "vaginal delivery"],
+  cardiac:    ["echocardiogra", "outflow tract", "aortic arch", "aorta"],
 };
 
 export const PLANE_TO_GROUP = {
-  "trans-thalamic":"brain",transthalamic:"brain","trans-cerebellar":"brain",transcerebellar:"brain",
-  transventricular:"brain",biparietal:"brain","four chamber":"cardiac","4-chamber":"cardiac",
-  "4 chamber":"cardiac","outflow tract":"cardiac","three vessel":"cardiac",aortic:"cardiac",
-  nuchal:"nt_nasal","longitudinal view of femur":"femur","femur length":"femur",
-  "abdominal circumference":"body_full","ac plane":"body_full","crown-rump":"crl","crown rump":"crl",
-  doppler:"doppler",waveform:"doppler","abdominal organ":"doppler","stomach bubble":"doppler",
-  "cervical length":"pelvimetry",
+  "trans-thalamic": "brain", transthalamic: "brain",
+  "trans-cerebellar": "brain", transcerebellar: "brain",
+  transventricular: "brain", biparietal: "brain",
+  "four chamber": "cardiac", "4-chamber": "cardiac",
+  "4 chamber": "cardiac", "outflow tract": "cardiac",
+  "three vessel": "cardiac", aortic: "cardiac",
+  nuchal: "nt_nasal",
+  "longitudinal view of femur": "femur", "femur length": "femur",
+  "abdominal circumference": "body_full", "ac plane": "body_full",
+  "crown-rump": "crl", "crown rump": "crl",
+  doppler: "doppler", waveform: "doppler",
+  "abdominal organ": "doppler", "stomach bubble": "doppler",
+  "cervical length": "pelvimetry",
 };
 
 export const NEGATIVE_MEASUREMENT_PATTERNS = [
-  "not measurable","not visible","cannot be measured","unable to measure",
-  "not assessed","not obtained","not clearly","inadequate",
+  "not measurable", "not visible", "cannot be measured",
+  "unable to measure", "not assessed", "not obtained",
+  "not clearly", "inadequate",
 ];
 
+// ---------------------------------------------------------------------------
 // Routing group map
+// ---------------------------------------------------------------------------
+
 export const ROUTING_GROUP_MAP = {
-  brain:["Brain, CSP, LV","Brain"],
-  cardiac:["cardiac, thorax","cardiac, thorax"],
-  nt_nasal:["NT, nasal_bone, nasal_skin, nasal_tip","NT"],
-  crl:["B, CRL, H, NB",null],
-  doppler:["artery, liver, stomach, vein","artery, liver, stomach, vein"],
-  pelvimetry:["fetal_head, pubic_symphysis","fetal_head, pubic_symphysis"],
-  body_pose:["abdomen, arm, head, legs",null],
-  femur:["legs",null],
-  crl_kp:["CRL_KP, ScaleBarKpoints",null],
-  nt_kp:["NTKpoints, ScaleBarKpoints",null],
+  brain:      ["Brain, CSP, LV",                          "Brain"],
+  cardiac:    ["cardiac, thorax",                          "cardiac, thorax"],
+  nt_nasal:   ["NT, nasal_bone, nasal_skin, nasal_tip",   "NT"],
+  crl:        ["B, CRL, H, NB",                           null],
+  doppler:    ["artery, liver, stomach, vein",             "artery, liver, stomach, vein"],
+  pelvimetry: ["fetal_head, pubic_symphysis",              "fetal_head, pubic_symphysis"],
+  body_pose:  ["abdomen, arm, head, legs",                 null],
+  femur:      ["legs",                                     null],
+  crl_kp:     ["CRL_KP, ScaleBarKpoints",                  null],
+  nt_kp:      ["NTKpoints, ScaleBarKpoints",               null],
 };
 
-// Color palette
+// ---------------------------------------------------------------------------
+// Color palette for annotation rendering
+// ---------------------------------------------------------------------------
+
 const _RAW_PALETTE = [
-  "#E6194B","#3CB44B","#FFE119","#4363D8","#F58231",
-  "#911EB4","#42D4F4","#F032E6","#BFEF45","#FABED4",
-  "#469990","#DCBEFF","#9A6324","#FFFAC8","#800000",
-  "#AAFFC3","#808000","#FFD8B1","#000075","#A9A9A9",
+  "#E6194B", "#3CB44B", "#FFE119", "#4363D8", "#F58231",
+  "#911EB4", "#42D4F4", "#F032E6", "#BFEF45", "#FABED4",
+  "#469990", "#DCBEFF", "#9A6324", "#FFFAC8", "#800000",
+  "#AAFFC3", "#808000", "#FFD8B1", "#000075", "#A9A9A9",
 ];
 
 const ALL_CLASSES_SORTED = [
-  "AB","B","Brain","C","CRL","CRL_KP","CSP","DP","G","H","LV",
-  "MDS","MLS","MX","NB","NT","NTAPS","NTKpoints","RBP",
-  "ScaleBar","ScaleBarKpoints",
-  "abdomen","arm","artery","cardiac","fetal_head","head","legs","liver",
-  "nasal_bone","nasal_skin","nasal_tip","pubic_symphysis","stomach","thorax","vein",
+  "AB", "B", "Brain", "C", "CRL", "CRL_KP", "CSP", "DP", "G", "H", "LV",
+  "MDS", "MLS", "MX", "NB", "NT", "NTAPS", "NTKpoints", "RBP",
+  "ScaleBar", "ScaleBarKpoints",
+  "abdomen", "arm", "artery", "cardiac",
+  "fetal_head", "head", "legs", "liver",
+  "nasal_bone", "nasal_skin", "nasal_tip",
+  "pubic_symphysis", "stomach", "thorax", "vein",
 ].sort();
 
 export const CLASS_COLORS = {};
@@ -192,7 +259,10 @@ ALL_CLASSES_SORTED.forEach((cls, i) => {
 
 export { _RAW_PALETTE };
 
+// ---------------------------------------------------------------------------
 // Inference settings
+// ---------------------------------------------------------------------------
+
 export const TEMPERATURE = 0.1;
 export const MAX_NEW_TOKENS = 1024;
 export const COORD_SCALE = 1000;

js/model.js

@@ -46,11 +46,17 @@ export async function loadModel(onProgress, device = "webgpu") {
     console.log(`[FADA] Loading model on device=${device}...`);
     activeDevice = device;
 
-    model = await Qwen3_5ForConditionalGeneration.from_pretrained(MODEL_ID, {
+    // Use fp32 for vision_encoder on WASM (no fp16 benefit) or try fp16 on WebGPU with fallback
+    const visionDtype = device === "wasm" ? "fp32" : "fp16";
+    if (device === "wasm") {
+      console.log("[FADA] WASM backend selected, using fp32 for vision encoder");
+    }
+
+    const buildModelConfig = (vDtype) => ({
       revision: MODEL_REVISION,
       dtype: {
         embed_tokens: "q4",
-        vision_encoder: "fp16",
+        vision_encoder: vDtype,
         decoder_model_merged: "q4",
       },
       device: device,
@@ -66,8 +72,30 @@ export async function loadModel(onProgress, device = "webgpu") {
         }
       },
     });
-    console.log("[FADA] Model loaded successfully on", device);
 
+    try {
+      model = await Qwen3_5ForConditionalGeneration.from_pretrained(
+        MODEL_ID,
+        buildModelConfig(visionDtype)
+      );
+    } catch (err) {
+      const isFp16Error =
+        visionDtype === "fp16" &&
+        err?.message?.toLowerCase().includes("does not support fp16");
+
+      if (isFp16Error) {
+        console.warn("[FADA] FP16 not supported on this device, retrying with FP32 vision encoder...");
+        onProgress?.({ status: "loading", file: "model", note: "FP16 not supported, loading vision encoder in FP32 mode..." });
+        model = await Qwen3_5ForConditionalGeneration.from_pretrained(
+          MODEL_ID,
+          buildModelConfig("fp32")
+        );
+      } else {
+        throw err;
+      }
+    }
+
+    console.log("[FADA] Model loaded successfully on", device);
     onProgress?.({ status: "ready" });
   })();