File size: 20,688 Bytes
9620ce1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9cb86c0
 
9620ce1
9cb86c0
 
9620ce1
 
 
 
9cb86c0
9620ce1
 
 
9cb86c0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9620ce1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8f656d
9620ce1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9cb86c0
 
 
 
 
 
 
9620ce1
 
9cb86c0
9620ce1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
(() => {
  "use strict";

  const MAX_README_BYTES = 2_000_000;
  const REPO_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*\/[A-Za-z0-9][A-Za-z0-9._-]*$/;
  const checks = [
    ["license", "License", ["license", "license_name", "license_link"], [], [], "Confirm the license identifier or link that governs reuse."],
    ["language", "Language coverage", ["language", "languages"], ["\\blanguages?\\b"], [], "Document the languages represented and any known coverage limits."],
    ["task", "Task and model type", ["pipeline_tag", "library_name", "tags"], ["\\bmodel description\\b", "\\btask\\b"], [], "Clarify the task, model family, and expected input/output shape."],
    ["base_model", "Base model or provenance", ["base_model"], ["\\bmodel description\\b", "\\bmodel details\\b", "\\bprovenance\\b"], ["\\bbase model\\b", "\\bfine[- ]?tuned from\\b", "\\bderived from\\b"], "Name the base model or explain the model's provenance."],
    ["intended_uses", "Intended uses", [], ["\\bintended uses?\\b", "\\buse cases?\\b", "\\buses?\\b", "\\bhow to use\\b"], ["\\bintended for\\b", "\\bcan be used\\b", "\\buse cases?\\b"], "Describe supported use cases and the users or settings considered."],
    ["out_of_scope", "Out-of-scope uses", [], ["\\bout[- ]of[- ]scope\\b", "\\bmisuse\\b", "\\bnot intended\\b", "\\bprohibited uses?\\b"], ["\\bshould not\\b", "\\bnot intended\\b", "\\bout[- ]of[- ]scope\\b"], "Identify unsupported or inappropriate uses, where relevant."],
    ["limitations", "Limitations, risks, or biases", [], ["\\blimitations?\\b", "\\brisks?\\b", "\\bbias(?:es)?\\b", "\\bethical considerations?\\b"], ["\\blimitations?\\b", "\\brisks?\\b", "\\bbias(?:es|ed)?\\b"], "Describe known limitations, biases, or risks and how they were observed."],
    ["training_data", "Training data", ["datasets", "dataset"], ["\\btraining data\\b", "\\bdatasets?\\b", "\\btraining details\\b"], ["\\btrained on\\b", "\\btraining dataset\\b"], "Name or characterize the training data and include source links when possible."],
    ["evaluation", "Evaluation evidence", ["model-index", "eval_results", "metrics"], ["\\bevaluation\\b", "\\bresults?\\b", "\\bbenchmarks?\\b", "\\bmetrics?\\b"], ["\\bbenchmark\\b", "\\baccuracy\\b", "\\bf1\\b", "\\bbleu\\b", "\\brouge\\b"], "Link evaluation datasets, metrics, values, and their source or procedure."],
    ["reproducibility", "Training and reproducibility details", ["training_args"], ["\\btraining procedure\\b", "\\btraining details\\b", "\\bhyperparameters?\\b", "\\breproducibility\\b"], ["\\blearning rate\\b", "\\bbatch size\\b", "\\btraining steps?\\b"], "Add the training procedure, key parameters, code, or experiment links."],
    ["citation", "Citation or supporting paper", ["citation"], ["\\bcitation\\b", "\\breferences?\\b", "\\bpaper\\b"], [], "Add a citation, paper, technical report, or supporting reference if one exists."]
  ];

  const form = document.querySelector("#review-form");
  const input = document.querySelector("#repo-id");
  const notice = document.querySelector("#notice");
  const results = document.querySelector("#results");
  const summary = document.querySelector("#summary");
  const evidencePanel = document.querySelector("#evidence-panel");
  const worksheetPanel = document.querySelector("#worksheet-panel");
  const discussionPanel = document.querySelector("#discussion-panel");
  const downloadButton = document.querySelector("#download-review");
  const modelOptions = document.querySelector("#model-options");
  const pickerStatus = document.querySelector("#picker-status");
  let latestReview = null;
  let searchTimer = null;
  let searchNonce = 0;

  function cleanRepoId(value) {
    let id = value.trim();
    if (id.startsWith("https://huggingface.co/")) id = id.slice("https://huggingface.co/".length).split("/").slice(0, 2).join("/");
    if (!REPO_ID.test(id)) throw new Error("Select a model from the live results, or enter its exact owner/model-name ID.");
    return id;
  }

  function numberLabel(value) {
    const number = Number(value || 0);
    if (number >= 1_000_000) return `${(number / 1_000_000).toFixed(number >= 10_000_000 ? 0 : 1)}M`;
    if (number >= 1_000) return `${(number / 1_000).toFixed(number >= 10_000 ? 0 : 1)}K`;
    return String(number);
  }

  function hideModelOptions() {
    modelOptions.hidden = true;
    input.setAttribute("aria-expanded", "false");
  }

  function showModelOptions(models, label) {
    modelOptions.replaceChildren();
    pickerStatus.textContent = label;
    if (!models.length) { hideModelOptions(); return; }
    models.forEach((model) => {
      const id = model.id || model.modelId;
      if (!id) return;
      const option = make("button", "model-option");
      option.type = "button";
      option.setAttribute("role", "option");
      option.setAttribute("aria-label", `Select ${id}`);
      const detail = model.pipeline_tag ? `${model.pipeline_tag} · ${numberLabel(model.downloads)} downloads` : `${numberLabel(model.downloads)} downloads`;
      const copy = make("span");
      copy.append(make("strong", "", id), make("span", "", detail));
      option.append(copy);
      if (model.pipeline_tag) option.append(make("span", "task", model.pipeline_tag));
      option.addEventListener("click", () => { input.value = id; modelOptions.replaceChildren(); hideModelOptions(); pickerStatus.textContent = `Selected: ${id}`; input.focus(); });
      modelOptions.append(option);
    });
    if (!modelOptions.childElementCount) { hideModelOptions(); return; }
    modelOptions.hidden = false;
    input.setAttribute("aria-expanded", "true");
  }

  async function searchModels(query = "") {
    const nonce = ++searchNonce;
    pickerStatus.textContent = query ? "Searching public models…" : "Loading live public models…";
    const params = new URLSearchParams({ limit: "8", sort: "downloads", direction: "-1" });
    if (query) params.set("search", query);
    try {
      const response = await fetch(`https://huggingface.co/api/models?${params}`);
      if (!response.ok) throw new Error("unavailable");
      const models = (await response.json()).filter((model) => !model.private && !model.gated && !model.disabled);
      if (nonce !== searchNonce) return;
      showModelOptions(models, query ? "Live public matches" : "Popular public models — start typing to narrow the list");
    } catch (_) {
      if (nonce !== searchNonce) return;
      pickerStatus.textContent = "Live model picker is unavailable right now. You can still enter an exact public model ID.";
      hideModelOptions();
    }
  }

  function meaningful(value) {
    if (value === null || value === undefined || value === false) return false;
    if (typeof value === "string") return Boolean(value.trim());
    if (Array.isArray(value)) return value.length > 0;
    if (typeof value === "object") return Object.keys(value).length > 0;
    return true;
  }

  function compact(value, limit = 320) {
    const text = (typeof value === "object" ? JSON.stringify(value) : String(value)).replace(/\s+/g, " ").trim();
    return text.length > limit ? `${text.slice(0, limit - 1)}…` : text;
  }

  function frontmatter(readme) {
    const lines = readme.split("\n");
    if (lines[0]?.trim() !== "---") return {};
    const end = lines.findIndex((line, index) => index > 0 && line.trim() === "---");
    if (end < 0) return {};
    const metadata = {};
    let current = null;
    for (const line of lines.slice(1, end)) {
      if (!line.trim() || line.trimStart().startsWith("#")) continue;
      const list = line.match(/^\s*-\s+(.+?)\s*$/);
      if (list && current) { if (Array.isArray(metadata[current])) metadata[current].push(list[1].replace(/^['"]|['"]$/g, "")); continue; }
      const key = line.match(/^([A-Za-z0-9_-]+):(?:\s*(.*))?$/);
      if (!key) continue;
      current = key[1];
      const value = (key[2] || "").trim();
      metadata[current] = value ? value.replace(/^['"]|['"]$/g, "") : [];
    }
    return metadata;
  }

  function sections(readme) {
    const lines = readme.split("\n");
    const headings = [];
    let fenced = false;
    lines.forEach((line, index) => {
      if (/^\s*(```|~~~)/.test(line)) { fenced = !fenced; return; }
      if (fenced) return;
      const match = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/);
      if (match) headings.push({ start: index + 1, level: match[1].length, title: match[2].trim() });
    });
    return headings.map((heading, index) => {
      let end = lines.length;
      for (const next of headings.slice(index + 1)) { if (next.level <= heading.level) { end = next.start - 1; break; } }
      return { ...heading, body: lines.slice(heading.start, end).join("\n").trim() };
    });
  }

  function sourceLink(repo, revision, line) {
    const link = `https://huggingface.co/${repo}/blob/${revision}/README.md`;
    return line ? `${link}#L${line}` : link;
  }

  function analyze(repo, api, readme) {
    const revision = api.sha || "main";
    const metadata = { ...frontmatter(readme), ...(api.cardData || {}) };
    if (!metadata.pipeline_tag && api.pipeline_tag) metadata.pipeline_tag = api.pipeline_tag;
    if (!metadata.library_name && api.library_name) metadata.library_name = api.library_name;
    if (!metadata.tags && api.tags) metadata.tags = api.tags;
    const parsedSections = sections(readme);
    const lines = readme.split("\n");
    const findings = checks.map(([key, label, keys, headings, keywords, prompt]) => {
      const evidence = [];
      const foundKeys = keys.filter((item) => meaningful(metadata[item]));
      foundKeys.slice(0, 2).forEach((item) => evidence.push({ type: "Model card metadata", text: `${item}: ${compact(metadata[item])}`, url: sourceLink(repo, revision) }));
      const matchedSections = parsedSections.filter((section) => headings.some((pattern) => new RegExp(pattern, "i").test(section.title)));
      if (matchedSections[0]) {
        const item = matchedSections[0];
        evidence.push({ type: "Model card section", text: compact(`${item.title}: ${item.body}`, 420), url: sourceLink(repo, revision, item.start) });
      }
      if (!evidence.length && keywords.length) {
        const foundLine = lines.findIndex((line) => line.trim() && !line.trim().startsWith("#") && keywords.some((pattern) => new RegExp(pattern, "i").test(line)));
        if (foundLine >= 0) evidence.push({ type: "Model card text", text: compact(lines[foundLine], 420), url: sourceLink(repo, revision, foundLine + 1) });
      }
      let status = evidence.length ? "documented" : "not_found";
      let text = evidence.length ? "Relevant information was found in the public card." : "Relevant information was not found by this review.";
      if (key === "evaluation" && matchedSections.length && !foundKeys.length) { status = "review"; text = "Evaluation text was found, but structured evaluation metadata was not found."; }
      if (key === "task" && foundKeys.length === 1 && foundKeys[0] === "tags") { status = "review"; text = "Relevant tags were found; the narrative description may still need review."; }
      return { key, label, status, text, evidence, prompt };
    });
    return { repo, revision, sourceUrl: sourceLink(repo, revision), findings, analyzedAt: new Date().toISOString() };
  }

  function counts(review) { return review.findings.reduce((all, finding) => ({ ...all, [finding.status]: (all[finding.status] || 0) + 1 }), { documented: 0, review: 0, not_found: 0 }); }
  function statusLabel(status) { return status === "not_found" ? "Not found" : status[0].toUpperCase() + status.slice(1); }
  function make(tag, className, text) { const element = document.createElement(tag); if (className) element.className = className; if (text !== undefined) element.textContent = text; return element; }

  function renderSummary(review) {
    const count = counts(review);
    summary.replaceChildren();
    const header = make("section", "result-header");
    const heading = make("div");
    heading.append(make("p", "eyebrow", "PUBLIC MODEL CARD REVIEW"), make("h2", "", review.repo));
    const link = make("a", "", "Open reviewed revision ↗"); link.href = review.sourceUrl; link.target = "_blank"; link.rel = "noopener"; heading.append(link);
    const strip = make("div", "count-strip");
    [[count.documented, "documented"], [count.review, "review"], [count.not_found, "not found"]].forEach(([number, label]) => { const entry = make("div"); entry.append(make("strong", "", number), make("span", "", label)); strip.append(entry); });
    header.append(heading, strip);
    const note = make("p", "scope-note", "A narrow result: “Not found” means the review did not locate relevant information. It is not a compliance grade or a claim of maintainer error.");
    const grid = make("section", "finding-grid");
    review.findings.forEach((finding) => { const card = make("article", `finding-card ${finding.status.replace("_", "-")}`); const top = make("div", "finding-topline"); top.append(make("h3", "", finding.label), make("span", "status-pill", statusLabel(finding.status))); card.append(top, make("p", "", finding.text), make("small", "", finding.evidence.length ? `${finding.evidence.length} source item${finding.evidence.length === 1 ? "" : "s"}` : "Maintainer input needed")); grid.append(card); });
    summary.append(header, note, grid);
  }

  function renderEvidence(review) {
    evidencePanel.replaceChildren();
    review.findings.forEach((finding) => { const item = make("article", "evidence-item"); item.append(make("h3", "", `${finding.label}${statusLabel(finding.status)}`), make("p", "", finding.text)); if (finding.evidence.length) { finding.evidence.forEach((evidence) => { const link = make("a", "", evidence.type); link.href = evidence.url; link.target = "_blank"; link.rel = "noopener"; const excerpt = make("div", "excerpt", evidence.text); item.append(link, excerpt); }); } else { item.append(make("p", "", `Question for the maintainer: ${finding.prompt}`)); } evidencePanel.append(item); });
  }

  function worksheet(review) {
    const unresolved = review.findings.filter((finding) => finding.status !== "documented");
    const lines = [`# Suggested model card additions for ${review.repo}`, "", "> Review worksheet—not a drop-in patch. Replace every TODO with maintainer-confirmed information.", ""];
    if (!unresolved.length) return lines.concat(["The selected checks found relevant documentation for every reviewed area.", "A human should still verify accuracy, currency, and applicability."]).join("\n");
    unresolved.forEach((finding) => lines.push(`## ${finding.label}`, "", `<!-- TODO: ${finding.prompt} -->`, ""));
    return lines.concat(["## Review notes", "", "- Confirm that every proposed addition is accurate with the maintainer.", "- Link primary datasets, evaluations, papers, and code where available.", "- Remove sections that are not applicable instead of filling them speculatively."]).join("\n");
  }

  function discussion(review) {
    const unresolved = review.findings.filter((finding) => finding.status !== "documented");
    const labels = unresolved.slice(0, 4).map((finding) => finding.label.toLowerCase()).join(", ");
    const observation = unresolved.length ? `While reading the public card, I could not confidently locate documentation for ${labels}${unresolved.length > 4 ? `, and ${unresolved.length - 4} other area(s)` : ""}. These may be in another source or may not apply to this model.` : "The review found relevant documentation for each checked area. I would still appreciate confirmation that the information is current.";
    return ["## Possible documentation contribution", "", `Hi—thank you for sharing \`${review.repo}\`.`, "", observation, "", "Would a small documentation contribution be useful? I would be glad to prepare one, but I do not want to infer training, evaluation, licensing, or intended-use details that only the maintainers can confirm.", "", `Card reviewed: ${review.sourceUrl}`, "", "_Drafted for human review. This message has not been posted._"].join("\n");
  }

  function fullReview(review, worksheetText, discussionText) {
    const count = counts(review);
    const lines = [`# One Good Commit review: ${review.repo}`, "", `- Source: ${review.sourceUrl}`, `- Revision: \`${review.revision}\``, `- Analyzed: ${review.analyzedAt}`, `- Documented: ${count.documented}`, `- Review: ${count.review}`, `- Not found: ${count.not_found}`, "", "> “Not found” means this deterministic review did not locate relevant information. It is not a claim of noncompliance, inaccuracy, or maintainer error.", "", "## Evidence", ""];
    review.findings.forEach((finding) => { lines.push(`### ${finding.label}${statusLabel(finding.status)}`, "", finding.text, ""); if (finding.evidence.length) finding.evidence.forEach((evidence) => lines.push(`- **${evidence.type}** ([source](${evidence.url}))`, "", `  > ${evidence.text}`, "")); else lines.push(`- Maintainer question: ${finding.prompt}`, ""); });
    return lines.concat(["---", "", worksheetText, "", "---", "", discussionText]).join("\n");
  }

  function showError(message) { notice.textContent = message; notice.hidden = false; results.hidden = true; }
  function clearNotice() { notice.hidden = true; notice.textContent = ""; }

  async function review(value) {
    let repo;
    try { repo = cleanRepoId(value); } catch (error) { showError(error.message); return; }
    clearNotice(); results.hidden = true;
    const button = form.querySelector("button"); button.disabled = true; button.textContent = "Reading public card…";
    try {
      const encoded = repo.split("/").map(encodeURIComponent).join("/");
      const apiResponse = await fetch(`https://huggingface.co/api/models/${encoded}`);
      if ([401, 403, 404].includes(apiResponse.status)) throw new Error("That model ID is not available as a public Hub repository. Check the ID or choose a public, ungated model.");
      if (!apiResponse.ok) throw new Error("The Hub API could not be reached. Try again shortly.");
      const api = await apiResponse.json();
      if (api.private || api.gated) throw new Error("That model is private or gated. This version reviews public cards only.");
      const revision = api.sha || "main";
      const cardResponse = await fetch(`https://huggingface.co/${encoded}/resolve/${encodeURIComponent(revision)}/README.md`);
      if (!cardResponse.ok) throw new Error("The repository does not expose a README.md on its selected revision.");
      const readme = await cardResponse.text();
      if (readme.length > MAX_README_BYTES) throw new Error("The model card is larger than the 2 MB review limit.");
      latestReview = analyze(repo, api, readme);
      const worksheetText = worksheet(latestReview); const discussionText = discussion(latestReview);
      renderSummary(latestReview); renderEvidence(latestReview);
      worksheetPanel.replaceChildren(make("pre", "worksheet", worksheetText));
      discussionPanel.replaceChildren(make("pre", "discussion", discussionText));
      latestReview.download = fullReview(latestReview, worksheetText, discussionText);
      results.hidden = false;
    } catch (error) { showError(error.message || "An unexpected error occurred. No contribution was created or posted."); }
    finally { button.disabled = false; button.textContent = "Review card"; }
  }

  form.addEventListener("submit", (event) => { event.preventDefault(); review(input.value); });
  input.addEventListener("input", () => {
    window.clearTimeout(searchTimer);
    const query = input.value.trim();
    searchTimer = window.setTimeout(() => searchModels(query), 250);
  });
  input.addEventListener("focus", () => { if (modelOptions.childElementCount) { modelOptions.hidden = false; input.setAttribute("aria-expanded", "true"); } });
  document.addEventListener("click", (event) => { if (!event.target.closest("#review-form") && !event.target.closest("#model-options")) hideModelOptions(); });
  document.querySelectorAll("[role=tab]").forEach((tab) => tab.addEventListener("click", () => { document.querySelectorAll("[role=tab]").forEach((item) => item.setAttribute("aria-selected", String(item === tab))); document.querySelectorAll("[role=tabpanel]").forEach((panel) => { panel.hidden = panel.id !== tab.dataset.tab; }); }));
  downloadButton.addEventListener("click", () => { if (!latestReview?.download) return; const blob = new Blob([latestReview.download], { type: "text/markdown;charset=utf-8" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = `${latestReview.repo.replace("/", "--")}-review.md`; link.click(); URL.revokeObjectURL(link.href); });
  searchModels();
})();