rafmacalaba commited on
Commit
c9986d8
·
1 Parent(s): b72d00c

feat: per-annotator validation support for multi-user overlap

Browse files

- Validate API stores validations in per-annotator 'validations' array
- Each annotator's validation is independent; re-validating updates in-place
- AnnotationPanel shows only current user's validation status
- ProgressBar counts only current user's verified mentions
- Next-page prompt checks only current user's unverified mentions
- Tag edits (dataset_tag) still go at top level

app/api/validate/route.js CHANGED
@@ -64,15 +64,43 @@ export async function PUT(request) {
64
  return NextResponse.json({ error: `Dataset index ${dataset_index} out of range` }, { status: 400 });
65
  }
66
 
67
- // Merge updates into the dataset entry.
68
- // Fields like dataset_tag go at top level.
69
- // Fields like human_validated, human_verdict, annotator, validated_at go at top level.
70
- // Only dataset_name sub-fields (text, confidence, etc.) go inside dataset_name.
71
  const currentEntry = pagesData[pageIdx].datasets[dataset_index];
72
- pagesData[pageIdx].datasets[dataset_index] = {
73
- ...currentEntry,
74
- ...updates,
75
- };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
 
77
  // Save back
78
  if (isHFSpace()) {
 
64
  return NextResponse.json({ error: `Dataset index ${dataset_index} out of range` }, { status: 400 });
65
  }
66
 
67
+ // Per-annotator validation: store in a `validations` array.
68
+ // Each annotator gets their own entry; re-validating updates in-place.
 
 
69
  const currentEntry = pagesData[pageIdx].datasets[dataset_index];
70
+ const annotator = updates.annotator || 'unknown';
71
+
72
+ // Separate validation fields from other updates (like dataset_tag edits)
73
+ const validationFields = ['human_validated', 'human_verdict', 'human_notes', 'annotator', 'validated_at'];
74
+ const isValidation = validationFields.some(f => f in updates);
75
+
76
+ if (isValidation) {
77
+ const validations = currentEntry.validations || [];
78
+ const existingIdx = validations.findIndex(v => v.annotator === annotator);
79
+ const validationEntry = {
80
+ human_validated: updates.human_validated,
81
+ human_verdict: updates.human_verdict,
82
+ human_notes: updates.human_notes || null,
83
+ annotator,
84
+ validated_at: updates.validated_at || new Date().toISOString(),
85
+ };
86
+
87
+ if (existingIdx >= 0) {
88
+ validations[existingIdx] = validationEntry;
89
+ } else {
90
+ validations.push(validationEntry);
91
+ }
92
+
93
+ pagesData[pageIdx].datasets[dataset_index] = {
94
+ ...currentEntry,
95
+ validations,
96
+ };
97
+ } else {
98
+ // Non-validation updates (e.g. dataset_tag edit) go at top level
99
+ pagesData[pageIdx].datasets[dataset_index] = {
100
+ ...currentEntry,
101
+ ...updates,
102
+ };
103
+ }
104
 
105
  // Save back
106
  if (isHFSpace()) {
app/components/AnnotationPanel.js CHANGED
@@ -85,10 +85,13 @@ export default function AnnotationPanel({
85
  const tag = ds.dataset_tag || 'named';
86
  const style = TAG_STYLES[tag] || TAG_STYLES.named;
87
  const isHuman = !!ds.annotator;
88
- const isValidated = ds.dataset_name?.human_validated;
89
- const humanVerdict = ds.dataset_name?.human_verdict;
90
- const humanNotes = ds.dataset_name?.human_notes;
91
- const validatedBy = ds.dataset_name?.annotator;
 
 
 
92
  const judgeVerdict = ds.dataset_name?.judge_verdict;
93
  const judgeTag = ds.dataset_name?.judge_tag;
94
  const isValidating = validatingIdx === i;
@@ -153,11 +156,11 @@ export default function AnnotationPanel({
153
  </span>
154
  )}
155
 
156
- {/* Existing validation status */}
157
  {isValidated && (
158
  <div className={`validation-status ${humanVerdict ? 'correct' : 'wrong'}`}>
159
  {humanVerdict ? '✅ Validated correct' : '❌ Marked incorrect'}
160
- <span className="validation-by"> by {validatedBy}</span>
161
  {humanNotes && (
162
  <p className="validation-notes">Note: {humanNotes}</p>
163
  )}
 
85
  const tag = ds.dataset_tag || 'named';
86
  const style = TAG_STYLES[tag] || TAG_STYLES.named;
87
  const isHuman = !!ds.annotator;
88
+
89
+ // Per-annotator validation: look up current user's entry
90
+ const myValidation = (ds.validations || []).find(v => v.annotator === annotatorName);
91
+ const isValidated = myValidation?.human_validated === true;
92
+ const humanVerdict = myValidation?.human_verdict;
93
+ const humanNotes = myValidation?.human_notes;
94
+
95
  const judgeVerdict = ds.dataset_name?.judge_verdict;
96
  const judgeTag = ds.dataset_name?.judge_tag;
97
  const isValidating = validatingIdx === i;
 
156
  </span>
157
  )}
158
 
159
+ {/* Existing validation status (your own) */}
160
  {isValidated && (
161
  <div className={`validation-status ${humanVerdict ? 'correct' : 'wrong'}`}>
162
  {humanVerdict ? '✅ Validated correct' : '❌ Marked incorrect'}
163
+ <span className="validation-by"> by you</span>
164
  {humanNotes && (
165
  <p className="validation-notes">Note: {humanNotes}</p>
166
  )}
app/components/ProgressBar.js CHANGED
@@ -6,6 +6,7 @@ export default function ProgressBar({
6
  currentDoc,
7
  pageIdx,
8
  currentPageDatasets,
 
9
  }) {
10
  if (!documents || documents.length === 0) return null;
11
 
@@ -17,9 +18,12 @@ export default function ProgressBar({
17
  const totalPages = currentDoc?.annotatable_pages?.length ?? 0;
18
  const currentPage = totalPages > 0 ? pageIdx + 1 : 0;
19
 
20
- // 3. Mentions progress: verified vs total on current page
21
  const totalMentions = currentPageDatasets?.length ?? 0;
22
- const verifiedMentions = currentPageDatasets?.filter(ds => ds.human_validated === true).length ?? 0;
 
 
 
23
 
24
  return (
25
  <div className="progress-container">
 
6
  currentDoc,
7
  pageIdx,
8
  currentPageDatasets,
9
+ annotatorName,
10
  }) {
11
  if (!documents || documents.length === 0) return null;
12
 
 
18
  const totalPages = currentDoc?.annotatable_pages?.length ?? 0;
19
  const currentPage = totalPages > 0 ? pageIdx + 1 : 0;
20
 
21
+ // 3. Mentions progress: verified by CURRENT USER vs total on current page
22
  const totalMentions = currentPageDatasets?.length ?? 0;
23
+ const verifiedMentions = currentPageDatasets?.filter(ds => {
24
+ const myValidation = (ds.validations || []).find(v => v.annotator === annotatorName);
25
+ return myValidation?.human_validated === true;
26
+ }).length ?? 0;
27
 
28
  return (
29
  <div className="progress-container">
app/page.js CHANGED
@@ -176,10 +176,13 @@ export default function Home() {
176
  };
177
 
178
  const handleNextPage = () => {
179
- const unverified = currentPageDatasets.filter(ds => ds.human_validated !== true).length;
 
 
 
180
  if (unverified > 0) {
181
  const proceed = confirm(
182
- `⚠️ There are ${unverified} unverified data mention${unverified > 1 ? 's' : ''} on this page.\n\nDo you want to proceed to the next page?`
183
  );
184
  if (!proceed) return;
185
  }
@@ -435,6 +438,7 @@ export default function Home() {
435
  currentDoc={currentDoc}
436
  pageIdx={pageIdx}
437
  currentPageDatasets={currentPageDatasets}
 
438
  />
439
  <div className="top-bar-user">
440
  {annotatorName ? (
 
176
  };
177
 
178
  const handleNextPage = () => {
179
+ const unverified = currentPageDatasets.filter(ds => {
180
+ const myValidation = (ds.validations || []).find(v => v.annotator === annotatorName);
181
+ return myValidation?.human_validated !== true;
182
+ }).length;
183
  if (unverified > 0) {
184
  const proceed = confirm(
185
+ `⚠️ You have ${unverified} unverified data mention${unverified > 1 ? 's' : ''} on this page.\n\nDo you want to proceed to the next page?`
186
  );
187
  if (!proceed) return;
188
  }
 
438
  currentDoc={currentDoc}
439
  pageIdx={pageIdx}
440
  currentPageDatasets={currentPageDatasets}
441
+ annotatorName={annotatorName}
442
  />
443
  <div className="top-bar-user">
444
  {annotatorName ? (