Upload routes/tutorial-refinity.js with huggingface_hub

routes/tutorial-refinity.js

@@ -259,8 +259,29 @@ router.post('/compare-comments-with-corrections', async (req, res) => {
     
     // Sort by position
     annotationBasedItems.sort((a, b) => a.start - b.start || a.end - b.end);
-    
-    const mergedAnnotations = annotationBasedItems;
+
+    // De-dupe identical annotations (same span/category/correction/flags).
+    // Without this, duplicated DB rows (or re-annotated identical spans) can cause the
+    // generator to literally duplicate the span text in the document body.
+    const mergedAnnotations = (() => {
+      const out = [];
+      const seen = new Set();
+      for (const it of annotationBasedItems) {
+        const key = [
+          it.start ?? '',
+          it.end ?? '',
+          String(it.category || ''),
+          String(it.replacementText || ''),
+          it.isDeleted ? '1' : '0',
+          it.isInsertion ? '1' : '0',
+          it.isMove ? '1' : '0',
+        ].join('|');
+        if (seen.has(key)) continue;
+        seen.add(key);
+        out.push(it);
+      }
+      return out;
+    })();
 
     const lines = [];
     let pos = 0;