new v2

frontend/src/App.tsx

@@ -5,21 +5,21 @@ import {
   StopCircle, RefreshCw, CheckCircle2, PenTool, ClipboardCheck, Info, Clock, FileText,
   Tag, Home, Cpu, FlaskConical, Target, Trash2, ArrowUpRight, CheckSquare, Square,
   Layers, Activity, Zap, BrainCircuit, Network, Archive, Plus, Edit3, RotateCcw,
-  Bot, Trophy, HelpCircle, Settings, Calculator
+  Bot, Trophy, HelpCircle, Settings, Calculator, ChevronDown, ChevronUp
 } from 'lucide-react';
 
 function App() {
   const[activeTab, setActiveTab] = useState('home');
   const [logs, setLogs] = useState<string>('System Ready.\n');
-  const [isProcessing, setIsProcessing] = useState(false);
+  const[isProcessing, setIsProcessing] = useState(false);
   const logContainerRef = useRef<HTMLDivElement>(null);
   
   // Processing Config State
-  const[modelProvider, setModelProvider] = useState('nrp');
+  const [modelProvider, setModelProvider] = useState('nrp');
   const [apiKey, setApiKey] = useState('');
-  const [baseUrl, setBaseUrl] = useState('https://ellm.nrp-nautilus.io/v1'); // NRP Default
-  const [modelName, setModelName] = useState('qwen3'); // NRP Default
-  const [projectId, setProjectId] = useState('');
+  const[baseUrl, setBaseUrl] = useState('https://ellm.nrp-nautilus.io/v1'); // NRP Default
+  const[modelName, setModelName] = useState('qwen3'); // Default
+  const[projectId, setProjectId] = useState('');
   const [location, setLocation] = useState('us-central1');
   const [includeComments, setIncludeComments] = useState(false);
   const[reasoningMethod, setReasoningMethod] = useState('cot');
@@ -35,6 +35,7 @@ function App() {
   // Data States
   const [queueList, setQueueList] = useState<any[]>([]);
   const[selectedQueueItems, setSelectedQueueItems] = useState<Set<string>>(new Set());
+  const[expandedQueueItems, setExpandedQueueItems] = useState<Set<string>>(new Set());
   const[lastQueueIndex, setLastQueueIndex] = useState<number | null>(null);
   
   const[singleLinkInput, setSingleLinkInput] = useState(''); 
@@ -43,10 +44,10 @@ function App() {
   const[profilePosts, setProfilePosts] = useState<any[]>([]);
   const [communityDatasets, setCommunityDatasets] = useState<any[]>([]);
   const [communityAnalysis, setCommunityAnalysis] = useState<any>(null);
-  const [integrityBoard, setIntegrityBoard] = useState<any[]>([]);
+  const[integrityBoard, setIntegrityBoard] = useState<any[]>([]);
   
-  const [datasetList, setDatasetList] = useState<any[]>([]);
-  const[selectedItems, setSelectedItems] = useState<Set<string>>(new Set());
+  const[datasetList, setDatasetList] = useState<any[]>([]);
+  const [selectedItems, setSelectedItems] = useState<Set<string>>(new Set());
   const [lastDatasetIndex, setLastDatasetIndex] = useState<number | null>(null);
 
   const [benchmarks, setBenchmarks] = useState<any>(null);
@@ -79,12 +80,20 @@ function App() {
   const [agentConfig, setAgentConfig] = useState({ use_search: true, use_code: false });
 
   // Resampling configuration
-  const [resampleCount, setResampleCount] = useState<number>(1);
+  const[resampleCount, setResampleCount] = useState<number>(1);
   
   // Drag Selection references
   const isDraggingQueueRef = useRef(false);
   const isDraggingDatasetRef = useRef(false);
 
+  // Quick Demo State
+  const[demoLink, setDemoLink] = useState('');
+  const [demoLogs, setDemoLogs] = useState('');
+  const [demoIsProcessing, setDemoIsProcessing] = useState(false);
+  const[demoResult, setDemoResult] = useState<any>(null);
+  const[showDemoConfig, setShowDemoConfig] = useState(false);
+  const demoLogContainerRef = useRef<HTMLDivElement>(null);
+
   useEffect(() => {
     const handleMouseUp = () => {
         isDraggingQueueRef.current = false;
@@ -123,12 +132,16 @@ function App() {
     
     setSelectedItems(new Set());
     setLastDatasetIndex(null);
-  }, [activeTab, refreshTrigger]);
+  },[activeTab, refreshTrigger]);
 
   useEffect(() => {
     if (logContainerRef.current) logContainerRef.current.scrollTop = logContainerRef.current.scrollHeight;
   }, [logs]);
 
+  useEffect(() => {
+    if (demoLogContainerRef.current) demoLogContainerRef.current.scrollTop = demoLogContainerRef.current.scrollHeight;
+  },[demoLogs]);
+
   useEffect(() => {
       if (activeTab === 'agent' && agentMessages.length === 0) {
           callAgent(agentMethod, { 
@@ -274,6 +287,14 @@ function App() {
       setLastQueueIndex(index);
   };
 
+  const toggleQueueExpand = (e: React.MouseEvent, link: string) => {
+      e.stopPropagation();
+      const newSet = new Set(expandedQueueItems);
+      if (newSet.has(link)) newSet.delete(link);
+      else newSet.add(link);
+      setExpandedQueueItems(newSet);
+  };
+
   const promoteSelected = async () => {
       if (selectedItems.size === 0) return alert("No items selected.");
       if (!confirm(`Promote ${selectedItems.size} items to Ground Truth?`)) return;
@@ -437,6 +458,20 @@ function App() {
       } catch(e) { alert("Error adding link"); }
   };
 
+  const addSingleLinkDirect = async (link: string) => {
+      if(!link) return;
+      try {
+          const res = await fetch('/queue/add', {
+              method: 'POST', headers: {'Content-Type': 'application/json'},
+              body: JSON.stringify({ link: link })
+          });
+          const d = await res.json();
+          if(d.status === 'success') {
+              setRefreshTrigger(p => p+1);
+          } else { alert(d.message); }
+      } catch(e) { alert("Error adding link"); }
+  };
+
   const clearProcessed = async () => {
       if(!confirm("Remove all 'Processed' items from the queue?")) return;
       try {
@@ -527,6 +562,84 @@ function App() {
       setRefreshTrigger(p => p+1);
   };
 
+  const runDemo = async () => {
+      if (!demoLink) return;
+      setDemoIsProcessing(true);
+      setDemoLogs('[SYSTEM] Preparing pipeline for single execution...\n');
+      setDemoResult(null);
+
+      try {
+          // 1. Ensure the link is added and forcefully requeued so it is 'Pending'
+          await fetch('/queue/add', {
+              method: 'POST', headers: {'Content-Type': 'application/json'},
+              body: JSON.stringify({ link: demoLink })
+          });
+          await fetch('/queue/requeue', {
+              method: 'POST', headers: {'Content-Type': 'application/json'},
+              body: JSON.stringify({ links: [demoLink] })
+          });
+
+          // 2. Setup FormData exactly like normal queue
+          const fd = new FormData();
+          const activeProvider = modelProvider === 'gcloud' ? 'vertex' : modelProvider;
+          fd.append('model_selection', activeProvider); 
+          fd.append('gemini_api_key', apiKey);
+          fd.append('gemini_model_name', modelName); 
+          fd.append('vertex_project_id', projectId);
+          fd.append('vertex_location', location); 
+          fd.append('vertex_model_name', modelName); 
+          fd.append('vertex_api_key', apiKey);
+          fd.append('nrp_api_key', apiKey);
+          fd.append('nrp_model_name', modelName);
+          fd.append('nrp_base_url', baseUrl);
+          fd.append('include_comments', includeComments.toString()); 
+          fd.append('reasoning_method', reasoningMethod);
+          fd.append('prompt_template', promptTemplate); 
+          fd.append('custom_query', customQuery);
+          fd.append('max_reprompts', maxRetries.toString());
+
+          setDemoLogs(prev => prev + '[SYSTEM] Sending analysis payload to model server...\n');
+          
+          // 3. Read Stream
+          const runRes = await fetch('/queue/run', { method: 'POST', body: fd });
+          const reader = runRes.body!.pipeThrough(new TextDecoderStream()).getReader();
+
+          while (true) {
+              const { value, done } = await reader.read();
+              if (done) break;
+              if (value.includes('event: close')) break;
+              const clean = value.replace(/data: /g, '').trim();
+              if (clean) setDemoLogs(prev => prev + clean + '\n');
+          }
+
+          // 4. Look up result from dataset
+          setDemoLogs(prev => prev + '\n[SYSTEM] Fetching structured object result...\n');
+          const dsRes = await fetch('/dataset/list');
+          const dsList = await dsRes.json();
+
+          const normalize = (l: string) => l.split('?')[0].replace('https://', '').replace('http://', '').replace('www.', '').replace(/\/$/, '');
+          const targetLink = normalize(demoLink);
+
+          // Sort descending to ensure we get the latest processed run
+          dsList.sort((a: any, b: any) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
+
+          const match = dsList.find((d: any) => normalize(d.link || '') === targetLink && d.source !== 'Manual');
+          
+          if (match) {
+              setDemoResult(match);
+              setDemoLogs(prev => prev + '[SYSTEM] Result rendered successfully.\n');
+          } else {
+              setDemoLogs(prev => prev + '[SYSTEM] Error: Could not find parsed result. Processing might have failed.\n');
+          }
+
+      } catch (err: any) {
+          setDemoLogs(prev => prev + `\n[ERROR] ${err.message}\n`);
+      } finally {
+          setDemoIsProcessing(false);
+          setRefreshTrigger(p => p+1); 
+      }
+  }
+
   const callAgent = async (method: string, payloadParams: any) => {
       return fetch(agentEndpoint, {
           method: 'POST',
@@ -602,7 +715,18 @@ function App() {
 
   return (
     <div className="flex h-screen w-full bg-[#09090b] text-slate-200 font-sans overflow-hidden">
-      
+      <datalist id="modelSuggestions">
+          <option value="gemini-1.5-pro-preview-0409" />
+          <option value="gemini-2.0-flash-exp" />
+          <option value="qwen3" />
+          <option value="gpt-oss" />
+          <option value="kimi" />
+          <option value="glm-4.7" />
+          <option value="minimax-m2" />
+          <option value="glm-v" />
+          <option value="gemma3" />
+      </datalist>
+
       {/* SIDEBAR */}
       <div className="w-[280px] flex flex-col border-r border-slate-800/60 bg-[#0c0c0e]">
         <div className="h-16 flex items-center px-6 border-b border-slate-800/60">
@@ -640,6 +764,196 @@ function App() {
             {/* HOME TAB */}
             {activeTab === 'home' && (
                 <div className="h-full overflow-y-auto space-y-8 max-w-5xl pr-2">
+                    
+                    {/* QUICK DEMO SECTION */}
+                    <div className="bg-slate-900/50 border border-slate-800 rounded-xl p-6 shadow-sm">
+                        <div className="flex justify-between items-start mb-4">
+                            <div>
+                                <h2 className="text-xl font-bold text-white flex items-center gap-2">
+                                    <Zap className="w-5 h-5 text-amber-400" /> Try LiarMP4 - Quick Demo
+                                </h2>
+                                <p className="text-sm text-slate-400 mt-1">
+                                    Test the Multimodal Factuality Pipeline on a single video URL.
+                                </p>
+                            </div>
+                            <button
+                                onClick={() => setShowDemoConfig(!showDemoConfig)}
+                                className="text-xs font-bold text-slate-500 hover:text-slate-300 flex items-center gap-1 bg-slate-950 px-3 py-1.5 rounded border border-slate-800"
+                            >
+                                <Settings className="w-4 h-4" />
+                                {showDemoConfig ? "Hide Config" : "Show Config"}
+                            </button>
+                        </div>
+
+                        {showDemoConfig && (
+                            <div className="bg-slate-950 p-4 rounded-lg border border-slate-800 mb-4 grid grid-cols-2 gap-6 animate-in fade-in zoom-in-95 duration-200">
+                                <div className="space-y-3">
+                                    <label className="text-[10px] text-slate-500 uppercase font-bold block border-b border-slate-800 pb-1">LLM Provider Config</label>
+                                    <select value={modelProvider} onChange={e => setModelProvider(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white">
+                                        <option value="vertex">Vertex AI (Enterprise)</option>
+                                        <option value="gemini">Gemini API (Public)</option>
+                                        <option value="gcloud">Google Cloud (Project + API Key)</option>
+                                        <option value="nrp">NRP (Nautilus Envoy Gateway)</option>
+                                    </select>
+                                    
+                                    {modelProvider === 'nrp' && (
+                                        <>
+                                            <input value={baseUrl} onChange={e => setBaseUrl(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white" placeholder="Base URL"/>
+                                            <input type="password" value={apiKey} onChange={e => setApiKey(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white" placeholder="API Token"/>
+                                        </>
+                                    )}
+                                    {modelProvider === 'gemini' && (
+                                         <input type="password" value={apiKey} onChange={e => setApiKey(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white" placeholder="API Key"/>
+                                    )}
+                                    {(modelProvider === 'vertex' || modelProvider === 'gcloud') && (
+                                         <>
+                                            <input value={projectId} onChange={e => setProjectId(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white" placeholder="Project ID"/>
+                                            <input value={location} onChange={e => setLocation(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white" placeholder="Location"/>
+                                            {modelProvider === 'gcloud' && <input type="password" value={apiKey} onChange={e => setApiKey(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white" placeholder="API Key"/>}
+                                         </>
+                                    )}
+                                    <input list="modelSuggestions" value={modelName} onChange={e => setModelName(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white" placeholder="Model Name (e.g. gemini-1.5-pro)"/>
+                                </div>
+                                <div className="space-y-3">
+                                    <label className="text-[10px] text-slate-500 uppercase font-bold block border-b border-slate-800 pb-1">Inference Strategy</label>
+                                    <select value={reasoningMethod} onChange={e => setReasoningMethod(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white">
+                                        <option value="cot">Standard Chain of Thought</option>
+                                        <option value="fcot">Fractal Chain of Thought</option>
+                                    </select>
+                                    <select value={promptTemplate} onChange={e => setPromptTemplate(e.target.value)} className="w-full bg-slate-900 border border-slate-700 rounded p-2 text-xs text-white">
+                                        {availablePrompts.length > 0 ? availablePrompts.map(p => (
+                                            <option key={p.id} value={p.id}>{p.name}</option>
+                                        )) : <option value="standard">Standard</option>}
+                                    </select>
+                                </div>
+                            </div>
+                        )}
+
+                        <div className="flex gap-2 mb-4">
+                            <input
+                                type="text"
+                                value={demoLink}
+                                onChange={(e) => setDemoLink(e.target.value)}
+                                placeholder="Enter X/Twitter Video URL here..."
+                                className="flex-1 bg-slate-950 border border-slate-700 rounded-lg p-3 text-sm text-white focus:outline-none focus:border-indigo-500"
+                                disabled={demoIsProcessing}
+                            />
+                            <button
+                                onClick={runDemo}
+                                disabled={demoIsProcessing || !demoLink}
+                                className="bg-indigo-600 hover:bg-indigo-500 disabled:bg-slate-700 text-white px-6 py-3 rounded-lg font-bold flex items-center gap-2 transition"
+                            >
+                                {demoIsProcessing ? <RefreshCw className="w-4 h-4 animate-spin" /> : <Play className="w-4 h-4 fill-white" />}
+                                {demoIsProcessing ? "Processing..." : "Analyze"}
+                            </button>
+                        </div>
+
+                        {(demoIsProcessing || demoLogs) && !demoResult && (
+                            <div className="bg-black border border-slate-800 rounded-lg p-4 font-mono text-[10px] text-emerald-500 h-64 overflow-y-auto whitespace-pre-wrap animate-in fade-in duration-300" ref={demoLogContainerRef}>
+                                {demoLogs || "Initializing pipeline..."}
+                            </div>
+                        )}
+
+                        {demoResult && (
+                            <div className="mt-6 border-t border-slate-800 pt-6 animate-in fade-in slide-in-from-bottom-4 duration-500">
+                                <div className="flex justify-between items-start mb-6">
+                                    <div>
+                                        <h3 className="text-xl font-bold text-white flex items-center gap-2">
+                                            <ShieldCheck className="w-6 h-6 text-emerald-400" />
+                                            <span className="text-indigo-400 uppercase tracking-wider text-[16px] bg-indigo-500/10 px-2 py-0.5 rounded border border-indigo-500/20">[{demoResult.config_model || 'AI'}]
+                                            </span>
+                                            Analysis Complete
+                                        </h3>
+                                        <div className="text-xs text-slate-400 mt-2 font-mono">
+                                            ID: {demoResult.id} | Prompt: {demoResult.config_prompt} | Reasoning: {demoResult.config_reasoning}
+                                        </div>
+                                    </div>
+                                    <div className="bg-slate-950 border border-slate-800 px-6 py-2 rounded-lg text-center">
+                                        <div className="text-[10px] uppercase text-slate-500 font-bold mb-1">Final Score</div>
+                                        <div className={`text-4xl font-bold font-mono ${demoResult.final_veracity_score < 50 ? 'text-red-400' : 'text-emerald-400'}`}>
+                                            {demoResult.final_veracity_score}
+                                        </div>
+                                    </div>
+                                </div>
+
+                                <div className="grid grid-cols-3 gap-6 mb-6">
+                                    <div className="col-span-2 space-y-4">
+                                        <div className="bg-slate-950 p-4 rounded-lg border border-slate-800">
+                                            <div className="text-xs font-bold text-indigo-400 uppercase mb-2">Video Context</div>
+                                            <div className="text-sm text-slate-300 italic leading-relaxed">"{demoResult.caption || 'No specific context found'}"</div>
+                                        </div>
+                                        
+                                        <div className="bg-slate-950 p-4 rounded-lg border border-slate-800">
+                                            <div className="text-xs font-bold text-amber-400 uppercase mb-2">AI Reasoning</div>
+                                            <div className="text-sm text-slate-300 whitespace-pre-wrap leading-relaxed">{demoResult.reasoning || 'No reasoning provided'}</div>
+                                        </div>
+
+                                        {demoResult.tags && (
+                                            <div className="flex flex-wrap gap-2">
+                                                {demoResult.tags.split(',').map((t: string) => (
+                                                    <span key={t} className="px-2 py-1 bg-slate-800 border border-slate-700 rounded text-xs text-slate-400 font-mono">{t.trim()}</span>
+                                                ))}
+                                            </div>
+                                        )}
+                                    </div>
+
+                                    <div className="space-y-4">
+                                        <div className="bg-slate-950 p-4 rounded-lg border border-slate-800">
+                                            <div className="text-xs font-bold text-sky-400 uppercase mb-3">Veracity Vectors</div>
+                                            <div className="space-y-2">
+                                                {[
+                                                    { label: 'Visual Integrity', score: parseFloat(demoResult.visual_score) || 0 },
+                                                    { label: 'Audio Integrity', score: parseFloat(demoResult.audio_score) || 0 },
+                                                    { label: 'Source Credibility', score: parseFloat(demoResult.source_score) || 0 },
+                                                    { label: 'Logical Consistency', score: parseFloat(demoResult.logic_score) || 0 },
+                                                    { label: 'Emotional Manip.', score: parseFloat(demoResult.emotion_score) || 0 },
+                                                ].map((v) => (
+                                                    <div key={v.label} className="flex justify-between items-center text-xs">
+                                                        <span className="text-slate-400 w-32 truncate">{v.label}</span>
+                                                        <div className="flex items-center gap-2 flex-1 ml-2">
+                                                            <div className="flex-1 h-1.5 bg-slate-800 rounded-full overflow-hidden">
+                                                                <div className={`h-full ${v.score < 5 ? 'bg-red-400' : v.score < 8 ? 'bg-amber-400' : 'bg-emerald-400'}`} style={{ width: `${(v.score / 10) * 100}%` }} />
+                                                            </div>
+                                                            <span className="font-mono text-slate-300 w-6 text-right">{v.score}</span>
+                                                        </div>
+                                                    </div>
+                                                ))}
+                                            </div>
+                                        </div>
+
+                                        <div className="bg-slate-950 p-4 rounded-lg border border-slate-800">
+                                            <div className="text-xs font-bold text-pink-400 uppercase mb-3">Modality Alignment</div>
+                                            <div className="space-y-2">
+                                                {[
+                                                    { label: 'Video ↔ Audio', score: parseFloat(demoResult.align_video_audio) || 0 },
+                                                    { label: 'Video ↔ Caption', score: parseFloat(demoResult.align_video_caption) || 0 },
+                                                    { label: 'Audio ↔ Caption', score: parseFloat(demoResult.align_audio_caption) || 0 },
+                                                ].map((v) => (
+                                                    <div key={v.label} className="flex justify-between items-center text-xs">
+                                                        <span className="text-slate-400 w-32 truncate">{v.label}</span>
+                                                        <div className="flex items-center gap-2 flex-1 ml-2">
+                                                            <div className="flex-1 h-1.5 bg-slate-800 rounded-full overflow-hidden">
+                                                                <div className={`h-full ${v.score < 5 ? 'bg-red-400' : v.score < 8 ? 'bg-amber-400' : 'bg-emerald-400'}`} style={{ width: `${(v.score / 10) * 100}%` }} />
+                                                            </div>
+                                                            <span className="font-mono text-slate-300 w-6 text-right">{v.score}</span>
+                                                        </div>
+                                                    </div>
+                                                ))}
+                                            </div>
+                                        </div>
+                                    </div>
+                                </div>
+
+                                <div className="flex justify-end">
+                                    <button onClick={() => {setDemoResult(null); setDemoLogs(''); setDemoLink('');}} className="text-xs text-slate-500 hover:text-white flex items-center gap-1 border border-slate-800 px-3 py-1.5 rounded bg-slate-950">
+                                        <RotateCcw className="w-3 h-3"/> Reset Demo
+                                    </button>
+                                </div>
+                            </div>
+                        )}
+                    </div>
+
+                    {/* EXISTING HOME TAB CONTENT */}
                     <div className="grid grid-cols-3 gap-6">
                         <div className="col-span-2 bg-slate-900/50 border border-slate-800 rounded-xl p-6">
                             <h2 className="text-xl font-bold text-white mb-4">Philosophy & Methodology</h2>
@@ -799,35 +1113,29 @@ function App() {
                                         <div className="space-y-1">
                                             <input type="password" value={apiKey} onChange={e => setApiKey(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white placeholder-slate-600" placeholder="NRP API Token"/>
                                         </div>
-                                        <div className="space-y-1">
-                                            <select value={modelName} onChange={e => setModelName(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white">
-                                                <option value="qwen3">qwen3 (Multimodal)</option>
-                                                <option value="gpt-oss">gpt-oss</option>
-                                                <option value="kimi">kimi</option>
-                                                <option value="glm-4.7">glm-4.7</option>
-                                                <option value="minimax-m2">minimax-m2</option>
-                                                <option value="glm-v">glm-v (Multimodal)</option>
-                                                <option value="gemma3">gemma3 (Multimodal)</option>
-                                            </select>
-                                        </div>
                                     </>
                                 )}
-                                
-                                {modelProvider !== 'nrp' && (
-                                    <div className="space-y-1">
-                                        <input value={modelName} onChange={e => setModelName(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white placeholder-slate-600" placeholder="Model Name"/>
-                                    </div>
-                                )}
 
-                                <select value={reasoningMethod} onChange={e => setReasoningMethod(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white">
-                                    <option value="cot">Standard Chain of Thought</option>
-                                    <option value="fcot">Fractal Chain of Thought</option>
-                                </select>
-                                <select value={promptTemplate} onChange={e => setPromptTemplate(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white">
-                                    {availablePrompts.length > 0 ? availablePrompts.map(p => (
-                                        <option key={p.id} value={p.id}>{p.name}</option>
-                                    )) : <option value="standard">Standard</option>}
-                                </select>
+                                <div className="space-y-1">
+                                    <label className="text-[10px] text-slate-500">Model Name</label>
+                                    <input list="modelSuggestions" value={modelName} onChange={e => setModelName(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white placeholder-slate-600" placeholder="Model Name"/>
+                                </div>
+
+                                <div className="space-y-1 mt-2">
+                                    <label className="text-[10px] text-slate-500">Reasoning Method</label>
+                                    <select value={reasoningMethod} onChange={e => setReasoningMethod(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white">
+                                        <option value="cot">Standard Chain of Thought</option>
+                                        <option value="fcot">Fractal Chain of Thought</option>
+                                    </select>
+                                </div>
+                                <div className="space-y-1">
+                                    <label className="text-[10px] text-slate-500">Prompt Persona</label>
+                                    <select value={promptTemplate} onChange={e => setPromptTemplate(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white">
+                                        {availablePrompts.length > 0 ? availablePrompts.map(p => (
+                                            <option key={p.id} value={p.id}>{p.name}</option>
+                                        )) : <option value="standard">Standard</option>}
+                                    </select>
+                                </div>
                             </div>
                         </div>
 
@@ -1004,27 +1312,12 @@ function App() {
                             </>
                         )}
 
-                        {modelProvider === 'nrp' ? (
-                            <div className="space-y-1">
-                                <label className="text-[10px] text-slate-500">Model Name</label>
-                                <select value={modelName} onChange={e => setModelName(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white">
-                                    <option value="qwen3">qwen3 (Multimodal)</option>
-                                    <option value="gpt-oss">gpt-oss</option>
-                                    <option value="kimi">kimi</option>
-                                    <option value="glm-4.7">glm-4.7</option>
-                                    <option value="minimax-m2">minimax-m2</option>
-                                    <option value="glm-v">glm-v (Multimodal)</option>
-                                    <option value="gemma3">gemma3 (Multimodal)</option>
-                                </select>
-                            </div>
-                        ) : (
-                            <div className="space-y-1">
-                                <label className="text-[10px] text-slate-500">Model Name</label>
-                                <input value={modelName} onChange={e => setModelName(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white"/>
-                            </div>
-                        )}
-
                         <div className="space-y-1">
+                            <label className="text-[10px] text-slate-500">Model Name</label>
+                            <input list="modelSuggestions" value={modelName} onChange={e => setModelName(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white" placeholder="Model Name"/>
+                        </div>
+
+                        <div className="space-y-1 mt-2">
                             <label className="text-[10px] text-slate-500">Reasoning Method</label>
                             <select value={reasoningMethod} onChange={e => setReasoningMethod(e.target.value)} className="w-full bg-slate-950 border border-slate-700 rounded p-2 text-xs text-white">
                                 <option value="cot">Standard Chain of Thought</option>
@@ -1076,34 +1369,68 @@ function App() {
                                 </thead>
                                 <tbody>
                                     {queueList.map((q, i, arr) => (
-                                        <tr 
-                                            key={i} 
-                                            className={`border-t border-slate-800/50 hover:bg-white/5 ${selectedQueueItems.has(q.link) ? 'bg-indigo-900/20' : ''}`}
-                                            onMouseDown={(e) => {
-                                                isDraggingQueueRef.current = true;
-                                                toggleQueueSelection(e, q.link, i, arr);
-                                            }}
-                                            onMouseEnter={() => {
-                                                if (isDraggingQueueRef.current && !selectedQueueItems.has(q.link)) {
-                                                    setSelectedQueueItems(prev => new Set(prev).add(q.link));
-                                                    setLastQueueIndex(i);
-                                                }
-                                            }}
-                                        >
-                                            <td className="p-3 cursor-pointer">
-                                                {selectedQueueItems.has(q.link) ? <CheckSquare className="w-4 h-4 text-indigo-400"/> : <Square className="w-4 h-4 text-slate-600"/>}
-                                            </td>
-                                            <td className="p-3 font-bold text-[10px]">{q.task_type === 'Verify' ? <span className="text-amber-400">VERIFY</span> : <span className="text-slate-500">INGEST</span>}</td>
-                                            <td className="p-3 text-sky-500 font-mono break-all">{q.link}</td>
-                                            <td className="p-3">
-                                                {q.status === 'Processed' ? 
-                                                    <span className="text-emerald-500 flex items-center gap-1"><CheckCircle2 className="w-3 h-3"/> Done</span> : 
-                                                    q.status === 'Error' ? 
-                                                    <span className="text-red-500 flex items-center gap-1"><AlertCircle className="w-3 h-3"/> Error</span> :
-                                                    <span className="text-amber-500">Pending</span>
-                                                }
-                                            </td>
-                                        </tr>
+                                        <React.Fragment key={i}>
+                                            <tr 
+                                                className={`border-t border-slate-800/50 hover:bg-white/5 ${selectedQueueItems.has(q.link) ? 'bg-indigo-900/20' : ''}`}
+                                                onMouseDown={(e) => {
+                                                    isDraggingQueueRef.current = true;
+                                                    toggleQueueSelection(e, q.link, i, arr);
+                                                }}
+                                                onMouseEnter={() => {
+                                                    if (isDraggingQueueRef.current && !selectedQueueItems.has(q.link)) {
+                                                        setSelectedQueueItems(prev => new Set(prev).add(q.link));
+                                                        setLastQueueIndex(i);
+                                                    }
+                                                }}
+                                            >
+                                                <td className="p-3 cursor-pointer">
+                                                    {selectedQueueItems.has(q.link) ? <CheckSquare className="w-4 h-4 text-indigo-400"/> : <Square className="w-4 h-4 text-slate-600"/>}
+                                                </td>
+                                                <td className="p-3 font-bold text-[10px]">{q.task_type === 'Verify' ? <span className="text-amber-400">VERIFY</span> : <span className="text-slate-500">INGEST</span>}</td>
+                                                <td className="p-3">
+                                                    <div className="flex items-center gap-2">
+                                                        <span className="text-sky-500 font-mono break-all">{q.link}</span>
+                                                        {q.comments && q.comments.length > 0 && (
+                                                            <button 
+                                                                onClick={(e) => toggleQueueExpand(e, q.link)} 
+                                                                className="px-2 py-1 bg-slate-800 rounded text-[10px] text-slate-300 hover:bg-slate-700 flex items-center gap-1 z-10"
+                                                            >
+                                                                {expandedQueueItems.has(q.link) ? <ChevronUp className="w-3 h-3"/> : <ChevronDown className="w-3 h-3"/>}
+                                                                {q.comments.length} Comments
+                                                            </button>
+                                                        )}
+                                                    </div>
+                                                </td>
+                                                <td className="p-3">
+                                                    {q.status === 'Processed' ? 
+                                                        <span className="text-emerald-500 flex items-center gap-1"><CheckCircle2 className="w-3 h-3"/> Done</span> : 
+                                                        q.status === 'Error' ? 
+                                                        <span className="text-red-500 flex items-center gap-1"><AlertCircle className="w-3 h-3"/> Error</span> :
+                                                        <span className="text-amber-500">Pending</span>
+                                                    }
+                                                </td>
+                                            </tr>
+                                            {expandedQueueItems.has(q.link) && q.comments && q.comments.length > 0 && (
+                                                <tr className="bg-slate-900/40">
+                                                    <td colSpan={4} className="p-0">
+                                                        <div className="px-12 py-3 border-l-2 border-indigo-500 ml-4 space-y-2">
+                                                            {q.comments.map((c: any, ci: number) => (
+                                                                <div key={ci} className="flex flex-col gap-1 p-2 bg-slate-900 border border-slate-800 rounded">
+                                                                    <div className="flex justify-between items-center">
+                                                                        <div className="text-[10px] font-bold text-indigo-400">{c.author}</div>
+                                                                        <div className="flex gap-2">
+                                                                            {c.link && <a href={c.link} target="_blank" rel="noreferrer" className="text-[10px] text-sky-400 hover:underline">View Post</a>}
+                                                                            {c.link && <button onClick={() => addSingleLinkDirect(c.link)} className="text-[10px] bg-slate-800 hover:bg-slate-700 text-slate-300 px-2 py-0.5 rounded flex items-center gap-1"><Plus className="w-3 h-3"/> Queue Comment</button>}
+                                                                        </div>
+                                                                    </div>
+                                                                    <div className="text-xs text-slate-400 line-clamp-2">{c.text}</div>
+                                                                </div>
+                                                            ))}
+                                                        </div>
+                                                    </td>
+                                                </tr>
+                                            )}
+                                        </React.Fragment>
                                     ))}
                                 </tbody>
                             </table>
@@ -1406,7 +1733,7 @@ function App() {
                                                      <span className="capitalize text-slate-300 font-bold">{k.replace('v', 'Video-').replace('a', 'Audio-').replace('c', 'Caption')}</span>
                                                      <span className="text-emerald-400 font-mono font-bold">{(manualScores as any)[k]}/10</span>
                                                  </div>
-                                                 <input type="range" min="1" max="10" value={(manualScores as any)[k]} onChange={e => setManualScores({...manualScores, [k]: parseInt(e.target.value)})} className="w-full accent-emerald-500"/>
+                                                 <input type="range" min="1" max="10" value={(manualScores as any)[k]} onChange={e => setManualScores({...manualScores,[k]: parseInt(e.target.value)})} className="w-full accent-emerald-500"/>
                                              </div>
                                          ))}
                                      </div>
@@ -1486,7 +1813,7 @@ function App() {
                 </div>
             )}
 
-            {/* COMMUNITY AND ANALYTICS TABS */}
+            {/* COMMUNITY AND ANALYTICS TABS (UNCHANGED) */}
             {activeTab === 'community' && (
                 <div className="flex h-full gap-6">
                     <div className="w-1/3 bg-slate-900/50 border border-slate-800 rounded-xl overflow-auto">

main.go

@@ -33,7 +33,17 @@ func main() {
 			strings.HasPrefix(r.URL.Path, "/download-dataset") ||
 			strings.HasPrefix(r.URL.Path, "/extension") ||
 			strings.HasPrefix(r.URL.Path, "/manage") ||
-			strings.HasPrefix(r.URL.Path, "/queue") {
+			strings.HasPrefix(r.URL.Path, "/queue") ||
+			strings.HasPrefix(r.URL.Path, "/a2a") ||
+			strings.HasPrefix(r.URL.Path, "/health") ||
+			strings.HasPrefix(r.URL.Path, "/benchmarks") ||
+			strings.HasPrefix(r.URL.Path, "/config") ||
+			strings.HasPrefix(r.URL.Path, "/tags") ||
+			strings.HasPrefix(r.URL.Path, "/manual") ||
+			strings.HasPrefix(r.URL.Path, "/profiles") ||
+			strings.HasPrefix(r.URL.Path, "/community") ||
+			strings.HasPrefix(r.URL.Path, "/dataset") ||
+			strings.HasPrefix(r.URL.Path, "/analyze") {
 
 			log.Printf("Proxying %s to Python Backend...", r.URL.Path)
 			proxy.ServeHTTP(w, r)
@@ -57,4 +67,4 @@ func main() {
 	if err := http.ListenAndServe(":"+port, nil); err != nil {
 		log.Fatal(err)
 	}
-}
+}

src/app.py

@@ -1,5 +1,11 @@
 import os
 import sys
+
+# --- FIX: Ensure 'src' is in sys.path so sibling imports work ---
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.append(current_dir)
+
 import asyncio
 import subprocess
 from pathlib import Path
@@ -10,9 +16,6 @@ import datetime
 import json
 import hashlib
 import re
-import glob
-import shutil
-import time
 from fastapi import FastAPI, Request, Form, UploadFile, File, Body, HTTPException
 from fastapi.responses import HTMLResponse, StreamingResponse, PlainTextResponse, Response, FileResponse, JSONResponse
 from fastapi.templating import Jinja2Templates
@@ -22,29 +25,17 @@ import yt_dlp
 import inference_logic
 import factuality_logic
 import transcription
-from factuality_logic import parse_vtt
-from toon_parser import parse_veracity_toon
+import user_analysis_logic 
+import agent_logic 
+import common_utils
 
-try:
-    import mlcroissant as mlc
-    CROISSANT_AVAILABLE = True
-except ImportError:
-    try:
-        import croissant as mlc
-        CROISSANT_AVAILABLE = True
-    except ImportError:
-        mlc = None
-        CROISSANT_AVAILABLE = False
-
-# Configure Logging with High Verbosity
-logging.basicConfig(
-    level=logging.INFO, 
-    format="%(asctime)s - %(levelname)s - %(message)s",
-    handlers=[logging.StreamHandler(sys.stdout)]
-)
-logger = logging.getLogger("vChat")
+from toon_parser import parse_veracity_toon
+from labeling_logic import PROMPT_VARIANTS, LABELING_PROMPT_TEMPLATE, FCOT_MACRO_PROMPT
+import benchmarking
 
-LITE_MODE = os.getenv("LITE_MODE", "false").lower() == "true"
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+LITE_MODE = os.getenv("LITE_MODE", "true").lower() == "true"
 
 app = FastAPI()
 
@@ -56,469 +47,718 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-# HF Spaces specific path
+# --- CRITICAL: Mount A2A Agent Application FIRST ---
+agent_mount_status = "pending"
+try:
+    logger.info("Attempting to build A2A Agent App...")
+    a2a_agent_app = agent_logic.create_a2a_app()
+    if a2a_agent_app:
+        app.mount("/a2a", a2a_agent_app)
+        agent_mount_status = "success"
+        logger.info("✅ A2A Agent App successfully mounted at /a2a")
+    else:
+        logger.warning("⚠️ Agent factory returned None. Mounting internal fallback.")
+        from fastapi import FastAPI as InnerFastAPI
+        fallback = InnerFastAPI()
+        @fallback.post("/")
+        @fallback.post("/jsonrpc")
+        async def fallback_endpoint(request: Request):
+            return {"jsonrpc": "2.0", "result": {"text": "Fallback Agent (Factory returned None)", "data": {"status": "fallback"}}}
+        app.mount("/a2a", fallback)
+        agent_mount_status = "fallback_none"
+except Exception as e:
+    logger.critical(f"❌ Failed to mount A2A Agent: {e}", exc_info=True)
+    from fastapi import FastAPI as InnerFastAPI
+    fallback = InnerFastAPI()
+    @fallback.post("/")
+    @fallback.post("/jsonrpc")
+    async def fallback_endpoint(request: Request):
+        return {"jsonrpc": "2.0", "result": {"text": f"Emergency Agent (Mount Error: {str(e)})", "data": {"status": "error"}}}
+    app.mount("/a2a", fallback)
+    agent_mount_status = f"error_{str(e)}"
+
+# --- Static Files & Frontend ---
 STATIC_DIR = "/app/static"
 if not os.path.isdir(STATIC_DIR):
-    # Fallback if running locally
-    STATIC_DIR = "static" 
-    os.makedirs(STATIC_DIR, exist_ok=True)
-
+    if os.path.isdir("/usr/share/vchat/static"):
+        STATIC_DIR = "/usr/share/vchat/static"
+    elif os.path.isdir("frontend/dist"):
+        STATIC_DIR = "frontend/dist"
+    else:
+        STATIC_DIR = "static"
+        os.makedirs(STATIC_DIR, exist_ok=True)
+    
 app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
-templates = Jinja2Templates(directory=STATIC_DIR)
 
-# Ensure data directories exist (HF Spaces writable locations)
-os.makedirs("data/videos", exist_ok=True)
-os.makedirs("data", exist_ok=True)
-os.makedirs("data/labels", exist_ok=True)
-os.makedirs("data/prompts", exist_ok=True) 
-os.makedirs("data/responses", exist_ok=True)
-os.makedirs("metadata", exist_ok=True)
+# --- FIX: Explicitly mount assets for Vite support ---
+assets_path = os.path.join(STATIC_DIR, "assets")
+if os.path.exists(assets_path):
+    app.mount("/assets", StaticFiles(directory=assets_path), name="assets")
 
-STOP_QUEUE_SIGNAL = False
+if not os.path.isdir("data/videos"):
+    os.makedirs("data/videos", exist_ok=True)
+app.mount("/videos", StaticFiles(directory="data/videos"), name="videos")
 
-PROMPT_VARIANTS = {
-    "standard": {"description": "Standard Analysis", "instruction": "You are an AI Factuality Assessment Agent operating under the 'Ali Arsanjani Factuality Factors' framework. Cross-reference claims and assess structural validity."},
-    "skeptic": {"description": "High Skepticism", "instruction": "You are a highly skeptical investigator. Question all assumptions, identify the weakest links in the narrative, and aggressively penalize logical fallacies."}
-}
+templates = Jinja2Templates(directory=STATIC_DIR)
 
-@app.on_event("startup")
-async def startup_event():
-    logger.info("Application starting up...")
-    try:
-        transcription.load_model()
-    except Exception as e:
-        logger.warning(f"Could not load Whisper model: {e}")
+# Ensure all data directories exist
+for d in["data", "data/videos", "data/labels", "data/prompts", "data/responses", "metadata", "data/profiles", "data/comments", "data/mnl_labeled", "models/sandbox_autogluon"]:
+    os.makedirs(d, exist_ok=True)
 
-    if not LITE_MODE:
-        try:
-            inference_logic.load_models()
-        except Exception as e:
-            logger.fatal(f"Could not load local inference models. Error: {e}", exc_info=True)
-    else:
-        logger.info("Running in LITE mode (API Only).")
+try:
+    csv.field_size_limit(sys.maxsize)
+except OverflowError:
+    csv.field_size_limit(2147483647)
 
-@app.get("/", response_class=HTMLResponse)
-async def read_root(request: Request):
-    custom_model_available = False
-    if not LITE_MODE:
-        custom_model_available = inference_logic.peft_model is not None
-    if not (Path(STATIC_DIR) / "index.html").exists():
-        return HTMLResponse(content="Frontend not found.", status_code=404)
-    return templates.TemplateResponse("index.html", {
-        "request": request,
-        "custom_model_available": custom_model_available,
-        "lite_mode": LITE_MODE
-    })
+STOP_QUEUE_SIGNAL = False
 
-@app.get("/config/prompts")
-async def list_prompts():
-    return[{"id": k, "name": v['description']} for k, v in PROMPT_VARIANTS.items()]
+# --- CONSTANTS ---
+QUEUE_COLUMNS =["link", "ingest_timestamp", "status", "task_type"]
 
-@app.get("/model-architecture", response_class=PlainTextResponse)
-async def get_model_architecture():
-    if LITE_MODE: return "Running in LITE mode."
-    if inference_logic.base_model: return str(inference_logic.base_model)
-    return "Model not loaded."
+GROUND_TRUTH_FIELDS =[
+    "id", "link", "timestamp", "caption", 
+    "visual_integrity_score", "audio_integrity_score", "source_credibility_score", 
+    "logical_consistency_score", "emotional_manipulation_score",
+    "video_audio_score", "video_caption_score", "audio_caption_score",
+    "final_veracity_score", "final_reasoning",
+    "stats_likes", "stats_shares", "stats_comments", "stats_platform",
+    "tags", "classification", "source"
+]
 
-@app.get("/download-dataset")
-async def download_dataset():
-    file_path = Path("data/dataset.csv")
-    if file_path.exists():
-        return FileResponse(path=file_path, filename="dataset.csv", media_type='text/csv')
-    return Response("Dataset not found.", status_code=404)
+DATASET_COLUMNS =[
+    "id", "link", "timestamp", "caption", 
+    "final_veracity_score", "visual_score", "audio_score", "source_score", "logic_score", "emotion_score", 
+    "align_video_audio", "align_video_caption", "align_audio_caption",
+    "classification", "reasoning", "tags", "raw_toon",
+    "config_type", "config_model", "config_prompt", "config_reasoning", "config_params"
+]
 
-progress_message = ""
-def progress_hook(d):
-    global progress_message
-    if d['status'] == 'downloading':
-        progress_message = f"Downloading: {d.get('_percent_str', 'N/A')} at {d.get('_speed_str', 'N/A')}\r"
-    elif d['status'] == 'finished':
-        progress_message = f"\nDownload finished. Preparing video assets...\n"
-
-def get_cookies_path():
-    """Look for cookies file in known locations for better yt-dlp support."""
-    candidates =["cookies.txt", "data/cookies.txt", "/app/cookies.txt"]
-    for c in candidates:
-        if os.path.exists(c):
-            return os.path.abspath(c)
-    return None
-
-async def run_subprocess_async(command: list[str]):
-    cmd_str = ' '.join(command)
-    logger.info(f"[Subprocess] Running: {cmd_str}")
-    process = await asyncio.create_subprocess_exec(*command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = await process.communicate()
-    
-    if process.returncode != 0:
-        err_msg = stderr.decode()
-        logger.error(f"[Subprocess] Failed ({process.returncode}): {err_msg}")
-        raise RuntimeError(f"Command failed: {err_msg}")
-    logger.info(f"[Subprocess] Success.")
-    return stdout.decode()
-
-def extract_tweet_id(url: str) -> str | None:
-    match = re.search(r"(?:twitter|x)\.com/[^/]+/status/(\d+)", url)
-    if match: return match.group(1)
-    return None
-
-def normalize_link(link: str) -> str:
-    if not link: return ""
-    return link.split('?')[0].strip().rstrip('/').replace('http://', '').replace('https://', '').replace('www.', '')
-
-def check_if_processed(link: str) -> bool:
-    target_id = extract_tweet_id(link)
-    link_clean = normalize_link(link)
-    
+def ensure_csv_schema(file_path: Path, fieldnames: list):
+    if not file_path.exists(): return
+    try:
+        rows =[]
+        with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
+            start_pos = f.tell()
+            line = f.readline()
+            if not line: return
+            existing_header =[h.strip() for h in line.split(',')]
+            missing =[col for col in fieldnames if col not in existing_header]
+            if not missing: return
+            f.seek(start_pos)
+            dict_reader = csv.DictReader(f)
+            rows = list(dict_reader)
+
+        with open(file_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
+            writer.writeheader()
+            for row in rows: writer.writerow(row)
+    except Exception as e: logger.error(f"Schema migration error: {e}")
+
+def get_processed_indices():
+    processed_ids = set()
+    processed_links = set()
     for filename in["data/dataset.csv", "data/manual_dataset.csv"]:
         path = Path(filename)
-        if not path.exists(): continue
-        try:
-            with open(path, 'r', encoding='utf-8', errors='ignore') as f:
-                sample = f.read(2048)
-                f.seek(0)
-                try: has_header = csv.Sniffer().has_header(sample)
-                except: has_header = True
-
-                if has_header:
-                    reader = csv.DictReader(f)
-                    for row in reader:
-                        row_link = normalize_link(row.get('link', ''))
-                        if row_link == link_clean: return True
-                        row_id = row.get('id', '')
-                        if target_id and row_id == target_id: return True
-                else:
-                    reader = csv.reader(f)
-                    for row in reader:
-                        if not row: continue
-                        if link_clean in [normalize_link(r) for r in row]: return True
-                        if target_id and target_id in row: return True
-        except Exception:
-            continue
-    return False
-
-def update_queue_status_in_file(link: str, new_status: str):
+        for row in common_utils.robust_read_csv(path):
+            if row.get('id'): processed_ids.add(row.get('id'))
+            if row.get('link'): processed_links.add(common_utils.normalize_link(row.get('link')))
+    return processed_ids, processed_links
+
+def check_if_processed(link: str, processed_ids=None, processed_links=None) -> bool:
+    target_id = common_utils.extract_tweet_id(link)
+    link_clean = common_utils.normalize_link(link)
+    if processed_ids is None or processed_links is None:
+        p_ids, p_links = get_processed_indices()
+    else: p_ids, p_links = processed_ids, processed_links
+    return (target_id and target_id in p_ids) or (link_clean and link_clean in p_links)
+
+def update_queue_status(link: str, status: str, task_type: str = None):
     q_path = Path("data/batch_queue.csv")
     if not q_path.exists(): return
     rows =[]
+    updated = False
+    norm_target = common_utils.normalize_link(link)
     with open(q_path, 'r', encoding='utf-8') as f:
-        reader = csv.reader(f)
-        try: header = next(reader)
-        except StopIteration: return
+        reader = csv.DictReader(f)
+        fieldnames = list(reader.fieldnames) if reader.fieldnames else list(QUEUE_COLUMNS)
+        for f_name in QUEUE_COLUMNS:
+            if f_name not in fieldnames: fieldnames.append(f_name)
+            
         for row in reader:
-            if row and normalize_link(row[0]) == normalize_link(link):
-                if len(row) > 2: row[2] = new_status
-                else: row.extend([new_status, "Ingest"])
+            if "task_type" not in row or not row["task_type"]: row["task_type"] = "Ingest"
+            if common_utils.normalize_link(row.get("link", "")) == norm_target:
+                if task_type is None or row["task_type"] == task_type:
+                    row["status"] = status
+                    updated = True
             rows.append(row)
-    with open(q_path, 'w', newline='', encoding='utf-8') as f:
+            
+    if updated:
+        with open(q_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
+            writer.writeheader()
+            writer.writerows(rows)
+
+def log_queue_error(link: str, error_msg: str, task_type: str = None):
+    p = Path("data/queue_errors.csv")
+    with open(p, 'a', newline='', encoding='utf-8') as f:
         writer = csv.writer(f)
-        writer.writerow(header)
-        writer.writerows(rows)
-
-async def prepare_video_assets_async(url: str) -> dict:
-    global progress_message
-    loop = asyncio.get_event_loop()
-    is_local = not (url.startswith("http://") or url.startswith("https://"))
-    video_id = "unknown"
-    transcript_path = None
-    
-    logger.info(f"Preparing assets for URL: {url}")
+        if not p.exists() or p.stat().st_size == 0: writer.writerow(["link", "timestamp", "error"])
+        writer.writerow([link, datetime.datetime.now().isoformat(), error_msg])
+    update_queue_status(link, "Error", task_type)
 
-    if is_local:
-        original_path = Path(url)
-        if not original_path.exists(): raise FileNotFoundError(f"File not found: {url}")
-        video_id = hashlib.md5(str(url).encode('utf-8')).hexdigest()[:16]
-        metadata = {"id": video_id, "link": url, "caption": original_path.stem}
-    else:
-        tweet_id = extract_tweet_id(url)
-        video_id = tweet_id if tweet_id else hashlib.md5(url.encode('utf-8')).hexdigest()[:16]
-        sanitized_check = Path(f"data/videos/{video_id}_fixed.mp4")
-        
-        cookies_path = get_cookies_path()
-        ydl_opts = {
-            'format': 'best[ext=mp4]/best', 
-            'outtmpl': 'data/videos/%(id)s.%(ext)s',
-            'progress_hooks': [progress_hook], 
-            'quiet': False, 
-            'no_warnings': False,
-            'noplaylist': True, 
-            'no_overwrites': True,
-            'writesubtitles': True, 
-            'writeautomaticsub': True, 
-            'subtitleslangs': ['en'],
-            'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        }
-        
-        if cookies_path:
-            ydl_opts['cookiefile'] = cookies_path
-            logger.info(f"Using cookies from {cookies_path}")
-
-        if sanitized_check.exists():
-            logger.info(f"Video {video_id} already cached at {sanitized_check}")
-            original_path = sanitized_check
-            metadata = {"id": video_id, "link": url, "caption": "Cached Video"}
-        else:
-            try:
-                logger.info(f"Starting yt-dlp download for {video_id}...")
-                with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-                    info = await loop.run_in_executor(None, lambda: ydl.extract_info(url, download=True))
-                    original_path = Path(ydl.prepare_filename(info))
-                    metadata = {
-                        "id": info.get("id", video_id), "link": info.get("webpage_url", url),
-                        "caption": info.get("description", info.get("title", "N/A")).encode('ascii', 'ignore').decode('ascii').strip()[:500],
-                        "postdatetime": info.get("upload_date", "N/A")
-                    }
-                    video_id = info.get("id", video_id)
-                logger.info("yt-dlp download successful.")
-            except yt_dlp.utils.DownloadError as e:
-                logger.error(f"yt-dlp download error: {e}")
-                if "No video could be found" in str(e):
-                    raise ValueError(f"No video content found at {url}")
-                raise RuntimeError(f"Download failed: {str(e)}")
-            except Exception as e:
-                logger.error(f"Unexpected yt-dlp error: {e}")
-                raise RuntimeError(f"Download failed: {str(e)}")
-
-        transcript_path = next(Path("data/videos").glob(f"{video_id}*.en.vtt"), None)
-        if not transcript_path: transcript_path = next(Path("data/videos").glob(f"{video_id}*.vtt"), None)
-
-    sanitized_path = Path(f"data/videos/{video_id}_fixed.mp4")
-    
-    # --- FFmpeg Sanitization Logic with Robust Fallback ---
-    if not sanitized_path.exists() and original_path.exists():
-        logger.info(f"Sanitizing video {video_id} (Original: {original_path})...")
-        ffmpeg_bin = shutil.which('ffmpeg')
-        if not ffmpeg_bin: raise RuntimeError("FFmpeg binary not found in system path!")
-        
-        try:
-            await run_subprocess_async([ffmpeg_bin, "-i", str(original_path), "-c:v", "libx264", "-c:a", "aac", "-pix_fmt", "yuv420p", "-y", str(sanitized_path)])
-            logger.info("Sanitization (re-encode) successful.")
-        except Exception as e:
-            logger.warning(f"Re-encode failed ({e}). Attempting Stream Copy...")
-            try:
-                await run_subprocess_async([ffmpeg_bin, "-i", str(original_path), "-c", "copy", "-y", str(sanitized_path)])
-                logger.info("Sanitization (copy) successful.")
-            except Exception as e2:
-                logger.error(f"Sanitization failed completely: {e2}")
-                if original_path.suffix == '.mp4':
-                    logger.warning("Using original file as sanitized file.")
-                    shutil.copy(original_path, sanitized_path)
-                else:
-                    raise RuntimeError("Could not produce a valid MP4 file.")
-
-    # --- Audio Extraction ---
-    audio_path = sanitized_path.with_suffix('.wav')
-    if not audio_path.exists() and sanitized_path.exists():
-        logger.info(f"Extracting audio to {audio_path}...")
-        try:
-            await run_subprocess_async(["ffmpeg", "-i", str(sanitized_path), "-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", "-y", str(audio_path)])
-            logger.info("Audio extraction successful.")
-        except Exception as e: 
-            logger.error(f"Audio extraction failed: {e}")
-    
-    # --- Transcription ---
-    if not transcript_path and audio_path.exists() and transcription.transcription_model is not None:
-        logger.info("Generating transcript via Whisper...")
-        transcript_path = await loop.run_in_executor(None, transcription.generate_transcript, str(audio_path))
-    elif not transcript_path:
-        logger.info("Skipping local transcription (Whisper not loaded or audio missing).")
+@app.on_event("startup")
+async def startup_event():
+    ensure_csv_schema(Path("data/dataset.csv"), DATASET_COLUMNS)
+    ensure_csv_schema(Path("data/manual_dataset.csv"), GROUND_TRUTH_FIELDS)
+    ensure_csv_schema(Path("data/batch_queue.csv"), QUEUE_COLUMNS)
+    if not LITE_MODE:
+        try: inference_logic.load_models()
+        except Exception: pass
 
-    return {"video": str(sanitized_path), "transcript": str(transcript_path) if transcript_path else None, "metadata": metadata}
+@app.get("/health")
+async def health_check():
+    return {"status": "ok", "agent_mount": agent_mount_status}
 
-def safe_int(value):
-    try:
-        clean = re.sub(r'[^\d]', '', str(value))
-        return int(clean) if clean else 0
-    except Exception:
-        return 0
+@app.get("/benchmarks/stats")
+async def get_benchmark_stats():
+    return benchmarking.calculate_benchmarks()
+
+@app.get("/benchmarks/leaderboard")
+async def get_benchmark_leaderboard():
+    return benchmarking.generate_leaderboard()
+
+@app.post("/benchmarks/train_predictive")
+async def run_predictive_training(config: dict = Body(...)):
+    return benchmarking.train_predictive_sandbox(config)
+
+@app.get("/config/prompts")
+async def list_prompts():
+    return [{"id": k, "name": v['description']} for k, v in PROMPT_VARIANTS.items()]
+
+@app.get("/config/tags")
+async def list_configured_tags():
+    path = Path("data/tags.json")
+    if path.exists():
+        with open(path, 'r') as f: return json.load(f)
+    return {}
+
+@app.post("/config/tags")
+async def save_configured_tags(tags: dict = Body(...)):
+    path = Path("data/tags.json")
+    with open(path, 'w', encoding='utf-8') as f: json.dump(tags, f, indent=2)
+    return {"status": "success"}
+
+@app.get("/tags/list")
+async def list_all_tags():
+    tags_count = {}
+    path = Path("data/dataset.csv")
+    if path.exists():
+        for row in common_utils.robust_read_csv(path):
+            t_str = row.get("tags", "")
+            if t_str:
+                for t in t_str.split(','):
+                    t = t.strip()
+                    if t: tags_count[t] = tags_count.get(t, 0) + 1
+    sorted_tags = sorted(tags_count.items(), key=lambda x: x[1], reverse=True)
+    return [{"name": k, "count": v} for k, v in sorted_tags]
 
-async def generate_and_save_croissant_metadata(row_data: dict) -> str:
+@app.post("/extension/ingest")
+async def extension_ingest_link(request: Request):
     try:
-        sanitized_data = {
-            "id": str(row_data.get("id", "")),
-            "link": str(row_data.get("link", "")),
-            "visual_integrity_score": safe_int(row_data.get("visual_integrity_score")),
-            "final_veracity_score": safe_int(row_data.get("final_veracity_score"))
-        }
-        video_id = sanitized_data["id"]
-        timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
-        croissant_json = {
-          "@context": "https://schema.org/",
-          "@type": "Dataset",
-          "name": f"vchat-label-{video_id}",
-          "description": f"Veracity analysis labels for video {video_id}",
-          "url": sanitized_data["link"],
-          "variableMeasured": sanitized_data
-        }
-        path = Path("metadata") / f"{video_id}_{timestamp}.json"
-        path.write_text(json.dumps(croissant_json, indent=2))
-        return str(path)
-    except Exception:
-        return "N/A (Error)"
+        data = await request.json()
+        link = data.get("link")
+        comments = data.get("comments",[])
+        if not link:
+            raise HTTPException(status_code=400, detail="Link required")
+        
+        q_path = Path("data/batch_queue.csv")
+        existing = set()
+        if q_path.exists():
+            for r in common_utils.robust_read_csv(q_path): existing.add(common_utils.normalize_link(r.get('link')))
+            
+        normalized = common_utils.normalize_link(link)
+        if normalized not in existing:
+            with open(q_path, 'a', newline='', encoding='utf-8') as f:
+                writer = csv.DictWriter(f, fieldnames=QUEUE_COLUMNS, extrasaction='ignore')
+                if not q_path.exists() or q_path.stat().st_size == 0: writer.writeheader()
+                writer.writerow({"link": link.strip(), "ingest_timestamp": datetime.datetime.now().isoformat(), "status": "Pending", "task_type": "Ingest"})
+        
+        if comments:
+            tid = common_utils.extract_tweet_id(link) or hashlib.md5(link.encode()).hexdigest()[:10]
+            context_path = Path(f"data/comments/{tid}_ingest.json")
+            with open(context_path, 'w', encoding='utf-8') as f:
+                json.dump({
+                    "link": link,
+                    "timestamp": datetime.datetime.now().isoformat(),
+                    "comments": comments
+                }, f, indent=2)
+            logger.info(f"Saved {len(comments)} comments for ingestion context: {tid}")
 
-async def get_labels_for_link(video_url: str, gemini_config: dict, vertex_config: dict, nrp_config: dict, model_selection: str, include_comments: bool, reasoning_method: str = "cot", system_persona: str = ""):
+        return {"status": "success", "link": link, "comments_saved": len(comments)}
+    except Exception as e:
+        logger.error(f"Ingest Error: {e}")
+        return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
+
+@app.post("/manual/promote")
+async def promote_to_ground_truth(request: Request):
     try:
-        yield f"Downloading assets for {video_url}..."
+        data = await request.json()
+        target_ids = data.get("ids",[])
+        if not target_ids and data.get("id"): target_ids = [data.get("id")]
         
-        try:
-            paths = await prepare_video_assets_async(video_url)
-        except ValueError as ve:
-            yield f"Skipped: {str(ve)}"
-            logger.warning(f"Skipping {video_url}: {ve}")
-            return
-        except Exception as e:
-            yield f"Error preparing assets: {str(e)}"
-            logger.error(f"Asset prep failed for {video_url}: {e}")
-            return
-
-        video_path = paths["video"]
-        transcript_text = parse_vtt(paths["transcript"]) if paths["transcript"] else "No transcript (Audio/Video Analysis only)."
-        caption = paths["metadata"].get("caption", "")
+        if not target_ids: return JSONResponse({"status": "error", "message": "No IDs provided"}, status_code=400)
+
+        ai_path = Path("data/dataset.csv")
+        ai_rows = {}
+        if ai_path.exists():
+            for row in common_utils.robust_read_csv(ai_path):
+                if row.get('id'): ai_rows[str(row['id'])] = row
         
-        yield f"Assets ready. Running inference ({model_selection}, {reasoning_method.upper()})..."
-        logger.info(f"Starting inference pipeline for {video_url} (Transcript len: {len(transcript_text)})")
+        manual_path = Path("data/manual_dataset.csv")
+        manual_exists = manual_path.exists()
+        existing_ids = set()
+        if manual_exists:
+            for row in common_utils.robust_read_csv(manual_path):
+                if row.get('id'): existing_ids.add(str(row['id']))
+
+        new_rows =[]
+        promoted_count = 0
+        for tid in target_ids:
+            tid_str = str(tid)
+            if tid_str in existing_ids: continue 
+            found_row = ai_rows.get(tid_str)
+            if found_row:
+                mapped_row = {
+                    "id": found_row.get("id"), "link": found_row.get("link"),
+                    "timestamp": datetime.datetime.now().isoformat(), "caption": found_row.get("caption"),
+                    "visual_integrity_score": found_row.get("visual_score", 0),
+                    "audio_integrity_score": found_row.get("audio_score", 0),
+                    "source_credibility_score": 5, "logical_consistency_score": found_row.get("logic_score", 0),
+                    "emotional_manipulation_score": 5, "video_audio_score": 5, 
+                    "video_caption_score": found_row.get("align_video_caption", 0), "audio_caption_score": 5,
+                    "final_veracity_score": found_row.get("final_veracity_score", 0),
+                    "final_reasoning": found_row.get("reasoning", ""),
+                    "stats_likes": 0, "stats_shares": 0, "stats_comments": 0, "stats_platform": "twitter",
+                    "tags": found_row.get("tags", ""), "classification": found_row.get("classification", "None"),
+                    "source": "manual_promoted"
+                }
+                new_rows.append(mapped_row)
+                promoted_count += 1
+                existing_ids.add(tid_str)
+
+        if not new_rows: return {"status": "success", "promoted_count": 0}
+
+        mode = 'a' if manual_exists else 'w'
+        with open(manual_path, mode, newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=GROUND_TRUTH_FIELDS, extrasaction='ignore')
+            if not manual_exists or manual_path.stat().st_size == 0: writer.writeheader()
+            for r in new_rows: writer.writerow(r)
+
+        return {"status": "success", "promoted_count": promoted_count}
+    except Exception as e: return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
+
+@app.post("/manual/delete")
+async def delete_ground_truth(request: Request):
+    try:
+        data = await request.json()
+        target_ids = data.get("ids",[])
+        if not target_ids and data.get("id"): target_ids = [data.get("id")]
+        if not target_ids: raise HTTPException(status_code=400)
         
-        final_labels = None
-        raw_toon = ""
-        prompt_used = ""
+        target_ids =[str(t) for t in target_ids]
+        manual_path = Path("data/manual_dataset.csv")
+        if not manual_path.exists(): return {"status": "error", "message": "File not found"}
+
+        rows =[]
+        deleted_count = 0
+        with open(manual_path, 'r', encoding='utf-8') as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                if str(row.get('id')) in target_ids: deleted_count += 1
+                else: rows.append(row)
+        with open(manual_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=GROUND_TRUTH_FIELDS)
+            writer.writeheader()
+            writer.writerows(rows)
+            
+        return {"status": "success", "deleted_count": deleted_count}
+    except Exception as e: return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
+
+@app.post("/manual/verify_queue")
+async def verify_queue_items(request: Request):
+    try:
+        data = await request.json()
+        target_ids = data.get("ids",[])
+        resample_count = max(1, min(data.get("resample_count", 1), 100))
+        if not target_ids: return JSONResponse({"status": "error", "message": "No IDs provided"}, status_code=400)
         
-        if model_selection == 'gemini':
-            pipeline = inference_logic.run_gemini_labeling_pipeline
-            config = gemini_config
-        elif model_selection == 'vertex':
-            pipeline = inference_logic.run_vertex_labeling_pipeline
-            config = vertex_config
-        elif model_selection == 'nrp':
-            if hasattr(inference_logic, 'run_nrp_labeling_pipeline'):
-                pipeline = inference_logic.run_nrp_labeling_pipeline
-            else:
-                yield "NRP Pipeline not supported in this version."
-                return
-            config = nrp_config
-        else:
-            yield f"Unknown model selection: {model_selection}"
-            return
+        manual_path = Path("data/manual_dataset.csv")
+        links_to_queue =[]
+        if manual_path.exists():
+            for row in common_utils.robust_read_csv(manual_path):
+                if str(row.get('id')) in target_ids:
+                    links_to_queue.append(row.get('link'))
         
-        try:
-            async for msg in pipeline(video_path, caption, transcript_text, config, include_comments, reasoning_method, system_persona, paths["metadata"]["id"]):
-                if isinstance(msg, dict) and "parsed_data" in msg:
-                    final_labels = msg["parsed_data"]
-                    raw_toon = msg.get("raw_toon", "")
-                    prompt_used = msg.get("prompt_used", "")
-                    logger.info("Inference successful. Data parsed.")
-                elif isinstance(msg, str): 
-                    yield msg
-                elif isinstance(msg, dict) and "error" in msg:
-                    yield f"API Error: {msg['error']}"
-        except Exception as pipe_err:
-            logger.error(f"Pipeline crashed: {pipe_err}")
-            yield f"Critical Pipeline Failure: {pipe_err}"
-            return
-
-        if not final_labels: 
-            logger.error(f"Inference pipeline completed but returned no labels for {video_url}")
-            yield "No labels generated. Check logs."
-            return
+        if not links_to_queue:
+            return {"status": "error", "message": "No matching links found in Ground Truth."}
+
+        q_path = Path("data/batch_queue.csv")
+        added_count = 0
+        with open(q_path, 'a', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=QUEUE_COLUMNS, extrasaction='ignore')
+            if not q_path.exists() or q_path.stat().st_size == 0: writer.writeheader()
+            for link in links_to_queue:
+                for _ in range(resample_count):
+                    writer.writerow({
+                        "link": link.strip(), 
+                        "ingest_timestamp": datetime.datetime.now().isoformat(), 
+                        "status": "Pending", 
+                        "task_type": "Verify"
+                    })
+                    added_count += 1
         
-        final_labels["meta_info"] = {
-            "prompt_used": prompt_used,
-            "model_selection": model_selection,
-            "reasoning_method": reasoning_method
-        }
+        return {"status": "success", "queued_count": added_count, "message": f"Added {added_count} items to queue for verification pipeline."}
+    except Exception as e:
+        return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
+
+@app.get("/profiles/list")
+async def list_profiles():
+    profiles_dir = Path("data/profiles")
+    profiles =[]
+    if not profiles_dir.exists(): return profiles
+    try:
+        for d in profiles_dir.iterdir():
+            if d.is_dir():
+                hist = d / "history.csv"
+                count = 0
+                if hist.exists():
+                    with open(hist, 'r', encoding='utf-8', errors='ignore') as f: count = sum(1 for _ in f) - 1
+                profiles.append({"username": d.name, "posts_count": max(0, count)})
+    except Exception: pass
+    return sorted(profiles, key=lambda x: x['username'])
+
+@app.get("/profiles/{username}/posts")
+async def get_profile_posts(username: str):
+    csv_path = Path(f"data/profiles/{username}/history.csv")
+    posts =[]
+    if not csv_path.exists(): return posts
+    p_ids, p_links = get_processed_indices()
+    try:
+        for row in common_utils.robust_read_csv(csv_path):
+            link = row.get('link', '')
+            is_labeled = False
+            t_id = common_utils.extract_tweet_id(link)
+            if t_id and t_id in p_ids: is_labeled = True
+            elif common_utils.normalize_link(link) in p_links: is_labeled = True
+            row['is_labeled'] = is_labeled
+            posts.append(row)
+    except Exception: pass
+    return posts
+
+@app.post("/extension/ingest_user_history")
+async def ingest_user_history(request: Request):
+    try:
+        data = await request.json()
+        username = data.get("username")
+        posts = data.get("posts",[])
+        if not username or not posts: raise HTTPException(status_code=400)
+        profile_dir = Path(f"data/profiles/{username}")
+        profile_dir.mkdir(parents=True, exist_ok=True)
+        csv_path = profile_dir / "history.csv"
+        file_exists = csv_path.exists()
+        existing = set()
+        if file_exists:
+            for row in common_utils.robust_read_csv(csv_path): existing.add(row.get('link'))
         
-        vec = final_labels.get("veracity_vectors", {})
-        mod = final_labels.get("modalities", {})
-        fin = final_labels.get("final_assessment", {})
+        with open(csv_path, 'a', newline='', encoding='utf-8') as f:
+            fieldnames =["link", "timestamp", "text", "is_reply", "metric_replies", "metric_reposts", "metric_likes", "metric_views", "ingested_at"]
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+            if not file_exists: writer.writeheader()
+            ts = datetime.datetime.now().isoformat()
+            count = 0
+            for p in posts:
+                if p['link'] not in existing:
+                    m = p.get('metrics', {})
+                    writer.writerow({
+                        "link": p.get('link'), "timestamp": p.get('timestamp'),
+                        "text": p.get('text', '').replace('\n', ' '), "is_reply": p.get('is_reply', False),
+                        "metric_replies": m.get('replies', 0), "metric_reposts": m.get('reposts', 0),
+                        "metric_likes": m.get('likes', 0), "metric_views": m.get('views', 0),
+                        "ingested_at": ts
+                    })
+                    count += 1
+        return {"status": "success", "new_posts": count}
+    except Exception as e: raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/extension/save_comments")
+async def extension_save_comments(request: Request):
+    try:
+        data = await request.json()
+        link = data.get("link")
+        comments = data.get("comments",[])
+        if not link: raise HTTPException(status_code=400)
+        tweet_id = common_utils.extract_tweet_id(link) or hashlib.md5(link.encode()).hexdigest()[:10]
+        csv_path = Path(f"data/comments/{tweet_id}.csv")
+        with open(csv_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=["author", "text", "link", "timestamp"])
+            writer.writeheader()
+            ts = datetime.datetime.now().isoformat()
+            for c in comments:
+                writer.writerow({
+                    "author": c.get("author", "Unknown"), 
+                    "text": c.get("text", "").replace("\n", " "), 
+                    "link": c.get("link", ""),
+                    "timestamp": ts
+                })
+        return {"status": "success", "count": len(comments)}
+    except Exception as e: raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/extension/save_manual")
+@app.post("/manual/save")
+async def save_manual_label(request: Request):
+    try:
+        data = await request.json()
+        link = data.get("link")
+        if not link: 
+            return JSONResponse({"status": "error", "message": "Link required"}, status_code=400)
         
+        tweet_id = common_utils.extract_tweet_id(link) or hashlib.md5(link.encode()).hexdigest()[:10]
+        labels = data.get("labels", data)
+
         row = {
-            "id": paths["metadata"]["id"],
-            "link": paths["metadata"]["link"],
-            "caption": caption,
-            "postdatetime": paths["metadata"].get("postdatetime", ""),
-            "collecttime": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-            "videotranscriptionpath": paths["transcript"] or "",
-            "visual_integrity_score": vec.get("visual_integrity_score", "0"),
-            "audio_integrity_score": vec.get("audio_integrity_score", "0"),
-            "source_credibility_score": vec.get("source_credibility_score", "0"),
-            "logical_consistency_score": vec.get("logical_consistency_score", "0"),
-            "emotional_manipulation_score": vec.get("emotional_manipulation_score", "0"),
-            "video_audio_score": mod.get("video_audio_score", "0"),
-            "video_caption_score": mod.get("video_caption_score", "0"),
-            "audio_caption_score": mod.get("audio_caption_score", "0"),
-            "final_veracity_score": fin.get("veracity_score_total", "0"),
-            "final_reasoning": fin.get("reasoning", "")
+            "id": tweet_id, "link": link, "timestamp": datetime.datetime.now().isoformat(),
+            "caption": data.get("caption", ""),
+            "visual_integrity_score": labels.get("visual_integrity_score", 0),
+            "audio_integrity_score": labels.get("audio_integrity_score", 0),
+            "source_credibility_score": labels.get("source_credibility_score", 0),
+            "logical_consistency_score": labels.get("logical_consistency_score", 0),
+            "emotional_manipulation_score": labels.get("emotional_manipulation_score", 5),
+            "video_audio_score": labels.get("video_audio_score", 0),
+            "video_caption_score": labels.get("video_caption_score", 0),
+            "audio_caption_score": labels.get("audio_caption_score", 0),
+            "final_veracity_score": labels.get("final_veracity_score", 0),
+            "final_reasoning": labels.get("reasoning", labels.get("final_reasoning", "")),
+            "stats_likes": 0, "stats_shares": 0, "stats_comments": 0, "stats_platform": "twitter",
+            "tags": data.get("tags", labels.get("tags", "")),
+            "classification": labels.get("classification", "None"),
+            "source": "Manual"
+        }
+
+        tag_str = str(row["tags"])
+        tag_list =[t.strip() for t in tag_str.split(',') if t.strip()]
+
+        deep_json = {
+            "veracity_vectors": {
+                "visual_integrity_score": str(row["visual_integrity_score"]),
+                "audio_integrity_score": str(row["audio_integrity_score"]),
+                "source_credibility_score": str(row["source_credibility_score"]),
+                "logical_consistency_score": str(row["logical_consistency_score"]),
+                "emotional_manipulation_score": str(row["emotional_manipulation_score"])
+            },
+            "modalities": {
+                "video_audio_score": str(row["video_audio_score"]),
+                "video_caption_score": str(row["video_caption_score"]),
+                "audio_caption_score": str(row["audio_caption_score"])
+            },
+            "video_context_summary": row["caption"], 
+            "tags": tag_list,
+            "factuality_factors": {
+                "claim_accuracy": "Manual",
+                "evidence_gap": "Manual Verification",
+                "grounding_check": "Manual Verification"
+            },
+            "disinformation_analysis": {
+                "classification": row["classification"],
+                "intent": "Manual Labeling",
+                "threat_vector": "Manual Labeling"
+            },
+            "final_assessment": {
+                "veracity_score_total": str(row["final_veracity_score"]),
+                "reasoning": row["final_reasoning"]
+            },
+            "raw_parsed_structure": {
+                "summary": {"text": row["caption"]},
+                "tags": {"keywords": row["tags"]},
+                "final": {"score": str(row["final_veracity_score"]), "reasoning": row["final_reasoning"]}
+            },
+            "meta_info": {
+                "id": tweet_id,
+                "timestamp": row["timestamp"],
+                "link": link,
+                "model_selection": "Manual"
+            }
         }
-        yield {"csv_row": row, "full_json": final_labels, "raw_toon": raw_toon}
 
+        json_path_direct = Path(f"data/labels/{tweet_id}.json")
+        with open(json_path_direct, 'w', encoding='utf-8') as jf:
+            json.dump(deep_json, jf, indent=2, ensure_ascii=False)
+        
+        with open(Path(f"data/mnl_labeled/{tweet_id}.json"), 'w', encoding='utf-8') as jf:
+            json.dump(row, jf, indent=2, ensure_ascii=False)
+
+        manual_path = Path("data/manual_dataset.csv")
+        exists = manual_path.exists()
+        ensure_csv_schema(manual_path, GROUND_TRUTH_FIELDS)
+
+        rows =[]
+        found = False
+        if exists:
+            for r in common_utils.robust_read_csv(manual_path):
+                if str(r.get('id')) == str(tweet_id):
+                    clean_row = {k: row.get(k, "") for k in GROUND_TRUTH_FIELDS}
+                    rows.append(clean_row)
+                    found = True
+                else:
+                    rows.append(r)
+        
+        if not found:
+            clean_row = {k: row.get(k, "") for k in GROUND_TRUTH_FIELDS}
+            rows.append(clean_row)
+            
+        with open(manual_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=GROUND_TRUTH_FIELDS, extrasaction='ignore')
+            writer.writeheader()
+            writer.writerows(rows)
+        
+        update_queue_status(link, "Processed")
+        return {"status": "success", "id": tweet_id}
     except Exception as e:
-        logger.error(f"Fatal error in get_labels_for_link: {e}", exc_info=True)
-        yield {"error": str(e)}
+        logger.error(f"Save Manual Error: {e}")
+        return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
 
-@app.get("/queue/list")
-async def get_queue_list():
-    queue_path = Path("data/batch_queue.csv")
-    if not queue_path.exists(): return[]
-    items =[]
-    with open(queue_path, 'r', encoding='utf-8') as f:
-        reader = csv.reader(f)
-        try: next(reader)
-        except: pass
-        for row in reader:
-            if len(row) > 0:
-                link = row[0]
-                status = row[2] if len(row) > 2 else ("Processed" if check_if_processed(link) else "Pending")
-                items.append({
-                    "link": link, 
-                    "timestamp": row[1] if len(row) > 1 else "", 
-                    "status": status,
-                    "task_type": row[3] if len(row) > 3 else "Ingest"
-                })
-    return items
+@app.get("/community/list_datasets")
+async def list_community_datasets():
+    path = Path("data/comments")
+    files =[]
+    if path.exists():
+        for f in path.glob("*.csv"):
+            files.append({"id": f.stem, "count": sum(1 for _ in open(f, encoding='utf-8'))-1})
+    return files
+
+@app.post("/community/analyze")
+async def analyze_community(dataset_id: str = Body(..., embed=True)):
+    path = Path(f"data/comments/{dataset_id}.csv")
+    if not path.exists(): raise HTTPException(status_code=404)
+    comments = list(common_utils.robust_read_csv(path))
+    if not comments: return {"score": 0, "verdict": "No Data"}
+    s_keys =["fake", "lie", "staged", "bs", "propaganda", "ai", "deepfake"]
+    t_keys =["true", "real", "confirmed", "fact", "source", "proof"]
+    s_count = sum(1 for c in comments if any(k in c['text'].lower() for k in s_keys))
+    t_count = sum(1 for c in comments if any(k in c['text'].lower() for k in t_keys))
+    score = max(0, min(100, 50 + (t_count * 2) - (s_count * 5)))
+    verdict = "Community Skepticism" if score < 30 else "Community Verification" if score > 70 else "Neutral/Mixed"
+    return {"dataset_id": dataset_id, "trust_score": score, "verdict": verdict, "details": {"skeptical_comments": s_count, "trusting_comments": t_count}}
+
+@app.get("/dataset/list")
+async def get_dataset_list():
+    dataset =[]
+    m_path = Path("data/manual_dataset.csv")
+    manual_ids = set()
+    if m_path.exists():
+         for row in common_utils.robust_read_csv(m_path):
+             row['source'] = 'Manual'
+             if row.get('id'): manual_ids.add(str(row['id']))
+             dataset.append(row)
+    
+    path = Path("data/dataset.csv")
+    if path.exists():
+         for row in common_utils.robust_read_csv(path):
+             tid = str(row.get('id', ''))
+             if tid not in manual_ids:
+                 row['source'] = 'AI'
+                 dataset.append(row)
+    return sorted(dataset, key=lambda x: x.get('timestamp', ''), reverse=True)
+
+@app.get("/analytics/account_integrity")
+async def get_account_integrity():
+    id_map = {}
+    prof_dir = Path("data/profiles")
+    if prof_dir.exists():
+        for d in prof_dir.iterdir():
+            for row in common_utils.robust_read_csv(d/"history.csv"):
+                tid = common_utils.extract_tweet_id(row.get('link',''))
+                if tid: id_map[tid] = d.name
+
+    scores_map = {}
+    for fname in ["data/dataset.csv", "data/manual_dataset.csv"]:
+        for row in common_utils.robust_read_csv(Path(fname)):
+            tid = row.get('id')
+            sc = row.get('final_veracity_score', '0')
+            try: val = float(re.sub(r'[^\d.]', '', str(sc)))
+            except: val = 0
+            
+            auth = id_map.get(tid, "Unknown")
+            if auth != "Unknown":
+                if auth not in scores_map: scores_map[auth] =[]
+                scores_map[auth].append(val)
+    
+    return sorted([{"username": k, "avg_veracity": round(sum(v)/len(v),1), "posts_labeled": len(v)} for k,v in scores_map.items()], key=lambda x: x['avg_veracity'], reverse=True)
 
 @app.post("/queue/add")
-async def add_queue_item(payload: dict = Body(...)):
-    link = payload.get("link")
-    if not link:
-        return {"status": "error", "message": "Link required"}
+async def add_queue_item(link: str = Body(..., embed=True)):
     q_path = Path("data/batch_queue.csv")
     existing = set()
     if q_path.exists():
-        with open(q_path, 'r', encoding='utf-8') as f:
-            reader = csv.reader(f)
-            for row in reader:
-                if row: existing.add(normalize_link(row[0]))
-    
-    if normalize_link(link) in existing: 
-        return {"status": "ignored", "message": "Link already in queue"}
+        for r in common_utils.robust_read_csv(q_path): existing.add(common_utils.normalize_link(r.get('link')))
+        
+    normalized = common_utils.normalize_link(link)
+    if not normalized: raise HTTPException(status_code=400, detail="Invalid link")
+    if normalized in existing: return {"status": "ignored", "message": "Link already in queue"}
         
     with open(q_path, 'a', newline='', encoding='utf-8') as f:
-        writer = csv.writer(f)
-        if not q_path.exists() or q_path.stat().st_size == 0:
-            writer.writerow(["link", "ingest_timestamp", "status", "task_type"])
-        writer.writerow([link.strip(), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "Pending", "Ingest"])
+        writer = csv.DictWriter(f, fieldnames=QUEUE_COLUMNS, extrasaction='ignore')
+        if not q_path.exists() or q_path.stat().st_size == 0: writer.writeheader()
+        writer.writerow({"link": link.strip(), "ingest_timestamp": datetime.datetime.now().isoformat(), "status": "Pending", "task_type": "Ingest"})
     return {"status": "success", "link": link}
 
+@app.post("/queue/upload_csv")
+async def upload_csv(file: UploadFile = File(...)):
+    contents = await file.read()
+    lines = contents.decode('utf-8').splitlines()
+    q_path = Path("data/batch_queue.csv")
+    existing = set()
+    if q_path.exists():
+        for r in common_utils.robust_read_csv(q_path): existing.add(common_utils.normalize_link(r.get('link')))
+    
+    added = 0
+    with open(q_path, 'a', newline='', encoding='utf-8') as f:
+        writer = csv.DictWriter(f, fieldnames=QUEUE_COLUMNS, extrasaction='ignore')
+        if not q_path.exists() or q_path.stat().st_size == 0: writer.writeheader()
+        for line in lines:
+            if 'http' in line:
+                raw = line.split(',')[0].strip()
+                if common_utils.normalize_link(raw) not in existing:
+                    writer.writerow({"link": raw, "ingest_timestamp": datetime.datetime.now().isoformat(), "status": "Pending", "task_type": "Ingest"})
+                    added += 1
+    return {"status": "success", "added_count": added}
+
+@app.post("/queue/stop")
+async def stop_processing():
+    global STOP_QUEUE_SIGNAL
+    STOP_QUEUE_SIGNAL = True
+    return {"status": "success", "message": "Stopping queue processing..."}
+
 @app.post("/queue/clear_processed")
 async def clear_processed_queue():
     q_path = Path("data/batch_queue.csv")
     if not q_path.exists(): return {"status": "success", "removed_count": 0}
-    
+    p_ids, p_links = get_processed_indices()
     kept_rows =[]
     removed_count = 0
-    with open(q_path, 'r', encoding='utf-8') as f:
-        reader = csv.reader(f)
-        header = next(reader, ["link", "ingest_timestamp", "status", "task_type"])
-        for row in reader:
-            if not row: continue
-            link = row[0]
-            status = row[2] if len(row) > 2 else "Pending"
-            if status == "Processed" or check_if_processed(link):
-                removed_count += 1
-            else:
-                kept_rows.append(row)
-                
+    for row in common_utils.robust_read_csv(q_path):
+        link = row.get("link")
+        status = row.get("status", "Pending")
+        task_type = row.get("task_type", "Ingest")
+        
+        is_done = False
+        if status == "Processed": is_done = True
+        elif task_type != "Verify" and check_if_processed(link, p_ids, p_links): is_done = True
+        
+        if is_done: removed_count += 1
+        else: kept_rows.append(row)
     with open(q_path, 'w', newline='', encoding='utf-8') as f:
-        writer = csv.writer(f)
-        writer.writerow(header)
+        writer = csv.DictWriter(f, fieldnames=QUEUE_COLUMNS, extrasaction='ignore')
+        writer.writeheader()
         writer.writerows(kept_rows)
     return {"status": "success", "removed_count": removed_count}
 
@@ -526,24 +766,17 @@ async def clear_processed_queue():
 async def delete_queue_items(request: Request):
     try:
         data = await request.json()
-        target_links = set(normalize_link(l) for l in data.get("links",[]))
+        target_links = set(common_utils.normalize_link(l) for l in data.get("links",[]))
         q_path = Path("data/batch_queue.csv")
         if not q_path.exists(): return {"status": "success", "count": 0}
         kept_rows =[]
         deleted_count = 0
-        with open(q_path, 'r', encoding='utf-8') as f:
-            reader = csv.reader(f)
-            header = next(reader,["link", "ingest_timestamp", "status", "task_type"])
-            for row in reader:
-                if not row: continue
-                link = row[0]
-                if normalize_link(link) in target_links:
-                    deleted_count += 1
-                else:
-                    kept_rows.append(row)
+        for row in common_utils.robust_read_csv(q_path):
+            if common_utils.normalize_link(row.get('link')) in target_links: deleted_count += 1
+            else: kept_rows.append(row)
         with open(q_path, 'w', newline='', encoding='utf-8') as f:
-            writer = csv.writer(f)
-            writer.writerow(header)
+            writer = csv.DictWriter(f, fieldnames=QUEUE_COLUMNS, extrasaction='ignore')
+            writer.writeheader()
             writer.writerows(kept_rows)
         return {"status": "success", "count": deleted_count}
     except Exception as e: return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
@@ -552,89 +785,109 @@ async def delete_queue_items(request: Request):
 async def requeue_items(request: Request):
     try:
         data = await request.json()
-        target_links = set(normalize_link(l) for l in data.get("links",[]))
+        target_links = set(common_utils.normalize_link(l) for l in data.get("links",[]))
         q_path = Path("data/batch_queue.csv")
         if not q_path.exists(): return {"status": "success", "count": 0}
         rows =[]
         requeued_count = 0
-        with open(q_path, 'r', encoding='utf-8') as f:
-            reader = csv.reader(f)
-            header = next(reader,["link", "ingest_timestamp", "status", "task_type"])
-            for row in reader:
-                if not row: continue
-                link = row[0]
-                if normalize_link(link) in target_links:
-                    if len(row) > 2: row[2] = "Pending"
-                    requeued_count += 1
-                rows.append(row)
+        for row in common_utils.robust_read_csv(q_path):
+            if common_utils.normalize_link(row.get('link')) in target_links: 
+                row['status'] = 'Pending'
+                requeued_count += 1
+            rows.append(row)
         with open(q_path, 'w', newline='', encoding='utf-8') as f:
-            writer = csv.writer(f)
-            writer.writerow(header)
+            writer = csv.DictWriter(f, fieldnames=QUEUE_COLUMNS, extrasaction='ignore')
+            writer.writeheader()
             writer.writerows(rows)
         return {"status": "success", "count": requeued_count}
     except Exception as e: return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
 
-@app.post("/queue/stop")
-async def stop_queue_processing():
-    global STOP_QUEUE_SIGNAL
-    logger.info("Received Stop Signal from User.")
-    STOP_QUEUE_SIGNAL = True
-    return {"status": "stopping"}
-
-@app.post("/queue/upload_csv")
-async def upload_csv_to_queue(file: UploadFile = File(...)):
+@app.post("/dataset/delete")
+async def delete_dataset_items(request: Request):
     try:
-        content = await file.read()
-        try:
-            decoded = content.decode('utf-8').splitlines()
-        except UnicodeDecodeError:
-             decoded = content.decode('latin-1').splitlines()
-
-        reader = csv.reader(decoded)
-        links_to_add =[]
-        header = next(reader, None)
-        if not header: return {"status": "empty file"}
-
-        link_idx = 0
-        header_lower = [h.lower() for h in header]
-        
-        if "link" in header_lower: link_idx = header_lower.index("link")
-        elif "url" in header_lower: link_idx = header_lower.index("url")
-        elif len(header) > 0 and header[0].strip().startswith("http"):
-            links_to_add.append(header[0])
-            link_idx = 0
-        
-        for row in reader:
-            if len(row) > link_idx and row[link_idx].strip():
-                links_to_add.append(row[link_idx].strip())
+        data = await request.json()
+        target_ids = data.get("ids",[])
+        if not target_ids: raise HTTPException(status_code=400)
+        target_ids = set(str(t) for t in target_ids)
 
-        queue_path = Path("data/batch_queue.csv")
-        existing_links = set()
-        if queue_path.exists():
-            with open(queue_path, 'r', encoding='utf-8') as f:
-                existing_links = set(f.read().splitlines())
+        path = Path("data/dataset.csv")
+        if not path.exists(): return {"status": "success", "count": 0}
 
-        added_count = 0
-        with open(queue_path, 'a', newline='', encoding='utf-8') as f:
-            writer = csv.writer(f)
-            if not queue_path.exists() or queue_path.stat().st_size == 0:
-                writer.writerow(["link", "ingest_timestamp", "status", "task_type"])
+        rows =[]
+        deleted_count = 0
+        for row in common_utils.robust_read_csv(path):
+            if str(row.get('id')) in target_ids:
+                deleted_count += 1
+            else:
+                rows.append(row)
+
+        with open(path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=DATASET_COLUMNS, extrasaction='ignore')
+            writer.writeheader()
+            writer.writerows(rows)
             
-            for link in links_to_add:
-                duplicate = False
-                for line in existing_links:
-                    if normalize_link(link) in line: 
-                        duplicate = True
-                        break
-                if duplicate: continue
-
-                writer.writerow([link, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "Pending", "Ingest"])
-                added_count += 1
-                
-        return {"status": "success", "added": added_count}
+        return {"status": "success", "deleted_count": deleted_count}
     except Exception as e:
-        logging.error(f"Upload CSV error: {e}")
-        return JSONResponse(status_code=400, content={"error": str(e), "status": "failed"})
+        return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
+
+@app.post("/analyze/user_context")
+async def analyze_user_context(request: Request):
+    try:
+        data = await request.json()
+        rep = await user_analysis_logic.generate_user_profile_report(data.get("username"))
+        return {"status": "success", "report": rep}
+    except Exception as e: return JSONResponse({"error": str(e)}, status_code=500)
+
+@app.get("/download-dataset")
+async def download_dataset():
+    file_path = Path("data/dataset.csv")
+    if file_path.exists():
+        return FileResponse(path=file_path, filename="dataset.csv", media_type='text/csv')
+    return Response("Dataset not found.", status_code=404)
+
+@app.get("/model-architecture", response_class=PlainTextResponse)
+async def get_model_architecture():
+    if LITE_MODE: return "Running in LITE mode."
+    if inference_logic.base_model: return str(inference_logic.base_model)
+    return "Model not loaded."
+
+@app.get("/", response_class=HTMLResponse)
+async def read_root(request: Request):
+    return templates.TemplateResponse("index.html", {"request": request})
+
+@app.get("/queue/list")
+async def get_queue_list():
+    q_path = Path("data/batch_queue.csv")
+    items =[]
+    p_ids, p_links = get_processed_indices()
+    for row in common_utils.robust_read_csv(q_path):
+        if row:
+            l = row.get("link")
+            status = row.get("status", "Pending")
+            task_type = row.get("task_type") or "Ingest"
+            
+            if status == "Pending" and task_type != "Verify" and check_if_processed(l, p_ids, p_links): status = "Processed"
+            
+            # Fetch associated comments to display in the dropdown
+            comments =[]
+            tid = common_utils.extract_tweet_id(l) or hashlib.md5(l.encode()).hexdigest()[:10]
+            c_path = Path(f"data/comments/{tid}_ingest.json")
+            if c_path.exists():
+                try:
+                    with open(c_path, 'r') as f:
+                        c_data = json.load(f)
+                        comments = c_data.get('comments',[])
+                except Exception:
+                    pass
+            
+            items.append({
+                "link": l, 
+                "timestamp": row.get("ingest_timestamp",""), 
+                "status": status, 
+                "task_type": task_type,
+                "comments": comments
+            })
+    return items
 
 @app.post("/queue/run")
 async def run_queue_processing(
@@ -642,293 +895,138 @@ async def run_queue_processing(
     gemini_api_key: str = Form(""), gemini_model_name: str = Form(""),
     vertex_project_id: str = Form(""), vertex_location: str = Form(""), vertex_model_name: str = Form(""), vertex_api_key: str = Form(""),
     nrp_api_key: str = Form(""), nrp_model_name: str = Form(""), nrp_base_url: str = Form("https://ellm.nrp-nautilus.io/v1"),
-    include_comments: bool = Form(False),
-    reasoning_method: str = Form("cot"),
-    prompt_template: str = Form("standard"),
-    custom_query: str = Form(""),
-    max_reprompts: int = Form(1)
+    include_comments: bool = Form(False), reasoning_method: str = Form("cot"), prompt_template: str = Form("standard"),
+    custom_query: str = Form(""), max_reprompts: int = Form(1)
 ):
     global STOP_QUEUE_SIGNAL
     STOP_QUEUE_SIGNAL = False
+    
     gemini_config = {"api_key": gemini_api_key, "model_name": gemini_model_name, "max_retries": max_reprompts}
-    vertex_config = {"project_id": vertex_project_id, "location": vertex_location, "model_name": vertex_model_name, "api_key": vertex_api_key, "max_retries": max_reprompts}
+    vertex_config = {"project_id": vertex_project_id, "location": vertex_location, "model_name": vertex_model_name, "api_key": vertex_api_key, "max_retries": max_reprompts, "use_search": True}
     nrp_config = {"api_key": nrp_api_key, "model_name": nrp_model_name, "base_url": nrp_base_url, "max_retries": max_reprompts}
-    
+
     sel_p = PROMPT_VARIANTS.get(prompt_template, PROMPT_VARIANTS['standard'])
     system_persona_txt = sel_p['instruction']
-    if custom_query.strip(): system_persona_txt += f"\n\nSPECIAL INSTRUCTION: {custom_query}"
+    if custom_query.strip(): system_persona_txt += f"\n\nSPECIAL INSTRUCTION FOR THIS BATCH: {custom_query}"
     
+    if model_selection == 'vertex':
+        active_config = vertex_config
+        active_model_name = vertex_model_name
+    elif model_selection == 'nrp':
+        active_config = nrp_config
+        active_model_name = nrp_model_name
+    else:
+        active_config = gemini_config
+        active_model_name = gemini_model_name
+
+    config_params_dict = {
+        "reprompts": max_reprompts,
+        "include_comments": include_comments,
+        "agent_active": False 
+    }
+    config_params_str = json.dumps(config_params_dict)
+
     async def queue_stream():
-        queue_path = Path("data/batch_queue.csv")
-        if not queue_path.exists():
-            yield "data: Queue empty.\n\n"
-            return
+        q_path = Path("data/batch_queue.csv")
+        items =[r for r in common_utils.robust_read_csv(q_path) if r.get("link") and r.get("status", "Pending") == "Pending"]
+        p_ids, p_links = get_processed_indices()
+        yield f"data:[SYSTEM] Persona: {sel_p['description']}\n\n"
         
-        items =[]
-        with open(queue_path, 'r', encoding='utf-8') as f:
-            reader = csv.reader(f)
-            try: next(reader) 
-            except: pass
-            for row in reader:
-                if row and len(row) > 0:
-                    status = row[2] if len(row) > 2 else "Pending"
-                    if status == "Pending":
-                        items.append(row[0])
+        for item in items:
+            link = item.get("link")
+            task_type = item.get("task_type") or "Ingest"
             
-        processed_count = 0
-        total = len(items)
-        
-        logger.info(f"Starting batch queue processing for {total} items.")
-
-        for i, link in enumerate(items):
-            if STOP_QUEUE_SIGNAL:
-                yield "data: [SYSTEM] Stopped by user.\n\n"
-                logger.info("Stopping queue loop.")
+            if STOP_QUEUE_SIGNAL: 
+                yield f"data:[SYSTEM] Stopping by user request.\n\n"
                 break
             
-            if check_if_processed(link):
-                yield f"data: [SKIP] {link} processed.\n\n"
-                update_queue_status_in_file(link, "Processed")
-                continue
-            
-            yield f"data: [START] {i+1}/{total}: {link}\n\n"
-            final_data = None
-            
-            # Streaming results from pipeline
-            async for res in get_labels_for_link(link, gemini_config, vertex_config, nrp_config, model_selection, include_comments, reasoning_method, system_persona_txt):
-                if isinstance(res, str): 
-                    msg = res.replace('\n', ' ')
-                    yield f"data: {msg}\n\n"
-                if isinstance(res, dict):
-                    if "error" in res:
-                        yield f"data:[ERROR DETAIL] {res['error']}\n\n"
-                    if "csv_row" in res: 
-                        final_data = res
-            
-            if final_data:
-                row = final_data["csv_row"]
-                vid_id = row["id"]
-                ts = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
-                
-                # Save artifacts
-                json_path = f"data/labels/{vid_id}_{ts}_labels.json"
-                with open(json_path, 'w') as f: json.dump(final_data["full_json"], f, indent=2)
-                with open(f"data/labels/{vid_id}_{ts}.toon", 'w') as f: f.write(final_data["raw_toon"])
-                
-                prompt_content = final_data.get("full_json", {}).get("meta_info", {}).get("prompt_used", "")
-                if prompt_content:
-                    with open(f"data/prompts/{vid_id}_{ts}_prompt.txt", 'w', encoding='utf-8') as f:
-                        f.write(prompt_content)
-
-                raw_response = final_data.get("raw_toon", "")
-                if raw_response:
-                    with open(f"data/responses/{vid_id}.txt", 'w', encoding='utf-8') as f:
-                        f.write(raw_response)
-
-                row["metadatapath"] = await generate_and_save_croissant_metadata(row)
-                row["json_path"] = json_path
-                
-                dpath = Path("data/dataset.csv")
-                exists = dpath.exists()
-                with open(dpath, 'a', newline='', encoding='utf-8') as f:
-                    writer = csv.DictWriter(f, fieldnames=list(row.keys()), extrasaction='ignore')
-                    if not exists: writer.writeheader()
-                    writer.writerow(row)
-                
-                update_queue_status_in_file(link, "Processed")
-                processed_count += 1
-                yield f"data: [SUCCESS] Labeled.\n\n"
-            else:
-                update_queue_status_in_file(link, "Error")
-                yield f"data: [FAIL] Failed to label. Check logs.\n\n"
-                
-        yield f"data: Batch Complete. +{processed_count} videos labeled.\n\n"
-        yield "event: close\ndata: Done\n\n"
-
-    return StreamingResponse(queue_stream(), media_type="text/event-stream")
+            if task_type != "Verify" and check_if_processed(link, p_ids, p_links): 
+                 update_queue_status(link, "Processed", task_type)
+                 continue
+                 
+            gt_data = None
+            if task_type == "Verify":
+                 manual_path = Path("data/manual_dataset.csv")
+                 if manual_path.exists():
+                      for row in common_utils.robust_read_csv(manual_path):
+                           if common_utils.normalize_link(row.get('link', '')) == common_utils.normalize_link(link):
+                                gt_data = row
+                                break
 
-@app.post("/extension/ingest")
-async def extension_ingest(request: Request):
-    try:
-        data = await request.json()
-        link = data.get("link")
-        if not link: raise HTTPException(status_code=400, detail="No link")
-        q_path = Path("data/batch_queue.csv")
-        file_exists = q_path.exists()
-        
-        if file_exists:
-            with open(q_path, 'r', encoding='utf-8') as f:
-                if link in f.read(): 
-                    return {"status": "queued", "msg": "Duplicate"}
-                    
-        with open(q_path, 'a', newline='', encoding='utf-8') as f:
-            writer = csv.writer(f)
-            if not file_exists: writer.writerow(["link", "ingest_timestamp", "status", "task_type"])
-            writer.writerow([link, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "Pending", "Ingest"])
+            yield f"data: [START] {link} (Type: {task_type})\n\n"
+            tid = common_utils.extract_tweet_id(link) or hashlib.md5(link.encode()).hexdigest()[:10]
+            assets = await common_utils.prepare_video_assets(link, tid)
             
-        return {"status": "queued", "link": link}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+            if not assets or (not assets.get('video') and not assets.get('caption')):
+                log_queue_error(link, "Download/Fetch Error", task_type)
+                yield f"data:   - Download Error.\n\n"
+                continue
 
-@app.post("/extension/save_comments")
-async def extension_save_comments(request: Request):
-    try:
-        data = await request.json()
-        link = data.get("link")
-        comments = data.get("comments",[])
-        if not link or not comments: raise HTTPException(status_code=400, detail="Missing data")
-        
-        csv_path = Path("data/comments.csv")
-        exists = csv_path.exists()
-        fieldnames = ["link", "author", "comment_text", "timestamp"]
-        
-        with open(csv_path, 'a', newline='', encoding='utf-8') as f:
-            writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
-            if not exists: writer.writeheader()
+            trans = common_utils.parse_vtt(assets['transcript']) if assets.get('transcript') else "No transcript (Audio/Video missing)."
+            video_file = assets.get('video')
+            if not video_file: 
+                yield f"data:   - No video found. Text-only analysis.\n\n"
+                video_file = None 
+            else: yield f"data:   - Video found. Inferencing...\n\n"
             
-            ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            for c in comments:
-                row = {"link": link, "timestamp": ts}
-                if isinstance(c, dict):
-                    row["author"] = c.get("author", "Unknown")
-                    row["comment_text"] = c.get("text", "").strip()
-                else:
-                    row["author"] = "Unknown"
-                    row["comment_text"] = str(c).strip()
-                
-                if row["comment_text"]:
-                    writer.writerow(row)
-                    
-        return {"status": "saved", "count": len(comments)}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+            comments_path = Path(f"data/comments/{tid}_ingest.json")
+            current_system_persona = system_persona_txt
+            if comments_path.exists():
+                try:
+                    with open(comments_path, 'r') as f:
+                        c_data = json.load(f)
+                        comments = c_data.get('comments',[])
+                        if comments:
+                            yield f"data:   - Found {len(comments)} comments. Generating Community Context...\n\n"
+                            community_summary = await inference_logic.generate_community_summary(comments, model_selection, active_config)
+                            current_system_persona += f"\n\n### COMMUNITY NOTES / CONTEXT (from Comments):\n{community_summary}\n\nUse this community context to cross-reference claims but remain objective."
+                            yield f"data:   - Context Generated.\n\n"
+                except Exception as e:
+                    logger.error(f"Error processing comments for context: {e}")
 
-@app.post("/extension/save_manual")
-async def extension_save_manual(request: Request):
-    try:
-        data = await request.json()
-        link = data.get("link")
-        labels = data.get("labels", {})
-        stats = data.get("stats", {})
-        if not link: raise HTTPException(status_code=400, detail="No link")
-        
-        video_id = extract_tweet_id(link) or hashlib.md5(link.encode()).hexdigest()[:16]
-        
-        row_data = {
-            "id": video_id,
-            "link": link,
-            "caption": data.get("caption", ""),
-            "collecttime": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-            "source": "manual_extension",
-            "visual_integrity_score": labels.get("visual_integrity_score", 0),
-            "audio_integrity_score": labels.get("audio_integrity_score", 0),
-            "source_credibility_score": labels.get("source_credibility_score", 0),
-            "logical_consistency_score": labels.get("logical_consistency_score", 0),
-            "emotional_manipulation_score": labels.get("emotional_manipulation_score", 0),
-            "video_audio_score": labels.get("video_audio_score", 0),
-            "video_caption_score": labels.get("video_caption_score", 0),
-            "audio_caption_score": labels.get("audio_caption_score", 0),
-            "final_veracity_score": labels.get("final_veracity_score", 0),
-            "final_reasoning": labels.get("reasoning", ""),
-            "stats_likes": stats.get("likes", 0),
-            "stats_shares": stats.get("shares", 0),
-            "stats_comments": stats.get("comments", 0),
-            "stats_platform": stats.get("platform", "unknown")
-        }
-        
-        dpath = Path("data/manual_dataset.csv")
-        exists = dpath.exists()
-        fieldnames = list(row_data.keys())
-        
-        with open(dpath, 'a', newline='', encoding='utf-8') as f:
-            writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
-            if not exists: writer.writeheader()
-            writer.writerow(row_data)
-            
-        return {"status": "saved"}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-@app.get("/manage/list")
-async def list_data():
-    data =[]
-    def read_csv(path, source_type):
-        if not path.exists(): return
-        with open(path, 'r', encoding='utf-8', errors='ignore') as f:
-            reader = csv.DictReader(f)
-            for row in reader:
-                if not row.get('id') or row['id'].strip() == "":
-                    link = row.get('link', '')
-                    tid = extract_tweet_id(link)
-                    row['id'] = tid if tid else hashlib.md5(link.encode()).hexdigest()[:16]
+            res_data = None
+            if model_selection == 'gemini':
+                async for chunk in inference_logic.run_gemini_labeling_pipeline(video_file, assets['caption'], trans, gemini_config, include_comments, reasoning_method, current_system_persona, request_id=tid):
+                    if isinstance(chunk, str): yield f"data:   - {chunk}\n\n"
+                    else: res_data = chunk
+            elif model_selection == 'vertex':
+                async for chunk in inference_logic.run_vertex_labeling_pipeline(video_file, assets['caption'], trans, vertex_config, include_comments, reasoning_method, current_system_persona, request_id=tid):
+                    if isinstance(chunk, str): yield f"data:   - {chunk}\n\n"
+                    else: res_data = chunk
+            elif model_selection == 'nrp':
+                async for chunk in inference_logic.run_nrp_labeling_pipeline(video_file, assets['caption'], trans, nrp_config, include_comments, reasoning_method, current_system_persona, request_id=tid):
+                    if isinstance(chunk, str): yield f"data:   - {chunk}\n\n"
+                    else: res_data = chunk
+
+            if res_data and "parsed_data" in res_data:
+                parsed = res_data["parsed_data"]
+                d_path = Path("data/dataset.csv")
+                ensure_csv_schema(d_path, DATASET_COLUMNS)
+                exists = d_path.exists()
                 
-                json_content = None
-                if row.get('json_path') and os.path.exists(row['json_path']):
-                     try:
-                         with open(row['json_path'], 'r') as jf: json_content = json.load(jf)
-                     except: pass
+                ai_score_val = parsed['final_assessment'].get('veracity_score_total', 0)
+                try: ai_score = float(ai_score_val)
+                except: ai_score = 0
                 
-                row['source_type'] = source_type
-                row['json_data'] = json_content
-                data.append(row)
-
-    read_csv(Path("data/dataset.csv"), "auto")
-    read_csv(Path("data/manual_dataset.csv"), "manual")
-    data.sort(key=lambda x: x.get('collecttime', ''), reverse=True)
-    return data
-
-@app.delete("/manage/delete")
-async def delete_data(id: str = "", link: str = ""):
-    if not id and not link: raise HTTPException(status_code=400, detail="Must provide ID or Link")
-    deleted_count = 0
-    target_id = id
-    
-    def remove_from_csv(path):
-        nonlocal deleted_count, target_id
-        if not path.exists(): return
-        rows =[]
-        found_in_file = False
-        with open(path, 'r', encoding='utf-8', errors='ignore') as f:
-            reader = csv.DictReader(f)
-            fieldnames = reader.fieldnames
-            for row in reader:
-                is_match = False
-                if id and row.get('id') == id: is_match = True
-                elif link and row.get('link') == link: is_match = True
-                if is_match:
-                    found_in_file = True
-                    deleted_count += 1
-                    if not target_id: target_id = row.get('id')
-                else: rows.append(row)
-        if found_in_file:
-            with open(path, 'w', newline='', encoding='utf-8') as f:
-                writer = csv.DictWriter(f, fieldnames=fieldnames)
-                writer.writeheader()
-                writer.writerows(rows)
-
-    remove_from_csv(Path("data/dataset.csv"))
-    remove_from_csv(Path("data/manual_dataset.csv"))
-    if target_id:
-        for p in Path("data/labels").glob(f"{target_id}_*"): p.unlink(missing_ok=True)
-        for p in Path("metadata").glob(f"{target_id}_*"): p.unlink(missing_ok=True)
-    return {"status": "deleted", "count": deleted_count}
-
-@app.post("/label_video")
-async def label_video_endpoint(
-    video_url: str = Form(...), model_selection: str = Form(...),
-    gemini_api_key: str = Form(""), gemini_model_name: str = Form(""),
-    vertex_project_id: str = Form(""), vertex_location: str = Form(""), vertex_model_name: str = Form(""), vertex_api_key: str = Form(""),
-    nrp_api_key: str = Form(""), nrp_model_name: str = Form(""), nrp_base_url: str = Form("https://ellm.nrp-nautilus.io/v1"),
-    include_comments: bool = Form(False),
-    reasoning_method: str = Form("cot")
-):
-    gemini_config = {"api_key": gemini_api_key, "model_name": gemini_model_name}
-    vertex_config = {"project_id": vertex_project_id, "location": vertex_location, "model_name": vertex_model_name, "api_key": vertex_api_key}
-    nrp_config = {"api_key": nrp_api_key, "model_name": nrp_model_name, "base_url": nrp_base_url}
-    
-    async def stream():
-        async for msg in get_labels_for_link(video_url, gemini_config, vertex_config, nrp_config, model_selection, include_comments, reasoning_method):
-             if isinstance(msg, str): yield f"data: {msg}\n\n"
-             if isinstance(msg, dict) and "csv_row" in msg: yield "data: Done. Labels generated.\n\n"
-        yield "event: close\ndata: Done.\n\n"
-    return StreamingResponse(stream(), media_type="text/event-stream")
+                if task_type == "Verify" and gt_data is not None:
+                     gt_final = float(gt_data.get('final_veracity_score', 0))
+                     delta = abs(ai_score - gt_final)
+                     vec_ai = parsed.get('veracity_vectors', {})
+                     mod_ai = parsed.get('modalities', {})
+                     
+                     def s_float(v):
+                         try: return float(v)
+                         except: return 0.0
+
+                     yield f"data:   -[VERIFICATION PIPELINE] Configuration Analysis:\n"
+                     yield f"data:      Model: {active_model_name} | Provider: {model_selection}\n"
+                     yield f"data:      Reasoning: {reasoning_method} | Prompt: {prompt_template} | Reprompts: {max_reprompts}\n"
+                     yield f"data:   -[VERIFICATION SCORES COMPARISON (AI vs Ground Truth)]\n"
+                     yield f"data:      Visual Integrity   : AI {s_float(vec_ai.get('visual_integrity_score'))} | GT {s_float(gt_data.get('visual_integrity_score'))}\n"
+                     yield f"data:      Audio Integrity    : AI {s_float(vec_ai.get('audio_integrity_score'))} | GT {s_float(gt_data.get('audio_integrity_score'))}\n"
+                     yield f"data:      Source Credibility : AI {s_float(vec_ai.get('source_credibility_score'))} | GT {s_float(gt_data.get('source_credibility_score'))}\n"
+                     yield f"data:      Logical Consistency: AI {s_float(vec_ai.get('logical_consistency_score'))} | GT {s_float(gt_data.get('logical_consistency_score'))}\n"
+                     yield f"data:      Emotional Manipul. : AI {s_float(vec_ai.get('emotional_manipulation_score'))} | GT {s_float(gt_data.get('emotional_manipulation_score'))}\n"
+                     yield f"data:      Video-Audio Align  : AI {s_float(mod_ai.get('video_audio_score'))} | GT {s_float(gt_data.get('video_audio_score'))}\n"
+                     yield f"data:      Video-Caption Align: AI {s_float(mod_ai.get('video_caption_score'))} | GT {s_float(gt_data.get('video_caption_score'))}\n"
+                     yield f"data:      Audio-Caption Align: AI {s_float(mod_ai.get('audio_caption_score'))} | GT {s_float(gt_