CircuitScope / frontend /src /components /PatchingLab.js
Gaurav711's picture
feat: complete CircuitScope v3 upgrades, Pearson validation checks, and elite README
e5cc7e7
Raw
History Blame Contribute Delete
28 kB
import React, { useState, useCallback, useMemo } from 'react';
import Plot from '../utils/PlotlyWrapper';
import { Send, Loader2, AlertCircle } from 'lucide-react';
import patchingData from '../data/ioi_patching_results.json';
const BACKEND_URL = process.env.REACT_APP_BACKEND_URL || '';
const PLOTLY_LAYOUT_DEFAULTS = {
paper_bgcolor: 'rgba(0,0,0,0)',
plot_bgcolor: 'rgba(0,0,0,0)',
font: { family: "'Inter', sans-serif", color: '#E8EEF8', size: 12 },
margin: { l: 60, r: 20, t: 30, b: 60 },
xaxis: {
gridcolor: 'rgba(30,43,69,0.55)',
zerolinecolor: 'rgba(42,58,88,0.7)',
tickfont: { color: '#8A9BC4', size: 10 },
},
yaxis: {
gridcolor: 'rgba(30,43,69,0.55)',
zerolinecolor: 'rgba(42,58,88,0.7)',
tickfont: { color: '#8A9BC4', size: 10 },
},
};
const FIELD_COLORS = {
causal_hotspot: '#FF5063',
critical_layers: '#4A9EFF',
io_token: '#00E676',
subject_token: '#FFB347',
mechanism: '#9B59F5',
};
const FIELD_LABELS = {
causal_hotspot: 'Causal Hotspot',
critical_layers: 'Critical Layers',
io_token: 'IO Token',
subject_token: 'Subject Token',
mechanism: 'Mechanism',
};
export const PatchingLab = () => {
const [prompt, setPrompt] = useState('When Alice and Bob went to the park, Bob gave flowers to');
const [loading, setLoading] = useState(false);
const [currentData, setCurrentData] = useState(patchingData);
const [error, setError] = useState(null);
const [modelStatus, setModelStatus] = useState(() => {
return sessionStorage.getItem('cs_model_status') || 'demo';
});
const [validationLayer, setValidationLayer] = useState(6);
const [validationHead, setValidationHead] = useState(9);
const [valLoading, setValLoading] = useState(false);
const [valData, setValData] = useState(null);
const [valError, setValError] = useState(null);
const handleValidateAttribution = useCallback(async () => {
setValLoading(true);
setValError(null);
try {
const res = await fetch(`${BACKEND_URL}/api/validate-attribution`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
prompt: prompt.trim(),
layer: Number(validationLayer),
head: Number(validationHead),
}),
});
if (!res.ok) {
throw new Error(`Validation check failed: ${res.status}`);
}
const data = await res.json();
setValData(data);
} catch (e) {
setValError(e.message || 'Failed to validate attribution linearity');
} finally {
setValLoading(false);
}
}, [prompt, validationLayer, validationHead]);
React.useEffect(() => {
handleValidateAttribution();
}, [prompt, validationLayer, validationHead, handleValidateAttribution]);
React.useEffect(() => {
const handleStatusChange = (e) => {
setModelStatus(e.detail);
};
window.addEventListener('cs_model_status_change', handleStatusChange);
return () => {
window.removeEventListener('cs_model_status_change', handleStatusChange);
};
}, []);
const handleAnalyze = useCallback(async () => {
if (!prompt.trim() || prompt.trim().length < 5) return;
if (modelStatus !== 'active') {
setError('Live model inference is not active. The backend must be active to run custom prompts.');
return;
}
setLoading(true);
setError(null);
try {
const res = await fetch(`${BACKEND_URL}/api/patch`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ prompt: prompt.trim() }),
});
if (!res.ok) {
const errData = await res.json().catch(() => ({}));
throw new Error(errData.detail || `Request failed (${res.status})`);
}
const data = await res.json();
setCurrentData(data);
} catch (e) {
setError(e.message || 'Failed to perform patching analysis');
} finally {
setLoading(false);
}
}, [prompt, modelStatus]);
const handleReset = useCallback(() => {
setCurrentData(patchingData);
setPrompt('When Mary and John went to the store, John gave a bottle of milk to');
setError(null);
}, []);
const heatmapTrace = useMemo(() => ({
z: currentData.values,
x: currentData.tokens,
y: currentData.layers.map(l => `Layer ${l}`),
type: 'heatmap',
colorscale: [
[0, '#060810'], [0.1, '#0D1525'], [0.2, '#0E2040'], [0.3, '#0C3060'],
[0.4, '#0A4080'], [0.5, '#0850A0'], [0.6, '#0A70C0'], [0.7, '#00B0D0'],
[0.8, '#00D0C0'], [1.0, '#00E8A0'],
],
hovertemplate: 'Token: %{x}<br>Layer: %{y}<br>Recovery: %{z:.2f}<extra></extra>',
colorbar: {
title: { text: 'Recovery', font: { color: '#8A9BC4', size: 11 } },
tickfont: { color: '#8A9BC4', size: 10 },
bordercolor: '#1E2B45',
},
}), [currentData]);
return (
<section id="patching" className="scroll-mt-section" data-testid="section-patching-lab" style={{ padding: '80px 0' }}>
<div className="section-container">
<div className="mb-2"><span className="badge-amber">Activation Patching</span></div>
<h2 style={{ fontFamily: "'Space Grotesk', sans-serif", fontWeight: 600, fontSize: 30, color: '#E8EEF8', marginBottom: 8 }}>
Causal Tracing: Where Does Information Live?
</h2>
<p style={{ fontSize: 15, color: '#8A9BC4', maxWidth: 680, marginBottom: 32, lineHeight: 1.75 }}>
Activation patching = running a corrupted prompt but injecting clean activations one position at a time. Where patching restores correct output is where the relevant computation lives.
</p>
{/* Conceptual Explainer */}
<div className="grid grid-cols-1 md:grid-cols-3 gap-4 mb-8">
<div className="research-card" style={{ borderTop: '2px solid #00E676' }}>
<div style={{ fontSize: 12, fontWeight: 600, color: '#00E676', marginBottom: 6 }}>Clean Run</div>
<div className="code-block" style={{ fontSize: 12 }}>
<span style={{ color: '#8A9BC4' }}>"Mary and John went to store,</span><br />
<span style={{ color: '#8A9BC4' }}> John gave milk to"</span><br />
<span style={{ color: '#00E676' }}>Output: Mary ✓</span>
</div>
</div>
<div className="flex items-center justify-center">
<div style={{ textAlign: 'center', fontSize: 12, color: '#FFB347', lineHeight: 1.6 }}>
Patch activations<br />Clean → Corrupted<br />at position X, layer Y<br />
<span style={{ color: '#00D9C0' }}>If output recovers →<br />X,Y stores the info</span>
</div>
</div>
<div className="research-card" style={{ borderTop: '2px solid #FF5063' }}>
<div style={{ fontSize: 12, fontWeight: 600, color: '#FF5063', marginBottom: 6 }}>Corrupted Run</div>
<div className="code-block" style={{ fontSize: 12 }}>
<span style={{ color: '#8A9BC4' }}>"John and Mary went to store,</span><br />
<span style={{ color: '#8A9BC4' }}> John gave milk to"</span><br />
<span style={{ color: '#FF5063' }}>Output: John ✗ (wrong)</span>
</div>
</div>
</div>
<div className="grid grid-cols-1 lg:grid-cols-5 gap-6">
{/* Heatmap */}
<div className="lg:col-span-3">
<div data-testid="patching-heatmap" className="research-card plotly-container" style={{ overflowX: 'auto' }}>
<div style={{ minWidth: 500 }}>
<Plot
data={[heatmapTrace]}
layout={{
...PLOTLY_LAYOUT_DEFAULTS,
title: { text: 'Activation Patching Results (IOI Task)', font: { color: '#E8EEF8', size: 14, family: "'Space Grotesk', sans-serif" } },
xaxis: { ...PLOTLY_LAYOUT_DEFAULTS.xaxis, title: { text: 'Token Position', font: { color: '#8A9BC4', size: 11 } } },
yaxis: { ...PLOTLY_LAYOUT_DEFAULTS.yaxis, title: { text: 'Layer', font: { color: '#8A9BC4', size: 11 } }, autorange: 'reversed' },
height: 420,
}}
config={{ displayModeBar: false, responsive: true }}
style={{ width: '100%' }}
/>
</div>
{/* Hotspot annotations */}
<div className="mt-4 border-t border-[#1E2B45] pt-4">
<div style={{ fontSize: 13, fontWeight: 600, color: '#E8EEF8', marginBottom: 12 }}>Identified Causal Hotspots</div>
<div className="space-y-2">
{currentData.hotspots.map((h, i) => (
<div key={i} className="flex items-start gap-2" style={{ fontSize: 12 }}>
<span style={{ color: '#00D9C0', fontFamily: "'JetBrains Mono', monospace", fontSize: 11, minWidth: 80 }}>
L{h.layer}, "{h.token}"
</span>
<span style={{ color: '#FFB347', fontFamily: "'JetBrains Mono', monospace", fontSize: 11, minWidth: 40 }}>
{h.recovery.toFixed(2)}
</span>
<span style={{ color: '#8A9BC4' }}>{h.interpretation}</span>
</div>
))}
</div>
</div>
</div>
</div>
{/* Live Demo / Control Panel */}
<div className="lg:col-span-2 space-y-4">
<div data-testid="patching-control-panel" className="research-card" style={{ borderLeft: '3px solid #00D9C0' }}>
<div className="flex items-center justify-between mb-4">
<h3 style={{ fontFamily: "'Space Grotesk', sans-serif", fontWeight: 600, fontSize: 18, color: '#E8EEF8' }}>
Causal Tracing Lab
</h3>
{/* Live Model Status Indicator */}
<div className="flex items-center gap-1.5 px-2.5 py-0.5 rounded-full border text-[10px] font-mono tracking-wider font-semibold transition-all duration-300"
style={{
background:
modelStatus === 'active' ? 'rgba(0,217,192,0.08)' :
modelStatus === 'loading' ? 'rgba(74,158,255,0.08)' :
'rgba(245,158,11,0.08)',
borderColor:
modelStatus === 'active' ? 'rgba(0,217,192,0.25)' :
modelStatus === 'loading' ? 'rgba(74,158,255,0.25)' :
'rgba(245,158,11,0.25)',
color:
modelStatus === 'active' ? '#00D9C0' :
modelStatus === 'loading' ? '#4A9EFF' :
'#F59E0B',
}}
>
<span className="relative flex h-1.5 w-1.5">
<span className={`animate-ping absolute inline-flex h-full w-full rounded-full opacity-75 ${
modelStatus === 'active' ? 'bg-[#00D9C0]' :
modelStatus === 'loading' ? 'bg-[#4A9EFF]' :
'bg-[#F59E0B]'
}`}></span>
<span className={`relative inline-flex rounded-full h-1.5 w-1.5 ${
modelStatus === 'active' ? 'bg-[#00D9C0]' :
modelStatus === 'loading' ? 'bg-[#4A9EFF]' :
'bg-[#F59E0B]'
}`}></span>
</span>
{modelStatus === 'active' ? 'LIVE MODEL' :
modelStatus === 'loading' ? 'LOADING...' :
'DEMO MODE'}
</div>
</div>
{modelStatus !== 'active' ? (
<div className="mb-4 p-3 rounded-lg border text-xs leading-relaxed"
style={{
background: 'rgba(245,158,11,0.04)',
borderColor: 'rgba(245,158,11,0.15)',
color: '#D1A354'
}}
>
<strong>Running in Demo Mode:</strong> To unlock dynamic activation patching on custom prompt inputs, boot up the local backend server (`python3 -m uvicorn server:app`). Currently displaying high-fidelity pre-computed baseline graphs.
</div>
) : (
<p style={{ fontSize: 12, color: '#8A9BC4', marginBottom: 12 }}>
Type a prompt with two distinct names. The backend will perform a full causal trace using the CPU-loaded GPT-2 Small model.
</p>
)}
<textarea
data-testid="patching-prompt-input"
value={prompt}
onChange={e => setPrompt(e.target.value)}
rows={3}
disabled={modelStatus !== 'active'}
style={{
width: '100%',
background: modelStatus === 'active' ? '#121729' : '#0B0E1A',
border: '1px solid #1E2B45',
borderRadius: 8,
padding: '10px 12px',
color: modelStatus === 'active' ? '#E8EEF8' : '#4A5A7A',
fontSize: 13,
fontFamily: "'JetBrains Mono', monospace",
resize: 'none',
outline: 'none',
opacity: modelStatus === 'active' ? 1 : 0.6
}}
placeholder="When Alice and Bob went to the park, Bob gave flowers to"
/>
<div className="flex gap-2 mt-3">
<button
data-testid="patching-run-button"
className="btn-primary flex-1 flex items-center justify-center gap-2"
onClick={handleAnalyze}
disabled={loading || prompt.trim().length < 5 || modelStatus !== 'active'}
style={{ opacity: (loading || modelStatus !== 'active') ? 0.5 : 1 }}
>
{loading ? <><Loader2 size={16} className="animate-spin" /> Patching...</> : <><Send size={16} /> Run Live Patching</>}
</button>
<button
onClick={handleReset}
className="btn-ghost"
style={{ fontSize: 12 }}
>
Reset
</button>
</div>
{error && (
<div className="mt-3 p-3 rounded-lg flex items-start gap-2" style={{ background: 'rgba(255,80,99,0.1)', border: '1px solid rgba(255,80,99,0.25)' }}>
<AlertCircle size={16} style={{ color: '#FF5063', marginTop: 2, flexShrink: 0 }} />
<span style={{ fontSize: 12, color: '#FF5063' }}>{error}</span>
</div>
)}
</div>
{/* Baseline Metrics Grid */}
<div className="research-card">
<div style={{ fontSize: 13, fontWeight: 600, color: '#E8EEF8', marginBottom: 12 }}>Baseline Model Statistics</div>
<div className="grid grid-cols-3 gap-2">
<div className="p-3 rounded-lg border border-[#1E2B45]" style={{ background: '#090C15' }}>
<div style={{ fontSize: 10, color: '#8A9BC4', marginBottom: 4 }}>Clean Diff</div>
<div style={{ fontSize: 16, fontWeight: 700, color: '#00E676', fontFamily: "'JetBrains Mono', monospace" }}>
{currentData.baseline?.clean ? currentData.baseline.clean.toFixed(2) : '3.56'}
</div>
</div>
<div className="p-3 rounded-lg border border-[#1E2B45]" style={{ background: '#090C15' }}>
<div style={{ fontSize: 10, color: '#8A9BC4', marginBottom: 4 }}>Corrupt Diff</div>
<div style={{ fontSize: 16, fontWeight: 700, color: '#FF5063', fontFamily: "'JetBrains Mono', monospace" }}>
{currentData.baseline?.corrupted ? currentData.baseline.corrupted.toFixed(2) : '0.84'}
</div>
</div>
<div className="p-3 rounded-lg border border-[#1E2B45]" style={{ background: '#090C15' }}>
<div style={{ fontSize: 10, color: '#8A9BC4', marginBottom: 4 }}>Recovered Diff</div>
<div style={{ fontSize: 16, fontWeight: 700, color: '#4A9EFF', fontFamily: "'JetBrains Mono', monospace" }}>
{currentData.baseline?.circuit_recovered ? currentData.baseline.circuit_recovered.toFixed(2) : '3.10'}
</div>
</div>
</div>
</div>
{/* Linearity Validation Indicator Card */}
<div className="research-card transition-all duration-300" style={{ borderTop: '3px solid #9B59F5', background: 'rgba(15, 10, 25, 0.45)' }}>
<div className="flex items-center justify-between mb-3">
<h3 style={{ fontFamily: "'Space Grotesk', sans-serif", fontWeight: 600, fontSize: 15, color: '#E8EEF8' }}>
Attribution Linearity Validation
</h3>
<span className="badge-teal" style={{ background: 'rgba(155, 89, 245, 0.1)', color: '#9B59F5', borderColor: 'rgba(155, 89, 245, 0.3)', fontSize: 10 }}>
Local Linearity Check
</span>
</div>
<p style={{ fontSize: 11, color: '#8A9BC4', marginBottom: 12, lineHeight: 1.5 }}>
Linear Taylor patching assumes local linearity. This tool computes the <strong>Pearson Correlation ($r$)</strong> between Taylor approximation and actual causal activation patching across the 12 heads.
</p>
{/* Selectors for Layer & Head */}
<div className="grid grid-cols-2 gap-3 mb-4">
<div>
<label style={{ display: 'block', fontSize: 10, color: '#8A9BC4', marginBottom: 4 }}>Verify Layer</label>
<select
value={validationLayer}
onChange={e => setValidationLayer(Number(e.target.value))}
style={{
width: '100%',
background: '#121729',
border: '1px solid #1E2B45',
borderRadius: 6,
padding: '4px 8px',
color: '#E8EEF8',
fontSize: 11,
fontFamily: "'Space Grotesk', sans-serif"
}}
>
{Array.from({ length: 12 }, (_, i) => (
<option key={i} value={i}>Layer {i}</option>
))}
</select>
</div>
<div>
<label style={{ display: 'block', fontSize: 10, color: '#8A9BC4', marginBottom: 4 }}>Active Head Reference</label>
<select
value={validationHead}
onChange={e => setValidationHead(Number(e.target.value))}
style={{
width: '100%',
background: '#121729',
border: '1px solid #1E2B45',
borderRadius: 6,
padding: '4px 8px',
color: '#E8EEF8',
fontSize: 11,
fontFamily: "'Space Grotesk', sans-serif"
}}
>
{Array.from({ length: 12 }, (_, i) => (
<option key={i} value={i}>Head {i}</option>
))}
</select>
</div>
</div>
{valLoading && !valData ? (
<div className="flex items-center justify-center py-6">
<Loader2 className="animate-spin text-[#9B59F5] mr-2" size={18} />
<span style={{ fontSize: 12, color: '#8A9BC4' }}>Calculating Pearson correlation coefficient...</span>
</div>
) : valError ? (
<div className="p-3 rounded-lg flex items-start gap-2" style={{ background: 'rgba(255,80,99,0.1)', border: '1px solid rgba(255,80,99,0.25)' }}>
<AlertCircle size={16} style={{ color: '#FF5063', marginTop: 2, flexShrink: 0 }} />
<span style={{ fontSize: 11, color: '#FF5063' }}>{valError}</span>
</div>
) : valData ? (
<div className="space-y-3">
{/* Gauge and Metric */}
<div className="flex items-center justify-between p-2.5 rounded-lg border border-[#1E2B45]" style={{ background: '#090C15' }}>
<div>
<div style={{ fontSize: 10, color: '#8A9BC4' }}>Pearson Correlation</div>
<div style={{ fontSize: 20, fontWeight: 700, fontFamily: "'JetBrains Mono', monospace", color: valData.pearson_r > 0.8 ? '#00D9C0' : valData.pearson_r > 0.5 ? '#FFB347' : '#FF5063' }}>
r = {valData.pearson_r.toFixed(3)}
</div>
</div>
<div style={{ textAlign: 'right' }}>
<div style={{ fontSize: 10, color: '#8A9BC4' }}>Verdict</div>
<div style={{
fontSize: 12,
fontWeight: 700,
color: valData.pearson_r > 0.8 ? '#00D9C0' : valData.pearson_r > 0.5 ? '#FFB347' : '#FF5063'
}}>
{valData.verdict}
</div>
</div>
</div>
{/* Visual Gauge Bar */}
<div>
<div className="flex justify-between text-[9px] text-[#8A9BC4] mb-1">
<span>Non-Linear (0.0)</span>
<span>Moderate (0.5)</span>
<span>High (1.0)</span>
</div>
<div style={{ height: 6, background: '#121729', borderRadius: 3, overflow: 'hidden', border: '1px solid #1E2B45' }}>
<div style={{
height: '100%',
width: `${Math.max(0, Math.min(100, valData.pearson_r * 100))}%`,
background: valData.pearson_r > 0.8 ? 'linear-gradient(90deg, #9B59F5 0%, #00D9C0 100%)' : valData.pearson_r > 0.5 ? 'linear-gradient(90deg, #9B59F5 0%, #FFB347 100%)' : 'linear-gradient(90deg, #9B59F5 0%, #FF5063 100%)',
borderRadius: 3,
transition: 'width 0.4s ease-out'
}}></div>
</div>
</div>
{/* Interpretation */}
<div style={{ fontSize: 11, color: '#8A9BC4', lineHeight: 1.5, background: 'rgba(255,255,255,0.02)', padding: 10, borderRadius: 6, border: '1px solid #1E2B45' }}>
{valData.interpretation}
</div>
{/* Mini Dot Comparison Chart */}
<div>
<div style={{ fontSize: 10, fontWeight: 600, color: '#E8EEF8', marginBottom: 6 }}>Attribution Comparison (Taylor vs. True Causal)</div>
<div className="space-y-1.5 max-h-[120px] overflow-y-auto pr-1">
{valData.taylor_values.map((t, idx) => {
const c = valData.causal_values[idx];
return (
<div key={idx} className="flex items-center justify-between text-[10px]" style={{ fontFamily: "'JetBrains Mono', monospace" }}>
<span style={{ color: '#8A9BC4', minWidth: 32 }}>H{idx}</span>
<div className="flex-1 mx-2 flex items-center h-2 relative" style={{ background: 'rgba(30,43,69,0.3)', borderRadius: 4 }}>
{/* Taylor approximation dot (Purple) */}
<div style={{
position: 'absolute',
left: `${Math.max(0, Math.min(95, t * 100))}%`,
width: 6,
height: 6,
borderRadius: '50%',
background: '#9B59F5',
zIndex: 2,
transform: 'translateY(-50%)',
top: '50%'
}} title={`Taylor: ${t.toFixed(2)}`} />
{/* Actual causal dot (Teal) */}
<div style={{
position: 'absolute',
left: `${Math.max(0, Math.min(95, c * 100))}%`,
width: 6,
height: 6,
borderRadius: '50%',
background: '#00D9C0',
zIndex: 3,
transform: 'translateY(-50%)',
top: '50%',
border: '1px solid #060810'
}} title={`Causal: ${c.toFixed(2)}`} />
</div>
<span style={{ color: '#8A9BC4', minWidth: 60, textAlign: 'right' }}>
T:{t.toFixed(1)} C:{c.toFixed(1)}
</span>
</div>
);
})}
</div>
<div className="flex justify-end gap-3 text-[9px] text-[#8A9BC4] mt-1.5">
<div className="flex items-center gap-1">
<div style={{ width: 6, height: 6, borderRadius: '50%', background: '#9B59F5' }}></div>
<span>Taylor</span>
</div>
<div className="flex items-center gap-1">
<div style={{ width: 6, height: 6, borderRadius: '50%', background: '#00D9C0' }}></div>
<span>True Causal</span>
</div>
</div>
</div>
</div>
) : null}
</div>
{/* Technique Cards */}
<div className="space-y-3">
{[
{ title: 'Activation Patching', badge: 'Causal intervention', color: '#FFB347', desc: 'Run corrupted prompt. Replace one activation with clean value. Measure metric recovery.', math: 'ΔLD = LD(patched) - LD(corrupted)' },
{ title: 'Path Patching', badge: 'Causal flow', color: '#4A9EFF', desc: 'More precise. Patches along specific paths between components. Identifies HOW information flows.' },
{ title: 'Direct Logit Attribution', badge: 'Linear algebra', color: '#9B59F5', desc: 'Decomposes model output into per-head contributions via W_O and W_U projection matrices.' },
].map((t, i) => (
<div key={i} className="research-card" style={{ padding: '12px 16px' }}>
<div className="flex items-center gap-2 mb-1">
<span style={{ fontSize: 13, fontWeight: 600, color: '#E8EEF8' }}>{t.title}</span>
<span className="badge-teal" style={{ background: `${t.color}15`, color: t.color, borderColor: `${t.color}40`, fontSize: 10 }}>{t.badge}</span>
</div>
<p style={{ fontSize: 12, color: '#8A9BC4', lineHeight: 1.5 }}>{t.desc}</p>
{t.math && <code style={{ fontSize: 11, color: '#00D9C0', marginTop: 4, display: 'block' }}>{t.math}</code>}
</div>
))}
</div>
</div>
</div>
</div>
</section>
);
};