Add experiment notes feature — attach research documents to experiments

New data type (experiment_notes) with CRUD endpoints, a Notes tab in
experiment detail view, and a full-page markdown reader. Import script
reads from note_sources directories configured in experiment.yaml.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

backend/api/experiments.py

@@ -15,7 +15,7 @@ _cache: dict[str, list[dict]] = {}
 _cache_loaded: set[str] = set()
 _lock = threading.Lock()
 
-FILES = ["experiments", "runs", "sub_experiments"]
+FILES = ["experiments", "runs", "sub_experiments", "experiment_notes"]
 
 
 def _ensure_local_dir():
@@ -102,16 +102,19 @@ def list_experiments():
     experiments = _get("experiments")
     runs = _get("runs")
     subs = _get("sub_experiments")
+    notes = _get("experiment_notes")
 
     # Enrich with counts
     result = []
     for exp in experiments:
         exp_runs = [r for r in runs if r.get("experiment_id") == exp["id"]]
         exp_subs = [s for s in subs if s.get("experiment_id") == exp["id"]]
+        exp_notes = [n for n in notes if n.get("experiment_id") == exp["id"]]
         result.append({
             **exp,
             "run_count": len(exp_runs),
             "sub_count": len(exp_subs),
+            "note_count": len(exp_notes),
         })
     return jsonify(result)
 
@@ -165,8 +168,9 @@ def get_experiment(exp_id):
 
     runs = [r for r in _get("runs") if r.get("experiment_id") == exp_id]
     subs = [s for s in _get("sub_experiments") if s.get("experiment_id") == exp_id]
+    notes = [n for n in _get("experiment_notes") if n.get("experiment_id") == exp_id]
 
-    return jsonify({**exp, "runs": runs, "sub_experiments": subs})
+    return jsonify({**exp, "runs": runs, "sub_experiments": subs, "experiment_notes": notes})
 
 
 @bp.route("/<exp_id>", methods=["PUT"])
@@ -194,11 +198,13 @@ def delete_experiment(exp_id):
     experiments = [e for e in experiments if e["id"] != exp_id]
     _set("experiments", experiments)
 
-    # Also delete associated runs and subs
+    # Also delete associated runs, subs, and notes
     runs = [r for r in _get("runs") if r.get("experiment_id") != exp_id]
     _set("runs", runs)
     subs = [s for s in _get("sub_experiments") if s.get("experiment_id") != exp_id]
     _set("sub_experiments", subs)
+    notes = [n for n in _get("experiment_notes") if n.get("experiment_id") != exp_id]
+    _set("experiment_notes", notes)
 
     return jsonify({"status": "ok"})
 
@@ -328,6 +334,71 @@ def delete_sub(exp_id, sub_id):
     return jsonify({"status": "ok"})
 
 
+# --- Experiment Notes ---
+
+@bp.route("/<exp_id>/notes", methods=["POST"])
+def create_note(exp_id):
+    experiments = _get("experiments")
+    if not any(e["id"] == exp_id for e in experiments):
+        return jsonify({"error": "experiment not found"}), 404
+
+    data = request.get_json()
+    title = data.get("title", "").strip()
+    if not title:
+        return jsonify({"error": "title is required"}), 400
+
+    note_id = data.get("id", f"{exp_id}__note_{uuid.uuid4().hex[:8]}")
+
+    note = {
+        "id": note_id,
+        "experiment_id": exp_id,
+        "title": title,
+        "filename": data.get("filename", ""),
+        "content_md": data.get("content_md", ""),
+        "created": _now(),
+        "updated": _now(),
+    }
+
+    notes = _get("experiment_notes")
+    notes.append(note)
+    _set("experiment_notes", notes)
+    return jsonify(note), 201
+
+
+@bp.route("/<exp_id>/notes/<note_id>", methods=["GET"])
+def get_note(exp_id, note_id):
+    notes = _get("experiment_notes")
+    note = next((n for n in notes if n["id"] == note_id and n["experiment_id"] == exp_id), None)
+    if not note:
+        return jsonify({"error": "not found"}), 404
+    return jsonify(note)
+
+
+@bp.route("/<exp_id>/notes/<note_id>", methods=["PUT"])
+def update_note(exp_id, note_id):
+    data = request.get_json()
+    notes = _get("experiment_notes")
+
+    for note in notes:
+        if note["id"] == note_id and note["experiment_id"] == exp_id:
+            for key in ["title", "content_md"]:
+                if key in data:
+                    note[key] = data[key]
+            note["updated"] = _now()
+            _set("experiment_notes", notes)
+            return jsonify(note)
+
+    return jsonify({"error": "not found"}), 404
+
+
+@bp.route("/<exp_id>/notes/<note_id>", methods=["DELETE"])
+def delete_note(exp_id, note_id):
+    notes = _get("experiment_notes")
+    notes = [n for n in notes if not (n["id"] == note_id and n["experiment_id"] == exp_id)]
+    _set("experiment_notes", notes)
+    return jsonify({"status": "ok"})
+
+
 # --- Sync & Import ---
 
 @bp.route("/sync", methods=["POST"])

frontend/src/experiments/ExperimentsApp.tsx

@@ -2,6 +2,7 @@ import { useExperimentsState } from "./store";
 import ExperimentList from "./components/ExperimentList";
 import ExperimentDetail from "./components/ExperimentDetail";
 import SubExperimentView from "./components/SubExperimentView";
+import NoteView from "./components/NoteView";
 
 export default function ExperimentsApp() {
   const state = useExperimentsState();
@@ -28,6 +29,16 @@ export default function ExperimentsApp() {
     );
   }
 
+  if (state.view.kind === "note" && state.currentNote && state.currentDetail) {
+    return (
+      <NoteView
+        note={state.currentNote}
+        experimentName={state.currentDetail.name}
+        onBack={() => state.navigateToDetail(state.view.kind === "note" ? state.view.expId : "")}
+      />
+    );
+  }
+
   if (state.view.kind === "sub" && state.currentSub && state.currentDetail) {
     return (
       <SubExperimentView
@@ -45,6 +56,7 @@ export default function ExperimentsApp() {
         experiment={state.currentDetail}
         onBack={state.navigateToList}
         onSelectSub={(subId) => state.navigateToSub(state.view.kind === "detail" ? state.view.expId : "", subId)}
+        onSelectNote={(noteId) => state.navigateToNote(state.view.kind === "detail" ? state.view.expId : "", noteId)}
         onRefresh={state.refreshDetail}
       />
     );

frontend/src/experiments/api.ts

@@ -1,4 +1,4 @@
-import type { Experiment, ExperimentDetail, RunRecord, SubExperiment } from "./types";
+import type { Experiment, ExperimentDetail, RunRecord, SubExperiment, ExperimentNote } from "./types";
 
 const BASE = "/api/experiments";
 
@@ -77,6 +77,25 @@ export const experimentsApi = {
     return fetchJSON<{ status: string }>(`${BASE}/${expId}/subs/${subId}`, { method: "DELETE" });
   },
 
+  // Notes
+  createNote(expId: string, data: Partial<ExperimentNote>) {
+    return fetchJSON<ExperimentNote>(`${BASE}/${expId}/notes`, {
+      method: "POST",
+      body: JSON.stringify(data),
+    });
+  },
+
+  updateNote(expId: string, noteId: string, data: Partial<ExperimentNote>) {
+    return fetchJSON<ExperimentNote>(`${BASE}/${expId}/notes/${noteId}`, {
+      method: "PUT",
+      body: JSON.stringify(data),
+    });
+  },
+
+  deleteNote(expId: string, noteId: string) {
+    return fetchJSON<{ status: string }>(`${BASE}/${expId}/notes/${noteId}`, { method: "DELETE" });
+  },
+
   sync() {
     return fetchJSON<{ status: string }>(`${BASE}/sync`, { method: "POST" });
   },

frontend/src/experiments/components/ExperimentDetail.tsx

@@ -18,16 +18,17 @@ const RUN_STATUS_COLORS: Record<string, string> = {
   failed: "text-red-400",
 };
 
-type Tab = "overview" | "runs" | "datasets" | "subs";
+type Tab = "overview" | "runs" | "datasets" | "subs" | "notes";
 
 interface Props {
   experiment: ExperimentDetailType;
   onBack: () => void;
   onSelectSub: (subId: string) => void;
+  onSelectNote: (noteId: string) => void;
   onRefresh: () => void;
 }
 
-export default function ExperimentDetail({ experiment, onBack, onSelectSub, onRefresh }: Props) {
+export default function ExperimentDetail({ experiment, onBack, onSelectSub, onSelectNote, onRefresh }: Props) {
   const [tab, setTab] = useState<Tab>("overview");
   const [editing, setEditing] = useState(false);
   const [notes, setNotes] = useState(experiment.notes || "");
@@ -95,6 +96,7 @@ export default function ExperimentDetail({ experiment, onBack, onSelectSub, onRe
     { id: "runs", label: "Runs", count: experiment.runs?.length || 0 },
     { id: "datasets", label: "Datasets", count: experiment.hf_repos?.length || 0 },
     { id: "subs", label: "Sub-experiments", count: experiment.sub_experiments?.length || 0 },
+    { id: "notes", label: "Notes", count: experiment.experiment_notes?.length || 0 },
   ];
 
   return (
@@ -479,6 +481,40 @@ export default function ExperimentDetail({ experiment, onBack, onSelectSub, onRe
             )}
           </div>
         )}
+
+        {tab === "notes" && (
+          <div>
+            <div className="flex justify-between items-center mb-4">
+              <h2 className="text-sm font-medium text-gray-300">Research Notes</h2>
+            </div>
+
+            {(experiment.experiment_notes || []).length === 0 ? (
+              <p className="text-sm text-gray-500">No research notes attached.</p>
+            ) : (
+              <div className="grid gap-2">
+                {experiment.experiment_notes.map((note) => (
+                  <button
+                    key={note.id}
+                    onClick={() => onSelectNote(note.id)}
+                    className="w-full text-left bg-gray-900 hover:bg-gray-800 border border-gray-800 hover:border-gray-700 rounded p-3 transition-colors"
+                  >
+                    <div className="flex items-center justify-between">
+                      <div>
+                        <span className="text-sm text-gray-200">{note.title}</span>
+                        {note.filename && (
+                          <p className="text-xs text-gray-500 mt-0.5 font-mono">{note.filename}</p>
+                        )}
+                      </div>
+                      <span className="text-xs text-gray-600">
+                        {note.updated ? new Date(note.updated).toLocaleDateString() : ""}
+                      </span>
+                    </div>
+                  </button>
+                ))}
+              </div>
+            )}
+          </div>
+        )}
       </div>
     </div>
   );

frontend/src/experiments/components/NoteView.tsx

@@ -0,0 +1,47 @@
+import type { ExperimentNote } from "../types";
+import Markdown from "./Markdown";
+
+interface Props {
+  note: ExperimentNote;
+  experimentName: string;
+  onBack: () => void;
+}
+
+export default function NoteView({ note, experimentName, onBack }: Props) {
+  return (
+    <div className="h-full flex flex-col">
+      {/* Breadcrumb + header */}
+      <div className="px-6 py-4 border-b border-gray-800">
+        <div className="flex items-center gap-2 text-sm mb-3">
+          <button onClick={onBack} className="text-gray-400 hover:text-gray-200 transition-colors">
+            &larr; {experimentName}
+          </button>
+          <span className="text-gray-600">/</span>
+          <span className="text-gray-300">{note.title}</span>
+        </div>
+        <h1 className="text-lg font-semibold text-gray-200">{note.title}</h1>
+        {note.filename && (
+          <p className="text-xs text-gray-500 mt-1 font-mono">{note.filename}</p>
+        )}
+      </div>
+
+      {/* Content */}
+      <div className="flex-1 overflow-y-auto p-6">
+        <div className="max-w-4xl">
+          <div className="bg-gray-900 rounded p-6">
+            {note.content_md ? (
+              <Markdown content={note.content_md} />
+            ) : (
+              <span className="text-sm text-gray-600 italic">No content.</span>
+            )}
+          </div>
+
+          <div className="mt-6 flex gap-4 text-xs text-gray-600">
+            {note.created && <span>Created: {new Date(note.created).toLocaleDateString()}</span>}
+            {note.updated && <span>Updated: {new Date(note.updated).toLocaleDateString()}</span>}
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}

frontend/src/experiments/store.ts

@@ -1,16 +1,18 @@
 import { useState, useCallback, useEffect } from "react";
-import type { Experiment, ExperimentDetail, SubExperiment } from "./types";
+import type { Experiment, ExperimentDetail, SubExperiment, ExperimentNote } from "./types";
 import { experimentsApi } from "./api";
 
 export type View =
   | { kind: "list" }
   | { kind: "detail"; expId: string }
-  | { kind: "sub"; expId: string; subId: string };
+  | { kind: "sub"; expId: string; subId: string }
+  | { kind: "note"; expId: string; noteId: string };
 
 export function useExperimentsState() {
   const [experiments, setExperiments] = useState<Experiment[]>([]);
   const [currentDetail, setCurrentDetail] = useState<ExperimentDetail | null>(null);
   const [currentSub, setCurrentSub] = useState<SubExperiment | null>(null);
+  const [currentNote, setCurrentNote] = useState<ExperimentNote | null>(null);
   const [view, setView] = useState<View>({ kind: "list" });
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState<string | null>(null);
@@ -36,6 +38,7 @@ export function useExperimentsState() {
     setView({ kind: "list" });
     setCurrentDetail(null);
     setCurrentSub(null);
+    setCurrentNote(null);
     loadExperiments();
   }, [loadExperiments]);
 
@@ -62,8 +65,17 @@ export function useExperimentsState() {
     }
   }, [currentDetail]);
 
+  const navigateToNote = useCallback((expId: string, noteId: string) => {
+    if (!currentDetail) return;
+    const note = (currentDetail.experiment_notes || []).find(n => n.id === noteId);
+    if (note) {
+      setCurrentNote(note);
+      setView({ kind: "note", expId, noteId });
+    }
+  }, [currentDetail]);
+
   const refreshDetail = useCallback(async () => {
-    if (view.kind === "detail" || view.kind === "sub") {
+    if (view.kind === "detail" || view.kind === "sub" || view.kind === "note") {
       const expId = view.expId;
       try {
         const detail = await experimentsApi.get(expId);
@@ -78,6 +90,7 @@ export function useExperimentsState() {
     experiments,
     currentDetail,
     currentSub,
+    currentNote,
     view,
     loading,
     error,
@@ -85,6 +98,7 @@ export function useExperimentsState() {
     navigateToList,
     navigateToDetail,
     navigateToSub,
+    navigateToNote,
     refreshDetail,
     loadExperiments,
   };

frontend/src/experiments/types.ts

@@ -30,6 +30,7 @@ export interface Experiment {
   updated: string;
   run_count?: number;
   sub_count?: number;
+  note_count?: number;
 }
 
 export interface RunRecord {
@@ -57,7 +58,18 @@ export interface SubExperiment {
   updated: string;
 }
 
+export interface ExperimentNote {
+  id: string;
+  experiment_id: string;
+  title: string;
+  filename: string;
+  content_md: string;
+  created: string;
+  updated: string;
+}
+
 export interface ExperimentDetail extends Experiment {
   runs: RunRecord[];
   sub_experiments: SubExperiment[];
+  experiment_notes: ExperimentNote[];
 }

scripts/import_experiments.py

@@ -200,13 +200,40 @@ def load_experiment(exp_dir: Path) -> tuple[dict, list[dict], list[dict]]:
             }
             sub_experiments.append(sub)
 
-    return experiment, runs, sub_experiments
+    # Experiment notes from note_sources
+    experiment_notes = []
+    for source_dir in config.get("note_sources", []):
+        source_path = Path(source_dir).expanduser()
+        if not source_path.exists():
+            continue
+        for md_file in sorted(source_path.glob("*.md")):
+            # Skip claude.md instruction files
+            if md_file.name.lower() == "claude.md":
+                continue
+            note_title = md_file.stem.replace("_", " ").replace("-", " ").title()
+            with open(md_file) as f:
+                note_content = f.read()
+
+            note_id = f"{name}__note_{md_file.stem}"
+            note = {
+                "id": note_id,
+                "experiment_id": name,
+                "title": note_title,
+                "filename": md_file.name,
+                "content_md": note_content,
+                "created": config.get("created", ""),
+                "updated": config.get("updated", ""),
+            }
+            experiment_notes.append(note)
+
+    return experiment, runs, sub_experiments, experiment_notes
 
 
 def main():
     all_experiments = []
     all_runs = []
     all_subs = []
+    all_notes = []
 
     for exp_dir in sorted(EXPERIMENTS_DIR.iterdir()):
         if not exp_dir.is_dir():
@@ -215,13 +242,14 @@ def main():
             continue
 
         print(f"Loading: {exp_dir.name}")
-        exp, runs, subs = load_experiment(exp_dir)
+        exp, runs, subs, notes = load_experiment(exp_dir)
         all_experiments.append(exp)
         all_runs.extend(runs)
         all_subs.extend(subs)
-        print(f"  -> {len(runs)} runs, {len(subs)} sub-experiments, {len(exp.get('hf_repos', []))} HF repos")
+        all_notes.extend(notes)
+        print(f"  -> {len(runs)} runs, {len(subs)} sub-experiments, {len(notes)} notes, {len(exp.get('hf_repos', []))} HF repos")
 
-    print(f"\nTotal: {len(all_experiments)} experiments, {len(all_runs)} runs, {len(all_subs)} sub-experiments")
+    print(f"\nTotal: {len(all_experiments)} experiments, {len(all_runs)} runs, {len(all_subs)} sub-experiments, {len(all_notes)} notes")
 
     # Upload to HF
     api = HfApi()
@@ -230,7 +258,7 @@ def main():
     except Exception:
         pass
 
-    for name, data in [("experiments", all_experiments), ("runs", all_runs), ("sub_experiments", all_subs)]:
+    for name, data in [("experiments", all_experiments), ("runs", all_runs), ("sub_experiments", all_subs), ("experiment_notes", all_notes)]:
         with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
             json.dump(data, f, indent=2, default=str)
             tmp = f.name