ajit3259 commited on
Commit
95d7ec3
Β·
1 Parent(s): 035fc3c

feat: engineering seed, BGE-base embed, FORCE_RESEED startup flag

Browse files
Files changed (3) hide show
  1. app.py +7 -1
  2. config.py +1 -1
  3. seed.py +342 -235
app.py CHANGED
@@ -56,8 +56,14 @@ UPLOADS_DIR.mkdir(parents=True, exist_ok=True)
56
  CONTENT_DIR.mkdir(exist_ok=True)
57
  init_db()
58
 
 
 
 
 
 
 
59
  _gc = get_captures
60
- if len(_gc(limit=1)) == 0:
61
  try:
62
  import seed
63
  seed.run()
 
56
  CONTENT_DIR.mkdir(exist_ok=True)
57
  init_db()
58
 
59
+ _force_reseed = bool(os.getenv("FORCE_RESEED"))
60
+ if _force_reseed and DB_PATH.exists():
61
+ print("[startup] FORCE_RESEED=1 β€” deleting existing DB and reseeding")
62
+ DB_PATH.unlink()
63
+ init_db()
64
+
65
  _gc = get_captures
66
+ if _force_reseed or len(_gc(limit=1)) == 0:
67
  try:
68
  import seed
69
  seed.run()
config.py CHANGED
@@ -17,4 +17,4 @@ LM_MODEL = os.getenv("LM_MODEL", "")
17
  # PATH B β€” HF Transformers (used when LM_STUDIO_URL is not set)
18
  HF_MODEL = os.getenv("HF_MODEL", "nvidia/Nemotron-Mini-4B-Instruct")
19
  HF_VL_MODEL = os.getenv("HF_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
20
- EMBED_MODEL = os.getenv("EMBED_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
 
17
  # PATH B β€” HF Transformers (used when LM_STUDIO_URL is not set)
18
  HF_MODEL = os.getenv("HF_MODEL", "nvidia/Nemotron-Mini-4B-Instruct")
19
  HF_VL_MODEL = os.getenv("HF_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
20
+ EMBED_MODEL = os.getenv("EMBED_MODEL", "BAAI/bge-base-en-v1.5")
seed.py CHANGED
@@ -1,297 +1,404 @@
1
- """Pre-populate the KB with curated captures so Space visitors see a working knowledge graph."""
2
- from db import init_db, save_capture, update_capture, get_all_embeddings, get_captures
 
3
  from lm import embed, find_related
4
 
5
  init_db()
6
 
 
 
 
 
7
  SEEDS = [
8
- # ── Memory & learning ──────────────────────────────────────────────────────
9
- {
10
- "type": "text", "intent": "learn",
11
- "raw": "Spaced repetition works because of the forgetting curve β€” you review just before you forget, which strengthens the memory trace each time.",
12
- "summary": "Spaced repetition exploits the forgetting curve: reviewing material just before forgetting it each time creates progressively stronger memory traces.",
13
- "tags": ["memory", "learning", "spaced repetition", "cognition"],
14
- },
15
- {
16
- "type": "text", "intent": "learn",
17
- "raw": "The default mode network activates during mind-wandering and is linked to creativity and insight. Boredom isn't wasted time.",
18
- "summary": "Mind-wandering activates the default mode network, which underlies creativity and insight β€” structured boredom has real cognitive value.",
19
- "tags": ["neuroscience", "creativity", "boredom", "default mode network"],
20
- },
21
- {
22
- "type": "text", "intent": "learn",
23
- "raw": "Attention is a zero-sum resource. Every notification you respond to is borrowed from the task you were doing.",
24
- "summary": "Attention is finite and non-recoverable: each notification incurs a hidden switching cost far larger than the interruption itself.",
25
- "tags": ["focus", "productivity", "attention", "deep work"],
26
- },
27
- {
28
- "type": "text", "intent": "learn",
29
- "raw": "Compounding applies to knowledge too. Learning something adjacent to what you already know is faster than learning something unrelated.",
30
- "summary": "Knowledge compounds: learning adjacent concepts is faster because existing mental models reduce the cognitive load of integrating new information.",
31
- "tags": ["learning", "compounding", "mental models", "knowledge"],
32
- },
33
- {
34
- "type": "text", "intent": "learn",
35
- "raw": "The Feynman technique: explain a concept as if teaching a child. Where you stumble is exactly where your understanding has gaps.",
36
- "summary": "Feynman technique: teach a concept in simple terms to surface gaps β€” where explanation breaks down reveals exactly what you don't understand yet.",
37
- "tags": ["learning", "Feynman", "mental models", "teaching"],
38
- },
39
- {
40
- "type": "text", "intent": "learn",
41
- "raw": "Interleaved practice β€” mixing different problem types in one session β€” beats blocked practice for long-term retention despite feeling harder.",
42
- "summary": "Interleaved practice (mixing problem types) produces better long-term retention than blocked practice, even though it feels more difficult during learning.",
43
- "tags": ["learning", "practice", "retention", "cognitive science"],
44
- },
45
 
46
- # ── AI / LLMs ─────────────────────────────────────────────────────────────
47
- {
48
- "type": "text", "intent": "learn",
49
- "raw": "RAG grounds LLM outputs in external documents, reducing hallucination and allowing knowledge to be updated without retraining.",
50
- "summary": "RAG reduces LLM hallucination by retrieving relevant documents at inference time, grounding responses in external, updateable knowledge rather than frozen weights.",
51
- "tags": ["RAG", "LLM", "retrieval", "AI engineering"],
52
- },
53
  {
 
54
  "type": "text", "intent": "learn",
55
- "raw": "Mixture of Experts routes each token to a subset of expert FFN layers, keeping compute constant while scaling parameters.",
56
- "summary": "MoE models route each token to a small subset of expert layers, allowing parameter scaling without proportional compute increase.",
57
- "tags": ["MoE", "LLM architecture", "efficiency", "AI"],
58
- },
59
- {
 
 
 
 
 
 
 
 
 
60
  "type": "text", "intent": "learn",
61
- "raw": "Quantization reduces model weight precision (FP16 β†’ INT4) with minimal accuracy loss, enabling much larger models to fit in consumer VRAM.",
62
- "summary": "Quantization (FP16β†’INT4) cuts model memory footprint by 4x with minimal accuracy loss, making large models runnable on consumer hardware.",
63
- "tags": ["quantization", "local LLM", "VRAM", "efficiency"],
64
- },
65
- {
66
- "type": "image", "intent": "learn",
67
- "raw": None,
68
- "summary": "Transformer attention mechanism diagram: queries, keys, and values compute scaled dot-product attention to weight which tokens to attend to.",
69
- "tags": ["transformers", "attention", "deep learning", "LLM internals"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  },
 
 
 
71
  {
 
72
  "type": "link", "intent": "learn",
73
  "source_url": "https://karpathy.github.io/2019/04/25/recipe/",
 
74
  "raw": "https://karpathy.github.io/2019/04/25/recipe/",
75
- "summary": "Karpathy's neural net training recipe: start simple, overfit a single batch first, add complexity incrementally, visualize everything.",
76
- "tags": ["Karpathy", "deep learning", "training", "debugging"],
77
- },
78
- {
 
 
 
 
 
 
 
 
79
  "type": "text", "intent": "learn",
80
- "raw": "Constitutional AI (CAI) trains models to critique and revise their own outputs against a set of principles, reducing the need for human-labelled harm examples.",
81
- "summary": "Constitutional AI trains models to self-critique against explicit principles, reducing reliance on human-labelled harmful output examples in RLHF.",
82
- "tags": ["RLHF", "alignment", "constitutional AI", "Anthropic"],
83
- },
84
- {
85
- "type": "text", "intent": "learn",
86
- "raw": "Context length scaling in LLMs is limited by quadratic attention complexity. Sliding window, sparse, and linear attention mechanisms trade some quality for longer context.",
87
- "summary": "LLM context length is limited by O(nΒ²) attention complexity; sliding window, sparse, and linear attention trade off some quality for scalable context.",
88
- "tags": ["LLM", "context window", "attention", "efficiency"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  },
90
 
91
- # ── PKM / second brain ─────────────────────────────────────────────────────
 
92
  {
 
93
  "type": "text", "intent": "learn",
94
- "raw": "Second brain principle: your brain is for having ideas, not storing them. Offload facts and references so you can focus on synthesis.",
95
- "summary": "The second brain principle: offload fact storage to an external system so cognitive resources are freed for synthesis and connection-making.",
96
- "tags": ["PKM", "second brain", "productivity", "knowledge management"],
97
- },
98
- {
 
 
 
 
 
 
 
 
 
99
  "type": "link", "intent": "learn",
100
- "source_url": "https://www.lesswrong.com/posts/RcZCwxFiZzE6X7nsv/what-is-rationality",
101
- "raw": "https://www.lesswrong.com/posts/RcZCwxFiZzE6X7nsv/what-is-rationality",
102
- "summary": "Rationality is about having beliefs that accurately reflect reality (epistemic) and taking actions that best achieve your goals given those beliefs (instrumental).",
103
- "tags": ["rationality", "epistemics", "decision making", "LessWrong"],
104
- },
105
- {
106
- "type": "link", "intent": "act",
107
- "source_url": "https://obsidian.md/plugins",
108
- "raw": "https://obsidian.md/plugins",
109
- "summary": "Explore Obsidian plugin ecosystem β€” particularly Dataview for querying notes as a database and Smart Connections for semantic similarity.",
110
- "tags": ["Obsidian", "PKM", "plugins", "knowledge management"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  },
 
 
 
112
  {
 
113
  "type": "text", "intent": "learn",
114
- "raw": "Evergreen notes are atomic, concept-level notes that evolve over time rather than being tied to a source or date. They form a graph of ideas.",
115
- "summary": "Evergreen notes are atomic, concept-level notes designed to evolve over time β€” they form a living, interconnected graph of understanding rather than a archive.",
116
- "tags": ["PKM", "evergreen notes", "Zettelkasten", "writing"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
  },
118
 
119
- # ── Productivity & focus ───────────────────────────────────────────────────
 
120
  {
 
121
  "type": "text", "intent": "learn",
122
- "raw": "Deep work (Cal Newport): the ability to focus without distraction on cognitively demanding tasks is becoming rare and simultaneously more valuable.",
123
- "summary": "Deep work β€” distraction-free, cognitively demanding focus β€” is becoming rarer as it becomes more economically valuable in an age of constant connectivity.",
124
- "tags": ["deep work", "focus", "Cal Newport", "productivity"],
125
- },
126
- {
 
 
 
 
 
 
 
 
 
127
  "type": "text", "intent": "learn",
128
- "raw": "Time blocking dedicates fixed calendar slots to specific work categories, preventing reactive task-switching and protecting time for deep work.",
129
- "summary": "Time blocking allocates fixed calendar slots to work categories, structuring the day to protect deep work time against reactive task-switching.",
130
- "tags": ["productivity", "time blocking", "scheduling", "deep work"],
131
- },
132
- {
133
- "type": "text", "intent": "act",
134
- "raw": "Implement a weekly review: 30 minutes every Sunday to process inbox, review goals, and plan the coming week.",
135
- "summary": "Schedule a 30-minute weekly review every Sunday to process inbox, review current goals, and plan the coming week intentionally.",
136
- "tags": ["productivity", "weekly review", "GTD", "habits"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  },
138
 
139
- # ── Engineering / tooling ──────────────────────────────────────────────────
140
- {
141
- "type": "text", "intent": "reference",
142
- "raw": "ffmpeg convert video to gif: ffmpeg -i input.mp4 -vf 'fps=10,scale=640:-1' -loop 0 output.gif",
143
- "summary": "ffmpeg: convert MP4 to GIF at 10fps, 640px wide: `ffmpeg -i input.mp4 -vf 'fps=10,scale=640:-1' -loop 0 output.gif`",
144
- "tags": ["ffmpeg", "CLI", "video", "reference"],
145
- },
146
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
147
  "type": "text", "intent": "reference",
148
- "raw": "SQLite FTS5: CREATE VIRTUAL TABLE t USING fts5(content); SELECT * FROM t WHERE t MATCH 'query';",
149
- "summary": "SQLite FTS5 quick reference: create with `CREATE VIRTUAL TABLE t USING fts5(content)`, query with `SELECT * FROM t WHERE t MATCH 'query'`.",
150
- "tags": ["SQLite", "FTS", "search", "database"],
151
- },
152
- {
153
- "type": "image", "intent": "reference",
154
- "raw": None,
155
- "summary": "Git rebase interactive cheatsheet β€” pick, squash, fixup, reword, drop commands with short descriptions of each operation.",
156
- "tags": ["git", "rebase", "cheatsheet", "reference"],
157
- },
158
- {
159
- "type": "link", "intent": "act",
160
- "source_url": "https://github.com/simonw/llm",
161
- "raw": "https://github.com/simonw/llm",
162
- "summary": "Simon Willison's `llm` CLI: run prompts against local and cloud models from the terminal, with plugins for Ollama and LM Studio.",
163
- "tags": ["CLI", "LLM", "tooling", "Python"],
164
- },
165
- {
166
- "type": "text", "intent": "act",
167
- "raw": "Try quantized Mistral 7B in LM Studio β€” reportedly faster than Gemma on Apple Silicon due to different attention pattern.",
168
- "summary": "Evaluate quantized Mistral 7B in LM Studio for speed vs Gemma on Apple Silicon β€” different attention patterns may favour Mistral.",
169
- "tags": ["LM Studio", "Mistral", "Apple Silicon", "benchmarking"],
170
  },
171
  {
 
172
  "type": "text", "intent": "act",
173
- "raw": "Set up Tailscale on iPhone so the local KB is accessible from anywhere without exposing it publicly.",
174
- "summary": "Configure Tailscale on iPhone to access the local Mycelium server from any device without a public server.",
175
- "tags": ["Tailscale", "networking", "privacy", "mobile"],
176
- },
177
- {
178
- "type": "text", "intent": "learn",
179
- "raw": "CRDT (conflict-free replicated data type) allows distributed nodes to update shared state independently and merge without coordination.",
180
- "summary": "CRDTs allow distributed systems to update shared state independently and merge without coordination, enabling conflict-free collaboration.",
181
- "tags": ["CRDT", "distributed systems", "sync", "data structures"],
182
- },
183
- {
184
- "type": "text", "intent": "learn",
185
- "raw": "Event sourcing stores every state change as an immutable event rather than the current state, enabling time travel, audit logs, and replay.",
186
- "summary": "Event sourcing stores each state change as an immutable event, enabling time-travel debugging, full audit logs, and state reconstruction by replaying events.",
187
- "tags": ["event sourcing", "architecture", "distributed systems", "audit"],
188
  },
189
 
190
- # ── Health & wellbeing ─────────────────────────────────────────────────────
 
191
  {
 
192
  "type": "text", "intent": "learn",
193
- "raw": "Sleep is the single most effective thing you can do to enhance memory consolidation. The hippocampus replays memories during slow-wave sleep.",
194
- "summary": "Sleep is the most effective memory consolidation tool: the hippocampus replays and transfers memories to cortex during slow-wave sleep.",
195
- "tags": ["sleep", "memory", "neuroscience", "health"],
196
- },
197
- {
 
 
 
 
 
 
 
 
 
198
  "type": "text", "intent": "learn",
199
- "raw": "Deliberate cold exposure (ice bath or cold shower) triggers norepinephrine release, improving mood, focus, and resilience with 2-3 sessions per week.",
200
- "summary": "Deliberate cold exposure triggers norepinephrine release, improving mood and focus β€” 2-3 sessions per week, 2-3 minutes each is the studied protocol.",
201
- "tags": ["cold exposure", "norepinephrine", "focus", "health"],
202
- },
203
- {
204
- "type": "text", "intent": "act",
205
- "raw": "Track protein intake for 2 weeks β€” aiming for 0.8g/lb body weight to support cognitive function and lean mass.",
206
- "summary": "Track daily protein intake for 2 weeks, targeting 0.8g per pound of body weight to support cognition and lean mass maintenance.",
207
- "tags": ["nutrition", "protein", "health", "tracking"],
208
- },
209
-
210
- # ── Ephemeral ─────────────────────────────────────────────────────────────
211
- {
212
- "type": "text", "intent": "ephemeral",
213
- "raw": "lol this dog just ran into a glass door on a reel",
214
- "summary": "Funny reel of a dog running into a glass door.",
215
- "tags": ["funny"],
216
- },
217
- {
 
 
 
 
 
 
 
 
 
 
 
218
  "type": "link", "intent": "ephemeral",
219
  "source_url": "https://neal.fun/deep-sea/",
 
220
  "raw": "https://neal.fun/deep-sea/",
221
- "summary": "Interactive deep sea depth visualisation β€” scroll through ocean depth with creatures at each level.",
222
- "tags": ["fun", "ocean", "interactive"],
223
- },
224
- {
225
- "type": "text", "intent": "ephemeral",
226
- "raw": "The vibes on the terrace this evening were immaculate. Perfect weather for doing nothing.",
227
- "summary": "A pleasant evening on the terrace β€” noted for the feeling, not the information.",
228
- "tags": ["life", "mood"],
229
- },
230
-
231
- # ── Reading list ───────────────────────────────────────────────────────────
232
- {
233
- "type": "text", "intent": "act",
234
- "raw": "Read Thinking Fast and Slow by Kahneman β€” covers System 1 / System 2 and cognitive biases relevant to decision making.",
235
- "summary": "Read Kahneman's Thinking Fast and Slow for a grounded account of System 1/2 thinking and the cognitive biases that distort decisions.",
236
- "tags": ["books", "reading list", "decision making", "cognitive bias"],
237
- },
238
- {
239
- "type": "text", "intent": "act",
240
- "raw": "Read Attention Is All You Need paper β€” I keep referencing transformers without having read the original.",
241
- "summary": "Read the original Attention Is All You Need paper to build a grounded understanding of transformer architecture.",
242
- "tags": ["transformers", "paper", "deep learning", "reading list"],
243
- },
244
- {
245
- "type": "text", "intent": "act",
246
- "raw": "Read The Body Keeps the Score β€” understanding trauma and the nervous system seems foundational for a lot of adjacent topics.",
247
- "summary": "Read The Body Keeps the Score for a foundational understanding of trauma, nervous system regulation, and somatic experience.",
248
- "tags": ["books", "reading list", "neuroscience", "health"],
249
  },
250
  ]
251
 
252
 
253
  def run():
254
- existing_raws = {c["raw"] for c in get_captures(limit=1000) if c.get("raw")}
255
  all_embs = get_all_embeddings()
256
  inserted = []
257
 
258
  for s in SEEDS:
259
- raw = s.get("raw") or s.get("source_url") or s.get("summary")
260
- if raw in existing_raws:
261
- print(f" skip (exists): {s['summary'][:60]}")
262
- continue
263
 
264
- cid = save_capture(
265
- type=s["type"],
266
- raw=s.get("raw"),
267
- source_url=s.get("source_url"),
268
- file_path=None,
269
- )
270
-
271
- emb = embed(s["summary"])
272
- related = find_related(emb, all_embs, exclude_id=cid)
273
- update_capture(cid, s["summary"], s["tags"], s["intent"], emb, related)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
274
 
275
  if emb:
276
  all_embs.append((cid, emb))
277
- existing_raws.add(raw)
278
-
279
- print(f" [{s['intent']:10}] {s['summary'][:70]}")
280
  inserted.append(cid)
 
281
 
282
- print(f"\nDone. Inserted {len(inserted)} captures.")
283
-
284
- # Backfill related_ids now that all embeddings exist
285
- print("\nBackfilling related_ids across all captures…")
286
- all_embs = get_all_embeddings()
287
- for c in get_captures(limit=1000):
288
- emb_row = next((e for cid, e in all_embs if cid == c["id"]), None)
289
- if not emb_row:
290
- continue
291
- related = find_related(emb_row, all_embs, exclude_id=c["id"])
292
- update_capture(c["id"], c["summary"], c.get("tags", []), c.get("intent"), emb_row, related)
293
 
294
- print("Backfill complete.")
295
 
296
 
297
  if __name__ == "__main__":
 
1
+ """Seed the KB with Ajit's engineering captures spread across the past week."""
2
+ import sqlite3
3
+ from db import init_db, get_all_embeddings, DB_PATH
4
  from lm import embed, find_related
5
 
6
  init_db()
7
 
8
+ # ── Seed content ───────────────────────────────────────────────────────────────
9
+ # Spread across June 7–13, 2026. All summaries/claims/questions are pre-written
10
+ # so seeding is deterministic and doesn't require the LLM.
11
+
12
  SEEDS = [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
 
14
+ # ── June 7 ─────────────────────────────────────────────────────────────────
15
+
 
 
 
 
 
16
  {
17
+ "date": "2026-06-07 09:12",
18
  "type": "text", "intent": "learn",
19
+ "title": "KV Cache is Why Context Has a Cost",
20
+ "raw": "Every token in an LLM's context window requires a KV cache entry. That cache grows O(n) in memory and causes O(nΒ²) compute in attention. 128k context sounds great until you're paying for it.",
21
+ "your_take": "This is why RAG beats long-context even when models support 128k β€” retrieval is cheaper than attending over everything",
22
+ "summary": "KV cache memory grows linearly with context length while attention compute grows quadratically β€” making long-context expensive even when technically supported.",
23
+ "claims": [
24
+ "KV cache memory scales O(n) with context length",
25
+ "Attention compute scales O(nΒ²), making 128k context very expensive",
26
+ "RAG is often cheaper than long-context because retrieval avoids full attention over all tokens",
27
+ ],
28
+ "tags": ["LLM", "KV cache", "attention", "inference", "AI engineering"],
29
+ "recall_question": "What are the two separate scaling costs of increasing LLM context length according to this note",
30
+ },
31
+ {
32
+ "date": "2026-06-07 14:33",
33
  "type": "text", "intent": "learn",
34
+ "title": "Embedding Models Are Not Interchangeable",
35
+ "raw": "Switching embedding models without re-embedding your entire corpus is silent data corruption. Cosine similarity is only meaningful within the same model family. BGE and MiniLM scores are not comparable.",
36
+ "your_take": "Got burned by this building Mycelium β€” MiniLM embeddings in prod, switched models, search was quietly broken for days",
37
+ "summary": "Embedding model outputs are not interchangeable β€” cosine similarity scores are only meaningful within a single model's vector space; mixing models silently corrupts retrieval.",
38
+ "claims": [
39
+ "Cosine similarity scores are only comparable within the same embedding model",
40
+ "Switching models without re-embedding corrupts semantic search silently",
41
+ "There is no way to 'convert' embeddings between model families",
42
+ ],
43
+ "tags": ["embeddings", "vector search", "AI engineering", "retrieval"],
44
+ "recall_question": "Why does switching embedding models without re-embedding corrupt semantic search according to this note",
45
+ },
46
+ {
47
+ "date": "2026-06-07 17:05",
48
+ "type": "link", "intent": "reference",
49
+ "source_url": "https://huggingface.co/spaces/mteb/leaderboard",
50
+ "title": "MTEB Leaderboard",
51
+ "raw": "https://huggingface.co/spaces/mteb/leaderboard",
52
+ "summary": "MTEB (Massive Text Embedding Benchmark) leaderboard ranks embedding models across retrieval, clustering, classification, and semantic similarity tasks β€” the definitive reference for choosing an embedding model.",
53
+ "claims": [
54
+ "MTEB covers retrieval, clustering, classification and semantic similarity tasks",
55
+ "BGE-base-en-v1.5 consistently ranks near the top for retrieval tasks at its size",
56
+ ],
57
+ "tags": ["MTEB", "embeddings", "benchmarks", "reference", "AI"],
58
+ "recall_question": "What four task types does the MTEB benchmark cover for evaluating embedding models",
59
  },
60
+
61
+ # ── June 8 ─────────────────────────────────────────────────────────────────
62
+
63
  {
64
+ "date": "2026-06-08 08:45",
65
  "type": "link", "intent": "learn",
66
  "source_url": "https://karpathy.github.io/2019/04/25/recipe/",
67
+ "title": "Karpathy's Neural Net Training Recipe",
68
  "raw": "https://karpathy.github.io/2019/04/25/recipe/",
69
+ "your_take": "The 'overfit one batch first' trick alone has saved me hours of debugging",
70
+ "summary": "Karpathy's systematic recipe for training neural networks: start simple, overfit a single batch before scaling, add complexity incrementally, visualize loss curves obsessively.",
71
+ "claims": [
72
+ "Overfit a single batch first to verify the model can learn before scaling",
73
+ "Add one variable at a time β€” changing multiple things simultaneously makes debugging impossible",
74
+ "Visualize everything: loss curves, activations, gradients β€” bugs hide in plain sight",
75
+ ],
76
+ "tags": ["Karpathy", "deep learning", "debugging", "training", "recipe"],
77
+ "recall_question": "What is the first step in Karpathy's neural net debugging recipe and why does it work",
78
+ },
79
+ {
80
+ "date": "2026-06-08 11:22",
81
  "type": "text", "intent": "learn",
82
+ "title": "Prototype to Production Gap",
83
+ "raw": "The gap between a working prototype and production-ready software is mostly edge cases and operational concerns, not features. A demo that works 80% of the time is worthless in prod.",
84
+ "your_take": "Every side project I've shipped taught me this the hard way β€” the last 20% takes longer than the first 80%",
85
+ "summary": "The prototype-to-production gap is dominated by edge case handling and operational concerns, not feature work β€” a system working 80% of the time is unusable in production.",
86
+ "claims": [
87
+ "Production failures come from the 20% of inputs a prototype wasn't tested on",
88
+ "Operational concerns (monitoring, error handling, retries) add more time than features",
89
+ "A demo working 80% of the time is a demo, not a product",
90
+ ],
91
+ "tags": ["engineering", "production", "software development", "shipping"],
92
+ "recall_question": "What does this note say dominates the gap between prototype and production software",
93
+ },
94
+ {
95
+ "date": "2026-06-08 16:40",
96
+ "type": "text", "intent": "act",
97
+ "title": "Evaluate BGE-base for Embedding Switch",
98
+ "raw": "Try BAAI/bge-base-en-v1.5 as the embedding model β€” MTEB shows it beats MiniLM by ~10 points on retrieval. Same 768-dim output so the similarity threshold can stay roughly the same.",
99
+ "your_take": "No trust_remote_code needed unlike nomic β€” cleaner for production",
100
+ "summary": "Switch to BAAI/bge-base-en-v1.5 for embeddings β€” outperforms all-MiniLM-L6-v2 by ~10 MTEB points on retrieval tasks at the same 768-dim output size.",
101
+ "claims": [
102
+ "BGE-base-en-v1.5 scores ~10 MTEB points higher than MiniLM on retrieval",
103
+ "Both output 768-dim vectors so the similarity threshold needs minimal adjustment",
104
+ ],
105
+ "tags": ["embeddings", "BGE", "retrieval", "AI engineering"],
106
+ "recall_question": "What specific advantage does BGE-base-en-v1.5 have over MiniLM according to this note",
107
  },
108
 
109
+ # ── June 9 ─────────────────────────────────────────────────────────────────
110
+
111
  {
112
+ "date": "2026-06-09 09:55",
113
  "type": "text", "intent": "learn",
114
+ "title": "Event Sourcing β€” What You Give Up",
115
+ "raw": "Event sourcing gives you audit logs and time-travel debugging for free. The tradeoff: read models are eventually consistent and you need projections for every query pattern. Works well when writes dominate reads and history matters.",
116
+ "your_take": "Worth it for financial systems or anything with compliance requirements β€” overkill for most CRUD apps",
117
+ "summary": "Event sourcing provides free audit logs and time-travel debugging at the cost of eventual consistency and requiring projections per query pattern β€” best when write history matters more than read simplicity.",
118
+ "claims": [
119
+ "Event sourcing makes audit logs and time-travel debugging free by design",
120
+ "Read models are eventually consistent β€” queries need pre-built projections",
121
+ "Best fit is systems where write history matters: finance, compliance, collaborative editing",
122
+ ],
123
+ "tags": ["event sourcing", "system design", "architecture", "distributed systems"],
124
+ "recall_question": "What are the two main costs of using event sourcing according to this note",
125
+ },
126
+ {
127
+ "date": "2026-06-09 13:10",
128
  "type": "link", "intent": "learn",
129
+ "source_url": "https://eugeneyan.com/writing/llm-patterns/",
130
+ "title": "Eugene Yan β€” LLM Patterns for Production",
131
+ "raw": "https://eugeneyan.com/writing/llm-patterns/",
132
+ "your_take": "The evals section is the most underrated β€” everyone ships LLM apps without evals then wonders why they can't improve them",
133
+ "summary": "Eugene Yan's taxonomy of production LLM patterns: RAG, fine-tuning, caching, guardrails, and evals β€” the evals pattern is most often skipped but most critical for iterating on quality.",
134
+ "claims": [
135
+ "Production LLM systems need five patterns: RAG, fine-tuning, caching, guardrails, evals",
136
+ "Evals are the most skipped but most critical pattern for improving LLM quality over time",
137
+ "Caching semantic-similar queries can cut LLM costs by 30-50% in high-traffic apps",
138
+ ],
139
+ "tags": ["LLM", "production", "RAG", "evals", "AI engineering"],
140
+ "recall_question": "Which LLM production pattern does Eugene Yan say is most often skipped and why is it critical",
141
+ },
142
+ {
143
+ "date": "2026-06-09 18:30",
144
+ "type": "text", "intent": "reference",
145
+ "title": "SQLite FTS5 Quick Reference",
146
+ "raw": "SQLite FTS5 setup: CREATE VIRTUAL TABLE search USING fts5(content, title); INSERT INTO search SELECT content, title FROM captures; SELECT * FROM search WHERE search MATCH 'query' ORDER BY rank;",
147
+ "summary": "SQLite FTS5 quick reference: virtual table with fts5, populated via INSERT SELECT, queried with MATCH and sorted by rank β€” no external search engine needed for keyword search.",
148
+ "claims": [
149
+ "FTS5 requires a virtual table β€” regular columns are not full-text indexed",
150
+ "The rank column in FTS5 queries sorts by relevance automatically",
151
+ ],
152
+ "tags": ["SQLite", "FTS5", "search", "reference", "database"],
153
+ "recall_question": "What SQL keyword does SQLite FTS5 use for full-text queries and how is relevance handled",
154
  },
155
+
156
+ # ── June 10 ────────────────────────────────────────────────────────────────
157
+
158
  {
159
+ "date": "2026-06-10 09:00",
160
  "type": "text", "intent": "learn",
161
+ "title": "Show the Problem Before the Solution",
162
+ "raw": "The best product demos show the problem for 30 seconds before showing the solution. Watching someone struggle with the thing you solved creates instant empathy and makes the demo memorable.",
163
+ "your_take": "Every founder talk I've liked does this β€” the ones that skip to features feel like ads",
164
+ "summary": "Effective product demos open with 30 seconds of the problem, not the solution β€” the audience needs to feel the pain before the fix can be satisfying.",
165
+ "claims": [
166
+ "30 seconds of problem context makes a demo more memorable than 3 minutes of features",
167
+ "Skipping to features means the audience has no emotional frame for why they should care",
168
+ ],
169
+ "tags": ["demos", "product", "storytelling", "communication"],
170
+ "recall_question": "What does this note say effective product demos show in the first 30 seconds and why",
171
+ },
172
+ {
173
+ "date": "2026-06-10 11:45",
174
+ "type": "link", "intent": "learn",
175
+ "source_url": "https://simonwillison.net/2024/Apr/17/ai-for-data-journalism/",
176
+ "title": "Simon Willison β€” Practical AI Use",
177
+ "raw": "https://simonwillison.net/2024/Apr/17/ai-for-data-journalism/",
178
+ "your_take": "Simon is the best writer on practical AI use β€” he actually builds things and writes about what breaks",
179
+ "summary": "Simon Willison on using LLMs practically: the value is in automating the tedious middle 80% of a task, not replacing the judgment at the edges β€” human-in-the-loop remains essential.",
180
+ "claims": [
181
+ "LLMs are best at automating tedious middle work, not replacing edge-case judgment",
182
+ "Human-in-the-loop is not a limitation but a feature for high-stakes AI applications",
183
+ "The practical value of AI tools comes from combining them with domain expertise",
184
+ ],
185
+ "tags": ["Simon Willison", "LLM", "practical AI", "journalism", "human-in-the-loop"],
186
+ "recall_question": "What does Simon Willison say LLMs are best at automating and what should humans still handle",
187
+ },
188
+ {
189
+ "date": "2026-06-10 20:15",
190
+ "type": "text", "intent": "ephemeral",
191
+ "title": "ZeroGPU Debug War Story",
192
+ "raw": "Spent 3 hours debugging a ZeroGPU cold start failure. Traced it through worker init, CUDA init, spaces wrappers. Turned out to be a module-level import of Qwen2_5_VLForConditionalGeneration that failed silently and put the Space in a restart loop. Fix was moving the import inside the function with a try/except. 2 lines.",
193
+ "summary": "3-hour ZeroGPU debug traced to a module-level import failing silently and causing a restart loop β€” fixed by moving the import inside the function with a try/except fallback.",
194
+ "claims": [
195
+ "Module-level imports that fail put HF Spaces in a silent restart loop",
196
+ ],
197
+ "tags": ["ZeroGPU", "debugging", "HF Spaces", "Python"],
198
+ "recall_question": "What caused the ZeroGPU restart loop in this debugging session",
199
  },
200
 
201
+ # ── June 11 ────────────────────────────────────────────────────────────────
202
+
203
  {
204
+ "date": "2026-06-11 08:30",
205
  "type": "text", "intent": "learn",
206
+ "title": "INT4 Quantization Tradeoffs",
207
+ "raw": "INT4 quantization cuts model size by 4x with roughly 2-3% accuracy loss on most benchmarks. The hidden cost: latency variance. Some tokens take much longer to decode because of dequantization overhead, making p99 latency worse than median.",
208
+ "your_take": "vLLM handles the variance better than naive implementations β€” worth benchmarking before assuming quantization is drop-in",
209
+ "summary": "INT4 quantization reduces model size 4x with ~2-3% accuracy loss, but introduces latency variance from dequantization overhead β€” p99 latency can be much worse than median.",
210
+ "claims": [
211
+ "INT4 quantization cuts model size 4x with only 2-3% accuracy loss on most tasks",
212
+ "Dequantization overhead causes latency variance β€” p99 latency is much worse than median",
213
+ "vLLM's implementation handles quantization variance better than naive approaches",
214
+ ],
215
+ "tags": ["quantization", "INT4", "LLM", "inference", "latency"],
216
+ "recall_question": "What hidden latency cost does INT4 quantization introduce that median benchmarks miss",
217
+ },
218
+ {
219
+ "date": "2026-06-11 10:20",
220
  "type": "text", "intent": "learn",
221
+ "title": "Memory is Reconstructive Not Reproductive",
222
+ "raw": "Spaced repetition works because memory is reconstructive, not reproductive. You're not playing back a recording β€” you're rebuilding the memory each time from fragments, which strengthens the neural pathway. Active recall beats re-reading because it forces reconstruction.",
223
+ "your_take": "This is the neuroscience behind why Mycelium's recall questions work β€” you're exercising reconstruction, not recognition",
224
+ "summary": "Memory is reconstructive: each recall rebuilds the memory from fragments rather than playing it back, which is why active recall strengthens retention more than passive re-reading.",
225
+ "claims": [
226
+ "Memory is reconstructive β€” each recall rebuilds rather than replays the memory",
227
+ "Reconstruction during recall strengthens the neural pathway more than passive review",
228
+ "Active recall beats re-reading because recognition is much easier than reconstruction",
229
+ ],
230
+ "tags": ["memory", "neuroscience", "spaced repetition", "learning", "active recall"],
231
+ "recall_question": "Why does active recall strengthen memory more than passive re-reading according to the reconstructive memory model",
232
+ },
233
+ {
234
+ "date": "2026-06-11 15:00",
235
+ "type": "link", "intent": "act",
236
+ "source_url": "https://github.com/vllm-project/vllm",
237
+ "title": "vLLM β€” Production LLM Inference",
238
+ "raw": "https://github.com/vllm-project/vllm",
239
+ "summary": "vLLM is an open-source LLM inference engine optimized for throughput with continuous batching, PagedAttention for KV cache management, and support for quantized models β€” the standard for self-hosted production inference.",
240
+ "claims": [
241
+ "PagedAttention manages KV cache like virtual memory β€” eliminates fragmentation",
242
+ "Continuous batching increases GPU utilization by mixing requests of different lengths",
243
+ ],
244
+ "tags": ["vLLM", "inference", "production", "LLM", "open source"],
245
+ "recall_question": "What two techniques does vLLM use to improve throughput compared to naive inference servers",
246
  },
247
 
248
+ # ── June 12 ────────────────────────────────────────────────────────────────
249
+
 
 
 
 
 
250
  {
251
+ "date": "2026-06-12 09:10",
252
+ "type": "text", "intent": "learn",
253
+ "title": "Solo Building Means Ruthless Scoping",
254
+ "raw": "Building a product solo means you're PM, designer, engineer, and support simultaneously. The only survival strategy: scope down until the core loop works end-to-end, then add. Partial features are technical debt disguised as progress.",
255
+ "your_take": "Mycelium started with 10 planned screens β€” shipped with 4 that actually work. Right call.",
256
+ "summary": "Solo product development requires ruthless scoping β€” the core loop must work end-to-end before adding features, because partial features are technical debt disguised as progress.",
257
+ "claims": [
258
+ "Solo builders must play PM, designer, engineer and support simultaneously",
259
+ "The core loop working end-to-end is the only valid milestone when building alone",
260
+ "Partial features create more debt than value β€” they commit future-you to maintenance without delivering user value",
261
+ ],
262
+ "tags": ["solo building", "product", "scoping", "indie hacking", "engineering"],
263
+ "recall_question": "What does this note say is the only valid milestone for solo builders before adding new features",
264
+ },
265
+ {
266
+ "date": "2026-06-12 11:30",
267
  "type": "text", "intent": "reference",
268
+ "title": "ffmpeg: MP4 to GIF",
269
+ "raw": "ffmpeg -i input.mp4 -vf 'fps=10,scale=640:-1:flags=lanczos' -loop 0 output.gif",
270
+ "summary": "ffmpeg command to convert MP4 to GIF: 10fps, 640px wide with Lanczos scaling, infinite loop. Add -ss 00:00:02 -t 5 to trim to 5 seconds starting at 2s.",
271
+ "claims": [
272
+ "Lanczos filter (-flags=lanczos) produces sharper GIFs than the default bilinear filter",
273
+ ],
274
+ "tags": ["ffmpeg", "GIF", "video", "CLI", "reference"],
275
+ "recall_question": "What ffmpeg filter flag produces sharper GIF output than the default according to this note",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
276
  },
277
  {
278
+ "date": "2026-06-12 16:45",
279
  "type": "text", "intent": "act",
280
+ "title": "Read The Mom Test",
281
+ "raw": "Read 'The Mom Test' by Rob Fitzpatrick before doing any more user interviews. Every question I ask is leading β€” I'm confirming what I want to hear, not learning what users actually do.",
282
+ "summary": "Read 'The Mom Test' to fix leading user interview questions β€” current interviews confirm existing assumptions rather than revealing actual user behavior.",
283
+ "claims": [
284
+ "Leading questions in user interviews produce false validation β€” you hear what you want to hear",
285
+ ],
286
+ "tags": ["user research", "product", "reading", "interviews"],
287
+ "recall_question": "What problem with the current user interview approach does this note identify",
 
 
 
 
 
 
 
288
  },
289
 
290
+ # ── June 13 ────────────────────────────────────────────────────────────────
291
+
292
  {
293
+ "date": "2026-06-13 08:00",
294
  "type": "text", "intent": "learn",
295
+ "title": "RAG vs Fine-Tuning Decision Frame",
296
+ "raw": "RAG when: knowledge changes frequently, sources need to be citable, context fits in the window. Fine-tune when: behavior/style needs to change, task is narrow and stable, latency matters more than accuracy. Most apps should try RAG first β€” it's reversible.",
297
+ "your_take": "Fine-tuning a model for knowledge is almost always the wrong call β€” that's what RAG is for",
298
+ "summary": "RAG is preferred for dynamic, citable knowledge; fine-tuning for stable narrow tasks requiring style/behavior changes β€” most applications should start with RAG because it's reversible.",
299
+ "claims": [
300
+ "RAG is better when knowledge changes frequently or sources need to be citable",
301
+ "Fine-tuning is better when behavior or style must change for a narrow stable task",
302
+ "RAG should be the default first attempt β€” it's reversible, fine-tuning is not",
303
+ ],
304
+ "tags": ["RAG", "fine-tuning", "LLM", "AI engineering", "system design"],
305
+ "recall_question": "What does this note say is the key reason RAG should be tried before fine-tuning for most applications",
306
+ },
307
+ {
308
+ "date": "2026-06-13 10:30",
309
  "type": "text", "intent": "learn",
310
+ "title": "Second Brain Principle",
311
+ "raw": "Your brain is optimized for having ideas and making connections, not for storage and retrieval. Every fact you try to hold in working memory is competing with the thing you're actually trying to think about. Offload storage, reclaim synthesis.",
312
+ "your_take": "This is the entire premise of Mycelium β€” capture fast, think later, let the system surface what matters",
313
+ "summary": "The second brain principle: offload fact storage to an external system to free cognitive resources for synthesis and creative connection-making β€” storage and retrieval are not what brains are for.",
314
+ "claims": [
315
+ "Working memory used for storage reduces capacity available for active thinking",
316
+ "Offloading facts to external systems frees cognitive resources for synthesis",
317
+ "Brains are optimized for generating and connecting ideas, not for storage and retrieval",
318
+ ],
319
+ "tags": ["second brain", "PKM", "cognition", "productivity", "knowledge management"],
320
+ "recall_question": "What cognitive function does this note say is the brain's actual strength versus what it should offload",
321
+ },
322
+ {
323
+ "date": "2026-06-13 14:20",
324
+ "type": "link", "intent": "learn",
325
+ "source_url": "https://www.benkuhn.net/speed/",
326
+ "title": "Ben Kuhn β€” In Praise of Fast Things",
327
+ "raw": "https://www.benkuhn.net/speed/",
328
+ "your_take": "The 'fast tools change how you think' argument is underrated β€” I notice this with LLM autocomplete vs slow chat interfaces",
329
+ "summary": "Ben Kuhn's argument for speed in tools: fast tools don't just save time, they change how you think β€” below certain latency thresholds, you maintain flow state and explore more freely.",
330
+ "claims": [
331
+ "Fast tools change how you think, not just how quickly β€” they enable exploration that slow tools discourage",
332
+ "There are latency thresholds below which qualitatively different work becomes possible",
333
+ "A 10x faster tool often enables more than 10x the value by changing usage patterns",
334
+ ],
335
+ "tags": ["speed", "tools", "productivity", "engineering", "latency"],
336
+ "recall_question": "What does Ben Kuhn say fast tools enable beyond time savings according to this note",
337
+ },
338
+ {
339
+ "date": "2026-06-13 17:00",
340
  "type": "link", "intent": "ephemeral",
341
  "source_url": "https://neal.fun/deep-sea/",
342
+ "title": "The Deep Sea β€” Interactive Depth",
343
  "raw": "https://neal.fun/deep-sea/",
344
+ "summary": "Interactive visualization of ocean depth β€” scroll from the surface to the Mariana Trench encountering creatures at each depth level. Oddly calming.",
345
+ "claims": [],
346
+ "tags": ["fun", "ocean", "interactive", "visualization"],
347
+ "recall_question": None,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
348
  },
349
  ]
350
 
351
 
352
  def run():
353
+ print(f"Seeding {len(SEEDS)} captures into {DB_PATH}…\n")
354
  all_embs = get_all_embeddings()
355
  inserted = []
356
 
357
  for s in SEEDS:
358
+ summary = s["summary"]
359
+ emb = embed(summary)
360
+ related = find_related(emb, all_embs, exclude_id=0) # placeholder id
 
361
 
362
+ with sqlite3.connect(DB_PATH) as conn:
363
+ cur = conn.execute(
364
+ """INSERT INTO captures
365
+ (type, raw, source_url, your_take, summary, title, tags, intent,
366
+ embedding, related_ids, recall_question, claims,
367
+ created_at, reviewed)
368
+ VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,0)""",
369
+ (
370
+ s["type"],
371
+ s.get("raw"),
372
+ s.get("source_url"),
373
+ s.get("your_take"),
374
+ summary,
375
+ s.get("title"),
376
+ __import__("json").dumps(s.get("tags", [])),
377
+ s["intent"],
378
+ __import__("json").dumps(emb) if emb else None,
379
+ __import__("json").dumps([]), # will backfill below
380
+ s.get("recall_question"),
381
+ __import__("json").dumps(s.get("claims", [])),
382
+ s["date"],
383
+ ),
384
+ )
385
+ cid = cur.lastrowid
386
 
387
  if emb:
388
  all_embs.append((cid, emb))
 
 
 
389
  inserted.append(cid)
390
+ print(f" [{s['intent']:10}] {s.get('title', summary[:50])}")
391
 
392
+ # backfill related_ids now that all embeddings exist
393
+ print("\nBackfilling related_ids…")
394
+ import json
395
+ for cid, emb in [(cid, emb) for cid, emb in all_embs if cid in inserted]:
396
+ related = find_related(emb, all_embs, exclude_id=cid)
397
+ with sqlite3.connect(DB_PATH) as conn:
398
+ conn.execute("UPDATE captures SET related_ids=? WHERE id=?",
399
+ (json.dumps(related), cid))
 
 
 
400
 
401
+ print(f"\nDone β€” {len(inserted)} captures seeded with embeddings and connections.")
402
 
403
 
404
  if __name__ == "__main__":