enrich: replace 'review-pending' placeholders with curated descriptions on 28 discovered tools

forge/enrich.py

@@ -0,0 +1,79 @@
+"""Replace 'Discovered via Bright Data scraping (review-pending)' placeholders with
+real curated descriptions for the 28 well-known ML tools node-discovery surfaced.
+
+The BD-scrape pass for integration snippets + citation URLs is a separate step
+(enrich_with_examples) that burns credits meaningfully — for now, this just
+gets the live demo off placeholder text.
+
+Run:  python -m forge.enrich
+"""
+from __future__ import annotations
+
+from . import db
+
+# Curated one-liners. Specific, factual, no marketing fluff.
+DESCRIPTIONS = {
+    # frameworks
+    "accelerate": "HuggingFace library that wraps PyTorch training to add multi-GPU, mixed-precision, and DeepSpeed/FSDP support with minimal code changes.",
+    "megatron": "NVIDIA's framework for training large transformer models with tensor + pipeline parallelism. Heavy-iron, multi-node scale.",
+    "pytorch": "The dynamic-graph deep-learning framework underneath nearly everything in modern ML training.",
+    "pytorchfsdp": "PyTorch's native Fully-Sharded Data Parallel — shards parameters, gradients, and optimizer state across GPUs. Alternative to DeepSpeed ZeRO.",
+    "tensorrtmodeloptimizer": "NVIDIA library for compressing models with PTQ, sparsity, and distillation before TensorRT deployment.",
+    "tensorrtllm": "NVIDIA's runtime for fast LLM inference on its GPUs. Inference-time, not training.",
+    "transformerengine": "NVIDIA library for FP8 training/inference on Hopper+ GPUs. Drop-in replacement for transformer blocks.",
+    "transformers": "HuggingFace's model library — the de-facto interface for loading and fine-tuning open LLMs.",
+    # optimizers
+    "adagrad": "Adaptive gradient optimizer that scales the per-parameter LR by accumulated squared gradients. Mostly superseded by AdamW for LLMs.",
+    "pagedoptimizers": "bitsandbytes optimizer that pages optimizer state between GPU and CPU RAM to handle memory spikes during QLoRA training.",
+    "rmsprop": "Adaptive optimizer that divides the gradient by a running average of its magnitude. Direct ancestor of Adam.",
+    "sgd": "Stochastic gradient descent — the classical optimizer. Still competitive with proper LR + momentum, but rarely the default for LLM finetuning.",
+    "shampoo": "Second-order optimizer using Kronecker-factored preconditioning. Promising at scale but compute-heavy.",
+    # quantization
+    "4bitquantization": "Storing model weights in 4 bits. The basis of QLoRA (via bitsandbytes NF4).",
+    "doublequantization": "QLoRA technique that quantizes the quantization constants themselves, saving an extra ~0.4 bits per parameter.",
+    "fp8": "8-bit floating-point format on Hopper/Blackwell GPUs for training + inference. Used via Transformer Engine.",
+    "ptq": "Post-training quantization — quantize weights after training without retraining. AWQ and GPTQ are PTQ methods.",
+    "bitsandbytes": "The library implementing 4-bit (NF4) and 8-bit weight quantization that powers QLoRA on consumer GPUs.",
+    # schedulers
+    "cosineannealinglr": "PyTorch's cosine decay scheduler. Anneals LR from max to min over the run.",
+    "reducelronplateau": "Scheduler that drops the LR when a tracked metric stops improving.",
+    "steplr": "Scheduler that multiplies LR by gamma every N epochs. Simple staircase decay.",
+    # techniques
+    "dora": "Weight-Decomposed LoRA — separates a weight update into magnitude and direction. Drop-in upgrade to LoRA with negligible overhead.",
+    "flashattention": "Memory-efficient exact attention kernel that avoids materializing the full attention matrix. Essential at long contexts.",
+    "peft": "HuggingFace library implementing parameter-efficient finetuning methods (LoRA, QLoRA, DoRA, IA3, etc.).",
+    "relora": "Restart-LoRA — periodically merges LoRA adapters into the base and re-initializes new ones, enabling higher effective rank than vanilla LoRA.",
+    "tensorparallelism": "Sharding a single layer's matmul across GPUs along the hidden dim. Standard for large-model training/inference.",
+    "mixedprecision": "Training with bf16/fp16 weights and fp32 master weights — saves memory and speeds up matmul without losing convergence.",
+    "pruning": "Setting unimportant weights to zero to reduce model size or compute. Structured pruning is GPU-friendly.",
+}
+
+
+def run(path=db.DB_PATH):
+    conn = db.connect(path)
+    updated, missed = [], []
+    rows = conn.execute(
+        "SELECT id, canonical, name FROM nodes WHERE tags_json LIKE '%discovered%'"
+    ).fetchall()
+    for r in rows:
+        desc = DESCRIPTIONS.get(r["canonical"])
+        if desc:
+            conn.execute("UPDATE nodes SET description=? WHERE id=?", (desc, r["id"]))
+            updated.append(r["name"])
+        else:
+            missed.append((r["canonical"], r["name"]))
+    conn.commit()
+    return updated, missed
+
+
+def main():
+    updated, missed = run()
+    print(f"updated {len(updated)} discovered-tool descriptions")
+    if missed:
+        print(f"no curated description for: {missed}")
+    else:
+        print("no placeholders remain.")
+
+
+if __name__ == "__main__":
+    main()