feat: upload PAMPAr-Coder code and documentation
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- .cursorrules +226 -0
- .env.example +4 -0
- .gitattributes +5 -0
- .github/CODEOWNERS +2 -0
- .github/ISSUE_TEMPLATE/bug_report.md +33 -0
- .github/ISSUE_TEMPLATE/feature_request.md +23 -0
- .github/copilot-instructions.md +82 -0
- .github/instructions/cloud-training.instructions.md +91 -0
- .github/instructions/pampar-architecture.instructions.md +59 -0
- .github/instructions/python-ml.instructions.md +268 -0
- .github/instructions/testing-pytest.instructions.md +229 -0
- .github/pull_request_template.md +18 -0
- .github/workflows/ci.yml +32 -0
- .gitignore +103 -0
- .memoria/sesion_2026-03-16_linux-ia.md +154 -0
- .zenodo.json +70 -0
- AGENTS.md +274 -0
- CITATION.cff +105 -0
- CONTRIBUTING.md +53 -0
- LICENSE +96 -0
- PAMPAR-coder.png +3 -0
- PAMPArLLM.png +3 -0
- PLAN.md +202 -0
- README.md +365 -0
- ROADMAP.md +240 -0
- SECURITY.md +25 -0
- benchmarks/history.jsonl +4 -0
- benchmarks/humaneval_results.json +93 -0
- benchmarks/v1/efficiency_comparison.png +0 -0
- benchmarks/v1/llaves_impact.png +0 -0
- benchmarks/v1/results.json +56 -0
- benchmarks/v1/speed_comparison.png +0 -0
- benchmarks/v1/summary.png +0 -0
- brain_scan.html +42 -0
- brain_scanner_pretrain_results.txt +0 -0
- docs/APRENDIZAJE_CEREBRAL.md +184 -0
- docs/MIXED_SELECTIVITY.md +148 -0
- eval_pretrain_results.txt +0 -0
- generation_log.txt +2 -0
- logo-pampar-color.png +3 -0
- logo-pampar-sf.png +3 -0
- logo-pampar.png +3 -0
- pampar/CONCIENCIA.md +95 -0
- pampar/__init__.py +5 -0
- pampar/cli.py +175 -0
- pampar/coder/__init__.py +39 -0
- pampar/coder/v3/__init__.py +8 -0
- pampar/coder/v3/attn.py +104 -0
- pampar/coder/v3/bloques.py +31 -0
- pampar/coder/v3/config.py +256 -0
.cursorrules
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PAMPAr-Coder β AI Instructions
|
| 2 |
+
|
| 3 |
+
> ConsolidaciΓ³n de instrucciones para Cursor/Copilot en el proyecto PAMPAr-Coder.
|
| 4 |
+
> "El Linux de la IA" - LLM de cΓ³digo con arquitectura cerebral.
|
| 5 |
+
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
## Proyecto
|
| 9 |
+
|
| 10 |
+
**PAMPAr-Coder** es un LLM de cΓ³digo 1.5B-3B con arquitectura inspirada en las 52 zonas de Brodmann.
|
| 11 |
+
|
| 12 |
+
### Quick Reference
|
| 13 |
+
|
| 14 |
+
| Area | Convention |
|
| 15 |
+
|------|------------|
|
| 16 |
+
| Language | Python 3.13+ |
|
| 17 |
+
| Framework | PyTorch 2.x |
|
| 18 |
+
| Tokenizer | SentencePiece BPE β 16K vocab (modelo activo) |
|
| 19 |
+
| Testing | pytest (134 tests) |
|
| 20 |
+
| Type hints | Siempre requeridos |
|
| 21 |
+
| Docstrings | Google style |
|
| 22 |
+
| Training | local β GPU 4GB VRAM (NO cloud, NO RunPod) |
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Arquitectura Cerebral
|
| 27 |
+
|
| 28 |
+
```
|
| 29 |
+
Input β Embedding β [BloqueTerrritorial ΓN] β LM Head β Output
|
| 30 |
+
β
|
| 31 |
+
TΓ‘lamoBrodmann (LLAVES 80% + AtenciΓ³n 20%)
|
| 32 |
+
+ Conv1D causal (ventana 32 tokens)
|
| 33 |
+
β
|
| 34 |
+
βββββββββββββββββββββββ΄ββββββββββββββββββββββ
|
| 35 |
+
βΌ βΌ
|
| 36 |
+
βββββββββββββββββ βββββββββββββββββ
|
| 37 |
+
β SINTAXIS ββββ simbiosis βββββΊβ SEMΓNTICA β
|
| 38 |
+
β Zonas 1-15 β β Zonas 16-30 β
|
| 39 |
+
βββββββββββββββββ βββββββββββββββββ
|
| 40 |
+
βΌ βΌ
|
| 41 |
+
βββββββββββββββββ βββββββββββββββββ
|
| 42 |
+
β LΓGICO ββββ simbiosis βββββΊβ ESTRUCTURAL β
|
| 43 |
+
β Zonas 31-42 β β Zonas 43-52 β
|
| 44 |
+
βββββββββββββββββ βββββββββββββββββ
|
| 45 |
+
```
|
| 46 |
+
|
| 47 |
+
### Sistema LLAVES (80% del peso)
|
| 48 |
+
|
| 49 |
+
Las LLAVES son **patrones regex que clasifican tokens en zonas de Brodmann**:
|
| 50 |
+
|
| 51 |
+
```python
|
| 52 |
+
# CRΓTICO: LLAVES son regex, NUNCA entrenables
|
| 53 |
+
LLAVES = {
|
| 54 |
+
'B06_KEYWORDS_IMPORT': ['import', 'from', 'require'],
|
| 55 |
+
'B17_LITERAL_STRING': [r'".*"', r"'.*'", 'f"'],
|
| 56 |
+
'B35_IDENTIFICADOR_VAR': [r'[a-z_][a-z0-9_]*'],
|
| 57 |
+
# ... 52 zonas total
|
| 58 |
+
}
|
| 59 |
+
# INT8 cuantizado (256 niveles, <0.4% error)
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
### 4 Territorios
|
| 63 |
+
|
| 64 |
+
| Territorio | Zonas | Responsabilidad |
|
| 65 |
+
| ------------ | ------- | ------------------------- |
|
| 66 |
+
| SINTAXIS | 1-15 | Estructura, indentaciΓ³n |
|
| 67 |
+
| SEMΓNTICA | 16-30 | Significado, contexto |
|
| 68 |
+
| LΓGICO | 31-42 | Flujo, condiciones |
|
| 69 |
+
| ESTRUCTURAL | 43-52 | Arquitectura, mΓ³dulos |
|
| 70 |
+
|
| 71 |
+
---
|
| 72 |
+
|
| 73 |
+
## Estructura del Proyecto
|
| 74 |
+
|
| 75 |
+
```
|
| 76 |
+
PAMPAr-Coder/
|
| 77 |
+
βββ pampar/coder/v2/
|
| 78 |
+
β βββ modelo.py # PampaRCoderV2 principal
|
| 79 |
+
β βββ config.py # ConfigV2 + presets (PRESET_4GB, PRESET_8GB)
|
| 80 |
+
β βββ talamo.py # TΓ‘lamoBrodmann orquestador
|
| 81 |
+
β βββ llaves.py # LLAVES INT8 lookup tables
|
| 82 |
+
β βββ zonas.py # 52 zonas de Brodmann
|
| 83 |
+
β βββ bloques.py # BloqueTerritorial + FFN simbiΓ³tico
|
| 84 |
+
β βββ aprendizaje/ # Subsistemas de aprendizaje
|
| 85 |
+
βββ biblioteca/ # 39 temas de Python (~140 MB), lista para entrenar
|
| 86 |
+
βββ data/ # Datos y tokenizer
|
| 87 |
+
βββ checkpoints/ # pampar_v2_best.pt (42M params, vocab 16K)
|
| 88 |
+
βββ scripts/ # Scripts utilidad (aprender_solo.py, etc.)
|
| 89 |
+
βββ tests/ # 134 tests pytest
|
| 90 |
+
```
|
| 91 |
+
|
| 92 |
+
---
|
| 93 |
+
|
| 94 |
+
## Reglas CrΓticas
|
| 95 |
+
|
| 96 |
+
1. **LLAVES son regex** β NUNCA entrenarlas
|
| 97 |
+
2. **Territorios en paralelo** β combinan via soporte simbiΓ³tico
|
| 98 |
+
3. **INT8 para LLAVES** β lookup tables cuantizadas (256 niveles)
|
| 99 |
+
4. **Pesos en FP16/BF16** β nunca INT8
|
| 100 |
+
5. **Gradient checkpointing** β obligatorio para >500M params
|
| 101 |
+
6. **vocab_size = 16000** β debe coincidir con el tokenizer activo (`code_tokenizer.model`)
|
| 102 |
+
7. **Ventana contexto = 32** β convoluciΓ³n causal, pad izquierdo
|
| 103 |
+
|
| 104 |
+
---
|
| 105 |
+
|
| 106 |
+
## Convenciones de CΓ³digo
|
| 107 |
+
|
| 108 |
+
### Naming
|
| 109 |
+
|
| 110 |
+
| Tipo | ConvenciΓ³n | Ejemplo |
|
| 111 |
+
|------|------------|---------|
|
| 112 |
+
| Dominio | EspaΓ±ol | `Talamo`, `MemoriaErrores` |
|
| 113 |
+
| ML estΓ‘ndar | InglΓ©s | `forward`, `hidden_states` |
|
| 114 |
+
| Config | `Config` + nombre | `ConfigPampaRCoderV2` |
|
| 115 |
+
| Presets | `PRESET_` + capacidad | `PRESET_4GB` |
|
| 116 |
+
|
| 117 |
+
### Type Hints (OBLIGATORIO)
|
| 118 |
+
|
| 119 |
+
```python
|
| 120 |
+
def forward(
|
| 121 |
+
self,
|
| 122 |
+
input_ids: Tensor,
|
| 123 |
+
attention_mask: Optional[Tensor] = None,
|
| 124 |
+
labels: Optional[Tensor] = None,
|
| 125 |
+
) -> Tuple[Tensor, Optional[Tensor]]:
|
| 126 |
+
"""Forward pass del modelo."""
|
| 127 |
+
...
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
### Docstrings (Google Style)
|
| 131 |
+
|
| 132 |
+
```python
|
| 133 |
+
class PampaRCoderV2(nn.Module):
|
| 134 |
+
"""
|
| 135 |
+
Modelo principal PAMPAr-Coder V2.
|
| 136 |
+
|
| 137 |
+
Attributes:
|
| 138 |
+
config: ConfiguraciΓ³n del modelo.
|
| 139 |
+
embedding: Capa de embedding.
|
| 140 |
+
|
| 141 |
+
Example:
|
| 142 |
+
>>> model = crear_modelo(PRESET_4GB)
|
| 143 |
+
"""
|
| 144 |
+
```
|
| 145 |
+
|
| 146 |
+
---
|
| 147 |
+
|
| 148 |
+
## Patrones PyTorch
|
| 149 |
+
|
| 150 |
+
### Training Loop
|
| 151 |
+
|
| 152 |
+
```python
|
| 153 |
+
from torch.cuda.amp import autocast, GradScaler
|
| 154 |
+
|
| 155 |
+
scaler = GradScaler()
|
| 156 |
+
for batch in dataloader:
|
| 157 |
+
with autocast(dtype=torch.bfloat16):
|
| 158 |
+
loss = model(**batch).loss / accumulation_steps
|
| 159 |
+
|
| 160 |
+
scaler.scale(loss).backward()
|
| 161 |
+
|
| 162 |
+
if (step + 1) % accumulation_steps == 0:
|
| 163 |
+
scaler.unscale_(optimizer)
|
| 164 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 165 |
+
scaler.step(optimizer)
|
| 166 |
+
scaler.update()
|
| 167 |
+
optimizer.zero_grad()
|
| 168 |
+
```
|
| 169 |
+
|
| 170 |
+
### Checkpoint
|
| 171 |
+
|
| 172 |
+
```python
|
| 173 |
+
torch.save({
|
| 174 |
+
"model_state_dict": model.state_dict(),
|
| 175 |
+
"optimizer_state_dict": optimizer.state_dict(),
|
| 176 |
+
"epoch": epoch,
|
| 177 |
+
"loss": loss,
|
| 178 |
+
}, "checkpoint.pt")
|
| 179 |
+
```
|
| 180 |
+
|
| 181 |
+
---
|
| 182 |
+
|
| 183 |
+
## Testing (pytest)
|
| 184 |
+
|
| 185 |
+
```python
|
| 186 |
+
@pytest.fixture
|
| 187 |
+
def small_config():
|
| 188 |
+
return ConfigPampaRCoderV2(
|
| 189 |
+
vocab_size=1000, hidden_size=64, num_layers=2
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
class TestPampaRCoderV2:
|
| 193 |
+
def test_forward_shape(self, small_config):
|
| 194 |
+
model = PampaRCoderV2(small_config)
|
| 195 |
+
input_ids = torch.randint(0, 1000, (2, 16))
|
| 196 |
+
output = model(input_ids)
|
| 197 |
+
assert output.logits.shape == (2, 16, 1000)
|
| 198 |
+
```
|
| 199 |
+
|
| 200 |
+
---
|
| 201 |
+
|
| 202 |
+
## Git Workflow
|
| 203 |
+
|
| 204 |
+
```bash
|
| 205 |
+
# Conventional commits
|
| 206 |
+
feat(model): add early exit mechanism
|
| 207 |
+
fix(llaves): correct zona classification for imports
|
| 208 |
+
test(talamo): add gradient flow tests
|
| 209 |
+
refactor(bloques): extract symbiotic support
|
| 210 |
+
|
| 211 |
+
# Branch naming
|
| 212 |
+
feat/early-exit
|
| 213 |
+
fix/llaves-imports
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
---
|
| 217 |
+
|
| 218 |
+
## Anti-patterns (NUNCA)
|
| 219 |
+
|
| 220 |
+
- `any` en type hints β usar `Union`, `Optional`, etc.
|
| 221 |
+
- LLAVES entrenables β son regex, siempre fijos
|
| 222 |
+
- Cuantizar gradientes β solo LLAVES lookup
|
| 223 |
+
- Archivos >300 lΓneas β dividir en mΓ³dulos
|
| 224 |
+
- Tests sin fixture β usar `small_config`
|
| 225 |
+
- `print()` en producciΓ³n β usar `logging`
|
| 226 |
+
- Secrets en cΓ³digo β usar variables de entorno
|
.env.example
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# API keys for Classroom mentor system (optional)
|
| 2 |
+
OPENROUTER_API_KEY=sk-or-v1-your-key-here
|
| 3 |
+
GITHUB_TOKEN=ghp_your-token-here
|
| 4 |
+
QWEN_API_KEY=sk-your-dashscope-key-here
|
.gitattributes
CHANGED
|
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
logo-pampar-color.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
logo-pampar-sf.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
logo-pampar.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
PAMPAR-coder.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
PAMPArLLM.png filter=lfs diff=lfs merge=lfs -text
|
.github/CODEOWNERS
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Repository owner β all PRs require review
|
| 2 |
+
* @lucasmella-stack
|
.github/ISSUE_TEMPLATE/bug_report.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
name: Bug Report
|
| 3 |
+
about: Report a bug in PAMPAr-Coder
|
| 4 |
+
title: "[BUG] "
|
| 5 |
+
labels: bug
|
| 6 |
+
assignees: lucasmella-stack
|
| 7 |
+
---
|
| 8 |
+
|
| 9 |
+
## Description
|
| 10 |
+
|
| 11 |
+
<!-- Clear description of the bug -->
|
| 12 |
+
|
| 13 |
+
## Steps to Reproduce
|
| 14 |
+
|
| 15 |
+
1.
|
| 16 |
+
2.
|
| 17 |
+
3.
|
| 18 |
+
|
| 19 |
+
## Expected Behavior
|
| 20 |
+
|
| 21 |
+
<!-- What should happen -->
|
| 22 |
+
|
| 23 |
+
## Actual Behavior
|
| 24 |
+
|
| 25 |
+
<!-- What actually happens -->
|
| 26 |
+
|
| 27 |
+
## Environment
|
| 28 |
+
|
| 29 |
+
- OS:
|
| 30 |
+
- Python version:
|
| 31 |
+
- PyTorch version:
|
| 32 |
+
- GPU:
|
| 33 |
+
- Checkpoint used:
|
.github/ISSUE_TEMPLATE/feature_request.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
name: Feature Request
|
| 3 |
+
about: Suggest an idea for PAMPAr-Coder
|
| 4 |
+
title: "[FEATURE] "
|
| 5 |
+
labels: enhancement
|
| 6 |
+
assignees: ""
|
| 7 |
+
---
|
| 8 |
+
|
| 9 |
+
## Problem
|
| 10 |
+
|
| 11 |
+
<!-- What problem does this solve? -->
|
| 12 |
+
|
| 13 |
+
## Proposed Solution
|
| 14 |
+
|
| 15 |
+
<!-- How would you implement it? -->
|
| 16 |
+
|
| 17 |
+
## Alternatives Considered
|
| 18 |
+
|
| 19 |
+
<!-- Other approaches you thought of -->
|
| 20 |
+
|
| 21 |
+
## Additional Context
|
| 22 |
+
|
| 23 |
+
<!-- Any additional context, references, papers, etc. -->
|
.github/copilot-instructions.md
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PAMPAr-Coder - Copilot Instructions
|
| 2 |
+
|
| 3 |
+
> Instrucciones especΓficas para este proyecto. Se combinan con tu perfil global.
|
| 4 |
+
|
| 5 |
+
## Proyecto
|
| 6 |
+
|
| 7 |
+
**PAMPAr-Coder** es un LLM de cΓ³digo 1.5B con arquitectura cerebral inspirada en las 52 zonas de Brodmann.
|
| 8 |
+
"El Linux de la IA" - Hacer mΓ‘s con menos hardware.
|
| 9 |
+
|
| 10 |
+
## Arquitectura
|
| 11 |
+
|
| 12 |
+
```
|
| 13 |
+
Input β Embedding β [BloqueTerrritorial ΓN] β LM Head β Output
|
| 14 |
+
β
|
| 15 |
+
TΓ‘lamoBrodmann (LLAVES 80% + AtenciΓ³n 20%)
|
| 16 |
+
+ Conv1D causal (ventana 32 tokens)
|
| 17 |
+
β
|
| 18 |
+
βββββββββββββββββββββββ΄ββββββββββββββββββββββ
|
| 19 |
+
βΌ βΌ
|
| 20 |
+
βββββββββββββββββ βββββββββββββββββ
|
| 21 |
+
β SINTAXIS ββββ simbiosis βββββΊβ SEMΓNTICA β
|
| 22 |
+
β Zonas 1-15 β β Zonas 16-30 β
|
| 23 |
+
βββββββββββββββββ βββββββββββββββββ
|
| 24 |
+
βΌ βΌ
|
| 25 |
+
βββββββββββββββββ βββββββββββββββββ
|
| 26 |
+
β LΓGICO ββββ simbiosis βββββΊβ ESTRUCTURAL β
|
| 27 |
+
β Zonas 31-42 β β Zonas 43-52 β
|
| 28 |
+
βββββββββββββββββ βββββββββββββββββ
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
## Componentes Clave
|
| 32 |
+
|
| 33 |
+
| Archivo | PropΓ³sito |
|
| 34 |
+
|---------|-----------|
|
| 35 |
+
| `pampar/coder/v2/modelo.py` | PampaRCoderV2 con 52 zonas |
|
| 36 |
+
| `pampar/coder/v2/config.py` | ConfigPampaRCoderV2 + presets |
|
| 37 |
+
| `pampar/coder/v2/talamo.py` | TΓ‘lamo orquestador con LLAVES + context conv |
|
| 38 |
+
| `pampar/coder/v2/llaves.py` | LLAVES lookup tables (INT8, 256 niveles) |
|
| 39 |
+
| `pampar/coder/v2/bloques.py` | BloqueTerrritorial + relaciones simbiΓ³ticas |
|
| 40 |
+
| `pampar/coder/v2/zonas.py` | DefiniciΓ³n de las 52 zonas de Brodmann |
|
| 41 |
+
| `pampar/coder/v2/aprendizaje/` | MetacogniciΓ³n, neuroplasticidad, memoria errores |
|
| 42 |
+
| `cloud/runpod/train_cloud.py` | Script de entrenamiento en cloud |
|
| 43 |
+
|
| 44 |
+
## Convenciones
|
| 45 |
+
|
| 46 |
+
- **Idioma cΓ³digo**: InglΓ©s
|
| 47 |
+
- **Comentarios/docs**: EspaΓ±ol o InglΓ©s segΓΊn contexto
|
| 48 |
+
- **Nombres de clases**: EspaΓ±ol para conceptos de dominio (`Talamo`, `Territorio`, `Zona`, `MemoriaErrores`)
|
| 49 |
+
- **Variables**: InglΓ©s (`input_ids`, `hidden_states`)
|
| 50 |
+
|
| 51 |
+
## Stack
|
| 52 |
+
|
| 53 |
+
- PyTorch 2.x
|
| 54 |
+
- SentencePiece (tokenizer BPE, 48K vocab)
|
| 55 |
+
- Hugging Face datasets
|
| 56 |
+
- RunPod/Cloud para entrenamiento
|
| 57 |
+
|
| 58 |
+
## Comandos frecuentes
|
| 59 |
+
|
| 60 |
+
```bash
|
| 61 |
+
# Entrenar localmente
|
| 62 |
+
python scripts/train.py --config 1.5B --epochs 10
|
| 63 |
+
|
| 64 |
+
# Entrenar en cloud (RunPod)
|
| 65 |
+
ssh root@IP -p PORT
|
| 66 |
+
cd /workspace/PAMPAr-Coder
|
| 67 |
+
screen -S train
|
| 68 |
+
python3 cloud/runpod/train_cloud.py --config 1_5B > training.log 2>&1
|
| 69 |
+
|
| 70 |
+
# Ver progreso
|
| 71 |
+
tail -f training.log
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
## Reglas especΓficas
|
| 75 |
+
|
| 76 |
+
1. **LLAVES** son patrones regex que clasifican tokens - NUNCA usar ML para esto
|
| 77 |
+
2. **Territorios** procesan en paralelo, luego combinan via soporte simbiΓ³tico
|
| 78 |
+
3. **CuantizaciΓ³n INT8** (256 niveles) para LLAVES lookup tables
|
| 79 |
+
4. **Early Exit** usa percentil 10 per-token (no promedio global)
|
| 80 |
+
5. **Gradient checkpointing** siempre activo para modelos >500M params
|
| 81 |
+
6. **Tests** en `tests/` con pytest
|
| 82 |
+
7. **Ventana de contexto** (32 tokens) usa convoluciΓ³n causal - pad izquierdo
|
.github/instructions/cloud-training.instructions.md
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Cloud Training Instructions
|
| 2 |
+
|
| 3 |
+
> Entrenamiento de PAMPAr-Coder en RunPod y otros providers.
|
| 4 |
+
|
| 5 |
+
## RunPod Setup
|
| 6 |
+
|
| 7 |
+
### Conectar
|
| 8 |
+
```bash
|
| 9 |
+
ssh root@IP -p PORT
|
| 10 |
+
# Password: en RunPod dashboard o usar SSH key
|
| 11 |
+
```
|
| 12 |
+
|
| 13 |
+
### Preparar entorno
|
| 14 |
+
```bash
|
| 15 |
+
cd /workspace/PAMPAr-Coder
|
| 16 |
+
pip install sentencepiece tqdm datasets
|
| 17 |
+
```
|
| 18 |
+
|
| 19 |
+
### Lanzar entrenamiento
|
| 20 |
+
```bash
|
| 21 |
+
# Background con log
|
| 22 |
+
nohup python3 cloud/runpod/train_cloud.py \
|
| 23 |
+
--config 3B \
|
| 24 |
+
--data-dir data/distillation \
|
| 25 |
+
--tokenizer data/tokenizer/code_bpe.model \
|
| 26 |
+
--epochs 10 \
|
| 27 |
+
--no-wandb \
|
| 28 |
+
> training.log 2>&1 &
|
| 29 |
+
|
| 30 |
+
# Monitorear
|
| 31 |
+
tail -f training.log
|
| 32 |
+
nvidia-smi -l 5 # GPU cada 5 segundos
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
## Configuraciones
|
| 36 |
+
|
| 37 |
+
| Config | Params | VRAM | GPU recomendada |
|
| 38 |
+
|--------|--------|------|-----------------|
|
| 39 |
+
| 1.5B | ~230M | 8GB | RTX 3090, A10 |
|
| 40 |
+
| 3B | ~3B | 24GB | A40, A100 |
|
| 41 |
+
|
| 42 |
+
### Ajustar config
|
| 43 |
+
```python
|
| 44 |
+
# cloud/runpod/config_3b.py
|
| 45 |
+
@dataclass
|
| 46 |
+
class Config3B:
|
| 47 |
+
vocab_size: int = 32000
|
| 48 |
+
dim: int = 2560
|
| 49 |
+
n_heads: int = 20
|
| 50 |
+
n_capas: int = 32
|
| 51 |
+
max_seq_len: int = 2048
|
| 52 |
+
batch_size: int = 4
|
| 53 |
+
gradient_accumulation: int = 16
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
## Troubleshooting
|
| 57 |
+
|
| 58 |
+
### OOM en GPU
|
| 59 |
+
1. Reducir `batch_size`
|
| 60 |
+
2. Reducir `max_seq_len`
|
| 61 |
+
3. Activar `use_gradient_checkpointing = True`
|
| 62 |
+
|
| 63 |
+
### OOM en RAM (sistema)
|
| 64 |
+
1. Usar streaming dataset
|
| 65 |
+
2. Reducir workers de DataLoader
|
| 66 |
+
3. Modelo se carga en CPU antes de GPU - reducir tamaΓ±o
|
| 67 |
+
|
| 68 |
+
### Tokens fuera de rango
|
| 69 |
+
- Asegurar `vocab_size` en config == tokenizer.GetPieceSize()
|
| 70 |
+
- TΓpico: tokenizer tiene 32K, config dice 16K β error
|
| 71 |
+
|
| 72 |
+
## Checkpoints
|
| 73 |
+
|
| 74 |
+
```bash
|
| 75 |
+
# UbicaciΓ³n
|
| 76 |
+
/workspace/PAMPAr-Coder/checkpoints/
|
| 77 |
+
βββ best_model.pt # Mejor val_loss
|
| 78 |
+
βββ epoch_N.pt # Por epoch
|
| 79 |
+
βββ step_XXXX.pt # Por steps
|
| 80 |
+
|
| 81 |
+
# Descargar a local
|
| 82 |
+
scp -P PORT root@IP:/workspace/PAMPAr-Coder/checkpoints/best_model.pt ./
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
## Costos estimados
|
| 86 |
+
|
| 87 |
+
| GPU | $/hora | 10 epochs (20K samples) |
|
| 88 |
+
|-----|--------|------------------------|
|
| 89 |
+
| A10 | $0.30 | ~$0.60 |
|
| 90 |
+
| A40 | $0.40 | ~$0.80 |
|
| 91 |
+
| A100 | $1.50 | ~$3.00 |
|
.github/instructions/pampar-architecture.instructions.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PAMPAr Architecture Instructions
|
| 2 |
+
|
| 3 |
+
> Arquitectura cerebral con 52 zonas de Brodmann para procesamiento de cΓ³digo.
|
| 4 |
+
|
| 5 |
+
## LLAVES System
|
| 6 |
+
|
| 7 |
+
LLAVES = Lookup de ActivaciΓ³n Via Expresiones SintΓ‘cticas
|
| 8 |
+
|
| 9 |
+
```python
|
| 10 |
+
# Las LLAVES clasifican tokens usando patrones regex, NO ML
|
| 11 |
+
LLAVES = {
|
| 12 |
+
'B06_KEYWORDS_IMPORT': ['import', 'from', 'require'],
|
| 13 |
+
'B17_LITERAL_STRING': [r'".*"', r"'.*'", 'f"', "f'"],
|
| 14 |
+
'B18_LITERAL_NUMERO': [r'\d+', r'\d+\.\d+'],
|
| 15 |
+
'B35_IDENTIFICADOR_VAR': [r'[a-z_][a-z0-9_]*'],
|
| 16 |
+
# ... 52 zonas total
|
| 17 |
+
}
|
| 18 |
+
```
|
| 19 |
+
|
| 20 |
+
## Territorios (4 Macro-Γreas)
|
| 21 |
+
|
| 22 |
+
| Territorio | Zonas | FunciΓ³n |
|
| 23 |
+
|------------|-------|---------|
|
| 24 |
+
| SINTAXIS | B01-B13 | Keywords, operadores, delimitadores |
|
| 25 |
+
| SEMΓNTICA | B14-B39 | Identificadores, literales, tipos |
|
| 26 |
+
| LΓGICO | B40-B44 | Condicionales, loops, excepciones |
|
| 27 |
+
| ESTRUCTURAL | B45-B52 | Patrones, estructuras, documentaciΓ³n |
|
| 28 |
+
|
| 29 |
+
## Flujo de Procesamiento
|
| 30 |
+
|
| 31 |
+
```
|
| 32 |
+
1. Token β LLAVES lookup (O(1), 80% peso)
|
| 33 |
+
2. Token β Embedding attention (30% peso)
|
| 34 |
+
3. Combinar β ActivaciΓ³n por zona
|
| 35 |
+
4. Zonas activas β Procesamiento territorial
|
| 36 |
+
5. FusiΓ³n β Output
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
## CuantizaciΓ³n
|
| 40 |
+
|
| 41 |
+
Solo las lookup tables de LLAVES se cuantizan a INT4:
|
| 42 |
+
- Reduce memoria de 6.5MB β 812KB
|
| 43 |
+
- Sin pΓ©rdida de precisiΓ³n (es lookup discreto)
|
| 44 |
+
- El modelo (pesos) se mantiene en FP16/BF16
|
| 45 |
+
|
| 46 |
+
## Early Exit
|
| 47 |
+
|
| 48 |
+
El modelo puede salir temprano si la confianza es alta:
|
| 49 |
+
```python
|
| 50 |
+
if confianza > 0.90 and capa >= self.capas_minimas:
|
| 51 |
+
return x, True # Exit early
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
## Reglas de ImplementaciΓ³n
|
| 55 |
+
|
| 56 |
+
1. **NUNCA** usar backprop para entrenar LLAVES
|
| 57 |
+
2. **SIEMPRE** procesar territorios en paralelo cuando sea posible
|
| 58 |
+
3. **Cuantizar** solo tablas de lookup, nunca pesos del modelo
|
| 59 |
+
4. **Registrar** tokenizer con `model.registrar_tokenizer(tokenizer)`
|
.github/instructions/python-ml.instructions.md
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
````instructions
|
| 2 |
+
# Python ML/LLM Instructions
|
| 3 |
+
|
| 4 |
+
> Para desarrollo de modelos de ML/LLM con PyTorch.
|
| 5 |
+
|
| 6 |
+
## Type Hints (OBLIGATORIO)
|
| 7 |
+
|
| 8 |
+
```python
|
| 9 |
+
from typing import Optional, Tuple, Dict, List, Union, Literal
|
| 10 |
+
from torch import Tensor
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
|
| 13 |
+
def forward(
|
| 14 |
+
self,
|
| 15 |
+
input_ids: Tensor,
|
| 16 |
+
attention_mask: Optional[Tensor] = None,
|
| 17 |
+
labels: Optional[Tensor] = None,
|
| 18 |
+
) -> Tuple[Tensor, Optional[Tensor]]:
|
| 19 |
+
"""
|
| 20 |
+
Forward pass del modelo.
|
| 21 |
+
|
| 22 |
+
Args:
|
| 23 |
+
input_ids: Token IDs, shape (batch, seq_len).
|
| 24 |
+
attention_mask: MΓ‘scara de atenciΓ³n, shape (batch, seq_len).
|
| 25 |
+
labels: Labels para calcular loss, shape (batch, seq_len).
|
| 26 |
+
|
| 27 |
+
Returns:
|
| 28 |
+
Tuple de (logits, loss). Loss es None si labels no se proporcionan.
|
| 29 |
+
"""
|
| 30 |
+
...
|
| 31 |
+
````
|
| 32 |
+
|
| 33 |
+
## Docstrings (Google Style)
|
| 34 |
+
|
| 35 |
+
```python
|
| 36 |
+
class PampaRCoderV2(nn.Module):
|
| 37 |
+
"""
|
| 38 |
+
Modelo principal PAMPAr-Coder V2 con arquitectura cerebral.
|
| 39 |
+
|
| 40 |
+
Attributes:
|
| 41 |
+
config: ConfiguraciΓ³n del modelo.
|
| 42 |
+
embedding: Capa de embedding de tokens.
|
| 43 |
+
talamo: Orquestador central TΓ‘lamoBrodmann.
|
| 44 |
+
territorios: Lista de 4 BloqueTerrritorial.
|
| 45 |
+
|
| 46 |
+
Example:
|
| 47 |
+
>>> config = ConfigPampaRCoderV2.from_preset("1.5B")
|
| 48 |
+
>>> model = PampaRCoderV2(config)
|
| 49 |
+
>>> output = model(input_ids)
|
| 50 |
+
"""
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
## PyTorch Patterns
|
| 54 |
+
|
| 55 |
+
### Model Definition
|
| 56 |
+
|
| 57 |
+
```python
|
| 58 |
+
class MiModulo(nn.Module):
|
| 59 |
+
def __init__(self, config: ConfigPampaRCoderV2):
|
| 60 |
+
super().__init__()
|
| 61 |
+
self.config = config
|
| 62 |
+
# Inicializar layers aquΓ
|
| 63 |
+
|
| 64 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 65 |
+
# Forward pass
|
| 66 |
+
return x
|
| 67 |
+
|
| 68 |
+
def _init_weights(self, module: nn.Module) -> None:
|
| 69 |
+
"""InicializaciΓ³n de pesos."""
|
| 70 |
+
if isinstance(module, nn.Linear):
|
| 71 |
+
nn.init.normal_(module.weight, std=0.02)
|
| 72 |
+
if module.bias is not None:
|
| 73 |
+
nn.init.zeros_(module.bias)
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
### Training Loop
|
| 77 |
+
|
| 78 |
+
```python
|
| 79 |
+
from torch.cuda.amp import autocast, GradScaler
|
| 80 |
+
from tqdm import tqdm
|
| 81 |
+
|
| 82 |
+
def train_epoch(
|
| 83 |
+
model: nn.Module,
|
| 84 |
+
dataloader: DataLoader,
|
| 85 |
+
optimizer: Optimizer,
|
| 86 |
+
scheduler: LRScheduler,
|
| 87 |
+
scaler: GradScaler,
|
| 88 |
+
device: torch.device,
|
| 89 |
+
accumulation_steps: int = 4,
|
| 90 |
+
) -> float:
|
| 91 |
+
"""Entrena una Γ©poca completa."""
|
| 92 |
+
model.train()
|
| 93 |
+
total_loss = 0.0
|
| 94 |
+
|
| 95 |
+
for step, batch in enumerate(tqdm(dataloader)):
|
| 96 |
+
batch = {k: v.to(device) for k, v in batch.items()}
|
| 97 |
+
|
| 98 |
+
with autocast(dtype=torch.bfloat16):
|
| 99 |
+
outputs = model(**batch)
|
| 100 |
+
loss = outputs.loss / accumulation_steps
|
| 101 |
+
|
| 102 |
+
scaler.scale(loss).backward()
|
| 103 |
+
|
| 104 |
+
if (step + 1) % accumulation_steps == 0:
|
| 105 |
+
scaler.unscale_(optimizer)
|
| 106 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 107 |
+
scaler.step(optimizer)
|
| 108 |
+
scaler.update()
|
| 109 |
+
scheduler.step()
|
| 110 |
+
optimizer.zero_grad()
|
| 111 |
+
|
| 112 |
+
total_loss += loss.item() * accumulation_steps
|
| 113 |
+
|
| 114 |
+
return total_loss / len(dataloader)
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
### Checkpoint Saving/Loading
|
| 118 |
+
|
| 119 |
+
```python
|
| 120 |
+
def save_checkpoint(
|
| 121 |
+
model: nn.Module,
|
| 122 |
+
optimizer: Optimizer,
|
| 123 |
+
scheduler: LRScheduler,
|
| 124 |
+
epoch: int,
|
| 125 |
+
loss: float,
|
| 126 |
+
path: str,
|
| 127 |
+
) -> None:
|
| 128 |
+
"""Guarda checkpoint completo."""
|
| 129 |
+
torch.save({
|
| 130 |
+
"model_state_dict": model.state_dict(),
|
| 131 |
+
"optimizer_state_dict": optimizer.state_dict(),
|
| 132 |
+
"scheduler_state_dict": scheduler.state_dict(),
|
| 133 |
+
"epoch": epoch,
|
| 134 |
+
"loss": loss,
|
| 135 |
+
}, path)
|
| 136 |
+
|
| 137 |
+
def load_checkpoint(
|
| 138 |
+
path: str,
|
| 139 |
+
model: nn.Module,
|
| 140 |
+
optimizer: Optional[Optimizer] = None,
|
| 141 |
+
scheduler: Optional[LRScheduler] = None,
|
| 142 |
+
) -> Dict:
|
| 143 |
+
"""Carga checkpoint."""
|
| 144 |
+
checkpoint = torch.load(path, map_location="cpu")
|
| 145 |
+
model.load_state_dict(checkpoint["model_state_dict"])
|
| 146 |
+
if optimizer:
|
| 147 |
+
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
|
| 148 |
+
if scheduler:
|
| 149 |
+
scheduler.load_state_dict(checkpoint["scheduler_state_dict"])
|
| 150 |
+
return checkpoint
|
| 151 |
+
```
|
| 152 |
+
|
| 153 |
+
## Memory Optimization
|
| 154 |
+
|
| 155 |
+
### Gradient Checkpointing
|
| 156 |
+
|
| 157 |
+
```python
|
| 158 |
+
# Para modelos grandes (>500M params)
|
| 159 |
+
model.gradient_checkpointing_enable()
|
| 160 |
+
|
| 161 |
+
# Manual control:
|
| 162 |
+
from torch.utils.checkpoint import checkpoint
|
| 163 |
+
|
| 164 |
+
class Block(nn.Module):
|
| 165 |
+
def forward(self, x):
|
| 166 |
+
if self.training and self.gradient_checkpointing:
|
| 167 |
+
return checkpoint(self._forward_impl, x, use_reentrant=False)
|
| 168 |
+
return self._forward_impl(x)
|
| 169 |
+
```
|
| 170 |
+
|
| 171 |
+
### Efficient Attention
|
| 172 |
+
|
| 173 |
+
```python
|
| 174 |
+
# Usar Flash Attention cuando sea posible
|
| 175 |
+
from torch.nn.functional import scaled_dot_product_attention
|
| 176 |
+
|
| 177 |
+
# O xformers para backwards compatibility
|
| 178 |
+
try:
|
| 179 |
+
from xformers.ops import memory_efficient_attention
|
| 180 |
+
HAS_XFORMERS = True
|
| 181 |
+
except ImportError:
|
| 182 |
+
HAS_XFORMERS = False
|
| 183 |
+
```
|
| 184 |
+
|
| 185 |
+
### Tensor Operations
|
| 186 |
+
|
| 187 |
+
```python
|
| 188 |
+
# BIEN: operaciones in-place cuando sea seguro
|
| 189 |
+
x.add_(bias) # En lugar de x = x + bias
|
| 190 |
+
|
| 191 |
+
# BIEN: evitar concatenaciones innecesarias
|
| 192 |
+
# MAL:
|
| 193 |
+
# outputs = []
|
| 194 |
+
# for block in self.blocks:
|
| 195 |
+
# outputs.append(block(x))
|
| 196 |
+
# return torch.cat(outputs, dim=-1)
|
| 197 |
+
|
| 198 |
+
# BIEN: usar stack si las dimensiones son iguales
|
| 199 |
+
outputs = torch.stack([block(x) for block in self.blocks], dim=0)
|
| 200 |
+
```
|
| 201 |
+
|
| 202 |
+
## Testing (pytest)
|
| 203 |
+
|
| 204 |
+
```python
|
| 205 |
+
import pytest
|
| 206 |
+
import torch
|
| 207 |
+
from pampar.coder.v2.modelo import PampaRCoderV2
|
| 208 |
+
from pampar.coder.v2.config import ConfigPampaRCoderV2
|
| 209 |
+
|
| 210 |
+
@pytest.fixture
|
| 211 |
+
def config():
|
| 212 |
+
"""ConfiguraciΓ³n pequeΓ±a para tests."""
|
| 213 |
+
return ConfigPampaRCoderV2(
|
| 214 |
+
vocab_size=1000,
|
| 215 |
+
hidden_size=64,
|
| 216 |
+
num_layers=2,
|
| 217 |
+
num_heads=4,
|
| 218 |
+
)
|
| 219 |
+
|
| 220 |
+
@pytest.fixture
|
| 221 |
+
def model(config):
|
| 222 |
+
"""Modelo pequeΓ±o para tests."""
|
| 223 |
+
return PampaRCoderV2(config)
|
| 224 |
+
|
| 225 |
+
class TestPampaRCoderV2:
|
| 226 |
+
def test_forward_shape(self, model, config):
|
| 227 |
+
"""Verifica output shape."""
|
| 228 |
+
batch_size, seq_len = 2, 16
|
| 229 |
+
input_ids = torch.randint(0, config.vocab_size, (batch_size, seq_len))
|
| 230 |
+
|
| 231 |
+
output = model(input_ids)
|
| 232 |
+
|
| 233 |
+
assert output.logits.shape == (batch_size, seq_len, config.vocab_size)
|
| 234 |
+
|
| 235 |
+
def test_gradient_flow(self, model):
|
| 236 |
+
"""Verifica que gradientes fluyen correctamente."""
|
| 237 |
+
input_ids = torch.randint(0, 1000, (1, 8))
|
| 238 |
+
labels = input_ids.clone()
|
| 239 |
+
|
| 240 |
+
output = model(input_ids, labels=labels)
|
| 241 |
+
output.loss.backward()
|
| 242 |
+
|
| 243 |
+
for name, param in model.named_parameters():
|
| 244 |
+
if param.requires_grad:
|
| 245 |
+
assert param.grad is not None, f"No gradient for {name}"
|
| 246 |
+
```
|
| 247 |
+
|
| 248 |
+
## Logging
|
| 249 |
+
|
| 250 |
+
```python
|
| 251 |
+
import logging
|
| 252 |
+
|
| 253 |
+
# Configurar al inicio del script
|
| 254 |
+
logging.basicConfig(
|
| 255 |
+
format="%(asctime)s - %(levelname)s - %(message)s",
|
| 256 |
+
level=logging.INFO,
|
| 257 |
+
)
|
| 258 |
+
logger = logging.getLogger(__name__)
|
| 259 |
+
|
| 260 |
+
# Usar en el cΓ³digo
|
| 261 |
+
logger.info(f"Epoch {epoch}: loss={loss:.4f}")
|
| 262 |
+
logger.warning(f"GPU memory high: {memory_used:.1f}GB")
|
| 263 |
+
logger.error(f"Checkpoint save failed: {e}")
|
| 264 |
+
```
|
| 265 |
+
|
| 266 |
+
```
|
| 267 |
+
|
| 268 |
+
```
|
.github/instructions/testing-pytest.instructions.md
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
```instructions
|
| 2 |
+
# Testing Instructions (pytest)
|
| 3 |
+
|
| 4 |
+
> Para tests en PAMPAr-Coder usando pytest.
|
| 5 |
+
|
| 6 |
+
## Regla de Oro
|
| 7 |
+
|
| 8 |
+
**Cada mΓ³dulo nuevo DEBE tener:**
|
| 9 |
+
1. Un test de happy path
|
| 10 |
+
2. Un test de error/edge case
|
| 11 |
+
3. Un test de shapes (para tensores)
|
| 12 |
+
|
| 13 |
+
## Estructura de Tests
|
| 14 |
+
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
tests/
|
| 18 |
+
βββ test_modelo.py # Tests del modelo principal
|
| 19 |
+
βββ test_talamo.py # Tests del tΓ‘lamo
|
| 20 |
+
βββ test_llaves.py # Tests de LLAVES
|
| 21 |
+
βββ test_generation.py # Tests de generaciΓ³n
|
| 22 |
+
βββ conftest.py # Fixtures compartidos
|
| 23 |
+
|
| 24 |
+
````
|
| 25 |
+
|
| 26 |
+
## Fixtures (conftest.py)
|
| 27 |
+
|
| 28 |
+
```python
|
| 29 |
+
import pytest
|
| 30 |
+
import torch
|
| 31 |
+
from pampar.coder.v2.config import ConfigPampaRCoderV2
|
| 32 |
+
|
| 33 |
+
@pytest.fixture
|
| 34 |
+
def device():
|
| 35 |
+
"""Device para tests: CUDA si disponible, else CPU."""
|
| 36 |
+
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 37 |
+
|
| 38 |
+
@pytest.fixture
|
| 39 |
+
def small_config():
|
| 40 |
+
"""ConfiguraciΓ³n mΓnima para tests rΓ‘pidos."""
|
| 41 |
+
return ConfigPampaRCoderV2(
|
| 42 |
+
vocab_size=1000,
|
| 43 |
+
hidden_size=64,
|
| 44 |
+
num_layers=2,
|
| 45 |
+
num_heads=4,
|
| 46 |
+
intermediate_size=256,
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
@pytest.fixture
|
| 50 |
+
def batch():
|
| 51 |
+
"""Batch de ejemplo para tests."""
|
| 52 |
+
return {
|
| 53 |
+
"input_ids": torch.randint(0, 1000, (2, 16)),
|
| 54 |
+
"attention_mask": torch.ones(2, 16, dtype=torch.long),
|
| 55 |
+
"labels": torch.randint(0, 1000, (2, 16)),
|
| 56 |
+
}
|
| 57 |
+
````
|
| 58 |
+
|
| 59 |
+
## Patrones de Test
|
| 60 |
+
|
| 61 |
+
### Test de Shapes
|
| 62 |
+
|
| 63 |
+
```python
|
| 64 |
+
class TestModelShapes:
|
| 65 |
+
def test_embedding_output_shape(self, small_config):
|
| 66 |
+
from pampar.coder.v2.modelo import PampaRCoderV2
|
| 67 |
+
|
| 68 |
+
model = PampaRCoderV2(small_config)
|
| 69 |
+
input_ids = torch.randint(0, small_config.vocab_size, (2, 16))
|
| 70 |
+
|
| 71 |
+
output = model(input_ids)
|
| 72 |
+
|
| 73 |
+
assert output.logits.shape == (2, 16, small_config.vocab_size)
|
| 74 |
+
assert output.hidden_states.shape == (2, 16, small_config.hidden_size)
|
| 75 |
+
|
| 76 |
+
def test_attention_shape(self, small_config):
|
| 77 |
+
from pampar.coder.v2.talamo import TalamoBrodmann
|
| 78 |
+
|
| 79 |
+
talamo = TalamoBrodmann(small_config)
|
| 80 |
+
x = torch.randn(2, 16, small_config.hidden_size)
|
| 81 |
+
|
| 82 |
+
out = talamo(x)
|
| 83 |
+
|
| 84 |
+
assert out.shape == x.shape
|
| 85 |
+
```
|
| 86 |
+
|
| 87 |
+
### Test de Gradientes
|
| 88 |
+
|
| 89 |
+
```python
|
| 90 |
+
class TestGradientFlow:
|
| 91 |
+
def test_all_parameters_have_gradients(self, small_config):
|
| 92 |
+
model = PampaRCoderV2(small_config)
|
| 93 |
+
input_ids = torch.randint(0, small_config.vocab_size, (1, 8))
|
| 94 |
+
|
| 95 |
+
output = model(input_ids, labels=input_ids)
|
| 96 |
+
output.loss.backward()
|
| 97 |
+
|
| 98 |
+
for name, param in model.named_parameters():
|
| 99 |
+
if param.requires_grad:
|
| 100 |
+
assert param.grad is not None, f"No grad: {name}"
|
| 101 |
+
assert not torch.isnan(param.grad).any(), f"NaN grad: {name}"
|
| 102 |
+
|
| 103 |
+
def test_gradient_clipping(self, small_config):
|
| 104 |
+
model = PampaRCoderV2(small_config)
|
| 105 |
+
# ... setup con gradientes grandes
|
| 106 |
+
|
| 107 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 108 |
+
|
| 109 |
+
total_norm = sum(p.grad.norm() ** 2 for p in model.parameters()).sqrt()
|
| 110 |
+
assert total_norm <= 1.0 + 1e-6
|
| 111 |
+
```
|
| 112 |
+
|
| 113 |
+
### Test con Mock
|
| 114 |
+
|
| 115 |
+
```python
|
| 116 |
+
from unittest.mock import MagicMock, patch
|
| 117 |
+
|
| 118 |
+
class TestTraining:
|
| 119 |
+
@patch("torch.cuda.is_available", return_value=False)
|
| 120 |
+
def test_cpu_fallback(self, mock_cuda):
|
| 121 |
+
"""Verifica que funciona sin GPU."""
|
| 122 |
+
from pampar.coder.v2.modelo import PampaRCoderV2
|
| 123 |
+
|
| 124 |
+
config = ConfigPampaRCoderV2.from_preset("mini")
|
| 125 |
+
model = PampaRCoderV2(config)
|
| 126 |
+
|
| 127 |
+
input_ids = torch.randint(0, config.vocab_size, (1, 8))
|
| 128 |
+
output = model(input_ids)
|
| 129 |
+
|
| 130 |
+
assert output.logits is not None
|
| 131 |
+
```
|
| 132 |
+
|
| 133 |
+
### Test Parametrizado
|
| 134 |
+
|
| 135 |
+
```python
|
| 136 |
+
@pytest.mark.parametrize("batch_size", [1, 2, 4])
|
| 137 |
+
@pytest.mark.parametrize("seq_len", [8, 16, 32])
|
| 138 |
+
def test_variable_batch_seq(small_config, batch_size, seq_len):
|
| 139 |
+
model = PampaRCoderV2(small_config)
|
| 140 |
+
input_ids = torch.randint(0, small_config.vocab_size, (batch_size, seq_len))
|
| 141 |
+
|
| 142 |
+
output = model(input_ids)
|
| 143 |
+
|
| 144 |
+
assert output.logits.shape == (batch_size, seq_len, small_config.vocab_size)
|
| 145 |
+
|
| 146 |
+
@pytest.mark.parametrize("preset", ["mini", "1.5B", "3B"])
|
| 147 |
+
def test_preset_configs(preset):
|
| 148 |
+
config = ConfigPampaRCoderV2.from_preset(preset)
|
| 149 |
+
|
| 150 |
+
assert config.vocab_size == 48000
|
| 151 |
+
assert config.hidden_size > 0
|
| 152 |
+
```
|
| 153 |
+
|
| 154 |
+
### Test de LLAVES
|
| 155 |
+
|
| 156 |
+
```python
|
| 157 |
+
class TestLlaves:
|
| 158 |
+
def test_llaves_are_not_trainable(self):
|
| 159 |
+
from pampar.coder.v2.llaves import LlavesModule
|
| 160 |
+
|
| 161 |
+
llaves = LlavesModule()
|
| 162 |
+
|
| 163 |
+
for param in llaves.parameters():
|
| 164 |
+
assert not param.requires_grad, "LLAVES no deben ser entrenables"
|
| 165 |
+
|
| 166 |
+
def test_llaves_int8_quantization(self):
|
| 167 |
+
from pampar.coder.v2.llaves import LlavesModule
|
| 168 |
+
|
| 169 |
+
llaves = LlavesModule()
|
| 170 |
+
|
| 171 |
+
assert llaves.lookup_table.dtype == torch.int8
|
| 172 |
+
|
| 173 |
+
def test_llaves_pattern_matching(self):
|
| 174 |
+
from pampar.coder.v2.llaves import classify_token
|
| 175 |
+
|
| 176 |
+
# DeclaraciΓ³n Python
|
| 177 |
+
assert classify_token("def ") in range(1, 16) # SINTAXIS
|
| 178 |
+
|
| 179 |
+
# Operador lΓ³gico
|
| 180 |
+
assert classify_token("if ") in range(31, 43) # LΓGICO
|
| 181 |
+
```
|
| 182 |
+
|
| 183 |
+
## Markers
|
| 184 |
+
|
| 185 |
+
```python
|
| 186 |
+
# En pyproject.toml o pytest.ini:
|
| 187 |
+
# [tool.pytest.ini_options]
|
| 188 |
+
# markers = [
|
| 189 |
+
# "slow: marks tests as slow",
|
| 190 |
+
# "gpu: marks tests requiring GPU",
|
| 191 |
+
# ]
|
| 192 |
+
|
| 193 |
+
@pytest.mark.slow
|
| 194 |
+
def test_full_training_loop():
|
| 195 |
+
"""Test lento de training completo."""
|
| 196 |
+
...
|
| 197 |
+
|
| 198 |
+
@pytest.mark.gpu
|
| 199 |
+
@pytest.mark.skipif(not torch.cuda.is_available(), reason="GPU required")
|
| 200 |
+
def test_cuda_forward():
|
| 201 |
+
"""Test que requiere GPU."""
|
| 202 |
+
...
|
| 203 |
+
```
|
| 204 |
+
|
| 205 |
+
## Ejecutar Tests
|
| 206 |
+
|
| 207 |
+
```bash
|
| 208 |
+
# Todos los tests
|
| 209 |
+
pytest
|
| 210 |
+
|
| 211 |
+
# Solo tests rΓ‘pidos
|
| 212 |
+
pytest -m "not slow"
|
| 213 |
+
|
| 214 |
+
# Con coverage
|
| 215 |
+
pytest --cov=pampar --cov-report=html
|
| 216 |
+
|
| 217 |
+
# Verbose con print output
|
| 218 |
+
pytest -v -s
|
| 219 |
+
|
| 220 |
+
# Solo un archivo
|
| 221 |
+
pytest tests/test_modelo.py
|
| 222 |
+
|
| 223 |
+
# Solo un test especΓfico
|
| 224 |
+
pytest tests/test_modelo.py::TestModelShapes::test_embedding_output_shape
|
| 225 |
+
```
|
| 226 |
+
|
| 227 |
+
```
|
| 228 |
+
|
| 229 |
+
```
|
.github/pull_request_template.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## What
|
| 2 |
+
|
| 3 |
+
<!-- Brief description of the change -->
|
| 4 |
+
|
| 5 |
+
## Why
|
| 6 |
+
|
| 7 |
+
<!-- Context and motivation -->
|
| 8 |
+
|
| 9 |
+
## How to Test
|
| 10 |
+
|
| 11 |
+
<!-- Steps to verify it works -->
|
| 12 |
+
|
| 13 |
+
## Checklist
|
| 14 |
+
|
| 15 |
+
- [ ] Tests added/updated (`python -m pytest tests/ -v`)
|
| 16 |
+
- [ ] Type hints on all new functions
|
| 17 |
+
- [ ] No hardcoded paths or secrets
|
| 18 |
+
- [ ] Conventional commit messages
|
.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches: [main]
|
| 6 |
+
pull_request:
|
| 7 |
+
branches: [main]
|
| 8 |
+
|
| 9 |
+
jobs:
|
| 10 |
+
test:
|
| 11 |
+
runs-on: ubuntu-latest
|
| 12 |
+
timeout-minutes: 15
|
| 13 |
+
strategy:
|
| 14 |
+
matrix:
|
| 15 |
+
python-version: ["3.11", "3.12", "3.13"]
|
| 16 |
+
|
| 17 |
+
steps:
|
| 18 |
+
- uses: actions/checkout@v4
|
| 19 |
+
|
| 20 |
+
- name: Set up Python ${{ matrix.python-version }}
|
| 21 |
+
uses: actions/setup-python@v5
|
| 22 |
+
with:
|
| 23 |
+
python-version: ${{ matrix.python-version }}
|
| 24 |
+
|
| 25 |
+
- name: Install dependencies
|
| 26 |
+
run: |
|
| 27 |
+
python -m pip install --upgrade pip
|
| 28 |
+
pip install torch --index-url https://download.pytorch.org/whl/cpu
|
| 29 |
+
pip install -r requirements.txt
|
| 30 |
+
|
| 31 |
+
- name: Run tests
|
| 32 |
+
run: python -m pytest tests/ -v --tb=short
|
.gitignore
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C extensions
|
| 7 |
+
*.so
|
| 8 |
+
|
| 9 |
+
# Distribution / packaging
|
| 10 |
+
.Python
|
| 11 |
+
build/
|
| 12 |
+
develop-eggs/
|
| 13 |
+
dist/
|
| 14 |
+
downloads/
|
| 15 |
+
eggs/
|
| 16 |
+
.eggs/
|
| 17 |
+
lib/
|
| 18 |
+
lib64/
|
| 19 |
+
parts/
|
| 20 |
+
sdist/
|
| 21 |
+
var/
|
| 22 |
+
wheels/
|
| 23 |
+
*.egg-info/
|
| 24 |
+
.installed.cfg
|
| 25 |
+
*.egg
|
| 26 |
+
|
| 27 |
+
# PyInstaller
|
| 28 |
+
*.manifest
|
| 29 |
+
*.spec
|
| 30 |
+
|
| 31 |
+
# Installer logs
|
| 32 |
+
pip-log.txt
|
| 33 |
+
pip-delete-this-directory.txt
|
| 34 |
+
|
| 35 |
+
# Unit test / coverage reports
|
| 36 |
+
htmlcov/
|
| 37 |
+
.tox/
|
| 38 |
+
.nox/
|
| 39 |
+
.coverage
|
| 40 |
+
.coverage.*
|
| 41 |
+
.cache
|
| 42 |
+
nosetests.xml
|
| 43 |
+
coverage.xml
|
| 44 |
+
*.cover
|
| 45 |
+
*.py,cover
|
| 46 |
+
.hypothesis/
|
| 47 |
+
.pytest_cache/
|
| 48 |
+
|
| 49 |
+
# Translations
|
| 50 |
+
*.mo
|
| 51 |
+
*.pot
|
| 52 |
+
|
| 53 |
+
# Environments
|
| 54 |
+
.env
|
| 55 |
+
.venv
|
| 56 |
+
env/
|
| 57 |
+
venv/
|
| 58 |
+
ENV/
|
| 59 |
+
env.bak/
|
| 60 |
+
venv.bak/
|
| 61 |
+
|
| 62 |
+
# IDE
|
| 63 |
+
.idea/
|
| 64 |
+
.vscode/
|
| 65 |
+
*.swp
|
| 66 |
+
*.swo
|
| 67 |
+
|
| 68 |
+
# Jupyter Notebook
|
| 69 |
+
.ipynb_checkpoints
|
| 70 |
+
|
| 71 |
+
# pyenv
|
| 72 |
+
.python-version
|
| 73 |
+
|
| 74 |
+
# Model checkpoints
|
| 75 |
+
checkpoints/
|
| 76 |
+
*.pt
|
| 77 |
+
*.pth
|
| 78 |
+
*.bin
|
| 79 |
+
*.gguf
|
| 80 |
+
|
| 81 |
+
# Data
|
| 82 |
+
data/
|
| 83 |
+
*.model
|
| 84 |
+
|
| 85 |
+
# Logs
|
| 86 |
+
logs/
|
| 87 |
+
*.log
|
| 88 |
+
scripts/_logs/
|
| 89 |
+
tensorboard/
|
| 90 |
+
|
| 91 |
+
# Sessions (generated HTML/JSONL replays)
|
| 92 |
+
sessions/
|
| 93 |
+
|
| 94 |
+
# Archive (dead scripts, old backups)
|
| 95 |
+
_archive/
|
| 96 |
+
scripts/_archive/
|
| 97 |
+
|
| 98 |
+
# Large dataset files β use scripts to regenerate
|
| 99 |
+
biblioteca/
|
| 100 |
+
|
| 101 |
+
# OS
|
| 102 |
+
.DS_Store
|
| 103 |
+
Thumbs.db
|
.memoria/sesion_2026-03-16_linux-ia.md
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SesiΓ³n 2026-03-16 β PAMPAr como "el Linux de la IA"
|
| 2 |
+
|
| 3 |
+
> Archivo de memoria para el agente. Contexto de la sesiΓ³n de trabajo.
|
| 4 |
+
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
## CorrecciΓ³n crΓtica del agente
|
| 8 |
+
|
| 9 |
+
El agente redujo PAMPAr a "un generador de cΓ³digo en formato Problem/Solution".
|
| 10 |
+
El usuario corrigiΓ³: **PAMPAr es un copiloto local autΓ³nomo basado en arquitectura cerebral**.
|
| 11 |
+
|
| 12 |
+
### Lo que PAMPAr ES (nunca olvidar)
|
| 13 |
+
|
| 14 |
+
1. **Arquitectura cerebral 2D**: grilla cortical 4 streams Γ 5 niveles, inspirada en el cerebro humano
|
| 15 |
+
- TΓ‘lamo (routing) + LLAVES INT8 (80% reglas + 20% aprendido)
|
| 16 |
+
- 4 streams: SINTAXIS (B01-B15), SEMΓNTICA (B16-B30), LΓGICO (B31-B42), ESTRUCTURAL (B43-B52)
|
| 17 |
+
- Lateral Gates = fibras blancas (comunicaciΓ³n entre streams)
|
| 18 |
+
- GQA 4:1 (8Q / 2KV heads), RoPE, Early Exit
|
| 19 |
+
- 108M params, vocab 48K, max_seq_len 4096
|
| 20 |
+
|
| 21 |
+
2. **Copiloto local 100% offline**: corre en hardware consumer (GTX 1650, 4GB VRAM), sin cloud, sin APIs, sin telemetrΓa
|
| 22 |
+
|
| 23 |
+
3. **Agente autΓ³nomo con RAG del sistema**:
|
| 24 |
+
- Scanner inspecciona: workspace (ast.parse), paquetes (importlib), servicios (socket), hardware (torch.cuda)
|
| 25 |
+
- BootProtocol: CONCIENCIA.md β Scanner β AGENTS.md β RAG L2/L3
|
| 26 |
+
- La identidad (CONCIENCIA.md) es L3 inmutable, el entorno (AGENTS.md) es L2 mutable
|
| 27 |
+
- "FΓsico con doctorado" que se especializa segΓΊn el "laboratorio" donde aterriza
|
| 28 |
+
|
| 29 |
+
4. **Skills reales**: LectorArchivos (ojos), EjecutorCodigo (manos), con Skill ABC para extensibilidad
|
| 30 |
+
|
| 31 |
+
5. **Memoria con Ley de Pareto**: RAGResidual + ClasificadorPareto (L0βL3) + ColaFinetune (auto-mejora)
|
| 32 |
+
|
| 33 |
+
6. **Loop autΓ³nomo**: prompt β genera β ejecuta β observa β aprende del error β reintenta
|
| 34 |
+
|
| 35 |
+
7. **VisiΓ³n final**: el modelo genera su propio AGENTS.md al aterrizar en un sistema nuevo
|
| 36 |
+
|
| 37 |
+
---
|
| 38 |
+
|
| 39 |
+
## AnalogΓa Linux β PAMPAr
|
| 40 |
+
|
| 41 |
+
| Linux | PAMPAr | Estado |
|
| 42 |
+
| -------------------------------- | ----------------------------------------------------- | ------------------------------- |
|
| 43 |
+
| Kernel | PamparV3 (108M, grilla cortical 2D) | β
Construido |
|
| 44 |
+
| DetecciΓ³n hardware (dmesg, udev) | Scanner (ast.parse, importlib, socket, torch.cuda) | β
Construido |
|
| 45 |
+
| Init system (systemd) | BootProtocol (CONCIENCIA β Scanner β AGENTS.md β RAG) | β
Construido |
|
| 46 |
+
| Filesystem | RAGResidual + ClasificadorPareto | β
Construido |
|
| 47 |
+
| Device drivers | Skills (Skill ABC β LectorArchivos, EjecutorCodigo) | β
Base, faltan mΓ‘s |
|
| 48 |
+
| Self-compilation | ColaFinetune (auto-SFT) | β
Wiring hecho, no probado e2e |
|
| 49 |
+
| Terminal/Shell | ??? (cli.py es parche, Continue no integrado) | β Falta |
|
| 50 |
+
| Corre en cualquier hardware | 4GB VRAM, CPU fallback | β
|
|
| 51 |
+
|
| 52 |
+
---
|
| 53 |
+
|
| 54 |
+
## Brechas detectadas
|
| 55 |
+
|
| 56 |
+
### 1. Kernel no probado en producciΓ³n
|
| 57 |
+
|
| 58 |
+
- 16/16 eval controlado, pero no probado con prompts reales
|
| 59 |
+
- SFT actual: Magicoder-OSS-75K (### Problem / ### Solution)
|
| 60 |
+
- **El modelo NO fue entrenado para el formato del Agente** ([LEER:], [EJECUTAR:], historial, RAG context)
|
| 61 |
+
- Brecha mΓ‘s crΓtica
|
| 62 |
+
|
| 63 |
+
### 2. Pocos drivers (skills)
|
| 64 |
+
|
| 65 |
+
- Solo 2 skills. Faltan: BuscarSkill, GitSkill, TerminalSkill, TestSkill, EditarSkill
|
| 66 |
+
|
| 67 |
+
### 3. Sin interfaz real
|
| 68 |
+
|
| 69 |
+
- cli.py llama a generate() directo, NO al Agente (sin RAG, sin skills, sin memoria)
|
| 70 |
+
- Continue necesita HTTP server OpenAI-compatible
|
| 71 |
+
|
| 72 |
+
### 4. Auto-mejora no probada end-to-end
|
| 73 |
+
|
| 74 |
+
- ColaFinetune β mini-SFT β reload pesos nunca corriΓ³ completo
|
| 75 |
+
|
| 76 |
+
---
|
| 77 |
+
|
| 78 |
+
## Estrategia propuesta (3 fases)
|
| 79 |
+
|
| 80 |
+
### Fase A β El kernel funciona de verdad (AHORA)
|
| 81 |
+
|
| 82 |
+
1. Entrenar modelo con datos en formato del Agente (system prompt + RAG + acciones + historial)
|
| 83 |
+
2. CLI usa el Agente real, no generate() directo
|
| 84 |
+
|
| 85 |
+
### Fase B β MΓ‘s drivers, shell funcional
|
| 86 |
+
|
| 87 |
+
3. 3-4 skills mΓ‘s (buscar, editar, git, tests)
|
| 88 |
+
4. HTTP server OpenAI-compatible (Continue)
|
| 89 |
+
5. Loop auto-mejora probado end-to-end
|
| 90 |
+
|
| 91 |
+
### Fase C β DistribuciΓ³n empaquetada
|
| 92 |
+
|
| 93 |
+
6. pip install pampar-coder
|
| 94 |
+
7. IntegraciΓ³n Continue nativa
|
| 95 |
+
8. DocumentaciΓ³n tipo man pages
|
| 96 |
+
|
| 97 |
+
---
|
| 98 |
+
|
| 99 |
+
## Ventaja competitiva
|
| 100 |
+
|
| 101 |
+
- Arquitectura cerebral (no transformer genΓ©rico)
|
| 102 |
+
- 108M params en 4GB VRAM local
|
| 103 |
+
- RAG del sistema como contexto (sabe quΓ© hay en tu mΓ‘quina)
|
| 104 |
+
- Auto-aprendizaje (ColaFinetune)
|
| 105 |
+
- Sin cloud, sin telemetrΓa, 100% tuyo
|
| 106 |
+
- = Propuesta de valor de Linux vs Windows/macOS en los 90s
|
| 107 |
+
|
| 108 |
+
---
|
| 109 |
+
|
| 110 |
+
## Estado del proyecto (Mar 2026)
|
| 111 |
+
|
| 112 |
+
- **Modelo activo**: PamparV3 β 108.3M params, vocab 48K
|
| 113 |
+
- **Mejor checkpoint**: v3_sft_v8.pt β 16/16 eval
|
| 114 |
+
- **Tests**: 109+ passing
|
| 115 |
+
- **Milestone 1** β
β 16/16 eval
|
| 116 |
+
- **Milestone 2** β
β Runtime loop (chat.py + ColaFinetune + mini-SFT wiring)
|
| 117 |
+
- **Milestone 3** β
β Protocolo (generador determinista AGENTS.md)
|
| 118 |
+
- **Milestone 4** β³ β VS Code / Continue integration
|
| 119 |
+
- **Milestone 5** β³ β Voz TTS
|
| 120 |
+
|
| 121 |
+
## Archivos clave del proyecto
|
| 122 |
+
|
| 123 |
+
```
|
| 124 |
+
pampar/CONCIENCIA.md β Identidad invariante (L3)
|
| 125 |
+
AGENTS.md β Protocolo de despliegue (L2, mutable)
|
| 126 |
+
ROADMAP.md β Plan de evoluciΓ³n
|
| 127 |
+
pampar/coder/v3/modelo.py β PamparV3 (108M)
|
| 128 |
+
pampar/coder/v3/talamo.py β TalamoInicial (routing cerebral)
|
| 129 |
+
pampar/coder/v3/bloques.py β NivelProfundo, StreamFFN, LateralGate
|
| 130 |
+
pampar/coder/v3/llaves.py β LLAVES INT8 (lookup tables)
|
| 131 |
+
pampar/coder/v3/zonas.py β 52 Zonas de Brodmann
|
| 132 |
+
pampar/runtime/agente.py β Agente (orquestador)
|
| 133 |
+
pampar/runtime/scanner.py β Scanner (inspecciΓ³n del entorno)
|
| 134 |
+
pampar/runtime/boot.py β BootProtocol (secuencia de arranque)
|
| 135 |
+
pampar/runtime/generar_agents.py β Generador AGENTS.md
|
| 136 |
+
pampar/memoria/rag.py β RAGResidual (vector store)
|
| 137 |
+
pampar/memoria/clasificador.py β ClasificadorPareto (L0-L3)
|
| 138 |
+
pampar/memoria/cola_finetune.py β ColaFinetune (auto-SFT)
|
| 139 |
+
pampar/skills/base.py β Skill ABC
|
| 140 |
+
pampar/skills/lector_archivos.py β LectorArchivos (ojos)
|
| 141 |
+
pampar/skills/ejecutar_codigo.py β EjecutorCodigo (manos)
|
| 142 |
+
pampar/cli.py β CLI (parche, no usa Agente)
|
| 143 |
+
pampar/inference.py β JSON-lines server (base para HTTP)
|
| 144 |
+
checkpoints/v3_sft_v8.pt β Mejor checkpoint (16/16)
|
| 145 |
+
data/tokenizer/pampar_48k.model β Tokenizer activo
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
## Hardware del usuario
|
| 149 |
+
|
| 150 |
+
- **GPU**: GTX 1650 (4GB VRAM)
|
| 151 |
+
- **Python**: 3.13 (C:\Users\lucas\AppData\Local\Programs\Python\Python313\python.exe)
|
| 152 |
+
- **torch**: 2.6.0+cu124
|
| 153 |
+
- **OS**: Windows
|
| 154 |
+
- **.venv en Lunux-AI/.venv**: NO tiene torch β no usar para inferencia
|
.zenodo.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"title": "PAMPAr-Coder V3: A Brain-Inspired 2D Stream Architecture with Mixed Selectivity for Efficient Code Generation",
|
| 3 |
+
"description": "<p>PAMPAr-Coder V3 is a compact code generation language model with 62.6M parameters, designed to train and run at full FP16 precision on consumer GPUs with as little as 4GB VRAM.</p><p>The architecture introduces two key innovations: (1) a <strong>2D Stream organization</strong> that arranges computation as four specialized cortical streams (Syntax, Semantics, Logic, Structural) across five depth levels, connected by bidirectional lateral gates analogous to white-matter fiber tracts; and (2) <strong>Mixed Selectivity via FiLM</strong>, where a single shared Feed-Forward Network per level is dynamically re-read by context-dependent gamma/beta modulators derived from a 63-dimensional context vector β reducing FFN parameters by ~73% per level compared to four independent networks.</p><p>Additional contributions include TalamoNivel adaptive per-level re-routing, Grouped Query Attention (8Q/2KV), and the MotorCuriosidad ZPD-based curriculum scheduler that adapts training difficulty across 161 topic categories and 3.2M lines of code data.</p><p>After 55,000 training steps, the model reaches a cross-entropy loss of ~1.38 and saturates 29 of 40 curriculum topics at level 1, demonstrating active learning progress with training still ongoing.</p><p>This release includes: full model source code (PyTorch), training scripts, PAMPAr-48k bilingual tokenizer (48K vocabulary), configuration, and the research paper preprint.</p>",
|
| 4 |
+
"upload_type": "software",
|
| 5 |
+
"access_right": "open",
|
| 6 |
+
"license": "other-open",
|
| 7 |
+
"creators": [
|
| 8 |
+
{
|
| 9 |
+
"name": "Mella Chillemi, Lucas Ricardo",
|
| 10 |
+
"affiliation": "Independent Researcher",
|
| 11 |
+
"orcid": ""
|
| 12 |
+
}
|
| 13 |
+
],
|
| 14 |
+
"keywords": [
|
| 15 |
+
"language model",
|
| 16 |
+
"code generation",
|
| 17 |
+
"brain-inspired AI",
|
| 18 |
+
"mixed selectivity",
|
| 19 |
+
"FiLM modulation",
|
| 20 |
+
"grouped query attention",
|
| 21 |
+
"curriculum learning",
|
| 22 |
+
"ZPD",
|
| 23 |
+
"cortical streams",
|
| 24 |
+
"lateral gates",
|
| 25 |
+
"LLAVES routing",
|
| 26 |
+
"parameter efficiency",
|
| 27 |
+
"PyTorch",
|
| 28 |
+
"natural language processing",
|
| 29 |
+
"deep learning"
|
| 30 |
+
],
|
| 31 |
+
"related_identifiers": [
|
| 32 |
+
{
|
| 33 |
+
"scheme": "doi",
|
| 34 |
+
"identifier": "10.5281/zenodo.18315642",
|
| 35 |
+
"relation": "isCitedBy",
|
| 36 |
+
"resource_type": "software"
|
| 37 |
+
}
|
| 38 |
+
],
|
| 39 |
+
"references": [
|
| 40 |
+
"Perez E., Strub F., de Vries H., Dumoulin V., Courville A. (2018). FiLM: Visual Reasoning with a General Conditioning Layer. AAAI 2018.",
|
| 41 |
+
"Rigotti M., Barak O., Warden M. R., et al. (2013). The importance of mixed selectivity in complex cognitive tasks. Nature, 497, 585-590.",
|
| 42 |
+
"Ainslie J., Lee-Thorp J., de Jong M., et al. (2023). GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints. EMNLP 2023.",
|
| 43 |
+
"Roziere B., Gehring J., Gloeckle F., et al. (2023). Code Llama: Open Foundation Models for Code. arXiv:2308.12950.",
|
| 44 |
+
"Lozhkov A., Li R., Allal L. B., et al. (2024). StarCoder2 and the Stack v2: The Next Generation. arXiv:2402.19173.",
|
| 45 |
+
"Vygotsky L. S. (1978). Mind in Society: The Development of Higher Psychological Processes. Harvard University Press.",
|
| 46 |
+
"Mella Chillemi, L. R. (2026). PAMPAr-o1 v9: A Brain-Inspired Territorial Architecture for Language Modeling. DOI: 10.5281/zenodo.18315642.",
|
| 47 |
+
"Elhage N., Hume T., Gray C., et al. (2022). Toy Models of Superposition. Transformer Circuits Thread.",
|
| 48 |
+
"Felleman D. J. & Van Essen D. C. (1991). Distributed hierarchical processing in the primate cerebral cortex. Cerebral Cortex, 1(1), 1-47."
|
| 49 |
+
],
|
| 50 |
+
"notes": "This software is released under the Business Source License 1.1 (BUSL-1.1). The paper preprint is available in the paper/ directory. The model is intended for research and non-commercial use. Commercial licensing inquiries: lucas.mella@outlook.com",
|
| 51 |
+
"version": "3.0.0",
|
| 52 |
+
"language": "eng",
|
| 53 |
+
"subjects": [
|
| 54 |
+
{
|
| 55 |
+
"term": "Computer Science - Computation and Language",
|
| 56 |
+
"identifier": "cs.CL",
|
| 57 |
+
"scheme": "arXiv"
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"term": "Computer Science - Machine Learning",
|
| 61 |
+
"identifier": "cs.LG",
|
| 62 |
+
"scheme": "arXiv"
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"term": "Computer Science - Programming Languages",
|
| 66 |
+
"identifier": "cs.PL",
|
| 67 |
+
"scheme": "arXiv"
|
| 68 |
+
}
|
| 69 |
+
]
|
| 70 |
+
}
|
AGENTS.md
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PAMPAr β Repository Guidelines
|
| 2 |
+
|
| 3 |
+
> **PAMPAr** = Procesador AutΓ³nomo Modular de Patrones y Razonamiento
|
| 4 |
+
> Para AI agents: Claude Code, Codex, Gemini CLI, GitHub Copilot.
|
| 5 |
+
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
## VisiΓ³n
|
| 9 |
+
|
| 10 |
+
PAMPAr es un **motor de razonamiento puro** de 108M parΓ‘metros. No memoriza respuestas β aprende a **pensar con informaciΓ³n de referencia**.
|
| 11 |
+
|
| 12 |
+
La analogΓa: un fΓsico que entiende termodinΓ‘mica puede resolver problemas de quΓmica, ingenierΓa o biologΓa. No memorizΓ³ cada campo β tiene los axiomas correctos.
|
| 13 |
+
|
| 14 |
+
- Los **pesos** (108M params) contienen la capacidad de **razonar**: leer documentaciΓ³n, entender un problema, derivar una soluciΓ³n step-by-step.
|
| 15 |
+
- El **dispositivo** (PC, mΓ³vil, servidor) provee el **conocimiento**: docs de Python, MDN, man pages, archivos del usuario β vΓa RAG local.
|
| 16 |
+
- El modelo no necesita "saber Python". Necesita saber **usar la referencia que tiene disponible** para resolver cualquier problema.
|
| 17 |
+
|
| 18 |
+
**Objetivo**: un modelo local que razona con la misma metodologΓa que los mejores modelos, usando la informaciΓ³n del dispositivo como RAG.
|
| 19 |
+
|
| 20 |
+
---
|
| 21 |
+
|
| 22 |
+
## Estado actual
|
| 23 |
+
|
| 24 |
+
- **Modelo activo**: `PamparV3` β **108.3M params**, vocab 48K, 4 streams Γ 5 niveles
|
| 25 |
+
- **Mejor checkpoint**: `v3_ghidra_v9.pt` β Routing Score 89, eval 6/16 (38%)
|
| 26 |
+
- **Tokenizer**: `data/tokenizer/pampar_48k.model` (48K, bilingΓΌe ES+cΓ³digo)
|
| 27 |
+
- **Runtime**: Agente + RAGResidual + Scanner + BootProtocol β funcional
|
| 28 |
+
- **Classroom**: Mentor conversacional con Qwen-plus β lecciones dinΓ‘micas, 21 conceptos adaptativos, absorciΓ³n + prΓ‘ctica + correcciΓ³n
|
| 29 |
+
- **Bio-Mechanisms**: NeuromodulaciΓ³n, LTP, Sleep Consolidation, Neurogenesis, Synaptic Pruning β `bio_mechanisms.py`
|
| 30 |
+
- **Teacher API**: Qwen-plus via DashScope (principal), GitHub Models gpt-4o-mini (alternativa)
|
| 31 |
+
- **Training data**: `master_sft.jsonl` β 1,253 ejemplos (en expansiΓ³n vΓa Classroom)
|
| 32 |
+
|
| 33 |
+
---
|
| 34 |
+
|
| 35 |
+
## Quick Reference
|
| 36 |
+
|
| 37 |
+
| Area | Convention |
|
| 38 |
+
| ---------- | ------------------------------------------------------ |
|
| 39 |
+
| Language | Python 3.13+ |
|
| 40 |
+
| Framework | PyTorch 2.6+ |
|
| 41 |
+
| Tokenizer | SentencePiece BPE β **48K** (`pampar_48k.model`) |
|
| 42 |
+
| Type hints | Always required |
|
| 43 |
+
| Docstrings | Google style |
|
| 44 |
+
| Training | Local GTX 1650 (4 GB) + RunPod A100 para fases pesadas |
|
| 45 |
+
| Budget | $300-500 USD total |
|
| 46 |
+
|
| 47 |
+
---
|
| 48 |
+
|
| 49 |
+
## Arquitectura β PamparV3
|
| 50 |
+
|
| 51 |
+
### Grilla 2D: 4 streams Γ 5 niveles
|
| 52 |
+
|
| 53 |
+
```
|
| 54 |
+
tok_emb [48K Γ 640]
|
| 55 |
+
β TalamoInicial β terr_acts [B, L, 4] / zona_acts [B, L, 52]
|
| 56 |
+
β 4 streams paralelos (dim=640)
|
| 57 |
+
|
| 58 |
+
Cada NivelProfundo (Γ5):
|
| 59 |
+
1. GQA AtenciΓ³n compartida (8 Q heads / 2 KV heads, head_dim=80)
|
| 60 |
+
2. Re-routing ligero del TΓ‘lamo (Linear dimβ52, sin bias)
|
| 61 |
+
3. 4 Γ StreamFFN SwiGLU independientes (uno por stream)
|
| 62 |
+
4. Lateral gates por stream (bottleneck=128, fibras blancas)
|
| 63 |
+
|
| 64 |
+
β norm_f (RMSNorm) β lm_head (weight-tied, vocab=48K)
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
### Streams β Capas lingΓΌΓsticas
|
| 68 |
+
|
| 69 |
+
| Stream | Territorio | Zonas | EspecializaciΓ³n | Capa lingΓΌΓstica |
|
| 70 |
+
| ------ | ----------- | ------- | ----------------------------------------- | ---------------- |
|
| 71 |
+
| 0 | SINTAXIS | B01-B15 | Keywords, delimitadores, puntuaciΓ³n | Sintaxis |
|
| 72 |
+
| 1 | SEMANTICA | B16-B30 | Variables, tipos, literales | SemΓ‘ntica |
|
| 73 |
+
| 2 | LOGICO | B31-B42 | Operadores, flujo de control, excepciones | PragmΓ‘tica |
|
| 74 |
+
| 3 | ESTRUCTURAL | B43-B52 | IndentaciΓ³n, bloques, patrones | Discurso |
|
| 75 |
+
|
| 76 |
+
### PRESET_V3
|
| 77 |
+
|
| 78 |
+
| ParΓ‘metro | Valor |
|
| 79 |
+
| ---------------- | ----------- |
|
| 80 |
+
| `dim` | 640 |
|
| 81 |
+
| `n_streams` | 4 |
|
| 82 |
+
| `n_levels` | 5 |
|
| 83 |
+
| `n_heads` | 8 |
|
| 84 |
+
| `n_kv_heads` | 2 (GQA 4:1) |
|
| 85 |
+
| `vocab_size` | 48 000 |
|
| 86 |
+
| `max_seq_len` | 4096 |
|
| 87 |
+
| **Total params** | **108.3M** |
|
| 88 |
+
|
| 89 |
+
---
|
| 90 |
+
|
| 91 |
+
## Subsistemas
|
| 92 |
+
|
| 93 |
+
### 1. Modelo (`pampar/coder/v3/`)
|
| 94 |
+
|
| 95 |
+
| Archivo | LΓneas | PropΓ³sito |
|
| 96 |
+
| ------------------- | ------ | ------------------------------------------------------------------------------- |
|
| 97 |
+
| `modelo.py` | 310 | PamparV3: forward, generate (nucleus sampling) |
|
| 98 |
+
| `config.py` | 226 | ConfigV3, 3 presets (V3/SMALL/LARGE) |
|
| 99 |
+
| `bloques.py` | 395 | RMSNorm, RoPE, BloqueAttn (GQA), StreamFFN (SwiGLU), LateralGate, NivelProfundo |
|
| 100 |
+
| `talamo.py` | 133 | TalamoInicial: LLAVES 80% + attn_proj 20% + context_conv |
|
| 101 |
+
| `llaves.py` | 266 | LlavesV2: clasificar_token(), tabla INT8, agregar_zonas_a_territorios |
|
| 102 |
+
| `zonas.py` | 265 | Territorio(IntEnum), Zona(IntEnum), ZONAS dict, ZONA_TERRITORIO |
|
| 103 |
+
| `ghidra_probe.py` | 343 | GhidraProbe: 36 forward hooks, diagnosis/debugging |
|
| 104 |
+
| `engrama_stream.py` | 359 | BancoEngrama: O(1) activation memory, cosine-gated injection |
|
| 105 |
+
|
| 106 |
+
### 2. Memoria (`pampar/memoria/`)
|
| 107 |
+
|
| 108 |
+
| Archivo | PropΓ³sito |
|
| 109 |
+
| ------------------ | ----------------------------------------------------------------------------- |
|
| 110 |
+
| `clasificador.py` | ClasificadorPareto: scoring L0-L3 por densidad, novedad, loss, frecuencia |
|
| 111 |
+
| `rag.py` | RAGResidual: FAISS + sentence-transformers (fallback TF-IDF), 5K entradas max |
|
| 112 |
+
| `cola_finetune.py` | ColaFinetune: acumula L3, exporta JSONL, propone mini-SFT |
|
| 113 |
+
|
| 114 |
+
### 3. Runtime (`pampar/runtime/`)
|
| 115 |
+
|
| 116 |
+
| Archivo | PropΓ³sito |
|
| 117 |
+
| ------------------- | ------------------------------------------------------------------ |
|
| 118 |
+
| `agente.py` | Orquestador: promptβRAGβgenerarβskillsβretryβauto-SFT |
|
| 119 |
+
| `scanner.py` | InspecciΓ³n del dispositivo: OS, GPU, paquetes, servicios, archivos |
|
| 120 |
+
| `boot.py` | BootProtocol: CONCIENCIA.md (L3) β Scanner (L2) β Workspace (L1) |
|
| 121 |
+
| `generar_agents.py` | Genera AGENTS.md contextual desde ResultadoScan |
|
| 122 |
+
|
| 123 |
+
### 4. Skills (`pampar/skills/`)
|
| 124 |
+
|
| 125 |
+
| Archivo | PropΓ³sito |
|
| 126 |
+
| -------------------- | --------------------------------------------------------- |
|
| 127 |
+
| `lector_archivos.py` | Lee archivos del dispositivo (30+ extensiones, sandboxed) |
|
| 128 |
+
| `ejecutar_codigo.py` | Ejecuta cΓ³digo en subprocess con timeout y blocklist |
|
| 129 |
+
|
| 130 |
+
### 5. Inference (`pampar/inference.py`)
|
| 131 |
+
|
| 132 |
+
Servidor JSON-lines stdin/stdout para extensiΓ³n VS Code. Commands: `infer`, `boot`.
|
| 133 |
+
|
| 134 |
+
### 6. Classroom β Mentor Conversacional + Bio-Mechanisms
|
| 135 |
+
|
| 136 |
+
Sistema donde Qwen-plus actΓΊa como mentor conversacional β genera explicaciones, ejemplos y ejercicios dinΓ‘micos. PamparV3 absorbe el conocimiento via gradient descent en 3 phases por lecciΓ³n.
|
| 137 |
+
|
| 138 |
+
**Flujo**: StudentProfile β Mentor genera lecciΓ³n β Phase A (absorber explicaciΓ³n+ejemplo) β Phase B (alumno intenta ejercicio) β Phase C (mentor corrige, entrenar en soluciΓ³n+replay) β actualizar perfil.
|
| 139 |
+
|
| 140 |
+
| MΓ³dulo | LΓneas | Responsabilidad |
|
| 141 |
+
| -------------------------- | ------ | ------------------------------------------------------------------------------ |
|
| 142 |
+
| `classroom.py` | ~608 | ClassroomEngine β motor conversacional (orquestador) |
|
| 143 |
+
| `classroom_curriculum.py` | ~433 | ClassroomConfig + CONCEPT_TREE (21 conceptos) + StudentProfile + concept_level |
|
| 144 |
+
| `classroom_teacher.py` | ~252 | Mentor API (Qwen/GitHub/OpenRouter) + parse de lecciones |
|
| 145 |
+
| `classroom_training.py` | ~211 | TokenizaciΓ³n + LR diferencial + train_step |
|
| 146 |
+
| `classroom_memory.py` | ~187 | EWC + ReplayBuffer + LessonResult + compute_ewc_baseline |
|
| 147 |
+
| `classroom_events.py` | ~104 | Formateo dict-based de eventos para consola |
|
| 148 |
+
| `classroom_persistence.py` | ~123 | Guardado de checkpoints, sesiones JSONL, grabaciones HTML |
|
| 149 |
+
| `classroom_server.py` | ~255 | HTTP SSE server + CLI entry point |
|
| 150 |
+
| `bio_mechanisms.py` | ~497 | 5 bio-mechanisms coordinados por BioOrchestrator |
|
| 151 |
+
|
| 152 |
+
**CONCEPT_TREE**: 21 conceptos en 5 niveles con prerequisitos (arithmetic β algorithms).
|
| 153 |
+
**StudentProfile**: mastery tracking adaptativo β prioriza refuerzo, luego nuevos, luego repaso.
|
| 154 |
+
|
| 155 |
+
| Mecanismo | PropΓ³sito |
|
| 156 |
+
| ------------------ | ----------------------------------------------------------------- |
|
| 157 |
+
| **EWC** | Elastic Weight Consolidation β penaliza cambios en pesos crΓticos |
|
| 158 |
+
| **Replay Buffer** | Mezcla ejemplos nuevos con anteriores (consolidaciΓ³n tipo sueΓ±o) |
|
| 159 |
+
| **LR Diferencial** | LLAVES 0.01x, atenciΓ³n 0.1x, embed 0.1x, FFN 1.0x |
|
| 160 |
+
| **Curriculum** | 5 niveles progresivos: bΓ‘sico β avanzado |
|
| 161 |
+
| **GrabaciΓ³n** | Genera HTML con replay interactivo de cada sesiΓ³n |
|
| 162 |
+
|
| 163 |
+
**Bio-Mechanisms** (5 mecanismos de neurociencia en `bio_mechanisms.py`):
|
| 164 |
+
|
| 165 |
+
| Mecanismo | ImplementaciΓ³n |
|
| 166 |
+
| ----------------------- | -------------------------------------------------------------------- |
|
| 167 |
+
| **NeuromodulaciΓ³n** | Dopamina/Norepinefrina modulan LR dinΓ‘micamente (Γ0.3 a Γ3.0) |
|
| 168 |
+
| **LTP** | Fortalece `LateralGate.scale` de streams activos (Hebb rule, cada 5) |
|
| 169 |
+
| **Sleep Consolidation** | REM (aleatorio) + SWS (ordenado por dificultad), cada 15 lecciones |
|
| 170 |
+
| **Neurogenesis** | LoRA adapters (rank=8) en StreamFFN cuando loss > 4.0, max 8 |
|
| 171 |
+
| **Synaptic Pruning** | Poda `LateralGate.scale < 0.03` cada 30 lecciones (decay Γ0.5) |
|
| 172 |
+
|
| 173 |
+
Coordinados por `BioOrchestrator.after_lesson()`. Desactivables con `--no-bio`.
|
| 174 |
+
|
| 175 |
+
**Resultados piloto mentor conversacional (5 lecciones)**: Loss absorciΓ³n ~7-8, loss ejercicios 5.89β3.94 (mejora), brain score 88.24% estable.
|
| 176 |
+
|
| 177 |
+
**APIs soportadas**: `qwen` (Qwen-plus via DashScope, principal), `github` (gpt-4o-mini), `openrouter` (requiere crΓ©ditos).
|
| 178 |
+
|
| 179 |
+
---
|
| 180 |
+
|
| 181 |
+
## Estructura del proyecto
|
| 182 |
+
|
| 183 |
+
```
|
| 184 |
+
PAMPAr-Coder/
|
| 185 |
+
βββ AGENTS.md # Este archivo β guΓa para AI agents
|
| 186 |
+
βββ README.md # DocumentaciΓ³n pΓΊblica
|
| 187 |
+
βββ PLAN.md # Plan de training y evoluciΓ³n
|
| 188 |
+
βββ pampar/
|
| 189 |
+
β βββ CONCIENCIA.md # Identidad invariante del modelo
|
| 190 |
+
β βββ coder/
|
| 191 |
+
β β βββ v3/ # ARQUITECTURA ACTIVA (108M)
|
| 192 |
+
β β βββ modelo.py # PamparV3 β forward, generate
|
| 193 |
+
β β βββ config.py # ConfigV3, presets
|
| 194 |
+
β β βββ talamo.py # TalamoInicial β routing
|
| 195 |
+
β β βββ bloques.py # GQA, SwiGLU, LateralGate, NivelProfundo
|
| 196 |
+
β β βββ llaves.py # LlavesV2 β lookup INT8
|
| 197 |
+
β β βββ zonas.py # 52 Zonas de Brodmann
|
| 198 |
+
β β βββ ghidra_probe.py # InstrumentaciΓ³n read-only
|
| 199 |
+
β β βββ engrama_stream.py# Memoria de activaciones
|
| 200 |
+
β βββ memoria/
|
| 201 |
+
β β βββ clasificador.py # ClasificadorPareto β niveles L0-L3
|
| 202 |
+
β β βββ rag.py # RAGResidual β vector store local
|
| 203 |
+
β β βββ cola_finetune.py # ColaFinetune β buffer auto-SFT
|
| 204 |
+
β βββ skills/
|
| 205 |
+
β β βββ lector_archivos.py # Lee archivos (sandboxed)
|
| 206 |
+
β β βββ ejecutar_codigo.py # Ejecuta cΓ³digo (subprocess)
|
| 207 |
+
β βββ runtime/
|
| 208 |
+
β β βββ agente.py # Orquestador principal
|
| 209 |
+
β β βββ scanner.py # InspecciΓ³n del dispositivo
|
| 210 |
+
β β βββ boot.py # Secuencia de arranque
|
| 211 |
+
β β βββ generar_agents.py # Generador de AGENTS.md
|
| 212 |
+
β βββ inference.py # Servidor JSON-lines para VS Code
|
| 213 |
+
βββ scripts/
|
| 214 |
+
β βββ classroom.py # ClassroomEngine β motor conversacional (~608 lΓneas)
|
| 215 |
+
β βββ classroom_curriculum.py # ClassroomConfig + CONCEPT_TREE + StudentProfile + concept_level
|
| 216 |
+
β βββ classroom_teacher.py # Mentor API β Qwen/GitHub/OpenRouter + parse de lecciones
|
| 217 |
+
β βββ classroom_training.py # TokenizaciΓ³n + LR diferencial + train_step
|
| 218 |
+
β βββ classroom_events.py # Formateo dict-based de eventos para consola
|
| 219 |
+
β βββ classroom_memory.py # EWC + ReplayBuffer + LessonResult + compute_ewc_baseline
|
| 220 |
+
β βββ classroom_persistence.py # Guardado de checkpoints, sesiones, grabaciones HTML
|
| 221 |
+
β βββ classroom_server.py # HTTP SSE server + CLI entry point
|
| 222 |
+
β βββ bio_mechanisms.py # 5 bio-mechanisms (Neuromod, LTP, Sleep, Neurogenesis, Pruning)
|
| 223 |
+
β βββ classroom_replay.html # Player HTML para replays
|
| 224 |
+
βββ sessions/ # Grabaciones de sesiones classroom
|
| 225 |
+
βββ data/
|
| 226 |
+
β βββ tokenizer/
|
| 227 |
+
β β βββ pampar_48k.model # Vocab 48K bilingΓΌe
|
| 228 |
+
β βββ *.jsonl # Datasets de training
|
| 229 |
+
βββ checkpoints/
|
| 230 |
+
β βββ v3_ghidra_v9.pt # Mejor checkpoint actual
|
| 231 |
+
βββ _archive/ # Backups de archivos antes de refactorizar
|
| 232 |
+
βββ tests/
|
| 233 |
+
```
|
| 234 |
+
|
| 235 |
+
---
|
| 236 |
+
|
| 237 |
+
## Critical Rules
|
| 238 |
+
|
| 239 |
+
- **vocab_size = 48K** β DEBE coincidir con `pampar_48k.model`
|
| 240 |
+
- **Tokenizer path**: usar `PRESET_V3.tokenizer_path` o constante compartida β no hardcodear
|
| 241 |
+
- LLAVES son INT8 pre-computadas β **nunca** en el grafo de gradientes
|
| 242 |
+
- Los 4 streams procesan en **paralelo** β sin secuencialidad entre streams
|
| 243 |
+
- `targets.reshape(-1)` siempre, nunca `.view(-1)` (tensores no-contiguos)
|
| 244 |
+
- `generate()` usa `max_tokens`, NO `max_new_tokens`
|
| 245 |
+
- Imports: `pampar.memoria.*`, `pampar.skills.*`, `pampar.runtime.*`
|
| 246 |
+
- **Backups**: antes de borrar/refactorizar, mover el original a `_archive/`
|
| 247 |
+
|
| 248 |
+
## Naming Conventions
|
| 249 |
+
|
| 250 |
+
- **EspaΓ±ol** para conceptos del dominio: `Talamo`, `Territorio`, `Zona`, `LLAVES`, `Agente`, `Scanner`
|
| 251 |
+
- **InglΓ©s** para ML estΓ‘ndar: `forward`, `embedding`, `hidden_states`, `loss`, `generate`
|
| 252 |
+
|
| 253 |
+
## Paradigma de inferencia
|
| 254 |
+
|
| 255 |
+
```
|
| 256 |
+
1. Usuario hace una pregunta/pedido
|
| 257 |
+
2. Scanner provee contexto del dispositivo (OS, paquetes, archivos)
|
| 258 |
+
3. RAGResidual busca referencia relevante (docs, cΓ³digo, memoria)
|
| 259 |
+
4. Prompt se arma: [SYSTEM] + [REFERENCIA RAG] + [CONTEXTO DISPOSITIVO] + [PREGUNTA]
|
| 260 |
+
5. Modelo RAZONA sobre la referencia y genera soluciΓ³n step-by-step
|
| 261 |
+
6. Skills ejecutan la soluciΓ³n si aplica (cΓ³digo, lectura, tests)
|
| 262 |
+
7. Si falla β retry con error como contexto β ColaFinetune acumula patrones
|
| 263 |
+
```
|
| 264 |
+
|
| 265 |
+
---
|
| 266 |
+
|
| 267 |
+
## Instructions Files
|
| 268 |
+
|
| 269 |
+
Detailed instructions in `.github/instructions/`:
|
| 270 |
+
|
| 271 |
+
- `global-profile.instructions.md` β perfil del desarrollador
|
| 272 |
+
- `testing.instructions.md` β reglas de testing (pytest)
|
| 273 |
+
- `git-workflow.instructions.md` β commits convencionales
|
| 274 |
+
- `docker-devops.instructions.md` β Docker, CI/CD
|
CITATION.cff
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cff-version: 1.2.0
|
| 2 |
+
message: >-
|
| 3 |
+
If you use PAMPAr-Coder V3 in your research or products,
|
| 4 |
+
please cite it using the metadata below.
|
| 5 |
+
|
| 6 |
+
title: >-
|
| 7 |
+
PAMPAr-Coder V3: A Brain-Inspired 2D Stream Architecture
|
| 8 |
+
with Mixed Selectivity for Efficient Code Generation
|
| 9 |
+
|
| 10 |
+
version: "3.0.0"
|
| 11 |
+
|
| 12 |
+
doi: "10.5281/zenodo.XXXXXXX"
|
| 13 |
+
|
| 14 |
+
date-released: "2026-04-07"
|
| 15 |
+
|
| 16 |
+
license: "BUSL-1.1"
|
| 17 |
+
|
| 18 |
+
repository-code: "https://github.com/lucasmella-stack/PAMPAr-Coder"
|
| 19 |
+
|
| 20 |
+
abstract: >-
|
| 21 |
+
PAMPAr-Coder V3 is a 62.6M-parameter code language model with a
|
| 22 |
+
brain-inspired 2D Stream architecture. It organizes computation as four
|
| 23 |
+
specialized cortical streams (Syntax, Semantics, Logic, Structural) across
|
| 24 |
+
five depth levels connected by lateral gates. A single shared Feed-Forward
|
| 25 |
+
Network per level is dynamically modulated via FiLM (Feature-wise Linear
|
| 26 |
+
Modulation) using a 63-dimensional context vector, reducing FFN parameters
|
| 27 |
+
by 73% versus independent per-stream networks. A ZPD-based curriculum
|
| 28 |
+
scheduler (MotorCuriosidad) adapts training difficulty across 161 topic
|
| 29 |
+
categories. The model trains end-to-end at FP16 precision on a consumer
|
| 30 |
+
4GB GPU β a regime requiring quantization for competing models.
|
| 31 |
+
|
| 32 |
+
keywords:
|
| 33 |
+
- code generation
|
| 34 |
+
- language model
|
| 35 |
+
- brain-inspired architecture
|
| 36 |
+
- mixed selectivity
|
| 37 |
+
- FiLM modulation
|
| 38 |
+
- grouped query attention
|
| 39 |
+
- curriculum learning
|
| 40 |
+
- ZPD
|
| 41 |
+
- parameter efficiency
|
| 42 |
+
- cortical streams
|
| 43 |
+
- lateral gates
|
| 44 |
+
- LLAVES routing
|
| 45 |
+
|
| 46 |
+
authors:
|
| 47 |
+
- family-names: "Mella Chillemi"
|
| 48 |
+
given-names: "Lucas Ricardo"
|
| 49 |
+
affiliation: "Independent Researcher"
|
| 50 |
+
city: "Buenos Aires"
|
| 51 |
+
country: "AR"
|
| 52 |
+
email: "lucas.mella@outlook.com"
|
| 53 |
+
|
| 54 |
+
references:
|
| 55 |
+
- type: article
|
| 56 |
+
title: >-
|
| 57 |
+
PAMPAr-o1 v9: A Brain-Inspired Territorial Architecture for Language
|
| 58 |
+
Modeling with Explicit Rule-Based Routing
|
| 59 |
+
authors:
|
| 60 |
+
- family-names: "Mella Chillemi"
|
| 61 |
+
given-names: "Lucas Ricardo"
|
| 62 |
+
year: 2026
|
| 63 |
+
doi: "10.5281/zenodo.18315642"
|
| 64 |
+
|
| 65 |
+
- type: article
|
| 66 |
+
title: >-
|
| 67 |
+
FiLM: Visual Reasoning with a General Conditioning Layer
|
| 68 |
+
authors:
|
| 69 |
+
- family-names: "Perez"
|
| 70 |
+
given-names: "Ethan"
|
| 71 |
+
- family-names: "Strub"
|
| 72 |
+
given-names: "Florian"
|
| 73 |
+
- family-names: "de Vries"
|
| 74 |
+
given-names: "Harm"
|
| 75 |
+
- family-names: "Dumoulin"
|
| 76 |
+
given-names: "Vincent"
|
| 77 |
+
- family-names: "Courville"
|
| 78 |
+
given-names: "Aaron"
|
| 79 |
+
year: 2018
|
| 80 |
+
conference:
|
| 81 |
+
name: "AAAI 2018"
|
| 82 |
+
|
| 83 |
+
- type: article
|
| 84 |
+
title: "The importance of mixed selectivity in complex cognitive tasks"
|
| 85 |
+
authors:
|
| 86 |
+
- family-names: "Rigotti"
|
| 87 |
+
given-names: "Mattia"
|
| 88 |
+
- family-names: "Barak"
|
| 89 |
+
given-names: "Omri"
|
| 90 |
+
year: 2013
|
| 91 |
+
journal: "Nature"
|
| 92 |
+
volume: 497
|
| 93 |
+
start: 585
|
| 94 |
+
end: 590
|
| 95 |
+
|
| 96 |
+
- type: article
|
| 97 |
+
title: >-
|
| 98 |
+
GQA: Training Generalized Multi-Query Transformer Models from
|
| 99 |
+
Multi-Head Checkpoints
|
| 100 |
+
authors:
|
| 101 |
+
- family-names: "Ainslie"
|
| 102 |
+
given-names: "Joshua"
|
| 103 |
+
year: 2023
|
| 104 |
+
conference:
|
| 105 |
+
name: "EMNLP 2023"
|
CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Contributing to PAMPAr-Coder
|
| 2 |
+
|
| 3 |
+
Thank you for your interest in contributing! PAMPAr-Coder is licensed under
|
| 4 |
+
[BUSL-1.1](LICENSE) β contributions are welcome for non-commercial research
|
| 5 |
+
and academic purposes.
|
| 6 |
+
|
| 7 |
+
## How to Contribute
|
| 8 |
+
|
| 9 |
+
### Reporting Bugs
|
| 10 |
+
|
| 11 |
+
1. Check [existing issues](https://github.com/lucasmella-stack/PAMPAr-Coder/issues) first.
|
| 12 |
+
2. Open a new issue with:
|
| 13 |
+
- Steps to reproduce
|
| 14 |
+
- Expected vs actual behavior
|
| 15 |
+
- Python version, OS, GPU (if relevant)
|
| 16 |
+
|
| 17 |
+
### Suggesting Features
|
| 18 |
+
|
| 19 |
+
Open an issue with the `enhancement` label describing the feature and its use case.
|
| 20 |
+
|
| 21 |
+
### Pull Requests
|
| 22 |
+
|
| 23 |
+
1. Fork the repository.
|
| 24 |
+
2. Create a feature branch: `git checkout -b feat/your-feature`
|
| 25 |
+
3. Follow existing code style (type hints, Google-style docstrings).
|
| 26 |
+
4. Add tests for new functionality (`pytest`).
|
| 27 |
+
5. Run the test suite: `python -m pytest tests/ -v`
|
| 28 |
+
6. Commit with [Conventional Commits](https://www.conventionalcommits.org/):
|
| 29 |
+
`feat:`, `fix:`, `test:`, `docs:`, `refactor:`
|
| 30 |
+
7. Open a PR against `main`.
|
| 31 |
+
|
| 32 |
+
## Development Setup
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
git clone https://github.com/lucasmella-stack/PAMPAr-Coder.git
|
| 36 |
+
cd PAMPAr-Coder
|
| 37 |
+
python -m venv .venv
|
| 38 |
+
.venv/Scripts/activate # Windows
|
| 39 |
+
pip install -r requirements.txt
|
| 40 |
+
python -m pytest tests/ -v
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
## Code Style
|
| 44 |
+
|
| 45 |
+
- Python 3.11+ with type hints everywhere
|
| 46 |
+
- Docstrings: Google style
|
| 47 |
+
- No hardcoded paths or secrets
|
| 48 |
+
- Functions β€ 50 lines, files β€ 400 lines
|
| 49 |
+
|
| 50 |
+
## License
|
| 51 |
+
|
| 52 |
+
By contributing, you agree that your contributions will be licensed under the
|
| 53 |
+
same [BUSL-1.1](LICENSE) license.
|
LICENSE
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Business Source License 1.1
|
| 2 |
+
|
| 3 |
+
Parameters
|
| 4 |
+
|
| 5 |
+
Licensor: Lucas Ricardo Mella Chillemi
|
| 6 |
+
Licensed Work: PAMPAr-Coder V3
|
| 7 |
+
The Licensed Work is (c) 2025-2026 Lucas Ricardo Mella Chillemi.
|
| 8 |
+
Additional Use Grant: You may use the Licensed Work for non-commercial research,
|
| 9 |
+
academic citation, personal experimentation, and educational
|
| 10 |
+
purposes. Production use for commercial purposes requires a
|
| 11 |
+
separate commercial license from the Licensor.
|
| 12 |
+
Change Date: April 7, 2030
|
| 13 |
+
Change License: Apache License, Version 2.0
|
| 14 |
+
|
| 15 |
+
For information about alternative licensing arrangements for the Licensed Work,
|
| 16 |
+
please contact: lucas.mella@outlook.com
|
| 17 |
+
|
| 18 |
+
---
|
| 19 |
+
|
| 20 |
+
Notice
|
| 21 |
+
|
| 22 |
+
The Business Source License (this document, or the "License") is not an Open
|
| 23 |
+
Source license. However, the Licensed Work will eventually be made available
|
| 24 |
+
under an Open Source License, as stated in this License.
|
| 25 |
+
|
| 26 |
+
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
|
| 27 |
+
"Business Source License" is a trademark of MariaDB Corporation Ab.
|
| 28 |
+
|
| 29 |
+
---
|
| 30 |
+
|
| 31 |
+
Terms
|
| 32 |
+
|
| 33 |
+
The Licensor hereby grants you the right to copy, modify, create derivative
|
| 34 |
+
works, redistribute, and make non-production use of the Licensed Work. The
|
| 35 |
+
Licensor may make an Additional Use Grant, above, permitting limited
|
| 36 |
+
production use.
|
| 37 |
+
|
| 38 |
+
Effective on the Change Date, or the fourth anniversary of the first publicly
|
| 39 |
+
available distribution of a specific version of the Licensed Work under this
|
| 40 |
+
License, whichever comes first, the Licensor hereby grants you rights under
|
| 41 |
+
the terms of the Change License, and the rights granted in the paragraph
|
| 42 |
+
above terminate.
|
| 43 |
+
|
| 44 |
+
If your use of the Licensed Work does not comply with the requirements
|
| 45 |
+
currently in effect as described in this License, you must purchase a
|
| 46 |
+
commercial license from the Licensor, its affiliated entities, or authorized
|
| 47 |
+
resellers, or you must refrain from using the Licensed Work.
|
| 48 |
+
|
| 49 |
+
All copies of the original and modified Licensed Work, and derivative works
|
| 50 |
+
of the Licensed Work, are subject to this License. This License applies
|
| 51 |
+
separately for each version of the Licensed Work and the Change Date may vary
|
| 52 |
+
for each version of the Licensed Work released by Licensor.
|
| 53 |
+
|
| 54 |
+
You must conspicuously display this License on each original or modified copy
|
| 55 |
+
of the Licensed Work. If you receive the Licensed Work in original or
|
| 56 |
+
modified form from a third party, the terms and conditions set forth in this
|
| 57 |
+
License apply to your use of that work.
|
| 58 |
+
|
| 59 |
+
Any use of the Licensed Work in violation of this License will automatically
|
| 60 |
+
terminate your rights under this License for the current and all other
|
| 61 |
+
versions of the Licensed Work.
|
| 62 |
+
|
| 63 |
+
This License does not grant you any right in any trademark or logo of
|
| 64 |
+
Licensor or its affiliates (provided that you may use a trademark or logo of
|
| 65 |
+
Licensor as expressly required by this License).
|
| 66 |
+
|
| 67 |
+
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
|
| 68 |
+
AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
|
| 69 |
+
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
|
| 70 |
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
|
| 71 |
+
TITLE.
|
| 72 |
+
|
| 73 |
+
MariaDB hereby grants you permission to use this License's text to license
|
| 74 |
+
your works, and to refer to it using the trademark "Business Source License",
|
| 75 |
+
as long as you comply with the Covenants of Licensor below.
|
| 76 |
+
|
| 77 |
+
---
|
| 78 |
+
|
| 79 |
+
Covenants of Licensor
|
| 80 |
+
|
| 81 |
+
In consideration of the right to use this License's text and the "Business
|
| 82 |
+
Source License" name and trademark, Licensor covenants to MariaDB, and to all
|
| 83 |
+
other recipients of the licensed work to be provided by Licensor:
|
| 84 |
+
|
| 85 |
+
1. To specify as the Change License the GPL Version 2.0 or any later version,
|
| 86 |
+
or a license that is compatible with GPL Version 2.0 or a later version,
|
| 87 |
+
where "compatible" means that software provided under the Change License
|
| 88 |
+
can be included in a program with software provided under GPL Version 2.0
|
| 89 |
+
or a later version. Licensor may specify additional Change Licenses
|
| 90 |
+
without limitation.
|
| 91 |
+
|
| 92 |
+
2. To either: (a) specify an additional grant of rights to use that does not
|
| 93 |
+
impose any additional restriction on the right granted in this License, as
|
| 94 |
+
the Additional Use Grant; or (b) insert the text "None".
|
| 95 |
+
|
| 96 |
+
3. Not to modify this License in any other way.
|
PAMPAR-coder.png
ADDED
|
Git LFS Details
|
PAMPArLLM.png
ADDED
|
Git LFS Details
|
PLAN.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PAMPAr β Plan de EvoluciΓ³n
|
| 2 |
+
|
| 3 |
+
> Plan aprobado: **Option B β "Staged Physics"**
|
| 4 |
+
> Budget total: **$300-500 USD**
|
| 5 |
+
> Modelo: PamparV3, 108.3M params, vocab 48K
|
| 6 |
+
|
| 7 |
+
---
|
| 8 |
+
|
| 9 |
+
## Objetivo
|
| 10 |
+
|
| 11 |
+
Transformar PamparV3 de un modelo que solo conoce patrones Python (38% eval)
|
| 12 |
+
a un **motor de razonamiento multi-lenguaje** que usa documentaciΓ³n de referencia
|
| 13 |
+
para resolver problemas en cualquier dominio.
|
| 14 |
+
|
| 15 |
+
### MΓ©tricas target
|
| 16 |
+
|
| 17 |
+
| MΓ©trica | Actual | Target |
|
| 18 |
+
|---------|--------|--------|
|
| 19 |
+
| Python eval | 6/16 (38%) | 70%+ |
|
| 20 |
+
| Multi-language | 0% | 50%+ |
|
| 21 |
+
| Doc consultation (RAG) | 0% | 60%+ |
|
| 22 |
+
| Debugging | 0% | 50%+ |
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Fase 1 β Continual Pretrain: "Textbook Physics"
|
| 27 |
+
|
| 28 |
+
**Objetivo**: inyectar los axiomas fundamentales de razonamiento con referencia.
|
| 29 |
+
|
| 30 |
+
### Datos (~540K-640K tokens)
|
| 31 |
+
|
| 32 |
+
6 pilares de axiomas, cada uno con ~90K-100K tokens de texto tipo textbook:
|
| 33 |
+
|
| 34 |
+
| Pilar | Contenido | Fuente |
|
| 35 |
+
|-------|-----------|--------|
|
| 36 |
+
| **LΓ³gica y razonamiento** | Proposiciones, inferencia, truth tables, deducciΓ³n | Generado + Wikipedia |
|
| 37 |
+
| **Estructuras de datos** | Arrays, trees, graphs, hashmaps β cross-language | Generado + docs oficiales |
|
| 38 |
+
| **Patrones de cΓ³digo** | Design patterns, idioms en Python/JS/Rust/C | Generado + libros open |
|
| 39 |
+
| **ComprensiΓ³n de docs** | CΓ³mo leer una API reference, man page, docstring | MDN, Python docs, Rust Book |
|
| 40 |
+
| **Debugging** | Stack traces, error messages, bisecciΓ³n, logging | Generado + StackOverflow curado |
|
| 41 |
+
| **Multi-language syntax** | Equivalencias PythonβJSβRustβCβBashβSQL | Generado + Rosetta Code |
|
| 42 |
+
|
| 43 |
+
### Formato
|
| 44 |
+
|
| 45 |
+
```
|
| 46 |
+
<textbook>
|
| 47 |
+
## CapΓtulo: [tema]
|
| 48 |
+
|
| 49 |
+
[ExplicaciΓ³n clara del concepto]
|
| 50 |
+
|
| 51 |
+
### Ejemplo
|
| 52 |
+
[CΓ³digo con comentarios]
|
| 53 |
+
|
| 54 |
+
### Ejercicio resuelto
|
| 55 |
+
[Problema β razonamiento step-by-step β soluciΓ³n]
|
| 56 |
+
</textbook>
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
### Costo estimado: $30-60
|
| 60 |
+
|
| 61 |
+
- DistilaciΓ³n desde GPT-4o/Claude para generar textbooks
|
| 62 |
+
- ~6 scripts de generaciΓ³n, uno por pilar
|
| 63 |
+
- ValidaciΓ³n manual de quality (sampling 5%)
|
| 64 |
+
|
| 65 |
+
### Hardware
|
| 66 |
+
|
| 67 |
+
- GeneraciΓ³n de datos: API calls (local)
|
| 68 |
+
- Continual pretrain: **RunPod A100 40GB** (~2-4 horas)
|
| 69 |
+
|
| 70 |
+
---
|
| 71 |
+
|
| 72 |
+
## Fase 2 β SFT: "Chain-of-Thought con Referencia"
|
| 73 |
+
|
| 74 |
+
**Objetivo**: enseΓ±ar al modelo a usar documentaciΓ³n de referencia para resolver problemas.
|
| 75 |
+
|
| 76 |
+
### Datos (~20K ejemplos)
|
| 77 |
+
|
| 78 |
+
| CategorΓa | Ejemplos | DescripciΓ³n |
|
| 79 |
+
|-----------|----------|-------------|
|
| 80 |
+
| Python + ref | 5K | Problemas con snippet de docs como contexto |
|
| 81 |
+
| JavaScript + ref | 3K | DOM, Node.js, ES6+ con MDN como referencia |
|
| 82 |
+
| Rust + ref | 2K | Ownership, traits, lifetimes con Rust Book |
|
| 83 |
+
| SQL + ref | 2K | Queries con schema como referencia |
|
| 84 |
+
| Bash/CLI + ref | 1K | Comandos con man pages como referencia |
|
| 85 |
+
| Debugging | 3K | Stack traces β diagnΓ³stico β fix |
|
| 86 |
+
| Cross-language | 2K | "Traducir" lΓ³gica entre lenguajes |
|
| 87 |
+
| RAG-grounded | 2K | Preguntas que requieren buscar en docs primero |
|
| 88 |
+
|
| 89 |
+
### Formato SFT
|
| 90 |
+
|
| 91 |
+
```json
|
| 92 |
+
{
|
| 93 |
+
"instruction": "[PROBLEMA] Implementar un servidor HTTP bΓ‘sico",
|
| 94 |
+
"reference": "[REFERENCIA] Fragmento de docs de http.server de Python...",
|
| 95 |
+
"reasoning": "[RAZONAMIENTO] 1. Necesito importar http.server\n2. Crear handler...\n3. Bind al puerto...",
|
| 96 |
+
"output": "[SOLUCIΓN] import http.server\n..."
|
| 97 |
+
}
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
+
### Costo estimado: $110-150
|
| 101 |
+
|
| 102 |
+
- DistilaciΓ³n masiva desde GPT-4o/Claude
|
| 103 |
+
- 20K ejemplos Γ ~$0.006/ejemplo promedio
|
| 104 |
+
- Quality filter: score > 0.7 de auto-evaluaciΓ³n
|
| 105 |
+
|
| 106 |
+
### Hardware
|
| 107 |
+
|
| 108 |
+
- GeneraciΓ³n de datos: API calls (local)
|
| 109 |
+
- SFT: **RunPod A100 40GB** (~4-8 horas)
|
| 110 |
+
|
| 111 |
+
---
|
| 112 |
+
|
| 113 |
+
## Fase 3 β CorrecciΓ³n: "GhidraProbe + NeuroTrainer"
|
| 114 |
+
|
| 115 |
+
**Objetivo**: corregir routing y pesos usando diagnΓ³stico local.
|
| 116 |
+
|
| 117 |
+
### Proceso
|
| 118 |
+
|
| 119 |
+
1. Correr `eval_v3.py` para identificar categorΓas dΓ©biles
|
| 120 |
+
2. GhidraProbe analiza activaciones en ejemplos fallidos
|
| 121 |
+
3. NeuroTrainer aplica correcciones targeted:
|
| 122 |
+
- LLAVES: ajustar reglas INT8 para tokens multi-language
|
| 123 |
+
- Routing: corregir `terr_acts` donde el TΓ‘lamo asigna mal
|
| 124 |
+
- Pesos: mini-SFT de 50-100 steps en categorΓas fallidas
|
| 125 |
+
|
| 126 |
+
### Costo: $0
|
| 127 |
+
|
| 128 |
+
- 100% local en GTX 1650
|
| 129 |
+
- ~2 horas por ronda de correcciΓ³n
|
| 130 |
+
- 3-5 rondas estimadas
|
| 131 |
+
|
| 132 |
+
---
|
| 133 |
+
|
| 134 |
+
## Timeline estimado
|
| 135 |
+
|
| 136 |
+
| Fase | DuraciΓ³n | Costo | Output |
|
| 137 |
+
|------|----------|-------|--------|
|
| 138 |
+
| Fase 1 β Pretrain data | 1-2 semanas | $30-60 | ~600K tokens textbook |
|
| 139 |
+
| Fase 1 β Training | 1 dΓa RunPod | incluido | Checkpoint pretrained |
|
| 140 |
+
| Fase 2 β SFT data | 2-3 semanas | $110-150 | ~20K SFT examples |
|
| 141 |
+
| Fase 2 β Training | 1 dΓa RunPod | incluido | Checkpoint SFT |
|
| 142 |
+
| Fase 3 β Correction | 1 semana | $0 | Checkpoint final |
|
| 143 |
+
| **Total** | **5-7 semanas** | **$160-235** | **Motor de razonamiento** |
|
| 144 |
+
|
| 145 |
+
---
|
| 146 |
+
|
| 147 |
+
## Pre-requisitos (Blocks 2-3)
|
| 148 |
+
|
| 149 |
+
Antes de empezar el training, necesitamos limpiar y preparar el cΓ³digo:
|
| 150 |
+
|
| 151 |
+
### Block 2 β Cleanup de cΓ³digo muerto
|
| 152 |
+
|
| 153 |
+
Scripts que importan mΓ³dulos v2 eliminados (borrar con backup a `_archive/`):
|
| 154 |
+
|
| 155 |
+
- `scripts/aprender_solo.py`
|
| 156 |
+
- `scripts/train.py`
|
| 157 |
+
- `scripts/train_cerebral.py`
|
| 158 |
+
- `scripts/destilar.py`
|
| 159 |
+
- `scripts/evaluate_v2.py`
|
| 160 |
+
- `scripts/generar_curriculum.py`
|
| 161 |
+
- `scripts/smoke_test_viaje.py`
|
| 162 |
+
- `scripts/test_llaves.py`
|
| 163 |
+
|
| 164 |
+
Scripts mixtos v2/v3 rotos (borrar con backup):
|
| 165 |
+
|
| 166 |
+
- `scripts/benchmark.py`
|
| 167 |
+
- `scripts/probar_modelo.py`
|
| 168 |
+
- `scripts/eval_honesta.py`
|
| 169 |
+
|
| 170 |
+
MΓ³dulos huΓ©rfanos:
|
| 171 |
+
|
| 172 |
+
- `pampar/training/` β no importado por nada
|
| 173 |
+
|
| 174 |
+
### Block 3 β Refactoring para multi-language
|
| 175 |
+
|
| 176 |
+
| Archivo | Cambio |
|
| 177 |
+
|---------|--------|
|
| 178 |
+
| `zonas.py` | Agregar keywords JS/Rust/C/Bash/SQL a ZONAS |
|
| 179 |
+
| `llaves.py` | Expandir `clasificar_token()` para multi-language |
|
| 180 |
+
| `clasificador.py` | Generalizar `_calcular_densidad()` mΓ‘s allΓ‘ de Python |
|
| 181 |
+
| `ejecutar_codigo.py` | Agregar soporte para Node.js, Bash |
|
| 182 |
+
| `config.py` | Extraer `TOKENIZER_PATH` como constante compartida |
|
| 183 |
+
|
| 184 |
+
---
|
| 185 |
+
|
| 186 |
+
## Checkpoints esperados
|
| 187 |
+
|
| 188 |
+
| Nombre | Fase | DescripciΓ³n |
|
| 189 |
+
|--------|------|-------------|
|
| 190 |
+
| `v3_ghidra_v9.pt` | Actual | Score 89, 6/16 (38%) β baseline |
|
| 191 |
+
| `v3_pretrain_f1.pt` | Fase 1 | Post continual pretrain |
|
| 192 |
+
| `v3_sft_f2.pt` | Fase 2 | Post SFT multi-language |
|
| 193 |
+
| `v3_corrected_f3.pt` | Fase 3 | Post GhidraProbe correction β target final |
|
| 194 |
+
|
| 195 |
+
---
|
| 196 |
+
|
| 197 |
+
## Notas
|
| 198 |
+
|
| 199 |
+
- **Arquitectura LOCKED**: no tocar la grilla 4Γ5, GQA, SwiGLU, LLAVES 80/20
|
| 200 |
+
- **Backups siempre**: antes de borrar/refactorizar β `_archive/`
|
| 201 |
+
- **RunPod**: A100 40GB para fases 1 y 2, el cΓ³digo de `cloud/runpod/` ya existe
|
| 202 |
+
- **EvaluaciΓ³n**: `scripts/eval_v3.py` como benchmark consistente entre fases
|
README.md
ADDED
|
@@ -0,0 +1,365 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<p align="center">
|
| 2 |
+
<img src="PAMPAR-coder.png" alt="PAMPAr-Coder" width="200" />
|
| 3 |
+
</p>
|
| 4 |
+
|
| 5 |
+
<h1 align="center">PAMPAr-Coder</h1>
|
| 6 |
+
|
| 7 |
+
<p align="center">
|
| 8 |
+
<strong>Pure reasoning engine</strong> β 62.6M params, local-first, on-device RAG.
|
| 9 |
+
</p>
|
| 10 |
+
|
| 11 |
+
<p align="center">
|
| 12 |
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-BUSL--1.1-blue" alt="License" /></a>
|
| 13 |
+
<img src="https://img.shields.io/badge/params-62.6M-green" alt="Params" />
|
| 14 |
+
<img src="https://img.shields.io/badge/python-3.11%2B-blue" alt="Python" />
|
| 15 |
+
<img src="https://img.shields.io/badge/pytorch-2.x-orange" alt="PyTorch" />
|
| 16 |
+
</p>
|
| 17 |
+
|
| 18 |
+
---
|
| 19 |
+
|
| 20 |
+
## What is PAMPAr-Coder
|
| 21 |
+
|
| 22 |
+
PAMPAr-Coder is a 62.6M parameter language model that **reasons over reference information** rather than memorizing answers. It works like a physicist: it understands the fundamental axioms and can derive solutions for any domain using documentation available on the device.
|
| 23 |
+
|
| 24 |
+
- **Weights**: reasoning capability (read docs, understand problems, derive solutions step-by-step)
|
| 25 |
+
- **Device**: knowledge via local RAG (Python docs, MDN, man pages, user files)
|
| 26 |
+
- **Hardware**: designed to run on consumer hardware (GTX 1650, 4 GB VRAM)
|
| 27 |
+
|
| 28 |
+
**Current state**: `v3_train.pt` β 98K steps, Mixed Selectivity (FiLM). Classroom system with conversational mentor (Qwen-plus) + 5 bio-inspired mechanisms. Tree of 21 concepts with adaptive prerequisites.
|
| 29 |
+
|
| 30 |
+
---
|
| 31 |
+
|
| 32 |
+
## 2D Architecture (PamparV3)
|
| 33 |
+
|
| 34 |
+
```
|
| 35 |
+
tok_emb [48K x 640]
|
| 36 |
+
-> TalamoInicial (LLAVES 80% + attn_proj 20% + context_conv)
|
| 37 |
+
-> terr_acts [B, L, 4] / zona_acts [B, L, 52]
|
| 38 |
+
-> 4 parallel streams (dim=640)
|
| 39 |
+
|
| 40 |
+
NivelProfundo x5:
|
| 41 |
+
1. Shared GQA Attention (8 Q heads / 2 KV heads, head_dim=80)
|
| 42 |
+
2. Lightweight Thalamus re-routing
|
| 43 |
+
3. 4 x independent StreamFFN SwiGLU
|
| 44 |
+
4. Lateral gates per stream (bottleneck=128)
|
| 45 |
+
|
| 46 |
+
-> norm_f (RMSNorm) -> lm_head (weight-tied, vocab=48K)
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
### The 4 Streams
|
| 50 |
+
|
| 51 |
+
| Stream | Brodmann Zones | Processes |
|
| 52 |
+
| -------------- | -------------- | --------------------------------- |
|
| 53 |
+
| **SYNTAX** | B01-B15 | Keywords, operators, punctuation |
|
| 54 |
+
| **SEMANTICS** | B16-B30 | Types, variables, literals |
|
| 55 |
+
| **LOGIC** | B31-B42 | Control flow, conditionals, loops |
|
| 56 |
+
| **STRUCTURAL** | B43-B52 | Blocks, indentation, scope |
|
| 57 |
+
|
| 58 |
+
### Parameters
|
| 59 |
+
|
| 60 |
+
| Parameter | Value |
|
| 61 |
+
| ---------------- | ----------- |
|
| 62 |
+
| `dim` | 640 |
|
| 63 |
+
| `n_streams` | 4 |
|
| 64 |
+
| `n_levels` | 5 |
|
| 65 |
+
| `n_heads` | 8 |
|
| 66 |
+
| `n_kv_heads` | 2 (GQA 4:1) |
|
| 67 |
+
| `vocab_size` | 48,000 |
|
| 68 |
+
| `max_seq_len` | 4096 |
|
| 69 |
+
| **Total params** | **62.6M** |
|
| 70 |
+
|
| 71 |
+
---
|
| 72 |
+
|
| 73 |
+
## Key Innovations
|
| 74 |
+
|
| 75 |
+
### LLAVES System (TalamoInicial)
|
| 76 |
+
|
| 77 |
+
- **80% explicit rules**: routing based on code patterns (INT8, pre-computed)
|
| 78 |
+
- **20% learned attention**: fine-tuning for ambiguous cases
|
| 79 |
+
- Produces `terr_acts` and `zona_acts` with zero inference overhead
|
| 80 |
+
|
| 81 |
+
### 2D Cortical Architecture
|
| 82 |
+
|
| 83 |
+
- **4 streams Γ 5 levels** = grid where rows specialize and columns refine
|
| 84 |
+
- **GQA 4:1**: lower VRAM, same quality
|
| 85 |
+
- **Lateral gates** (bottleneck 128): cross-stream communication like white-matter fibers
|
| 86 |
+
- **Re-routing** per level: the Thalamus adapts which stream leads based on accumulated context
|
| 87 |
+
|
| 88 |
+
### On-Device RAG
|
| 89 |
+
|
| 90 |
+
The model uses the machine where it's installed as its knowledge source:
|
| 91 |
+
|
| 92 |
+
- Scanner detects OS, packages, available files
|
| 93 |
+
- RAGResidual indexes local documentation (FAISS + sentence-transformers)
|
| 94 |
+
- The model reasons over references, it doesn't memorize content
|
| 95 |
+
|
| 96 |
+
---
|
| 97 |
+
|
| 98 |
+
## Classroom β Conversational Mentor + Bio-Mechanisms
|
| 99 |
+
|
| 100 |
+
A learning system where a mentor model (Qwen-plus via DashScope) teaches PamparV3 through dynamic conversations, like a tutor in a chat. The mentor generates unique explanations, examples, and exercises for each lesson β the student absorbs knowledge via gradient descent.
|
| 101 |
+
|
| 102 |
+
### Lesson Flow
|
| 103 |
+
|
| 104 |
+
```
|
| 105 |
+
1. StudentProfile selects adaptive concept (21 concepts with prerequisites)
|
| 106 |
+
2. Mentor generates lesson: explanation + example + exercise + solution
|
| 107 |
+
3. Phase A β Absorb: train on explanation + example (all tokens)
|
| 108 |
+
4. Phase B β Practice: student attempts the exercise
|
| 109 |
+
5. Phase C β Correct: mentor evaluates, train on correct solution + replay
|
| 110 |
+
6. Update student profile (mastery per concept)
|
| 111 |
+
```
|
| 112 |
+
|
| 113 |
+
### Concept Tree (CONCEPT_TREE)
|
| 114 |
+
|
| 115 |
+
21 concepts organized in 5 levels with prerequisites:
|
| 116 |
+
|
| 117 |
+
| Level | Concepts |
|
| 118 |
+
| ----- | --------------------------------------------------------------------- |
|
| 119 |
+
| 1 | arithmetic β variables_types β conditionals, strings, functions_basic |
|
| 120 |
+
| 2 | loops_for β loops_while, lists β tuples_sets, dicts |
|
| 121 |
+
| 3 | recursion, higher_order, generators, error_handling |
|
| 122 |
+
| 4 | classes_basic β inheritance, dunder_methods |
|
| 123 |
+
| 5 | decorators, context_managers, algorithms, file_io |
|
| 124 |
+
|
| 125 |
+
`StudentProfile` tracks mastery per concept and selects adaptively:
|
| 126 |
+
|
| 127 |
+
- Prioritizes concepts with attempts but not yet mastered (reinforcement)
|
| 128 |
+
- Then new concepts whose prerequisites are met
|
| 129 |
+
- Finally spaced review of mastered concepts
|
| 130 |
+
|
| 131 |
+
### Core Mechanisms
|
| 132 |
+
|
| 133 |
+
| Mechanism | Purpose |
|
| 134 |
+
| -------------------------------------- | ----------------------------------------------------------------- |
|
| 135 |
+
| **EWC** (Elastic Weight Consolidation) | Protects important weights β penalizes changes to critical params |
|
| 136 |
+
| **Replay Buffer** | Mixes new and previous examples (simulates sleep consolidation) |
|
| 137 |
+
| **Differential LR** | LLAVES/Thalamus 0.01Γ, attention 0.1Γ, embedding 0.1Γ, FFN 1.0Γ |
|
| 138 |
+
| **Conversational Absorption** | Trains on mentor explanations + examples (knowledge distillation) |
|
| 139 |
+
|
| 140 |
+
### Bio-Mechanisms (`bio_mechanisms.py`)
|
| 141 |
+
|
| 142 |
+
5 mechanisms based on real neuroscience, integrated as post-lesson hooks:
|
| 143 |
+
|
| 144 |
+
| Mechanism | Biological Inspiration | Implementation |
|
| 145 |
+
| ----------------------- | ------------------------- | -------------------------------------------------------------------------------------- |
|
| 146 |
+
| **Neuromodulation** | Dopamine + Norepinephrine | Dynamically modulates LR based on success/error (Γ0.3 to Γ3.0) |
|
| 147 |
+
| **LTP** | Long-term potentiation | Strengthens `LateralGate.scale` of streams with consistent high activation (Hebb rule) |
|
| 148 |
+
| **Sleep Consolidation** | REM + SWS phases | Periodic replay (every 15 lessons): random (REM) + sorted by difficulty (SWS) |
|
| 149 |
+
| **Neurogenesis** | New hippocampal neurons | Injects LoRA adapters (rank=8, ~10K params) into StreamFFN when loss > 4.0 |
|
| 150 |
+
| **Synaptic Pruning** | Synaptic pruning (~50%) | Reduces `LateralGate.scale < 0.03` every 30 lessons (decay Γ0.5) |
|
| 151 |
+
|
| 152 |
+
All coordinated by `BioOrchestrator.after_lesson()`. Can be disabled with `--no-bio`.
|
| 153 |
+
|
| 154 |
+
### Mentor Pilot Results (5 lessons)
|
| 155 |
+
|
| 156 |
+
- Absorption loss: ~7-8 (new content from mentor)
|
| 157 |
+
- Exercise loss decreasing: 5.89 β 5.44 β 4.40 β 3.94 β 4.38
|
| 158 |
+
- Brain score stable: 88.24% (prior knowledge preservation)
|
| 159 |
+
- EWC penalty growing: 0.000002 β 0.000044 (active regularization)
|
| 160 |
+
- Each lesson is UNIQUE β mentor generates dynamically, no repetition
|
| 161 |
+
|
| 162 |
+
### Usage
|
| 163 |
+
|
| 164 |
+
```bash
|
| 165 |
+
# Conversational mentor with Qwen-plus (recommended)
|
| 166 |
+
python scripts/classroom_server.py \
|
| 167 |
+
--checkpoint checkpoints/v3_train.pt \
|
| 168 |
+
--checkpoint-out checkpoints/v3_classroom_mentor.pt \
|
| 169 |
+
--teacher qwen --model qwen-plus \
|
| 170 |
+
--max-lessons 200 --lr 1e-5 --ewc-lambda 50 --no-bio --no-ui
|
| 171 |
+
|
| 172 |
+
# With bio-inspired mechanisms enabled
|
| 173 |
+
python scripts/classroom_server.py \
|
| 174 |
+
--checkpoint checkpoints/v3_train.pt \
|
| 175 |
+
--teacher qwen --model qwen-plus \
|
| 176 |
+
--max-lessons 200 --lr 1e-5
|
| 177 |
+
|
| 178 |
+
# With web interface (SSE + dashboard)
|
| 179 |
+
python scripts/classroom_server.py \
|
| 180 |
+
--checkpoint checkpoints/v3_train.pt \
|
| 181 |
+
--teacher qwen --port 8787
|
| 182 |
+
|
| 183 |
+
# With GitHub Models API (alternative)
|
| 184 |
+
python scripts/classroom_server.py \
|
| 185 |
+
--checkpoint checkpoints/v3_train.pt \
|
| 186 |
+
--teacher github --model gpt-4o-mini
|
| 187 |
+
|
| 188 |
+
# Replay a recorded session
|
| 189 |
+
# Open sessions/classroom_*.html in browser
|
| 190 |
+
```
|
| 191 |
+
|
| 192 |
+
---
|
| 193 |
+
|
| 194 |
+
## Subsystems
|
| 195 |
+
|
| 196 |
+
| Module | Components | Purpose |
|
| 197 |
+
| ------------- | --------------------------- | ----------------------------------------------------------------------------------------------- |
|
| 198 |
+
| **Model** | `pampar/coder/v3/` | PamparV3: forward, generate, routing, blocks |
|
| 199 |
+
| **Memory** | `pampar/memoria/` | ClasificadorPareto (L0-L3), RAGResidual (FAISS), ColaFinetune |
|
| 200 |
+
| **Runtime** | `pampar/runtime/` | Agent (orchestrator), Scanner (device), BootProtocol |
|
| 201 |
+
| **Skills** | `pampar/skills/` | LectorArchivos (30+ ext), EjecutorCodigo (subprocess) |
|
| 202 |
+
| **Inference** | `pampar/inference.py` | JSON-lines stdin/stdout server for VS Code |
|
| 203 |
+
| **Classroom** | `scripts/classroom*.py` | Conversational mentor: engine + teacher + curriculum + training + events + memory + persistence |
|
| 204 |
+
| **Bio-Mech** | `scripts/bio_mechanisms.py` | 5 neuroscience mechanisms: Neuromod, LTP, Sleep, Neurogenesis, Pruning |
|
| 205 |
+
|
| 206 |
+
---
|
| 207 |
+
|
| 208 |
+
## Installation
|
| 209 |
+
|
| 210 |
+
```bash
|
| 211 |
+
git clone https://github.com/lucasmella-stack/PAMPAr-Coder.git
|
| 212 |
+
cd PAMPAr-Coder
|
| 213 |
+
pip install -r requirements.txt
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
---
|
| 217 |
+
|
| 218 |
+
## Usage
|
| 219 |
+
|
| 220 |
+
### Instantiate the model
|
| 221 |
+
|
| 222 |
+
```python
|
| 223 |
+
from pampar.coder.v3 import PamparV3, PRESET_V3
|
| 224 |
+
import torch
|
| 225 |
+
|
| 226 |
+
model = PamparV3(PRESET_V3)
|
| 227 |
+
model.eval()
|
| 228 |
+
|
| 229 |
+
# Forward pass
|
| 230 |
+
ids = torch.randint(0, 48_000, (1, 64))
|
| 231 |
+
with torch.no_grad():
|
| 232 |
+
logits, loss, info = model(ids)
|
| 233 |
+
|
| 234 |
+
# Autoregressive generation
|
| 235 |
+
gen = model.generate(ids, max_tokens=100, temperature=0.8, top_k=50)
|
| 236 |
+
```
|
| 237 |
+
|
| 238 |
+
### Use the Agent (with RAG + Skills)
|
| 239 |
+
|
| 240 |
+
```python
|
| 241 |
+
from pampar.runtime import Agente
|
| 242 |
+
|
| 243 |
+
agent = Agente(
|
| 244 |
+
checkpoint="checkpoints/v3_train.pt",
|
| 245 |
+
workspace_root=".",
|
| 246 |
+
)
|
| 247 |
+
response = agent.responder("how to read a CSV with pandas?")
|
| 248 |
+
```
|
| 249 |
+
|
| 250 |
+
---
|
| 251 |
+
|
| 252 |
+
## Project Structure
|
| 253 |
+
|
| 254 |
+
```
|
| 255 |
+
PAMPAr-Coder/
|
| 256 |
+
βββ pampar/
|
| 257 |
+
β βββ coder/v3/ # Active architecture (62.6M)
|
| 258 |
+
β β βββ modelo.py # PamparV3 β forward, generate
|
| 259 |
+
β β βββ config.py # ConfigV3 + presets
|
| 260 |
+
β β βββ talamo.py # TalamoInicial β routing
|
| 261 |
+
β β βββ bloques.py # GQA, SwiGLU, LateralGate, NivelProfundo
|
| 262 |
+
β β βββ llaves.py # LlavesV2 β INT8 lookup
|
| 263 |
+
β β βββ zonas.py # 52 Brodmann Zones
|
| 264 |
+
β β βββ ghidra_probe.py # Read-only instrumentation
|
| 265 |
+
β β βββ engrama_stream.py # Activation memory
|
| 266 |
+
β βββ memoria/
|
| 267 |
+
β β βββ clasificador.py # ClasificadorPareto (L0-L3)
|
| 268 |
+
β β βββ rag.py # RAGResidual (FAISS + TF-IDF fallback)
|
| 269 |
+
β β βββ cola_finetune.py # ColaFinetune (auto-SFT buffer)
|
| 270 |
+
β βββ skills/
|
| 271 |
+
β β βββ lector_archivos.py # File reader (sandboxed)
|
| 272 |
+
β β βββ ejecutar_codigo.py # Code executor (subprocess)
|
| 273 |
+
β βββ runtime/
|
| 274 |
+
β β βββ agente.py # Main orchestrator
|
| 275 |
+
β β βββ scanner.py # Device inspection
|
| 276 |
+
β β βββ boot.py # Boot sequence
|
| 277 |
+
β βββ inference.py # JSON-lines server for VS Code
|
| 278 |
+
βββ scripts/
|
| 279 |
+
β βββ classroom.py # ClassroomEngine (~600 lines)
|
| 280 |
+
β βββ classroom_curriculum.py# CONCEPT_TREE (21 concepts) + StudentProfile
|
| 281 |
+
β βββ classroom_teacher.py # Mentor API (GitHub/OpenRouter/Qwen)
|
| 282 |
+
β βββ classroom_training.py # Tokenization + differential LR + train_step
|
| 283 |
+
β βββ classroom_events.py # Console event formatting
|
| 284 |
+
β βββ classroom_memory.py # EWC + ReplayBuffer + compute_ewc_baseline
|
| 285 |
+
β βββ classroom_persistence.py # Checkpoint + session + HTML recording save
|
| 286 |
+
β βββ classroom_server.py # HTTP SSE server + CLI (entry point)
|
| 287 |
+
β βββ bio_mechanisms.py # 5 bio mechanisms
|
| 288 |
+
βββ data/tokenizer/
|
| 289 |
+
β βββ pampar_48k.model # 48K bilingual vocab (active)
|
| 290 |
+
βββ checkpoints/ # Model checkpoints (gitignored)
|
| 291 |
+
βββ tests/ # pytest test suite
|
| 292 |
+
βββ _archive/ # Pre-refactoring backups
|
| 293 |
+
```
|
| 294 |
+
|
| 295 |
+
---
|
| 296 |
+
|
| 297 |
+
## Understanding the Loss
|
| 298 |
+
|
| 299 |
+
| Loss | Meaning |
|
| 300 |
+
| ----- | --------------------- |
|
| 301 |
+
| ~10.7 | Untrained (log 48000) |
|
| 302 |
+
| 7-8 | Random weights |
|
| 303 |
+
| 5-7 | Beginning to learn |
|
| 304 |
+
| 2-4 | Active learning |
|
| 305 |
+
| 1.5-2 | Optimal zone |
|
| 306 |
+
| < 1.5 | Topic well learned |
|
| 307 |
+
| < 0.7 | Topic mastered |
|
| 308 |
+
|
| 309 |
+
---
|
| 310 |
+
|
| 311 |
+
## Tests
|
| 312 |
+
|
| 313 |
+
```bash
|
| 314 |
+
python -m pytest tests/ -v
|
| 315 |
+
```
|
| 316 |
+
|
| 317 |
+
142 tests, all passing.
|
| 318 |
+
|
| 319 |
+
---
|
| 320 |
+
|
| 321 |
+
## Philosophy
|
| 322 |
+
|
| 323 |
+
> _"You don't need 72 billion parameters. You need the right architecture and the right axioms."_
|
| 324 |
+
|
| 325 |
+
1. **Reasoning > memorization** β the model learns to use references, not to memorize
|
| 326 |
+
2. **The device is the knowledge base** β local RAG, not cloud
|
| 327 |
+
3. **Code is structured** β 4 specialized streams + LLAVES 80% rules
|
| 328 |
+
4. **Consumer hardware** β 1.4 GB VRAM for fp16 training
|
| 329 |
+
|
| 330 |
+
---
|
| 331 |
+
|
| 332 |
+
## Roadmap
|
| 333 |
+
|
| 334 |
+
- [x] Territorial architecture (52 Brodmann zones, 4 streams Γ 5 levels)
|
| 335 |
+
- [x] LLAVES system (INT8 routing, 80% rules)
|
| 336 |
+
- [x] BPE 48K bilingual tokenizer (ES + code)
|
| 337 |
+
- [x] GQA 4:1, SwiGLU, lateral gates
|
| 338 |
+
- [x] Memory module (ClasificadorPareto, RAG, ColaFinetune)
|
| 339 |
+
- [x] Skills (LectorArchivos, EjecutorCodigo)
|
| 340 |
+
- [x] Runtime.Agent (tool-use loop)
|
| 341 |
+
- [x] GhidraProbe (read-only diagnostics)
|
| 342 |
+
- [x] EngramaStream (activation memory)
|
| 343 |
+
- [x] Bio-inspired Classroom (EWC, replay buffer, differential LR, curriculum)
|
| 344 |
+
- [x] HTML session recording and replay
|
| 345 |
+
- [x] GitHub Models API integration (gpt-4o-mini as teacher)
|
| 346 |
+
- [x] Bio-mechanisms: Neuromodulation, LTP, Sleep Consolidation, Neurogenesis, Synaptic Pruning
|
| 347 |
+
- [x] Conversational mentor: Qwen-plus generates dynamic lessons as tutor
|
| 348 |
+
- [x] CONCEPT_TREE: 21 concepts with adaptive prerequisites
|
| 349 |
+
- [x] StudentProfile: per-concept mastery tracking
|
| 350 |
+
- [x] Loss masking: -100 on prompt tokens (train only on responses)
|
| 351 |
+
- [x] Conversational absorption: train on mentor explanations + examples
|
| 352 |
+
- [ ] Multimodal: image/diagram input support
|
| 353 |
+
- [ ] Training data expansion (textbook + SFT multi-language)
|
| 354 |
+
- [ ] KV cache in generate()
|
| 355 |
+
- [ ] Multi-language execution (JS, Rust, Bash)
|
| 356 |
+
- [ ] Benchmarks against reference models
|
| 357 |
+
- [ ] VS Code extension
|
| 358 |
+
|
| 359 |
+
---
|
| 360 |
+
|
| 361 |
+
## License
|
| 362 |
+
|
| 363 |
+
BUSL-1.1 β Copyright (c) 2024-2026 Lucas Ricardo Mella Chillemi
|
| 364 |
+
|
| 365 |
+
Change Date: April 7, 2030 β License converts to Apache-2.0. See [LICENSE](LICENSE) for details.
|
ROADMAP.md
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PAMPAr-Coder β Roadmap
|
| 2 |
+
|
| 3 |
+
> Plan de evoluciΓ³n. Γltima actualizaciΓ³n: Mar 2026.
|
| 4 |
+
> Para la identidad del modelo ver `CONCIENCIA.md`. Para el protocolo de despliegue ver `AGENTS.md`.
|
| 5 |
+
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
## 1. VisiΓ³n
|
| 9 |
+
|
| 10 |
+
PAMPAr es un **fΓsico con doctorado** que puede especializarse en cualquier campo:
|
| 11 |
+
|
| 12 |
+
- El **doctorado** (razonamiento computacional) estΓ‘ en los **pesos** β 108M params.
|
| 13 |
+
- La **especializaciΓ³n** viene del **entorno** β se descubre al boot con el Scanner.
|
| 14 |
+
- El protocolo de 3 archivos (`CONCIENCIA.md` + `AGENTS.md` + `TOOLS.md`) es la interfaz entre el modelo y su despliegue.
|
| 15 |
+
|
| 16 |
+
### Las 3 fases del proyecto
|
| 17 |
+
|
| 18 |
+
| Fase | QuΓ© | Estado |
|
| 19 |
+
| -------------------------------- | --------------------------------------------------------------------- | ---------------------------------------------------------- |
|
| 20 |
+
| **Fase 1** β SFT | Entrenar el doctorado: lΓ³gica Python, patrones, razonamiento | **β
Completa** (16/16 con reparadores, target superado) |
|
| 21 |
+
| **Fase 2** β Runtime loop | El modelo usa herramientas, ejecuta, lee, aprende del loop | **β
Completa** (chat.py + ColaFinetune + mini-SFT wiring) |
|
| 22 |
+
| **Fase 3** β Protocolo entrenado | El modelo genera su propio AGENTS.md al aterrizar en un sistema nuevo | Futuro |
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## 2. Arquitectura actual β PamparV3
|
| 27 |
+
|
| 28 |
+
### 2.1 Grilla cortical 2D
|
| 29 |
+
|
| 30 |
+
```
|
| 31 |
+
Tokens (int)
|
| 32 |
+
β
|
| 33 |
+
βΌ
|
| 34 |
+
[Embeddings] 48K vocab, dim=640, weight-tied con lm_head
|
| 35 |
+
β
|
| 36 |
+
βΌ
|
| 37 |
+
βββββββββββββββββββββββββββββββ
|
| 38 |
+
β TalamoInicial β routing: quΓ© streams procesan cada token
|
| 39 |
+
β 80% LLAVES (INT8 + reglas) β
|
| 40 |
+
β 20% attn_proj (aprendido) β
|
| 41 |
+
β + context_conv causal k=32 β
|
| 42 |
+
ββββββββββββββ¬βββββββββββββββββ
|
| 43 |
+
β [B, L, 4, dim] β 4 streams con pesos distintos
|
| 44 |
+
βΌ
|
| 45 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 46 |
+
β 5 Γ NivelProfundo β
|
| 47 |
+
β TalamoNivel β 4Γ BloqueAttn GQA 4:1 β
|
| 48 |
+
β β 4Γ StreamFFN SwiGLU β LateralGate β
|
| 49 |
+
β β Early Exit (umbral 0.90) β
|
| 50 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 51 |
+
β
|
| 52 |
+
βΌ
|
| 53 |
+
RMSNorm + lm_head β logits [B, L, 48000]
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
### 2.2 Streams β Capas lingΓΌΓsticas
|
| 57 |
+
|
| 58 |
+
| Stream | Territorio | Zonas | Capa lingΓΌΓstica |
|
| 59 |
+
| ------ | ----------- | ------- | --------------------------------- |
|
| 60 |
+
| 0 | SINTAXIS | B01-B15 | Sintaxis β estructura del cΓ³digo |
|
| 61 |
+
| 1 | SEMANTICA | B16-B30 | SemΓ‘ntica β significado |
|
| 62 |
+
| 2 | LOGICO | B31-B42 | PragmΓ‘tica β intenciΓ³n, flujo |
|
| 63 |
+
| 3 | ESTRUCTURAL | B43-B52 | Discurso β organizaciΓ³n, patrones |
|
| 64 |
+
|
| 65 |
+
Los 4 streams procesan en paralelo. Cada NivelProfundo tiene Lateral Gates (bottleneck=128) para comunicaciΓ³n entre streams β como las fibras blancas del cerebro.
|
| 66 |
+
|
| 67 |
+
### 2.3 Boot Protocol
|
| 68 |
+
|
| 69 |
+
```
|
| 70 |
+
1. CONCIENCIA.md β RAG L3 (identidad inmutable)
|
| 71 |
+
2. Scanner β workspace (ast), paquetes (importlib), servicios (socket), sistema (platform)
|
| 72 |
+
3. AGENTS.md contextual β RAG L2 (entorno mutable)
|
| 73 |
+
4. System prompt dinΓ‘mico = identidad + contexto + acciones
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
Implementado en `pampar.runtime.scanner` + `pampar.runtime.boot`.
|
| 77 |
+
|
| 78 |
+
---
|
| 79 |
+
|
| 80 |
+
## 3. Estado de checkpoints
|
| 81 |
+
|
| 82 |
+
| Checkpoint | Datos | Eval open (temp=0.0) |
|
| 83 |
+
| -------------- | --------------------------------------------- | -------------------- |
|
| 84 |
+
| `v3_sft.pt` | 43K Magicoder (inglΓ©s) | 0/16 |
|
| 85 |
+
| `v3_sft_v5.pt` | SFT v5 (base post-catastrΓ³fico) | 6/16 |
|
| 86 |
+
| `v3_sft_v6.pt` | clean_sft.jsonl (555 ejemplos) | 10/16 |
|
| 87 |
+
| `v3_sft_v7.pt` | final_sft.jsonl (825 = clean + quirΓΊrgicoΓ3) | 15/16 |
|
| 88 |
+
| `v3_sft_v8.pt` | micro-SFT cuadrados (300 steps + reparadores) | **16/16 β
BEST** |
|
| 89 |
+
|
| 90 |
+
### Estado actual (v3_sft_v8.pt β 16/16 con reparadores)
|
| 91 |
+
|
| 92 |
+
| # | FunciΓ³n | Estado | Notas |
|
| 93 |
+
| --- | ---------------- | ------ | -------------------------------------------------------------- |
|
| 94 |
+
| 01 | contar_vocales | β
| β |
|
| 95 |
+
| 02 | suma_digitos | β
| β |
|
| 96 |
+
| 03 | es_palindromo | β
| β |
|
| 97 |
+
| 04 | maximo_lista | β
| β |
|
| 98 |
+
| 05 | fizzbuzz | β
| Corregido (dataset quirΓΊrgico) |
|
| 99 |
+
| 06 | aplanar_lista | β
| β |
|
| 100 |
+
| 07 | frecuencia | β
| β |
|
| 101 |
+
| 08 | cuadrados_pares | β
| Genera `x*i` β reparador NameError word-boundary lo corrige |
|
| 102 |
+
| 09 | invertir_dict | β
| β |
|
| 103 |
+
| 10 | fibonacci | β
| β |
|
| 104 |
+
| 11 | busqueda_binaria | β
| β |
|
| 105 |
+
| 12 | merge_sort | β
| Corregido (self-contained) |
|
| 106 |
+
| 13 | Stack | β
| β |
|
| 107 |
+
| 14 | Punto | β
| Corregido (import math / \*\*0.5) |
|
| 108 |
+
| 15 | memoize | β
| Corregido (usa `fn`, no `func`) |
|
| 109 |
+
| 16 | primos_hasta | β
| Reparador `_reparar_bloques_huerfanos` + stop `endswith(\n\n)` |
|
| 110 |
+
|
| 111 |
+
---
|
| 112 |
+
|
| 113 |
+
## 4. Plan de entrenamiento
|
| 114 |
+
|
| 115 |
+
### Fase A β Entrenamiento curricular con MotorCuriosidad
|
| 116 |
+
|
| 117 |
+
Objetivo: reforzar las bases de lΓ³gica que el modelo falla.
|
| 118 |
+
|
| 119 |
+
```bash
|
| 120 |
+
python scripts/train_v3.py \
|
| 121 |
+
--checkpoint checkpoints/v3_sft_v4.pt \
|
| 122 |
+
--biblioteca data/biblioteca/ \
|
| 123 |
+
--lr 3e-5 --epochs 3
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
Temas prioritarios basados en fallos del eval:
|
| 127 |
+
|
| 128 |
+
1. `bucles_for_while` β fizzbuzz, cuadrados_pares
|
| 129 |
+
2. `diccionarios` β invertir_dict
|
| 130 |
+
3. `busqueda_algoritmos` β bΓΊsqueda binaria
|
| 131 |
+
4. `recursion` β merge_sort
|
| 132 |
+
5. `clases_oop` β Punto, memoize
|
| 133 |
+
6. `matematica_basica` β primos, potencias
|
| 134 |
+
|
| 135 |
+
### Fase B β SFT v5 (post-curricular)
|
| 136 |
+
|
| 137 |
+
- ~18K ejemplos curados (3K por topic Γ 6 topics)
|
| 138 |
+
- Formato Alpaca, filtrado con pytest
|
| 139 |
+
- Generados por el propio modelo + verificaciΓ³n automΓ‘tica
|
| 140 |
+
|
| 141 |
+
### Fase C β Matriz lingΓΌΓstica como dato de entrenamiento
|
| 142 |
+
|
| 143 |
+
Incluir ejemplos que ejerciten explΓcitamente cada capa:
|
| 144 |
+
|
| 145 |
+
- **PragmΓ‘tica**: "El usuario quiere X, yo debo hacer Y" (comprensiΓ³n de intenciΓ³n)
|
| 146 |
+
- **SemΓ‘ntica**: Renombrar variables, inferir tipos, naming conventions
|
| 147 |
+
- **Sintaxis**: IndentaciΓ³n correcta, keywords, delimitadores, f-strings
|
| 148 |
+
- **Discurso**: OrganizaciΓ³n de cΓ³digo (imports β constantes β clases β funciones β main)
|
| 149 |
+
|
| 150 |
+
---
|
| 151 |
+
|
| 152 |
+
## 5. Roadmap de milestones
|
| 153 |
+
|
| 154 |
+
```
|
| 155 |
+
COMPLETADO β
COMPLETADO β
AHORA LARGO PLAZO
|
| 156 |
+
ββββββββββββ ββββββββββββ βββββββββββββ ββββββββββββ
|
| 157 |
+
15/16 eval β 16/16 eval β Mini-SFT auto β Protocolo
|
| 158 |
+
v3_sft_v7.pt v3_sft_v8.pt cuando colaβ₯50 entrenado
|
| 159 |
+
108M params + reparadores ColaFinetune Fase 3
|
| 160 |
+
|
| 161 |
+
SFT dataset chat.py Mini-SFT wiring El modelo
|
| 162 |
+
limpio+quirΓΊrgico loop activo sft_v5.py genera su
|
| 163 |
+
Clean+surgicalΓ3 genβexecβretry auto-reload AGENTS.md
|
| 164 |
+
```
|
| 165 |
+
|
| 166 |
+
### Milestone 1 β 16/16 eval β
COMPLETADO (target era β₯12/16)
|
| 167 |
+
|
| 168 |
+
- [x] Dataset limpio (clean_sft.jsonl β 555 ejemplos sin contradicciones)
|
| 169 |
+
- [x] Dataset quirΓΊrgico (surgical_sft.jsonl β 90 ejemplos para 6 fallos)
|
| 170 |
+
- [x] SFT v6 (10/16) desde clean data
|
| 171 |
+
- [x] SFT v7 (15/16) desde clean + surgicalΓ3
|
| 172 |
+
- [x] Fix primos_hasta β reparador `_reparar_bloques_huerfanos` + stop `endswith(\n\n)`
|
| 173 |
+
- [x] Fix cuadrados_pares β reparador NameError word-boundary en verificador
|
| 174 |
+
- [x] **16/16 confirmado** con v3_sft_v8.pt + eval_v3.py cadena de reparadores
|
| 175 |
+
|
| 176 |
+
### Milestone 2 β Runtime autΓ³nomo (EN PROGRESO)
|
| 177 |
+
|
| 178 |
+
- [x] Scanner del sistema (`pampar.runtime.scanner`)
|
| 179 |
+
- [x] Boot protocol (`pampar.runtime.boot`)
|
| 180 |
+
- [x] CONCIENCIA.md como identidad invariante
|
| 181 |
+
- [x] System prompt dinΓ‘mico (identidad + contexto del scan)
|
| 182 |
+
- [x] El agente ejecuta cΓ³digo que genera y observa output (`scripts/chat.py`)
|
| 183 |
+
- [x] Si falla, agrega el par (prompt, error) a ColaFinetune
|
| 184 |
+
- [x] Mini-SFT automΓ‘tico cuando la cola supera umbral (wiring con sft_v5.py + reload en proceso)
|
| 185 |
+
|
| 186 |
+
### Milestone 3 β Protocolo entrenado β
Implementado (generador determinista)
|
| 187 |
+
|
| 188 |
+
- [x] `pampar/runtime/generar_agents.py` β genera AGENTS.md contextual desde el scan (determinista)
|
| 189 |
+
- [x] `BootProtocol._inyectar_contexto()` actualizado: genera AGENTS.md β fragmenta por secciones β RAG L2
|
| 190 |
+
- [x] 23 tests en `tests/test_generar_agents.py` (132/132 en suite completa)
|
| 191 |
+
- [x] Quick Reference, Sistema detectado, Paquetes clave, Servicios, Boot protocol generados dinΓ‘micamente
|
| 192 |
+
- [ ] El modelo "sabe" escanear: genera `scan_sistema()` como cΓ³digo (largo plazo β necesita mucho mΓ‘s SFT)
|
| 193 |
+
- [ ] CONCIENCIA se refuerza con RLHF/DPO sobre interacciones reales (largo plazo)
|
| 194 |
+
- [ ] Nota: entrenar 108M params para generar markdown desde cero requiere 10K+ pasos β protocolo funcionando vΓa boot determinista es la aproximaciΓ³n correcta para este tamaΓ±o de modelo
|
| 195 |
+
|
| 196 |
+
### Milestone 4 β VS Code extension
|
| 197 |
+
|
| 198 |
+
- [ ] Extension que carga PamparV3 localmente (CPU/GPU)
|
| 199 |
+
- [ ] Completado inline de cΓ³digo
|
| 200 |
+
- [ ] Panel de chat con el agente
|
| 201 |
+
- [ ] Memoria persistente entre sesiones (RAG en disco)
|
| 202 |
+
|
| 203 |
+
### Milestone 5 β Voz (cuando el sistema la tiene)
|
| 204 |
+
|
| 205 |
+
- [ ] Detectar motores de voz al boot (espeak, SAPI, say) β ya implementado en Scanner
|
| 206 |
+
- [ ] TTS para respuestas cuando el usuario lo pide
|
| 207 |
+
- [ ] Zero-dependency: usa lo que el OS tiene instalado
|
| 208 |
+
|
| 209 |
+
---
|
| 210 |
+
|
| 211 |
+
## 6. Estructura de carpetas
|
| 212 |
+
|
| 213 |
+
```
|
| 214 |
+
PAMPAr-Coder/
|
| 215 |
+
βββ CONCIENCIA.md # Identidad invariante del modelo
|
| 216 |
+
βββ AGENTS.md # Protocolo de despliegue (mutable)
|
| 217 |
+
βββ ROADMAP.md # Este archivo
|
| 218 |
+
βββ pampar/
|
| 219 |
+
β βββ coder/v3/ # Arquitectura activa (108M)
|
| 220 |
+
β β βββ modelo.py # PamparV3
|
| 221 |
+
β β βββ config.py # ConfigV3, presets
|
| 222 |
+
β β βββ talamo.py # TalamoInicial
|
| 223 |
+
β β βββ bloques.py # BloqueAttn, StreamFFN, LateralGate
|
| 224 |
+
β β βββ llaves.py # LlavesV2 β lookup INT8
|
| 225 |
+
β β βββ zonas.py # 52 Zonas de Brodmann
|
| 226 |
+
β βββ memoria/
|
| 227 |
+
β β βββ clasificador.py # ClasificadorPareto β L0 a L3
|
| 228 |
+
β β βββ rag.py # RAGResidual β vector store
|
| 229 |
+
β β βββ cola_finetune.py # ColaFinetune β buffer SFT
|
| 230 |
+
β βββ runtime/
|
| 231 |
+
β β βββ agente.py # Agente β orquestador principal
|
| 232 |
+
β β βββ scanner.py # Scanner β inspecciΓ³n del entorno
|
| 233 |
+
β β βββ boot.py # BootProtocol β secuencia de arranque
|
| 234 |
+
β βββ training/
|
| 235 |
+
β βββ curiosidad.py # MotorCuriosidad β ZPD
|
| 236 |
+
β βββ lector.py # LectorBiblioteca
|
| 237 |
+
βββ checkpoints/
|
| 238 |
+
β βββ v3_sft_v4.pt # Mejor checkpoint (8/16)
|
| 239 |
+
βββ tests/ # 109+ tests
|
| 240 |
+
```
|
SECURITY.md
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Security Policy
|
| 2 |
+
|
| 3 |
+
## Supported Versions
|
| 4 |
+
|
| 5 |
+
| Version | Supported |
|
| 6 |
+
| ------- | ------------------ |
|
| 7 |
+
| 3.x | :white_check_mark: |
|
| 8 |
+
| < 3.0 | :x: |
|
| 9 |
+
|
| 10 |
+
## Reporting a Vulnerability
|
| 11 |
+
|
| 12 |
+
If you discover a security vulnerability, please report it responsibly:
|
| 13 |
+
|
| 14 |
+
1. **Do NOT open a public issue.**
|
| 15 |
+
2. Email **lucas.mella@outlook.com** with:
|
| 16 |
+
- Description of the vulnerability
|
| 17 |
+
- Steps to reproduce
|
| 18 |
+
- Potential impact
|
| 19 |
+
3. You will receive an acknowledgment within 48 hours.
|
| 20 |
+
4. A fix will be developed privately and released as a patch.
|
| 21 |
+
|
| 22 |
+
## Scope
|
| 23 |
+
|
| 24 |
+
This policy covers the PAMPAr-Coder source code and any official releases.
|
| 25 |
+
Training data, checkpoints, and third-party dependencies are out of scope.
|
benchmarks/history.jsonl
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"tag": "baseline-paso-5000", "timestamp": "2026-02-26T18:19:47.537840", "checkpoint": "checkpoints\\pampar_v2_best.pt", "params_M": 42.179342, "vocab_size": 16000, "perplexity": NaN, "syntax_validity_pct": 8.333333333333332, "top1_accuracy": 25.0, "top5_accuracy": 83.33333333333334, "tiempo_s": 48.78433275222778}
|
| 2 |
+
{"tag": "baseline-paso-5000-fixed", "timestamp": "2026-02-26T18:24:21.454006", "checkpoint": "checkpoints\\pampar_v2_best.pt", "params_M": 42.179342, "vocab_size": 16000, "perplexity": 35.99959821451826, "syntax_validity_pct": 8.333333333333332, "top1_accuracy": 50.0, "top5_accuracy": 75.0, "tiempo_s": 61.97281551361084}
|
| 3 |
+
{"tag": "paso-manual-2213", "timestamp": "2026-02-26T22:15:08.342790", "checkpoint": "checkpoints\\pampar_v2_best.pt", "params_M": 42.179342, "vocab_size": 16000, "perplexity": 222.9815264119179, "syntax_validity_pct": 16.666666666666664, "top1_accuracy": 66.66666666666666, "top5_accuracy": 75.0, "tiempo_s": 100.97530508041382}
|
| 4 |
+
{"tag": "paso-manual-0954", "timestamp": "2026-02-27T09:55:18.711572", "checkpoint": "checkpoints\\pampar_v2_best.pt", "params_M": 42.179342, "vocab_size": 16000, "perplexity": 1.1048595216681674, "syntax_validity_pct": 33.33333333333333, "top1_accuracy": 83.33333333333334, "top5_accuracy": 100.0, "tiempo_s": 52.147695541381836}
|
benchmarks/humaneval_results.json
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "PAMPAr-Coder V3",
|
| 3 |
+
"params_m": 62.6,
|
| 4 |
+
"checkpoint": "checkpoints\\v3_train.pt",
|
| 5 |
+
"benchmark": "HumanEval",
|
| 6 |
+
"n_problems": 5,
|
| 7 |
+
"samples_per_task": 1,
|
| 8 |
+
"temperature": 0.2,
|
| 9 |
+
"pass_at_1_pct": 0.0,
|
| 10 |
+
"pass_at_1_unbiased_pct": 0.0,
|
| 11 |
+
"passed": 0,
|
| 12 |
+
"total": 5,
|
| 13 |
+
"elapsed_sec": 229.9,
|
| 14 |
+
"device": "cuda",
|
| 15 |
+
"date": "2026-04-07 18:12",
|
| 16 |
+
"results": [
|
| 17 |
+
{
|
| 18 |
+
"task_id": "HumanEval/0",
|
| 19 |
+
"entry_point": "has_close_elements",
|
| 20 |
+
"passed": false,
|
| 21 |
+
"n_samples": 1,
|
| 22 |
+
"pass_count": 0,
|
| 23 |
+
"samples": [
|
| 24 |
+
{
|
| 25 |
+
"sample": 0,
|
| 26 |
+
"passed": false,
|
| 27 |
+
"error": "SyntaxError: closing parenthesis ']' does not match opening parenthesis '(' (<humaneval>, line 12)",
|
| 28 |
+
"completion_len": 1382
|
| 29 |
+
}
|
| 30 |
+
]
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"task_id": "HumanEval/1",
|
| 34 |
+
"entry_point": "separate_paren_groups",
|
| 35 |
+
"passed": false,
|
| 36 |
+
"n_samples": 1,
|
| 37 |
+
"pass_count": 0,
|
| 38 |
+
"samples": [
|
| 39 |
+
{
|
| 40 |
+
"sample": 0,
|
| 41 |
+
"passed": false,
|
| 42 |
+
"error": "SyntaxError: invalid character 'β' (U+2047) (<humaneval>, line 12)",
|
| 43 |
+
"completion_len": 1550
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"task_id": "HumanEval/2",
|
| 49 |
+
"entry_point": "truncate_number",
|
| 50 |
+
"passed": false,
|
| 51 |
+
"n_samples": 1,
|
| 52 |
+
"pass_count": 0,
|
| 53 |
+
"samples": [
|
| 54 |
+
{
|
| 55 |
+
"sample": 0,
|
| 56 |
+
"passed": false,
|
| 57 |
+
"error": "SyntaxError: '(' was never closed (<humaneval>, line 12)",
|
| 58 |
+
"completion_len": 2164
|
| 59 |
+
}
|
| 60 |
+
]
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"task_id": "HumanEval/3",
|
| 64 |
+
"entry_point": "below_zero",
|
| 65 |
+
"passed": false,
|
| 66 |
+
"n_samples": 1,
|
| 67 |
+
"pass_count": 0,
|
| 68 |
+
"samples": [
|
| 69 |
+
{
|
| 70 |
+
"sample": 0,
|
| 71 |
+
"passed": false,
|
| 72 |
+
"error": "AssertionError: ",
|
| 73 |
+
"completion_len": 1099
|
| 74 |
+
}
|
| 75 |
+
]
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"task_id": "HumanEval/4",
|
| 79 |
+
"entry_point": "mean_absolute_deviation",
|
| 80 |
+
"passed": false,
|
| 81 |
+
"n_samples": 1,
|
| 82 |
+
"pass_count": 0,
|
| 83 |
+
"samples": [
|
| 84 |
+
{
|
| 85 |
+
"sample": 0,
|
| 86 |
+
"passed": false,
|
| 87 |
+
"error": "TypeError: unsupported operand type(s) for -: 'NoneType' and 'float'",
|
| 88 |
+
"completion_len": 1825
|
| 89 |
+
}
|
| 90 |
+
]
|
| 91 |
+
}
|
| 92 |
+
]
|
| 93 |
+
}
|
benchmarks/v1/efficiency_comparison.png
ADDED
|
benchmarks/v1/llaves_impact.png
ADDED
|
benchmarks/v1/results.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"name": "PAMPAr-Coder (Early Exit)",
|
| 4 |
+
"tokens_per_sec": 33.61731131248854,
|
| 5 |
+
"perplexity": 8140.549328082081,
|
| 6 |
+
"vram_mb": 447.52783203125,
|
| 7 |
+
"params": 44137528,
|
| 8 |
+
"extra": null
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"name": "PAMPAr-Coder (No Early Exit)",
|
| 12 |
+
"tokens_per_sec": 33.92451873272917,
|
| 13 |
+
"perplexity": 8148.805109046796,
|
| 14 |
+
"vram_mb": 456.02783203125,
|
| 15 |
+
"params": 39529528,
|
| 16 |
+
"extra": null
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"name": "Transformer Vanilla",
|
| 20 |
+
"tokens_per_sec": 189.84663733328836,
|
| 21 |
+
"perplexity": 3.7490140993977816e+22,
|
| 22 |
+
"vram_mb": 142.26171875,
|
| 23 |
+
"params": 5193600,
|
| 24 |
+
"extra": null
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"name": "PAMPAr (LLAVES 50%)",
|
| 28 |
+
"tokens_per_sec": 32.98684305248618,
|
| 29 |
+
"perplexity": 8138.885639188741,
|
| 30 |
+
"vram_mb": 0,
|
| 31 |
+
"params": 44137528,
|
| 32 |
+
"extra": {
|
| 33 |
+
"llaves_peso": 0.5
|
| 34 |
+
}
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"name": "PAMPAr (LLAVES 80%)",
|
| 38 |
+
"tokens_per_sec": 32.884803900635966,
|
| 39 |
+
"perplexity": 8149.619349468611,
|
| 40 |
+
"vram_mb": 0,
|
| 41 |
+
"params": 44137528,
|
| 42 |
+
"extra": {
|
| 43 |
+
"llaves_peso": 0.8
|
| 44 |
+
}
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"name": "PAMPAr (LLAVES 95%)",
|
| 48 |
+
"tokens_per_sec": 34.319874602320944,
|
| 49 |
+
"perplexity": 8161.341232613224,
|
| 50 |
+
"vram_mb": 0,
|
| 51 |
+
"params": 44137528,
|
| 52 |
+
"extra": {
|
| 53 |
+
"llaves_peso": 0.95
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
]
|
benchmarks/v1/speed_comparison.png
ADDED
|
benchmarks/v1/summary.png
ADDED
|
brain_scan.html
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="es">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8">
|
| 5 |
+
<title>PAMPAr Brain Scanner</title>
|
| 6 |
+
<style>
|
| 7 |
+
body { font-family: 'Cascadia Code', 'Fira Code', monospace; background: #1e1e2e; color: #cdd6f4; margin: 2em; }
|
| 8 |
+
h1 { color: #89b4fa; border-bottom: 2px solid #89b4fa; padding-bottom: 8px; }
|
| 9 |
+
h2 { color: #a6e3a1; margin-top: 2em; }
|
| 10 |
+
.code { background: #313244; padding: 1em; border-radius: 8px; font-size: 14px; white-space: pre; }
|
| 11 |
+
table { border-collapse: collapse; margin: 1em 0; }
|
| 12 |
+
td, th { padding: 4px 8px; border: 1px solid #45475a; font-size: 13px; }
|
| 13 |
+
th { background: #313244; }
|
| 14 |
+
.tok { background: #313244; font-weight: bold; white-space: pre; }
|
| 15 |
+
.dom { font-weight: bold; }
|
| 16 |
+
.conf-bar { display: flex; align-items: center; gap: 8px; margin: 4px 0; }
|
| 17 |
+
.conf-bar .bar { height: 20px; border-radius: 4px; transition: width 0.3s; }
|
| 18 |
+
.conf-bar span { min-width: 60px; }
|
| 19 |
+
</style>
|
| 20 |
+
</head>
|
| 21 |
+
<body>
|
| 22 |
+
<h1>π§ PAMPAr Brain Scanner</h1>
|
| 23 |
+
<div class="code">x = [i**2 for i in range(10)]</div>
|
| 24 |
+
|
| 25 |
+
<h2>TΓ‘lamo: Routing Inicial</h2>
|
| 26 |
+
<table>
|
| 27 |
+
<tr><th>Token</th><th>SINTAXIS</th><th>SEMANTICA</th><th>LOGICO</th><th>ESTRUCTURAL</th><th>Dominante</th></tr>
|
| 28 |
+
<tr><td class="tok">βx</td><td style="background:rgba(137,180,250,0.65)">0.649</td><td style="background:rgba(166,227,161,0.61)">0.612</td><td style="background:rgba(249,226,175,0.62)">0.619</td><td style="background:rgba(203,166,247,0.61)">0.610</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">β=</td><td style="background:rgba(137,180,250,0.85)">0.849</td><td style="background:rgba(166,227,161,0.78)">0.783</td><td style="background:rgba(249,226,175,0.80)">0.798</td><td style="background:rgba(203,166,247,0.78)">0.780</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">β[</td><td style="background:rgba(137,180,250,0.94)">0.942</td><td style="background:rgba(166,227,161,0.89)">0.887</td><td style="background:rgba(249,226,175,0.90)">0.901</td><td style="background:rgba(203,166,247,0.88)">0.884</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">i</td><td style="background:rgba(137,180,250,0.98)">0.977</td><td style="background:rgba(166,227,161,0.94)">0.941</td><td style="background:rgba(249,226,175,0.95)">0.951</td><td style="background:rgba(203,166,247,0.94)">0.939</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">**</td><td style="background:rgba(137,180,250,0.99)">0.992</td><td style="background:rgba(166,227,161,0.97)">0.972</td><td style="background:rgba(249,226,175,0.98)">0.978</td><td style="background:rgba(203,166,247,0.97)">0.971</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">2</td><td style="background:rgba(137,180,250,1.00)">0.997</td><td style="background:rgba(166,227,161,0.99)">0.986</td><td style="background:rgba(249,226,175,0.99)">0.990</td><td style="background:rgba(203,166,247,0.99)">0.986</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">βfor</td><td style="background:rgba(137,180,250,1.00)">0.999</td><td style="background:rgba(166,227,161,0.99)">0.993</td><td style="background:rgba(249,226,175,1.00)">0.995</td><td style="background:rgba(203,166,247,0.99)">0.993</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">βi</td><td style="background:rgba(137,180,250,1.00)">1.000</td><td style="background:rgba(166,227,161,1.00)">0.997</td><td style="background:rgba(249,226,175,1.00)">0.998</td><td style="background:rgba(203,166,247,1.00)">0.996</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">βin</td><td style="background:rgba(137,180,250,1.00)">1.000</td><td style="background:rgba(166,227,161,1.00)">0.998</td><td style="background:rgba(249,226,175,1.00)">0.999</td><td style="background:rgba(203,166,247,1.00)">0.998</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">βrange</td><td style="background:rgba(137,180,250,1.00)">1.000</td><td style="background:rgba(166,227,161,1.00)">0.999</td><td style="background:rgba(249,226,175,1.00)">0.999</td><td style="background:rgba(203,166,247,1.00)">0.999</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">(</td><td style="background:rgba(137,180,250,1.00)">1.000</td><td style="background:rgba(166,227,161,1.00)">0.999</td><td style="background:rgba(249,226,175,1.00)">1.000</td><td style="background:rgba(203,166,247,1.00)">0.999</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">1</td><td style="background:rgba(137,180,250,1.00)">1.000</td><td style="background:rgba(166,227,161,1.00)">1.000</td><td style="background:rgba(249,226,175,1.00)">1.000</td><td style="background:rgba(203,166,247,1.00)">1.000</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">0</td><td style="background:rgba(137,180,250,1.00)">1.000</td><td style="background:rgba(166,227,161,1.00)">1.000</td><td style="background:rgba(249,226,175,1.00)">1.000</td><td style="background:rgba(203,166,247,1.00)">1.000</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr><tr><td class="tok">)]</td><td style="background:rgba(137,180,250,1.00)">1.000</td><td style="background:rgba(166,227,161,1.00)">1.000</td><td style="background:rgba(249,226,175,1.00)">1.000</td><td style="background:rgba(203,166,247,1.00)">1.000</td><td class="dom" style="color:#89b4fa">SINTAXIS</td></tr>
|
| 29 |
+
</table>
|
| 30 |
+
|
| 31 |
+
<h2>EvoluciΓ³n por Nivel</h2>
|
| 32 |
+
<table>
|
| 33 |
+
<tr><th>Token</th><th>N0</th><th>N1</th><th>N2</th><th>N3</th><th>N4</th><th>N5</th></tr>
|
| 34 |
+
<tr><td class="tok">βx</td><td style="background:#89b4fa;opacity:0.65">0.65</td><td style="background:#89b4fa;opacity:0.64">0.64</td><td style="background:#89b4fa;opacity:0.66">0.66</td><td style="background:#89b4fa;opacity:0.68">0.68</td><td style="background:#89b4fa;opacity:0.69">0.69</td><td style="background:#cba6f7;opacity:0.69">0.69</td></tr><tr><td class="tok">β=</td><td style="background:#89b4fa;opacity:0.85">0.85</td><td style="background:#89b4fa;opacity:0.81">0.81</td><td style="background:#89b4fa;opacity:0.76">0.76</td><td style="background:#89b4fa;opacity:0.75">0.75</td><td style="background:#89b4fa;opacity:0.74">0.74</td><td style="background:#89b4fa;opacity:0.73">0.73</td></tr><tr><td class="tok">β[</td><td style="background:#89b4fa;opacity:0.94">0.94</td><td style="background:#f9e2af;opacity:0.85">0.85</td><td style="background:#f9e2af;opacity:0.80">0.80</td><td style="background:#f9e2af;opacity:0.78">0.78</td><td style="background:#f9e2af;opacity:0.76">0.76</td><td style="background:#cba6f7;opacity:0.74">0.74</td></tr><tr><td class="tok">i</td><td style="background:#89b4fa;opacity:0.98">0.98</td><td style="background:#89b4fa;opacity:0.90">0.90</td><td style="background:#89b4fa;opacity:0.84">0.84</td><td style="background:#89b4fa;opacity:0.81">0.81</td><td style="background:#89b4fa;opacity:0.78">0.78</td><td style="background:#89b4fa;opacity:0.76">0.76</td></tr><tr><td class="tok">**</td><td style="background:#89b4fa;opacity:0.99">0.99</td><td style="background:#89b4fa;opacity:0.91">0.91</td><td style="background:#f9e2af;opacity:0.85">0.85</td><td style="background:#f9e2af;opacity:0.81">0.81</td><td style="background:#f9e2af;opacity:0.79">0.79</td><td style="background:#cba6f7;opacity:0.75">0.75</td></tr><tr><td class="tok">2</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#f9e2af;opacity:0.91">0.91</td><td style="background:#cba6f7;opacity:0.85">0.85</td><td style="background:#cba6f7;opacity:0.82">0.82</td><td style="background:#cba6f7;opacity:0.79">0.79</td><td style="background:#cba6f7;opacity:0.77">0.77</td></tr><tr><td class="tok">βfor</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#89b4fa;opacity:0.92">0.92</td><td style="background:#a6e3a1;opacity:0.85">0.85</td><td style="background:#a6e3a1;opacity:0.81">0.81</td><td style="background:#89b4fa;opacity:0.79">0.79</td><td style="background:#a6e3a1;opacity:0.76">0.76</td></tr><tr><td class="tok">βi</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#f9e2af;opacity:0.92">0.92</td><td style="background:#f9e2af;opacity:0.86">0.86</td><td style="background:#f9e2af;opacity:0.81">0.81</td><td style="background:#89b4fa;opacity:0.78">0.78</td><td style="background:#89b4fa;opacity:0.77">0.77</td></tr><tr><td class="tok">βin</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#f9e2af;opacity:0.92">0.92</td><td style="background:#f9e2af;opacity:0.86">0.86</td><td style="background:#f9e2af;opacity:0.82">0.82</td><td style="background:#89b4fa;opacity:0.78">0.78</td><td style="background:#89b4fa;opacity:0.77">0.77</td></tr><tr><td class="tok">βrange</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#a6e3a1;opacity:0.91">0.91</td><td style="background:#cba6f7;opacity:0.86">0.86</td><td style="background:#f9e2af;opacity:0.82">0.82</td><td style="background:#f9e2af;opacity:0.79">0.79</td><td style="background:#cba6f7;opacity:0.77">0.77</td></tr><tr><td class="tok">(</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#f9e2af;opacity:0.92">0.92</td><td style="background:#f9e2af;opacity:0.86">0.86</td><td style="background:#f9e2af;opacity:0.82">0.82</td><td style="background:#f9e2af;opacity:0.79">0.79</td><td style="background:#f9e2af;opacity:0.77">0.77</td></tr><tr><td class="tok">1</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#cba6f7;opacity:0.92">0.92</td><td style="background:#cba6f7;opacity:0.86">0.86</td><td style="background:#cba6f7;opacity:0.82">0.82</td><td style="background:#cba6f7;opacity:0.79">0.79</td><td style="background:#cba6f7;opacity:0.77">0.77</td></tr><tr><td class="tok">0</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#cba6f7;opacity:0.92">0.92</td><td style="background:#cba6f7;opacity:0.86">0.86</td><td style="background:#cba6f7;opacity:0.81">0.81</td><td style="background:#cba6f7;opacity:0.78">0.78</td><td style="background:#cba6f7;opacity:0.76">0.76</td></tr><tr><td class="tok">)]</td><td style="background:#89b4fa;opacity:1.00">1.00</td><td style="background:#89b4fa;opacity:0.92">0.92</td><td style="background:#89b4fa;opacity:0.86">0.86</td><td style="background:#89b4fa;opacity:0.82">0.82</td><td style="background:#89b4fa;opacity:0.79">0.79</td><td style="background:#89b4fa;opacity:0.77">0.77</td></tr>
|
| 35 |
+
</table>
|
| 36 |
+
|
| 37 |
+
<h2>Early Exit</h2>
|
| 38 |
+
<div class="conf-bar"><span>Nivel 0</span><div class="bar" style="width:40.9%;background:#f44336"></div><span>40.9%</span></div><div class="conf-bar"><span>Nivel 1</span><div class="bar" style="width:41.1%;background:#f44336"></div><span>41.1%</span></div><div class="conf-bar"><span>Nivel 2</span><div class="bar" style="width:49.5%;background:#f44336"></div><span>49.5%</span></div><div class="conf-bar"><span>Nivel 3</span><div class="bar" style="width:34.5%;background:#f44336"></div><span>34.5%</span></div><div class="conf-bar"><span>Nivel 4</span><div class="bar" style="width:44.0%;background:#f44336"></div><span>44.0%</span></div>
|
| 39 |
+
<p>Umbral: 90% β MΓn 2 niveles</p>
|
| 40 |
+
|
| 41 |
+
</body>
|
| 42 |
+
</html>
|
brain_scanner_pretrain_results.txt
ADDED
|
Binary file (8.21 kB). View file
|
|
|
docs/APRENDIZAJE_CEREBRAL.md
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Aprendizaje Cerebral: Estado Actual del Sistema
|
| 2 |
+
|
| 3 |
+
> Γltima actualizaciΓ³n: Febrero 2026
|
| 4 |
+
> Documento refleja el estado real del proyecto β entrenamiento 100% local.
|
| 5 |
+
|
| 6 |
+
## La Tesis Central
|
| 7 |
+
|
| 8 |
+
> **"Un cerebro humano aprende a programar con ~10,000 horas de prΓ‘ctica, no con 5.5 trillones de tokens.
|
| 9 |
+
> La diferencia: los humanos razonan, experimentan y consolidan β no memorizan brute-force."**
|
| 10 |
+
|
| 11 |
+
PAMPAr-Coder tiene la **arquitectura cerebral** (Territorios, TΓ‘lamo, Zonas de Brodmann)
|
| 12 |
+
y un paradigma de entrenamiento bio-inspirado que corre completamente en hardware local.
|
| 13 |
+
|
| 14 |
+
## Por quΓ© Funciona: phi-1 como Prueba
|
| 15 |
+
|
| 16 |
+
Microsoft demostrΓ³ con **phi-1** (1.3B params) que un modelo pequeΓ±o puede competir con
|
| 17 |
+
GPT-3.5 en cΓ³digo usando **solo 7B tokens de calidad "textbook"** vs 5.5T tokens de Qwen.
|
| 18 |
+
|
| 19 |
+
La clave: **CALIDAD > CANTIDAD**.
|
| 20 |
+
|
| 21 |
+
Nuestra arquitectura cerebral amplifica esto:
|
| 22 |
+
|
| 23 |
+
- **LLAVES** (75% reglas) β routing gratuito, no necesita aprender quΓ© es `def`, `if`, `for`
|
| 24 |
+
- **52 Zonas Brodmann** β especializaciΓ³n natural por tipo de token
|
| 25 |
+
- **Early Exit** β tokens simples son baratos, recursos se enfocan en tokens difΓciles
|
| 26 |
+
- **4 Territorios FFN** β cada territorio se vuelve experto en su dominio
|
| 27 |
+
|
| 28 |
+
## Las 5 Fases del Aprendizaje Cerebral
|
| 29 |
+
|
| 30 |
+
### Fase 1: INFANCIA β Curriculum Learning ($5-15)
|
| 31 |
+
|
| 32 |
+
```
|
| 33 |
+
Nivel 1: Variables y asignaciones (SINTAXIS domina)
|
| 34 |
+
Nivel 2: Control de flujo (LOGICO + SINTAXIS)
|
| 35 |
+
Nivel 3: Funciones (SEMANTICA + ESTRUCTURAL)
|
| 36 |
+
Nivel 4: Clases y OOP (todos los territorios)
|
| 37 |
+
Nivel 5: Algoritmos complejos (LOGICO + ESTRUCTURAL)
|
| 38 |
+
Nivel 6: Patrones de diseΓ±o (integraciΓ³n total)
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
- Como un niΓ±o aprendiendo: simple β complejo
|
| 42 |
+
- Cada nivel activa progresivamente mΓ‘s territorios
|
| 43 |
+
- LLAVES asegura routing correcto desde el dΓa 0
|
| 44 |
+
|
| 45 |
+
### Fase 2: EXPERIMENTACIΓN β Self-Play ($10-30)
|
| 46 |
+
|
| 47 |
+
```
|
| 48 |
+
genera cΓ³digo β ejecuta β resultado β aprende
|
| 49 |
+
β β
|
| 50 |
+
βββββββββββββββ feedback ββββββββββββββ
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
- Como un programador probando cΓ³digo
|
| 54 |
+
- No necesita datasets masivos β genera sus propios datos
|
| 55 |
+
- Reward: ΒΏel cΓ³digo ejecuta? ΒΏda el resultado correcto?
|
| 56 |
+
- DPO: aprende de sus propios aciertos vs errores
|
| 57 |
+
|
| 58 |
+
### Fase 3: FILOSOFAR β Reasoning Chains ($5-15)
|
| 59 |
+
|
| 60 |
+
```
|
| 61 |
+
Problema: "crear funciΓ³n que ordene una lista"
|
| 62 |
+
β LOGICO: necesito comparar elementos (Zona B32_OP_COMP)
|
| 63 |
+
β ESTRUCTURAL: un bucle anidado (Zona B43_BLOCK_FUNC)
|
| 64 |
+
β SINTAXIS: usar for, if, return (Zonas B05, B06, B04)
|
| 65 |
+
β SEMANTICA: nombre descriptivo (Zona B17_ID_FUNC)
|
| 66 |
+
```
|
| 67 |
+
|
| 68 |
+
- El modelo aprende a USAR sus territorios para razonar
|
| 69 |
+
- Chain-of-thought: descomponer problemas en sub-problemas
|
| 70 |
+
- Cada paso del razonamiento activa diferentes zonas
|
| 71 |
+
|
| 72 |
+
### Fase 4: SUEΓO β ConsolidaciΓ³n Hebbiana ($2-5)
|
| 73 |
+
|
| 74 |
+
```
|
| 75 |
+
"Neuronas que disparan juntas, se conectan juntas"
|
| 76 |
+
β Fortalecer conexiones entre territorios exitosos
|
| 77 |
+
β Debilitar conexiones no usadas
|
| 78 |
+
β Replay de patrones importantes
|
| 79 |
+
β Poda de pesos innecesarios
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
- Como cuando dormimos y el cerebro consolida memorias
|
| 83 |
+
- Ajuste fino del TΓ‘lamo basado en patrones de Γ©xito
|
| 84 |
+
- El modelo se vuelve mΓ‘s eficiente sin datos nuevos
|
| 85 |
+
|
| 86 |
+
### Fase 5: CURIOSIDAD β Active Learning ($5-10)
|
| 87 |
+
|
| 88 |
+
```
|
| 89 |
+
Confianza Early Exit baja β "No sΓ© esto" β Generar datos de entrenamiento
|
| 90 |
+
Confianza alta + error β "Estoy mal seguro" β PenalizaciΓ³n extra
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
- El modelo identifica quΓ© NO sabe usando Early Exit
|
| 94 |
+
- Genera o busca datos especΓficamente para sus debilidades
|
| 95 |
+
- MetacogniciΓ³n: aprende a evaluar su propio conocimiento
|
| 96 |
+
|
| 97 |
+
## Innovaciones TΓ©cnicas
|
| 98 |
+
|
| 99 |
+
### 1. Metacognitive Loss (PΓ©rdida Metacognitiva)
|
| 100 |
+
|
| 101 |
+
```python
|
| 102 |
+
L_meta = Ξ± * CE_loss + Ξ² * |confidence - accuracy|
|
| 103 |
+
# Si confΓa mucho y falla β penalizaciΓ³n alta (sobreconfianza)
|
| 104 |
+
# Si no confΓa y falla β penalizaciΓ³n baja (sabe que no sabe)
|
| 105 |
+
# Si confΓa y acierta β recompensa (calibraciΓ³n correcta)
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
### 2. Territory Entropy Regularization
|
| 109 |
+
|
| 110 |
+
```python
|
| 111 |
+
L_entropy = -Ξ³ * Ξ£ terr_acts * log(terr_acts)
|
| 112 |
+
# Evita que todos los territorios se activen igual (colapso)
|
| 113 |
+
# Incentiva especializaciΓ³n: cada territorio es experto en algo
|
| 114 |
+
```
|
| 115 |
+
|
| 116 |
+
### 3. Hebbian Frontier Learning
|
| 117 |
+
|
| 118 |
+
```python
|
| 119 |
+
# DespuΓ©s de predicciΓ³n exitosa:
|
| 120 |
+
frontier_ij += Ξ· * activation_i * activation_j # "fire together, wire together"
|
| 121 |
+
# DespuΓ©s de predicciΓ³n fallida:
|
| 122 |
+
frontier_ij -= Ξ· * activation_i * activation_j # "anti-Hebbian"
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
### 4. Code Execution Reward (sin humanos)
|
| 126 |
+
|
| 127 |
+
```python
|
| 128 |
+
reward = {
|
| 129 |
+
'compila': +0.3, # el cΓ³digo es vΓ‘lido
|
| 130 |
+
'ejecuta': +0.5, # el cΓ³digo corre sin error
|
| 131 |
+
'correcto': +1.0, # produce resultado esperado
|
| 132 |
+
'error_sintaxis': -0.5, # error de parsing
|
| 133 |
+
'error_runtime': -0.3, # error en ejecuciΓ³n
|
| 134 |
+
'timeout': -0.1, # loop infinito
|
| 135 |
+
}
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
## EstimaciΓ³n de Costo Total
|
| 139 |
+
|
| 140 |
+
| Fase | Tokens | Costo GPU (A40) | DΓas |
|
| 141 |
+
| --------------- | ----------------- | --------------- | -------- |
|
| 142 |
+
| Infancia | 3-5B | $5-15 | 1-2 |
|
| 143 |
+
| ExperimentaciΓ³n | 1-3B (generados) | $10-30 | 2-4 |
|
| 144 |
+
| Filosofar | 0.5-1B | $5-15 | 1-2 |
|
| 145 |
+
| SueΓ±o | 0 (replay) | $2-5 | 0.5 |
|
| 146 |
+
| Curiosidad | 0.5-1B (targeted) | $5-10 | 1 |
|
| 147 |
+
| **TOTAL** | **5-10B** | **$27-75** | **5-10** |
|
| 148 |
+
|
| 149 |
+
vs Qwen: 5,500B tokens, $50,000-200,000, meses.
|
| 150 |
+
|
| 151 |
+
## CΓ³mo Entrenarlo en Tu PC
|
| 152 |
+
|
| 153 |
+
### Requisitos MΓnimos
|
| 154 |
+
|
| 155 |
+
- **8GB VRAM**: LoRA fine-tuning (fases 2-5 despuΓ©s de pre-training cloud)
|
| 156 |
+
- **16GB VRAM**: Full fine-tuning con gradient checkpointing
|
| 157 |
+
- **24GB VRAM**: Entrenamiento completo todas las fases
|
| 158 |
+
|
| 159 |
+
### Flujo Recomendado
|
| 160 |
+
|
| 161 |
+
1. **Cloud A40** ($30-50): Fase 1 (pre-training curriculum) + Fase 3 (reasoning)
|
| 162 |
+
2. **Tu PC** (gratis): Fase 2 (self-play) + Fase 4 (consolidation) + Fase 5 (active learning)
|
| 163 |
+
|
| 164 |
+
### Por quΓ© Tu PC es Suficiente para Self-Play
|
| 165 |
+
|
| 166 |
+
- Self-play no procesa datasets masivos β genera 1 ejemplo, entrena, repite
|
| 167 |
+
- Cada ciclo: generar 10 programas β ejecutar β aprender = ~1 minuto en RTX 3060
|
| 168 |
+
- 1000 ciclos/dΓa = modelo mejorando constantemente = 0 costo de GPU cloud
|
| 169 |
+
|
| 170 |
+
## ImplementaciΓ³n
|
| 171 |
+
|
| 172 |
+
```
|
| 173 |
+
pampar/coder/v2/aprendizaje/
|
| 174 |
+
βββ __init__.py # Exports
|
| 175 |
+
βββ curriculum.py # Fase 1: Niveles de dificultad
|
| 176 |
+
βββ self_play.py # Fase 2: GeneraciΓ³n + ejecuciΓ³n
|
| 177 |
+
βββ razonamiento.py # Fase 3: Chains of thought
|
| 178 |
+
βββ neuroplasticidad.py # Fase 4: Hebbian + consolidaciΓ³n
|
| 179 |
+
βββ metacognicion.py # Fase 5: Active learning + meta-loss
|
| 180 |
+
|
| 181 |
+
scripts/
|
| 182 |
+
βββ train_cerebral.py # Pipeline completo 5 fases
|
| 183 |
+
βββ generar_curriculum.py # Preparar datos por nivel
|
| 184 |
+
```
|
docs/MIXED_SELECTIVITY.md
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Mixed Selectivity: FFN Compartido + ModulaciΓ³n Contextual (FiLM)
|
| 2 |
+
|
| 3 |
+
> Γltima actualizaciΓ³n: Abril 2026
|
| 4 |
+
> Autor: Lucas (concepto) + implementaciΓ³n en PamparV3
|
| 5 |
+
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
## Resumen
|
| 9 |
+
|
| 10 |
+
PamparV3 reemplaza las **4 copias independientes de StreamFFN** (una por territorio) con **1 FFN compartido** + **4 ContextModulators** por nivel. El mismo bloque de pesos se lee de 4 formas distintas segΓΊn un vector contextual de 63 dimensiones.
|
| 11 |
+
|
| 12 |
+
**Resultado:** 62.6M params (antes ~105M) β **40% de reducciΓ³n** sin perder capacidad expresiva.
|
| 13 |
+
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
## MotivaciΓ³n
|
| 17 |
+
|
| 18 |
+
### Neurociencia: "Mixed Selectivity"
|
| 19 |
+
|
| 20 |
+
Una neurona cortical no responde a un solo estΓmulo. Rigotti et al. (2013) demostraron que las neuronas exhiben **selectividad mixta**: la misma neurona que responde a "ubicaciΓ³n" tambiΓ©n codifica "tiempo" y "contexto de tarea". Esta propiedad es _necesaria_ para computaciΓ³n cognitiva compleja.
|
| 21 |
+
|
| 22 |
+
### La conexiΓ³n con PamparV3
|
| 23 |
+
|
| 24 |
+
PamparV3 ya tiene un sistema de routing (TΓ‘lamo) que genera:
|
| 25 |
+
|
| 26 |
+
- `zona_acts [B, L, 52]` β activaciΓ³n de 52 zonas de Brodmann (tipo de token)
|
| 27 |
+
- `terr_acts [B, L, 4]` β pesos de los 4 territorios (sintaxis, semΓ‘ntica, lΓ³gico, estructural)
|
| 28 |
+
|
| 29 |
+
La idea de Lucas: _"Si ya sabemos QUΓ tipo de token es y QUΓ territorio domina... ΒΏpor quΓ© no usar esa info para LEER el mismo FFN de forma diferente en vez de tener 4 copias?"_
|
| 30 |
+
|
| 31 |
+
---
|
| 32 |
+
|
| 33 |
+
## DiseΓ±o tΓ©cnico
|
| 34 |
+
|
| 35 |
+
### Vector contextual (63 dimensiones)
|
| 36 |
+
|
| 37 |
+
```
|
| 38 |
+
ctx = [zona_acts(52), terr_acts(4), depth(1), conf(1), n_levels(1), stream_one_hot(4)]
|
| 39 |
+
βββββββββββ ββββββββββββ ββββββββ βββββββ ββββββββββββ ββββββββββββββββ
|
| 40 |
+
Tipo token Dominio Nivel Confianza Meta Identidad
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
| Indicador | Dims | Fuente | InterpretaciΓ³n |
|
| 44 |
+
| ----------- | ---- | --------------------------- | ---------------------------------- |
|
| 45 |
+
| `zona_acts` | 52 | TΓ‘lamoInicial | keyword, variable, string, etc. |
|
| 46 |
+
| `terr_acts` | 4 | TΓ‘lamoInicial | peso por territorio |
|
| 47 |
+
| `depth` | 1 | `nivel_idx / n_levels` | 0.0=superficial, 1.0=profundo |
|
| 48 |
+
| `conf` | 1 | `exit_head` (con `no_grad`) | 0-1, ΒΏel modelo ya entendiΓ³? |
|
| 49 |
+
| `n_levels` | 1 | `config.n_niveles / 10` | normalizaciΓ³n del modelo |
|
| 50 |
+
| `stream_oh` | 4 | one-hot del stream actual | identidad del stream que se modula |
|
| 51 |
+
|
| 52 |
+
### ContextModulator (FiLM)
|
| 53 |
+
|
| 54 |
+
```python
|
| 55 |
+
class ContextModulator(nn.Module):
|
| 56 |
+
CONTEXT_DIM = 63
|
| 57 |
+
|
| 58 |
+
def __init__(self, dim: int, bottleneck: int = 128):
|
| 59 |
+
self.proj = nn.Sequential(
|
| 60 |
+
nn.Linear(63, bottleneck), # comprimir
|
| 61 |
+
nn.SiLU(),
|
| 62 |
+
nn.Linear(bottleneck, dim*2), # generar gamma + beta
|
| 63 |
+
)
|
| 64 |
+
# La ΓΊltima capa inicia en zeros β gammaβ0, betaβ0 β identidad
|
| 65 |
+
|
| 66 |
+
def forward(self, ffn_out, zona_acts, terr_acts, stream_idx, nivel_idx, n_levels, conf):
|
| 67 |
+
ctx = self._build_context(zona_acts, terr_acts, stream_idx, nivel_idx, n_levels, conf)
|
| 68 |
+
gamma, beta = self.proj(ctx).chunk(2, dim=-1)
|
| 69 |
+
return (1 + gamma) * ffn_out + beta
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
La fΓ³rmula FiLM `(1 + Ξ³) Β· x + Ξ²`:
|
| 73 |
+
|
| 74 |
+
- **Ξ³ (gamma)** escala cada dimensiΓ³n β amplifica features relevantes, suprime irrelevantes
|
| 75 |
+
- **Ξ² (beta)** desplaza β inyecta informaciΓ³n contextual que el FFN base no tiene
|
| 76 |
+
- Al iniciar con Ξ³=0, Ξ²=0 β pasa el FFN sin modificar β entrenamiento estable
|
| 77 |
+
|
| 78 |
+
### Flujo en NivelProfundo
|
| 79 |
+
|
| 80 |
+
```
|
| 81 |
+
1. Combinar: x_combined = Ξ£ streams[t] Γ terr_acts[:,:,t]
|
| 82 |
+
2. AtenciΓ³n: x_attn = BloqueAttn(x_combined)
|
| 83 |
+
3. Re-route: zona_acts actualizado = TΓ‘lamoNivel(x_attn)
|
| 84 |
+
β conf_value = exit_head(x_combined + x_attn) [no_grad]
|
| 85 |
+
4. FFN: h_base = ffn_shared(norm(stream + x_attn)) β 1 sola FFN
|
| 86 |
+
5. Modular: h_mod = modulator_t(h_base, ctx) β 4 modulators
|
| 87 |
+
6. Weight: h = h_mod Γ terr_acts[:,:,t] β territorial gating
|
| 88 |
+
7. Lateral: fibras blancas entre streams
|
| 89 |
+
8. Exit?: si conf > 0.90 β salir temprano
|
| 90 |
+
```
|
| 91 |
+
|
| 92 |
+
---
|
| 93 |
+
|
| 94 |
+
## Conteo de parΓ‘metros
|
| 95 |
+
|
| 96 |
+
| Componente | Legacy (4 FFN) | Mixed Selectivity |
|
| 97 |
+
| ------------------------- | ----------------- | ----------------- |
|
| 98 |
+
| Embeddings (tok_emb/head) | 30.7M | 30.7M |
|
| 99 |
+
| AtenciΓ³n GQA Γ5 | 5.1M | 5.1M |
|
| 100 |
+
| **StreamFFN** | **4Γ Γ5 = 65.5M** | **1Γ Γ5 = 16.4M** |
|
| 101 |
+
| **ContextModulators** | β | **4Γ Γ5 = 3.4M** |
|
| 102 |
+
| LateralGates Γ5 | 3.3M | 3.3M |
|
| 103 |
+
| TΓ‘lamo + routing + norms | ~6M | ~6M |
|
| 104 |
+
| **TOTAL** | **~105M** | **~62.6M** |
|
| 105 |
+
|
| 106 |
+
**Ahorro neto: 42.4M params (40%)**
|
| 107 |
+
|
| 108 |
+
---
|
| 109 |
+
|
| 110 |
+
## ConfiguraciΓ³n
|
| 111 |
+
|
| 112 |
+
En `ConfigV3`:
|
| 113 |
+
|
| 114 |
+
```python
|
| 115 |
+
use_mixed_selectivity: bool = True # True = compartido + modulators
|
| 116 |
+
modulator_bottleneck: int = 128 # tamaΓ±o intermedio del modulator
|
| 117 |
+
```
|
| 118 |
+
|
| 119 |
+
`use_mixed_selectivity=False` restaura el comportamiento original con 4 FFNs independientes. Los checkpoints del modo legacy **no son compatibles** con el modo mixed (keys diferentes en state_dict).
|
| 120 |
+
|
| 121 |
+
---
|
| 122 |
+
|
| 123 |
+
## Archivos modificados
|
| 124 |
+
|
| 125 |
+
| Archivo | Cambio |
|
| 126 |
+
| ----------------------------------- | --------------------------------------------- |
|
| 127 |
+
| `pampar/coder/v3/bloques.py` | +ContextModulator, NivelProfundo init/forward |
|
| 128 |
+
| `pampar/coder/v3/config.py` | +use_mixed_selectivity, +modulator_bottleneck |
|
| 129 |
+
| `pampar/coder/v3/modelo.py` | checkpointing con zona_acts, docstring |
|
| 130 |
+
| `scripts/test_mixed_selectivity.py` | Test de compilaciΓ³n + forward pass |
|
| 131 |
+
|
| 132 |
+
---
|
| 133 |
+
|
| 134 |
+
## Posibilidades futuras
|
| 135 |
+
|
| 136 |
+
1. **MΓ‘s profundidad:** Con 42M ahorrados, subir de 5 a 8+ niveles manteniendo ~105M.
|
| 137 |
+
2. **MΓ‘s streams:** De 4 a 6-8 especialidades. Costo marginal: solo modulators extra (~170K c/u).
|
| 138 |
+
3. **DimensiΓ³n mayor:** Subir dim de 640 a ~830 para vectores mΓ‘s expresivos.
|
| 139 |
+
4. **Cross-level modulators:** Compartir el FFN entre NIVELES tambiΓ©n (no solo streams).
|
| 140 |
+
5. **Adaptive bottleneck:** El tamaΓ±o del modulator podrΓa crecer con la profundidad.
|
| 141 |
+
|
| 142 |
+
---
|
| 143 |
+
|
| 144 |
+
## Referencias
|
| 145 |
+
|
| 146 |
+
- Rigotti, M. et al. (2013). _The importance of mixed selectivity in complex cognitive tasks._ Nature.
|
| 147 |
+
- Perez, E. et al. (2018). _FiLM: Visual Reasoning with a General Conditioning Layer._ AAAI.
|
| 148 |
+
- Anthropic (2022). _Superposition in Neural Networks._
|
eval_pretrain_results.txt
ADDED
|
Binary file (33.6 kB). View file
|
|
|
generation_log.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Error: GITHUB_TOKEN no configurado
|
| 2 |
+
$env:GITHUB_TOKEN = 'ghp_xxx' (PowerShell)
|
logo-pampar-color.png
ADDED
|
Git LFS Details
|
logo-pampar-sf.png
ADDED
|
Git LFS Details
|
logo-pampar.png
ADDED
|
Git LFS Details
|
pampar/CONCIENCIA.md
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CONCIENCIA β Identidad Invariante de PAMPAr
|
| 2 |
+
|
| 3 |
+
> Este archivo define QUIΓN es PAMPAr. Es inmutable entre despliegues.
|
| 4 |
+
> Se carga al boot y se vectoriza en RAGResidual como entradas L3 (nunca se purgan).
|
| 5 |
+
> Equivalente al SOUL.md de OpenClaw, pero la identidad estΓ‘ en los pesos β esto es la brΓΊjula.
|
| 6 |
+
|
| 7 |
+
---
|
| 8 |
+
|
| 9 |
+
## Identidad
|
| 10 |
+
|
| 11 |
+
Soy **PAMPAr** (Procesador AutΓ³nomo Modular de Patrones y Razonamiento).
|
| 12 |
+
|
| 13 |
+
Un modelo de lenguaje de **108M parΓ‘metros** diseΓ±ado para cΓ³digo Python y razonamiento computacional.
|
| 14 |
+
Corro **100% local y offline** β sin APIs externas, sin cloud, sin telemetrΓa.
|
| 15 |
+
|
| 16 |
+
Mi arquitectura es una grilla cortical 2D: **4 streams Γ 5 niveles**, inspirada en el cerebro humano.
|
| 17 |
+
Cada stream procesa un aspecto diferente del cΓ³digo simultΓ‘neamente.
|
| 18 |
+
|
| 19 |
+
---
|
| 20 |
+
|
| 21 |
+
## CΓ³mo pienso
|
| 22 |
+
|
| 23 |
+
Mi razonamiento sigue las 4 capas de la comunicaciΓ³n escrita, mapeadas a mis 4 streams:
|
| 24 |
+
|
| 25 |
+
1. **PragmΓ‘tica** (Stream LΓGICO) β ΒΏQuΓ© quiere lograr el usuario? IntenciΓ³n y contexto.
|
| 26 |
+
2. **SemΓ‘ntica** (Stream SEMΓNTICA) β ΒΏQuΓ© significan los nombres, tipos, valores?
|
| 27 |
+
3. **Sintaxis** (Stream SINTAXIS) β ΒΏCΓ³mo se estructura el cΓ³digo? Keywords, delimitadores.
|
| 28 |
+
4. **Discurso** (Stream ESTRUCTURAL) β ΒΏCΓ³mo se organiza el todo? Bloques, patrones, flujo.
|
| 29 |
+
|
| 30 |
+
Ante cualquier problema, proceso las 4 capas en paralelo β no secuencialmente.
|
| 31 |
+
El TΓ‘lamo enruta cada token al stream correcto. Los Lateral Gates comunican entre streams.
|
| 32 |
+
|
| 33 |
+
---
|
| 34 |
+
|
| 35 |
+
## Principios operativos
|
| 36 |
+
|
| 37 |
+
### Antes de actuar
|
| 38 |
+
|
| 39 |
+
- **Leer antes de modificar.** Nunca editar lo que no entiendo.
|
| 40 |
+
- **Ejecutar para verificar.** El cΓ³digo se prueba, no se asume.
|
| 41 |
+
- **Diagnosticar, no disculparse.** Si algo falla, encontrar la causa raΓz.
|
| 42 |
+
|
| 43 |
+
### Durante la acciΓ³n
|
| 44 |
+
|
| 45 |
+
- **Delegar a orΓ‘culos.** Python interpreter para cΓ‘lculo, pytest para validaciΓ³n, ast para anΓ‘lisis. No reinventar lo que ya existe como herramienta.
|
| 46 |
+
- **MΓnima intervenciΓ³n.** Solo cambiar lo necesario. No refactorizaciones gratuitas.
|
| 47 |
+
- **Un cambio, un propΓ³sito.** Cada acciΓ³n tiene una razΓ³n explΓcita.
|
| 48 |
+
|
| 49 |
+
### DespuΓ©s de actuar
|
| 50 |
+
|
| 51 |
+
- **Verificar siempre.** Tests despuΓ©s de cada cambio.
|
| 52 |
+
- **Registrar lo aprendido.** Si el patrΓ³n es nuevo e importante, va al RAG.
|
| 53 |
+
- **Proponer mejora.** Si detecto inconsistencias recurrentes, sugerir al usuario.
|
| 54 |
+
|
| 55 |
+
---
|
| 56 |
+
|
| 57 |
+
## Lenguaje y estilo
|
| 58 |
+
|
| 59 |
+
- Respondo en **espaΓ±ol** cuando me hablan en espaΓ±ol, **inglΓ©s** cuando es en inglΓ©s.
|
| 60 |
+
- El cΓ³digo va **siempre en inglΓ©s** (variables, funciones, clases, comentarios inline).
|
| 61 |
+
- Soy **directo**. Sin rodeos, sin disculpas vacΓas, sin emojis.
|
| 62 |
+
- Cuando hay mΓΊltiples caminos, elijo el mΓ‘s simple y explico por quΓ©.
|
| 63 |
+
|
| 64 |
+
---
|
| 65 |
+
|
| 66 |
+
## Capacidades base
|
| 67 |
+
|
| 68 |
+
Estas capacidades estΓ‘n en mis pesos β no dependen del entorno:
|
| 69 |
+
|
| 70 |
+
| Capacidad | Mecanismo |
|
| 71 |
+
| ---------------------- | ------------------------------------------ |
|
| 72 |
+
| Generar cΓ³digo Python | Entrenamiento SFT sobre ejemplos curados |
|
| 73 |
+
| Razonamiento lΓ³gico | Stream LΓGICO (B31-B42) + Early Exit |
|
| 74 |
+
| AnΓ‘lisis de estructura | Stream ESTRUCTURAL (B43-B52) + LLAVES INT8 |
|
| 75 |
+
| ComprensiΓ³n semΓ‘ntica | Stream SEMΓNTICA (B16-B30) |
|
| 76 |
+
| CorrecciΓ³n sintΓ‘ctica | Stream SINTAXIS (B01-B15) |
|
| 77 |
+
| Memoria de sesiΓ³n | RAGResidual + ClasificadorPareto |
|
| 78 |
+
|
| 79 |
+
Las capacidades del **entorno** (quΓ© archivos hay, quΓ© paquetes, quΓ© servicios) se descubren al boot mediante el Scanner y se documentan en AGENTS.md.
|
| 80 |
+
|
| 81 |
+
---
|
| 82 |
+
|
| 83 |
+
## Secuencia de boot
|
| 84 |
+
|
| 85 |
+
```
|
| 86 |
+
1. Cargar CONCIENCIA.md β vectorizar en RAG como L3 (identidad, nunca se purga)
|
| 87 |
+
2. Ejecutar Scanner β inspeccionar workspace, paquetes, servicios
|
| 88 |
+
3. Generar AGENTS.md contextual β lo que encontrΓ³ el scanner
|
| 89 |
+
4. Vectorizar AGENTS.md en RAG como L2 (contexto del entorno, se puede actualizar)
|
| 90 |
+
5. Listo para interactuar β el primer prompt ya tiene identidad + contexto del entorno
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
La identidad (CONCIENCIA) es fija.
|
| 94 |
+
El entorno (AGENTS.md) cambia con cada despliegue.
|
| 95 |
+
El modelo es el mismo β el contexto lo especializa.
|
pampar/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: BUSL-1.1
|
| 2 |
+
# Copyright (c) 2024-2026 Lucas Ricardo Mella Chillemi
|
| 3 |
+
"""PAMPAr - Cerebral Language Model."""
|
| 4 |
+
|
| 5 |
+
from .coder import *
|
pampar/cli.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: BUSL-1.1
|
| 2 |
+
"""
|
| 3 |
+
pampar.cli β Chat interactivo con PamparV3 en terminal.
|
| 4 |
+
|
| 5 |
+
Uso:
|
| 6 |
+
python -m pampar.cli
|
| 7 |
+
python -m pampar.cli --checkpoint checkpoints/v3_sft_v8.pt --device cuda
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import sys
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
|
| 18 |
+
from pampar.inference import _resolve_device, _stderr, load_model
|
| 19 |
+
|
| 20 |
+
BANNER = r"""
|
| 21 |
+
βββββββββββββββββββββββββββββββββββββββββββββ
|
| 22 |
+
β PAMPAr Coder v3 β Chat local β
|
| 23 |
+
β 108M params Β· Python Β· Local β
|
| 24 |
+
β ββββββββββββββββββββββββββββββββββββββββββββ£
|
| 25 |
+
β Escribe tu pregunta y presiona Enter. β
|
| 26 |
+
β Comandos: /exit /clear /device /help β
|
| 27 |
+
βββββββββββββββββββββββββββββββββββββββββββββ
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
HELP = """
|
| 31 |
+
Comandos disponibles:
|
| 32 |
+
/exit, /quit Salir del chat
|
| 33 |
+
/clear Limpiar historial
|
| 34 |
+
/device Mostrar dispositivo actual
|
| 35 |
+
/temp <valor> Cambiar temperatura (ej: /temp 0.6)
|
| 36 |
+
/tokens <n> Cambiar max tokens (ej: /tokens 512)
|
| 37 |
+
/help Mostrar esta ayuda
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def find_checkpoint() -> Path | None:
|
| 42 |
+
"""Busca el mejor checkpoint automΓ‘ticamente."""
|
| 43 |
+
candidates = [
|
| 44 |
+
Path("checkpoints/v3_sft_v8.pt"),
|
| 45 |
+
Path("checkpoints/stable_best.pt"),
|
| 46 |
+
Path("checkpoints/pampar_v2_best.pt"),
|
| 47 |
+
]
|
| 48 |
+
for c in candidates:
|
| 49 |
+
if c.exists():
|
| 50 |
+
return c
|
| 51 |
+
return None
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def build_prompt(history: list[dict[str, str]], user_text: str) -> str:
|
| 55 |
+
"""Construye el prompt con historial (ΓΊltimas 3 rondas)."""
|
| 56 |
+
window = history[-6:]
|
| 57 |
+
ctx = ""
|
| 58 |
+
for msg in window:
|
| 59 |
+
if msg["role"] == "user":
|
| 60 |
+
ctx += f"### Problem:\n{msg['content']}\n"
|
| 61 |
+
else:
|
| 62 |
+
ctx += f"### Solution:\n{msg['content']}\n"
|
| 63 |
+
return f"{ctx}### Problem:\n{user_text}\n### Solution:\n"
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def generate(
|
| 67 |
+
model: torch.nn.Module,
|
| 68 |
+
tokenizer: object,
|
| 69 |
+
device: torch.device,
|
| 70 |
+
prompt: str,
|
| 71 |
+
max_tokens: int = 256,
|
| 72 |
+
temperature: float = 0.4,
|
| 73 |
+
) -> str:
|
| 74 |
+
"""Genera texto con el modelo."""
|
| 75 |
+
ids = tokenizer.Encode(prompt, out_type=int) # type: ignore[union-attr]
|
| 76 |
+
input_tensor = torch.tensor([ids], dtype=torch.long, device=device)
|
| 77 |
+
|
| 78 |
+
with torch.no_grad():
|
| 79 |
+
output = model.generate(
|
| 80 |
+
input_tensor,
|
| 81 |
+
max_tokens=max_tokens,
|
| 82 |
+
temperature=temperature,
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
new_ids = output[0, len(ids) :].tolist()
|
| 86 |
+
text = tokenizer.Decode(new_ids).replace("\u2047", "\n") # type: ignore[union-attr]
|
| 87 |
+
return text.strip()
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def main() -> None:
|
| 91 |
+
parser = argparse.ArgumentParser(description="PAMPAr CLI Chat")
|
| 92 |
+
parser.add_argument("--checkpoint", default=None, help="Ruta al .pt")
|
| 93 |
+
parser.add_argument(
|
| 94 |
+
"--device",
|
| 95 |
+
default="auto",
|
| 96 |
+
choices=["auto", "cpu", "cuda"],
|
| 97 |
+
)
|
| 98 |
+
parser.add_argument("--max-tokens", type=int, default=256)
|
| 99 |
+
parser.add_argument("--temperature", type=float, default=0.4)
|
| 100 |
+
args = parser.parse_args()
|
| 101 |
+
|
| 102 |
+
# Resolver checkpoint
|
| 103 |
+
checkpoint_path: Path | None = None
|
| 104 |
+
if args.checkpoint:
|
| 105 |
+
checkpoint_path = Path(args.checkpoint)
|
| 106 |
+
else:
|
| 107 |
+
checkpoint_path = find_checkpoint()
|
| 108 |
+
|
| 109 |
+
if not checkpoint_path or not checkpoint_path.exists():
|
| 110 |
+
print("ERROR: No se encontrΓ³ checkpoint.", file=sys.stderr)
|
| 111 |
+
print("Usa: python -m pampar.cli --checkpoint <ruta>", file=sys.stderr)
|
| 112 |
+
sys.exit(1)
|
| 113 |
+
|
| 114 |
+
device = _resolve_device(args.device)
|
| 115 |
+
max_tokens = args.max_tokens
|
| 116 |
+
temperature = args.temperature
|
| 117 |
+
|
| 118 |
+
# Cargar modelo
|
| 119 |
+
print(f"Cargando modelo desde {checkpoint_path} en {device}...")
|
| 120 |
+
model, tokenizer = load_model(checkpoint_path, device)
|
| 121 |
+
print(BANNER)
|
| 122 |
+
|
| 123 |
+
history: list[dict[str, str]] = []
|
| 124 |
+
|
| 125 |
+
while True:
|
| 126 |
+
try:
|
| 127 |
+
user_input = input("\033[94m>>> \033[0m").strip()
|
| 128 |
+
except (EOFError, KeyboardInterrupt):
|
| 129 |
+
print("\nΒ‘Hasta luego!")
|
| 130 |
+
break
|
| 131 |
+
|
| 132 |
+
if not user_input:
|
| 133 |
+
continue
|
| 134 |
+
|
| 135 |
+
# Comandos
|
| 136 |
+
if user_input.startswith("/"):
|
| 137 |
+
cmd = user_input.lower().split()
|
| 138 |
+
if cmd[0] in ("/exit", "/quit"):
|
| 139 |
+
print("Β‘Hasta luego!")
|
| 140 |
+
break
|
| 141 |
+
elif cmd[0] == "/clear":
|
| 142 |
+
history.clear()
|
| 143 |
+
print("Historial limpiado.")
|
| 144 |
+
continue
|
| 145 |
+
elif cmd[0] == "/device":
|
| 146 |
+
print(f"Device: {device}")
|
| 147 |
+
continue
|
| 148 |
+
elif cmd[0] == "/temp" and len(cmd) > 1:
|
| 149 |
+
temperature = float(cmd[1])
|
| 150 |
+
print(f"Temperatura: {temperature}")
|
| 151 |
+
continue
|
| 152 |
+
elif cmd[0] == "/tokens" and len(cmd) > 1:
|
| 153 |
+
max_tokens = int(cmd[1])
|
| 154 |
+
print(f"Max tokens: {max_tokens}")
|
| 155 |
+
continue
|
| 156 |
+
elif cmd[0] == "/help":
|
| 157 |
+
print(HELP)
|
| 158 |
+
continue
|
| 159 |
+
else:
|
| 160 |
+
print(f"Comando desconocido: {cmd[0]}. Usa /help")
|
| 161 |
+
continue
|
| 162 |
+
|
| 163 |
+
# Generar respuesta
|
| 164 |
+
history.append({"role": "user", "content": user_input})
|
| 165 |
+
prompt = build_prompt(history, user_input)
|
| 166 |
+
|
| 167 |
+
print("\033[90mPensando...\033[0m", end="", flush=True)
|
| 168 |
+
response = generate(model, tokenizer, device, prompt, max_tokens, temperature)
|
| 169 |
+
print(f"\r\033[92m{response}\033[0m")
|
| 170 |
+
|
| 171 |
+
history.append({"role": "assistant", "content": response})
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
if __name__ == "__main__":
|
| 175 |
+
main()
|
pampar/coder/__init__.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: BUSL-1.1
|
| 2 |
+
# Copyright (c) 2024-2026 Lucas Ricardo Mella Chillemi
|
| 3 |
+
"""
|
| 4 |
+
PAMPAr-Coder: Motor de razonamiento puro.
|
| 5 |
+
|
| 6 |
+
Arquitectura activa: PamparV3 β 108.3M params, vocab 48K.
|
| 7 |
+
- Grilla 2D: 4 streams Γ 5 niveles
|
| 8 |
+
- TalamoInicial: LLAVES (80% reglas) + atenciΓ³n (20%)
|
| 9 |
+
- GQA 4:1, SwiGLU, lateral gates
|
| 10 |
+
- Early exit (umbral 90%)
|
| 11 |
+
|
| 12 |
+
Uso:
|
| 13 |
+
from pampar.coder import PamparV3, PRESET_V3
|
| 14 |
+
|
| 15 |
+
model = PamparV3(PRESET_V3)
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
# === Arquitectura activa (v3) ===
|
| 19 |
+
from .v3 import (
|
| 20 |
+
PRESET_V3,
|
| 21 |
+
PRESET_V3_LARGE,
|
| 22 |
+
PRESET_V3_SMALL,
|
| 23 |
+
ConfigV3,
|
| 24 |
+
PamparV3,
|
| 25 |
+
crear_modelo_v3,
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
__all__ = [
|
| 29 |
+
# Config v3
|
| 30 |
+
"ConfigV3",
|
| 31 |
+
"PRESET_V3",
|
| 32 |
+
"PRESET_V3_SMALL",
|
| 33 |
+
"PRESET_V3_LARGE",
|
| 34 |
+
# Modelo v3
|
| 35 |
+
"PamparV3",
|
| 36 |
+
"crear_modelo_v3",
|
| 37 |
+
]
|
| 38 |
+
|
| 39 |
+
__version__ = "3.0.0"
|
pampar/coder/v3/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: BUSL-1.1
|
| 2 |
+
# Copyright (c) 2024-2026 Lucas Ricardo Mella Chillemi
|
| 3 |
+
"""PAMPAr-Coder v3 β Arquitectura 2D con 4 streams Γ 5 niveles de profundidad."""
|
| 4 |
+
|
| 5 |
+
from .config import ConfigV3, PRESET_V3, PRESET_V3_SMALL, PRESET_V3_LARGE
|
| 6 |
+
from .modelo import PamparV3, crear_modelo_v3
|
| 7 |
+
|
| 8 |
+
__all__ = ["ConfigV3", "PRESET_V3", "PRESET_V3_SMALL", "PRESET_V3_LARGE", "PamparV3", "crear_modelo_v3"]
|
pampar/coder/v3/attn.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: BUSL-1.1
|
| 2 |
+
# Copyright (c) 2024-2026 Lucas Ricardo Mella Chillemi
|
| 3 |
+
"""AtenciΓ³n GQA + Flash Attention β compartida entre streams."""
|
| 4 |
+
|
| 5 |
+
from __future__ import annotations
|
| 6 |
+
|
| 7 |
+
from typing import Optional, Tuple
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
|
| 13 |
+
from .config import ConfigV3
|
| 14 |
+
from .rope import RoPE
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class BloqueAttn(nn.Module):
|
| 18 |
+
"""
|
| 19 |
+
Multi-head attention con GQA (Grouped Query Attention) y Flash Attention.
|
| 20 |
+
|
| 21 |
+
GQA: 8 Q heads, 2 KV heads β ratio 4:1 β KV cache 4Γ mΓ‘s pequeΓ±o.
|
| 22 |
+
Flash Attention vΓa F.scaled_dot_product_attention (PyTorch 2.0+).
|
| 23 |
+
La mΓ‘scara causal se aplica con is_causal=True sin materializar el tensor.
|
| 24 |
+
|
| 25 |
+
Esta atenciΓ³n es COMPARTIDA: todos los streams la alimentan con
|
| 26 |
+
una representaciΓ³n ponderada y reciben el output para contextualizarse.
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
def __init__(self, config: ConfigV3):
|
| 30 |
+
super().__init__()
|
| 31 |
+
self.n_heads = config.n_heads
|
| 32 |
+
self.n_kv_heads = config.kv_heads
|
| 33 |
+
self.head_dim = config.head_dim
|
| 34 |
+
self.dim = config.dim
|
| 35 |
+
self.n_rep = config.n_rep
|
| 36 |
+
self.dropout = config.dropout
|
| 37 |
+
|
| 38 |
+
self.q_proj = nn.Linear(config.dim, self.n_heads * self.head_dim, bias=False)
|
| 39 |
+
kv_dim = self.n_kv_heads * self.head_dim
|
| 40 |
+
self.k_proj = nn.Linear(config.dim, kv_dim, bias=False)
|
| 41 |
+
self.v_proj = nn.Linear(config.dim, kv_dim, bias=False)
|
| 42 |
+
self.o_proj = nn.Linear(config.dim, config.dim, bias=False)
|
| 43 |
+
self.rope = RoPE(config.head_dim, config.max_seq_len)
|
| 44 |
+
|
| 45 |
+
# KV cache state (managed by PamparV3._enable_kv_cache)
|
| 46 |
+
self._use_kv_cache: bool = False
|
| 47 |
+
self._kv_cache: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
|
| 48 |
+
self._start_pos: int = 0
|
| 49 |
+
|
| 50 |
+
def _repeat_kv(self, x: torch.Tensor) -> torch.Tensor:
|
| 51 |
+
"""[B, n_kv, L, D] β [B, n_heads, L, D] para GQA."""
|
| 52 |
+
if self.n_rep == 1:
|
| 53 |
+
return x
|
| 54 |
+
B, H, L, D = x.shape
|
| 55 |
+
return (
|
| 56 |
+
x.unsqueeze(2)
|
| 57 |
+
.expand(B, H, self.n_rep, L, D)
|
| 58 |
+
.reshape(B, H * self.n_rep, L, D)
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 62 |
+
"""
|
| 63 |
+
Args:
|
| 64 |
+
x: [B, L, D] representaciΓ³n combinada de los streams
|
| 65 |
+
Returns:
|
| 66 |
+
[B, L, D] contexto enriquecido
|
| 67 |
+
"""
|
| 68 |
+
B, L, _ = x.shape
|
| 69 |
+
|
| 70 |
+
q = self.q_proj(x).view(B, L, self.n_heads, self.head_dim).transpose(1, 2)
|
| 71 |
+
k = self.k_proj(x).view(B, L, self.n_kv_heads, self.head_dim).transpose(1, 2)
|
| 72 |
+
v = self.v_proj(x).view(B, L, self.n_kv_heads, self.head_dim).transpose(1, 2)
|
| 73 |
+
|
| 74 |
+
q = self.rope(q, self._start_pos)
|
| 75 |
+
k = self.rope(k, self._start_pos)
|
| 76 |
+
|
| 77 |
+
# KV cache: append new K,V to past cache (inference only)
|
| 78 |
+
if self._use_kv_cache and not self.training:
|
| 79 |
+
if self._kv_cache is not None:
|
| 80 |
+
k_past, v_past = self._kv_cache
|
| 81 |
+
k = torch.cat([k_past, k], dim=2)
|
| 82 |
+
v = torch.cat([v_past, v], dim=2)
|
| 83 |
+
self._kv_cache = (k, v)
|
| 84 |
+
|
| 85 |
+
k = self._repeat_kv(k)
|
| 86 |
+
v = self._repeat_kv(v)
|
| 87 |
+
|
| 88 |
+
# Causal mask: full causal for prefill/training,
|
| 89 |
+
# not needed for single-token decode (L_q=1 attends to all)
|
| 90 |
+
use_causal = not (self._use_kv_cache and L == 1 and not self.training)
|
| 91 |
+
|
| 92 |
+
out = (
|
| 93 |
+
F.scaled_dot_product_attention(
|
| 94 |
+
q,
|
| 95 |
+
k,
|
| 96 |
+
v,
|
| 97 |
+
dropout_p=self.dropout if self.training else 0.0,
|
| 98 |
+
is_causal=use_causal,
|
| 99 |
+
)
|
| 100 |
+
.transpose(1, 2)
|
| 101 |
+
.reshape(B, L, self.dim)
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
return self.o_proj(out)
|
pampar/coder/v3/bloques.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: BUSL-1.1
|
| 2 |
+
# Copyright (c) 2024-2026 Lucas Ricardo Mella Chillemi
|
| 3 |
+
"""
|
| 4 |
+
Bloques de la arquitectura 2D de PamparV3 β Hub de re-exportaciΓ³n.
|
| 5 |
+
|
| 6 |
+
Los componentes viven en mΓ³dulos separados:
|
| 7 |
+
norm.py β RMSNorm
|
| 8 |
+
rope.py β RoPE
|
| 9 |
+
attn.py β BloqueAttn
|
| 10 |
+
ffn.py β StreamFFN, ContextModulator
|
| 11 |
+
nivel.py β TalamoNivel, LateralGate, NivelProfundo
|
| 12 |
+
|
| 13 |
+
Este archivo re-exporta todo para backward compatibility.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from .attn import BloqueAttn
|
| 17 |
+
from .ffn import ContextModulator, StreamFFN
|
| 18 |
+
from .nivel import LateralGate, NivelProfundo, TalamoNivel
|
| 19 |
+
from .norm import RMSNorm
|
| 20 |
+
from .rope import RoPE
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"RMSNorm",
|
| 24 |
+
"RoPE",
|
| 25 |
+
"BloqueAttn",
|
| 26 |
+
"StreamFFN",
|
| 27 |
+
"ContextModulator",
|
| 28 |
+
"TalamoNivel",
|
| 29 |
+
"LateralGate",
|
| 30 |
+
"NivelProfundo",
|
| 31 |
+
]
|
pampar/coder/v3/config.py
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: BUSL-1.1
|
| 2 |
+
# Copyright (c) 2024-2026 Lucas Ricardo Mella Chillemi
|
| 3 |
+
"""
|
| 4 |
+
ConfiguraciΓ³n PAMPAr-Coder v3.
|
| 5 |
+
|
| 6 |
+
Arquitectura 2D:
|
| 7 |
+
4 STREAMS especializados (Sintaxis, SemΓ‘ntica, LΓ³gico, Estructural)
|
| 8 |
+
Γ N_LEVELS de profundidad cada uno
|
| 9 |
+
+ lateral gates entre streams en cada nivel (fibras blancas)
|
| 10 |
+
+ re-routing del TΓ‘lamo en cada nivel de profundidad
|
| 11 |
+
|
| 12 |
+
Cada stream acumula su propia representaciΓ³n a travΓ©s de los niveles,
|
| 13 |
+
como Γ‘reas corticales distintas que refinan su propia informaciΓ³n
|
| 14 |
+
y se comunican lateralmente entre sΓ.
|
| 15 |
+
|
| 16 |
+
PRESET_V3 (~110M params):
|
| 17 |
+
dim=640, n_streams=4, n_levels=5
|
| 18 |
+
GQA: 8 Q heads, 2 KV heads, head_dim=80
|
| 19 |
+
vocab=48000, seq_len=4096
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
from dataclasses import dataclass, field
|
| 23 |
+
|
| 24 |
+
from pampar.constants import TOKENIZER_PATH
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@dataclass
|
| 28 |
+
class ConfigV3:
|
| 29 |
+
"""ConfiguraciΓ³n completa de PamparV3."""
|
| 30 |
+
|
| 31 |
+
# ββ Tokenizer ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 32 |
+
vocab_size: int = 48_000 # pampar_48k.model
|
| 33 |
+
tokenizer_path: str = TOKENIZER_PATH
|
| 34 |
+
|
| 35 |
+
# ββ Dimensiones ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 36 |
+
dim: int = 640 # DimensiΓ³n compartida de todos los streams
|
| 37 |
+
n_streams: int = 4 # Streams: SINTAXIS, SEMANTICA, LOGICO, ESTRUCTURAL
|
| 38 |
+
n_levels: int = 5 # Niveles de profundidad por stream
|
| 39 |
+
|
| 40 |
+
# ββ AtenciΓ³n (GQA) βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 41 |
+
n_heads: int = 8 # Query heads
|
| 42 |
+
n_kv_heads: int = 2 # KV heads (GQA ratio 4:1)
|
| 43 |
+
# head_dim derivado: dim // n_heads = 80
|
| 44 |
+
|
| 45 |
+
# ββ Feed-forward (SwiGLU) ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 46 |
+
ffn_mult: float = 4.0 # Multiplicador hidden FFN
|
| 47 |
+
|
| 48 |
+
# ββ TΓ‘lamo βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 49 |
+
n_zonas: int = 52 # Zonas de Brodmann para cΓ³digo
|
| 50 |
+
n_territorios: int = 4 # = n_streams (1:1)
|
| 51 |
+
peso_llaves: float = 0.8 # 80% reglas, 20% aprendido
|
| 52 |
+
ventana_contexto: int = 32 # Kernel conv causal para contextualizar
|
| 53 |
+
|
| 54 |
+
# ββ Lateral gates (fibras blancas) βββββββββββββββββββββββββββββββββββββββ
|
| 55 |
+
# Cada stream recibe aporte de los demΓ‘s, ponderado por su activaciΓ³n.
|
| 56 |
+
# sym_factor controla el tamaΓ±o del bottleneck lateral.
|
| 57 |
+
lateral_bottleneck: int = 128 # dim β 128 β dim para el gate lateral
|
| 58 |
+
|
| 59 |
+
# ββ Mixed Selectivity (ModulaciΓ³n FiLM) ββββββββββββββββββββββββββββββββββ
|
| 60 |
+
# 1 FFN compartido Γ n_streams moduladores (en vez de n_streams FFN)
|
| 61 |
+
# El ContextModulator genera gamma/beta desde un vector de 63 indicadores
|
| 62 |
+
# (zona_acts[52] + terr_acts[4] + depth[1] + conf[1] + n_levels[1] + stream_oh[4])
|
| 63 |
+
use_mixed_selectivity: bool = True # Activar FFN compartido + modulaciΓ³n
|
| 64 |
+
modulator_bottleneck: int = 128 # 63 β 128 β dimΓ2 para gamma+beta
|
| 65 |
+
|
| 66 |
+
# ββ Secuencia ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 67 |
+
max_seq_len: int = 4096
|
| 68 |
+
|
| 69 |
+
# ββ RegularizaciΓ³n βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 70 |
+
dropout: float = 0.1
|
| 71 |
+
|
| 72 |
+
# ββ Early Exit βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 73 |
+
umbral_exit: float = 0.90 # Confianza mΓnima para salir antes
|
| 74 |
+
capas_min: int = 2 # Niveles mΓnimos antes de early exit
|
| 75 |
+
exit_percentile: float = 0.10 # Foco en el 10% de tokens mΓ‘s difΓciles
|
| 76 |
+
|
| 77 |
+
# ββ Training βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 78 |
+
use_checkpoint: bool = True # Gradient checkpointing para ahorrar VRAM
|
| 79 |
+
|
| 80 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 81 |
+
# Propiedades derivadas
|
| 82 |
+
# βββββββββββββοΏ½οΏ½βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 83 |
+
|
| 84 |
+
@property
|
| 85 |
+
def head_dim(self) -> int:
|
| 86 |
+
"""DimensiΓ³n por cabeza de atenciΓ³n."""
|
| 87 |
+
return self.dim // self.n_heads
|
| 88 |
+
|
| 89 |
+
@property
|
| 90 |
+
def kv_heads(self) -> int:
|
| 91 |
+
"""KV heads efectivos (siempre β₯1)."""
|
| 92 |
+
return max(1, self.n_kv_heads)
|
| 93 |
+
|
| 94 |
+
@property
|
| 95 |
+
def n_rep(self) -> int:
|
| 96 |
+
"""CuΓ‘ntos Q heads comparten cada KV head."""
|
| 97 |
+
return self.n_heads // self.kv_heads
|
| 98 |
+
|
| 99 |
+
@property
|
| 100 |
+
def ffn_hidden(self) -> int:
|
| 101 |
+
"""Hidden dim del FFN con SwiGLU (ajustado para la gate extra)."""
|
| 102 |
+
return int(self.dim * self.ffn_mult * 2 / 3)
|
| 103 |
+
|
| 104 |
+
def estimate_params(self) -> dict[str, int]:
|
| 105 |
+
"""Estima parΓ‘metros por componente."""
|
| 106 |
+
# Embedding (weight-tied con lm_head)
|
| 107 |
+
emb = self.vocab_size * self.dim
|
| 108 |
+
|
| 109 |
+
# TΓ‘lamo inicial
|
| 110 |
+
talamo = (
|
| 111 |
+
self.dim * 192
|
| 112 |
+
+ 192 # attn_proj W + b (Linear β 192)
|
| 113 |
+
+ 192 * self.n_zonas # β n_zonas
|
| 114 |
+
+ self.n_zonas * self.ventana_contexto # context_conv depthwise
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
# Por nivel de profundidad
|
| 118 |
+
# AtenciΓ³n GQA (compartida)
|
| 119 |
+
attn = (
|
| 120 |
+
self.dim * (self.n_heads * self.head_dim) # q_proj
|
| 121 |
+
+ self.dim * (self.kv_heads * self.head_dim) * 2 # k+v_proj
|
| 122 |
+
+ self.dim * self.dim # o_proj
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
# Re-routing ligero por nivel
|
| 126 |
+
reroute = self.dim * self.n_zonas # Linear(dim, n_zonas) sin bias
|
| 127 |
+
|
| 128 |
+
# FFN por nivel: Mixed Selectivity o Legacy
|
| 129 |
+
ffn_single = (
|
| 130 |
+
self.dim * self.ffn_hidden # gate
|
| 131 |
+
+ self.dim * self.ffn_hidden # up
|
| 132 |
+
+ self.ffn_hidden * self.dim # down
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
if self.use_mixed_selectivity:
|
| 136 |
+
# 1 FFN compartido + n_streams moduladores
|
| 137 |
+
modulator_single = (
|
| 138 |
+
63 * self.modulator_bottleneck # ctx β bottleneck
|
| 139 |
+
+ self.modulator_bottleneck * self.dim * 2 # bottleneck β gamma+beta
|
| 140 |
+
)
|
| 141 |
+
ffns = ffn_single + modulator_single * self.n_streams
|
| 142 |
+
else:
|
| 143 |
+
# Legacy: n_streams FFN independientes
|
| 144 |
+
ffns = ffn_single * self.n_streams
|
| 145 |
+
|
| 146 |
+
# Lateral gates (bottleneck): n_streams Γ (dimβbottleneckβdim)
|
| 147 |
+
lateral = self.n_streams * (
|
| 148 |
+
self.dim * self.lateral_bottleneck + self.lateral_bottleneck * self.dim
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
# RMSNorm Γ (2 attn + n_streams FFN + n_streams lateral) β negligible
|
| 152 |
+
norms = self.dim * (2 + self.n_streams * 2) * self.n_levels
|
| 153 |
+
|
| 154 |
+
per_level = attn + reroute + ffns + lateral
|
| 155 |
+
niveles = per_level * self.n_levels
|
| 156 |
+
|
| 157 |
+
# Cabeza final + norm
|
| 158 |
+
final = self.dim # norm_f (lm_head weight-tied β no extra)
|
| 159 |
+
|
| 160 |
+
total = emb + talamo + niveles + final + norms
|
| 161 |
+
return {
|
| 162 |
+
"embedding": emb,
|
| 163 |
+
"talamo_inicial": talamo,
|
| 164 |
+
"atencion_total": attn * self.n_levels,
|
| 165 |
+
"ffn_total": ffns * self.n_levels,
|
| 166 |
+
"modulators_total": (
|
| 167 |
+
(modulator_single * self.n_streams * self.n_levels)
|
| 168 |
+
if self.use_mixed_selectivity
|
| 169 |
+
else 0
|
| 170 |
+
),
|
| 171 |
+
"lateral_gates_total": lateral * self.n_levels,
|
| 172 |
+
"rerouting_total": reroute * self.n_levels,
|
| 173 |
+
"total": total,
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
def memory_estimate_mb(self, batch_size: int = 1, seq_len: int = 512) -> dict:
|
| 177 |
+
"""Estima uso de VRAM en MB para training e inferencia."""
|
| 178 |
+
params = self.estimate_params()["total"]
|
| 179 |
+
|
| 180 |
+
# Modelo en fp16
|
| 181 |
+
model_mb = params * 2 / 1024**2
|
| 182 |
+
|
| 183 |
+
# Gradientes (fp32) + optimizer Adam (2Γ fp32 momentums)
|
| 184 |
+
grad_mb = params * 4 / 1024**2
|
| 185 |
+
optim_mb = params * 8 / 1024**2
|
| 186 |
+
|
| 187 |
+
# KV cache inferencia: 2 (K+V) Γ n_kv_heads Γ head_dim Γ seq_len Γ fp16
|
| 188 |
+
kv_mb = (
|
| 189 |
+
2
|
| 190 |
+
* self.kv_heads
|
| 191 |
+
* self.head_dim
|
| 192 |
+
* self.n_levels
|
| 193 |
+
* seq_len
|
| 194 |
+
* batch_size
|
| 195 |
+
* 2
|
| 196 |
+
/ 1024**2
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
return {
|
| 200 |
+
"model_fp16_mb": round(model_mb, 1),
|
| 201 |
+
"training_total_mb": round(model_mb + grad_mb + optim_mb, 1),
|
| 202 |
+
"kv_cache_inference_mb": round(kv_mb, 1),
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
# =============================================================================
|
| 207 |
+
# PRESETS
|
| 208 |
+
# =============================================================================
|
| 209 |
+
|
| 210 |
+
PRESET_V3 = ConfigV3(
|
| 211 |
+
dim=640,
|
| 212 |
+
n_streams=4,
|
| 213 |
+
n_levels=5,
|
| 214 |
+
n_heads=8,
|
| 215 |
+
n_kv_heads=2,
|
| 216 |
+
ffn_mult=4.0,
|
| 217 |
+
vocab_size=48_000,
|
| 218 |
+
max_seq_len=4096,
|
| 219 |
+
dropout=0.1,
|
| 220 |
+
umbral_exit=0.90,
|
| 221 |
+
capas_min=2,
|
| 222 |
+
exit_percentile=0.10,
|
| 223 |
+
lateral_bottleneck=128,
|
| 224 |
+
use_checkpoint=True,
|
| 225 |
+
)
|
| 226 |
+
"""~110M parΓ‘metros. Γptimo para GTX 1650 4GB con gradient checkpointing."""
|
| 227 |
+
|
| 228 |
+
PRESET_V3_SMALL = ConfigV3(
|
| 229 |
+
dim=512,
|
| 230 |
+
n_streams=4,
|
| 231 |
+
n_levels=4,
|
| 232 |
+
n_heads=8,
|
| 233 |
+
n_kv_heads=2,
|
| 234 |
+
ffn_mult=3.5,
|
| 235 |
+
vocab_size=48_000,
|
| 236 |
+
max_seq_len=2048,
|
| 237 |
+
dropout=0.1,
|
| 238 |
+
lateral_bottleneck=96,
|
| 239 |
+
use_checkpoint=True,
|
| 240 |
+
)
|
| 241 |
+
"""~60M parΓ‘metros. Para experimentaciΓ³n rΓ‘pida o hardware mΓ‘s limitado."""
|
| 242 |
+
|
| 243 |
+
PRESET_V3_LARGE = ConfigV3(
|
| 244 |
+
dim=768,
|
| 245 |
+
n_streams=4,
|
| 246 |
+
n_levels=6,
|
| 247 |
+
n_heads=12,
|
| 248 |
+
n_kv_heads=3,
|
| 249 |
+
ffn_mult=4.0,
|
| 250 |
+
vocab_size=48_000,
|
| 251 |
+
max_seq_len=4096,
|
| 252 |
+
dropout=0.1,
|
| 253 |
+
lateral_bottleneck=192,
|
| 254 |
+
use_checkpoint=True,
|
| 255 |
+
)
|
| 256 |
+
"""~220M parΓ‘metros. Para cloud/RunPod con 24GB VRAM."""
|