File size: 1,388 Bytes
34c4a38 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | {
"project": "CyberCoder-7B-v1",
"description": "Cybersecurity-focused code model with structured JSON output",
"architecture_decisions": {
"base_model": {
"choice": "Qwen/Qwen2.5-Coder-7B-Instruct",
"rationale": "SOTA open code model at 7B scale. Strong on HumanEval, MBPP, LiveCodeBench. Apache 2.0 license."
},
"training_method": {
"choice": "SFT with LoRA (r=64, alpha=128)",
"rationale": "CyberPal 2.0 recipe. LoRA allows training on single A10G/A100.",
"hyperparameters": {
"learning_rate": 4e-5,
"warmup_ratio": 0.15,
"num_epochs": 2,
"max_seq_length": 4096,
"batch_size_effective": 16,
"optimizer": "AdamW",
"scheduler": "cosine"
}
}
},
"scaling_roadmap": {
"phase_1": "7B LoRA SFT (current) - $4-16, 2-4hrs on A10G",
"phase_2": "7B full SFT with 200K+ examples - $32-64, 12-24hrs on A100",
"phase_3": "32B LoRA SFT - $192-768, 24-48hrs on 8xA100",
"phase_4": "100B+ sparse MoE (frontier) - $5-50M, 2-4 months on 1000+ H100s"
},
"research_references": [
{"paper": "CyberPal 2.0", "arxiv": "2510.14113"},
{"paper": "Foundation-Sec-8B", "arxiv": "2504.21039"},
{"paper": "SWE-Master", "arxiv": "2602.03411"},
{"paper": "RL-Struct (JSON output)", "arxiv": "2512.00319"},
{"paper": "DeepSeek-V3", "arxiv": "2412.19437"}
]
}
|