File size: 1,697 Bytes
87bd3a1 88d2e34 5bfcec7 88d2e34 5bfcec7 88d2e34 5bfcec7 88d2e34 5bfcec7 88d2e34 5bfcec7 88d2e34 5bfcec7 88d2e34 5bfcec7 88d2e34 5bfcec7 88d2e34 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | license: mit
language:
- en
library_name: pytorch
pipeline_tag: text-generation
tags:
- text-generation
- causal-lm
- tiny-transformer
- edge-ai
- onnx
- int8
- quantization
- knowledge-distillation
- tinybert-style
- efficient-ml
- mobile-ai
model_name: Krull-Micro
base_model: Krull-Micro (distilled from causal LM teacher)
datasets:
- custom
metrics:
- perplexity
model-index:
- name: Krull-Micro
results:
- task:
type: text-generation
dataset:
name: Custom corpus
type: custom
metrics:
- name: Perplexity
type: perplexity
value: null
config:
architecture: decoder-only transformer
hidden_size: 192
num_hidden_layers: 2
num_attention_heads: 4
intermediate_size: 512
max_position_embeddings: 256
vocab_size: 8000
attention_type: causal
distillation: full (embedding + hidden + attention + logits)
inference:
parameters:
max_new_tokens: 64
temperature: 0.8
top_k: 50
top_p: 0.95
deployment:
framework: onnxruntime
quantization: int8
edge_ready: true
memory_footprint: ~8-20MB
training:
method: knowledge distillation
teacher_model: causal language model (e.g., GPT-style)
losses:
- embedding_mse
- hidden_state_mse
- attention_mse
- kl_divergence
- cross_entropy
limitations:
- Small model capacity may reduce long-range coherence
- Sensitive to training corpus quality
- Not suitable for high-stakes or safety-critical use
intended_use:
- Edge text generation
- Embedded AI systems
- Mobile applications
- Low-latency inference
authors:
- name: Igor Machado
version: 0.1.0 |