| { |
| "model_type": "tiny_gpt_qa", |
| "task": "question_answering", |
| "language": "pt-BR", |
| "domain": "clash_royale", |
|
|
| "architecture": { |
| "vocab_size": 800, |
| "d_model": 64, |
| "n_heads": 4, |
| "n_layers": 2, |
| "d_ff": 192, |
| "max_position_embeddings": 64, |
| "dropout": 0.1, |
| "activation": "gelu", |
| "tie_word_embeddings": true, |
| "positional_encoding": "sinusoidal", |
| "attention_type": "causal_self_attention", |
| "normalization": "layer_norm", |
| "norm_position": "post" |
| }, |
|
|
| "special_tokens": { |
| "pad_token": "<pad>", |
| "bos_token": "<bos>", |
| "eos_token": "<eos>", |
| "sep_token": "<sep>", |
| "unk_token": "<unk>", |
| "pad_token_id": 0, |
| "bos_token_id": 1, |
| "eos_token_id": 2, |
| "sep_token_id": 3, |
| "unk_token_id": 4 |
| }, |
|
|
| "tokenizer": { |
| "type": "BPE", |
| "pre_tokenizer": "Whitespace", |
| "vocab_size": 800, |
| "min_frequency": 1, |
| "file": "tokenizer.json" |
| }, |
|
|
| "training": { |
| "batch_size": 8, |
| "epochs": 300, |
| "optimizer": "AdamW", |
| "learning_rate": 3e-3, |
| "weight_decay": 0.01, |
| "loss_function": "cross_entropy", |
| "grad_clip_norm": 1.0, |
| "seed": 42, |
| "hardware": "2x NVIDIA T4", |
| "parallelism": "torch.nn.DataParallel" |
| }, |
|
|
| "inference": { |
| "max_new_tokens": 40, |
| "temperature": 0.7, |
| "top_k": 10, |
| "sampling": "top_k_multinomial" |
| }, |
|
|
| "estimated_parameters": 500000, |
|
|
| "input_format": { |
| "template": "<bos> {pergunta} <sep> {resposta} <eos>", |
| "max_len": 64 |
| }, |
|
|
| "checkpoint": { |
| "file": "clash_royale_qa_model.pt", |
| "contains": [ |
| "model_state_dict", |
| "vocab_size", |
| "d_model", |
| "n_heads", |
| "n_layers", |
| "d_ff", |
| "max_len", |
| "pad_id", |
| "bos_id", |
| "eos_id", |
| "sep_id" |
| ] |
| } |
| } |
