config.json

{
  "architectures": [
    "PalmModel"
  ],
  "attn_dropout": 0.0,
  "causal": true,
  "cross_attend": false,
  "cross_entropy_ignore_index": 0,
  "default_start_token_id": null,
  "depth": 5,
  "dim": 64,
  "dim_head": 64,
  "ff_dropout": 0.0,
  "ff_mult": 4,
  "finetune_scopes": [],
  "heads": 8,
  "init_std": 0.02,
  "lora_r": 8,
  "num_tokens": 50401,
  "rotary_xpos_scale_base": 512,
  "torch_dtype": "float32",
  "transformers_version": "4.25.1"
}