config.json

{
  "model_type": "gpt2",
  "n_positions": 8192,
  "n_ctx": 8192,
  "n_embd": 768,
  "n_layer": 12,
  "n_head": 12,
  "activation_function": "gelu_new",
  "resid_pdrop": 0.1,
  "embd_pdrop": 0.1,
  "attn_pdrop": 0.1,
  "layer_norm_epsilon": 1e-05,
  "initializer_range": 0.02,
  "summary_type": "cls_index",
  "summary_use_proj": true,
  "summary_activation": null,
  "summary_proj_to_labels": true,
  "summary_first_dropout": 0.1,
  "scale_attn_weights": true,
  "use_cache": true,
  "scale_attn_by_inverse_layer_idx": false,
  "reorder_and_upcast_attn": false,
  "multilingual": true,
  "supported_languages": [
    "ru",
    "es",
    "hi",
    "th"
  ],
  "spiritual_categories": [
    "icaros",
    "jiv_jago",
    "meditation",
    "yoga",
    "mantra",
    "buddhism",
    "freedome",
    "love_destiny"
  ],
  "base_model": "nativemind/shridhar_8k",
  "lora_config": {
    "r": 4,
    "lora_alpha": 8,
    "lora_dropout": 0.05,
    "bias": "none",
    "task_type": "CAUSAL_LM",
    "target_modules": [
      "c_attn",
      "c_proj"
    ]
  }
}