Hugging Face's logo

OktoSeek
/
oktoscript

English
ai
training
dsl
oktoscript
oktoseek
okto
automation
ai-pipelines
ai-governance
Model card Files
xet
 Community
oktoscript
File size: 2,169 Bytes
5fc8c9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# okto_version: "1.1"
PROJECT "LoRAChatbot"
DESCRIPTION "Fine-tuning a chatbot using LoRA adapters for efficient training"
VERSION "1.0"
AUTHOR "OktoSeek"

DATASET {

    mix_datasets: [
        { path: "dataset/base_conversations.jsonl", weight: 60 },
        { path: "dataset/specialized_qa.jsonl", weight: 30 },
        { path: "dataset/domain_specific.jsonl", weight: 10 }
    ]

    dataset_percent: 75

    sampling: "weighted"

    shuffle: true

    format: "jsonl"

    type: "chat"

    language: "en"
}

MODEL {

    base: "oktoseek/base-llm-7b"

    architecture: "transformer"

    parameters: 7B

    context_window: 4096

    precision: "fp16"
}

FT_LORA {

    base_model: "oktoseek/base-llm-7b"

    train_dataset: "dataset/main.jsonl"

    lora_rank: 8

    lora_alpha: 32

    dataset_percent: 75

    mix_datasets: [
        { path: "dataset/base_conversations.jsonl", weight: 60 },
        { path: "dataset/specialized_qa.jsonl", weight: 30 },
        { path: "dataset/domain_specific.jsonl", weight: 10 }
    ]

    epochs: 5

    batch_size: 4

    learning_rate: 0.00003

    device: "cuda"

    target_modules: ["q_proj", "v_proj", "k_proj", "o_proj"]
}

METRICS {
    loss
    perplexity
    accuracy
    f1
    rouge_l
}

VALIDATE {

    on_validation: true

    frequency: 1

    save_best_model: true

    metric_to_monitor: "loss"
}

MONITOR {

    level: "full"

    log_metrics: [
        "loss",
        "val_loss",
        "accuracy",
        "perplexity"
    ]

    log_system: [
        "gpu_memory_used",
        "gpu_memory_free",
        "cpu_usage",
        "ram_used",
        "temperature"
    ]

    log_speed: [
        "tokens_per_second",
        "samples_per_second"
    ]

    refresh_interval: 2s

    export_to: "runs/lora-chatbot/system.json"

    dashboard: true
}

EXPORT {

    format: ["gguf", "okm", "safetensors"]

    path: "export/"

    quantization: "int8"
}

LOGGING {

    save_logs: true

    metrics_file: "runs/lora-chatbot/metrics.json"

    training_file: "runs/lora-chatbot/training_logs.json"

    log_level: "info"

    log_every: 10
}