zerofata
/

MS3.2-PaintedFantasy-24B

+---
+library_name: transformers
+license: apache-2.0
+datasets:
+- zerofata/Roleplay-Anime-Characters
+- zerofata/Instruct-Anime-CreativeWriting
+- zerofata/Summaries-Anime-FandomPages
+base_model:
+- mistralai/Mistral-Small-3.2-24B-Instruct-2506
+---
+<!DOCTYPE html>
+<style>
+body {
+  font-family: sans-serif;
+  color: #e8f4f8;
+  line-height: 1.6;
+  margin: 0;
+  padding: 0;
+  background-color: #0a1628;
+}
+.lemonade-text {
+  color: #4fc3f7;
+  position: relative;
+  z-index: 2;
+  margin-left: 0.2em;
+  text-shadow: 0 0 15px #4fc3f7;
+}
+/* Section styling */
+.section-container {
+  background-color: rgba(10, 22, 40, 0.8);
+  margin-bottom: 30px;
+  position: relative;
+  overflow: hidden;
+  border-bottom: 1px solid #4fc3f7;
+  box-shadow: 0 4px 15px rgba(79, 195, 247, 0.1);
+}
+.section-header {
+  display: flex;
+  align-items: center;
+  background-color: rgba(79, 195, 247, 0.06);
+  padding: 10px 20px;
+}
+.section-indicator {
+  width: 8px;
+  height: 20px;
+  background-color: #4fc3f7;
+  margin-right: 15px;
+  box-shadow: 0 0 8px rgba(79, 195, 247, 0.4);
+}
+.section-title {
+  font-family: 'Georgia', 'Times New Roman', serif;
+  color: #e8f4f8;
+  font-size: 1.4rem;
+  margin: 0;
+  letter-spacing: 1px;
+  font-weight: 400;
+  text-transform: capitalize;
+}
+.section-content {
+  padding: 20px;
+  font-family: sans-serif;
+  color: #e8f4f8;
+  line-height: 1.6;
+}
+/* Title styling */
+.title-container {
+  background-color: #051017;
+  position: relative;
+  overflow: hidden;
+  margin-bottom: 40px;
+  border-left: 3px solid #4fc3f7;
+  box-shadow: 0 6px 20px rgba(79, 195, 247, 0.15);
+}
+.title-wrapper {
+  position: relative;
+  z-index: 2;
+  padding: 25px 20px 30px 30px;
+  font-family: 'Georgia', 'Times New Roman', serif;
+}
+.title-main {
+  color: #e8f4f8;
+  font-size: 2.5rem;
+  font-weight: 700;
+  margin: 0;
+  letter-spacing: 2px;
+  display: inline-block;
+  position: relative;
+  text-transform: uppercase;
+}
+.title-prefix {
+  position: relative;
+  z-index: 2;
+}
+.title-subtitle {
+  padding-left: 15px;
+  margin-top: 5px;
+  margin-left: 5px;
+}
+.subtitle-text {
+  color: #b39ddb;
+  font-size: 1.2rem;
+  font-family: 'Georgia', 'Times New Roman', serif;
+  font-weight: 300;
+  letter-spacing: 3px;
+  text-transform: uppercase;
+  display: inline-block;
+}
+.glitchy-overlay {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-image: repeating-linear-gradient(0deg, rgba(0,0,0,0) 0, rgba(79, 195, 247, 0.08) 1px, rgba(0,0,0,0) 2px);
+  z-index: 1;
+}
+/* Data box styling */
+.data-box {
+  background-color: rgba(5, 16, 23, 0.6);
+  padding: 15px;
+  border-left: 2px solid #4fc3f7;
+  margin-bottom: 20px;
+  box-shadow: 0 2px 10px rgba(79, 195, 247, 0.1);
+}
+.data-row {
+  display: flex;
+  margin-bottom: 8px;
+}
+.data-arrow {
+  color: #4fc3f7;
+  width: 20px;
+  display: inline-block;
+}
+.data-label {
+  color: #b39ddb;
+  width: 80px;
+  display: inline-block;
+}
+/* Subheading styling */
+.subheading {
+  color: #b39ddb;
+  font-size: 1.1rem;
+  margin-top: 20px;
+  margin-bottom: 15px;
+  font-weight: 400;
+  border-bottom: 1px dashed rgba(179, 157, 219, 0.4);
+  display: inline-block;
+  text-transform: uppercase;
+  letter-spacing: 1px;
+  font-family: 'Georgia', 'Times New Roman', serif;
+}
+/* Links */
+a {
+  color: #b39ddb;
+  text-decoration: none;
+}
+a:hover {
+  text-decoration: underline;
+  color: #f8bbd9;
+}
+/* Container */
+.container {
+  max-width: 1200px;
+  margin: 20px auto;
+  padding: 40px 20px;
+  background-color: #051017;
+  background-image:
+    radial-gradient(circle at 20% 80%, rgba(79, 195, 247, 0.03) 0%, transparent 50%),
+    radial-gradient(circle at 80% 20%, rgba(179, 157, 219, 0.03) 0%, transparent 50%),
+    radial-gradient(circle at 40% 40%, rgba(248, 187, 217, 0.02) 0%, transparent 50%);
+  min-height: calc(100vh - 40px);
+  border: 1px solid #4fc3f7;
+  border-radius: 8px;
+  box-shadow: 0 8px 32px rgba(79, 195, 247, 0.15);
+}
+/* Dropdown styling */
+.dropdown-container {
+  margin-top: 20px;
+}
+.dropdown-summary {
+  cursor: pointer;
+  padding: 10px 0;
+  border-bottom: 1px dashed rgba(179, 157, 219, 0.4);
+  color: #b39ddb;
+  font-size: 1.1rem;
+  font-weight: 400;
+  text-transform: uppercase;
+  letter-spacing: 1px;
+  font-family: 'Georgia', 'Times New Roman', serif;
+  list-style: none;
+  display: flex;
+  align-items: center;
+}
+.dropdown-summary::-webkit-details-marker {
+  display: none;
+}
+.dropdown-arrow {
+  color: #4fc3f7;
+  margin-right: 10px;
+  transition: transform 0.3s ease;
+}
+.dropdown-container[open] .dropdown-arrow {
+  transform: rotate(90deg);
+}
+.dropdown-content {
+  margin-top: 15px;
+  padding: 15px;
+  background-color: rgba(5, 16, 23, 0.6);
+  border-left: 2px solid #4fc3f7;
+  box-shadow: 0 2px 10px rgba(79, 195, 247, 0.1);
+}
+.config-title {
+  color: #4fc3f7;
+  font-size: 1rem;
+  margin-bottom: 10px;
+  font-family: 'Georgia', 'Times New Roman', serif;
+  text-transform: uppercase;
+  letter-spacing: 1px;
+}
+</style>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Apocalypse Lemonade</title>
+  <link href="https://fonts.googleapis.com/css2?family=Orbitron:wght@400;500;600;700&family=JetBrains+Mono:wght@100;300;400;700&display=swap" rel="stylesheet">
+</head>
+<body>
+<div class="container">
+  <div class="title-container">
+    <!-- Glitchy overlay -->
+    <div class="glitchy-overlay"></div>
+    <!-- Main title -->
+    <div class="title-wrapper">
+      <h1 class="title-main">
+        <span class="title-prefix">APOCALYPSE</span>
+        <span class="lemonade-text">LEMONADE</span> <!-- Static text with glow -->
+      </h1>
+      <div class="title-subtitle">
+        <span class="subtitle-text">Mistral Small 3.2 24B</span>
+      </div>
+    </div>
+  </div>
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/65b19c6c638328850e12d38c/0Ka08CdFUIJtYctBeBATo.png)
+  <div class="section-container">
+    <div class="section-header">
+      <div class="section-indicator"></div>
+      <h2 class="section-title">Overview</h2>
+    </div>
+    <div class="section-content">
+      <p>Experimental release.</p>
+      <p>First Mistral model and first small model I've tuned in general.</p>
+      <p>This is an uncensored creative model intended to excel at character driven RP / ERP.</p>
+      <p>This model is designed to provide longer, narrative heavy responses where characters are portrayed accurately and proactively.</p>
+      <p>Mistral Small 3.2 surprised me. Normally I find small models annoying inconsistent, but this has been relatively creative and stable, all things considered. Possibly due to this being a QLora finetune, the model has similar issues to default MS3.2 (repetition and system prompt adherance), but I think the outputs are nice.</p>
+    </div>
+  </div>
+  <div class="section-container">
+    <div class="section-header">
+      <div class="section-indicator"></div>
+      <h2 class="section-title">SillyTavern Settings</h2>
+    </div>
+    <div class="section-content">
+      <h3 class="subheading">Recommended Roleplay Format</h3>
+      <div class="data-box">
+        <div class="data-row">
+            <span class="data-arrow">></span>
+            <span class="data-label">Actions:</span>
+            <span>In plaintext</span>
+        </div>
+      <div class="data-row">
+            <span class="data-arrow">></span>
+            <span class="data-label">Dialogue:</span>
+            <span>"In quotes"</span>
+      </div>
+      <div class="data-row">
+            <span class="data-arrow">></span>
+            <span class="data-label">Thoughts:</span>
+            <span>*In asterisks*</span>
+      </div>
+      </div>
+      <h3 class="subheading">Recommended Samplers</h3>
+      <div class="data-box">
+        <div class="data-row">
+          <span class="data-arrow">></span>
+          <span class="data-label">Temp:</span>
+          <span>0.8</span>
+        </div>
+        <div class="data-row">
+          <span class="data-arrow">></span>
+          <span class="data-label">MinP:</span>
+          <span>0.04 - 0.05</span>
+        </div>
+        <div class="data-row">
+          <span class="data-arrow">></span>
+          <span class="data-label">TopP:</span>
+          <span>0.95 - 1.0</span>
+        </div>
+        <div class="data-row">
+          <span class="data-arrow">></span>
+          <span class="data-label">Dry:</span>
+          <span>0.8, 1.75, 4</span>
+        </div>
+      </div>
+      <h3 class="subheading">Instruct</h3>
+      <div class="data-box">
+        <p style="margin: 0;">Mistral v11 (or v7) Tekken</p>
+      </div>
+    </div>
+  </div>
+  <div class="section-container">
+    <div class="section-header">
+      <div class="section-indicator"></div>
+      <h2 class="section-title">Quantizations</h2>
+    </div>
+    <div class="section-content">
+      <div style="margin-bottom: 20px;">
+        <h3 class="subheading">GGUF</h3>
+        <div class="data-box">
+          <div class="data-row">
+            <span style="color: #4fc3f7; display: inline-block; margin-right: 10px;">> </span><a href="">iMatrix (TBC)</a>
+          </div>
+        </div>
+      </div>
+      <div>
+        <h3 class="subheading">EXL3</h3>
+        <div class="data-box">
+          <div class="data-row">
+            <span style="color: #4fc3f7; display: inline-block; margin-right: 10px;">> </span><a href="">4bpw</a>
+          </div>
+          <div class="data-row">
+            <span style="color: #4fc3f7; display: inline-block; margin-right: 10px;">> </span><a href="">4.5bpw</a>
+          </div>
+          <div class="data-row">
+            <span style="color: #4fc3f7; display: inline-block; margin-right: 10px;">> </span><a href="">4.65bpw</a>
+          </div>
+          <div class="data-row">
+            <span style="color: #4fc3f7; display: inline-block; margin-right: 10px;">> </span><a href="">6bpw</a>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+  <div class="section-container">
+    <div class="section-header">
+      <div class="section-indicator"></div>
+      <h2 class="section-title">Training Process</h2>
+    </div>
+    <div class="section-content">
+      <p>Training process: Pretrain > SFT > DPO > DPO 2</p>
+      <p>Did a small pretrain on some light novels and Frieren wiki data as a test. Hasn't seemed to hurt the model and model has shown some small improvements in the lore of series that were included.</p>
+      <p>The model then went through the standard SFT using a dataset of approx 3.6 million tokens, 700 RP conversations, 1000 creative writing / instruct samples and about 100 summaries. The bulk of this data has been made public.</p>
+      <p>Finally DPO was used to make the model a little more consistent. The first stage of DPO focused on instruction following and the second tried to burn out some Mistral-isms.</p>
+      <div class="dropdown-container">
+        <details>
+          <summary class="dropdown-summary">
+            <span class="dropdown-arrow">></span>
+            Axolotl configs
+          </summary>
+          <div class="dropdown-content">
+            <p>Not optimized for cost / performance efficiency, YMMV.</p>
+            <div class="config-title">SFT 1*H100</div>
+            <pre style="background-color: rgba(5, 16, 23, 0.4); border-radius: 6px; padding: 15px; overflow-x: auto; border: 1px solid #4fc3f7; position: relative;"><code style="font-family: 'Fira Code', 'Courier New', Courier, monospace; font-size: 0.9em; color: #e8f4f8;">&#35; ====================
+&#35; MODEL CONFIGURATION
+&#35; ====================
+base_model: ./MS3-2-Pretrain/merged
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+<br>
+&#35; ====================
+&#35; DATASET CONFIGURATION
+&#35; ====================
+datasets:
+  - path: ./dataset.jsonl
+    type: chat_template
+    split: train
+    chat_template_strategy: tokenizer
+    field_messages: messages
+    message_property_mappings:
+      role: role
+      content: content
+    roles:
+      user: ["user"]
+      assistant: ["assistant"]
+      system: ["system"]
+dataset_prepared_path:
+train_on_inputs: false  &#35; Only train on assistant responses
+&#35; ====================
+&#35; QLORA CONFIGURATION
+&#35; ====================
+adapter: qlora
+load_in_4bit: true
+lora_r: 128
+lora_alpha: 128
+lora_dropout: 0.1
+lora_target_linear: true
+&#35; lora_modules_to_save:  &#35; Uncomment only if you added NEW tokens
+&#35; ====================
+&#35; TRAINING PARAMETERS
+&#35; ====================
+num_epochs: 3
+micro_batch_size: 4
+gradient_accumulation_steps: 2
+learning_rate: 1e-5
+optimizer: paged_adamw_8bit
+lr_scheduler: rex
+warmup_ratio: 0.05
+weight_decay: 0.01
+max_grad_norm: 1.0
+&#35; ====================
+&#35; SEQUENCE &amp; PACKING
+&#35; ====================
+sequence_len: 8192
+sample_packing: true
+eval_sample_packing: false
+pad_to_sequence_len: true
+&#35; ====================
+&#35; HARDWARE OPTIMIZATIONS
+&#35; ====================
+bf16: auto
+flash_attention: true
+gradient_checkpointing: true
+&#35; ====================
+&#35; EVALUATION &amp; CHECKPOINTING
+&#35; ====================
+save_strategy: steps
+save_steps: 5
+save_total_limit: 5  &#35; Keep best + last few checkpoints
+load_best_model_at_end: true
+greater_is_better: false
+&#35; ====================
+&#35; LOGGING &amp; OUTPUT
+&#35; ====================
+output_dir: ./MS3-2-SFT-2
+logging_steps: 2
+save_safetensors: true
+&#35; ====================
+&#35; WANDB TRACKING
+&#35; ====================
+wandb_project: MS3-2-SFT
+&#35; wandb_entity: your_entity
+wandb_name: run_name</code></pre>
+          </div>
+        </details>
+      </div>
+    </div>
+  </div>
+</div>
+</body>
+</html>