PEFT LoRA adapter for MOSS-Image-Base: eight

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images/example.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,102 @@
+---
+license: apache-2.0
+base_model: OpenMOSS-Team/MOSS-Image-Base
+base_model_relation: adapter
+library_name: peft
+pipeline_tag: text-to-image
+language:
+  - sv
+tags:
+  - lora
+  - peft
+  - moss-image
+  - handwriting
+  - historical-documents
+  - archives
+  - htr
+  - synthetic-data
+widget:
+  - text: "A flat, evenly lit archival scan of a handwritten page from the Bergskollegium records of Stora Kopparbergs bergslag."
+    output:
+      url: images/example.png
+---
+
+# diffu-moss-hand-eight
+
+A LoRA adapter for [MOSS-Image-Base](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base) that writes in
+eight archival hands in one adapter, each chosen by naming its collection in the prompt. It is part of [diffu](https://github.com/Borg93/diffu2), which draws synthetic archive pages with exact
+ALTO and PAGE labels, for training handwriting and page-reading models.
+
+![A window drawn with this adapter](images/example.png)
+
+*A page window drawn by MOSS-Image-Base with this adapter; not a scan.*
+
+## Files
+
+| File | What |
+|---|---|
+| `adapter_config.json` | a PEFT `LoraConfig`: rank 32, alpha 32, on 265 Linear layers of the DiT |
+| `adapter_model.safetensors` | the LoRA weights (bfloat16), PEFT's key layout |
+
+The adapter covers the DiT's 42 joint blocks (`q`, `k`, `v`, `o`, `ffn_up_gate`, `ffn_down`), its text refiner, its two
+embedders and its final layer. The module names are MOSS-Image's own, so it applies to any implementation that keeps
+them; diffu's `MossTransformer` does.
+
+## Use
+
+With PEFT, merged into the DiT, which diffu then draws with like any MOSS-Image model:
+
+```python
+from peft import PeftModel
+
+from diffu_moss.transformer import MossTransformer
+
+dit = MossTransformer.from_pretrained("<MOSS-Image-Base snapshot>/transformer")
+dit = PeftModel.from_pretrained(dit, "Gabriel/diffu-moss-hand-eight").merge_and_unload()
+dit.save_pretrained("dit-eight")
+```
+
+```bash
+diffu-moss-init --base <MOSS-Image-Base snapshot> --transformer dit-eight --out moss-eight
+```
+
+Base snapshot: `OpenMOSS-Team/MOSS-Image-Base` at revision `85975793d8b3d432f1b6e27f352abaa4c2a4c0d3`. The adapter takes
+no sample page: the hand comes from the adapter and the prompt.
+
+### Prompting
+
+The hand is chosen by the collection the prompt's caption names, as in training (160 windows of each):
+
+| Hand | Caption names it as |
+|---|---|
+| Göteborg register | "…a handwritten page from a Göteborg register, dated 1790–1794…" |
+| Jämtlands domsaga | "…a handwritten page from the court records of Jämtlands domsaga…" |
+| Stora Kopparbergs bergslag | "…a handwritten page from the Bergskollegium records of Stora Kopparbergs bergslag…" |
+| Landskapshandlingar | "…a handwritten page from the landskapshandlingar provincial accounts…" |
+| Överståthållarämbetet | "…a handwritten page from the records of Överståthållarämbetet…" |
+| Påhlman's travel journals | "…a handwritten page from Carl Fredrik Påhlman's travel journals, dated 1824–1825…" |
+| Riksregistraturet | "…a handwritten page from Riksregistraturet, the royal chancery register…" |
+| Trolldomskommissionen | "…a handwritten page from the records of the witchcraft commissions (trolldomskommissionen), dated 1676…" |
+
+Draw as MOSS-Image is drawn: about 1 MP, 50 steps, guidance 3, shift 5, block 22 skipped in the unconditional pass.
+
+## Training
+
+- Data: 1,280 windows of about 1 MP cut from pages of Swedish archival handwriting ground truth, each
+  captioned as a MOSS Layout prompt (the collection named, every line's exact text and box).
+- Trainer: KohakuMossTrainer, a LoRA (LyCORIS, algo `lora`), rank 32, alpha 32; AdamW at 1e-4,
+  2,000 steps; MOSS-Image's own noise-level sampling; text encoder and VAE frozen.
+- Format: converted from the trainer's LyCORIS export to PEFT's layout. The conversion was checked layer by layer: every
+  key lands on a layer PEFT builds from this config, none is missing or left over, and scaling x B @ A equals the
+  trained (alpha / r) x up @ down exactly.
+
+## Limitations
+
+- Eight hands only, each from one collection; other hands
+  and kinds of page are untested.
+- The drawn text must be checked: diffu reads every drawn line back and tags a line it cannot confirm as unclear in
+  the labels.
+
+## Licence
+
+Apache-2.0, as MOSS-Image-Base.

adapter_config.json

@@ -0,0 +1,309 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "OpenMOSS-Team/MOSS-Image-Base",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": false,
+  "init_lora_weights": true,
+  "kasa_config": null,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32.0,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.21.2",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": "85975793d8b3d432f1b6e27f352abaa4c2a4c0d3",
+  "target_modules": [
+    "layers.19.proj.k",
+    "layers.28.proj.v",
+    "layers.7.proj.q",
+    "layers.16.proj.o",
+    "layers.38.proj.o",
+    "layers.38.proj.q",
+    "layers.27.proj.ffn_down",
+    "layers.37.proj.ffn_up_gate",
+    "layers.6.proj.ffn_down",
+    "layers.14.proj.ffn_up_gate",
+    "layers.4.proj.k",
+    "layers.15.proj.ffn_up_gate",
+    "layers.31.proj.ffn_down",
+    "layers.7.proj.k",
+    "layers.6.proj.q",
+    "layers.26.proj.v",
+    "layers.34.proj.k",
+    "layers.18.proj.o",
+    "layers.19.proj.v",
+    "layers.24.proj.v",
+    "layers.32.proj.ffn_down",
+    "layers.35.proj.ffn_up_gate",
+    "layers.37.proj.q",
+    "layers.9.proj.k",
+    "layers.10.proj.k",
+    "layers.4.proj.ffn_down",
+    "layers.13.proj.ffn_down",
+    "text_embedder.blocks.0.qkv",
+    "layers.6.proj.o",
+    "layers.28.proj.ffn_down",
+    "layers.15.proj.v",
+    "layers.10.proj.ffn_down",
+    "layers.33.proj.ffn_down",
+    "layers.27.proj.o",
+    "layers.5.proj.ffn_down",
+    "layers.41.proj.k",
+    "layers.29.proj.q",
+    "layers.28.proj.q",
+    "layers.34.proj.v",
+    "layers.11.proj.ffn_down",
+    "layers.14.proj.k",
+    "layers.27.proj.v",
+    "layers.9.proj.v",
+    "layers.39.proj.ffn_up_gate",
+    "layers.10.proj.v",
+    "layers.21.proj.ffn_down",
+    "layers.5.proj.ffn_up_gate",
+    "layers.25.proj.o",
+    "layers.36.proj.ffn_down",
+    "layers.2.proj.v",
+    "layers.17.proj.q",
+    "layers.29.proj.v",
+    "layers.17.proj.v",
+    "layers.32.proj.o",
+    "layers.20.proj.q",
+    "layers.3.proj.q",
+    "layers.14.proj.q",
+    "layers.35.proj.v",
+    "layers.28.proj.o",
+    "layers.1.proj.ffn_down",
+    "layers.3.proj.v",
+    "layers.27.proj.ffn_up_gate",
+    "layers.17.proj.ffn_down",
+    "layers.18.proj.v",
+    "layers.24.proj.q",
+    "layers.29.proj.ffn_down",
+    "layers.11.proj.o",
+    "layers.5.proj.k",
+    "layers.5.proj.v",
+    "layers.8.proj.v",
+    "layers.4.proj.q",
+    "layers.37.proj.o",
+    "layers.35.proj.ffn_down",
+    "layers.4.proj.o",
+    "layers.33.proj.v",
+    "layers.33.proj.ffn_up_gate",
+    "layers.23.proj.o",
+    "layers.35.proj.k",
+    "video_embedder",
+    "layers.0.proj.k",
+    "layers.14.proj.v",
+    "layers.22.proj.ffn_down",
+    "layers.8.proj.q",
+    "layers.4.proj.ffn_up_gate",
+    "layers.30.proj.k",
+    "layers.26.proj.ffn_up_gate",
+    "layers.8.proj.ffn_up_gate",
+    "layers.10.proj.ffn_up_gate",
+    "layers.20.proj.k",
+    "layers.36.proj.ffn_up_gate",
+    "layers.33.proj.q",
+    "layers.38.proj.k",
+    "layers.33.proj.o",
+    "layers.39.proj.q",
+    "layers.0.proj.ffn_up_gate",
+    "layers.25.proj.v",
+    "layers.24.proj.ffn_up_gate",
+    "layers.32.proj.q",
+    "layers.31.proj.v",
+    "layers.19.proj.o",
+    "layers.8.proj.o",
+    "layers.37.proj.k",
+    "text_embedder.blocks.1.ffn_gate",
+    "text_embedder.blocks.1.ffn_down",
+    "layers.31.proj.o",
+    "layers.26.proj.o",
+    "layers.11.proj.q",
+    "layers.33.proj.k",
+    "layers.38.proj.v",
+    "layers.41.proj.v",
+    "layers.0.proj.q",
+    "layers.22.proj.k",
+    "text_embedder.blocks.1.proj",
+    "final_linear_video",
+    "layers.12.proj.o",
+    "layers.41.proj.ffn_up_gate",
+    "layers.7.proj.v",
+    "layers.38.proj.ffn_up_gate",
+    "layers.16.proj.v",
+    "layers.24.proj.ffn_down",
+    "layers.13.proj.v",
+    "layers.20.proj.o",
+    "text_embedder.proj",
+    "layers.40.proj.o",
+    "layers.29.proj.ffn_up_gate",
+    "text_embedder.blocks.1.ffn_up",
+    "layers.40.proj.q",
+    "layers.25.proj.ffn_up_gate",
+    "layers.16.proj.k",
+    "layers.1.proj.v",
+    "layers.32.proj.ffn_up_gate",
+    "layers.40.proj.ffn_down",
+    "layers.39.proj.v",
+    "layers.2.proj.k",
+    "layers.28.proj.ffn_up_gate",
+    "layers.5.proj.o",
+    "layers.2.proj.q",
+    "layers.11.proj.k",
+    "layers.18.proj.k",
+    "layers.13.proj.o",
+    "layers.13.proj.q",
+    "layers.34.proj.o",
+    "layers.6.proj.k",
+    "layers.30.proj.v",
+    "layers.20.proj.ffn_up_gate",
+    "text_embedder.blocks.0.ffn_down",
+    "layers.31.proj.q",
+    "layers.22.proj.o",
+    "layers.27.proj.k",
+    "layers.9.proj.ffn_down",
+    "layers.9.proj.o",
+    "layers.22.proj.v",
+    "layers.8.proj.k",
+    "layers.12.proj.q",
+    "layers.23.proj.v",
+    "layers.21.proj.ffn_up_gate",
+    "layers.14.proj.o",
+    "layers.16.proj.ffn_down",
+    "layers.1.proj.ffn_up_gate",
+    "layers.1.proj.o",
+    "layers.36.proj.k",
+    "layers.32.proj.v",
+    "layers.6.proj.v",
+    "layers.16.proj.ffn_up_gate",
+    "layers.19.proj.ffn_up_gate",
+    "layers.2.proj.ffn_down",
+    "layers.21.proj.k",
+    "layers.19.proj.ffn_down",
+    "layers.26.proj.q",
+    "layers.5.proj.q",
+    "layers.17.proj.o",
+    "layers.39.proj.o",
+    "layers.21.proj.q",
+    "layers.32.proj.k",
+    "layers.12.proj.v",
+    "layers.27.proj.q",
+    "layers.40.proj.k",
+    "layers.26.proj.k",
+    "layers.34.proj.ffn_down",
+    "layers.7.proj.ffn_down",
+    "layers.28.proj.k",
+    "layers.30.proj.o",
+    "layers.12.proj.k",
+    "layers.15.proj.o",
+    "layers.16.proj.q",
+    "layers.21.proj.v",
+    "layers.23.proj.ffn_down",
+    "layers.37.proj.v",
+    "layers.31.proj.ffn_up_gate",
+    "layers.6.proj.ffn_up_gate",
+    "layers.36.proj.q",
+    "text_embedder.blocks.1.qkv",
+    "layers.39.proj.k",
+    "layers.12.proj.ffn_up_gate",
+    "layers.0.proj.ffn_down",
+    "layers.25.proj.k",
+    "layers.23.proj.k",
+    "layers.10.proj.q",
+    "layers.4.proj.v",
+    "layers.9.proj.q",
+    "layers.0.proj.o",
+    "layers.41.proj.q",
+    "layers.30.proj.ffn_up_gate",
+    "layers.40.proj.ffn_up_gate",
+    "layers.3.proj.ffn_down",
+    "layers.41.proj.o",
+    "layers.2.proj.ffn_up_gate",
+    "layers.15.proj.q",
+    "layers.35.proj.o",
+    "layers.3.proj.ffn_up_gate",
+    "layers.17.proj.ffn_up_gate",
+    "layers.9.proj.ffn_up_gate",
+    "layers.40.proj.v",
+    "layers.13.proj.k",
+    "layers.18.proj.q",
+    "layers.23.proj.ffn_up_gate",
+    "layers.23.proj.q",
+    "layers.17.proj.k",
+    "layers.20.proj.v",
+    "layers.10.proj.o",
+    "layers.18.proj.ffn_down",
+    "layers.7.proj.ffn_up_gate",
+    "layers.21.proj.o",
+    "layers.25.proj.ffn_down",
+    "layers.11.proj.v",
+    "layers.25.proj.q",
+    "layers.36.proj.v",
+    "layers.29.proj.k",
+    "layers.30.proj.ffn_down",
+    "layers.36.proj.o",
+    "layers.14.proj.ffn_down",
+    "layers.1.proj.k",
+    "layers.34.proj.q",
+    "layers.12.proj.ffn_down",
+    "layers.15.proj.k",
+    "layers.34.proj.ffn_up_gate",
+    "layers.30.proj.q",
+    "layers.31.proj.k",
+    "layers.26.proj.ffn_down",
+    "layers.7.proj.o",
+    "layers.8.proj.ffn_down",
+    "layers.41.proj.ffn_down",
+    "layers.3.proj.k",
+    "layers.29.proj.o",
+    "layers.35.proj.q",
+    "layers.13.proj.ffn_up_gate",
+    "layers.39.proj.ffn_down",
+    "text_embedder.blocks.0.ffn_gate",
+    "layers.37.proj.ffn_down",
+    "layers.11.proj.ffn_up_gate",
+    "layers.3.proj.o",
+    "layers.15.proj.ffn_down",
+    "layers.22.proj.q",
+    "layers.18.proj.ffn_up_gate",
+    "layers.24.proj.o",
+    "layers.22.proj.ffn_up_gate",
+    "layers.20.proj.ffn_down",
+    "layers.24.proj.k",
+    "layers.2.proj.o",
+    "layers.0.proj.v",
+    "layers.38.proj.ffn_down",
+    "layers.1.proj.q",
+    "layers.19.proj.q",
+    "text_embedder.blocks.0.ffn_up",
+    "text_embedder.blocks.0.proj"
+  ],
+  "target_parameters": null,
+  "task_type": null,
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9b291e6920fe2e0d2e1be1da99d1e40ed58f21491ada54b35d5f61111d2c767
+size 156254296