Spaces:
Running
Running
| { | |
| "schema_version": 1, | |
| "peft_version": "0.19.2.dev0", | |
| "baselines": { | |
| "metamathqa": { | |
| "experiment_name": "full-finetuning/llama-3.2-3B-lr_0.00001", | |
| "test accuracy": 0.49507202426080366, | |
| "forgetting": 0.5911064147949219, | |
| "peak_memory_bytes": 38549848064, | |
| "train_time_sec": 2154.846133010811, | |
| "num_trainable_params": 3212749824, | |
| "adapter_file_size_bytes": 6425499648, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "full-finetuning/flux2-klein-default", | |
| "test dino_similarity": 0.5767437418301901, | |
| "drift": null, | |
| "peak_memory_bytes": 39873150976, | |
| "train_time_sec": 292.64274606599975, | |
| "num_trainable_params": 3875544576, | |
| "adapter_file_size_bytes": 7751089152, | |
| "num_runs": 1 | |
| } | |
| }, | |
| "methods": { | |
| "ADALORA": { | |
| "config_class": "AdaLoraConfig", | |
| "model_class": "AdaLoraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": true, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": true | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "bnb_4bit", | |
| "bnb_8bit", | |
| "gptq" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": false, | |
| "source": "probe", | |
| "note": "ValueError: AdaLoraModel supports only 1 trainable adapter. When using multiple adapters, set inference_mode to True for all adapters except the one you want to train." | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "probe", | |
| "note": "a second adapter could not be added: ValueError: AdaLoraModel supports only 1 trainable adapter. When using multiple adapters, set inference_mode to True for all adapters except the one you want to train." | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": true | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [ | |
| "alpha_pattern", | |
| "layer_replication", | |
| "rank_pattern", | |
| "use_dora", | |
| "use_rslora" | |
| ], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://openreview.net/forum?id=lq62uWRJjiY", | |
| "source": "file_check", | |
| "note": "from the model class docstring" | |
| } | |
| }, | |
| "description": "AdaLoRA (Adaptive LoRA) is a method for optimizing the number of trainable parameters to assign to weight matrices and layers, unlike LoRA, which distributes parameters evenly across all modules. More parameters are budgeted for important weight matrices and layers while less important ones receive fewer parameters.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/adalora", | |
| "paper_url": "https://openreview.net/forum?id=lq62uWRJjiY", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "adalora/llama-3.2-3B-rank32", | |
| "test accuracy": 0.38362395754359363, | |
| "forgetting": 0.0754537582397461, | |
| "peak_memory_bytes": 22796042240, | |
| "train_time_sec": 1097.5522341161268, | |
| "num_trainable_params": 18353664, | |
| "adapter_file_size_bytes": 35131056, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "adalora/flux2-klein-default", | |
| "test dino_similarity": 0.6824255585670471, | |
| "drift": 0.26671119034290314, | |
| "peak_memory_bytes": 11630804992, | |
| "train_time_sec": 718.568363055, | |
| "num_trainable_params": 38341120, | |
| "adapter_file_size_bytes": 68503160, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "ADAMSS": { | |
| "config_class": "AdamssConfig", | |
| "model_class": "AdamssModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "unknown", | |
| "source": "probe", | |
| "note": "could not instantiate AdamssConfig for probing (consider adding an entry to PROBE_CONFIG_OVERRIDES): ImportError: scikit-learn is required for AdaMSS. Please install it with: pip install scikit-learn" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": "unknown", | |
| "source": "probe", | |
| "note": "could not instantiate AdamssConfig for probing (consider adding an entry to PROBE_CONFIG_OVERRIDES): ImportError: scikit-learn is required for AdaMSS. Please install it with: pip install scikit-learn" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "introspection", | |
| "note": "merge() is implemented, but probing failed: could not instantiate AdamssConfig for probing (consider adding an entry to PROBE_CONFIG_OVERRIDES): ImportError: scikit-learn is required for AdaMSS. Please install it with: pip install scikit-learn" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "based on presence of a supports_lora_conversion override; probing failed: could not instantiate AdamssConfig for probing (consider adding an entry to PROBE_CONFIG_OVERRIDES): ImportError: scikit-learn is required for AdaMSS. Please install it with: pip install scikit-learn" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://openreview.net/forum?id=8ZdWmpYxT0", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "AdaMSS (AdaMSS: Adaptive Multi-Subspace Approach for Parameter-Efficient Fine-Tuning) is a parameter-efficient fine-tuning method that decomposes weight matrices using SVD and clusters the decomposed space into multiple trainable subspaces. Each subspace learns independent low-rank updates while the original weights remain frozen.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/adamss", | |
| "paper_url": "https://openreview.net/forum?id=8ZdWmpYxT0", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "adamss/llama-3.2-3B-rank32", | |
| "test accuracy": 0.3366186504927976, | |
| "forgetting": 0.1917562484741211, | |
| "peak_memory_bytes": 20323500032, | |
| "train_time_sec": 4596.08915475203, | |
| "num_trainable_params": 293888, | |
| "adapter_file_size_bytes": 70459192, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "ADAPTION_PROMPT": { | |
| "config_class": "AdaptionPromptConfig", | |
| "model_class": "AdaptionPromptModel", | |
| "features": { | |
| "category": { | |
| "value": "other", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "quantization_backends": { | |
| "value": "unknown", | |
| "source": "introspection", | |
| "note": "no known quantization signal" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "only supported by layer-wrapping adapter methods" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "method does not implement merging" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": false, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2303.16199.", | |
| "source": "file_check", | |
| "note": "from the model class docstring" | |
| } | |
| }, | |
| "description": "Llama-Adapter is a PEFT method specifically designed for turning Llama into an instruction-following model. The Llama model is frozen and only a set of adaptation prompts prefixed to the input instruction tokens are learned.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/llama_adapter", | |
| "paper_url": "https://huggingface.co/papers/2303.16199.", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "adaptionprompt/llama-3.2-3B-lr_0.0005", | |
| "test accuracy": 0.21228203184230476, | |
| "forgetting": -0.9543380737304688, | |
| "peak_memory_bytes": 22445817856, | |
| "train_time_sec": 1180.5937879550038, | |
| "num_trainable_params": 8601628, | |
| "adapter_file_size_bytes": 17210384, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "BEFT": { | |
| "config_class": "BeftConfig", | |
| "model_class": "BeftModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://arxiv.org/abs/2509.15974", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "BEFT is a parameter efficient fine-tuning algorithm (PEFT) that only fine-tunes the added bias terms of value projections from pretrained transformer models. BEFT demonstrates that fine-tuning the added bias terms of value projections from pretrained transformers generally leads to a higher downstream performance in low-data regimes than fine-tuning the added bias terms of query/key projections.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/beft", | |
| "paper_url": "https://arxiv.org/abs/2509.15974", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "beft/llama-3.2-3B-target-v_proj", | |
| "test accuracy": 0.3290371493555724, | |
| "forgetting": -0.029061317443847656, | |
| "peak_memory_bytes": 20174602240, | |
| "train_time_sec": 895.6792339479998, | |
| "num_trainable_params": 28672, | |
| "adapter_file_size_bytes": 118192, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "BOFT": { | |
| "config_class": "BOFTConfig", | |
| "model_class": "BOFTModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": true, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "aqlm", | |
| "awq", | |
| "bnb_4bit", | |
| "bnb_8bit", | |
| "eetq", | |
| "gptq", | |
| "hqq", | |
| "inc", | |
| "torchao" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "Orthogonal Butterfly (BOFT) is a generic method designed for finetuning foundation models. It improves the parameter efficiency of the finetuning paradigm -- Orthogonal Finetuning (OFT), by taking inspiration from Cooley-Tukey fast Fourier transform, showing favorable results across finetuning different foundation models, including large vision transformers, large language models and text-to-image diffusion models.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/boft", | |
| "paper_url": null, | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "boft/llama-3.2-3B-default", | |
| "test accuracy": 0.36239575435936316, | |
| "forgetting": 0.3084230422973633, | |
| "peak_memory_bytes": 24410849280, | |
| "train_time_sec": 7190.453845723998, | |
| "num_trainable_params": 802816, | |
| "adapter_file_size_bytes": 3225360, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "boft/flux2-klein-default", | |
| "test dino_similarity": 0.6449412306149801, | |
| "drift": 0.2069137692451477, | |
| "peak_memory_bytes": 17368612864, | |
| "train_time_sec": 2524.314185989001, | |
| "num_trainable_params": 2119680, | |
| "adapter_file_size_bytes": 8500008, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "C3A": { | |
| "config_class": "C3AConfig", | |
| "model_class": "C3AModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2407.19342", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "C3A is a parameter-efficient fine-tuning technique that leverages Circular Convolution to achieve high rank adaptation within reasonable resource limits.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/c3a", | |
| "paper_url": "https://huggingface.co/papers/2407.19342", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "c3a/llama-3.2-3B-default", | |
| "test accuracy": 0.5026535253980288, | |
| "forgetting": 0.4295167922973633, | |
| "peak_memory_bytes": 22166896640, | |
| "train_time_sec": 1153.291345822101, | |
| "num_trainable_params": 5505024, | |
| "adapter_file_size_bytes": 22027512, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "c3a/flux2-klein-default", | |
| "test dino_similarity": 0.6366289258003235, | |
| "drift": 0.3616682291030884, | |
| "peak_memory_bytes": 11586764800, | |
| "train_time_sec": 1141.61083238, | |
| "num_trainable_params": 11427840, | |
| "adapter_file_size_bytes": 45722624, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "CARTRIDGE": { | |
| "config_class": "CartridgeConfig", | |
| "model_class": "CartridgeEncoder", | |
| "features": { | |
| "category": { | |
| "value": "prompt_learning", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "quantization_backends": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "only supported by layer-wrapping adapter methods" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "virtual tokens cannot be merged into base weights" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "Cartridges are a prompt-learning method that stores a compressed long-context representation as a parameterized KV-cache prefix. The core idea comes from the paper Cartridges: Lightweight and general-purpose long context representations via self-study.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/cartridges", | |
| "paper_url": null, | |
| "benchmarks": {} | |
| }, | |
| "CPT": { | |
| "config_class": "CPTConfig", | |
| "model_class": "CPTEmbedding", | |
| "features": { | |
| "category": { | |
| "value": "prompt_learning", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "quantization_backends": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "only supported by layer-wrapping adapter methods" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "virtual tokens cannot be merged into base weights" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2410.17222", | |
| "source": "file_check", | |
| "note": "from the config class docstring" | |
| } | |
| }, | |
| "description": "Context-Aware Prompt Tuning (CPT) is designed to enhance few-shot classification by refining only context embeddings. This approach combines ideas from In-Context Learning (ICL), Prompt Tuning (PT), and adversarial optimization, focusing on making model adaptation both parameter-efficient and effective.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/cpt", | |
| "paper_url": "https://huggingface.co/papers/2410.17222", | |
| "benchmarks": {} | |
| }, | |
| "DELORA": { | |
| "config_class": "DeloraConfig", | |
| "model_class": "DeloraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [ | |
| "rank_pattern" | |
| ], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2503.18225", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "DeLoRA is a parameter-efficient fine-tuning technique that implicitly maintains a Frobenius boundary with respect to the pretrained weights by normalizing and scaling learnable low-rank matrices. This effectively decouples the learning of directions (BA term) and magnitude (boundary term) of the weight updates, avoiding catastrophic shifts in the adapted weights and enhancing robustness to hyperparameter choices.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/delora", | |
| "paper_url": "https://huggingface.co/papers/2503.18225", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "delora/llama-3.2-3B-rank32", | |
| "test accuracy": 0.5026535253980288, | |
| "forgetting": 0.515233039855957, | |
| "peak_memory_bytes": 22353543168, | |
| "train_time_sec": 1058.4298028770136, | |
| "num_trainable_params": 9175096, | |
| "adapter_file_size_bytes": 37417520, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "delora/flux2-klein-default", | |
| "test dino_similarity": 0.5392374992370605, | |
| "drift": 0.46095240488648415, | |
| "peak_memory_bytes": 10563354624, | |
| "train_time_sec": 631.5103333749994, | |
| "num_trainable_params": 38338640, | |
| "adapter_file_size_bytes": 154627416, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "FOURIERFT": { | |
| "config_class": "FourierFTConfig", | |
| "model_class": "FourierFTModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2405.03003", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "FourierFT is a parameter-efficient fine-tuning technique that leverages Discrete Fourier Transform to compress the model's tunable weights. This method outperforms LoRA in the GLUE benchmark and common ViT classification tasks using much less parameters.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/fourierft", | |
| "paper_url": "https://huggingface.co/papers/2405.03003", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "fourierft/llama-3.2-3B-n_frequency-5000", | |
| "test accuracy": 0.11675511751326763, | |
| "forgetting": 0.40204334259033203, | |
| "peak_memory_bytes": 23674748928, | |
| "train_time_sec": 1735.5117494451115, | |
| "num_trainable_params": 280000, | |
| "adapter_file_size_bytes": 1127472, | |
| "num_runs": 2 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "fourierft/flux2-klein-default", | |
| "test dino_similarity": 0.3470671971638997, | |
| "drift": 0.020928651094436646, | |
| "peak_memory_bytes": 12935233536, | |
| "train_time_sec": 2988.2643262210013, | |
| "num_trainable_params": 160000, | |
| "adapter_file_size_bytes": 651152, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "FROD": { | |
| "config_class": "FrodConfig", | |
| "model_class": "FrodModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "FRoD is a parameter-efficient fine-tuning method that combines a shared full-rank basis with sparse learnable rotational degrees. The adapter update is expressed through fixed projection tensors and trainable coefficients, which allows FRoD to apply full-rank updates while keeping the number of trained parameters small.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/frod", | |
| "paper_url": null, | |
| "benchmarks": {} | |
| }, | |
| "GLORA": { | |
| "config_class": "GloraConfig", | |
| "model_class": "GloraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2306.07967", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "Generalized Low-Rank Adaptation (GLoRA) is a PEFT method that generalizes LoRA and related approaches. GLoRA decomposes updates into configurable paths (A, B, C, D, E), where each path can use low-rank, vector, constant, or disabled parameterization depending on the path.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/glora", | |
| "paper_url": "https://huggingface.co/papers/2306.07967", | |
| "benchmarks": {} | |
| }, | |
| "GRALORA": { | |
| "config_class": "GraloraConfig", | |
| "model_class": "GraloraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2505.20355", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "Unlike standard LoRA, which applies a single low-rank adapter across the entire feature space, GraLoRA introduces a structured and fine-grained adaptation scheme. It divides the adaptation space into a grid of $\ud835\udc58^2$ smaller, independent adapter pairs, each responsible for a localized subset of the input and output dimensions.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/gralora", | |
| "paper_url": "https://huggingface.co/papers/2505.20355", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "gralora/llama-3.2-3B-rank32", | |
| "test accuracy": 0.5011372251705838, | |
| "forgetting": 0.49779701232910156, | |
| "peak_memory_bytes": 22307405824, | |
| "train_time_sec": 1078.0007609191234, | |
| "num_trainable_params": 9175040, | |
| "adapter_file_size_bytes": 36714960, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "gralora/flux2-klein-default", | |
| "test dino_similarity": 0.7006240288416544, | |
| "drift": 0.2746447026729584, | |
| "peak_memory_bytes": 10729029632, | |
| "train_time_sec": 727.7293240400013, | |
| "num_trainable_params": 38338560, | |
| "adapter_file_size_bytes": 153376720, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "HIRA": { | |
| "config_class": "HiraConfig", | |
| "model_class": "HiraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": true, | |
| "Conv1d": true, | |
| "Conv2d": true, | |
| "Conv3d": true, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "bnb_4bit", | |
| "bnb_8bit" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [ | |
| "rank_pattern" | |
| ], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "High-Rank Adaptation (HiRA) is a PEFT method that extends the LoRA approach by applying an element-wise modulation on the original weight matrix. Instead of adding a low-rank update directly, HiRA computes:", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/hira", | |
| "paper_url": null, | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "hira/llama-3.2-3B-rank32-lr4.5e-3", | |
| "test accuracy": 0.5072024260803639, | |
| "forgetting": 0.4813060760498047, | |
| "peak_memory_bytes": 23599251456, | |
| "train_time_sec": 1301.0097959969953, | |
| "num_trainable_params": 9175040, | |
| "adapter_file_size_bytes": 36714432, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "HRA": { | |
| "config_class": "HRAConfig", | |
| "model_class": "HRAModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": true, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2405.17484", | |
| "source": "file_check", | |
| "note": "from the model class docstring" | |
| } | |
| }, | |
| "description": "HRA provides a new perspective connecting LoRA to OFT, which means it can harness the advantages of both strategies, by leveraging Householder reflections to reduce parameters and computation costs while penalizing the loss of pre-training knowledge. It consistently achieves better performance with fewer trainable parameters and outperforms state-of-the-art adapters across different models, including large language models (LLMs) and conditional image generators.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/hra", | |
| "paper_url": "https://huggingface.co/papers/2405.17484", | |
| "benchmarks": { | |
| "image-gen": { | |
| "experiment_name": "hra/flux2-klein-default", | |
| "test dino_similarity": 0.6441149314244589, | |
| "drift": 0.2597343921661377, | |
| "peak_memory_bytes": 47359983616, | |
| "train_time_sec": 9824.567679766005, | |
| "num_trainable_params": 9830400, | |
| "adapter_file_size_bytes": 39332240, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "IA3": { | |
| "config_class": "IA3Config", | |
| "model_class": "IA3Model", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": true, | |
| "Conv3d": true, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "bnb_4bit", | |
| "bnb_8bit" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2205.05638", | |
| "source": "file_check", | |
| "note": "from the model class docstring" | |
| } | |
| }, | |
| "description": "Infused Adapter by Inhibiting and Amplifying Inner Activations, or IA3, is a method that adds three learned vectors to rescale the keys and values of the self-attention and encoder-decoder attention layers, and the intermediate activation of the position-wise feed-forward network.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/ia3", | |
| "paper_url": "https://huggingface.co/papers/2205.05638", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "ia3/llama-3.2-3B-lr_0.001", | |
| "test accuracy": 0.41243366186504926, | |
| "forgetting": 0.2050151824951172, | |
| "peak_memory_bytes": 23240638464, | |
| "train_time_sec": 969.7345388169051, | |
| "num_trainable_params": 286720, | |
| "adapter_file_size_bytes": 1157064, | |
| "num_runs": 2 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "ia3/flux2-klein-default", | |
| "test dino_similarity": 0.6476706067721049, | |
| "drift": 0.22948315739631653, | |
| "peak_memory_bytes": 9586081792, | |
| "train_time_sec": 481.21032093000395, | |
| "num_trainable_params": 307200, | |
| "adapter_file_size_bytes": 1239104, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "LILY": { | |
| "config_class": "LilyConfig", | |
| "model_class": "LilyModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "probe", | |
| "note": "NotImplementedError: This method is not supported for Lily." | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2407.09946", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "Lily is a parameter-efficient fine-tuning technique that introduces cross-layer weight sharing for adapter matrices. Instead of learning an independent AB pair per layer as in LoRA, Lily uses **locally shared A adapters** (each A is shared across a block of `stride_A` consecutive layers) and **globally shared B experts** (a small pool of `num_B` B adapters is shared across all layers).", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/lily", | |
| "paper_url": "https://huggingface.co/papers/2407.09946", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "lily/llama-3.2-3B-rank140-mlp-a2-b2-s8.0", | |
| "test accuracy": 0.5489006823351024, | |
| "forgetting": -0.09576702117919922, | |
| "peak_memory_bytes": 25620905984, | |
| "train_time_sec": 1277.8012849430233, | |
| "num_trainable_params": 9485280, | |
| "adapter_file_size_bytes": 835056616, | |
| "num_runs": 2 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "lily/flux2-klein-default", | |
| "test dino_similarity": 0.648700992266337, | |
| "drift": 0.2646396607160568, | |
| "peak_memory_bytes": 10624172032, | |
| "train_time_sec": 672.2888688070016, | |
| "num_trainable_params": 14554112, | |
| "adapter_file_size_bytes": 267441872, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "LN_TUNING": { | |
| "config_class": "LNTuningConfig", | |
| "model_class": "LNTuningModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": true, | |
| "Conv1d": true, | |
| "Conv2d": true, | |
| "Conv3d": true, | |
| "LayerNorm": true, | |
| "MultiheadAttention": true, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2312.11420", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "LayerNorm Tuning (LN Tuning) is a PEFT method that only fine-tunes the parameters of the LayerNorm layers in a model. The paper has tested the performance of this method on large language models and has shown that it can achieve strong performance with a significant reduction in the number of trainable parameters and GPU memory usage.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/layernorm_tuning", | |
| "paper_url": "https://huggingface.co/papers/2312.11420", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "ln_tuning/llama-3.2-3B-default", | |
| "test accuracy": 0.0, | |
| "forgetting": -0.032855987548828125, | |
| "peak_memory_bytes": 21179138048, | |
| "train_time_sec": 933.4248713181005, | |
| "num_trainable_params": 175104, | |
| "adapter_file_size_bytes": 358288, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "ln_tuning/flux2-klein-default", | |
| "test dino_similarity": 0.34151384234428406, | |
| "drift": 0.0, | |
| "peak_memory_bytes": 9609150464, | |
| "train_time_sec": 458.36126944199714, | |
| "num_trainable_params": 6400, | |
| "adapter_file_size_bytes": 19856, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "LOHA": { | |
| "config_class": "LoHaConfig", | |
| "model_class": "LoHaModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": true, | |
| "Conv2d": true, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [ | |
| "alpha_pattern", | |
| "rank_pattern" | |
| ], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2108.06098", | |
| "source": "file_check", | |
| "note": "from the model class docstring" | |
| } | |
| }, | |
| "description": "Low-Rank Hadamard Product (LoHa), is similar to LoRA except it approximates the large weight matrix with more low-rank matrices and combines them with the Hadamard product. This method is even more parameter-efficient than LoRA and achieves comparable performance.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/loha", | |
| "paper_url": "https://huggingface.co/papers/2108.06098", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "loha/llama-3.2-3B-rank32", | |
| "test accuracy": 0.4184988627748294, | |
| "forgetting": 0.07485389709472656, | |
| "peak_memory_bytes": 23907532800, | |
| "train_time_sec": 1444.0684821169707, | |
| "num_trainable_params": 18350080, | |
| "adapter_file_size_bytes": 73429560, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "loha/flux2-klein-default", | |
| "test dino_similarity": 0.6618489821751913, | |
| "drift": 0.26028451323509216, | |
| "peak_memory_bytes": 12691963904, | |
| "train_time_sec": 2281.250607946, | |
| "num_trainable_params": 76677120, | |
| "adapter_file_size_bytes": 306753024, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "LOKR": { | |
| "config_class": "LoKrConfig", | |
| "model_class": "LoKrModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": true, | |
| "Conv2d": true, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [ | |
| "alpha_pattern", | |
| "rank_pattern" | |
| ], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "Low-Rank Kronecker Product (LoKr), is a LoRA-variant method that approximates the large weight matrix with two low-rank matrices and combines them with the Kronecker product. LoKr also provides an optional third low-rank matrix to provide better control during fine-tuning.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/lokr", | |
| "paper_url": null, | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "lokr/llama-3.2-3B-rank32", | |
| "test accuracy": 0.379833206974981, | |
| "forgetting": 0.3052024841308594, | |
| "peak_memory_bytes": 23571988480, | |
| "train_time_sec": 1317.615543427004, | |
| "num_trainable_params": 279552, | |
| "adapter_file_size_bytes": 1131984, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "lokr/flux2-klein-default", | |
| "test dino_similarity": 0.5805431207021078, | |
| "drift": 0.16636252403259277, | |
| "peak_memory_bytes": 10856955904, | |
| "train_time_sec": 2097.732945282998, | |
| "num_trainable_params": 783360, | |
| "adapter_file_size_bytes": 3160032, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "LORA": { | |
| "config_class": "LoraConfig", | |
| "model_class": "LoraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": true, | |
| "Conv1d": true, | |
| "Conv2d": true, | |
| "Conv3d": true, | |
| "LayerNorm": false, | |
| "MultiheadAttention": true, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": true | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "aqlm", | |
| "awq", | |
| "bnb_4bit", | |
| "bnb_8bit", | |
| "eetq", | |
| "gptq", | |
| "hqq", | |
| "inc", | |
| "torchao" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "introspection", | |
| "note": "already a LoRA adapter" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "hotswapping": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": true | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [ | |
| "alpha_pattern", | |
| "layer_replication", | |
| "rank_pattern", | |
| "use_dora", | |
| "use_rslora" | |
| ], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2106.09685.", | |
| "source": "file_check", | |
| "note": "from the model class docstring" | |
| } | |
| }, | |
| "description": "Low-Rank Adaptation (LoRA) is a PEFT method that decomposes a large matrix into two smaller low-rank matrices. This drastically reduces the number of parameters that need to be fine-tuned.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/lora", | |
| "paper_url": "https://huggingface.co/papers/2106.09685.", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "lora/llama-3.2-3B-rank64-rslora", | |
| "test accuracy": 0.5322213798332069, | |
| "forgetting": 0.6632366180419922, | |
| "peak_memory_bytes": 22554869760, | |
| "train_time_sec": 982.3692580849747, | |
| "num_trainable_params": 18350080, | |
| "adapter_file_size_bytes": 73415408, | |
| "num_runs": 7 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "lora/flux2-klein-default", | |
| "test dino_similarity": 0.697154720624288, | |
| "drift": 0.26200442016124725, | |
| "peak_memory_bytes": 10708058112, | |
| "train_time_sec": 759.1968001689999, | |
| "num_trainable_params": 38338560, | |
| "adapter_file_size_bytes": 153377056, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "MISS": { | |
| "config_class": "MissConfig", | |
| "model_class": "MissModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "aqlm", | |
| "awq", | |
| "bnb_4bit", | |
| "bnb_8bit", | |
| "eetq", | |
| "gptq", | |
| "hqq", | |
| "inc", | |
| "torchao" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://arxiv.org/abs/2409.15371", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "MiSS (Matrix Shard Sharing) is a PEFT method that achieves a good balance between model performance and computational efficiency. It requires only a single trainable matrix and introduces a shard-sharing mechanism distinct from LoRA.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/miss", | |
| "paper_url": "https://arxiv.org/abs/2409.15371", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "miss/llama-3.2-3B-bat", | |
| "test accuracy": 0.5163002274450341, | |
| "forgetting": 0.2831449508666992, | |
| "peak_memory_bytes": 25209864192, | |
| "train_time_sec": 1423.5636942129931, | |
| "num_trainable_params": 7340032, | |
| "adapter_file_size_bytes": 29367552, | |
| "num_runs": 3 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "miss/flux2-klein-default", | |
| "test dino_similarity": 0.6565384666124979, | |
| "drift": 0.3087518662214279, | |
| "peak_memory_bytes": 10561257472, | |
| "train_time_sec": 626.3718940060062, | |
| "num_trainable_params": 28508160, | |
| "adapter_file_size_bytes": 114043792, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "MULTITASK_PROMPT_TUNING": { | |
| "config_class": "MultitaskPromptTuningConfig", | |
| "model_class": "MultitaskPromptEmbedding", | |
| "features": { | |
| "category": { | |
| "value": "prompt_learning", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "quantization_backends": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "only supported by layer-wrapping adapter methods" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "virtual tokens cannot be merged into base weights" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2303.02861", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "Multitask prompt tuning decomposes the soft prompts of each task into a single learned transferable prompt instead of a separate prompt for each task. The single learned prompt can be adapted for each task by multiplicative low rank updates.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/multitask_prompt_tuning", | |
| "paper_url": "https://huggingface.co/papers/2303.02861", | |
| "benchmarks": {} | |
| }, | |
| "OFT": { | |
| "config_class": "OFTConfig", | |
| "model_class": "OFTModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": true, | |
| "Conv1d": false, | |
| "Conv2d": true, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "aqlm", | |
| "awq", | |
| "bnb_4bit", | |
| "bnb_8bit", | |
| "eetq", | |
| "gptq", | |
| "hqq", | |
| "inc" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2306.07280", | |
| "source": "file_check", | |
| "note": "from the model class docstring" | |
| } | |
| }, | |
| "description": "Orthogonal Finetuning (OFT) and OFTv2 is a method developed for adapting text-to-image diffusion models. It works by reparameterizing the pretrained weight matrices with its orthogonal matrix to preserve information in the pretrained model.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/oft", | |
| "paper_url": "https://huggingface.co/papers/2306.07280", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "oft/llama-3.2-3B-rank32", | |
| "test accuracy": 0.5034116755117514, | |
| "forgetting": 0.5187549591064453, | |
| "peak_memory_bytes": 22315794432, | |
| "train_time_sec": 1118.0740923878038, | |
| "num_trainable_params": 8171520, | |
| "adapter_file_size_bytes": 32693568, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "oft/flux2-klein-default", | |
| "test dino_similarity": 0.7077637513478597, | |
| "drift": 0.26190416514873505, | |
| "peak_memory_bytes": 9669967872, | |
| "train_time_sec": 523.9673415769976, | |
| "num_trainable_params": 4761600, | |
| "adapter_file_size_bytes": 19057488, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "OSF": { | |
| "config_class": "OSFConfig", | |
| "model_class": "OSFModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [ | |
| "rank_pattern" | |
| ], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2504.07097", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "Orthogonal Subspace Fine-tuning (OSF) is a PEFT method designed for continual learning that constrains parameter updates to be orthogonal to previously important directions. This approach enables full fine-tuning while preventing catastrophic forgetting without requiring additional parameters or storing previous gradients.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/osf", | |
| "paper_url": "https://huggingface.co/papers/2504.07097", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "osf/llama-3.2-3B-rank128", | |
| "test accuracy": 0.44579226686884005, | |
| "forgetting": 0.3507108688354492, | |
| "peak_memory_bytes": 38600179712, | |
| "train_time_sec": 2726.1649636220536, | |
| "num_trainable_params": 194535936, | |
| "adapter_file_size_bytes": 389155304, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "PEANUT": { | |
| "config_class": "PeanutConfig", | |
| "model_class": "PeanutModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://arxiv.org/abs/2410.01870", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "PEANuT is a parameter-efficient fine-tuning technique that introduces weight-aware neural tweakers to generate adapter updates from the frozen pretrained weights themselves. Instead of learning a purely linear low-rank update as in LoRA, PEANuT conditions the adapter transformation on the base weight, which makes the update rule more expressive while keeping the number of trainable parameters small.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/peanut", | |
| "paper_url": "https://arxiv.org/abs/2410.01870", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "peanut/llama-3.2-3B-rank32-relu-depth0-s2.0", | |
| "test accuracy": 0.4533737680060652, | |
| "forgetting": 0.3534202575683594, | |
| "peak_memory_bytes": 25216155648, | |
| "train_time_sec": 1324.3875514060055, | |
| "num_trainable_params": 7340032, | |
| "adapter_file_size_bytes": 29375392, | |
| "num_runs": 2 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "peanut/flux2-klein-default", | |
| "test dino_similarity": 0.7129100759824117, | |
| "drift": 0.31749118864536285, | |
| "peak_memory_bytes": 12398362624, | |
| "train_time_sec": 1981.0113854429874, | |
| "num_trainable_params": 57016320, | |
| "adapter_file_size_bytes": 228088552, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "POLY": { | |
| "config_class": "PolyConfig", | |
| "model_class": "PolyModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "no tuner layer class implements merge()" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "Polytropon is a multitask model with a number of different LoRA adapters in its \"inventory\". The model learns the correct combination of adapters from the inventory with a routing function to choose the best subset of modules for a specific task.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/poly", | |
| "paper_url": null, | |
| "benchmarks": {} | |
| }, | |
| "PREFIX_TUNING": { | |
| "config_class": "PrefixTuningConfig", | |
| "model_class": "PrefixEncoder", | |
| "features": { | |
| "category": { | |
| "value": "prompt_learning", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "quantization_backends": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "only supported by layer-wrapping adapter methods" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "virtual tokens cannot be merged into base weights" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "Prefix tuning prefixes a series of task-specific vectors to the input sequence that can be learned while keeping the pretrained model frozen. The prefix parameters are inserted in all of the model layers.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/prefix_tuning", | |
| "paper_url": null, | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "prefixtuning/llama-3.2-3B-tokens50-init_prefix-lr_0.005", | |
| "test accuracy": 0.400303260045489, | |
| "forgetting": 0.5316238403320312, | |
| "peak_memory_bytes": 20598226944, | |
| "train_time_sec": 876.9671407810383, | |
| "num_trainable_params": 2867200, | |
| "adapter_file_size_bytes": 11468928, | |
| "num_runs": 3 | |
| } | |
| } | |
| }, | |
| "PROMPT_TUNING": { | |
| "config_class": "PromptTuningConfig", | |
| "model_class": "PromptEmbedding", | |
| "features": { | |
| "category": { | |
| "value": "prompt_learning", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "quantization_backends": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "only supported by layer-wrapping adapter methods" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "virtual tokens cannot be merged into base weights" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "Prompt tuning adds a task-specific, virtual prompt to the input that consists of trainable vectors in the embedding space. The virtual token parameters are updated independently of the pretrained model parameters which are frozen.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/prompt_tuning", | |
| "paper_url": null, | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001", | |
| "test accuracy": 0.3555724033358605, | |
| "forgetting": 2.631608009338379, | |
| "peak_memory_bytes": 24423432192, | |
| "train_time_sec": 1432.4597771912813, | |
| "num_trainable_params": 614400, | |
| "adapter_file_size_bytes": 2457728, | |
| "num_runs": 3 | |
| } | |
| } | |
| }, | |
| "PSOFT": { | |
| "config_class": "PsoftConfig", | |
| "model_class": "PsoftModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://openreview.net/forum?id=FSHrinMArK.", | |
| "source": "file_check", | |
| "note": "from the config class docstring" | |
| } | |
| }, | |
| "description": "PSOFT is an Orthogonal Fine-Tuning (OFT)-based parameter-efficient fine-tuning method that preserves the geometric relationships of pre-trained weight column vectors while achieving a balanced trade-off between performance and multi-dimensional efficiency, including parameter count, memory usage, and computational cost. By restricting orthogonal transformations to a low-rank principal subspace derived from pre-trained weights, PSOFT bridges the gap between LoRA and OFT, providing both theoretical guarantees and practical adaptability.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/psoft", | |
| "paper_url": "https://openreview.net/forum?id=FSHrinMArK.", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "psoft/llama-3.2-3B-default", | |
| "test accuracy": 0.4488248673237301, | |
| "forgetting": 0.5123701095581055, | |
| "peak_memory_bytes": 21990735872, | |
| "train_time_sec": 1505.385764200997, | |
| "num_trainable_params": 1856512, | |
| "adapter_file_size_bytes": 7448144, | |
| "num_runs": 2 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "psoft/flux2-klein-default", | |
| "test dino_similarity": 0.5984091758728027, | |
| "drift": 0.15463966131210327, | |
| "peak_memory_bytes": 10536091648, | |
| "train_time_sec": 768.7374397489912, | |
| "num_trainable_params": 44800, | |
| "adapter_file_size_bytes": 211320, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "PVERA": { | |
| "config_class": "PveraConfig", | |
| "model_class": "PveraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "bnb_4bit", | |
| "bnb_8bit" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2512.07703", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "PVeRA is a parameter-efficient fine-tuning technique that is base on VeRA, in the family of the LoRA-based adapters. It keeps the very low parameter budget of VeRA, but increases the performance by learning a distribution of latent adaptations.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/pvera", | |
| "paper_url": "https://huggingface.co/papers/2512.07703", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "pvera/llama-3.2-3B-default", | |
| "test accuracy": 0.3305534495830174, | |
| "forgetting": 0.29680824279785156, | |
| "peak_memory_bytes": 22152216576, | |
| "train_time_sec": 1069.4061620029825, | |
| "num_trainable_params": 143360, | |
| "adapter_file_size_bytes": 10025160, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "pvera/flux2-klein-default", | |
| "test dino_similarity": 0.6553937196731567, | |
| "drift": 0.23013418912887573, | |
| "peak_memory_bytes": 10567548928, | |
| "train_time_sec": 729.4813956100115, | |
| "num_trainable_params": 896000, | |
| "adapter_file_size_bytes": 9504248, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "P_TUNING": { | |
| "config_class": "PromptEncoderConfig", | |
| "model_class": "PromptEncoder", | |
| "features": { | |
| "category": { | |
| "value": "prompt_learning", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "quantization_backends": { | |
| "value": "not_applicable", | |
| "source": "introspection", | |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "only supported by layer-wrapping adapter methods" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "virtual tokens cannot be merged into base weights" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "method does not wrap target layers" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "P-tuning is designed for natural language understanding (NLU) tasks and all language models.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/p_tuning", | |
| "paper_url": null, | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "ptuning/llama-3.2-3B-default", | |
| "test accuracy": 0.3821076573161486, | |
| "forgetting": 3.1548709869384766, | |
| "peak_memory_bytes": 20950548480, | |
| "train_time_sec": 959.7272375969915, | |
| "num_trainable_params": 28382208, | |
| "adapter_file_size_bytes": 245880, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "RANDLORA": { | |
| "config_class": "RandLoraConfig", | |
| "model_class": "RandLoraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "bnb_4bit", | |
| "bnb_8bit" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "RandLora is a parameter-efficient fine-tuning technique that is similar to LoRA and VeRA but performs full rank updates to improve performance. RandLora can be particularly useful when adapting large model to hard tasks that require complex updates while preserving the parameter efficiency of LoRA.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/randlora", | |
| "paper_url": null, | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "randlora/llama-3.2-3B-default", | |
| "test accuracy": 0.49962092494313876, | |
| "forgetting": 0.46999263763427734, | |
| "peak_memory_bytes": 22791847936, | |
| "train_time_sec": 1465.4105980930617, | |
| "num_trainable_params": 9289728, | |
| "adapter_file_size_bytes": 2211281240, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "randlora/flux2-klein-default", | |
| "test dino_similarity": 0.6052322387695312, | |
| "drift": 0.4385121874511242, | |
| "peak_memory_bytes": 11456741376, | |
| "train_time_sec": 2124.7402266980134, | |
| "num_trainable_params": 23838720, | |
| "adapter_file_size_bytes": 99999999, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "ROAD": { | |
| "config_class": "RoadConfig", | |
| "model_class": "RoadModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "bnb_4bit", | |
| "bnb_8bit" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2409.00119", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "RoAd is a parameter\u2011efficient fine\u2011tuning technique that adapts large language models by learning a small set of 2\u00d72 rotation matrices (and optional scaling factors) applied to pairs of hidden dimensions. RoAd achieves competitive or superior performance compared to other PEFT methods with under 0.1% trainable parameters.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/road", | |
| "paper_url": "https://huggingface.co/papers/2409.00119", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "road/llama-3.2-3B-lr_0.001", | |
| "test accuracy": 0.3866565579984837, | |
| "forgetting": 0.1559743881225586, | |
| "peak_memory_bytes": 22806528000, | |
| "train_time_sec": 986.1049144260469, | |
| "num_trainable_params": 229376, | |
| "adapter_file_size_bytes": 931480, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "road/flux2-klein-default", | |
| "test dino_similarity": 0.6731533209482828, | |
| "drift": 0.2640690803527832, | |
| "peak_memory_bytes": 11345592320, | |
| "train_time_sec": 1011.2359452090022, | |
| "num_trainable_params": 890880, | |
| "adapter_file_size_bytes": 3584880, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "SHIRA": { | |
| "config_class": "ShiraConfig", | |
| "model_class": "ShiraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": true, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2406.13175", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "Sparse High Rank Adapters or SHiRA is an alternate type of adapter and has been found to have significant advantages over the low rank adapters. Specifically, SHiRA achieves better accuracy than LoRA for a variety of vision and language tasks.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/shira", | |
| "paper_url": "https://huggingface.co/papers/2406.13175", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "shira/llama-3.2-3B-lr_0.0003-random_seed_42", | |
| "test accuracy": 0.5276724791508719, | |
| "forgetting": 0.3599224090576172, | |
| "peak_memory_bytes": 21678260224, | |
| "train_time_sec": 1026.581762216112, | |
| "num_trainable_params": 9175040, | |
| "adapter_file_size_bytes": 110115520, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "shira/flux2-klein-default", | |
| "test dino_similarity": 0.7041612068812052, | |
| "drift": 0.2558731734752655, | |
| "peak_memory_bytes": 11054088192, | |
| "train_time_sec": 791.5365517199971, | |
| "num_trainable_params": 38338560, | |
| "adapter_file_size_bytes": 460085512, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "TINYLORA": { | |
| "config_class": "TinyLoraConfig", | |
| "model_class": "TinyLoraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": true, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://arxiv.org/abs/2602.04118", | |
| "source": "file_check", | |
| "note": "from the config class docstring" | |
| } | |
| }, | |
| "description": "TinyLoRA is an extremely parameter-efficient fine-tuning technique that builds upon the LoRA-XS approach by using SVD decomposition of frozen weights and projecting a tiny trainable vector through fixed random tensors. When combined with reinforcement learning (RL) training methods like GRPO, TinyLoRA can achieve competitive performance with as few as 1-13 trainable parameters.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/tinylora", | |
| "paper_url": "https://arxiv.org/abs/2602.04118", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "tinylora/llama-3.2-3B-default", | |
| "test accuracy": 0.009855951478392721, | |
| "forgetting": -0.7508955001831055, | |
| "peak_memory_bytes": 20126367744, | |
| "train_time_sec": 1005.6939271469892, | |
| "num_trainable_params": 3584, | |
| "adapter_file_size_bytes": 2391912, | |
| "num_runs": 2 | |
| } | |
| } | |
| }, | |
| "TRAINABLE_TOKENS": { | |
| "config_class": "TrainableTokensConfig", | |
| "model_class": "TrainableTokensModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": true, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": false, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": false, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": null, | |
| "source": "file_check", | |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" | |
| } | |
| }, | |
| "description": "The Trainable Tokens method provides a way to target specific token embeddings for fine-tuning without resorting to training the full embedding matrix or using an adapter on the embedding matrix. It is based on the initial implementation from here.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/trainable_tokens", | |
| "paper_url": null, | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "trainable_tokens/llama-3.2-3B-sos+eos", | |
| "test accuracy": 0.2721758908263836, | |
| "forgetting": 0.20132732391357422, | |
| "peak_memory_bytes": 20967325696, | |
| "train_time_sec": 997.0611474359757, | |
| "num_trainable_params": 6144, | |
| "adapter_file_size_bytes": 49424, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "UNILORA": { | |
| "config_class": "UniLoraConfig", | |
| "model_class": "UniLoraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2506.00799", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "Uni-LoRA is a PEFT method that shares a compact trainable vector bank across low-rank adapter weights. Instead of learning every LoRA matrix element independently, UniLoRA deterministically projects entries into shared `theta_d` values and learns the shared parameters used by the adapter update.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/unilora", | |
| "paper_url": "https://huggingface.co/papers/2506.00799", | |
| "benchmarks": {} | |
| }, | |
| "VBLORA": { | |
| "config_class": "VBLoRAConfig", | |
| "model_class": "VBLoRAModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2405.15179", | |
| "source": "file_check", | |
| "note": "from the config class docstring" | |
| } | |
| }, | |
| "description": "VB-LoRA is a parameter-efficient fine-tuning technique that extends LoRA by learning a fine-grained parameter-sharing scheme at the sub-vector level, achieving significantly higher parameter efficiency. This makes VB-LoRA especially useful in scenarios where storage and transmission costs are critical.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/vblora", | |
| "paper_url": "https://huggingface.co/papers/2405.15179", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "vblora/llama-3.2-3B-default", | |
| "test accuracy": 0.354814253222138, | |
| "forgetting": -0.010824203491210938, | |
| "peak_memory_bytes": 22168993792, | |
| "train_time_sec": 1188.2950591499684, | |
| "num_trainable_params": 1212416, | |
| "adapter_file_size_bytes": 4864912, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "vblora/flux2-klein-default", | |
| "test dino_similarity": 0.5697349011898041, | |
| "drift": 0.1362656056880951, | |
| "peak_memory_bytes": 10705960960, | |
| "train_time_sec": 701.1592478580133, | |
| "num_trainable_params": 38404096, | |
| "adapter_file_size_bytes": 153639928, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "VERA": { | |
| "config_class": "VeraConfig", | |
| "model_class": "VeraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [ | |
| "aqlm", | |
| "awq", | |
| "bnb_4bit", | |
| "bnb_8bit", | |
| "eetq", | |
| "gptq", | |
| "hqq", | |
| "inc", | |
| "torchao" | |
| ], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2310.11454", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "VeRA is a parameter-efficient fine-tuning technique that is similar to LoRA but requires even fewer extra parameters while promising similar or even better performance. As such, it is particularly useful when the parameter budget is very limited, e.g.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/vera", | |
| "paper_url": "https://huggingface.co/papers/2310.11454", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "vera/llama-3.2-3B-default", | |
| "test accuracy": 0.37604245640636846, | |
| "forgetting": 0.1016998291015625, | |
| "peak_memory_bytes": 21617442816, | |
| "train_time_sec": 1015.8542694449425, | |
| "num_trainable_params": 129024, | |
| "adapter_file_size_bytes": 6821968, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "vera/flux2-klein-default", | |
| "test dino_similarity": 0.6626973748207092, | |
| "drift": 0.23100578784942627, | |
| "peak_memory_bytes": 10565451776, | |
| "train_time_sec": 720.1334591520208, | |
| "num_trainable_params": 893440, | |
| "adapter_file_size_bytes": 8314200, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "WAVEFT": { | |
| "config_class": "WaveFTConfig", | |
| "model_class": "WaveFTModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": { | |
| "Linear": true, | |
| "Embedding": false, | |
| "Conv1d": false, | |
| "Conv2d": false, | |
| "Conv3d": false, | |
| "LayerNorm": false, | |
| "MultiheadAttention": false, | |
| "Conv1D (transformers)": true, | |
| "nn.Parameter": false | |
| }, | |
| "source": "probe", | |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": true, | |
| "source": "probe" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": true, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2505.12532", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "WaveFT is a novel parameter-efficient fine-tuning (PEFT) method that introduces sparse updates in the **wavelet domain** of residual matrices. Unlike LoRA, which is constrained by discrete low-rank choices, WaveFT enables fine-grained control over the number of trainable parameters by directly learning a sparse set of coefficients in the transformed space.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/waveft", | |
| "paper_url": "https://huggingface.co/papers/2505.12532", | |
| "benchmarks": { | |
| "metamathqa": { | |
| "experiment_name": "waveft/llama-3.2-3B-n_frequency-5000", | |
| "test accuracy": 0.4200151630022744, | |
| "forgetting": -0.13996505737304688, | |
| "peak_memory_bytes": 24895291392, | |
| "train_time_sec": 2067.484345029923, | |
| "num_trainable_params": 280000, | |
| "adapter_file_size_bytes": 1127304, | |
| "num_runs": 1 | |
| }, | |
| "image-gen": { | |
| "experiment_name": "waveft/flux2-klein-default", | |
| "test dino_similarity": 0.6139253179232279, | |
| "drift": 0.20272931456565857, | |
| "peak_memory_bytes": 12985565184, | |
| "train_time_sec": 2970.379196393995, | |
| "num_trainable_params": 207360, | |
| "adapter_file_size_bytes": 840360, | |
| "num_runs": 1 | |
| } | |
| } | |
| }, | |
| "XLORA": { | |
| "config_class": "XLoraConfig", | |
| "model_class": "XLoraModel", | |
| "features": { | |
| "category": { | |
| "value": "adapter", | |
| "source": "introspection" | |
| }, | |
| "target_layer_types": { | |
| "value": "unknown", | |
| "source": "probe", | |
| "note": "not probed: requires pre-trained LoRA adapter checkpoints to instantiate" | |
| }, | |
| "quantization_backends": { | |
| "value": [], | |
| "source": "file_check" | |
| }, | |
| "multiple_adapters": { | |
| "value": "unknown", | |
| "source": "probe", | |
| "note": "not probed: requires pre-trained LoRA adapter checkpoints to instantiate" | |
| }, | |
| "mixed_adapter_batches": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "merging": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "no tuner layer class implements merge()" | |
| }, | |
| "peft_mixed_model": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "lora_conversion": { | |
| "value": false, | |
| "source": "introspection", | |
| "note": "based on presence of a supports_lora_conversion override; probing failed: not probed: requires pre-trained LoRA adapter checkpoints to instantiate" | |
| }, | |
| "add_weighted_adapter": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "hotswapping": { | |
| "value": false, | |
| "source": "introspection" | |
| }, | |
| "auxiliary_modules": { | |
| "value": { | |
| "modules_to_save": false, | |
| "trainable_token_indices": false | |
| }, | |
| "source": "introspection" | |
| }, | |
| "extras": { | |
| "value": [], | |
| "source": "introspection" | |
| }, | |
| "paper_url": { | |
| "value": "https://huggingface.co/papers/2402.07148", | |
| "source": "file_check", | |
| "note": "from the docs intro" | |
| } | |
| }, | |
| "description": "Mixture of LoRA Experts (X-LoRA) is a PEFT method enabling sparse or dense mixture of LoRA experts based on a high granularity (token, layer, sequence) scalings matrix. This leverages frozen LoRA adapters and a frozen base model to drastically reduces the number of parameters that need to be fine-tuned.", | |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/xlora", | |
| "paper_url": "https://huggingface.co/papers/2402.07148", | |
| "benchmarks": {} | |
| } | |
| } | |
| } | |