bitsandbytes/pr_2043/en/reference/optim/ademamix.md

AdEMAMix

AdEMAMix is a variant of the Adam optimizer.

bitsandbytes also supports paged optimizers which take advantage of CUDAs unified memory to transfer memory from the GPU to the CPU when GPU memory is exhausted.

AdEMAMix[[api-class]][[bitsandbytes.optim.AdEMAMix]]

bitsandbytes.optim.AdEMAMix[[bitsandbytes.optim.AdEMAMix]]

bitsandbytes.optim.AdEMAMix(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, optim_bits: typing.Literal[8, 32] = 32, min_8bit_size: int = 4096, is_paged: bool = False)

Source

init[[bitsandbytes.optim.AdEMAMix.init]]

__init__(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, optim_bits: typing.Literal[8, 32] = 32, min_8bit_size: int = 4096, is_paged: bool = False)

Source

AdEMAMix8bit[[bitsandbytes.optim.AdEMAMix8bit]]

bitsandbytes.optim.AdEMAMix8bit[[bitsandbytes.optim.AdEMAMix8bit]]

bitsandbytes.optim.AdEMAMix8bit(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, min_8bit_size: int = 4096, is_paged: bool = False)

Source

init[[bitsandbytes.optim.AdEMAMix8bit.init]]

__init__(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, min_8bit_size: int = 4096, is_paged: bool = False)

Source

AdEMAMix32bit[[bitsandbytes.optim.AdEMAMix32bit]]

bitsandbytes.optim.AdEMAMix32bit[[bitsandbytes.optim.AdEMAMix32bit]]

bitsandbytes.optim.AdEMAMix32bit(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, min_8bit_size: int = 4096, is_paged: bool = False)

Source

init[[bitsandbytes.optim.AdEMAMix32bit.init]]

__init__(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, min_8bit_size: int = 4096, is_paged: bool = False)

Source

PagedAdEMAMix[[bitsandbytes.optim.PagedAdEMAMix]]

bitsandbytes.optim.PagedAdEMAMix[[bitsandbytes.optim.PagedAdEMAMix]]

bitsandbytes.optim.PagedAdEMAMix(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, optim_bits: typing.Literal[8, 32] = 32, min_8bit_size: int = 4096)

Source

init[[bitsandbytes.optim.PagedAdEMAMix.init]]

__init__(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, optim_bits: typing.Literal[8, 32] = 32, min_8bit_size: int = 4096)

Source

PagedAdEMAMix8bit[[bitsandbytes.optim.PagedAdEMAMix8bit]]

bitsandbytes.optim.PagedAdEMAMix8bit[[bitsandbytes.optim.PagedAdEMAMix8bit]]

bitsandbytes.optim.PagedAdEMAMix8bit(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, min_8bit_size: int = 4096)

Source

init[[bitsandbytes.optim.PagedAdEMAMix8bit.init]]

__init__(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, min_8bit_size: int = 4096)

Source

PagedAdEMAMix32bit[[bitsandbytes.optim.PagedAdEMAMix32bit]]

bitsandbytes.optim.PagedAdEMAMix32bit[[bitsandbytes.optim.PagedAdEMAMix32bit]]

bitsandbytes.optim.PagedAdEMAMix32bit(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, min_8bit_size: int = 4096)

Source

init[[bitsandbytes.optim.PagedAdEMAMix32bit.init]]

__init__(params: Iterable, lr: float = 0.001, betas: tuple = (0.9, 0.999, 0.9999), alpha: float = 5.0, t_alpha: typing.Optional[int] = None, t_beta3: typing.Optional[int] = None, eps: float = 1e-08, weight_decay: float = 0.01, min_8bit_size: int = 4096)

Source