EveryDream2trainer/optimizer.json

{
    "doc": {
        "optimizer": "adamw, adamw8bit, lion",
        "optimizer_desc": "'adamw' in standard 32bit, 'adamw8bit' is bitsandbytes, 'lion' is lucidrains",
        "lr": "learning rate, if null wil use CLI or main JSON config value",
        "betas": "exponential decay rates for the moment estimates",
        "epsilon": "value added to denominator for numerical stability, unused for lion",
        "weight_decay": "weight decay (L2 penalty)",
        "text_encoder_lr_scale": "if set, scale the text encoder's LR by this much relative to the unet LR"
    },
    "optimizer": "adamw8bit",
    "lr": 1e-6,
    "betas": [0.9, 0.999],
    "epsilon": 1e-8,
    "weight_decay": 0.010,
    "text_encoder_lr_scale": 1.0
}