Mirrors
/
stable-diffusion-webui
mirror of https://github.com/AUTOMATIC1111/stable-diffusion-webui.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15283 from AUTOMATIC1111/dora-weight-decompose 

Use correct DoRA implementation
This commit is contained in:
AUTOMATIC1111 2024-03-16 20:20:08 +03:00 committed by GitHub
parent bf35c66183 8dcb8faf5d
commit df8c09bcb3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
extensions-builtin/Lora/network.py
View File

@ -153,7 +153,7 @@ class NetworkModule:
self.scale = weights.w["scale"].item() if "scale" in weights.w else None
self.dora_scale = weights.w.get("dora_scale", None)
self.dora_mean_dim = tuple(i for i in range(len(self.shape)) if i != 1)
self.dora_norm_dims = len(self.shape) - 1
def multiplier(self):
if 'transformer' in self.sd_key[:20]:
@ -170,10 +170,22 @@ class NetworkModule:
return 1.0
def apply_weight_decompose(self, updown, orig_weight):
orig_weight = orig_weight.to(updown)
# Match the device/dtype
orig_weight = orig_weight.to(updown.dtype)
dora_scale = self.dora_scale.to(device=orig_weight.device, dtype=updown.dtype)
updown = updown.to(orig_weight.device)
merged_scale1 = updown + orig_weight
merged_scale1_norm = (
merged_scale1.transpose(0, 1)
.reshape(merged_scale1.shape[1], -1)
.norm(dim=1, keepdim=True)
.reshape(merged_scale1.shape[1], *[1] * self.dora_norm_dims)
.transpose(0, 1)
)
dora_merged = (
merged_scale1 / merged_scale1.mean(dim=self.dora_mean_dim, keepdim=True) * self.dora_scale
merged_scale1 * (dora_scale / merged_scale1_norm)
)
final_updown = dora_merged - orig_weight
return final_updown