Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: EETQLinear with bias in layers.py (#1176)

This commit is contained in:
star 2023-10-19 18:15:05 +08:00 committed by GitHub
parent 9179605e1e
commit 648ea06430
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
server/text_generation_server/utils/layers.py
View File

@ -155,10 +155,7 @@ class EETQLinear(nn.Module):
device = weight.device device = weight.device
weight = torch.t(weight).contiguous().cpu() weight = torch.t(weight).contiguous().cpu()
weight, scale = quant_weights(weight, torch.int8, False) weight, scale = quant_weights(weight, torch.int8, False)
if bias:
bias = weights.get_tensor(f"{prefix}.bias")
else:
bias = None
self.weight = weight.cuda(device) self.weight = weight.cuda(device)
self.scale = scale.cuda(device) self.scale = scale.cuda(device)
self.bias = bias.cuda(device) if bias is not None else None self.bias = bias.cuda(device) if bias is not None else None