From 648ea06430366a735c92b0c688b09b022ad84438 Mon Sep 17 00:00:00 2001 From: star Date: Thu, 19 Oct 2023 18:15:05 +0800 Subject: [PATCH] fix: EETQLinear with bias in layers.py (#1176) --- server/text_generation_server/utils/layers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index f38f130e..7bb95dd2 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -155,10 +155,7 @@ class EETQLinear(nn.Module): device = weight.device weight = torch.t(weight).contiguous().cpu() weight, scale = quant_weights(weight, torch.int8, False) - if bias: - bias = weights.get_tensor(f"{prefix}.bias") - else: - bias = None + self.weight = weight.cuda(device) self.scale = scale.cuda(device) self.bias = bias.cuda(device) if bias is not None else None