Modify exllama weight

2023-08-03 23:20:59 +02:00 · 2023-08-03 23:20:59 +02:00 · a9838bba2f
parent 8af4a7a0b0
commit a9838bba2f
1 changed files with 1 additions and 1 deletions
--- a/server/text_generation_server/utils/layers.py
+++ b/server/text_generation_server/utils/layers.py
@ -199,7 +199,7 @@ def get_linear(weight, bias, quantize):
        )
    elif quantize == "gptq":
        try:
-            qweight, qzeros, scales, g_idx, bits, groupsize = weight
+            qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama = weight
        except Exception:
            raise NotImplementedError(
                f"The passed weight is not `gptq` compatible, loader needs to be updated."