Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(server): fix bnb quantization for CausalLM models (#385)

This commit is contained in:
OlivierDehaene 2023-05-31 11:48:28 +02:00 committed by GitHub
parent 87dc034b59
commit 337afb2842
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/models/bloom.py
View File

@ -245,6 +245,8 @@ class BLOOMSharded(BLOOM):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq":
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None:

2
server/text_generation_server/models/galactica.py
View File

@ -364,6 +364,8 @@ class GalacticaSharded(Galactica):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq":
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None:

2
server/text_generation_server/models/gpt_neox.py
View File

@ -210,6 +210,8 @@ class GPTNeoxSharded(CausalLM):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq":
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None:

9
server/text_generation_server/models/opt.py
View File

@ -166,7 +166,7 @@ class OPTSharded(OPT):
tensor = tensor.contiguous().to(dtype)
if quantize:
if quantize == "bitsandbytes":
if not HAS_BITS_AND_BYTES:
raise ImportError(
"bitsandbytes is not available on your machine either because it is not installed "
@ -216,9 +216,14 @@ class OPTSharded(OPT):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq":
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None:
tensor = tensor.to(device)
else:
raise ValueError(f"Unexpected quantize `{quantize}`")
module._parameters[param_name] = tensor
if name == "model.decoder.embed_tokens.weight":

3
server/text_generation_server/models/t5.py
View File

@ -222,7 +222,8 @@ class T5Sharded(Seq2SeqLM):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq" and not module_name.endswith("wo"):
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None or module_name.endswith("wo"):