diff --git a/Cargo.lock b/Cargo.lock index 67ad867..bd5994a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2557,7 +2557,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.8.1" +version = "0.8.2" dependencies = [ "average", "clap", @@ -2577,7 +2577,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.8.1" +version = "0.8.2" dependencies = [ "futures", "grpc-metadata", @@ -2593,7 +2593,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.8.1" +version = "0.8.2" dependencies = [ "clap", "ctrlc", @@ -2609,7 +2609,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.8.1" +version = "0.8.2" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index 3190b64..b28286f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.8.1" +version = "0.8.2" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index 040c8e8..e5ef0e3 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.8.1" + "version": "0.8.2" }, "paths": { "/": { diff --git a/server/pyproject.toml b/server/pyproject.toml index 7400a05..d381eac 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.8.1" +version = "0.8.2" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 78b6872..fc92d03 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -138,7 +138,9 @@ def get_model( trust_remote_code=trust_remote_code, ) - config_dict, _ = PretrainedConfig.get_config_dict(model_id, revision=revision, trust_remote_code=trust_remote_code) + config_dict, _ = PretrainedConfig.get_config_dict( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) model_type = config_dict["model_type"] if model_type == "gpt_bigcode": diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 7907e2c..e1c893d 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -87,7 +87,9 @@ class FlashSantacoder(FlashCausalLM): ): for filename in filenames: with safe_open( - filename, framework="pt", device=str(device) if quantize is None else "cpu" + filename, + framework="pt", + device=str(device) if quantize is None else "cpu", ) as f: for key in f.keys(): value = f.get_tensor(key) @@ -148,11 +150,13 @@ class FlashSantacoder(FlashCausalLM): module._parameters[param_name][: value.shape[0]] = value elif "kv_attn.weight" in key: module._parameters[param_name][ - model.transformer.head_size * model.transformer.num_heads : + model.transformer.head_size + * model.transformer.num_heads : ] = value elif "kv_attn.bias" in key: module._parameters[param_name][ - model.transformer.head_size * model.transformer.num_heads : + model.transformer.head_size + * model.transformer.num_heads : ] = value else: if current_parameter_tensor.shape != value.shape: