This commit is contained in:
OlivierDehaene 2023-03-30 17:28:14 +02:00 committed by GitHub
parent 84722f3e33
commit fef1a1c381
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 12 additions and 16 deletions

6
Cargo.lock generated
View File

@ -2212,7 +2212,7 @@ dependencies = [
[[package]]
name = "text-generation-client"
version = "0.4.2"
version = "0.4.3"
dependencies = [
"futures",
"grpc-metadata",
@ -2229,7 +2229,7 @@ dependencies = [
[[package]]
name = "text-generation-launcher"
version = "0.4.2"
version = "0.4.3"
dependencies = [
"clap 4.1.8",
"ctrlc",
@ -2244,7 +2244,7 @@ dependencies = [
[[package]]
name = "text-generation-router"
version = "0.4.2"
version = "0.4.3"
dependencies = [
"async-stream",
"axum",

2
benchmark/Cargo.lock generated
View File

@ -2069,7 +2069,7 @@ dependencies = [
[[package]]
name = "text-generation-client"
version = "0.4.2"
version = "0.4.3"
dependencies = [
"futures",
"grpc-metadata",

View File

@ -11,7 +11,7 @@
"name": "Apache 2.0",
"url": "https://www.apache.org/licenses/LICENSE-2.0"
},
"version": "0.4.2"
"version": "0.4.3"
},
"paths": {
"/generate": {

View File

@ -1,6 +1,6 @@
[package]
name = "text-generation-launcher"
version = "0.4.2"
version = "0.4.3"
edition = "2021"
authors = ["Olivier Dehaene"]
description = "Text Generation Launcher"

View File

@ -1,6 +1,6 @@
[package]
name = "text-generation-router"
version = "0.4.2"
version = "0.4.3"
edition = "2021"
authors = ["Olivier Dehaene"]
description = "Text Generation Webserver"

View File

@ -1,6 +1,6 @@
[package]
name = "text-generation-client"
version = "0.4.2"
version = "0.4.3"
edition = "2021"
[dependencies]

View File

@ -37,7 +37,7 @@ struct Args {
max_waiting_tokens: usize,
#[clap(default_value = "3000", long, short, env)]
port: u16,
#[clap(default_value = "/tmp/text-generation-server-0", long, env)]
#[clap(default_value = "/tmp/text-generation-0", long, env)]
master_shard_uds_path: String,
#[clap(default_value = "bigscience/bloom", long, env)]
tokenizer_name: String,
@ -76,8 +76,6 @@ fn main() -> Result<(), std::io::Error> {
panic!("validation_workers must be > 0");
}
init_logging(otlp_endpoint, json_output);
// CORS allowed origins
// map to go inside the option and then map to parse from String to HeaderValue
// Finally, convert to AllowOrigin
@ -91,21 +89,17 @@ fn main() -> Result<(), std::io::Error> {
// Tokenizer instance
// This will only be used to validate payloads
tracing::info!("Loading tokenizer");
let local_path = Path::new(&tokenizer_name);
let tokenizer =
if local_path.exists() && local_path.is_dir() && local_path.join("tokenizer.json").exists()
{
// Load local tokenizer
tracing::info!("Found local tokenizer");
Tokenizer::from_file(local_path.join("tokenizer.json")).unwrap()
} else {
// Download and instantiate tokenizer
// We need to download it outside of the Tokio runtime
tracing::info!("Downloading tokenizer");
Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap()
};
tracing::info!("Tokenizer loaded");
// Launch Tokio runtime
tokio::runtime::Builder::new_multi_thread()
@ -113,6 +107,8 @@ fn main() -> Result<(), std::io::Error> {
.build()
.unwrap()
.block_on(async {
init_logging(otlp_endpoint, json_output);
// Get pipeline tag
let model_info = reqwest::get(format!(
"https://huggingface.co/api/models/{tokenizer_name}"

View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "text-generation-server"
version = "0.4.2"
version = "0.4.3"
description = "Text Generation Inference Python gRPC Server"
authors = ["Olivier Dehaene <olivier@huggingface.co>"]