Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(launcher): add router parameters to launcher (#95)

This commit is contained in:
OlivierDehaene 2023-03-03 16:01:25 +01:00 committed by GitHub
parent e3ded361b2
commit 240c4187fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
launcher/src/main.rs
View File

@ -29,8 +29,12 @@ struct Args {
quantize: bool,
#[clap(default_value = "128", long, env)]
max_concurrent_requests: usize,
#[clap(default_value = "4", long, env)]
max_stop_sequences: usize,
#[clap(default_value = "1000", long, env)]
max_input_length: usize,
#[clap(default_value = "1512", long, env)]
max_total_tokens: usize,
#[clap(default_value = "32", long, env)]
max_batch_size: usize,
#[clap(default_value = "20", long, env)]
@ -79,7 +83,9 @@ fn main() -> ExitCode {
num_shard,
quantize,
max_concurrent_requests,
max_stop_sequences,
max_input_length,
max_total_tokens,
max_batch_size,
max_waiting_tokens,
port,
@ -299,8 +305,12 @@ fn main() -> ExitCode {
"text-generation-router".to_string(),
"--max-concurrent-requests".to_string(),
max_concurrent_requests.to_string(),
"--max-stop-sequences".to_string(),
max_stop_sequences.to_string(),
"--max-input-length".to_string(),
max_input_length.to_string(),
"--max-total-tokens".to_string(),
max_total_tokens.to_string(),
"--max-batch-size".to_string(),
max_batch_size.to_string(),
"--max-waiting-tokens".to_string(),