Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Max token capacity metric (#2595) 

* adding max_token_capacity_metric

* added tgi to name of metric

* Adding max capacity metric.

* Add description for the metrics

---------

Co-authored-by: Edwinhr716 <Edandres249@gmail.com>
This commit is contained in:
Nicolas Patry 2024-10-02 16:32:36 +02:00 committed by GitHub
parent d18ed5cfc5
commit 0204946d26
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backends/v3/src/lib.rs
View File

@ -100,6 +100,7 @@ pub async fn connect_backend(
.map_err(V3Error::Warmup)?, .map_err(V3Error::Warmup)?,
)?; )?;
tracing::info!("Setting max batch total tokens to {max_batch_total_tokens}"); tracing::info!("Setting max batch total tokens to {max_batch_total_tokens}");
metrics::gauge!("tgi_batch_max_total_tokens").set(max_batch_total_tokens);
let backend_info = BackendInfo { let backend_info = BackendInfo {
waiting_served_ratio, waiting_served_ratio,

5
router/src/server.rs
View File

@ -1937,6 +1937,11 @@ async fn start(
metrics::Unit::Count, metrics::Unit::Count,
"Maximum tokens for the current batch" "Maximum tokens for the current batch"
); );
metrics::describe_gauge!(
"tgi_batch_total_tokens",
metrics::Unit::Count,
"Maximum amount of tokens in total."
);
metrics::describe_histogram!( metrics::describe_histogram!(
"tgi_request_max_new_tokens", "tgi_request_max_new_tokens",
metrics::Unit::Count, metrics::Unit::Count,