diff --git a/backends/trtllm/src/looper.rs b/backends/trtllm/src/looper.rs index beae8e8e..1411a8ea 100644 --- a/backends/trtllm/src/looper.rs +++ b/backends/trtllm/src/looper.rs @@ -8,6 +8,7 @@ use hashbrown::HashMap; use log::warn; use tokenizers::{Encoding, Tokenizer}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +use tokio::sync::TryAcquireError; use tokio::task::{spawn_blocking, JoinHandle}; use tokio::time::Instant; use tokio_stream::wrappers::UnboundedReceiverStream; @@ -121,7 +122,7 @@ fn executor_status_looper( let what = e.to_string(); error!(error = what.as_str(), "Failed to schedule request"); - let err = Err(InferError::SchedulingError(what)); + let err = Err(InferError::Overloaded(TryAcquireError::NoPermits)); if let Err(_) = ctx.streamer.send(err) { error!("Failed to send back error to the client"); } diff --git a/router/src/infer/mod.rs b/router/src/infer/mod.rs index 39b6f8cf..896f4f43 100644 --- a/router/src/infer/mod.rs +++ b/router/src/infer/mod.rs @@ -357,8 +357,6 @@ pub enum InferError { ToolError(String), #[error("Stream event serialization error")] StreamSerializationError(String), - #[error("Scheduling error: {0}")] - SchedulingError(String), } impl InferError { @@ -373,7 +371,6 @@ impl InferError { InferError::MissingTemplateVariable(_) => "missing_template_variable", InferError::ToolError(_) => "tool_error", InferError::StreamSerializationError(_) => "stream_serialization_error", - InferError::SchedulingError(_) => "schedling" } } }