From 5f52e2e38ed6f73a1264dc2e5807ab07937b2b75 Mon Sep 17 00:00:00 2001 From: "Wang, Yi A" Date: Tue, 19 Nov 2024 23:27:45 -0800 Subject: [PATCH] entries.len() could > batch.size in prefill, so need to filter as well. Signed-off-by: Wang, Yi A --- backends/v3/src/backend.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/backends/v3/src/backend.rs b/backends/v3/src/backend.rs index a5c0f512..d83a38e6 100644 --- a/backends/v3/src/backend.rs +++ b/backends/v3/src/backend.rs @@ -369,11 +369,6 @@ async fn filter_batch( ) -> Option { let mut batch = next_batch?; - // No need to filter - if batch.size as usize == entries.len() { - return Some(batch); - } - let id = batch.id; // Retain only requests that are still in entries