diff --git a/router/src/validation.rs b/router/src/validation.rs index ff659b3..f6da191 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -131,7 +131,7 @@ fn validation_worker( } // Get the number of tokens in the input - match tokenizer.encode(request.inputs.clone(), false) { + match tokenizer.encode(request.inputs.clone(), true) { Ok(inputs) => { let input_length = inputs.len();