Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: include add_special_tokens in kserve request (#2859) 

merging as this patch is already used, and fully limit to the kserve feature
This commit is contained in:
drbh 2024-12-19 16:55:17 -05:00 committed by GitHub
parent ab5f616920
commit 23bc38b10d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
router/src/kserve.rs
View File

@ -205,6 +205,7 @@ pub async fn kserve_model_infer(
let generate_request = GenerateRequest { let generate_request = GenerateRequest {
inputs: str_input.to_string(), inputs: str_input.to_string(),
parameters: payload.parameters.clone(), parameters: payload.parameters.clone(),
add_special_tokens: true,
}; };
let infer = infer.clone(); let infer = infer.clone();
let compute_type = compute_type.clone(); let compute_type = compute_type.clone();
@ -212,7 +213,7 @@ pub async fn kserve_model_infer(
async move { async move {
generate_internal(infer, compute_type, Json(generate_request), span) generate_internal(infer, compute_type, Json(generate_request), span)
.await .await
.map(|(_, Json(generation))| { .map(|(_, _, Json(generation))| {
let generation_as_bytes = generation.generated_text.as_bytes().to_vec(); let generation_as_bytes = generation.generated_text.as_bytes().to_vec();
OutputChunk { OutputChunk {
name: output.name.clone(), name: output.name.clone(),