From 6e3220529df5906ae586031873b7865e9923040b Mon Sep 17 00:00:00 2001 From: drbh Date: Fri, 1 Nov 2024 20:40:05 -0400 Subject: [PATCH] fix: create position ids for text only input (#2714) * fix: create position ids for text only input * fix: prefer repeat over expand to avoid clone --- .../models/custom_modeling/qwen2_vl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/custom_modeling/qwen2_vl.py b/server/text_generation_server/models/custom_modeling/qwen2_vl.py index 5936c6fe..73325c88 100644 --- a/server/text_generation_server/models/custom_modeling/qwen2_vl.py +++ b/server/text_generation_server/models/custom_modeling/qwen2_vl.py @@ -468,7 +468,12 @@ class Qwen2VLForConditionalGeneration(nn.Module): llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) position_ids[:, i, :] = llm_positions.to(position_ids.device) - + else: + position_ids = ( + torch.arange(batch_input_ids.shape[1], device=batch_input_ids.device) + .view(1, 1, -1) + .repeat(3, batch_input_ids.shape[0], 1) + ) return position_ids def forward(