From 6becab5d3fefa8dd2a15abb3378d4ba4255f0d75 Mon Sep 17 00:00:00 2001
From: "Wang, Yi A" <yi.a.wang@intel.com>
Date: Mon, 4 Nov 2024 18:06:23 -0800
Subject: [PATCH] torch has xpu support in 2.5

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
---
 server/text_generation_server/models/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 6c633521..259c703a 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -413,7 +413,9 @@ def get_model(
 
     if dtype is None:
         if quantize in ["awq", "exl2", "gptq", "marlin"]:
-            if SYSTEM == "ipex" and not hasattr(torch, "xpu"):
+            if SYSTEM == "ipex" and not (
+                hasattr(torch, "xpu") and torch.xpu.is_available()
+            ):
                 dtype = torch.bfloat16
             else:
                 # These quantizers only work with float16 params.