diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 6c633521..259c703a 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -413,7 +413,9 @@ def get_model(
 
     if dtype is None:
         if quantize in ["awq", "exl2", "gptq", "marlin"]:
-            if SYSTEM == "ipex" and not hasattr(torch, "xpu"):
+            if SYSTEM == "ipex" and not (
+                hasattr(torch, "xpu") and torch.xpu.is_available()
+            ):
                 dtype = torch.bfloat16
             else:
                 # These quantizers only work with float16 params.