Fixing Phi3.

2024-06-01 08:47:00 +00:00 · 2024-06-01 08:47:00 +00:00 · 799a193b10
parent 08b3eac2ce
commit 799a193b10
1 changed files with 2 additions and 1 deletions
--- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
@ -52,7 +52,8 @@ if SYSTEM == "rocm":


 def load_attention(config, prefix, weights):
-    bias = config.attention_bias
+    # Only defined in granite.
+    bias = getattr(config, "attention_bias", False)

    # if specific model type, load the correct attention
    if config.model_type == "phi3":