From 3d7f4f41bb8f59c07ee8ef8e4bf807f4ddb947fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Kaunism=C3=A4ki?= Date: Mon, 29 Jul 2024 16:09:25 +0200 Subject: [PATCH] patch-error-on-invalid-grammar (#2282) * quick fix * allow silent failure * explicit todo that this is only short term --- server/text_generation_server/utils/logits_process.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/logits_process.py b/server/text_generation_server/utils/logits_process.py index 6b915437..9abd886f 100644 --- a/server/text_generation_server/utils/logits_process.py +++ b/server/text_generation_server/utils/logits_process.py @@ -521,7 +521,13 @@ class GrammarLogitProcessor(LogitsProcessor): def _cached_compile_fsm(grammar_type, schema, tokenizer): start_time = time.time() if grammar_type == GrammarType.GRAMMAR_TYPE_JSON: - schema = build_regex_from_schema(schema) + try: + schema = build_regex_from_schema(schema) + # TODO: this is only here short term to avoid crashing the python server, mid term we want this in the rust/router layer + except Exception as e: + logger.error(f"Error compiling FSM, grammar won't be enforced \n{e}") + # allows everything + schema = "(.*?)" elif grammar_type == GrammarType.GRAMMAR_TYPE_REGEX: pass # schema is already a regex just here for clarity fsm = RegexFSM(schema, tokenizer)