From 5e2932552c2c0fdcf5c14aceb71ef4ba4b883bc8 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Thu, 29 Aug 2024 11:35:18 +0200
Subject: [PATCH] Revert the Cohere tokenizer change (for now using a revision
 instead).

---
 router/src/server.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/router/src/server.rs b/router/src/server.rs
index 6b7fff81..f273a786 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -1748,7 +1748,7 @@ pub async fn run(
         let mut tokenizer = Tokenizer::from_file(filename).ok();
         if let Some(tokenizer) = &mut tokenizer {
             if let Some(class) = &tokenizer_config.tokenizer_class {
-                if class == "LlamaTokenizer" || class == "LlamaTokenizerFast" || class == "CohereTokenizerFast"{
+                if class == "LlamaTokenizer" || class == "LlamaTokenizerFast"{
                     if let Ok(post_processor) = create_post_processor(tokenizer, &tokenizer_config) {
                         tracing::info!("Overriding LlamaTokenizer with TemplateProcessing to follow python override defined in https://github.com/huggingface/transformers/blob/4aa17d00690b7f82c95bb2949ea57e22c35b4336/src/transformers/models/llama/tokenization_llama_fast.py#L203-L205");
                         tokenizer.with_post_processor(post_processor);