fix: show warning with tokenizer config parsing error (#1488)
This tiny PR just prints the parsing error when a tokenizer config fails to load. This is helpful when a chat_template wont load due to formatting issues https://github.com/huggingface/text-generation-inference/pull/1427#issuecomment-1909226388
This commit is contained in:
parent
17b7b75e65
commit
13dd8e2361
|
@ -462,7 +462,12 @@ pub async fn get_tokenizer_config(api_repo: &ApiRepo) -> Option<HubTokenizerConf
|
|||
let reader = BufReader::new(file);
|
||||
|
||||
// Read the JSON contents of the file as an instance of 'HubTokenizerConfig'.
|
||||
let tokenizer_config: HubTokenizerConfig = serde_json::from_reader(reader).ok()?;
|
||||
let tokenizer_config: HubTokenizerConfig = serde_json::from_reader(reader)
|
||||
.map_err(|e| {
|
||||
tracing::warn!("Unable to parse tokenizer config: {}", e);
|
||||
e
|
||||
})
|
||||
.ok()?;
|
||||
|
||||
Some(tokenizer_config)
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue