Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(backend): minor refactor

This commit is contained in:
Morgan Funtowicz 2024-10-23 14:12:32 +02:00
parent 92bb113653
commit d4b5be10f9
2 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backends/llamacpp/csrc/backend.cpp
View File

@ -123,7 +123,7 @@ namespace huggingface::tgi::backends::llama {
#else
const auto status = llama_decode(ctx, batch);
#endif
if (status == LLAMA_SUCCESS) {
if (LLAMA_SUCCESS(status)) {
// Sample the new token
auto new_token_id = llama_sampler_sample(*sampler, ctx, -1);
generated.emplace_back(new_token_id);
@ -133,7 +133,6 @@ namespace huggingface::tgi::backends::llama {
batch = llama_batch_get_one(&new_token_id, 1);
}
}
generated.shrink_to_fit();
return generated;
}
}

2
backends/llamacpp/csrc/backend.hpp
View File

@ -10,7 +10,7 @@
#include <memory>
#include <llama.h>
#define LLAMA_SUCCESS 0
#define LLAMA_SUCCESS(x) x == 0
namespace huggingface::tgi::backends::llama {
enum TgiLlamaCppBackendError {