Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(backend): use c++ defined types for llama.cpp

This commit is contained in:
Morgan Funtowicz 2024-11-29 23:38:27 +01:00
parent c9f6c3a8f7
commit e0dda9b614
2 changed files with 2 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backends/llamacpp/csrc/backend.cpp
View File

@ -36,7 +36,7 @@ namespace huggingface::tgi::backends::llamacpp {
llama_sampler_chain_add(sampler, llama_sampler_init_temp(temperature));
llama_sampler_chain_add(sampler, llama_sampler_init_dist(seed));
return {sampler, llama_sampler_deleter};
return llama_sampler_ptr(sampler);
}
std::expected<llama_batch, backend_error_t> get_batch_from_prompt(std::span<llama_token> prompt) {

8
backends/llamacpp/csrc/backend.hpp
View File

@ -17,18 +17,12 @@
#include <vector>
#include <llama.h>
#include <llama-cpp.h>
#include <thread>
#define LLAMA_SUCCESS(x) x == 0
namespace huggingface::tgi::backends::llamacpp {
static constexpr auto llama_context_deleter = [](llama_context *pContext) { llama_free(pContext); };
typedef std::unique_ptr<llama_context, decltype(llama_context_deleter)> llama_context_ptr;
static constexpr auto llama_sampler_deleter = [](llama_sampler *pSampler) { llama_sampler_free(pSampler); };
typedef std::unique_ptr<llama_sampler, decltype(llama_sampler_deleter)> llama_sampler_ptr;
typedef std::function<bool(llama_token, float_t, bool, size_t)> llama_decode_callback;
static constexpr auto llama_void_callback = [](llama_token, float_t, bool, size_t) -> bool { return false; };