Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(backend): better map exception throw on C++ side

This commit is contained in:
Morgan Funtowicz 2024-11-29 23:34:16 +01:00
parent db41776a0e
commit c9f6c3a8f7
3 changed files with 39 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backends/llamacpp/csrc/backend.cpp
View File

@ -125,6 +125,12 @@ namespace huggingface::tgi::backends::llamacpp {
// Update the batch for the next generation // Update the batch for the next generation
sampling_index = update_batch_for_decoding(batch, new_token_id, position); sampling_index = update_batch_for_decoding(batch, new_token_id, position);
position += 1; position += 1;
} else {
if (status == 1) {
return backend_error_t::NO_KV_SLOT_AVAILABLE;
} else {
return backend_error_t::DECODING_ERROR;
}
} }
} }

4
backends/llamacpp/csrc/backend.hpp
View File

@ -37,7 +37,9 @@ namespace huggingface::tgi::backends::llamacpp {
*/ */
enum backend_error_t : uint8_t { enum backend_error_t : uint8_t {
// Provided model filepath doesnt exist // Provided model filepath doesnt exist
MODEL_FILE_DOESNT_EXIST = 1 MODEL_FILE_DOESNT_EXIST = 1,
NO_KV_SLOT_AVAILABLE = 2,
DECODING_ERROR = 3
}; };
/** /**

33
backends/llamacpp/csrc/ffi.hpp
View File

@ -32,7 +32,6 @@ namespace huggingface::tgi::backends::llamacpp {
#include "backends/llamacpp/src/lib.rs.h" #include "backends/llamacpp/src/lib.rs.h"
#include "rust/cxx.h" #include "rust/cxx.h"
namespace huggingface::tgi::backends::llamacpp { namespace huggingface::tgi::backends::llamacpp {
/** /**
@ -56,7 +55,12 @@ namespace huggingface::tgi::backends::llamacpp {
* llama.cpp backend specific exception mapped from `backend_exception_t` to throw at the FFI level and * llama.cpp backend specific exception mapped from `backend_exception_t` to throw at the FFI level and
* allow automatic implementation of Result<_, Exception> from C++ to Rust * allow automatic implementation of Result<_, Exception> from C++ to Rust
*/ */
class llama_cpp_backend_exception_t : std::exception {}; class llama_cpp_backend_exception_t : std::exception {
public:
backend_error_t error;
llama_cpp_backend_exception_t(const backend_error_t error): error(error) {};
};
/** /**
* Llama.cpp frontend over the worker interfacing with Rust FFI layer * Llama.cpp frontend over the worker interfacing with Rust FFI layer
@ -119,7 +123,7 @@ namespace huggingface::tgi::backends::llamacpp {
if(const auto result = worker_.generate(generation_context, context_forwarding_callback); result.has_value()) [[likely]] { if(const auto result = worker_.generate(generation_context, context_forwarding_callback); result.has_value()) [[likely]] {
return *result; return *result;
} else { } else {
throw llama_cpp_backend_exception_t {}; throw llama_cpp_backend_exception_t(result.error());
} }
} }
}; };
@ -232,5 +236,28 @@ namespace huggingface::tgi::backends::llamacpp {
} }
} }
// Error handle converting to rust Result<T, CxxError>
template <typename Try, typename Fail>
static void trycatch(Try &&func, Fail &&fail) noexcept try {
func();
} catch (const huggingface::tgi::backends::llamacpp::llama_cpp_backend_exception_t &e) {
switch (e.error) {
case huggingface::tgi::backends::llamacpp::backend_error_t::MODEL_FILE_DOESNT_EXIST: {
fail("Specified model path doesn't exist.");
break;
}
case huggingface::tgi::backends::llamacpp::backend_error_t::NO_KV_SLOT_AVAILABLE: {
fail("Keys/Values cache is full, no slot available for the new batch.");
break;
}
case huggingface::tgi::backends::llamacpp::backend_error_t::DECODING_ERROR: {
fail("An error what detected during the generation.");
break;
}
}
fail();
}
#endif //TGI_LLAMA_CPP_BACKEND_FFI_HPP #endif //TGI_LLAMA_CPP_BACKEND_FFI_HPP