Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Fix Cmakelist to allow building on Darwin platform (#2785) 

* feat: Fix Cmakelist to allow building on Darwin platform
* fix: Fix tokenizer in llama.cpp Dockerfile
This commit is contained in:
Hugo Larcher 2024-11-29 00:31:36 +01:00 committed by GitHub
parent b10eaab9f3
commit 59b0ef3018
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 30 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Dockerfile.llamacpp
View File

@ -66,11 +66,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
openssl \
python3.11-dev \
python3.11-venv \
ibgomp1
libgomp1
COPY --from=builder /usr/src/text-generation-inference/target/release-opt/text-generation-backend-llamacpp /usr/src/text-generation-inference/text-generation-launcher
COPY --from=builder /usr/src/text-generation-inference/dist /usr/
COPY --from=builder /usr/src/text-generation-inference/backends/llamacpp/requirements.txt requirements.txt
RUN /usr/bin/python3.11 -m venv /usr/src/text-generation-inference/venv
ENV PATH="/usr/src/text-generation-inference/venv/bin:$PATH"
RUN pip3 install --no-cache-dir -r requirements.txt
ENV PORT=8080
WORKDIR /usr/src/text-generation-inference
ENTRYPOINT ["text-generation-launcher"]

8
backends/llamacpp/CMakeLists.txt
View File

@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.24)
project(tgi-llama-cpp-backend VERSION 1.0.0)
set(CMAKE_CXX_STANDARD 23)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
include(FetchContent)
@ -10,13 +11,18 @@ set(LLAMA_CPP_TARGET_CUDA_ARCHS "75-real;80-real;86-real;89-real;90-real" CACHE
option(LLAMA_CPP_BUILD_OFFLINE_RUNNER "Flag to build the standalone c++ backend runner")
option(LLAMA_CPP_BUILD_CUDA "Flag to build CUDA enabled inference through llama.cpp")
if (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" AND ${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
if (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" AND (${CMAKE_SYSTEM_NAME} STREQUAL "Linux" OR ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin"))
message(STATUS "Targeting libc++")
set(CMAKE_CXX_FLAGS -stdlib=libc++ ${CMAKE_CXX_FLAGS})
else ()
message(STATUS "Not using libc++ ${CMAKE_CXX_COMPILER_ID} ${CMAKE_SYSTEM_NAME}")
endif ()
# add linker options for Darwin
if (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L$HOMEBREW_PREFIX/opt/llvm/lib/c++ -L$HOMEBREW_PREFIX/opt/llvm/lib/unwind -lunwind")
endif ()
# Add dependencies
include(cmake/numa.cmake)
include(cmake/spdlog.cmake)

17
backends/llamacpp/README.md Normal file
View File

@ -0,0 +1,17 @@
## Compiling with MacOS
To compile the Llama.cpp backend on MacOS, you need to install `clang` and `cmake` via Homebrew:
```bash
brew install llvm cmake
```
You then need to configure CMakelists.txt to use the newly installed clang compiler.
You can do this by configuring your IDE or adding the following lines to the top of the file:
```cmake
set(CMAKE_C_COMPILER /opt/homebrew/opt/llvm/bin/clang)
set(CMAKE_CXX_COMPILER /opt/homebrew/opt/llvm/bin/clang++)
```
CMakelist.txt assumes that Homebrew installs libc++ in `$HOMEBREW_PREFIX/opt/llvm/lib/c++`.

1
backends/llamacpp/requirements.txt Normal file
View File

@ -0,0 +1 @@
transformers==4.45.2 ; python_version >= "3.9" and python_version < "3.13"