From 59b0ef30189c55e52b20f229d0d39ea74a5bd02d Mon Sep 17 00:00:00 2001
From: Hugo Larcher <hugo.larcher@huggingface.co>
Date: Fri, 29 Nov 2024 00:31:36 +0100
Subject: [PATCH] feat: Fix Cmakelist to allow building on Darwin platform
 (#2785)

* feat: Fix Cmakelist to allow building on Darwin platform
* fix: Fix tokenizer in llama.cpp Dockerfile
---
 Dockerfile.llamacpp                |  7 +++++--
 backends/llamacpp/CMakeLists.txt   |  8 +++++++-
 backends/llamacpp/README.md        | 17 +++++++++++++++++
 backends/llamacpp/requirements.txt |  1 +
 4 files changed, 30 insertions(+), 3 deletions(-)
 create mode 100644 backends/llamacpp/README.md
 create mode 100644 backends/llamacpp/requirements.txt

diff --git a/Dockerfile.llamacpp b/Dockerfile.llamacpp
index 92b1882a..e8896ad4 100644
--- a/Dockerfile.llamacpp
+++ b/Dockerfile.llamacpp
@@ -66,11 +66,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     openssl \
     python3.11-dev \
     python3.11-venv \
-    ibgomp1
+    libgomp1
 
 COPY --from=builder /usr/src/text-generation-inference/target/release-opt/text-generation-backend-llamacpp /usr/src/text-generation-inference/text-generation-launcher
 COPY --from=builder /usr/src/text-generation-inference/dist /usr/
-
+COPY --from=builder /usr/src/text-generation-inference/backends/llamacpp/requirements.txt requirements.txt
+RUN /usr/bin/python3.11 -m venv /usr/src/text-generation-inference/venv
+ENV PATH="/usr/src/text-generation-inference/venv/bin:$PATH"
+RUN pip3 install --no-cache-dir -r requirements.txt
 ENV PORT=8080
 WORKDIR /usr/src/text-generation-inference
 ENTRYPOINT ["text-generation-launcher"]
\ No newline at end of file
diff --git a/backends/llamacpp/CMakeLists.txt b/backends/llamacpp/CMakeLists.txt
index 05fce922..6599fd69 100644
--- a/backends/llamacpp/CMakeLists.txt
+++ b/backends/llamacpp/CMakeLists.txt
@@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.24)
 
 project(tgi-llama-cpp-backend VERSION 1.0.0)
 set(CMAKE_CXX_STANDARD 23)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 include(FetchContent)
 
@@ -10,13 +11,18 @@ set(LLAMA_CPP_TARGET_CUDA_ARCHS "75-real;80-real;86-real;89-real;90-real" CACHE
 option(LLAMA_CPP_BUILD_OFFLINE_RUNNER "Flag to build the standalone c++ backend runner")
 option(LLAMA_CPP_BUILD_CUDA "Flag to build CUDA enabled inference through llama.cpp")
 
-if (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" AND ${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
+if (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" AND (${CMAKE_SYSTEM_NAME} STREQUAL "Linux" OR ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin"))
     message(STATUS "Targeting libc++")
     set(CMAKE_CXX_FLAGS -stdlib=libc++ ${CMAKE_CXX_FLAGS})
 else ()
     message(STATUS "Not using libc++ ${CMAKE_CXX_COMPILER_ID} ${CMAKE_SYSTEM_NAME}")
 endif ()
 
+# add linker options for Darwin
+if (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}  -L$HOMEBREW_PREFIX/opt/llvm/lib/c++ -L$HOMEBREW_PREFIX/opt/llvm/lib/unwind -lunwind")
+endif ()
+
 # Add dependencies
 include(cmake/numa.cmake)
 include(cmake/spdlog.cmake)
diff --git a/backends/llamacpp/README.md b/backends/llamacpp/README.md
new file mode 100644
index 00000000..0931339c
--- /dev/null
+++ b/backends/llamacpp/README.md
@@ -0,0 +1,17 @@
+## Compiling with MacOS
+
+To compile the Llama.cpp backend on MacOS, you need to install `clang` and `cmake` via Homebrew:
+
+```bash
+brew install llvm cmake
+```
+
+You then need to configure CMakelists.txt to use the newly installed clang compiler.
+You can do this by configuring your IDE or adding the following lines to the top of the file:
+
+```cmake
+set(CMAKE_C_COMPILER /opt/homebrew/opt/llvm/bin/clang)
+set(CMAKE_CXX_COMPILER /opt/homebrew/opt/llvm/bin/clang++)
+```
+
+CMakelist.txt assumes that Homebrew installs libc++ in `$HOMEBREW_PREFIX/opt/llvm/lib/c++`.
\ No newline at end of file
diff --git a/backends/llamacpp/requirements.txt b/backends/llamacpp/requirements.txt
new file mode 100644
index 00000000..2372d58b
--- /dev/null
+++ b/backends/llamacpp/requirements.txt
@@ -0,0 +1 @@
+transformers==4.45.2 ; python_version >= "3.9" and python_version < "3.13"
\ No newline at end of file