From e37154a87902dca1653458d3eecb385d08ed661b Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Tue, 4 Dec 2018 22:01:03 +0100 Subject: [PATCH] build: protobuf dependency fixes, libusb build - docker protobuf dependencies, cross-compilation - device/trezor protobuf build fixes, try_compile - libusb built under all platforms, used by trezor for direct connect --- .travis.yml | 7 +- Dockerfile | 17 +++- cmake/CheckTrezor.cmake | 95 ++++++++++++++++++++- cmake/test-protobuf.cpp | 43 ++++++++++ cmake/test-protobuf.proto | 7 ++ contrib/depends/README.md | 10 +++ contrib/depends/packages/libusb.mk | 20 +++-- contrib/depends/packages/native_protobuf.mk | 28 ++++++ contrib/depends/packages/packages.mk | 6 +- contrib/depends/toolchain.cmake.in | 6 ++ src/device_trezor/CMakeLists.txt | 8 -- src/wallet/CMakeLists.txt | 6 ++ 12 files changed, 228 insertions(+), 25 deletions(-) create mode 100644 cmake/test-protobuf.cpp create mode 100644 cmake/test-protobuf.proto create mode 100644 contrib/depends/packages/native_protobuf.mk diff --git a/.travis.yml b/.travis.yml index ca3a24ae8..ffb31fc08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,9 @@ env: - DOCKER_PACKAGES="build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" matrix: # ARM v7 - - HOST=arm-linux-gnueabihf PACKAGES="gperf g++-arm-linux-gnueabihf" + - HOST=arm-linux-gnueabihf PACKAGES="python3 gperf g++-arm-linux-gnueabihf" # ARM v8 - - HOST=aarch64-linux-gnu PACKAGES="gperf g++-aarch64-linux-gnu" + - HOST=aarch64-linux-gnu PACKAGES="python3 gperf g++-aarch64-linux-gnu" # i686 Win - HOST=i686-w64-mingw32 PACKAGES="python3 nsis g++-mingw-w64-i686" # i686 Linux @@ -33,7 +33,7 @@ env: # Win64 - HOST=x86_64-w64-mingw32 PACKAGES="cmake python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 bc" RUN_TESTS=true # x86_64 Linux - - HOST=x86_64-unknown-linux-gnu PACKAGES="gperf cmake python3-zmq protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev" RUN_TESTS=true + - HOST=x86_64-unknown-linux-gnu PACKAGES="gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" RUN_TESTS=true # Cross-Mac - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" OSX_SDK=10.11 @@ -52,6 +52,7 @@ before_script: - if [ -n "$OSX_SDK" -a ! -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - if [ -n "$OSX_SDK" -a -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C contrib/depends/SDKs -xf contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-g++ \$(which $HOST-g++-posix)"; fi + - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-gcc \$(which $HOST-gcc-posix)"; fi - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC bash -c "CONFIG_SHELL= make $MAKEJOBS -C contrib/depends HOST=$HOST $DEP_OPTS"; fi script: - git submodule init && git submodule update diff --git a/Dockerfile b/Dockerfile index 0728c6ce8..9fe7cfb8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,8 @@ RUN set -ex && \ automake \ bzip2 \ xsltproc \ - gperf + gperf \ + unzip WORKDIR /usr/local @@ -147,6 +148,20 @@ RUN set -ex \ && make \ && make install +# Protobuf +ARG PROTOBUF_VERSION=v3.6.1 +ARG PROTOBUF_HASH=48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 +RUN set -ex \ + && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \ + && cd protobuf \ + && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \ + && git submodule update --init --recursive \ + && ./autogen.sh \ + && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && make \ + && make install \ + && ldconfig + WORKDIR /src COPY . . diff --git a/cmake/CheckTrezor.cmake b/cmake/CheckTrezor.cmake index bcd3cfc2c..71214363d 100644 --- a/cmake/CheckTrezor.cmake +++ b/cmake/CheckTrezor.cmake @@ -2,13 +2,55 @@ OPTION(USE_DEVICE_TREZOR "Trezor support compilation" ON) OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" ON) OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" OFF) +# Helper function to fix cmake < 3.6.0 FindProtobuf variables +function(_trezor_protobuf_fix_vars) + if(${CMAKE_VERSION} VERSION_LESS "3.6.0") + foreach(UPPER + PROTOBUF_SRC_ROOT_FOLDER + PROTOBUF_IMPORT_DIRS + PROTOBUF_DEBUG + PROTOBUF_LIBRARY + PROTOBUF_PROTOC_LIBRARY + PROTOBUF_INCLUDE_DIR + PROTOBUF_PROTOC_EXECUTABLE + PROTOBUF_LIBRARY_DEBUG + PROTOBUF_PROTOC_LIBRARY_DEBUG + PROTOBUF_LITE_LIBRARY + PROTOBUF_LITE_LIBRARY_DEBUG + ) + if (DEFINED ${UPPER}) + string(REPLACE "PROTOBUF_" "Protobuf_" Camel ${UPPER}) + if (NOT DEFINED ${Camel}) + set(${Camel} ${${UPPER}} PARENT_SCOPE) + endif() + endif() + endforeach() + endif() +endfunction() + # Use Trezor master switch if (USE_DEVICE_TREZOR) # Protobuf is required to build protobuf messages for Trezor include(FindProtobuf OPTIONAL) find_package(Protobuf) - if(NOT Protobuf_FOUND) + _trezor_protobuf_fix_vars() + + # Protobuf handling the cache variables set in docker. + if(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR) message(STATUS "Could not find Protobuf") + elseif(NOT Protobuf_LIBRARY OR NOT EXISTS "${Protobuf_LIBRARY}") + message(STATUS "Protobuf library not found: ${Protobuf_LIBRARY}") + unset(Protobuf_FOUND) + elseif(NOT Protobuf_PROTOC_EXECUTABLE OR NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + message(STATUS "Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}") + unset(Protobuf_FOUND) + elseif(NOT Protobuf_INCLUDE_DIR OR NOT EXISTS "${Protobuf_INCLUDE_DIR}") + message(STATUS "Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}") + unset(Protobuf_FOUND) + else() + message(STATUS "Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}") + set(Protobuf_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR}) + set(Protobuf_FOUND 1) # override found if all rquired info was provided by variables endif() else() @@ -37,9 +79,32 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR) endif() endif() -# Try to build protobuf messages +# Protobuf compilation test if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON) - set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIRS}") + execute_process(COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I "${CMAKE_SOURCE_DIR}/cmake" -I "${Protobuf_INCLUDE_DIR}" "${CMAKE_SOURCE_DIR}/cmake/test-protobuf.proto" --cpp_out ${CMAKE_BINARY_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) + if(RET) + message(STATUS "Protobuf test generation failed: ${OUT} ${ERR}") + endif() + + try_compile(Protobuf_COMPILE_TEST_PASSED + "${CMAKE_BINARY_DIR}" + SOURCES + "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc" + "${CMAKE_SOURCE_DIR}/cmake/test-protobuf.cpp" + CMAKE_FLAGS + "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}" + "-DCMAKE_CXX_STANDARD=11" + LINK_LIBRARIES ${Protobuf_LIBRARY} + OUTPUT_VARIABLE OUTPUT + ) + if(NOT Protobuf_COMPILE_TEST_PASSED) + message(STATUS "Protobuf Compilation test failed: ${OUTPUT}.") + endif() +endif() + +# Try to build protobuf messages +if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_TEST_PASSED) + set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIR}") set(ENV{PROTOBUF_PROTOC_EXECUTABLE} "${Protobuf_PROTOC_EXECUTABLE}") execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) if(RET) @@ -47,9 +112,10 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON) "OUT: ${OUT}, ERR: ${ERR}." "Please read src/device_trezor/trezor/tools/README.md") else() - message(STATUS "Trezor protobuf messages regenerated ${OUT}") + message(STATUS "Trezor protobuf messages regenerated out: \"${OUT}.\"") set(DEVICE_TREZOR_READY 1) add_definitions(-DDEVICE_TREZOR_READY=1) + add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0) if(CMAKE_BUILD_TYPE STREQUAL "Debug") add_definitions(-DTREZOR_DEBUG=1) @@ -75,5 +141,26 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON) include_directories(${LibUSB_INCLUDE_DIRS}) endif() endif() + + set(TREZOR_LIBUSB_LIBRARIES "") + if(LibUSB_COMPILE_TEST_PASSED) + list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES}) + message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}") + endif() + + if (BUILD_GUI_DEPS) + set(TREZOR_DEP_LIBS "") + set(TREZOR_DEP_LINKER "") + + if (Protobuf_LIBRARY) + list(APPEND TREZOR_DEP_LIBS ${Protobuf_LIBRARY}) + string(APPEND TREZOR_DEP_LINKER " -lprotobuf") + endif() + + if (TREZOR_LIBUSB_LIBRARIES) + list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES}) + string(APPEND TREZOR_DEP_LINKER " -lusb-1.0") + endif() + endif() endif() endif() diff --git a/cmake/test-protobuf.cpp b/cmake/test-protobuf.cpp new file mode 100644 index 000000000..532df7cbe --- /dev/null +++ b/cmake/test-protobuf.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include "test-protobuf.pb.h" + +int main(int argc, char *argv[]) { + google::protobuf::UnknownFieldSet ufs; + ufs.ClearAndFreeMemory(); + + Success sc; + sc.set_message("test"); + sc.SerializeToOstream(&std::cerr); + return 0; +} diff --git a/cmake/test-protobuf.proto b/cmake/test-protobuf.proto new file mode 100644 index 000000000..5300aea35 --- /dev/null +++ b/cmake/test-protobuf.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; + +import "google/protobuf/descriptor.proto"; + +message Success { + optional string message = 1; +} diff --git a/contrib/depends/README.md b/contrib/depends/README.md index dd2824569..7019c5166 100644 --- a/contrib/depends/README.md +++ b/contrib/depends/README.md @@ -59,6 +59,16 @@ Download it from apple, or search for it on github. Create a new directoty calle directory and place the entire MacOSX10.11.sdk folder in it. The depends build will then pick it up automatically (without requiring SDK_PATH). + +#Mingw builds + +Building for 32/64bit mingw requires switching alternatives to a posix mode + +```bash +update-alternatives --set x86_64-w64-mingw32-g++ x86_64-w64-mingw32-g++-posix +update-alternatives --set x86_64-w64-mingw32-gcc x86_64-w64-mingw32-gcc-posix +``` + ### Other documentation - [description.md](description.md): General description of the depends system diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk index 6d60cce26..d865d2a17 100644 --- a/contrib/depends/packages/libusb.mk +++ b/contrib/depends/packages/libusb.mk @@ -11,13 +11,21 @@ endef define $(package)_set_vars $(package)_config_opts=--disable-shared $(package)_config_opts_linux=--with-pic --disable-udev + $(package)_config_opts_mingw32=--disable-udev + $(package)_config_opts_darwin=--disable-udev endef -define $(package)_config_cmds - cp -f $(BASEDIR)/config.guess config.guess &&\ - cp -f $(BASEDIR)/config.sub config.sub &&\ - $($(package)_autoconf) -endef +ifneq ($(host_os),darwin) + define $(package)_config_cmds + cp -f $(BASEDIR)/config.guess config.guess &&\ + cp -f $(BASEDIR)/config.sub config.sub &&\ + $($(package)_autoconf) + endef +else + define $(package)_config_cmds + $($(package)_autoconf) + endef +endif define $(package)_build_cmd $(MAKE) @@ -27,5 +35,5 @@ define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef -define $(package)_postprocess_cmds cp -f lib/libusb-1.0.a lib/libusb.a +define $(package)_postprocess_cmds cp -f lib/libusb-1.0.a lib/libusb.a endef diff --git a/contrib/depends/packages/native_protobuf.mk b/contrib/depends/packages/native_protobuf.mk new file mode 100644 index 000000000..83e602341 --- /dev/null +++ b/contrib/depends/packages/native_protobuf.mk @@ -0,0 +1,28 @@ +package=protobuf3 +$(package)_version=3.6.1 +$(package)_download_path=https://github.com/protocolbuffers/protobuf/releases/download/v$($(package)_version)/ +$(package)_file_name=protobuf-cpp-$($(package)_version).tar.gz +$(package)_sha256_hash=b3732e471a9bb7950f090fd0457ebd2536a9ba0891b7f3785919c654fe2a2529 +$(package)_cxxflags=-std=c++11 + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --prefix=$(build_prefix) + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) -C src libprotobuf.la all +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install install-libLTLIBRARIES install-nobase_includeHEADERS &&\ + $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA +endef + +define $(package)_postprocess_cmds + rm lib/libprotoc.a +endef diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index f814c14d6..a25e54947 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,12 +1,12 @@ -packages:=boost openssl libevent zeromq cppzmq zlib expat ldns cppzmq readline libiconv qt hidapi -native_packages := native_ccache +packages:=boost openssl libevent zeromq cppzmq zlib expat ldns cppzmq readline libiconv qt hidapi protobuf libusb +native_packages := native_ccache native_protobuf wallet_packages=bdb darwin_native_packages = native_biplist native_ds_store native_mac_alias darwin_packages = sodium-darwin -linux_packages = eudev libusb +linux_packages = eudev ifeq ($(host_os),linux) packages += unwind diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index f59b7b5ee..547b59108 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -21,6 +21,12 @@ SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib) SET(LIBUSB-1.0_LIBRARY @prefix@/lib/libusb-1.0.a) SET(LIBUDEV_LIBRARY @prefix@/lib/libudev.a) +SET(Protobuf_FOUND 1) +SET(Protobuf_PROTOC_EXECUTABLE @prefix@/native/bin/protoc CACHE FILEPATH "Path to the native protoc") +SET(Protobuf_INCLUDE_DIR @prefix@/include CACHE PATH "Protobuf include dir") +SET(Protobuf_INCLUDE_DIRS @prefix@/include CACHE PATH "Protobuf include dir") +SET(Protobuf_LIBRARY @prefix@/lib/libprotobuf.a CACHE FILEPATH "Protobuf library") + SET(ZMQ_INCLUDE_PATH @prefix@/include) SET(ZMQ_LIB @prefix@/lib/libzmq.a) diff --git a/src/device_trezor/CMakeLists.txt b/src/device_trezor/CMakeLists.txt index b27c843b6..7f979389a 100644 --- a/src/device_trezor/CMakeLists.txt +++ b/src/device_trezor/CMakeLists.txt @@ -67,14 +67,6 @@ set(trezor_private_headers) if(DEVICE_TREZOR_READY) message(STATUS "Trezor support enabled") - add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0) - - set(TREZOR_LIBUSB_LIBRARIES "") - if(LibUSB_COMPILE_TEST_PASSED) - list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES}) - message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}") - endif() - monero_private_headers(device_trezor ${device_private_headers}) diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index 4e3fb1ae5..64e84020e 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -134,6 +134,12 @@ if (BUILD_GUI_DEPS) endif() install(TARGETS wallet_merged ARCHIVE DESTINATION ${lib_folder}) + + install(FILES ${TREZOR_DEP_LIBS} + DESTINATION ${lib_folder}) + file(WRITE "trezor_link_flags.txt" ${TREZOR_DEP_LINKER}) + install(FILES "trezor_link_flags.txt" + DESTINATION ${lib_folder}) endif() add_subdirectory(api)