build: protobuf dependency fixes, libusb build

- docker protobuf dependencies, cross-compilation
- device/trezor protobuf build fixes, try_compile
- libusb built under all platforms, used by trezor for direct connect

.travis.yml

@@ -23,9 +23,9 @@ env:
     - DOCKER_PACKAGES="build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache"
   matrix:
 # ARM v7
-    - HOST=arm-linux-gnueabihf PACKAGES="gperf g++-arm-linux-gnueabihf"
+    - HOST=arm-linux-gnueabihf PACKAGES="python3 gperf g++-arm-linux-gnueabihf"
 # ARM v8
-    - HOST=aarch64-linux-gnu PACKAGES="gperf g++-aarch64-linux-gnu"
+    - HOST=aarch64-linux-gnu PACKAGES="python3 gperf g++-aarch64-linux-gnu"
 # i686 Win
     - HOST=i686-w64-mingw32 PACKAGES="python3 nsis g++-mingw-w64-i686"
 # i686 Linux
@@ -33,7 +33,7 @@ env:
 # Win64
     - HOST=x86_64-w64-mingw32 PACKAGES="cmake python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 bc" RUN_TESTS=true
 # x86_64 Linux
-    - HOST=x86_64-unknown-linux-gnu PACKAGES="gperf cmake python3-zmq protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev" RUN_TESTS=true
+    - HOST=x86_64-unknown-linux-gnu PACKAGES="gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" RUN_TESTS=true
 # Cross-Mac
     - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" OSX_SDK=10.11
 
@@ -52,6 +52,7 @@ before_script:
     - if [ -n "$OSX_SDK" -a ! -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
     - if [ -n "$OSX_SDK" -a -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C contrib/depends/SDKs -xf contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
     - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-g++ \$(which $HOST-g++-posix)"; fi
+    - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-gcc \$(which $HOST-gcc-posix)"; fi
     - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC bash -c "CONFIG_SHELL= make $MAKEJOBS -C contrib/depends HOST=$HOST $DEP_OPTS"; fi
 script:
     - git submodule init && git submodule update

Dockerfile

@@ -20,7 +20,8 @@ RUN set -ex && \
         automake \
         bzip2 \
         xsltproc \
-        gperf
+        gperf \
+        unzip
 
 WORKDIR /usr/local
 
@@ -147,6 +148,20 @@ RUN set -ex \
     && make \
     && make install
 
+# Protobuf
+ARG PROTOBUF_VERSION=v3.6.1
+ARG PROTOBUF_HASH=48cb18e5c419ddd23d9badcfe4e9df7bde1979b2
+RUN set -ex \
+    && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \
+    && cd protobuf \
+    && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \
+    && git submodule update --init --recursive \
+    && ./autogen.sh \
+    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \
+    && make \
+    && make install \
+    && ldconfig
+
 WORKDIR /src
 COPY . .
 

cmake/CheckTrezor.cmake

@@ -2,13 +2,55 @@ OPTION(USE_DEVICE_TREZOR "Trezor support compilation" ON)
 OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" ON)
 OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" OFF)
 
+# Helper function to fix cmake < 3.6.0 FindProtobuf variables
+function(_trezor_protobuf_fix_vars)
+    if(${CMAKE_VERSION} VERSION_LESS "3.6.0")
+        foreach(UPPER
+                PROTOBUF_SRC_ROOT_FOLDER
+                PROTOBUF_IMPORT_DIRS
+                PROTOBUF_DEBUG
+                PROTOBUF_LIBRARY
+                PROTOBUF_PROTOC_LIBRARY
+                PROTOBUF_INCLUDE_DIR
+                PROTOBUF_PROTOC_EXECUTABLE
+                PROTOBUF_LIBRARY_DEBUG
+                PROTOBUF_PROTOC_LIBRARY_DEBUG
+                PROTOBUF_LITE_LIBRARY
+                PROTOBUF_LITE_LIBRARY_DEBUG
+                )
+            if (DEFINED ${UPPER})
+                string(REPLACE "PROTOBUF_" "Protobuf_" Camel ${UPPER})
+                if (NOT DEFINED ${Camel})
+                    set(${Camel} ${${UPPER}} PARENT_SCOPE)
+                endif()
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
 # Use Trezor master switch
 if (USE_DEVICE_TREZOR)
     # Protobuf is required to build protobuf messages for Trezor
     include(FindProtobuf OPTIONAL)
     find_package(Protobuf)
-    if(NOT Protobuf_FOUND)
+    _trezor_protobuf_fix_vars()
+
+    # Protobuf handling the cache variables set in docker.
+    if(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR)
         message(STATUS "Could not find Protobuf")
+    elseif(NOT Protobuf_LIBRARY OR NOT EXISTS "${Protobuf_LIBRARY}")
+        message(STATUS "Protobuf library not found: ${Protobuf_LIBRARY}")
+        unset(Protobuf_FOUND)
+    elseif(NOT Protobuf_PROTOC_EXECUTABLE OR NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+        message(STATUS "Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}")
+        unset(Protobuf_FOUND)
+    elseif(NOT Protobuf_INCLUDE_DIR OR NOT EXISTS "${Protobuf_INCLUDE_DIR}")
+        message(STATUS "Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}")
+        unset(Protobuf_FOUND)
+    else()
+        message(STATUS "Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}")
+        set(Protobuf_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR})
+        set(Protobuf_FOUND 1)  # override found if all rquired info was provided by variables
     endif()
 
 else()
@@ -37,9 +79,32 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR)
     endif()
 endif()
 
-# Try to build protobuf messages
+# Protobuf compilation test
 if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
-    set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIRS}")
+    execute_process(COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I "${CMAKE_SOURCE_DIR}/cmake" -I "${Protobuf_INCLUDE_DIR}" "${CMAKE_SOURCE_DIR}/cmake/test-protobuf.proto" --cpp_out ${CMAKE_BINARY_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
+    if(RET)
+        message(STATUS "Protobuf test generation failed: ${OUT} ${ERR}")
+    endif()
+
+    try_compile(Protobuf_COMPILE_TEST_PASSED
+        "${CMAKE_BINARY_DIR}"
+        SOURCES
+        "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc"
+        "${CMAKE_SOURCE_DIR}/cmake/test-protobuf.cpp"
+        CMAKE_FLAGS
+        "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}"
+        "-DCMAKE_CXX_STANDARD=11"
+        LINK_LIBRARIES ${Protobuf_LIBRARY}
+        OUTPUT_VARIABLE OUTPUT
+    )
+    if(NOT Protobuf_COMPILE_TEST_PASSED)
+        message(STATUS "Protobuf Compilation test failed: ${OUTPUT}.")
+    endif()
+endif()
+
+# Try to build protobuf messages
+if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_TEST_PASSED)
+    set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIR}")
     set(ENV{PROTOBUF_PROTOC_EXECUTABLE} "${Protobuf_PROTOC_EXECUTABLE}")
     execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
     if(RET)
@@ -47,9 +112,10 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
                 "OUT: ${OUT}, ERR: ${ERR}."
                 "Please read src/device_trezor/trezor/tools/README.md")
     else()
-        message(STATUS "Trezor protobuf messages regenerated ${OUT}")
+        message(STATUS "Trezor protobuf messages regenerated out: \"${OUT}.\"")
         set(DEVICE_TREZOR_READY 1)
         add_definitions(-DDEVICE_TREZOR_READY=1)
+        add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0)
 
         if(CMAKE_BUILD_TYPE STREQUAL "Debug")
             add_definitions(-DTREZOR_DEBUG=1)
@@ -75,5 +141,26 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
                 include_directories(${LibUSB_INCLUDE_DIRS})
             endif()
         endif()
+
+        set(TREZOR_LIBUSB_LIBRARIES "")
+        if(LibUSB_COMPILE_TEST_PASSED)
+            list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES})
+            message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}")
+        endif()
+
+        if (BUILD_GUI_DEPS)
+            set(TREZOR_DEP_LIBS "")
+            set(TREZOR_DEP_LINKER "")
+
+            if (Protobuf_LIBRARY)
+                list(APPEND TREZOR_DEP_LIBS ${Protobuf_LIBRARY})
+                string(APPEND TREZOR_DEP_LINKER " -lprotobuf")
+            endif()
+
+            if (TREZOR_LIBUSB_LIBRARIES)
+                list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES})
+                string(APPEND TREZOR_DEP_LINKER " -lusb-1.0")
+            endif()
+        endif()
     endif()
 endif()

cmake/test-protobuf.cpp

@@ -0,0 +1,43 @@
+// Copyright (c) 2014-2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <string>
+#include <iostream>
+#include <google/protobuf/message.h>
+#include <google/protobuf/unknown_field_set.h>
+#include "test-protobuf.pb.h"
+
+int main(int argc, char *argv[]) {
+  google::protobuf::UnknownFieldSet ufs;
+  ufs.ClearAndFreeMemory();
+
+  Success sc;
+  sc.set_message("test");
+  sc.SerializeToOstream(&std::cerr);
+  return 0;
+}

cmake/test-protobuf.proto

@@ -0,0 +1,7 @@
+syntax = "proto2";
+
+import "google/protobuf/descriptor.proto";
+
+message Success {
+    optional string message = 1;
+}

contrib/depends/README.md

@@ -59,6 +59,16 @@ Download it from apple, or search for it on github. Create a new directoty calle
 directory and place the entire MacOSX10.11.sdk folder in it. The depends build will then pick it up automatically
 (without requiring SDK_PATH). 
 
+
+#Mingw builds
+
+Building for 32/64bit mingw requires switching alternatives to a posix mode
+
+```bash
+update-alternatives --set x86_64-w64-mingw32-g++ x86_64-w64-mingw32-g++-posix
+update-alternatives --set x86_64-w64-mingw32-gcc x86_64-w64-mingw32-gcc-posix
+```
+
 ### Other documentation
 
 - [description.md](description.md): General description of the depends system

contrib/depends/packages/libusb.mk

@@ -11,13 +11,21 @@ endef
 define $(package)_set_vars
   $(package)_config_opts=--disable-shared
   $(package)_config_opts_linux=--with-pic --disable-udev
+  $(package)_config_opts_mingw32=--disable-udev
+  $(package)_config_opts_darwin=--disable-udev
 endef
 
-define $(package)_config_cmds
+ifneq ($(host_os),darwin)
+  define $(package)_config_cmds
     cp -f $(BASEDIR)/config.guess config.guess &&\
     cp -f $(BASEDIR)/config.sub config.sub &&\
     $($(package)_autoconf)
-endef
+  endef
+else
+  define $(package)_config_cmds
+    $($(package)_autoconf)
+  endef
+endif
 
 define $(package)_build_cmd
   $(MAKE)

contrib/depends/packages/native_protobuf.mk

@@ -0,0 +1,28 @@
+package=protobuf3
+$(package)_version=3.6.1
+$(package)_download_path=https://github.com/protocolbuffers/protobuf/releases/download/v$($(package)_version)/
+$(package)_file_name=protobuf-cpp-$($(package)_version).tar.gz
+$(package)_sha256_hash=b3732e471a9bb7950f090fd0457ebd2536a9ba0891b7f3785919c654fe2a2529
+$(package)_cxxflags=-std=c++11
+
+define $(package)_set_vars
+  $(package)_config_opts=--disable-shared --prefix=$(build_prefix)
+  $(package)_config_opts_linux=--with-pic
+endef
+
+define $(package)_config_cmds
+  $($(package)_autoconf)
+endef
+
+define $(package)_build_cmds
+  $(MAKE) -C src libprotobuf.la all
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install install-libLTLIBRARIES install-nobase_includeHEADERS &&\
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA
+endef
+
+define $(package)_postprocess_cmds
+  rm lib/libprotoc.a
+endef

contrib/depends/packages/packages.mk

@@ -1,12 +1,12 @@
-packages:=boost openssl libevent zeromq cppzmq zlib expat ldns cppzmq readline libiconv qt hidapi
+packages:=boost openssl libevent zeromq cppzmq zlib expat ldns cppzmq readline libiconv qt hidapi protobuf libusb
-native_packages := native_ccache
+native_packages := native_ccache native_protobuf
 
 wallet_packages=bdb
 
 darwin_native_packages = native_biplist native_ds_store native_mac_alias
 darwin_packages = sodium-darwin
 
-linux_packages = eudev libusb
+linux_packages = eudev
 
 ifeq ($(host_os),linux)
 packages += unwind

contrib/depends/toolchain.cmake.in

@@ -21,6 +21,12 @@ SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib)
 SET(LIBUSB-1.0_LIBRARY @prefix@/lib/libusb-1.0.a)
 SET(LIBUDEV_LIBRARY @prefix@/lib/libudev.a)
 
+SET(Protobuf_FOUND 1)
+SET(Protobuf_PROTOC_EXECUTABLE @prefix@/native/bin/protoc CACHE FILEPATH "Path to the native protoc")
+SET(Protobuf_INCLUDE_DIR @prefix@/include CACHE PATH "Protobuf include dir")
+SET(Protobuf_INCLUDE_DIRS @prefix@/include CACHE PATH "Protobuf include dir")
+SET(Protobuf_LIBRARY @prefix@/lib/libprotobuf.a CACHE FILEPATH "Protobuf library")
+
 SET(ZMQ_INCLUDE_PATH @prefix@/include)
 SET(ZMQ_LIB @prefix@/lib/libzmq.a)
 

src/device_trezor/CMakeLists.txt

@@ -67,14 +67,6 @@ set(trezor_private_headers)
 if(DEVICE_TREZOR_READY)
     message(STATUS "Trezor support enabled")
 
-    add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0)
-
-    set(TREZOR_LIBUSB_LIBRARIES "")
-    if(LibUSB_COMPILE_TEST_PASSED)
-        list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES})
-        message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}")
-    endif()
-
     monero_private_headers(device_trezor
             ${device_private_headers})
 

src/wallet/CMakeLists.txt

@@ -134,6 +134,12 @@ if (BUILD_GUI_DEPS)
     endif()
     install(TARGETS wallet_merged
         ARCHIVE DESTINATION ${lib_folder})
+
+    install(FILES ${TREZOR_DEP_LIBS}
+            DESTINATION ${lib_folder})
+    file(WRITE "trezor_link_flags.txt" ${TREZOR_DEP_LINKER})
+    install(FILES "trezor_link_flags.txt"
+            DESTINATION ${lib_folder})
 endif()
 
 add_subdirectory(api)