diff --git a/.github/workflows/wasm-package.yml b/.github/workflows/wasm-package.yml index 6015c1751..c753c15db 100644 --- a/.github/workflows/wasm-package.yml +++ b/.github/workflows/wasm-package.yml @@ -10,7 +10,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v5 @@ -24,7 +24,7 @@ jobs: # bullseye_deps.sh is written for Debian, but works as-is on Ubuntu too. # Skip installing Java (-j), the Android NDK (-n) and MinGW (-w) which # we don't need for building the NPM package. - - run: sudo contrib/bullseye_deps.sh -j -n -w + - run: sudo contrib/trixie_deps.sh -j -n -w # Build NPM package into a tgz file (pack internally triggers the build/prepare script) - run: cd src/wasm_package && npm ci && npm pack --foreground-scripts diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 76f6068df..6678e3b9f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -33,7 +33,8 @@ jobs: - name: Install MSVC if: runner.os == 'Windows' - uses: ilammy/msvc-dev-cmd@v1.13.0 + shell: cmd + run: call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - name: Install qemu aarch64 if: runner.os == 'Linux' @@ -44,7 +45,7 @@ jobs: - name: Build Wheels uses: pypa/cibuildwheel@v3.1.4 env: - CIBW_SKIP: "cp38* cp314* *-win32 *-musllinux_aarch64 pp* gp* cp3??t-*" + CIBW_SKIP: "cp38* cp315* *-win32 *-musllinux_aarch64 pp* gp* cp3??t-*" CIBW_ARCHS_LINUX: "x86_64 aarch64" CIBW_ARCHS_MACOS: "x86_64 arm64 universal2" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19ed7b705..2a51179fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,15 +4,19 @@ variables: stages: - test - release + - docker_build -test_with_valgrind: +.test_template: stage: test - image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c + image: blockstream/wallycore@sha256:060af7536b6d197b269f4e8bcab0e30fe082f656dd4738cf64ad9fd90b93e1d7 + tags: + - ga + +test_with_valgrind: + extends: .test_template parallel: matrix: - CONFIGURE_ARGS: [--enable-elements=yes,--enable-elements=no,--enable-minimal=yes] - tags: - - ga artifacts: reports: codequality: valgrind.json @@ -20,7 +24,6 @@ test_with_valgrind: - ./tools/cleanup.sh && ./tools/autogen.sh - CFLAGS='-Werror' ./configure --enable-export-all --enable-swig-python --enable-swig-java $CONFIGURE_ARGS --enable-shared --disable-static - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2)) - - make check -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2)) - for t in $(ls src/.libs/test_* | egrep -v '_clear|xml|json' | tr '\n' ' '); do LD_LIBRARY_PATH=./src/.libs/ valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose --xml=yes --xml-file=$t.xml $t; done - for t in $(ls src/.libs/test_* | egrep -v '_clear|xml|json' | tr '\n' ' '); do valgrind-codequality --input-file $t.xml --output-file $t.json; done - for t in $(ls src/test/test_*.py | tr '\n' ' '); do WALLY_SKIP_EXPENSIVE_TESTS=1 PYTHONMALLOC=malloc PYTHONDEVMODE=1 MALLOC_CHECK_=3 valgrind --tool=memcheck --leak-check=no --verbose --xml=yes --xml-file=$t.xml python $t; done @@ -28,33 +31,27 @@ test_with_valgrind: - jq '[.[]|.[]]' -s ./src/.libs/test_*.json src/test/test_*.json > valgrind.json || true test_asan_ubsan_gcc: - stage: test - image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c + extends: .test_template parallel: matrix: - CONFIGURE_ARGS: [ --enable-elements=no, --enable-elements=no --enable-minimal=yes, "", --enable-minimal=yes ] - tags: - - ga script: - ./tools/cleanup.sh && ./tools/autogen.sh - CC=gcc CFLAGS="-O2 -fsanitize=address -fsanitize=bounds -fsanitize=undefined -fsanitize=alignment -fsanitize-address-use-after-scope -fno-sanitize-recover=all" ./configure --enable-export-all --enable-swig-python --enable-swig-java $CONFIGURE_ARGS --enable-shared --disable-static --disable-clear-tests --disable-asm - - sed -i 's/^PYTHON = /PYTHON = LD_PRELOAD=\/usr\/lib\/gcc\/x86_64-linux-gnu\/10\/libasan.so /g' src/Makefile - - sed -i 's/^JAVA = /JAVA = LD_PRELOAD=\/usr\/lib\/gcc\/x86_64-linux-gnu\/10\/libasan.so /g' src/Makefile + - sed -i 's/^PYTHON = /PYTHON = LD_PRELOAD=\/usr\/lib\/gcc\/x86_64-linux-gnu\/14\/libasan.so /g' src/Makefile + - sed -i 's/^JAVA = /JAVA = LD_PRELOAD=\/usr\/lib\/gcc\/x86_64-linux-gnu\/14\/libasan.so /g' src/Makefile - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2)) - ASAN_OPTIONS=abort_on_error=1:fast_unwind_on_malloc=0:detect_leaks=0 UBSAN_OPTIONS=print_stacktrace=1 make check V=1 test_scan_build_clang: - stage: test - image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c + extends: .test_template parallel: matrix: - CONFIGURE_ARGS: [ --enable-elements=no, --enable-elements=no --enable-minimal=yes, "", --enable-minimal=yes ] - tags: - - ga script: - ./tools/cleanup.sh && ./tools/autogen.sh - - CC=clang scan-build-11 ./configure --enable-export-all --enable-swig-python --enable-swig-java --disable-clear-tests --disable-asm $CONFIGURE_ARGS - - scan-build-11 --keep-cc --exclude src/secp256k1/ --status-bugs --keep-empty -o scan-build-output make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2)) + - CC=clang scan-build-19 ./configure --enable-export-all --enable-swig-python --enable-swig-java --disable-clear-tests --disable-asm $CONFIGURE_ARGS + - scan-build-19 --keep-cc --exclude src/secp256k1/ --status-bugs --keep-empty -o scan-build-output make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2)) artifacts: expire_in: 3 days name: scan-build-output @@ -63,10 +60,7 @@ test_scan_build_clang: - scan-build-output/ test_cmake: - stage: test - image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c - tags: - - ga + extends: .test_template script: - mv _cmake cmake - mv _CMakeLists.txt CMakeLists.txt @@ -82,7 +76,6 @@ test_cmake: - cd build-cmake - ctest --output-on-failure - cd - - - pip install gcovr - gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml --root ${CI_PROJECT_DIR} coverage: /^\s*lines:\s*\d+.\d+\%/ artifacts: @@ -93,33 +86,24 @@ test_cmake: path: coverage.xml test_amalgamation: - stage: test - image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c + extends: .test_template parallel: matrix: - CC: [gcc, clang ] BUILD_ARGS: [ "", -DBUILD_MINIMAL, -DBUILD_ELEMENTS, -DBUILD_ELEMENTS -DBUILD_MINIMAL ] - tags: - - ga script: - touch config.h - $CC $BUILD_ARGS -Wall -W -Wextra -Werror -I. -I./src -I./src/ccan -I./src/secp256k1/include src/ctest/amalgamation_compile_test.c test_mingw_static_build: - stage: test - image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c - tags: - - ga + extends: .test_template script: - ./tools/cleanup.sh && ./tools/autogen.sh - CC=x86_64-w64-mingw32-gcc ./configure --host=x86_64-w64-mingw32 --disable-swig-python --disable-swig-java --disable-shared --enable-static - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2)) test_no_elements_abi: - stage: test - image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c - tags: - - ga + extends: .test_template artifacts: script: - ./tools/cleanup.sh && ./tools/autogen.sh @@ -127,25 +111,25 @@ test_no_elements_abi: - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2)) build_wally_release_files: + extends: .test_template stage: release needs: [test_mingw_static_build,test_with_valgrind,test_asan_ubsan_gcc,test_scan_build_clang,test_cmake,test_amalgamation] - image: greenaddress/wallycore@sha256:956b107d688f549c6e3884424991b7d3d34d84173990d43046fd760d7918db7c artifacts: expire_in: 7 days name: wallycore-bindings when: on_success paths: - dist/* - tags: - - ga script: - - python3 -m build + - python3 -m build --wheel + - mv dist/*.whl ${CI_PROJECT_DIR} + - python3 -m build --sdist - virtualenv -p python3 .smoketest - source .smoketest/bin/activate - - pip install --find-links=./dist wallycore + - pip install --find-links=${CI_PROJECT_DIR} wallycore - python -c "import wallycore as w; assert w.hex_from_bytes(w.hex_to_bytes('ff')) == 'ff'" - deactivate - - rm -rf .smoketest dist/*.whl + - rm -rf .smoketest ${CI_PROJECT_DIR}/*.whl - mv dist wally_dist - ./tools/build_android_libraries.sh - mv release wallycore-android-jni @@ -166,3 +150,39 @@ build_wally_release_files: - cd ../.. - mv wally_dist/* dist/ - rmdir wally_dist + +.docker_build_template: + stage: docker_build + needs: [] + when: manual + variables: + DOCKER_HOST: tcp://localhost:2375 + DOCKER_TLS_CERTDIR: "" + DOCKER_BUILDKIT: 1 + BUILDX_GIT_INFO: false + image: docker:23 + services: + - docker:23-dind + tags: + - cloud + retry: + max: 2 + when: [runner_system_failure, unknown_failure, stuck_or_timeout_failure] + script: + - echo "$DOCKER_HUB_TOKEN" | docker login --username "$DOCKER_HUB_USER" --password-stdin + - docker buildx create --use --platform=linux/amd64,linux/arm64 --name wally-multi-builder --buildkitd-flags '--allow-insecure-entitlement network.host' + - docker buildx inspect --bootstrap + - cd contrib + - docker buildx build + --network=host + -t blockstream/wallycore:${CI_COMMIT_SHA} + ${DOCKER_BUILD_ARGS} + . + --progress=plain + --push + +# Debian stable +build_docker_trixie: + extends: .docker_build_template + variables: + DOCKER_BUILD_ARGS: -f Dockerfile_trixie --platform linux/amd64,linux/arm64 diff --git a/_cmake/config.h.in b/_cmake/config.h.in index 692b63e81..fd045ba27 100644 --- a/_cmake/config.h.in +++ b/_cmake/config.h.in @@ -7,8 +7,7 @@ /* Define to 1 if you have the header file. */ #cmakedefine HAVE_ASM_PAGE_H @HAVE_ASM_PAGE_H@ - -/* Define to 1 if you have the header file. */ +/* Define to 1 if you have the header file. */ #cmakedefine HAVE_BYTESWAP_H @HAVE_BYTESWAP_H@ /* Define to 1 if you have the `explicit_bzero' function. */ @@ -41,6 +40,9 @@ /* Define if we have unaligned access */ #cmakedefine HAVE_UNALIGNED_ACCESS @HAVE_UNALIGNED_ACCESS@ +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_UNISTD_H @HAVE_UNISTD_H@ + /* Name of package */ #cmakedefine PACKAGE @PACKAGE@ @@ -65,17 +67,7 @@ /* Version number of package */ #cmakedefine VERSION @VERSION@ -/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most - significant byte first (like Motorola and SPARC, unlike Intel). */ -#if defined AC_APPLE_UNIVERSAL_BUILD -# if defined __BIG_ENDIAN__ -# define WORDS_BIGENDIAN 1 -# endif -#else -# ifndef WORDS_BIGENDIAN #cmakedefine WORDS_BIGENDIAN @WORDS_BIGENDIAN@ -# endif -#endif #if defined (_WIN32) && !defined(_SSIZE_T_DECLARED) && !defined(_ssize_t) && !defined(ssize_t) #if defined(_WIN64) diff --git a/_cmake/utils.cmake b/_cmake/utils.cmake index 490e96448..7532d708c 100644 --- a/_cmake/utils.cmake +++ b/_cmake/utils.cmake @@ -4,13 +4,18 @@ include(CheckFunctionExists) include(CheckCSourceRuns) function(generate_config_file) + # FIXME: AC_APPLE_UNIVERSAL_BUILD + check_include_file("asm/page.h" HAVE_ASM_PAGE_H) + check_include_file("byteswap.h" HAVE_BYTESWAP_H) check_function_exists("explicit_bzero" HAVE_EXPLICIT_BZERO) check_function_exists("explicit_memset" HAVE_EXPLICIT_MEMSET) check_c_source_compiles( "int main(void) {int a = 42; int *pnt = &a; __asm__ __volatile__ (\"\" : : \"r\"(pnt) : \"memory\");}" HAVE_INLINE_ASM ) + check_include_file("mbedtls/sha256.h" HAVE_MBEDTLS_SHA256_H) check_include_file("mbedtls/sha512.h" HAVE_MBEDTLS_SHA512_H) + check_function_exists("memset_s" HAVE_MEMSET_S) check_function_exists("mmap" HAVE_MMAP) check_function_exists("posix_memalign" HAVE_POSIX_MEMALIGN) check_include_file("sys/mman.h" HAVE_SYS_MMAN_H) @@ -21,6 +26,7 @@ function(generate_config_file) "int main(void){static int a[2];return *((int*)(((char*)a)+1)) != 0;}" HAVE_UNALIGNED_ACCESS ) endif() + check_include_file("unistd.h" HAVE_UNISTD_H) set(PACKAGE \"${CMAKE_PROJECT_NAME}\") set(PACKAGE_BUGREPORT \"\") set(PACKAGE_NAME \"${CMAKE_PROJECT_NAME}\") @@ -30,5 +36,11 @@ function(generate_config_file) set(PACKAGE_VERSION \"${CMAKE_PROJECT_VERSION}\") set(VERSION \"${CMAKE_PROJECT_VERSION}\") + if(CMAKE_C_BYTE_ORDER STREQUAL "BIG_ENDIAN") + set(WORDS_BIGENDIAN TRUE) + else() + set(WORDS_BIGENDIAN FALSE) + endif() + configure_file(cmake/config.h.in config.h) endfunction() diff --git a/configure.ac b/configure.ac index a4637cc93..bf83da840 100644 --- a/configure.ac +++ b/configure.ac @@ -343,8 +343,8 @@ AC_ARG_ENABLE(python-manylinux, [python_manylinux=$enableval], [python_manylinux=no]) AM_CONDITIONAL([PYTHON_MANYLINUX], [test "x$python_manylinux" = "xyes"]) -AX_PYTHON_DEVEL([>= '2.7.0']) -AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != ""]) +AX_PYTHON_DEVEL([>= '3.9.0'], [true]) +AM_CONDITIONAL([HAVE_PYTHON], [test "x$ax_python_devel_found" = "xyes"]) AC_ARG_ENABLE(wasm-interface, @@ -422,7 +422,7 @@ AM_CONDITIONAL([HAVE_JAVAC], [test "x$JAVAC" != "x"]) AM_CONDITIONAL([RUN_JAVA_TESTS], dnl Only run tests if we have java-swig, compiler and interpreter [test "x$tests" = xyes -a "x$swig_java" = xyes -a -n "$JAVAC" -a -n "$JAVA"]) -JAVAC_TARGET=1.8 +JAVAC_TARGET=17 AC_SUBST([JAVAC_TARGET]) AC_SUBST([AM_CFLAGS]) diff --git a/contrib/Dockerfile_bookworm b/contrib/Dockerfile_bookworm index 417fce19a..c3b186fe9 100644 --- a/contrib/Dockerfile_bookworm +++ b/contrib/Dockerfile_bookworm @@ -1,5 +1,5 @@ # -# Dockerfile for wally builds on Debian bookworm (stable). +# Dockerfile for wally builds on Debian bookworm (oldstable). # build from this directory with e.g: # docker buildx build --platform linux/arm64,linux/amd64 -f Dockerfile_bullseye -t greenaddress/wallycore:bookworm . # diff --git a/contrib/Dockerfile_bullseye b/contrib/Dockerfile_bullseye index 277292a59..443ec302d 100644 --- a/contrib/Dockerfile_bullseye +++ b/contrib/Dockerfile_bullseye @@ -1,5 +1,5 @@ # -# Dockerfile for wally builds on Debian bullseye (oldstable). +# Dockerfile for wally builds on Debian bullseye (oldoldstable). # build from this directory with e.g: # DOCKER_BUILDKIT=1 docker build . -t greenaddress/wallycore -f Dockerfile_bullseye # and for linux/arm64: diff --git a/contrib/Dockerfile_trixie b/contrib/Dockerfile_trixie new file mode 100644 index 000000000..436e7fc63 --- /dev/null +++ b/contrib/Dockerfile_trixie @@ -0,0 +1,16 @@ +# +# Dockerfile for wally builds on Debian trixie (stable). +# build from this directory with e.g: +# DOCKER_BUILDKIT=1 docker build . -t blockstream/wallycore -f Dockerfile_trixie +# and for linux/arm64: +# DOCKER_BUILDKIT=1 docker build . -t blockstream/wallycore -f Dockerfile_trixie --platform linux/arm64 --build-arg TARGETARCH=arm64 +# +FROM debian:trixie@sha256:35b8ff74ead4880f22090b617372daff0ccae742eb5674455d542bef71ef1999 +WORKDIR /root +COPY trixie_deps.sh ./deps.sh +COPY requirements.txt ./contrib/requirements.txt +ARG TARGETARCH=amd64 +ENV TARGETARCH=${TARGETARCH} +ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-${TARGETARCH} +RUN ./deps.sh && rm ./deps.sh +ENV ANDROID_NDK=/opt/android-ndk-r26b diff --git a/contrib/trixie_deps.sh b/contrib/trixie_deps.sh new file mode 100755 index 000000000..e2d665151 --- /dev/null +++ b/contrib/trixie_deps.sh @@ -0,0 +1,78 @@ +#! /usr/bin/env bash +# Install required dependencies for building wally +# Options: +# -e : Don't install emsdk (used for JS builds) +# -j : Don't install Java SDK (used for Java builds) +# -n : Don't install Android NDK (used for Android builds) +# -w : Don't install MinGW (used for Windows cross compiles) +set -e + +skip_emsdk= +skip_ndk= +skip_java= +skip_windows= +while getopts enjw name +do + case $name in + e) skip_emsdk=1;; + n) skip_ndk=1;; + j) skip_java=1;; + w) skip_windows=1;; + *) echo "Invalid flag"; exit 1;; + esac +done +shift $(($OPTIND - 1)) + +apt update -qq +apt upgrade -yqq + +java_packages= +if [ -z "$skip_java" ]; then + jdk_package="openjdk-21-jdk" + jdk_install_dir="java-21-openjdk-$TARGETARCH" + jre_package="openjdk-21-jre" +fi +windows_packages= +if [ -z "$skip_windows" ]; then + windows_packages="g++-mingw-w64-x86-64" +fi +apt install --no-install-recommends unzip autoconf automake autotools-dev pkg-config build-essential libtool python3{,-dev,-pip,-virtualenv} python{,-dev}-is-python3 clang{,-format,-tidy} git swig curl cmake libssl-dev libtool-bin $jdk_package $jre_package $windows_packages valgrind jq -yqq + +# Note --break-system-packages to allow installing our requirements system-wide +pip install valgrind-codequality gcovr -r contrib/requirements.txt --break-system-packages + +pushd /opt + +if [ -z "$skip_ndk" ]; then + curl -L -o ndk.zip https://dl.google.com/android/repository/android-ndk-r26b-linux.zip + echo "ad73c0370f0b0a87d1671ed2fd5a9ac9acfd1eb5c43a7fbfbd330f85d19dd632 ndk.zip" | shasum -a 256 -c + unzip ndk.zip + rm ndk.zip +fi + +if [ -z "$skip_emsdk" ]; then + # Install node 24 + curl -fsSL https://deb.nodesource.com/setup_24.x | bash - + apt install nodejs -yqq + # Install emsdk + git clone https://github.com/emscripten-core/emsdk + cd emsdk + EMSDK_VERSION=3.1.57 + ./emsdk install ${EMSDK_VERSION} + ./emsdk activate ${EMSDK_VERSION} + # Force emsdk to use the installed node version instead of its own + sed -i "s/^NODE_JS = .*$/NODE_JS = 'node'/g" /opt/emsdk/.emscripten + # Make emsdk commands available + source ./emsdk_env.sh +fi + +if [ -f /.dockerenv ]; then + # Installing dependencies into a docker image. + # Purge uneeded files to keep the image as small as possible. + apt remove --purge curl unzip -yqq + apt -yqq autoremove + apt -yqq clean + rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/* /usr/share/locale/* /usr/share/man /usr/share/doc /lib/xtables/libip6* /root/.cache +fi + +popd diff --git a/src/Makefile.am b/src/Makefile.am index 08abbc504..82674dbc1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -127,7 +127,8 @@ endif # RUN_JAVA_TESTS $(JAVA_CLASSES) &: $(JAVA_CLASSES:.class=.java) $(AM_V_at)$(JAVAC) -implicit:none -source $(JAVAC_TARGET) -target $(JAVAC_TARGET) -sourcepath $(sjs) $^ -JAVA_TEST = @LDPATH_VAR@=.libs $(JAVA) -Djava.library.path=.libs -classpath $(sjs) com.blockstream.test. +JAVA_TEST_OPT = --enable-native-access=ALL-UNNAMED +JAVA_TEST = @LDPATH_VAR@=.libs $(JAVA) $(JAVA_TEST_OPT) -Djava.library.path=.libs -classpath $(sjs) com.blockstream.test. endif # HAVE_JAVAC @@ -295,9 +296,27 @@ test_elements_tx_LDADD += $(PYTHON_LIBS) endif endif -check-local: check-libwallycore check-swig-python check-swig-java +check-local: check-libwallycore check-swig-java check-swig-python $(AM_V_at)! grep '^int ' $(top_srcdir)/include/*.h # Missing WALLY_CORE_API +if RUN_JAVA_TESTS +check-swig-java: $(SWIG_JAVA_TEST_DEPS) .libs/libwallycore.$(platform_dso_ext) + $(AM_V_at)! grep 'native int wally_' $(sjs)/$(cblw)/Wally.java # Unwrapped Java calls + $(AM_V_at)! grep 'native Object wally_' $(sjs)/$(cblw)/Wally.java # Unwrapped Java calls +if BUILD_ELEMENTS + $(AM_V_at)$(JAVA_TEST)test_assets + $(AM_V_at)$(JAVA_TEST)test_elements_tx + $(AM_V_at)$(JAVA_TEST)test_pegs +endif + $(AM_V_at)$(JAVA_TEST)test_bip32 + $(AM_V_at)$(JAVA_TEST)test_descriptor + $(AM_V_at)$(JAVA_TEST)test_mnemonic + $(AM_V_at)$(JAVA_TEST)test_scripts + $(AM_V_at)$(JAVA_TEST)test_tx +else # RUN_JAVA_TESTS +check-swig-java: ; +endif # RUN_JAVA_TESTS + if SHARED_BUILD_ENABLED if RUN_PYTHON_TESTS check-libwallycore: $(PYTHON_TEST_DEPS) @@ -337,7 +356,8 @@ if BUILD_ELEMENTS endif if USE_SWIG_PYTHON -check-swig-python: +check-swig-python: check-libwallycore check-swig-java + $(AM_V_at)rm -rf $(top_builddir)/venv $(AM_V_at)$(PYTHON) -m virtualenv $(top_builddir)/venv $(AM_V_at)$(top_builddir)/venv/bin/python -m pip install $(top_srcdir) $(AM_V_at)$(top_builddir)/venv/bin/python swig_python/contrib/aes.py @@ -362,27 +382,8 @@ else # RUN_PYTHON_TESTS check-libwallycore: ; endif # RUN_PYTHON_TESTS -if RUN_JAVA_TESTS -check-swig-java: $(SWIG_JAVA_TEST_DEPS) - $(AM_V_at)! grep 'native int wally_' $(sjs)/$(cblw)/Wally.java # Unwrapped Java calls - $(AM_V_at)! grep 'native Object wally_' $(sjs)/$(cblw)/Wally.java # Unwrapped Java calls -if BUILD_ELEMENTS - $(AM_V_at)$(JAVA_TEST)test_assets - $(AM_V_at)$(JAVA_TEST)test_elements_tx - $(AM_V_at)$(JAVA_TEST)test_pegs -endif - $(AM_V_at)$(JAVA_TEST)test_bip32 - $(AM_V_at)$(JAVA_TEST)test_descriptor - $(AM_V_at)$(JAVA_TEST)test_mnemonic - $(AM_V_at)$(JAVA_TEST)test_scripts - $(AM_V_at)$(JAVA_TEST)test_tx -else # RUN_JAVA_TESTS -check-swig-java: ; -endif # RUN_JAVA_TESTS - endif # SHARED_BUILD_ENABLED -.PHONY: check-libwallycore check-swig-python check-swig-java clean-swig-python clean-swig-java +.PHONY: check-libwallycore check-swig-java check-swig-python clean-swig-java clean-swig-python else # RUN_TESTS -.PHONY: clean-swig-python clean-swig-java +.PHONY: clean-swig-java clean-swig-python endif # RUN_TESTS - diff --git a/src/ccan/ccan/base64/base64.c b/src/ccan/ccan/base64/base64.c index 439655d5d..ea2ce8727 100644 --- a/src/ccan/ccan/base64/base64.c +++ b/src/ccan/ccan/base64/base64.c @@ -214,44 +214,66 @@ ssize_t base64_decode_using_maps(const base64_maps_t *maps, * base64_maps_rfc4648 - pregenerated maps struct for rfc4648 */ const base64_maps_t base64_maps_rfc4648 = { - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", - - "\xff\xff\xff\xff\xff" /* 0 */ \ - "\xff\xff\xff\xff\xff" /* 5 */ \ - "\xff\xff\xff\xff\xff" /* 10 */ \ - "\xff\xff\xff\xff\xff" /* 15 */ \ - "\xff\xff\xff\xff\xff" /* 20 */ \ - "\xff\xff\xff\xff\xff" /* 25 */ \ - "\xff\xff\xff\xff\xff" /* 30 */ \ - "\xff\xff\xff\xff\xff" /* 35 */ \ - "\xff\xff\xff\x3e\xff" /* 40 */ \ - "\xff\xff\x3f\x34\x35" /* 45 */ \ - "\x36\x37\x38\x39\x3a" /* 50 */ \ - "\x3b\x3c\x3d\xff\xff" /* 55 */ \ - "\xff\xff\xff\xff\xff" /* 60 */ \ - "\x00\x01\x02\x03\x04" /* 65 A */ \ - "\x05\x06\x07\x08\x09" /* 70 */ \ - "\x0a\x0b\x0c\x0d\x0e" /* 75 */ \ - "\x0f\x10\x11\x12\x13" /* 80 */ \ - "\x14\x15\x16\x17\x18" /* 85 */ \ - "\x19\xff\xff\xff\xff" /* 90 */ \ - "\xff\xff\x1a\x1b\x1c" /* 95 */ \ - "\x1d\x1e\x1f\x20\x21" /* 100 */ \ - "\x22\x23\x24\x25\x26" /* 105 */ \ - "\x27\x28\x29\x2a\x2b" /* 110 */ \ - "\x2c\x2d\x2e\x2f\x30" /* 115 */ \ - "\x31\x32\x33\xff\xff" /* 120 */ \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */ \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */ \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */ \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */ \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" \ - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */ + { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/' + }, { + '\xff', '\xff', '\xff', '\xff', '\xff', /* 0 */ + '\xff', '\xff', '\xff', '\xff', '\xff', /* 5 */ + '\xff', '\xff', '\xff', '\xff', '\xff', /* 10 */ + '\xff', '\xff', '\xff', '\xff', '\xff', /* 15 */ + '\xff', '\xff', '\xff', '\xff', '\xff', /* 20 */ + '\xff', '\xff', '\xff', '\xff', '\xff', /* 25 */ + '\xff', '\xff', '\xff', '\xff', '\xff', /* 30 */ + '\xff', '\xff', '\xff', '\xff', '\xff', /* 35 */ + '\xff', '\xff', '\xff', '\x3e', '\xff', /* 40 */ + '\xff', '\xff', '\x3f', '\x34', '\x35', /* 45 */ + '\x36', '\x37', '\x38', '\x39', '\x3a', /* 50 */ + '\x3b', '\x3c', '\x3d', '\xff', '\xff', /* 55 */ + '\xff', '\xff', '\xff', '\xff', '\xff', /* 60 */ + '\x00', '\x01', '\x02', '\x03', '\x04', /* 65 A */ + '\x05', '\x06', '\x07', '\x08', '\x09', /* 70 */ + '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', /* 75 */ + '\x0f', '\x10', '\x11', '\x12', '\x13', /* 80 */ + '\x14', '\x15', '\x16', '\x17', '\x18', /* 85 */ + '\x19', '\xff', '\xff', '\xff', '\xff', /* 90 */ + '\xff', '\xff', '\x1a', '\x1b', '\x1c', /* 95 */ + '\x1d', '\x1e', '\x1f', '\x20', '\x21', /* 100 */ + '\x22', '\x23', '\x24', '\x25', '\x26', /* 105 */ + '\x27', '\x28', '\x29', '\x2a', '\x2b', /* 110 */ + '\x2c', '\x2d', '\x2e', '\x2f', '\x30', /* 115 */ + '\x31', '\x32', '\x33', '\xff', '\xff', /* 120 */ + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', /* 125 */ + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', /* 155 */ + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', /* 185 */ + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', /* 215 */ + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', + '\xff', '\xff', '\xff', '\xff', '\xff', '\xff' /* 245 */ + } }; diff --git a/src/secp256k1 b/src/secp256k1 index 615262261..45f6f0f15 160000 --- a/src/secp256k1 +++ b/src/secp256k1 @@ -1 +1 @@ -Subproject commit 6152622613fdf1c5af6f31f74c427c4e9ee120ce +Subproject commit 45f6f0f158c5ae80a2c8a53398ea4adbf19af6dc diff --git a/tools/build-aux/m4/ax_python_devel.m4 b/tools/build-aux/m4/ax_python_devel.m4 index 8269c547e..cbed7f318 100644 --- a/tools/build-aux/m4/ax_python_devel.m4 +++ b/tools/build-aux/m4/ax_python_devel.m4 @@ -4,7 +4,7 @@ # # SYNOPSIS # -# AX_PYTHON_DEVEL([version]) +# AX_PYTHON_DEVEL([version[,optional]]) # # DESCRIPTION # @@ -23,6 +23,11 @@ # version number. Don't use "PYTHON_VERSION" for this: that environment # variable is declared as precious and thus reserved for the end-user. # +# By default this will fail if it does not detect a development version of +# python. If you want it to continue, set optional to true, like +# AX_PYTHON_DEVEL([], [true]). The ax_python_devel_found variable will be +# "no" if it fails. +# # This macro should work for all versions of Python >= 2.1.0. As an end # user, you can disable the check for the python version by setting the # PYTHON_NOVERSIONCHECK environment variable to something else than the @@ -67,10 +72,18 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 21 +#serial 37 AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL]) AC_DEFUN([AX_PYTHON_DEVEL],[ + # Get whether it's optional + if test -z "$2"; then + ax_python_devel_optional=false + else + ax_python_devel_optional=$2 + fi + ax_python_devel_found=yes + # # Allow the use of a (user set) custom python version # @@ -81,21 +94,26 @@ AC_DEFUN([AX_PYTHON_DEVEL],[ AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]]) if test -z "$PYTHON"; then - AC_MSG_ERROR([Cannot find python$PYTHON_VERSION in your system path]) + AC_MSG_WARN([Cannot find python$PYTHON_VERSION in your system path]) + if ! $ax_python_devel_optional; then + AC_MSG_ERROR([Giving up, python development not available]) + fi + ax_python_devel_found=no PYTHON_VERSION="" fi - # - # Check for a version of Python >= 2.1.0 - # - AC_MSG_CHECKING([for a version of Python >= '2.1.0']) - ac_supports_python_ver=`$PYTHON -c "import sys; \ + if test $ax_python_devel_found = yes; then + # + # Check for a version of Python >= 2.1.0 + # + AC_MSG_CHECKING([for a version of Python >= '2.1.0']) + ac_supports_python_ver=`$PYTHON -c "import sys; \ ver = sys.version.split ()[[0]]; \ print (ver >= '2.1.0')"` - if test "$ac_supports_python_ver" != "True"; then + if test "$ac_supports_python_ver" != "True"; then if test -z "$PYTHON_NOVERSIONCHECK"; then AC_MSG_RESULT([no]) - AC_MSG_FAILURE([ + AC_MSG_WARN([ This version of the AC@&t@_PYTHON_DEVEL macro doesn't work properly with versions of Python before 2.1.0. You may need to re-run configure, setting the @@ -104,58 +122,119 @@ PYTHON_EXTRA_LIBS and PYTHON_EXTRA_LDFLAGS by hand. Moreover, to disable this check, set PYTHON_NOVERSIONCHECK to something else than an empty string. ]) + if ! $ax_python_devel_optional; then + AC_MSG_FAILURE([Giving up]) + fi + ax_python_devel_found=no + PYTHON_VERSION="" else AC_MSG_RESULT([skip at user request]) fi - else + else AC_MSG_RESULT([yes]) + fi fi - # - # if the macro parameter ``version'' is set, honour it - # - if test -n "$1"; then + if test $ax_python_devel_found = yes; then + # + # If the macro parameter ``version'' is set, honour it. + # A Python shim class, VPy, is used to implement correct version comparisons via + # string expressions, since e.g. a naive textual ">= 2.7.3" won't work for + # Python 2.7.10 (the ".1" being evaluated as less than ".3"). + # + if test -n "$1"; then AC_MSG_CHECKING([for a version of Python $1]) - ac_supports_python_ver=`$PYTHON -c "import sys; \ - ver = sys.version.split ()[[0]]; \ + cat << EOF > ax_python_devel_vpy.py +class VPy: + def vtup(self, s): + return tuple(map(int, s.strip().replace("rc", ".").split("."))) + def __init__(self): + import sys + self.vpy = tuple(sys.version_info)[[:3]] + def __eq__(self, s): + return self.vpy == self.vtup(s) + def __ne__(self, s): + return self.vpy != self.vtup(s) + def __lt__(self, s): + return self.vpy < self.vtup(s) + def __gt__(self, s): + return self.vpy > self.vtup(s) + def __le__(self, s): + return self.vpy <= self.vtup(s) + def __ge__(self, s): + return self.vpy >= self.vtup(s) +EOF + ac_supports_python_ver=`$PYTHON -c "import ax_python_devel_vpy; \ + ver = ax_python_devel_vpy.VPy(); \ print (ver $1)"` + rm -rf ax_python_devel_vpy*.py* __pycache__/ax_python_devel_vpy*.py* if test "$ac_supports_python_ver" = "True"; then - AC_MSG_RESULT([yes]) + AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) - AC_MSG_ERROR([this package requires Python $1. + AC_MSG_WARN([this package requires Python $1. If you have it installed, but it isn't the default Python interpreter in your system path, please pass the PYTHON_VERSION variable to configure. See ``configure --help'' for reference. ]) + if ! $ax_python_devel_optional; then + AC_MSG_ERROR([Giving up]) + fi + ax_python_devel_found=no PYTHON_VERSION="" fi + fi fi - # - # Check if you have distutils, else fail - # - AC_MSG_CHECKING([for the distutils Python package]) - ac_distutils_result=`$PYTHON -c "import distutils" 2>&1` - if test $? -eq 0; then + if test $ax_python_devel_found = yes; then + # + # Check if you have distutils, else fail + # + AC_MSG_CHECKING([for the sysconfig Python package]) + ac_sysconfig_result=`$PYTHON -c "import sysconfig" 2>&1` + if test $? -eq 0; then AC_MSG_RESULT([yes]) - else + IMPORT_SYSCONFIG="import sysconfig" + else AC_MSG_RESULT([no]) - AC_MSG_ERROR([cannot import Python module "distutils". + + AC_MSG_CHECKING([for the distutils Python package]) + ac_sysconfig_result=`$PYTHON -c "from distutils import sysconfig" 2>&1` + if test $? -eq 0; then + AC_MSG_RESULT([yes]) + IMPORT_SYSCONFIG="from distutils import sysconfig" + else + AC_MSG_WARN([cannot import Python module "distutils". Please check your Python installation. The error was: -$ac_distutils_result]) - PYTHON_VERSION="" +$ac_sysconfig_result]) + if ! $ax_python_devel_optional; then + AC_MSG_ERROR([Giving up]) + fi + ax_python_devel_found=no + PYTHON_VERSION="" + fi + fi fi - # - # Check for Python include path - # - AC_MSG_CHECKING([for Python include path]) - if test -z "$PYTHON_CPPFLAGS"; then - python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc ());"` - plat_python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc (plat_specific=1));"` + if test $ax_python_devel_found = yes; then + # + # Check for Python include path + # + AC_MSG_CHECKING([for Python include path]) + if test -z "$PYTHON_CPPFLAGS"; then + if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then + # sysconfig module has different functions + python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_path ('include'));"` + plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_path ('platinclude'));"` + else + # old distutils way + python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_python_inc ());"` + plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_python_inc (plat_specific=1));"` + fi if test -n "${python_path}"; then if test "${plat_python_path}" != "${python_path}"; then python_path="-I$python_path -I$plat_python_path" @@ -164,21 +243,22 @@ $ac_distutils_result]) fi fi PYTHON_CPPFLAGS=$python_path - fi - AC_MSG_RESULT([$PYTHON_CPPFLAGS]) + fi + AC_MSG_RESULT([$PYTHON_CPPFLAGS]) + AC_SUBST([PYTHON_CPPFLAGS]) - # - # Check for Python library path - # - AC_MSG_CHECKING([for Python library path]) - if test -z "$PYTHON_LIBS"; then + # + # Check for Python library path + # + AC_MSG_CHECKING([for Python library path]) + if test -z "$PYTHON_LIBS"; then # (makes two attempts to ensure we've got a version number # from the interpreter) ac_python_version=`cat</dev/null; then - # On OSX we can hit https://bugs.python.org/issue3588 - PYTHON_EXTRA_LIBS=`$PYTHON-config --ldflags` - fi - fi - AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS]) + # Hack for macos, it sticks this in here. + PYTHON_EXTRA_LDFLAGS=`echo $PYTHON_EXTRA_LDFLAGS | sed 's/CoreFoundation.*$/CoreFoundation/'` + fi + AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS]) + AC_SUBST(PYTHON_EXTRA_LDFLAGS) - # - # final check to see if everything compiles alright - # - AC_MSG_CHECKING([consistency of all components of python development environment]) - # save current global flags - ac_save_LIBS="$LIBS" - ac_save_LDFLAGS="$LDFLAGS" - ac_save_CPPFLAGS="$CPPFLAGS" - LIBS="$ac_save_LIBS $PYTHON_LIBS $PYTHON_EXTRA_LIBS $PYTHON_EXTRA_LIBS" - LDFLAGS="$ac_save_LDFLAGS $PYTHON_EXTRA_LDFLAGS" - CPPFLAGS="$ac_save_CPPFLAGS $PYTHON_CPPFLAGS" - AC_LANG_PUSH([C]) - AC_LINK_IFELSE([ + # + # final check to see if everything compiles alright + # + AC_MSG_CHECKING([consistency of all components of python development environment]) + # save current global flags + ac_save_LIBS="$LIBS" + ac_save_LDFLAGS="$LDFLAGS" + ac_save_CPPFLAGS="$CPPFLAGS" + LIBS="$ac_save_LIBS $PYTHON_LIBS $PYTHON_EXTRA_LIBS" + LDFLAGS="$ac_save_LDFLAGS $PYTHON_EXTRA_LDFLAGS" + CPPFLAGS="$ac_save_CPPFLAGS $PYTHON_CPPFLAGS" + AC_LANG_PUSH([C]) + AC_LINK_IFELSE([ AC_LANG_PROGRAM([[#include ]], [[Py_Initialize();]]) ],[pythonexists=yes],[pythonexists=no]) - AC_LANG_POP([C]) - # turn back to default flags - CPPFLAGS="$ac_save_CPPFLAGS" - LIBS="$ac_save_LIBS" - LDFLAGS="$ac_save_LDFLAGS" + AC_LANG_POP([C]) + # turn back to default flags + CPPFLAGS="$ac_save_CPPFLAGS" + LIBS="$ac_save_LIBS" + LDFLAGS="$ac_save_LDFLAGS" - AC_MSG_RESULT([$pythonexists]) + AC_MSG_RESULT([$pythonexists]) - if test ! "x$pythonexists" = "xyes"; then - if test "x$python_manylinux" = "xyes"; then + if test ! "x$pythonexists" = "xyes"; then + if test "x$python_manylinux" = "xyes"; then # Ignore linking errors for manylinux builds (no libpython.so) pythonexists="yes" - else - AC_MSG_RESULT([ + else + + AC_MSG_WARN([ Could not link test program to Python. Maybe the main Python library has been installed in some non-standard library path. If so, pass it to configure, via the LIBS environment variable. @@ -320,20 +458,16 @@ EOD` You probably have to install the development version of the Python package for your distribution. The exact name of this package varies among them. ============================================================================ - ]) - PYTHON_VERSION="" - PYTHON_CPPFLAGS="" - PYTHON_LIBS="" - PYTHON_SITE_PKG="" - PYTHON_EXTRA_LDFLAGS="" - PYTHON_EXTRA_LIBS="" - fi + ]) + if ! $ax_python_devel_optional; then + AC_MSG_ERROR([Giving up]) + fi + ax_python_devel_found=no + PYTHON_VERSION="" + fi + fi fi - AC_SUBST([PYTHON_CPPFLAGS]) - AC_SUBST([PYTHON_LIBS]) - AC_SUBST([PYTHON_SITE_PKG]) - AC_SUBST(PYTHON_EXTRA_LDFLAGS) - AC_SUBST(PYTHON_EXTRA_LIBS) + # # all done! #