From 187d602a7c5b7cc03b2d88e513bfddd4611c5484 Mon Sep 17 00:00:00 2001 From: Matt Whitlock Date: Thu, 31 Jul 2025 17:22:27 -0400 Subject: [PATCH] don't embed Python code into libwallycore.so Embedding SWIG Python code into libwallycore.so ties the library to a specific version of Python, thereby precluding concurrent installation of the wallycore Python module for multiple implementations of Python on the same system if the Python modules are linked with a system-wide libwallycore.so. On Gentoo at least, linking libwallycore.so with libpython3.14.so and then attempting to use the wallycore Python module from Python 3.11, 3.12, or 3.13 causes an immediate segfault. Rather than injecting Python-specific glue into libwallycore.so, we can and should keep it contained within the Python native extension library that we build for each Python implementation. The Python wheel actually already compiles swig_python_wrap.c and links it into the native extension library, so linking it into libwallycore.so as well is redundant and harmful. * Remove libswig_python.la from libwallycore_la_LIBADD, and expunge its existence from Makefile.am entirely since it's now unused. * The Python wheel build for each Python implementation compiles swig_python_wrap.c against the headers for that particular Python implementation, ensuring that the resulting native extension library matches the ABI of the Python implementation for which it is installed. This may be a different Python implementation than the one found by Autoconf, the only relevance of which now is in finding the interpreter with which to run the tests. * Since we no longer link libwallycore.so with libpython*.so, there should no longer be any manylinux compatibility issues, so drop the --enable-python-manylinux Autoconf option and the PYTHON_MANYLINUX Automake conditional. This also means that the test programs no longer need to link with $(PYTHON_LIBS) since libwallycore.la now never has any dependence on Python (implicit or explicit). * Note that the Python native extension libraries do not explicitly link with libpython*.so either. This is correct, as the dynamic linker resolves their undefined Py* symbols when it loads them into the Python interpreter. * Remove check-libwallycore and check-swig-java prerequisites from the check-swig-python target, as these are three orthogonal test suites with no interdependencies. After applying these changes, the Gentoo ebuild for libwally-core 1.5.4 now successfully runs the SWIG Python tests on all currently supported versions of Python (3.12 through 3.14), all using the same libwallycore.so.6. --- README.md | 3 --- configure.ac | 13 +++++------ setup.py | 2 +- src/Makefile.am | 62 ++++++++++--------------------------------------- 4 files changed, 19 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index d830eb97b..3aa37473a 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,6 @@ $ brew install swig - `--enable-swig-python`. Enable the [SWIG](http://www.swig.org/) Python interface. The resulting shared library can be imported from Python using the generated interface file `src/swig_python/wallycore/__init__.py`. (default: no). -- `--enable-python-manylinux`. Enable [manylinux](https://github.com/pypa/manylinux) - support for building [PyPI](https://pypi.org/) compatible python wheels. Using - the resulting library in non-python programs requires linking with `libpython.so`. - `--enable-swig-java`. Enable the [SWIG](http://www.swig.org/) Java (JNI) interface. After building, see `src/swig_java/src/com/blockstream/libwally/Wally.java` for the Java interface definition (default: no). diff --git a/configure.ac b/configure.ac index 9dd4fa54e..eb7662ecd 100644 --- a/configure.ac +++ b/configure.ac @@ -338,11 +338,10 @@ AC_SUBST([libsecp256k1_LIBS]) # # Python facilities # -AC_ARG_ENABLE(python-manylinux, - AS_HELP_STRING([--enable-python-manylinux],[enable manylinux Python compatibility (default: no)]), - [python_manylinux=$enableval], [python_manylinux=no]) -AM_CONDITIONAL([PYTHON_MANYLINUX], [test "x$python_manylinux" = "xyes"]) - +dnl We set the 'optional' flag here because manylinux cibuildwheel containers +dnl don't include libpython*.so, and thus AX_PYTHON_DEVEL fails its final sanity +dnl check. We actually don't link against libpython*.so anyway, so all we care +dnl about is that we get a good value of $PYTHON_CPPFLAGS to pass to SWIG. AX_PYTHON_DEVEL([>= '3.9.0'], [true]) AM_CONDITIONAL([HAVE_PYTHON], [test "x$ax_python_devel_found" = "xyes"]) @@ -365,10 +364,10 @@ AC_ARG_ENABLE(swig-python, [swig_python=$enableval], [swig_python=no]) AM_CONDITIONAL([USE_SWIG_PYTHON], [test "x$swig_python" = "xyes"]) -AM_CONDITIONAL([RUN_PYTHON_TESTS], [test "$PYTHON" != "" -a "x$pythonexists" = "xyes"]) +AM_CONDITIONAL([RUN_PYTHON_TESTS], [test -n "$PYTHON"]) if test "x$swig_python" = "xyes"; then - if test "x$pythonexists" != "xyes"; then + if test -z "$PYTHON_CPPFLAGS"; then AC_MSG_FAILURE([ERROR: No usable Python was found for swig-python]) fi if test "x$elements_abi" = "xno"; then diff --git a/setup.py b/setup.py index b76e8cc85..279b8b653 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def _call(args, cwd=ABS_PATH): _call(['git', 'submodule', 'update', '--init', '--recursive']) CONFIGURE_ARGS = [ - '--with-pic', '--enable-swig-python', '--enable-python-manylinux', + '--with-pic', '--enable-swig-python', '--disable-swig-java', '--disable-tests', '--disable-dependency-tracking' ] diff --git a/src/Makefile.am b/src/Makefile.am index 977593a51..ec2cb4820 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -38,15 +38,7 @@ TOOLS_EXTRA_ARGS := endif if USE_SWIG_PYTHON -noinst_LTLIBRARIES += libswig_python.la -libswig_python_la_SOURCES = swig_python/swig_python_wrap.c - -libswig_python_la_CFLAGS = -I$(top_srcdir) $(libsecp256k1_CFLAGS) $(AM_CFLAGS) $(SWIG_PYTHON_CPPFLAGS) $(SWIG_WARN_CFLAGS) $(NOALIAS_CFLAGS) -if PYTHON_MANYLINUX -else -libswig_python_la_LIBADD = $(PYTHON_LIBS) -endif # PYTHON_MANYLINUX - +BUILT_SOURCES = swig_python/swig_python_wrap.c # Append our extra wrapper code to the package swig_python/swig_python_wrap.c : swig_python/swig.i swig_python/python_extra.py_in $(AM_V_at)$(SWIG) $(SWIG_PYTHON_OPT) $(SWIG_GEN_FLAGS) -outdir swig_python -o $@ $< && \ @@ -60,20 +52,6 @@ clean-local: clean-swig-python endif # USE_SWIG_PYTHON if RUN_PYTHON_TESTS -# Python requires the shared library to be named _wallycore.so -# for 'import' to work. -if IS_OSX -platform_dso_ext = dylib -else -if IS_MINGW -platform_dso_ext = dll -else -platform_dso_ext = so -endif # IS_MINGW -endif # IS_OSX -.libs/_wallycore.so: .libs/libwallycore.$(platform_dso_ext) - $(AM_V_at)ln -sfn libwallycore.$(platform_dso_ext) $@ -PYTHON_TEST_DEPS = .libs/_wallycore.so PYTHON_TEST = PYTHONDONTWRITEBYTECODE=1 $(PYTHON) endif @@ -225,31 +203,26 @@ endif TESTS = noinst_PROGRAMS = + if RUN_TESTS TESTS += test_bech32 noinst_PROGRAMS += test_bech32 test_bech32_SOURCES = ctest/test_bech32.c test_bech32_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) test_bech32_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@ -if PYTHON_MANYLINUX -test_bech32_LDADD += $(PYTHON_LIBS) -endif + TESTS += test_psbt noinst_PROGRAMS += test_psbt test_psbt_SOURCES = ctest/test_psbt.c ccan/ccan/str/hex/hex.c test_psbt_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) -I$(srcdir)/ccan test_psbt_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@ -if PYTHON_MANYLINUX -test_psbt_LDADD += $(PYTHON_LIBS) -endif + TESTS += test_psbt_limits noinst_PROGRAMS += test_psbt_limits test_psbt_limits_SOURCES = ctest/test_psbt_limits.c ccan/ccan/str/hex/hex.c test_psbt_limits_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) -I$(srcdir)/ccan test_psbt_limits_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@ -if PYTHON_MANYLINUX -test_psbt_limits_LDADD += $(PYTHON_LIBS) -endif + if USE_PTHREAD TESTS += test_clear noinst_PROGRAMS += test_clear @@ -257,50 +230,39 @@ test_clear_SOURCES = ctest/test_clear.c test_clear_CFLAGS = -I$(top_srcdir)/include $(PTHREAD_CFLAGS) $(AM_CFLAGS) $(NOOPT_CFLAGS) $(NOBUILTIN_CFLAGS) test_clear_LIBS = $(PTHREAD_LIBS) test_clear_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@ -if PYTHON_MANYLINUX -test_clear_LDADD += $(PYTHON_LIBS) -endif endif + TESTS += test_coinselection noinst_PROGRAMS += test_coinselection test_coinselection_SOURCES = ctest/test_coinselection.c test_coinselection_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) test_coinselection_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@ -if PYTHON_MANYLINUX -test_coinselection_LDADD += $(PYTHON_LIBS) -endif + TESTS += test_tx noinst_PROGRAMS += test_tx test_tx_SOURCES = ctest/test_tx.c test_tx_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) test_tx_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@ -if PYTHON_MANYLINUX -test_tx_LDADD += $(PYTHON_LIBS) -endif + TESTS += test_descriptor noinst_PROGRAMS += test_descriptor test_descriptor_SOURCES = ctest/test_descriptor.c test_descriptor_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) test_descriptor_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@ -if PYTHON_MANYLINUX -test_descriptor_LDADD += $(PYTHON_LIBS) -endif + if BUILD_ELEMENTS TESTS += test_elements_tx noinst_PROGRAMS += test_elements_tx test_elements_tx_SOURCES = ctest/test_elements_tx.c test_elements_tx_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) test_elements_tx_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@ -if PYTHON_MANYLINUX -test_elements_tx_LDADD += $(PYTHON_LIBS) -endif endif check-local: check-libwallycore check-swig-java check-swig-python $(AM_V_at)! grep '^int ' $(top_srcdir)/include/*.h # Missing WALLY_CORE_API if RUN_JAVA_TESTS -check-swig-java: $(SWIG_JAVA_TEST_DEPS) .libs/libwallycore.$(platform_dso_ext) +check-swig-java: $(SWIG_JAVA_TEST_DEPS) libwallycore.la $(AM_V_at)! grep 'native int wally_' $(sjs)/$(cblw)/Wally.java # Unwrapped Java calls $(AM_V_at)! grep 'native Object wally_' $(sjs)/$(cblw)/Wally.java # Unwrapped Java calls if BUILD_ELEMENTS @@ -319,7 +281,7 @@ endif # RUN_JAVA_TESTS if SHARED_BUILD_ENABLED if RUN_PYTHON_TESTS -check-libwallycore: $(PYTHON_TEST_DEPS) +check-libwallycore: $(AM_V_at)$(PYTHON_TEST) test/test_address.py $(AM_V_at)$(PYTHON_TEST) test/test_aes.py $(AM_V_at)$(PYTHON_TEST) test/test_anti_exfil.py @@ -356,7 +318,7 @@ if BUILD_ELEMENTS endif if USE_SWIG_PYTHON -check-swig-python: check-libwallycore check-swig-java +check-swig-python: $(AM_V_at)rm -rf $(top_builddir)/venv $(AM_V_at)$(PYTHON) -m virtualenv $(top_builddir)/venv $(AM_V_at)$(top_builddir)/venv/bin/python -m pip install $(top_srcdir)