bpo-45800: Move pyexpat build setup into configure (GH-29547)

Makefile.pre.in

@@ -537,6 +537,33 @@ LIBMPDEC_HEADERS= \
 
 LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
 
+##########################################################################
+# pyexpat's expat library
+
+LIBEXPAT_OBJS= \
+		Modules/expat/xmlparse.o \
+		Modules/expat/xmlrole.o \
+		Modules/expat/xmltok.o
+
+LIBEXPAT_HEADERS= \
+		Modules/expat/ascii.h \
+		Modules/expat/asciitab.h \
+		Modules/expat/expat.h \
+		Modules/expat/expat_config.h \
+		Modules/expat/expat_external.h \
+		Modules/expat/iasciitab.h \
+		Modules/expat/internal.h \
+		Modules/expat/latin1tab.h \
+		Modules/expat/nametab.h \
+		Modules/expat/pyexpatns.h \
+		Modules/expat/siphash.h \
+		Modules/expat/utf8tab.h \
+		Modules/expat/xmlrole.h \
+		Modules/expat/xmltok.h \
+		Modules/expat/xmltok_impl.h
+
+LIBEXPAT_A= Modules/expat/libexpat.a
+
 #########################################################################
 # Rules
 
@@ -688,7 +715,7 @@ $(srcdir)/Modules/_blake2/blake2s_impl.c: $(srcdir)/Modules/_blake2/blake2b_impl
 # -s, --silent or --quiet is always the first char.
 # Under BSD make, MAKEFLAGS might be " -s -v x=y".
 # Ignore macros passed by GNU make, passed after --
-sharedmods: $(BUILDPYTHON) pybuilddir.txt @LIBMPDEC_INTERNAL@
+sharedmods: $(BUILDPYTHON) pybuilddir.txt @LIBMPDEC_INTERNAL@ @LIBEXPAT_INTERNAL@
 	@case "`echo X $$MAKEFLAGS | sed 's/^X //;s/ -- .*//'`" in \
 	    *\ -s*|s*) quiet="-q";; \
 	    *) quiet="";; \
@@ -828,6 +855,27 @@ $(LIBMPDEC_A): $(LIBMPDEC_OBJS)
 	-rm -f $@
 	$(AR) $(ARFLAGS) $@ $(LIBMPDEC_OBJS)
 
+##########################################################################
+# Build static libexpat.a
+LIBEXPAT_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBEXPAT_CFLAGS@
+
+# for setup.py
+EXPAT_CFLAGS=@LIBEXPAT_CFLAGS@
+EXPAT_LDFLAGS=@LIBEXPAT_LDFLAGS@
+
+Modules/expat/xmlparse.o: $(srcdir)/Modules/expat/xmlparse.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS)
+	$(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmlparse.c
+
+Modules/expat/xmlrole.o: $(srcdir)/Modules/expat/xmlrole.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS)
+	$(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmlrole.c
+
+Modules/expat/xmltok.o: $(srcdir)/Modules/expat/xmltok.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS)
+	$(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmltok.c
+
+$(LIBEXPAT_A): $(LIBEXPAT_OBJS)
+	-rm -f $@
+	$(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS)
+
 # create relative links from build/lib.platform/egg.so to Modules/egg.so
 # pybuilddir.txt is created too late. We cannot use it in Makefile
 # targets. ln --relative is not portable.
@@ -2407,12 +2455,12 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
 
 MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h
 MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h
-MODULE_PYEXPAT_DEPS=$(srcdir)/Modules/expat/ascii.h $(srcdir)/Modules/expat/asciitab.h $(srcdir)/Modules/expat/expat.h $(srcdir)/Modules/expat/expat_config.h $(srcdir)/Modules/expat/expat_external.h $(srcdir)/Modules/expat/internal.h $(srcdir)/Modules/expat/latin1tab.h $(srcdir)/Modules/expat/utf8tab.h $(srcdir)/Modules/expat/xmlrole.h $(srcdir)/Modules/expat/xmltok.h $(srcdir)/Modules/expat/xmltok_impl.h
+MODULE_PYEXPAT_DEPS=$(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@
 MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h
 MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-dispatch.c $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2-kat.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b-test.c $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2bp-test.c $(srcdir)/Modules/_blake2/impl/blake2bp.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s-test.c $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/impl/blake2sp-test.c $(srcdir)/Modules/_blake2/impl/blake2sp.c $(srcdir)/Modules/hashlib.h
 MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h
 MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h $(LIBMPDEC_HEADERS) @LIBMPDEC_INTERNAL@
-MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/expat/ascii.h $(srcdir)/Modules/expat/asciitab.h $(srcdir)/Modules/expat/expat.h $(srcdir)/Modules/expat/expat_config.h $(srcdir)/Modules/expat/expat_external.h $(srcdir)/Modules/expat/internal.h $(srcdir)/Modules/expat/latin1tab.h $(srcdir)/Modules/expat/utf8tab.h $(srcdir)/Modules/expat/xmlparse.c $(srcdir)/Modules/expat/xmlrole.c $(srcdir)/Modules/expat/xmlrole.h $(srcdir)/Modules/expat/xmltok.c $(srcdir)/Modules/expat/xmltok.h $(srcdir)/Modules/expat/xmltok_impl.h $(srcdir)/Modules/pyexpat.c
+MODULE__ELEMENTTREE_DEPS=$(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@
 MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
 MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
 MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h

Misc/NEWS.d/next/Build/2021-11-13-16-40-05.bpo-45800.5Hz6nr.rst

@@ -0,0 +1,2 @@
+Settings for :mod:`pyexpat` C extension are now detected by ``configure``.
+The bundled ``expat`` library is built in ``Makefile``.

Modules/Setup

@@ -172,8 +172,8 @@ time timemodule.c
 #select selectmodule.c
 
 # XML
-#_elementtree -I$(srcdir)/Modules/expat _elementtree.c
-#pyexpat -I$(srcdir)/Modules/expat  expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c
+#_elementtree _elementtree.c $(EXPAT_CFLAGS)
+#pyexpat pyexpat.c $(EXPAT_CFLAGS) $(EXPAT_LDFLAGS)
 
 # hashing builtins
 #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c

configure

@@ -664,6 +664,9 @@ LIBMPDEC_INTERNAL
 LIBMPDEC_LDFLAGS
 LIBMPDEC_CFLAGS
 LIBFFI_INCLUDEDIR
+LIBEXPAT_INTERNAL
+LIBEXPAT_LDFLAGS
+LIBEXPAT_CFLAGS
 TZPATH
 SHLIBS
 CFLAGSFORSHARED
@@ -10715,6 +10718,24 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_system_expat" >&5
 $as_echo "$with_system_expat" >&6; }
 
+if test "x$with_system_expat" = xyes; then :
+
+  LIBEXPAT_CFLAGS=""
+  LIBEXPAT_LDFLAGS="-lexpat"
+  LIBEXPAT_INTERNAL=
+
+else
+
+  LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat"
+  LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)"
+  LIBEXPAT_INTERNAL="\$(LIBEXPAT_A)"
+
+fi
+
+
+
+
+
 # Check for use of the system libffi library
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-ffi" >&5
 $as_echo_n "checking for --with-system-ffi... " >&6; }

configure.ac

@@ -3017,6 +3017,20 @@ AC_ARG_WITH(system_expat,
 
 AC_MSG_RESULT($with_system_expat)
 
+AS_VAR_IF([with_system_expat], [yes], [
+  LIBEXPAT_CFLAGS=""
+  LIBEXPAT_LDFLAGS="-lexpat"
+  LIBEXPAT_INTERNAL=
+], [
+  LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat"
+  LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)"
+  LIBEXPAT_INTERNAL="\$(LIBEXPAT_A)"
+])
+
+AC_SUBST([LIBEXPAT_CFLAGS])
+AC_SUBST([LIBEXPAT_LDFLAGS])
+AC_SUBST([LIBEXPAT_INTERNAL])
+
 # Check for use of the system libffi library
 AC_MSG_CHECKING(for --with-system-ffi)
 AC_ARG_WITH(system_ffi,

setup.py

@@ -364,6 +364,57 @@ def find_module_file(module, dirlist):
     return os.path.join(dirs[0], module)
 
 
+def parse_cflags(flags):
+    """Parse a string with compiler flags (-I, -D, -U, extra)
+
+    Distutils appends extra args to the compiler arguments. Some flags like
+    -I must appear earlier. Otherwise the pre-processor picks up files
+    from system inclue directories.
+    """
+    include_dirs = []
+    define_macros = []
+    undef_macros = []
+    extra_compile_args = []
+    if flags is not None:
+        # shlex.split(None) reads from stdin
+        for token in shlex.split(flags):
+            switch = token[0:2]
+            value = token[2:]
+            if switch == '-I':
+                include_dirs.append(value)
+            elif switch == '-D':
+                key, _, val = value.partition("=")
+                if not val:
+                    val = None
+                define_macros.append((key, val))
+            elif switch == '-U':
+                undef_macros.append(value)
+            else:
+                extra_compile_args.append(token)
+
+    return include_dirs, define_macros, undef_macros, extra_compile_args
+
+
+def parse_ldflags(flags):
+    """Parse a string with linker flags (-L, -l, extra)"""
+    library_dirs = []
+    libraries = []
+    extra_link_args = []
+    if flags is not None:
+        # shlex.split(None) reads from stdin
+        for token in shlex.split(flags):
+            switch = token[0:2]
+            value = token[2:]
+            if switch == '-L':
+                library_dirs.append(value)
+            elif switch == '-l':
+                libraries.append(value)
+            else:
+                extra_link_args.append(token)
+
+    return library_dirs, libraries, extra_link_args
+
+
 class PyBuildExt(build_ext):
 
     def __init__(self, dist):
@@ -1469,59 +1520,39 @@ def detect_expat_elementtree(self):
         #
         # More information on Expat can be found at www.libexpat.org.
         #
-        if '--with-system-expat' in sysconfig.get_config_var("CONFIG_ARGS"):
-            expat_inc = []
-            extra_compile_args = []
-            expat_lib = ['expat']
-            expat_sources = []
-            expat_depends = []
-        else:
-            expat_inc = [os.path.join(self.srcdir, 'Modules', 'expat')]
-            extra_compile_args = []
-            # bpo-44394: libexpat uses isnan() of math.h and needs linkage
-            # against the libm
-            expat_lib = ['m']
-            expat_sources = ['expat/xmlparse.c',
-                             'expat/xmlrole.c',
-                             'expat/xmltok.c']
-            expat_depends = ['expat/ascii.h',
-                             'expat/asciitab.h',
-                             'expat/expat.h',
-                             'expat/expat_config.h',
-                             'expat/expat_external.h',
-                             'expat/internal.h',
-                             'expat/latin1tab.h',
-                             'expat/utf8tab.h',
-                             'expat/xmlrole.h',
-                             'expat/xmltok.h',
-                             'expat/xmltok_impl.h'
-                             ]
-
-            cc = sysconfig.get_config_var('CC').split()[0]
-            ret = run_command(
-                      '"%s" -Werror -Wno-unreachable-code -E -xc /dev/null >/dev/null 2>&1' % cc)
-            if ret == 0:
-                extra_compile_args.append('-Wno-unreachable-code')
+        cflags = parse_cflags(sysconfig.get_config_var("EXPAT_CFLAGS"))
+        include_dirs, define_macros, undef_macros, extra_compile_args = cflags
+        # ldflags includes either system libexpat or full path to
+        # our static libexpat.a.
+        ldflags = parse_ldflags(sysconfig.get_config_var("EXPAT_LDFLAGS"))
+        library_dirs, libraries, extra_link_args = ldflags
+
+        expat_depends = []
+        libexpat_a = sysconfig.get_config_var("LIBEXPAT_A")
+        if libexpat_a:
+            expat_depends.append(libexpat_a)
 
         self.add(Extension('pyexpat',
+                           include_dirs=include_dirs,
+                           define_macros=define_macros,
+                           undef_macros=undef_macros,
                            extra_compile_args=extra_compile_args,
-                           include_dirs=expat_inc,
-                           libraries=expat_lib,
-                           sources=['pyexpat.c'] + expat_sources,
+                           library_dirs=library_dirs,
+                           libraries=libraries,
+                           extra_link_args=extra_link_args,
+                           sources=['pyexpat.c'],
                            depends=expat_depends))
 
         # Fredrik Lundh's cElementTree module.  Note that this also
         # uses expat (via the CAPI hook in pyexpat).
-
-        if os.path.isfile(os.path.join(self.srcdir, 'Modules', '_elementtree.c')):
-            self.add(Extension('_elementtree',
-                               include_dirs=expat_inc,
-                               libraries=expat_lib,
-                               sources=['_elementtree.c'],
-                               depends=['pyexpat.c', *expat_sources,
-                                        *expat_depends]))
-        else:
-            self.missing.append('_elementtree')
+        self.add(Extension('_elementtree',
+                           include_dirs=include_dirs,
+                           define_macros=define_macros,
+                           undef_macros=undef_macros,
+                           extra_compile_args=extra_compile_args,
+                           # no EXPAT_LDFLAGS
+                           sources=['_elementtree.c'],
+                           depends=['pyexpat.c', *expat_depends]))
 
     def detect_multibytecodecs(self):
         # Hye-Shik Chang's CJKCodecs modules.
@@ -2019,24 +2050,27 @@ def detect_decimal(self):
         # Stefan Krah's _decimal module
         sources = ['_decimal/_decimal.c']
         depends = ['_decimal/docstrings.h']
-        define_macros = []
-
-        cflags = sysconfig.get_config_var("DECIMAL_CFLAGS")
-        extra_compile_args = shlex.split(cflags) if cflags else None
+
+        cflags = parse_cflags(sysconfig.get_config_var("DECIMAL_CFLAGS"))
+        include_dirs, define_macros, undef_macros, extra_compile_args = cflags
         # ldflags includes either system libmpdec or full path to
         # our static libmpdec.a.
-        ldflags = sysconfig.get_config_var("DECIMAL_LDFLAGS")
-        extra_link_args = shlex.split(ldflags) if ldflags else None
-        
+        ldflags = parse_ldflags(sysconfig.get_config_var("DECIMAL_LDFLAGS"))
+        library_dirs, libraries, extra_link_args = ldflags
+
         libmpdec_a = sysconfig.get_config_var("LIBMPDEC_A")
         if libmpdec_a:
             depends.append(libmpdec_a)
 
         # Uncomment for extra functionality:
         #define_macros.append(('EXTRA_FUNCTIONALITY', 1))
         self.add(Extension('_decimal',
+                           include_dirs=include_dirs,
                            define_macros=define_macros,
+                           undef_macros=undef_macros,
                            extra_compile_args=extra_compile_args,
+                           library_dirs=library_dirs,
+                           libraries=libraries,
                            extra_link_args=extra_link_args,
                            sources=sources,
                            depends=depends))