Skip to content

Commit 3af290e

Browse files
claudeclaude
committed
Link only the bootstrap modules into the build helper programs
Programs/_freeze_module and _bootstrap_python were linked with $(LIBRARY_OBJS_OMIT_FROZEN), i.e. every static module from all Setup files (Setup.local, Setup.stdlib with MODULE_BUILDTYPE=static, ...). That makes it impossible for a builtin module to depend on anything the helpers produce: a frozen module header, Modules/getpath.o, or generated sources. A Setup.local module including a header generated by $(FREEZE_MODULE) is a dependency cycle ("Circular ... dependency dropped"), and one referencing a getpath.o symbol fails to link, since _freeze_module links Modules/getpath_noop.o instead. CPython itself needed a workaround for the latter: a stub of _Py_Get_Getpath_CodeObject() in getpath_noop.c for _testinternalcapi in MODULE_BUILDTYPE=static builds. Mirror what PCbuild/_freeze_module.vcxproj already does on Windows (a fixed source list plus PC/config_minimal.c): link the helpers with the core objects plus the modules of Modules/Setup.bootstrap only. * makesetup: new "-b Setup" option marking a Setup file as the bootstrap modules. Their objects and libraries are substituted for _BOOTSTRAP_MODOBJS_ / _BOOTSTRAP_MODLIBS_, and a second table, config_bootstrap.c, is generated with only those modules. A bootstrap module overridden by an earlier rule (Setup.local) keeps the overriding static module for the helpers too. * Makefile.pre.in: BOOTSTRAP_LIBRARY_OBJS (core + Modules/config_bootstrap.o + $(BOOTSTRAP_MODOBJS)) replaces LIBRARY_OBJS_OMIT_FROZEN in the _freeze_module and _bootstrap_python links. getbuildinfo gets a separate object for the helpers, as its prerequisites include $(MODOBJS). * configure.ac / configure: pass -b Modules/Setup.bootstrap and move config_bootstrap.c into Modules/ (configure hand-edited in sync). * Tools/c-analyzer: whitelist _PyImport_Inittab in the generated Modules/config_bootstrap.c, as for Modules/config.c. The final libpython/python link is unchanged. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YD8kPnXaAMJZk2LvSHPXYA
1 parent 7b12a41 commit 3af290e

6 files changed

Lines changed: 138 additions & 37 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ Modules/Setup.config
102102
Modules/Setup.local
103103
Modules/Setup.stdlib
104104
Modules/config.c
105+
Modules/config_bootstrap.c
105106
Modules/ld_so_aix
106107
Programs/_freeze_module
107108
Programs/_testembed

‎Makefile.pre.in‎

Lines changed: 54 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ MODSHARED_NAMES= _MODSHARED_NAMES_
2525
MODDISABLED_NAMES= _MODDISABLED_NAMES_
2626
MODOBJS= _MODOBJS_
2727
MODLIBS= _MODLIBS_
28+
# Objects and libraries of the bootstrap modules (Modules/Setup.bootstrap)
29+
# only. These are the only static modules linked into the build helper
30+
# programs (Programs/_freeze_module, _bootstrap_python), so that other
31+
# builtin modules (Modules/Setup.local, MODULE_BUILDTYPE=static) may depend
32+
# on products of those helpers (frozen modules, getpath).
33+
BOOTSTRAP_MODOBJS= _BOOTSTRAP_MODOBJS_
34+
BOOTSTRAP_MODLIBS= _BOOTSTRAP_MODLIBS_
2835

2936
# === Variables set by configure
3037
VERSION= @VERSION@
@@ -359,11 +366,14 @@ COVERAGE_REPORT_OPTIONS=--rc lcov_branch_coverage=1 --branch-coverage --title "C
359366

360367
##########################################################################
361368
# Modules
362-
MODULE_OBJS= \
363-
Modules/config.o \
369+
MODULE_OBJS_COMMON= \
364370
Modules/main.o \
365371
Modules/gcmodule.o
366372

373+
MODULE_OBJS= \
374+
Modules/config.o \
375+
$(MODULE_OBJS_COMMON)
376+
367377
IO_H= Modules/_io/_iomodule.h
368378

369379
IO_OBJS= \
@@ -614,6 +624,19 @@ LIBRARY_OBJS= \
614624
Modules/getpath.o \
615625
Python/frozen.o
616626

627+
# objects that get linked into the build helper programs
628+
# (Programs/_freeze_module, _bootstrap_python): the core, plus the bootstrap
629+
# modules only (see BOOTSTRAP_MODOBJS). Does not include frozen modules nor
630+
# getpath, which the helpers are used to build.
631+
BOOTSTRAP_LIBRARY_OBJS= \
632+
Modules/getbuildinfo_bootstrap.o \
633+
$(PARSER_OBJS) \
634+
$(OBJECT_OBJS) \
635+
$(PYTHON_OBJS) \
636+
Modules/config_bootstrap.o \
637+
$(MODULE_OBJS_COMMON) \
638+
$(BOOTSTRAP_MODOBJS)
639+
617640
LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@
618641

619642
##########################################################################
@@ -1613,7 +1636,7 @@ Modules/Setup.bootstrap: $(srcdir)/Modules/Setup.bootstrap.in config.status
16131636
Modules/Setup.stdlib: $(srcdir)/Modules/Setup.stdlib.in config.status
16141637
./config.status $@
16151638

1616-
Makefile Modules/config.c: Makefile.pre \
1639+
Makefile Modules/config.c Modules/config_bootstrap.c: Makefile.pre \
16171640
$(srcdir)/Modules/config.c.in \
16181641
$(MAKESETUP) \
16191642
$(srcdir)/Modules/Setup \
@@ -1624,9 +1647,9 @@ Makefile Modules/config.c: Makefile.pre \
16241647
-s Modules \
16251648
Modules/Setup.local \
16261649
Modules/Setup.stdlib \
1627-
Modules/Setup.bootstrap \
1650+
-b Modules/Setup.bootstrap \
16281651
$(srcdir)/Modules/Setup
1629-
@mv config.c Modules
1652+
@mv config.c config_bootstrap.c Modules
16301653
@echo "The Makefile was updated, you may need to re-run make."
16311654

16321655
.PHONY: regen-test-frozenmain
@@ -1661,9 +1684,9 @@ BOOTSTRAP_HEADERS = \
16611684

16621685
Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS)
16631686

1664-
_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
1665-
$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
1666-
Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS)
1687+
_bootstrap_python: $(BOOTSTRAP_LIBRARY_OBJS) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
1688+
$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(BOOTSTRAP_LIBRARY_OBJS) \
1689+
Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(BOOTSTRAP_MODLIBS) $(SYSLIBS)
16671690
# Dummy pybuilddir.txt is needed for _bootstrap_python to be runnable
16681691
@echo "none" > ./pybuilddir.txt
16691692

@@ -1681,6 +1704,9 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu
16811704
#
16821705
# Standard build process:
16831706
# 1) compile minimal core objects for Py_Compile*() and PyMarshal_Write*().
1707+
# Only the bootstrap modules (Modules/Setup.bootstrap) are part of these
1708+
# objects (see BOOTSTRAP_LIBRARY_OBJS), so other builtin modules may
1709+
# depend on the output of the following steps.
16841710
# 2) build Programs/_freeze_module binary.
16851711
# 3) create frozen module headers for importlib and getpath.
16861712
# 4) build _bootstrap_python binary.
@@ -1753,8 +1779,8 @@ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
17531779

17541780
Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile
17551781

1756-
Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN)
1757-
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
1782+
Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(BOOTSTRAP_LIBRARY_OBJS)
1783+
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(BOOTSTRAP_LIBRARY_OBJS) $(LIBS) $(BOOTSTRAP_MODLIBS) $(SYSLIBS)
17581784

17591785
# We manually freeze getpath.py rather than through freeze_modules
17601786
Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
@@ -1907,11 +1933,24 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
19071933
$(MODOBJS) \
19081934
$(DTRACE_OBJS) \
19091935
$(srcdir)/Modules/getbuildinfo.c
1910-
$(CC) -c $(PY_CORE_CFLAGS) \
1936+
$(CC) -c $(PY_CORE_CFLAGS) $(GETBUILDINFO_CFLAGS) \
1937+
-o $@ $(srcdir)/Modules/getbuildinfo.c
1938+
1939+
# Separate object for the build helper programs: it must not depend on
1940+
# $(MODOBJS), which may in turn depend on the helpers' output.
1941+
Modules/getbuildinfo_bootstrap.o: $(PARSER_OBJS) \
1942+
$(OBJECT_OBJS) \
1943+
$(PYTHON_OBJS) \
1944+
$(MODULE_OBJS_COMMON) \
1945+
$(BOOTSTRAP_MODOBJS) \
1946+
$(srcdir)/Modules/getbuildinfo.c
1947+
$(CC) -c $(PY_CORE_CFLAGS) $(GETBUILDINFO_CFLAGS) \
1948+
-o $@ $(srcdir)/Modules/getbuildinfo.c
1949+
1950+
GETBUILDINFO_CFLAGS= \
19111951
-DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \
19121952
-DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \
1913-
-DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
1914-
-o $@ $(srcdir)/Modules/getbuildinfo.c
1953+
-DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\""
19151954

19161955
Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h Makefile $(PYTHON_HEADERS)
19171956
$(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \
@@ -2279,7 +2318,7 @@ regen-slots: Python/slots.toml
22792318
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_slots.py \
22802319
--generate-all
22812320

2282-
$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o $(PLATFORM_MAIN_OBJS): $(PYTHON_HEADERS)
2321+
$(LIBRARY_OBJS) $(MODOBJS) Modules/config_bootstrap.o Programs/python.o $(PLATFORM_MAIN_OBJS): $(PYTHON_HEADERS)
22832322

22842323

22852324
######################################################################
@@ -3365,7 +3404,7 @@ clean: clean-profile clean-jit-stencils
33653404
clobber: clean
33663405
-rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \
33673406
tags TAGS \
3368-
config.cache config.log pyconfig.h Modules/config.c
3407+
config.cache config.log pyconfig.h Modules/config.c Modules/config_bootstrap.c
33693408
-rm -rf build platform
33703409
-rm -rf $(PYTHONFRAMEWORKDIR)
33713410
-rm -rf Platforms/Apple/iOS/Frameworks

‎Modules/makesetup‎

Lines changed: 78 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,22 @@ set -e
44
# Convert templates into Makefile and config.c, based on the module
55
# definitions found in the file Setup.
66
#
7-
# Usage: makesetup [-s dir] [-c file] [-m file] [Setup] ... [-n [Setup] ...]
7+
# Usage: makesetup [-s dir] [-c file] [-m file] [Setup] ... [-b Setup] ...
8+
# [-n [Setup] ...]
89
#
910
# Options:
1011
# -s directory: alternative source directory (default .)
1112
# -l directory: library source directory (default derived from $0)
1213
# -c file: alternative config.c template (default $libdir/config.c.in)
13-
# -c -: don't write config.c
14+
# -c -: don't write config.c (nor config_bootstrap.c)
1415
# -m file: alternative Makefile template (default ./Makefile.pre)
1516
# -m -: don't write Makefile
1617
#
1718
# Remaining arguments are one or more Setup files (default ./Setup).
19+
# The Setup file following a -b option holds the *bootstrap* modules:
20+
# the static modules that the build helper programs (_freeze_module,
21+
# _bootstrap_python) are linked with. They are also regular modules of
22+
# the final build.
1823
# Setup files after a -n option are used for their variables, modules
1924
# and libraries but not for their .o files.
2025
#
@@ -28,6 +33,9 @@ set -e
2833
# + insert 'extern PyObject* PyInit_<module>(void);' before MARKER 1
2934
# + insert '{"<module>", PyInit_<module>},' before MARKER 2
3035
#
36+
# Copying config.c.in to config_bootstrap.c (only with a -b Setup file):
37+
# - same as config.c, but only for the modules of the -b Setup file
38+
#
3139
# Copying Makefile.pre to Makefile:
3240
# - insert an identifying comment at the start
3341
# - replace _MODBUILT_NAMES_ by the list of *static* and *shared* modules
@@ -37,6 +45,8 @@ set -e
3745
# - replace _MODOBJS_ by the list of objects from Setup (except for
3846
# Setup files after a -n option)
3947
# - replace _MODLIBS_ by the list of libraries from Setup
48+
# - replace _BOOTSTRAP_MODOBJS_ and _BOOTSTRAP_MODLIBS_ by the objects and
49+
# libraries of the -b Setup file
4050
# - for each object file mentioned in Setup, append a rule
4151
# '<file>.o: <file>.c; <build commands>' to the end of the Makefile
4252
# - for each module mentioned in Setup, append a rule
@@ -47,13 +57,14 @@ set -e
4757
# Loop over command line options
4858
usage='
4959
usage: makesetup [-s srcdir] [-l libdir] [-c config.c.in] [-m Makefile.pre]
50-
[Setup] ... [-n [Setup] ...]'
60+
[Setup] ... [-b Setup] ... [-n [Setup] ...]'
5161
srcdir='.'
5262
libdir=''
5363
config=''
5464
makepre=''
5565
noobjects=''
5666
doconfig=yes
67+
bootstrap=
5768
while :
5869
do
5970
case $1 in
@@ -93,7 +104,8 @@ do
93104
echo '' # Add a linebreak so we don't choke on files missing EOL.
94105
case $i in
95106
-n) echo '*noobjects*';;
96-
*) echo '*doconfig*'; cat "$i";;
107+
-b) echo '*bootstrap*';;
108+
*) echo '*doconfig*'; cat "$i"; echo ''; echo '*nobootstrap*';;
97109
esac
98110
done |
99111
sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
@@ -114,6 +126,10 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
114126
LIBS=
115127
LOCALLIBS=
116128
BASELIBS=
129+
BOOTMODS=
130+
BOOTOBJS=
131+
BOOTLIBS=
132+
havebootstrap=
117133
while read line
118134
do
119135
# to handle backslashes for sh's that don't automatically
@@ -136,6 +152,8 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
136152
noobjects=yes;
137153
continue;;
138154
'*doconfig*') doconfig=yes; continue;;
155+
'*bootstrap*') bootstrap=yes; havebootstrap=yes; continue;;
156+
'*nobootstrap*') bootstrap=; continue;;
139157
'*static*') doconfig=yes; continue;;
140158
'*noconfig*') doconfig=no; continue;;
141159
'*shared*') doconfig=no; continue;;
@@ -204,6 +222,18 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
204222
# Detected multiple rules for a module, first rule wins. This
205223
# allows users to disable modules in Setup.local.
206224
echo 1>&2 "maksetup: '$mod' was handled by previous rule."
225+
case $bootstrap in
226+
yes)
227+
# This module is in the bootstrap Setup file, but an
228+
# earlier Setup file (e.g. Setup.local) already defined
229+
# it, and that definition won. Give the build helpers
230+
# that earlier definition, so they use the same module
231+
# as the final interpreter.
232+
eval "BOOTMODS=\"\$BOOTMODS \${STATICMOD_$mod}\""
233+
eval "BOOTOBJS=\"\$BOOTOBJS \${STATICOBJS_$mod}\""
234+
eval "BOOTLIBS=\"\$BOOTLIBS \${STATICLIBS_$mod}\""
235+
;;
236+
esac
207237
continue 2;;
208238
esac
209239
CONFIGURED="$CONFIGURED,${mod},"
@@ -213,6 +243,12 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
213243
LIBS="$LIBS $libs"
214244
MODS="$MODS $mods"
215245
BUILT="$BUILT $mods"
246+
case $bootstrap in
247+
yes)
248+
BOOTLIBS="$BOOTLIBS $libs"
249+
BOOTMODS="$BOOTMODS $mods"
250+
;;
251+
esac
216252
;;
217253
no)
218254
BUILT="$BUILT $mods"
@@ -265,7 +301,21 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
265301
echo "$rule" >>$rulesf
266302
done
267303
case $doconfig in
268-
yes) OBJS="$OBJS $objs";;
304+
yes)
305+
OBJS="$OBJS $objs"
306+
case $bootstrap in
307+
yes) BOOTOBJS="$BOOTOBJS $objs";;
308+
esac
309+
# Record each static module's objects and libs by name, so
310+
# the bootstrap handling above can find a definition that
311+
# won over a later line in the bootstrap Setup file.
312+
for mod in $mods
313+
do
314+
eval "STATICMOD_$mod='$mod'"
315+
eval "STATICOBJS_$mod='$objs'"
316+
eval "STATICLIBS_$mod='$libs'"
317+
done
318+
;;
269319
esac
270320
for mod in $mods
271321
do
@@ -295,24 +345,31 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
295345
DEFS="BASEMODLIBS=$BASELIBS$NL$DEFS"
296346
DEFS="LOCALMODLIBS=$LOCALLIBS$NL$DEFS"
297347

298-
EXTDECLS=
299-
INITBITS=
300-
for mod in $MODS
301-
do
302-
EXTDECLS="${EXTDECLS}extern PyObject* PyInit_$mod(void);$NL"
303-
INITBITS="${INITBITS} {\"$mod\", PyInit_$mod},$NL"
304-
done
305-
306-
307-
case $config in
308-
-) ;;
309-
*) sed -e "
348+
# Generate a config.c from the template for the given modules
349+
genconfig() {
350+
out=$1; shift
351+
EXTDECLS=
352+
INITBITS=
353+
for mod
354+
do
355+
EXTDECLS="${EXTDECLS}extern PyObject* PyInit_$mod(void);$NL"
356+
INITBITS="${INITBITS} {\"$mod\", PyInit_$mod},$NL"
357+
done
358+
sed -e "
310359
1i$NL/* Generated automatically from $config by makesetup. */
311360
/MARKER 1/i$NL$EXTDECLS
312361
313362
/MARKER 2/i$NL$INITBITS
314363
315-
" $config >config.c
364+
" $config >$out
365+
}
366+
367+
case $config in
368+
-) ;;
369+
*) genconfig config.c $MODS
370+
case $havebootstrap in
371+
yes) genconfig config_bootstrap.c $BOOTMODS;;
372+
esac
316373
;;
317374
esac
318375

@@ -331,6 +388,9 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
331388
echo "s%_MODBUILT_NAMES_%$BUILT%" >>$sedf
332389
echo "s%_MODSHARED_NAMES_%$BUILT_SHARED%" >>$sedf
333390
echo "s%_MODDISABLED_NAMES_%$DISABLED%" >>$sedf
391+
# _BOOTSTRAP_* first: _MODOBJS_ is a substring of _BOOTSTRAP_MODOBJS_
392+
echo "s%_BOOTSTRAP_MODOBJS_%$BOOTOBJS%" >>$sedf
393+
echo "s%_BOOTSTRAP_MODLIBS_%$BOOTLIBS%" >>$sedf
334394
echo "s%_MODOBJS_%$OBJS%" >>$sedf
335395
echo "s%_MODLIBS_%$LIBS%" >>$sedf
336396
echo "/Definitions added by makesetup/r $sedr" >>$sedf

‎Tools/c-analyzer/cpython/ignored.tsv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ Modules/cmathmodule.c - sinh_special_values -
314314
Modules/cmathmodule.c - sqrt_special_values -
315315
Modules/cmathmodule.c - tanh_special_values -
316316
Modules/config.c - _PyImport_Inittab -
317+
Modules/config_bootstrap.c - _PyImport_Inittab -
317318
Modules/faulthandler.c - faulthandler_handlers -
318319
Modules/getnameinfo.c - gni_afdl -
319320
Modules/hmacmodule.c - py_hmac_static_hinfo -

‎configure‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎configure.ac‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9070,12 +9070,12 @@ fi
90709070
AC_MSG_NOTICE([creating Makefile])
90719071
$SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
90729072
-s Modules \
9073-
Modules/Setup.local Modules/Setup.stdlib Modules/Setup.bootstrap $srcdir/Modules/Setup
9073+
Modules/Setup.local Modules/Setup.stdlib -b Modules/Setup.bootstrap $srcdir/Modules/Setup
90749074
if test $? -ne 0; then
90759075
AC_MSG_ERROR([makesetup failed])
90769076
fi
90779077

9078-
mv config.c Modules
9078+
mv config.c config_bootstrap.c Modules
90799079

90809080
if test -z "$PKG_CONFIG"; then
90819081
AC_MSG_WARN([pkg-config is missing. Some dependencies may not be detected correctly.])

0 commit comments

Comments
 (0)