nixpkgs/pkgs/development/interpreters/python/cpython/default.nix
Greg Price 480c8d1991 cpython: Optimize dynamic symbol tables, for a 6% speedup.
I took a close look at how Debian builds the Python interpreter,
because I noticed it ran substantially faster than the one in nixpkgs
and I was curious why.

One thing that I found made a material difference in performance was
this pair of linker flags (passed to the compiler):

    -Wl,-O1 -Wl,-Bsymbolic-functions

In other words, effectively the linker gets passed the flags:

    -O1 -Bsymbolic-functions

Doing the same thing in nixpkgs turns out to make the interpreter
run about 6% faster, which is quite a big win for such an easy
change.  So, let's apply it.

---

I had not known there was a `-O1` flag for the *linker*!
But indeed there is.

These flags are unrelated to "link-time optimization" (LTO), despite
the latter's name.  LTO means doing classic compiler optimizations
on the actual code, at the linking step when it becomes possible to
do them with cross-object-file information.  These two flags, by
contrast, cause the linker to make certain optimizations within the
scope of its job as the linker.

Documentation is here, though sparse:
  https://sourceware.org/binutils/docs-2.31/ld/Options.html

The meaning of -O1 was explained in more detail in this LWN article:
  https://lwn.net/Articles/192624/
Apparently it makes the resulting symbol table use a bigger hash
table, so the load factor is smaller and lookups are faster.  Cool.

As for -Bsymbolic-functions, the documentation indicates that it's a
way of saving lookups through the symbol table entirely.  There can
apparently be situations where it changes the behavior of a program,
specifically if the program relies on linker tricks to provide
customization features:
  https://bugs.launchpad.net/ubuntu/+source/xfe/+bug/644645
  https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=637184#35
But I'm pretty sure CPython doesn't permit that kind of trick: you
don't load a shared object that tries to redefine some symbol found
in the interpreter core.

The stronger reason I'm confident using -Bsymbolic-functions is
safe, though, is empirical.  Both Debian and Ubuntu have been
shipping a Python built this way since forever -- it was introduced
for the Python 2.4 and 2.5 in Ubuntu "hardy", and Debian "lenny",
released in 2008 and 2009.  In those 12 years they haven't seen a
need to drop this flag; and I've been unable to locate any reports
of trouble related to it, either on the Web in general or on the
Debian bug tracker.  (There are reports of a handful of other
programs breaking with it, but not Python/CPython.)  So that seems
like about as thorough testing as one could hope for.

---

As for the performance impact: I ran CPython upstream's preferred
benchmark suite, "pyperformance", in the same way as described in
the previous commit.  On top of that commit's change, the results
across the 60 benchmarks in the suite are:

The median is 6% faster.

The middle half (aka interquartile range) is from 4% to 8% faster.

Out of 60 benchmarks, 3 come out slower, by 1-4%.  At the other end,
5 are at least 10% faster, and one is 17% faster.

So, that's quite a material speedup!  I don't know how big the
effect of these flags is for other software; but certainly CPython
tends to do plenty of dynamic linking, as that's how it loads
extension modules, which are ubiquitous in the stdlib as well as
popular third-party libraries.  So perhaps that helps explain why
optimizing the dynamic linker has such an impact.
2020-05-13 21:24:30 -07:00

331 lines
12 KiB
Nix

{ stdenv, fetchurl, fetchpatch
, bzip2
, expat
, libffi
, gdbm
, lzma
, ncurses
, openssl
, readline
, sqlite
, tcl ? null, tk ? null, tix ? null, libX11 ? null, xorgproto ? null, x11Support ? false
, zlib
, self
, configd
, autoreconfHook
, python-setup-hook
, nukeReferences
# For the Python package set
, packageOverrides ? (self: super: {})
, buildPackages
, pythonForBuild ? buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}
, sourceVersion
, sha256
, passthruFun
, bash
, stripConfig ? false
, stripIdlelib ? false
, stripTests ? false
, stripTkinter ? false
, rebuildBytecode ? true
, stripBytecode ? false
}:
assert x11Support -> tcl != null
&& tk != null
&& xorgproto != null
&& libX11 != null;
with stdenv.lib;
let
passthru = passthruFun rec {
inherit self sourceVersion packageOverrides;
implementation = "cpython";
libPrefix = "python${pythonVersion}";
executable = libPrefix;
pythonVersion = with sourceVersion; "${major}.${minor}";
sitePackages = "lib/${libPrefix}/site-packages";
inherit hasDistutilsCxxPatch pythonForBuild;
};
version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
nativeBuildInputs = [
autoreconfHook
nukeReferences
] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
buildPackages.stdenv.cc
pythonForBuild
];
buildInputs = filter (p: p != null) ([
zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ]
++ optionals x11Support [ tcl tk libX11 xorgproto ]
++ optionals stdenv.isDarwin [ configd ]);
hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
inherit pythonForBuild;
pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
"$out/bin/python"
else pythonForBuild.interpreter;
in with passthru; stdenv.mkDerivation {
pname = "python3";
inherit version;
inherit buildInputs nativeBuildInputs;
src = fetchurl {
url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz";
inherit sha256;
};
prePatch = optionalString stdenv.isDarwin ''
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
'' + optionalString (stdenv.isDarwin && x11Support) ''
substituteInPlace setup.py --replace /Library/Frameworks /no-such-path
'';
patches = [
# Disable the use of ldconfig in ctypes.util.find_library (since
# ldconfig doesn't work on NixOS), and don't use
# ctypes.util.find_library during the loading of the uuid module
# (since it will do a futile invocation of gcc (!) to find
# libuuid, slowing down program startup a lot).
(./. + "/${sourceVersion.major}.${sourceVersion.minor}/no-ldconfig.patch")
] ++ optionals stdenv.isLinux [
# Optimize symbol tables for the sake of dynamic linking.
# Significant for Python because of extension modules.
(
if pythonAtLeast "3.8" then
fetchpatch {
url = "https://salsa.debian.org/cpython-team/python3/-/raw/3.8.3rc1-1/debian/patches/link-opt.diff";
sha256 = "0va85318nahnqgydwjs7723h8gx41inbdawdy6v4hiykzgc8s7vs";
}
else if isPy37 then
fetchurl {
url = "https://salsa.debian.org/cpython-team/python3/-/raw/3.7.6-1/debian/patches/link-opt.diff";
sha256 = "1aqvsc0p3sxnfsi8jz7537wl6v95v26ba4nflwvmn5lxlc3y3g13";
}
else if isPy36 then
fetchpatch {
url = "https://salsa.debian.org/cpython-team/python3/-/raw/3.6.8-1/debian/patches/link-opt.diff";
sha256 = "1nhdrgla75ily9gk7xx0crxa7ynqzks0djxk36sa3lgg5w8vjvyr";
}
else
fetchpatch {
url = "https://salsa.debian.org/cpython-team/python3/-/raw/27103a32e/debian/patches/link-opt.diff";
sha256 = "0vp36276ndbrwr7882vg7vjd61c8mv7bqgal6bbh2fimp6zlkdhv";
}
)
] ++ optionals (isPy35 || isPy36) [
# Determinism: Write null timestamps when compiling python files.
./3.5/force_bytecode_determinism.patch
] ++ optionals isPy35 [
# Backports support for LD_LIBRARY_PATH from 3.6
./3.5/ld_library_path.patch
] ++ optionals (isPy37 || isPy38) [
# Fix darwin build https://bugs.python.org/issue34027
./3.7/darwin-libutil.patch
] ++ optionals (pythonOlder "3.8") [
# Backport from CPython 3.8 of a good list of tests to run for PGO.
(
if isPy36 || isPy37 then
./3.6/profile-task.patch
else
./3.5/profile-task.patch
)
] ++ optionals (isPy3k && hasDistutilsCxxPatch) [
# Fix for http://bugs.python.org/issue1222585
# Upstream distutils is calling C compiler to compile C++ code, which
# only works for GCC and Apple Clang. This makes distutils to call C++
# compiler when needed.
(
if isPy35 then
./3.5/python-3.x-distutils-C++.patch
else if isPy37 || isPy38 then
./3.7/python-3.x-distutils-C++.patch
else
fetchpatch {
url = "https://bugs.python.org/file48016/python-3.x-distutils-C++.patch";
sha256 = "1h18lnpx539h5lfxyk379dxwr8m2raigcjixkf133l4xy3f4bzi2";
}
)
];
postPatch = ''
'' + optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
'';
CPPFLAGS = concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs);
LDFLAGS = concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs);
LIBS = "${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED=0;
configureFlags = [
"--enable-optimizations"
"--enable-shared"
"--without-ensurepip"
"--with-system-expat"
"--with-system-ffi"
] ++ optionals (pythonOlder "3.7") [
# This is unconditionally true starting in CPython 3.7.
"--with-threads"
] ++ optionals (sqlite != null && isPy3k) [
"--enable-loadable-sqlite-extensions"
] ++ optionals (openssl != null) [
"--with-openssl=${openssl.dev}"
] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
"ac_cv_buggy_getaddrinfo=no"
# Assume little-endian IEEE 754 floating point when cross compiling
"ac_cv_little_endian_double=yes"
"ac_cv_big_endian_double=no"
"ac_cv_mixed_endian_double=no"
"ac_cv_x87_double_rounding=yes"
"ac_cv_tanh_preserves_zero_sign=yes"
# Generally assume that things are present and work
"ac_cv_posix_semaphores_enabled=yes"
"ac_cv_broken_sem_getvalue=no"
"ac_cv_wchar_t_signed=yes"
"ac_cv_rshift_extends_sign=yes"
"ac_cv_broken_nice=no"
"ac_cv_broken_poll=no"
"ac_cv_working_tzset=yes"
"ac_cv_have_long_long_format=yes"
"ac_cv_have_size_t_format=yes"
"ac_cv_computed_gotos=yes"
"ac_cv_file__dev_ptmx=yes"
"ac_cv_file__dev_ptc=yes"
] ++ optionals stdenv.hostPlatform.isLinux [
# Never even try to use lchmod on linux,
# don't rely on detecting glibc-isms.
"ac_cv_func_lchmod=no"
];
preConfigure = ''
for i in /usr /sw /opt /pkg; do # improve purity
substituteInPlace ./setup.py --replace $i /no-such-path
done
'' + optionalString stdenv.isDarwin ''
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
export MACOSX_DEPLOYMENT_TARGET=10.6
'' + optionalString (isPy3k && pythonOlder "3.7") ''
# Determinism: The interpreter is patched to write null timestamps when compiling Python files
# so Python doesn't try to update the bytecode when seeing frozen timestamps in Nix's store.
export DETERMINISTIC_BUILD=1;
'' + optionalString stdenv.hostPlatform.isMusl ''
export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000"
'';
setupHook = python-setup-hook sitePackages;
postInstall = ''
# needed for some packages, especially packages that backport functionality
# to 2.x from 3.x
for item in $out/lib/${libPrefix}/test/*; do
if [[ "$item" != */test_support.py*
&& "$item" != */test/support
&& "$item" != */test/libregrtest
&& "$item" != */test/regrtest.py* ]]; then
rm -rf "$item"
else
echo $item
fi
done
touch $out/lib/${libPrefix}/test/__init__.py
ln -s "$out/include/${executable}m" "$out/include/${executable}"
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Get rid of retained dependencies on -dev packages, and remove
# some $TMPDIR references to improve binary reproducibility.
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
for i in $out/lib/${libPrefix}/_sysconfigdata*.py $out/lib/${libPrefix}/config-${sourceVersion.major}${sourceVersion.minor}*/Makefile; do
sed -i $i -e "s|$TMPDIR|/no-such-path|g"
done
# Further get rid of references. https://github.com/NixOS/nixpkgs/issues/51668
find $out/lib/python*/config-* -type f -print -exec nuke-refs -e $out '{}' +
find $out/lib -name '_sysconfigdata*.py*' -print -exec nuke-refs -e $out '{}' +
'' + optionalString stripConfig ''
rm -R $out/bin/python*-config $out/lib/python*/config-*
'' + optionalString stripIdlelib ''
# Strip IDLE (and turtledemo, which uses it)
rm -R $out/bin/idle* $out/lib/python*/{idlelib,turtledemo}
'' + optionalString stripTkinter ''
rm -R $out/lib/python*/tkinter
'' + optionalString stripTests ''
# Strip tests
rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
'' + ''
# Include a sitecustomize.py file
cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
'' + optionalString rebuildBytecode ''
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
# Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
# This is automatically used when `SOURCE_DATE_EPOCH` is set.
find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
'' + optionalString stripBytecode ''
find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
'';
preFixup = stdenv.lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
# Ensure patch-shebangs uses shebangs of host interpreter.
export PATH=${stdenv.lib.makeBinPath [ "$out" bash ]}:$PATH
'';
# Enforce that we don't have references to the OpenSSL -dev package, which we
# explicitly specify in our configure flags above.
disallowedReferences =
stdenv.lib.optionals (openssl != null) [ openssl.dev ]
++ stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
# Ensure we don't have references to build-time packages.
# These typically end up in shebangs.
pythonForBuild buildPackages.bash
];
inherit passthru;
enableParallelBuilding = true;
meta = {
homepage = http://python.org;
description = "A high-level dynamically-typed programming language";
longDescription = ''
Python is a remarkably powerful dynamic programming language that
is used in a wide variety of application domains. Some of its key
distinguishing features include: clear, readable syntax; strong
introspection capabilities; intuitive object orientation; natural
expression of procedural code; full modularity, supporting
hierarchical packages; exception-based error handling; and very
high level dynamic data types.
'';
license = licenses.psfl;
platforms = with platforms; linux ++ darwin;
maintainers = with maintainers; [ fridh ];
};
}