Merge pull request #22585 from FRidh/repr

Python: deterministic interpreters
This commit is contained in:
Frederik Rietdijk 2017-02-26 14:52:07 +01:00 committed by GitHub
commit 04c41e753b
7 changed files with 154 additions and 27 deletions

View file

@ -641,6 +641,19 @@ community to help save time. No tool is preferred at the moment.
- [pypi2nix](https://github.com/garbas/pypi2nix) by Rok Garbas
- [pypi2nix](https://github.com/offlinehacker/pypi2nix) by Jaka Hudoklin
### Deterministic builds
Python 2.7, 3.5 and 3.6 are now built deterministically and 3.4 mostly.
Minor modifications had to be made to the interpreters in order to generate
deterministic bytecode. This has security implications and is relevant for
those using Python in a `nix-shell`.
When the environment variable `DETERMINISTIC_BUILD` is set, all bytecode will have timestamp 1.
The `buildPythonPackage` function sets `DETERMINISTIC_BUILD` as well as
[PYTHONHASHSEED](https://docs.python.org/3.5/using/cmdline.html#envvar-PYTHONHASHSEED).
Both are also exported in `nix-shell`.
## FAQ
### How can I install a working Python environment?

View file

@ -253,6 +253,17 @@ following incompatible changes:</para>
</para>
</listitem>
<listitem>
<para>
Python 2.7, 3.5 and 3.6 are now built deterministically and 3.4 mostly.
Minor modifications had to be made to the interpreters in order to generate
deterministic bytecode. This has security implications and is relevant for
those using Python in a <literal>nix-shell</literal>. See the Nixpkgs manual
for details.
</para>
</listitem>
</itemizedlist>

View file

@ -178,6 +178,17 @@ in stdenv.mkDerivation {
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';
passthru = let

View file

@ -1,5 +1,7 @@
{ stdenv, fetchurl
, bzip2
, expat
, libffi
, gdbm
, lzma
, ncurses
@ -50,21 +52,43 @@ in stdenv.mkDerivation {
NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
# This way python doesn't try to update them when we freeze timestamps in nix store.
DETERMINISTIC_BUILD=1;
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED=0;
prePatch = optionalString stdenv.isDarwin ''
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
'';
postPatch = optionalString (x11Support && (tix != null)) ''
postPatch = ''
# Determinism
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
# Determinism. This is done unconditionally
substituteInPlace "Lib/importlib/_bootstrap.py" --replace "source_mtime = int(source_stats['mtime'])" "source_mtime = 1"
'' + optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
''
# Avoid picking up getentropy() from glibc >= 2.25, as that would break
# on older kernels. http://bugs.python.org/issue29157
+ optionalString stdenv.isLinux
''
+ optionalString stdenv.isLinux ''
substituteInPlace Python/random.c --replace 'defined(HAVE_GETENTROPY)' '0'
cat Python/random.c
'';
'';
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
configureFlags = [
"--enable-shared"
"--with-threads"
"--without-ensurepip"
"--with-system-expat"
"--with-system-ffi"
];
preConfigure = ''
for i in /usr /sw /opt /pkg; do # improve purity
@ -74,12 +98,6 @@ in stdenv.mkDerivation {
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
export MACOSX_DEPLOYMENT_TARGET=10.6
''}
configureFlagsArray=( --enable-shared --with-threads
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
)
'';
setupHook = ./setup-hook.sh;
@ -102,6 +120,10 @@ in stdenv.mkDerivation {
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pip3" "$out/bin/pip"
@ -109,6 +131,13 @@ in stdenv.mkDerivation {
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';
postFixup = ''

View file

@ -1,5 +1,7 @@
{ stdenv, fetchurl, fetchpatch
, bzip2
, expat
, libffi
, gdbm
, lzma
, ncurses
@ -32,7 +34,7 @@ let
sitePackages = "lib/${libPrefix}/site-packages";
buildInputs = filter (p: p != null) [
zlib bzip2 lzma gdbm sqlite readline ncurses openssl ]
zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ]
++ optionals x11Support [ tcl tk libX11 xproto ]
++ optionals stdenv.isDarwin [ CF configd ];
@ -50,6 +52,12 @@ in stdenv.mkDerivation {
NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
# This way python doesn't try to update them when we freeze timestamps in nix store.
DETERMINISTIC_BUILD=1;
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED=0;
prePatch = optionalString stdenv.isDarwin ''
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
@ -63,10 +71,27 @@ in stdenv.mkDerivation {
})
];
postPatch = optionalString (x11Support && (tix != null)) ''
postPatch = ''
# Determinism
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
# Determinism. This is done unconditionally
substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1"
'' + optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
'';
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
configureFlags = [
"--enable-shared"
"--with-threads"
"--without-ensurepip"
"--with-system-expat"
"--with-system-ffi"
];
preConfigure = ''
for i in /usr /sw /opt /pkg; do # improve purity
substituteInPlace ./setup.py --replace $i /no-such-path
@ -75,12 +100,6 @@ in stdenv.mkDerivation {
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
export MACOSX_DEPLOYMENT_TARGET=10.6
''}
configureFlagsArray=( --enable-shared --with-threads
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
)
'';
setupHook = ./setup-hook.sh;
@ -103,6 +122,10 @@ in stdenv.mkDerivation {
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pip3" "$out/bin/pip"
@ -110,6 +133,13 @@ in stdenv.mkDerivation {
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';
postFixup = ''

View file

@ -1,6 +1,8 @@
{ stdenv, fetchurl, fetchpatch
, glibc
, bzip2
, expat
, libffi
, gdbm
, lzma
, ncurses
@ -50,6 +52,12 @@ in stdenv.mkDerivation {
NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
# This way python doesn't try to update them when we freeze timestamps in nix store.
DETERMINISTIC_BUILD=1;
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED=0;
prePatch = optionalString stdenv.isDarwin ''
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
@ -63,10 +71,27 @@ in stdenv.mkDerivation {
})
];
postPatch = optionalString (x11Support && (tix != null)) ''
postPatch = ''
# Determinism
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
# Determinism. This is done unconditionally
substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1"
'' + optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
'';
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
configureFlags = [
"--enable-shared"
"--with-threads"
"--without-ensurepip"
"--with-system-expat"
"--with-system-ffi"
];
preConfigure = ''
for i in /usr /sw /opt /pkg; do # improve purity
substituteInPlace ./setup.py --replace $i /no-such-path
@ -75,12 +100,6 @@ in stdenv.mkDerivation {
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
export MACOSX_DEPLOYMENT_TARGET=10.6
''}
configureFlagsArray=( --enable-shared --with-threads
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
)
'';
setupHook = ./setup-hook.sh;
@ -103,6 +122,10 @@ in stdenv.mkDerivation {
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pip3" "$out/bin/pip"
@ -110,6 +133,13 @@ in stdenv.mkDerivation {
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'';
passthru = let

View file

@ -57,9 +57,12 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // {
inherit pythonPath;
# patch python interpreter to write null timestamps when compiling python files
# this way python doesn't try to update them when we freeze timestamps in nix store
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
# This way python doesn't try to update them when we freeze timestamps in nix store.
DETERMINISTIC_BUILD=1;
# Determinism: We fix the hashes of str, bytes and datetime objects.
PYTHONHASHSEED = 0;
buildInputs = [ wrapPython ] ++ buildInputs ++ pythonPath
++ [ (ensureNewerSourcesHook { year = "1980"; }) ]