Python 3.x: do not regenerate _sysconfigdata

This commit fixes several issues:

- as reported in https://github.com/NixOS/nixpkgs/issues/24924 it was
possible that the file _sysconfigdata.pyc was generated after the actual
build of the CPython interpreter. We forgot to regenerate that file
during the build. This is now fixed

- the expression of the 3.3 interpreter now also includes some of the
determinism patches even though the output isn't yet reproducible. The
reason for adding them is that this makes the expressions of the
different interpreters more similar.

- references to -dev packages are now also removed in the 3.6 package,
thereby reducing its closure size

4 files changed, 44 insertions, 29 deletions

diff --git a/pkgs/development/interpreters/python/cpython/3.3/default.nix b/pkgs/development/interpreters/python/cpython/3.3/default.nix
index 3ab0de96704c..c561a1ed750a 100644
--- a/pkgs/development/interpreters/python/cpython/3.3/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.3/default.nix
@@ -50,7 +50,17 @@ in stdenv.mkDerivation {
 
   NIX_LDFLAGS = stdenv.lib.optionalString stdenv.isLinux "-lgcc_s";
 
-  postPatch = optionalString (x11Support && (tix != null)) ''
+  # Determinism: The interpreter is patched to write null timestamps when compiling python files.
+  # This way python doesn't try to update them when we freeze timestamps in nix store.
+  DETERMINISTIC_BUILD=1;
+  # Determinism: We fix the hashes of str, bytes and datetime objects.
+  PYTHONHASHSEED=0;
+
+  postPatch = ''
+    # Determinism
+    substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
+#     # We do not patch `Lib/importlib/_bootstrap_external.py` because it does not exist.
+  '' + optionalString (x11Support && (tix != null)) ''
     substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
   '';
 
@@ -80,12 +90,17 @@ in stdenv.mkDerivation {
       fi
     done
     touch $out/lib/python${majorVersion}/test/__init__.py
+
     ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}"
     paxmark E $out/bin/python${majorVersion}
 
     # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
     echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
 
+    # Determinism: Windows installers were not deterministic.
+    # We're also not interested in building Windows installers.
+    find "$out" -name 'wininst*.exe' | xargs -r rm -f
+
     # Use Python3 as default python
     ln -s "$out/bin/idle3" "$out/bin/idle"
     ln -s "$out/bin/pip3" "$out/bin/pip"
@@ -93,18 +108,20 @@ in stdenv.mkDerivation {
     ln -s "$out/bin/python3" "$out/bin/python"
     ln -s "$out/bin/python3-config" "$out/bin/python-config"
     ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
-  '';
 
-  postFixup = ''
     # Get rid of retained dependencies on -dev packages, and remove
     # some $TMPDIR references to improve binary reproducibility.
+    # Note that the .pyc file of _sysconfigdata.py should be regenerated!
     for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
       sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
     done
 
-    # FIXME: should regenerate this.
-    rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
-
+    # Determinism: rebuild all bytecode
+    # We exclude lib2to3 because that's Python 2 code which fails
+    # We rebuild three times, once for each optimization level
+    find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
   '';
 
   passthru = let
diff --git a/pkgs/development/interpreters/python/cpython/3.4/default.nix b/pkgs/development/interpreters/python/cpython/3.4/default.nix
index 58e4f21bb4a1..b2a4d849c942 100644
--- a/pkgs/development/interpreters/python/cpython/3.4/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.4/default.nix
@@ -132,24 +132,19 @@ in stdenv.mkDerivation {
     ln -s "$out/bin/python3-config" "$out/bin/python-config"
     ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
 
-    # Determinism: rebuild all bytecode
-    # We exclude lib2to3 because that's Python 2 code which fails
-    # We rebuild three times, once for each optimization level
-    find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
-  '';
-
-  postFixup = ''
     # Get rid of retained dependencies on -dev packages, and remove
     # some $TMPDIR references to improve binary reproducibility.
+    # Note that the .pyc file of _sysconfigdata.py should be regenerated!
     for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
       sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
     done
 
-    # FIXME: should regenerate this.
-    rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
-
+    # Determinism: rebuild all bytecode
+    # We exclude lib2to3 because that's Python 2 code which fails
+    # We rebuild three times, once for each optimization level
+    find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
   '';
 
   passthru = let
diff --git a/pkgs/development/interpreters/python/cpython/3.5/default.nix b/pkgs/development/interpreters/python/cpython/3.5/default.nix
index 082f6ff67897..76f445f7a509 100644
--- a/pkgs/development/interpreters/python/cpython/3.5/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.5/default.nix
@@ -134,23 +134,19 @@ in stdenv.mkDerivation {
     ln -s "$out/bin/python3-config" "$out/bin/python-config"
     ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
 
-    # Determinism: rebuild all bytecode
-    # We exclude lib2to3 because that's Python 2 code which fails
-    # We rebuild three times, once for each optimization level
-    find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
-  '';
-
-  postFixup = ''
     # Get rid of retained dependencies on -dev packages, and remove
     # some $TMPDIR references to improve binary reproducibility.
+    # Note that the .pyc file of _sysconfigdata.py should be regenerated!
     for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
       sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
     done
 
-    # FIXME: should regenerate this.
-    rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
+    # Determinism: rebuild all bytecode
+    # We exclude lib2to3 because that's Python 2 code which fails
+    # We rebuild three times, once for each optimization level
+    find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
   '';
 
   passthru = let
diff --git a/pkgs/development/interpreters/python/cpython/3.6/default.nix b/pkgs/development/interpreters/python/cpython/3.6/default.nix
index 2acca2b8a3cf..3c96bf38b6aa 100644
--- a/pkgs/development/interpreters/python/cpython/3.6/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.6/default.nix
@@ -134,6 +134,13 @@ in stdenv.mkDerivation {
     ln -s "$out/bin/python3-config" "$out/bin/python-config"
     ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
 
+    # Get rid of retained dependencies on -dev packages, and remove
+    # some $TMPDIR references to improve binary reproducibility.
+    # Note that the .pyc file of _sysconfigdata.py should be regenerated!
+    for i in $out/lib/python${majorVersion}/_sysconfigdata*.py $out/lib/python${majorVersion}/config-${majorVersion}m*/Makefile; do
+      sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
+    done
+
     # Determinism: rebuild all bytecode
     # We exclude lib2to3 because that's Python 2 code which fails
     # We rebuild three times, once for each optimization level