From acd32a4caf533ca6202afad07c085450649dbe4a Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Fri, 19 May 2017 14:21:58 +0200 Subject: Python: set DETERMINISTIC_BUILD and PYTHONHASHSEED in setupHook The Python interpreters are patched so they can build .pyc bytecode free of certain indeterminism. When building Python packages we currently set ``` compiling python files. in nix store. DETERMINISTIC_BUILD=1; PYTHONHASHSEED = 0; ``` Instead if setting these environment variables in the function that builds the package, this commit sets the variables instead in the Python setup hook. That way, whenever Python is included in a derivation, these variables are set. See also the issue https://github.com/NixOS/nixpkgs/issues/25707. --- .../interpreters/python/cpython/2.7/default.nix | 3 ++- .../interpreters/python/cpython/2.7/setup-hook.sh | 15 --------------- .../interpreters/python/cpython/3.3/default.nix | 2 +- .../interpreters/python/cpython/3.3/setup-hook.sh | 15 --------------- .../interpreters/python/cpython/3.4/default.nix | 3 ++- .../interpreters/python/cpython/3.4/setup-hook.sh | 15 --------------- .../interpreters/python/cpython/3.5/default.nix | 3 ++- .../interpreters/python/cpython/3.5/setup-hook.sh | 15 --------------- .../interpreters/python/cpython/3.6/default.nix | 3 ++- .../interpreters/python/cpython/3.6/setup-hook.sh | 15 --------------- .../interpreters/python/mk-python-derivation.nix | 7 ------- .../interpreters/python/pypy/2.7/default.nix | 7 ++++--- .../interpreters/python/pypy/2.7/setup-hook.sh | 15 --------------- pkgs/development/interpreters/python/setup-hook.nix | 13 +++++++++++++ pkgs/development/interpreters/python/setup-hook.sh | 21 +++++++++++++++++++++ pkgs/top-level/all-packages.nix | 3 +++ 16 files changed, 50 insertions(+), 105 deletions(-) delete mode 100644 pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh delete mode 100644 pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh delete mode 100644 pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh delete mode 100644 pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh delete mode 100644 pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh delete mode 100644 pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh create mode 100644 pkgs/development/interpreters/python/setup-hook.nix create mode 100644 pkgs/development/interpreters/python/setup-hook.sh (limited to 'pkgs') diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index 8426902414a7..c5fa05651bbe 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -15,6 +15,7 @@ , expat , libffi , CF, configd, coreutils +, python-setup-hook # For the Python package set , pkgs, packageOverrides ? (self: super: {}) }: @@ -150,7 +151,7 @@ in stdenv.mkDerivation { NIX_CFLAGS_COMPILE = optionalString stdenv.isDarwin "-msse2"; DETERMINISTIC_BUILD = 1; - setupHook = ./setup-hook.sh; + setupHook = python-setup-hook sitePackages; postPatch = optionalString (x11Support && (tix != null)) '' substituteInPlace "Lib/lib-tk/Tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" diff --git a/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh b/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh deleted file mode 100644 index 4770eea886f6..000000000000 --- a/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh +++ /dev/null @@ -1,15 +0,0 @@ -addPythonPath() { - addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python2.7/site-packages -} - -toPythonPath() { - local paths="$1" - local result= - for i in $paths; do - p="$i/lib/python2.7/site-packages" - result="${result}${result:+:}$p" - done - echo $result -} - -envHooks+=(addPythonPath) diff --git a/pkgs/development/interpreters/python/cpython/3.3/default.nix b/pkgs/development/interpreters/python/cpython/3.3/default.nix index c561a1ed750a..061176335c4f 100644 --- a/pkgs/development/interpreters/python/cpython/3.3/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.3/default.nix @@ -77,7 +77,7 @@ in stdenv.mkDerivation { ) ''; - setupHook = ./setup-hook.sh; + setupHook = python-setup-hook sitePackages; postInstall = '' # needed for some packages, especially packages that backport functionality diff --git a/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh deleted file mode 100644 index 82a8c0abd32f..000000000000 --- a/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh +++ /dev/null @@ -1,15 +0,0 @@ -addPythonPath() { - addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.3/site-packages -} - -toPythonPath() { - local paths="$1" - local result= - for i in $paths; do - p="$i/lib/python3.3/site-packages" - result="${result}${result:+:}$p" - done - echo $result -} - -envHooks+=(addPythonPath) diff --git a/pkgs/development/interpreters/python/cpython/3.4/default.nix b/pkgs/development/interpreters/python/cpython/3.4/default.nix index b2a4d849c942..4d20a21a4d13 100644 --- a/pkgs/development/interpreters/python/cpython/3.4/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.4/default.nix @@ -13,6 +13,7 @@ , callPackage , self , CF, configd +, python-setup-hook # For the Python package set , pkgs, packageOverrides ? (self: super: {}) }: @@ -100,7 +101,7 @@ in stdenv.mkDerivation { ''} ''; - setupHook = ./setup-hook.sh; + setupHook = python-setup-hook sitePackages; postInstall = '' # needed for some packages, especially packages that backport functionality diff --git a/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh deleted file mode 100644 index fddcc0b73fe8..000000000000 --- a/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh +++ /dev/null @@ -1,15 +0,0 @@ -addPythonPath() { - addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.4/site-packages -} - -toPythonPath() { - local paths="$1" - local result= - for i in $paths; do - p="$i/lib/python3.4/site-packages" - result="${result}${result:+:}$p" - done - echo $result -} - -envHooks+=(addPythonPath) diff --git a/pkgs/development/interpreters/python/cpython/3.5/default.nix b/pkgs/development/interpreters/python/cpython/3.5/default.nix index 76f445f7a509..a4aec241a1aa 100644 --- a/pkgs/development/interpreters/python/cpython/3.5/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.5/default.nix @@ -13,6 +13,7 @@ , callPackage , self , CF, configd +, python-setup-hook # For the Python package set , pkgs, packageOverrides ? (self: super: {}) }: @@ -102,7 +103,7 @@ in stdenv.mkDerivation { ''} ''; - setupHook = ./setup-hook.sh; + setupHook = python-setup-hook sitePackages; postInstall = '' # needed for some packages, especially packages that backport functionality diff --git a/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh deleted file mode 100644 index 2836ad7e8f56..000000000000 --- a/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh +++ /dev/null @@ -1,15 +0,0 @@ -addPythonPath() { - addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.5/site-packages -} - -toPythonPath() { - local paths="$1" - local result= - for i in $paths; do - p="$i/lib/python3.5/site-packages" - result="${result}${result:+:}$p" - done - echo $result -} - -envHooks+=(addPythonPath) diff --git a/pkgs/development/interpreters/python/cpython/3.6/default.nix b/pkgs/development/interpreters/python/cpython/3.6/default.nix index 150786191669..d2d922ce4955 100644 --- a/pkgs/development/interpreters/python/cpython/3.6/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.6/default.nix @@ -14,6 +14,7 @@ , callPackage , self , CF, configd +, python-setup-hook # For the Python package set , pkgs, packageOverrides ? (self: super: {}) }: @@ -94,7 +95,7 @@ in stdenv.mkDerivation { ''} ''; - setupHook = ./setup-hook.sh; + setupHook = python-setup-hook sitePackages; postInstall = '' # needed for some packages, especially packages that backport functionality diff --git a/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh deleted file mode 100644 index 26a0d57bc87c..000000000000 --- a/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh +++ /dev/null @@ -1,15 +0,0 @@ -addPythonPath() { - addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.6/site-packages -} - -toPythonPath() { - local paths="$1" - local result= - for i in $paths; do - p="$i/lib/python3.6/site-packages" - result="${result}${result:+:}$p" - done - echo $result -} - -envHooks+=(addPythonPath) diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 5d710fcad882..1a388bfe4afa 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -57,13 +57,6 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // { inherit pythonPath; - - # Determinism: The interpreter is patched to write null timestamps when compiling python files. - # This way python doesn't try to update them when we freeze timestamps in nix store. - DETERMINISTIC_BUILD=1; - # Determinism: We fix the hashes of str, bytes and datetime objects. - PYTHONHASHSEED = 0; - buildInputs = [ wrapPython ] ++ buildInputs ++ pythonPath ++ [ (ensureNewerSourcesHook { year = "1980"; }) ] ++ (lib.optional (lib.hasSuffix "zip" attrs.src.name or "") unzip) diff --git a/pkgs/development/interpreters/python/pypy/2.7/default.nix b/pkgs/development/interpreters/python/pypy/2.7/default.nix index 456a078874cd..7552c6cd2853 100644 --- a/pkgs/development/interpreters/python/pypy/2.7/default.nix +++ b/pkgs/development/interpreters/python/pypy/2.7/default.nix @@ -1,6 +1,7 @@ { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, pkgconfig, libffi , sqlite, openssl, ncurses, python, expat, tcl, tk, tix, xlibsWrapper, libX11 , makeWrapper, callPackage, self, gdbm, db +, python-setup-hook # For the Python package set , pkgs, packageOverrides ? (self: super: {}) }: @@ -14,6 +15,7 @@ let pythonVersion = "2.7"; version = "${majorVersion}.${minorVersion}${minorVersionSuffix}"; libPrefix = "pypy${majorVersion}"; + sitePackages = "site-packages"; in stdenv.mkDerivation rec { name = "pypy-${version}"; @@ -67,7 +69,7 @@ in stdenv.mkDerivation rec { ${python.interpreter} rpython/bin/rpython --make-jobs="$NIX_BUILD_CORES" -Ojit --batch pypy/goal/targetpypystandalone.py --withmod-_minimal_curses --withmod-unicodedata --withmod-thread --withmod-bz2 --withmod-_multiprocessing ''; - setupHook = ./setup-hook.sh; + setupHook = python-setup-hook sitePackages; postBuild = '' cd ./lib_pypy @@ -125,12 +127,11 @@ in stdenv.mkDerivation rec { passthru = let pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;}; in rec { - inherit zlibSupport libPrefix; + inherit zlibSupport libPrefix sitePackages; executable = "pypy"; isPypy = true; buildEnv = callPackage ../../wrapper.nix { python = self; }; interpreter = "${self}/bin/${executable}"; - sitePackages = "site-packages"; withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; pkgs = pythonPackages; }; diff --git a/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh b/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh deleted file mode 100644 index e9081d1eaa53..000000000000 --- a/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh +++ /dev/null @@ -1,15 +0,0 @@ -addPythonPath() { - addToSearchPathWithCustomDelimiter : PYTHONPATH $1/site-packages -} - -toPythonPath() { - local paths="$1" - local result= - for i in $paths; do - p="$i/site-packages" - result="${result}${result:+:}$p" - done - echo $result -} - -envHooks+=(addPythonPath) diff --git a/pkgs/development/interpreters/python/setup-hook.nix b/pkgs/development/interpreters/python/setup-hook.nix new file mode 100644 index 000000000000..b66bd1cc5f69 --- /dev/null +++ b/pkgs/development/interpreters/python/setup-hook.nix @@ -0,0 +1,13 @@ +{ runCommand }: + +sitePackages: + +let + hook = ./setup-hook.sh; +in runCommand "python-setup-hook.sh" { + inherit sitePackages; +} '' + cp ${hook} hook.sh + substituteAllInPlace hook.sh + mv hook.sh $out +'' diff --git a/pkgs/development/interpreters/python/setup-hook.sh b/pkgs/development/interpreters/python/setup-hook.sh new file mode 100644 index 000000000000..dda9bed39f85 --- /dev/null +++ b/pkgs/development/interpreters/python/setup-hook.sh @@ -0,0 +1,21 @@ +addPythonPath() { + addToSearchPathWithCustomDelimiter : PYTHONPATH $1/@sitePackages@ +} + +toPythonPath() { + local paths="$1" + local result= + for i in $paths; do + p="$i/@sitePackages@" + result="${result}${result:+:}$p" + done + echo $result +} + +envHooks+=(addPythonPath) + +# Determinism: The interpreter is patched to write null timestamps when compiling python files. +# This way python doesn't try to update them when we freeze timestamps in nix store. +export DETERMINISTIC_BUILD=1; +# Determinism: We fix the hashes of str, bytes and datetime objects. +export PYTHONHASHSEED=0; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index ea47d41fc348..ae7e30d0ff89 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6162,6 +6162,9 @@ with pkgs; self = python36; }; + # Should eventually be moved inside Python interpreters. + python-setup-hook = callPackage ../development/interpreters/python/setup-hook.nix { }; + pypy = pypy27; pypy27 = callPackage ../development/interpreters/python/pypy/2.7 { -- cgit 1.4.1