diff options
Diffstat (limited to 'nixpkgs/pkgs/development/interpreters/python')
55 files changed, 3667 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/development/interpreters/python/build-python-package-common.nix b/nixpkgs/pkgs/development/interpreters/python/build-python-package-common.nix new file mode 100644 index 000000000000..0f8e088d434a --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/build-python-package-common.nix @@ -0,0 +1,31 @@ +# This function provides generic bits to install a Python wheel. + +{ python +}: + +{ buildInputs ? [] +# Additional flags to pass to "pip install". +, installFlags ? [] +, ... } @ attrs: + +attrs // { + buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ]; + + configurePhase = attrs.configurePhase or '' + runHook preConfigure + runHook postConfigure + ''; + + installPhase = attrs.installPhase or '' + runHook preInstall + + mkdir -p "$out/${python.sitePackages}" + export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" + + pushd dist + ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild + popd + + runHook postInstall + ''; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/build-python-package-flit.nix b/nixpkgs/pkgs/development/interpreters/python/build-python-package-flit.nix new file mode 100644 index 000000000000..911078a5e340 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/build-python-package-flit.nix @@ -0,0 +1,22 @@ +# This function provides specific bits for building a flit-based Python package. + +{ python +, flit +}: + +{ ... } @ attrs: + +attrs // { + buildInputs = [ flit ]; + buildPhase = attrs.buildPhase or '' + runHook preBuild + flit build --format wheel + runHook postBuild + ''; + + # Flit packages, like setuptools packages, might have tests. + installCheckPhase = attrs.checkPhase or '' + ${python.interpreter} -m unittest discover + ''; + doCheck = attrs.doCheck or true; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/build-python-package-setuptools.nix b/nixpkgs/pkgs/development/interpreters/python/build-python-package-setuptools.nix new file mode 100644 index 000000000000..4c66fdec5f6b --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/build-python-package-setuptools.nix @@ -0,0 +1,55 @@ +# This function provides specific bits for building a setuptools-based Python package. + +{ lib +, python +}: + +{ +# passed to "python setup.py build_ext" +# https://github.com/pypa/pip/issues/881 + setupPyBuildFlags ? [] +# Execute before shell hook +, preShellHook ? "" +# Execute after shell hook +, postShellHook ? "" +, ... } @ attrs: + +let + # use setuptools shim (so that setuptools is imported before distutils) + # pip does the same thing: https://github.com/pypa/pip/pull/3265 + setuppy = ./run_setup.py; + +in attrs // { + # we copy nix_run_setup over so it's executed relative to the root of the source + # many project make that assumption + buildPhase = attrs.buildPhase or '' + runHook preBuild + cp ${setuppy} nix_run_setup + ${python.pythonForBuild.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel + runHook postBuild + ''; + + installCheckPhase = attrs.checkPhase or '' + runHook preCheck + ${python.pythonForBuild.interpreter} nix_run_setup test + runHook postCheck + ''; + + # Python packages that are installed with setuptools + # are typically distributed with tests. + # With Python it's a common idiom to run the tests + # after the software has been installed. + doCheck = attrs.doCheck or true; + + shellHook = attrs.shellHook or '' + ${preShellHook} + if test -e setup.py; then + tmp_path=$(mktemp -d) + export PATH="$tmp_path/bin:$PATH" + export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH" + mkdir -p $tmp_path/${python.pythonForBuild.sitePackages} + ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2 + fi + ${postShellHook} + ''; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/build-python-package-wheel.nix b/nixpkgs/pkgs/development/interpreters/python/build-python-package-wheel.nix new file mode 100644 index 000000000000..7be0a4c304a3 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/build-python-package-wheel.nix @@ -0,0 +1,20 @@ +# This function provides specific bits for building a wheel-based Python package. + +{ +}: + +{ ... } @ attrs: + +attrs // { + unpackPhase = '' + mkdir dist + cp $src dist/"''${src#*-}" + ''; + + # Wheels are pre-compiled + buildPhase = attrs.buildPhase or ":"; + installCheckPhase = attrs.checkPhase or ":"; + + # Wheels don't have any checks to run + doCheck = attrs.doCheck or false; +} \ No newline at end of file diff --git a/nixpkgs/pkgs/development/interpreters/python/build-python-package.nix b/nixpkgs/pkgs/development/interpreters/python/build-python-package.nix new file mode 100644 index 000000000000..b664cf0b14f4 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/build-python-package.nix @@ -0,0 +1,46 @@ +# This function provides a generic Python package builder, +# and can build packages that use distutils, setuptools or flit. + +{ lib +, config +, python +, wrapPython +, setuptools +, unzip +, ensureNewerSourcesForZipFilesHook +, toPythonModule +, namePrefix +, flit +, writeScript +, update-python-libraries +}: + +let + setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; }; + flit-specific = import ./build-python-package-flit.nix { inherit python flit; }; + wheel-specific = import ./build-python-package-wheel.nix { }; + common = import ./build-python-package-common.nix { inherit python; }; + mkPythonDerivation = import ./mk-python-derivation.nix { + inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook; + inherit toPythonModule namePrefix writeScript update-python-libraries; + }; +in + +{ +# Several package formats are supported. +# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel. +# "wheel" : Install from a pre-compiled wheel. +# "flit" : Install a flit package. This builds a wheel. +# "other" : Provide your own buildPhase and installPhase. +format ? "setuptools" +, ... } @ attrs: + +let + formatspecific = + if format == "setuptools" then common (setuptools-specific attrs) + else if format == "flit" then common (flit-specific attrs) + else if format == "wheel" then common (wheel-specific attrs) + else if format == "other" then {} + else throw "Unsupported format ${format}"; + +in mkPythonDerivation ( attrs // formatspecific ) diff --git a/nixpkgs/pkgs/development/interpreters/python/catch_conflicts/README.md b/nixpkgs/pkgs/development/interpreters/python/catch_conflicts/README.md new file mode 100644 index 000000000000..d144b80e3385 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/catch_conflicts/README.md @@ -0,0 +1,13 @@ + + +catch_conflicts.py +================== + +The file catch_conflicts.py is in a subdirectory because, if it isn't, the +/nix/store/ directory is added to sys.path causing a delay when building. + +Pointers: + +- https://docs.python.org/3/library/sys.html#sys.path + +- https://github.com/NixOS/nixpkgs/pull/23600 diff --git a/nixpkgs/pkgs/development/interpreters/python/catch_conflicts/catch_conflicts.py b/nixpkgs/pkgs/development/interpreters/python/catch_conflicts/catch_conflicts.py new file mode 100644 index 000000000000..bb82900c65a9 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/catch_conflicts/catch_conflicts.py @@ -0,0 +1,30 @@ +import pkg_resources +import collections +import sys + +do_abort = False +packages = collections.defaultdict(list) + +for f in sys.path: + for req in pkg_resources.find_distributions(f): + if req not in packages[req.project_name]: + # some exceptions inside buildPythonPackage + if req.project_name in ['setuptools', 'pip', 'wheel']: + continue + packages[req.project_name].append(req) + + +for name, duplicates in packages.items(): + if len(duplicates) > 1: + do_abort = True + print("Found duplicated packages in closure for dependency '{}': ".format(name)) + for dup in duplicates: + print(" " + repr(dup)) + +if do_abort: + print("") + print( + 'Package duplicates found in closure, see above. Usually this ' + 'happens if two packages depend on different version ' + 'of the same dependency.') + sys.exit(1) diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.5.2-ctypes-util-find_library.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.5.2-ctypes-util-find_library.patch new file mode 100644 index 000000000000..22bc0f7ced0a --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.5.2-ctypes-util-find_library.patch @@ -0,0 +1,34 @@ +--- origsrc/Lib/ctypes/util.py 2007-09-14 15:05:26.000000000 -0500 ++++ src/Lib/ctypes/util.py 2008-11-25 17:54:47.319296200 -0600 +@@ -41,6 +41,20 @@ + continue + return None + ++elif sys.platform == "cygwin": ++ def find_library(name): ++ for libdir in ['/usr/lib', '/usr/local/lib']: ++ for libext in ['lib%s.dll.a' % name, 'lib%s.a' % name]: ++ implib = os.path.join(libdir, libext) ++ if not os.path.exists(implib): ++ continue ++ cmd = "dlltool -I " + implib + " 2>/dev/null" ++ res = os.popen(cmd).read().replace("\n","") ++ if not res: ++ continue ++ return res ++ return None ++ + elif os.name == "posix": + # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump + import re, tempfile, errno +@@ -157,6 +173,10 @@ + print cdll.LoadLibrary("libcrypto.dylib") + print cdll.LoadLibrary("libSystem.dylib") + print cdll.LoadLibrary("System.framework/System") ++ elif sys.platform == "cygwin": ++ print cdll.LoadLibrary("cygbz2-1.dll") ++ print find_library("crypt") ++ print cdll.LoadLibrary("cygcrypt-0.dll") + else: + print cdll.LoadLibrary("libm.so") + print cdll.LoadLibrary("libcrypt.so") diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.5.2-tkinter-x11.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.5.2-tkinter-x11.patch new file mode 100644 index 000000000000..28b6dafc3f15 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.5.2-tkinter-x11.patch @@ -0,0 +1,27 @@ +--- origsrc/setup.py 2008-02-04 17:41:02.000000000 -0600 ++++ src/setup.py 2008-07-02 02:11:28.671875000 -0500 +@@ -1277,12 +1279,6 @@ + include_dirs.append('/usr/X11/include') + added_lib_dirs.append('/usr/X11/lib') + +- # If Cygwin, then verify that X is installed before proceeding +- if host_platform == 'cygwin': +- x11_inc = find_file('X11/Xlib.h', [], include_dirs) +- if x11_inc is None: +- return +- + # Check for BLT extension + if self.compiler.find_library_file(lib_dirs + added_lib_dirs, + 'BLT8.0'): +@@ -1300,9 +1296,8 @@ + if host_platform in ['aix3', 'aix4']: + libs.append('ld') + +- # Finally, link with the X11 libraries (not appropriate on cygwin) +- if host_platform != "cygwin": +- libs.append('X11') ++ # Finally, link with the X11 libraries ++ libs.append('X11') + + ext = Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'], + define_macros=[('WITH_APPINIT', 1)] + defs, diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.2-ssl-threads.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.2-ssl-threads.patch new file mode 100644 index 000000000000..bef137efda7b --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.2-ssl-threads.patch @@ -0,0 +1,13 @@ +--- origsrc/Modules/_ssl.c 2009-01-26 10:55:41.000000000 -0600 ++++ src/Modules/_ssl.c 2009-08-20 00:04:59.346816700 -0500 +@@ -15,6 +15,10 @@ + + #include "Python.h" + ++#ifdef __CYGWIN__ ++#undef WITH_THREAD ++#endif ++ + #ifdef WITH_THREAD + #include "pythread.h" + #define PySSL_BEGIN_ALLOW_THREADS { \ diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-FD_SETSIZE.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-FD_SETSIZE.patch new file mode 100644 index 000000000000..d1dae8c47dc1 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-FD_SETSIZE.patch @@ -0,0 +1,41 @@ +--- Python-2.6.5.orig/Modules/selectmodule.c 2012-02-02 22:35:21.835125000 -0500 ++++ Python-2.6.5/Modules/selectmodule.c 2012-02-02 22:41:41.210125000 -0500 +@@ -6,6 +6,21 @@ + >= 0. + */ + ++/* Windows #defines FD_SETSIZE to 64 if FD_SETSIZE isn't already defined. ++ 64 is too small (too many people have bumped into that limit). ++ Here we boost it. ++ ++ Cygwin also defines FD_SETSIZE to 64, so also increase the limit on ++ Cygwin. We must do this before sys/types.h is included, which otherwise ++ sets FD_SETSIZE to the default. ++ ++ Users who want even more than the boosted limit should #define ++ FD_SETSIZE higher before this; e.g., via compiler /D switch. ++*/ ++#if (defined(MS_WINDOWS) || defined(__CYGWIN__)) && !defined(FD_SETSIZE) ++#define FD_SETSIZE 512 ++#endif ++ + #include "Python.h" + #include <structmember.h> + +@@ -16,16 +31,6 @@ + #undef HAVE_BROKEN_POLL + #endif + +-/* Windows #defines FD_SETSIZE to 64 if FD_SETSIZE isn't already defined. +- 64 is too small (too many people have bumped into that limit). +- Here we boost it. +- Users who want even more than the boosted limit should #define +- FD_SETSIZE higher before this; e.g., via compiler /D switch. +-*/ +-#if defined(MS_WINDOWS) && !defined(FD_SETSIZE) +-#define FD_SETSIZE 512 +-#endif +- + #if defined(HAVE_POLL_H) + #include <poll.h> + #elif defined(HAVE_SYS_POLL_H) diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-export-PySignal_SetWakeupFd.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-export-PySignal_SetWakeupFd.patch new file mode 100644 index 000000000000..ea696978236a --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-export-PySignal_SetWakeupFd.patch @@ -0,0 +1,11 @@ +--- origsrc/Include/pyerrors.h 2008-06-08 23:58:54.000000000 -0500 ++++ src/Include/pyerrors.h 2010-05-12 04:19:31.535297200 -0500 +@@ -232,7 +232,7 @@ PyAPI_FUNC(int) PyErr_CheckSignals(void) + PyAPI_FUNC(void) PyErr_SetInterrupt(void); + + /* In signalmodule.c */ +-int PySignal_SetWakeupFd(int fd); ++PyAPI_FUNC(int) PySignal_SetWakeupFd(int fd); + + /* Support for adding program text to SyntaxErrors */ + PyAPI_FUNC(void) PyErr_SyntaxLocation(const char *, int); diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-ncurses-abi6.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-ncurses-abi6.patch new file mode 100644 index 000000000000..e1cf5ad4bbf9 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.6.5-ncurses-abi6.patch @@ -0,0 +1,16 @@ +--- origsrc/Include/py_curses.h 2009-09-06 16:23:05.000000000 -0500 ++++ src/Include/py_curses.h 2010-04-14 15:21:23.008971400 -0500 +@@ -17,6 +17,13 @@ + #define NCURSES_OPAQUE 0 + #endif /* __APPLE__ */ + ++#ifdef __CYGWIN__ ++/* the following define is necessary for Cygwin; without it, the ++ Cygwin-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python ++ can't get at the WINDOW flags field. */ ++#define NCURSES_INTERNALS ++#endif /* __CYGWIN__ */ ++ + #ifdef __FreeBSD__ + /* + ** On FreeBSD, [n]curses.h and stdlib.h/wchar.h use different guards diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-dbm.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-dbm.patch new file mode 100644 index 000000000000..bfaeb37c287d --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-dbm.patch @@ -0,0 +1,27 @@ +--- origsrc/setup.py.orig 2012-11-27 10:20:47.442395900 -0500 ++++ src/setup.py 2012-11-27 10:53:15.583020900 -0500 +@@ -1141,7 +1141,7 @@ + + dbm_order = ['gdbm'] + # The standard Unix dbm module: +- if host_platform not in ['cygwin']: ++ if host_platform not in ['win32']: + config_args = [arg.strip("'") + for arg in sysconfig.get_config_var("CONFIG_ARGS").split()] + dbm_args = [arg for arg in config_args +@@ -1192,6 +1192,15 @@ + ], + libraries = gdbm_libs) + break ++ if find_file("ndbm.h", inc_dirs, []) is not None: ++ print("building dbm using gdbm") ++ dbmext = Extension( ++ 'dbm', ['dbmmodule.c'], ++ define_macros=[ ++ ('HAVE_NDBM_H', None), ++ ], ++ libraries = gdbm_libs) ++ break + elif cand == "bdb": + if db_incs is not None: + print "building dbm using bdb" diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-dylib.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-dylib.patch new file mode 100644 index 000000000000..6e1fc8b53e83 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-dylib.patch @@ -0,0 +1,10 @@ +--- origsrc/Lib/distutils/unixccompiler.py.orig 2012-11-27 07:44:15.409993500 -0500 ++++ src/Lib/distutils/unixccompiler.py 2012-11-27 08:09:57.801770900 -0500 +@@ -141,6 +141,7 @@ + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + if sys.platform == "cygwin": + exe_extension = ".exe" ++ dylib_lib_extension = ".dll.a" + + def preprocess(self, source, + output_file=None, macros=None, include_dirs=None, diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-getpath-exe-extension.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-getpath-exe-extension.patch new file mode 100644 index 000000000000..68f6921ba6aa --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-getpath-exe-extension.patch @@ -0,0 +1,31 @@ +--- origsrc/Modules/getpath.c.orig 2012-11-27 12:07:56.098645900 -0500 ++++ src/Modules/getpath.c 2012-11-27 12:10:11.254895900 -0500 +@@ -436,6 +436,28 @@ + if (isxfile(progpath)) + break; + ++#ifdef __CYGWIN__ ++ /* ++ * Cygwin automatically removes the ".exe" extension from argv[0] ++ * to make programs feel like they are in a more Unix-like ++ * environment. Unfortunately, this can make it problemmatic for ++ * Cygwin to distinguish between a directory and an executable with ++ * the same name excluding the ".exe" extension. For example, the ++ * Cygwin Python build directory has a "Python" directory and a ++ * "python.exe" executable. This causes isxfile() to erroneously ++ * return false. If isdir() returns true and there is enough space ++ * to append the ".exe" extension, then we try again with the ++ * extension appended. ++ */ ++#define EXE ".exe" ++ if (isdir(progpath) && strlen(progpath) + strlen(EXE) <= MAXPATHLEN) ++ { ++ strcat(progpath, EXE); ++ if (isxfile(progpath)) ++ break; ++ } ++#endif /* __CYGWIN__ */ ++ + if (!delim) { + progpath[0] = '\0'; + break; diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-no-libm.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-no-libm.patch new file mode 100644 index 000000000000..55281db6768f --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/2.7.3-no-libm.patch @@ -0,0 +1,11 @@ +--- origsrc/setup.py.orig 2012-11-27 09:28:34.051770900 -0500 ++++ src/setup.py 2012-11-27 09:28:47.239270900 -0500 +@@ -470,7 +470,7 @@ + + # Check for MacOS X, which doesn't need libm.a at all + math_libs = ['m'] +- if host_platform in ['darwin', 'beos']: ++ if host_platform in ['darwin', 'beos', 'cygwin']: + math_libs = [] + + # XXX Omitted modules: gl, pure, dl, SGI-specific modules diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/boot.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/boot.nix new file mode 100644 index 000000000000..9e38e8250748 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/boot.nix @@ -0,0 +1,103 @@ +{ stdenv, fetchurl, configd, CF, coreutils }: + +with stdenv.lib; + +let + + mkPaths = paths: { + C_INCLUDE_PATH = makeSearchPathOutput "dev" "include" paths; + LIBRARY_PATH = makeLibraryPath paths; + }; + +in + +stdenv.mkDerivation rec { + name = "python-boot-${version}"; + version = "2.7.12"; + libPrefix = "python2.7"; + + src = fetchurl { + url = "https://www.python.org/ftp/python/2.7.12/Python-${version}.tar.xz"; + sha256 = "0y7rl603vmwlxm6ilkhc51rx2mfj14ckcz40xxgs0ljnvlhp30yp"; + }; + + inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH; + + LDFLAGS = optionalString (!stdenv.isDarwin) "-lgcc_s"; + NIX_CFLAGS_COMPILE = optionalString stdenv.isDarwin "-msse2"; + + buildInputs = optionals stdenv.isDarwin [ CF configd ]; + + patches = + [ # Look in C_INCLUDE_PATH and LIBRARY_PATH for stuff. + ./search-path.patch + + # Python recompiles a Python if the mtime stored *in* the + # pyc/pyo file differs from the mtime of the source file. This + # doesn't work in Nix because Nix changes the mtime of files in + # the Nix store to 1. So treat that as a special case. + ./nix-store-mtime.patch + + # patch python to put zero timestamp into pyc + # if DETERMINISTIC_BUILD env var is set + ./deterministic-build.patch + ]; + + # Hack hack hack to stop shit from failing from a missing _scproxy on Darwin. Since + # we only use this python for bootstrappy things, it doesn't really matter if it + # doesn't have perfect proxy support in urllib :) this just makes it fall back on env + # vars instead of attempting to read the proxy configuration automatically, so not a + # huge loss even if for whatever reason we did want proxy support. + postPatch = '' + substituteInPlace Lib/urllib.py --replace "if sys.platform == 'darwin'" "if False" + ''; + + DETERMINISTIC_BUILD = 1; + + preConfigure = '' + # Purity. + for i in /usr /sw /opt /pkg; do + substituteInPlace ./setup.py --replace $i /no-such-path + done + '' + optionalString (stdenv ? cc && stdenv.cc.libc != null) '' + for i in Lib/plat-*/regen; do + substituteInPlace $i --replace /usr/include/ ${stdenv.cc.libc}/include/ + done + '' + optionalString stdenv.isDarwin '' + substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"' + substituteInPlace Lib/multiprocessing/__init__.py \ + --replace 'os.popen(comm)' 'os.popen("${coreutils}/bin/nproc")' + ''; + + configureFlags = [ "--enable-shared" "--with-threads" "--enable-unicode=ucs4" ] + ++ optionals stdenv.isCygwin [ "ac_cv_func_bind_textdomain_codeset=yes" ] + ++ optionals stdenv.isDarwin [ "--disable-toolbox-glue" ]; + + postInstall = + '' + ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz} + + rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev + ''; + + enableParallelBuilding = true; + + passthru.pkgs = builtins.throw "python-boot does not support packages, this package is only intended for bootstrapping." {}; + + meta = { + homepage = http://python.org; + description = "A high-level dynamically-typed programming language"; + longDescription = '' + Python is a remarkably powerful dynamic programming language that + is used in a wide variety of application domains. Some of its key + distinguishing features include: clear, readable syntax; strong + introspection capabilities; intuitive object orientation; natural + expression of procedural code; full modularity, supporting + hierarchical packages; exception-based error handling; and very + high level dynamic data types. + ''; + license = stdenv.lib.licenses.psfl; + platforms = stdenv.lib.platforms.all; + maintainers = with stdenv.lib.maintainers; [ lnl7 domenkozar ]; + }; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/cross-compile.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/cross-compile.patch new file mode 100644 index 000000000000..c83b56437a49 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/cross-compile.patch @@ -0,0 +1,32 @@ +--- ./setup.py.orig 2018-04-29 15:47:33.000000000 -0700 ++++ ./setup.py 2018-11-11 09:41:58.097682221 -0800 +@@ -458,8 +458,6 @@ + if not cross_compiling: + add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') + add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') +- if cross_compiling: +- self.add_gcc_paths() + self.add_multiarch_paths() + + # Add paths specified in the environment variables LDFLAGS and +@@ -517,7 +515,10 @@ + # be assumed that no additional -I,-L directives are needed. + inc_dirs = self.compiler.include_dirs[:] + lib_dirs = self.compiler.library_dirs[:] +- if not cross_compiling: ++ if cross_compiling: ++ inc_dirs = [] ++ lib_dirs = [] ++ else: + for d in ( + '/usr/include', + ): +@@ -582,6 +584,8 @@ class PyBuildExt(build_ext): + # Some modules that are normally always on: + #exts.append( Extension('_weakref', ['_weakref.c']) ) + ++ self.compiler.library_dirs = lib_dirs + [ '.' ] ++ + # array objects + exts.append( Extension('array', ['arraymodule.c']) ) + diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix new file mode 100644 index 000000000000..249c4ac9cf79 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -0,0 +1,289 @@ +{ stdenv, fetchurl, fetchpatch +, bzip2 +, expat +, libffi +, gdbm +, db +, ncurses +, openssl +, readline +, sqlite +, tcl ? null, tk ? null, tix ? null, xlibsWrapper ? null, libX11 ? null, x11Support ? false +, zlib +, callPackage +, self +, CF, configd, coreutils +, python-setup-hook +# Some proprietary libs assume UCS2 unicode, especially on darwin :( +, ucsEncoding ? 4 +# For the Python package set +, packageOverrides ? (self: super: {}) +, buildPackages +, sourceVersion +, sha256 +, passthruFun +}: + +assert x11Support -> tcl != null + && tk != null + && xlibsWrapper != null + && libX11 != null; + +with stdenv.lib; + +let + + pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}; + + passthru = passthruFun rec { + inherit self sourceVersion packageOverrides; + implementation = "cpython"; + libPrefix = "python${pythonVersion}"; + executable = libPrefix; + pythonVersion = with sourceVersion; "${major}.${minor}"; + sitePackages = "lib/${libPrefix}/site-packages"; + inherit pythonForBuild; + } // { + inherit ucsEncoding; + }; + + version = with sourceVersion; "${major}.${minor}.${patch}${suffix}"; + + src = fetchurl { + url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz"; + inherit sha256; + }; + + hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false); + patches = + [ # Look in C_INCLUDE_PATH and LIBRARY_PATH for stuff. + ./search-path.patch + + # Python recompiles a Python if the mtime stored *in* the + # pyc/pyo file differs from the mtime of the source file. This + # doesn't work in Nix because Nix changes the mtime of files in + # the Nix store to 1. So treat that as a special case. + ./nix-store-mtime.patch + + # patch python to put zero timestamp into pyc + # if DETERMINISTIC_BUILD env var is set + ./deterministic-build.patch + + # Fix python bug #27177 (https://bugs.python.org/issue27177) + # The issue is that `match.group` only recognizes python integers + # instead of everything that has `__index__`. + # This bug was fixed upstream, but not backported to 2.7 + (fetchpatch { + name = "re_match_index.patch"; + url = "https://bugs.python.org/file43084/re_match_index.patch"; + sha256 = "0l9rw6r5r90iybdkp3hhl2pf0h0s1izc68h5d3ywrm92pq32wz57"; + }) + + # "`type_getattro()` calls `tp_descr_get(self, obj, type)` without actually owning a reference to "self". + # In very rare cases, this can cause a segmentation fault if "self" is deleted by the descriptor." + # https://github.com/python/cpython/pull/6118 + (fetchpatch { + name = "type_getattro.patch"; + url = "file://${./type_getattro.patch}"; + sha256 = "11v9yx20hs3jmw0wggzvmw39qs4mxay4kb8iq2qjydwy9ya61nrd"; + }) + + (fetchpatch { + name = "CVE-2018-1000802.patch"; + url = "https://github.com/python/cpython/pull/8985.patch"; + sha256 = "1c8nq2c9sjqa8ipl62hiandg6a7lzrwwfhi3ky6jd3pxgyalrh97"; + }) + ] ++ optionals (x11Support && stdenv.isDarwin) [ + ./use-correct-tcl-tk-on-darwin.patch + ] ++ optionals stdenv.isLinux [ + + # Disable the use of ldconfig in ctypes.util.find_library (since + # ldconfig doesn't work on NixOS), and don't use + # ctypes.util.find_library during the loading of the uuid module + # (since it will do a futile invocation of gcc (!) to find + # libuuid, slowing down program startup a lot). + ./no-ldconfig.patch + + ] ++ optionals stdenv.hostPlatform.isCygwin [ + ./2.5.2-ctypes-util-find_library.patch + ./2.5.2-tkinter-x11.patch + ./2.6.2-ssl-threads.patch + ./2.6.5-export-PySignal_SetWakeupFd.patch + ./2.6.5-FD_SETSIZE.patch + ./2.6.5-ncurses-abi6.patch + ./2.7.3-dbm.patch + ./2.7.3-dylib.patch + ./2.7.3-getpath-exe-extension.patch + ./2.7.3-no-libm.patch + ] ++ optionals hasDistutilsCxxPatch [ + + # Patch from http://bugs.python.org/issue1222585 adapted to work with + # `patch -p1' and with a last hunk removed + # Upstream distutils is calling C compiler to compile C++ code, which + # only works for GCC and Apple Clang. This makes distutils to call C++ + # compiler when needed. + ./python-2.7-distutils-C++.patch + ] ++ optional (stdenv.hostPlatform != stdenv.buildPlatform) [ + ./cross-compile.patch + ]; + + preConfigure = '' + # Purity. + for i in /usr /sw /opt /pkg; do + substituteInPlace ./setup.py --replace $i /no-such-path + done + '' + optionalString (stdenv ? cc && stdenv.cc.libc != null) '' + for i in Lib/plat-*/regen; do + substituteInPlace $i --replace /usr/include/ ${stdenv.cc.libc}/include/ + done + '' + optionalString stdenv.isDarwin '' + substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"' + substituteInPlace Lib/multiprocessing/__init__.py \ + --replace 'os.popen(comm)' 'os.popen("${coreutils}/bin/nproc")' + ''; + + configureFlags = [ + "--enable-shared" + "--with-threads" + "--enable-unicode=ucs${toString ucsEncoding}" + ] ++ optionals (stdenv.hostPlatform.isCygwin || stdenv.hostPlatform.isAarch64) [ + "--with-system-ffi" + ] ++ optionals stdenv.hostPlatform.isCygwin [ + "--with-system-expat" + "ac_cv_func_bind_textdomain_codeset=yes" + ] ++ optionals stdenv.isDarwin [ + "--disable-toolbox-glue" + ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "PYTHON_FOR_BUILD=${getBin buildPackages.python}/bin/python" + "ac_cv_buggy_getaddrinfo=no" + # Assume little-endian IEEE 754 floating point when cross compiling + "ac_cv_little_endian_double=yes" + "ac_cv_big_endian_double=no" + "ac_cv_mixed_endian_double=no" + "ac_cv_x87_double_rounding=yes" + "ac_cv_tanh_preserves_zero_sign=yes" + # Generally assume that things are present and work + "ac_cv_posix_semaphores_enabled=yes" + "ac_cv_broken_sem_getvalue=no" + "ac_cv_wchar_t_signed=yes" + "ac_cv_rshift_extends_sign=yes" + "ac_cv_broken_nice=no" + "ac_cv_broken_poll=no" + "ac_cv_working_tzset=yes" + "ac_cv_have_long_long_format=yes" + "ac_cv_have_size_t_format=yes" + "ac_cv_computed_gotos=yes" + "ac_cv_file__dev_ptmx=yes" + "ac_cv_file__dev_ptc=yes" + ] + # Never even try to use lchmod on linux, + # don't rely on detecting glibc-isms. + ++ optional stdenv.hostPlatform.isLinux "ac_cv_func_lchmod=no"; + + buildInputs = + optional (stdenv ? cc && stdenv.cc.libc != null) stdenv.cc.libc ++ + [ bzip2 openssl zlib ] + ++ optional (stdenv.hostPlatform.isCygwin || stdenv.hostPlatform.isAarch64) libffi + ++ optional stdenv.hostPlatform.isCygwin expat + ++ [ db gdbm ncurses sqlite readline ] + ++ optionals x11Support [ tcl tk xlibsWrapper libX11 ] + ++ optionals stdenv.isDarwin ([ CF ] ++ optional (configd != null) configd); + nativeBuildInputs = + optionals (stdenv.hostPlatform != stdenv.buildPlatform) + [ buildPackages.stdenv.cc buildPackages.python ]; + + mkPaths = paths: { + C_INCLUDE_PATH = makeSearchPathOutput "dev" "include" paths; + LIBRARY_PATH = makeLibraryPath paths; + }; + + # Python 2.7 needs this + crossCompileEnv = stdenv.lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform) + { _PYTHON_HOST_PLATFORM = stdenv.hostPlatform.config; }; + + # Build the basic Python interpreter without modules that have + # external dependencies. + +in with passthru; stdenv.mkDerivation ({ + pname = "python"; + inherit version; + + inherit src patches buildInputs nativeBuildInputs preConfigure configureFlags; + + LDFLAGS = stdenv.lib.optionalString (!stdenv.isDarwin) "-lgcc_s"; + inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH; + + NIX_CFLAGS_COMPILE = optionalString stdenv.isDarwin "-msse2" + + optionalString stdenv.hostPlatform.isMusl " -DTHREAD_STACK_SIZE=0x100000"; + DETERMINISTIC_BUILD = 1; + + setupHook = python-setup-hook sitePackages; + + postPatch = optionalString (x11Support && (tix != null)) '' + substituteInPlace "Lib/lib-tk/Tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" + ''; + + postInstall = + '' + # needed for some packages, especially packages that backport + # functionality to 2.x from 3.x + for item in $out/lib/${libPrefix}/test/*; do + if [[ "$item" != */test_support.py* + && "$item" != */test/support + && "$item" != */test/regrtest.py* ]]; then + rm -rf "$item" + else + echo $item + fi + done + touch $out/lib/${libPrefix}/test/__init__.py + ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb + ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb${sourceVersion.major}.${sourceVersion.minor} + ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz} + + # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484 + echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py + + rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev + + # Determinism: Windows installers were not deterministic. + # We're also not interested in building Windows installers. + find "$out" -name 'wininst*.exe' | xargs -r rm -f + '' + optionalString (stdenv.hostPlatform == stdenv.buildPlatform) + '' + # Determinism: rebuild all bytecode + # We exclude lib2to3 because that's Python 2 code which fails + # We rebuild three times, once for each optimization level + find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i - + find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i - + find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i - + '' + optionalString stdenv.hostPlatform.isCygwin '' + cp libpython2.7.dll.a $out/lib + ''; + + inherit passthru; + + enableParallelBuilding = true; + + doCheck = false; # expensive, and fails + + meta = { + homepage = http://python.org; + description = "A high-level dynamically-typed programming language"; + longDescription = '' + Python is a remarkably powerful dynamic programming language that + is used in a wide variety of application domains. Some of its key + distinguishing features include: clear, readable syntax; strong + introspection capabilities; intuitive object orientation; natural + expression of procedural code; full modularity, supporting + hierarchical packages; exception-based error handling; and very + high level dynamic data types. + ''; + license = stdenv.lib.licenses.psfl; + platforms = stdenv.lib.platforms.all; + maintainers = with stdenv.lib.maintainers; [ fridh ]; + # Higher priority than Python 3.x so that `/bin/python` points to `/bin/python2` + # in case both 2 and 3 are installed. + priority = -100; + }; + } // crossCompileEnv) diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/deterministic-build.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/deterministic-build.patch new file mode 100644 index 000000000000..98d9d339fa11 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/deterministic-build.patch @@ -0,0 +1,36 @@ +diff -ur orig/Lib/py_compile.py new/Lib/py_compile.py +--- orig/Lib/py_compile.py ++++ new/Lib/py_compile.py +@@ -122,7 +122,10 @@ + cfile = file + (__debug__ and 'c' or 'o') + with open(cfile, 'wb') as fc: + fc.write('\0\0\0\0') +- wr_long(fc, timestamp) ++ if "DETERMINISTIC_BUILD" in os.environ: ++ fc.write('\0\0\0\0') ++ else: ++ wr_long(fc, timestamp) + marshal.dump(codeobject, fc) + fc.flush() + fc.seek(0, 0) +diff -ur orig/Python/import.c new/Python/import.c +--- orig/Python/import.c ++++ new/Python/import.c +@@ -939,10 +939,12 @@ + return; + } + /* Now write the true mtime (as a 32-bit field) */ +- fseek(fp, 4L, 0); +- assert(mtime <= 0xFFFFFFFF); +- PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); +- fflush(fp); ++ if (Py_GETENV("DETERMINISTIC_BUILD") == NULL) { ++ fseek(fp, 4L, 0); ++ assert(mtime <= 0xFFFFFFFF); ++ PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); ++ fflush(fp); ++ } + fclose(fp); + if (Py_VerboseFlag) + PySys_WriteStderr("# wrote %s\n", cpathname); + diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/nix-store-mtime.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/nix-store-mtime.patch new file mode 100644 index 000000000000..83f3fea1931b --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/nix-store-mtime.patch @@ -0,0 +1,12 @@ +diff -ru -x '*~' Python-2.7.1-orig/Python/import.c Python-2.7.1/Python/import.c +--- Python-2.7.1-orig/Python/import.c 2010-05-20 20:37:55.000000000 +0200 ++++ Python-2.7.1/Python/import.c 2011-01-04 15:55:11.000000000 +0100 +@@ -751,7 +751,7 @@ + return NULL; + } + pyc_mtime = PyMarshal_ReadLongFromFile(fp); +- if (pyc_mtime != mtime) { ++ if (pyc_mtime != mtime && mtime != 1) { + if (Py_VerboseFlag) + PySys_WriteStderr("# %s has bad mtime\n", cpathname); + fclose(fp); diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/no-ldconfig.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/no-ldconfig.patch new file mode 100644 index 000000000000..2a6b2a20dd19 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/no-ldconfig.patch @@ -0,0 +1,117 @@ +From 6b0f329a9f37110020ca02b35c8125391ef282b7 Mon Sep 17 00:00:00 2001 +From: Frederik Rietdijk <fridh@fridh.nl> +Date: Sat, 24 Dec 2016 15:56:10 +0100 +Subject: [PATCH] no ldconfig + +--- + Lib/ctypes/util.py | 35 +---------------------------------- + Lib/uuid.py | 47 ----------------------------------------------- + 2 files changed, 1 insertion(+), 81 deletions(-) + +diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py +index ab10ec5..f253e34 100644 +--- a/Lib/ctypes/util.py ++++ b/Lib/ctypes/util.py +@@ -235,40 +235,7 @@ elif os.name == "posix": + else: + + def _findSoname_ldconfig(name): +- import struct +- if struct.calcsize('l') == 4: +- machine = os.uname()[4] + '-32' +- else: +- machine = os.uname()[4] + '-64' +- mach_map = { +- 'x86_64-64': 'libc6,x86-64', +- 'ppc64-64': 'libc6,64bit', +- 'sparc64-64': 'libc6,64bit', +- 's390x-64': 'libc6,64bit', +- 'ia64-64': 'libc6,IA-64', +- } +- abi_type = mach_map.get(machine, 'libc6') +- +- # XXX assuming GLIBC's ldconfig (with option -p) +- expr = r'\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type) +- +- env = dict(os.environ) +- env['LC_ALL'] = 'C' +- env['LANG'] = 'C' +- null = open(os.devnull, 'wb') +- try: +- with null: +- p = subprocess.Popen(['/sbin/ldconfig', '-p'], +- stderr=null, +- stdout=subprocess.PIPE, +- env=env) +- except OSError: # E.g. command not found +- return None +- [data, _] = p.communicate() +- res = re.search(expr, data) +- if not res: +- return None +- return res.group(1) ++ return None + + def find_library(name): + return _findSoname_ldconfig(name) or _get_soname(_findLib_gcc(name)) +diff --git a/Lib/uuid.py b/Lib/uuid.py +index 7432032..05eeee5 100644 +--- a/Lib/uuid.py ++++ b/Lib/uuid.py +@@ -441,53 +441,6 @@ def _netbios_getnode(): + + # If ctypes is available, use it to find system routines for UUID generation. + _uuid_generate_time = _UuidCreate = None +-try: +- import ctypes, ctypes.util +- import sys +- +- # The uuid_generate_* routines are provided by libuuid on at least +- # Linux and FreeBSD, and provided by libc on Mac OS X. +- _libnames = ['uuid'] +- if not sys.platform.startswith('win'): +- _libnames.append('c') +- for libname in _libnames: +- try: +- lib = ctypes.CDLL(ctypes.util.find_library(libname)) +- except: +- continue +- if hasattr(lib, 'uuid_generate_time'): +- _uuid_generate_time = lib.uuid_generate_time +- break +- del _libnames +- +- # The uuid_generate_* functions are broken on MacOS X 10.5, as noted +- # in issue #8621 the function generates the same sequence of values +- # in the parent process and all children created using fork (unless +- # those children use exec as well). +- # +- # Assume that the uuid_generate functions are broken from 10.5 onward, +- # the test can be adjusted when a later version is fixed. +- if sys.platform == 'darwin': +- import os +- if int(os.uname()[2].split('.')[0]) >= 9: +- _uuid_generate_time = None +- +- # On Windows prior to 2000, UuidCreate gives a UUID containing the +- # hardware address. On Windows 2000 and later, UuidCreate makes a +- # random UUID and UuidCreateSequential gives a UUID containing the +- # hardware address. These routines are provided by the RPC runtime. +- # NOTE: at least on Tim's WinXP Pro SP2 desktop box, while the last +- # 6 bytes returned by UuidCreateSequential are fixed, they don't appear +- # to bear any relationship to the MAC address of any network device +- # on the box. +- try: +- lib = ctypes.windll.rpcrt4 +- except: +- lib = None +- _UuidCreate = getattr(lib, 'UuidCreateSequential', +- getattr(lib, 'UuidCreate', None)) +-except: +- pass + + def _unixdll_getnode(): + """Get the hardware address on Unix using ctypes.""" +-- +2.11.0 + diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/python-2.7-distutils-C++.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/python-2.7-distutils-C++.patch new file mode 100644 index 000000000000..90c21d5e60ca --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/python-2.7-distutils-C++.patch @@ -0,0 +1,260 @@ +--- a/Lib/distutils/cygwinccompiler.py ++++ b/Lib/distutils/cygwinccompiler.py +@@ -117,8 +117,10 @@ + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = "gcc" ++ self.linker_dll_cxx = "g++" + else: + self.linker_dll = "dllwrap" ++ self.linker_dll_cxx = "dllwrap" + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static +@@ -132,9 +134,13 @@ + self.set_executables(compiler='gcc -mcygwin -O -Wall', + compiler_so='gcc -mcygwin -mdll -O -Wall', + compiler_cxx='g++ -mcygwin -O -Wall', ++ compiler_so_cxx='g++ -mcygwin -mdll -O -Wall', + linker_exe='gcc -mcygwin', + linker_so=('%s -mcygwin %s' % +- (self.linker_dll, shared_option))) ++ (self.linker_dll, shared_option)), ++ linker_exe_cxx='g++ -mcygwin', ++ linker_so_cxx=('%s -mcygwin %s' % ++ (self.linker_dll_cxx, shared_option))) + + # cygwin and mingw32 need different sets of libraries + if self.gcc_version == "2.91.57": +@@ -160,8 +166,12 @@ + raise CompileError, msg + else: # for other files use the C-compiler + try: +- self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + +- extra_postargs) ++ if self.detect_language(src) == 'c++': ++ self.spawn(self.compiler_so_cxx + cc_args + [src, '-o', obj] + ++ extra_postargs) ++ else: ++ self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + ++ extra_postargs) + except DistutilsExecError, msg: + raise CompileError, msg + +@@ -327,9 +337,14 @@ + self.set_executables(compiler='gcc%s -O -Wall' % no_cygwin, + compiler_so='gcc%s -mdll -O -Wall' % no_cygwin, + compiler_cxx='g++%s -O -Wall' % no_cygwin, ++ compiler_so_cxx='g++%s -mdll -O -Wall' % no_cygwin, + linker_exe='gcc%s' % no_cygwin, + linker_so='%s%s %s %s' + % (self.linker_dll, no_cygwin, ++ shared_option, entry_point), ++ linker_exe_cxx='g++%s' % no_cygwin, ++ linker_so_cxx='%s%s %s %s' ++ % (self.linker_dll_cxx, no_cygwin, + shared_option, entry_point)) + # Maybe we should also append -mthreads, but then the finished + # dlls need another dll (mingwm10.dll see Mingw32 docs) +--- a/Lib/distutils/emxccompiler.py ++++ b/Lib/distutils/emxccompiler.py +@@ -65,8 +65,12 @@ + # XXX optimization, warnings etc. should be customizable. + self.set_executables(compiler='gcc -Zomf -Zmt -O3 -fomit-frame-pointer -mprobe -Wall', + compiler_so='gcc -Zomf -Zmt -O3 -fomit-frame-pointer -mprobe -Wall', ++ compiler_cxx='g++ -Zomf -Zmt -O3 -fomit-frame-pointer -mprobe -Wall', ++ compiler_so_cxx='g++ -Zomf -Zmt -O3 -fomit-frame-pointer -mprobe -Wall', + linker_exe='gcc -Zomf -Zmt -Zcrtdll', +- linker_so='gcc -Zomf -Zmt -Zcrtdll -Zdll') ++ linker_so='gcc -Zomf -Zmt -Zcrtdll -Zdll', ++ linker_exe_cxx='g++ -Zomf -Zmt -Zcrtdll', ++ linker_so_cxx='g++ -Zomf -Zmt -Zcrtdll -Zdll') + + # want the gcc library statically linked (so that we don't have + # to distribute a version dependent on the compiler we have) +@@ -83,8 +87,12 @@ + raise CompileError, msg + else: # for other files use the C-compiler + try: +- self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + +- extra_postargs) ++ if self.detect_language(src) == 'c++': ++ self.spawn(self.compiler_so_cxx + cc_args + [src, '-o', obj] + ++ extra_postargs) ++ else: ++ self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + ++ extra_postargs) + except DistutilsExecError, msg: + raise CompileError, msg + +--- a/Lib/distutils/sysconfig.py ++++ b/Lib/distutils/sysconfig.py +@@ -170,10 +170,12 @@ + _osx_support.customize_compiler(_config_vars) + _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' + +- (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \ +- get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS', +- 'CCSHARED', 'LDSHARED', 'SO', 'AR', +- 'ARFLAGS') ++ (cc, cxx, ccshared, ldshared, ldcxxshared, so_ext, ar, ar_flags) = \ ++ get_config_vars('CC', 'CXX', 'CCSHARED', 'LDSHARED', 'LDCXXSHARED', ++ 'SO', 'AR', 'ARFLAGS') ++ ++ cflags = '' ++ cxxflags = '' + + if 'CC' in os.environ: + newcc = os.environ['CC'] +@@ -188,19 +190,27 @@ + cxx = os.environ['CXX'] + if 'LDSHARED' in os.environ: + ldshared = os.environ['LDSHARED'] ++ if 'LDCXXSHARED' in os.environ: ++ ldcxxshared = os.environ['LDCXXSHARED'] + if 'CPP' in os.environ: + cpp = os.environ['CPP'] + else: + cpp = cc + " -E" # not always + if 'LDFLAGS' in os.environ: + ldshared = ldshared + ' ' + os.environ['LDFLAGS'] ++ ldcxxshared = ldcxxshared + ' ' + os.environ['LDFLAGS'] + if 'CFLAGS' in os.environ: +- cflags = opt + ' ' + os.environ['CFLAGS'] ++ cflags = os.environ['CFLAGS'] + ldshared = ldshared + ' ' + os.environ['CFLAGS'] ++ if 'CXXFLAGS' in os.environ: ++ cxxflags = os.environ['CXXFLAGS'] ++ ldcxxshared = ldcxxshared + ' ' + os.environ['CXXFLAGS'] + if 'CPPFLAGS' in os.environ: + cpp = cpp + ' ' + os.environ['CPPFLAGS'] + cflags = cflags + ' ' + os.environ['CPPFLAGS'] ++ cxxflags = cxxflags + ' ' + os.environ['CPPFLAGS'] + ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] ++ ldcxxshared = ldcxxshared + ' ' + os.environ['CPPFLAGS'] + if 'AR' in os.environ: + ar = os.environ['AR'] + if 'ARFLAGS' in os.environ: +@@ -209,13 +219,17 @@ + archiver = ar + ' ' + ar_flags + + cc_cmd = cc + ' ' + cflags ++ cxx_cmd = cxx + ' ' + cxxflags + compiler.set_executables( + preprocessor=cpp, + compiler=cc_cmd, + compiler_so=cc_cmd + ' ' + ccshared, +- compiler_cxx=cxx, ++ compiler_cxx=cxx_cmd, ++ compiler_so_cxx=cxx_cmd + ' ' + ccshared, + linker_so=ldshared, + linker_exe=cc, ++ linker_so_cxx=ldcxxshared, ++ linker_exe_cxx=cxx, + archiver=archiver) + + compiler.shared_lib_extension = so_ext +--- a/Lib/distutils/unixccompiler.py ++++ b/Lib/distutils/unixccompiler.py +@@ -55,14 +55,17 @@ + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). +- executables = {'preprocessor' : None, +- 'compiler' : ["cc"], +- 'compiler_so' : ["cc"], +- 'compiler_cxx' : ["cc"], +- 'linker_so' : ["cc", "-shared"], +- 'linker_exe' : ["cc"], +- 'archiver' : ["ar", "-cr"], +- 'ranlib' : None, ++ executables = {'preprocessor' : None, ++ 'compiler' : ["cc"], ++ 'compiler_so' : ["cc"], ++ 'compiler_cxx' : ["c++"], ++ 'compiler_so_cxx' : ["c++"], ++ 'linker_so' : ["cc", "-shared"], ++ 'linker_exe' : ["cc"], ++ 'linker_so_cxx' : ["c++", "-shared"], ++ 'linker_exe_cxx' : ["c++"], ++ 'archiver' : ["ar", "-cr"], ++ 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": +@@ -112,12 +115,19 @@ + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + compiler_so = self.compiler_so ++ compiler_so_cxx = self.compiler_so_cxx + if sys.platform == 'darwin': + compiler_so = _osx_support.compiler_fixup(compiler_so, + cc_args + extra_postargs) ++ compiler_so_cxx = _osx_support.compiler_fixup(compiler_so_cxx, ++ cc_args + extra_postargs) + try: +- self.spawn(compiler_so + cc_args + [src, '-o', obj] + +- extra_postargs) ++ if self.detect_language(src) == 'c++': ++ self.spawn(compiler_so_cxx + cc_args + [src, '-o', obj] + ++ extra_postargs) ++ else: ++ self.spawn(compiler_so + cc_args + [src, '-o', obj] + ++ extra_postargs) + except DistutilsExecError, msg: + raise CompileError, msg + +@@ -174,23 +184,16 @@ + ld_args.extend(extra_postargs) + self.mkpath(os.path.dirname(output_filename)) + try: +- if target_desc == CCompiler.EXECUTABLE: +- linker = self.linker_exe[:] ++ if target_lang == "c++": ++ if target_desc == CCompiler.EXECUTABLE: ++ linker = self.linker_exe_cxx[:] ++ else: ++ linker = self.linker_so_cxx[:] + else: +- linker = self.linker_so[:] +- if target_lang == "c++" and self.compiler_cxx: +- # skip over environment variable settings if /usr/bin/env +- # is used to set up the linker's environment. +- # This is needed on OSX. Note: this assumes that the +- # normal and C++ compiler have the same environment +- # settings. +- i = 0 +- if os.path.basename(linker[0]) == "env": +- i = 1 +- while '=' in linker[i]: +- i = i + 1 +- +- linker[i] = self.compiler_cxx[i] ++ if target_desc == CCompiler.EXECUTABLE: ++ linker = self.linker_exe[:] ++ else: ++ linker = self.linker_so[:] + + if sys.platform == 'darwin': + linker = _osx_support.compiler_fixup(linker, ld_args) +--- a/Lib/_osx_support.py ++++ b/Lib/_osx_support.py +@@ -14,13 +14,13 @@ + # configuration variables that may contain universal build flags, + # like "-arch" or "-isdkroot", that may need customization for + # the user environment +-_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', +- 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', +- 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', +- 'PY_CORE_CFLAGS') ++_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'CPPFLAGS', ++ 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'LDCXXSHARED', ++ 'CC', 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', ++ 'PY_CPPFLAGS', 'PY_CORE_CFLAGS') + + # configuration variables that may contain compiler calls +-_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') ++_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'LDCXXSHARED', 'CC', 'CXX') + + # prefix added to original configuration variable names + _INITPRE = '_OSX_SUPPORT_INITIAL_' \ No newline at end of file diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/search-path.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/search-path.patch new file mode 100644 index 000000000000..2e7b7526c0ce --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/search-path.patch @@ -0,0 +1,27 @@ +diff -rc Python-2.4.4-orig/setup.py Python-2.4.4/setup.py +*** Python-2.4.4-orig/setup.py 2006-10-08 19:41:25.000000000 +0200 +--- Python-2.4.4/setup.py 2007-05-27 16:04:54.000000000 +0200 +*************** +*** 279,288 **** + # Check for AtheOS which has libraries in non-standard locations + if platform == 'atheos': + lib_dirs += ['/system/libs', '/atheos/autolnk/lib'] +- lib_dirs += os.getenv('LIBRARY_PATH', '').split(os.pathsep) + inc_dirs += ['/system/include', '/atheos/autolnk/include'] +- inc_dirs += os.getenv('C_INCLUDE_PATH', '').split(os.pathsep) + + # OSF/1 and Unixware have some stuff in /usr/ccs/lib (like -ldb) + if platform in ['osf1', 'unixware7', 'openunix8']: + lib_dirs += ['/usr/ccs/lib'] +--- 279,289 ---- + # Check for AtheOS which has libraries in non-standard locations + if platform == 'atheos': + lib_dirs += ['/system/libs', '/atheos/autolnk/lib'] + inc_dirs += ['/system/include', '/atheos/autolnk/include'] + ++ lib_dirs += os.getenv('LIBRARY_PATH', '').split(os.pathsep) ++ inc_dirs += os.getenv('C_INCLUDE_PATH', '').split(os.pathsep) ++ + # OSF/1 and Unixware have some stuff in /usr/ccs/lib (like -ldb) + if platform in ['osf1', 'unixware7', 'openunix8']: + lib_dirs += ['/usr/ccs/lib'] diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/type_getattro.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/type_getattro.patch new file mode 100644 index 000000000000..c69c86b642b4 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/type_getattro.patch @@ -0,0 +1,62 @@ +--- a/Objects/typeobject.c ++++ b/Objects/typeobject.c +@@ -3137,6 +3137,7 @@ + PyTypeObject *metatype = Py_TYPE(type); + PyObject *meta_attribute, *attribute; + descrgetfunc meta_get; ++ PyObject* res; + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, +@@ -3158,6 +3159,7 @@ + meta_attribute = _PyType_Lookup(metatype, name); + + if (meta_attribute != NULL) { ++ Py_INCREF(meta_attribute); + meta_get = Py_TYPE(meta_attribute)->tp_descr_get; + + if (meta_get != NULL && PyDescr_IsData(meta_attribute)) { +@@ -3165,10 +3167,11 @@ + * writes. Assume the attribute is not overridden in + * type's tp_dict (and bases): call the descriptor now. + */ +- return meta_get(meta_attribute, (PyObject *)type, +- (PyObject *)metatype); ++ res = meta_get(meta_attribute, (PyObject *)type, ++ (PyObject *)metatype); ++ Py_DECREF(meta_attribute); ++ return res; + } +- Py_INCREF(meta_attribute); + } + + /* No data descriptor found on metatype. Look in tp_dict of this +@@ -3176,6 +3179,7 @@ + attribute = _PyType_Lookup(type, name); + if (attribute != NULL) { + /* Implement descriptor functionality, if any */ ++ Py_INCREF(attribute); + descrgetfunc local_get = Py_TYPE(attribute)->tp_descr_get; + + Py_XDECREF(meta_attribute); +@@ -3183,11 +3187,12 @@ + if (local_get != NULL) { + /* NULL 2nd argument indicates the descriptor was + * found on the target object itself (or a base) */ +- return local_get(attribute, (PyObject *)NULL, +- (PyObject *)type); ++ res = local_get(attribute, (PyObject *)NULL, ++ (PyObject *)type); ++ Py_DECREF(attribute); ++ return res; + } + +- Py_INCREF(attribute); + return attribute; + } + +--- /dev/null ++++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-14-21-42-17.bpo-25750.lxgkQz.rst +@@ -0,0 +1,2 @@ ++Fix rare Python crash due to bad refcounting in ``type_getattro()`` if a ++descriptor deletes itself from the class. diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/use-correct-tcl-tk-on-darwin.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/use-correct-tcl-tk-on-darwin.patch new file mode 100644 index 000000000000..b73f62b97ec5 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/use-correct-tcl-tk-on-darwin.patch @@ -0,0 +1,48 @@ +diff --git a/setup.py b/setup.py +index 2779658..902d0eb 100644 +--- a/setup.py ++++ b/setup.py +@@ -1699,9 +1699,6 @@ class PyBuildExt(build_ext): + # Rather than complicate the code below, detecting and building + # AquaTk is a separate method. Only one Tkinter will be built on + # Darwin - either AquaTk, if it is found, or X11 based Tk. +- if (host_platform == 'darwin' and +- self.detect_tkinter_darwin(inc_dirs, lib_dirs)): +- return + + # Assume we haven't found any of the libraries or include files + # The versions with dots are used on Unix, and the versions without +@@ -1747,22 +1744,6 @@ class PyBuildExt(build_ext): + if dir not in include_dirs: + include_dirs.append(dir) + +- # Check for various platform-specific directories +- if host_platform == 'sunos5': +- include_dirs.append('/usr/openwin/include') +- added_lib_dirs.append('/usr/openwin/lib') +- elif os.path.exists('/usr/X11R6/include'): +- include_dirs.append('/usr/X11R6/include') +- added_lib_dirs.append('/usr/X11R6/lib64') +- added_lib_dirs.append('/usr/X11R6/lib') +- elif os.path.exists('/usr/X11R5/include'): +- include_dirs.append('/usr/X11R5/include') +- added_lib_dirs.append('/usr/X11R5/lib') +- else: +- # Assume default location for X11 +- include_dirs.append('/usr/X11/include') +- added_lib_dirs.append('/usr/X11/lib') +- + # If Cygwin, then verify that X is installed before proceeding + if host_platform == 'cygwin': + x11_inc = find_file('X11/Xlib.h', [], include_dirs) +@@ -1786,10 +1767,6 @@ class PyBuildExt(build_ext): + if host_platform in ['aix3', 'aix4']: + libs.append('ld') + +- # Finally, link with the X11 libraries (not appropriate on cygwin) +- if host_platform != "cygwin": +- libs.append('X11') +- + ext = Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'], + define_macros=[('WITH_APPINIT', 1)] + defs, + include_dirs = include_dirs, diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/ld_library_path.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/ld_library_path.patch new file mode 100644 index 000000000000..013c2d266eff --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/ld_library_path.patch @@ -0,0 +1,51 @@ +From 918201682127ed8a270a4bd1a448b490019e4ada Mon Sep 17 00:00:00 2001 +From: Frederik Rietdijk <fridh@fridh.nl> +Date: Thu, 14 Sep 2017 10:00:31 +0200 +Subject: [PATCH] ctypes.util: support LD_LIBRARY_PATH + +Backports support for LD_LIBRARY_PATH from 3.6 +--- + Lib/ctypes/util.py | 26 +++++++++++++++++++++++++- + 1 file changed, 25 insertions(+), 1 deletion(-) + +diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py +index e9957d7951..9926f6c881 100644 +--- a/Lib/ctypes/util.py ++++ b/Lib/ctypes/util.py +@@ -219,8 +219,32 @@ elif os.name == "posix": + def _findSoname_ldconfig(name): + return None + ++ def _findLib_ld(name): ++ # See issue #9998 for why this is needed ++ expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name) ++ cmd = ['ld', '-t'] ++ libpath = os.environ.get('LD_LIBRARY_PATH') ++ if libpath: ++ for d in libpath.split(':'): ++ cmd.extend(['-L', d]) ++ cmd.extend(['-o', os.devnull, '-l%s' % name]) ++ result = None ++ try: ++ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, ++ stderr=subprocess.PIPE, ++ universal_newlines=True) ++ out, _ = p.communicate() ++ res = re.search(expr, os.fsdecode(out)) ++ if res: ++ result = res.group(0) ++ except Exception as e: ++ pass # result will be None ++ return result ++ + def find_library(name): +- return _findSoname_ldconfig(name) or _get_soname(_findLib_gcc(name)) ++ # See issue #9998 ++ return _findSoname_ldconfig(name) or \ ++ _get_soname(_findLib_gcc(name) or _findLib_ld(name)) + + ################################################################ + # test code +-- +2.14.1 + diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/no-ldconfig.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/no-ldconfig.patch new file mode 100644 index 000000000000..9718b1d8dceb --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/no-ldconfig.patch @@ -0,0 +1,164 @@ +From 590c46bb04f79ab611b2f8fd682dd7e43a01f268 Mon Sep 17 00:00:00 2001 +From: Frederik Rietdijk <fridh@fridh.nl> +Date: Mon, 28 Aug 2017 09:24:06 +0200 +Subject: [PATCH] Don't use ldconfig and speed up uuid load + +--- + Lib/ctypes/util.py | 70 ++---------------------------------------------------- + Lib/uuid.py | 49 -------------------------------------- + 2 files changed, 2 insertions(+), 117 deletions(-) + +diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py +index 7684eab81d..e9957d7951 100644 +--- a/Lib/ctypes/util.py ++++ b/Lib/ctypes/util.py +@@ -95,46 +95,7 @@ elif os.name == "posix": + import re, tempfile + + def _findLib_gcc(name): +- # Run GCC's linker with the -t (aka --trace) option and examine the +- # library name it prints out. The GCC command will fail because we +- # haven't supplied a proper program with main(), but that does not +- # matter. +- expr = os.fsencode(r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)) +- +- c_compiler = shutil.which('gcc') +- if not c_compiler: +- c_compiler = shutil.which('cc') +- if not c_compiler: +- # No C compiler available, give up +- return None +- +- temp = tempfile.NamedTemporaryFile() +- try: +- args = [c_compiler, '-Wl,-t', '-o', temp.name, '-l' + name] +- +- env = dict(os.environ) +- env['LC_ALL'] = 'C' +- env['LANG'] = 'C' +- try: +- proc = subprocess.Popen(args, +- stdout=subprocess.PIPE, +- stderr=subprocess.STDOUT, +- env=env) +- except OSError: # E.g. bad executable +- return None +- with proc: +- trace = proc.stdout.read() +- finally: +- try: +- temp.close() +- except FileNotFoundError: +- # Raised if the file was already removed, which is the normal +- # behaviour of GCC if linking fails +- pass +- res = re.search(expr, trace) +- if not res: +- return None +- return os.fsdecode(res.group(0)) ++ return None + + + if sys.platform == "sunos5": +@@ -256,34 +217,7 @@ elif os.name == "posix": + else: + + def _findSoname_ldconfig(name): +- import struct +- if struct.calcsize('l') == 4: +- machine = os.uname().machine + '-32' +- else: +- machine = os.uname().machine + '-64' +- mach_map = { +- 'x86_64-64': 'libc6,x86-64', +- 'ppc64-64': 'libc6,64bit', +- 'sparc64-64': 'libc6,64bit', +- 's390x-64': 'libc6,64bit', +- 'ia64-64': 'libc6,IA-64', +- } +- abi_type = mach_map.get(machine, 'libc6') +- +- # XXX assuming GLIBC's ldconfig (with option -p) +- regex = os.fsencode( +- '\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type)) +- try: +- with subprocess.Popen(['/sbin/ldconfig', '-p'], +- stdin=subprocess.DEVNULL, +- stderr=subprocess.DEVNULL, +- stdout=subprocess.PIPE, +- env={'LC_ALL': 'C', 'LANG': 'C'}) as p: +- res = re.search(regex, p.stdout.read()) +- if res: +- return os.fsdecode(res.group(1)) +- except OSError: +- pass ++ return None + + def find_library(name): + return _findSoname_ldconfig(name) or _get_soname(_findLib_gcc(name)) +diff --git a/Lib/uuid.py b/Lib/uuid.py +index e96e7e034c..31160ace95 100644 +--- a/Lib/uuid.py ++++ b/Lib/uuid.py +@@ -455,58 +455,9 @@ def _netbios_getnode(): + continue + return int.from_bytes(bytes, 'big') + +-# Thanks to Thomas Heller for ctypes and for his help with its use here. + +-# If ctypes is available, use it to find system routines for UUID generation. +-# XXX This makes the module non-thread-safe! + _uuid_generate_time = _UuidCreate = None +-try: +- import ctypes, ctypes.util +- import sys + +- # The uuid_generate_* routines are provided by libuuid on at least +- # Linux and FreeBSD, and provided by libc on Mac OS X. +- _libnames = ['uuid'] +- if not sys.platform.startswith('win'): +- _libnames.append('c') +- for libname in _libnames: +- try: +- lib = ctypes.CDLL(ctypes.util.find_library(libname)) +- except Exception: +- continue +- if hasattr(lib, 'uuid_generate_time'): +- _uuid_generate_time = lib.uuid_generate_time +- break +- del _libnames +- +- # The uuid_generate_* functions are broken on MacOS X 10.5, as noted +- # in issue #8621 the function generates the same sequence of values +- # in the parent process and all children created using fork (unless +- # those children use exec as well). +- # +- # Assume that the uuid_generate functions are broken from 10.5 onward, +- # the test can be adjusted when a later version is fixed. +- if sys.platform == 'darwin': +- import os +- if int(os.uname().release.split('.')[0]) >= 9: +- _uuid_generate_time = None +- +- # On Windows prior to 2000, UuidCreate gives a UUID containing the +- # hardware address. On Windows 2000 and later, UuidCreate makes a +- # random UUID and UuidCreateSequential gives a UUID containing the +- # hardware address. These routines are provided by the RPC runtime. +- # NOTE: at least on Tim's WinXP Pro SP2 desktop box, while the last +- # 6 bytes returned by UuidCreateSequential are fixed, they don't appear +- # to bear any relationship to the MAC address of any network device +- # on the box. +- try: +- lib = ctypes.windll.rpcrt4 +- except: +- lib = None +- _UuidCreate = getattr(lib, 'UuidCreateSequential', +- getattr(lib, 'UuidCreate', None)) +-except: +- pass + + def _unixdll_getnode(): + """Get the hardware address on Unix using ctypes.""" +-- +2.14.1 + diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/use-correct-tcl-tk-on-darwin.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/use-correct-tcl-tk-on-darwin.patch new file mode 100644 index 000000000000..b73f62b97ec5 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.5/use-correct-tcl-tk-on-darwin.patch @@ -0,0 +1,48 @@ +diff --git a/setup.py b/setup.py +index 2779658..902d0eb 100644 +--- a/setup.py ++++ b/setup.py +@@ -1699,9 +1699,6 @@ class PyBuildExt(build_ext): + # Rather than complicate the code below, detecting and building + # AquaTk is a separate method. Only one Tkinter will be built on + # Darwin - either AquaTk, if it is found, or X11 based Tk. +- if (host_platform == 'darwin' and +- self.detect_tkinter_darwin(inc_dirs, lib_dirs)): +- return + + # Assume we haven't found any of the libraries or include files + # The versions with dots are used on Unix, and the versions without +@@ -1747,22 +1744,6 @@ class PyBuildExt(build_ext): + if dir not in include_dirs: + include_dirs.append(dir) + +- # Check for various platform-specific directories +- if host_platform == 'sunos5': +- include_dirs.append('/usr/openwin/include') +- added_lib_dirs.append('/usr/openwin/lib') +- elif os.path.exists('/usr/X11R6/include'): +- include_dirs.append('/usr/X11R6/include') +- added_lib_dirs.append('/usr/X11R6/lib64') +- added_lib_dirs.append('/usr/X11R6/lib') +- elif os.path.exists('/usr/X11R5/include'): +- include_dirs.append('/usr/X11R5/include') +- added_lib_dirs.append('/usr/X11R5/lib') +- else: +- # Assume default location for X11 +- include_dirs.append('/usr/X11/include') +- added_lib_dirs.append('/usr/X11/lib') +- + # If Cygwin, then verify that X is installed before proceeding + if host_platform == 'cygwin': + x11_inc = find_file('X11/Xlib.h', [], include_dirs) +@@ -1786,10 +1767,6 @@ class PyBuildExt(build_ext): + if host_platform in ['aix3', 'aix4']: + libs.append('ld') + +- # Finally, link with the X11 libraries (not appropriate on cygwin) +- if host_platform != "cygwin": +- libs.append('X11') +- + ext = Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'], + define_macros=[('WITH_APPINIT', 1)] + defs, + include_dirs = include_dirs, diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.6/no-ldconfig.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.6/no-ldconfig.patch new file mode 100644 index 000000000000..0f829860a5b7 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.6/no-ldconfig.patch @@ -0,0 +1,163 @@ +From 105621b99cc30615c79b5aa3d12d6732e14b0d59 Mon Sep 17 00:00:00 2001 +From: Frederik Rietdijk <fridh@fridh.nl> +Date: Mon, 28 Aug 2017 09:24:06 +0200 +Subject: [PATCH] Don't use ldconfig and speed up uuid load + +--- + Lib/ctypes/util.py | 70 ++---------------------------------------------------- + Lib/uuid.py | 48 ------------------------------------- + 2 files changed, 2 insertions(+), 116 deletions(-) + +diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py +index 339ae8aa8a..2944985c30 100644 +--- a/Lib/ctypes/util.py ++++ b/Lib/ctypes/util.py +@@ -85,46 +85,7 @@ elif os.name == "posix": + import re, tempfile + + def _findLib_gcc(name): +- # Run GCC's linker with the -t (aka --trace) option and examine the +- # library name it prints out. The GCC command will fail because we +- # haven't supplied a proper program with main(), but that does not +- # matter. +- expr = os.fsencode(r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)) +- +- c_compiler = shutil.which('gcc') +- if not c_compiler: +- c_compiler = shutil.which('cc') +- if not c_compiler: +- # No C compiler available, give up +- return None +- +- temp = tempfile.NamedTemporaryFile() +- try: +- args = [c_compiler, '-Wl,-t', '-o', temp.name, '-l' + name] +- +- env = dict(os.environ) +- env['LC_ALL'] = 'C' +- env['LANG'] = 'C' +- try: +- proc = subprocess.Popen(args, +- stdout=subprocess.PIPE, +- stderr=subprocess.STDOUT, +- env=env) +- except OSError: # E.g. bad executable +- return None +- with proc: +- trace = proc.stdout.read() +- finally: +- try: +- temp.close() +- except FileNotFoundError: +- # Raised if the file was already removed, which is the normal +- # behaviour of GCC if linking fails +- pass +- res = re.search(expr, trace) +- if not res: +- return None +- return os.fsdecode(res.group(0)) ++ return None + + + if sys.platform == "sunos5": +@@ -246,34 +207,7 @@ elif os.name == "posix": + else: + + def _findSoname_ldconfig(name): +- import struct +- if struct.calcsize('l') == 4: +- machine = os.uname().machine + '-32' +- else: +- machine = os.uname().machine + '-64' +- mach_map = { +- 'x86_64-64': 'libc6,x86-64', +- 'ppc64-64': 'libc6,64bit', +- 'sparc64-64': 'libc6,64bit', +- 's390x-64': 'libc6,64bit', +- 'ia64-64': 'libc6,IA-64', +- } +- abi_type = mach_map.get(machine, 'libc6') +- +- # XXX assuming GLIBC's ldconfig (with option -p) +- regex = r'\s+(lib%s\.[^\s]+)\s+\(%s' +- regex = os.fsencode(regex % (re.escape(name), abi_type)) +- try: +- with subprocess.Popen(['/sbin/ldconfig', '-p'], +- stdin=subprocess.DEVNULL, +- stderr=subprocess.DEVNULL, +- stdout=subprocess.PIPE, +- env={'LC_ALL': 'C', 'LANG': 'C'}) as p: +- res = re.search(regex, p.stdout.read()) +- if res: +- return os.fsdecode(res.group(1)) +- except OSError: +- pass ++ return None + + def _findLib_ld(name): + # See issue #9998 for why this is needed +diff --git a/Lib/uuid.py b/Lib/uuid.py +index 200c800b34..31160ace95 100644 +--- a/Lib/uuid.py ++++ b/Lib/uuid.py +@@ -455,57 +455,9 @@ def _netbios_getnode(): + continue + return int.from_bytes(bytes, 'big') + +-# Thanks to Thomas Heller for ctypes and for his help with its use here. + +-# If ctypes is available, use it to find system routines for UUID generation. +-# XXX This makes the module non-thread-safe! + _uuid_generate_time = _UuidCreate = None +-try: +- import ctypes, ctypes.util +- import sys + +- # The uuid_generate_* routines are provided by libuuid on at least +- # Linux and FreeBSD, and provided by libc on Mac OS X. +- _libnames = ['uuid'] +- if not sys.platform.startswith('win'): +- _libnames.append('c') +- for libname in _libnames: +- try: +- lib = ctypes.CDLL(ctypes.util.find_library(libname)) +- except Exception: +- continue +- if hasattr(lib, 'uuid_generate_time'): +- _uuid_generate_time = lib.uuid_generate_time +- break +- del _libnames +- +- # The uuid_generate_* functions are broken on MacOS X 10.5, as noted +- # in issue #8621 the function generates the same sequence of values +- # in the parent process and all children created using fork (unless +- # those children use exec as well). +- # +- # Assume that the uuid_generate functions are broken from 10.5 onward, +- # the test can be adjusted when a later version is fixed. +- if sys.platform == 'darwin': +- if int(os.uname().release.split('.')[0]) >= 9: +- _uuid_generate_time = None +- +- # On Windows prior to 2000, UuidCreate gives a UUID containing the +- # hardware address. On Windows 2000 and later, UuidCreate makes a +- # random UUID and UuidCreateSequential gives a UUID containing the +- # hardware address. These routines are provided by the RPC runtime. +- # NOTE: at least on Tim's WinXP Pro SP2 desktop box, while the last +- # 6 bytes returned by UuidCreateSequential are fixed, they don't appear +- # to bear any relationship to the MAC address of any network device +- # on the box. +- try: +- lib = ctypes.windll.rpcrt4 +- except: +- lib = None +- _UuidCreate = getattr(lib, 'UuidCreateSequential', +- getattr(lib, 'UuidCreate', None)) +-except: +- pass + + def _unixdll_getnode(): + """Get the hardware address on Unix using ctypes.""" +-- +2.14.1 + diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.6/use-correct-tcl-tk-on-darwin.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.6/use-correct-tcl-tk-on-darwin.patch new file mode 100644 index 000000000000..b73f62b97ec5 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.6/use-correct-tcl-tk-on-darwin.patch @@ -0,0 +1,48 @@ +diff --git a/setup.py b/setup.py +index 2779658..902d0eb 100644 +--- a/setup.py ++++ b/setup.py +@@ -1699,9 +1699,6 @@ class PyBuildExt(build_ext): + # Rather than complicate the code below, detecting and building + # AquaTk is a separate method. Only one Tkinter will be built on + # Darwin - either AquaTk, if it is found, or X11 based Tk. +- if (host_platform == 'darwin' and +- self.detect_tkinter_darwin(inc_dirs, lib_dirs)): +- return + + # Assume we haven't found any of the libraries or include files + # The versions with dots are used on Unix, and the versions without +@@ -1747,22 +1744,6 @@ class PyBuildExt(build_ext): + if dir not in include_dirs: + include_dirs.append(dir) + +- # Check for various platform-specific directories +- if host_platform == 'sunos5': +- include_dirs.append('/usr/openwin/include') +- added_lib_dirs.append('/usr/openwin/lib') +- elif os.path.exists('/usr/X11R6/include'): +- include_dirs.append('/usr/X11R6/include') +- added_lib_dirs.append('/usr/X11R6/lib64') +- added_lib_dirs.append('/usr/X11R6/lib') +- elif os.path.exists('/usr/X11R5/include'): +- include_dirs.append('/usr/X11R5/include') +- added_lib_dirs.append('/usr/X11R5/lib') +- else: +- # Assume default location for X11 +- include_dirs.append('/usr/X11/include') +- added_lib_dirs.append('/usr/X11/lib') +- + # If Cygwin, then verify that X is installed before proceeding + if host_platform == 'cygwin': + x11_inc = find_file('X11/Xlib.h', [], include_dirs) +@@ -1786,10 +1767,6 @@ class PyBuildExt(build_ext): + if host_platform in ['aix3', 'aix4']: + libs.append('ld') + +- # Finally, link with the X11 libraries (not appropriate on cygwin) +- if host_platform != "cygwin": +- libs.append('X11') +- + ext = Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'], + define_macros=[('WITH_APPINIT', 1)] + defs, + include_dirs = include_dirs, diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.7/no-ldconfig.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.7/no-ldconfig.patch new file mode 100644 index 000000000000..a1f9d68eb166 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.7/no-ldconfig.patch @@ -0,0 +1,100 @@ +From 597e73f2a4b2f0b508127931b36d5540d6941823 Mon Sep 17 00:00:00 2001 +From: Frederik Rietdijk <fridh@fridh.nl> +Date: Mon, 28 Aug 2017 09:24:06 +0200 +Subject: [PATCH] Don't use ldconfig + +--- + Lib/ctypes/util.py | 70 ++---------------------------------------------------- + 1 file changed, 2 insertions(+), 68 deletions(-) + +diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py +index 5e8b31a854..7b45ce6c15 100644 +--- a/Lib/ctypes/util.py ++++ b/Lib/ctypes/util.py +@@ -94,46 +94,7 @@ elif os.name == "posix": + import re, tempfile + + def _findLib_gcc(name): +- # Run GCC's linker with the -t (aka --trace) option and examine the +- # library name it prints out. The GCC command will fail because we +- # haven't supplied a proper program with main(), but that does not +- # matter. +- expr = os.fsencode(r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)) +- +- c_compiler = shutil.which('gcc') +- if not c_compiler: +- c_compiler = shutil.which('cc') +- if not c_compiler: +- # No C compiler available, give up +- return None +- +- temp = tempfile.NamedTemporaryFile() +- try: +- args = [c_compiler, '-Wl,-t', '-o', temp.name, '-l' + name] +- +- env = dict(os.environ) +- env['LC_ALL'] = 'C' +- env['LANG'] = 'C' +- try: +- proc = subprocess.Popen(args, +- stdout=subprocess.PIPE, +- stderr=subprocess.STDOUT, +- env=env) +- except OSError: # E.g. bad executable +- return None +- with proc: +- trace = proc.stdout.read() +- finally: +- try: +- temp.close() +- except FileNotFoundError: +- # Raised if the file was already removed, which is the normal +- # behaviour of GCC if linking fails +- pass +- res = re.search(expr, trace) +- if not res: +- return None +- return os.fsdecode(res.group(0)) ++ return None + + + if sys.platform == "sunos5": +@@ -255,34 +216,7 @@ elif os.name == "posix": + else: + + def _findSoname_ldconfig(name): +- import struct +- if struct.calcsize('l') == 4: +- machine = os.uname().machine + '-32' +- else: +- machine = os.uname().machine + '-64' +- mach_map = { +- 'x86_64-64': 'libc6,x86-64', +- 'ppc64-64': 'libc6,64bit', +- 'sparc64-64': 'libc6,64bit', +- 's390x-64': 'libc6,64bit', +- 'ia64-64': 'libc6,IA-64', +- } +- abi_type = mach_map.get(machine, 'libc6') +- +- # XXX assuming GLIBC's ldconfig (with option -p) +- regex = r'\s+(lib%s\.[^\s]+)\s+\(%s' +- regex = os.fsencode(regex % (re.escape(name), abi_type)) +- try: +- with subprocess.Popen(['/sbin/ldconfig', '-p'], +- stdin=subprocess.DEVNULL, +- stderr=subprocess.DEVNULL, +- stdout=subprocess.PIPE, +- env={'LC_ALL': 'C', 'LANG': 'C'}) as p: +- res = re.search(regex, p.stdout.read()) +- if res: +- return os.fsdecode(res.group(1)) +- except OSError: +- pass ++ return None + + def _findLib_ld(name): + # See issue #9998 for why this is needed +-- +2.15.0 + diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/default.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/default.nix new file mode 100644 index 000000000000..ed6c8c9893b4 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/default.nix @@ -0,0 +1,260 @@ +{ stdenv, fetchurl, fetchpatch +, bzip2 +, expat +, libffi +, gdbm +, lzma +, ncurses +, openssl +, readline +, sqlite +, tcl ? null, tk ? null, tix ? null, libX11 ? null, xorgproto ? null, x11Support ? false +, zlib +, callPackage +, self +, CF, configd +, python-setup-hook +, nukeReferences +# For the Python package set +, packageOverrides ? (self: super: {}) +, buildPackages +, sourceVersion +, sha256 +, passthruFun +, bash +}: + +assert x11Support -> tcl != null + && tk != null + && xorgproto != null + && libX11 != null; +with stdenv.lib; + +let + + passthru = passthruFun rec { + inherit self sourceVersion packageOverrides; + implementation = "cpython"; + libPrefix = "python${pythonVersion}"; + executable = libPrefix; + pythonVersion = with sourceVersion; "${major}.${minor}"; + sitePackages = "lib/${libPrefix}/site-packages"; + inherit pythonForBuild; + }; + + version = with sourceVersion; "${major}.${minor}.${patch}${suffix}"; + + nativeBuildInputs = [ + nukeReferences + ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + buildPackages.stdenv.cc + pythonForBuild + ]; + + buildInputs = filter (p: p != null) [ + zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ] + ++ optionals x11Support [ tcl tk libX11 xorgproto ] + ++ optionals stdenv.isDarwin [ CF configd ]; + + hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false); + + pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}; + + pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then + "$out/bin/python" + else pythonForBuild.interpreter; + +in with passthru; stdenv.mkDerivation { + pname = "python3"; + inherit version; + + inherit buildInputs nativeBuildInputs; + + src = fetchurl { + url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz"; + inherit sha256; + }; + + prePatch = optionalString stdenv.isDarwin '' + substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"' + substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' ' + ''; + + patches = [ + # Disable the use of ldconfig in ctypes.util.find_library (since + # ldconfig doesn't work on NixOS), and don't use + # ctypes.util.find_library during the loading of the uuid module + # (since it will do a futile invocation of gcc (!) to find + # libuuid, slowing down program startup a lot). + (./. + "/${sourceVersion.major}.${sourceVersion.minor}/no-ldconfig.patch") + ] ++ optionals isPy35 [ + # Backports support for LD_LIBRARY_PATH from 3.6 + ./3.5/ld_library_path.patch + ] ++ optionals isPy37 [ + # Fix darwin build https://bugs.python.org/issue34027 + (fetchpatch { + url = https://bugs.python.org/file47666/darwin-libutil.patch; + sha256 = "0242gihnw3wfskl4fydp2xanpl8k5q7fj4dp7dbbqf46a4iwdzpa"; + }) + ] ++ optionals (isPy3k && hasDistutilsCxxPatch) [ + # Fix for http://bugs.python.org/issue1222585 + # Upstream distutils is calling C compiler to compile C++ code, which + # only works for GCC and Apple Clang. This makes distutils to call C++ + # compiler when needed. + (fetchpatch { + url = "https://bugs.python.org/file48016/python-3.x-distutils-C++.patch"; + sha256 = "1h18lnpx539h5lfxyk379dxwr8m2raigcjixkf133l4xy3f4bzi2"; + }) + ]; + + postPatch = '' + '' + optionalString (x11Support && (tix != null)) '' + substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" + ''; + + CPPFLAGS = "${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"; + LDFLAGS = "${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"; + LIBS = "${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"; + NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s"; + # Determinism: We fix the hashes of str, bytes and datetime objects. + PYTHONHASHSEED=0; + + configureFlags = [ + "--enable-shared" + "--with-threads" + "--without-ensurepip" + "--with-system-expat" + "--with-system-ffi" + "--with-openssl=${openssl.dev}" + ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "ac_cv_buggy_getaddrinfo=no" + # Assume little-endian IEEE 754 floating point when cross compiling + "ac_cv_little_endian_double=yes" + "ac_cv_big_endian_double=no" + "ac_cv_mixed_endian_double=no" + "ac_cv_x87_double_rounding=yes" + "ac_cv_tanh_preserves_zero_sign=yes" + # Generally assume that things are present and work + "ac_cv_posix_semaphores_enabled=yes" + "ac_cv_broken_sem_getvalue=no" + "ac_cv_wchar_t_signed=yes" + "ac_cv_rshift_extends_sign=yes" + "ac_cv_broken_nice=no" + "ac_cv_broken_poll=no" + "ac_cv_working_tzset=yes" + "ac_cv_have_long_long_format=yes" + "ac_cv_have_size_t_format=yes" + "ac_cv_computed_gotos=yes" + "ac_cv_file__dev_ptmx=yes" + "ac_cv_file__dev_ptc=yes" + ] ++ optionals stdenv.hostPlatform.isLinux [ + # Never even try to use lchmod on linux, + # don't rely on detecting glibc-isms. + "ac_cv_func_lchmod=no" + ]; + + preConfigure = '' + for i in /usr /sw /opt /pkg; do # improve purity + substituteInPlace ./setup.py --replace $i /no-such-path + done + '' + optionalString stdenv.isDarwin '' + export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2" + export MACOSX_DEPLOYMENT_TARGET=10.6 + '' + optionalString (isPy3k && pythonOlder "3.7") '' + # Determinism: The interpreter is patched to write null timestamps when compiling python files. + # This way python does not try to update them when we freeze timestamps in nix store. + export DETERMINISTIC_BUILD=1; + '' + optionalString stdenv.hostPlatform.isMusl '' + export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000" + ''; + + setupHook = python-setup-hook sitePackages; + + postInstall = '' + # needed for some packages, especially packages that backport functionality + # to 2.x from 3.x + for item in $out/lib/${libPrefix}/test/*; do + if [[ "$item" != */test_support.py* + && "$item" != */test/support + && "$item" != */test/libregrtest + && "$item" != */test/regrtest.py* ]]; then + rm -rf "$item" + else + echo $item + fi + done + touch $out/lib/${libPrefix}/test/__init__.py + + ln -s "$out/include/${executable}m" "$out/include/${executable}" + + # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484 + echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py + + # Determinism: Windows installers were not deterministic. + # We're also not interested in building Windows installers. + find "$out" -name 'wininst*.exe' | xargs -r rm -f + + # Use Python3 as default python + ln -s "$out/bin/idle3" "$out/bin/idle" + ln -s "$out/bin/pydoc3" "$out/bin/pydoc" + ln -s "$out/bin/python3" "$out/bin/python" + ln -s "$out/bin/python3-config" "$out/bin/python-config" + ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc" + + # Get rid of retained dependencies on -dev packages, and remove + # some $TMPDIR references to improve binary reproducibility. + # Note that the .pyc file of _sysconfigdata.py should be regenerated! + for i in $out/lib/${libPrefix}/_sysconfigdata*.py $out/lib/${libPrefix}/config-${sourceVersion.major}${sourceVersion.minor}*/Makefile; do + sed -i $i -e "s|$TMPDIR|/no-such-path|g" + done + + # Further get rid of references. https://github.com/NixOS/nixpkgs/issues/51668 + find $out/lib/python*/config-* -type f -print -exec nuke-refs -e $out '{}' + + find $out/lib -name '_sysconfigdata*.py*' -print -exec nuke-refs -e $out '{}' + + + # Determinism: rebuild all bytecode + # We exclude lib2to3 because that's Python 2 code which fails + # We rebuild three times, once for each optimization level + # Python 3.7 implements PEP 552, introducing support for deterministic bytecode. + # This is automatically used when `SOURCE_DATE_EPOCH` is set. + find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i - + find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i - + find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i - + ''; + + preFixup = stdenv.lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) '' + # Ensure patch-shebangs uses shebangs of host interpreter. + export PATH=${stdenv.lib.makeBinPath [ "$out" bash ]}:$PATH + ''; + + # Enforce that we don't have references to the OpenSSL -dev package, which we + # explicitly specify in our configure flags above. + disallowedReferences = [ + openssl.dev + ] ++ stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + # Ensure we don't have references to build-time packages. + # These typically end up in shebangs. + pythonForBuild buildPackages.bash + ]; + + inherit passthru; + + enableParallelBuilding = true; + + meta = { + homepage = http://python.org; + description = "A high-level dynamically-typed programming language"; + longDescription = '' + Python is a remarkably powerful dynamic programming language that + is used in a wide variety of application domains. Some of its key + distinguishing features include: clear, readable syntax; strong + introspection capabilities; intuitive object orientation; natural + expression of procedural code; full modularity, supporting + hierarchical packages; exception-based error handling; and very + high level dynamic data types. + ''; + license = licenses.psfl; + platforms = with platforms; linux ++ darwin; + maintainers = with maintainers; [ fridh ]; + }; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-html.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-html.nix new file mode 100644 index 000000000000..7727b984d969 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-html.nix @@ -0,0 +1,18 @@ +# This file was generated and will be overwritten by ./generate.sh + +{ stdenv, fetchurl, lib }: + +stdenv.mkDerivation rec { + name = "python27-docs-html-2.7.3"; + src = fetchurl { + url = http://docs.python.org/ftp/python/doc/2.7.3/python-2.7.3-docs-html.tar.bz2; + sha256 = "1hg92n0mzl9w6j33b2h0bf2vy6fsxnpxfdc3qw760vcm0y00155j"; + }; + installPhase = '' + mkdir -p $out/share/doc/python27 + cp -R ./ $out/share/doc/python27/html + ''; + meta = { + maintainers = [ ]; + }; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-pdf-a4.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-pdf-a4.nix new file mode 100644 index 000000000000..0c9673b7d7a9 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-pdf-a4.nix @@ -0,0 +1,18 @@ +# This file was generated and will be overwritten by ./generate.sh + +{ stdenv, fetchurl, lib }: + +stdenv.mkDerivation rec { + name = "python27-docs-pdf-a4-2.7.3"; + src = fetchurl { + url = http://docs.python.org/ftp/python/doc/2.7.3/python-2.7.3-docs-pdf-a4.tar.bz2; + sha256 = "13da88panq5b6qfhf8k4dgqgxkg4ydcac5cx69a3f35s1w90xdjr"; + }; + installPhase = '' + mkdir -p $out/share/doc/python27 + cp -R ./ $out/share/doc/python27/pdf-a4 + ''; + meta = { + maintainers = [ ]; + }; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-pdf-letter.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-pdf-letter.nix new file mode 100644 index 000000000000..9b0018e7cd78 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-pdf-letter.nix @@ -0,0 +1,18 @@ +# This file was generated and will be overwritten by ./generate.sh + +{ stdenv, fetchurl, lib }: + +stdenv.mkDerivation rec { + name = "python27-docs-pdf-letter-2.7.3"; + src = fetchurl { + url = http://docs.python.org/ftp/python/doc/2.7.3/python-2.7.3-docs-pdf-letter.tar.bz2; + sha256 = "0x41phsdrpivhzkchswsliyx3a10n7gzc9irkrw6rz22j81bfydg"; + }; + installPhase = '' + mkdir -p $out/share/doc/python27 + cp -R ./ $out/share/doc/python27/pdf-letter + ''; + meta = { + maintainers = [ ]; + }; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-text.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-text.nix new file mode 100644 index 000000000000..b7bcfb51d29a --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/2.7-text.nix @@ -0,0 +1,18 @@ +# This file was generated and will be overwritten by ./generate.sh + +{ stdenv, fetchurl, lib }: + +stdenv.mkDerivation rec { + name = "python27-docs-text-2.7.3"; + src = fetchurl { + url = http://docs.python.org/ftp/python/doc/2.7.3/python-2.7.3-docs-text.tar.bz2; + sha256 = "1rxlb3jhh3892y65i45nk1y2lx981fr22a5hmfkp9gvjvdykjnzp"; + }; + installPhase = '' + mkdir -p $out/share/doc/python27 + cp -R ./ $out/share/doc/python27/text + ''; + meta = { + maintainers = [ ]; + }; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/docs/default.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/default.nix new file mode 100644 index 000000000000..60e0a0bf1e6e --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/default.nix @@ -0,0 +1,29 @@ +{ stdenv, fetchurl, lib }: + +let +pythonDocs = { + html = { + recurseForDerivations = true; + python27 = import ./2.7-html.nix { + inherit stdenv fetchurl lib; + }; + }; + pdf_a4 = { + recurseForDerivations = true; + python27 = import ./2.7-pdf-a4.nix { + inherit stdenv fetchurl lib; + }; + }; + pdf_letter = { + recurseForDerivations = true; + python27 = import ./2.7-pdf-letter.nix { + inherit stdenv fetchurl lib; + }; + }; + text = { + recurseForDerivations = true; + python27 = import ./2.7-text.nix { + inherit stdenv fetchurl lib; + }; + }; +}; in pythonDocs diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/docs/generate.sh b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/generate.sh new file mode 100755 index 000000000000..bebefc10ccd0 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/generate.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +TYPES="html pdf-a4 pdf-letter text" +URL=http://docs.python.org/ftp/python/doc/VERSION/python-VERSION-docs-TYPE.tar.bz2 +VERSIONS=$(curl http://www.python.org/download/releases/ 2>/dev/null | grep "releases/[123456789]"| cut -d/ -f4 |grep -v "^[12].[012345]" |grep -v "^1.6.1") +echo "Generating expressions for: +${VERSIONS} +" + + +cat >default.nix <<EOF +{ stdenv, fetchurl, lib }: + +let +pythonDocs = { +EOF + +for type in $TYPES; do + cat >>default.nix <<EOF + ${type/-/_} = { + recurseForDerivations = true; +EOF + + for version in $VERSIONS; do + major=$(echo -n ${version}| cut -d. -f1) + minor=$(echo -n ${version}| cut -d. -f2) + outfile=${major}.${minor}-${type}.nix + hash= + if [ -e ${outfile} ]; then + currentversion=$(grep "url =" ${outfile} |cut -d/ -f7) + if [ ${version} = ${currentversion} ]; then + hash=$(grep sha256 ${outfile} | cut -d'"' -f2) + fi + fi + echo "Generating ${outfile}" + url=$(echo -n $URL |sed -e "s,VERSION,${version},g" -e "s,TYPE,${type},") + sha=$(nix-prefetch-url ${url} ${hash}) + + sed -e "s,VERSION,${version}," \ + -e "s,MAJOR,${major}," \ + -e "s,MINOR,${minor}," \ + -e "s,TYPE,${type}," \ + -e "s,URL,${url}," \ + -e "s,SHA,${sha}," < template.nix > ${outfile} + + attrname=python${major}${minor} + cat >>default.nix <<EOF + ${attrname} = import ./${major}.${minor}-${type}.nix { + inherit stdenv fetchurl lib; + }; +EOF + + echo "done." + echo + done + echo " };" >> default.nix +done + +echo "}; in pythonDocs" >> default.nix diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/docs/template.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/template.nix new file mode 100644 index 000000000000..4a0c84fb272e --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/cpython/docs/template.nix @@ -0,0 +1,18 @@ +# This file was generated and will be overwritten by ./generate.sh + +{ stdenv, fetchurl, lib }: + +stdenv.mkDerivation rec { + name = "pythonMAJORMINOR-docs-TYPE-VERSION"; + src = fetchurl { + url = URL; + sha256 = "SHA"; + }; + installPhase = '' + mkdir -p $out/share/doc/pythonMAJORMINOR + cp -R ./ $out/share/doc/pythonMAJORMINOR/TYPE + ''; + meta = { + maintainers = [ ]; + }; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/default.nix b/nixpkgs/pkgs/development/interpreters/python/default.nix new file mode 100644 index 000000000000..f1461d784be0 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/default.nix @@ -0,0 +1,156 @@ +{ pkgs, lib }: + +with pkgs; + +(let + + # Common passthru for all Python interpreters. + passthruFun = + { implementation + , libPrefix + , executable + , sourceVersion + , pythonVersion + , packageOverrides + , sitePackages + , pythonForBuild + , self + }: let + pythonPackages = callPackage ../../../top-level/python-packages.nix { + python = self; + overrides = packageOverrides; + }; + in rec { + isPy27 = pythonVersion == "2.7"; + isPy33 = pythonVersion == "3.3"; # TODO: remove + isPy34 = pythonVersion == "3.4"; # TODO: remove + isPy35 = pythonVersion == "3.5"; + isPy36 = pythonVersion == "3.6"; + isPy37 = pythonVersion == "3.7"; + isPy2 = lib.strings.substring 0 1 pythonVersion == "2"; + isPy3 = lib.strings.substring 0 1 pythonVersion == "3"; + isPy3k = isPy3; + isPyPy = interpreter == "pypy"; + + buildEnv = callPackage ./wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; + withPackages = import ./with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; + interpreter = "${self}/bin/${executable}"; + inherit executable implementation libPrefix pythonVersion sitePackages; + inherit sourceVersion; + pythonAtLeast = lib.versionAtLeast pythonVersion; + pythonOlder = lib.versionOlder pythonVersion; + inherit pythonForBuild; + }; + +in { + + python27 = callPackage ./cpython/2.7 { + self = python27; + sourceVersion = { + major = "2"; + minor = "7"; + patch = "15"; + suffix = ""; + }; + sha256 = "0x2mvz9dp11wj7p5ccvmk9s0hzjk2fa1m462p395l4r6bfnb3n92"; + inherit (darwin) CF configd; + inherit passthruFun; + }; + + python35 = callPackage ./cpython { + self = python35; + sourceVersion = { + major = "3"; + minor = "5"; + patch = "6"; + suffix = ""; + }; + sha256 = "0pqmf51zy2lzhbaj4yya2py2qr653j9152d0rg3p7wi1yl2dwp7m"; + inherit (darwin) CF configd; + inherit passthruFun; + }; + + python36 = callPackage ./cpython { + self = python36; + sourceVersion = { + major = "3"; + minor = "6"; + patch = "8"; + suffix = ""; + }; + sha256 = "14qi6n5gpcjnwy165wi9hkfcmbadc95ny6bxxldknxwmx50n4i1m"; + inherit (darwin) CF configd; + inherit passthruFun; + }; + + python37 = callPackage ./cpython { + self = python37; + sourceVersion = { + major = "3"; + minor = "7"; + patch = "2"; + suffix = ""; + }; + sha256 = "1fzi9d2gibh0wzwidyckzbywsxcsbckgsl05ryxlifxia77fhgyq"; + inherit (darwin) CF configd; + inherit passthruFun; + }; + + pypy27 = callPackage ./pypy { + self = pypy27; + sourceVersion = { + major = "6"; + minor = "0"; + patch = "0"; + }; + sha256 = "1qjwpc8n68sxxlfg36s5vn1h2gdfvvd6lxvr4lzbvfwhzrgqahsw"; + pythonVersion = "2.7"; + db = db.override { dbmSupport = true; }; + python = python27; + inherit passthruFun; + }; + + pypy35 = callPackage ./pypy { + self = pypy35; + sourceVersion = { + major = "6"; + minor = "0"; + patch = "0"; + }; + sha256 = "0lwq8nn0r5yj01bwmkk5p7xvvrp4s550l8184mkmn74d3gphrlwg"; + pythonVersion = "3.5"; + db = db.override { dbmSupport = true; }; + python = python27; + inherit passthruFun; + }; + + pypy27_prebuilt = callPackage ./pypy/prebuilt.nix { + # Not included at top-level + self = pythonInterpreters.pypy27_prebuilt; + sourceVersion = { + major = "6"; + minor = "0"; + patch = "0"; + }; + sha256 = "0rxgnp3fm18b87ln8bbjr13g2fsf4ka4abkaim6m03y9lwmr9gvc"; # linux64 + pythonVersion = "2.7"; + inherit passthruFun; + ncurses = ncurses5; + }; + + pypy35_prebuilt = callPackage ./pypy/prebuilt.nix { + # Not included at top-level + self = pythonInterpreters.pypy35_prebuilt; + sourceVersion = { + major = "6"; + minor = "0"; + patch = "0"; + }; + sha256 = "0j3h08s7wpglghasmym3baycpif5jshvmk9rpav4pwwy5clzmzsc"; # linux64 + pythonVersion = "3.5"; + inherit passthruFun; + ncurses = ncurses5; + }; + +}) \ No newline at end of file diff --git a/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix b/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix new file mode 100644 index 000000000000..4951ae4499f8 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -0,0 +1,122 @@ +# Generic builder. + +{ lib +, config +, python +, wrapPython +, setuptools +, unzip +, ensureNewerSourcesForZipFilesHook +# Whether the derivation provides a Python module or not. +, toPythonModule +, namePrefix +, writeScript +, update-python-libraries +}: + +{ name ? "${attrs.pname}-${attrs.version}" + +# Build-time dependencies for the package +, nativeBuildInputs ? [] + +# Run-time dependencies for the package +, buildInputs ? [] + +# Dependencies needed for running the checkPhase. +# These are added to buildInputs when doCheck = true. +, checkInputs ? [] + +# propagate build dependencies so in case we have A -> B -> C, +# C can import package A propagated by B +, propagatedBuildInputs ? [] + +# DEPRECATED: use propagatedBuildInputs +, pythonPath ? [] + +# used to disable derivation, useful for specific python versions +, disabled ? false + +# Raise an error if two packages are installed with the same name +, catchConflicts ? true + +# Additional arguments to pass to the makeWrapper function, which wraps +# generated binaries. +, makeWrapperArgs ? [] + +# Skip wrapping of python programs altogether +, dontWrapPythonPrograms ? false + +# Remove bytecode from bin folder. +# When a Python script has the extension `.py`, bytecode is generated +# Typically, executables in bin have no extension, so no bytecode is generated. +# However, some packages do provide executables with extensions, and thus bytecode is generated. +, removeBinBytecode ? true + +, meta ? {} + +, passthru ? {} + +, doCheck ? config.doCheckByDefault or false + +, ... } @ attrs: + + +# Keep extra attributes from `attrs`, e.g., `patchPhase', etc. +if disabled +then throw "${name} not supported for interpreter ${python.executable}" +else + +let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attrs [ + "disabled" "checkInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" + ] // { + + name = namePrefix + name; + + nativeBuildInputs = [ + python + wrapPython + ensureNewerSourcesForZipFilesHook + setuptools +# ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools + ] ++ lib.optionals (lib.hasSuffix "zip" (attrs.src.name or "")) [ + unzip + ] ++ nativeBuildInputs; + + buildInputs = buildInputs ++ pythonPath; + + # Propagate python and setuptools. We should stop propagating setuptools. + propagatedBuildInputs = propagatedBuildInputs ++ [ python setuptools ]; + + # Python packages don't have a checkPhase, only an installCheckPhase + doCheck = false; + doInstallCheck = doCheck; + installCheckInputs = checkInputs; + + postFixup = lib.optionalString (!dontWrapPythonPrograms) '' + wrapPythonPrograms + '' + lib.optionalString removeBinBytecode '' + if [ -d "$out/bin" ]; then + rm -rf "$out/bin/__pycache__" # Python 3 + find "$out/bin" -type f -name "*.pyc" -delete # Python 2 + fi + '' + lib.optionalString catchConflicts '' + # Check if we have two packages with the same name in the closure and fail. + # If this happens, something went wrong with the dependencies specs. + # Intentionally kept in a subdirectory, see catch_conflicts/README.md. + ${python.pythonForBuild.interpreter} ${./catch_conflicts}/catch_conflicts.py + '' + attrs.postFixup or ''''; + + # Python packages built through cross-compilation are always for the host platform. + disallowedReferences = lib.optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonForBuild ]; + + meta = { + # default to python's platforms + platforms = python.meta.platforms; + isBuildPythonPackage = python.meta.platforms; + } // meta; +})); + +passthru.updateScript = let + filename = builtins.head (lib.splitString ":" self.meta.position); + in attrs.passthru.updateScript or [ update-python-libraries filename ]; +in lib.extendDerivation true passthru self diff --git a/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix b/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix new file mode 100644 index 000000000000..193f134e5c6c --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix @@ -0,0 +1,148 @@ +{ stdenv, substituteAll, fetchurl +, zlib ? null, zlibSupport ? true, bzip2, pkgconfig, libffi +, sqlite, openssl, ncurses, python, expat, tcl, tk, tix, xlibsWrapper, libX11 +, callPackage, self, gdbm, db, lzma +, python-setup-hook +# For the Python package set +, packageOverrides ? (self: super: {}) +, sourceVersion +, pythonVersion +, sha256 +, passthruFun +}: + +assert zlibSupport -> zlib != null; + +with stdenv.lib; + +let + isPy3k = substring 0 1 pythonVersion == "3"; + passthru = passthruFun rec { + inherit self sourceVersion pythonVersion packageOverrides; + implementation = "pypy"; + libPrefix = "pypy${pythonVersion}"; + executable = "pypy${if isPy3k then "3" else ""}"; + pythonForBuild = self; # No cross-compiling for now. + sitePackages = "site-packages"; + }; + pname = passthru.executable; + version = with sourceVersion; "${major}.${minor}.${patch}"; + pythonForPypy = python.withPackages (ppkgs: [ ppkgs.pycparser ]); + +in with passthru; stdenv.mkDerivation rec { + inherit pname version; + + src = fetchurl { + url = "https://bitbucket.org/pypy/pypy/get/release-pypy${pythonVersion}-v${version}.tar.bz2"; + inherit sha256; + }; + + nativeBuildInputs = [ pkgconfig ]; + buildInputs = [ + bzip2 openssl pythonForPypy libffi ncurses expat sqlite tk tcl xlibsWrapper libX11 gdbm db + ] ++ optionals isPy3k [ + lzma + ] ++ optionals (stdenv ? cc && stdenv.cc.libc != null) [ + stdenv.cc.libc + ] ++ optionals zlibSupport [ + zlib + ]; + + hardeningDisable = optional stdenv.isi686 "pic"; + + C_INCLUDE_PATH = makeSearchPathOutput "dev" "include" buildInputs; + LIBRARY_PATH = makeLibraryPath buildInputs; + LD_LIBRARY_PATH = makeLibraryPath (filter (x : x.outPath != stdenv.cc.libc.outPath or "") buildInputs); + + patches = [ + (substituteAll { + src = ./tk_tcl_paths.patch; + inherit tk tcl; + tk_dev = tk.dev; + tcl_dev = tcl; + tk_libprefix = tk.libPrefix; + tcl_libprefix = tcl.libPrefix; + }) + ]; + + postPatch = '' + substituteInPlace "lib-python/${if isPy3k then "3/tkinter/tix.py" else "2.7/lib-tk/Tix.py"}" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" + + # hint pypy to find nix ncurses + substituteInPlace pypy/module/_minimal_curses/fficurses.py \ + --replace "/usr/include/ncurses/curses.h" "${ncurses.dev}/include/curses.h" \ + --replace "ncurses/curses.h" "${ncurses.dev}/include/curses.h" \ + --replace "ncurses/term.h" "${ncurses.dev}/include/term.h" \ + --replace "libraries=['curses']" "libraries=['ncurses']" + + sed -i "s@libraries=\['sqlite3'\]\$@libraries=['sqlite3'], include_dirs=['${sqlite.dev}/include'], library_dirs=['${sqlite.out}/lib']@" lib_pypy/_sqlite3_build.py + ''; + + buildPhase = '' + ${pythonForPypy.interpreter} rpython/bin/rpython \ + --make-jobs="$NIX_BUILD_CORES" \ + -Ojit \ + --batch pypy/goal/targetpypystandalone.py + ''; + + setupHook = python-setup-hook sitePackages; + + doCheck = true; + checkPhase = let + disabledTests = [ + # disable shutils because it assumes gid 0 exists + "test_shutil" + # disable socket because it has two actual network tests that fail + "test_socket" + ] ++ optionals (!isPy3k) [ + # disable test_urllib2net, test_urllib2_localnet, and test_urllibnet because they require networking (example.com) + "test_urllib2net" + "test_urllibnet" + "test_urllib2_localnet" + ] ++ optionals isPy3k [ + # disable asyncio due to https://github.com/NixOS/nix/issues/1238 + "test_asyncio" + # disable os due to https://github.com/NixOS/nixpkgs/issues/10496 + "test_os" + # disable pathlib due to https://bitbucket.org/pypy/pypy/pull-requests/594 + "test_pathlib" + # disable tarfile because it assumes gid 0 exists + "test_tarfile" + ]; + in '' + export TERMINFO="${ncurses.out}/share/terminfo/"; + export TERM="xterm"; + export HOME="$TMPDIR"; + + ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${concatStringsSep " or " disabledTests})' lib-python + ''; + + installPhase = '' + mkdir -p $out/{bin,include,lib,${executable}-c} + + cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c + cp lib${executable}-c.so $out/lib/ + ln -s $out/${executable}-c/${executable}-c $out/bin/${executable} + + # other packages expect to find stuff according to libPrefix + ln -s $out/${executable}/include $out/include/${libPrefix} + ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix} + + # verify cffi modules + $out/bin/${executable} -c ${if isPy3k then "'import tkinter;import sqlite3;import curses;import lzma'" else "'import Tkinter;import sqlite3;import curses'"} + + # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484 + echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py + ''; + + inherit passthru; + enableParallelBuilding = true; # almost no parallelization without STM + + meta = with stdenv.lib; { + homepage = http://pypy.org/; + description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})"; + license = licenses.mit; + platforms = [ "i686-linux" "x86_64-linux" ]; + maintainers = with maintainers; [ andersk ]; + }; +} diff --git a/nixpkgs/pkgs/development/interpreters/python/pypy/prebuilt.nix b/nixpkgs/pkgs/development/interpreters/python/pypy/prebuilt.nix new file mode 100644 index 000000000000..ee556ba05bf5 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/pypy/prebuilt.nix @@ -0,0 +1,123 @@ +{ stdenv +, fetchurl +, python-setup-hook +, self +, which +# Dependencies +, bzip2 +, zlib +, openssl +, expat +, libffi +, ncurses +, tcl +, tk +# For the Python package set +, packageOverrides ? (self: super: {}) +, sourceVersion +, pythonVersion +, sha256 +, passthruFun +}: + +# This version of PyPy is primarily added to speed-up translation of +# our PyPy source build when developing that expression. + +with stdenv.lib; + +let + isPy3k = majorVersion == "3"; + passthru = passthruFun rec { + inherit self sourceVersion pythonVersion packageOverrides; + implementation = "pypy"; + libPrefix = "pypy${pythonVersion}"; + executable = "pypy${if isPy3k then "3" else ""}"; + pythonForBuild = self; # Not possible to cross-compile with. + sitePackages = "site-packages"; + }; + pname = "${passthru.executable}_prebuilt"; + version = with sourceVersion; "${major}.${minor}.${patch}"; + + majorVersion = substring 0 1 pythonVersion; + + setupHook = python-setup-hook sitePackages; + + deps = [ + bzip2 + zlib + openssl + expat + libffi + ncurses + tcl + tk + ]; + +in with passthru; stdenv.mkDerivation { + inherit pname version; + + src = fetchurl { + url= "https://bitbucket.org/pypy/pypy/downloads/pypy${majorVersion}-v${version}-linux64.tar.bz2"; + inherit sha256; + }; + + buildInputs = [ which ]; + + installPhase = '' + mkdir -p $out/lib + echo "Moving files to $out" + mv -t $out bin include lib-python lib_pypy site-packages + + mv $out/bin/libpypy*-c.so $out/lib/ + + rm $out/bin/*.debug + + echo "Patching binaries" + interpreter=$(patchelf --print-interpreter $(readlink -f $(which patchelf))) + patchelf --set-interpreter $interpreter \ + --set-rpath $out/lib \ + $out/bin/pypy* + + pushd $out + find {lib,lib_pypy*} -name "*.so" -exec patchelf --replace-needed "libbz2.so.1.0" "libbz2.so.1" {} \; + find {lib,lib_pypy*} -name "*.so" -exec patchelf --set-rpath ${stdenv.lib.makeLibraryPath deps} {} \; + + echo "Removing bytecode" + find . -name "__pycache__" -type d -depth -exec rm -rf {} \; + popd + ''; + + doInstallCheck = true; + + # Check whether importing of (extension) modules functions + installCheckPhase = let + modules = [ + "ssl" + "sys" + "curses" + ] ++ optionals (!isPy3k) [ + "Tkinter" + ] ++ optionals isPy3k [ + "tkinter" + ]; + imports = concatMapStringsSep "; " (x: "import ${x}") modules; + in '' + echo "Testing whether we can import modules" + $out/bin/${executable} -c '${imports}' + ''; + + setupHook = python-setup-hook sitePackages; + + donPatchElf = true; + dontStrip = true; + + inherit passthru; + + meta = with stdenv.lib; { + homepage = http://pypy.org/; + description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})"; + license = licenses.mit; + platforms = [ "x86_64-linux" ]; + }; + +} \ No newline at end of file diff --git a/nixpkgs/pkgs/development/interpreters/python/pypy/tk_tcl_paths.patch b/nixpkgs/pkgs/development/interpreters/python/pypy/tk_tcl_paths.patch new file mode 100644 index 000000000000..92bbfc557b35 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/pypy/tk_tcl_paths.patch @@ -0,0 +1,17 @@ +--- pypy-pypy-84a2f3e6a7f8.org/lib_pypy/_tkinter/tklib_build.py 2017-10-03 11:49:20.000000000 +0100 ++++ pypy-pypy-84a2f3e6a7f8/lib_pypy/_tkinter/tklib_build.py 2017-11-21 13:20:51.398607530 +0000 +@@ -24,11 +24,11 @@ + else: + # On some Linux distributions, the tcl and tk libraries are + # stored in /usr/include, so we must check this case also +- libdirs = [] ++ libdirs = ["@tcl@/lib", "@tk@/lib"] + found = False + for _ver in ['', '8.6', '8.5']: +- incdirs = ['/usr/include/tcl' + _ver] +- linklibs = ['tcl' + _ver, 'tk' + _ver] ++ incdirs = ['@tcl_dev@/include', '@tk_dev@/include'] ++ linklibs = ['@tcl_libprefix@', '@tk_libprefix@'] + if os.path.isdir(incdirs[0]): + found = True + break diff --git a/nixpkgs/pkgs/development/interpreters/python/run_setup.py b/nixpkgs/pkgs/development/interpreters/python/run_setup.py new file mode 100644 index 000000000000..e3a530eb0cb6 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/run_setup.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +import setuptools +import tokenize + +__file__='setup.py'; + +exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec')) diff --git a/nixpkgs/pkgs/development/interpreters/python/setup-hook.nix b/nixpkgs/pkgs/development/interpreters/python/setup-hook.nix new file mode 100644 index 000000000000..b66bd1cc5f69 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/setup-hook.nix @@ -0,0 +1,13 @@ +{ runCommand }: + +sitePackages: + +let + hook = ./setup-hook.sh; +in runCommand "python-setup-hook.sh" { + inherit sitePackages; +} '' + cp ${hook} hook.sh + substituteAllInPlace hook.sh + mv hook.sh $out +'' diff --git a/nixpkgs/pkgs/development/interpreters/python/setup-hook.sh b/nixpkgs/pkgs/development/interpreters/python/setup-hook.sh new file mode 100644 index 000000000000..77ec9e9ac0bf --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/setup-hook.sh @@ -0,0 +1,24 @@ +addPythonPath() { + addToSearchPathWithCustomDelimiter : PYTHONPATH $1/@sitePackages@ +} + +toPythonPath() { + local paths="$1" + local result= + for i in $paths; do + p="$i/@sitePackages@" + result="${result}${result:+:}$p" + done + echo $result +} + +addEnvHooks "$hostOffset" addPythonPath + +# Determinism: The interpreter is patched to write null timestamps when compiling python files. +# This way python doesn't try to update them when we freeze timestamps in nix store. +export DETERMINISTIC_BUILD=1; +# Determinism: We fix the hashes of str, bytes and datetime objects. +export PYTHONHASHSEED=0; +# Determinism. Whenever Python is included, it should not check user site-packages. +# This option is only relevant when the sandbox is disabled. +export PYTHONNOUSERSITE=1; diff --git a/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/default.nix b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/default.nix new file mode 100644 index 000000000000..762ca2bdd34b --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/default.nix @@ -0,0 +1,12 @@ +{ python3, runCommand, git }: + +runCommand "update-python-libraries" { + buildInputs = [ + (python3.withPackages(ps: with ps; [ packaging requests toolz ])) + git + ]; +} '' + cp ${./update-python-libraries.py} $out + patchShebangs $out + substituteInPlace $out --replace 'GIT = "git"' 'GIT = "${git}/bin/git"' +'' \ No newline at end of file diff --git a/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py new file mode 100755 index 000000000000..9292a9307030 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 + +""" +Update a Python package expression by passing in the `.nix` file, or the directory containing it. +You can pass in multiple files or paths. + +You'll likely want to use +`` + $ ./update-python-libraries ../../pkgs/development/python-modules/* +`` +to update all libraries in that folder. +""" + +import argparse +import logging +import os +import re +import requests +import toolz +from concurrent.futures import ThreadPoolExecutor as Pool +from packaging.version import Version as _Version +from packaging.version import InvalidVersion +from packaging.specifiers import SpecifierSet +import collections +import subprocess + +INDEX = "https://pypi.io/pypi" +"""url of PyPI""" + +EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl'] +"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned.""" + +PRERELEASES = False + +GIT = "git" + +import logging +logging.basicConfig(level=logging.INFO) + + +class Version(_Version, collections.abc.Sequence): + + def __init__(self, version): + super().__init__(version) + # We cannot use `str(Version(0.04.21))` because that becomes `0.4.21` + # https://github.com/avian2/unidecode/issues/13#issuecomment-354538882 + self.raw_version = version + + def __getitem__(self, i): + return self._version.release[i] + + def __len__(self): + return len(self._version.release) + + def __iter__(self): + yield from self._version.release + + +def _get_values(attribute, text): + """Match attribute in text and return all matches. + + :returns: List of matches. + """ + regex = '{}\s+=\s+"(.*)";'.format(attribute) + regex = re.compile(regex) + values = regex.findall(text) + return values + +def _get_unique_value(attribute, text): + """Match attribute in text and return unique match. + + :returns: Single match. + """ + values = _get_values(attribute, text) + n = len(values) + if n > 1: + raise ValueError("found too many values for {}".format(attribute)) + elif n == 1: + return values[0] + else: + raise ValueError("no value found for {}".format(attribute)) + +def _get_line_and_value(attribute, text): + """Match attribute in text. Return the line and the value of the attribute.""" + regex = '({}\s+=\s+"(.*)";)'.format(attribute) + regex = re.compile(regex) + value = regex.findall(text) + n = len(value) + if n > 1: + raise ValueError("found too many values for {}".format(attribute)) + elif n == 1: + return value[0] + else: + raise ValueError("no value found for {}".format(attribute)) + + +def _replace_value(attribute, value, text): + """Search and replace value of attribute in text.""" + old_line, old_value = _get_line_and_value(attribute, text) + new_line = old_line.replace(old_value, value) + new_text = text.replace(old_line, new_line) + return new_text + +def _fetch_page(url): + r = requests.get(url) + if r.status_code == requests.codes.ok: + return r.json() + else: + raise ValueError("request for {} failed".format(url)) + + +SEMVER = { + 'major' : 0, + 'minor' : 1, + 'patch' : 2, +} + + +def _determine_latest_version(current_version, target, versions): + """Determine latest version, given `target`. + """ + current_version = Version(current_version) + + def _parse_versions(versions): + for v in versions: + try: + yield Version(v) + except InvalidVersion: + pass + + versions = _parse_versions(versions) + + index = SEMVER[target] + + ceiling = list(current_version[0:index]) + if len(ceiling) == 0: + ceiling = None + else: + ceiling[-1]+=1 + ceiling = Version(".".join(map(str, ceiling))) + + # We do not want prereleases + versions = SpecifierSet(prereleases=PRERELEASES).filter(versions) + + if ceiling is not None: + versions = SpecifierSet(f"<{ceiling}").filter(versions) + + return (max(sorted(versions))).raw_version + + +def _get_latest_version_pypi(package, extension, current_version, target): + """Get latest version and hash from PyPI.""" + url = "{}/{}/json".format(INDEX, package) + json = _fetch_page(url) + + versions = json['releases'].keys() + version = _determine_latest_version(current_version, target, versions) + + try: + releases = json['releases'][version] + except KeyError as e: + raise KeyError('Could not find version {} for {}'.format(version, package)) from e + for release in releases: + if release['filename'].endswith(extension): + # TODO: In case of wheel we need to do further checks! + sha256 = release['digests']['sha256'] + break + else: + sha256 = None + return version, sha256 + + +def _get_latest_version_github(package, extension, current_version, target): + raise ValueError("updating from GitHub is not yet supported.") + + +FETCHERS = { + 'fetchFromGitHub' : _get_latest_version_github, + 'fetchPypi' : _get_latest_version_pypi, + 'fetchurl' : _get_latest_version_pypi, +} + + +DEFAULT_SETUPTOOLS_EXTENSION = 'tar.gz' + + +FORMATS = { + 'setuptools' : DEFAULT_SETUPTOOLS_EXTENSION, + 'wheel' : 'whl' +} + +def _determine_fetcher(text): + # Count occurences of fetchers. + nfetchers = sum(text.count('src = {}'.format(fetcher)) for fetcher in FETCHERS.keys()) + if nfetchers == 0: + raise ValueError("no fetcher.") + elif nfetchers > 1: + raise ValueError("multiple fetchers.") + else: + # Then we check which fetcher to use. + for fetcher in FETCHERS.keys(): + if 'src = {}'.format(fetcher) in text: + return fetcher + + +def _determine_extension(text, fetcher): + """Determine what extension is used in the expression. + + If we use: + - fetchPypi, we check if format is specified. + - fetchurl, we determine the extension from the url. + - fetchFromGitHub we simply use `.tar.gz`. + """ + if fetcher == 'fetchPypi': + try: + src_format = _get_unique_value('format', text) + except ValueError as e: + src_format = None # format was not given + + try: + extension = _get_unique_value('extension', text) + except ValueError as e: + extension = None # extension was not given + + if extension is None: + if src_format is None: + src_format = 'setuptools' + elif src_format == 'flit': + raise ValueError("Don't know how to update a Flit package.") + extension = FORMATS[src_format] + + elif fetcher == 'fetchurl': + url = _get_unique_value('url', text) + extension = os.path.splitext(url)[1] + if 'pypi' not in url: + raise ValueError('url does not point to PyPI.') + + elif fetcher == 'fetchFromGitHub': + raise ValueError('updating from GitHub is not yet implemented.') + + return extension + + +def _update_package(path, target): + + # Read the expression + with open(path, 'r') as f: + text = f.read() + + # Determine pname. + pname = _get_unique_value('pname', text) + + # Determine version. + version = _get_unique_value('version', text) + + # First we check how many fetchers are mentioned. + fetcher = _determine_fetcher(text) + + extension = _determine_extension(text, fetcher) + + new_version, new_sha256 = FETCHERS[fetcher](pname, extension, version, target) + + if new_version == version: + logging.info("Path {}: no update available for {}.".format(path, pname)) + return False + elif Version(new_version) <= Version(version): + raise ValueError("downgrade for {}.".format(pname)) + if not new_sha256: + raise ValueError("no file available for {}.".format(pname)) + + text = _replace_value('version', new_version, text) + text = _replace_value('sha256', new_sha256, text) + + with open(path, 'w') as f: + f.write(text) + + logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version)) + + result = { + 'path' : path, + 'target': target, + 'pname': pname, + 'old_version' : version, + 'new_version' : new_version, + #'fetcher' : fetcher, + } + + return result + + +def _update(path, target): + + # We need to read and modify a Nix expression. + if os.path.isdir(path): + path = os.path.join(path, 'default.nix') + + # If a default.nix does not exist, we quit. + if not os.path.isfile(path): + logging.info("Path {}: does not exist.".format(path)) + return False + + # If file is not a Nix expression, we quit. + if not path.endswith(".nix"): + logging.info("Path {}: does not end with `.nix`.".format(path)) + return False + + try: + return _update_package(path, target) + except ValueError as e: + logging.warning("Path {}: {}".format(path, e)) + return False + + +def _commit(path, pname, old_version, new_version, **kwargs): + """Commit result. + """ + + msg = f'python: {pname}: {old_version} -> {new_version}' + + try: + subprocess.check_call([GIT, 'add', path]) + subprocess.check_call([GIT, 'commit', '-m', msg]) + except subprocess.CalledProcessError as e: + subprocess.check_call([GIT, 'checkout', path]) + raise subprocess.CalledProcessError(f'Could not commit {path}') from e + + return True + + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument('package', type=str, nargs='+') + parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major') + parser.add_argument('--commit', action='store_true', help='Create a commit for each package update') + + args = parser.parse_args() + target = args.target + + packages = list(map(os.path.abspath, args.package)) + + logging.info("Updating packages...") + + # Use threads to update packages concurrently + with Pool() as p: + results = list(p.map(lambda pkg: _update(pkg, target), packages)) + + logging.info("Finished updating packages.") + + # Commits are created sequentially. + if args.commit: + logging.info("Committing updates...") + list(map(lambda x: _commit(**x), filter(bool, results))) + logging.info("Finished committing updates") + + count = sum(map(bool, results)) + logging.info("{} package(s) updated".format(count)) + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/nixpkgs/pkgs/development/interpreters/python/with-packages.nix b/nixpkgs/pkgs/development/interpreters/python/with-packages.nix new file mode 100644 index 000000000000..e1de0b2ee4ca --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/with-packages.nix @@ -0,0 +1,3 @@ +{ buildEnv, pythonPackages }: + +f: let packages = f pythonPackages; in buildEnv.override { extraLibs = packages; } diff --git a/nixpkgs/pkgs/development/interpreters/python/wrap-python.nix b/nixpkgs/pkgs/development/interpreters/python/wrap-python.nix new file mode 100644 index 000000000000..6a19a2152419 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/wrap-python.nix @@ -0,0 +1,52 @@ +{ lib +, python +, makeSetupHook +, makeWrapper }: + +with lib; + +makeSetupHook { + deps = makeWrapper; + substitutions.sitePackages = python.sitePackages; + substitutions.executable = python.interpreter; + substitutions.python = python.pythonForBuild; + substitutions.pythonHost = python; + substitutions.magicalSedExpression = let + # Looks weird? Of course, it's between single quoted shell strings. + # NOTE: Order DOES matter here, so single character quotes need to be + # at the last position. + quoteVariants = [ "'\"'''\"'" "\"\"\"" "\"" "'\"'\"'" ]; # hey Vim: '' + + mkStringSkipper = labelNum: quote: let + label = "q${toString labelNum}"; + isSingle = elem quote [ "\"" "'\"'\"'" ]; + endQuote = if isSingle then "[^\\\\]${quote}" else quote; + in '' + /^[a-z]?${quote}/ { + /${quote}${quote}|${quote}.*${endQuote}/{n;br} + :${label}; n; /^${quote}/{n;br}; /${endQuote}/{n;br}; b${label} + } + ''; + + # This preamble does two things: + # * Sets argv[0] to the original application's name; otherwise it would be .foo-wrapped. + # Python doesn't support `exec -a`. + # * Adds all required libraries to sys.path via `site.addsitedir`. It also handles *.pth files. + preamble = '' + import sys + import site + import functools + sys.argv[0] = '"'$(readlink -f "$f")'"' + functools.reduce(lambda k, p: site.addsitedir(p, k), ['"$([ -n "$program_PYTHONPATH" ] && (echo "'$program_PYTHONPATH'" | sed "s|:|','|g") || true)"'], site._init_pathinfo()) + ''; + + in '' + 1 { + :r + /\\$|,$/{N;br} + /__future__|^ |^ *(#.*)?$/{n;br} + ${concatImapStrings mkStringSkipper quoteVariants} + /^[^# ]/i ${replaceStrings ["\n"] [";"] preamble} + } + ''; +} ./wrap.sh diff --git a/nixpkgs/pkgs/development/interpreters/python/wrap.sh b/nixpkgs/pkgs/development/interpreters/python/wrap.sh new file mode 100644 index 000000000000..b2d65422db45 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/wrap.sh @@ -0,0 +1,128 @@ +# Wrapper around wrapPythonProgramsIn, below. The $pythonPath +# variable is passed in from the buildPythonPackage function. +wrapPythonPrograms() { + wrapPythonProgramsIn "$out/bin" "$out $pythonPath" +} + +# Builds environment variables like PYTHONPATH and PATH walking through closure +# of dependencies. +buildPythonPath() { + local pythonPath="$1" + local path + + # Create an empty table of python paths (see doc on _addToPythonPath + # for how this is used). Build up the program_PATH and program_PYTHONPATH + # variables. + declare -A pythonPathsSeen=() + program_PYTHONPATH= + program_PATH= + pythonPathsSeen["@pythonHost@"]=1 + addToSearchPath program_PATH @pythonHost@/bin + for path in $pythonPath; do + _addToPythonPath $path + done +} + +# Patches a Python script so that it has correct libraries path and executable +# name. +patchPythonScript() { + local f="$1" + + # The magicalSedExpression will invoke a "$(basename "$f")", so + # if you change $f to something else, be sure to also change it + # in pkgs/top-level/python-packages.nix! + # It also uses $program_PYTHONPATH. + sed -i "$f" -re '@magicalSedExpression@' +} + +# Transforms any binaries generated by the setup.py script, replacing them +# with an executable shell script which will set some environment variables +# and then call into the original binary (which has been given a .wrapped +# suffix). +wrapPythonProgramsIn() { + local dir="$1" + local pythonPath="$2" + local f + + buildPythonPath "$pythonPath" + + # Find all regular files in the output directory that are executable. + if [ -d "$dir" ]; then + find "$dir" -type f -perm -0100 -print0 | while read -d "" f; do + # Rewrite "#! .../env python" to "#! /nix/store/.../python". + # Strip suffix, like "3" or "2.7m" -- we don't have any choice on which + # Python to use besides one with this hook anyway. + if head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'; then + sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#!@executable@^" + fi + + if head -n1 "$f" | grep -q '#!.*'; then + # Cross-compilation hack: ensure shebangs are for the host + echo "Rewriting $(head -n 1 $f) to #!@pythonHost@" + sed -i "$f" -e "1 s^#!@python@^#!@pythonHost@^" + fi + + # catch /python and /.python-wrapped + if head -n1 "$f" | grep -q '/\.\?\(python\|pypy\)'; then + # dont wrap EGG-INFO scripts since they are called from python + if echo "$f" | grep -qv EGG-INFO/scripts; then + echo "wrapping \`$f'..." + patchPythonScript "$f" + # wrapProgram creates the executable shell script described + # above. The script will set PYTHONPATH and PATH variables.! + # (see pkgs/build-support/setup-hooks/make-wrapper.sh) + local -a wrap_args=("$f" + --prefix PATH ':' "$program_PATH" + --set PYTHONNOUSERSITE "true" + ) + + # Add any additional arguments provided by makeWrapperArgs + # argument to buildPythonPackage. + local -a user_args="($makeWrapperArgs)" + local -a wrapProgramArgs=("${wrap_args[@]}" "${user_args[@]}") + wrapProgram "${wrapProgramArgs[@]}" + fi + fi + done + fi +} + +# Adds the lib and bin directories to the PYTHONPATH and PATH variables, +# respectively. Recurses on any paths declared in +# `propagated-build-inputs`, while avoiding duplicating paths by +# flagging the directories it has visited in `pythonPathsSeen`. +_addToPythonPath() { + local dir="$1" + # Stop if we've already visited here. + if [ -n "${pythonPathsSeen[$dir]}" ]; then return; fi + pythonPathsSeen[$dir]=1 + # addToSearchPath is defined in stdenv/generic/setup.sh. It will have + # the effect of calling `export program_X=$dir/...:$program_X`. + addToSearchPath program_PYTHONPATH $dir/@sitePackages@ + addToSearchPath program_PATH $dir/bin + + # Inspect the propagated inputs (if they exist) and recur on them. + local prop="$dir/nix-support/propagated-build-inputs" + if [ -e $prop ]; then + local new_path + for new_path in $(cat $prop); do + _addToPythonPath $new_path + done + fi +} + +createBuildInputsPth() { + local category="$1" + local inputs="$2" + if [ foo"$inputs" != foo ]; then + for x in $inputs; do + if $(echo -n $x |grep -q python-recursive-pth-loader); then + continue + fi + if test -d "$x"/@sitePackages@; then + echo $x/@sitePackages@ \ + >> "$out"/@sitePackages@/${name}-nix-python-$category.pth + fi + done + fi +} diff --git a/nixpkgs/pkgs/development/interpreters/python/wrapper.nix b/nixpkgs/pkgs/development/interpreters/python/wrapper.nix new file mode 100644 index 000000000000..27d01e4e1f51 --- /dev/null +++ b/nixpkgs/pkgs/development/interpreters/python/wrapper.nix @@ -0,0 +1,63 @@ +{ stdenv, python, buildEnv, makeWrapper +, extraLibs ? [] +, extraOutputsToInstall ? [] +, postBuild ? "" +, ignoreCollisions ? false +, requiredPythonModules +# Wrap executables with the given argument. +, makeWrapperArgs ? [] +, }: + +# Create a python executable that knows about additional packages. +let + env = let + paths = requiredPythonModules (extraLibs ++ [ python ] ) ; + in buildEnv { + name = "${python.name}-env"; + + inherit paths; + inherit ignoreCollisions; + extraOutputsToInstall = [ "out" ] ++ extraOutputsToInstall; + + postBuild = '' + . "${makeWrapper}/nix-support/setup-hook" + + if [ -L "$out/bin" ]; then + unlink "$out/bin" + fi + mkdir -p "$out/bin" + + for path in ${stdenv.lib.concatStringsSep " " paths}; do + if [ -d "$path/bin" ]; then + cd "$path/bin" + for prg in *; do + if [ -f "$prg" ]; then + rm -f "$out/bin/$prg" + if [ -x "$prg" ]; then + makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set PYTHONHOME "$out" --set PYTHONNOUSERSITE "true" ${stdenv.lib.concatStringsSep " " makeWrapperArgs} + fi + fi + done + fi + done + '' + postBuild; + + inherit (python) meta; + + passthru = python.passthru // { + interpreter = "${env}/bin/${python.executable}"; + inherit python; + env = stdenv.mkDerivation { + name = "interactive-${python.name}-environment"; + nativeBuildInputs = [ env ]; + + buildCommand = '' + echo >&2 "" + echo >&2 "*** Python 'env' attributes are intended for interactive nix-shell sessions, not for building! ***" + echo >&2 "" + exit 1 + ''; + }; + }; + }; +in env |