diff options
Diffstat (limited to 'pkgs/development/interpreters/python')
15 files changed, 144 insertions, 11 deletions
diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix index 23a3d5d84099..abf2e223a4d1 100644 --- a/pkgs/development/interpreters/python/cpython/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -28,6 +28,7 @@ , stripTkinter ? false , rebuildBytecode ? true , stripBytecode ? false +, includeSiteCustomize ? true , static ? false }: @@ -238,7 +239,7 @@ in with passthru; stdenv.mkDerivation { '' + optionalString stripTests '' # Strip tests rm -R $out/lib/python*/test $out/lib/python*/**/test{,s} - '' + '' + '' + optionalString includeSiteCustomize '' # Include a sitecustomize.py file cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py '' + optionalString rebuildBytecode '' diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix index 94f04b73f498..099632b1e853 100644 --- a/pkgs/development/interpreters/python/default.nix +++ b/pkgs/development/interpreters/python/default.nix @@ -96,10 +96,10 @@ in { sourceVersion = { major = "3"; minor = "7"; - patch = "6"; + patch = "7"; suffix = ""; }; - sha256 = "0gskry19ylw91p38pdq36qcgk6h3x5i4ia0ik977kw2943kwr8jm"; + sha256 = "0di1y2cna823qgk6sd2lvpjdm3g2qikdd50i2bjd330dpzqsk806"; inherit (darwin) configd; inherit passthruFun; }; @@ -147,6 +147,7 @@ in { stripTkinter = true; rebuildBytecode = false; stripBytecode = true; + includeSiteCustomize = false; }).overrideAttrs(old: { pname = "python3-minimal"; meta = old.meta // { diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index 47690320e81e..4d736426f3bf 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -89,6 +89,14 @@ in rec { }; } ./python-imports-check-hook.sh) {}; + pythonNamespacesHook = callPackage ({}: + makeSetupHook { + name = "python-namespaces-hook.sh"; + substitutions = { + inherit pythonSitePackages; + }; + } ./python-namespaces-hook.sh) {}; + pythonRemoveBinBytecodeHook = callPackage ({ }: makeSetupHook { name = "python-remove-bin-bytecode-hook"; diff --git a/pkgs/development/interpreters/python/hooks/python-namespaces-hook.sh b/pkgs/development/interpreters/python/hooks/python-namespaces-hook.sh new file mode 100644 index 000000000000..50f21819d176 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/python-namespaces-hook.sh @@ -0,0 +1,40 @@ +# Clean up __init__.py's found in namespace directories +echo "Sourcing python-namespaces-hook" + +pythonNamespacesHook() { + echo "Executing pythonNamespacesHook" + + for namespace in ${pythonNamespaces[@]}; do + echo "Enforcing PEP420 namespace: ${namespace}" + + # split namespace into segments. "azure.mgmt" -> "azure mgmt" + IFS='.' read -ra pathSegments <<< $namespace + constructedPath=$out/@pythonSitePackages@ + + # Need to remove the __init__.py at each namespace level + # E.g `azure/__init__.py` and `azure/mgmt/__init__.py` + # The __pycache__ entry also needs to be removed + for pathSegment in ${pathSegments[@]}; do + constructedPath=${constructedPath}/${pathSegment} + pathToRemove=${constructedPath}/__init__.py + pycachePath=${constructedPath}/__pycache__/__init__* + + if [ -f "$pathToRemove" ]; then + echo "Removing $pathToRemove" + rm "$pathToRemove" + fi + + if [ -f "$pycachePath" ]; then + echo "Removing $pycachePath" + rm "$pycachePath" + fi + done + done + + echo "Finished executing pythonNamespacesHook" +} + +if [ -z "${dontUsePythonNamespacesHook-}" -a -n "${pythonNamespaces-}" ]; then + postFixupHooks+=(pythonNamespacesHook) +fi + diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 180bc63857c2..22938a455852 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -16,6 +16,7 @@ , pipInstallHook , pythonCatchConflictsHook , pythonImportsCheckHook +, pythonNamespacesHook , pythonRemoveBinBytecodeHook , pythonRemoveTestsDirHook , setuptoolsBuildHook @@ -131,6 +132,9 @@ let ] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ # This is a test, however, it should be ran independent of the checkPhase and checkInputs pythonImportsCheckHook + ] ++ lib.optionals (python.pythonAtLeast "3.3") [ + # Optionally enforce PEP420 for python3 + pythonNamespacesHook ] ++ nativeBuildInputs; buildInputs = buildInputs ++ pythonPath; diff --git a/pkgs/development/interpreters/python/sitecustomize.py b/pkgs/development/interpreters/python/sitecustomize.py index e03b244dbc04..72ce951328f1 100644 --- a/pkgs/development/interpreters/python/sitecustomize.py +++ b/pkgs/development/interpreters/python/sitecustomize.py @@ -21,6 +21,19 @@ paths = os.environ.pop('NIX_PYTHONPATH', None) if paths: functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo()) -executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None) -if 'PYTHONEXECUTABLE' not in os.environ and executable: - sys.executable = executable +# Check whether we are in a venv. +# Note Python 2 does not support base_prefix so we assume we are not in a venv. +in_venv = sys.version_info.major == 3 and sys.prefix != sys.base_prefix + +if not in_venv: + executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None) + prefix = os.environ.pop('NIX_PYTHONPREFIX', None) + + if 'PYTHONEXECUTABLE' not in os.environ and executable is not None: + sys.executable = executable + if prefix is not None: + # Because we cannot check with Python 2 whether we are in a venv, + # creating a venv from a Nix env won't work as well with Python 2. + # Also, note that sysconfig does not like it when sys.prefix is set to None + sys.prefix = sys.exec_prefix = prefix + site.PREFIXES.insert(0, prefix) diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix index 37fbe6701148..55065c45d576 100644 --- a/pkgs/development/interpreters/python/tests.nix +++ b/pkgs/development/interpreters/python/tests.nix @@ -2,6 +2,7 @@ , runCommand , substituteAll , lib +, callPackage }: let @@ -36,6 +37,7 @@ let is_venv = "True"; is_nixenv = "False"; }; + # Venv built using Python Nix environment (python.buildEnv) # TODO: Cannot create venv from a nix env # Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1. @@ -49,6 +51,14 @@ let # }; }; + # All PyPy package builds are broken at the moment + integrationTests = lib.optionalAttrs (python.isPy3k && (!python.isPyPy)) rec { + # Before the addition of NIX_PYTHONPREFIX mypy was broken with typed packages + nix-pythonprefix-mypy = callPackage ./tests/test_nix_pythonprefix { + interpreter = python; + }; + }; + testfun = name: attrs: runCommand "${python.name}-tests-${name}" ({ inherit (python) pythonVersion; } // attrs) '' @@ -60,4 +70,4 @@ let touch $out/success ''; -in lib.mapAttrs testfun envs \ No newline at end of file +in lib.mapAttrs testfun envs // integrationTests diff --git a/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/default.nix b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/default.nix new file mode 100644 index 000000000000..05798cbaf1b8 --- /dev/null +++ b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/default.nix @@ -0,0 +1,25 @@ +{ interpreter, writeText, runCommandNoCC }: + +let + + python = let + packageOverrides = self: super: { + typeddep = super.callPackage ./typeddep {}; + }; + in interpreter.override {inherit packageOverrides; self = python;}; + + pythonEnv = python.withPackages(ps: [ + ps.typeddep + ps.mypy + ]); + + pythonScript = writeText "myscript.py" '' + from typeddep import util + s: str = util.echo("hello") + print(s) + ''; + +in runCommandNoCC "${interpreter.name}-site-prefix-mypy-test" {} '' + ${pythonEnv}/bin/mypy ${pythonScript} + touch $out +'' diff --git a/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/default.nix b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/default.nix new file mode 100644 index 000000000000..06219a69fcaf --- /dev/null +++ b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/default.nix @@ -0,0 +1,11 @@ +{ buildPythonPackage }: + + +buildPythonPackage { + + pname = "typeddep"; + version = "1.3.3.7"; + + src = ./.; + +} diff --git a/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/setup.py b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/setup.py new file mode 100644 index 000000000000..25bac69ea09a --- /dev/null +++ b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup + +setup(**{ + 'name': 'typeddep', + 'version': '1.3.3.7', + 'description': 'Minimal repro to test mypy and site prefixes with Nix', + 'long_description': None, + 'author': 'adisbladis', + 'author_email': 'adisbladis@gmail.com', + 'maintainer': None, + 'maintainer_email': None, + 'url': None, + 'packages': ['typeddep'], + 'package_data': {'': ['*']}, + 'install_requires': [], + 'entry_points': {}, + 'python_requires': '>=3.7,<4.0', +}) diff --git a/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/__init__.py b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/__init__.py diff --git a/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/py.typed b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/py.typed diff --git a/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/util.py b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/util.py new file mode 100644 index 000000000000..c1c3ffe74777 --- /dev/null +++ b/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/util.py @@ -0,0 +1,2 @@ +def echo(s: str) -> str: + return s diff --git a/pkgs/development/interpreters/python/tests/test_python.py b/pkgs/development/interpreters/python/tests/test_python.py index f631a172ccc6..011978c62547 100644 --- a/pkgs/development/interpreters/python/tests/test_python.py +++ b/pkgs/development/interpreters/python/tests/test_python.py @@ -27,7 +27,7 @@ class TestCasePython(unittest.TestCase): def test_interpreter(self): self.assertEqual(sys.executable, INTERPRETER) - @unittest.skipIf(IS_NIXENV or IS_PYPY, "Prefix is incorrect and needs to be fixed.") + @unittest.skipIf(IS_PYPY, "Prefix is incorrect and needs to be fixed.") def test_prefix(self): self.assertEqual(sys.prefix, ENV) self.assertEqual(sys.prefix, sys.exec_prefix) @@ -35,9 +35,9 @@ class TestCasePython(unittest.TestCase): def test_site_prefix(self): self.assertTrue(sys.prefix in site.PREFIXES) - @unittest.skipIf(sys.version_info.major==2, "Python 2 does not have base_prefix") + @unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix") def test_base_prefix(self): - if IS_VENV: + if IS_VENV or IS_NIXENV: self.assertNotEqual(sys.prefix, sys.base_prefix) else: self.assertEqual(sys.prefix, sys.base_prefix) diff --git a/pkgs/development/interpreters/python/wrapper.nix b/pkgs/development/interpreters/python/wrapper.nix index b437584024fe..dffad6b98f5e 100644 --- a/pkgs/development/interpreters/python/wrapper.nix +++ b/pkgs/development/interpreters/python/wrapper.nix @@ -37,7 +37,7 @@ let if [ -f "$prg" ]; then rm -f "$out/bin/$prg" if [ -x "$prg" ]; then - makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${if permitUserSite then "" else ''--set PYTHONNOUSERSITE "true"''} ${stdenv.lib.concatStringsSep " " makeWrapperArgs} + makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONPREFIX "$out" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${if permitUserSite then "" else ''--set PYTHONNOUSERSITE "true"''} ${stdenv.lib.concatStringsSep " " makeWrapperArgs} fi fi done |