diff options
author | adisbladis <adisbladis@gmail.com> | 2020-05-06 09:04:51 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-05-06 09:04:51 +0200 |
commit | d7c358645ece3d0c164199336aa46111b85526a0 (patch) | |
tree | 330438954733932af7b1caa68ad2567457f7d71f | |
parent | f1eef414d1310d5a482ac764072b93cdb2a83e41 (diff) | |
parent | 36debc367e69a5e03fef7f1ec9a5d86d6fa7c3ce (diff) | |
download | nixlib-d7c358645ece3d0c164199336aa46111b85526a0.tar nixlib-d7c358645ece3d0c164199336aa46111b85526a0.tar.gz nixlib-d7c358645ece3d0c164199336aa46111b85526a0.tar.bz2 nixlib-d7c358645ece3d0c164199336aa46111b85526a0.tar.lz nixlib-d7c358645ece3d0c164199336aa46111b85526a0.tar.xz nixlib-d7c358645ece3d0c164199336aa46111b85526a0.tar.zst nixlib-d7c358645ece3d0c164199336aa46111b85526a0.zip |
Merge pull request #86988 from adisbladis/poetry2nix-1_8_0
poetry2nix: 1.7.1 -> 1.8.0
13 files changed, 1314 insertions, 1008 deletions
diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix b/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix index fbcee749b7cd..b96316e59d83 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix @@ -1,4 +1,4 @@ -{ pkgs ? import <nixpkgs> {} +{ pkgs ? import <nixpkgs> { } , lib ? pkgs.lib , version }: @@ -33,8 +33,8 @@ pkgs.stdenv.mkDerivation { mv poetry2nix $out/bin wrapProgram $out/bin/poetry2nix --prefix PATH ":" ${lib.makeBinPath [ - pkgs.nix-prefetch-git - ]} + pkgs.nix-prefetch-git + ]} runHook postInstall ''; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix index e7f718519a59..a7fb382c7553 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -1,22 +1,20 @@ -{ pkgs ? import <nixpkgs> {} +{ pkgs ? import <nixpkgs> { } , lib ? pkgs.lib , poetry ? null , poetryLib ? import ./lib.nix { inherit lib pkgs; } }: let - inherit (poetryLib) isCompatible readTOML; + inherit (poetryLib) isCompatible readTOML moduleName; # Poetry2nix version - version = "1.7.1"; + version = "1.8.0"; /* The default list of poetry2nix override overlays */ defaultPoetryOverrides = (import ./overrides.nix { inherit pkgs lib; }); - mkEvalPep508 = import ./pep508.nix { inherit lib poetryLib; stdenv = pkgs.stdenv; }; - getFunctorFn = fn: if builtins.typeOf fn == "set" then fn.__functor else fn; # Map SPDX identifiers to license names @@ -34,95 +32,99 @@ let , overrides ? [ defaultPoetryOverrides ] , python ? pkgs.python3 , pwd ? projectDir + , preferWheels ? false }@attrs: - let - poetryPkg = poetry.override { inherit python; }; - - pyProject = readTOML pyproject; - poetryLock = readTOML poetrylock; - lockFiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock; - - specialAttrs = [ - "overrides" - "poetrylock" - "projectDir" - "pwd" - ]; - passedAttrs = builtins.removeAttrs attrs specialAttrs; - - evalPep508 = mkEvalPep508 python; - - # Filter packages by their PEP508 markers & pyproject interpreter version - partitions = let + let + poetryPkg = poetry.override { inherit python; }; + pyProject = readTOML pyproject; + poetryLock = readTOML poetrylock; + lockFiles = + let + lockfiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock; + in + lib.listToAttrs (lib.mapAttrsToList (n: v: { name = moduleName n; value = v; }) lockfiles); + specialAttrs = [ + "overrides" + "poetrylock" + "projectDir" + "pwd" + "preferWheels" + ]; + passedAttrs = builtins.removeAttrs attrs specialAttrs; + evalPep508 = mkEvalPep508 python; + + # Filter packages by their PEP508 markers & pyproject interpreter version + partitions = + let supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true; in - lib.partition supportsPythonVersion poetryLock.package; - - compatible = partitions.right; - incompatible = partitions.wrong; - - # Create an overriden version of pythonPackages - # - # We need to avoid mixing multiple versions of pythonPackages in the same - # closure as python can only ever have one version of a dependency - baseOverlay = self: super: - let - getDep = depName: self.${depName}; - - lockPkgs = builtins.listToAttrs ( - builtins.map ( - pkgMeta: rec { - name = pkgMeta.name; - value = self.mkPoetryDep ( - pkgMeta // { - inherit pwd; - source = pkgMeta.source or null; - files = lockFiles.${name}; - pythonPackages = self; - sourceSpec = pyProject.tool.poetry.dependencies.${name} or pyProject.tool.poetry.dev-dependencies.${name}; - } - ); - } - ) compatible + lib.partition supportsPythonVersion poetryLock.package; + compatible = partitions.right; + incompatible = partitions.wrong; + + # Create an overriden version of pythonPackages + # + # We need to avoid mixing multiple versions of pythonPackages in the same + # closure as python can only ever have one version of a dependency + baseOverlay = self: super: + let + getDep = depName: self.${depName}; + lockPkgs = builtins.listToAttrs + ( + builtins.map + ( + pkgMeta: rec { + name = moduleName pkgMeta.name; + value = self.mkPoetryDep + ( + pkgMeta // { + inherit pwd preferWheels; + source = pkgMeta.source or null; + files = lockFiles.${name}; + pythonPackages = self; + sourceSpec = pyProject.tool.poetry.dependencies.${name} or pyProject.tool.poetry.dev-dependencies.${name}; + } + ); + } + ) compatible ); - in - lockPkgs; - overlays = builtins.map getFunctorFn ( + in + lockPkgs; + overlays = builtins.map getFunctorFn + ( [ ( self: super: let - hooks = self.callPackage ./hooks {}; + hooks = self.callPackage ./hooks { }; in - { - mkPoetryDep = self.callPackage ./mk-poetry-dep.nix { - inherit pkgs lib python poetryLib; - }; - poetry = poetryPkg; - # The canonical name is setuptools-scm - setuptools-scm = super.setuptools_scm; - - inherit (hooks) removePathDependenciesHook poetry2nixFixupHook; - } + { + mkPoetryDep = self.callPackage ./mk-poetry-dep.nix { + inherit pkgs lib python poetryLib; + }; + poetry = poetryPkg; + # The canonical name is setuptools-scm + setuptools-scm = super.setuptools_scm; + + inherit (hooks) pipBuildHook removePathDependenciesHook poetry2nixFixupHook; + } ) # Null out any filtered packages, we don't want python.pkgs from nixpkgs - (self: super: builtins.listToAttrs (builtins.map (x: { name = x.name; value = null; }) incompatible)) + (self: super: builtins.listToAttrs (builtins.map (x: { name = moduleName x.name; value = null; }) incompatible)) # Create poetry2nix layer baseOverlay ] ++ # User provided overrides overrides ); - - packageOverrides = lib.foldr lib.composeExtensions (self: super: {}) overlays; - - py = python.override { inherit packageOverrides; self = py; }; - in - { - python = py; - poetryPackages = map (pkg: py.pkgs.${pkg.name}) compatible; - poetryLock = poetryLock; - inherit pyProject; - }; + packageOverrides = lib.foldr lib.composeExtensions (self: super: { }) overlays; + py = python.override { inherit packageOverrides; self = py; }; + in + { + python = py; + poetryPackages = map (pkg: py.pkgs.${moduleName pkg.name}) compatible; + poetryLock = poetryLock; + inherit pyProject; + }; /* Returns a package with a python interpreter and all packages specified in the poetry.lock lock file. @@ -136,15 +138,17 @@ let , overrides ? [ defaultPoetryOverrides ] , pwd ? projectDir , python ? pkgs.python3 + , preferWheels ? false }: - let - py = mkPoetryPackages ( + let + py = mkPoetryPackages + ( { - inherit pyproject poetrylock overrides python pwd; + inherit pyproject poetrylock overrides python pwd preferWheels; } ); - in - py.python.withPackages (_: py.poetryPackages); + in + py.python.withPackages (_: py.poetryPackages); /* Creates a Python application from pyproject.toml and poetry.lock */ mkPoetryApplication = @@ -153,79 +157,79 @@ let , pyproject ? projectDir + "/pyproject.toml" , poetrylock ? projectDir + "/poetry.lock" , overrides ? [ defaultPoetryOverrides ] - , meta ? {} + , meta ? { } , python ? pkgs.python3 , pwd ? projectDir + , preferWheels ? false , ... }@attrs: - let - poetryPython = mkPoetryPackages { - inherit pyproject poetrylock overrides python pwd; - }; - py = poetryPython.python; - - inherit (poetryPython) pyProject; - - specialAttrs = [ - "overrides" - "poetrylock" - "projectDir" - "pwd" - "pyproject" - ]; - passedAttrs = builtins.removeAttrs attrs specialAttrs; - - # Get dependencies and filter out depending on interpreter version - getDeps = depAttr: - let - compat = isCompatible py.pythonVersion; - deps = pyProject.tool.poetry.${depAttr} or {}; - depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps); - in - builtins.map ( - dep: - let - pkg = py.pkgs."${dep}"; - constraints = deps.${dep}.python or ""; - isCompat = compat constraints; - in - if isCompat then pkg else null - ) depAttrs; - - getInputs = attr: attrs.${attr} or []; - mkInput = attr: extraInputs: getInputs attr ++ extraInputs; - - buildSystemPkgs = poetryLib.getBuildSystemPkgs { - inherit pyProject; - pythonPackages = py.pkgs; - }; - in - py.pkgs.buildPythonApplication ( - passedAttrs // { - pname = pyProject.tool.poetry.name; - version = pyProject.tool.poetry.version; - - inherit src; - - format = "pyproject"; - - buildInputs = mkInput "buildInputs" buildSystemPkgs; - propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]); - nativeBuildInputs = mkInput "nativeBuildInputs" [ pkgs.yj py.pkgs.removePathDependenciesHook ]; - checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies"); - - passthru = { - python = py; - }; - - meta = meta // { - inherit (pyProject.tool.poetry) description homepage; - inherit (py.meta) platforms; - license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown"); - }; - - } - ); + let + poetryPython = mkPoetryPackages { + inherit pyproject poetrylock overrides python pwd preferWheels; + }; + py = poetryPython.python; + + inherit (poetryPython) pyProject; + specialAttrs = [ + "overrides" + "poetrylock" + "projectDir" + "pwd" + "pyproject" + "preferWheels" + ]; + passedAttrs = builtins.removeAttrs attrs specialAttrs; + + # Get dependencies and filter out depending on interpreter version + getDeps = depAttr: + let + compat = isCompatible (poetryLib.getPythonVersion py); + deps = pyProject.tool.poetry.${depAttr} or { }; + depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps); + in + builtins.map + ( + dep: + let + pkg = py.pkgs."${dep}"; + constraints = deps.${dep}.python or ""; + isCompat = compat constraints; + in if isCompat then pkg else null + ) depAttrs; + getInputs = attr: attrs.${attr} or [ ]; + mkInput = attr: extraInputs: getInputs attr ++ extraInputs; + buildSystemPkgs = poetryLib.getBuildSystemPkgs { + inherit pyProject; + pythonPackages = py.pkgs; + }; + in + py.pkgs.buildPythonApplication + ( + passedAttrs // { + pname = moduleName pyProject.tool.poetry.name; + version = pyProject.tool.poetry.version; + + inherit src; + + format = "pyproject"; + + buildInputs = mkInput "buildInputs" buildSystemPkgs; + propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]); + nativeBuildInputs = mkInput "nativeBuildInputs" [ pkgs.yj py.pkgs.removePathDependenciesHook ]; + checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies"); + + passthru = { + python = py; + }; + + meta = meta // { + inherit (pyProject.tool.poetry) description homepage; + inherit (py.meta) platforms; + license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown"); + }; + + } + ); /* Poetry2nix CLI used to supplement SHA-256 hashes for git dependencies */ cli = import ./cli.nix { inherit pkgs lib version; }; @@ -247,7 +251,7 @@ in defaultSet = defaultPoetryOverrides self super; customSet = fn self super; in - defaultSet // customSet; + defaultSet // customSet; }; /* diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/fetch-wheel.sh b/pkgs/development/tools/poetry2nix/poetry2nix/fetch-wheel.sh new file mode 100644 index 000000000000..97f54b23416d --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/fetch-wheel.sh @@ -0,0 +1,24 @@ +source $stdenv/setup +set -euo pipefail + +curl="curl \ + --location \ + --max-redirs 20 \ + --retry 2 \ + --disable-epsv \ + --cookie-jar cookies \ + --insecure \ + --speed-time 5 \ + -# \ + --fail \ + $curlOpts \ + $NIX_CURL_FLAGS" + +echo "Trying to fetch wheel with predicted URL: $predictedURL" + +$curl $predictedURL --output $out && exit 0 + +echo "Predicted URL '$predictedURL' failed, querying pypi.org" +$curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url +url=$(cat url) +$curl -k $url --output $out diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix index 12d2bc962065..ae5867b83372 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix @@ -2,31 +2,48 @@ , callPackage , makeSetupHook , yj +, wheel +, pip }: let pythonInterpreter = python.pythonForBuild.interpreter; + pythonSitePackages = python.sitePackages; in { - removePathDependenciesHook = callPackage ( - {}: + removePathDependenciesHook = callPackage + ( + {}: makeSetupHook { name = "remove-path-dependencies.sh"; - deps = []; + deps = [ ]; substitutions = { inherit pythonInterpreter; yj = "${yj}/bin/yj"; pyprojectPatchScript = "${./pyproject-without-path.py}"; }; } ./remove-path-dependencies.sh - ) {}; + ) { }; - poetry2nixFixupHook = callPackage ( - {}: + pipBuildHook = callPackage + ( + { pip, wheel }: + makeSetupHook { + name = "pip-build-hook.sh"; + deps = [ pip wheel ]; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + }; + } ./pip-build-hook.sh + ) { }; + + poetry2nixFixupHook = callPackage + ( + {}: makeSetupHook { name = "fixup-hook.sh"; - deps = []; + deps = [ ]; } ./fixup-hook.sh - ) {}; + ) { }; } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pip-build-hook.sh b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pip-build-hook.sh new file mode 100644 index 000000000000..fa7b698fb510 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pip-build-hook.sh @@ -0,0 +1,50 @@ +# Setup hook to use for pip projects +echo "Sourcing pip-build-hook" + +pipBuildPhase() { + echo "Executing pipBuildPhase" + runHook preBuild + + # Prefer using setup.py to avoid build-system dependencies if we have a setup.py + if [ -z "${dontPreferSetupPy-}" ]; then + if test -e setup.py && test -e pyproject.toml; then + echo "Removing pyproject.toml..." + rm -f pyproject.toml + fi + fi + + mkdir -p dist + echo "Creating a wheel..." + @pythonInterpreter@ -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist . + echo "Finished creating a wheel..." + + runHook postBuild + echo "Finished executing pipBuildPhase" +} + +pipShellHook() { + echo "Executing pipShellHook" + runHook preShellHook + + # Long-term setup.py should be dropped. + if [ -e pyproject.toml ]; then + tmp_path=$(mktemp -d) + export PATH="$tmp_path/bin:$PATH" + export PYTHONPATH="$tmp_path/@pythonSitePackages@:$PYTHONPATH" + mkdir -p "$tmp_path/@pythonSitePackages@" + @pythonInterpreter@ -m pip install -e . --prefix "$tmp_path" >&2 + fi + + runHook postShellHook + echo "Finished executing pipShellHook" +} + +if [ -z "${dontUsePipBuild-}" ] && [ -z "${buildPhase-}" ]; then + echo "Using pipBuildPhase" + buildPhase=pipBuildPhase +fi + +if [ -z "${shellHook-}" ]; then + echo "Using pipShellHook" + shellHook=pipShellHook +fi diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-path.py b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-path.py index bb61e4a5cb4b..5d8fbcfe6b4d 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-path.py +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-path.py @@ -6,7 +6,14 @@ import sys data = json.load(sys.stdin) -for dep in data['tool']['poetry']['dependencies'].values(): + +def get_deep(o, path): + for p in path.split('.'): + o = o.get(p, {}) + return o + + +for dep in get_deep(data, 'tool.poetry.dependencies').values(): if isinstance(dep, dict): try: del dep['path']; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix index f4497afb8bc9..ed47837ee439 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix @@ -8,6 +8,20 @@ let genList (i: if i == idx then value else (builtins.elemAt list i)) (length list) ); + # Do some canonicalisation of module names + moduleName = name: lib.toLower (lib.replaceStrings [ "_" "." ] [ "-" "-" ] name); + + # Get a full semver pythonVersion from a python derivation + getPythonVersion = python: + let + pyVer = lib.splitVersion python.pythonVersion ++ [ "0" ]; + ver = lib.splitVersion python.version; + major = l: lib.elemAt l 0; + minor = l: lib.elemAt l 1; + joinVersion = v: lib.concatStringsSep "." v; + in + joinVersion ( if major pyVer == major ver && minor pyVer == minor ver then ver else pyVer); + # Compare a semver expression with a version isCompatible = version: let @@ -18,41 +32,41 @@ let }; splitRe = "(" + (builtins.concatStringsSep "|" (builtins.map (x: lib.replaceStrings [ "|" ] [ "\\|" ] x) (lib.attrNames operators))) + ")"; in - expr: + expr: + let + tokens = builtins.filter (x: x != "") (builtins.split splitRe expr); + combine = acc: v: let - tokens = builtins.filter (x: x != "") (builtins.split splitRe expr); - combine = acc: v: - let - isOperator = builtins.typeOf v == "list"; - operator = if isOperator then (builtins.elemAt v 0) else acc.operator; - in - if isOperator then (acc // { inherit operator; }) else { - inherit operator; - state = operators."${operator}" acc.state (satisfiesSemver version v); - }; - initial = { operator = "&&"; state = true; }; + isOperator = builtins.typeOf v == "list"; + operator = if isOperator then (builtins.elemAt v 0) else acc.operator; in - if expr == "" then true else (builtins.foldl' combine initial tokens).state; - + if isOperator then (acc // { inherit operator; }) else { + inherit operator; + state = operators."${operator}" acc.state (satisfiesSemver version v); + }; + initial = { operator = "&&"; state = true; }; + in if expr == "" then true else (builtins.foldl' combine initial tokens).state; fromTOML = builtins.fromTOML or ( - toml: builtins.fromJSON ( - builtins.readFile ( - pkgs.runCommand "from-toml" - { - inherit toml; - allowSubstitutes = false; - preferLocalBuild = true; - } - '' - ${pkgs.remarshal}/bin/remarshal \ - -if toml \ - -i <(echo "$toml") \ - -of json \ - -o $out - '' + toml: builtins.fromJSON + ( + builtins.readFile + ( + pkgs.runCommand "from-toml" + { + inherit toml; + allowSubstitutes = false; + preferLocalBuild = true; + } + '' + ${pkgs.remarshal}/bin/remarshal \ + -if toml \ + -i <(echo "$toml") \ + -of json \ + -o $out + '' + ) ) - ) ); readTOML = path: fromTOML (builtins.readFile path); @@ -63,10 +77,61 @@ let let ml = pkgs.pythonManylinuxPackages; in - if lib.strings.hasInfix "manylinux1" f then { pkg = [ ml.manylinux1 ]; str = "1"; } - else if lib.strings.hasInfix "manylinux2010" f then { pkg = [ ml.manylinux2010 ]; str = "2010"; } - else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; } - else { pkg = []; str = null; }; + if lib.strings.hasInfix "manylinux1" f then { pkg = [ ml.manylinux1 ]; str = "1"; } + else if lib.strings.hasInfix "manylinux2010" f then { pkg = [ ml.manylinux2010 ]; str = "2010"; } + else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; } + else { pkg = [ ]; str = null; }; + + # Predict URL from the PyPI index. + # Args: + # pname: package name + # file: filename including extension + # hash: SRI hash + # kind: Language implementation and version tag + predictURLFromPypi = lib.makeOverridable + ( + { pname, file, hash, kind }: + "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}" + ); + + + # Fetch the wheels from the PyPI index. + # We need to first get the proper URL to the wheel. + # Args: + # pname: package name + # file: filename including extension + # hash: SRI hash + # kind: Language implementation and version tag + fetchWheelFromPypi = lib.makeOverridable + ( + { pname, file, hash, kind, curlOpts ? "" }: + let + version = builtins.elemAt (builtins.split "-" file) 2; + in + (pkgs.stdenvNoCC.mkDerivation { + name = file; + nativeBuildInputs = [ + pkgs.curl + pkgs.jq + ]; + isWheel = true; + system = "builtin"; + + preferLocalBuild = true; + impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ + "NIX_CURL_FLAGS" + ]; + + predictedURL = predictURLFromPypi { inherit pname file hash kind; }; + inherit pname file version curlOpts; + + builder = ./fetch-wheel.sh; + + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = hash; + }) + ); # Fetch the artifacts from the PyPI index. Since we get all # info we need from the lock file we don't use nixpkgs' fetchPyPi @@ -78,25 +143,27 @@ let # file: filename including extension # hash: SRI hash # kind: Language implementation and version tag https://www.python.org/dev/peps/pep-0427/#file-name-convention - fetchFromPypi = lib.makeOverridable ( - { pname, file, hash, kind }: - pkgs.fetchurl { - url = "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}"; - inherit hash; - } - ); - + fetchFromPypi = lib.makeOverridable + ( + { pname, file, hash, kind }: + if lib.strings.hasSuffix "whl" file then fetchWheelFromPypi { inherit pname file hash kind; } + else + pkgs.fetchurl { + url = predictURLFromPypi { inherit pname file hash kind; }; + inherit hash; + } + ); getBuildSystemPkgs = { pythonPackages , pyProject }: - let - buildSystem = lib.getAttrFromPath [ "build-system" "build-backend" ] pyProject; - drvAttr = builtins.elemAt (builtins.split "\\.|:" buildSystem) 0; - in - if buildSystem == "" then [] else ( - [ pythonPackages.${drvAttr} or (throw "unsupported build system ${buildSystem}") ] - ); + let + buildSystem = lib.attrByPath [ "build-system" "build-backend" ] "" pyProject; + drvAttr = moduleName (builtins.elemAt (builtins.split "\\.|:" buildSystem) 0); + in + if buildSystem == "" then [ ] else ( + [ pythonPackages.${drvAttr} or (throw "unsupported build system ${buildSystem}") ] + ); # Find gitignore files recursively in parent directory stopping with .git findGitIgnores = path: @@ -105,9 +172,9 @@ let gitIgnore = path + "/.gitignore"; isGitRoot = builtins.pathExists (path + "/.git"); hasGitIgnore = builtins.pathExists gitIgnore; - gitIgnores = if hasGitIgnore then [ gitIgnore ] else []; + gitIgnores = if hasGitIgnore then [ gitIgnore ] else [ ]; in - lib.optionals (builtins.toString path != "/" && ! isGitRoot) (findGitIgnores parent) ++ gitIgnores; + lib.optionals (builtins.toString path != "/" && ! isGitRoot) (findGitIgnores parent) ++ gitIgnores; /* Provides a source filtering mechanism that: @@ -124,22 +191,25 @@ let || (type == "regular" && ! lib.strings.hasSuffix ".pyc" name) ; in - lib.cleanSourceWith { - filter = lib.cleanSourceFilter; - src = lib.cleanSourceWith { - filter = pkgs.nix-gitignore.gitignoreFilterPure pycacheFilter gitIgnores src; - inherit src; - }; + lib.cleanSourceWith { + filter = lib.cleanSourceFilter; + src = lib.cleanSourceWith { + filter = pkgs.nix-gitignore.gitignoreFilterPure pycacheFilter gitIgnores src; + inherit src; }; + }; in { inherit fetchFromPypi + fetchWheelFromPypi getManyLinuxDeps isCompatible readTOML getBuildSystemPkgs satisfiesSemver cleanPythonSources + moduleName + getPythonVersion ; } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix index 5e71190d28fe..2467dad31e1a 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix @@ -10,74 +10,68 @@ , version , files , source -, dependencies ? {} +, dependencies ? { } , pythonPackages , python-versions , pwd , sourceSpec , supportedExtensions ? lib.importJSON ./extensions.json +, preferWheels ? false , ... }: -pythonPackages.callPackage ( - { preferWheel ? false - , ... - }@args: +pythonPackages.callPackage + ( + { preferWheel ? preferWheels + , ... + }@args: let - inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi; + inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi moduleName; inherit (import ./pep425.nix { inherit lib python; inherit (pkgs) stdenv; }) selectWheel ; - - fileCandidates = let - supportedRegex = ("^.*?(" + builtins.concatStringsSep "|" supportedExtensions + ")"); - matchesVersion = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." ] [ "\\." ] version + ".*$") fname != null; - hasSupportedExtension = fname: builtins.match supportedRegex fname != null; - isCompatibleEgg = fname: ! lib.strings.hasSuffix ".egg" fname || lib.strings.hasSuffix "py${python.pythonVersion}.egg" fname; - in + fileCandidates = + let + supportedRegex = ("^.*?(" + builtins.concatStringsSep "|" supportedExtensions + ")"); + matchesVersion = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." ] [ "\\." ] version + ".*$") fname != null; + hasSupportedExtension = fname: builtins.match supportedRegex fname != null; + isCompatibleEgg = fname: ! lib.strings.hasSuffix ".egg" fname || lib.strings.hasSuffix "py${python.pythonVersion}.egg" fname; + in builtins.filter (f: matchesVersion f.file && hasSupportedExtension f.file && isCompatibleEgg f.file) files; - toPath = s: pwd + "/${s}"; - isSource = source != null; isGit = isSource && source.type == "git"; isLocal = isSource && source.type == "directory"; - localDepPath = toPath source.url; pyProject = poetryLib.readTOML (localDepPath + "/pyproject.toml"); - buildSystemPkgs = poetryLib.getBuildSystemPkgs { inherit pythonPackages pyProject; }; - - fileInfo = let - isBdist = f: lib.strings.hasSuffix "whl" f.file; - isSdist = f: ! isBdist f && ! isEgg f; - isEgg = f: lib.strings.hasSuffix ".egg" f.file; - - binaryDist = selectWheel fileCandidates; - sourceDist = builtins.filter isSdist fileCandidates; - eggs = builtins.filter isEgg fileCandidates; - - entries = (if preferWheel then binaryDist ++ sourceDist else sourceDist ++ binaryDist) ++ eggs; - - lockFileEntry = builtins.head entries; - - _isEgg = isEgg lockFileEntry; - in + fileInfo = + let + isBdist = f: lib.strings.hasSuffix "whl" f.file; + isSdist = f: ! isBdist f && ! isEgg f; + isEgg = f: lib.strings.hasSuffix ".egg" f.file; + binaryDist = selectWheel fileCandidates; + sourceDist = builtins.filter isSdist fileCandidates; + eggs = builtins.filter isEgg fileCandidates; + entries = ( if preferWheel then binaryDist ++ sourceDist else sourceDist ++ binaryDist) ++ eggs; + lockFileEntry = builtins.head entries; + _isEgg = isEgg lockFileEntry; + in rec { inherit (lockFileEntry) file hash; name = file; format = if _isEgg then "egg" else if lib.strings.hasSuffix ".whl" name then "wheel" - else "setuptools"; + else "pyproject"; kind = if _isEgg then python.pythonVersion - else if format == "setuptools" then "source" + else if format == "pyproject" then "source" else (builtins.elemAt (lib.strings.splitString "-" name) 2); }; @@ -88,63 +82,65 @@ pythonPackages.callPackage ( "toml" # Toml is an extra for setuptools-scm ]; baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm; - - format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format; + format = if isLocal then "pyproject" else if isGit then "pyproject" else fileInfo.format; in - - buildPythonPackage { - pname = name; - version = version; - - inherit format; - - doCheck = false; # We never get development deps - - # Stripping pre-built wheels lead to `ELF load command address/offset not properly aligned` - dontStrip = format == "wheel"; - - nativeBuildInputs = [ - pythonPackages.poetry2nixFixupHook - ] - ++ lib.optional (!isSource && (getManyLinuxDeps fileInfo.name).str != null) autoPatchelfHook - ++ lib.optional (format == "pyproject") pythonPackages.removePathDependenciesHook - ; - - buildInputs = ( - baseBuildInputs - ++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg - ++ lib.optional isLocal buildSystemPkgs - ); - - propagatedBuildInputs = let - compat = isCompatible python.pythonVersion; - deps = lib.filterAttrs (n: v: v) ( - lib.mapAttrs ( - n: v: - let - constraints = v.python or ""; - in - compat constraints - ) dependencies - ); + buildPythonPackage { + pname = moduleName name; + version = version; + + inherit format; + + doCheck = false; # We never get development deps + + # Stripping pre-built wheels lead to `ELF load command address/offset not properly aligned` + dontStrip = format == "wheel"; + + nativeBuildInputs = [ + pythonPackages.poetry2nixFixupHook + ] + ++ lib.optional (!isSource && (getManyLinuxDeps fileInfo.name).str != null) autoPatchelfHook + ++ lib.optional (format == "pyproject") pythonPackages.removePathDependenciesHook + ; + + buildInputs = ( + baseBuildInputs + ++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg + ++ lib.optional isLocal buildSystemPkgs + ); + + propagatedBuildInputs = + let + compat = isCompatible (poetryLib.getPythonVersion python); + deps = lib.filterAttrs (n: v: v) + ( + lib.mapAttrs + ( + n: v: + let + constraints = v.python or ""; + in + compat constraints + ) dependencies + ); depAttrs = lib.attrNames deps; in - builtins.map (n: pythonPackages.${lib.toLower n}) depAttrs; + builtins.map (n: pythonPackages.${moduleName n}) depAttrs; - meta = { - broken = ! isCompatible python.pythonVersion python-versions; - license = []; - inherit (python.meta) platforms; - }; + meta = { + broken = ! isCompatible (poetryLib.getPythonVersion python) python-versions; + license = [ ]; + inherit (python.meta) platforms; + }; - passthru = { - inherit args; - }; + passthru = { + inherit args; + }; - # We need to retrieve kind from the interpreter and the filename of the package - # Interpreters should declare what wheel types they're compatible with (python type + ABI) - # Here we can then choose a file based on that info. - src = if isGit then ( + # We need to retrieve kind from the interpreter and the filename of the package + # Interpreters should declare what wheel types they're compatible with (python type + ABI) + # Here we can then choose a file based on that info. + src = + if isGit then ( builtins.fetchGit { inherit (source) url; rev = source.reference; @@ -154,6 +150,5 @@ pythonPackages.callPackage ( pname = name; inherit (fileInfo) file hash kind; }; - } - -) {} + } + ) { } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 04340ac4e89f..d18e6b8293f3 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -1,4 +1,4 @@ -{ pkgs ? import <nixpkgs> {} +{ pkgs ? import <nixpkgs> { } , lib ? pkgs.lib , stdenv ? pkgs.stdenv }: @@ -6,202 +6,256 @@ self: super: { - astroid = super.astroid.overridePythonAttrs ( - old: rec { - buildInputs = old.buildInputs ++ [ self.pytest-runner ]; - doCheck = false; - } - ); + astroid = super.astroid.overridePythonAttrs + ( + old: rec { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); - av = super.av.overridePythonAttrs ( - old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.pkgconfig - ]; - buildInputs = old.buildInputs ++ [ pkgs.ffmpeg_4 ]; - } - ); + av = super.av.overridePythonAttrs + ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.pkgconfig + ]; + buildInputs = old.buildInputs ++ [ pkgs.ffmpeg_4 ]; + } + ); - bcrypt = super.bcrypt.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ pkgs.libffi ]; - } - ); + bcrypt = super.bcrypt.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.libffi ]; + } + ); cffi = # cffi is bundled with pypy if self.python.implementation == "pypy" then null else ( - super.cffi.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ pkgs.libffi ]; - } - ) + super.cffi.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.libffi ]; + } + ) ); - cftime = super.cftime.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ - self.cython - ]; - } - ); + cftime = super.cftime.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ + self.cython + ]; + } + ); - cryptography = super.cryptography.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ pkgs.openssl ]; - } - ); + configparser = super.configparser.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ + self.toml + ]; - django = ( - super.django.overridePythonAttrs ( + postPatch = '' + substituteInPlace setup.py --replace 'setuptools.setup()' 'setuptools.setup(version="${old.version}")' + ''; + } + ); + + cryptography = super.cryptography.overridePythonAttrs + ( old: { - propagatedNativeBuildInputs = (old.propagatedNativeBuildInputs or []) - ++ [ pkgs.gettext ]; + buildInputs = old.buildInputs ++ [ pkgs.openssl ]; } - ) - ); + ); - django-bakery = super.django-bakery.overridePythonAttrs ( - old: { - configurePhase = '' - if ! test -e LICENSE; then - touch LICENSE - fi - '' + (old.configurePhase or ""); - } + django = ( + super.django.overridePythonAttrs + ( + old: { + propagatedNativeBuildInputs = (old.propagatedNativeBuildInputs or [ ]) + ++ [ pkgs.gettext ]; + } + ) ); - dlib = super.dlib.overridePythonAttrs ( - old: { - # Parallel building enabled - inherit (pkgs.python.pkgs.dlib) patches; + django-bakery = super.django-bakery.overridePythonAttrs + ( + old: { + configurePhase = '' + if ! test -e LICENSE; then + touch LICENSE + fi + '' + (old.configurePhase or ""); + } + ); + + dlib = super.dlib.overridePythonAttrs + ( + old: { + # Parallel building enabled + inherit (pkgs.python.pkgs.dlib) patches; - enableParallelBuilding = true; - dontUseCmakeConfigure = true; + enableParallelBuilding = true; + dontUseCmakeConfigure = true; - nativeBuildInputs = old.nativeBuildInputs ++ pkgs.dlib.nativeBuildInputs; - buildInputs = old.buildInputs ++ pkgs.dlib.buildInputs; - } - ); + nativeBuildInputs = old.nativeBuildInputs ++ pkgs.dlib.nativeBuildInputs; + buildInputs = old.buildInputs ++ pkgs.dlib.buildInputs; + } + ); # Environment markers are not always included (depending on how a dep was defined) enum34 = if self.pythonAtLeast "3.4" then null else super.enum34; - faker = super.faker.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ self.pytest-runner ]; - doCheck = false; - } - ); + faker = super.faker.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); - fancycompleter = super.fancycompleter.overridePythonAttrs ( - old: { - postPatch = '' - substituteInPlace setup.py \ - --replace 'setup_requires="setupmeta"' 'setup_requires=[]' \ - --replace 'versioning="devcommit"' 'version="${old.version}"' - ''; - } - ); + fancycompleter = super.fancycompleter.overridePythonAttrs + ( + old: { + postPatch = '' + substituteInPlace setup.py \ + --replace 'setup_requires="setupmeta"' 'setup_requires=[]' \ + --replace 'versioning="devcommit"' 'version="${old.version}"' + ''; + } + ); - fastparquet = super.fastparquet.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ self.pytest-runner ]; - } - ); + fastparquet = super.fastparquet.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + } + ); - grandalf = super.grandalf.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ self.pytest-runner ]; - doCheck = false; - } - ); + grandalf = super.grandalf.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); - h5py = super.h5py.overridePythonAttrs ( - old: rec { - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; - buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkgconfig self.cython ]; - configure_flags = "--hdf5=${pkgs.hdf5}"; - postConfigure = '' - ${self.python.executable} setup.py configure ${configure_flags} - ''; - } - ); + h5py = super.h5py.overridePythonAttrs + ( + old: + if old.format != "wheel" then rec { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; + buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkgconfig self.cython ]; + configure_flags = "--hdf5=${pkgs.hdf5}"; + postConfigure = '' + ${self.python.executable} setup.py configure ${configure_flags} + ''; + } else old + ); - horovod = super.horovod.overridePythonAttrs ( - old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; - } - ); + horovod = super.horovod.overridePythonAttrs + ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; + } + ); # importlib-metadata has an incomplete dependency specification - importlib-metadata = super.importlib-metadata.overridePythonAttrs ( - old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ lib.optional self.python.isPy2 self.pathlib2; - } - ); + importlib-metadata = super.importlib-metadata.overridePythonAttrs + ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ lib.optional self.python.isPy2 self.pathlib2; + } + ); - jupyter = super.jupyter.overridePythonAttrs ( - old: rec { - # jupyter is a meta-package. Everything relevant comes from the - # dependencies. It does however have a jupyter.py file that conflicts - # with jupyter-core so this meta solves this conflict. - meta.priority = 100; - } - ); + isort = super.isort.overridePythonAttrs + ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.setuptools ]; + } + ); - lap = super.lap.overridePythonAttrs ( - old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ - self.numpy - ]; - } - ); + jupyter = super.jupyter.overridePythonAttrs + ( + old: rec { + # jupyter is a meta-package. Everything relevant comes from the + # dependencies. It does however have a jupyter.py file that conflicts + # with jupyter-core so this meta solves this conflict. + meta.priority = 100; + } + ); - llvmlite = super.llvmlite.overridePythonAttrs ( - old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.llvm ]; + kiwisolver = super.kiwisolver.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ + # cppy is at the time of writing not in nixpkgs + (self.cppy or null) + ]; + } + ); - # Disable static linking - # https://github.com/numba/llvmlite/issues/93 - postPatch = '' - substituteInPlace ffi/Makefile.linux --replace "-static-libstdc++" "" + lap = super.lap.overridePythonAttrs + ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + self.numpy + ]; + } + ); - substituteInPlace llvmlite/tests/test_binding.py --replace "test_linux" "nope" - ''; + llvmlite = super.llvmlite.overridePythonAttrs + ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.llvm ]; - # Set directory containing llvm-config binary - preConfigure = '' - export LLVM_CONFIG=${pkgs.llvm}/bin/llvm-config - ''; + # Disable static linking + # https://github.com/numba/llvmlite/issues/93 + postPatch = '' + substituteInPlace ffi/Makefile.linux --replace "-static-libstdc++" "" - __impureHostDeps = pkgs.stdenv.lib.optionals pkgs.stdenv.isDarwin [ "/usr/lib/libm.dylib" ]; + substituteInPlace llvmlite/tests/test_binding.py --replace "test_linux" "nope" + ''; - passthru = old.passthru // { llvm = pkgs.llvm; }; - } - ); + # Set directory containing llvm-config binary + preConfigure = '' + export LLVM_CONFIG=${pkgs.llvm}/bin/llvm-config + ''; - lockfile = super.lockfile.overridePythonAttrs ( - old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pbr ]; - } - ); + __impureHostDeps = pkgs.stdenv.lib.optionals pkgs.stdenv.isDarwin [ "/usr/lib/libm.dylib" ]; - lxml = super.lxml.overridePythonAttrs ( - old: { - nativeBuildInputs = with pkgs; old.nativeBuildInputs ++ [ pkgconfig libxml2.dev libxslt.dev ]; - buildInputs = with pkgs; old.buildInputs ++ [ libxml2 libxslt ]; - } - ); + passthru = old.passthru // { llvm = pkgs.llvm; }; + } + ); - markupsafe = super.markupsafe.overridePythonAttrs ( - old: { - src = old.src.override { pname = builtins.replaceStrings [ "markupsafe" ] [ "MarkupSafe" ] old.pname; }; - } - ); + lockfile = super.lockfile.overridePythonAttrs + ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pbr ]; + } + ); + + lxml = super.lxml.overridePythonAttrs + ( + old: { + nativeBuildInputs = with pkgs; old.nativeBuildInputs ++ [ pkgconfig libxml2.dev libxslt.dev ]; + buildInputs = with pkgs; old.buildInputs ++ [ libxml2 libxslt ]; + } + ); + + markupsafe = super.markupsafe.overridePythonAttrs + ( + old: { + src = old.src.override { pname = builtins.replaceStrings [ "markupsafe" ] [ "MarkupSafe" ] old.pname; }; + } + ); - matplotlib = super.matplotlib.overridePythonAttrs ( - old: + matplotlib = super.matplotlib.overridePythonAttrs + ( + old: let enableGhostscript = old.passthru.enableGhostscript or false; enableGtk3 = old.passthru.enableTk or false; @@ -210,484 +264,583 @@ self: super: inherit (pkgs.darwin.apple_sdk.frameworks) Cocoa; in - { - NIX_CFLAGS_COMPILE = stdenv.lib.optionalString stdenv.isDarwin "-I${pkgs.libcxx}/include/c++/v1"; + { + NIX_CFLAGS_COMPILE = stdenv.lib.optionalString stdenv.isDarwin "-I${pkgs.libcxx}/include/c++/v1"; - XDG_RUNTIME_DIR = "/tmp"; + XDG_RUNTIME_DIR = "/tmp"; - buildInputs = old.buildInputs + buildInputs = old.buildInputs ++ lib.optional enableGhostscript pkgs.ghostscript ++ lib.optional stdenv.isDarwin [ Cocoa ]; - nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.pkgconfig - ]; + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.pkgconfig + ]; - propagatedBuildInputs = old.propagatedBuildInputs ++ [ - pkgs.libpng - pkgs.freetype - ] + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + pkgs.libpng + pkgs.freetype + ] ++ stdenv.lib.optionals enableGtk3 [ pkgs.cairo self.pycairo pkgs.gtk3 pkgs.gobject-introspection self.pygobject3 ] ++ stdenv.lib.optionals enableTk [ pkgs.tcl pkgs.tk self.tkinter pkgs.libX11 ] ++ stdenv.lib.optionals enableQt [ self.pyqt5 ] - ; + ; - inherit (super.matplotlib) patches; - } - ); + inherit (super.matplotlib) patches; + } + ); # Calls Cargo at build time for source builds and is really tricky to package maturin = super.maturin.override { preferWheel = true; }; - mccabe = super.mccabe.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ self.pytest-runner ]; - doCheck = false; - } - ); + mccabe = super.mccabe.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); - netcdf4 = super.netcdf4.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ - self.cython - ]; - - propagatedBuildInputs = old.propagatedBuildInputs ++ [ - pkgs.zlib - pkgs.netcdf - pkgs.hdf5 - pkgs.curl - pkgs.libjpeg - ]; - - # Variables used to configure the build process - USE_NCCONFIG = "0"; - HDF5_DIR = lib.getDev pkgs.hdf5; - NETCDF4_DIR = pkgs.netcdf; - CURL_DIR = pkgs.curl.dev; - JPEG_DIR = pkgs.libjpeg.dev; - } - ); + netcdf4 = super.netcdf4.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ + self.cython + ]; - numpy = super.numpy.overridePythonAttrs ( - old: + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + pkgs.zlib + pkgs.netcdf + pkgs.hdf5 + pkgs.curl + pkgs.libjpeg + ]; + + # Variables used to configure the build process + USE_NCCONFIG = "0"; + HDF5_DIR = lib.getDev pkgs.hdf5; + NETCDF4_DIR = pkgs.netcdf; + CURL_DIR = pkgs.curl.dev; + JPEG_DIR = pkgs.libjpeg.dev; + } + ); + + numpy = super.numpy.overridePythonAttrs + ( + old: let blas = old.passthru.args.blas or pkgs.openblasCompat; - lapack = old.passthru.args.lapack or pkgs.openblasCompat; + blasImplementation = lib.nameFromURL blas.name "-"; cfg = pkgs.writeTextFile { name = "site.cfg"; - text = (lib.generators.toINI {} { - ${blas.implementation} = { - include_dirs = "${blas}/include:${lapack}/include"; - library_dirs = "${blas}/lib:${lapack}/lib"; - }; - }); + text = ( + lib.generators.toINI { } { + ${blasImplementation} = { + include_dirs = "${blas}/include"; + library_dirs = "${blas}/lib"; + } // lib.optionalAttrs (blasImplementation == "mkl") { + mkl_libs = "mkl_rt"; + lapack_libs = ""; + }; + } + ); }; in - { - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; - buildInputs = old.buildInputs ++ [ blas self.cython ]; - enableParallelBuilding = true; - preBuild = '' - ln -s ${cfg} site.cfg - ''; - passthru = old.passthru // { - blsaImplementation = blas.implementation; - inherit blas cfg; - }; - } - ); + { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; + buildInputs = old.buildInputs ++ [ blas self.cython ]; + enableParallelBuilding = true; + preBuild = '' + ln -s ${cfg} site.cfg + ''; + passthru = old.passthru // { + blas = blas; + inherit blasImplementation cfg; + }; + } + ); - openexr = super.openexr.overridePythonAttrs ( - old: rec { - buildInputs = old.buildInputs ++ [ pkgs.openexr pkgs.ilmbase ]; - NIX_CFLAGS_COMPILE = [ "-I${pkgs.openexr.dev}/include/OpenEXR" "-I${pkgs.ilmbase.dev}/include/OpenEXR" ]; - } - ); + openexr = super.openexr.overridePythonAttrs + ( + old: rec { + buildInputs = old.buildInputs ++ [ pkgs.openexr pkgs.ilmbase ]; + NIX_CFLAGS_COMPILE = [ "-I${pkgs.openexr.dev}/include/OpenEXR" "-I${pkgs.ilmbase.dev}/include/OpenEXR" ]; + } + ); - peewee = super.peewee.overridePythonAttrs ( - old: + peewee = super.peewee.overridePythonAttrs + ( + old: let withPostgres = old.passthru.withPostgres or false; withMysql = old.passthru.withMysql or false; in - { - buildInputs = old.buildInputs ++ [ self.cython pkgs.sqlite ]; - propagatedBuildInputs = old.propagatedBuildInputs + { + buildInputs = old.buildInputs ++ [ self.cython pkgs.sqlite ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ lib.optional withPostgres self.psycopg2 ++ lib.optional withMysql self.mysql-connector; - } - ); + } + ); - pillow = super.pillow.overridePythonAttrs ( - old: { - nativeBuildInputs = [ pkgs.pkgconfig ] ++ old.nativeBuildInputs; - buildInputs = with pkgs; [ freetype libjpeg zlib libtiff libwebp tcl lcms2 ] ++ old.buildInputs; - } - ); + pillow = super.pillow.overridePythonAttrs + ( + old: { + nativeBuildInputs = [ pkgs.pkgconfig ] ++ old.nativeBuildInputs; + buildInputs = with pkgs; [ freetype libjpeg zlib libtiff libwebp tcl lcms2 ] ++ old.buildInputs; + } + ); - psycopg2 = super.psycopg2.overridePythonAttrs ( - old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ]; - } - ); + psycopg2 = super.psycopg2.overridePythonAttrs + ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ]; + } + ); - psycopg2-binary = super.psycopg2-binary.overridePythonAttrs ( - old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ]; - } - ); + psycopg2-binary = super.psycopg2-binary.overridePythonAttrs + ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ]; + } + ); - pyarrow = super.pyarrow.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ - self.cython - ]; - } - ); + pyarrow = + if lib.versionAtLeast super.pyarrow.version "0.16.0" then super.pyarrow.overridePythonAttrs + ( + old: + let + parseMinor = drv: lib.concatStringsSep "." (lib.take 2 (lib.splitVersion drv.version)); + _arrow-cpp = pkgs.arrow-cpp.override { inherit (self) python; }; + ARROW_HOME = _arrow-cpp; + arrowCppVersion = parseMinor pkgs.arrow-cpp; + pyArrowVersion = parseMinor super.pyarrow; + errorMessage = "arrow-cpp version (${arrowCppVersion}) mismatches pyarrow version (${pyArrowVersion})"; + in + if arrowCppVersion != pyArrowVersion then throw errorMessage else { - pycairo = ( - drv: ( - drv.overridePythonAttrs ( - _: { - format = "other"; - } - ) - ).overridePythonAttrs ( - old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ + self.cython + pkgs.pkgconfig + pkgs.cmake + ]; - nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.meson - pkgs.ninja - pkgs.pkgconfig - ]; + preBuild = '' + export PYARROW_PARALLEL=$NIX_BUILD_CORES + ''; - propagatedBuildInputs = old.propagatedBuildInputs ++ [ - pkgs.cairo - pkgs.xlibsWrapper - ]; + PARQUET_HOME = _arrow-cpp; + inherit ARROW_HOME; - mesonFlags = [ "-Dpython=${if self.isPy3k then "python3" else "python"}" ]; - } - ) - ) super.pycairo; + buildInputs = old.buildInputs ++ [ + pkgs.arrow-cpp + ]; - pycocotools = super.pycocotools.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ - self.cython - self.numpy - ]; - } - ); + PYARROW_BUILD_TYPE = "release"; + PYARROW_WITH_PARQUET = true; + PYARROW_CMAKE_OPTIONS = [ + "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib" - pygobject = super.pygobject.overridePythonAttrs ( - old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; - buildInputs = old.buildInputs ++ [ pkgs.glib pkgs.gobject-introspection ]; - } - ); + # This doesn't use setup hook to call cmake so we need to workaround #54606 + # ourselves + "-DCMAKE_POLICY_DEFAULT_CMP0025=NEW" + ]; - pylint = super.pylint.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ self.pytest-runner ]; - doCheck = false; - } - ); + dontUseCmakeConfigure = true; + } + ) else super.pyarrow.overridePythonAttrs + ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ + self.cython + ]; + } + ); - pyopenssl = super.pyopenssl.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ pkgs.openssl ]; - } - ); + pycairo = + ( + drv: ( + drv.overridePythonAttrs + ( + _: { + format = "other"; + } + ) + ).overridePythonAttrs + ( + old: { - pyqt5 = let - drv = super.pyqt5; - withConnectivity = drv.passthru.args.withConnectivity or false; - withMultimedia = drv.passthru.args.withMultimedia or false; - withWebKit = drv.passthru.args.withWebKit or false; - withWebSockets = drv.passthru.args.withWebSockets or false; - in - super.pyqt5.overridePythonAttrs ( - old: { - format = "other"; + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.meson + pkgs.ninja + pkgs.pkgconfig + ]; - nativeBuildInputs = old.nativeBuildInputs ++ [ - pkgs.pkgconfig - pkgs.qt5.qmake - pkgs.xorg.lndir - pkgs.qt5.qtbase - pkgs.qt5.qtsvg - pkgs.qt5.qtdeclarative - pkgs.qt5.qtwebchannel - # self.pyqt5-sip - self.sip - ] - ++ lib.optional withConnectivity pkgs.qt5.qtconnectivity - ++ lib.optional withMultimedia pkgs.qt5.qtmultimedia - ++ lib.optional withWebKit pkgs.qt5.qtwebkit - ++ lib.optional withWebSockets pkgs.qt5.qtwebsockets - ; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + pkgs.cairo + pkgs.xlibsWrapper + ]; - buildInputs = old.buildInputs ++ [ - pkgs.dbus - pkgs.qt5.qtbase - pkgs.qt5.qtsvg - pkgs.qt5.qtdeclarative - self.sip - ] - ++ lib.optional withConnectivity pkgs.qt5.qtconnectivity - ++ lib.optional withWebKit pkgs.qt5.qtwebkit - ++ lib.optional withWebSockets pkgs.qt5.qtwebsockets - ; + mesonFlags = [ "-Dpython=${ if self.isPy3k then "python3" else "python"}" ]; + } + ) + ) super.pycairo; - # Fix dbus mainloop - patches = pkgs.python3.pkgs.pyqt5.patches or []; + pycocotools = super.pycocotools.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ + self.cython + self.numpy + ]; + } + ); - configurePhase = '' - runHook preConfigure + pygobject = super.pygobject.overridePythonAttrs + ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; + buildInputs = old.buildInputs ++ [ pkgs.glib pkgs.gobject-introspection ]; + } + ); - export PYTHONPATH=$PYTHONPATH:$out/${self.python.sitePackages} + pylint = super.pylint.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ self.pytest-runner ]; + doCheck = false; + } + ); - mkdir -p $out/${self.python.sitePackages}/dbus/mainloop - ${self.python.executable} configure.py -w \ - --confirm-license \ - --no-qml-plugin \ - --bindir=$out/bin \ - --destdir=$out/${self.python.sitePackages} \ - --stubsdir=$out/${self.python.sitePackages}/PyQt5 \ - --sipdir=$out/share/sip/PyQt5 \ - --designer-plugindir=$out/plugins/designer + pyopenssl = super.pyopenssl.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.openssl ]; + } + ); - runHook postConfigure - ''; + pyqt5 = + let + drv = super.pyqt5; + withConnectivity = drv.passthru.args.withConnectivity or false; + withMultimedia = drv.passthru.args.withMultimedia or false; + withWebKit = drv.passthru.args.withWebKit or false; + withWebSockets = drv.passthru.args.withWebSockets or false; + in + super.pyqt5.overridePythonAttrs + ( + old: { + format = "other"; - postInstall = '' - ln -s ${self.pyqt5-sip}/${self.python.sitePackages}/PyQt5/sip.* $out/${self.python.sitePackages}/PyQt5/ - for i in $out/bin/*; do - wrapProgram $i --prefix PYTHONPATH : "$PYTHONPATH" - done - - # Let's make it a namespace package - cat << EOF > $out/${self.python.sitePackages}/PyQt5/__init__.py - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) - EOF - ''; + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.pkgconfig + pkgs.qt5.qmake + pkgs.xorg.lndir + pkgs.qt5.qtbase + pkgs.qt5.qtsvg + pkgs.qt5.qtdeclarative + pkgs.qt5.qtwebchannel + # self.pyqt5-sip + self.sip + ] + ++ lib.optional withConnectivity pkgs.qt5.qtconnectivity + ++ lib.optional withMultimedia pkgs.qt5.qtmultimedia + ++ lib.optional withWebKit pkgs.qt5.qtwebkit + ++ lib.optional withWebSockets pkgs.qt5.qtwebsockets + ; - installCheckPhase = let - modules = [ - "PyQt5" - "PyQt5.QtCore" - "PyQt5.QtQml" - "PyQt5.QtWidgets" - "PyQt5.QtGui" + buildInputs = old.buildInputs ++ [ + pkgs.dbus + pkgs.qt5.qtbase + pkgs.qt5.qtsvg + pkgs.qt5.qtdeclarative + self.sip ] - ++ lib.optional withWebSockets "PyQt5.QtWebSockets" - ++ lib.optional withWebKit "PyQt5.QtWebKit" - ++ lib.optional withMultimedia "PyQt5.QtMultimedia" - ++ lib.optional withConnectivity "PyQt5.QtConnectivity" + ++ lib.optional withConnectivity pkgs.qt5.qtconnectivity + ++ lib.optional withWebKit pkgs.qt5.qtwebkit + ++ lib.optional withWebSockets pkgs.qt5.qtwebsockets ; - imports = lib.concatMapStrings (module: "import ${module};") modules; - in - '' - echo "Checking whether modules can be imported..." - ${self.python.interpreter} -c "${imports}" + # Fix dbus mainloop + patches = pkgs.python3.pkgs.pyqt5.patches or [ ]; + + configurePhase = '' + runHook preConfigure + + export PYTHONPATH=$PYTHONPATH:$out/${self.python.sitePackages} + + mkdir -p $out/${self.python.sitePackages}/dbus/mainloop + ${self.python.executable} configure.py -w \ + --confirm-license \ + --no-qml-plugin \ + --bindir=$out/bin \ + --destdir=$out/${self.python.sitePackages} \ + --stubsdir=$out/${self.python.sitePackages}/PyQt5 \ + --sipdir=$out/share/sip/PyQt5 \ + --designer-plugindir=$out/plugins/designer + + runHook postConfigure ''; - doCheck = true; + postInstall = '' + ln -s ${self.pyqt5-sip}/${self.python.sitePackages}/PyQt5/sip.* $out/${self.python.sitePackages}/PyQt5/ + for i in $out/bin/*; do + wrapProgram $i --prefix PYTHONPATH : "$PYTHONPATH" + done + + # Let's make it a namespace package + cat << EOF > $out/${self.python.sitePackages}/PyQt5/__init__.py + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) + EOF + ''; - enableParallelBuilding = true; + installCheckPhase = + let + modules = [ + "PyQt5" + "PyQt5.QtCore" + "PyQt5.QtQml" + "PyQt5.QtWidgets" + "PyQt5.QtGui" + ] + ++ lib.optional withWebSockets "PyQt5.QtWebSockets" + ++ lib.optional withWebKit "PyQt5.QtWebKit" + ++ lib.optional withMultimedia "PyQt5.QtMultimedia" + ++ lib.optional withConnectivity "PyQt5.QtConnectivity" + ; + imports = lib.concatMapStrings (module: "import ${module};") modules; + in + '' + echo "Checking whether modules can be imported..." + ${self.python.interpreter} -c "${imports}" + ''; + + doCheck = true; + + enableParallelBuilding = true; + } + ); + + pytest-datadir = super.pytest-datadir.overridePythonAttrs + ( + old: { + postInstall = '' + rm -f $out/LICENSE + ''; } ); - pytest-datadir = super.pytest-datadir.overridePythonAttrs ( - old: { - postInstall = '' - rm -f $out/LICENSE - ''; - } - ); - - pytest = super.pytest.overridePythonAttrs ( - old: { - doCheck = false; - } - ); + pytest = super.pytest.overridePythonAttrs + ( + old: { + doCheck = false; + } + ); pytest-runner = super.pytest-runner or super.pytestrunner; - python-prctl = super.python-prctl.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ - pkgs.libcap - ]; - } - ); + python-jose = super.python-jose.overridePythonAttrs + ( + old: { + postPath = '' + substituteInPlace setup.py --replace "'pytest-runner'," "" + substituteInPlace setup.py --replace "'pytest-runner'" "" + ''; + } + ); - pyzmq = super.pyzmq.overridePythonAttrs ( - old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.zeromq ]; - } - ); + python-prctl = super.python-prctl.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ + pkgs.libcap + ]; + } + ); - rockset = super.rockset.overridePythonAttrs ( - old: rec { - postPatch = '' - cp ./setup_rockset.py ./setup.py - ''; - } - ); + pyzmq = super.pyzmq.overridePythonAttrs + ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.zeromq ]; + } + ); - scaleapi = super.scaleapi.overridePythonAttrs ( - old: { - postPatch = '' - substituteInPlace setup.py --replace "install_requires = ['requests>=2.4.2', 'enum34']" "install_requires = ['requests>=2.4.2']" || true - ''; - } - ); + rockset = super.rockset.overridePythonAttrs + ( + old: rec { + postPatch = '' + cp ./setup_rockset.py ./setup.py + ''; + } + ); - pandas = super.pandas.overridePythonAttrs ( - old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ self.cython ]; - } - ); + scaleapi = super.scaleapi.overridePythonAttrs + ( + old: { + postPatch = '' + substituteInPlace setup.py --replace "install_requires = ['requests>=2.4.2', 'enum34']" "install_requires = ['requests>=2.4.2']" || true + ''; + } + ); + + pandas = super.pandas.overridePythonAttrs + ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ self.cython ]; + } + ); # Pybind11 is an undeclared dependency of scipy that we need to pick from nixpkgs # Make it not fail with infinite recursion - pybind11 = super.pybind11.overridePythonAttrs ( - old: { - cmakeFlags = (old.cmakeFlags or []) ++ [ - "-DPYBIND11_TEST=off" - ]; - doCheck = false; # Circular test dependency - } - ); - - scipy = super.scipy.overridePythonAttrs ( - old: { - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; - propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pybind11 ]; - setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; - enableParallelBuilding = true; - buildInputs = old.buildInputs ++ [ self.numpy.blas ]; - preConfigure = '' - sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py - export NPY_NUM_BUILD_JOBS=$NIX_BUILD_CORES - ''; - preBuild = '' - ln -s ${self.numpy.cfg} site.cfg - ''; - } - ); + pybind11 = super.pybind11.overridePythonAttrs + ( + old: { + cmakeFlags = (old.cmakeFlags or [ ]) ++ [ + "-DPYBIND11_TEST=off" + ]; + doCheck = false; # Circular test dependency + } + ); - scikit-learn = super.scikit-learn.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ - pkgs.gfortran - pkgs.glibcLocales - ] ++ lib.optionals stdenv.cc.isClang [ - pkgs.llvmPackages.openmp - ]; + scipy = super.scipy.overridePythonAttrs + ( + old: + if old.format != "wheel" then { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pybind11 ]; + setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; + enableParallelBuilding = true; + buildInputs = old.buildInputs ++ [ self.numpy.blas ]; + preConfigure = '' + sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py + export NPY_NUM_BUILD_JOBS=$NIX_BUILD_CORES + ''; + preBuild = '' + ln -s ${self.numpy.cfg} site.cfg + ''; + } else old + ); - nativeBuildInputs = old.nativeBuildInputs ++ [ - self.cython - ]; + scikit-learn = super.scikit-learn.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ [ + pkgs.gfortran + pkgs.glibcLocales + ] ++ lib.optionals stdenv.cc.isClang [ + pkgs.llvmPackages.openmp + ]; - enableParallelBuilding = true; - } - ); + nativeBuildInputs = old.nativeBuildInputs ++ [ + self.cython + ]; - shapely = super.shapely.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ [ pkgs.geos self.cython ]; - inherit (pkgs.python3.pkgs.shapely) patches GEOS_LIBRARY_PATH; - } - ); + enableParallelBuilding = true; + } + ); - shellingham = if lib.versionAtLeast super.shellingham.version "1.3.2" then ( - super.shellingham.overridePythonAttrs ( + shapely = super.shapely.overridePythonAttrs + ( old: { - format = "pyproject"; + buildInputs = old.buildInputs ++ [ pkgs.geos self.cython ]; + inherit (pkgs.python3.pkgs.shapely) patches GEOS_LIBRARY_PATH; } - ) - ) else super.shellingham; + ); - tables = super.tables.overridePythonAttrs ( - old: { - HDF5_DIR = "${pkgs.hdf5}"; - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; - propagatedBuildInputs = old.nativeBuildInputs ++ [ pkgs.hdf5 self.numpy self.numexpr ]; - } - ); + shellingham = + if lib.versionAtLeast super.shellingham.version "1.3.2" then ( + super.shellingham.overridePythonAttrs + ( + old: { + format = "pyproject"; + } + ) + ) else super.shellingham; - tensorpack = super.tensorpack.overridePythonAttrs ( - old: { - postPatch = '' - substituteInPlace setup.cfg --replace "# will call find_packages()" "" - ''; - } - ); + tables = super.tables.overridePythonAttrs + ( + old: { + HDF5_DIR = "${pkgs.hdf5}"; + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; + propagatedBuildInputs = old.nativeBuildInputs ++ [ pkgs.hdf5 self.numpy self.numexpr ]; + } + ); - urwidtrees = super.urwidtrees.overridePythonAttrs ( - old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ - self.urwid - ]; - } - ); + tensorpack = super.tensorpack.overridePythonAttrs + ( + old: { + postPatch = '' + substituteInPlace setup.cfg --replace "# will call find_packages()" "" + ''; + } + ); - vose-alias-method = super.vose-alias-method.overridePythonAttrs ( - old: { - postInstall = '' - rm -f $out/LICENSE - ''; - } - ); + urwidtrees = super.urwidtrees.overridePythonAttrs + ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + self.urwid + ]; + } + ); - uvloop = super.uvloop.overridePythonAttrs ( - old: { - buildInputs = old.buildInputs ++ lib.optionals stdenv.isDarwin [ - pkgs.darwin.apple_sdk.frameworks.ApplicationServices - pkgs.darwin.apple_sdk.frameworks.CoreServices - ]; - } - ); + vose-alias-method = super.vose-alias-method.overridePythonAttrs + ( + old: { + postInstall = '' + rm -f $out/LICENSE + ''; + } + ); + + uvloop = super.uvloop.overridePythonAttrs + ( + old: { + buildInputs = old.buildInputs ++ lib.optionals stdenv.isDarwin [ + pkgs.darwin.apple_sdk.frameworks.ApplicationServices + pkgs.darwin.apple_sdk.frameworks.CoreServices + ]; + } + ); # Stop infinite recursion by using bootstrapped pkg from nixpkgs wheel = ( pkgs.python3.pkgs.override { python = self.python; } - ).wheel.overridePythonAttrs ( - _: { - inherit (super.wheel) pname name version src; - } - ); + ).wheel.overridePythonAttrs + ( + old: + if old.format == "other" then old else { + inherit (super.wheel) pname name version src; + } + ); zipp = ( if lib.versionAtLeast super.zipp.version "2.0.0" then ( - super.zipp.overridePythonAttrs ( - old: { - prePatch = '' - substituteInPlace setup.py --replace \ - 'setuptools.setup()' \ - 'setuptools.setup(version="${super.zipp.version}")' - ''; - } - ) + super.zipp.overridePythonAttrs + ( + old: { + prePatch = '' + substituteInPlace setup.py --replace \ + 'setuptools.setup()' \ + 'setuptools.setup(version="${super.zipp.version}")' + ''; + } + ) ) else super.zipp - ).overridePythonAttrs ( - old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ - self.toml - ]; - } - ); + ).overridePythonAttrs + ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + self.toml + ]; + } + ); } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix index cda4e8c78d4d..496c4ff15818 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix @@ -12,8 +12,7 @@ let major = builtins.elemAt ver 0; minor = builtins.elemAt ver 1; in - "cp${major}${minor}"; - + "cp${major}${minor}"; abiTag = "${pythonTag}m"; # @@ -24,13 +23,13 @@ let entries = splitString "-" str; p = removeSuffix ".whl" (builtins.elemAt entries 4); in - { - pkgName = builtins.elemAt entries 0; - pkgVer = builtins.elemAt entries 1; - pyVer = builtins.elemAt entries 2; - abi = builtins.elemAt entries 3; - platform = p; - }; + { + pkgName = builtins.elemAt entries 0; + pkgVer = builtins.elemAt entries 1; + pyVer = builtins.elemAt entries 2; + abi = builtins.elemAt entries 3; + platform = p; + }; # # Builds list of acceptable osx wheel files @@ -42,9 +41,9 @@ let v = lib.lists.head versions; vs = lib.lists.tail versions; in - if (builtins.length versions == 0) - then [] - else (builtins.filter (x: hasInfix v x.file) candidates) ++ (findBestMatches vs candidates); + if (builtins.length versions == 0) + then [ ] + else (builtins.filter (x: hasInfix v x.file) candidates) ++ (findBestMatches vs candidates); # pyver = "cpXX" # x = "cpXX" | "py2" | "py3" | "py2.py3" @@ -53,7 +52,7 @@ let normalize = y: ''cp${lib.strings.removePrefix "cp" (lib.strings.removePrefix "py" y)}''; isCompat = p: x: lib.strings.hasPrefix (normalize x) p; in - lib.lists.any (isCompat pyver) (lib.strings.splitString "." x); + lib.lists.any (isCompat pyver) (lib.strings.splitString "." x); # # Selects the best matching wheel file from a list of files @@ -61,42 +60,37 @@ let selectWheel = files: let filesWithoutSources = (builtins.filter (x: hasSuffix ".whl" x.file) files); - isPyAbiCompatible = pyabi: x: x == "none" || pyabi == x; - withPython = ver: abi: x: (isPyVersionCompatible ver x.pyVer) && (isPyAbiCompatible abi x.abi); - - withPlatform = if isLinux - then ( - x: x.platform == "manylinux1_${stdenv.platform.kernelArch}" - || x.platform == "manylinux2010_${stdenv.platform.kernelArch}" - || x.platform == "manylinux2014_${stdenv.platform.kernelArch}" - || x.platform == "any" - ) - else (x: hasInfix "macosx" x.platform || x.platform == "any"); - + withPlatform = + if isLinux + then ( + x: x.platform == "manylinux1_${stdenv.platform.kernelArch}" + || x.platform == "manylinux2010_${stdenv.platform.kernelArch}" + || x.platform == "manylinux2014_${stdenv.platform.kernelArch}" + || x.platform == "any" + ) + else (x: hasInfix "macosx" x.platform || x.platform == "any"); filterWheel = x: let f = toWheelAttrs x.file; in - (withPython pythonTag abiTag f) && (withPlatform f); - + (withPython pythonTag abiTag f) && (withPlatform f); filtered = builtins.filter filterWheel filesWithoutSources; - choose = files: let osxMatches = [ "10_12" "10_11" "10_10" "10_9" "any" ]; linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "any" ]; - chooseLinux = x: lib.singleton (builtins.head (findBestMatches linuxMatches x)); - chooseOSX = x: lib.singleton (builtins.head (findBestMatches osxMatches x)); + chooseLinux = x: lib.take 1 (findBestMatches linuxMatches x); + chooseOSX = x: lib.take 1 (findBestMatches osxMatches x); in - if isLinux - then chooseLinux files - else chooseOSX files; + if isLinux + then chooseLinux files + else chooseOSX files; in - if (builtins.length filtered == 0) - then [] - else choose (filtered); + if (builtins.length filtered == 0) + then [ ] + else choose (filtered); in { inherit selectWheel toWheelAttrs isPyVersionCompatible; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix index 67ffdddb4bd8..773e0a60e093 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix @@ -7,7 +7,6 @@ let # Strip leading/trailing whitespace from string stripStr = s: lib.elemAt (builtins.split "^ *" (lib.elemAt (builtins.split " *$" s) 0)) 2; - findSubExpressionsFun = acc: c: ( if c == "(" then ( let @@ -15,23 +14,23 @@ let isOpen = acc.openP == 0; startPos = if isOpen then posNew else acc.startPos; in - acc // { - inherit startPos; - exprs = acc.exprs ++ [ (substr acc.exprPos (acc.pos - 1) acc.expr) ]; - pos = posNew; - openP = acc.openP + 1; - } + acc // { + inherit startPos; + exprs = acc.exprs ++ [ (substr acc.exprPos (acc.pos - 1) acc.expr) ]; + pos = posNew; + openP = acc.openP + 1; + } ) else if c == ")" then ( let openP = acc.openP - 1; exprs = findSubExpressions (substr acc.startPos acc.pos acc.expr); in - acc // { - inherit openP; - pos = acc.pos + 1; - exprs = if openP == 0 then acc.exprs ++ [ exprs ] else acc.exprs; - exprPos = if openP == 0 then acc.pos + 1 else acc.exprPos; - } + acc // { + inherit openP; + pos = acc.pos + 1; + exprs = if openP == 0 then acc.exprs ++ [ exprs ] else acc.exprs; + exprPos = if openP == 0 then acc.pos + 1 else acc.exprPos; + } ) else acc // { pos = acc.pos + 1; } ); @@ -39,7 +38,7 @@ let findSubExpressions = expr: let acc = builtins.foldl' findSubExpressionsFun { - exprs = []; + exprs = [ ]; expr = expr; pos = 0; openP = 0; @@ -47,18 +46,16 @@ let startPos = 0; } (lib.stringToCharacters expr); tailExpr = (substr acc.exprPos acc.pos expr); - tailExprs = if tailExpr != "" then [ tailExpr ] else []; + tailExprs = if tailExpr != "" then [ tailExpr ] else [ ]; in - acc.exprs ++ tailExprs; - + acc.exprs ++ tailExprs; parseExpressions = exprs: let splitCond = ( s: builtins.map - (x: stripStr (if builtins.typeOf x == "list" then (builtins.elemAt x 0) else x)) + (x: stripStr ( if builtins.typeOf x == "list" then (builtins.elemAt x 0) else x)) (builtins.split " (and|or) " (s + " ")) ); - mapfn = expr: ( if (builtins.match "^ ?$" expr != null) then null # Filter empty else if (builtins.elem expr [ "and" "or" ]) then { @@ -70,14 +67,12 @@ let value = expr; } ); - parse = expr: builtins.filter (x: x != null) (builtins.map mapfn (splitCond expr)); in - builtins.foldl' ( - acc: v: acc ++ ( - if builtins.typeOf v == "string" then parse v else [ (parseExpressions v) ] - ) - ) [] exprs; + builtins.foldl' + ( + acc: v: acc ++ ( if builtins.typeOf v == "string" then parse v else [ (parseExpressions v) ]) + ) [ ] exprs; # Transform individual expressions to structured expressions # This function also performs variable substitution, replacing environment markers with their explicit values @@ -94,9 +89,10 @@ let else throw "Unsupported platform" ); platform_machine = stdenv.platform.kernelArch; - platform_python_implementation = let - impl = python.passthru.implementation; - in + platform_python_implementation = + let + impl = python.passthru.implementation; + in ( if impl == "cpython" then "CPython" else if impl == "pypy" then "PyPy" @@ -115,34 +111,32 @@ let implementation_version = python.version; extra = ""; }; - substituteVar = value: if builtins.hasAttr value variables then (builtins.toJSON variables."${value}") else value; - processVar = value: builtins.foldl' (acc: v: v acc) value [ stripStr substituteVar ]; in - if builtins.typeOf exprs == "set" then ( - if exprs.type == "expr" then ( - let - mVal = ''[a-zA-Z0-9\'"_\. ]+''; - mOp = "in|[!=<>]+"; - e = stripStr exprs.value; - m = builtins.map stripStr (builtins.match ''^(${mVal}) *(${mOp}) *(${mVal})$'' e); - in - { - type = "expr"; - value = { - op = builtins.elemAt m 1; - values = [ - (processVar (builtins.elemAt m 0)) - (processVar (builtins.elemAt m 2)) - ]; - }; - } - ) else exprs - ) else builtins.map transformExpressions exprs; + if builtins.typeOf exprs == "set" then ( + if exprs.type == "expr" then ( + let + mVal = ''[a-zA-Z0-9\'"_\. ]+''; + mOp = "in|[!=<>]+"; + e = stripStr exprs.value; + m = builtins.map stripStr (builtins.match ''^(${mVal}) *(${mOp}) *(${mVal})$'' e); + in + { + type = "expr"; + value = { + op = builtins.elemAt m 1; + values = [ + (processVar (builtins.elemAt m 0)) + (processVar (builtins.elemAt m 2)) + ]; + }; + } + ) else exprs + ) else builtins.map transformExpressions exprs; # Recursively eval all expressions evalExpressions = exprs: @@ -165,32 +159,33 @@ let let parts = builtins.splitVersion c; pruned = lib.take ((builtins.length parts) - 1) parts; - upper = builtins.toString ( - (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1 - ); + upper = builtins.toString + ( + (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1 + ); upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned); in - op.">=" v c && op."<" v upperConstraint; + op.">=" v c && op."<" v upperConstraint; "===" = x: y: x == y; "in" = x: y: let values = builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " (unmarshal y)); in - builtins.elem (unmarshal x) values; + builtins.elem (unmarshal x) values; }; in - if builtins.typeOf exprs == "set" then ( - if exprs.type == "expr" then ( - let - expr = exprs; - result = (op."${expr.value.op}") (builtins.elemAt expr.value.values 0) (builtins.elemAt expr.value.values 1); - in - { - type = "value"; - value = result; - } - ) else exprs - ) else builtins.map evalExpressions exprs; + if builtins.typeOf exprs == "set" then ( + if exprs.type == "expr" then ( + let + expr = exprs; + result = (op."${expr.value.op}") (builtins.elemAt expr.value.values 0) (builtins.elemAt expr.value.values 1); + in + { + type = "value"; + value = result; + } + ) else exprs + ) else builtins.map evalExpressions exprs; # Now that we have performed an eval all that's left to do is to concat the graph into a single bool reduceExpressions = exprs: @@ -217,18 +212,18 @@ let cond = "and"; } v; in - acc // { - value = cond."${acc.cond}" acc.value ret.value; - } + acc // { + value = cond."${acc.cond}" acc.value ret.value; + } ) else throw "Unsupported type" ); in - ( - builtins.foldl' reduceExpressionsFun { - value = true; - cond = "and"; - } exprs - ).value; + ( + builtins.foldl' reduceExpressionsFun { + value = true; + cond = "and"; + } exprs + ).value; in e: builtins.foldl' (acc: v: v acc) e [ findSubExpressions diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix b/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix index 07dcbbc5eacb..e86b1d3ac660 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix @@ -1,28 +1,28 @@ { lib, ireplace }: let inherit (builtins) elemAt match; - - operators = let - matchWildCard = s: match "([^\*])(\.[\*])" s; - mkComparison = ret: version: v: builtins.compareVersions version v == ret; - mkIdxComparison = idx: version: v: - let - ver = builtins.splitVersion v; - minor = builtins.toString (lib.toInt (elemAt ver idx) + 1); - upper = builtins.concatStringsSep "." (ireplace idx minor ver); - in + operators = + let + matchWildCard = s: match "([^\*])(\.[\*])" s; + mkComparison = ret: version: v: builtins.compareVersions version v == ret; + mkIdxComparison = idx: version: v: + let + ver = builtins.splitVersion v; + minor = builtins.toString (lib.toInt (elemAt ver idx) + 1); + upper = builtins.concatStringsSep "." (ireplace idx minor ver); + in operators.">=" version v && operators."<" version upper; - dropWildcardPrecision = f: version: constraint: - let - m = matchWildCard constraint; - hasWildcard = m != null; - c = if hasWildcard then (elemAt m 0) else constraint; - v = - if hasWildcard then (builtins.substring 0 (builtins.stringLength c) version) - else version; - in + dropWildcardPrecision = f: version: constraint: + let + m = matchWildCard constraint; + hasWildcard = m != null; + c = if hasWildcard then (elemAt m 0) else constraint; + v = + if hasWildcard then (builtins.substring 0 (builtins.stringLength c) version) + else version; + in f v c; - in + in { # Prefix operators "==" = dropWildcardPrecision (mkComparison 0); @@ -39,24 +39,23 @@ let # Prune constraint parts = builtins.splitVersion c; pruned = lib.take ((builtins.length parts) - 1) parts; - upper = builtins.toString ( - (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1 - ); + upper = builtins.toString + ( + (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1 + ); upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned); in - operators.">=" v c && operators."<" v upperConstraint; + operators.">=" v c && operators."<" v upperConstraint; # Infix operators "-" = version: v: operators.">=" version v.vl && operators."<=" version v.vu; # Arbitrary equality clause, just run simple comparison "===" = v: c: v == c; # }; - re = { operators = "([=><!~\^]+)"; version = "([0-9\.\*x]+)"; }; - parseConstraint = constraint: let constraintStr = builtins.replaceStrings [ " " ] [ "" ] constraint; @@ -65,26 +64,24 @@ let # There is also an infix operator to match ranges mIn = match "${re.version} *(-) *${re.version}" constraintStr; in - ( - if mPre != null then { - op = elemAt mPre 0; - v = elemAt mPre 1; - } - # Infix operators are range matches - else if mIn != null then { - op = elemAt mIn 1; - v = { - vl = (elemAt mIn 0); - vu = (elemAt mIn 2); - }; - } - else throw "Constraint \"${constraintStr}\" could not be parsed" - ); - + ( + if mPre != null then { + op = elemAt mPre 0; + v = elemAt mPre 1; + } + # Infix operators are range matches + else if mIn != null then { + op = elemAt mIn 1; + v = { + vl = (elemAt mIn 0); + vu = (elemAt mIn 2); + }; + } + else throw "Constraint \"${constraintStr}\" could not be parsed" + ); satisfiesSemver = version: constraint: let inherit (parseConstraint constraint) op v; - in - if constraint == "*" then true else operators."${op}" version v; + in if constraint == "*" then true else operators."${op}" version v; in { inherit satisfiesSemver; } diff --git a/pkgs/development/tools/poetry2nix/update b/pkgs/development/tools/poetry2nix/update index 4840c23c3b81..5bea272f8a9a 100755 --- a/pkgs/development/tools/poetry2nix/update +++ b/pkgs/development/tools/poetry2nix/update @@ -14,7 +14,7 @@ curl -L -s https://github.com/nix-community/poetry2nix/archive/master.tar.gz | t mv poetry2nix-master/* . mkdir build -cp *.nix *.json *.py build/ +cp *.* build/ cp -r hooks bin build/ rm build/shell.nix build/generate.py build/overlay.nix build/flake.nix |