From 92d53362d4da7e19b6a7a6b1b936a0a34a17c0c9 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sat, 24 Mar 2018 10:47:41 -0400 Subject: Move all nixpkgs doc files in to the doc directory This makes a makefile-driven developer workflow nicer. --- doc/default.nix | 40 +- doc/introduction.chapter.md | 51 ++ doc/introduction.md | 51 -- doc/languages-frameworks/emscripten.md | 185 ----- doc/languages-frameworks/emscripten.section.md | 185 +++++ doc/languages-frameworks/haskell.md | 953 ---------------------- doc/languages-frameworks/haskell.section.md | 953 ++++++++++++++++++++++ doc/languages-frameworks/idris.section.md | 39 + doc/languages-frameworks/index.xml | 16 +- doc/languages-frameworks/node.section.md | 51 ++ doc/languages-frameworks/python.md | 1026 ------------------------ doc/languages-frameworks/python.section.md | 1026 ++++++++++++++++++++++++ doc/languages-frameworks/r.section.md | 120 +++ doc/languages-frameworks/rust.md | 309 ------- doc/languages-frameworks/rust.section.md | 309 +++++++ doc/languages-frameworks/vim.md | 135 ---- doc/languages-frameworks/vim.section.md | 135 ++++ doc/manual.xml | 2 +- doc/shell.md | 20 - doc/shell.section.md | 20 + pkgs/development/idris-modules/README.md | 40 +- pkgs/development/node-packages/README.md | 52 +- pkgs/development/r-modules/README.md | 121 +-- 23 files changed, 2921 insertions(+), 2918 deletions(-) create mode 100644 doc/introduction.chapter.md delete mode 100644 doc/introduction.md delete mode 100644 doc/languages-frameworks/emscripten.md create mode 100644 doc/languages-frameworks/emscripten.section.md delete mode 100644 doc/languages-frameworks/haskell.md create mode 100644 doc/languages-frameworks/haskell.section.md create mode 100644 doc/languages-frameworks/idris.section.md create mode 100644 doc/languages-frameworks/node.section.md delete mode 100644 doc/languages-frameworks/python.md create mode 100644 doc/languages-frameworks/python.section.md create mode 100644 doc/languages-frameworks/r.section.md delete mode 100644 doc/languages-frameworks/rust.md create mode 100644 doc/languages-frameworks/rust.section.md delete mode 100644 doc/languages-frameworks/vim.md create mode 100644 doc/languages-frameworks/vim.section.md delete mode 100644 doc/shell.md create mode 100644 doc/shell.section.md diff --git a/doc/default.nix b/doc/default.nix index ec458634a42c..1adcd7968204 100644 --- a/doc/default.nix +++ b/doc/default.nix @@ -45,45 +45,45 @@ pkgs.stdenv.mkDerivation { cp -s '${sources-langs}'/* ./languages-frameworks '' + toDocbook { - inputFile = ./introduction.md; - outputFile = "introduction.xml"; + inputFile = ./introduction.chapter.md; + outputFile = "introduction.chapter.xml"; useChapters = true; } + toDocbook { - inputFile = ./shell.md; - outputFile = "shell.xml"; + inputFile = ./shell.section.md; + outputFile = "shell.section.xml"; } + toDocbook { - inputFile = ./languages-frameworks/python.md; - outputFile = "./languages-frameworks/python.xml"; + inputFile = ./languages-frameworks/python.section.md; + outputFile = "./languages-frameworks/python.section.xml"; } + toDocbook { - inputFile = ./languages-frameworks/haskell.md; - outputFile = "./languages-frameworks/haskell.xml"; + inputFile = ./languages-frameworks/haskell.section.md; + outputFile = "./languages-frameworks/haskell.section.xml"; } + toDocbook { - inputFile = ../pkgs/development/idris-modules/README.md; - outputFile = "languages-frameworks/idris.xml"; + inputFile = ./languages-frameworks/idris.section.md; + outputFile = "languages-frameworks/idris.section.xml"; } + toDocbook { - inputFile = ../pkgs/development/node-packages/README.md; - outputFile = "languages-frameworks/node.xml"; + inputFile = ./languages-frameworks/node.section.md; + outputFile = "languages-frameworks/node.section.xml"; } + toDocbook { - inputFile = ../pkgs/development/r-modules/README.md; - outputFile = "languages-frameworks/r.xml"; + inputFile = ./languages-frameworks/r.section.md; + outputFile = "languages-frameworks/r.section.xml"; } + toDocbook { - inputFile = ./languages-frameworks/rust.md; - outputFile = "./languages-frameworks/rust.xml"; + inputFile = ./languages-frameworks/rust.section.md; + outputFile = "./languages-frameworks/rust.section.xml"; } + toDocbook { - inputFile = ./languages-frameworks/vim.md; - outputFile = "./languages-frameworks/vim.xml"; + inputFile = ./languages-frameworks/vim.section.md; + outputFile = "./languages-frameworks/vim.section.xml"; } + toDocbook { - inputFile = ./languages-frameworks/emscripten.md; - outputFile = "./languages-frameworks/emscripten.xml"; + inputFile = ./languages-frameworks/emscripten.section.md; + outputFile = "./languages-frameworks/emscripten.section.xml"; } + '' echo ${lib.nixpkgsVersion} > .version diff --git a/doc/introduction.chapter.md b/doc/introduction.chapter.md new file mode 100644 index 000000000000..8b03221c64b1 --- /dev/null +++ b/doc/introduction.chapter.md @@ -0,0 +1,51 @@ +--- +title: Introduction +author: Frederik Rietdijk +date: 2015-11-25 +--- + +# Introduction + +The Nix Packages collection (Nixpkgs) is a set of thousands of packages for the +[Nix package manager](http://nixos.org/nix/), released under a +[permissive MIT/X11 license](https://github.com/NixOS/nixpkgs/blob/master/COPYING). +Packages are available for several platforms, and can be used with the Nix +package manager on most GNU/Linux distributions as well as NixOS. + +This manual primarily describes how to write packages for the Nix Packages collection +(Nixpkgs). Thus it’s mainly for packagers and developers who want to add packages to +Nixpkgs. If you like to learn more about the Nix package manager and the Nix +expression language, then you are kindly referred to the [Nix manual](http://nixos.org/nix/manual/). + +## Overview of Nixpkgs + +Nix expressions describe how to build packages from source and are collected in +the [nixpkgs repository](https://github.com/NixOS/nixpkgs). Also included in the +collection are Nix expressions for +[NixOS modules](http://nixos.org/nixos/manual/index.html#sec-writing-modules). +With these expressions the Nix package manager can build binary packages. + +Packages, including the Nix packages collection, are distributed through +[channels](http://nixos.org/nix/manual/#sec-channels). The collection is +distributed for users of Nix on non-NixOS distributions through the channel +`nixpkgs`. Users of NixOS generally use one of the `nixos-*` channels, e.g. +`nixos-16.03`, which includes all packages and modules for the stable NixOS +16.03. The purpose of stable NixOS releases are generally only given +security updates. More up to date packages and modules are available via the +`nixos-unstable` channel. + +Both `nixos-unstable` and `nixpkgs` follow the `master` branch of the Nixpkgs +repository, although both do lag the `master` branch by generally +[a couple of days](http://howoldis.herokuapp.com/). Updates to a channel are +distributed as soon as all tests for that channel pass, e.g. +[this table](http://hydra.nixos.org/job/nixpkgs/trunk/unstable#tabs-constituents) +shows the status of tests for the `nixpkgs` channel. + +The tests are conducted by a cluster called [Hydra](http://nixos.org/hydra/), +which also builds binary packages from the Nix expressions in Nixpkgs for +`x86_64-linux`, `i686-linux` and `x86_64-darwin`. +The binaries are made available via a [binary cache](https://cache.nixos.org). + +The current Nix expressions of the channels are available in the +[`nixpkgs-channels`](https://github.com/NixOS/nixpkgs-channels) repository, +which has branches corresponding to the available channels. diff --git a/doc/introduction.md b/doc/introduction.md deleted file mode 100644 index 8b03221c64b1..000000000000 --- a/doc/introduction.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Introduction -author: Frederik Rietdijk -date: 2015-11-25 ---- - -# Introduction - -The Nix Packages collection (Nixpkgs) is a set of thousands of packages for the -[Nix package manager](http://nixos.org/nix/), released under a -[permissive MIT/X11 license](https://github.com/NixOS/nixpkgs/blob/master/COPYING). -Packages are available for several platforms, and can be used with the Nix -package manager on most GNU/Linux distributions as well as NixOS. - -This manual primarily describes how to write packages for the Nix Packages collection -(Nixpkgs). Thus it’s mainly for packagers and developers who want to add packages to -Nixpkgs. If you like to learn more about the Nix package manager and the Nix -expression language, then you are kindly referred to the [Nix manual](http://nixos.org/nix/manual/). - -## Overview of Nixpkgs - -Nix expressions describe how to build packages from source and are collected in -the [nixpkgs repository](https://github.com/NixOS/nixpkgs). Also included in the -collection are Nix expressions for -[NixOS modules](http://nixos.org/nixos/manual/index.html#sec-writing-modules). -With these expressions the Nix package manager can build binary packages. - -Packages, including the Nix packages collection, are distributed through -[channels](http://nixos.org/nix/manual/#sec-channels). The collection is -distributed for users of Nix on non-NixOS distributions through the channel -`nixpkgs`. Users of NixOS generally use one of the `nixos-*` channels, e.g. -`nixos-16.03`, which includes all packages and modules for the stable NixOS -16.03. The purpose of stable NixOS releases are generally only given -security updates. More up to date packages and modules are available via the -`nixos-unstable` channel. - -Both `nixos-unstable` and `nixpkgs` follow the `master` branch of the Nixpkgs -repository, although both do lag the `master` branch by generally -[a couple of days](http://howoldis.herokuapp.com/). Updates to a channel are -distributed as soon as all tests for that channel pass, e.g. -[this table](http://hydra.nixos.org/job/nixpkgs/trunk/unstable#tabs-constituents) -shows the status of tests for the `nixpkgs` channel. - -The tests are conducted by a cluster called [Hydra](http://nixos.org/hydra/), -which also builds binary packages from the Nix expressions in Nixpkgs for -`x86_64-linux`, `i686-linux` and `x86_64-darwin`. -The binaries are made available via a [binary cache](https://cache.nixos.org). - -The current Nix expressions of the channels are available in the -[`nixpkgs-channels`](https://github.com/NixOS/nixpkgs-channels) repository, -which has branches corresponding to the available channels. diff --git a/doc/languages-frameworks/emscripten.md b/doc/languages-frameworks/emscripten.md deleted file mode 100644 index 24c49ec1409c..000000000000 --- a/doc/languages-frameworks/emscripten.md +++ /dev/null @@ -1,185 +0,0 @@ -# User's Guide to Emscripten in Nixpkgs - -[Emscripten](https://github.com/kripken/emscripten): An LLVM-to-JavaScript Compiler - -This section of the manual covers how to use `emscripten` in nixpkgs. - -Minimal requirements: - -* nix -* nixpkgs - -Modes of use of `emscripten`: - -* **Imperative usage** (on the command line): - - If you want to work with `emcc`, `emconfigure` and `emmake` as you are used to from Ubuntu and similar distributions you can use these commands: - - * `nix-env -i emscripten` - * `nix-shell -p emscripten` - -* **Declarative usage**: - - This mode is far more power full since this makes use of `nix` for dependency management of emscripten libraries and targets by using the `mkDerivation` which is implemented by `pkgs.emscriptenStdenv` and `pkgs.buildEmscriptenPackage`. The source for the packages is in `pkgs/top-level/emscripten-packages.nix` and the abstraction behind it in `pkgs/development/em-modules/generic/default.nix`. - * build and install all packages: - * `nix-env -iA emscriptenPackages` - - * dev-shell for zlib implementation hacking: - * `nix-shell -A emscriptenPackages.zlib` - - -## Imperative usage - -A few things to note: - -* `export EMCC_DEBUG=2` is nice for debugging -* `~/.emscripten`, the build artifact cache sometimes creates issues and needs to be removed from time to time - - -## Declarative usage - -Let's see two different examples from `pkgs/top-level/emscripten-packages.nix`: - -* `pkgs.zlib.override` -* `pkgs.buildEmscriptenPackage` - -Both are interesting concepts. - -A special requirement of the `pkgs.buildEmscriptenPackage` is the `doCheck = true` is a default meaning that each emscriptenPackage requires a `checkPhase` implemented. - -* Use `export EMCC_DEBUG=2` from within a emscriptenPackage's `phase` to get more detailed debug output what is going wrong. -* ~/.emscripten cache is requiring us to set `HOME=$TMPDIR` in individual phases. This makes compilation slower but also makes it more deterministic. - -### Usage 1: pkgs.zlib.override - -This example uses `zlib` from nixpkgs but instead of compiling **C** to **ELF** it compiles **C** to **JS** since we were using `pkgs.zlib.override` and changed stdenv to `pkgs.emscriptenStdenv`. A few adaptions and hacks were set in place to make it working. One advantage is that when `pkgs.zlib` is updated, it will automatically update this package as well. However, this can also be the downside... - -See the `zlib` example: - - zlib = (pkgs.zlib.override { - stdenv = pkgs.emscriptenStdenv; - }).overrideDerivation - (old: rec { - buildInputs = old.buildInputs ++ [ pkgconfig ]; - # we need to reset this setting! - NIX_CFLAGS_COMPILE=""; - configurePhase = '' - # FIXME: Some tests require writing at $HOME - HOME=$TMPDIR - runHook preConfigure - - #export EMCC_DEBUG=2 - emconfigure ./configure --prefix=$out --shared - - runHook postConfigure - ''; - dontStrip = true; - outputs = [ "out" ]; - buildPhase = '' - emmake make - ''; - installPhase = '' - emmake make install - ''; - checkPhase = '' - echo "================= testing zlib using node =================" - - echo "Compiling a custom test" - set -x - emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 test/example.c -DZ_SOLO \ - libz.so.${old.version} -I . -o example.js - - echo "Using node to execute the test" - ${pkgs.nodejs}/bin/node ./example.js - - set +x - if [ $? -ne 0 ]; then - echo "test failed for some reason" - exit 1; - else - echo "it seems to work! very good." - fi - echo "================= /testing zlib using node =================" - ''; - - postPatch = pkgs.stdenv.lib.optionalString pkgs.stdenv.isDarwin '' - substituteInPlace configure \ - --replace '/usr/bin/libtool' 'ar' \ - --replace 'AR="libtool"' 'AR="ar"' \ - --replace 'ARFLAGS="-o"' 'ARFLAGS="-r"' - ''; - }); - -### Usage 2: pkgs.buildEmscriptenPackage - -This `xmlmirror` example features a emscriptenPackage which is defined completely from this context and no `pkgs.zlib.override` is used. - - xmlmirror = pkgs.buildEmscriptenPackage rec { - name = "xmlmirror"; - - buildInputs = [ pkgconfig autoconf automake libtool gnumake libxml2 nodejs openjdk json_c ]; - nativeBuildInputs = [ pkgconfig zlib ]; - - src = pkgs.fetchgit { - url = "https://gitlab.com/odfplugfest/xmlmirror.git"; - rev = "4fd7e86f7c9526b8f4c1733e5c8b45175860a8fd"; - sha256 = "1jasdqnbdnb83wbcnyrp32f36w3xwhwp0wq8lwwmhqagxrij1r4b"; - }; - - configurePhase = '' - rm -f fastXmlLint.js* - # a fix for ERROR:root:For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was 234217728 - # https://gitlab.com/odfplugfest/xmlmirror/issues/8 - sed -e "s/TOTAL_MEMORY=234217728/TOTAL_MEMORY=268435456/g" -i Makefile.emEnv - # https://github.com/kripken/emscripten/issues/6344 - # https://gitlab.com/odfplugfest/xmlmirror/issues/9 - sed -e "s/\$(JSONC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(LIBXML20_LDFLAGS)/\$(JSONC_LDFLAGS) \$(LIBXML20_LDFLAGS) \$(ZLIB_LDFLAGS) /g" -i Makefile.emEnv - # https://gitlab.com/odfplugfest/xmlmirror/issues/11 - sed -e "s/-o fastXmlLint.js/-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -o fastXmlLint.js/g" -i Makefile.emEnv - ''; - - buildPhase = '' - HOME=$TMPDIR - make -f Makefile.emEnv - ''; - - outputs = [ "out" "doc" ]; - - installPhase = '' - mkdir -p $out/share - mkdir -p $doc/share/${name} - - cp Demo* $out/share - cp -R codemirror-5.12 $out/share - cp fastXmlLint.js* $out/share - cp *.xsd $out/share - cp *.js $out/share - cp *.xhtml $out/share - cp *.html $out/share - cp *.json $out/share - cp *.rng $out/share - cp README.md $doc/share/${name} - ''; - checkPhase = '' - - ''; - }; - -### Declarative debugging - -Use `nix-shell -I nixpkgs=/some/dir/nixpkgs -A emscriptenPackages.libz` and from there you can go trough the individual steps. This makes it easy to build a good `unit test` or list the files of the project. - -1. `nix-shell -I nixpkgs=/some/dir/nixpkgs -A emscriptenPackages.libz` -2. `cd /tmp/` -3. `unpackPhase` -4. cd libz-1.2.3 -5. `configurePhase` -6. `buildPhase` -7. ... happy hacking... - -## Summary - -Using this toolchain makes it easy to leverage `nix` from NixOS, MacOSX or even Windows (WSL+ubuntu+nix). This toolchain is reproducible, behaves like the rest of the packages from nixpkgs and contains a set of well working examples to learn and adapt from. - -If in trouble, ask the maintainers. - diff --git a/doc/languages-frameworks/emscripten.section.md b/doc/languages-frameworks/emscripten.section.md new file mode 100644 index 000000000000..24c49ec1409c --- /dev/null +++ b/doc/languages-frameworks/emscripten.section.md @@ -0,0 +1,185 @@ +# User's Guide to Emscripten in Nixpkgs + +[Emscripten](https://github.com/kripken/emscripten): An LLVM-to-JavaScript Compiler + +This section of the manual covers how to use `emscripten` in nixpkgs. + +Minimal requirements: + +* nix +* nixpkgs + +Modes of use of `emscripten`: + +* **Imperative usage** (on the command line): + + If you want to work with `emcc`, `emconfigure` and `emmake` as you are used to from Ubuntu and similar distributions you can use these commands: + + * `nix-env -i emscripten` + * `nix-shell -p emscripten` + +* **Declarative usage**: + + This mode is far more power full since this makes use of `nix` for dependency management of emscripten libraries and targets by using the `mkDerivation` which is implemented by `pkgs.emscriptenStdenv` and `pkgs.buildEmscriptenPackage`. The source for the packages is in `pkgs/top-level/emscripten-packages.nix` and the abstraction behind it in `pkgs/development/em-modules/generic/default.nix`. + * build and install all packages: + * `nix-env -iA emscriptenPackages` + + * dev-shell for zlib implementation hacking: + * `nix-shell -A emscriptenPackages.zlib` + + +## Imperative usage + +A few things to note: + +* `export EMCC_DEBUG=2` is nice for debugging +* `~/.emscripten`, the build artifact cache sometimes creates issues and needs to be removed from time to time + + +## Declarative usage + +Let's see two different examples from `pkgs/top-level/emscripten-packages.nix`: + +* `pkgs.zlib.override` +* `pkgs.buildEmscriptenPackage` + +Both are interesting concepts. + +A special requirement of the `pkgs.buildEmscriptenPackage` is the `doCheck = true` is a default meaning that each emscriptenPackage requires a `checkPhase` implemented. + +* Use `export EMCC_DEBUG=2` from within a emscriptenPackage's `phase` to get more detailed debug output what is going wrong. +* ~/.emscripten cache is requiring us to set `HOME=$TMPDIR` in individual phases. This makes compilation slower but also makes it more deterministic. + +### Usage 1: pkgs.zlib.override + +This example uses `zlib` from nixpkgs but instead of compiling **C** to **ELF** it compiles **C** to **JS** since we were using `pkgs.zlib.override` and changed stdenv to `pkgs.emscriptenStdenv`. A few adaptions and hacks were set in place to make it working. One advantage is that when `pkgs.zlib` is updated, it will automatically update this package as well. However, this can also be the downside... + +See the `zlib` example: + + zlib = (pkgs.zlib.override { + stdenv = pkgs.emscriptenStdenv; + }).overrideDerivation + (old: rec { + buildInputs = old.buildInputs ++ [ pkgconfig ]; + # we need to reset this setting! + NIX_CFLAGS_COMPILE=""; + configurePhase = '' + # FIXME: Some tests require writing at $HOME + HOME=$TMPDIR + runHook preConfigure + + #export EMCC_DEBUG=2 + emconfigure ./configure --prefix=$out --shared + + runHook postConfigure + ''; + dontStrip = true; + outputs = [ "out" ]; + buildPhase = '' + emmake make + ''; + installPhase = '' + emmake make install + ''; + checkPhase = '' + echo "================= testing zlib using node =================" + + echo "Compiling a custom test" + set -x + emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 test/example.c -DZ_SOLO \ + libz.so.${old.version} -I . -o example.js + + echo "Using node to execute the test" + ${pkgs.nodejs}/bin/node ./example.js + + set +x + if [ $? -ne 0 ]; then + echo "test failed for some reason" + exit 1; + else + echo "it seems to work! very good." + fi + echo "================= /testing zlib using node =================" + ''; + + postPatch = pkgs.stdenv.lib.optionalString pkgs.stdenv.isDarwin '' + substituteInPlace configure \ + --replace '/usr/bin/libtool' 'ar' \ + --replace 'AR="libtool"' 'AR="ar"' \ + --replace 'ARFLAGS="-o"' 'ARFLAGS="-r"' + ''; + }); + +### Usage 2: pkgs.buildEmscriptenPackage + +This `xmlmirror` example features a emscriptenPackage which is defined completely from this context and no `pkgs.zlib.override` is used. + + xmlmirror = pkgs.buildEmscriptenPackage rec { + name = "xmlmirror"; + + buildInputs = [ pkgconfig autoconf automake libtool gnumake libxml2 nodejs openjdk json_c ]; + nativeBuildInputs = [ pkgconfig zlib ]; + + src = pkgs.fetchgit { + url = "https://gitlab.com/odfplugfest/xmlmirror.git"; + rev = "4fd7e86f7c9526b8f4c1733e5c8b45175860a8fd"; + sha256 = "1jasdqnbdnb83wbcnyrp32f36w3xwhwp0wq8lwwmhqagxrij1r4b"; + }; + + configurePhase = '' + rm -f fastXmlLint.js* + # a fix for ERROR:root:For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was 234217728 + # https://gitlab.com/odfplugfest/xmlmirror/issues/8 + sed -e "s/TOTAL_MEMORY=234217728/TOTAL_MEMORY=268435456/g" -i Makefile.emEnv + # https://github.com/kripken/emscripten/issues/6344 + # https://gitlab.com/odfplugfest/xmlmirror/issues/9 + sed -e "s/\$(JSONC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(LIBXML20_LDFLAGS)/\$(JSONC_LDFLAGS) \$(LIBXML20_LDFLAGS) \$(ZLIB_LDFLAGS) /g" -i Makefile.emEnv + # https://gitlab.com/odfplugfest/xmlmirror/issues/11 + sed -e "s/-o fastXmlLint.js/-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -o fastXmlLint.js/g" -i Makefile.emEnv + ''; + + buildPhase = '' + HOME=$TMPDIR + make -f Makefile.emEnv + ''; + + outputs = [ "out" "doc" ]; + + installPhase = '' + mkdir -p $out/share + mkdir -p $doc/share/${name} + + cp Demo* $out/share + cp -R codemirror-5.12 $out/share + cp fastXmlLint.js* $out/share + cp *.xsd $out/share + cp *.js $out/share + cp *.xhtml $out/share + cp *.html $out/share + cp *.json $out/share + cp *.rng $out/share + cp README.md $doc/share/${name} + ''; + checkPhase = '' + + ''; + }; + +### Declarative debugging + +Use `nix-shell -I nixpkgs=/some/dir/nixpkgs -A emscriptenPackages.libz` and from there you can go trough the individual steps. This makes it easy to build a good `unit test` or list the files of the project. + +1. `nix-shell -I nixpkgs=/some/dir/nixpkgs -A emscriptenPackages.libz` +2. `cd /tmp/` +3. `unpackPhase` +4. cd libz-1.2.3 +5. `configurePhase` +6. `buildPhase` +7. ... happy hacking... + +## Summary + +Using this toolchain makes it easy to leverage `nix` from NixOS, MacOSX or even Windows (WSL+ubuntu+nix). This toolchain is reproducible, behaves like the rest of the packages from nixpkgs and contains a set of well working examples to learn and adapt from. + +If in trouble, ask the maintainers. + diff --git a/doc/languages-frameworks/haskell.md b/doc/languages-frameworks/haskell.md deleted file mode 100644 index 1623e0d276f2..000000000000 --- a/doc/languages-frameworks/haskell.md +++ /dev/null @@ -1,953 +0,0 @@ ---- -title: User's Guide for Haskell in Nixpkgs -author: Peter Simons -date: 2015-06-01 ---- -# User's Guide to the Haskell Infrastructure - - -## How to install Haskell packages - -Nixpkgs distributes build instructions for all Haskell packages registered on -[Hackage](http://hackage.haskell.org/), but strangely enough normal Nix package -lookups don't seem to discover any of them, except for the default version of ghc, cabal-install, and stack: -``` -$ nix-env -i alex -error: selector ‘alex’ matches no derivations -$ nix-env -qa ghc -ghc-7.10.2 -``` - -The Haskell package set is not registered in the top-level namespace because it -is *huge*. If all Haskell packages were visible to these commands, then -name-based search/install operations would be much slower than they are now. We -avoided that by keeping all Haskell-related packages in a separate attribute -set called `haskellPackages`, which the following command will list: -``` -$ nix-env -f "" -qaP -A haskellPackages -haskellPackages.a50 a50-0.5 -haskellPackages.abacate haskell-abacate-0.0.0.0 -haskellPackages.abcBridge haskell-abcBridge-0.12 -haskellPackages.afv afv-0.1.1 -haskellPackages.alex alex-3.1.4 -haskellPackages.Allure Allure-0.4.101.1 -haskellPackages.alms alms-0.6.7 -[... some 8000 entries omitted ...] -``` - -To install any of those packages into your profile, refer to them by their -attribute path (first column): -```shell -nix-env -f "" -iA haskellPackages.Allure ... -``` - -The attribute path of any Haskell packages corresponds to the name of that -particular package on Hackage: the package `cabal-install` has the attribute -`haskellPackages.cabal-install`, and so on. (Actually, this convention causes -trouble with packages like `3dmodels` and `4Blocks`, because these names are -invalid identifiers in the Nix language. The issue of how to deal with these -rare corner cases is currently unresolved.) - -Haskell packages whose Nix name (second column) begins with a `haskell-` prefix -are packages that provide a library whereas packages without that prefix -provide just executables. Libraries may provide executables too, though: the -package `haskell-pandoc`, for example, installs both a library and an -application. You can install and use Haskell executables just like any other -program in Nixpkgs, but using Haskell libraries for development is a bit -trickier and we'll address that subject in great detail in section [How to -create a development environment]. - -Attribute paths are deterministic inside of Nixpkgs, but the path necessary to -reach Nixpkgs varies from system to system. We dodged that problem by giving -`nix-env` an explicit `-f ""` parameter, but if you call `nix-env` -without that flag, then chances are the invocation fails: -``` -$ nix-env -iA haskellPackages.cabal-install -error: attribute ‘haskellPackages’ in selection path - ‘haskellPackages.cabal-install’ not found -``` - -On NixOS, for example, Nixpkgs does *not* exist in the top-level namespace by -default. To figure out the proper attribute path, it's easiest to query for the -path of a well-known Nixpkgs package, i.e.: -``` -$ nix-env -qaP coreutils -nixos.coreutils coreutils-8.23 -``` - -If your system responds like that (most NixOS installations will), then the -attribute path to `haskellPackages` is `nixos.haskellPackages`. Thus, if you -want to use `nix-env` without giving an explicit `-f` flag, then that's the way -to do it: -```shell -nix-env -qaP -A nixos.haskellPackages -nix-env -iA nixos.haskellPackages.cabal-install -``` - -Our current default compiler is GHC 7.10.x and the `haskellPackages` set -contains packages built with that particular version. Nixpkgs contains the -latest major release of every GHC since 6.10.4, however, and there is a whole -family of package sets available that defines Hackage packages built with each -of those compilers, too: -```shell -nix-env -f "" -qaP -A haskell.packages.ghc6123 -nix-env -f "" -qaP -A haskell.packages.ghc763 -``` - -The name `haskellPackages` is really just a synonym for -`haskell.packages.ghc7102`, because we prefer that package set internally and -recommend it to our users as their default choice, but ultimately you are free -to compile your Haskell packages with any GHC version you please. The following -command displays the complete list of available compilers: -``` -$ nix-env -f "" -qaP -A haskell.compiler -haskell.compiler.ghc6104 ghc-6.10.4 -haskell.compiler.ghc6123 ghc-6.12.3 -haskell.compiler.ghc704 ghc-7.0.4 -haskell.compiler.ghc722 ghc-7.2.2 -haskell.compiler.ghc742 ghc-7.4.2 -haskell.compiler.ghc763 ghc-7.6.3 -haskell.compiler.ghc784 ghc-7.8.4 -haskell.compiler.ghc7102 ghc-7.10.2 -haskell.compiler.ghcHEAD ghc-7.11.20150402 -haskell.compiler.ghcNokinds ghc-nokinds-7.11.20150704 -haskell.compiler.ghcjs ghcjs-0.1.0 -haskell.compiler.jhc jhc-0.8.2 -haskell.compiler.uhc uhc-1.1.9.0 -``` - -We have no package sets for `jhc` or `uhc` yet, unfortunately, but for every -version of GHC listed above, there exists a package set based on that compiler. -Also, the attributes `haskell.compiler.ghcXYC` and -`haskell.packages.ghcXYC.ghc` are synonymous for the sake of convenience. - -## How to create a development environment - -### How to install a compiler - -A simple development environment consists of a Haskell compiler and one or both -of the tools `cabal-install` and `stack`. We saw in section -[How to install Haskell packages] how you can install those programs into your -user profile: -```shell -nix-env -f "" -iA haskellPackages.ghc haskellPackages.cabal-install -``` - -Instead of the default package set `haskellPackages`, you can also use the more -precise name `haskell.compiler.ghc7102`, which has the advantage that it refers -to the same GHC version regardless of what Nixpkgs considers "default" at any -given time. - -Once you've made those tools available in `$PATH`, it's possible to build -Hackage packages the same way people without access to Nix do it all the time: -```shell -cabal get lens-4.11 && cd lens-4.11 -cabal install -j --dependencies-only -cabal configure -cabal build -``` - -If you enjoy working with Cabal sandboxes, then that's entirely possible too: -just execute the command -```shell -cabal sandbox init -``` -before installing the required dependencies. - -The `nix-shell` utility makes it easy to switch to a different compiler -version; just enter the Nix shell environment with the command -```shell -nix-shell -p haskell.compiler.ghc784 -``` -to bring GHC 7.8.4 into `$PATH`. Alternatively, you can use Stack instead of -`nix-shell` directly to select compiler versions and other build tools -per-project. It uses `nix-shell` under the hood when Nix support is turned on. -See [How to build a Haskell project using Stack]. - -If you're using `cabal-install`, re-running `cabal configure` inside the spawned -shell switches your build to use that compiler instead. If you're working on -a project that doesn't depend on any additional system libraries outside of GHC, -then it's even sufficient to just run the `cabal configure` command inside of -the shell: -```shell -nix-shell -p haskell.compiler.ghc784 --command "cabal configure" -``` - -Afterwards, all other commands like `cabal build` work just fine in any shell -environment, because the configure phase recorded the absolute paths to all -required tools like GHC in its build configuration inside of the `dist/` -directory. Please note, however, that `nix-collect-garbage` can break such an -environment because the Nix store paths created by `nix-shell` aren't "alive" -anymore once `nix-shell` has terminated. If you find that your Haskell builds -no longer work after garbage collection, then you'll have to re-run `cabal -configure` inside of a new `nix-shell` environment. - -### How to install a compiler with libraries - -GHC expects to find all installed libraries inside of its own `lib` directory. -This approach works fine on traditional Unix systems, but it doesn't work for -Nix, because GHC's store path is immutable once it's built. We cannot install -additional libraries into that location. As a consequence, our copies of GHC -don't know any packages except their own core libraries, like `base`, -`containers`, `Cabal`, etc. - -We can register additional libraries to GHC, however, using a special build -function called `ghcWithPackages`. That function expects one argument: a -function that maps from an attribute set of Haskell packages to a list of -packages, which determines the libraries known to that particular version of -GHC. For example, the Nix expression `ghcWithPackages (pkgs: [pkgs.mtl])` -generates a copy of GHC that has the `mtl` library registered in addition to -its normal core packages: -``` -$ nix-shell -p "haskellPackages.ghcWithPackages (pkgs: [pkgs.mtl])" - -[nix-shell:~]$ ghc-pkg list mtl -/nix/store/zy79...-ghc-7.10.2/lib/ghc-7.10.2/package.conf.d: - mtl-2.2.1 -``` - -This function allows users to define their own development environment by means -of an override. After adding the following snippet to `~/.config/nixpkgs/config.nix`, -```nix -{ - packageOverrides = super: let self = super.pkgs; in - { - myHaskellEnv = self.haskell.packages.ghc7102.ghcWithPackages - (haskellPackages: with haskellPackages; [ - # libraries - arrows async cgi criterion - # tools - cabal-install haskintex - ]); - }; -} -``` -it's possible to install that compiler with `nix-env -f "" -iA -myHaskellEnv`. If you'd like to switch that development environment to a -different version of GHC, just replace the `ghc7102` bit in the previous -definition with the appropriate name. Of course, it's also possible to define -any number of these development environments! (You can't install two of them -into the same profile at the same time, though, because that would result in -file conflicts.) - -The generated `ghc` program is a wrapper script that re-directs the real -GHC executable to use a new `lib` directory --- one that we specifically -constructed to contain all those packages the user requested: -``` -$ cat $(type -p ghc) -#! /nix/store/xlxj...-bash-4.3-p33/bin/bash -e -export NIX_GHC=/nix/store/19sm...-ghc-7.10.2/bin/ghc -export NIX_GHCPKG=/nix/store/19sm...-ghc-7.10.2/bin/ghc-pkg -export NIX_GHC_DOCDIR=/nix/store/19sm...-ghc-7.10.2/share/doc/ghc/html -export NIX_GHC_LIBDIR=/nix/store/19sm...-ghc-7.10.2/lib/ghc-7.10.2 -exec /nix/store/j50p...-ghc-7.10.2/bin/ghc "-B$NIX_GHC_LIBDIR" "$@" -``` - -The variables `$NIX_GHC`, `$NIX_GHCPKG`, etc. point to the *new* store path -`ghcWithPackages` constructed specifically for this environment. The last line -of the wrapper script then executes the real `ghc`, but passes the path to the -new `lib` directory using GHC's `-B` flag. - -The purpose of those environment variables is to work around an impurity in the -popular [ghc-paths](http://hackage.haskell.org/package/ghc-paths) library. That -library promises to give its users access to GHC's installation paths. Only, -the library can't possible know that path when it's compiled, because the path -GHC considers its own is determined only much later, when the user configures -it through `ghcWithPackages`. So we [patched -ghc-paths](https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/haskell-modules/patches/ghc-paths-nix.patch) -to return the paths found in those environment variables at run-time rather -than trying to guess them at compile-time. - -To make sure that mechanism works properly all the time, we recommend that you -set those variables to meaningful values in your shell environment, too, i.e. -by adding the following code to your `~/.bashrc`: -```bash -if type >/dev/null 2>&1 -p ghc; then - eval "$(egrep ^export "$(type -p ghc)")" -fi -``` - -If you are certain that you'll use only one GHC environment which is located in -your user profile, then you can use the following code, too, which has the -advantage that it doesn't contain any paths from the Nix store, i.e. those -settings always remain valid even if a `nix-env -u` operation updates the GHC -environment in your profile: -```bash -if [ -e ~/.nix-profile/bin/ghc ]; then - export NIX_GHC="$HOME/.nix-profile/bin/ghc" - export NIX_GHCPKG="$HOME/.nix-profile/bin/ghc-pkg" - export NIX_GHC_DOCDIR="$HOME/.nix-profile/share/doc/ghc/html" - export NIX_GHC_LIBDIR="$HOME/.nix-profile/lib/ghc-$($NIX_GHC --numeric-version)" -fi -``` - -### How to install a compiler with libraries, hoogle and documentation indexes - -If you plan to use your environment for interactive programming, not just -compiling random Haskell code, you might want to replace `ghcWithPackages` in -all the listings above with `ghcWithHoogle`. - -This environment generator not only produces an environment with GHC and all -the specified libraries, but also generates a `hoogle` and `haddock` indexes -for all the packages, and provides a wrapper script around `hoogle` binary that -uses all those things. A precise name for this thing would be -"`ghcWithPackagesAndHoogleAndDocumentationIndexes`", which is, regrettably, too -long and scary. - -For example, installing the following environment -```nix -{ - packageOverrides = super: let self = super.pkgs; in - { - myHaskellEnv = self.haskellPackages.ghcWithHoogle - (haskellPackages: with haskellPackages; [ - # libraries - arrows async cgi criterion - # tools - cabal-install haskintex - ]); - }; -} -``` -allows one to browse module documentation index [not too dissimilar to -this](https://downloads.haskell.org/~ghc/latest/docs/html/libraries/index.html) -for all the specified packages and their dependencies by directing a browser of -choice to `~/.nix-profiles/share/doc/hoogle/index.html` (or -`/run/current-system/sw/share/doc/hoogle/index.html` in case you put it in -`environment.systemPackages` in NixOS). - -After you've marveled enough at that try adding the following to your -`~/.ghc/ghci.conf` -``` -:def hoogle \s -> return $ ":! hoogle search -cl --count=15 \"" ++ s ++ "\"" -:def doc \s -> return $ ":! hoogle search -cl --info \"" ++ s ++ "\"" -``` -and test it by typing into `ghci`: -``` -:hoogle a -> a -:doc a -> a -``` - -Be sure to note the links to `haddock` files in the output. With any modern and -properly configured terminal emulator you can just click those links to -navigate there. - -Finally, you can run -```shell -hoogle server -p 8080 --local -``` -and navigate to http://localhost:8080/ for your own local -[Hoogle](https://www.haskell.org/hoogle/). - -### How to build a Haskell project using Stack - -[Stack](http://haskellstack.org) is a popular build tool for Haskell projects. -It has first-class support for Nix. Stack can optionally use Nix to -automatically select the right version of GHC and other build tools to build, -test and execute apps in an existing project downloaded from somewhere on the -Internet. Pass the `--nix` flag to any `stack` command to do so, e.g. -```shell -git clone --recursive http://github.com/yesodweb/wai -cd wai -stack --nix build -``` - -If you want `stack` to use Nix by default, you can add a `nix` section to the -`stack.yaml` file, as explained in the [Stack documentation][stack-nix-doc]. For -example: -```yaml -nix: - enable: true - packages: [pkgconfig zeromq zlib] -``` - -The example configuration snippet above tells Stack to create an ad hoc -environment for `nix-shell` as in the below section, in which the `pkgconfig`, -`zeromq` and `zlib` packages from Nixpkgs are available. All `stack` commands -will implicitly be executed inside this ad hoc environment. - -Some projects have more sophisticated needs. For examples, some ad hoc -environments might need to expose Nixpkgs packages compiled in a certain way, or -with extra environment variables. In these cases, you'll need a `shell` field -instead of `packages`: -```yaml -nix: - enable: true - shell-file: shell.nix -``` - -For more on how to write a `shell.nix` file see the below section. You'll need -to express a derivation. Note that Nixpkgs ships with a convenience wrapper -function around `mkDerivation` called `haskell.lib.buildStackProject` to help you -create this derivation in exactly the way Stack expects. All of the same inputs -as `mkDerivation` can be provided. For example, to build a Stack project that -including packages that link against a version of the R library compiled with -special options turned on: -```nix -with (import { }); - -let R = pkgs.R.override { enableStrictBarrier = true; }; -in -haskell.lib.buildStackProject { - name = "HaskellR"; - buildInputs = [ R zeromq zlib ]; -} -``` - -You can select a particular GHC version to compile with by setting the -`ghc` attribute as an argument to `buildStackProject`. Better yet, let -Stack choose what GHC version it wants based on the snapshot specified -in `stack.yaml` (only works with Stack >= 1.1.3): -```nix -{nixpkgs ? import { }, ghc ? nixpkgs.ghc}: - -with nixpkgs; - -let R = pkgs.R.override { enableStrictBarrier = true; }; -in -haskell.lib.buildStackProject { - name = "HaskellR"; - buildInputs = [ R zeromq zlib ]; - inherit ghc; -} -``` - -[stack-nix-doc]: http://docs.haskellstack.org/en/stable/nix_integration.html - -### How to create ad hoc environments for `nix-shell` - -The easiest way to create an ad hoc development environment is to run -`nix-shell` with the appropriate GHC environment given on the command-line: -```shell -nix-shell -p "haskellPackages.ghcWithPackages (pkgs: with pkgs; [mtl pandoc])" -``` - -For more sophisticated use-cases, however, it's more convenient to save the -desired configuration in a file called `shell.nix` that looks like this: -```nix -{ nixpkgs ? import {}, compiler ? "ghc7102" }: -let - inherit (nixpkgs) pkgs; - ghc = pkgs.haskell.packages.${compiler}.ghcWithPackages (ps: with ps; [ - monad-par mtl - ]); -in -pkgs.stdenv.mkDerivation { - name = "my-haskell-env-0"; - buildInputs = [ ghc ]; - shellHook = "eval $(egrep ^export ${ghc}/bin/ghc)"; -} -``` - -Now run `nix-shell` --- or even `nix-shell --pure` --- to enter a shell -environment that has the appropriate compiler in `$PATH`. If you use `--pure`, -then add all other packages that your development environment needs into the -`buildInputs` attribute. If you'd like to switch to a different compiler -version, then pass an appropriate `compiler` argument to the expression, i.e. -`nix-shell --argstr compiler ghc784`. - -If you need such an environment because you'd like to compile a Hackage package -outside of Nix --- i.e. because you're hacking on the latest version from Git ----, then the package set provides suitable nix-shell environments for you -already! Every Haskell package has an `env` attribute that provides a shell -environment suitable for compiling that particular package. If you'd like to -hack the `lens` library, for example, then you just have to check out the -source code and enter the appropriate environment: -``` -$ cabal get lens-4.11 && cd lens-4.11 -Downloading lens-4.11... -Unpacking to lens-4.11/ - -$ nix-shell "" -A haskellPackages.lens.env -[nix-shell:/tmp/lens-4.11]$ -``` - -At point, you can run `cabal configure`, `cabal build`, and all the other -development commands. Note that you need `cabal-install` installed in your -`$PATH` already to use it here --- the `nix-shell` environment does not provide -it. - -## How to create Nix builds for your own private Haskell packages - -If your own Haskell packages have build instructions for Cabal, then you can -convert those automatically into build instructions for Nix using the -`cabal2nix` utility, which you can install into your profile by running -`nix-env -i cabal2nix`. - -### How to build a stand-alone project - -For example, let's assume that you're working on a private project called -`foo`. To generate a Nix build expression for it, change into the project's -top-level directory and run the command: -```shell -cabal2nix . > foo.nix -``` -Then write the following snippet into a file called `default.nix`: -```nix -{ nixpkgs ? import {}, compiler ? "ghc7102" }: -nixpkgs.pkgs.haskell.packages.${compiler}.callPackage ./foo.nix { } -``` - -Finally, store the following code in a file called `shell.nix`: -```nix -{ nixpkgs ? import {}, compiler ? "ghc7102" }: -(import ./default.nix { inherit nixpkgs compiler; }).env -``` - -At this point, you can run `nix-build` to have Nix compile your project and -install it into a Nix store path. The local directory will contain a symlink -called `result` after `nix-build` returns that points into that location. Of -course, passing the flag `--argstr compiler ghc763` allows switching the build -to any version of GHC currently supported. - -Furthermore, you can call `nix-shell` to enter an interactive development -environment in which you can use `cabal configure` and `cabal build` to develop -your code. That environment will automatically contain a proper GHC derivation -with all the required libraries registered as well as all the system-level -libraries your package might need. - -If your package does not depend on any system-level libraries, then it's -sufficient to run -```shell -nix-shell --command "cabal configure" -``` -once to set up your build. `cabal-install` determines the absolute paths to all -resources required for the build and writes them into a config file in the -`dist/` directory. Once that's done, you can run `cabal build` and any other -command for that project even outside of the `nix-shell` environment. This -feature is particularly nice for those of us who like to edit their code with -an IDE, like Emacs' `haskell-mode`, because it's not necessary to start Emacs -inside of nix-shell just to make it find out the necessary settings for -building the project; `cabal-install` has already done that for us. - -If you want to do some quick-and-dirty hacking and don't want to bother setting -up a `default.nix` and `shell.nix` file manually, then you can use the -`--shell` flag offered by `cabal2nix` to have it generate a stand-alone -`nix-shell` environment for you. With that feature, running -```shell -cabal2nix --shell . > shell.nix -nix-shell --command "cabal configure" -``` -is usually enough to set up a build environment for any given Haskell package. -You can even use that generated file to run `nix-build`, too: -```shell -nix-build shell.nix -``` - -### How to build projects that depend on each other - -If you have multiple private Haskell packages that depend on each other, then -you'll have to register those packages in the Nixpkgs set to make them visible -for the dependency resolution performed by `callPackage`. First of all, change -into each of your projects top-level directories and generate a `default.nix` -file with `cabal2nix`: -```shell -cd ~/src/foo && cabal2nix . > default.nix -cd ~/src/bar && cabal2nix . > default.nix -``` -Then edit your `~/.config/nixpkgs/config.nix` file to register those builds in the -default Haskell package set: -```nix -{ - packageOverrides = super: let self = super.pkgs; in - { - haskellPackages = super.haskellPackages.override { - overrides = self: super: { - foo = self.callPackage ../src/foo {}; - bar = self.callPackage ../src/bar {}; - }; - }; - }; -} -``` -Once that's accomplished, `nix-env -f "" -qA haskellPackages` will -show your packages like any other package from Hackage, and you can build them -```shell -nix-build "" -A haskellPackages.foo -``` -or enter an interactive shell environment suitable for building them: -```shell -nix-shell "" -A haskellPackages.bar.env -``` - -## Miscellaneous Topics - -### How to build with profiling enabled - -Every Haskell package set takes a function called `overrides` that you can use -to manipulate the package as much as you please. One useful application of this -feature is to replace the default `mkDerivation` function with one that enables -library profiling for all packages. To accomplish that add the following -snippet to your `~/.config/nixpkgs/config.nix` file: -```nix -{ - packageOverrides = super: let self = super.pkgs; in - { - profiledHaskellPackages = self.haskellPackages.override { - overrides = self: super: { - mkDerivation = args: super.mkDerivation (args // { - enableLibraryProfiling = true; - }); - }; - }; - }; -} -``` -Then, replace instances of `haskellPackages` in the `cabal2nix`-generated -`default.nix` or `shell.nix` files with `profiledHaskellPackages`. - -### How to override package versions in a compiler-specific package set - -Nixpkgs provides the latest version of -[`ghc-events`](http://hackage.haskell.org/package/ghc-events), which is 0.4.4.0 -at the time of this writing. This is fine for users of GHC 7.10.x, but GHC -7.8.4 cannot compile that binary. Now, one way to solve that problem is to -register an older version of `ghc-events` in the 7.8.x-specific package set. -The first step is to generate Nix build instructions with `cabal2nix`: -```shell -cabal2nix cabal://ghc-events-0.4.3.0 > ~/.nixpkgs/ghc-events-0.4.3.0.nix -``` -Then add the override in `~/.config/nixpkgs/config.nix`: -```nix -{ - packageOverrides = super: let self = super.pkgs; in - { - haskell = super.haskell // { - packages = super.haskell.packages // { - ghc784 = super.haskell.packages.ghc784.override { - overrides = self: super: { - ghc-events = self.callPackage ./ghc-events-0.4.3.0.nix {}; - }; - }; - }; - }; - }; -} -``` - -This code is a little crazy, no doubt, but it's necessary because the intuitive -version -```nix -{ # ... - - haskell.packages.ghc784 = super.haskell.packages.ghc784.override { - overrides = self: super: { - ghc-events = self.callPackage ./ghc-events-0.4.3.0.nix {}; - }; - }; -} -``` -doesn't do what we want it to: that code replaces the `haskell` package set in -Nixpkgs with one that contains only one entry,`packages`, which contains only -one entry `ghc784`. This override loses the `haskell.compiler` set, and it -loses the `haskell.packages.ghcXYZ` sets for all compilers but GHC 7.8.4. To -avoid that problem, we have to perform the convoluted little dance from above, -iterating over each step in hierarchy. - -Once it's accomplished, however, we can install a variant of `ghc-events` -that's compiled with GHC 7.8.4: -```shell -nix-env -f "" -iA haskell.packages.ghc784.ghc-events -``` -Unfortunately, it turns out that this build fails again while executing the -test suite! Apparently, the release archive on Hackage is missing some data -files that the test suite requires, so we cannot run it. We accomplish that by -re-generating the Nix expression with the `--no-check` flag: -```shell -cabal2nix --no-check cabal://ghc-events-0.4.3.0 > ~/.nixpkgs/ghc-events-0.4.3.0.nix -``` -Now the builds succeeds. - -Of course, in the concrete example of `ghc-events` this whole exercise is not -an ideal solution, because `ghc-events` can analyze the output emitted by any -version of GHC later than 6.12 regardless of the compiler version that was used -to build the `ghc-events` executable, so strictly speaking there's no reason to -prefer one built with GHC 7.8.x in the first place. However, for users who -cannot use GHC 7.10.x at all for some reason, the approach of downgrading to an -older version might be useful. - -### How to recover from GHC's infamous non-deterministic library ID bug - -GHC and distributed build farms don't get along well: - - - https://ghc.haskell.org/trac/ghc/ticket/4012 - -When you see an error like this one -``` -package foo-0.7.1.0 is broken due to missing package -text-1.2.0.4-98506efb1b9ada233bb5c2b2db516d91 -``` -then you have to download and re-install `foo` and all its dependents from -scratch: -```shell -nix-store -q --referrers /nix/store/*-haskell-text-1.2.0.4 \ - | xargs -L 1 nix-store --repair-path -``` - -If you're using additional Hydra servers other than `hydra.nixos.org`, then it -might be necessary to purge the local caches that store data from those -machines to disable these binary channels for the duration of the previous -command, i.e. by running: -```shell -rm ~/.cache/nix/binary-cache*.sqlite -``` - -### Builds on Darwin fail with `math.h` not found - -Users of GHC on Darwin have occasionally reported that builds fail, because the -compiler complains about a missing include file: -``` -fatal error: 'math.h' file not found -``` -The issue has been discussed at length in [ticket -6390](https://github.com/NixOS/nixpkgs/issues/6390), and so far no good -solution has been proposed. As a work-around, users who run into this problem -can configure the environment variables -```shell -export NIX_CFLAGS_COMPILE="-idirafter /usr/include" -export NIX_CFLAGS_LINK="-L/usr/lib" -``` -in their `~/.bashrc` file to avoid the compiler error. - -### Builds using Stack complain about missing system libraries - -``` --- While building package zlib-0.5.4.2 using: - runhaskell -package=Cabal-1.22.4.0 -clear-package-db [... lots of flags ...] -Process exited with code: ExitFailure 1 -Logs have been written to: /home/foo/src/stack-ide/.stack-work/logs/zlib-0.5.4.2.log - -Configuring zlib-0.5.4.2... -Setup.hs: Missing dependency on a foreign library: -* Missing (or bad) header file: zlib.h -This problem can usually be solved by installing the system package that -provides this library (you may need the "-dev" version). If the library is -already installed but in a non-standard location then you can use the flags ---extra-include-dirs= and --extra-lib-dirs= to specify where it is. -If the header file does exist, it may contain errors that are caught by the C -compiler at the preprocessing stage. In this case you can re-run configure -with the verbosity flag -v3 to see the error messages. -``` - -When you run the build inside of the nix-shell environment, the system -is configured to find `libz.so` without any special flags -- the compiler -and linker "just know" how to find it. Consequently, Cabal won't record -any search paths for `libz.so` in the package description, which means -that the package works fine inside of nix-shell, but once you leave the -shell the shared object can no longer be found. That issue is by no -means specific to Stack: you'll have that problem with any other -Haskell package that's built inside of nix-shell but run outside of that -environment. - -You can remedy this issue in several ways. The easiest is to add a `nix` section -to the `stack.yaml` like the following: -```yaml -nix: - enable: true - packages: [ zlib ] -``` - -Stack's Nix support knows to add `${zlib.out}/lib` and `${zlib.dev}/include` -as an `--extra-lib-dirs` and `extra-include-dirs`, respectively. -Alternatively, you can achieve the same effect by hand. First of all, run -``` -$ nix-build --no-out-link "" -A zlib -/nix/store/alsvwzkiw4b7ip38l4nlfjijdvg3fvzn-zlib-1.2.8 -``` -to find out the store path of the system's zlib library. Now, you can - - 1. add that path (plus a "/lib" suffix) to your `$LD_LIBRARY_PATH` - environment variable to make sure your system linker finds `libz.so` - automatically. It's no pretty solution, but it will work. - - 2. As a variant of (1), you can also install any number of system - libraries into your user's profile (or some other profile) and point - `$LD_LIBRARY_PATH` to that profile instead, so that you don't have to - list dozens of those store paths all over the place. - - 3. The solution I prefer is to call stack with an appropriate - --extra-lib-dirs flag like so: - ```shell - stack --extra-lib-dirs=/nix/store/alsvwzkiw4b7ip38l4nlfjijdvg3fvzn-zlib-1.2.8/lib build - ``` - -Typically, you'll need `--extra-include-dirs` as well. It's possible -to add those flag to the project's `stack.yaml` or your user's -global `~/.stack/global/stack.yaml` file so that you don't have to -specify them manually every time. But again, you're likely better off -using Stack's Nix support instead. - -The same thing applies to `cabal configure`, of course, if you're -building with `cabal-install` instead of Stack. - -### Creating statically linked binaries - -There are two levels of static linking. The first option is to configure the -build with the Cabal flag `--disable-executable-dynamic`. In Nix expressions, -this can be achieved by setting the attribute: -``` -enableSharedExecutables = false; -``` -That gives you a binary with statically linked Haskell libraries and -dynamically linked system libraries. - -To link both Haskell libraries and system libraries statically, the additional -flags `--ghc-option=-optl=-static --ghc-option=-optl=-pthread` need to be used. -In Nix, this is accomplished with: -``` -configureFlags = [ "--ghc-option=-optl=-static" "--ghc-option=-optl=-pthread" ]; -``` - -It's important to realize, however, that most system libraries in Nix are -built as shared libraries only, i.e. there is just no static library -available that Cabal could link! - -### Building GHC with integer-simple - -By default GHC implements the Integer type using the -[GNU Multiple Precision Arithmetic (GMP) library](https://gmplib.org/). -The implementation can be found in the -[integer-gmp](http://hackage.haskell.org/package/integer-gmp) package. - -A potential problem with this is that GMP is licensed under the -[GNU Lesser General Public License (LGPL)](http://www.gnu.org/copyleft/lesser.html), -a kind of "copyleft" license. According to the terms of the LGPL, paragraph 5, -you may distribute a program that is designed to be compiled and dynamically -linked with the library under the terms of your choice (i.e., commercially) but -if your program incorporates portions of the library, if it is linked -statically, then your program is a "derivative"--a "work based on the -library"--and according to paragraph 2, section c, you "must cause the whole of -the work to be licensed" under the terms of the LGPL (including for free). - -The LGPL licensing for GMP is a problem for the overall licensing of binary -programs compiled with GHC because most distributions (and builds) of GHC use -static libraries. (Dynamic libraries are currently distributed only for macOS.) -The LGPL licensing situation may be worse: even though -[The Glasgow Haskell Compiler License](https://www.haskell.org/ghc/license) -is essentially a "free software" license (BSD3), according to -paragraph 2 of the LGPL, GHC must be distributed under the terms of the LGPL! - -To work around these problems GHC can be build with a slower but LGPL-free -alternative implemention for Integer called -[integer-simple](http://hackage.haskell.org/package/integer-simple). - -To get a GHC compiler build with `integer-simple` instead of `integer-gmp` use -the attribute: `haskell.compiler.integer-simple."${ghcVersion}"`. -For example: -``` -$ nix-build -E '(import {}).haskell.compiler.integer-simple.ghc802' -... -$ result/bin/ghc-pkg list | grep integer - integer-simple-0.1.1.1 -``` -The following command displays the complete list of GHC compilers build with `integer-simple`: -``` -$ nix-env -f "" -qaP -A haskell.compiler.integer-simple -haskell.compiler.integer-simple.ghc7102 ghc-7.10.2 -haskell.compiler.integer-simple.ghc7103 ghc-7.10.3 -haskell.compiler.integer-simple.ghc722 ghc-7.2.2 -haskell.compiler.integer-simple.ghc742 ghc-7.4.2 -haskell.compiler.integer-simple.ghc783 ghc-7.8.3 -haskell.compiler.integer-simple.ghc784 ghc-7.8.4 -haskell.compiler.integer-simple.ghc801 ghc-8.0.1 -haskell.compiler.integer-simple.ghc802 ghc-8.0.2 -haskell.compiler.integer-simple.ghcHEAD ghc-8.1.20170106 -``` - -To get a package set supporting `integer-simple` use the attribute: -`haskell.packages.integer-simple."${ghcVersion}"`. For example -use the following to get the `scientific` package build with `integer-simple`: -```shell -nix-build -A haskell.packages.integer-simple.ghc802.scientific -``` - -### Quality assurance - -The `haskell.lib` library includes a number of functions for checking for -various imperfections in Haskell packages. It's useful to apply these functions -to your own Haskell packages and integrate that in a Continuous Integration -server like [hydra](https://nixos.org/hydra/) to assure your packages maintain a -minimum level of quality. This section discusses some of these functions. - -#### failOnAllWarnings - -Applying `haskell.lib.failOnAllWarnings` to a Haskell package enables the -`-Wall` and `-Werror` GHC options to turn all warnings into build failures. - -#### buildStrictly - -Applying `haskell.lib.buildStrictly` to a Haskell package calls -`failOnAllWarnings` on the given package to turn all warnings into build -failures. Additionally the source of your package is gotten from first invoking -`cabal sdist` to ensure all needed files are listed in the Cabal file. - -#### checkUnusedPackages - -Applying `haskell.lib.checkUnusedPackages` to a Haskell package invokes -the [packunused](http://hackage.haskell.org/package/packunused) tool on the -package. `packunused` complains when it finds packages listed as build-depends -in the Cabal file which are redundant. For example: - -``` -$ nix-build -E 'let pkgs = import {}; in pkgs.haskell.lib.checkUnusedPackages {} pkgs.haskellPackages.scientific' -these derivations will be built: - /nix/store/3lc51cxj2j57y3zfpq5i69qbzjpvyci1-scientific-0.3.5.1.drv -... -detected package components -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - library - - testsuite(s): test-scientific - - benchmark(s): bench-scientific* - -(component names suffixed with '*' are not configured to be built) - -library -~~~~~~~ - -The following package dependencies seem redundant: - - - ghc-prim-0.5.0.0 - -testsuite(test-scientific) -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -no redundant packages dependencies found - -builder for ‘/nix/store/3lc51cxj2j57y3zfpq5i69qbzjpvyci1-scientific-0.3.5.1.drv’ failed with exit code 1 -error: build of ‘/nix/store/3lc51cxj2j57y3zfpq5i69qbzjpvyci1-scientific-0.3.5.1.drv’ failed -``` - -As you can see, `packunused` finds out that although the testsuite component has -no redundant dependencies the library component of `scientific-0.3.5.1` depends -on `ghc-prim` which is unused in the library. - -## Other resources - - - The Youtube video [Nix Loves Haskell](https://www.youtube.com/watch?v=BsBhi_r-OeE) - provides an introduction into Haskell NG aimed at beginners. The slides are - available at http://cryp.to/nixos-meetup-3-slides.pdf and also -- in a form - ready for cut & paste -- at - https://github.com/NixOS/cabal2nix/blob/master/doc/nixos-meetup-3-slides.md. - - - Another Youtube video is [Escaping Cabal Hell with Nix](https://www.youtube.com/watch?v=mQd3s57n_2Y), - which discusses the subject of Haskell development with Nix but also provides - a basic introduction to Nix as well, i.e. it's suitable for viewers with - almost no prior Nix experience. - - - Oliver Charles wrote a very nice [Tutorial how to develop Haskell packages with Nix](http://wiki.ocharles.org.uk/Nix). - - - The *Journey into the Haskell NG infrastructure* series of postings - describe the new Haskell infrastructure in great detail: - - - [Part 1](https://nixos.org/nix-dev/2015-January/015591.html) - explains the differences between the old and the new code and gives - instructions how to migrate to the new setup. - - - [Part 2](https://nixos.org/nix-dev/2015-January/015608.html) - looks in-depth at how to tweak and configure your setup by means of - overrides. - - - [Part 3](https://nixos.org/nix-dev/2015-April/016912.html) - describes the infrastructure that keeps the Haskell package set in Nixpkgs - up-to-date. diff --git a/doc/languages-frameworks/haskell.section.md b/doc/languages-frameworks/haskell.section.md new file mode 100644 index 000000000000..1623e0d276f2 --- /dev/null +++ b/doc/languages-frameworks/haskell.section.md @@ -0,0 +1,953 @@ +--- +title: User's Guide for Haskell in Nixpkgs +author: Peter Simons +date: 2015-06-01 +--- +# User's Guide to the Haskell Infrastructure + + +## How to install Haskell packages + +Nixpkgs distributes build instructions for all Haskell packages registered on +[Hackage](http://hackage.haskell.org/), but strangely enough normal Nix package +lookups don't seem to discover any of them, except for the default version of ghc, cabal-install, and stack: +``` +$ nix-env -i alex +error: selector ‘alex’ matches no derivations +$ nix-env -qa ghc +ghc-7.10.2 +``` + +The Haskell package set is not registered in the top-level namespace because it +is *huge*. If all Haskell packages were visible to these commands, then +name-based search/install operations would be much slower than they are now. We +avoided that by keeping all Haskell-related packages in a separate attribute +set called `haskellPackages`, which the following command will list: +``` +$ nix-env -f "" -qaP -A haskellPackages +haskellPackages.a50 a50-0.5 +haskellPackages.abacate haskell-abacate-0.0.0.0 +haskellPackages.abcBridge haskell-abcBridge-0.12 +haskellPackages.afv afv-0.1.1 +haskellPackages.alex alex-3.1.4 +haskellPackages.Allure Allure-0.4.101.1 +haskellPackages.alms alms-0.6.7 +[... some 8000 entries omitted ...] +``` + +To install any of those packages into your profile, refer to them by their +attribute path (first column): +```shell +nix-env -f "" -iA haskellPackages.Allure ... +``` + +The attribute path of any Haskell packages corresponds to the name of that +particular package on Hackage: the package `cabal-install` has the attribute +`haskellPackages.cabal-install`, and so on. (Actually, this convention causes +trouble with packages like `3dmodels` and `4Blocks`, because these names are +invalid identifiers in the Nix language. The issue of how to deal with these +rare corner cases is currently unresolved.) + +Haskell packages whose Nix name (second column) begins with a `haskell-` prefix +are packages that provide a library whereas packages without that prefix +provide just executables. Libraries may provide executables too, though: the +package `haskell-pandoc`, for example, installs both a library and an +application. You can install and use Haskell executables just like any other +program in Nixpkgs, but using Haskell libraries for development is a bit +trickier and we'll address that subject in great detail in section [How to +create a development environment]. + +Attribute paths are deterministic inside of Nixpkgs, but the path necessary to +reach Nixpkgs varies from system to system. We dodged that problem by giving +`nix-env` an explicit `-f ""` parameter, but if you call `nix-env` +without that flag, then chances are the invocation fails: +``` +$ nix-env -iA haskellPackages.cabal-install +error: attribute ‘haskellPackages’ in selection path + ‘haskellPackages.cabal-install’ not found +``` + +On NixOS, for example, Nixpkgs does *not* exist in the top-level namespace by +default. To figure out the proper attribute path, it's easiest to query for the +path of a well-known Nixpkgs package, i.e.: +``` +$ nix-env -qaP coreutils +nixos.coreutils coreutils-8.23 +``` + +If your system responds like that (most NixOS installations will), then the +attribute path to `haskellPackages` is `nixos.haskellPackages`. Thus, if you +want to use `nix-env` without giving an explicit `-f` flag, then that's the way +to do it: +```shell +nix-env -qaP -A nixos.haskellPackages +nix-env -iA nixos.haskellPackages.cabal-install +``` + +Our current default compiler is GHC 7.10.x and the `haskellPackages` set +contains packages built with that particular version. Nixpkgs contains the +latest major release of every GHC since 6.10.4, however, and there is a whole +family of package sets available that defines Hackage packages built with each +of those compilers, too: +```shell +nix-env -f "" -qaP -A haskell.packages.ghc6123 +nix-env -f "" -qaP -A haskell.packages.ghc763 +``` + +The name `haskellPackages` is really just a synonym for +`haskell.packages.ghc7102`, because we prefer that package set internally and +recommend it to our users as their default choice, but ultimately you are free +to compile your Haskell packages with any GHC version you please. The following +command displays the complete list of available compilers: +``` +$ nix-env -f "" -qaP -A haskell.compiler +haskell.compiler.ghc6104 ghc-6.10.4 +haskell.compiler.ghc6123 ghc-6.12.3 +haskell.compiler.ghc704 ghc-7.0.4 +haskell.compiler.ghc722 ghc-7.2.2 +haskell.compiler.ghc742 ghc-7.4.2 +haskell.compiler.ghc763 ghc-7.6.3 +haskell.compiler.ghc784 ghc-7.8.4 +haskell.compiler.ghc7102 ghc-7.10.2 +haskell.compiler.ghcHEAD ghc-7.11.20150402 +haskell.compiler.ghcNokinds ghc-nokinds-7.11.20150704 +haskell.compiler.ghcjs ghcjs-0.1.0 +haskell.compiler.jhc jhc-0.8.2 +haskell.compiler.uhc uhc-1.1.9.0 +``` + +We have no package sets for `jhc` or `uhc` yet, unfortunately, but for every +version of GHC listed above, there exists a package set based on that compiler. +Also, the attributes `haskell.compiler.ghcXYC` and +`haskell.packages.ghcXYC.ghc` are synonymous for the sake of convenience. + +## How to create a development environment + +### How to install a compiler + +A simple development environment consists of a Haskell compiler and one or both +of the tools `cabal-install` and `stack`. We saw in section +[How to install Haskell packages] how you can install those programs into your +user profile: +```shell +nix-env -f "" -iA haskellPackages.ghc haskellPackages.cabal-install +``` + +Instead of the default package set `haskellPackages`, you can also use the more +precise name `haskell.compiler.ghc7102`, which has the advantage that it refers +to the same GHC version regardless of what Nixpkgs considers "default" at any +given time. + +Once you've made those tools available in `$PATH`, it's possible to build +Hackage packages the same way people without access to Nix do it all the time: +```shell +cabal get lens-4.11 && cd lens-4.11 +cabal install -j --dependencies-only +cabal configure +cabal build +``` + +If you enjoy working with Cabal sandboxes, then that's entirely possible too: +just execute the command +```shell +cabal sandbox init +``` +before installing the required dependencies. + +The `nix-shell` utility makes it easy to switch to a different compiler +version; just enter the Nix shell environment with the command +```shell +nix-shell -p haskell.compiler.ghc784 +``` +to bring GHC 7.8.4 into `$PATH`. Alternatively, you can use Stack instead of +`nix-shell` directly to select compiler versions and other build tools +per-project. It uses `nix-shell` under the hood when Nix support is turned on. +See [How to build a Haskell project using Stack]. + +If you're using `cabal-install`, re-running `cabal configure` inside the spawned +shell switches your build to use that compiler instead. If you're working on +a project that doesn't depend on any additional system libraries outside of GHC, +then it's even sufficient to just run the `cabal configure` command inside of +the shell: +```shell +nix-shell -p haskell.compiler.ghc784 --command "cabal configure" +``` + +Afterwards, all other commands like `cabal build` work just fine in any shell +environment, because the configure phase recorded the absolute paths to all +required tools like GHC in its build configuration inside of the `dist/` +directory. Please note, however, that `nix-collect-garbage` can break such an +environment because the Nix store paths created by `nix-shell` aren't "alive" +anymore once `nix-shell` has terminated. If you find that your Haskell builds +no longer work after garbage collection, then you'll have to re-run `cabal +configure` inside of a new `nix-shell` environment. + +### How to install a compiler with libraries + +GHC expects to find all installed libraries inside of its own `lib` directory. +This approach works fine on traditional Unix systems, but it doesn't work for +Nix, because GHC's store path is immutable once it's built. We cannot install +additional libraries into that location. As a consequence, our copies of GHC +don't know any packages except their own core libraries, like `base`, +`containers`, `Cabal`, etc. + +We can register additional libraries to GHC, however, using a special build +function called `ghcWithPackages`. That function expects one argument: a +function that maps from an attribute set of Haskell packages to a list of +packages, which determines the libraries known to that particular version of +GHC. For example, the Nix expression `ghcWithPackages (pkgs: [pkgs.mtl])` +generates a copy of GHC that has the `mtl` library registered in addition to +its normal core packages: +``` +$ nix-shell -p "haskellPackages.ghcWithPackages (pkgs: [pkgs.mtl])" + +[nix-shell:~]$ ghc-pkg list mtl +/nix/store/zy79...-ghc-7.10.2/lib/ghc-7.10.2/package.conf.d: + mtl-2.2.1 +``` + +This function allows users to define their own development environment by means +of an override. After adding the following snippet to `~/.config/nixpkgs/config.nix`, +```nix +{ + packageOverrides = super: let self = super.pkgs; in + { + myHaskellEnv = self.haskell.packages.ghc7102.ghcWithPackages + (haskellPackages: with haskellPackages; [ + # libraries + arrows async cgi criterion + # tools + cabal-install haskintex + ]); + }; +} +``` +it's possible to install that compiler with `nix-env -f "" -iA +myHaskellEnv`. If you'd like to switch that development environment to a +different version of GHC, just replace the `ghc7102` bit in the previous +definition with the appropriate name. Of course, it's also possible to define +any number of these development environments! (You can't install two of them +into the same profile at the same time, though, because that would result in +file conflicts.) + +The generated `ghc` program is a wrapper script that re-directs the real +GHC executable to use a new `lib` directory --- one that we specifically +constructed to contain all those packages the user requested: +``` +$ cat $(type -p ghc) +#! /nix/store/xlxj...-bash-4.3-p33/bin/bash -e +export NIX_GHC=/nix/store/19sm...-ghc-7.10.2/bin/ghc +export NIX_GHCPKG=/nix/store/19sm...-ghc-7.10.2/bin/ghc-pkg +export NIX_GHC_DOCDIR=/nix/store/19sm...-ghc-7.10.2/share/doc/ghc/html +export NIX_GHC_LIBDIR=/nix/store/19sm...-ghc-7.10.2/lib/ghc-7.10.2 +exec /nix/store/j50p...-ghc-7.10.2/bin/ghc "-B$NIX_GHC_LIBDIR" "$@" +``` + +The variables `$NIX_GHC`, `$NIX_GHCPKG`, etc. point to the *new* store path +`ghcWithPackages` constructed specifically for this environment. The last line +of the wrapper script then executes the real `ghc`, but passes the path to the +new `lib` directory using GHC's `-B` flag. + +The purpose of those environment variables is to work around an impurity in the +popular [ghc-paths](http://hackage.haskell.org/package/ghc-paths) library. That +library promises to give its users access to GHC's installation paths. Only, +the library can't possible know that path when it's compiled, because the path +GHC considers its own is determined only much later, when the user configures +it through `ghcWithPackages`. So we [patched +ghc-paths](https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/haskell-modules/patches/ghc-paths-nix.patch) +to return the paths found in those environment variables at run-time rather +than trying to guess them at compile-time. + +To make sure that mechanism works properly all the time, we recommend that you +set those variables to meaningful values in your shell environment, too, i.e. +by adding the following code to your `~/.bashrc`: +```bash +if type >/dev/null 2>&1 -p ghc; then + eval "$(egrep ^export "$(type -p ghc)")" +fi +``` + +If you are certain that you'll use only one GHC environment which is located in +your user profile, then you can use the following code, too, which has the +advantage that it doesn't contain any paths from the Nix store, i.e. those +settings always remain valid even if a `nix-env -u` operation updates the GHC +environment in your profile: +```bash +if [ -e ~/.nix-profile/bin/ghc ]; then + export NIX_GHC="$HOME/.nix-profile/bin/ghc" + export NIX_GHCPKG="$HOME/.nix-profile/bin/ghc-pkg" + export NIX_GHC_DOCDIR="$HOME/.nix-profile/share/doc/ghc/html" + export NIX_GHC_LIBDIR="$HOME/.nix-profile/lib/ghc-$($NIX_GHC --numeric-version)" +fi +``` + +### How to install a compiler with libraries, hoogle and documentation indexes + +If you plan to use your environment for interactive programming, not just +compiling random Haskell code, you might want to replace `ghcWithPackages` in +all the listings above with `ghcWithHoogle`. + +This environment generator not only produces an environment with GHC and all +the specified libraries, but also generates a `hoogle` and `haddock` indexes +for all the packages, and provides a wrapper script around `hoogle` binary that +uses all those things. A precise name for this thing would be +"`ghcWithPackagesAndHoogleAndDocumentationIndexes`", which is, regrettably, too +long and scary. + +For example, installing the following environment +```nix +{ + packageOverrides = super: let self = super.pkgs; in + { + myHaskellEnv = self.haskellPackages.ghcWithHoogle + (haskellPackages: with haskellPackages; [ + # libraries + arrows async cgi criterion + # tools + cabal-install haskintex + ]); + }; +} +``` +allows one to browse module documentation index [not too dissimilar to +this](https://downloads.haskell.org/~ghc/latest/docs/html/libraries/index.html) +for all the specified packages and their dependencies by directing a browser of +choice to `~/.nix-profiles/share/doc/hoogle/index.html` (or +`/run/current-system/sw/share/doc/hoogle/index.html` in case you put it in +`environment.systemPackages` in NixOS). + +After you've marveled enough at that try adding the following to your +`~/.ghc/ghci.conf` +``` +:def hoogle \s -> return $ ":! hoogle search -cl --count=15 \"" ++ s ++ "\"" +:def doc \s -> return $ ":! hoogle search -cl --info \"" ++ s ++ "\"" +``` +and test it by typing into `ghci`: +``` +:hoogle a -> a +:doc a -> a +``` + +Be sure to note the links to `haddock` files in the output. With any modern and +properly configured terminal emulator you can just click those links to +navigate there. + +Finally, you can run +```shell +hoogle server -p 8080 --local +``` +and navigate to http://localhost:8080/ for your own local +[Hoogle](https://www.haskell.org/hoogle/). + +### How to build a Haskell project using Stack + +[Stack](http://haskellstack.org) is a popular build tool for Haskell projects. +It has first-class support for Nix. Stack can optionally use Nix to +automatically select the right version of GHC and other build tools to build, +test and execute apps in an existing project downloaded from somewhere on the +Internet. Pass the `--nix` flag to any `stack` command to do so, e.g. +```shell +git clone --recursive http://github.com/yesodweb/wai +cd wai +stack --nix build +``` + +If you want `stack` to use Nix by default, you can add a `nix` section to the +`stack.yaml` file, as explained in the [Stack documentation][stack-nix-doc]. For +example: +```yaml +nix: + enable: true + packages: [pkgconfig zeromq zlib] +``` + +The example configuration snippet above tells Stack to create an ad hoc +environment for `nix-shell` as in the below section, in which the `pkgconfig`, +`zeromq` and `zlib` packages from Nixpkgs are available. All `stack` commands +will implicitly be executed inside this ad hoc environment. + +Some projects have more sophisticated needs. For examples, some ad hoc +environments might need to expose Nixpkgs packages compiled in a certain way, or +with extra environment variables. In these cases, you'll need a `shell` field +instead of `packages`: +```yaml +nix: + enable: true + shell-file: shell.nix +``` + +For more on how to write a `shell.nix` file see the below section. You'll need +to express a derivation. Note that Nixpkgs ships with a convenience wrapper +function around `mkDerivation` called `haskell.lib.buildStackProject` to help you +create this derivation in exactly the way Stack expects. All of the same inputs +as `mkDerivation` can be provided. For example, to build a Stack project that +including packages that link against a version of the R library compiled with +special options turned on: +```nix +with (import { }); + +let R = pkgs.R.override { enableStrictBarrier = true; }; +in +haskell.lib.buildStackProject { + name = "HaskellR"; + buildInputs = [ R zeromq zlib ]; +} +``` + +You can select a particular GHC version to compile with by setting the +`ghc` attribute as an argument to `buildStackProject`. Better yet, let +Stack choose what GHC version it wants based on the snapshot specified +in `stack.yaml` (only works with Stack >= 1.1.3): +```nix +{nixpkgs ? import { }, ghc ? nixpkgs.ghc}: + +with nixpkgs; + +let R = pkgs.R.override { enableStrictBarrier = true; }; +in +haskell.lib.buildStackProject { + name = "HaskellR"; + buildInputs = [ R zeromq zlib ]; + inherit ghc; +} +``` + +[stack-nix-doc]: http://docs.haskellstack.org/en/stable/nix_integration.html + +### How to create ad hoc environments for `nix-shell` + +The easiest way to create an ad hoc development environment is to run +`nix-shell` with the appropriate GHC environment given on the command-line: +```shell +nix-shell -p "haskellPackages.ghcWithPackages (pkgs: with pkgs; [mtl pandoc])" +``` + +For more sophisticated use-cases, however, it's more convenient to save the +desired configuration in a file called `shell.nix` that looks like this: +```nix +{ nixpkgs ? import {}, compiler ? "ghc7102" }: +let + inherit (nixpkgs) pkgs; + ghc = pkgs.haskell.packages.${compiler}.ghcWithPackages (ps: with ps; [ + monad-par mtl + ]); +in +pkgs.stdenv.mkDerivation { + name = "my-haskell-env-0"; + buildInputs = [ ghc ]; + shellHook = "eval $(egrep ^export ${ghc}/bin/ghc)"; +} +``` + +Now run `nix-shell` --- or even `nix-shell --pure` --- to enter a shell +environment that has the appropriate compiler in `$PATH`. If you use `--pure`, +then add all other packages that your development environment needs into the +`buildInputs` attribute. If you'd like to switch to a different compiler +version, then pass an appropriate `compiler` argument to the expression, i.e. +`nix-shell --argstr compiler ghc784`. + +If you need such an environment because you'd like to compile a Hackage package +outside of Nix --- i.e. because you're hacking on the latest version from Git +---, then the package set provides suitable nix-shell environments for you +already! Every Haskell package has an `env` attribute that provides a shell +environment suitable for compiling that particular package. If you'd like to +hack the `lens` library, for example, then you just have to check out the +source code and enter the appropriate environment: +``` +$ cabal get lens-4.11 && cd lens-4.11 +Downloading lens-4.11... +Unpacking to lens-4.11/ + +$ nix-shell "" -A haskellPackages.lens.env +[nix-shell:/tmp/lens-4.11]$ +``` + +At point, you can run `cabal configure`, `cabal build`, and all the other +development commands. Note that you need `cabal-install` installed in your +`$PATH` already to use it here --- the `nix-shell` environment does not provide +it. + +## How to create Nix builds for your own private Haskell packages + +If your own Haskell packages have build instructions for Cabal, then you can +convert those automatically into build instructions for Nix using the +`cabal2nix` utility, which you can install into your profile by running +`nix-env -i cabal2nix`. + +### How to build a stand-alone project + +For example, let's assume that you're working on a private project called +`foo`. To generate a Nix build expression for it, change into the project's +top-level directory and run the command: +```shell +cabal2nix . > foo.nix +``` +Then write the following snippet into a file called `default.nix`: +```nix +{ nixpkgs ? import {}, compiler ? "ghc7102" }: +nixpkgs.pkgs.haskell.packages.${compiler}.callPackage ./foo.nix { } +``` + +Finally, store the following code in a file called `shell.nix`: +```nix +{ nixpkgs ? import {}, compiler ? "ghc7102" }: +(import ./default.nix { inherit nixpkgs compiler; }).env +``` + +At this point, you can run `nix-build` to have Nix compile your project and +install it into a Nix store path. The local directory will contain a symlink +called `result` after `nix-build` returns that points into that location. Of +course, passing the flag `--argstr compiler ghc763` allows switching the build +to any version of GHC currently supported. + +Furthermore, you can call `nix-shell` to enter an interactive development +environment in which you can use `cabal configure` and `cabal build` to develop +your code. That environment will automatically contain a proper GHC derivation +with all the required libraries registered as well as all the system-level +libraries your package might need. + +If your package does not depend on any system-level libraries, then it's +sufficient to run +```shell +nix-shell --command "cabal configure" +``` +once to set up your build. `cabal-install` determines the absolute paths to all +resources required for the build and writes them into a config file in the +`dist/` directory. Once that's done, you can run `cabal build` and any other +command for that project even outside of the `nix-shell` environment. This +feature is particularly nice for those of us who like to edit their code with +an IDE, like Emacs' `haskell-mode`, because it's not necessary to start Emacs +inside of nix-shell just to make it find out the necessary settings for +building the project; `cabal-install` has already done that for us. + +If you want to do some quick-and-dirty hacking and don't want to bother setting +up a `default.nix` and `shell.nix` file manually, then you can use the +`--shell` flag offered by `cabal2nix` to have it generate a stand-alone +`nix-shell` environment for you. With that feature, running +```shell +cabal2nix --shell . > shell.nix +nix-shell --command "cabal configure" +``` +is usually enough to set up a build environment for any given Haskell package. +You can even use that generated file to run `nix-build`, too: +```shell +nix-build shell.nix +``` + +### How to build projects that depend on each other + +If you have multiple private Haskell packages that depend on each other, then +you'll have to register those packages in the Nixpkgs set to make them visible +for the dependency resolution performed by `callPackage`. First of all, change +into each of your projects top-level directories and generate a `default.nix` +file with `cabal2nix`: +```shell +cd ~/src/foo && cabal2nix . > default.nix +cd ~/src/bar && cabal2nix . > default.nix +``` +Then edit your `~/.config/nixpkgs/config.nix` file to register those builds in the +default Haskell package set: +```nix +{ + packageOverrides = super: let self = super.pkgs; in + { + haskellPackages = super.haskellPackages.override { + overrides = self: super: { + foo = self.callPackage ../src/foo {}; + bar = self.callPackage ../src/bar {}; + }; + }; + }; +} +``` +Once that's accomplished, `nix-env -f "" -qA haskellPackages` will +show your packages like any other package from Hackage, and you can build them +```shell +nix-build "" -A haskellPackages.foo +``` +or enter an interactive shell environment suitable for building them: +```shell +nix-shell "" -A haskellPackages.bar.env +``` + +## Miscellaneous Topics + +### How to build with profiling enabled + +Every Haskell package set takes a function called `overrides` that you can use +to manipulate the package as much as you please. One useful application of this +feature is to replace the default `mkDerivation` function with one that enables +library profiling for all packages. To accomplish that add the following +snippet to your `~/.config/nixpkgs/config.nix` file: +```nix +{ + packageOverrides = super: let self = super.pkgs; in + { + profiledHaskellPackages = self.haskellPackages.override { + overrides = self: super: { + mkDerivation = args: super.mkDerivation (args // { + enableLibraryProfiling = true; + }); + }; + }; + }; +} +``` +Then, replace instances of `haskellPackages` in the `cabal2nix`-generated +`default.nix` or `shell.nix` files with `profiledHaskellPackages`. + +### How to override package versions in a compiler-specific package set + +Nixpkgs provides the latest version of +[`ghc-events`](http://hackage.haskell.org/package/ghc-events), which is 0.4.4.0 +at the time of this writing. This is fine for users of GHC 7.10.x, but GHC +7.8.4 cannot compile that binary. Now, one way to solve that problem is to +register an older version of `ghc-events` in the 7.8.x-specific package set. +The first step is to generate Nix build instructions with `cabal2nix`: +```shell +cabal2nix cabal://ghc-events-0.4.3.0 > ~/.nixpkgs/ghc-events-0.4.3.0.nix +``` +Then add the override in `~/.config/nixpkgs/config.nix`: +```nix +{ + packageOverrides = super: let self = super.pkgs; in + { + haskell = super.haskell // { + packages = super.haskell.packages // { + ghc784 = super.haskell.packages.ghc784.override { + overrides = self: super: { + ghc-events = self.callPackage ./ghc-events-0.4.3.0.nix {}; + }; + }; + }; + }; + }; +} +``` + +This code is a little crazy, no doubt, but it's necessary because the intuitive +version +```nix +{ # ... + + haskell.packages.ghc784 = super.haskell.packages.ghc784.override { + overrides = self: super: { + ghc-events = self.callPackage ./ghc-events-0.4.3.0.nix {}; + }; + }; +} +``` +doesn't do what we want it to: that code replaces the `haskell` package set in +Nixpkgs with one that contains only one entry,`packages`, which contains only +one entry `ghc784`. This override loses the `haskell.compiler` set, and it +loses the `haskell.packages.ghcXYZ` sets for all compilers but GHC 7.8.4. To +avoid that problem, we have to perform the convoluted little dance from above, +iterating over each step in hierarchy. + +Once it's accomplished, however, we can install a variant of `ghc-events` +that's compiled with GHC 7.8.4: +```shell +nix-env -f "" -iA haskell.packages.ghc784.ghc-events +``` +Unfortunately, it turns out that this build fails again while executing the +test suite! Apparently, the release archive on Hackage is missing some data +files that the test suite requires, so we cannot run it. We accomplish that by +re-generating the Nix expression with the `--no-check` flag: +```shell +cabal2nix --no-check cabal://ghc-events-0.4.3.0 > ~/.nixpkgs/ghc-events-0.4.3.0.nix +``` +Now the builds succeeds. + +Of course, in the concrete example of `ghc-events` this whole exercise is not +an ideal solution, because `ghc-events` can analyze the output emitted by any +version of GHC later than 6.12 regardless of the compiler version that was used +to build the `ghc-events` executable, so strictly speaking there's no reason to +prefer one built with GHC 7.8.x in the first place. However, for users who +cannot use GHC 7.10.x at all for some reason, the approach of downgrading to an +older version might be useful. + +### How to recover from GHC's infamous non-deterministic library ID bug + +GHC and distributed build farms don't get along well: + + - https://ghc.haskell.org/trac/ghc/ticket/4012 + +When you see an error like this one +``` +package foo-0.7.1.0 is broken due to missing package +text-1.2.0.4-98506efb1b9ada233bb5c2b2db516d91 +``` +then you have to download and re-install `foo` and all its dependents from +scratch: +```shell +nix-store -q --referrers /nix/store/*-haskell-text-1.2.0.4 \ + | xargs -L 1 nix-store --repair-path +``` + +If you're using additional Hydra servers other than `hydra.nixos.org`, then it +might be necessary to purge the local caches that store data from those +machines to disable these binary channels for the duration of the previous +command, i.e. by running: +```shell +rm ~/.cache/nix/binary-cache*.sqlite +``` + +### Builds on Darwin fail with `math.h` not found + +Users of GHC on Darwin have occasionally reported that builds fail, because the +compiler complains about a missing include file: +``` +fatal error: 'math.h' file not found +``` +The issue has been discussed at length in [ticket +6390](https://github.com/NixOS/nixpkgs/issues/6390), and so far no good +solution has been proposed. As a work-around, users who run into this problem +can configure the environment variables +```shell +export NIX_CFLAGS_COMPILE="-idirafter /usr/include" +export NIX_CFLAGS_LINK="-L/usr/lib" +``` +in their `~/.bashrc` file to avoid the compiler error. + +### Builds using Stack complain about missing system libraries + +``` +-- While building package zlib-0.5.4.2 using: + runhaskell -package=Cabal-1.22.4.0 -clear-package-db [... lots of flags ...] +Process exited with code: ExitFailure 1 +Logs have been written to: /home/foo/src/stack-ide/.stack-work/logs/zlib-0.5.4.2.log + +Configuring zlib-0.5.4.2... +Setup.hs: Missing dependency on a foreign library: +* Missing (or bad) header file: zlib.h +This problem can usually be solved by installing the system package that +provides this library (you may need the "-dev" version). If the library is +already installed but in a non-standard location then you can use the flags +--extra-include-dirs= and --extra-lib-dirs= to specify where it is. +If the header file does exist, it may contain errors that are caught by the C +compiler at the preprocessing stage. In this case you can re-run configure +with the verbosity flag -v3 to see the error messages. +``` + +When you run the build inside of the nix-shell environment, the system +is configured to find `libz.so` without any special flags -- the compiler +and linker "just know" how to find it. Consequently, Cabal won't record +any search paths for `libz.so` in the package description, which means +that the package works fine inside of nix-shell, but once you leave the +shell the shared object can no longer be found. That issue is by no +means specific to Stack: you'll have that problem with any other +Haskell package that's built inside of nix-shell but run outside of that +environment. + +You can remedy this issue in several ways. The easiest is to add a `nix` section +to the `stack.yaml` like the following: +```yaml +nix: + enable: true + packages: [ zlib ] +``` + +Stack's Nix support knows to add `${zlib.out}/lib` and `${zlib.dev}/include` +as an `--extra-lib-dirs` and `extra-include-dirs`, respectively. +Alternatively, you can achieve the same effect by hand. First of all, run +``` +$ nix-build --no-out-link "" -A zlib +/nix/store/alsvwzkiw4b7ip38l4nlfjijdvg3fvzn-zlib-1.2.8 +``` +to find out the store path of the system's zlib library. Now, you can + + 1. add that path (plus a "/lib" suffix) to your `$LD_LIBRARY_PATH` + environment variable to make sure your system linker finds `libz.so` + automatically. It's no pretty solution, but it will work. + + 2. As a variant of (1), you can also install any number of system + libraries into your user's profile (or some other profile) and point + `$LD_LIBRARY_PATH` to that profile instead, so that you don't have to + list dozens of those store paths all over the place. + + 3. The solution I prefer is to call stack with an appropriate + --extra-lib-dirs flag like so: + ```shell + stack --extra-lib-dirs=/nix/store/alsvwzkiw4b7ip38l4nlfjijdvg3fvzn-zlib-1.2.8/lib build + ``` + +Typically, you'll need `--extra-include-dirs` as well. It's possible +to add those flag to the project's `stack.yaml` or your user's +global `~/.stack/global/stack.yaml` file so that you don't have to +specify them manually every time. But again, you're likely better off +using Stack's Nix support instead. + +The same thing applies to `cabal configure`, of course, if you're +building with `cabal-install` instead of Stack. + +### Creating statically linked binaries + +There are two levels of static linking. The first option is to configure the +build with the Cabal flag `--disable-executable-dynamic`. In Nix expressions, +this can be achieved by setting the attribute: +``` +enableSharedExecutables = false; +``` +That gives you a binary with statically linked Haskell libraries and +dynamically linked system libraries. + +To link both Haskell libraries and system libraries statically, the additional +flags `--ghc-option=-optl=-static --ghc-option=-optl=-pthread` need to be used. +In Nix, this is accomplished with: +``` +configureFlags = [ "--ghc-option=-optl=-static" "--ghc-option=-optl=-pthread" ]; +``` + +It's important to realize, however, that most system libraries in Nix are +built as shared libraries only, i.e. there is just no static library +available that Cabal could link! + +### Building GHC with integer-simple + +By default GHC implements the Integer type using the +[GNU Multiple Precision Arithmetic (GMP) library](https://gmplib.org/). +The implementation can be found in the +[integer-gmp](http://hackage.haskell.org/package/integer-gmp) package. + +A potential problem with this is that GMP is licensed under the +[GNU Lesser General Public License (LGPL)](http://www.gnu.org/copyleft/lesser.html), +a kind of "copyleft" license. According to the terms of the LGPL, paragraph 5, +you may distribute a program that is designed to be compiled and dynamically +linked with the library under the terms of your choice (i.e., commercially) but +if your program incorporates portions of the library, if it is linked +statically, then your program is a "derivative"--a "work based on the +library"--and according to paragraph 2, section c, you "must cause the whole of +the work to be licensed" under the terms of the LGPL (including for free). + +The LGPL licensing for GMP is a problem for the overall licensing of binary +programs compiled with GHC because most distributions (and builds) of GHC use +static libraries. (Dynamic libraries are currently distributed only for macOS.) +The LGPL licensing situation may be worse: even though +[The Glasgow Haskell Compiler License](https://www.haskell.org/ghc/license) +is essentially a "free software" license (BSD3), according to +paragraph 2 of the LGPL, GHC must be distributed under the terms of the LGPL! + +To work around these problems GHC can be build with a slower but LGPL-free +alternative implemention for Integer called +[integer-simple](http://hackage.haskell.org/package/integer-simple). + +To get a GHC compiler build with `integer-simple` instead of `integer-gmp` use +the attribute: `haskell.compiler.integer-simple."${ghcVersion}"`. +For example: +``` +$ nix-build -E '(import {}).haskell.compiler.integer-simple.ghc802' +... +$ result/bin/ghc-pkg list | grep integer + integer-simple-0.1.1.1 +``` +The following command displays the complete list of GHC compilers build with `integer-simple`: +``` +$ nix-env -f "" -qaP -A haskell.compiler.integer-simple +haskell.compiler.integer-simple.ghc7102 ghc-7.10.2 +haskell.compiler.integer-simple.ghc7103 ghc-7.10.3 +haskell.compiler.integer-simple.ghc722 ghc-7.2.2 +haskell.compiler.integer-simple.ghc742 ghc-7.4.2 +haskell.compiler.integer-simple.ghc783 ghc-7.8.3 +haskell.compiler.integer-simple.ghc784 ghc-7.8.4 +haskell.compiler.integer-simple.ghc801 ghc-8.0.1 +haskell.compiler.integer-simple.ghc802 ghc-8.0.2 +haskell.compiler.integer-simple.ghcHEAD ghc-8.1.20170106 +``` + +To get a package set supporting `integer-simple` use the attribute: +`haskell.packages.integer-simple."${ghcVersion}"`. For example +use the following to get the `scientific` package build with `integer-simple`: +```shell +nix-build -A haskell.packages.integer-simple.ghc802.scientific +``` + +### Quality assurance + +The `haskell.lib` library includes a number of functions for checking for +various imperfections in Haskell packages. It's useful to apply these functions +to your own Haskell packages and integrate that in a Continuous Integration +server like [hydra](https://nixos.org/hydra/) to assure your packages maintain a +minimum level of quality. This section discusses some of these functions. + +#### failOnAllWarnings + +Applying `haskell.lib.failOnAllWarnings` to a Haskell package enables the +`-Wall` and `-Werror` GHC options to turn all warnings into build failures. + +#### buildStrictly + +Applying `haskell.lib.buildStrictly` to a Haskell package calls +`failOnAllWarnings` on the given package to turn all warnings into build +failures. Additionally the source of your package is gotten from first invoking +`cabal sdist` to ensure all needed files are listed in the Cabal file. + +#### checkUnusedPackages + +Applying `haskell.lib.checkUnusedPackages` to a Haskell package invokes +the [packunused](http://hackage.haskell.org/package/packunused) tool on the +package. `packunused` complains when it finds packages listed as build-depends +in the Cabal file which are redundant. For example: + +``` +$ nix-build -E 'let pkgs = import {}; in pkgs.haskell.lib.checkUnusedPackages {} pkgs.haskellPackages.scientific' +these derivations will be built: + /nix/store/3lc51cxj2j57y3zfpq5i69qbzjpvyci1-scientific-0.3.5.1.drv +... +detected package components +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + - library + - testsuite(s): test-scientific + - benchmark(s): bench-scientific* + +(component names suffixed with '*' are not configured to be built) + +library +~~~~~~~ + +The following package dependencies seem redundant: + + - ghc-prim-0.5.0.0 + +testsuite(test-scientific) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +no redundant packages dependencies found + +builder for ‘/nix/store/3lc51cxj2j57y3zfpq5i69qbzjpvyci1-scientific-0.3.5.1.drv’ failed with exit code 1 +error: build of ‘/nix/store/3lc51cxj2j57y3zfpq5i69qbzjpvyci1-scientific-0.3.5.1.drv’ failed +``` + +As you can see, `packunused` finds out that although the testsuite component has +no redundant dependencies the library component of `scientific-0.3.5.1` depends +on `ghc-prim` which is unused in the library. + +## Other resources + + - The Youtube video [Nix Loves Haskell](https://www.youtube.com/watch?v=BsBhi_r-OeE) + provides an introduction into Haskell NG aimed at beginners. The slides are + available at http://cryp.to/nixos-meetup-3-slides.pdf and also -- in a form + ready for cut & paste -- at + https://github.com/NixOS/cabal2nix/blob/master/doc/nixos-meetup-3-slides.md. + + - Another Youtube video is [Escaping Cabal Hell with Nix](https://www.youtube.com/watch?v=mQd3s57n_2Y), + which discusses the subject of Haskell development with Nix but also provides + a basic introduction to Nix as well, i.e. it's suitable for viewers with + almost no prior Nix experience. + + - Oliver Charles wrote a very nice [Tutorial how to develop Haskell packages with Nix](http://wiki.ocharles.org.uk/Nix). + + - The *Journey into the Haskell NG infrastructure* series of postings + describe the new Haskell infrastructure in great detail: + + - [Part 1](https://nixos.org/nix-dev/2015-January/015591.html) + explains the differences between the old and the new code and gives + instructions how to migrate to the new setup. + + - [Part 2](https://nixos.org/nix-dev/2015-January/015608.html) + looks in-depth at how to tweak and configure your setup by means of + overrides. + + - [Part 3](https://nixos.org/nix-dev/2015-April/016912.html) + describes the infrastructure that keeps the Haskell package set in Nixpkgs + up-to-date. diff --git a/doc/languages-frameworks/idris.section.md b/doc/languages-frameworks/idris.section.md new file mode 100644 index 000000000000..005ed3602851 --- /dev/null +++ b/doc/languages-frameworks/idris.section.md @@ -0,0 +1,39 @@ +Idris packages +============== + +This directory contains build rules for idris packages. In addition, +it contains several functions to build and compose those packages. +Everything is exposed to the user via the `idrisPackages` attribute. + +callPackage +------------ + +This is like the normal nixpkgs callPackage function, specialized to +idris packages. + +builtins +--------- + +This is a list of all of the libraries that come packaged with Idris +itself. + +build-idris-package +-------------------- + +A function to build an idris package. Its sole argument is a set like +you might pass to `stdenv.mkDerivation`, except `build-idris-package` +sets several attributes for you. See `build-idris-package.nix` for +details. + +build-builtin-package +---------------------- + +A version of `build-idris-package` specialized to builtin libraries. +Mostly for internal use. + +with-packages +------------- + +Bundle idris together with a list of packages. Because idris currently +only supports a single directory in its library path, you must include +all desired libraries here, including `prelude` and `base`. \ No newline at end of file diff --git a/doc/languages-frameworks/index.xml b/doc/languages-frameworks/index.xml index 6743c131201f..a1c265f67484 100644 --- a/doc/languages-frameworks/index.xml +++ b/doc/languages-frameworks/index.xml @@ -17,20 +17,20 @@ such as Perl or Haskell. These are described in this chapter. - - + + - + - + - + - + - - + + diff --git a/doc/languages-frameworks/node.section.md b/doc/languages-frameworks/node.section.md new file mode 100644 index 000000000000..17a203ed12be --- /dev/null +++ b/doc/languages-frameworks/node.section.md @@ -0,0 +1,51 @@ +Node.js packages +================ +The `pkgs/development/node-packages` folder contains a generated collection of +[NPM packages](https://npmjs.com/) that can be installed with the Nix package +manager. + +As a rule of thumb, the package set should only provide *end user* software +packages, such as command-line utilities. Libraries should only be added to the +package set if there is a non-NPM package that requires it. + +When it is desired to use NPM libraries in a development project, use the +`node2nix` generator directly on the `package.json` configuration file of the +project. + +The package set also provides support for multiple Node.js versions. The policy +is that a new package should be added to the collection for the latest stable LTS +release (which is currently 6.x), unless there is an explicit reason to support +a different release. + +If your package uses native addons, you need to examine what kind of native +build system it uses. Here are some examples: + +* `node-gyp` +* `node-gyp-builder` +* `node-pre-gyp` + +After you have identified the correct system, you need to override your package +expression while adding in build system as a build input. For example, `dat` +requires `node-gyp-build`, so we override its expression in `default-v6.nix`: + +```nix +dat = nodePackages.dat.override (oldAttrs: { + buildInputs = oldAttrs.buildInputs ++ [ nodePackages.node-gyp-build ]; +}); +``` + +To add a package from NPM to nixpkgs: + + 1. Modify `pkgs/development/node-packages/node-packages-v6.json` to add, update + or remove package entries. (Or `pkgs/development/node-packages/node-packages-v4.json` + for packages depending on Node.js 4.x) + 2. Run the script: `(cd pkgs/development/node-packages && ./generate.sh)`. + 3. Build your new package to test your changes: + `cd /path/to/nixpkgs && nix-build -A nodePackages.`. + To build against a specific Node.js version (e.g. 4.x): + `nix-build -A nodePackages_4_x.` + 4. Add and commit all modified and generated files. + +For more information about the generation process, consult the +[README.md](https://github.com/svanderburg/node2nix) file of the `node2nix` +tool. diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md deleted file mode 100644 index 0f871395854c..000000000000 --- a/doc/languages-frameworks/python.md +++ /dev/null @@ -1,1026 +0,0 @@ -# Python - -## User Guide - -### Using Python - -#### Overview - -Several versions of the Python interpreter are available on Nix, as well as a -high amount of packages. The attribute `python` refers to the default -interpreter, which is currently CPython 2.7. It is also possible to refer to -specific versions, e.g. `python35` refers to CPython 3.5, and `pypy` refers to -the default PyPy interpreter. - -Python is used a lot, and in different ways. This affects also how it is -packaged. In the case of Python on Nix, an important distinction is made between -whether the package is considered primarily an application, or whether it should -be used as a library, i.e., of primary interest are the modules in -`site-packages` that should be importable. - -In the Nixpkgs tree Python applications can be found throughout, depending on -what they do, and are called from the main package set. Python libraries, -however, are in separate sets, with one set per interpreter version. - -The interpreters have several common attributes. One of these attributes is -`pkgs`, which is a package set of Python libraries for this specific -interpreter. E.g., the `toolz` package corresponding to the default interpreter -is `python.pkgs.toolz`, and the CPython 3.5 version is `python35.pkgs.toolz`. -The main package set contains aliases to these package sets, e.g. -`pythonPackages` refers to `python.pkgs` and `python35Packages` to -`python35.pkgs`. - -#### Installing Python and packages - -The Nix and NixOS manuals explain how packages are generally installed. In the -case of Python and Nix, it is important to make a distinction between whether the -package is considered an application or a library. - -Applications on Nix are typically installed into your user -profile imperatively using `nix-env -i`, and on NixOS declaratively by adding the -package name to `environment.systemPackages` in `/etc/nixos/configuration.nix`. -Dependencies such as libraries are automatically installed and should not be -installed explicitly. - -The same goes for Python applications and libraries. Python applications can be -installed in your profile. But Python libraries you would like to use for -development cannot be installed, at least not individually, because they won't -be able to find each other resulting in import errors. Instead, it is possible -to create an environment with `python.buildEnv` or `python.withPackages` where -the interpreter and other executables are able to find each other and all of the -modules. - -In the following examples we create an environment with Python 3.5, `numpy` and -`toolz`. As you may imagine, there is one limitation here, and that's that -you can install only one environment at a time. You will notice the complaints -about collisions when you try to install a second environment. - -##### Environment defined in separate `.nix` file - -Create a file, e.g. `build.nix`, with the following expression -```nix -with import {}; - -python35.withPackages (ps: with ps; [ numpy toolz ]) -``` -and install it in your profile with -```shell -nix-env -if build.nix -``` -Now you can use the Python interpreter, as well as the extra packages (`numpy`, -`toolz`) that you added to the environment. - -##### Environment defined in `~/.config/nixpkgs/config.nix` - -If you prefer to, you could also add the environment as a package override to the Nixpkgs set, e.g. -using `config.nix`, -```nix -{ # ... - - packageOverrides = pkgs: with pkgs; { - myEnv = python35.withPackages (ps: with ps; [ numpy toolz ]); - }; -} -``` -and install it in your profile with -```shell -nix-env -iA nixpkgs.myEnv -``` -The environment is is installed by referring to the attribute, and considering -the `nixpkgs` channel was used. - -##### Environment defined in `/etc/nixos/configuration.nix` - -For the sake of completeness, here's another example how to install the environment system-wide. - -```nix -{ # ... - - environment.systemPackages = with pkgs; [ - (python35.withPackages(ps: with ps; [ numpy toolz ])) - ]; -} -``` - -#### Temporary Python environment with `nix-shell` - -The examples in the previous section showed how to install a Python environment -into a profile. For development you may need to use multiple environments. -`nix-shell` gives the possibility to temporarily load another environment, akin -to `virtualenv`. - -There are two methods for loading a shell with Python packages. The first and recommended method -is to create an environment with `python.buildEnv` or `python.withPackages` and load that. E.g. -```sh -$ nix-shell -p 'python35.withPackages(ps: with ps; [ numpy toolz ])' -``` -opens a shell from which you can launch the interpreter -```sh -[nix-shell:~] python3 -``` -The other method, which is not recommended, does not create an environment and requires you to list the packages directly, - -```sh -$ nix-shell -p python35.pkgs.numpy python35.pkgs.toolz -``` -Again, it is possible to launch the interpreter from the shell. -The Python interpreter has the attribute `pkgs` which contains all Python libraries for that specific interpreter. - -##### Load environment from `.nix` expression -As explained in the Nix manual, `nix-shell` can also load an -expression from a `.nix` file. Say we want to have Python 3.5, `numpy` -and `toolz`, like before, in an environment. Consider a `shell.nix` file -with -```nix -with import {}; - -(python35.withPackages (ps: [ps.numpy ps.toolz])).env -``` -Executing `nix-shell` gives you again a Nix shell from which you can run Python. - -What's happening here? - -1. We begin with importing the Nix Packages collections. `import ` imports the `` function, `{}` calls it and the `with` statement brings all attributes of `nixpkgs` in the local scope. These attributes form the main package set. -2. Then we create a Python 3.5 environment with the `withPackages` function. -3. The `withPackages` function expects us to provide a function as an argument that takes the set of all python packages and returns a list of packages to include in the environment. Here, we select the packages `numpy` and `toolz` from the package set. - -##### Execute command with `--run` -A convenient option with `nix-shell` is the `--run` -option, with which you can execute a command in the `nix-shell`. We can -e.g. directly open a Python shell -```sh -$ nix-shell -p python35Packages.numpy python35Packages.toolz --run "python3" -``` -or run a script -```sh -$ nix-shell -p python35Packages.numpy python35Packages.toolz --run "python3 myscript.py" -``` - -##### `nix-shell` as shebang -In fact, for the second use case, there is a more convenient method. You can -add a [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) to your script -specifying which dependencies `nix-shell` needs. With the following shebang, you -can just execute `./myscript.py`, and it will make available all dependencies and -run the script in the `python3` shell. - -```py -#! /usr/bin/env nix-shell -#! nix-shell -i python3 -p "python3.withPackages(ps: [ps.numpy])" - -import numpy - -print(numpy.__version__) -``` - -### Developing with Python - -Now that you know how to get a working Python environment with Nix, it is time -to go forward and start actually developing with Python. We will first have a -look at how Python packages are packaged on Nix. Then, we will look at how you -can use development mode with your code. - -#### Packaging a library - -With Nix all packages are built by functions. The main function in Nix for -building Python libraries is `buildPythonPackage`. Let's see how we can build the -`toolz` package. - -```nix -{ # ... - - toolz = buildPythonPackage rec { - pname = "toolz"; - version = "0.7.4"; - - src = fetchPypi { - inherit pname version; - sha256 = "43c2c9e5e7a16b6c88ba3088a9bfc82f7db8e13378be7c78d6c14a5f8ed05afd"; - }; - - doCheck = false; - - meta = { - homepage = "http://github.com/pytoolz/toolz/"; - description = "List processing tools and functional utilities"; - license = licenses.bsd3; - maintainers = with maintainers; [ fridh ]; - }; - }; -} -``` - -What happens here? The function `buildPythonPackage` is called and as argument -it accepts a set. In this case the set is a recursive set, `rec`. One of the -arguments is the name of the package, which consists of a basename (generally -following the name on PyPi) and a version. Another argument, `src` specifies the -source, which in this case is fetched from PyPI using the helper function -`fetchPypi`. The argument `doCheck` is used to set whether tests should be run -when building the package. Furthermore, we specify some (optional) meta -information. The output of the function is a derivation. - -An expression for `toolz` can be found in the Nixpkgs repository. As explained -in the introduction of this Python section, a derivation of `toolz` is available -for each interpreter version, e.g. `python35.pkgs.toolz` refers to the `toolz` -derivation corresponding to the CPython 3.5 interpreter. -The above example works when you're directly working on -`pkgs/top-level/python-packages.nix` in the Nixpkgs repository. Often though, -you will want to test a Nix expression outside of the Nixpkgs tree. - -The following expression creates a derivation for the `toolz` package, -and adds it along with a `numpy` package to a Python environment. - -```nix -with import {}; - -( let - my_toolz = python35.pkgs.buildPythonPackage rec { - pname = "toolz"; - version = "0.7.4"; - - src = python35.pkgs.fetchPypi { - inherit pname version; - sha256 = "43c2c9e5e7a16b6c88ba3088a9bfc82f7db8e13378be7c78d6c14a5f8ed05afd"; - }; - - doCheck = false; - - meta = { - homepage = "http://github.com/pytoolz/toolz/"; - description = "List processing tools and functional utilities"; - }; - }; - - in python35.withPackages (ps: [ps.numpy my_toolz]) -).env -``` -Executing `nix-shell` will result in an environment in which you can use -Python 3.5 and the `toolz` package. As you can see we had to explicitly mention -for which Python version we want to build a package. - -So, what did we do here? Well, we took the Nix expression that we used earlier -to build a Python environment, and said that we wanted to include our own -version of `toolz`, named `my_toolz`. To introduce our own package in the scope -of `withPackages` we used a `let` expression. You can see that we used -`ps.numpy` to select numpy from the nixpkgs package set (`ps`). We did not take -`toolz` from the Nixpkgs package set this time, but instead took our own version -that we introduced with the `let` expression. - -#### Handling dependencies - -Our example, `toolz`, does not have any dependencies on other Python -packages or system libraries. According to the manual, `buildPythonPackage` -uses the arguments `buildInputs` and `propagatedBuildInputs` to specify dependencies. If something is -exclusively a build-time dependency, then the dependency should be included as a -`buildInput`, but if it is (also) a runtime dependency, then it should be added -to `propagatedBuildInputs`. Test dependencies are considered build-time dependencies. - -The following example shows which arguments are given to `buildPythonPackage` in -order to build [`datashape`](https://github.com/blaze/datashape). - -```nix -{ # ... - - datashape = buildPythonPackage rec { - pname = "datashape"; - version = "0.4.7"; - - src = fetchPypi { - inherit pname version; - sha256 = "14b2ef766d4c9652ab813182e866f493475e65e558bed0822e38bf07bba1a278"; - }; - - checkInputs = with self; [ pytest ]; - propagatedBuildInputs = with self; [ numpy multipledispatch dateutil ]; - - meta = { - homepage = https://github.com/ContinuumIO/datashape; - description = "A data description language"; - license = licenses.bsd2; - maintainers = with maintainers; [ fridh ]; - }; - }; -} -``` - -We can see several runtime dependencies, `numpy`, `multipledispatch`, and -`dateutil`. Furthermore, we have one `buildInput`, i.e. `pytest`. `pytest` is a -test runner and is only used during the `checkPhase` and is therefore not added -to `propagatedBuildInputs`. - -In the previous case we had only dependencies on other Python packages to consider. -Occasionally you have also system libraries to consider. E.g., `lxml` provides -Python bindings to `libxml2` and `libxslt`. These libraries are only required -when building the bindings and are therefore added as `buildInputs`. - -```nix -{ # ... - - lxml = buildPythonPackage rec { - pname = "lxml"; - version = "3.4.4"; - - src = fetchPypi { - inherit pname version; - sha256 = "16a0fa97hym9ysdk3rmqz32xdjqmy4w34ld3rm3jf5viqjx65lxk"; - }; - - buildInputs = with self; [ pkgs.libxml2 pkgs.libxslt ]; - - meta = { - description = "Pythonic binding for the libxml2 and libxslt libraries"; - homepage = http://lxml.de; - license = licenses.bsd3; - maintainers = with maintainers; [ sjourdois ]; - }; - }; -} -``` - -In this example `lxml` and Nix are able to work out exactly where the relevant -files of the dependencies are. This is not always the case. - -The example below shows bindings to The Fastest Fourier Transform in the West, commonly known as -FFTW. On Nix we have separate packages of FFTW for the different types of floats -(`"single"`, `"double"`, `"long-double"`). The bindings need all three types, -and therefore we add all three as `buildInputs`. The bindings don't expect to -find each of them in a different folder, and therefore we have to set `LDFLAGS` -and `CFLAGS`. - -```nix -{ # ... - - pyfftw = buildPythonPackage rec { - pname = "pyFFTW"; - version = "0.9.2"; - - src = fetchPypi { - inherit pname version; - sha256 = "f6bbb6afa93085409ab24885a1a3cdb8909f095a142f4d49e346f2bd1b789074"; - }; - - buildInputs = [ pkgs.fftw pkgs.fftwFloat pkgs.fftwLongDouble]; - - propagatedBuildInputs = with self; [ numpy scipy ]; - - # Tests cannot import pyfftw. pyfftw works fine though. - doCheck = false; - - preConfigure = '' - export LDFLAGS="-L${pkgs.fftw.dev}/lib -L${pkgs.fftwFloat.out}/lib -L${pkgs.fftwLongDouble.out}/lib" - export CFLAGS="-I${pkgs.fftw.dev}/include -I${pkgs.fftwFloat.dev}/include -I${pkgs.fftwLongDouble.dev}/include" - ''; - - meta = { - description = "A pythonic wrapper around FFTW, the FFT library, presenting a unified interface for all the supported transforms"; - homepage = http://hgomersall.github.com/pyFFTW/; - license = with licenses; [ bsd2 bsd3 ]; - maintainer = with maintainers; [ fridh ]; - }; - }; -} -``` -Note also the line `doCheck = false;`, we explicitly disabled running the test-suite. - - -#### Develop local package - -As a Python developer you're likely aware of [development mode](http://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode) (`python setup.py develop`); -instead of installing the package this command creates a special link to the project code. -That way, you can run updated code without having to reinstall after each and every change you make. -Development mode is also available. Let's see how you can use it. - -In the previous Nix expression the source was fetched from an url. We can also refer to a local source instead using -`src = ./path/to/source/tree;` - -If we create a `shell.nix` file which calls `buildPythonPackage`, and if `src` -is a local source, and if the local source has a `setup.py`, then development -mode is activated. - -In the following example we create a simple environment that -has a Python 3.5 version of our package in it, as well as its dependencies and -other packages we like to have in the environment, all specified with `propagatedBuildInputs`. -Indeed, we can just add any package we like to have in our environment to `propagatedBuildInputs`. - -```nix -with import {}; -with pkgs.python35Packages; - -buildPythonPackage rec { - name = "mypackage"; - src = ./path/to/package/source; - propagatedBuildInputs = [ pytest numpy pkgs.libsndfile ]; -} -``` - -It is important to note that due to how development mode is implemented on Nix it is not possible to have multiple packages simultaneously in development mode. - - -### Organising your packages - -So far we discussed how you can use Python on Nix, and how you can develop with -it. We've looked at how you write expressions to package Python packages, and we -looked at how you can create environments in which specified packages are -available. - -At some point you'll likely have multiple packages which you would -like to be able to use in different projects. In order to minimise unnecessary -duplication we now look at how you can maintain yourself a repository with your -own packages. The important functions here are `import` and `callPackage`. - -### Including a derivation using `callPackage` - -Earlier we created a Python environment using `withPackages`, and included the -`toolz` package via a `let` expression. -Let's split the package definition from the environment definition. - -We first create a function that builds `toolz` in `~/path/to/toolz/release.nix` - -```nix -{ pkgs, buildPythonPackage }: - -buildPythonPackage rec { - pname = "toolz"; - version = "0.7.4"; - - src = fetchPypi { - inherit pname version; - sha256 = "43c2c9e5e7a16b6c88ba3088a9bfc82f7db8e13378be7c78d6c14a5f8ed05afd"; - }; - - meta = { - homepage = "http://github.com/pytoolz/toolz/"; - description = "List processing tools and functional utilities"; - license = licenses.bsd3; - maintainers = with maintainers; [ fridh ]; - }; -} -``` - -It takes two arguments, `pkgs` and `buildPythonPackage`. -We now call this function using `callPackage` in the definition of our environment - -```nix -with import {}; - -( let - toolz = pkgs.callPackage /path/to/toolz/release.nix { - pkgs = pkgs; - buildPythonPackage = pkgs.python35Packages.buildPythonPackage; - }; - in pkgs.python35.withPackages (ps: [ ps.numpy toolz ]) -).env -``` - -Important to remember is that the Python version for which the package is made -depends on the `python` derivation that is passed to `buildPythonPackage`. Nix -tries to automatically pass arguments when possible, which is why generally you -don't explicitly define which `python` derivation should be used. In the above -example we use `buildPythonPackage` that is part of the set `python35Packages`, -and in this case the `python35` interpreter is automatically used. - - - -## Reference - -### Interpreters - -Versions 2.7, 3.3, 3.4, 3.5 and 3.6 of the CPython interpreter are available as -respectively `python27`, `python34`, `python35` and `python36`. The PyPy interpreter -is available as `pypy`. The aliases `python2` and `python3` correspond to respectively `python27` and -`python35`. The default interpreter, `python`, maps to `python2`. -The Nix expressions for the interpreters can be found in -`pkgs/development/interpreters/python`. - -All packages depending on any Python interpreter get appended -`out/{python.sitePackages}` to `$PYTHONPATH` if such directory -exists. - -#### Missing `tkinter` module standard library - -To reduce closure size the `Tkinter`/`tkinter` is available as a separate package, `pythonPackages.tkinter`. - -#### Attributes on interpreters packages - -Each interpreter has the following attributes: - -- `libPrefix`. Name of the folder in `${python}/lib/` for corresponding interpreter. -- `interpreter`. Alias for `${python}/bin/${executable}`. -- `buildEnv`. Function to build python interpreter environments with extra packages bundled together. See section *python.buildEnv function* for usage and documentation. -- `withPackages`. Simpler interface to `buildEnv`. See section *python.withPackages function* for usage and documentation. -- `sitePackages`. Alias for `lib/${libPrefix}/site-packages`. -- `executable`. Name of the interpreter executable, e.g. `python3.4`. -- `pkgs`. Set of Python packages for that specific interpreter. The package set can be modified by overriding the interpreter and passing `packageOverrides`. - -### Building packages and applications - -Python libraries and applications that use `setuptools` or -`distutils` are typically build with respectively the `buildPythonPackage` and -`buildPythonApplication` functions. These two functions also support installing a `wheel`. - -All Python packages reside in `pkgs/top-level/python-packages.nix` and all -applications elsewhere. In case a package is used as both a library and an application, -then the package should be in `pkgs/top-level/python-packages.nix` since only those packages are made -available for all interpreter versions. The preferred location for library expressions is in -`pkgs/development/python-modules`. It is important that these packages are -called from `pkgs/top-level/python-packages.nix` and not elsewhere, to guarantee -the right version of the package is built. - -Based on the packages defined in `pkgs/top-level/python-packages.nix` an -attribute set is created for each available Python interpreter. The available -sets are - -* `pkgs.python27Packages` -* `pkgs.python34Packages` -* `pkgs.python35Packages` -* `pkgs.python36Packages` -* `pkgs.pypyPackages` - -and the aliases - -* `pkgs.python2Packages` pointing to `pkgs.python27Packages` -* `pkgs.python3Packages` pointing to `pkgs.python36Packages` -* `pkgs.pythonPackages` pointing to `pkgs.python2Packages` - -#### `buildPythonPackage` function - -The `buildPythonPackage` function is implemented in -`pkgs/development/interpreters/python/build-python-package.nix` - -The following is an example: -```nix - -buildPythonPackage rec { - version = "3.3.1"; - pname = "pytest"; - - preCheck = '' - # don't test bash builtins - rm testing/test_argcomplete.py - ''; - - src = fetchPypi { - inherit pname version; - sha256 = "cf8436dc59d8695346fcd3ab296de46425ecab00d64096cebe79fb51ecb2eb93"; - }; - - checkInputs = [ hypothesis ]; - buildInputs = [ setuptools_scm ]; - propagatedBuildInputs = [ attrs py setuptools six pluggy ]; - - meta = with stdenv.lib; { - maintainers = with maintainers; [ domenkozar lovek323 madjar lsix ]; - description = "Framework for writing tests"; - }; -} - -``` - -The `buildPythonPackage` mainly does four things: - -* In the `buildPhase`, it calls `${python.interpreter} setup.py bdist_wheel` to - build a wheel binary zipfile. -* In the `installPhase`, it installs the wheel file using `pip install *.whl`. -* In the `postFixup` phase, the `wrapPythonPrograms` bash function is called to - wrap all programs in the `$out/bin/*` directory to include `$PATH` - environment variable and add dependent libraries to script's `sys.path`. -* In the `installCheck` phase, `${python.interpreter} setup.py test` is ran. - -As in Perl, dependencies on other Python packages can be specified in the -`buildInputs` and `propagatedBuildInputs` attributes. If something is -exclusively a build-time dependency, use `buildInputs`; if it’s (also) a runtime -dependency, use `propagatedBuildInputs`. - -By default tests are run because `doCheck = true`. Test dependencies, like -e.g. the test runner, should be added to `buildInputs`. - -By default `meta.platforms` is set to the same value -as the interpreter unless overridden otherwise. - -##### `buildPythonPackage` parameters - -All parameters from `mkDerivation` function are still supported. - -* `namePrefix`: Prepended text to `${name}` parameter. Defaults to `"python3.3-"` for Python 3.3, etc. Set it to `""` if you're packaging an application or a command line tool. -* `disabled`: If `true`, package is not build for particular python interpreter version. Grep around `pkgs/top-level/python-packages.nix` for examples. -* `setupPyBuildFlags`: List of flags passed to `setup.py build_ext` command. -* `pythonPath`: List of packages to be added into `$PYTHONPATH`. Packages in `pythonPath` are not propagated (contrary to `propagatedBuildInputs`). -* `preShellHook`: Hook to execute commands before `shellHook`. -* `postShellHook`: Hook to execute commands after `shellHook`. -* `makeWrapperArgs`: A list of strings. Arguments to be passed to `makeWrapper`, which wraps generated binaries. By default, the arguments to `makeWrapper` set `PATH` and `PYTHONPATH` environment variables before calling the binary. Additional arguments here can allow a developer to set environment variables which will be available when the binary is run. For example, `makeWrapperArgs = ["--set FOO BAR" "--set BAZ QUX"]`. -* `installFlags`: A list of strings. Arguments to be passed to `pip install`. To pass options to `python setup.py install`, use `--install-option`. E.g., `installFlags=["--install-option='--cpp_implementation'"]. -* `format`: Format of the source. Valid options are `setuptools` (default), `flit`, `wheel`, and `other`. `setuptools` is for when the source has a `setup.py` and `setuptools` is used to build a wheel, `flit`, in case `flit` should be used to build a wheel, and `wheel` in case a wheel is provided. In case you need to provide your own `buildPhase` and `installPhase` you can use `other`. -* `catchConflicts` If `true`, abort package build if a package name appears more than once in dependency tree. Default is `true`. -* `checkInputs` Dependencies needed for running the `checkPhase`. These are added to `buildInputs` when `doCheck = true`. - -##### Overriding Python packages - -The `buildPythonPackage` function has a `overridePythonAttrs` method that -can be used to override the package. In the following example we create an -environment where we have the `blaze` package using an older version of `pandas`. -We override first the Python interpreter and pass -`packageOverrides` which contains the overrides for packages in the package set. - -```nix -with import {}; - -(let - python = let - packageOverrides = self: super: { - pandas = super.pandas.overridePythonAttrs(old: rec { - version = "0.19.1"; - src = super.fetchPypi { - pname = "pandas"; - inherit version; - sha256 = "08blshqj9zj1wyjhhw3kl2vas75vhhicvv72flvf1z3jvapgw295"; - }; - }); - }; - in pkgs.python3.override {inherit packageOverrides;}; - -in python.withPackages(ps: [ps.blaze])).env -``` - -#### `buildPythonApplication` function - -The `buildPythonApplication` function is practically the same as `buildPythonPackage`. -The difference is that `buildPythonPackage` by default prefixes the names of the packages with the version of the interpreter. -Because with an application we're not interested in multiple version the prefix is dropped. - -#### python.buildEnv function - -Python environments can be created using the low-level `pkgs.buildEnv` function. -This example shows how to create an environment that has the Pyramid Web Framework. -Saving the following as `default.nix` -```nix -with import {}; - -python.buildEnv.override { - extraLibs = [ pkgs.pythonPackages.pyramid ]; - ignoreCollisions = true; -} -``` - -and running `nix-build` will create -``` -/nix/store/cf1xhjwzmdki7fasgr4kz6di72ykicl5-python-2.7.8-env -``` - -with wrapped binaries in `bin/`. - -You can also use the `env` attribute to create local environments with needed -packages installed. This is somewhat comparable to `virtualenv`. For example, -running `nix-shell` with the following `shell.nix` -```nix -with import {}; - -(python3.buildEnv.override { - extraLibs = with python3Packages; [ numpy requests ]; -}).env -``` - -will drop you into a shell where Python will have the -specified packages in its path. - - -##### `python.buildEnv` arguments - -* `extraLibs`: List of packages installed inside the environment. -* `postBuild`: Shell command executed after the build of environment. -* `ignoreCollisions`: Ignore file collisions inside the environment (default is `false`). - -#### python.withPackages function - -The `python.withPackages` function provides a simpler interface to the `python.buildEnv` functionality. -It takes a function as an argument that is passed the set of python packages and returns the list -of the packages to be included in the environment. Using the `withPackages` function, the previous -example for the Pyramid Web Framework environment can be written like this: -```nix -with import {}; - -python.withPackages (ps: [ps.pyramid]) -``` - -`withPackages` passes the correct package set for the specific interpreter version as an -argument to the function. In the above example, `ps` equals `pythonPackages`. -But you can also easily switch to using python3: -```nix -with import {}; - -python3.withPackages (ps: [ps.pyramid]) -``` - -Now, `ps` is set to `python3Packages`, matching the version of the interpreter. - -As `python.withPackages` simply uses `python.buildEnv` under the hood, it also supports the `env` -attribute. The `shell.nix` file from the previous section can thus be also written like this: -```nix -with import {}; - -(python36.withPackages (ps: [ps.numpy ps.requests])).env -``` - -In contrast to `python.buildEnv`, `python.withPackages` does not support the more advanced options -such as `ignoreCollisions = true` or `postBuild`. If you need them, you have to use `python.buildEnv`. - -Python 2 namespace packages may provide `__init__.py` that collide. In that case `python.buildEnv` -should be used with `ignoreCollisions = true`. - -### Development mode - -Development or editable mode is supported. To develop Python packages -`buildPythonPackage` has additional logic inside `shellPhase` to run `pip -install -e . --prefix $TMPDIR/`for the package. - -Warning: `shellPhase` is executed only if `setup.py` exists. - -Given a `default.nix`: -```nix -with import {}; - -buildPythonPackage { name = "myproject"; - -buildInputs = with pkgs.pythonPackages; [ pyramid ]; - -src = ./.; } -``` - -Running `nix-shell` with no arguments should give you -the environment in which the package would be built with -`nix-build`. - -Shortcut to setup environments with C headers/libraries and python packages: -```shell -nix-shell -p pythonPackages.pyramid zlib libjpeg git -``` - -Note: There is a boolean value `lib.inNixShell` set to `true` if nix-shell is invoked. - -### Tools - -Packages inside nixpkgs are written by hand. However many tools exist in -community to help save time. No tool is preferred at the moment. - -- [python2nix](https://github.com/proger/python2nix) by Vladimir Kirillov -- [pypi2nix](https://github.com/garbas/pypi2nix) by Rok Garbas -- [pypi2nix](https://github.com/offlinehacker/pypi2nix) by Jaka Hudoklin - -### Deterministic builds - -Python 2.7, 3.5 and 3.6 are now built deterministically and 3.4 mostly. -Minor modifications had to be made to the interpreters in order to generate -deterministic bytecode. This has security implications and is relevant for -those using Python in a `nix-shell`. - -When the environment variable `DETERMINISTIC_BUILD` is set, all bytecode will have timestamp 1. -The `buildPythonPackage` function sets `DETERMINISTIC_BUILD=1` and -[PYTHONHASHSEED=0](https://docs.python.org/3.5/using/cmdline.html#envvar-PYTHONHASHSEED). -Both are also exported in `nix-shell`. - - -### Automatic tests - -It is recommended to test packages as part of the build process. -Source distributions (`sdist`) often include test files, but not always. - -By default the command `python setup.py test` is run as part of the -`checkPhase`, but often it is necessary to pass a custom `checkPhase`. An -example of such a situation is when `py.test` is used. - -#### Common issues - -- Non-working tests can often be deselected. By default `buildPythonPackage` runs `python setup.py test`. - Most python modules follows the standard test protocol where the pytest runner can be used instead. - `py.test` supports a `-k` parameter to ignore test methods or classes: - - ```nix - buildPythonPackage { - # ... - # assumes the tests are located in tests - checkInputs = [ pytest ]; - checkPhase = '' - py.test -k 'not function_name and not other_function' tests - ''; - } - ``` -- Unicode issues can typically be fixed by including `glibcLocales` in `buildInputs` and exporting `LC_ALL=en_US.utf-8`. -- Tests that attempt to access `$HOME` can be fixed by using the following work-around before running tests (e.g. `preCheck`): `export HOME=$(mktemp -d)` - -## FAQ - -### How to solve circular dependencies? - -Consider the packages `A` and `B` that depend on each other. When packaging `B`, -a solution is to override package `A` not to depend on `B` as an input. The same -should also be done when packaging `A`. - -### How to override a Python package? - -We can override the interpreter and pass `packageOverrides`. -In the following example we rename the `pandas` package and build it. -```nix -with import {}; - -(let - python = let - packageOverrides = self: super: { - pandas = super.pandas.overridePythonAttrs(old: {name="foo";}); - }; - in pkgs.python35.override {inherit packageOverrides;}; - -in python.withPackages(ps: [ps.pandas])).env -``` -Using `nix-build` on this expression will build an environment that contains the -package `pandas` but with the new name `foo`. - -All packages in the package set will use the renamed package. -A typical use case is to switch to another version of a certain package. -For example, in the Nixpkgs repository we have multiple versions of `django` and `scipy`. -In the following example we use a different version of `scipy` and create an environment that uses it. -All packages in the Python package set will now use the updated `scipy` version. - -```nix -with import {}; - -( let - packageOverrides = self: super: { - scipy = super.scipy_0_17; - }; - in (pkgs.python35.override {inherit packageOverrides;}).withPackages (ps: [ps.blaze]) -).env -``` -The requested package `blaze` depends on `pandas` which itself depends on `scipy`. - -If you want the whole of Nixpkgs to use your modifications, then you can use `overlays` -as explained in this manual. In the following example we build a `inkscape` using a different version of `numpy`. -```nix -let - pkgs = import {}; - newpkgs = import pkgs.path { overlays = [ (pkgsself: pkgssuper: { - python27 = let - packageOverrides = self: super: { - numpy = super.numpy_1_10; - }; - in pkgssuper.python27.override {inherit packageOverrides;}; - } ) ]; }; -in newpkgs.inkscape -``` - -### `python setup.py bdist_wheel` cannot create .whl - -Executing `python setup.py bdist_wheel` in a `nix-shell `fails with -``` -ValueError: ZIP does not support timestamps before 1980 -``` - -This is because files from the Nix store (which have a timestamp of the UNIX epoch of January 1, 1970) are included in the .ZIP, but .ZIP archives follow the DOS convention of counting timestamps from 1980. - -The command `bdist_wheel` reads the `SOURCE_DATE_EPOCH` environment variable, which `nix-shell` sets to 1. Unsetting this variable or giving it a value corresponding to 1980 or later enables building wheels. - -Use 1980 as timestamp: -```shell -nix-shell --run "SOURCE_DATE_EPOCH=315532800 python3 setup.py bdist_wheel" -``` -or the current time: -```shell -nix-shell --run "SOURCE_DATE_EPOCH=$(date +%s) python3 setup.py bdist_wheel" -``` -or unset `SOURCE_DATE_EPOCH`: -```shell -nix-shell --run "unset SOURCE_DATE_EPOCH; python3 setup.py bdist_wheel" -``` - -### `install_data` / `data_files` problems - -If you get the following error: -``` -could not create '/nix/store/6l1bvljpy8gazlsw2aw9skwwp4pmvyxw-python-2.7.8/etc': -Permission denied -``` -This is a [known bug](https://github.com/pypa/setuptools/issues/130) in `setuptools`. -Setuptools `install_data` does not respect `--prefix`. An example of such package using the feature is `pkgs/tools/X11/xpra/default.nix`. -As workaround install it as an extra `preInstall` step: -```shell -${python.interpreter} setup.py install_data --install-dir=$out --root=$out -sed -i '/ = data\_files/d' setup.py -``` - -### Rationale of non-existent global site-packages - -On most operating systems a global `site-packages` is maintained. This however -becomes problematic if you want to run multiple Python versions or have multiple -versions of certain libraries for your projects. Generally, you would solve such -issues by creating virtual environments using `virtualenv`. - -On Nix each package has an isolated dependency tree which, in the case of -Python, guarantees the right versions of the interpreter and libraries or -packages are available. There is therefore no need to maintain a global `site-packages`. - -If you want to create a Python environment for development, then the recommended -method is to use `nix-shell`, either with or without the `python.buildEnv` -function. - -### How to consume python modules using pip in a virtualenv like I am used to on other Operating Systems ? - -This is an example of a `default.nix` for a `nix-shell`, which allows to consume a `virtualenv` environment, -and install python modules through `pip` the traditional way. - -Create this `default.nix` file, together with a `requirements.txt` and simply execute `nix-shell`. - -```nix -with import {}; -with pkgs.python27Packages; - -stdenv.mkDerivation { - name = "impurePythonEnv"; - buildInputs = [ - # these packages are required for virtualenv and pip to work: - # - python27Full - python27Packages.virtualenv - python27Packages.pip - # the following packages are related to the dependencies of your python - # project. - # In this particular example the python modules listed in the - # requirements.tx require the following packages to be installed locally - # in order to compile any binary extensions they may require. - # - taglib - openssl - git - libxml2 - libxslt - libzip - stdenv - zlib ]; - src = null; - shellHook = '' - # set SOURCE_DATE_EPOCH so that we can use python wheels - SOURCE_DATE_EPOCH=$(date +%s) - virtualenv --no-setuptools venv - export PATH=$PWD/venv/bin:$PATH - pip install -r requirements.txt - ''; -} -``` - -Note that the `pip install` is an imperative action. So every time `nix-shell` -is executed it will attempt to download the python modules listed in -requirements.txt. However these will be cached locally within the `virtualenv` -folder and not downloaded again. - -### How to override a Python package from `configuration.nix`? - -If you need to change a package's attribute(s) from `configuration.nix` you could do: - -```nix - nixpkgs.config.packageOverrides = superP: { - pythonPackages = superP.pythonPackages.override { - overrides = self: super: { - bepasty-server = super.bepasty-server.overrideAttrs ( oldAttrs: { - src = pkgs.fetchgit { - url = "https://github.com/bepasty/bepasty-server"; - sha256 = "9ziqshmsf0rjvdhhca55sm0x8jz76fsf2q4rwh4m6lpcf8wr0nps"; - rev = "e2516e8cf4f2afb5185337073607eb9e84a61d2d"; - }; - }); - }; - }; - }; -``` - -If you are using the `bepasty-server` package somewhere, for example in `systemPackages` or indirectly from `services.bepasty`, then a `nixos-rebuild switch` will rebuild the system but with the `bepasty-server` package using a different `src` attribute. This way one can modify `python` based software/libraries easily. Using `self` and `super` one can also alter dependencies (`buildInputs`) between the old state (`self`) and new state (`super`). - -### How to override a Python package using overlays? - -To alter a python package using overlays, you would use the following approach: - -```nix -self: super: -rec { - python = super.python.override { - packageOverrides = python-self: python-super: { - bepasty-server = python-super.bepasty-server.overrideAttrs ( oldAttrs: { - src = self.pkgs.fetchgit { - url = "https://github.com/bepasty/bepasty-server"; - sha256 = "9ziqshmsf0rjvdhhca55sm0x8jz76fsf2q4rwh4m6lpcf8wr0nps"; - rev = "e2516e8cf4f2afb5185337073607eb9e84a61d2d"; - }; - }); - }; - }; - pythonPackages = python.pkgs; -} -``` - -## Contributing - -### Contributing guidelines - -Following rules are desired to be respected: - -* Python libraries are called from `python-packages.nix` and packaged with `buildPythonPackage`. The expression of a library should be in `pkgs/development/python-modules//default.nix`. Libraries in `pkgs/top-level/python-packages.nix` are sorted quasi-alphabetically to avoid merge conflicts. -* Python applications live outside of `python-packages.nix` and are packaged with `buildPythonApplication`. -* Make sure libraries build for all Python interpreters. -* By default we enable tests. Make sure the tests are found and, in the case of libraries, are passing for all interpreters. If certain tests fail they can be disabled individually. Try to avoid disabling the tests altogether. In any case, when you disable tests, leave a comment explaining why. -* Commit names of Python libraries should reflect that they are Python libraries, so write for example `pythonPackages.numpy: 1.11 -> 1.12`. - diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md new file mode 100644 index 000000000000..0f871395854c --- /dev/null +++ b/doc/languages-frameworks/python.section.md @@ -0,0 +1,1026 @@ +# Python + +## User Guide + +### Using Python + +#### Overview + +Several versions of the Python interpreter are available on Nix, as well as a +high amount of packages. The attribute `python` refers to the default +interpreter, which is currently CPython 2.7. It is also possible to refer to +specific versions, e.g. `python35` refers to CPython 3.5, and `pypy` refers to +the default PyPy interpreter. + +Python is used a lot, and in different ways. This affects also how it is +packaged. In the case of Python on Nix, an important distinction is made between +whether the package is considered primarily an application, or whether it should +be used as a library, i.e., of primary interest are the modules in +`site-packages` that should be importable. + +In the Nixpkgs tree Python applications can be found throughout, depending on +what they do, and are called from the main package set. Python libraries, +however, are in separate sets, with one set per interpreter version. + +The interpreters have several common attributes. One of these attributes is +`pkgs`, which is a package set of Python libraries for this specific +interpreter. E.g., the `toolz` package corresponding to the default interpreter +is `python.pkgs.toolz`, and the CPython 3.5 version is `python35.pkgs.toolz`. +The main package set contains aliases to these package sets, e.g. +`pythonPackages` refers to `python.pkgs` and `python35Packages` to +`python35.pkgs`. + +#### Installing Python and packages + +The Nix and NixOS manuals explain how packages are generally installed. In the +case of Python and Nix, it is important to make a distinction between whether the +package is considered an application or a library. + +Applications on Nix are typically installed into your user +profile imperatively using `nix-env -i`, and on NixOS declaratively by adding the +package name to `environment.systemPackages` in `/etc/nixos/configuration.nix`. +Dependencies such as libraries are automatically installed and should not be +installed explicitly. + +The same goes for Python applications and libraries. Python applications can be +installed in your profile. But Python libraries you would like to use for +development cannot be installed, at least not individually, because they won't +be able to find each other resulting in import errors. Instead, it is possible +to create an environment with `python.buildEnv` or `python.withPackages` where +the interpreter and other executables are able to find each other and all of the +modules. + +In the following examples we create an environment with Python 3.5, `numpy` and +`toolz`. As you may imagine, there is one limitation here, and that's that +you can install only one environment at a time. You will notice the complaints +about collisions when you try to install a second environment. + +##### Environment defined in separate `.nix` file + +Create a file, e.g. `build.nix`, with the following expression +```nix +with import {}; + +python35.withPackages (ps: with ps; [ numpy toolz ]) +``` +and install it in your profile with +```shell +nix-env -if build.nix +``` +Now you can use the Python interpreter, as well as the extra packages (`numpy`, +`toolz`) that you added to the environment. + +##### Environment defined in `~/.config/nixpkgs/config.nix` + +If you prefer to, you could also add the environment as a package override to the Nixpkgs set, e.g. +using `config.nix`, +```nix +{ # ... + + packageOverrides = pkgs: with pkgs; { + myEnv = python35.withPackages (ps: with ps; [ numpy toolz ]); + }; +} +``` +and install it in your profile with +```shell +nix-env -iA nixpkgs.myEnv +``` +The environment is is installed by referring to the attribute, and considering +the `nixpkgs` channel was used. + +##### Environment defined in `/etc/nixos/configuration.nix` + +For the sake of completeness, here's another example how to install the environment system-wide. + +```nix +{ # ... + + environment.systemPackages = with pkgs; [ + (python35.withPackages(ps: with ps; [ numpy toolz ])) + ]; +} +``` + +#### Temporary Python environment with `nix-shell` + +The examples in the previous section showed how to install a Python environment +into a profile. For development you may need to use multiple environments. +`nix-shell` gives the possibility to temporarily load another environment, akin +to `virtualenv`. + +There are two methods for loading a shell with Python packages. The first and recommended method +is to create an environment with `python.buildEnv` or `python.withPackages` and load that. E.g. +```sh +$ nix-shell -p 'python35.withPackages(ps: with ps; [ numpy toolz ])' +``` +opens a shell from which you can launch the interpreter +```sh +[nix-shell:~] python3 +``` +The other method, which is not recommended, does not create an environment and requires you to list the packages directly, + +```sh +$ nix-shell -p python35.pkgs.numpy python35.pkgs.toolz +``` +Again, it is possible to launch the interpreter from the shell. +The Python interpreter has the attribute `pkgs` which contains all Python libraries for that specific interpreter. + +##### Load environment from `.nix` expression +As explained in the Nix manual, `nix-shell` can also load an +expression from a `.nix` file. Say we want to have Python 3.5, `numpy` +and `toolz`, like before, in an environment. Consider a `shell.nix` file +with +```nix +with import {}; + +(python35.withPackages (ps: [ps.numpy ps.toolz])).env +``` +Executing `nix-shell` gives you again a Nix shell from which you can run Python. + +What's happening here? + +1. We begin with importing the Nix Packages collections. `import ` imports the `` function, `{}` calls it and the `with` statement brings all attributes of `nixpkgs` in the local scope. These attributes form the main package set. +2. Then we create a Python 3.5 environment with the `withPackages` function. +3. The `withPackages` function expects us to provide a function as an argument that takes the set of all python packages and returns a list of packages to include in the environment. Here, we select the packages `numpy` and `toolz` from the package set. + +##### Execute command with `--run` +A convenient option with `nix-shell` is the `--run` +option, with which you can execute a command in the `nix-shell`. We can +e.g. directly open a Python shell +```sh +$ nix-shell -p python35Packages.numpy python35Packages.toolz --run "python3" +``` +or run a script +```sh +$ nix-shell -p python35Packages.numpy python35Packages.toolz --run "python3 myscript.py" +``` + +##### `nix-shell` as shebang +In fact, for the second use case, there is a more convenient method. You can +add a [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) to your script +specifying which dependencies `nix-shell` needs. With the following shebang, you +can just execute `./myscript.py`, and it will make available all dependencies and +run the script in the `python3` shell. + +```py +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p "python3.withPackages(ps: [ps.numpy])" + +import numpy + +print(numpy.__version__) +``` + +### Developing with Python + +Now that you know how to get a working Python environment with Nix, it is time +to go forward and start actually developing with Python. We will first have a +look at how Python packages are packaged on Nix. Then, we will look at how you +can use development mode with your code. + +#### Packaging a library + +With Nix all packages are built by functions. The main function in Nix for +building Python libraries is `buildPythonPackage`. Let's see how we can build the +`toolz` package. + +```nix +{ # ... + + toolz = buildPythonPackage rec { + pname = "toolz"; + version = "0.7.4"; + + src = fetchPypi { + inherit pname version; + sha256 = "43c2c9e5e7a16b6c88ba3088a9bfc82f7db8e13378be7c78d6c14a5f8ed05afd"; + }; + + doCheck = false; + + meta = { + homepage = "http://github.com/pytoolz/toolz/"; + description = "List processing tools and functional utilities"; + license = licenses.bsd3; + maintainers = with maintainers; [ fridh ]; + }; + }; +} +``` + +What happens here? The function `buildPythonPackage` is called and as argument +it accepts a set. In this case the set is a recursive set, `rec`. One of the +arguments is the name of the package, which consists of a basename (generally +following the name on PyPi) and a version. Another argument, `src` specifies the +source, which in this case is fetched from PyPI using the helper function +`fetchPypi`. The argument `doCheck` is used to set whether tests should be run +when building the package. Furthermore, we specify some (optional) meta +information. The output of the function is a derivation. + +An expression for `toolz` can be found in the Nixpkgs repository. As explained +in the introduction of this Python section, a derivation of `toolz` is available +for each interpreter version, e.g. `python35.pkgs.toolz` refers to the `toolz` +derivation corresponding to the CPython 3.5 interpreter. +The above example works when you're directly working on +`pkgs/top-level/python-packages.nix` in the Nixpkgs repository. Often though, +you will want to test a Nix expression outside of the Nixpkgs tree. + +The following expression creates a derivation for the `toolz` package, +and adds it along with a `numpy` package to a Python environment. + +```nix +with import {}; + +( let + my_toolz = python35.pkgs.buildPythonPackage rec { + pname = "toolz"; + version = "0.7.4"; + + src = python35.pkgs.fetchPypi { + inherit pname version; + sha256 = "43c2c9e5e7a16b6c88ba3088a9bfc82f7db8e13378be7c78d6c14a5f8ed05afd"; + }; + + doCheck = false; + + meta = { + homepage = "http://github.com/pytoolz/toolz/"; + description = "List processing tools and functional utilities"; + }; + }; + + in python35.withPackages (ps: [ps.numpy my_toolz]) +).env +``` +Executing `nix-shell` will result in an environment in which you can use +Python 3.5 and the `toolz` package. As you can see we had to explicitly mention +for which Python version we want to build a package. + +So, what did we do here? Well, we took the Nix expression that we used earlier +to build a Python environment, and said that we wanted to include our own +version of `toolz`, named `my_toolz`. To introduce our own package in the scope +of `withPackages` we used a `let` expression. You can see that we used +`ps.numpy` to select numpy from the nixpkgs package set (`ps`). We did not take +`toolz` from the Nixpkgs package set this time, but instead took our own version +that we introduced with the `let` expression. + +#### Handling dependencies + +Our example, `toolz`, does not have any dependencies on other Python +packages or system libraries. According to the manual, `buildPythonPackage` +uses the arguments `buildInputs` and `propagatedBuildInputs` to specify dependencies. If something is +exclusively a build-time dependency, then the dependency should be included as a +`buildInput`, but if it is (also) a runtime dependency, then it should be added +to `propagatedBuildInputs`. Test dependencies are considered build-time dependencies. + +The following example shows which arguments are given to `buildPythonPackage` in +order to build [`datashape`](https://github.com/blaze/datashape). + +```nix +{ # ... + + datashape = buildPythonPackage rec { + pname = "datashape"; + version = "0.4.7"; + + src = fetchPypi { + inherit pname version; + sha256 = "14b2ef766d4c9652ab813182e866f493475e65e558bed0822e38bf07bba1a278"; + }; + + checkInputs = with self; [ pytest ]; + propagatedBuildInputs = with self; [ numpy multipledispatch dateutil ]; + + meta = { + homepage = https://github.com/ContinuumIO/datashape; + description = "A data description language"; + license = licenses.bsd2; + maintainers = with maintainers; [ fridh ]; + }; + }; +} +``` + +We can see several runtime dependencies, `numpy`, `multipledispatch`, and +`dateutil`. Furthermore, we have one `buildInput`, i.e. `pytest`. `pytest` is a +test runner and is only used during the `checkPhase` and is therefore not added +to `propagatedBuildInputs`. + +In the previous case we had only dependencies on other Python packages to consider. +Occasionally you have also system libraries to consider. E.g., `lxml` provides +Python bindings to `libxml2` and `libxslt`. These libraries are only required +when building the bindings and are therefore added as `buildInputs`. + +```nix +{ # ... + + lxml = buildPythonPackage rec { + pname = "lxml"; + version = "3.4.4"; + + src = fetchPypi { + inherit pname version; + sha256 = "16a0fa97hym9ysdk3rmqz32xdjqmy4w34ld3rm3jf5viqjx65lxk"; + }; + + buildInputs = with self; [ pkgs.libxml2 pkgs.libxslt ]; + + meta = { + description = "Pythonic binding for the libxml2 and libxslt libraries"; + homepage = http://lxml.de; + license = licenses.bsd3; + maintainers = with maintainers; [ sjourdois ]; + }; + }; +} +``` + +In this example `lxml` and Nix are able to work out exactly where the relevant +files of the dependencies are. This is not always the case. + +The example below shows bindings to The Fastest Fourier Transform in the West, commonly known as +FFTW. On Nix we have separate packages of FFTW for the different types of floats +(`"single"`, `"double"`, `"long-double"`). The bindings need all three types, +and therefore we add all three as `buildInputs`. The bindings don't expect to +find each of them in a different folder, and therefore we have to set `LDFLAGS` +and `CFLAGS`. + +```nix +{ # ... + + pyfftw = buildPythonPackage rec { + pname = "pyFFTW"; + version = "0.9.2"; + + src = fetchPypi { + inherit pname version; + sha256 = "f6bbb6afa93085409ab24885a1a3cdb8909f095a142f4d49e346f2bd1b789074"; + }; + + buildInputs = [ pkgs.fftw pkgs.fftwFloat pkgs.fftwLongDouble]; + + propagatedBuildInputs = with self; [ numpy scipy ]; + + # Tests cannot import pyfftw. pyfftw works fine though. + doCheck = false; + + preConfigure = '' + export LDFLAGS="-L${pkgs.fftw.dev}/lib -L${pkgs.fftwFloat.out}/lib -L${pkgs.fftwLongDouble.out}/lib" + export CFLAGS="-I${pkgs.fftw.dev}/include -I${pkgs.fftwFloat.dev}/include -I${pkgs.fftwLongDouble.dev}/include" + ''; + + meta = { + description = "A pythonic wrapper around FFTW, the FFT library, presenting a unified interface for all the supported transforms"; + homepage = http://hgomersall.github.com/pyFFTW/; + license = with licenses; [ bsd2 bsd3 ]; + maintainer = with maintainers; [ fridh ]; + }; + }; +} +``` +Note also the line `doCheck = false;`, we explicitly disabled running the test-suite. + + +#### Develop local package + +As a Python developer you're likely aware of [development mode](http://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode) (`python setup.py develop`); +instead of installing the package this command creates a special link to the project code. +That way, you can run updated code without having to reinstall after each and every change you make. +Development mode is also available. Let's see how you can use it. + +In the previous Nix expression the source was fetched from an url. We can also refer to a local source instead using +`src = ./path/to/source/tree;` + +If we create a `shell.nix` file which calls `buildPythonPackage`, and if `src` +is a local source, and if the local source has a `setup.py`, then development +mode is activated. + +In the following example we create a simple environment that +has a Python 3.5 version of our package in it, as well as its dependencies and +other packages we like to have in the environment, all specified with `propagatedBuildInputs`. +Indeed, we can just add any package we like to have in our environment to `propagatedBuildInputs`. + +```nix +with import {}; +with pkgs.python35Packages; + +buildPythonPackage rec { + name = "mypackage"; + src = ./path/to/package/source; + propagatedBuildInputs = [ pytest numpy pkgs.libsndfile ]; +} +``` + +It is important to note that due to how development mode is implemented on Nix it is not possible to have multiple packages simultaneously in development mode. + + +### Organising your packages + +So far we discussed how you can use Python on Nix, and how you can develop with +it. We've looked at how you write expressions to package Python packages, and we +looked at how you can create environments in which specified packages are +available. + +At some point you'll likely have multiple packages which you would +like to be able to use in different projects. In order to minimise unnecessary +duplication we now look at how you can maintain yourself a repository with your +own packages. The important functions here are `import` and `callPackage`. + +### Including a derivation using `callPackage` + +Earlier we created a Python environment using `withPackages`, and included the +`toolz` package via a `let` expression. +Let's split the package definition from the environment definition. + +We first create a function that builds `toolz` in `~/path/to/toolz/release.nix` + +```nix +{ pkgs, buildPythonPackage }: + +buildPythonPackage rec { + pname = "toolz"; + version = "0.7.4"; + + src = fetchPypi { + inherit pname version; + sha256 = "43c2c9e5e7a16b6c88ba3088a9bfc82f7db8e13378be7c78d6c14a5f8ed05afd"; + }; + + meta = { + homepage = "http://github.com/pytoolz/toolz/"; + description = "List processing tools and functional utilities"; + license = licenses.bsd3; + maintainers = with maintainers; [ fridh ]; + }; +} +``` + +It takes two arguments, `pkgs` and `buildPythonPackage`. +We now call this function using `callPackage` in the definition of our environment + +```nix +with import {}; + +( let + toolz = pkgs.callPackage /path/to/toolz/release.nix { + pkgs = pkgs; + buildPythonPackage = pkgs.python35Packages.buildPythonPackage; + }; + in pkgs.python35.withPackages (ps: [ ps.numpy toolz ]) +).env +``` + +Important to remember is that the Python version for which the package is made +depends on the `python` derivation that is passed to `buildPythonPackage`. Nix +tries to automatically pass arguments when possible, which is why generally you +don't explicitly define which `python` derivation should be used. In the above +example we use `buildPythonPackage` that is part of the set `python35Packages`, +and in this case the `python35` interpreter is automatically used. + + + +## Reference + +### Interpreters + +Versions 2.7, 3.3, 3.4, 3.5 and 3.6 of the CPython interpreter are available as +respectively `python27`, `python34`, `python35` and `python36`. The PyPy interpreter +is available as `pypy`. The aliases `python2` and `python3` correspond to respectively `python27` and +`python35`. The default interpreter, `python`, maps to `python2`. +The Nix expressions for the interpreters can be found in +`pkgs/development/interpreters/python`. + +All packages depending on any Python interpreter get appended +`out/{python.sitePackages}` to `$PYTHONPATH` if such directory +exists. + +#### Missing `tkinter` module standard library + +To reduce closure size the `Tkinter`/`tkinter` is available as a separate package, `pythonPackages.tkinter`. + +#### Attributes on interpreters packages + +Each interpreter has the following attributes: + +- `libPrefix`. Name of the folder in `${python}/lib/` for corresponding interpreter. +- `interpreter`. Alias for `${python}/bin/${executable}`. +- `buildEnv`. Function to build python interpreter environments with extra packages bundled together. See section *python.buildEnv function* for usage and documentation. +- `withPackages`. Simpler interface to `buildEnv`. See section *python.withPackages function* for usage and documentation. +- `sitePackages`. Alias for `lib/${libPrefix}/site-packages`. +- `executable`. Name of the interpreter executable, e.g. `python3.4`. +- `pkgs`. Set of Python packages for that specific interpreter. The package set can be modified by overriding the interpreter and passing `packageOverrides`. + +### Building packages and applications + +Python libraries and applications that use `setuptools` or +`distutils` are typically build with respectively the `buildPythonPackage` and +`buildPythonApplication` functions. These two functions also support installing a `wheel`. + +All Python packages reside in `pkgs/top-level/python-packages.nix` and all +applications elsewhere. In case a package is used as both a library and an application, +then the package should be in `pkgs/top-level/python-packages.nix` since only those packages are made +available for all interpreter versions. The preferred location for library expressions is in +`pkgs/development/python-modules`. It is important that these packages are +called from `pkgs/top-level/python-packages.nix` and not elsewhere, to guarantee +the right version of the package is built. + +Based on the packages defined in `pkgs/top-level/python-packages.nix` an +attribute set is created for each available Python interpreter. The available +sets are + +* `pkgs.python27Packages` +* `pkgs.python34Packages` +* `pkgs.python35Packages` +* `pkgs.python36Packages` +* `pkgs.pypyPackages` + +and the aliases + +* `pkgs.python2Packages` pointing to `pkgs.python27Packages` +* `pkgs.python3Packages` pointing to `pkgs.python36Packages` +* `pkgs.pythonPackages` pointing to `pkgs.python2Packages` + +#### `buildPythonPackage` function + +The `buildPythonPackage` function is implemented in +`pkgs/development/interpreters/python/build-python-package.nix` + +The following is an example: +```nix + +buildPythonPackage rec { + version = "3.3.1"; + pname = "pytest"; + + preCheck = '' + # don't test bash builtins + rm testing/test_argcomplete.py + ''; + + src = fetchPypi { + inherit pname version; + sha256 = "cf8436dc59d8695346fcd3ab296de46425ecab00d64096cebe79fb51ecb2eb93"; + }; + + checkInputs = [ hypothesis ]; + buildInputs = [ setuptools_scm ]; + propagatedBuildInputs = [ attrs py setuptools six pluggy ]; + + meta = with stdenv.lib; { + maintainers = with maintainers; [ domenkozar lovek323 madjar lsix ]; + description = "Framework for writing tests"; + }; +} + +``` + +The `buildPythonPackage` mainly does four things: + +* In the `buildPhase`, it calls `${python.interpreter} setup.py bdist_wheel` to + build a wheel binary zipfile. +* In the `installPhase`, it installs the wheel file using `pip install *.whl`. +* In the `postFixup` phase, the `wrapPythonPrograms` bash function is called to + wrap all programs in the `$out/bin/*` directory to include `$PATH` + environment variable and add dependent libraries to script's `sys.path`. +* In the `installCheck` phase, `${python.interpreter} setup.py test` is ran. + +As in Perl, dependencies on other Python packages can be specified in the +`buildInputs` and `propagatedBuildInputs` attributes. If something is +exclusively a build-time dependency, use `buildInputs`; if it’s (also) a runtime +dependency, use `propagatedBuildInputs`. + +By default tests are run because `doCheck = true`. Test dependencies, like +e.g. the test runner, should be added to `buildInputs`. + +By default `meta.platforms` is set to the same value +as the interpreter unless overridden otherwise. + +##### `buildPythonPackage` parameters + +All parameters from `mkDerivation` function are still supported. + +* `namePrefix`: Prepended text to `${name}` parameter. Defaults to `"python3.3-"` for Python 3.3, etc. Set it to `""` if you're packaging an application or a command line tool. +* `disabled`: If `true`, package is not build for particular python interpreter version. Grep around `pkgs/top-level/python-packages.nix` for examples. +* `setupPyBuildFlags`: List of flags passed to `setup.py build_ext` command. +* `pythonPath`: List of packages to be added into `$PYTHONPATH`. Packages in `pythonPath` are not propagated (contrary to `propagatedBuildInputs`). +* `preShellHook`: Hook to execute commands before `shellHook`. +* `postShellHook`: Hook to execute commands after `shellHook`. +* `makeWrapperArgs`: A list of strings. Arguments to be passed to `makeWrapper`, which wraps generated binaries. By default, the arguments to `makeWrapper` set `PATH` and `PYTHONPATH` environment variables before calling the binary. Additional arguments here can allow a developer to set environment variables which will be available when the binary is run. For example, `makeWrapperArgs = ["--set FOO BAR" "--set BAZ QUX"]`. +* `installFlags`: A list of strings. Arguments to be passed to `pip install`. To pass options to `python setup.py install`, use `--install-option`. E.g., `installFlags=["--install-option='--cpp_implementation'"]. +* `format`: Format of the source. Valid options are `setuptools` (default), `flit`, `wheel`, and `other`. `setuptools` is for when the source has a `setup.py` and `setuptools` is used to build a wheel, `flit`, in case `flit` should be used to build a wheel, and `wheel` in case a wheel is provided. In case you need to provide your own `buildPhase` and `installPhase` you can use `other`. +* `catchConflicts` If `true`, abort package build if a package name appears more than once in dependency tree. Default is `true`. +* `checkInputs` Dependencies needed for running the `checkPhase`. These are added to `buildInputs` when `doCheck = true`. + +##### Overriding Python packages + +The `buildPythonPackage` function has a `overridePythonAttrs` method that +can be used to override the package. In the following example we create an +environment where we have the `blaze` package using an older version of `pandas`. +We override first the Python interpreter and pass +`packageOverrides` which contains the overrides for packages in the package set. + +```nix +with import {}; + +(let + python = let + packageOverrides = self: super: { + pandas = super.pandas.overridePythonAttrs(old: rec { + version = "0.19.1"; + src = super.fetchPypi { + pname = "pandas"; + inherit version; + sha256 = "08blshqj9zj1wyjhhw3kl2vas75vhhicvv72flvf1z3jvapgw295"; + }; + }); + }; + in pkgs.python3.override {inherit packageOverrides;}; + +in python.withPackages(ps: [ps.blaze])).env +``` + +#### `buildPythonApplication` function + +The `buildPythonApplication` function is practically the same as `buildPythonPackage`. +The difference is that `buildPythonPackage` by default prefixes the names of the packages with the version of the interpreter. +Because with an application we're not interested in multiple version the prefix is dropped. + +#### python.buildEnv function + +Python environments can be created using the low-level `pkgs.buildEnv` function. +This example shows how to create an environment that has the Pyramid Web Framework. +Saving the following as `default.nix` +```nix +with import {}; + +python.buildEnv.override { + extraLibs = [ pkgs.pythonPackages.pyramid ]; + ignoreCollisions = true; +} +``` + +and running `nix-build` will create +``` +/nix/store/cf1xhjwzmdki7fasgr4kz6di72ykicl5-python-2.7.8-env +``` + +with wrapped binaries in `bin/`. + +You can also use the `env` attribute to create local environments with needed +packages installed. This is somewhat comparable to `virtualenv`. For example, +running `nix-shell` with the following `shell.nix` +```nix +with import {}; + +(python3.buildEnv.override { + extraLibs = with python3Packages; [ numpy requests ]; +}).env +``` + +will drop you into a shell where Python will have the +specified packages in its path. + + +##### `python.buildEnv` arguments + +* `extraLibs`: List of packages installed inside the environment. +* `postBuild`: Shell command executed after the build of environment. +* `ignoreCollisions`: Ignore file collisions inside the environment (default is `false`). + +#### python.withPackages function + +The `python.withPackages` function provides a simpler interface to the `python.buildEnv` functionality. +It takes a function as an argument that is passed the set of python packages and returns the list +of the packages to be included in the environment. Using the `withPackages` function, the previous +example for the Pyramid Web Framework environment can be written like this: +```nix +with import {}; + +python.withPackages (ps: [ps.pyramid]) +``` + +`withPackages` passes the correct package set for the specific interpreter version as an +argument to the function. In the above example, `ps` equals `pythonPackages`. +But you can also easily switch to using python3: +```nix +with import {}; + +python3.withPackages (ps: [ps.pyramid]) +``` + +Now, `ps` is set to `python3Packages`, matching the version of the interpreter. + +As `python.withPackages` simply uses `python.buildEnv` under the hood, it also supports the `env` +attribute. The `shell.nix` file from the previous section can thus be also written like this: +```nix +with import {}; + +(python36.withPackages (ps: [ps.numpy ps.requests])).env +``` + +In contrast to `python.buildEnv`, `python.withPackages` does not support the more advanced options +such as `ignoreCollisions = true` or `postBuild`. If you need them, you have to use `python.buildEnv`. + +Python 2 namespace packages may provide `__init__.py` that collide. In that case `python.buildEnv` +should be used with `ignoreCollisions = true`. + +### Development mode + +Development or editable mode is supported. To develop Python packages +`buildPythonPackage` has additional logic inside `shellPhase` to run `pip +install -e . --prefix $TMPDIR/`for the package. + +Warning: `shellPhase` is executed only if `setup.py` exists. + +Given a `default.nix`: +```nix +with import {}; + +buildPythonPackage { name = "myproject"; + +buildInputs = with pkgs.pythonPackages; [ pyramid ]; + +src = ./.; } +``` + +Running `nix-shell` with no arguments should give you +the environment in which the package would be built with +`nix-build`. + +Shortcut to setup environments with C headers/libraries and python packages: +```shell +nix-shell -p pythonPackages.pyramid zlib libjpeg git +``` + +Note: There is a boolean value `lib.inNixShell` set to `true` if nix-shell is invoked. + +### Tools + +Packages inside nixpkgs are written by hand. However many tools exist in +community to help save time. No tool is preferred at the moment. + +- [python2nix](https://github.com/proger/python2nix) by Vladimir Kirillov +- [pypi2nix](https://github.com/garbas/pypi2nix) by Rok Garbas +- [pypi2nix](https://github.com/offlinehacker/pypi2nix) by Jaka Hudoklin + +### Deterministic builds + +Python 2.7, 3.5 and 3.6 are now built deterministically and 3.4 mostly. +Minor modifications had to be made to the interpreters in order to generate +deterministic bytecode. This has security implications and is relevant for +those using Python in a `nix-shell`. + +When the environment variable `DETERMINISTIC_BUILD` is set, all bytecode will have timestamp 1. +The `buildPythonPackage` function sets `DETERMINISTIC_BUILD=1` and +[PYTHONHASHSEED=0](https://docs.python.org/3.5/using/cmdline.html#envvar-PYTHONHASHSEED). +Both are also exported in `nix-shell`. + + +### Automatic tests + +It is recommended to test packages as part of the build process. +Source distributions (`sdist`) often include test files, but not always. + +By default the command `python setup.py test` is run as part of the +`checkPhase`, but often it is necessary to pass a custom `checkPhase`. An +example of such a situation is when `py.test` is used. + +#### Common issues + +- Non-working tests can often be deselected. By default `buildPythonPackage` runs `python setup.py test`. + Most python modules follows the standard test protocol where the pytest runner can be used instead. + `py.test` supports a `-k` parameter to ignore test methods or classes: + + ```nix + buildPythonPackage { + # ... + # assumes the tests are located in tests + checkInputs = [ pytest ]; + checkPhase = '' + py.test -k 'not function_name and not other_function' tests + ''; + } + ``` +- Unicode issues can typically be fixed by including `glibcLocales` in `buildInputs` and exporting `LC_ALL=en_US.utf-8`. +- Tests that attempt to access `$HOME` can be fixed by using the following work-around before running tests (e.g. `preCheck`): `export HOME=$(mktemp -d)` + +## FAQ + +### How to solve circular dependencies? + +Consider the packages `A` and `B` that depend on each other. When packaging `B`, +a solution is to override package `A` not to depend on `B` as an input. The same +should also be done when packaging `A`. + +### How to override a Python package? + +We can override the interpreter and pass `packageOverrides`. +In the following example we rename the `pandas` package and build it. +```nix +with import {}; + +(let + python = let + packageOverrides = self: super: { + pandas = super.pandas.overridePythonAttrs(old: {name="foo";}); + }; + in pkgs.python35.override {inherit packageOverrides;}; + +in python.withPackages(ps: [ps.pandas])).env +``` +Using `nix-build` on this expression will build an environment that contains the +package `pandas` but with the new name `foo`. + +All packages in the package set will use the renamed package. +A typical use case is to switch to another version of a certain package. +For example, in the Nixpkgs repository we have multiple versions of `django` and `scipy`. +In the following example we use a different version of `scipy` and create an environment that uses it. +All packages in the Python package set will now use the updated `scipy` version. + +```nix +with import {}; + +( let + packageOverrides = self: super: { + scipy = super.scipy_0_17; + }; + in (pkgs.python35.override {inherit packageOverrides;}).withPackages (ps: [ps.blaze]) +).env +``` +The requested package `blaze` depends on `pandas` which itself depends on `scipy`. + +If you want the whole of Nixpkgs to use your modifications, then you can use `overlays` +as explained in this manual. In the following example we build a `inkscape` using a different version of `numpy`. +```nix +let + pkgs = import {}; + newpkgs = import pkgs.path { overlays = [ (pkgsself: pkgssuper: { + python27 = let + packageOverrides = self: super: { + numpy = super.numpy_1_10; + }; + in pkgssuper.python27.override {inherit packageOverrides;}; + } ) ]; }; +in newpkgs.inkscape +``` + +### `python setup.py bdist_wheel` cannot create .whl + +Executing `python setup.py bdist_wheel` in a `nix-shell `fails with +``` +ValueError: ZIP does not support timestamps before 1980 +``` + +This is because files from the Nix store (which have a timestamp of the UNIX epoch of January 1, 1970) are included in the .ZIP, but .ZIP archives follow the DOS convention of counting timestamps from 1980. + +The command `bdist_wheel` reads the `SOURCE_DATE_EPOCH` environment variable, which `nix-shell` sets to 1. Unsetting this variable or giving it a value corresponding to 1980 or later enables building wheels. + +Use 1980 as timestamp: +```shell +nix-shell --run "SOURCE_DATE_EPOCH=315532800 python3 setup.py bdist_wheel" +``` +or the current time: +```shell +nix-shell --run "SOURCE_DATE_EPOCH=$(date +%s) python3 setup.py bdist_wheel" +``` +or unset `SOURCE_DATE_EPOCH`: +```shell +nix-shell --run "unset SOURCE_DATE_EPOCH; python3 setup.py bdist_wheel" +``` + +### `install_data` / `data_files` problems + +If you get the following error: +``` +could not create '/nix/store/6l1bvljpy8gazlsw2aw9skwwp4pmvyxw-python-2.7.8/etc': +Permission denied +``` +This is a [known bug](https://github.com/pypa/setuptools/issues/130) in `setuptools`. +Setuptools `install_data` does not respect `--prefix`. An example of such package using the feature is `pkgs/tools/X11/xpra/default.nix`. +As workaround install it as an extra `preInstall` step: +```shell +${python.interpreter} setup.py install_data --install-dir=$out --root=$out +sed -i '/ = data\_files/d' setup.py +``` + +### Rationale of non-existent global site-packages + +On most operating systems a global `site-packages` is maintained. This however +becomes problematic if you want to run multiple Python versions or have multiple +versions of certain libraries for your projects. Generally, you would solve such +issues by creating virtual environments using `virtualenv`. + +On Nix each package has an isolated dependency tree which, in the case of +Python, guarantees the right versions of the interpreter and libraries or +packages are available. There is therefore no need to maintain a global `site-packages`. + +If you want to create a Python environment for development, then the recommended +method is to use `nix-shell`, either with or without the `python.buildEnv` +function. + +### How to consume python modules using pip in a virtualenv like I am used to on other Operating Systems ? + +This is an example of a `default.nix` for a `nix-shell`, which allows to consume a `virtualenv` environment, +and install python modules through `pip` the traditional way. + +Create this `default.nix` file, together with a `requirements.txt` and simply execute `nix-shell`. + +```nix +with import {}; +with pkgs.python27Packages; + +stdenv.mkDerivation { + name = "impurePythonEnv"; + buildInputs = [ + # these packages are required for virtualenv and pip to work: + # + python27Full + python27Packages.virtualenv + python27Packages.pip + # the following packages are related to the dependencies of your python + # project. + # In this particular example the python modules listed in the + # requirements.tx require the following packages to be installed locally + # in order to compile any binary extensions they may require. + # + taglib + openssl + git + libxml2 + libxslt + libzip + stdenv + zlib ]; + src = null; + shellHook = '' + # set SOURCE_DATE_EPOCH so that we can use python wheels + SOURCE_DATE_EPOCH=$(date +%s) + virtualenv --no-setuptools venv + export PATH=$PWD/venv/bin:$PATH + pip install -r requirements.txt + ''; +} +``` + +Note that the `pip install` is an imperative action. So every time `nix-shell` +is executed it will attempt to download the python modules listed in +requirements.txt. However these will be cached locally within the `virtualenv` +folder and not downloaded again. + +### How to override a Python package from `configuration.nix`? + +If you need to change a package's attribute(s) from `configuration.nix` you could do: + +```nix + nixpkgs.config.packageOverrides = superP: { + pythonPackages = superP.pythonPackages.override { + overrides = self: super: { + bepasty-server = super.bepasty-server.overrideAttrs ( oldAttrs: { + src = pkgs.fetchgit { + url = "https://github.com/bepasty/bepasty-server"; + sha256 = "9ziqshmsf0rjvdhhca55sm0x8jz76fsf2q4rwh4m6lpcf8wr0nps"; + rev = "e2516e8cf4f2afb5185337073607eb9e84a61d2d"; + }; + }); + }; + }; + }; +``` + +If you are using the `bepasty-server` package somewhere, for example in `systemPackages` or indirectly from `services.bepasty`, then a `nixos-rebuild switch` will rebuild the system but with the `bepasty-server` package using a different `src` attribute. This way one can modify `python` based software/libraries easily. Using `self` and `super` one can also alter dependencies (`buildInputs`) between the old state (`self`) and new state (`super`). + +### How to override a Python package using overlays? + +To alter a python package using overlays, you would use the following approach: + +```nix +self: super: +rec { + python = super.python.override { + packageOverrides = python-self: python-super: { + bepasty-server = python-super.bepasty-server.overrideAttrs ( oldAttrs: { + src = self.pkgs.fetchgit { + url = "https://github.com/bepasty/bepasty-server"; + sha256 = "9ziqshmsf0rjvdhhca55sm0x8jz76fsf2q4rwh4m6lpcf8wr0nps"; + rev = "e2516e8cf4f2afb5185337073607eb9e84a61d2d"; + }; + }); + }; + }; + pythonPackages = python.pkgs; +} +``` + +## Contributing + +### Contributing guidelines + +Following rules are desired to be respected: + +* Python libraries are called from `python-packages.nix` and packaged with `buildPythonPackage`. The expression of a library should be in `pkgs/development/python-modules//default.nix`. Libraries in `pkgs/top-level/python-packages.nix` are sorted quasi-alphabetically to avoid merge conflicts. +* Python applications live outside of `python-packages.nix` and are packaged with `buildPythonApplication`. +* Make sure libraries build for all Python interpreters. +* By default we enable tests. Make sure the tests are found and, in the case of libraries, are passing for all interpreters. If certain tests fail they can be disabled individually. Try to avoid disabling the tests altogether. In any case, when you disable tests, leave a comment explaining why. +* Commit names of Python libraries should reflect that they are Python libraries, so write for example `pythonPackages.numpy: 1.11 -> 1.12`. + diff --git a/doc/languages-frameworks/r.section.md b/doc/languages-frameworks/r.section.md new file mode 100644 index 000000000000..c8f02bd14780 --- /dev/null +++ b/doc/languages-frameworks/r.section.md @@ -0,0 +1,120 @@ +R packages +========== + +## Installation + +Define an environment for R that contains all the libraries that you'd like to +use by adding the following snippet to your $HOME/.config/nixpkgs/config.nix file: + +```nix +{ + packageOverrides = super: let self = super.pkgs; in + { + + rEnv = super.rWrapper.override { + packages = with self.rPackages; [ + devtools + ggplot2 + reshape2 + yaml + optparse + ]; + }; + }; +} +``` + +Then you can use `nix-env -f "" -iA rEnv` to install it into your user +profile. The set of available libraries can be discovered by running the +command `nix-env -f "" -qaP -A rPackages`. The first column from that +output is the name that has to be passed to rWrapper in the code snipped above. + +However, if you'd like to add a file to your project source to make the +environment available for other contributors, you can create a `default.nix` +file like so: +```nix +let + pkgs = import {}; + stdenv = pkgs.stdenv; +in with pkgs; { + myProject = stdenv.mkDerivation { + name = "myProject"; + version = "1"; + src = if pkgs.lib.inNixShell then null else nix; + + buildInputs = with rPackages; [ + R + ggplot2 + knitr + ]; + }; +} +``` +and then run `nix-shell .` to be dropped into a shell with those packages +available. + +## RStudio + +RStudio uses a standard set of packages and ignores any custom R +environments or installed packages you may have. To create a custom +environment, see `rstudioWrapper`, which functions similarly to +`rWrapper`: + +```nix +{ + packageOverrides = super: let self = super.pkgs; in + { + + rstudioEnv = super.rstudioWrapper.override { + packages = with self.rPackages; [ + dplyr + ggplot2 + reshape2 + ]; + }; + }; +} +``` + +Then like above, `nix-env -f "" -iA rstudioEnv` will install +this into your user profile. + +Alternatively, you can create a self-contained `shell.nix` without the need to +modify any configuration files: + +```nix +{ pkgs ? import {} +}: + +pkgs.rstudioWrapper.override { + packages = with pkgs.rPackages; [ dplyr ggplot2 reshape2 ]; +} + +``` + +Executing `nix-shell` will then drop you into an environment equivalent to the +one above. If you need additional packages just add them to the list and +re-enter the shell. + +## Updating the package set + +```bash +nix-shell generate-shell.nix + +Rscript generate-r-packages.R cran > cran-packages.nix.new +mv cran-packages.nix.new cran-packages.nix + +Rscript generate-r-packages.R bioc > bioc-packages.nix.new +mv bioc-packages.nix.new bioc-packages.nix +``` + +`generate-r-packages.R ` reads `-packages.nix`, therefor the renaming. + + +## Testing if the Nix-expression could be evaluated + +```bash +nix-build test-evaluation.nix --dry-run +``` + +If this exits fine, the expression is ok. If not, you have to edit `default.nix` diff --git a/doc/languages-frameworks/rust.md b/doc/languages-frameworks/rust.md deleted file mode 100644 index f0498eac5b19..000000000000 --- a/doc/languages-frameworks/rust.md +++ /dev/null @@ -1,309 +0,0 @@ ---- -title: Rust -author: Matthias Beyer -date: 2017-03-05 ---- - -# User's Guide to the Rust Infrastructure - -To install the rust compiler and cargo put - -``` -rustc -cargo -``` - -into the `environment.systemPackages` or bring them into -scope with `nix-shell -p rustc cargo`. - -For daily builds (beta and nightly) use either rustup from -nixpkgs or use the [Rust nightlies -overlay](#using-the-rust-nightlies-overlay). - -## Compiling Rust applications with Cargo - -Rust applications are packaged by using the `buildRustPackage` helper from `rustPlatform`: - -``` -rustPlatform.buildRustPackage rec { - name = "ripgrep-${version}"; - version = "0.4.0"; - - src = fetchFromGitHub { - owner = "BurntSushi"; - repo = "ripgrep"; - rev = "${version}"; - sha256 = "0y5d1n6hkw85jb3rblcxqas2fp82h3nghssa4xqrhqnz25l799pj"; - }; - - cargoSha256 = "0q68qyl2h6i0qsz82z840myxlnjay8p1w5z7hfyr8fqp7wgwa9cx"; - - meta = with stdenv.lib; { - description = "A fast line-oriented regex search tool, similar to ag and ack"; - homepage = https://github.com/BurntSushi/ripgrep; - license = licenses.unlicense; - maintainers = [ maintainers.tailhook ]; - platforms = platforms.all; - }; -} -``` - -`buildRustPackage` requires a `cargoSha256` attribute which is computed over -all crate sources of this package. Currently it is obtained by inserting a -fake checksum into the expression and building the package once. The correct -checksum can be then take from the failed build. - -To install crates with nix there is also an experimental project called -[nixcrates](https://github.com/fractalide/nixcrates). - -## Compiling Rust crates using Nix instead of Cargo - -### Simple operation - -When run, `cargo build` produces a file called `Cargo.lock`, -containing pinned versions of all dependencies. Nixpkgs contains a -tool called `carnix` (`nix-env -iA nixos.carnix`), which can be used -to turn a `Cargo.lock` into a Nix expression. - -That Nix expression calls `rustc` directly (hence bypassing Cargo), -and can be used to compile a crate and all its dependencies. Here is -an example for a minimal `hello` crate: - - - $ cargo new hello - $ cd hello - $ cargo build - Compiling hello v0.1.0 (file:///tmp/hello) - Finished dev [unoptimized + debuginfo] target(s) in 0.20 secs - $ carnix -o hello.nix --src ./. Cargo.lock --standalone - $ nix-build hello.nix - -Now, the file produced by the call to `carnix`, called `hello.nix`, looks like: - -``` -# Generated by carnix 0.6.5: carnix -o hello.nix --src ./. Cargo.lock --standalone -{ lib, buildPlatform, buildRustCrate, fetchgit }: -let kernel = buildPlatform.parsed.kernel.name; - # ... (content skipped) -in -rec { - hello = f: hello_0_1_0 { features = hello_0_1_0_features { hello_0_1_0 = f; }; }; - hello_0_1_0_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate { - crateName = "hello"; - version = "0.1.0"; - authors = [ "pe@pijul.org " ]; - src = ./.; - inherit dependencies buildDependencies features; - }; - hello_0_1_0 = { features?(hello_0_1_0_features {}) }: hello_0_1_0_ {}; - hello_0_1_0_features = f: updateFeatures f (rec { - hello_0_1_0.default = (f.hello_0_1_0.default or true); - }) [ ]; -} -``` - -In particular, note that the argument given as `--src` is copied -verbatim to the source. If we look at a more complicated -dependencies, for instance by adding a single line `libc="*"` to our -`Cargo.toml`, we first need to run `cargo build` to update the -`Cargo.lock`. Then, `carnix` needs to be run again, and produces the -following nix file: - -``` -# Generated by carnix 0.6.5: carnix -o hello.nix --src ./. Cargo.lock --standalone -{ lib, buildPlatform, buildRustCrate, fetchgit }: -let kernel = buildPlatform.parsed.kernel.name; - # ... (content skipped) -in -rec { - hello = f: hello_0_1_0 { features = hello_0_1_0_features { hello_0_1_0 = f; }; }; - hello_0_1_0_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate { - crateName = "hello"; - version = "0.1.0"; - authors = [ "pe@pijul.org " ]; - src = ./.; - inherit dependencies buildDependencies features; - }; - libc_0_2_36_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate { - crateName = "libc"; - version = "0.2.36"; - authors = [ "The Rust Project Developers" ]; - sha256 = "01633h4yfqm0s302fm0dlba469bx8y6cs4nqc8bqrmjqxfxn515l"; - inherit dependencies buildDependencies features; - }; - hello_0_1_0 = { features?(hello_0_1_0_features {}) }: hello_0_1_0_ { - dependencies = mapFeatures features ([ libc_0_2_36 ]); - }; - hello_0_1_0_features = f: updateFeatures f (rec { - hello_0_1_0.default = (f.hello_0_1_0.default or true); - libc_0_2_36.default = true; - }) [ libc_0_2_36_features ]; - libc_0_2_36 = { features?(libc_0_2_36_features {}) }: libc_0_2_36_ { - features = mkFeatures (features.libc_0_2_36 or {}); - }; - libc_0_2_36_features = f: updateFeatures f (rec { - libc_0_2_36.default = (f.libc_0_2_36.default or true); - libc_0_2_36.use_std = - (f.libc_0_2_36.use_std or false) || - (f.libc_0_2_36.default or false) || - (libc_0_2_36.default or false); - }) []; -} -``` - -Here, the `libc` crate has no `src` attribute, so `buildRustCrate` -will fetch it from [crates.io](https://crates.io). A `sha256` -attribute is still needed for Nix purity. - -### Handling external dependencies - -Some crates require external libraries. For crates from -[crates.io](https://crates.io), such libraries can be specified in -`defaultCrateOverrides` package in nixpkgs itself. - -Starting from that file, one can add more overrides, to add features -or build inputs by overriding the hello crate in a seperate file. - -``` -with import {}; -((import ./hello.nix).hello {}).override { - crateOverrides = defaultCrateOverrides // { - hello = attrs: { buildInputs = [ openssl ]; }; - }; -} -``` - -Here, `crateOverrides` is expected to be a attribute set, where the -key is the crate name without version number and the value a function. -The function gets all attributes passed to `buildRustCrate` as first -argument and returns a set that contains all attribute that should be -overwritten. - -For more complicated cases, such as when parts of the crate's -derivation depend on the the crate's version, the `attrs` argument of -the override above can be read, as in the following example, which -patches the derivation: - -``` -with import {}; -((import ./hello.nix).hello {}).override { - crateOverrides = defaultCrateOverrides // { - hello = attrs: lib.optionalAttrs (lib.versionAtLeast attrs.version "1.0") { - postPatch = '' - substituteInPlace lib/zoneinfo.rs \ - --replace "/usr/share/zoneinfo" "${tzdata}/share/zoneinfo" - ''; - }; - }; -} -``` - -Another situation is when we want to override a nested -dependency. This actually works in the exact same way, since the -`crateOverrides` parameter is forwarded to the crate's -dependencies. For instance, to override the build inputs for crate -`libc` in the example above, where `libc` is a dependency of the main -crate, we could do: - -``` -with import {}; -((import hello.nix).hello {}).override { - crateOverrides = defaultCrateOverrides // { - libc = attrs: { buildInputs = []; }; - }; -} -``` - -### Options and phases configuration - -Actually, the overrides introduced in the previous section are more -general. A number of other parameters can be overridden: - -- The version of rustc used to compile the crate: - - ``` - (hello {}).override { rust = pkgs.rust; }; - ``` - -- Whether to build in release mode or debug mode (release mode by - default): - - ``` - (hello {}).override { release = false; }; - ``` - -- Whether to print the commands sent to rustc when building - (equivalent to `--verbose` in cargo: - - ``` - (hello {}).override { verbose = false; }; - ``` - -- Extra arguments to be passed to `rustc`: - - ``` - (hello {}).override { extraRustcOpts = "-Z debuginfo=2"; }; - ``` - -- Phases, just like in any other derivation, can be specified using - the following attributes: `preUnpack`, `postUnpack`, `prePatch`, - `patches`, `postPatch`, `preConfigure` (in the case of a Rust crate, - this is run before calling the "build" script), `postConfigure` - (after the "build" script),`preBuild`, `postBuild`, `preInstall` and - `postInstall`. As an example, here is how to create a new module - before running the build script: - - ``` - (hello {}).override { - preConfigure = '' - echo "pub const PATH=\"${hi.out}\";" >> src/path.rs" - ''; - }; - ``` - -### Features - -One can also supply features switches. For example, if we want to -compile `diesel_cli` only with the `postgres` feature, and no default -features, we would write: - -``` -(callPackage ./diesel.nix {}).diesel { - default = false; - postgres = true; -} -``` - -Where `diesel.nix` is the file generated by Carnix, as explained above. - -## Using the Rust nightlies overlay - -Mozilla provides an overlay for nixpkgs to bring a nightly version of Rust into scope. -This overlay can _also_ be used to install recent unstable or stable versions -of Rust, if desired. - -To use this overlay, clone -[nixpkgs-mozilla](https://github.com/mozilla/nixpkgs-mozilla), -and create a symbolic link to the file -[rust-overlay.nix](https://github.com/mozilla/nixpkgs-mozilla/blob/master/rust-overlay.nix) -in the `~/.config/nixpkgs/overlays` directory. - - $ git clone https://github.com/mozilla/nixpkgs-mozilla.git - $ mkdir -p ~/.config/nixpkgs/overlays - $ ln -s $(pwd)/nixpkgs-mozilla/rust-overlay.nix ~/.config/nixpkgs/overlays/rust-overlay.nix - -The latest version can be installed with the following command: - - $ nix-env -Ai nixos.latest.rustChannels.stable.rust - -Or using the attribute with nix-shell: - - $ nix-shell -p nixos.latest.rustChannels.stable.rust - -To install the beta or nightly channel, "stable" should be substituted by -"nightly" or "beta", or -use the function provided by this overlay to pull a version based on a -build date. - -The overlay automatically updates itself as it uses the same source as -[rustup](https://www.rustup.rs/). diff --git a/doc/languages-frameworks/rust.section.md b/doc/languages-frameworks/rust.section.md new file mode 100644 index 000000000000..f0498eac5b19 --- /dev/null +++ b/doc/languages-frameworks/rust.section.md @@ -0,0 +1,309 @@ +--- +title: Rust +author: Matthias Beyer +date: 2017-03-05 +--- + +# User's Guide to the Rust Infrastructure + +To install the rust compiler and cargo put + +``` +rustc +cargo +``` + +into the `environment.systemPackages` or bring them into +scope with `nix-shell -p rustc cargo`. + +For daily builds (beta and nightly) use either rustup from +nixpkgs or use the [Rust nightlies +overlay](#using-the-rust-nightlies-overlay). + +## Compiling Rust applications with Cargo + +Rust applications are packaged by using the `buildRustPackage` helper from `rustPlatform`: + +``` +rustPlatform.buildRustPackage rec { + name = "ripgrep-${version}"; + version = "0.4.0"; + + src = fetchFromGitHub { + owner = "BurntSushi"; + repo = "ripgrep"; + rev = "${version}"; + sha256 = "0y5d1n6hkw85jb3rblcxqas2fp82h3nghssa4xqrhqnz25l799pj"; + }; + + cargoSha256 = "0q68qyl2h6i0qsz82z840myxlnjay8p1w5z7hfyr8fqp7wgwa9cx"; + + meta = with stdenv.lib; { + description = "A fast line-oriented regex search tool, similar to ag and ack"; + homepage = https://github.com/BurntSushi/ripgrep; + license = licenses.unlicense; + maintainers = [ maintainers.tailhook ]; + platforms = platforms.all; + }; +} +``` + +`buildRustPackage` requires a `cargoSha256` attribute which is computed over +all crate sources of this package. Currently it is obtained by inserting a +fake checksum into the expression and building the package once. The correct +checksum can be then take from the failed build. + +To install crates with nix there is also an experimental project called +[nixcrates](https://github.com/fractalide/nixcrates). + +## Compiling Rust crates using Nix instead of Cargo + +### Simple operation + +When run, `cargo build` produces a file called `Cargo.lock`, +containing pinned versions of all dependencies. Nixpkgs contains a +tool called `carnix` (`nix-env -iA nixos.carnix`), which can be used +to turn a `Cargo.lock` into a Nix expression. + +That Nix expression calls `rustc` directly (hence bypassing Cargo), +and can be used to compile a crate and all its dependencies. Here is +an example for a minimal `hello` crate: + + + $ cargo new hello + $ cd hello + $ cargo build + Compiling hello v0.1.0 (file:///tmp/hello) + Finished dev [unoptimized + debuginfo] target(s) in 0.20 secs + $ carnix -o hello.nix --src ./. Cargo.lock --standalone + $ nix-build hello.nix + +Now, the file produced by the call to `carnix`, called `hello.nix`, looks like: + +``` +# Generated by carnix 0.6.5: carnix -o hello.nix --src ./. Cargo.lock --standalone +{ lib, buildPlatform, buildRustCrate, fetchgit }: +let kernel = buildPlatform.parsed.kernel.name; + # ... (content skipped) +in +rec { + hello = f: hello_0_1_0 { features = hello_0_1_0_features { hello_0_1_0 = f; }; }; + hello_0_1_0_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate { + crateName = "hello"; + version = "0.1.0"; + authors = [ "pe@pijul.org " ]; + src = ./.; + inherit dependencies buildDependencies features; + }; + hello_0_1_0 = { features?(hello_0_1_0_features {}) }: hello_0_1_0_ {}; + hello_0_1_0_features = f: updateFeatures f (rec { + hello_0_1_0.default = (f.hello_0_1_0.default or true); + }) [ ]; +} +``` + +In particular, note that the argument given as `--src` is copied +verbatim to the source. If we look at a more complicated +dependencies, for instance by adding a single line `libc="*"` to our +`Cargo.toml`, we first need to run `cargo build` to update the +`Cargo.lock`. Then, `carnix` needs to be run again, and produces the +following nix file: + +``` +# Generated by carnix 0.6.5: carnix -o hello.nix --src ./. Cargo.lock --standalone +{ lib, buildPlatform, buildRustCrate, fetchgit }: +let kernel = buildPlatform.parsed.kernel.name; + # ... (content skipped) +in +rec { + hello = f: hello_0_1_0 { features = hello_0_1_0_features { hello_0_1_0 = f; }; }; + hello_0_1_0_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate { + crateName = "hello"; + version = "0.1.0"; + authors = [ "pe@pijul.org " ]; + src = ./.; + inherit dependencies buildDependencies features; + }; + libc_0_2_36_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate { + crateName = "libc"; + version = "0.2.36"; + authors = [ "The Rust Project Developers" ]; + sha256 = "01633h4yfqm0s302fm0dlba469bx8y6cs4nqc8bqrmjqxfxn515l"; + inherit dependencies buildDependencies features; + }; + hello_0_1_0 = { features?(hello_0_1_0_features {}) }: hello_0_1_0_ { + dependencies = mapFeatures features ([ libc_0_2_36 ]); + }; + hello_0_1_0_features = f: updateFeatures f (rec { + hello_0_1_0.default = (f.hello_0_1_0.default or true); + libc_0_2_36.default = true; + }) [ libc_0_2_36_features ]; + libc_0_2_36 = { features?(libc_0_2_36_features {}) }: libc_0_2_36_ { + features = mkFeatures (features.libc_0_2_36 or {}); + }; + libc_0_2_36_features = f: updateFeatures f (rec { + libc_0_2_36.default = (f.libc_0_2_36.default or true); + libc_0_2_36.use_std = + (f.libc_0_2_36.use_std or false) || + (f.libc_0_2_36.default or false) || + (libc_0_2_36.default or false); + }) []; +} +``` + +Here, the `libc` crate has no `src` attribute, so `buildRustCrate` +will fetch it from [crates.io](https://crates.io). A `sha256` +attribute is still needed for Nix purity. + +### Handling external dependencies + +Some crates require external libraries. For crates from +[crates.io](https://crates.io), such libraries can be specified in +`defaultCrateOverrides` package in nixpkgs itself. + +Starting from that file, one can add more overrides, to add features +or build inputs by overriding the hello crate in a seperate file. + +``` +with import {}; +((import ./hello.nix).hello {}).override { + crateOverrides = defaultCrateOverrides // { + hello = attrs: { buildInputs = [ openssl ]; }; + }; +} +``` + +Here, `crateOverrides` is expected to be a attribute set, where the +key is the crate name without version number and the value a function. +The function gets all attributes passed to `buildRustCrate` as first +argument and returns a set that contains all attribute that should be +overwritten. + +For more complicated cases, such as when parts of the crate's +derivation depend on the the crate's version, the `attrs` argument of +the override above can be read, as in the following example, which +patches the derivation: + +``` +with import {}; +((import ./hello.nix).hello {}).override { + crateOverrides = defaultCrateOverrides // { + hello = attrs: lib.optionalAttrs (lib.versionAtLeast attrs.version "1.0") { + postPatch = '' + substituteInPlace lib/zoneinfo.rs \ + --replace "/usr/share/zoneinfo" "${tzdata}/share/zoneinfo" + ''; + }; + }; +} +``` + +Another situation is when we want to override a nested +dependency. This actually works in the exact same way, since the +`crateOverrides` parameter is forwarded to the crate's +dependencies. For instance, to override the build inputs for crate +`libc` in the example above, where `libc` is a dependency of the main +crate, we could do: + +``` +with import {}; +((import hello.nix).hello {}).override { + crateOverrides = defaultCrateOverrides // { + libc = attrs: { buildInputs = []; }; + }; +} +``` + +### Options and phases configuration + +Actually, the overrides introduced in the previous section are more +general. A number of other parameters can be overridden: + +- The version of rustc used to compile the crate: + + ``` + (hello {}).override { rust = pkgs.rust; }; + ``` + +- Whether to build in release mode or debug mode (release mode by + default): + + ``` + (hello {}).override { release = false; }; + ``` + +- Whether to print the commands sent to rustc when building + (equivalent to `--verbose` in cargo: + + ``` + (hello {}).override { verbose = false; }; + ``` + +- Extra arguments to be passed to `rustc`: + + ``` + (hello {}).override { extraRustcOpts = "-Z debuginfo=2"; }; + ``` + +- Phases, just like in any other derivation, can be specified using + the following attributes: `preUnpack`, `postUnpack`, `prePatch`, + `patches`, `postPatch`, `preConfigure` (in the case of a Rust crate, + this is run before calling the "build" script), `postConfigure` + (after the "build" script),`preBuild`, `postBuild`, `preInstall` and + `postInstall`. As an example, here is how to create a new module + before running the build script: + + ``` + (hello {}).override { + preConfigure = '' + echo "pub const PATH=\"${hi.out}\";" >> src/path.rs" + ''; + }; + ``` + +### Features + +One can also supply features switches. For example, if we want to +compile `diesel_cli` only with the `postgres` feature, and no default +features, we would write: + +``` +(callPackage ./diesel.nix {}).diesel { + default = false; + postgres = true; +} +``` + +Where `diesel.nix` is the file generated by Carnix, as explained above. + +## Using the Rust nightlies overlay + +Mozilla provides an overlay for nixpkgs to bring a nightly version of Rust into scope. +This overlay can _also_ be used to install recent unstable or stable versions +of Rust, if desired. + +To use this overlay, clone +[nixpkgs-mozilla](https://github.com/mozilla/nixpkgs-mozilla), +and create a symbolic link to the file +[rust-overlay.nix](https://github.com/mozilla/nixpkgs-mozilla/blob/master/rust-overlay.nix) +in the `~/.config/nixpkgs/overlays` directory. + + $ git clone https://github.com/mozilla/nixpkgs-mozilla.git + $ mkdir -p ~/.config/nixpkgs/overlays + $ ln -s $(pwd)/nixpkgs-mozilla/rust-overlay.nix ~/.config/nixpkgs/overlays/rust-overlay.nix + +The latest version can be installed with the following command: + + $ nix-env -Ai nixos.latest.rustChannels.stable.rust + +Or using the attribute with nix-shell: + + $ nix-shell -p nixos.latest.rustChannels.stable.rust + +To install the beta or nightly channel, "stable" should be substituted by +"nightly" or "beta", or +use the function provided by this overlay to pull a version based on a +build date. + +The overlay automatically updates itself as it uses the same source as +[rustup](https://www.rustup.rs/). diff --git a/doc/languages-frameworks/vim.md b/doc/languages-frameworks/vim.md deleted file mode 100644 index 1d6a4fe8da8d..000000000000 --- a/doc/languages-frameworks/vim.md +++ /dev/null @@ -1,135 +0,0 @@ ---- -title: User's Guide for Vim in Nixpkgs -author: Marc Weber -date: 2016-06-25 ---- -# User's Guide to Vim Plugins/Addons/Bundles/Scripts in Nixpkgs - -You'll get a vim(-your-suffix) in PATH also loading the plugins you want. -Loading can be deferred; see examples. - -Vim packages, VAM (=vim-addon-manager) and Pathogen are supported to load -packages. - -## Custom configuration - -Adding custom .vimrc lines can be done using the following code: - -``` -vim_configurable.customize { - name = "vim-with-plugins"; - - vimrcConfig.customRC = '' - set hidden - ''; -} -``` - -## Vim packages - -To store you plugins in Vim packages the following example can be used: - -``` -vim_configurable.customize { - vimrcConfig.packages.myVimPackage = with pkgs.vimPlugins; { - # loaded on launch - start = [ youcompleteme fugitive ]; - # manually loadable by calling `:packadd $plugin-name` - opt = [ phpCompletion elm-vim ]; - # To automatically load a plugin when opening a filetype, add vimrc lines like: - # autocmd FileType php :packadd phpCompletion - } -}; -``` - -## VAM - -### dependencies by Vim plugins - -VAM introduced .json files supporting dependencies without versioning -assuming that "using latest version" is ok most of the time. - -### Example - -First create a vim-scripts file having one plugin name per line. Example: - - "tlib" - {'name': 'vim-addon-sql'} - {'filetype_regex': '\%(vim)$', 'names': ['reload', 'vim-dev-plugin']} - -Such vim-scripts file can be read by VAM as well like this: - - call vam#Scripts(expand('~/.vim-scripts'), {}) - -Create a default.nix file: - - { nixpkgs ? import {}, compiler ? "ghc7102" }: - nixpkgs.vim_configurable.customize { name = "vim"; vimrcConfig.vam.pluginDictionaries = [ "vim-addon-vim2nix" ]; } - -Create a generate.vim file: - - ActivateAddons vim-addon-vim2nix - let vim_scripts = "vim-scripts" - call nix#ExportPluginsForNix({ - \ 'path_to_nixpkgs': eval('{"'.substitute(substitute(substitute($NIX_PATH, ':', ',', 'g'), '=',':', 'g'), '\([:,]\)', '"\1"',"g").'"}')["nixpkgs"], - \ 'cache_file': '/tmp/vim2nix-cache', - \ 'try_catch': 0, - \ 'plugin_dictionaries': ["vim-addon-manager"]+map(readfile(vim_scripts), 'eval(v:val)') - \ }) - -Then run - - nix-shell -p vimUtils.vim_with_vim2nix --command "vim -c 'source generate.vim'" - -You should get a Vim buffer with the nix derivations (output1) and vam.pluginDictionaries (output2). -You can add your vim to your system's configuration file like this and start it by "vim-my": - - my-vim = - let plugins = let inherit (vimUtils) buildVimPluginFrom2Nix; in { - copy paste output1 here - }; in vim_configurable.customize { - name = "vim-my"; - - vimrcConfig.vam.knownPlugins = plugins; # optional - vimrcConfig.vam.pluginDictionaries = [ - copy paste output2 here - ]; - - # Pathogen would be - # vimrcConfig.pathogen.knownPlugins = plugins; # plugins - # vimrcConfig.pathogen.pluginNames = ["tlib"]; - }; - - -Sample output1: - - "reload" = buildVimPluginFrom2Nix { # created by nix#NixDerivation - name = "reload"; - src = fetchgit { - url = "git://github.com/xolox/vim-reload"; - rev = "0a601a668727f5b675cb1ddc19f6861f3f7ab9e1"; - sha256 = "0vb832l9yxj919f5hfg6qj6bn9ni57gnjd3bj7zpq7d4iv2s4wdh"; - }; - dependencies = ["nim-misc"]; - - }; - [...] - -Sample output2: - - [ - ''vim-addon-manager'' - ''tlib'' - { "name" = ''vim-addon-sql''; } - { "filetype_regex" = ''\%(vim)$$''; "names" = [ ''reload'' ''vim-dev-plugin'' ]; } - ] - - -## Important repositories - -- [vim-pi](https://bitbucket.org/vimcommunity/vim-pi) is a plugin repository - from VAM plugin manager meant to be used by others as well used by - -- [vim2nix](http://github.com/MarcWeber/vim-addon-vim2nix) which generates the - .nix code - diff --git a/doc/languages-frameworks/vim.section.md b/doc/languages-frameworks/vim.section.md new file mode 100644 index 000000000000..1d6a4fe8da8d --- /dev/null +++ b/doc/languages-frameworks/vim.section.md @@ -0,0 +1,135 @@ +--- +title: User's Guide for Vim in Nixpkgs +author: Marc Weber +date: 2016-06-25 +--- +# User's Guide to Vim Plugins/Addons/Bundles/Scripts in Nixpkgs + +You'll get a vim(-your-suffix) in PATH also loading the plugins you want. +Loading can be deferred; see examples. + +Vim packages, VAM (=vim-addon-manager) and Pathogen are supported to load +packages. + +## Custom configuration + +Adding custom .vimrc lines can be done using the following code: + +``` +vim_configurable.customize { + name = "vim-with-plugins"; + + vimrcConfig.customRC = '' + set hidden + ''; +} +``` + +## Vim packages + +To store you plugins in Vim packages the following example can be used: + +``` +vim_configurable.customize { + vimrcConfig.packages.myVimPackage = with pkgs.vimPlugins; { + # loaded on launch + start = [ youcompleteme fugitive ]; + # manually loadable by calling `:packadd $plugin-name` + opt = [ phpCompletion elm-vim ]; + # To automatically load a plugin when opening a filetype, add vimrc lines like: + # autocmd FileType php :packadd phpCompletion + } +}; +``` + +## VAM + +### dependencies by Vim plugins + +VAM introduced .json files supporting dependencies without versioning +assuming that "using latest version" is ok most of the time. + +### Example + +First create a vim-scripts file having one plugin name per line. Example: + + "tlib" + {'name': 'vim-addon-sql'} + {'filetype_regex': '\%(vim)$', 'names': ['reload', 'vim-dev-plugin']} + +Such vim-scripts file can be read by VAM as well like this: + + call vam#Scripts(expand('~/.vim-scripts'), {}) + +Create a default.nix file: + + { nixpkgs ? import {}, compiler ? "ghc7102" }: + nixpkgs.vim_configurable.customize { name = "vim"; vimrcConfig.vam.pluginDictionaries = [ "vim-addon-vim2nix" ]; } + +Create a generate.vim file: + + ActivateAddons vim-addon-vim2nix + let vim_scripts = "vim-scripts" + call nix#ExportPluginsForNix({ + \ 'path_to_nixpkgs': eval('{"'.substitute(substitute(substitute($NIX_PATH, ':', ',', 'g'), '=',':', 'g'), '\([:,]\)', '"\1"',"g").'"}')["nixpkgs"], + \ 'cache_file': '/tmp/vim2nix-cache', + \ 'try_catch': 0, + \ 'plugin_dictionaries': ["vim-addon-manager"]+map(readfile(vim_scripts), 'eval(v:val)') + \ }) + +Then run + + nix-shell -p vimUtils.vim_with_vim2nix --command "vim -c 'source generate.vim'" + +You should get a Vim buffer with the nix derivations (output1) and vam.pluginDictionaries (output2). +You can add your vim to your system's configuration file like this and start it by "vim-my": + + my-vim = + let plugins = let inherit (vimUtils) buildVimPluginFrom2Nix; in { + copy paste output1 here + }; in vim_configurable.customize { + name = "vim-my"; + + vimrcConfig.vam.knownPlugins = plugins; # optional + vimrcConfig.vam.pluginDictionaries = [ + copy paste output2 here + ]; + + # Pathogen would be + # vimrcConfig.pathogen.knownPlugins = plugins; # plugins + # vimrcConfig.pathogen.pluginNames = ["tlib"]; + }; + + +Sample output1: + + "reload" = buildVimPluginFrom2Nix { # created by nix#NixDerivation + name = "reload"; + src = fetchgit { + url = "git://github.com/xolox/vim-reload"; + rev = "0a601a668727f5b675cb1ddc19f6861f3f7ab9e1"; + sha256 = "0vb832l9yxj919f5hfg6qj6bn9ni57gnjd3bj7zpq7d4iv2s4wdh"; + }; + dependencies = ["nim-misc"]; + + }; + [...] + +Sample output2: + + [ + ''vim-addon-manager'' + ''tlib'' + { "name" = ''vim-addon-sql''; } + { "filetype_regex" = ''\%(vim)$$''; "names" = [ ''reload'' ''vim-dev-plugin'' ]; } + ] + + +## Important repositories + +- [vim-pi](https://bitbucket.org/vimcommunity/vim-pi) is a plugin repository + from VAM plugin manager meant to be used by others as well used by + +- [vim2nix](http://github.com/MarcWeber/vim-addon-vim2nix) which generates the + .nix code + diff --git a/doc/manual.xml b/doc/manual.xml index eb0a24789d44..385079eb5785 100644 --- a/doc/manual.xml +++ b/doc/manual.xml @@ -9,7 +9,7 @@ - + diff --git a/doc/shell.md b/doc/shell.md deleted file mode 100644 index 079574d4ae86..000000000000 --- a/doc/shell.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: pkgs.mkShell -author: zimbatm -date: 2017-10-30 ---- - -pkgs.mkShell is a special kind of derivation that is only useful when using -it combined with nix-shell. It will in fact fail to instantiate when invoked -with nix-build. - -## Usage - -```nix -{ pkgs ? import {} }: -pkgs.mkShell { - # this will make all the build inputs from hello and gnutar available to the shell environment - inputsFrom = with pkgs; [ hello gnutar ]; - buildInputs = [ pkgs.gnumake ]; -} -``` diff --git a/doc/shell.section.md b/doc/shell.section.md new file mode 100644 index 000000000000..079574d4ae86 --- /dev/null +++ b/doc/shell.section.md @@ -0,0 +1,20 @@ +--- +title: pkgs.mkShell +author: zimbatm +date: 2017-10-30 +--- + +pkgs.mkShell is a special kind of derivation that is only useful when using +it combined with nix-shell. It will in fact fail to instantiate when invoked +with nix-build. + +## Usage + +```nix +{ pkgs ? import {} }: +pkgs.mkShell { + # this will make all the build inputs from hello and gnutar available to the shell environment + inputsFrom = with pkgs; [ hello gnutar ]; + buildInputs = [ pkgs.gnumake ]; +} +``` diff --git a/pkgs/development/idris-modules/README.md b/pkgs/development/idris-modules/README.md index 005ed3602851..80b7ccefbcfa 100644 --- a/pkgs/development/idris-modules/README.md +++ b/pkgs/development/idris-modules/README.md @@ -1,39 +1 @@ -Idris packages -============== - -This directory contains build rules for idris packages. In addition, -it contains several functions to build and compose those packages. -Everything is exposed to the user via the `idrisPackages` attribute. - -callPackage ------------- - -This is like the normal nixpkgs callPackage function, specialized to -idris packages. - -builtins ---------- - -This is a list of all of the libraries that come packaged with Idris -itself. - -build-idris-package --------------------- - -A function to build an idris package. Its sole argument is a set like -you might pass to `stdenv.mkDerivation`, except `build-idris-package` -sets several attributes for you. See `build-idris-package.nix` for -details. - -build-builtin-package ----------------------- - -A version of `build-idris-package` specialized to builtin libraries. -Mostly for internal use. - -with-packages -------------- - -Bundle idris together with a list of packages. Because idris currently -only supports a single directory in its library path, you must include -all desired libraries here, including `prelude` and `base`. \ No newline at end of file +Moved to [/doc/languages-frameworks/idris.section.md](/doc/languages-frameworks/idris.section.md) diff --git a/pkgs/development/node-packages/README.md b/pkgs/development/node-packages/README.md index 17a203ed12be..9760285a915e 100644 --- a/pkgs/development/node-packages/README.md +++ b/pkgs/development/node-packages/README.md @@ -1,51 +1 @@ -Node.js packages -================ -The `pkgs/development/node-packages` folder contains a generated collection of -[NPM packages](https://npmjs.com/) that can be installed with the Nix package -manager. - -As a rule of thumb, the package set should only provide *end user* software -packages, such as command-line utilities. Libraries should only be added to the -package set if there is a non-NPM package that requires it. - -When it is desired to use NPM libraries in a development project, use the -`node2nix` generator directly on the `package.json` configuration file of the -project. - -The package set also provides support for multiple Node.js versions. The policy -is that a new package should be added to the collection for the latest stable LTS -release (which is currently 6.x), unless there is an explicit reason to support -a different release. - -If your package uses native addons, you need to examine what kind of native -build system it uses. Here are some examples: - -* `node-gyp` -* `node-gyp-builder` -* `node-pre-gyp` - -After you have identified the correct system, you need to override your package -expression while adding in build system as a build input. For example, `dat` -requires `node-gyp-build`, so we override its expression in `default-v6.nix`: - -```nix -dat = nodePackages.dat.override (oldAttrs: { - buildInputs = oldAttrs.buildInputs ++ [ nodePackages.node-gyp-build ]; -}); -``` - -To add a package from NPM to nixpkgs: - - 1. Modify `pkgs/development/node-packages/node-packages-v6.json` to add, update - or remove package entries. (Or `pkgs/development/node-packages/node-packages-v4.json` - for packages depending on Node.js 4.x) - 2. Run the script: `(cd pkgs/development/node-packages && ./generate.sh)`. - 3. Build your new package to test your changes: - `cd /path/to/nixpkgs && nix-build -A nodePackages.`. - To build against a specific Node.js version (e.g. 4.x): - `nix-build -A nodePackages_4_x.` - 4. Add and commit all modified and generated files. - -For more information about the generation process, consult the -[README.md](https://github.com/svanderburg/node2nix) file of the `node2nix` -tool. +Moved to [/doc/languages-frameworks/node.section.md](/doc/languages-frameworks/node.section.md) diff --git a/pkgs/development/r-modules/README.md b/pkgs/development/r-modules/README.md index c8f02bd14780..e6fd09d76471 100644 --- a/pkgs/development/r-modules/README.md +++ b/pkgs/development/r-modules/README.md @@ -1,120 +1 @@ -R packages -========== - -## Installation - -Define an environment for R that contains all the libraries that you'd like to -use by adding the following snippet to your $HOME/.config/nixpkgs/config.nix file: - -```nix -{ - packageOverrides = super: let self = super.pkgs; in - { - - rEnv = super.rWrapper.override { - packages = with self.rPackages; [ - devtools - ggplot2 - reshape2 - yaml - optparse - ]; - }; - }; -} -``` - -Then you can use `nix-env -f "" -iA rEnv` to install it into your user -profile. The set of available libraries can be discovered by running the -command `nix-env -f "" -qaP -A rPackages`. The first column from that -output is the name that has to be passed to rWrapper in the code snipped above. - -However, if you'd like to add a file to your project source to make the -environment available for other contributors, you can create a `default.nix` -file like so: -```nix -let - pkgs = import {}; - stdenv = pkgs.stdenv; -in with pkgs; { - myProject = stdenv.mkDerivation { - name = "myProject"; - version = "1"; - src = if pkgs.lib.inNixShell then null else nix; - - buildInputs = with rPackages; [ - R - ggplot2 - knitr - ]; - }; -} -``` -and then run `nix-shell .` to be dropped into a shell with those packages -available. - -## RStudio - -RStudio uses a standard set of packages and ignores any custom R -environments or installed packages you may have. To create a custom -environment, see `rstudioWrapper`, which functions similarly to -`rWrapper`: - -```nix -{ - packageOverrides = super: let self = super.pkgs; in - { - - rstudioEnv = super.rstudioWrapper.override { - packages = with self.rPackages; [ - dplyr - ggplot2 - reshape2 - ]; - }; - }; -} -``` - -Then like above, `nix-env -f "" -iA rstudioEnv` will install -this into your user profile. - -Alternatively, you can create a self-contained `shell.nix` without the need to -modify any configuration files: - -```nix -{ pkgs ? import {} -}: - -pkgs.rstudioWrapper.override { - packages = with pkgs.rPackages; [ dplyr ggplot2 reshape2 ]; -} - -``` - -Executing `nix-shell` will then drop you into an environment equivalent to the -one above. If you need additional packages just add them to the list and -re-enter the shell. - -## Updating the package set - -```bash -nix-shell generate-shell.nix - -Rscript generate-r-packages.R cran > cran-packages.nix.new -mv cran-packages.nix.new cran-packages.nix - -Rscript generate-r-packages.R bioc > bioc-packages.nix.new -mv bioc-packages.nix.new bioc-packages.nix -``` - -`generate-r-packages.R ` reads `-packages.nix`, therefor the renaming. - - -## Testing if the Nix-expression could be evaluated - -```bash -nix-build test-evaluation.nix --dry-run -``` - -If this exits fine, the expression is ok. If not, you have to edit `default.nix` +Moved to [/doc/languages-frameworks/r.section.md](/doc/languages-frameworks/r.section.md) -- cgit 1.4.1