Merge commit '2d2f10475138b7206572dc3ec288184df2be022e'

17 files changed, 756 insertions, 63 deletions

diff --git a/nixpkgs/doc/Makefile b/nixpkgs/doc/Makefile
index c6aed62a9396..91b62fe138bc 100644
--- a/nixpkgs/doc/Makefile
+++ b/nixpkgs/doc/Makefile
@@ -9,8 +9,10 @@ debug:
 
 .PHONY: format
 format:
-	find . -iname '*.xml' -type f -print0 | xargs -0 -I{} -n1 \
-		xmlformat --config-file "$$XMLFORMAT_CONFIG" -i {}
+	find . -iname '*.xml' -type f | while read f; do \
+		echo $$f ;\
+		xmlformat --config-file "$$XMLFORMAT_CONFIG" -i $$f ;\
+	done
 
 .PHONY: fix-misc-xml
 fix-misc-xml:
diff --git a/nixpkgs/doc/coding-conventions.xml b/nixpkgs/doc/coding-conventions.xml
index a8a4557b461c..d2c7a1baae9c 100644
--- a/nixpkgs/doc/coding-conventions.xml
+++ b/nixpkgs/doc/coding-conventions.xml
@@ -814,7 +814,7 @@ args.stdenv.mkDerivation (args // {
 
   <para>
    There are multiple ways to fetch a package source in nixpkgs. The general
-   guideline is that you should package sources with a high degree of
+   guideline is that you should package reproducible sources with a high degree of
    availability. Right now there is only one fetcher which has mirroring
    support and that is <literal>fetchurl</literal>. Note that you should also
    prefer protocols which have a corresponding proxy environment variable.
@@ -876,6 +876,123 @@ src = fetchFromGitHub {
    </itemizedlist>
   </para>
  </section>
+ <section xml:id="sec-source-hashes">
+  <title>Obtaining source hash</title>
+
+  <para>
+   Preferred source hash type is sha256. There are several ways to get it.
+  </para>
+
+  <orderedlist>
+   <listitem>
+    <para>
+     Prefetch URL (with <literal>nix-prefetch-<replaceable>XXX</replaceable>
+     <replaceable>URL</replaceable></literal>, where
+     <replaceable>XXX</replaceable> is one of <literal>url</literal>,
+     <literal>git</literal>, <literal>hg</literal>, <literal>cvs</literal>,
+     <literal>bzr</literal>, <literal>svn</literal>). Hash is printed to
+     stdout.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     Prefetch by package source (with <literal>nix-prefetch-url
+     '&lt;nixpkgs&gt;' -A <replaceable>PACKAGE</replaceable>.src</literal>,
+     where <replaceable>PACKAGE</replaceable> is package attribute name). Hash
+     is printed to stdout.
+    </para>
+    <para>
+     This works well when you've upgraded existing package version and want to
+     find out new hash, but is useless if package can't be accessed by
+     attribute or package has multiple sources (<literal>.srcs</literal>,
+     architecture-dependent sources, etc).
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     Upstream provided hash: use it when upstream provides
+     <literal>sha256</literal> or <literal>sha512</literal> (when upstream
+     provides <literal>md5</literal>, don't use it, compute
+     <literal>sha256</literal> instead).
+    </para>
+    <para>
+     A little nuance is that <literal>nix-prefetch-*</literal> tools produce
+     hash encoded with <literal>base32</literal>, but upstream usually provides
+     hexadecimal (<literal>base16</literal>) encoding. Fetchers understand both
+     formats. Nixpkgs does not standardize on any one format.
+    </para>
+    <para>
+     You can convert between formats with nix-hash, for example:
+<screen>
+$ nix-hash --type sha256 --to-base32 <replaceable>HASH</replaceable>
+</screen>
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     Extracting hash from local source tarball can be done with
+     <literal>sha256sum</literal>. Use <literal>nix-prefetch-url
+     file:///path/to/tarball </literal> if you want base32 hash.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     Fake hash: set fake hash in package expression, perform build and extract
+     correct hash from error Nix prints.
+    </para>
+    <para>
+     For package updates it is enough to change one symbol to make hash fake.
+     For new packages, you can use <literal>lib.fakeSha256</literal>,
+     <literal>lib.fakeSha512</literal> or any other fake hash.
+    </para>
+    <para>
+     This is last resort method when reconstructing source URL is non-trivial
+     and <literal>nix-prefetch-url -A</literal> isn't applicable (for example,
+     <link xlink:href="https://github.com/NixOS/nixpkgs/blob/d2ab091dd308b99e4912b805a5eb088dd536adb9/pkgs/applications/video/kodi/default.nix#L73">
+     one of <literal>kodi</literal> dependencies</link>). The easiest way then
+     would be replace hash with a fake one and rebuild. Nix build will fail and
+     error message will contain desired hash.
+    </para>
+    <warning><para>This method has security problems. Check below for details.</para></warning>
+   </listitem>
+  </orderedlist>
+
+  <section xml:id="sec-source-hashes-security">
+   <title>Obtaining hashes securely</title>
+   <para>
+    Let's say Man-in-the-Middle (MITM) sits close to your network. Then instead of fetching
+    source you can fetch malware, and instead of source hash you get hash of malware. Here are
+    security considerations for this scenario:
+   </para>
+   <itemizedlist>
+    <listitem>
+     <para>
+      <literal>http://</literal> URLs are not secure to prefetch hash from;
+     </para>
+    </listitem>
+    <listitem>
+     <para>
+      hashes from upstream (in method 3) should be obtained via secure protocol;
+     </para>
+    </listitem>
+    <listitem>
+     <para>
+      <literal>https://</literal> URLs are secure in methods 1, 2, 3;
+     </para>
+    </listitem>
+    <listitem>
+     <para>
+      <literal>https://</literal> URLs are not secure in method 5. When obtaining hashes
+      with fake hash method, TLS checks are disabled. So
+      refetch source hash from several different networks to exclude MITM scenario.
+      Alternatively, use fake hash method to make Nix error, but instead of extracting
+      hash from error, extract <literal>https://</literal> URL and prefetch it
+      with method 1.
+     </para>
+    </listitem>
+   </itemizedlist>
+  </section>
+ </section>
  <section xml:id="sec-patches">
   <title>Patches</title>
 
diff --git a/nixpkgs/doc/configuration.xml b/nixpkgs/doc/configuration.xml
index 624a5bb270ab..8a5ff8dcb8e0 100644
--- a/nixpkgs/doc/configuration.xml
+++ b/nixpkgs/doc/configuration.xml
@@ -180,7 +180,11 @@
      code:
 <programlisting>
 {
-  allowUnfreePredicate = (pkg: builtins.elem (builtins.parseDrvName pkg.name).name [ "flashplayer" "vscode" ]);
+  allowUnfreePredicate = (pkg: builtins.elem
+    (builtins.parseDrvName pkg.name).name [
+      "flashplayer"
+      "vscode"
+    ]);
 }
 </programlisting>
     </para>
@@ -322,7 +326,18 @@
   packageOverrides = pkgs: with pkgs; {
     myPackages = pkgs.buildEnv {
       name = "my-packages";
-      paths = [ aspell bc coreutils gdb ffmpeg nixUnstable emscripten jq nox silver-searcher ];
+      paths = [
+        aspell
+        bc
+        coreutils
+        gdb
+        ffmpeg
+        nixUnstable
+        emscripten
+        jq
+        nox
+        silver-searcher
+      ];
     };
   };
 }
@@ -343,7 +358,18 @@
   packageOverrides = pkgs: with pkgs; {
     myPackages = pkgs.buildEnv {
       name = "my-packages";
-      paths = [ aspell bc coreutils gdb ffmpeg nixUnstable emscripten jq nox silver-searcher ];
+      paths = [
+        aspell
+        bc
+        coreutils
+        gdb
+        ffmpeg
+        nixUnstable
+        emscripten
+        jq
+        nox
+        silver-searcher
+      ];
       pathsToLink = [ "/share" "/bin" ];
     };
   };
@@ -378,7 +404,17 @@
   packageOverrides = pkgs: with pkgs; {
     myPackages = pkgs.buildEnv {
       name = "my-packages";
-      paths = [ aspell bc coreutils ffmpeg nixUnstable emscripten jq nox silver-searcher ];
+      paths = [
+        aspell
+        bc
+        coreutils
+        ffmpeg
+        nixUnstable
+        emscripten
+        jq
+        nox
+        silver-searcher
+      ];
       pathsToLink = [ "/share/man" "/share/doc" "/bin" ];
       extraOutputsToInstall = [ "man" "doc" ];
     };
diff --git a/nixpkgs/doc/default.nix b/nixpkgs/doc/default.nix
index 98b4b92be524..7ceaec28af38 100644
--- a/nixpkgs/doc/default.nix
+++ b/nixpkgs/doc/default.nix
@@ -2,8 +2,8 @@
 let
   lib = pkgs.lib;
   locationsXml = import ./lib-function-locations.nix { inherit pkgs nixpkgs; };
-in
-pkgs.stdenv.mkDerivation {
+  functionDocs = import ./lib-function-docs.nix { inherit locationsXml pkgs; };
+in pkgs.stdenv.mkDerivation {
   name = "nixpkgs-manual";
 
   buildInputs = with pkgs; [ pandoc libxml2 libxslt zip jing  xmlformat ];
@@ -32,6 +32,7 @@ pkgs.stdenv.mkDerivation {
   postPatch = ''
     rm -rf ./functions/library/locations.xml
     ln -s ${locationsXml} ./functions/library/locations.xml
+    ln -s ${functionDocs} ./functions/library/generated
     echo ${lib.version} > .version
   '';
 
diff --git a/nixpkgs/doc/functions.xml b/nixpkgs/doc/functions.xml
index 4193bb49f77a..0d6e2770e6e6 100644
--- a/nixpkgs/doc/functions.xml
+++ b/nixpkgs/doc/functions.xml
@@ -11,7 +11,10 @@
  <xi:include href="functions/overrides.xml" />
  <xi:include href="functions/generators.xml" />
  <xi:include href="functions/debug.xml" />
+ <xi:include href="functions/fetchers.xml" />
+ <xi:include href="functions/trivial-builders.xml" />
  <xi:include href="functions/fhs-environments.xml" />
  <xi:include href="functions/shell.xml" />
  <xi:include href="functions/dockertools.xml" />
+ <xi:include href="functions/prefer-remote-fetch.xml" />
 </chapter>
diff --git a/nixpkgs/doc/functions/dockertools.xml b/nixpkgs/doc/functions/dockertools.xml
index 501f46a967c3..ff446cbfffdd 100644
--- a/nixpkgs/doc/functions/dockertools.xml
+++ b/nixpkgs/doc/functions/dockertools.xml
@@ -24,7 +24,7 @@
 
   <para>
    This function is analogous to the <command>docker build</command> command,
-   in that can used to build a Docker-compatible repository tarball containing
+   in that it can be used to build a Docker-compatible repository tarball containing
    a single image with one or multiple layers. As such, the result is suitable
    for being loaded in Docker with <command>docker load</command>.
   </para>
@@ -190,11 +190,11 @@ buildImage {
     By default <function>buildImage</function> will use a static date of one
     second past the UNIX Epoch. This allows <function>buildImage</function> to
     produce binary reproducible images. When listing images with
-    <command>docker list images</command>, the newly created images will be
+    <command>docker images</command>, the newly created images will be
     listed like this:
    </para>
 <screen><![CDATA[
-$ docker image list
+$ docker images
 REPOSITORY   TAG      IMAGE ID       CREATED        SIZE
 hello        latest   08c791c7846e   48 years ago   25.2MB
 ]]></screen>
@@ -217,7 +217,7 @@ pkgs.dockerTools.buildImage {
     and now the Docker CLI will display a reasonable date and sort the images
     as expected:
 <screen><![CDATA[
-$ docker image list
+$ docker images
 REPOSITORY   TAG      IMAGE ID       CREATED              SIZE
 hello        latest   de2bf4786de6   About a minute ago   25.2MB
 ]]></screen>
@@ -402,7 +402,7 @@ pkgs.dockerTools.buildLayeredImage {
 
   <para>
    This function is analogous to the <command>docker pull</command> command, in
-   that can be used to pull a Docker image from a Docker registry. By default
+   that it can be used to pull a Docker image from a Docker registry. By default
    <link xlink:href="https://hub.docker.com/">Docker Hub</link> is used to pull
    images.
   </para>
@@ -484,7 +484,7 @@ sha256:20d9485b25ecfd89204e843a962c1bd70e9cc6858d65d7f5fadc340246e2116b
 
   <para>
    This function is analogous to the <command>docker export</command> command,
-   in that can used to flatten a Docker image that contains multiple layers. It
+   in that it can be used to flatten a Docker image that contains multiple layers. It
    is in fact the result of the merge of all the layers of the image. As such,
    the result is suitable for being imported in Docker with <command>docker
    import</command>.
@@ -557,7 +557,7 @@ buildImage {
 
   <para>
    Creating base files like <literal>/etc/passwd</literal> or
-   <literal>/etc/login.defs</literal> are necessary for shadow-utils to
+   <literal>/etc/login.defs</literal> is necessary for shadow-utils to
    manipulate users and groups.
   </para>
  </section>
diff --git a/nixpkgs/doc/functions/fetchers.xml b/nixpkgs/doc/functions/fetchers.xml
new file mode 100644
index 000000000000..b3bd2fe0f45e
--- /dev/null
+++ b/nixpkgs/doc/functions/fetchers.xml
@@ -0,0 +1,206 @@
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         xml:id="sec-pkgs-fetchers">
+ <title>Fetcher functions</title>
+
+ <para>
+   When using Nix, you will frequently need to download source code
+   and other files from the internet. Nixpkgs comes with a few helper
+   functions that allow you to fetch fixed-output derivations in a
+   structured way.
+ </para>
+
+ <para>
+   The two fetcher primitives are <function>fetchurl</function> and
+   <function>fetchzip</function>. Both of these have two required
+   arguments, a URL and a hash. The hash is typically
+   <literal>sha256</literal>, although many more hash algorithms are
+   supported. Nixpkgs contributors are currently recommended to use
+   <literal>sha256</literal>. This hash will be used by Nix to
+   identify your source. A typical usage of fetchurl is provided
+   below.
+ </para>
+
+ <programlisting><![CDATA[
+{ stdenv, fetchurl }:
+
+stdenv.mkDerivation {
+  name = "hello";
+  src = fetchurl {
+    url = "http://www.example.org/hello.tar.gz";
+    sha256 = "1111111111111111111111111111111111111111111111111111";
+  };
+}
+]]></programlisting>
+
+ <para>
+   The main difference between <function>fetchurl</function> and
+   <function>fetchzip</function> is in how they store the contents.
+   <function>fetchurl</function> will store the unaltered contents of
+   the URL within the Nix store. <function>fetchzip</function> on the
+   other hand will decompress the archive for you, making files and
+   directories directly accessible in the future.
+   <function>fetchzip</function> can only be used with archives.
+   Despite the name, <function>fetchzip</function> is not limited to
+   .zip files and can also be used with any tarball.
+ </para>
+
+ <para>
+   <function>fetchpatch</function> works very similarly to
+   <function>fetchurl</function> with the same arguments expected. It
+   expects patch files as a source and and performs normalization on
+   them before computing the checksum. For example it will remove
+   comments or other unstable parts that are sometimes added by
+   version control systems and can change over time.
+ </para>
+
+ <para>
+   Other fetcher functions allow you to add source code directly from
+   a VCS such as subversion or git. These are mostly straightforward
+   names based on the name of the command used with the VCS system.
+   Because they give you a working repository, they act most like
+   <function>fetchzip</function>.
+ </para>
+
+ <variablelist>
+   <varlistentry>
+     <term>
+      <literal>fetchsvn</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with Subversion. Expects <literal>url</literal> to a
+       Subversion directory, <literal>rev</literal>, and
+       <literal>sha256</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchgit</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with Git. Expects <literal>url</literal> to a Git repo,
+       <literal>rev</literal>, and <literal>sha256</literal>.
+       <literal>rev</literal> in this case can be full the git commit
+       id (SHA1 hash) or a tag name like
+       <literal>refs/tags/v1.0</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchfossil</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with Fossil. Expects <literal>url</literal> to a Fossil
+       archive, <literal>rev</literal>, and <literal>sha256</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchcvs</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with CVS. Expects <literal>cvsRoot</literal>,
+       <literal>tag</literal>, and <literal>sha256</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchhg</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with Mercurial. Expects <literal>url</literal>,
+       <literal>rev</literal>, and <literal>sha256</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+ </variablelist>
+
+ <para>
+   A number of fetcher functions wrap part of
+   <function>fetchurl</function> and <function>fetchzip</function>.
+   They are mainly convenience functions intended for commonly used
+   destinations of source code in Nixpkgs. These wrapper fetchers are
+   listed below.
+ </para>
+
+ <variablelist>
+   <varlistentry>
+     <term>
+      <literal>fetchFromGitHub</literal>
+     </term>
+     <listitem>
+      <para>
+        <function>fetchFromGitHub</function> expects four arguments.
+        <literal>owner</literal> is a string corresponding to the
+        GitHub user or organization that controls this repository.
+        <literal>repo</literal> corresponds to the name of the
+        software repository. These are located at the top of every
+        GitHub HTML page as
+        <literal>owner</literal>/<literal>repo</literal>.
+        <literal>rev</literal> corresponds to the Git commit hash or
+        tag (e.g <literal>v1.0</literal>) that will be downloaded from
+        Git. Finally, <literal>sha256</literal> corresponds to the
+        hash of the extracted directory. Again, other hash algorithms
+        are also available but <literal>sha256</literal> is currently
+        preferred.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchFromGitLab</literal>
+     </term>
+     <listitem>
+      <para>
+       This is used with GitLab repositories. The arguments expected
+       are very similar to fetchFromGitHub above.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchFromBitbucket</literal>
+     </term>
+     <listitem>
+      <para>
+       This is used with BitBucket repositories. The arguments expected
+       are very similar to fetchFromGitHub above.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchFromSavannah</literal>
+     </term>
+     <listitem>
+      <para>
+       This is used with Savannah repositories. The arguments expected
+       are very similar to fetchFromGitHub above.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchFromRepoOrCz</literal>
+     </term>
+     <listitem>
+      <para>
+       This is used with repo.or.cz repositories. The arguments
+       expected are very similar to fetchFromGitHub above.
+      </para>
+     </listitem>
+   </varlistentry>
+ </variablelist>
+
+
+</section>
diff --git a/nixpkgs/doc/functions/library.xml b/nixpkgs/doc/functions/library.xml
index 901423c52a18..b01de3c6e413 100644
--- a/nixpkgs/doc/functions/library.xml
+++ b/nixpkgs/doc/functions/library.xml
@@ -12,4 +12,13 @@
  <xi:include href="./library/asserts.xml" />
 
  <xi:include href="./library/attrsets.xml" />
+
+ <!-- These docs are generated via nixdoc. To add another generated
+      library function file to this list, the file
+      `lib-function-docs.nix` must also be updated. -->
+ <xi:include href="./library/generated/strings.xml" />
+ <xi:include href="./library/generated/trivial.xml" />
+ <xi:include href="./library/generated/lists.xml" />
+ <xi:include href="./library/generated/debug.xml" />
+ <xi:include href="./library/generated/options.xml" />
 </section>
diff --git a/nixpkgs/doc/functions/prefer-remote-fetch.xml b/nixpkgs/doc/functions/prefer-remote-fetch.xml
new file mode 100644
index 000000000000..85f08f4eae14
--- /dev/null
+++ b/nixpkgs/doc/functions/prefer-remote-fetch.xml
@@ -0,0 +1,27 @@
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/xinclude"
+         xml:id="sec-prefer-remote-fetch">
+ <title>prefer-remote-fetch overlay</title>
+
+ <para>
+  <function>prefer-remote-fetch</function> is an overlay that download sources
+  on remote builder. This is useful when the evaluating machine has a slow
+  upload while the builder can fetch faster directly from the source.
+  To use it, put the following snippet as a new overlay:
+  <programlisting>
+    self: super:
+      (super.prefer-remote-fetch self super)
+  </programlisting>
+
+  A full configuration example for that sets the overlay up for your own account,
+  could look like this
+
+  <programlisting>
+    $ mkdir ~/.config/nixpkgs/overlays/
+    $ cat &gt; ~/.config/nixpkgs/overlays/prefer-remote-fetch.nix &lt;&lt;EOF
+      self: super: super.prefer-remote-fetch self super
+    EOF
+  </programlisting>
+ </para>
+</section>
diff --git a/nixpkgs/doc/functions/trivial-builders.xml b/nixpkgs/doc/functions/trivial-builders.xml
new file mode 100644
index 000000000000..92a07aedb5b9
--- /dev/null
+++ b/nixpkgs/doc/functions/trivial-builders.xml
@@ -0,0 +1,124 @@
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         xml:id="sec-trivial-builders">
+ <title>Trivial builders</title>
+
+ <para>
+   Nixpkgs provides a couple of functions that help with building
+   derivations. The most important one,
+   <function>stdenv.mkDerivation</function>, has already been
+   documented above. The following functions wrap
+   <function>stdenv.mkDerivation</function>, making it easier to use
+   in certain cases.
+ </para>
+
+ <variablelist>
+  <varlistentry>
+   <term>
+    <literal>runCommand</literal>
+   </term>
+   <listitem>
+     <para>
+       This takes three arguments, <literal>name</literal>,
+       <literal>env</literal>, and <literal>buildCommand</literal>.
+       <literal>name</literal> is just the name that Nix will append
+       to the store path in the same way that
+       <literal>stdenv.mkDerivation</literal> uses its
+       <literal>name</literal> attribute. <literal>env</literal> is an
+       attribute set specifying environment variables that will be set
+       for this derivation. These attributes are then passed to the
+       wrapped <literal>stdenv.mkDerivation</literal>.
+       <literal>buildCommand</literal> specifies the commands that
+       will be run to create this derivation. Note that you will need
+       to create <literal>$out</literal> for Nix to register the
+       command as successful.
+     </para>
+     <para>
+       An example of using <literal>runCommand</literal> is provided
+       below.
+     </para>
+     <programlisting>
+       (import &lt;nixpkgs&gt; {}).runCommand "my-example" {} ''
+         echo My example command is running
+
+         mkdir $out
+
+         echo I can write data to the Nix store > $out/message
+
+         echo I can also run basic commands like:
+
+         echo ls
+         ls
+
+         echo whoami
+         whoami
+
+         echo date
+         date
+       ''
+     </programlisting>
+   </listitem>
+  </varlistentry>
+  <varlistentry>
+   <term>
+    <literal>runCommandCC</literal>
+   </term>
+   <listitem>
+     <para>
+       This works just like <literal>runCommand</literal>. The only
+       difference is that it also provides a C compiler in
+       <literal>buildCommand</literal>’s environment. To minimize your
+       dependencies, you should only use this if you are sure you will
+       need a C compiler as part of running your command.
+    </para>
+   </listitem>
+  </varlistentry>
+  <varlistentry>
+   <term>
+    <literal>writeTextFile</literal>, <literal>writeText</literal>,
+    <literal>writeTextDir</literal>, <literal>writeScript</literal>,
+    <literal>writeScriptBin</literal>
+   </term>
+   <listitem>
+     <para>
+       These functions write <literal>text</literal> to the Nix store.
+       This is useful for creating scripts from Nix expressions.
+       <literal>writeTextFile</literal> takes an attribute set and
+       expects two arguments, <literal>name</literal> and
+       <literal>text</literal>. <literal>name</literal> corresponds to
+       the name used in the Nix store path. <literal>text</literal>
+       will be the contents of the file. You can also set
+       <literal>executable</literal> to true to make this file have
+       the executable bit set.
+     </para>
+     <para>
+       Many more commands wrap <literal>writeTextFile</literal>
+       including <literal>writeText</literal>,
+       <literal>writeTextDir</literal>,
+       <literal>writeScript</literal>, and
+       <literal>writeScriptBin</literal>. These are convenience
+       functions over <literal>writeTextFile</literal>.
+     </para>
+   </listitem>
+  </varlistentry>
+  <varlistentry>
+   <term>
+    <literal>symlinkJoin</literal>
+   </term>
+   <listitem>
+    <para>
+     This can be used to put many derivations into the same directory
+     structure. It works by creating a new derivation and adding
+     symlinks to each of the paths listed. It expects two arguments,
+     <literal>name</literal>, and <literal>paths</literal>.
+     <literal>name</literal> is the name used in the Nix store path
+     for the created derivation. <literal>paths</literal> is a list of
+     paths that will be symlinked. These paths can be to Nix store
+     derivations or any other subdirectory contained within.
+    </para>
+   </listitem>
+  </varlistentry>
+ </variablelist>
+
+</section>
diff --git a/nixpkgs/doc/languages-frameworks/rust.section.md b/nixpkgs/doc/languages-frameworks/rust.section.md
index 4549bbd1686b..14b36f55f52f 100644
--- a/nixpkgs/doc/languages-frameworks/rust.section.md
+++ b/nixpkgs/doc/languages-frameworks/rust.section.md
@@ -303,11 +303,15 @@ with import <nixpkgs> {};
 
 stdenv.mkDerivation {
   name = "rust-env";
-  buildInputs = [
+  nativeBuildInputs = [
     rustc cargo
 
-    # Example Additional Dependencies
-    pkgconfig openssl
+    # Example Build-time Additional Dependencies
+    pkgconfig
+  ];
+  buildInputs = [
+    # Example Run-time Additional Dependencies
+    openssl
   ];
 
   # Set Environment Variables
diff --git a/nixpkgs/doc/languages-frameworks/vim.section.md b/nixpkgs/doc/languages-frameworks/vim.section.md
index 6ed60028ae20..e4c486a0e52e 100644
--- a/nixpkgs/doc/languages-frameworks/vim.section.md
+++ b/nixpkgs/doc/languages-frameworks/vim.section.md
@@ -46,6 +46,21 @@ neovim.override {
 }
 ```
 
+If you want to use `neovim-qt` as a graphical editor, you can configure it by overriding neovim in an overlay
+or passing it an overridden neovimn:
+
+```
+neovim-qt.override {
+  neovim = neovim.override {
+    configure = {
+      customRC = ''
+        # your custom configuration
+      '';
+    };
+  };
+}
+```
+
 ## Managing plugins with Vim packages
 
 To store you plugins in Vim packages (the native vim plugin manager, see `:help packages`) the following example can be used:
diff --git a/nixpkgs/doc/lib-function-docs.nix b/nixpkgs/doc/lib-function-docs.nix
new file mode 100644
index 000000000000..421f848d25ab
--- /dev/null
+++ b/nixpkgs/doc/lib-function-docs.nix
@@ -0,0 +1,26 @@
+# Generates the documentation for library functons via nixdoc. To add
+# another library function file to this list, the include list in the
+# file `doc/functions/library.xml` must also be updated.
+
+{ pkgs ? import ./.. {}, locationsXml }:
+
+with pkgs; stdenv.mkDerivation {
+  name = "nixpkgs-lib-docs";
+  src = ./../lib;
+
+  buildInputs = [ nixdoc ];
+  installPhase = ''
+    function docgen {
+      nixdoc -c "$1" -d "$2" -f "../lib/$1.nix"  > "$out/$1.xml"
+    }
+
+    mkdir -p $out
+    ln -s ${locationsXml} $out/locations.xml
+
+    docgen strings 'String manipulation functions'
+    docgen trivial 'Miscellaneous functions'
+    docgen lists 'List manipulation functions'
+    docgen debug 'Debugging functions'
+    docgen options 'NixOS / nixpkgs option handling'
+  '';
+}
diff --git a/nixpkgs/doc/package-notes.xml b/nixpkgs/doc/package-notes.xml
index 803d343aa096..e23593107d8d 100644
--- a/nixpkgs/doc/package-notes.xml
+++ b/nixpkgs/doc/package-notes.xml
@@ -307,23 +307,19 @@ packageOverrides = pkgs: {
 </screen>
   </para>
  </section>
+
  <section xml:id="sec-elm">
   <title>Elm</title>
 
   <para>
-   The Nix expressions for Elm reside in
-   <filename>pkgs/development/compilers/elm</filename>. They are generated
-   automatically by <command>update-elm.rb</command> script. One should specify
-   versions of Elm packages inside the script, clear the
-   <filename>packages</filename> directory and run the script from inside it.
-   <literal>elm-reactor</literal> is special because it also has Elm package
-   dependencies. The process is not automated very much for now -- you should
-   get the <literal>elm-reactor</literal> source tree (e.g. with
-   <command>nix-shell</command>) and run <command>elm2nix.rb</command> inside
-   it. Place the resulting <filename>package.nix</filename> file into
-   <filename>packages/elm-reactor-elm.nix</filename>.
+   To update Elm compiler, see <filename>nixpkgs/pkgs/development/compilers/elm/README.md</filename>.
+  </para>
+
+  <para>
+   To package Elm applications, <link xlink:href="https://github.com/hercules-ci/elm2nix#elm2nix">read about elm2nix</link>.
   </para>
  </section>
+
  <section xml:id="sec-shell-helpers">
   <title>Interactive shell helpers</title>
 
diff --git a/nixpkgs/doc/reviewing-contributions.xml b/nixpkgs/doc/reviewing-contributions.xml
index 5618567e3852..f541b7f22daa 100644
--- a/nixpkgs/doc/reviewing-contributions.xml
+++ b/nixpkgs/doc/reviewing-contributions.xml
@@ -605,11 +605,11 @@ policy.
 -->
 
   <para>
-   In a case a contributor leaves definitively the Nix community, he should
+   In a case a contributor definitively leaves the Nix community, they should
    create an issue or post on
    <link
    xlink:href="https://discourse.nixos.org">Discourse</link> with
-   references of packages and modules he maintains so the maintainership can be
+   references of packages and modules they maintain so the maintainership can be
    taken over by other contributors.
   </para>
  </section>
diff --git a/nixpkgs/doc/stdenv.xml b/nixpkgs/doc/stdenv.xml
index 1c18fab86696..3a51907eb8a6 100644
--- a/nixpkgs/doc/stdenv.xml
+++ b/nixpkgs/doc/stdenv.xml
@@ -1279,7 +1279,9 @@ makeFlags = [ "PREFIX=$(out)" ];
        <command>make</command>. You must use this instead of
        <varname>makeFlags</varname> if the arguments contain spaces, e.g.
 <programlisting>
-makeFlagsArray=(CFLAGS="-O0 -g" LDFLAGS="-lfoo -lbar")
+preBuild = ''
+  makeFlagsArray+=(CFLAGS="-O0 -g" LDFLAGS="-lfoo -lbar")
+'';
 </programlisting>
        Note that shell arrays cannot be passed through environment variables,
        so you cannot set <varname>makeFlagsArray</varname> in a derivation
@@ -2190,10 +2192,130 @@ addEnvHooks "$hostOffset" myBashFunction
   </para>
 
   <para>
-   Here are some packages that provide a setup hook. Since the mechanism is
-   modular, this probably isn't an exhaustive list. Then again, since the
-   mechanism is only to be used as a last resort, it might be.
-   <variablelist>
+    First, let’s cover some setup hooks that are part of Nixpkgs
+    default stdenv. This means that they are run for every package
+    built using <function>stdenv.mkDerivation</function>. Some of
+    these are platform specific, so they may run on Linux but not
+    Darwin or vice-versa.
+    <variablelist>
+    <varlistentry>
+     <term>
+      <literal>move-docs.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook moves any installed documentation to the
+       <literal>/share</literal> subdirectory directory. This includes
+       the man, doc and info directories. This is needed for legacy
+       programs that do not know how to use the
+       <literal>share</literal> subdirectory.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>compress-man-pages.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook compresses any man pages that have been
+       installed. The compression is done using the gzip program. This
+       helps to reduce the installed size of packages.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>strip.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This runs the strip command on installed binaries and
+       libraries. This removes unnecessary information like debug
+       symbols when they are not needed. This also helps to reduce the
+       installed size of packages.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>patch-shebangs.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook patches installed scripts to use the full path
+       to the shebang interpreter. A shebang interpreter is the first
+       commented line of a script telling the operating system which
+       program will run the script (e.g <literal>#!/bin/bash</literal>). In
+       Nix, we want an exact path to that interpreter to be used. This
+       often replaces <literal>/bin/sh</literal> with a path in the
+       Nix store.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>audit-tmpdir.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This verifies that no references are left from the install
+       binaries to the directory used to build those binaries. This
+       ensures that the binaries do not need things outside the Nix
+       store. This is currently supported in Linux only.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>multiple-outputs.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook adds configure flags that tell packages to
+       install files into any one of the proper outputs listed in
+       <literal>outputs</literal>. This behavior can be turned off by setting
+       <literal>setOutputFlags</literal> to false in the derivation
+       environment. See <xref linkend="chap-multiple-output"/> for
+       more information.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>move-sbin.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook moves any binaries installed in the sbin
+       subdirectory into bin. In addition, a link is provided from
+       sbin to bin for compatibility.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>move-lib64.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook moves any libraries installed in the lib64
+       subdirectory into lib. In addition, a link is provided from
+       lib64 to lib for compatibility.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>set-source-date-epoch-to-latest.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This sets <literal>SOURCE_DATE_EPOCH</literal> to the
+       modification time of the most recent file.
+       </para>
+     </listitem>
+    </varlistentry>
     <varlistentry>
      <term>
       Bintools Wrapper
@@ -2300,6 +2422,15 @@ addEnvHooks "$hostOffset" myBashFunction
       </para>
      </listitem>
     </varlistentry>
+    </variablelist>
+  </para>
+
+  <para>
+   Here are some more packages that provide a setup hook. Since the
+   list of hooks is extensible, this is not an exhaustive list the
+   mechanism is only to be used as a last resort, it might cover most
+   uses.
+   <variablelist>
     <varlistentry>
      <term>
       Perl
@@ -2435,30 +2566,6 @@ addEnvHooks "$hostOffset" myBashFunction
     </varlistentry>
     <varlistentry>
      <term>
-      paxctl
-     </term>
-     <listitem>
-      <para>
-       Defines the <varname>paxmark</varname> helper for setting per-executable
-       PaX flags on Linux (where it is available by default; on all other
-       platforms, <varname>paxmark</varname> is a no-op). For example, to
-       disable secure memory protections on the executable
-       <replaceable>foo</replaceable>
-<programlisting>
-      postFixup = ''
-        paxmark m $out/bin/<replaceable>foo</replaceable>
-      '';
-    </programlisting>
-       The <literal>m</literal> flag is the most common flag and is typically
-       required for applications that employ JIT compilation or otherwise need
-       to execute code generated at run-time. Disabling PaX protections should
-       be considered a last resort: if possible, problematic features should be
-       disabled or patched to work with PaX.
-      </para>
-     </listitem>
-    </varlistentry>
-    <varlistentry>
-     <term>
       autoPatchelfHook
      </term>
      <listitem>
diff --git a/nixpkgs/doc/style.css b/nixpkgs/doc/style.css
index 0db907815b6b..474dd32e3fb1 100644
--- a/nixpkgs/doc/style.css
+++ b/nixpkgs/doc/style.css
@@ -9,6 +9,7 @@
 body
 {
     font-family: "Nimbus Sans L", sans-serif;
+    font-size: 1em;
     background: white;
     margin: 2em 1em 2em 1em;
 }
@@ -28,6 +29,25 @@ h2 /* chapters, appendices, subtitle */
     font-size: 180%;
 }
 
+div.book
+{
+    text-align: center;
+}
+
+div.book > div
+{
+    /*
+     * based on https://medium.com/@zkareemz/golden-ratio-62b3b6d4282a
+     * we do 70 characters per line to fit code listings better
+     * 70 * (font-size / 1.618)
+     * expression for emacs:
+     * (* 70 (/ 1 1.618))
+     */
+    max-width: 43.2em;
+    text-align: left;
+    margin: auto;
+}
+
 /* Extra space between chapters, appendices. */
 div.chapter > div.titlepage h2, div.appendix > div.titlepage h2
 {
@@ -102,8 +122,8 @@ pre.screen, pre.programlisting
 {
     border: 1px solid #b0b0b0;
     padding: 3px 3px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
+    margin-left: 0.5em;
+    margin-right: 0.5em;
 
     background: #f4f4f8;
     font-family: monospace;