Merge commit 'b3c692172e5b5241b028a98e1977f9fb12eeaf42'

11 files changed, 422 insertions, 121 deletions

diff --git a/nixpkgs/doc/languages-frameworks/beam.section.md b/nixpkgs/doc/languages-frameworks/beam.section.md
index 934e6cdca121..fb608932dfc3 100644
--- a/nixpkgs/doc/languages-frameworks/beam.section.md
+++ b/nixpkgs/doc/languages-frameworks/beam.section.md
@@ -68,74 +68,128 @@ Erlang.mk functions similarly to Rebar3, except we use `buildErlangMk` instead o
 
 `mixRelease` is used to make a release in the mix sense. Dependencies will need to be fetched with `fetchMixDeps` and passed to it.
 
-#### mixRelease - Elixir Phoenix example {#mixrelease---elixir-phoenix-example}
+#### mixRelease - Elixir Phoenix example {#mix-release-elixir-phoenix-example}
 
-Here is how your `default.nix` file would look.
+there are 3 steps, frontend dependencies (javascript), backend dependencies (elixir) and the final derivation that puts both of those together
+
+##### mixRelease - Frontend dependencies (javascript) {#mix-release-javascript-deps}
+
+for phoenix projects, inside of nixpkgs you can either use yarn2nix (mkYarnModule) or node2nix. An example with yarn2nix can be found [here](https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/web-apps/plausible/default.nix#L39). An example with node2nix will follow. To package something outside of nixpkgs, you have alternatives like [npmlock2nix](https://github.com/nix-community/npmlock2nix) or [nix-npm-buildpackage](https://github.com/serokell/nix-npm-buildpackage)
+
+##### mixRelease - backend dependencies (mix) {#mix-release-mix-deps}
+
+There are 2 ways to package backend dependencies. With mix2nix and with a fixed-output-derivation (FOD).
+
+###### mix2nix {#mix2nix}
+
+mix2nix is a cli tool available in nixpkgs. it will generate a nix expression from a mix.lock file. It is quite standard in the 2nix tool series.
+
+Note that currently mix2nix can't handle git dependencies inside the mix.lock file. If you have git dependencies, you can either add them manually (see [example](https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/pleroma/default.nix#L20)) or use the FOD method.
+
+The advantage of using mix2nix is that nix will know your whole dependency graph. On a dependency update, this won't trigger a full rebuild and download of all the dependencies, where FOD will do so.
+
+practical steps:
+
+- run `mix2nix > mix_deps.nix` in the upstream repo.
+- pass `mixNixDeps = with pkgs; import ./mix_deps.nix { inherit lib beamPackages; };` as an argument to mixRelease.
+
+If there are git depencencies.
+
+- You'll need to fix the version artificially in mix.exs and regenerate the mix.lock with fixed version (on upstream). This will enable you to run `mix2nix > mix_deps.nix`.
+- From the mix_deps.nix file, remove the dependencies that had git versions and pass them as an override to the import function.
 
 ```nix
-with import <nixpkgs> { };
+  mixNixDeps = import ./mix.nix {
+    inherit beamPackages lib;
+    overrides = (final: prev: {
+      # mix2nix does not support git dependencies yet,
+      # so we need to add them manually
+      prometheus_ex = beamPackages.buildMix rec {
+        name = "prometheus_ex";
+        version = "3.0.5";
+
+        # Change the argument src with the git src that you actually need
+        src = fetchFromGitLab {
+          domain = "git.pleroma.social";
+          group = "pleroma";
+          owner = "elixir-libraries";
+          repo = "prometheus.ex";
+          rev = "a4e9beb3c1c479d14b352fd9d6dd7b1f6d7deee5";
+          sha256 = "1v0q4bi7sb253i8q016l7gwlv5562wk5zy3l2sa446csvsacnpjk";
+        };
+        # you can re-use the same beamDeps argument as generated
+        beamDeps = with final; [ prometheus ];
+      };
+  });
+};
+```
 
-let
-  packages = beam.packagesWith beam.interpreters.erlang;
-  src = builtins.fetchgit {
-    url = "ssh://git@github.com/your_id/your_repo";
-    rev = "replace_with_your_commit";
-  };
+You will need to run the build process once to fix the sha256 to correspond to your new git src.
 
-  pname = "your_project";
-  version = "0.0.1";
-  mixEnv = "prod";
+###### FOD {#fixed-output-derivation}
 
-  mixFodDeps = packages.fetchMixDeps {
+A fixed output derivation will download mix dependencies from the internet. To ensure reproducibility, a hash will be supplied. Note that mix is relatively reproducible. An FOD generating a different hash on each run hasn't been observed (as opposed to npm where the chances are relatively high). See [elixir_ls](https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/beam-modules/elixir_ls.nix) for a usage example of FOD.
+
+Practical steps
+
+- start with the following argument to mixRelease
+
+```nix
+  mixFodDeps = fetchMixDeps {
     pname = "mix-deps-${pname}";
-    inherit src mixEnv version;
-    # nix will complain and tell you the right value to replace this with
+    inherit src version;
     sha256 = lib.fakeSha256;
-    # if you have build time environment variables add them here
-    MY_ENV_VAR="my_value";
   };
+```
 
-  nodeDependencies = (pkgs.callPackage ./assets/default.nix { }).shell.nodeDependencies;
+The first build will complain about the sha256 value, you can replace with the suggested value after that.
 
-  frontEndFiles = stdenvNoCC.mkDerivation {
-    pname = "frontend-${pname}";
+Note that if after you've replaced the value, nix suggests another sha256, then mix is not fetching the dependencies reproducibly. An FOD will not work in that case and you will have to use mix2nix.
 
-    nativeBuildInputs = [ nodejs ];
+##### mixRelease - example {#mix-release-example}
 
-    inherit version src;
+Here is how your `default.nix` file would look for a phoenix project.
 
-    buildPhase = ''
-      cp -r ./assets $TEMPDIR
+```nix
+with import <nixpkgs> { };
 
-      mkdir -p $TEMPDIR/assets/node_modules/.cache
-      cp -r ${nodeDependencies}/lib/node_modules $TEMPDIR/assets
-      export PATH="${nodeDependencies}/bin:$PATH"
+let
+  # beam.interpreters.erlangR23 is available if you need a particular version
+  packages = beam.packagesWith beam.interpreters.erlang;
 
-      cd $TEMPDIR/assets
-      webpack --config ./webpack.config.js
-      cd ..
-    '';
+  pname = "your_project";
+  version = "0.0.1";
 
-    installPhase = ''
-      cp -r ./priv/static $out/
-    '';
+  src = builtins.fetchgit {
+    url = "ssh://git@github.com/your_id/your_repo";
+    rev = "replace_with_your_commit";
+  };
 
-    outputHashAlgo = "sha256";
-    outputHashMode = "recursive";
+  # if using mix2nix you can use the mixNixDeps attribute
+  mixFodDeps = packages.fetchMixDeps {
+    pname = "mix-deps-${pname}";
+    inherit src version;
     # nix will complain and tell you the right value to replace this with
-    outputHash = lib.fakeSha256;
-
-    impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+    sha256 = lib.fakeSha256;
+    # if you have build time environment variables add them here
+    MY_ENV_VAR="my_value";
   };
 
+  nodeDependencies = (pkgs.callPackage ./assets/default.nix { }).shell.nodeDependencies;
 
 in packages.mixRelease {
-  inherit src pname version mixEnv mixFodDeps;
+  inherit src pname version mixFodDeps;
   # if you have build time environment variables add them here
   MY_ENV_VAR="my_value";
-  preInstall = ''
-    mkdir -p ./priv/static
-    cp -r ${frontEndFiles} ./priv/static
+
+  postBuild = ''
+    ln -sf ${nodeDependencies}/lib/node_modules assets/node_modules
+    npm run deploy --prefix ./assets
+
+    # for external task you need a workaround for the no deps check flag
+    # https://github.com/phoenixframework/phoenix/issues/2690
+    mix do deps.loadpaths --no-deps-check, phx.digest
+    mix phx.digest --no-deps-check
   '';
 }
 ```
@@ -165,6 +219,8 @@ in
   systemd.services.${release_name} = {
     wantedBy = [ "multi-user.target" ];
     after = [ "network.target" "postgresql.service" ];
+    # note that if you are connecting to a postgres instance on a different host
+    # postgresql.service should not be included in the requires.
     requires = [ "network-online.target" "postgresql.service" ];
     description = "my app";
     environment = {
@@ -201,6 +257,7 @@ in
     path = [ pkgs.bash ];
   };
 
+  # in case you have migration scripts or you want to use a remote shell
   environment.systemPackages = [ release ];
 }
 ```
@@ -212,19 +269,14 @@ in
 Usually, we need to create a `shell.nix` file and do our development inside of the environment specified therein. Just install your version of Erlang and any other interpreters, and then use your normal build tools. As an example with Elixir:
 
 ```nix
-{ pkgs ? import "<nixpkgs"> {} }:
+{ pkgs ? import <nixpkgs> {} }:
 
 with pkgs;
-
 let
-
-  elixir = beam.packages.erlangR22.elixir_1_9;
-
+  elixir = beam.packages.erlangR24.elixir_1_12;
 in
 mkShell {
   buildInputs = [ elixir ];
-
-  ERL_INCLUDE_PATH="${erlang}/lib/erlang/usr/include";
 }
 ```
 
@@ -264,6 +316,7 @@ let
     # TODO: not sure how to make hex available without installing it afterwards.
     mix local.hex --if-missing
     export LANG=en_US.UTF-8
+    # keep your shell history in iex
     export ERL_AFLAGS="-kernel shell_history enabled"
 
     # postges related
diff --git a/nixpkgs/doc/languages-frameworks/coq.section.md b/nixpkgs/doc/languages-frameworks/coq.section.md
index 0674c5a4702e..39b60d83ac7d 100644
--- a/nixpkgs/doc/languages-frameworks/coq.section.md
+++ b/nixpkgs/doc/languages-frameworks/coq.section.md
@@ -33,6 +33,7 @@ The recommended way of defining a derivation for a Coq library, is to use the `c
 * `mlPlugin` (optional, defaults to `false`). Some extensions (plugins) might require OCaml and sometimes other OCaml packages. Standard dependencies can be added by setting the current option to `true`. For a finer grain control, the `coq.ocamlPackages` attribute can be used in `extraBuildInputs` to depend on the same package set Coq was built against.
 * `useDune2ifVersion` (optional, default to `(x: false)` uses Dune2 to build the package if the provided predicate evaluates to true on the version, e.g. `useDune2if = versions.isGe "1.1"`  will use dune if the version of the package is greater or equal to `"1.1"`,
 * `useDune2` (optional, defaults to `false`) uses Dune2 to build the package if set to true, the presence of this attribute overrides the behavior of the previous one.
+* `opam-name` (optional, defaults to `coq-` followed by the value of `pname`), name of the Dune package to build.
 * `enableParallelBuilding` (optional, defaults to `true`), since it is activated by default, we provide a way to disable it.
 * `extraInstallFlags` (optional), allows to extend `installFlags` which initializes the variable `COQMF_COQLIB` so as to install in the proper subdirectory. Indeed Coq libraries should be installed in `$(out)/lib/coq/${coq.coq-version}/user-contrib/`. Such directories are automatically added to the `$COQPATH` environment variable by the hook defined in the Coq derivation.
 * `setCOQBIN` (optional, defaults to `true`), by default, the environment variable `$COQBIN` is set to the current Coq's binary, but one can disable this behavior by setting it to `false`,
diff --git a/nixpkgs/doc/languages-frameworks/go.section.md b/nixpkgs/doc/languages-frameworks/go.section.md
index b20a8d0c3549..45d85f1f99ec 100644
--- a/nixpkgs/doc/languages-frameworks/go.section.md
+++ b/nixpkgs/doc/languages-frameworks/go.section.md
@@ -13,6 +13,7 @@ In the following is an example expression using `buildGoModule`, the following a
 
 - `vendorSha256`: is the hash of the output of the intermediate fetcher derivation. `vendorSha256` can also take `null` as an input. When `null` is used as a value, rather than fetching the dependencies and vendoring them, we use the vendoring included within the source repo. If you'd like to not have to update this field on dependency changes, run `go mod vendor` in your source repo and set `vendorSha256 = null;`
 - `runVend`: runs the vend command to generate the vendor directory. This is useful if your code depends on c code and go mod tidy does not include the needed sources to build.
+- `proxyVendor`: Fetches (go mod download) and proxies the vendor directory. This is useful if any dependency has case-insensitive conflicts which will produce platform dependant `vendorSha256` checksums.
 
 ```nix
 pet = buildGoModule rec {
@@ -112,16 +113,6 @@ done
 
 Both `buildGoModule` and `buildGoPackage` can be tweaked to behave slightly differently, if the following attributes are used:
 
-### `buildFlagsArray` and `buildFlags`: {#ex-goBuildFlags-noarray}
-
-These attributes set build flags supported by `go build`. We recommend using `buildFlagsArray`.
-
-```nix
-  buildFlagsArray = [
-    "-tags=release"
-  ];
-```
-
 ### `ldflags` {#var-go-ldflags}
 
 Arguments to pass to the Go linker tool via the `-ldflags` argument of `go build`. The most common use case for this argument is to make the resulting executable aware of its own version. For example:
@@ -134,6 +125,21 @@ Arguments to pass to the Go linker tool via the `-ldflags` argument of `go build
   ];
 ```
 
+### `tags` {#var-go-tags}
+
+Arguments to pass to the Go via the `-tags` argument of `go build`. For example:
+
+```nix
+  tags = [
+    "production"
+    "sqlite"
+  ];
+```
+
+```nix
+  tags = [ "production" ] ++ lib.optionals withSqlite [ "sqlite" ];
+```
+
 ### `deleteVendor` {#var-go-deleteVendor}
 
 Removes the pre-existing vendor directory. This should only be used if the dependencies included in the vendor folder are broken or incomplete.
diff --git a/nixpkgs/doc/languages-frameworks/index.xml b/nixpkgs/doc/languages-frameworks/index.xml
index 516bddf67fd4..49cdf94a44ab 100644
--- a/nixpkgs/doc/languages-frameworks/index.xml
+++ b/nixpkgs/doc/languages-frameworks/index.xml
@@ -20,9 +20,9 @@
  <xi:include href="idris.section.xml" />
  <xi:include href="ios.section.xml" />
  <xi:include href="java.section.xml" />
+ <xi:include href="javascript.section.xml" />
  <xi:include href="lua.section.xml" />
  <xi:include href="maven.section.xml" />
- <xi:include href="node.section.xml" />
  <xi:include href="ocaml.section.xml" />
  <xi:include href="perl.section.xml" />
  <xi:include href="php.section.xml" />
diff --git a/nixpkgs/doc/languages-frameworks/javascript.section.md b/nixpkgs/doc/languages-frameworks/javascript.section.md
new file mode 100644
index 000000000000..bf5742d6855e
--- /dev/null
+++ b/nixpkgs/doc/languages-frameworks/javascript.section.md
@@ -0,0 +1,256 @@
+# Javascript {#language-javascript}
+
+## Introduction {#javascript-introduction}
+
+This contains instructions on how to package javascript applications.
+
+The various tools available will be listed in the [tools-overview](#javascript-tools-overview). Some general principles for packaging will follow. Finally some tool specific instructions will be given.
+
+## Getting unstuck / finding code examples
+
+If you find you are lacking inspiration for packing javascript applications, the links below might prove useful.
+Searching online for prior art can be helpful if you are running into solved problems.
+
+### Github
+
+- Searching Nix files for `mkYarnPackage`: <https://github.com/search?q=mkYarnPackage+language%3ANix&type=code>
+
+- Searching just `flake.nix` files for `mkYarnPackage`: <https://github.com/search?q=mkYarnPackage+filename%3Aflake.nix&type=code>
+
+### Gitlab
+
+- Searching Nix files for `mkYarnPackage`: <https://gitlab.com/search?scope=blobs&search=mkYarnPackage+extension%3Anix>
+
+- Searching just `flake.nix` files for `mkYarnPackage`: <https://gitlab.com/search?scope=blobs&search=mkYarnPackage+filename%3Aflake.nix>
+
+## Tools overview {#javascript-tools-overview}
+
+## General principles {#javascript-general-principles}
+
+The following principles are given in order of importance with potential exceptions.
+
+### Try to use the same node version used upstream {#javascript-upstream-node-version}
+
+It is often not documented which node version is used upstream, but if it is, try to use the same version when packaging.
+
+This can be a problem if upstream is using the latest and greatest and you are trying to use an earlier version of node. Some cryptic errors regarding V8 may appear.
+
+An exception to this:
+
+### Try to respect the package manager originally used by upstream (and use the upstream lock file) {#javascript-upstream-package-manager}
+
+A lock file (package-lock.json, yarn.lock...) is supposed to make reproducible installations of node_modules for each tool.
+
+Guidelines of package managers, recommend to commit those lock files to the repos. If a particular lock file is present, it is a strong indication of which package manager is used upstream.
+
+It's better to try to use a nix tool that understand the lock file. Using a different tool might give you hard to understand error because different packages have been installed. An example of problems that could arise can be found [here](https://github.com/NixOS/nixpkgs/pull/126629). Upstream uses npm, but this is an attempt to package it with yarn2nix (that uses yarn.lock)
+
+Using a different tool forces to commit a lock file to the repository. Those files are fairly large, so when packaging for nixpkgs, this approach does not scale well.
+
+Exceptions to this rule are:
+
+- when you encounter one of the bugs from a nix tool. In each of the tool specific instructions, known problems will be detailed. If you have a problem with a particular tool, then it's best to try another tool, even if this means you will have to recreate a lock file and commit it to nixpkgs. In general yarn2nix has less known problems and so a simple search in nixpkgs will reveal many yarn.lock files committed
+- Some lock files contain particular version of a package that has been pulled off npm for some reason. In that case, you can recreate upstream lock (by removing the original and `npm install`, `yarn`, ...) and commit this to nixpkgs.
+- The only tool that supports workspaces (a feature of npm that helps manage sub-directories with different package.json from a single top level package.json) is yarn2nix. If upstream has workspaces you should try yarn2nix.
+
+### Try to use upstream package.json {#javascript-upstream-package-json}
+
+Exceptions to this rule are
+
+- Sometimes the upstream repo assumes some dependencies be installed globally. In that case you can add them manually to the upstream package.json (`yarn add xxx` or `npm install xxx`, ...). Dependencies that are installed locally can be executed with `npx` for cli tools. (e.g. `npx postcss ...`, this is how you can call those dependencies in the phases).
+- Sometimes there is a version conflict between some dependency requirements. In that case you can fix a version (by removing the `^`).
+- Sometimes the script defined in the package.json does not work as is. Some scripts for example use cli tools that might not be available, or cd in directory with a different package.json (for workspaces notably). In that case, it's perfectly fine to look at what the particular script is doing and break this down in the phases. In the build script you can see `build:*` calling in turns several other build scripts like `build:ui` or `build:server`. If one of those fails, you can try to separate those into:
+
+```Shell
+yarn build:ui
+yarn build:server
+# OR
+npm run build:ui
+npm run build:server
+```
+
+when you need to override a package.json. It's nice to use the one from the upstream src and do some explicit override. Here is an example.
+
+```nix
+patchedPackageJSON = final.runCommand "package.json" { } ''
+  ${jq}/bin/jq '.version = "0.4.0" |
+    .devDependencies."@jsdoc/cli" = "^0.2.5"
+    ${sonar-src}/package.json > $out
+'';
+```
+
+you will still need to commit the modified version of the lock files, but at least the overrides are explicit for everyone to see.
+
+### Using node_modules directly {#javascript-using-node_modules}
+
+each tool has an abstraction to just build the node_modules (dependencies) directory. you can always use the stdenv.mkDerivation with the node_modules to build the package (symlink the node_modules directory and then use the package build command). the node_modules abstraction can be also used to build some web framework frontends. For an example of this see how [plausible](https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/web-apps/plausible/default.nix) is built. mkYarnModules to make the derivation containing node_modules. Then when building the frontend you can just symlink the node_modules directory
+
+## javascript packages inside nixpkgs {#javascript-packages-nixpkgs}
+
+The `pkgs/development/node-packages` folder contains a generated collection of
+[NPM packages](https://npmjs.com/) that can be installed with the Nix package
+manager.
+
+As a rule of thumb, the package set should only provide _end user_ software
+packages, such as command-line utilities. Libraries should only be added to the
+package set if there is a non-NPM package that requires it.
+
+When it is desired to use NPM libraries in a development project, use the
+`node2nix` generator directly on the `package.json` configuration file of the
+project.
+
+The package set provides support for the official stable Node.js versions.
+The latest stable LTS release in `nodePackages`, as well as the latest stable
+Current release in `nodePackages_latest`.
+
+If your package uses native addons, you need to examine what kind of native
+build system it uses. Here are some examples:
+
+- `node-gyp`
+- `node-gyp-builder`
+- `node-pre-gyp`
+
+After you have identified the correct system, you need to override your package
+expression while adding in build system as a build input. For example, `dat`
+requires `node-gyp-build`, so [we override](https://github.com/NixOS/nixpkgs/blob/32f5e5da4a1b3f0595527f5195ac3a91451e9b56/pkgs/development/node-packages/default.nix#L37-L40) its expression in [`default.nix`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/node-packages/default.nix):
+
+```nix
+    dat = super.dat.override {
+      buildInputs = [ self.node-gyp-build pkgs.libtool pkgs.autoconf pkgs.automake ];
+      meta.broken = since "12";
+    };
+```
+
+To add a package from NPM to nixpkgs:
+
+1. Modify `pkgs/development/node-packages/node-packages.json` to add, update
+    or remove package entries to have it included in `nodePackages` and
+    `nodePackages_latest`.
+2. Run the script: `cd pkgs/development/node-packages && ./generate.sh`.
+3. Build your new package to test your changes:
+    `cd /path/to/nixpkgs && nix-build -A nodePackages.<new-or-updated-package>`.
+    To build against the latest stable Current Node.js version (e.g. 14.x):
+    `nix-build -A nodePackages_latest.<new-or-updated-package>`
+4. Add and commit all modified and generated files.
+
+For more information about the generation process, consult the
+[README.md](https://github.com/svanderburg/node2nix) file of the `node2nix`
+tool.
+
+## Tool specific instructions {#javascript-tool-specific}
+
+### node2nix {#javascript-node2nix}
+
+#### Preparation {#javascript-node2nix-preparation}
+
+you will need to generate a nix expression for the dependencies
+
+- don't forget the `-l package-lock.json` if there is a lock file
+- Most probably you will need the `--development` to include the `devDependencies`
+
+so the command will most likely be
+`node2nix --development -l package-lock.json`
+
+[link to the doc in the repo](https://github.com/svanderburg/node2nix)
+
+#### Pitfalls {#javascript-node2nix-pitfalls}
+
+- if upstream package.json does not have a "version" attribute, node2nix will crash. You will need to add it like shown in [the package.json section](#javascript-upstream-package-json)
+- node2nix has some [bugs](https://github.com/svanderburg/node2nix/issues/238). related to working with lock files from npm distributed with nodejs-16_x
+- node2nix does not like missing packages from npm. If you see something like `Cannot resolve version: vue-loader-v16@undefined` then you might want to try another tool. The package might have been pulled off of npm.
+
+### yarn2nix {#javascript-yarn2nix}
+
+#### Preparation {#javascript-yarn2nix-preparation}
+
+you will need at least a yarn.lock and yarn.nix file
+
+- generate a yarn.lock in upstream if it is not already there
+- `yarn2nix > yarn.nix` will generate the dependencies in a nix format
+
+#### mkYarnPackage {#javascript-yarn2nix-mkYarnPackage}
+
+this will by default try to generate a binary. For package only generating static assets (Svelte, Vue, React...), you will need to explicitly override the build step with your instructions. It's important to use the `--offline` flag. For example if you script is `"build": "something"` in package.json use
+
+```nix
+buildPhase = ''
+  yarn build --offline
+'';
+```
+
+The dist phase is also trying to build a binary, the only way to override it is with
+
+```nix
+distPhase = "true";
+```
+
+the configure phase can sometimes fail because it tries to be too clever.
+One common override is
+
+```nix
+configurePhase = "ln -s $node_modules node_modules";
+```
+
+#### mkYarnModules {#javascript-yarn2nix-mkYarnModules}
+
+this will generate a derivation including the node_modules. If you have to build a derivation for an integrated web framework (rails, phoenix..), this is probably the easiest way. [Plausible](https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/web-apps/plausible/default.nix#L39) offers a good example of how to do this.
+
+#### Overriding dependency behavior
+
+In the `mkYarnPackage` record the property `pkgConfig` can be used to override packages when you encounter problems building.
+
+For instance, say your package is throwing errors when trying to invoke node-sass: `ENOENT: no such file or directory, scandir '/build/source/node_modules/node-sass/vendor'`
+
+To fix this we will specify different versions of build inputs to use, as well as some post install steps to get the software built the way we want:
+
+```nix
+mkYarnPackage rec {
+  pkgConfig = {
+    node-sass = {
+      buildInputs = with final;[ python libsass pkg-config ];
+      postInstall = ''
+        LIBSASS_EXT=auto yarn --offline run build
+        rm build/config.gypi
+      '';
+    };
+  };
+}
+```
+
+#### Pitfalls {#javascript-yarn2nix-pitfalls}
+
+- if version is missing from upstream package.json, yarn will silently install nothing. In that case, you will need to override package.json as shown in the [package.json section](#javascript-upstream-package-json)
+
+- having trouble with node-gyp? Try adding these lines to the `yarnPreBuild` steps:
+
+  ```nix
+  yarnPreBuild = ''
+    mkdir -p $HOME/.node-gyp/${nodejs.version}
+    echo 9 > $HOME/.node-gyp/${nodejs.version}/installVersion
+    ln -sfv ${nodejs}/include $HOME/.node-gyp/${nodejs.version}
+    export npm_config_nodedir=${nodejs}
+  '';
+  ```
+
+  - The `echo 9` steps comes from this answer: <https://stackoverflow.com/a/49139496>
+  - Exporting the headers in `npm_config_nodedir` comes from this issue: <https://github.com/nodejs/node-gyp/issues/1191#issuecomment-301243919>
+
+## Outside of nixpkgs {#javascript-outside-nixpkgs}
+
+There are some other options available that can't be used inside nixpkgs. Those other options are written in nix. Importing them in nixpkgs will require moving the source code into nixpkgs. Using [Import From Derivation](https://nixos.wiki/wiki/Import_From_Derivation) is not allowed in hydra at present. If you are packaging something outside nixpkgs, those can be considered
+
+### npmlock2nix {#javascript-npmlock2nix}
+
+[npmlock2nix](https://github.com/nix-community/npmlock2nix) aims at building node_modules without code generation. It hasn't reached v1 yet, the api might be subject to change.
+
+#### Pitfalls {#javascript-npmlock2nix-pitfalls}
+
+- there are some [problems with npm v7](https://github.com/tweag/npmlock2nix/issues/45).
+
+### nix-npm-buildpackage {#javascript-nix-npm-buildpackage}
+
+[nix-npm-buildpackage](https://github.com/serokell/nix-npm-buildpackage) aims at building node_modules without code generation. It hasn't reached v1 yet, the api might change. It supports both package-lock.json and yarn.lock.
+
+#### Pitfalls {#javascript-nix-npm-buildpackage-pitfalls}
+
+- there are some [problems with npm v7](https://github.com/serokell/nix-npm-buildpackage/issues/33).
diff --git a/nixpkgs/doc/languages-frameworks/lua.section.md b/nixpkgs/doc/languages-frameworks/lua.section.md
index ea893ce3a4a7..41fe0de04b4d 100644
--- a/nixpkgs/doc/languages-frameworks/lua.section.md
+++ b/nixpkgs/doc/languages-frameworks/lua.section.md
@@ -139,11 +139,9 @@ the whitelist maintainers/scripts/luarocks-packages.csv and updated by running m
 [luarocks2nix](https://github.com/nix-community/luarocks) is a tool capable of generating nix derivations from both rockspec and src.rock (and favors the src.rock).
 The automation only goes so far though and some packages need to be customized.
 These customizations go in `pkgs/development/lua-modules/overrides.nix`.
-For instance if the rockspec defines `external_dependencies`, these need to be manually added in its rockspec file then it won't work.
+For instance if the rockspec defines `external_dependencies`, these need to be manually added to the overrides.nix.
 
 You can try converting luarocks packages to nix packages with the command `nix-shell -p luarocks-nix` and then `luarocks nix PKG_NAME`.
-Nix rely on luarocks to install lua packages, basically it runs:
-`luarocks make --deps-mode=none --tree $out`
 
 #### Packaging a library manually {#packaging-a-library-manually}
 
@@ -161,8 +159,8 @@ are not packaged for luarocks. You can see a few examples at `pkgs/top-level/lua
 
 ### Lua interpreters {#lua-interpreters}
 
-Versions 5.1, 5.2 and 5.3 of the lua interpreter are available as
-respectively `lua5_1`, `lua5_2` and `lua5_3`. Luajit is available too.
+Versions 5.1, 5.2, 5.3 and 5.4 of the lua interpreter are available as
+respectively `lua5_1`, `lua5_2`, `lua5_3` and `lua5_4`. Luajit is available too.
 The Nix expressions for the interpreters can be found in `pkgs/development/interpreters/lua-5`.
 
 #### Attributes on lua interpreters packages {#attributes-on-lua-interpreters-packages}
diff --git a/nixpkgs/doc/languages-frameworks/node.section.md b/nixpkgs/doc/languages-frameworks/node.section.md
deleted file mode 100644
index 847db22941fd..000000000000
--- a/nixpkgs/doc/languages-frameworks/node.section.md
+++ /dev/null
@@ -1,51 +0,0 @@
-# Node.js {#node.js}
-
-The `pkgs/development/node-packages` folder contains a generated collection of
-[NPM packages](https://npmjs.com/) that can be installed with the Nix package
-manager.
-
-As a rule of thumb, the package set should only provide *end user* software
-packages, such as command-line utilities. Libraries should only be added to the
-package set if there is a non-NPM package that requires it.
-
-When it is desired to use NPM libraries in a development project, use the
-`node2nix` generator directly on the `package.json` configuration file of the
-project.
-
-The package set provides support for the official stable Node.js versions.
-The latest stable LTS release in `nodePackages`, as well as the latest stable
-Current release in `nodePackages_latest`.
-
-If your package uses native addons, you need to examine what kind of native
-build system it uses. Here are some examples:
-
-* `node-gyp`
-* `node-gyp-builder`
-* `node-pre-gyp`
-
-After you have identified the correct system, you need to override your package
-expression while adding in build system as a build input. For example, `dat`
-requires `node-gyp-build`, so [we override](https://github.com/NixOS/nixpkgs/blob/32f5e5da4a1b3f0595527f5195ac3a91451e9b56/pkgs/development/node-packages/default.nix#L37-L40) its expression in [`default.nix`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/node-packages/default.nix):
-
-```nix
-    dat = super.dat.override {
-      buildInputs = [ self.node-gyp-build pkgs.libtool pkgs.autoconf pkgs.automake ];
-      meta.broken = since "12";
-    };
-```
-
-To add a package from NPM to nixpkgs:
-
- 1. Modify `pkgs/development/node-packages/node-packages.json` to add, update
-    or remove package entries to have it included in `nodePackages` and
-    `nodePackages_latest`.
- 2. Run the script: `(cd pkgs/development/node-packages && ./generate.sh)`.
- 3. Build your new package to test your changes:
-    `cd /path/to/nixpkgs && nix-build -A nodePackages.<new-or-updated-package>`.
-    To build against the latest stable Current Node.js version (e.g. 14.x):
-    `nix-build -A nodePackages_latest.<new-or-updated-package>`
- 4. Add and commit all modified and generated files.
-
-For more information about the generation process, consult the
-[README.md](https://github.com/svanderburg/node2nix) file of the `node2nix`
-tool.
diff --git a/nixpkgs/doc/languages-frameworks/perl.section.md b/nixpkgs/doc/languages-frameworks/perl.section.md
index dcb7dcb77b65..c992b9d658bb 100644
--- a/nixpkgs/doc/languages-frameworks/perl.section.md
+++ b/nixpkgs/doc/languages-frameworks/perl.section.md
@@ -122,7 +122,7 @@ ImageExifTool = buildPerlPackage {
   };
 
   buildInputs = lib.optional stdenv.isDarwin shortenPerlShebang;
-  postInstall = lib.optional stdenv.isDarwin ''
+  postInstall = lib.optionalString stdenv.isDarwin ''
     shortenPerlShebang $out/bin/exiftool
   '';
 };
diff --git a/nixpkgs/doc/languages-frameworks/python.section.md b/nixpkgs/doc/languages-frameworks/python.section.md
index 53df328f8d48..987ea1af9daa 100644
--- a/nixpkgs/doc/languages-frameworks/python.section.md
+++ b/nixpkgs/doc/languages-frameworks/python.section.md
@@ -1602,3 +1602,33 @@ The following rules are desired to be respected:
   If necessary, `pname` has to be given a different value within `fetchPypi`.
 * Attribute names in `python-packages.nix` should be sorted alphanumerically to
   avoid merge conflicts and ease locating attributes.
+
+## Package set maintenance
+
+The whole Python package set has a lot of packages that do not see regular
+updates, because they either are a very fragile component in the Python
+ecosystem, like for example the `hypothesis` package, or packages that have
+no maintainer, so maintenance falls back to the package set maintainers.
+
+### Updating packages in bulk
+
+There is a tool to update alot of python libraries in bulk, it exists at
+`maintainers/scripts/update-python-libraries` with this repository.
+
+It can quickly update minor or major versions for all packages selected
+and create update commits, and supports the `fetchPypi`, `fetchurl` and
+`fetchFromGitHub` fetchers. When updating lots of packages that are
+hosted on GitHub, exporting a `GITHUB_API_TOKEN` is highly recommended.
+
+Updating packages in bulk leads to lots of breakages, which is why a
+stabilization period on the `python-unstable` branch is required.
+
+Once the branch is sufficiently stable it should normally be merged
+into the `staging` branch.
+
+An exemplary call to update all python libraries between minor versions
+would be:
+
+```ShellSession
+$ maintainers/scripts/update-python-libraries --target minor --commit --use-pkgs-prefix pkgs/development/python-modules/**/default.nix
+```
diff --git a/nixpkgs/doc/languages-frameworks/rust.section.md b/nixpkgs/doc/languages-frameworks/rust.section.md
index f5c664bec20e..9de9a06465f8 100644
--- a/nixpkgs/doc/languages-frameworks/rust.section.md
+++ b/nixpkgs/doc/languages-frameworks/rust.section.md
@@ -129,7 +129,15 @@ rustPlatform.buildRustPackage rec {
 ```
 
 This will retrieve the dependencies using fixed-output derivations from
-the specified lockfile.
+the specified lockfile. Note that setting `cargoLock.lockFile` doesn't
+add a `Cargo.lock` to your `src`, and a `Cargo.lock` is still required
+to build a rust package. A simple fix is to use:
+
+```nix
+postPatch = ''
+  cp ${./Cargo.lock} Cargo.lock
+'';
+```
 
 The output hash of each dependency that uses a git source must be
 specified in the `outputHashes` attribute. For example:
@@ -144,7 +152,7 @@ rustPlatform.buildRustPackage rec {
     outputHashes = {
       "finalfusion-0.14.0" = "17f4bsdzpcshwh74w5z119xjy2if6l2wgyjy56v621skr2r8y904";
     };
-  }
+  };
 
   # ...
 }
diff --git a/nixpkgs/doc/languages-frameworks/vim.section.md b/nixpkgs/doc/languages-frameworks/vim.section.md
index 26a01a89aa51..cb5311fc95f1 100644
--- a/nixpkgs/doc/languages-frameworks/vim.section.md
+++ b/nixpkgs/doc/languages-frameworks/vim.section.md
@@ -309,7 +309,7 @@ Sample output2:
 
 ## Adding new plugins to nixpkgs {#adding-new-plugins-to-nixpkgs}
 
-Nix expressions for Vim plugins are stored in [pkgs/misc/vim-plugins](/pkgs/misc/vim-plugins). For the vast majority of plugins, Nix expressions are automatically generated by running [`./update.py`](/pkgs/misc/vim-plugins/update.py). This creates a [generated.nix](/pkgs/misc/vim-plugins/generated.nix) file based on the plugins listed in [vim-plugin-names](/pkgs/misc/vim-plugins/vim-plugin-names). Plugins are listed in alphabetical order in `vim-plugin-names` using the format `[github username]/[repository]`. For example https://github.com/scrooloose/nerdtree becomes `scrooloose/nerdtree`.
+Nix expressions for Vim plugins are stored in [pkgs/misc/vim-plugins](/pkgs/misc/vim-plugins). For the vast majority of plugins, Nix expressions are automatically generated by running [`./update.py`](/pkgs/misc/vim-plugins/update.py). This creates a [generated.nix](/pkgs/misc/vim-plugins/generated.nix) file based on the plugins listed in [vim-plugin-names](/pkgs/misc/vim-plugins/vim-plugin-names). Plugins are listed in alphabetical order in `vim-plugin-names` using the format `[github username]/[repository]@[gitref]`. For example https://github.com/scrooloose/nerdtree becomes `scrooloose/nerdtree`.
 
 Some plugins require overrides in order to function properly. Overrides are placed in [overrides.nix](/pkgs/misc/vim-plugins/overrides.nix). Overrides are most often required when a plugin requires some dependencies, or extra steps are required during the build process. For example `deoplete-fish` requires both `deoplete-nvim` and `vim-fish`, and so the following override was added: