diff options
| -rw-r--r-- | doc/languages-frameworks/beam.section.md | 217 | ||||
| -rw-r--r-- | pkgs/development/beam-modules/build-mix.nix | 100 | ||||
| -rw-r--r-- | pkgs/development/beam-modules/default.nix | 2 | ||||
| -rw-r--r-- | pkgs/development/beam-modules/fetch-mix-deps.nix | 41 | ||||
| -rwxr-xr-x | pkgs/development/beam-modules/mix-bootstrap | 108 | ||||
| -rw-r--r-- | pkgs/development/beam-modules/mix-release.nix | 106 |
6 files changed, 343 insertions, 231 deletions
diff --git a/doc/languages-frameworks/beam.section.md b/doc/languages-frameworks/beam.section.md index ad3b94880b5c..b5d39595c107 100644 --- a/doc/languages-frameworks/beam.section.md +++ b/doc/languages-frameworks/beam.section.md @@ -2,15 +2,15 @@ ## Introduction {#beam-introduction} -In this document and related Nix expressions, we use the term, *BEAM*, to describe the environment. BEAM is the name of the Erlang Virtual Machine and, as far as we're concerned, from a packaging perspective, all languages that run on the BEAM are interchangeable. That which varies, like the build system, is transparent to users of any given BEAM package, so we make no distinction. +In this document and related Nix expressions, we use the term, _BEAM_, to describe the environment. BEAM is the name of the Erlang Virtual Machine and, as far as we're concerned, from a packaging perspective, all languages that run on the BEAM are interchangeable. That which varies, like the build system, is transparent to users of any given BEAM package, so we make no distinction. ## Structure {#beam-structure} All BEAM-related expressions are available via the top-level `beam` attribute, which includes: - - `interpreters`: a set of compilers running on the BEAM, including multiple Erlang/OTP versions (`beam.interpreters.erlangR19`, etc), Elixir (`beam.interpreters.elixir`) and LFE (`beam.interpreters.lfe`). +- `interpreters`: a set of compilers running on the BEAM, including multiple Erlang/OTP versions (`beam.interpreters.erlangR19`, etc), Elixir (`beam.interpreters.elixir`) and LFE (Lisp Flavoured Erlang) (`beam.interpreters.lfe`). - - `packages`: a set of package builders (Mix and rebar3), each compiled with a specific Erlang/OTP version, e.g. `beam.packages.erlangR19`. +- `packages`: a set of package builders (Mix and rebar3), each compiled with a specific Erlang/OTP version, e.g. `beam.packages.erlangR19`. The default Erlang compiler, defined by `beam.interpreters.erlang`, is aliased as `erlang`. The default BEAM package set is defined by `beam.packages.erlang` and aliased at the top level as `beamPackages`. @@ -26,7 +26,9 @@ We provide a version of Rebar3, under `rebar3`. We also provide a helper to fetc ### Mix & Erlang.mk {#build-tools-other} -Both Mix and Erlang.mk work exactly as expected. There is a bootstrap process that needs to be run for both, however, which is supported by the `buildMix` and `buildErlangMk` derivations, respectively. +Erlang.mk works exactly as expected. There is a bootstrap process that needs to be run, which is supported by the `buildErlangMk` derivation. + +For Elixir applications use `mixRelease` to make a release. See examples for more details. ## How to Install BEAM Packages {#how-to-install-beam-packages} @@ -52,15 +54,150 @@ Erlang.mk functions similarly to Rebar3, except we use `buildErlangMk` instead o #### Mix Packages {#mix-packages} -Mix functions similarly to Rebar3, except we use `buildMix` instead of `buildRebar3`. +`mixRelease` is used to make a release in the mix sense. Dependencies will need to be fetched with `fetchMixDeps` and passed to it. + +#### mixRelease - Elixir Phoenix example + +Here is how your `default.nix` file would look. + +```nix +with import <nixpkgs> { }; + +let + packages = beam.packagesWith beam.interpreters.erlang; + src = builtins.fetchgit { + url = "ssh://git@github.com/your_id/your_repo"; + rev = "replace_with_your_commit"; + }; + + pname = "your_project"; + version = "0.0.1"; + mixEnv = "prod"; + + mixDeps = packages.fetchMixDeps { + pname = "mix-deps-${pname}"; + inherit src mixEnv version; + # nix will complain and tell you the right value to replace this with + sha256 = lib.fakeSha256; + # if you have build time environment variables add them here + MY_ENV_VAR="my_value"; + }; + + nodeDependencies = (pkgs.callPackage ./assets/default.nix { }).shell.nodeDependencies; + + frontEndFiles = stdenvNoCC.mkDerivation { + pname = "frontend-${pname}"; + + nativeBuildInputs = [ nodejs ]; + + inherit version src; + + buildPhase = '' + cp -r ./assets $TEMPDIR + + mkdir -p $TEMPDIR/assets/node_modules/.cache + cp -r ${nodeDependencies}/lib/node_modules $TEMPDIR/assets + export PATH="${nodeDependencies}/bin:$PATH" + + cd $TEMPDIR/assets + webpack --config ./webpack.config.js + cd .. + ''; + + installPhase = '' + cp -r ./priv/static $out/ + ''; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + # nix will complain and tell you the right value to replace this with + outputHash = lib.fakeSha256; + + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + }; + + +in packages.mixRelease { + inherit src pname version mixEnv mixDeps; + # if you have build time environment variables add them here + MY_ENV_VAR="my_value"; + preInstall = '' + mkdir -p ./priv/static + cp -r ${frontEndFiles} ./priv/static + ''; +} +``` + +Setup will require the following steps: + +- Move your secrets to runtime environment variables. For more information refer to the [runtime.exs docs](https://hexdocs.pm/mix/Mix.Tasks.Release.html#module-runtime-configuration). On a fresh Phoenix build that would mean that both `DATABASE_URL` and `SECRET_KEY` need to be moved to `runtime.exs`. +- `cd assets` and `nix-shell -p node2nix --run node2nix --development` will generate a Nix expression containing your frontend dependencies +- commit and push those changes +- you can now `nix-build .` +- To run the release, set the `RELEASE_TMP` environment variable to a directory that your program has write access to. It will be used to store the BEAM settings. -Alternatively, we can use `buildHex` as a shortcut: +#### Example of creating a service for an Elixir - Phoenix project + +In order to create a service with your release, you could add a `service.nix` +in your project with the following + +```nix +{config, pkgs, lib, ...}: + +let + release = pkgs.callPackage ./default.nix; + release_name = "app"; + working_directory = "/home/app"; +in +{ + systemd.services.${release_name} = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "postgresql.service" ]; + requires = [ "network-online.target" "postgresql.service" ]; + description = "my app"; + environment = { + # RELEASE_TMP is used to write the state of the + # VM configuration when the system is running + # it needs to be a writable directory + RELEASE_TMP = working_directory; + # can be generated in an elixir console with + # Base.encode32(:crypto.strong_rand_bytes(32)) + RELEASE_COOKIE = "my_cookie"; + MY_VAR = "my_var"; + }; + serviceConfig = { + Type = "exec"; + DynamicUser = true; + WorkingDirectory = working_directory; + # Implied by DynamicUser, but just to emphasize due to RELEASE_TMP + PrivateTmp = true; + ExecStart = '' + ${release}/bin/${release_name} start + ''; + ExecStop = '' + ${release}/bin/${release_name} stop + ''; + ExecReload = '' + ${release}/bin/${release_name} restart + ''; + Restart = "on-failure"; + RestartSec = 5; + StartLimitBurst = 3; + StartLimitInterval = 10; + }; + # disksup requires bash + path = [ pkgs.bash ]; + }; + + environment.systemPackages = [ release ]; +} +``` ## How to Develop {#how-to-develop} ### Creating a Shell {#creating-a-shell} -Usually, we need to create a `shell.nix` file and do our development inside of the environment specified therein. Just install your version of erlang and other interpreter, and then user your normal build tools. As an example with elixir: +Usually, we need to create a `shell.nix` file and do our development inside of the environment specified therein. Just install your version of Erlang and any other interpreters, and then use your normal build tools. As an example with Elixir: ```nix { pkgs ? import "<nixpkgs"> {} }: @@ -79,6 +216,68 @@ mkShell { } ``` -#### Building in a Shell (for Mix Projects) {#building-in-a-shell} +#### Elixir - Phoenix project + +Here is an example `shell.nix`. + +```nix +with import <nixpkgs> { }; + +let + # define packages to install + basePackages = [ + git + # replace with beam.packages.erlang.elixir_1_11 if you need + beam.packages.erlang.elixir + nodejs-15_x + postgresql_13 + # only used for frontend dependencies + # you are free to use yarn2nix as well + nodePackages.node2nix + # formatting js file + nodePackages.prettier + ]; + + inputs = basePackages ++ lib.optionals stdenv.isLinux [ inotify-tools ] + ++ lib.optionals stdenv.isDarwin + (with darwin.apple_sdk.frameworks; [ CoreFoundation CoreServices ]); + + # define shell startup command + hooks = '' + # this allows mix to work on the local directory + mkdir -p .nix-mix .nix-hex + export MIX_HOME=$PWD/.nix-mix + export HEX_HOME=$PWD/.nix-mix + export PATH=$MIX_HOME/bin:$HEX_HOME/bin:$PATH + # TODO: not sure how to make hex available without installing it afterwards. + mix local.hex --if-missing + export LANG=en_US.UTF-8 + export ERL_AFLAGS="-kernel shell_history enabled" + + # postges related + # keep all your db data in a folder inside the project + export PGDATA="$PWD/db" + + # phoenix related env vars + export POOL_SIZE=15 + export DB_URL="postgresql://postgres:postgres@localhost:5432/db" + export PORT=4000 + export MIX_ENV=dev + # add your project env vars here, word readable in the nix store. + export ENV_VAR="your_env_var" + ''; + +in mkShell { + buildInputs = inputs; + shellHook = hooks; +} +``` + +Initializing the project will require the following steps: -Using a `shell.nix` as described (see <xref linkend="creating-a-shell"/>) should just work. +- create the db directory `initdb ./db` (inside your mix project folder) +- create the postgres user `createuser postgres -ds` +- create the db `createdb db` +- start the postgres instance `pg_ctl -l "$PGDATA/server.log" start` +- add the `/db` folder to your `.gitignore` +- you can start your phoenix server and get a shell with `iex -S mix phx.server` diff --git a/pkgs/development/beam-modules/build-mix.nix b/pkgs/development/beam-modules/build-mix.nix deleted file mode 100644 index 45f5e3674421..000000000000 --- a/pkgs/development/beam-modules/build-mix.nix +++ /dev/null @@ -1,100 +0,0 @@ -{ stdenv, writeText, elixir, erlang, hex, lib }: - -{ name -, version -, src -, setupHook ? null -, buildInputs ? [] -, beamDeps ? [] -, postPatch ? "" -, compilePorts ? false -, installPhase ? null -, buildPhase ? null -, configurePhase ? null -, meta ? {} -, enableDebugInfo ? false -, ... }@attrs: - -with lib; - -let - - debugInfoFlag = lib.optionalString (enableDebugInfo || elixir.debugInfo) "--debug-info"; - - shell = drv: stdenv.mkDerivation { - name = "interactive-shell-${drv.name}"; - buildInputs = [ drv ]; - }; - - bootstrapper = ./mix-bootstrap; - - pkg = self: stdenv.mkDerivation ( attrs // { - name = "${name}-${version}"; - inherit version; - - dontStrip = true; - - inherit src; - - setupHook = if setupHook == null - then writeText "setupHook.sh" '' - addToSearchPath ERL_LIBS "$1/lib/erlang/lib" - '' - else setupHook; - - inherit buildInputs; - propagatedBuildInputs = [ hex elixir ] ++ beamDeps; - - configurePhase = if configurePhase == null - then '' - runHook preConfigure - ${erlang}/bin/escript ${bootstrapper} - runHook postConfigure - '' - else configurePhase ; - - - buildPhase = if buildPhase == null - then '' - runHook preBuild - - export HEX_OFFLINE=1 - export HEX_HOME=`pwd` - export MIX_ENV=prod - export MIX_NO_DEPS=1 - - mix compile ${debugInfoFlag} --no-deps-check - - runHook postBuild - '' - else buildPhase; - - installPhase = if installPhase == null - then '' - runHook preInstall - - MIXENV=prod - - if [ -d "_build/shared" ]; then - MIXENV=shared - fi - - mkdir -p "$out/lib/erlang/lib/${name}-${version}" - for reldir in src ebin priv include; do - fd="_build/$MIXENV/lib/${name}/$reldir" - [ -d "$fd" ] || continue - cp -Hrt "$out/lib/erlang/lib/${name}-${version}" "$fd" - success=1 - done - - runHook postInstall - '' - else installPhase; - - passthru = { - packageName = name; - env = shell self; - inherit beamDeps; - }; -}); -in fix pkg diff --git a/pkgs/development/beam-modules/default.nix b/pkgs/development/beam-modules/default.nix index f78c66d3805a..6e6da4fab037 100644 --- a/pkgs/development/beam-modules/default.nix +++ b/pkgs/development/beam-modules/default.nix @@ -33,7 +33,7 @@ let buildHex = callPackage ./build-hex.nix { }; buildErlangMk = callPackage ./build-erlang-mk.nix { }; fetchMixDeps = callPackage ./fetch-mix-deps.nix { }; - buildMix = callPackage ./build-mix.nix { }; + mixRelease = callPackage ./mix-release.nix { }; # BEAM-based languages. elixir = elixir_1_11; diff --git a/pkgs/development/beam-modules/fetch-mix-deps.nix b/pkgs/development/beam-modules/fetch-mix-deps.nix index 020a82ad70b7..73365bd1a93e 100644 --- a/pkgs/development/beam-modules/fetch-mix-deps.nix +++ b/pkgs/development/beam-modules/fetch-mix-deps.nix @@ -1,34 +1,49 @@ { stdenvNoCC, lib, elixir, hex, rebar, rebar3, cacert, git }: -{ name, version, sha256, src, mixEnv ? "prod", debug ? false, meta ? { } }: - -stdenvNoCC.mkDerivation ({ - name = "mix-deps-${name}-${version}"; - +{ pname +, version +, sha256 +, src +, mixEnv ? "prod" +, debug ? false +, meta ? { } +, ... +}@attrs: + +stdenvNoCC.mkDerivation (attrs // { nativeBuildInputs = [ elixir hex cacert git ]; - inherit src; - MIX_ENV = mixEnv; MIX_DEBUG = if debug then 1 else 0; DEBUG = if debug then 1 else 0; # for rebar3 - - configurePhase = '' + # the api with `mix local.rebar rebar path` makes a copy of the binary + MIX_REBAR = "${rebar}/bin/rebar"; + MIX_REBAR3 = "${rebar3}/bin/rebar3"; + # there is a persistent download failure with absinthe 1.6.3 + # those defaults reduce the failure rate + HEX_HTTP_CONCURRENCY = 1; + HEX_HTTP_TIMEOUT = 120; + + configurePhase = attrs.configurePhase or '' + runHook preConfigure export HEX_HOME="$TEMPDIR/.hex"; export MIX_HOME="$TEMPDIR/.mix"; - export MIX_DEPS_PATH="$out"; + export MIX_DEPS_PATH="$TEMPDIR/deps"; # Rebar - mix local.rebar rebar "${rebar}/bin/rebar" - mix local.rebar rebar3 "${rebar3}/bin/rebar3" export REBAR_GLOBAL_CONFIG_DIR="$TMPDIR/rebar3" export REBAR_CACHE_DIR="$TMPDIR/rebar3.cache" + runHook postConfigure ''; dontBuild = true; - installPhase = '' + installPhase = attrs.installPhase or '' + runHook preInstall mix deps.get --only ${mixEnv} + find "$TEMPDIR/deps" -path '*/.git/*' -a ! -name HEAD -exec rm -rf {} + + cp -r --no-preserve=mode,ownership,timestamps $TEMPDIR/deps $out + runHook postInstall ''; outputHashAlgo = "sha256"; diff --git a/pkgs/development/beam-modules/mix-bootstrap b/pkgs/development/beam-modules/mix-bootstrap deleted file mode 100755 index 7e31def71fa8..000000000000 --- a/pkgs/development/beam-modules/mix-bootstrap +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env escript -%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -%%! -smp enable -%%% --------------------------------------------------------------------------- -%%% @doc -%%% The purpose of this command is to prepare a mix project so that mix -%%% understands that the dependencies are all already installed. If you want a -%%% hygienic build on nix then you must run this command before running mix. I -%%% suggest that you add a `Makefile` to your project and have the bootstrap -%%% command be a dependency of the build commands. See the nix documentation for -%%% more information. -%%% -%%% This command designed to have as few dependencies as possible so that it can -%%% be a dependency of root level packages like mix. To that end it does many -%%% things in a fairly simplistic way. That is by design. -%%% -%%% ### Assumptions -%%% -%%% This command makes the following assumptions: -%%% -%%% * It is run in a nix-shell or nix-build environment -%%% * that all dependencies have been added to the ERL_LIBS -%%% Environment Variable - --record(data, {version - , erl_libs - , root - , name}). --define(LOCAL_HEX_REGISTRY, "registry.ets"). - -main(Args) -> - {ok, RequiredData} = gather_required_data_from_the_environment(Args), - ok = bootstrap_libs(RequiredData). - -%% @doc -%% This takes an app name in the standard OTP <name>-<version> format -%% and returns just the app name. Why? Because rebar doesn't -%% respect OTP conventions in some cases. --spec fixup_app_name(file:name()) -> string(). -fixup_app_name(Path) -> - BaseName = filename:basename(Path), - case string:split(BaseName, "-") of - [Name, _Version] -> Name; - Name -> Name - end. - - --spec gather_required_data_from_the_environment([string()]) -> {ok, #data{}}. -gather_required_data_from_the_environment(_) -> - {ok, #data{ version = guard_env("version") - , erl_libs = os:getenv("ERL_LIBS", []) - , root = code:root_dir() - , name = guard_env("name")}}. - --spec guard_env(string()) -> string(). -guard_env(Name) -> - case os:getenv(Name) of - false -> - stderr("Expected Environment variable ~s! Are you sure you are " - "running in a Nix environment? Either a nix-build, " - "nix-shell, etc?~n", [Name]), - erlang:halt(1); - Variable -> - Variable - end. - --spec bootstrap_libs(#data{}) -> ok. -bootstrap_libs(#data{erl_libs = ErlLibs}) -> - io:format("Bootstrapping dependent libraries~n"), - Target = "_build/prod/lib/", - Paths = string:tokens(ErlLibs, ":"), - CopiableFiles = - lists:foldl(fun(Path, Acc) -> - gather_directory_contents(Path) ++ Acc - end, [], Paths), - lists:foreach(fun (Path) -> - ok = link_app(Path, Target) - end, CopiableFiles). - --spec gather_directory_contents(string()) -> [{string(), string()}]. -gather_directory_contents(Path) -> - {ok, Names} = file:list_dir(Path), - lists:map(fun(AppName) -> - {filename:join(Path, AppName), fixup_app_name(AppName)} - end, Names). - -%% @doc -%% Makes a symlink from the directory pointed at by Path to a -%% directory of the same name in Target. So if we had a Path of -%% {`foo/bar/baz/bash`, `baz`} and a Target of `faz/foo/foos`, the symlink -%% would be `faz/foo/foos/baz`. --spec link_app({string(), string()}, string()) -> ok. -link_app({Path, TargetFile}, TargetDir) -> - Target = filename:join(TargetDir, TargetFile), - ok = make_symlink(Path, Target). - --spec make_symlink(string(), string()) -> ok. -make_symlink(Path, TargetFile) -> - file:delete(TargetFile), - ok = filelib:ensure_dir(TargetFile), - io:format("Making symlink from ~s to ~s~n", [Path, TargetFile]), - ok = file:make_symlink(Path, TargetFile). - -%% @doc -%% Write the result of the format string out to stderr. --spec stderr(string(), [term()]) -> ok. -stderr(FormatStr, Args) -> - io:put_chars(standard_error, io_lib:format(FormatStr, Args)). diff --git a/pkgs/development/beam-modules/mix-release.nix b/pkgs/development/beam-modules/mix-release.nix new file mode 100644 index 000000000000..320fcaa9c9b7 --- /dev/null +++ b/pkgs/development/beam-modules/mix-release.nix @@ -0,0 +1,106 @@ +{ stdenv, lib, elixir, erlang, findutils, hex, rebar, rebar3, fetchMixDeps, makeWrapper, git }: + +{ pname +, version +, src +, nativeBuildInputs ? [ ] +, meta ? { } +, enableDebugInfo ? false +, mixEnv ? "prod" +, compileFlags ? [ ] +, mixDeps ? null +, ... +}@attrs: +let + overridable = builtins.removeAttrs attrs [ "compileFlags" ]; + +in +stdenv.mkDerivation (overridable // { + nativeBuildInputs = nativeBuildInputs ++ [ erlang hex elixir makeWrapper git ]; + + MIX_ENV = mixEnv; + MIX_DEBUG = if enableDebugInfo then 1 else 0; + HEX_OFFLINE = 1; + DEBUG = if enableDebugInfo then 1 else 0; # for Rebar3 compilation + # the api with `mix local.rebar rebar path` makes a copy of the binary + MIX_REBAR = "${rebar}/bin/rebar"; + MIX_REBAR3 = "${rebar3}/bin/rebar3"; + + postUnpack = '' + export HEX_HOME="$TEMPDIR/hex" + export MIX_HOME="$TEMPDIR/mix" + # compilation of the dependencies will require + # that the dependency path is writable + # thus a copy to the TEMPDIR is inevitable here + export MIX_DEPS_PATH="$TEMPDIR/deps" + + # Rebar + export REBAR_GLOBAL_CONFIG_DIR="$TEMPDIR/rebar3" + export REBAR_CACHE_DIR="$TEMPDIR/rebar3.cache" + + ${lib.optionalString (mixDeps != null) '' + cp --no-preserve=mode -R "${mixDeps}" "$MIX_DEPS_PATH" + '' + } + + '' + (attrs.postUnpack or ""); + + configurePhase = attrs.configurePhase or '' + runHook preConfigure + + # this is needed for projects that have a specific compile step + # the dependency needs to be compiled in order for the task + # to be available + # Phoenix projects for example will need compile.phoenix + mix deps.compile --no-deps-check --skip-umbrella-children + + runHook postConfigure + ''; + + buildPhase = attrs.buildPhase or '' + runHook preBuild + + mix compile --no-deps-check ${lib.concatStringsSep " " compileFlags} + + runHook postBuild + ''; + + + installPhase = attrs.installPhase or '' + runHook preInstall + + mix release --no-deps-check --path "$out" + + runHook postInstall + ''; + + fixupPhase = '' + runHook preFixup + if [ -e "$out/bin/${pname}.bat" ]; then # absent in special cases, i.e. elixir-ls + rm "$out/bin/${pname}.bat" # windows file + fi + # contains secrets and should not be in the nix store + # TODO document how to handle RELEASE_COOKIE + # secrets should not be in the nix store. + # This is only used for connecting multiple nodes + if [ -e $out/releases/COOKIE ]; then # absent in special cases, i.e. elixir-ls + rm $out/releases/COOKIE + fi + # TODO remove the uneeded reference too erlang + # one possible way would be + # for f in $(${findutils}/bin/find $out -name start); do + # substituteInPlace $f \ + # --replace 'ROOTDIR=${erlang}/lib/erlang' 'ROOTDIR=""' + # done + # What is left to do is to check that erlang is not required on + # the host + + patchShebangs $out + runHook postFixup + ''; + # TODO figure out how to do a Fixed Output Derivation and add the output hash + # This doesn't play well at the moment with Phoenix projects + # for example that have frontend dependencies + + # disallowedReferences = [ erlang ]; +}) |