hydra: 2020-02-06 -> 2020-03-{24,27}

Upgrades Hydra to the latest master/flake branch. To perform this
upgrade, it's needed to do a non-trivial db-migration which provides a
massive performance-improvement[1].

The basic ideas behind multi-step upgrades of services between NixOS versions
have been gathered already[2]. For further context it's recommended to
read this first.

Basically, the following steps are needed:

* Upgrade to a non-breaking version of Hydra with the db-changes
  (columns are still nullable here). If `system.stateVersion` is set to
  something older than 20.03, the package will be selected
  automatically, otherwise `pkgs.hydra-migration` needs to be used.

* Run `hydra-backfill-ids` on the server.

* Deploy either `pkgs.hydra-unstable` (for Hydra master) or
  `pkgs.hydra-flakes` (for flakes-support) to activate the optimization.

The steps are also documented in the release-notes and in the module
using `warnings`.

`pkgs.hydra` has been removed as latest Hydra doesn't compile with
`pkgs.nixStable` and to ensure a graceful migration using the newly
introduced packages.

To verify the approach, a simple vm-test has been added which verifies
the migration steps.

[1] https://github.com/NixOS/hydra/pull/711
[2] https://github.com/NixOS/nixpkgs/pull/82353#issuecomment-598269471

1 files changed, 135 insertions, 0 deletions

diff --git a/pkgs/development/tools/misc/hydra/common.nix b/pkgs/development/tools/misc/hydra/common.nix
new file mode 100644
index 000000000000..7310ee949721
--- /dev/null
+++ b/pkgs/development/tools/misc/hydra/common.nix
@@ -0,0 +1,135 @@
+{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub
+, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx
+, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt
+, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json
+, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar
+, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook, src ? null, version ? null
+, migration ? false
+}:
+
+with stdenv;
+
+if lib.versions.major nix.version == "1"
+  then throw "This Hydra version doesn't support Nix 1.x"
+else
+
+let
+  perlDeps = buildEnv {
+    name = "hydra-perl-deps";
+    paths = with perlPackages; lib.closePropagation
+      [ ModulePluggable
+        CatalystActionREST
+        CatalystAuthenticationStoreDBIxClass
+        CatalystDevel
+        CatalystDispatchTypeRegex
+        CatalystPluginAccessLog
+        CatalystPluginAuthorizationRoles
+        CatalystPluginCaptcha
+        CatalystPluginSessionStateCookie
+        CatalystPluginSessionStoreFastMmap
+        CatalystPluginStackTrace
+        CatalystPluginUnicodeEncoding
+        CatalystTraitForRequestProxyBase
+        CatalystViewDownload
+        CatalystViewJSON
+        CatalystViewTT
+        CatalystXScriptServerStarman
+        CatalystXRoleApplicator
+        CryptRandPasswd
+        DBDPg
+        DBDSQLite
+        DataDump
+        DateTime
+        DigestSHA1
+        EmailMIME
+        EmailSender
+        FileSlurp
+        IOCompress
+        IPCRun
+        JSON
+        JSONAny
+        JSONXS
+        LWP
+        LWPProtocolHttps
+        NetAmazonS3
+        NetPrometheus
+        NetStatsd
+        PadWalker
+        Readonly
+        SQLSplitStatement
+        SetScalar
+        Starman
+        SysHostnameLong
+        TermSizeAny
+        TestMore
+        TextDiff
+        TextTable
+        XMLSimple
+        nix
+        nix.perl-bindings
+        git
+        boehmgc
+      ];
+  };
+in stdenv.mkDerivation rec {
+  pname = "hydra";
+
+  inherit stdenv src version;
+
+  buildInputs =
+    [ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx
+      gitAndTools.top-git mercurial /*darcs*/ subversion bazaar openssl bzip2 libxslt
+      perlDeps perl nix
+      postgresql # for running the tests
+      nlohmann_json
+      boost
+    ];
+
+  hydraPath = lib.makeBinPath (
+    [ sqlite subversion openssh nix coreutils findutils pixz
+      gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial /*darcs*/ gnused bazaar
+    ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
+
+  nativeBuildInputs = [ autoreconfHook pkgconfig ];
+
+  configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ];
+
+  NIX_CFLAGS_COMPILE = "-pthread";
+
+  shellHook = ''
+    PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH
+    PERL5LIB=$(pwd)/src/lib:$PERL5LIB;
+  '';
+
+  enableParallelBuilding = true;
+
+  preCheck = ''
+    patchShebangs .
+    export LOGNAME=''${LOGNAME:-foo}
+  '';
+
+  postInstall = ''
+    mkdir -p $out/nix-support
+    for i in $out/bin/*; do
+        read -n 4 chars < $i
+        if [[ $chars =~ ELF ]]; then continue; fi
+        wrapProgram $i \
+            --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
+            --prefix PATH ':' $out/bin:$hydraPath \
+            --set HYDRA_RELEASE ${version} \
+            --set HYDRA_HOME $out/libexec/hydra \
+            --set NIX_RELEASE ${nix.name or "unknown"}
+    done
+  ''; # */
+
+  dontStrip = true;
+
+  passthru = { inherit perlDeps migration; };
+
+  meta = with stdenv.lib; {
+    description = "Nix-based continuous build system";
+    license = licenses.gpl3;
+    platforms = platforms.linux;
+    maintainers = with maintainers; [ ma27 ];
+  };
+}