diff options
author | Maximilian Bosch <maximilian@mbosch.me> | 2020-03-27 22:09:13 +0100 |
---|---|---|
committer | Maximilian Bosch <maximilian@mbosch.me> | 2020-03-28 23:33:25 +0100 |
commit | bd5324c4fcb990715c64fc40c19341947e04bf17 (patch) | |
tree | 683924c8027bdbe86b7634fa85a58aafd92f1a0c /pkgs | |
parent | ea5c800175f3bec015ff04a68649bc1aa422c2e8 (diff) | |
download | nixlib-bd5324c4fcb990715c64fc40c19341947e04bf17.tar nixlib-bd5324c4fcb990715c64fc40c19341947e04bf17.tar.gz nixlib-bd5324c4fcb990715c64fc40c19341947e04bf17.tar.bz2 nixlib-bd5324c4fcb990715c64fc40c19341947e04bf17.tar.lz nixlib-bd5324c4fcb990715c64fc40c19341947e04bf17.tar.xz nixlib-bd5324c4fcb990715c64fc40c19341947e04bf17.tar.zst nixlib-bd5324c4fcb990715c64fc40c19341947e04bf17.zip |
hydra: 2020-02-06 -> 2020-03-{24,27}
Upgrades Hydra to the latest master/flake branch. To perform this upgrade, it's needed to do a non-trivial db-migration which provides a massive performance-improvement[1]. The basic ideas behind multi-step upgrades of services between NixOS versions have been gathered already[2]. For further context it's recommended to read this first. Basically, the following steps are needed: * Upgrade to a non-breaking version of Hydra with the db-changes (columns are still nullable here). If `system.stateVersion` is set to something older than 20.03, the package will be selected automatically, otherwise `pkgs.hydra-migration` needs to be used. * Run `hydra-backfill-ids` on the server. * Deploy either `pkgs.hydra-unstable` (for Hydra master) or `pkgs.hydra-flakes` (for flakes-support) to activate the optimization. The steps are also documented in the release-notes and in the module using `warnings`. `pkgs.hydra` has been removed as latest Hydra doesn't compile with `pkgs.nixStable` and to ensure a graceful migration using the newly introduced packages. To verify the approach, a simple vm-test has been added which verifies the migration steps. [1] https://github.com/NixOS/hydra/pull/711 [2] https://github.com/NixOS/nixpkgs/pull/82353#issuecomment-598269471
Diffstat (limited to 'pkgs')
-rw-r--r-- | pkgs/development/tools/misc/hydra/common.nix | 135 | ||||
-rw-r--r-- | pkgs/development/tools/misc/hydra/default.nix | 173 | ||||
-rw-r--r-- | pkgs/top-level/all-packages.nix | 3 |
3 files changed, 173 insertions, 138 deletions
diff --git a/pkgs/development/tools/misc/hydra/common.nix b/pkgs/development/tools/misc/hydra/common.nix new file mode 100644 index 000000000000..7310ee949721 --- /dev/null +++ b/pkgs/development/tools/misc/hydra/common.nix @@ -0,0 +1,135 @@ +{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub +, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx +, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt +, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json +, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar +, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook, src ? null, version ? null +, migration ? false +}: + +with stdenv; + +if lib.versions.major nix.version == "1" + then throw "This Hydra version doesn't support Nix 1.x" +else + +let + perlDeps = buildEnv { + name = "hydra-perl-deps"; + paths = with perlPackages; lib.closePropagation + [ ModulePluggable + CatalystActionREST + CatalystAuthenticationStoreDBIxClass + CatalystDevel + CatalystDispatchTypeRegex + CatalystPluginAccessLog + CatalystPluginAuthorizationRoles + CatalystPluginCaptcha + CatalystPluginSessionStateCookie + CatalystPluginSessionStoreFastMmap + CatalystPluginStackTrace + CatalystPluginUnicodeEncoding + CatalystTraitForRequestProxyBase + CatalystViewDownload + CatalystViewJSON + CatalystViewTT + CatalystXScriptServerStarman + CatalystXRoleApplicator + CryptRandPasswd + DBDPg + DBDSQLite + DataDump + DateTime + DigestSHA1 + EmailMIME + EmailSender + FileSlurp + IOCompress + IPCRun + JSON + JSONAny + JSONXS + LWP + LWPProtocolHttps + NetAmazonS3 + NetPrometheus + NetStatsd + PadWalker + Readonly + SQLSplitStatement + SetScalar + Starman + SysHostnameLong + TermSizeAny + TestMore + TextDiff + TextTable + XMLSimple + nix + nix.perl-bindings + git + boehmgc + ]; + }; +in stdenv.mkDerivation rec { + pname = "hydra"; + + inherit stdenv src version; + + buildInputs = + [ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx + gitAndTools.top-git mercurial /*darcs*/ subversion bazaar openssl bzip2 libxslt + perlDeps perl nix + postgresql # for running the tests + nlohmann_json + boost + ]; + + hydraPath = lib.makeBinPath ( + [ sqlite subversion openssh nix coreutils findutils pixz + gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial /*darcs*/ gnused bazaar + ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); + + nativeBuildInputs = [ autoreconfHook pkgconfig ]; + + configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; + + NIX_CFLAGS_COMPILE = "-pthread"; + + shellHook = '' + PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH + PERL5LIB=$(pwd)/src/lib:$PERL5LIB; + ''; + + enableParallelBuilding = true; + + preCheck = '' + patchShebangs . + export LOGNAME=''${LOGNAME:-foo} + ''; + + postInstall = '' + mkdir -p $out/nix-support + for i in $out/bin/*; do + read -n 4 chars < $i + if [[ $chars =~ ELF ]]; then continue; fi + wrapProgram $i \ + --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ + --prefix PATH ':' $out/bin:$hydraPath \ + --set HYDRA_RELEASE ${version} \ + --set HYDRA_HOME $out/libexec/hydra \ + --set NIX_RELEASE ${nix.name or "unknown"} + done + ''; # */ + + dontStrip = true; + + passthru = { inherit perlDeps migration; }; + + meta = with stdenv.lib; { + description = "Nix-based continuous build system"; + license = licenses.gpl3; + platforms = platforms.linux; + maintainers = with maintainers; [ ma27 ]; + }; +} diff --git a/pkgs/development/tools/misc/hydra/default.nix b/pkgs/development/tools/misc/hydra/default.nix index 72aecf20ca39..35c069211b3f 100644 --- a/pkgs/development/tools/misc/hydra/default.nix +++ b/pkgs/development/tools/misc/hydra/default.nix @@ -1,143 +1,42 @@ -{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub -, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx -, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt -, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json -, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar -, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook -}: - -with stdenv; - -if lib.versions.major nix.version == "1" - then throw "This Hydra version doesn't support Nix 1.x" -else - -let - perlDeps = buildEnv { - name = "hydra-perl-deps"; - paths = with perlPackages; lib.closePropagation - [ ModulePluggable - CatalystActionREST - CatalystAuthenticationStoreDBIxClass - CatalystDevel - CatalystDispatchTypeRegex - CatalystPluginAccessLog - CatalystPluginAuthorizationRoles - CatalystPluginCaptcha - CatalystPluginSessionStateCookie - CatalystPluginSessionStoreFastMmap - CatalystPluginStackTrace - CatalystPluginUnicodeEncoding - CatalystTraitForRequestProxyBase - CatalystViewDownload - CatalystViewJSON - CatalystViewTT - CatalystXScriptServerStarman - CatalystXRoleApplicator - CryptRandPasswd - DBDPg - DBDSQLite - DataDump - DateTime - DigestSHA1 - EmailMIME - EmailSender - FileSlurp - IOCompress - IPCRun - JSON - JSONAny - JSONXS - LWP - LWPProtocolHttps - NetAmazonS3 - NetPrometheus - NetStatsd - PadWalker - Readonly - SQLSplitStatement - SetScalar - Starman - SysHostnameLong - TermSizeAny - TestMore - TextDiff - TextTable - XMLSimple - nix - nix.perl-bindings - git - boehmgc - ]; +{ fetchFromGitHub, nixStable, nixUnstable, callPackage, nixFlakes }: + +{ + # Package for phase-1 of the db migration for Hydra. + # https://github.com/NixOS/hydra/pull/711 + hydra-migration = callPackage ./common.nix { + version = "2020-02-10"; + src = fetchFromGitHub { + owner = "NixOS"; + repo = "hydra"; + rev = "add4f610ce6f206fb44702b5a894d877b3a30e3a"; + sha256 = "1d8hdgjx2ys0zmixi2ydmimdq7ml20h1ji4amwawcyw59kssh6l3"; + }; + nix = nixStable; + migration = true; }; -in stdenv.mkDerivation rec { - pname = "hydra"; - version = "2020-02-06"; - - inherit stdenv; - src = fetchFromGitHub { - owner = "NixOS"; - repo = pname; - rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc"; - sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick"; + # Hydra from latest master (or flakes) branch. Contains breaking changes, + # so when having an older version, `pkgs.hydra-migration` should be deployed first. + + hydra-unstable = callPackage ./common.nix { + version = "2020-03-24"; + src = fetchFromGitHub { + owner = "NixOS"; + repo = "hydra"; + rev = "12cc46cdb36321acd4c982429a86eb0f8f3cc969"; + sha256 = "10ipxzdxr47c8w5jg69mbax2ykc7lb5fs9bbdd3iai9wzyfz17ln"; + }; + nix = nixUnstable; }; - buildInputs = - [ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx - gitAndTools.top-git mercurial darcs subversion bazaar openssl bzip2 libxslt - guile # optional, for Guile + Guix support - perlDeps perl nix - postgresql # for running the tests - nlohmann_json - boost - ]; - - hydraPath = lib.makeBinPath ( - [ sqlite subversion openssh nix coreutils findutils pixz - gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial darcs gnused bazaar - ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); - - nativeBuildInputs = [ autoreconfHook pkgconfig ]; - - configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; - - NIX_CFLAGS_COMPILE = "-pthread"; - - shellHook = '' - PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH - PERL5LIB=$(pwd)/src/lib:$PERL5LIB; - ''; - - enableParallelBuilding = true; - - preCheck = '' - patchShebangs . - export LOGNAME=''${LOGNAME:-foo} - ''; - - postInstall = '' - mkdir -p $out/nix-support - for i in $out/bin/*; do - read -n 4 chars < $i - if [[ $chars =~ ELF ]]; then continue; fi - wrapProgram $i \ - --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ - --prefix PATH ':' $out/bin:$hydraPath \ - --set HYDRA_RELEASE ${version} \ - --set HYDRA_HOME $out/libexec/hydra \ - --set NIX_RELEASE ${nix.name or "unknown"} - done - ''; # */ - - dontStrip = true; - - passthru.perlDeps = perlDeps; - - meta = with stdenv.lib; { - description = "Nix-based continuous build system"; - license = licenses.gpl3; - platforms = platforms.linux; - maintainers = with maintainers; [ ma27 ]; + hydra-flakes = callPackage ./common.nix { + version = "2020-03-27"; + src = fetchFromGitHub { + owner = "NixOS"; + repo = "hydra"; + rev = "a7540b141d085a7e78c21fda8e8c05907c659b34"; + sha256 = "08fs7593w5zs8vh4c66gvrxk6s840pp6hj8nwf51wsa27kg5a943"; + }; + nix = nixFlakes; }; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 0341fd97bc0b..bbd9acd37ee7 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -12249,7 +12249,8 @@ in hwloc = callPackage ../development/libraries/hwloc {}; - hydra = callPackage ../development/tools/misc/hydra { }; + inherit (callPackage ../development/tools/misc/hydra { }) + hydra-migration hydra-unstable hydra-flakes; hydra-cli = callPackage ../development/tools/misc/hydra-cli { }; |