diff options
Diffstat (limited to 'nixpkgs/nixos/modules/services/continuous-integration')
13 files changed, 3075 insertions, 0 deletions
diff --git a/nixpkgs/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixpkgs/nixos/modules/services/continuous-integration/buildbot/master.nix new file mode 100644 index 000000000000..a49f5f8100dc --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -0,0 +1,287 @@ +# NixOS module for Buildbot continous integration server. + +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.buildbot-master; + + python = cfg.package.pythonModule; + + escapeStr = s: escape ["'"] s; + + defaultMasterCfg = pkgs.writeText "master.cfg" '' + from buildbot.plugins import * + factory = util.BuildFactory() + c = BuildmasterConfig = dict( + workers = [${concatStringsSep "," cfg.workers}], + protocols = { 'pb': {'port': ${toString cfg.pbPort} } }, + title = '${escapeStr cfg.title}', + titleURL = '${escapeStr cfg.titleUrl}', + buildbotURL = '${escapeStr cfg.buildbotUrl}', + db = dict(db_url='${escapeStr cfg.dbUrl}'), + www = dict(port=${toString cfg.port}), + change_source = [ ${concatStringsSep "," cfg.changeSource} ], + schedulers = [ ${concatStringsSep "," cfg.schedulers} ], + builders = [ ${concatStringsSep "," cfg.builders} ], + services = [ ${concatStringsSep "," cfg.reporters} ], + ) + for step in [ ${concatStringsSep "," cfg.factorySteps} ]: + factory.addStep(step) + + ${cfg.extraConfig} + ''; + + tacFile = pkgs.writeText "buildbot-master.tac" '' + import os + + from twisted.application import service + from buildbot.master import BuildMaster + + basedir = '${cfg.buildbotDir}' + + configfile = '${cfg.masterCfg}' + + # Default umask for server + umask = None + + # note: this line is matched against to check that this is a buildmaster + # directory; do not edit it. + application = service.Application('buildmaster') + + m = BuildMaster(basedir, configfile, umask) + m.setServiceParent(application) + ''; + +in { + options = { + services.buildbot-master = { + + factorySteps = mkOption { + type = types.listOf types.str; + description = "Factory Steps"; + default = []; + example = [ + "steps.Git(repourl='git://github.com/buildbot/pyflakes.git', mode='incremental')" + "steps.ShellCommand(command=['trial', 'pyflakes'])" + ]; + }; + + changeSource = mkOption { + type = types.listOf types.str; + description = "List of Change Sources."; + default = []; + example = [ + "changes.GitPoller('git://github.com/buildbot/pyflakes.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" + ]; + }; + + enable = mkOption { + type = types.bool; + default = false; + description = "Whether to enable the Buildbot continuous integration server."; + }; + + extraConfig = mkOption { + type = types.str; + description = "Extra configuration to append to master.cfg"; + default = "c['buildbotNetUsageData'] = None"; + }; + + masterCfg = mkOption { + type = types.path; + description = "Optionally pass master.cfg path. Other options in this configuration will be ignored."; + default = defaultMasterCfg; + example = "/etc/nixos/buildbot/master.cfg"; + }; + + schedulers = mkOption { + type = types.listOf types.str; + description = "List of Schedulers."; + default = [ + "schedulers.SingleBranchScheduler(name='all', change_filter=util.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=['runtests'])" + "schedulers.ForceScheduler(name='force',builderNames=['runtests'])" + ]; + }; + + builders = mkOption { + type = types.listOf types.str; + description = "List of Builders."; + default = [ + "util.BuilderConfig(name='runtests',workernames=['example-worker'],factory=factory)" + ]; + }; + + workers = mkOption { + type = types.listOf types.str; + description = "List of Workers."; + default = [ "worker.Worker('example-worker', 'pass')" ]; + }; + + reporters = mkOption { + default = []; + type = types.listOf types.str; + description = "List of reporter objects used to present build status to various users."; + }; + + user = mkOption { + default = "buildbot"; + type = types.str; + description = "User the buildbot server should execute under."; + }; + + group = mkOption { + default = "buildbot"; + type = types.str; + description = "Primary group of buildbot user."; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = []; + description = "List of extra groups that the buildbot user should be a part of."; + }; + + home = mkOption { + default = "/home/buildbot"; + type = types.path; + description = "Buildbot home directory."; + }; + + buildbotDir = mkOption { + default = "${cfg.home}/master"; + type = types.path; + description = "Specifies the Buildbot directory."; + }; + + pbPort = mkOption { + default = 9989; + type = types.either types.str types.int; + example = "'tcp:9990:interface=127.0.0.1'"; + description = '' + The buildmaster will listen on a TCP port of your choosing + for connections from workers. + It can also use this port for connections from remote Change Sources, + status clients, and debug tools. + This port should be visible to the outside world, and you’ll need to tell + your worker admins about your choice. + If put in (single) quotes, this can also be used as a connection string, + as defined in the <link xlink:href="https://twistedmatrix.com/documents/current/core/howto/endpoints.html">ConnectionStrings guide</link>. + ''; + }; + + listenAddress = mkOption { + default = "0.0.0.0"; + type = types.str; + description = "Specifies the bind address on which the buildbot HTTP interface listens."; + }; + + buildbotUrl = mkOption { + default = "http://localhost:8010/"; + type = types.str; + description = "Specifies the Buildbot URL."; + }; + + title = mkOption { + default = "Buildbot"; + type = types.str; + description = "Specifies the Buildbot Title."; + }; + + titleUrl = mkOption { + default = "Buildbot"; + type = types.str; + description = "Specifies the Buildbot TitleURL."; + }; + + dbUrl = mkOption { + default = "sqlite:///state.sqlite"; + type = types.str; + description = "Specifies the database connection string."; + }; + + port = mkOption { + default = 8010; + type = types.int; + description = "Specifies port number on which the buildbot HTTP interface listens."; + }; + + package = mkOption { + type = types.package; + default = pkgs.python3Packages.buildbot-full; + defaultText = "pkgs.python3Packages.buildbot-full"; + description = "Package to use for buildbot."; + example = literalExample "pkgs.python3Packages.buildbot"; + }; + + packages = mkOption { + default = [ pkgs.git ]; + example = literalExample "[ pkgs.git ]"; + type = types.listOf types.package; + description = "Packages to add to PATH for the buildbot process."; + }; + + pythonPackages = mkOption { + type = types.functionTo (types.listOf types.package); + default = pythonPackages: with pythonPackages; [ ]; + defaultText = "pythonPackages: with pythonPackages; [ ]"; + description = "Packages to add the to the PYTHONPATH of the buildbot process."; + example = literalExample "pythonPackages: with pythonPackages; [ requests ]"; + }; + }; + }; + + config = mkIf cfg.enable { + users.groups = optionalAttrs (cfg.group == "buildbot") { + buildbot = { }; + }; + + users.users = optionalAttrs (cfg.user == "buildbot") { + buildbot = { + description = "Buildbot User."; + isNormalUser = true; + createHome = true; + home = cfg.home; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + }; + }; + + systemd.services.buildbot-master = { + description = "Buildbot Continuous Integration Server."; + after = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + path = cfg.packages ++ cfg.pythonPackages python.pkgs; + environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ cfg.package ])}/${python.sitePackages}"; + + preStart = '' + mkdir -vp "${cfg.buildbotDir}" + # Link the tac file so buildbot command line tools recognize the directory + ln -sf "${tacFile}" "${cfg.buildbotDir}/buildbot.tac" + ${cfg.package}/bin/buildbot create-master --db "${cfg.dbUrl}" "${cfg.buildbotDir}" + rm -f buildbot.tac.new master.cfg.sample + ''; + + serviceConfig = { + Type = "simple"; + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.home; + # NOTE: call twistd directly with stdout logging for systemd + ExecStart = "${python.pkgs.twisted}/bin/twistd -o --nodaemon --pidfile= --logfile - --python ${tacFile}"; + }; + }; + }; + + imports = [ + (mkRenamedOptionModule [ "services" "buildbot-master" "bpPort" ] [ "services" "buildbot-master" "pbPort" ]) + (mkRemovedOptionModule [ "services" "buildbot-master" "status" ] '' + Since Buildbot 0.9.0, status targets are deprecated and ignored. + Review your configuration and migrate to reporters (available at services.buildbot-master.reporters). + '') + ]; + + meta.maintainers = with lib.maintainers; [ nand0p mic92 lopsided98 ]; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixpkgs/nixos/modules/services/continuous-integration/buildbot/worker.nix new file mode 100644 index 000000000000..7b8a35f54bfa --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/buildbot/worker.nix @@ -0,0 +1,196 @@ +# NixOS module for Buildbot Worker. + +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.buildbot-worker; + + python = cfg.package.pythonModule; + + tacFile = pkgs.writeText "aur-buildbot-worker.tac" '' + import os + from io import open + + from buildbot_worker.bot import Worker + from twisted.application import service + + basedir = '${cfg.buildbotDir}' + + # note: this line is matched against to check that this is a worker + # directory; do not edit it. + application = service.Application('buildbot-worker') + + master_url_split = '${cfg.masterUrl}'.split(':') + buildmaster_host = master_url_split[0] + port = int(master_url_split[1]) + workername = '${cfg.workerUser}' + + with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file: + passwd = passwd_file.read().strip('\r\n') + keepalive = ${toString cfg.keepalive} + umask = None + maxdelay = 300 + numcpus = None + allow_shutdown = None + + s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + numcpus=numcpus, allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; + +in { + options = { + services.buildbot-worker = { + + enable = mkOption { + type = types.bool; + default = false; + description = "Whether to enable the Buildbot Worker."; + }; + + user = mkOption { + default = "bbworker"; + type = types.str; + description = "User the buildbot Worker should execute under."; + }; + + group = mkOption { + default = "bbworker"; + type = types.str; + description = "Primary group of buildbot Worker user."; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = []; + description = "List of extra groups that the Buildbot Worker user should be a part of."; + }; + + home = mkOption { + default = "/home/bbworker"; + type = types.path; + description = "Buildbot home directory."; + }; + + buildbotDir = mkOption { + default = "${cfg.home}/worker"; + type = types.path; + description = "Specifies the Buildbot directory."; + }; + + workerUser = mkOption { + default = "example-worker"; + type = types.str; + description = "Specifies the Buildbot Worker user."; + }; + + workerPass = mkOption { + default = "pass"; + type = types.str; + description = "Specifies the Buildbot Worker password."; + }; + + workerPassFile = mkOption { + type = types.path; + description = "File used to store the Buildbot Worker password"; + }; + + hostMessage = mkOption { + default = null; + type = types.nullOr types.str; + description = "Description of this worker"; + }; + + adminMessage = mkOption { + default = null; + type = types.nullOr types.str; + description = "Name of the administrator of this worker"; + }; + + masterUrl = mkOption { + default = "localhost:9989"; + type = types.str; + description = "Specifies the Buildbot Worker connection string."; + }; + + keepalive = mkOption { + default = 600; + type = types.int; + description = " + This is a number that indicates how frequently keepalive messages should be sent + from the worker to the buildmaster, expressed in seconds. + "; + }; + + package = mkOption { + type = types.package; + default = pkgs.python3Packages.buildbot-worker; + defaultText = "pkgs.python3Packages.buildbot-worker"; + description = "Package to use for buildbot worker."; + example = literalExample "pkgs.python2Packages.buildbot-worker"; + }; + + packages = mkOption { + default = with pkgs; [ git ]; + example = literalExample "[ pkgs.git ]"; + type = types.listOf types.package; + description = "Packages to add to PATH for the buildbot process."; + }; + }; + }; + + config = mkIf cfg.enable { + services.buildbot-worker.workerPassFile = mkDefault (pkgs.writeText "buildbot-worker-password" cfg.workerPass); + + users.groups = optionalAttrs (cfg.group == "bbworker") { + bbworker = { }; + }; + + users.users = optionalAttrs (cfg.user == "bbworker") { + bbworker = { + description = "Buildbot Worker User."; + isNormalUser = true; + createHome = true; + home = cfg.home; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + }; + }; + + systemd.services.buildbot-worker = { + description = "Buildbot Worker."; + after = [ "network.target" "buildbot-master.service" ]; + wantedBy = [ "multi-user.target" ]; + path = cfg.packages; + environment.PYTHONPATH = "${python.withPackages (p: [ cfg.package ])}/${python.sitePackages}"; + + preStart = '' + mkdir -vp "${cfg.buildbotDir}/info" + ${optionalString (cfg.hostMessage != null) '' + ln -sf "${pkgs.writeText "buildbot-worker-host" cfg.hostMessage}" "${cfg.buildbotDir}/info/host" + ''} + ${optionalString (cfg.adminMessage != null) '' + ln -sf "${pkgs.writeText "buildbot-worker-admin" cfg.adminMessage}" "${cfg.buildbotDir}/info/admin" + ''} + ''; + + serviceConfig = { + Type = "simple"; + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.home; + + # NOTE: call twistd directly with stdout logging for systemd + ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}"; + }; + + }; + }; + + meta.maintainers = with lib.maintainers; [ nand0p ]; + +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/buildkite-agents.nix b/nixpkgs/nixos/modules/services/continuous-integration/buildkite-agents.nix new file mode 100644 index 000000000000..b0045409ae60 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/buildkite-agents.nix @@ -0,0 +1,276 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.buildkite-agents; + + mkHookOption = { name, description, example ? null }: { + inherit name; + value = mkOption { + default = null; + inherit description; + type = types.nullOr types.lines; + } // (if example == null then {} else { inherit example; }); + }; + mkHookOptions = hooks: listToAttrs (map mkHookOption hooks); + + hooksDir = cfg: let + mkHookEntry = name: value: '' + cat > $out/${name} <<'EOF' + #! ${pkgs.runtimeShell} + set -e + ${value} + EOF + chmod 755 $out/${name} + ''; + in pkgs.runCommand "buildkite-agent-hooks" { preferLocalBuild = true; } '' + mkdir $out + ${concatStringsSep "\n" (mapAttrsToList mkHookEntry (filterAttrs (n: v: v != null) cfg.hooks))} + ''; + + buildkiteOptions = { name ? "", config, ... }: { + options = { + enable = mkOption { + default = true; + type = types.bool; + description = "Whether to enable this buildkite agent"; + }; + + package = mkOption { + default = pkgs.buildkite-agent; + defaultText = "pkgs.buildkite-agent"; + description = "Which buildkite-agent derivation to use"; + type = types.package; + }; + + dataDir = mkOption { + default = "/var/lib/buildkite-agent-${name}"; + description = "The workdir for the agent"; + type = types.str; + }; + + runtimePackages = mkOption { + default = [ pkgs.bash pkgs.gnutar pkgs.gzip pkgs.git pkgs.nix ]; + defaultText = "[ pkgs.bash pkgs.gnutar pkgs.gzip pkgs.git pkgs.nix ]"; + description = "Add programs to the buildkite-agent environment"; + type = types.listOf types.package; + }; + + tokenPath = mkOption { + type = types.path; + description = '' + The token from your Buildkite "Agents" page. + + A run-time path to the token file, which is supposed to be provisioned + outside of Nix store. + ''; + }; + + name = mkOption { + type = types.str; + default = "%hostname-${name}-%n"; + description = '' + The name of the agent as seen in the buildkite dashboard. + ''; + }; + + tags = mkOption { + type = types.attrsOf types.str; + default = {}; + example = { queue = "default"; docker = "true"; ruby2 ="true"; }; + description = '' + Tags for the agent. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + example = "debug=true"; + description = '' + Extra lines to be added verbatim to the configuration file. + ''; + }; + + privateSshKeyPath = mkOption { + type = types.nullOr types.path; + default = null; + ## maximum care is taken so that secrets (ssh keys and the CI token) + ## don't end up in the Nix store. + apply = final: if final == null then null else toString final; + + description = '' + OpenSSH private key + + A run-time path to the key file, which is supposed to be provisioned + outside of Nix store. + ''; + }; + + hooks = mkHookOptions [ + { name = "checkout"; + description = '' + The `checkout` hook script will replace the default checkout routine of the + bootstrap.sh script. You can use this hook to do your own SCM checkout + behaviour + ''; } + { name = "command"; + description = '' + The `command` hook script will replace the default implementation of running + the build command. + ''; } + { name = "environment"; + description = '' + The `environment` hook will run before all other commands, and can be used + to set up secrets, data, etc. Anything exported in hooks will be available + to the build script. + + Note: the contents of this file will be copied to the world-readable + Nix store. + ''; + example = '' + export SECRET_VAR=`head -1 /run/keys/secret` + ''; } + { name = "post-artifact"; + description = '' + The `post-artifact` hook will run just after artifacts are uploaded + ''; } + { name = "post-checkout"; + description = '' + The `post-checkout` hook will run after the bootstrap script has checked out + your projects source code. + ''; } + { name = "post-command"; + description = '' + The `post-command` hook will run after the bootstrap script has run your + build commands + ''; } + { name = "pre-artifact"; + description = '' + The `pre-artifact` hook will run just before artifacts are uploaded + ''; } + { name = "pre-checkout"; + description = '' + The `pre-checkout` hook will run just before your projects source code is + checked out from your SCM provider + ''; } + { name = "pre-command"; + description = '' + The `pre-command` hook will run just before your build command runs + ''; } + { name = "pre-exit"; + description = '' + The `pre-exit` hook will run just before your build job finishes + ''; } + ]; + + hooksPath = mkOption { + type = types.path; + default = hooksDir config; + defaultText = "generated from services.buildkite-agents.<name>.hooks"; + description = '' + Path to the directory storing the hooks. + Consider using <option>services.buildkite-agents.<name>.hooks.<name></option> + instead. + ''; + }; + + shell = mkOption { + type = types.str; + default = "${pkgs.bash}/bin/bash -e -c"; + description = '' + Command that buildkite-agent 3 will execute when it spawns a shell. + ''; + }; + }; + }; + enabledAgents = lib.filterAttrs (n: v: v.enable) cfg; + mapAgents = function: lib.mkMerge (lib.mapAttrsToList function enabledAgents); +in +{ + options.services.buildkite-agents = mkOption { + type = types.attrsOf (types.submodule buildkiteOptions); + default = {}; + description = '' + Attribute set of buildkite agents. + The attribute key is combined with the hostname and a unique integer to + create the final agent name. This can be overridden by setting the `name` + attribute. + ''; + }; + + config.users.users = mapAgents (name: cfg: { + "buildkite-agent-${name}" = { + name = "buildkite-agent-${name}"; + home = cfg.dataDir; + createHome = true; + description = "Buildkite agent user"; + extraGroups = [ "keys" ]; + isSystemUser = true; + group = "buildkite-agent-${name}"; + }; + }); + config.users.groups = mapAgents (name: cfg: { + "buildkite-agent-${name}" = {}; + }); + + config.systemd.services = mapAgents (name: cfg: { + "buildkite-agent-${name}" = + { description = "Buildkite Agent"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = cfg.runtimePackages ++ [ cfg.package pkgs.coreutils ]; + environment = config.networking.proxy.envVars // { + HOME = cfg.dataDir; + NIX_REMOTE = "daemon"; + }; + + ## NB: maximum care is taken so that secrets (ssh keys and the CI token) + ## don't end up in the Nix store. + preStart = let + sshDir = "${cfg.dataDir}/.ssh"; + tagStr = lib.concatStringsSep "," (lib.mapAttrsToList (name: value: "${name}=${value}") cfg.tags); + in + optionalString (cfg.privateSshKeyPath != null) '' + mkdir -m 0700 -p "${sshDir}" + cp -f "${toString cfg.privateSshKeyPath}" "${sshDir}/id_rsa" + chmod 600 "${sshDir}"/id_rsa + '' + '' + cat > "${cfg.dataDir}/buildkite-agent.cfg" <<EOF + token="$(cat ${toString cfg.tokenPath})" + name="${cfg.name}" + shell="${cfg.shell}" + tags="${tagStr}" + build-path="${cfg.dataDir}/builds" + hooks-path="${cfg.hooksPath}" + ${cfg.extraConfig} + EOF + ''; + + serviceConfig = + { ExecStart = "${cfg.package}/bin/buildkite-agent start --config ${cfg.dataDir}/buildkite-agent.cfg"; + User = "buildkite-agent-${name}"; + RestartSec = 5; + Restart = "on-failure"; + TimeoutSec = 10; + # set a long timeout to give buildkite-agent a chance to finish current builds + TimeoutStopSec = "2 min"; + KillMode = "mixed"; + }; + }; + }); + + config.assertions = mapAgents (name: cfg: [ + { assertion = cfg.hooksPath == (hooksDir cfg) || all (v: v == null) (attrValues cfg.hooks); + message = '' + Options `services.buildkite-agents.${name}.hooksPath' and + `services.buildkite-agents.${name}.hooks.<name>' are mutually exclusive. + ''; + } + ]); + + imports = [ + (mkRemovedOptionModule [ "services" "buildkite-agent"] "services.buildkite-agent has been upgraded from version 2 to version 3 and moved to an attribute set at services.buildkite-agents. Please consult the 20.03 release notes for more information.") + ]; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix b/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix new file mode 100644 index 000000000000..2c6d9530a6b8 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix @@ -0,0 +1,575 @@ +{ config, lib, pkgs, ... }: +with builtins; +with lib; +let + cfg = config.services.gitlab-runner; + hasDocker = config.virtualisation.docker.enable; + hashedServices = mapAttrs' + (name: service: nameValuePair + "${name}_${config.networking.hostName}_${ + substring 0 12 + (hashString "md5" (unsafeDiscardStringContext (toJSON service)))}" + service) + cfg.services; + configPath = "$HOME/.gitlab-runner/config.toml"; + configureScript = pkgs.writeShellScriptBin "gitlab-runner-configure" ( + if (cfg.configFile != null) then '' + mkdir -p $(dirname ${configPath}) + cp ${cfg.configFile} ${configPath} + # make config file readable by service + chown -R --reference=$HOME $(dirname ${configPath}) + '' else '' + export CONFIG_FILE=${configPath} + + mkdir -p $(dirname ${configPath}) + + # remove no longer existing services + gitlab-runner verify --delete + + # current and desired state + NEEDED_SERVICES=$(echo ${concatStringsSep " " (attrNames hashedServices)} | tr " " "\n") + REGISTERED_SERVICES=$(gitlab-runner list 2>&1 | grep 'Executor' | awk '{ print $1 }') + + # difference between current and desired state + NEW_SERVICES=$(grep -vxF -f <(echo "$REGISTERED_SERVICES") <(echo "$NEEDED_SERVICES") || true) + OLD_SERVICES=$(grep -vxF -f <(echo "$NEEDED_SERVICES") <(echo "$REGISTERED_SERVICES") || true) + + # register new services + ${concatStringsSep "\n" (mapAttrsToList (name: service: '' + if echo "$NEW_SERVICES" | grep -xq ${name}; then + bash -c ${escapeShellArg (concatStringsSep " \\\n " ([ + "set -a && source ${service.registrationConfigFile} &&" + "gitlab-runner register" + "--non-interactive" + "--name ${name}" + "--executor ${service.executor}" + "--limit ${toString service.limit}" + "--request-concurrency ${toString service.requestConcurrency}" + "--maximum-timeout ${toString service.maximumTimeout}" + ] ++ service.registrationFlags + ++ optional (service.buildsDir != null) + "--builds-dir ${service.buildsDir}" + ++ optional (service.cloneUrl != null) + "--clone-url ${service.cloneUrl}" + ++ optional (service.preCloneScript != null) + "--pre-clone-script ${service.preCloneScript}" + ++ optional (service.preBuildScript != null) + "--pre-build-script ${service.preBuildScript}" + ++ optional (service.postBuildScript != null) + "--post-build-script ${service.postBuildScript}" + ++ optional (service.tagList != [ ]) + "--tag-list ${concatStringsSep "," service.tagList}" + ++ optional service.runUntagged + "--run-untagged" + ++ optional service.protected + "--access-level ref_protected" + ++ optional service.debugTraceDisabled + "--debug-trace-disabled" + ++ map (e: "--env ${escapeShellArg e}") (mapAttrsToList (name: value: "${name}=${value}") service.environmentVariables) + ++ optionals (hasPrefix "docker" service.executor) ( + assert ( + assertMsg (service.dockerImage != null) + "dockerImage option is required for ${service.executor} executor (${name})"); + [ "--docker-image ${service.dockerImage}" ] + ++ optional service.dockerDisableCache + "--docker-disable-cache" + ++ optional service.dockerPrivileged + "--docker-privileged" + ++ map (v: "--docker-volumes ${escapeShellArg v}") service.dockerVolumes + ++ map (v: "--docker-extra-hosts ${escapeShellArg v}") service.dockerExtraHosts + ++ map (v: "--docker-allowed-images ${escapeShellArg v}") service.dockerAllowedImages + ++ map (v: "--docker-allowed-services ${escapeShellArg v}") service.dockerAllowedServices + ) + ))} && sleep 1 || exit 1 + fi + '') hashedServices)} + + # unregister old services + for NAME in $(echo "$OLD_SERVICES") + do + [ ! -z "$NAME" ] && gitlab-runner unregister \ + --name "$NAME" && sleep 1 + done + + # update global options + remarshal --if toml --of json ${configPath} \ + | jq -cM ${escapeShellArg (concatStringsSep " | " [ + ".check_interval = ${toJSON cfg.checkInterval}" + ".concurrent = ${toJSON cfg.concurrent}" + ".sentry_dsn = ${toJSON cfg.sentryDSN}" + ".listen_address = ${toJSON cfg.prometheusListenAddress}" + ".session_server.listen_address = ${toJSON cfg.sessionServer.listenAddress}" + ".session_server.advertise_address = ${toJSON cfg.sessionServer.advertiseAddress}" + ".session_server.session_timeout = ${toJSON cfg.sessionServer.sessionTimeout}" + "del(.[] | nulls)" + "del(.session_server[] | nulls)" + ])} \ + | remarshal --if json --of toml \ + | sponge ${configPath} + + # make config file readable by service + chown -R --reference=$HOME $(dirname ${configPath}) + ''); + startScript = pkgs.writeShellScriptBin "gitlab-runner-start" '' + export CONFIG_FILE=${configPath} + exec gitlab-runner run --working-directory $HOME + ''; +in +{ + options.services.gitlab-runner = { + enable = mkEnableOption "Gitlab Runner"; + configFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Configuration file for gitlab-runner. + + <option>configFile</option> takes precedence over <option>services</option>. + <option>checkInterval</option> and <option>concurrent</option> will be ignored too. + + This option is deprecated, please use <option>services</option> instead. + You can use <option>registrationConfigFile</option> and + <option>registrationFlags</option> + for settings not covered by this module. + ''; + }; + checkInterval = mkOption { + type = types.int; + default = 0; + example = literalExample "with lib; (length (attrNames config.services.gitlab-runner.services)) * 3"; + description = '' + Defines the interval length, in seconds, between new jobs check. + The default value is 3; + if set to 0 or lower, the default value will be used. + See <link xlink:href="https://docs.gitlab.com/runner/configuration/advanced-configuration.html#how-check_interval-works">runner documentation</link> for more information. + ''; + }; + concurrent = mkOption { + type = types.int; + default = 1; + example = literalExample "config.nix.maxJobs"; + description = '' + Limits how many jobs globally can be run concurrently. + The most upper limit of jobs using all defined runners. + 0 does not mean unlimited. + ''; + }; + sentryDSN = mkOption { + type = types.nullOr types.str; + default = null; + example = "https://public:private@host:port/1"; + description = '' + Data Source Name for tracking of all system level errors to Sentry. + ''; + }; + prometheusListenAddress = mkOption { + type = types.nullOr types.str; + default = null; + example = "localhost:8080"; + description = '' + Address (<host>:<port>) on which the Prometheus metrics HTTP server + should be listening. + ''; + }; + sessionServer = mkOption { + type = types.submodule { + options = { + listenAddress = mkOption { + type = types.nullOr types.str; + default = null; + example = "0.0.0.0:8093"; + description = '' + An internal URL to be used for the session server. + ''; + }; + advertiseAddress = mkOption { + type = types.nullOr types.str; + default = null; + example = "runner-host-name.tld:8093"; + description = '' + The URL that the Runner will expose to GitLab to be used + to access the session server. + Fallbacks to <option>listenAddress</option> if not defined. + ''; + }; + sessionTimeout = mkOption { + type = types.int; + default = 1800; + description = '' + How long in seconds the session can stay active after + the job completes (which will block the job from finishing). + ''; + }; + }; + }; + default = { }; + example = literalExample '' + { + listenAddress = "0.0.0.0:8093"; + } + ''; + description = '' + The session server allows the user to interact with jobs + that the Runner is responsible for. A good example of this is the + <link xlink:href="https://docs.gitlab.com/ee/ci/interactive_web_terminal/index.html">interactive web terminal</link>. + ''; + }; + gracefulTermination = mkOption { + type = types.bool; + default = false; + description = '' + Finish all remaining jobs before stopping. + If not set gitlab-runner will stop immediatly without waiting + for jobs to finish, which will lead to failed builds. + ''; + }; + gracefulTimeout = mkOption { + type = types.str; + default = "infinity"; + example = "5min 20s"; + description = '' + Time to wait until a graceful shutdown is turned into a forceful one. + ''; + }; + package = mkOption { + type = types.package; + default = pkgs.gitlab-runner; + defaultText = "pkgs.gitlab-runner"; + example = literalExample "pkgs.gitlab-runner_1_11"; + description = "Gitlab Runner package to use."; + }; + extraPackages = mkOption { + type = types.listOf types.package; + default = [ ]; + description = '' + Extra packages to add to PATH for the gitlab-runner process. + ''; + }; + services = mkOption { + description = "GitLab Runner services."; + default = { }; + example = literalExample '' + { + # runner for building in docker via host's nix-daemon + # nix store will be readable in runner, might be insecure + nix = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + dockerImage = "alpine"; + dockerVolumes = [ + "/nix/store:/nix/store:ro" + "/nix/var/nix/db:/nix/var/nix/db:ro" + "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro" + ]; + dockerDisableCache = true; + preBuildScript = pkgs.writeScript "setup-container" ''' + mkdir -p -m 0755 /nix/var/log/nix/drvs + mkdir -p -m 0755 /nix/var/nix/gcroots + mkdir -p -m 0755 /nix/var/nix/profiles + mkdir -p -m 0755 /nix/var/nix/temproots + mkdir -p -m 0755 /nix/var/nix/userpool + mkdir -p -m 1777 /nix/var/nix/gcroots/per-user + mkdir -p -m 1777 /nix/var/nix/profiles/per-user + mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root + mkdir -p -m 0700 "$HOME/.nix-defexpr" + + . ''${pkgs.nix}/etc/profile.d/nix.sh + + ''${pkgs.nix}/bin/nix-env -i ''${concatStringsSep " " (with pkgs; [ nix cacert git openssh ])} + + ''${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable + ''${pkgs.nix}/bin/nix-channel --update nixpkgs + '''; + environmentVariables = { + ENV = "/etc/profile"; + USER = "root"; + NIX_REMOTE = "daemon"; + PATH = "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin"; + NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"; + }; + tagList = [ "nix" ]; + }; + # runner for building docker images + docker-images = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + dockerImage = "docker:stable"; + dockerVolumes = [ + "/var/run/docker.sock:/var/run/docker.sock" + ]; + tagList = [ "docker-images" ]; + }; + # runner for executing stuff on host system (very insecure!) + # make sure to add required packages (including git!) + # to `environment.systemPackages` + shell = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + executor = "shell"; + tagList = [ "shell" ]; + }; + # runner for everything else + default = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + dockerImage = "debian:stable"; + }; + } + ''; + type = types.attrsOf (types.submodule { + options = { + registrationConfigFile = mkOption { + type = types.path; + description = '' + Absolute path to a file with environment variables + used for gitlab-runner registration. + A list of all supported environment variables can be found in + <literal>gitlab-runner register --help</literal>. + + Ones that you probably want to set is + + <literal>CI_SERVER_URL=<CI server URL></literal> + + <literal>REGISTRATION_TOKEN=<registration secret></literal> + ''; + }; + registrationFlags = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "--docker-helper-image my/gitlab-runner-helper" ]; + description = '' + Extra command-line flags passed to + <literal>gitlab-runner register</literal>. + Execute <literal>gitlab-runner register --help</literal> + for a list of supported flags. + ''; + }; + environmentVariables = mkOption { + type = types.attrsOf types.str; + default = { }; + example = { NAME = "value"; }; + description = '' + Custom environment variables injected to build environment. + For secrets you can use <option>registrationConfigFile</option> + with <literal>RUNNER_ENV</literal> variable set. + ''; + }; + executor = mkOption { + type = types.str; + default = "docker"; + description = '' + Select executor, eg. shell, docker, etc. + See <link xlink:href="https://docs.gitlab.com/runner/executors/README.html">runner documentation</link> for more information. + ''; + }; + buildsDir = mkOption { + type = types.nullOr types.path; + default = null; + example = "/var/lib/gitlab-runner/builds"; + description = '' + Absolute path to a directory where builds will be stored + in context of selected executor (Locally, Docker, SSH). + ''; + }; + cloneUrl = mkOption { + type = types.nullOr types.str; + default = null; + example = "http://gitlab.example.local"; + description = '' + Overwrite the URL for the GitLab instance. Used if the Runner can’t connect to GitLab on the URL GitLab exposes itself. + ''; + }; + dockerImage = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Docker image to be used. + ''; + }; + dockerVolumes = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "/var/run/docker.sock:/var/run/docker.sock" ]; + description = '' + Bind-mount a volume and create it + if it doesn't exist prior to mounting. + ''; + }; + dockerDisableCache = mkOption { + type = types.bool; + default = false; + description = '' + Disable all container caching. + ''; + }; + dockerPrivileged = mkOption { + type = types.bool; + default = false; + description = '' + Give extended privileges to container. + ''; + }; + dockerExtraHosts = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "other-host:127.0.0.1" ]; + description = '' + Add a custom host-to-IP mapping. + ''; + }; + dockerAllowedImages = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "ruby:*" "python:*" "php:*" "my.registry.tld:5000/*:*" ]; + description = '' + Whitelist allowed images. + ''; + }; + dockerAllowedServices = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "postgres:9" "redis:*" "mysql:*" ]; + description = '' + Whitelist allowed services. + ''; + }; + preCloneScript = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Runner-specific command script executed before code is pulled. + ''; + }; + preBuildScript = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Runner-specific command script executed after code is pulled, + just before build executes. + ''; + }; + postBuildScript = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Runner-specific command script executed after code is pulled + and just after build executes. + ''; + }; + tagList = mkOption { + type = types.listOf types.str; + default = [ ]; + description = '' + Tag list. + ''; + }; + runUntagged = mkOption { + type = types.bool; + default = false; + description = '' + Register to run untagged builds; defaults to + <literal>true</literal> when <option>tagList</option> is empty. + ''; + }; + limit = mkOption { + type = types.int; + default = 0; + description = '' + Limit how many jobs can be handled concurrently by this service. + 0 (default) simply means don't limit. + ''; + }; + requestConcurrency = mkOption { + type = types.int; + default = 0; + description = '' + Limit number of concurrent requests for new jobs from GitLab. + ''; + }; + maximumTimeout = mkOption { + type = types.int; + default = 0; + description = '' + What is the maximum timeout (in seconds) that will be set for + job when using this Runner. 0 (default) simply means don't limit. + ''; + }; + protected = mkOption { + type = types.bool; + default = false; + description = '' + When set to true Runner will only run on pipelines + triggered on protected branches. + ''; + }; + debugTraceDisabled = mkOption { + type = types.bool; + default = false; + description = '' + When set to true Runner will disable the possibility of + using the <literal>CI_DEBUG_TRACE</literal> feature. + ''; + }; + }; + }); + }; + }; + config = mkIf cfg.enable { + warnings = optional (cfg.configFile != null) "services.gitlab-runner.`configFile` is deprecated, please use services.gitlab-runner.`services`."; + environment.systemPackages = [ cfg.package ]; + systemd.services.gitlab-runner = { + description = "Gitlab Runner"; + documentation = [ "https://docs.gitlab.com/runner/" ]; + after = [ "network.target" ] + ++ optional hasDocker "docker.service"; + requires = optional hasDocker "docker.service"; + wantedBy = [ "multi-user.target" ]; + environment = config.networking.proxy.envVars // { + HOME = "/var/lib/gitlab-runner"; + }; + path = with pkgs; [ + bash + gawk + jq + moreutils + remarshal + util-linux + cfg.package + ] ++ cfg.extraPackages; + reloadIfChanged = true; + serviceConfig = { + # Set `DynamicUser` under `systemd.services.gitlab-runner.serviceConfig` + # to `lib.mkForce false` in your configuration to run this service as root. + # You can also set `User` and `Group` options to run this service as desired user. + # Make sure to restart service or changes won't apply. + DynamicUser = true; + StateDirectory = "gitlab-runner"; + SupplementaryGroups = optional hasDocker "docker"; + ExecStartPre = "!${configureScript}/bin/gitlab-runner-configure"; + ExecStart = "${startScript}/bin/gitlab-runner-start"; + ExecReload = "!${configureScript}/bin/gitlab-runner-configure"; + } // optionalAttrs (cfg.gracefulTermination) { + TimeoutStopSec = "${cfg.gracefulTimeout}"; + KillSignal = "SIGQUIT"; + KillMode = "process"; + }; + }; + # Enable docker if `docker` executor is used in any service + virtualisation.docker.enable = mkIf ( + any (s: s.executor == "docker") (attrValues cfg.services) + ) (mkDefault true); + }; + imports = [ + (mkRenamedOptionModule [ "services" "gitlab-runner" "packages" ] [ "services" "gitlab-runner" "extraPackages" ] ) + (mkRemovedOptionModule [ "services" "gitlab-runner" "configOptions" ] "Use services.gitlab-runner.services option instead" ) + (mkRemovedOptionModule [ "services" "gitlab-runner" "workDir" ] "You should move contents of workDir (if any) to /var/lib/gitlab-runner" ) + ]; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/gocd-agent/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/gocd-agent/default.nix new file mode 100644 index 000000000000..8cae08bf1fa0 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/gocd-agent/default.nix @@ -0,0 +1,208 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.gocd-agent; +in { + options = { + services.gocd-agent = { + enable = mkEnableOption "gocd-agent"; + + user = mkOption { + default = "gocd-agent"; + type = types.str; + description = '' + User the Go.CD agent should execute under. + ''; + }; + + group = mkOption { + default = "gocd-agent"; + type = types.str; + description = '' + If the default user "gocd-agent" is configured then this is the primary + group of that user. + ''; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "wheel" "docker" ]; + description = '' + List of extra groups that the "gocd-agent" user should be a part of. + ''; + }; + + packages = mkOption { + default = [ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]; + defaultText = "[ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]"; + type = types.listOf types.package; + description = '' + Packages to add to PATH for the Go.CD agent process. + ''; + }; + + agentConfig = mkOption { + default = ""; + type = types.str; + example = '' + agent.auto.register.resources=ant,java + agent.auto.register.environments=QA,Performance + agent.auto.register.hostname=Agent01 + ''; + description = '' + Agent registration configuration. + ''; + }; + + goServer = mkOption { + default = "https://127.0.0.1:8154/go"; + type = types.str; + description = '' + URL of the GoCD Server to attach the Go.CD Agent to. + ''; + }; + + workDir = mkOption { + default = "/var/lib/go-agent"; + type = types.str; + description = '' + Specifies the working directory in which the Go.CD agent java archive resides. + ''; + }; + + initialJavaHeapSize = mkOption { + default = "128m"; + type = types.str; + description = '' + Specifies the initial java heap memory size for the Go.CD agent java process. + ''; + }; + + maxJavaHeapMemory = mkOption { + default = "256m"; + type = types.str; + description = '' + Specifies the java maximum heap memory size for the Go.CD agent java process. + ''; + }; + + startupOptions = mkOption { + type = types.listOf types.str; + default = [ + "-Xms${cfg.initialJavaHeapSize}" + "-Xmx${cfg.maxJavaHeapMemory}" + "-Djava.io.tmpdir=/tmp" + "-Dcruise.console.publish.interval=10" + "-Djava.security.egd=file:/dev/./urandom" + ]; + description = '' + Specifies startup command line arguments to pass to Go.CD agent + java process. + ''; + }; + + extraOptions = mkOption { + default = [ ]; + type = types.listOf types.str; + example = [ + "-X debug" + "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5006" + "-verbose:gc" + "-Xloggc:go-agent-gc.log" + "-XX:+PrintGCTimeStamps" + "-XX:+PrintTenuringDistribution" + "-XX:+PrintGCDetails" + "-XX:+PrintGC" + ]; + description = '' + Specifies additional command line arguments to pass to Go.CD agent + java process. Example contains debug and gcLog arguments. + ''; + }; + + environment = mkOption { + default = { }; + type = with types; attrsOf str; + description = '' + Additional environment variables to be passed to the Go.CD agent process. + As a base environment, Go.CD agent receives NIX_PATH from + <option>environment.sessionVariables</option>, NIX_REMOTE is set to + "daemon". + ''; + }; + }; + }; + + config = mkIf cfg.enable { + users.groups = optionalAttrs (cfg.group == "gocd-agent") { + gocd-agent.gid = config.ids.gids.gocd-agent; + }; + + users.users = optionalAttrs (cfg.user == "gocd-agent") { + gocd-agent = { + description = "gocd-agent user"; + createHome = true; + home = cfg.workDir; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + uid = config.ids.uids.gocd-agent; + }; + }; + + systemd.services.gocd-agent = { + description = "GoCD Agent"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = + let + selectedSessionVars = + lib.filterAttrs (n: v: builtins.elem n [ "NIX_PATH" ]) + config.environment.sessionVariables; + in + selectedSessionVars // + { + NIX_REMOTE = "daemon"; + AGENT_WORK_DIR = cfg.workDir; + AGENT_STARTUP_ARGS = ''${concatStringsSep " " cfg.startupOptions}''; + LOG_DIR = cfg.workDir; + LOG_FILE = "${cfg.workDir}/go-agent-start.log"; + } // + cfg.environment; + + path = cfg.packages; + + script = '' + MPATH="''${PATH}"; + source /etc/profile + export PATH="''${MPATH}:''${PATH}"; + + if ! test -f ~/.nixpkgs/config.nix; then + mkdir -p ~/.nixpkgs/ + echo "{ allowUnfree = true; }" > ~/.nixpkgs/config.nix + fi + + mkdir -p config + rm -f config/autoregister.properties + ln -s "${pkgs.writeText "autoregister.properties" cfg.agentConfig}" config/autoregister.properties + + ${pkgs.git}/bin/git config --global --add http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + ${pkgs.jre}/bin/java ${concatStringsSep " " cfg.startupOptions} \ + ${concatStringsSep " " cfg.extraOptions} \ + -jar ${pkgs.gocd-agent}/go-agent/agent-bootstrapper.jar \ + -serverUrl ${cfg.goServer} + ''; + + serviceConfig = { + User = cfg.user; + WorkingDirectory = cfg.workDir; + RestartSec = 30; + Restart = "on-failure"; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/gocd-server/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/gocd-server/default.nix new file mode 100644 index 000000000000..4c829664a0a5 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/gocd-server/default.nix @@ -0,0 +1,197 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.gocd-server; +in { + options = { + services.gocd-server = { + enable = mkEnableOption "gocd-server"; + + user = mkOption { + default = "gocd-server"; + type = types.str; + description = '' + User the Go.CD server should execute under. + ''; + }; + + group = mkOption { + default = "gocd-server"; + type = types.str; + description = '' + If the default user "gocd-server" is configured then this is the primary group of that user. + ''; + }; + + extraGroups = mkOption { + default = [ ]; + type = types.listOf types.str; + example = [ "wheel" "docker" ]; + description = '' + List of extra groups that the "gocd-server" user should be a part of. + ''; + }; + + listenAddress = mkOption { + default = "0.0.0.0"; + example = "localhost"; + type = types.str; + description = '' + Specifies the bind address on which the Go.CD server HTTP interface listens. + ''; + }; + + port = mkOption { + default = 8153; + type = types.int; + description = '' + Specifies port number on which the Go.CD server HTTP interface listens. + ''; + }; + + sslPort = mkOption { + default = 8154; + type = types.int; + description = '' + Specifies port number on which the Go.CD server HTTPS interface listens. + ''; + }; + + workDir = mkOption { + default = "/var/lib/go-server"; + type = types.str; + description = '' + Specifies the working directory in which the Go.CD server java archive resides. + ''; + }; + + packages = mkOption { + default = [ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]; + defaultText = "[ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]"; + type = types.listOf types.package; + description = '' + Packages to add to PATH for the Go.CD server's process. + ''; + }; + + initialJavaHeapSize = mkOption { + default = "512m"; + type = types.str; + description = '' + Specifies the initial java heap memory size for the Go.CD server's java process. + ''; + }; + + maxJavaHeapMemory = mkOption { + default = "1024m"; + type = types.str; + description = '' + Specifies the java maximum heap memory size for the Go.CD server's java process. + ''; + }; + + startupOptions = mkOption { + type = types.listOf types.str; + default = [ + "-Xms${cfg.initialJavaHeapSize}" + "-Xmx${cfg.maxJavaHeapMemory}" + "-Dcruise.listen.host=${cfg.listenAddress}" + "-Duser.language=en" + "-Djruby.rack.request.size.threshold.bytes=30000000" + "-Duser.country=US" + "-Dcruise.config.dir=${cfg.workDir}/conf" + "-Dcruise.config.file=${cfg.workDir}/conf/cruise-config.xml" + "-Dcruise.server.port=${toString cfg.port}" + "-Dcruise.server.ssl.port=${toString cfg.sslPort}" + ]; + + description = '' + Specifies startup command line arguments to pass to Go.CD server + java process. + ''; + }; + + extraOptions = mkOption { + default = [ ]; + type = types.listOf types.str; + example = [ + "-X debug" + "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005" + "-verbose:gc" + "-Xloggc:go-server-gc.log" + "-XX:+PrintGCTimeStamps" + "-XX:+PrintTenuringDistribution" + "-XX:+PrintGCDetails" + "-XX:+PrintGC" + ]; + description = '' + Specifies additional command line arguments to pass to Go.CD server's + java process. Example contains debug and gcLog arguments. + ''; + }; + + environment = mkOption { + default = { }; + type = with types; attrsOf str; + description = '' + Additional environment variables to be passed to the gocd-server process. + As a base environment, gocd-server receives NIX_PATH from + <option>environment.sessionVariables</option>, NIX_REMOTE is set to + "daemon". + ''; + }; + }; + }; + + config = mkIf cfg.enable { + users.groups = optionalAttrs (cfg.group == "gocd-server") { + gocd-server.gid = config.ids.gids.gocd-server; + }; + + users.users = optionalAttrs (cfg.user == "gocd-server") { + gocd-server = { + description = "gocd-server user"; + createHome = true; + home = cfg.workDir; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + uid = config.ids.uids.gocd-server; + }; + }; + + systemd.services.gocd-server = { + description = "GoCD Server"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = + let + selectedSessionVars = + lib.filterAttrs (n: v: builtins.elem n [ "NIX_PATH" ]) + config.environment.sessionVariables; + in + selectedSessionVars // + { NIX_REMOTE = "daemon"; + } // + cfg.environment; + + path = cfg.packages; + + script = '' + ${pkgs.git}/bin/git config --global --add http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + ${pkgs.jre}/bin/java -server ${concatStringsSep " " cfg.startupOptions} \ + ${concatStringsSep " " cfg.extraOptions} \ + -jar ${pkgs.gocd-server}/go-server/go.jar + ''; + + serviceConfig = { + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.workDir; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/hail.nix b/nixpkgs/nixos/modules/services/continuous-integration/hail.nix new file mode 100644 index 000000000000..5d0c3f7b4ab3 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/hail.nix @@ -0,0 +1,61 @@ +{ config, lib, pkgs, ...}: + +with lib; + +let + cfg = config.services.hail; +in { + + + ###### interface + + options.services.hail = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enables the Hail Auto Update Service. Hail can automatically deploy artifacts + built by a Hydra Continous Integration server. A common use case is to provide + continous deployment for single services or a full NixOS configuration.''; + }; + profile = mkOption { + type = types.str; + default = "hail-profile"; + description = "The name of the Nix profile used by Hail."; + }; + hydraJobUri = mkOption { + type = types.str; + description = "The URI of the Hydra Job."; + }; + netrc = mkOption { + type = types.nullOr types.path; + description = "The netrc file to use when fetching data from Hydra."; + default = null; + }; + package = mkOption { + type = types.package; + default = pkgs.haskellPackages.hail; + defaultText = "pkgs.haskellPackages.hail"; + description = "Hail package to use."; + }; + }; + + + ###### implementation + + config = mkIf cfg.enable { + systemd.services.hail = { + description = "Hail Auto Update Service"; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + path = with pkgs; [ nix ]; + environment = { + HOME = "/var/lib/empty"; + }; + serviceConfig = { + ExecStart = "${cfg.package}/bin/hail --profile ${cfg.profile} --job-uri ${cfg.hydraJobUri}" + + lib.optionalString (cfg.netrc != null) " --netrc-file ${cfg.netrc}"; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix b/nixpkgs/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix new file mode 100644 index 000000000000..9f9b86ee61cb --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix @@ -0,0 +1,192 @@ +/* + +This file is for options that NixOS and nix-darwin have in common. + +Platform-specific code is in the respective default.nix files. + + */ + +{ config, lib, options, pkgs, ... }: +let + inherit (lib) + filterAttrs + literalExample + mkIf + mkOption + mkRemovedOptionModule + mkRenamedOptionModule + types + ; + + cfg = + config.services.hercules-ci-agent; + + format = pkgs.formats.toml { }; + + settingsModule = { config, ... }: { + freeformType = format.type; + options = { + baseDirectory = mkOption { + type = types.path; + default = "/var/lib/hercules-ci-agent"; + description = '' + State directory (secrets, work directory, etc) for agent + ''; + }; + concurrentTasks = mkOption { + description = '' + Number of tasks to perform simultaneously. + + A task is a single derivation build or an evaluation. + At minimum, you need 2 concurrent tasks for <literal>x86_64-linux</literal> + in your cluster, to allow for import from derivation. + + <literal>concurrentTasks</literal> can be around the CPU core count or lower if memory is + the bottleneck. + ''; + type = types.int; + default = 4; + }; + workDirectory = mkOption { + description = '' + The directory in which temporary subdirectories are created for task state. This includes sources for Nix evaluation. + ''; + type = types.path; + default = config.baseDirectory + "/work"; + defaultText = literalExample ''baseDirectory + "/work"''; + }; + staticSecretsDirectory = mkOption { + description = '' + This is the default directory to look for statically configured secrets like <literal>cluster-join-token.key</literal>. + ''; + type = types.path; + default = config.baseDirectory + "/secrets"; + defaultText = literalExample ''baseDirectory + "/secrets"''; + }; + clusterJoinTokenPath = mkOption { + description = '' + Location of the cluster-join-token.key file. + ''; + type = types.path; + default = config.staticSecretsDirectory + "/cluster-join-token.key"; + defaultText = literalExample ''staticSecretsDirectory + "/cluster-join-token.key"''; + # internal: It's a bit too detailed to show by default in the docs, + # but useful to define explicitly to allow reuse by other modules. + internal = true; + }; + binaryCachesPath = mkOption { + description = '' + Location of the binary-caches.json file. + ''; + type = types.path; + default = config.staticSecretsDirectory + "/binary-caches.json"; + defaultText = literalExample ''staticSecretsDirectory + "/binary-caches.json"''; + # internal: It's a bit too detailed to show by default in the docs, + # but useful to define explicitly to allow reuse by other modules. + internal = true; + }; + }; + }; + + # TODO (roberth, >=2022) remove + checkNix = + if !cfg.checkNix + then "" + else if lib.versionAtLeast config.nix.package.version "2.3.10" + then "" + else + pkgs.stdenv.mkDerivation { + name = "hercules-ci-check-system-nix-src"; + inherit (config.nix.package) src patches; + configurePhase = ":"; + buildPhase = '' + echo "Checking in-memory pathInfoCache expiry" + if ! grep 'PathInfoCacheValue' src/libstore/store-api.hh >/dev/null; then + cat 1>&2 <<EOF + + You are deploying Hercules CI Agent on a system with an incompatible + nix-daemon. Please make sure nix.package is set to a Nix version of at + least 2.3.10 or a master version more recent than Mar 12, 2020. + EOF + exit 1 + fi + ''; + installPhase = "touch $out"; + }; + +in +{ + imports = [ + (mkRenamedOptionModule [ "services" "hercules-ci-agent" "extraOptions" ] [ "services" "hercules-ci-agent" "settings" ]) + (mkRenamedOptionModule [ "services" "hercules-ci-agent" "baseDirectory" ] [ "services" "hercules-ci-agent" "settings" "baseDirectory" ]) + (mkRenamedOptionModule [ "services" "hercules-ci-agent" "concurrentTasks" ] [ "services" "hercules-ci-agent" "settings" "concurrentTasks" ]) + (mkRemovedOptionModule [ "services" "hercules-ci-agent" "patchNix" ] "Nix versions packaged in this version of Nixpkgs don't need a patched nix-daemon to work correctly in Hercules CI Agent clusters.") + ]; + + options.services.hercules-ci-agent = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enable to run Hercules CI Agent as a system service. + + <link xlink:href="https://hercules-ci.com">Hercules CI</link> is a + continuous integation service that is centered around Nix. + + Support is available at <link xlink:href="mailto:help@hercules-ci.com">help@hercules-ci.com</link>. + ''; + }; + checkNix = mkOption { + type = types.bool; + default = true; + description = '' + Whether to make sure that the system's Nix (nix-daemon) is compatible. + + If you set this to false, please keep up with the change log. + ''; + }; + package = mkOption { + description = '' + Package containing the bin/hercules-ci-agent executable. + ''; + type = types.package; + default = pkgs.hercules-ci-agent; + defaultText = literalExample "pkgs.hercules-ci-agent"; + }; + settings = mkOption { + description = '' + These settings are written to the <literal>agent.toml</literal> file. + + Not all settings are listed as options, can be set nonetheless. + + For the exhaustive list of settings, see <link xlink:href="https://docs.hercules-ci.com/hercules-ci/reference/agent-config/"/>. + ''; + type = types.submoduleWith { modules = [ settingsModule ]; }; + }; + + /* + Internal and/or computed values. + + These are written as options instead of let binding to allow sharing with + default.nix on both NixOS and nix-darwin. + */ + tomlFile = mkOption { + type = types.path; + internal = true; + defaultText = "generated hercules-ci-agent.toml"; + description = '' + The fully assembled config file. + ''; + }; + }; + + config = mkIf cfg.enable { + nix.extraOptions = lib.addContextFrom checkNix '' + # A store path that was missing at first may well have finished building, + # even shortly after the previous lookup. This *also* applies to the daemon. + narinfo-cache-negative-ttl = 0 + ''; + services.hercules-ci-agent.tomlFile = + format.generate "hercules-ci-agent.toml" cfg.settings; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix new file mode 100644 index 000000000000..e8a42e59de0d --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix @@ -0,0 +1,85 @@ +/* + +This file is for NixOS-specific options and configs. + +Code that is shared with nix-darwin goes in common.nix. + + */ + +{ pkgs, config, lib, ... }: +let + inherit (lib) mkIf mkDefault; + + cfg = config.services.hercules-ci-agent; + + command = "${cfg.package}/bin/hercules-ci-agent --config ${cfg.tomlFile}"; + testCommand = "${command} --test-configuration"; + +in +{ + imports = [ + ./common.nix + (lib.mkRenamedOptionModule [ "services" "hercules-ci-agent" "user" ] [ "systemd" "services" "hercules-ci-agent" "serviceConfig" "User" ]) + ]; + + config = mkIf cfg.enable { + systemd.services.hercules-ci-agent = { + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + path = [ config.nix.package ]; + startLimitBurst = 30 * 1000000; # practically infinite + serviceConfig = { + User = "hercules-ci-agent"; + ExecStart = command; + ExecStartPre = testCommand; + Restart = "on-failure"; + RestartSec = 120; + }; + }; + + # Changes in the secrets do not affect the unit in any way that would cause + # a restart, which is currently necessary to reload the secrets. + systemd.paths.hercules-ci-agent-restart-files = { + wantedBy = [ "hercules-ci-agent.service" ]; + pathConfig = { + Unit = "hercules-ci-agent-restarter.service"; + PathChanged = [ cfg.settings.clusterJoinTokenPath cfg.settings.binaryCachesPath ]; + }; + }; + systemd.services.hercules-ci-agent-restarter = { + serviceConfig.Type = "oneshot"; + script = '' + # Wait a bit, with the effect of bundling up file changes into a single + # run of this script and hopefully a single restart. + sleep 10 + if systemctl is-active --quiet hercules-ci-agent.service; then + if ${testCommand}; then + systemctl restart hercules-ci-agent.service + else + echo 1>&2 "WARNING: Not restarting agent because config is not valid at this time." + fi + else + echo 1>&2 "Not restarting hercules-ci-agent despite config file update, because it is not already active." + fi + ''; + }; + + # Trusted user allows simplified configuration and better performance + # when operating in a cluster. + nix.trustedUsers = [ config.systemd.services.hercules-ci-agent.serviceConfig.User ]; + services.hercules-ci-agent.settings.nixUserIsTrusted = true; + + users.users.hercules-ci-agent = { + home = cfg.settings.baseDirectory; + createHome = true; + group = "hercules-ci-agent"; + description = "Hercules CI Agent system user"; + isSystemUser = true; + }; + + users.groups.hercules-ci-agent = { }; + }; + + meta.maintainers = [ lib.maintainers.roberth ]; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/hydra/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/hydra/default.nix new file mode 100644 index 000000000000..0103cd723d2f --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/hydra/default.nix @@ -0,0 +1,497 @@ +{ config, pkgs, lib, ... }: + +with lib; + +let + + cfg = config.services.hydra; + + baseDir = "/var/lib/hydra"; + + hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig; + + hydraEnv = + { HYDRA_DBI = cfg.dbi; + HYDRA_CONFIG = "${baseDir}/hydra.conf"; + HYDRA_DATA = "${baseDir}"; + }; + + env = + { NIX_REMOTE = "daemon"; + SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03 + PGPASSFILE = "${baseDir}/pgpass"; + NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles; + } // optionalAttrs (cfg.smtpHost != null) { + EMAIL_SENDER_TRANSPORT = "SMTP"; + EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost; + } // hydraEnv // cfg.extraEnv; + + serverEnv = env // + { HYDRA_TRACKER = cfg.tracker; + XDG_CACHE_HOME = "${baseDir}/www/.cache"; + COLUMNS = "80"; + PGPASSFILE = "${baseDir}/pgpass-www"; # grrr + } // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; }); + + localDB = "dbi:Pg:dbname=hydra;user=hydra;"; + + haveLocalDB = cfg.dbi == localDB; + + hydra-package = + let + makeWrapperArgs = concatStringsSep " " (mapAttrsToList (key: value: "--set \"${key}\" \"${value}\"") hydraEnv); + in pkgs.buildEnv rec { + name = "hydra-env"; + buildInputs = [ pkgs.makeWrapper ]; + paths = [ cfg.package ]; + + postBuild = '' + if [ -L "$out/bin" ]; then + unlink "$out/bin" + fi + mkdir -p "$out/bin" + + for path in ${concatStringsSep " " paths}; do + if [ -d "$path/bin" ]; then + cd "$path/bin" + for prg in *; do + if [ -f "$prg" ]; then + rm -f "$out/bin/$prg" + if [ -x "$prg" ]; then + makeWrapper "$path/bin/$prg" "$out/bin/$prg" ${makeWrapperArgs} + fi + fi + done + fi + done + ''; + }; + +in + +{ + ###### interface + options = { + + services.hydra = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run Hydra services. + ''; + }; + + dbi = mkOption { + type = types.str; + default = localDB; + example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;"; + description = '' + The DBI string for Hydra database connection. + + NOTE: Attempts to set `application_name` will be overridden by + `hydra-TYPE` (where TYPE is e.g. `evaluator`, `queue-runner`, + etc.) in all hydra services to more easily distinguish where + queries are coming from. + ''; + }; + + package = mkOption { + type = types.package; + default = pkgs.hydra-unstable; + defaultText = "pkgs.hydra-unstable"; + description = "The Hydra package."; + }; + + hydraURL = mkOption { + type = types.str; + description = '' + The base URL for the Hydra webserver instance. Used for links in emails. + ''; + }; + + listenHost = mkOption { + type = types.str; + default = "*"; + example = "localhost"; + description = '' + The hostname or address to listen on or <literal>*</literal> to listen + on all interfaces. + ''; + }; + + port = mkOption { + type = types.int; + default = 3000; + description = '' + TCP port the web server should listen to. + ''; + }; + + minimumDiskFree = mkOption { + type = types.int; + default = 0; + description = '' + Threshold of minimum disk space (GiB) to determine if the queue runner should run or not. + ''; + }; + + minimumDiskFreeEvaluator = mkOption { + type = types.int; + default = 0; + description = '' + Threshold of minimum disk space (GiB) to determine if the evaluator should run or not. + ''; + }; + + notificationSender = mkOption { + type = types.str; + description = '' + Sender email address used for email notifications. + ''; + }; + + smtpHost = mkOption { + type = types.nullOr types.str; + default = null; + example = ["localhost"]; + description = '' + Hostname of the SMTP server to use to send email. + ''; + }; + + tracker = mkOption { + type = types.str; + default = ""; + description = '' + Piece of HTML that is included on all pages. + ''; + }; + + logo = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Path to a file containing the logo of your Hydra instance. + ''; + }; + + debugServer = mkOption { + type = types.bool; + default = false; + description = "Whether to run the server in debug mode."; + }; + + extraConfig = mkOption { + type = types.lines; + description = "Extra lines for the Hydra configuration."; + }; + + extraEnv = mkOption { + type = types.attrsOf types.str; + default = {}; + description = "Extra environment variables for Hydra."; + }; + + gcRootsDir = mkOption { + type = types.path; + default = "/nix/var/nix/gcroots/hydra"; + description = "Directory that holds Hydra garbage collector roots."; + }; + + buildMachinesFiles = mkOption { + type = types.listOf types.path; + default = optional (config.nix.buildMachines != []) "/etc/nix/machines"; + example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ]; + description = "List of files containing build machines."; + }; + + useSubstitutes = mkOption { + type = types.bool; + default = false; + description = '' + Whether to use binary caches for downloading store paths. Note that + binary substitutions trigger (a potentially large number of) additional + HTTP requests that slow down the queue monitor thread significantly. + Also, this Hydra instance will serve those downloaded store paths to + its users with its own signature attached as if it had built them + itself, so don't enable this feature unless your active binary caches + are absolute trustworthy. + ''; + }; + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + users.groups.hydra = { + gid = config.ids.gids.hydra; + }; + + users.users.hydra = + { description = "Hydra"; + group = "hydra"; + # We don't enable `createHome` here because the creation of the home directory is handled by the hydra-init service below. + home = baseDir; + useDefaultShell = true; + uid = config.ids.uids.hydra; + }; + + users.users.hydra-queue-runner = + { description = "Hydra queue runner"; + group = "hydra"; + useDefaultShell = true; + home = "${baseDir}/queue-runner"; # really only to keep SSH happy + uid = config.ids.uids.hydra-queue-runner; + }; + + users.users.hydra-www = + { description = "Hydra web server"; + group = "hydra"; + useDefaultShell = true; + uid = config.ids.uids.hydra-www; + }; + + nix.trustedUsers = [ "hydra-queue-runner" ]; + + services.hydra.extraConfig = + '' + using_frontend_proxy = 1 + base_uri = ${cfg.hydraURL} + notification_sender = ${cfg.notificationSender} + max_servers = 25 + ${optionalString (cfg.logo != null) '' + hydra_logo = ${cfg.logo} + ''} + gc_roots_dir = ${cfg.gcRootsDir} + use-substitutes = ${if cfg.useSubstitutes then "1" else "0"} + ''; + + environment.systemPackages = [ hydra-package ]; + + environment.variables = hydraEnv; + + nix.extraOptions = '' + keep-outputs = true + keep-derivations = true + + + '' + optionalString (versionOlder (getVersion config.nix.package.out) "2.4pre") '' + # The default (`true') slows Nix down a lot since the build farm + # has so many GC roots. + gc-check-reachability = false + ''; + + systemd.services.hydra-init = + { wantedBy = [ "multi-user.target" ]; + requires = optional haveLocalDB "postgresql.service"; + after = optional haveLocalDB "postgresql.service"; + environment = env // { + HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init"; + }; + preStart = '' + mkdir -p ${baseDir} + chown hydra.hydra ${baseDir} + chmod 0750 ${baseDir} + + ln -sf ${hydraConf} ${baseDir}/hydra.conf + + mkdir -m 0700 -p ${baseDir}/www + chown hydra-www.hydra ${baseDir}/www + + mkdir -m 0700 -p ${baseDir}/queue-runner + mkdir -m 0750 -p ${baseDir}/build-logs + chown hydra-queue-runner.hydra ${baseDir}/queue-runner ${baseDir}/build-logs + + ${optionalString haveLocalDB '' + if ! [ -e ${baseDir}/.db-created ]; then + ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra + ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -O hydra hydra + touch ${baseDir}/.db-created + fi + echo "create extension if not exists pg_trgm" | ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra + ''} + + if [ ! -e ${cfg.gcRootsDir} ]; then + + # Move legacy roots directory. + if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then + mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir} + fi + + mkdir -p ${cfg.gcRootsDir} + fi + + # Move legacy hydra-www roots. + if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then + find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \ + | xargs -r mv -f -t ${cfg.gcRootsDir}/ + rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots + fi + + chown hydra.hydra ${cfg.gcRootsDir} + chmod 2775 ${cfg.gcRootsDir} + ''; + serviceConfig.ExecStart = "${hydra-package}/bin/hydra-init"; + serviceConfig.PermissionsStartOnly = true; + serviceConfig.User = "hydra"; + serviceConfig.Type = "oneshot"; + serviceConfig.RemainAfterExit = true; + }; + + systemd.services.hydra-server = + { wantedBy = [ "multi-user.target" ]; + requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" ]; + environment = serverEnv // { + HYDRA_DBI = "${serverEnv.HYDRA_DBI};application_name=hydra-server"; + }; + restartTriggers = [ hydraConf ]; + serviceConfig = + { ExecStart = + "@${hydra-package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' " + + "-p ${toString cfg.port} --max_spare_servers 5 --max_servers 25 " + + "--max_requests 100 ${optionalString cfg.debugServer "-d"}"; + User = "hydra-www"; + PermissionsStartOnly = true; + Restart = "always"; + }; + }; + + systemd.services.hydra-queue-runner = + { wantedBy = [ "multi-user.target" ]; + requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" "network.target" ]; + path = [ hydra-package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ]; + restartTriggers = [ hydraConf ]; + environment = env // { + PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr + IN_SYSTEMD = "1"; # to get log severity levels + HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-queue-runner"; + }; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v"; + ExecStopPost = "${hydra-package}/bin/hydra-queue-runner --unlock"; + User = "hydra-queue-runner"; + Restart = "always"; + + # Ensure we can get core dumps. + LimitCORE = "infinity"; + WorkingDirectory = "${baseDir}/queue-runner"; + }; + }; + + systemd.services.hydra-evaluator = + { wantedBy = [ "multi-user.target" ]; + requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" "network.target" ]; + path = with pkgs; [ hydra-package nettools jq ]; + restartTriggers = [ hydraConf ]; + environment = env // { + HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator"; + }; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator"; + User = "hydra"; + Restart = "always"; + WorkingDirectory = baseDir; + }; + }; + + systemd.services.hydra-update-gc-roots = + { requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" ]; + environment = env // { + HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-update-gc-roots"; + }; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots"; + User = "hydra"; + }; + startAt = "2,14:15"; + }; + + systemd.services.hydra-send-stats = + { wantedBy = [ "multi-user.target" ]; + after = [ "hydra-init.service" ]; + environment = env // { + HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-send-stats"; + }; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats"; + User = "hydra"; + }; + }; + + systemd.services.hydra-notify = + { wantedBy = [ "multi-user.target" ]; + requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" ]; + restartTriggers = [ hydraConf ]; + environment = env // { + PGPASSFILE = "${baseDir}/pgpass-queue-runner"; + HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify"; + }; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify"; + # FIXME: run this under a less privileged user? + User = "hydra-queue-runner"; + Restart = "always"; + RestartSec = 5; + }; + }; + + # If there is less than a certain amount of free disk space, stop + # the queue/evaluator to prevent builds from failing or aborting. + systemd.services.hydra-check-space = + { script = + '' + if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then + echo "stopping Hydra queue runner due to lack of free space..." + systemctl stop hydra-queue-runner + fi + if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then + echo "stopping Hydra evaluator due to lack of free space..." + systemctl stop hydra-evaluator + fi + ''; + startAt = "*:0/5"; + }; + + # Periodically compress build logs. The queue runner compresses + # logs automatically after a step finishes, but this doesn't work + # if the queue runner is stopped prematurely. + systemd.services.hydra-compress-logs = + { path = [ pkgs.bzip2 ]; + script = + '' + find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f + ''; + startAt = "Sun 01:45"; + }; + + services.postgresql.enable = mkIf haveLocalDB true; + + services.postgresql.identMap = optionalString haveLocalDB + '' + hydra-users hydra hydra + hydra-users hydra-queue-runner hydra + hydra-users hydra-www hydra + hydra-users root hydra + # The postgres user is used to create the pg_trgm extension for the hydra database + hydra-users postgres postgres + ''; + + services.postgresql.authentication = optionalString haveLocalDB + '' + local hydra all ident map=hydra-users + ''; + + }; + +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/jenkins/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/default.nix new file mode 100644 index 000000000000..cdc3b4b5c58f --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/default.nix @@ -0,0 +1,228 @@ +{ config, lib, pkgs, ... }: +with lib; +let + cfg = config.services.jenkins; +in { + options = { + services.jenkins = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable the jenkins continuous integration server. + ''; + }; + + user = mkOption { + default = "jenkins"; + type = types.str; + description = '' + User the jenkins server should execute under. + ''; + }; + + group = mkOption { + default = "jenkins"; + type = types.str; + description = '' + If the default user "jenkins" is configured then this is the primary + group of that user. + ''; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "wheel" "dialout" ]; + description = '' + List of extra groups that the "jenkins" user should be a part of. + ''; + }; + + home = mkOption { + default = "/var/lib/jenkins"; + type = types.path; + description = '' + The path to use as JENKINS_HOME. If the default user "jenkins" is configured then + this is the home of the "jenkins" user. + ''; + }; + + listenAddress = mkOption { + default = "0.0.0.0"; + example = "localhost"; + type = types.str; + description = '' + Specifies the bind address on which the jenkins HTTP interface listens. + The default is the wildcard address. + ''; + }; + + port = mkOption { + default = 8080; + type = types.int; + description = '' + Specifies port number on which the jenkins HTTP interface listens. + The default is 8080. + ''; + }; + + prefix = mkOption { + default = ""; + example = "/jenkins"; + type = types.str; + description = '' + Specifies a urlPrefix to use with jenkins. + If the example /jenkins is given, the jenkins server will be + accessible using localhost:8080/jenkins. + ''; + }; + + package = mkOption { + default = pkgs.jenkins; + defaultText = "pkgs.jenkins"; + type = types.package; + description = "Jenkins package to use."; + }; + + packages = mkOption { + default = [ pkgs.stdenv pkgs.git pkgs.jdk11 config.programs.ssh.package pkgs.nix ]; + defaultText = "[ pkgs.stdenv pkgs.git pkgs.jdk11 config.programs.ssh.package pkgs.nix ]"; + type = types.listOf types.package; + description = '' + Packages to add to PATH for the jenkins process. + ''; + }; + + environment = mkOption { + default = { }; + type = with types; attrsOf str; + description = '' + Additional environment variables to be passed to the jenkins process. + As a base environment, jenkins receives NIX_PATH from + <option>environment.sessionVariables</option>, NIX_REMOTE is set to + "daemon" and JENKINS_HOME is set to the value of + <option>services.jenkins.home</option>. + This option has precedence and can be used to override those + mentioned variables. + ''; + }; + + plugins = mkOption { + default = null; + type = types.nullOr (types.attrsOf types.package); + description = '' + A set of plugins to activate. Note that this will completely + remove and replace any previously installed plugins. If you + have manually-installed plugins that you want to keep while + using this module, set this option to + <literal>null</literal>. You can generate this set with a + tool such as <literal>jenkinsPlugins2nix</literal>. + ''; + example = literalExample '' + import path/to/jenkinsPlugins2nix-generated-plugins.nix { inherit (pkgs) fetchurl stdenv; } + ''; + }; + + extraOptions = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "--debug=9" ]; + description = '' + Additional command line arguments to pass to Jenkins. + ''; + }; + + extraJavaOptions = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "-Xmx80m" ]; + description = '' + Additional command line arguments to pass to the Java run time (as opposed to Jenkins). + ''; + }; + }; + }; + + config = mkIf cfg.enable { + # server references the dejavu fonts + environment.systemPackages = [ + pkgs.dejavu_fonts + ]; + + users.groups = optionalAttrs (cfg.group == "jenkins") { + jenkins.gid = config.ids.gids.jenkins; + }; + + users.users = optionalAttrs (cfg.user == "jenkins") { + jenkins = { + description = "jenkins user"; + createHome = true; + home = cfg.home; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + uid = config.ids.uids.jenkins; + }; + }; + + systemd.services.jenkins = { + description = "Jenkins Continuous Integration Server"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = + let + selectedSessionVars = + lib.filterAttrs (n: v: builtins.elem n [ "NIX_PATH" ]) + config.environment.sessionVariables; + in + selectedSessionVars // + { JENKINS_HOME = cfg.home; + NIX_REMOTE = "daemon"; + } // + cfg.environment; + + path = cfg.packages; + + # Force .war (re)extraction, or else we might run stale Jenkins. + + preStart = + let replacePlugins = + if cfg.plugins == null + then "" + else + let pluginCmds = lib.attrsets.mapAttrsToList + (n: v: "cp ${v} ${cfg.home}/plugins/${n}.jpi") + cfg.plugins; + in '' + rm -r ${cfg.home}/plugins || true + mkdir -p ${cfg.home}/plugins + ${lib.strings.concatStringsSep "\n" pluginCmds} + ''; + in '' + rm -rf ${cfg.home}/war + ${replacePlugins} + ''; + + # For reference: https://wiki.jenkins.io/display/JENKINS/JenkinsLinuxStartupScript + script = '' + ${pkgs.jdk11}/bin/java ${concatStringsSep " " cfg.extraJavaOptions} -jar ${cfg.package}/webapps/jenkins.war --httpListenAddress=${cfg.listenAddress} \ + --httpPort=${toString cfg.port} \ + --prefix=${cfg.prefix} \ + -Djava.awt.headless=true \ + ${concatStringsSep " " cfg.extraOptions} + ''; + + postStart = '' + until [[ $(${pkgs.curl.bin}/bin/curl -L -s --head -w '\n%{http_code}' http://${cfg.listenAddress}:${toString cfg.port}${cfg.prefix} | tail -n1) =~ ^(200|403)$ ]]; do + sleep 1 + done + ''; + + serviceConfig = { + User = cfg.user; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/jenkins/job-builder.nix b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/job-builder.nix new file mode 100644 index 000000000000..5d1bfe4ec407 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/job-builder.nix @@ -0,0 +1,205 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + jenkinsCfg = config.services.jenkins; + cfg = config.services.jenkins.jobBuilder; + +in { + options = { + services.jenkins.jobBuilder = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether or not to enable the Jenkins Job Builder (JJB) service. It + allows defining jobs for Jenkins in a declarative manner. + + Jobs managed through the Jenkins WebUI (or by other means) are left + unchanged. + + Note that it really is declarative configuration; if you remove a + previously defined job, the corresponding job directory will be + deleted. + + Please see the Jenkins Job Builder documentation for more info: + <link xlink:href="http://docs.openstack.org/infra/jenkins-job-builder/"> + http://docs.openstack.org/infra/jenkins-job-builder/</link> + ''; + }; + + accessUser = mkOption { + default = ""; + type = types.str; + description = '' + User id in Jenkins used to reload config. + ''; + }; + + accessToken = mkOption { + default = ""; + type = types.str; + description = '' + User token in Jenkins used to reload config. + WARNING: This token will be world readable in the Nix store. To keep + it secret, use the <option>accessTokenFile</option> option instead. + ''; + }; + + accessTokenFile = mkOption { + default = ""; + type = types.str; + example = "/run/keys/jenkins-job-builder-access-token"; + description = '' + File containing the API token for the <option>accessUser</option> + user. + ''; + }; + + yamlJobs = mkOption { + default = ""; + type = types.lines; + example = '' + - job: + name: jenkins-job-test-1 + builders: + - shell: echo 'Hello world!' + ''; + description = '' + Job descriptions for Jenkins Job Builder in YAML format. + ''; + }; + + jsonJobs = mkOption { + default = [ ]; + type = types.listOf types.str; + example = literalExample '' + [ + ''' + [ { "job": + { "name": "jenkins-job-test-2", + "builders": [ "shell": "echo 'Hello world!'" ] + } + } + ] + ''' + ] + ''; + description = '' + Job descriptions for Jenkins Job Builder in JSON format. + ''; + }; + + nixJobs = mkOption { + default = [ ]; + type = types.listOf types.attrs; + example = literalExample '' + [ { job = + { name = "jenkins-job-test-3"; + builders = [ + { shell = "echo 'Hello world!'"; } + ]; + }; + } + ] + ''; + description = '' + Job descriptions for Jenkins Job Builder in Nix format. + + This is a trivial wrapper around jsonJobs, using builtins.toJSON + behind the scene. + ''; + }; + }; + }; + + config = mkIf (jenkinsCfg.enable && cfg.enable) { + assertions = [ + { assertion = + if cfg.accessUser != "" + then (cfg.accessToken != "" && cfg.accessTokenFile == "") || + (cfg.accessToken == "" && cfg.accessTokenFile != "") + else true; + message = '' + One of accessToken and accessTokenFile options must be non-empty + strings, but not both. Current values: + services.jenkins.jobBuilder.accessToken = "${cfg.accessToken}" + services.jenkins.jobBuilder.accessTokenFile = "${cfg.accessTokenFile}" + ''; + } + ]; + + systemd.services.jenkins-job-builder = { + description = "Jenkins Job Builder Service"; + # JJB can run either before or after jenkins. We chose after, so we can + # always use curl to notify (running) jenkins to reload its config. + after = [ "jenkins.service" ]; + wantedBy = [ "multi-user.target" ]; + + path = with pkgs; [ jenkins-job-builder curl ]; + + # Q: Why manipulate files directly instead of using "jenkins-jobs upload [...]"? + # A: Because this module is for administering a local jenkins install, + # and using local file copy allows us to not worry about + # authentication. + script = + let + yamlJobsFile = builtins.toFile "jobs.yaml" cfg.yamlJobs; + jsonJobsFiles = + map (x: (builtins.toFile "jobs.json" x)) + (cfg.jsonJobs ++ [(builtins.toJSON cfg.nixJobs)]); + jobBuilderOutputDir = "/run/jenkins-job-builder/output"; + # Stamp file is placed in $JENKINS_HOME/jobs/$JOB_NAME/ to indicate + # ownership. Enables tracking and removal of stale jobs. + ownerStamp = ".config-xml-managed-by-nixos-jenkins-job-builder"; + reloadScript = '' + echo "Asking Jenkins to reload config" + curl_opts="--silent --fail --show-error" + access_token=${if cfg.accessTokenFile != "" + then "$(cat '${cfg.accessTokenFile}')" + else cfg.accessToken} + jenkins_url="http://${cfg.accessUser}:$access_token@${jenkinsCfg.listenAddress}:${toString jenkinsCfg.port}${jenkinsCfg.prefix}" + crumb=$(curl $curl_opts "$jenkins_url"'/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)') + curl $curl_opts -X POST -H "$crumb" "$jenkins_url"/reload + ''; + in + '' + rm -rf ${jobBuilderOutputDir} + cur_decl_jobs=/run/jenkins-job-builder/declarative-jobs + rm -f "$cur_decl_jobs" + + # Create / update jobs + mkdir -p ${jobBuilderOutputDir} + for inputFile in ${yamlJobsFile} ${concatStringsSep " " jsonJobsFiles}; do + HOME="${jenkinsCfg.home}" "${pkgs.jenkins-job-builder}/bin/jenkins-jobs" --ignore-cache test -o "${jobBuilderOutputDir}" "$inputFile" + done + + for file in "${jobBuilderOutputDir}/"*; do + test -f "$file" || continue + jobname="$(basename $file)" + jobdir="${jenkinsCfg.home}/jobs/$jobname" + echo "Creating / updating job \"$jobname\"" + mkdir -p "$jobdir" + touch "$jobdir/${ownerStamp}" + cp "$file" "$jobdir/config.xml" + echo "$jobname" >> "$cur_decl_jobs" + done + + # Remove stale jobs + for file in "${jenkinsCfg.home}"/jobs/*/${ownerStamp}; do + test -f "$file" || continue + jobdir="$(dirname $file)" + jobname="$(basename "$jobdir")" + grep --quiet --line-regexp "$jobname" "$cur_decl_jobs" 2>/dev/null && continue + echo "Deleting stale job \"$jobname\"" + rm -rf "$jobdir" + done + '' + (if cfg.accessUser != "" then reloadScript else ""); + serviceConfig = { + User = jenkinsCfg.user; + RuntimeDirectory = "jenkins-job-builder"; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/jenkins/slave.nix b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/slave.nix new file mode 100644 index 000000000000..3c0e6f78e74c --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/slave.nix @@ -0,0 +1,68 @@ +{ config, lib, ... }: +with lib; +let + cfg = config.services.jenkinsSlave; + masterCfg = config.services.jenkins; +in { + options = { + services.jenkinsSlave = { + # todo: + # * assure the profile of the jenkins user has a JRE and any specified packages. This would + # enable ssh slaves. + # * Optionally configure the node as a jenkins ad-hoc slave. This would imply configuration + # properties for the master node. + enable = mkOption { + type = types.bool; + default = false; + description = '' + If true the system will be configured to work as a jenkins slave. + If the system is also configured to work as a jenkins master then this has no effect. + In progress: Currently only assures the jenkins user is configured. + ''; + }; + + user = mkOption { + default = "jenkins"; + type = types.str; + description = '' + User the jenkins slave agent should execute under. + ''; + }; + + group = mkOption { + default = "jenkins"; + type = types.str; + description = '' + If the default slave agent user "jenkins" is configured then this is + the primary group of that user. + ''; + }; + + home = mkOption { + default = "/var/lib/jenkins"; + type = types.path; + description = '' + The path to use as JENKINS_HOME. If the default user "jenkins" is configured then + this is the home of the "jenkins" user. + ''; + }; + }; + }; + + config = mkIf (cfg.enable && !masterCfg.enable) { + users.groups = optionalAttrs (cfg.group == "jenkins") { + jenkins.gid = config.ids.gids.jenkins; + }; + + users.users = optionalAttrs (cfg.user == "jenkins") { + jenkins = { + description = "jenkins user"; + createHome = true; + home = cfg.home; + group = cfg.group; + useDefaultShell = true; + uid = config.ids.uids.jenkins; + }; + }; + }; +} |