diff options
Diffstat (limited to 'nixpkgs/nixos/modules/services/continuous-integration')
11 files changed, 2360 insertions, 0 deletions
diff --git a/nixpkgs/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixpkgs/nixos/modules/services/continuous-integration/buildbot/master.nix new file mode 100644 index 000000000000..e3da3092d459 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -0,0 +1,268 @@ +# NixOS module for Buildbot continous integration server. + +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.buildbot-master; + + python = cfg.package.pythonModule; + + escapeStr = s: escape ["'"] s; + + defaultMasterCfg = pkgs.writeText "master.cfg" '' + from buildbot.plugins import * + factory = util.BuildFactory() + c = BuildmasterConfig = dict( + workers = [${concatStringsSep "," cfg.workers}], + protocols = { 'pb': {'port': ${toString cfg.bpPort} } }, + title = '${escapeStr cfg.title}', + titleURL = '${escapeStr cfg.titleUrl}', + buildbotURL = '${escapeStr cfg.buildbotUrl}', + db = dict(db_url='${escapeStr cfg.dbUrl}'), + www = dict(port=${toString cfg.port}), + change_source = [ ${concatStringsSep "," cfg.changeSource} ], + schedulers = [ ${concatStringsSep "," cfg.schedulers} ], + builders = [ ${concatStringsSep "," cfg.builders} ], + status = [ ${concatStringsSep "," cfg.status} ], + ) + for step in [ ${concatStringsSep "," cfg.factorySteps} ]: + factory.addStep(step) + + ${cfg.extraConfig} + ''; + + tacFile = pkgs.writeText "buildbot-master.tac" '' + import os + + from twisted.application import service + from buildbot.master import BuildMaster + + basedir = '${cfg.buildbotDir}' + + configfile = '${cfg.masterCfg}' + + # Default umask for server + umask = None + + # note: this line is matched against to check that this is a buildmaster + # directory; do not edit it. + application = service.Application('buildmaster') + + m = BuildMaster(basedir, configfile, umask) + m.setServiceParent(application) + ''; + +in { + options = { + services.buildbot-master = { + + factorySteps = mkOption { + type = types.listOf types.str; + description = "Factory Steps"; + default = []; + example = [ + "steps.Git(repourl='git://github.com/buildbot/pyflakes.git', mode='incremental')" + "steps.ShellCommand(command=['trial', 'pyflakes'])" + ]; + }; + + changeSource = mkOption { + type = types.listOf types.str; + description = "List of Change Sources."; + default = []; + example = [ + "changes.GitPoller('git://github.com/buildbot/pyflakes.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" + ]; + }; + + enable = mkOption { + type = types.bool; + default = false; + description = "Whether to enable the Buildbot continuous integration server."; + }; + + extraConfig = mkOption { + type = types.str; + description = "Extra configuration to append to master.cfg"; + default = "c['buildbotNetUsageData'] = None"; + }; + + masterCfg = mkOption { + type = types.path; + description = "Optionally pass master.cfg path. Other options in this configuration will be ignored."; + default = defaultMasterCfg; + example = "/etc/nixos/buildbot/master.cfg"; + }; + + schedulers = mkOption { + type = types.listOf types.str; + description = "List of Schedulers."; + default = [ + "schedulers.SingleBranchScheduler(name='all', change_filter=util.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=['runtests'])" + "schedulers.ForceScheduler(name='force',builderNames=['runtests'])" + ]; + }; + + builders = mkOption { + type = types.listOf types.str; + description = "List of Builders."; + default = [ + "util.BuilderConfig(name='runtests',workernames=['example-worker'],factory=factory)" + ]; + }; + + workers = mkOption { + type = types.listOf types.str; + description = "List of Workers."; + default = [ "worker.Worker('example-worker', 'pass')" ]; + }; + + status = mkOption { + default = []; + type = types.listOf types.str; + description = "List of status notification endpoints."; + }; + + user = mkOption { + default = "buildbot"; + type = types.str; + description = "User the buildbot server should execute under."; + }; + + group = mkOption { + default = "buildbot"; + type = types.str; + description = "Primary group of buildbot user."; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = []; + description = "List of extra groups that the buildbot user should be a part of."; + }; + + home = mkOption { + default = "/home/buildbot"; + type = types.path; + description = "Buildbot home directory."; + }; + + buildbotDir = mkOption { + default = "${cfg.home}/master"; + type = types.path; + description = "Specifies the Buildbot directory."; + }; + + bpPort = mkOption { + default = 9989; + type = types.int; + description = "Port where the master will listen to Buildbot Worker."; + }; + + listenAddress = mkOption { + default = "0.0.0.0"; + type = types.str; + description = "Specifies the bind address on which the buildbot HTTP interface listens."; + }; + + buildbotUrl = mkOption { + default = "http://localhost:8010/"; + type = types.str; + description = "Specifies the Buildbot URL."; + }; + + title = mkOption { + default = "Buildbot"; + type = types.str; + description = "Specifies the Buildbot Title."; + }; + + titleUrl = mkOption { + default = "Buildbot"; + type = types.str; + description = "Specifies the Buildbot TitleURL."; + }; + + dbUrl = mkOption { + default = "sqlite:///state.sqlite"; + type = types.str; + description = "Specifies the database connection string."; + }; + + port = mkOption { + default = 8010; + type = types.int; + description = "Specifies port number on which the buildbot HTTP interface listens."; + }; + + package = mkOption { + type = types.package; + default = pkgs.python3Packages.buildbot-full; + defaultText = "pkgs.python3Packages.buildbot-full"; + description = "Package to use for buildbot."; + example = literalExample "pkgs.python3Packages.buildbot"; + }; + + packages = mkOption { + default = [ pkgs.git ]; + example = literalExample "[ pkgs.git ]"; + type = types.listOf types.package; + description = "Packages to add to PATH for the buildbot process."; + }; + + pythonPackages = mkOption { + default = pythonPackages: with pythonPackages; [ ]; + defaultText = "pythonPackages: with pythonPackages; [ ]"; + description = "Packages to add the to the PYTHONPATH of the buildbot process."; + example = literalExample "pythonPackages: with pythonPackages; [ requests ]"; + }; + }; + }; + + config = mkIf cfg.enable { + users.groups = optionalAttrs (cfg.group == "buildbot") { + buildbot = { }; + }; + + users.users = optionalAttrs (cfg.user == "buildbot") { + buildbot = { + description = "Buildbot User."; + isNormalUser = true; + createHome = true; + home = cfg.home; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + }; + }; + + systemd.services.buildbot-master = { + description = "Buildbot Continuous Integration Server."; + after = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + path = cfg.packages ++ cfg.pythonPackages python.pkgs; + environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ cfg.package ])}/${python.sitePackages}"; + + preStart = '' + mkdir -vp "${cfg.buildbotDir}" + # Link the tac file so buildbot command line tools recognize the directory + ln -sf "${tacFile}" "${cfg.buildbotDir}/buildbot.tac" + ${cfg.package}/bin/buildbot create-master --db "${cfg.dbUrl}" "${cfg.buildbotDir}" + rm -f buildbot.tac.new master.cfg.sample + ''; + + serviceConfig = { + Type = "simple"; + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.home; + # NOTE: call twistd directly with stdout logging for systemd + ExecStart = "${python.pkgs.twisted}/bin/twistd -o --nodaemon --pidfile= --logfile - --python ${tacFile}"; + }; + }; + }; + + meta.maintainers = with lib.maintainers; [ nand0p mic92 ]; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixpkgs/nixos/modules/services/continuous-integration/buildbot/worker.nix new file mode 100644 index 000000000000..52f24b8cee3c --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/buildbot/worker.nix @@ -0,0 +1,187 @@ +# NixOS module for Buildbot Worker. + +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.buildbot-worker; + + python = cfg.package.pythonModule; + + tacFile = pkgs.writeText "aur-buildbot-worker.tac" '' + import os + from io import open + + from buildbot_worker.bot import Worker + from twisted.application import service + + basedir = '${cfg.buildbotDir}' + + # note: this line is matched against to check that this is a worker + # directory; do not edit it. + application = service.Application('buildbot-worker') + + master_url_split = '${cfg.masterUrl}'.split(':') + buildmaster_host = master_url_split[0] + port = int(master_url_split[1]) + workername = '${cfg.workerUser}' + + with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file: + passwd = passwd_file.read().strip('\r\n') + keepalive = 600 + umask = None + maxdelay = 300 + numcpus = None + allow_shutdown = None + + s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + numcpus=numcpus, allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; + +in { + options = { + services.buildbot-worker = { + + enable = mkOption { + type = types.bool; + default = false; + description = "Whether to enable the Buildbot Worker."; + }; + + user = mkOption { + default = "bbworker"; + type = types.str; + description = "User the buildbot Worker should execute under."; + }; + + group = mkOption { + default = "bbworker"; + type = types.str; + description = "Primary group of buildbot Worker user."; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = []; + description = "List of extra groups that the Buildbot Worker user should be a part of."; + }; + + home = mkOption { + default = "/home/bbworker"; + type = types.path; + description = "Buildbot home directory."; + }; + + buildbotDir = mkOption { + default = "${cfg.home}/worker"; + type = types.path; + description = "Specifies the Buildbot directory."; + }; + + workerUser = mkOption { + default = "example-worker"; + type = types.str; + description = "Specifies the Buildbot Worker user."; + }; + + workerPass = mkOption { + default = "pass"; + type = types.str; + description = "Specifies the Buildbot Worker password."; + }; + + workerPassFile = mkOption { + type = types.path; + description = "File used to store the Buildbot Worker password"; + }; + + hostMessage = mkOption { + default = null; + type = types.nullOr types.str; + description = "Description of this worker"; + }; + + adminMessage = mkOption { + default = null; + type = types.nullOr types.str; + description = "Name of the administrator of this worker"; + }; + + masterUrl = mkOption { + default = "localhost:9989"; + type = types.str; + description = "Specifies the Buildbot Worker connection string."; + }; + + package = mkOption { + type = types.package; + default = pkgs.python3Packages.buildbot-worker; + defaultText = "pkgs.python3Packages.buildbot-worker"; + description = "Package to use for buildbot worker."; + example = literalExample "pkgs.python2Packages.buildbot-worker"; + }; + + packages = mkOption { + default = with pkgs; [ git ]; + example = literalExample "[ pkgs.git ]"; + type = types.listOf types.package; + description = "Packages to add to PATH for the buildbot process."; + }; + }; + }; + + config = mkIf cfg.enable { + services.buildbot-worker.workerPassFile = mkDefault (pkgs.writeText "buildbot-worker-password" cfg.workerPass); + + users.groups = optionalAttrs (cfg.group == "bbworker") { + bbworker = { }; + }; + + users.users = optionalAttrs (cfg.user == "bbworker") { + bbworker = { + description = "Buildbot Worker User."; + isNormalUser = true; + createHome = true; + home = cfg.home; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + }; + }; + + systemd.services.buildbot-worker = { + description = "Buildbot Worker."; + after = [ "network.target" "buildbot-master.service" ]; + wantedBy = [ "multi-user.target" ]; + path = cfg.packages; + environment.PYTHONPATH = "${python.withPackages (p: [ cfg.package ])}/${python.sitePackages}"; + + preStart = '' + mkdir -vp "${cfg.buildbotDir}/info" + ${optionalString (cfg.hostMessage != null) '' + ln -sf "${pkgs.writeText "buildbot-worker-host" cfg.hostMessage}" "${cfg.buildbotDir}/info/host" + ''} + ${optionalString (cfg.adminMessage != null) '' + ln -sf "${pkgs.writeText "buildbot-worker-admin" cfg.adminMessage}" "${cfg.buildbotDir}/info/admin" + ''} + ''; + + serviceConfig = { + Type = "simple"; + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.home; + + # NOTE: call twistd directly with stdout logging for systemd + ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}"; + }; + + }; + }; + + meta.maintainers = with lib.maintainers; [ nand0p ]; + +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/buildkite-agents.nix b/nixpkgs/nixos/modules/services/continuous-integration/buildkite-agents.nix new file mode 100644 index 000000000000..b0045409ae60 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/buildkite-agents.nix @@ -0,0 +1,276 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.buildkite-agents; + + mkHookOption = { name, description, example ? null }: { + inherit name; + value = mkOption { + default = null; + inherit description; + type = types.nullOr types.lines; + } // (if example == null then {} else { inherit example; }); + }; + mkHookOptions = hooks: listToAttrs (map mkHookOption hooks); + + hooksDir = cfg: let + mkHookEntry = name: value: '' + cat > $out/${name} <<'EOF' + #! ${pkgs.runtimeShell} + set -e + ${value} + EOF + chmod 755 $out/${name} + ''; + in pkgs.runCommand "buildkite-agent-hooks" { preferLocalBuild = true; } '' + mkdir $out + ${concatStringsSep "\n" (mapAttrsToList mkHookEntry (filterAttrs (n: v: v != null) cfg.hooks))} + ''; + + buildkiteOptions = { name ? "", config, ... }: { + options = { + enable = mkOption { + default = true; + type = types.bool; + description = "Whether to enable this buildkite agent"; + }; + + package = mkOption { + default = pkgs.buildkite-agent; + defaultText = "pkgs.buildkite-agent"; + description = "Which buildkite-agent derivation to use"; + type = types.package; + }; + + dataDir = mkOption { + default = "/var/lib/buildkite-agent-${name}"; + description = "The workdir for the agent"; + type = types.str; + }; + + runtimePackages = mkOption { + default = [ pkgs.bash pkgs.gnutar pkgs.gzip pkgs.git pkgs.nix ]; + defaultText = "[ pkgs.bash pkgs.gnutar pkgs.gzip pkgs.git pkgs.nix ]"; + description = "Add programs to the buildkite-agent environment"; + type = types.listOf types.package; + }; + + tokenPath = mkOption { + type = types.path; + description = '' + The token from your Buildkite "Agents" page. + + A run-time path to the token file, which is supposed to be provisioned + outside of Nix store. + ''; + }; + + name = mkOption { + type = types.str; + default = "%hostname-${name}-%n"; + description = '' + The name of the agent as seen in the buildkite dashboard. + ''; + }; + + tags = mkOption { + type = types.attrsOf types.str; + default = {}; + example = { queue = "default"; docker = "true"; ruby2 ="true"; }; + description = '' + Tags for the agent. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + example = "debug=true"; + description = '' + Extra lines to be added verbatim to the configuration file. + ''; + }; + + privateSshKeyPath = mkOption { + type = types.nullOr types.path; + default = null; + ## maximum care is taken so that secrets (ssh keys and the CI token) + ## don't end up in the Nix store. + apply = final: if final == null then null else toString final; + + description = '' + OpenSSH private key + + A run-time path to the key file, which is supposed to be provisioned + outside of Nix store. + ''; + }; + + hooks = mkHookOptions [ + { name = "checkout"; + description = '' + The `checkout` hook script will replace the default checkout routine of the + bootstrap.sh script. You can use this hook to do your own SCM checkout + behaviour + ''; } + { name = "command"; + description = '' + The `command` hook script will replace the default implementation of running + the build command. + ''; } + { name = "environment"; + description = '' + The `environment` hook will run before all other commands, and can be used + to set up secrets, data, etc. Anything exported in hooks will be available + to the build script. + + Note: the contents of this file will be copied to the world-readable + Nix store. + ''; + example = '' + export SECRET_VAR=`head -1 /run/keys/secret` + ''; } + { name = "post-artifact"; + description = '' + The `post-artifact` hook will run just after artifacts are uploaded + ''; } + { name = "post-checkout"; + description = '' + The `post-checkout` hook will run after the bootstrap script has checked out + your projects source code. + ''; } + { name = "post-command"; + description = '' + The `post-command` hook will run after the bootstrap script has run your + build commands + ''; } + { name = "pre-artifact"; + description = '' + The `pre-artifact` hook will run just before artifacts are uploaded + ''; } + { name = "pre-checkout"; + description = '' + The `pre-checkout` hook will run just before your projects source code is + checked out from your SCM provider + ''; } + { name = "pre-command"; + description = '' + The `pre-command` hook will run just before your build command runs + ''; } + { name = "pre-exit"; + description = '' + The `pre-exit` hook will run just before your build job finishes + ''; } + ]; + + hooksPath = mkOption { + type = types.path; + default = hooksDir config; + defaultText = "generated from services.buildkite-agents.<name>.hooks"; + description = '' + Path to the directory storing the hooks. + Consider using <option>services.buildkite-agents.<name>.hooks.<name></option> + instead. + ''; + }; + + shell = mkOption { + type = types.str; + default = "${pkgs.bash}/bin/bash -e -c"; + description = '' + Command that buildkite-agent 3 will execute when it spawns a shell. + ''; + }; + }; + }; + enabledAgents = lib.filterAttrs (n: v: v.enable) cfg; + mapAgents = function: lib.mkMerge (lib.mapAttrsToList function enabledAgents); +in +{ + options.services.buildkite-agents = mkOption { + type = types.attrsOf (types.submodule buildkiteOptions); + default = {}; + description = '' + Attribute set of buildkite agents. + The attribute key is combined with the hostname and a unique integer to + create the final agent name. This can be overridden by setting the `name` + attribute. + ''; + }; + + config.users.users = mapAgents (name: cfg: { + "buildkite-agent-${name}" = { + name = "buildkite-agent-${name}"; + home = cfg.dataDir; + createHome = true; + description = "Buildkite agent user"; + extraGroups = [ "keys" ]; + isSystemUser = true; + group = "buildkite-agent-${name}"; + }; + }); + config.users.groups = mapAgents (name: cfg: { + "buildkite-agent-${name}" = {}; + }); + + config.systemd.services = mapAgents (name: cfg: { + "buildkite-agent-${name}" = + { description = "Buildkite Agent"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = cfg.runtimePackages ++ [ cfg.package pkgs.coreutils ]; + environment = config.networking.proxy.envVars // { + HOME = cfg.dataDir; + NIX_REMOTE = "daemon"; + }; + + ## NB: maximum care is taken so that secrets (ssh keys and the CI token) + ## don't end up in the Nix store. + preStart = let + sshDir = "${cfg.dataDir}/.ssh"; + tagStr = lib.concatStringsSep "," (lib.mapAttrsToList (name: value: "${name}=${value}") cfg.tags); + in + optionalString (cfg.privateSshKeyPath != null) '' + mkdir -m 0700 -p "${sshDir}" + cp -f "${toString cfg.privateSshKeyPath}" "${sshDir}/id_rsa" + chmod 600 "${sshDir}"/id_rsa + '' + '' + cat > "${cfg.dataDir}/buildkite-agent.cfg" <<EOF + token="$(cat ${toString cfg.tokenPath})" + name="${cfg.name}" + shell="${cfg.shell}" + tags="${tagStr}" + build-path="${cfg.dataDir}/builds" + hooks-path="${cfg.hooksPath}" + ${cfg.extraConfig} + EOF + ''; + + serviceConfig = + { ExecStart = "${cfg.package}/bin/buildkite-agent start --config ${cfg.dataDir}/buildkite-agent.cfg"; + User = "buildkite-agent-${name}"; + RestartSec = 5; + Restart = "on-failure"; + TimeoutSec = 10; + # set a long timeout to give buildkite-agent a chance to finish current builds + TimeoutStopSec = "2 min"; + KillMode = "mixed"; + }; + }; + }); + + config.assertions = mapAgents (name: cfg: [ + { assertion = cfg.hooksPath == (hooksDir cfg) || all (v: v == null) (attrValues cfg.hooks); + message = '' + Options `services.buildkite-agents.${name}.hooksPath' and + `services.buildkite-agents.${name}.hooks.<name>' are mutually exclusive. + ''; + } + ]); + + imports = [ + (mkRemovedOptionModule [ "services" "buildkite-agent"] "services.buildkite-agent has been upgraded from version 2 to version 3 and moved to an attribute set at services.buildkite-agents. Please consult the 20.03 release notes for more information.") + ]; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix b/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix new file mode 100644 index 000000000000..bd4cf6a37bad --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/gitlab-runner.nix @@ -0,0 +1,160 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.gitlab-runner; + configFile = + if (cfg.configFile == null) then + (pkgs.runCommand "config.toml" { + buildInputs = [ pkgs.remarshal ]; + preferLocalBuild = true; + } '' + remarshal -if json -of toml \ + < ${pkgs.writeText "config.json" (builtins.toJSON cfg.configOptions)} \ + > $out + '') + else + cfg.configFile; + hasDocker = config.virtualisation.docker.enable; +in +{ + options.services.gitlab-runner = { + enable = mkEnableOption "Gitlab Runner"; + + configFile = mkOption { + default = null; + description = '' + Configuration file for gitlab-runner. + Use this option in favor of configOptions to avoid placing CI tokens in the nix store. + + <option>configFile</option> takes precedence over <option>configOptions</option>. + + Warning: Not using <option>configFile</option> will potentially result in secrets + leaking into the WORLD-READABLE nix store. + ''; + type = types.nullOr types.path; + }; + + configOptions = mkOption { + description = '' + Configuration for gitlab-runner + <option>configFile</option> will take precedence over this option. + + Warning: all Configuration, especially CI token, will be stored in a + WORLD-READABLE file in the Nix Store. + + If you want to protect your CI token use <option>configFile</option> instead. + ''; + type = types.attrs; + example = { + concurrent = 2; + runners = [{ + name = "docker-nix-1.11"; + url = "https://CI/"; + token = "TOKEN"; + executor = "docker"; + builds_dir = ""; + docker = { + host = ""; + image = "nixos/nix:1.11"; + privileged = true; + disable_cache = true; + cache_dir = ""; + }; + }]; + }; + }; + + gracefulTermination = mkOption { + default = false; + type = types.bool; + description = '' + Finish all remaining jobs before stopping, restarting or reconfiguring. + If not set gitlab-runner will stop immediatly without waiting for jobs to finish, + which will lead to failed builds. + ''; + }; + + gracefulTimeout = mkOption { + default = "infinity"; + type = types.str; + example = "5min 20s"; + description = ''Time to wait until a graceful shutdown is turned into a forceful one.''; + }; + + workDir = mkOption { + default = "/var/lib/gitlab-runner"; + type = types.path; + description = "The working directory used"; + }; + + package = mkOption { + description = "Gitlab Runner package to use"; + default = pkgs.gitlab-runner; + defaultText = "pkgs.gitlab-runner"; + type = types.package; + example = literalExample "pkgs.gitlab-runner_1_11"; + }; + + packages = mkOption { + default = [ pkgs.bash pkgs.docker-machine ]; + defaultText = "[ pkgs.bash pkgs.docker-machine ]"; + type = types.listOf types.package; + description = '' + Packages to add to PATH for the gitlab-runner process. + ''; + }; + + }; + + config = mkIf cfg.enable { + systemd.services.gitlab-runner = { + path = cfg.packages; + environment = config.networking.proxy.envVars // { + # Gitlab runner will not start if the HOME variable is not set + HOME = cfg.workDir; + }; + description = "Gitlab Runner"; + after = [ "network.target" ] + ++ optional hasDocker "docker.service"; + requires = optional hasDocker "docker.service"; + wantedBy = [ "multi-user.target" ]; + reloadIfChanged = true; + restartTriggers = [ + config.environment.etc."gitlab-runner/config.toml".source + ]; + serviceConfig = { + StateDirectory = "gitlab-runner"; + ExecReload= "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + ExecStart = ''${cfg.package.bin}/bin/gitlab-runner run \ + --working-directory ${cfg.workDir} \ + --config /etc/gitlab-runner/config.toml \ + --service gitlab-runner \ + --user gitlab-runner \ + ''; + + } // optionalAttrs (cfg.gracefulTermination) { + TimeoutStopSec = "${cfg.gracefulTimeout}"; + KillSignal = "SIGQUIT"; + KillMode = "process"; + }; + }; + + # Make the gitlab-runner command availabe so users can query the runner + environment.systemPackages = [ cfg.package ]; + + # Make sure the config can be reloaded on change + environment.etc."gitlab-runner/config.toml".source = configFile; + + users.users.gitlab-runner = { + group = "gitlab-runner"; + extraGroups = optional hasDocker "docker"; + uid = config.ids.uids.gitlab-runner; + home = cfg.workDir; + createHome = true; + }; + + users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/gocd-agent/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/gocd-agent/default.nix new file mode 100644 index 000000000000..2e9e1c94857a --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/gocd-agent/default.nix @@ -0,0 +1,206 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.gocd-agent; +in { + options = { + services.gocd-agent = { + enable = mkEnableOption "gocd-agent"; + + user = mkOption { + default = "gocd-agent"; + type = types.str; + description = '' + User the Go.CD agent should execute under. + ''; + }; + + group = mkOption { + default = "gocd-agent"; + type = types.str; + description = '' + If the default user "gocd-agent" is configured then this is the primary + group of that user. + ''; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "wheel" "docker" ]; + description = '' + List of extra groups that the "gocd-agent" user should be a part of. + ''; + }; + + packages = mkOption { + default = [ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]; + defaultText = "[ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]"; + type = types.listOf types.package; + description = '' + Packages to add to PATH for the Go.CD agent process. + ''; + }; + + agentConfig = mkOption { + default = ""; + type = types.str; + example = '' + agent.auto.register.resources=ant,java + agent.auto.register.environments=QA,Performance + agent.auto.register.hostname=Agent01 + ''; + description = '' + Agent registration configuration. + ''; + }; + + goServer = mkOption { + default = "https://127.0.0.1:8154/go"; + type = types.str; + description = '' + URL of the GoCD Server to attach the Go.CD Agent to. + ''; + }; + + workDir = mkOption { + default = "/var/lib/go-agent"; + type = types.str; + description = '' + Specifies the working directory in which the Go.CD agent java archive resides. + ''; + }; + + initialJavaHeapSize = mkOption { + default = "128m"; + type = types.str; + description = '' + Specifies the initial java heap memory size for the Go.CD agent java process. + ''; + }; + + maxJavaHeapMemory = mkOption { + default = "256m"; + type = types.str; + description = '' + Specifies the java maximum heap memory size for the Go.CD agent java process. + ''; + }; + + startupOptions = mkOption { + default = [ + "-Xms${cfg.initialJavaHeapSize}" + "-Xmx${cfg.maxJavaHeapMemory}" + "-Djava.io.tmpdir=/tmp" + "-Dcruise.console.publish.interval=10" + "-Djava.security.egd=file:/dev/./urandom" + ]; + description = '' + Specifies startup command line arguments to pass to Go.CD agent + java process. + ''; + }; + + extraOptions = mkOption { + default = [ ]; + example = [ + "-X debug" + "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5006" + "-verbose:gc" + "-Xloggc:go-agent-gc.log" + "-XX:+PrintGCTimeStamps" + "-XX:+PrintTenuringDistribution" + "-XX:+PrintGCDetails" + "-XX:+PrintGC" + ]; + description = '' + Specifies additional command line arguments to pass to Go.CD agent + java process. Example contains debug and gcLog arguments. + ''; + }; + + environment = mkOption { + default = { }; + type = with types; attrsOf str; + description = '' + Additional environment variables to be passed to the Go.CD agent process. + As a base environment, Go.CD agent receives NIX_PATH from + <option>environment.sessionVariables</option>, NIX_REMOTE is set to + "daemon". + ''; + }; + }; + }; + + config = mkIf cfg.enable { + users.groups = optionalAttrs (cfg.group == "gocd-agent") { + gocd-agent.gid = config.ids.gids.gocd-agent; + }; + + users.users = optionalAttrs (cfg.user == "gocd-agent") { + gocd-agent = { + description = "gocd-agent user"; + createHome = true; + home = cfg.workDir; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + uid = config.ids.uids.gocd-agent; + }; + }; + + systemd.services.gocd-agent = { + description = "GoCD Agent"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = + let + selectedSessionVars = + lib.filterAttrs (n: v: builtins.elem n [ "NIX_PATH" ]) + config.environment.sessionVariables; + in + selectedSessionVars // + { + NIX_REMOTE = "daemon"; + AGENT_WORK_DIR = cfg.workDir; + AGENT_STARTUP_ARGS = ''${concatStringsSep " " cfg.startupOptions}''; + LOG_DIR = cfg.workDir; + LOG_FILE = "${cfg.workDir}/go-agent-start.log"; + } // + cfg.environment; + + path = cfg.packages; + + script = '' + MPATH="''${PATH}"; + source /etc/profile + export PATH="''${MPATH}:''${PATH}"; + + if ! test -f ~/.nixpkgs/config.nix; then + mkdir -p ~/.nixpkgs/ + echo "{ allowUnfree = true; }" > ~/.nixpkgs/config.nix + fi + + mkdir -p config + rm -f config/autoregister.properties + ln -s "${pkgs.writeText "autoregister.properties" cfg.agentConfig}" config/autoregister.properties + + ${pkgs.git}/bin/git config --global --add http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + ${pkgs.jre}/bin/java ${concatStringsSep " " cfg.startupOptions} \ + ${concatStringsSep " " cfg.extraOptions} \ + -jar ${pkgs.gocd-agent}/go-agent/agent-bootstrapper.jar \ + -serverUrl ${cfg.goServer} + ''; + + serviceConfig = { + User = cfg.user; + WorkingDirectory = cfg.workDir; + RestartSec = 30; + Restart = "on-failure"; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/gocd-server/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/gocd-server/default.nix new file mode 100644 index 000000000000..4fa41ac49edf --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/gocd-server/default.nix @@ -0,0 +1,194 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.gocd-server; +in { + options = { + services.gocd-server = { + enable = mkEnableOption "gocd-server"; + + user = mkOption { + default = "gocd-server"; + type = types.str; + description = '' + User the Go.CD server should execute under. + ''; + }; + + group = mkOption { + default = "gocd-server"; + type = types.str; + description = '' + If the default user "gocd-server" is configured then this is the primary group of that user. + ''; + }; + + extraGroups = mkOption { + default = [ ]; + example = [ "wheel" "docker" ]; + description = '' + List of extra groups that the "gocd-server" user should be a part of. + ''; + }; + + listenAddress = mkOption { + default = "0.0.0.0"; + example = "localhost"; + type = types.str; + description = '' + Specifies the bind address on which the Go.CD server HTTP interface listens. + ''; + }; + + port = mkOption { + default = 8153; + type = types.int; + description = '' + Specifies port number on which the Go.CD server HTTP interface listens. + ''; + }; + + sslPort = mkOption { + default = 8154; + type = types.int; + description = '' + Specifies port number on which the Go.CD server HTTPS interface listens. + ''; + }; + + workDir = mkOption { + default = "/var/lib/go-server"; + type = types.str; + description = '' + Specifies the working directory in which the Go.CD server java archive resides. + ''; + }; + + packages = mkOption { + default = [ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]; + defaultText = "[ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]"; + type = types.listOf types.package; + description = '' + Packages to add to PATH for the Go.CD server's process. + ''; + }; + + initialJavaHeapSize = mkOption { + default = "512m"; + type = types.str; + description = '' + Specifies the initial java heap memory size for the Go.CD server's java process. + ''; + }; + + maxJavaHeapMemory = mkOption { + default = "1024m"; + type = types.str; + description = '' + Specifies the java maximum heap memory size for the Go.CD server's java process. + ''; + }; + + startupOptions = mkOption { + default = [ + "-Xms${cfg.initialJavaHeapSize}" + "-Xmx${cfg.maxJavaHeapMemory}" + "-Dcruise.listen.host=${cfg.listenAddress}" + "-Duser.language=en" + "-Djruby.rack.request.size.threshold.bytes=30000000" + "-Duser.country=US" + "-Dcruise.config.dir=${cfg.workDir}/conf" + "-Dcruise.config.file=${cfg.workDir}/conf/cruise-config.xml" + "-Dcruise.server.port=${toString cfg.port}" + "-Dcruise.server.ssl.port=${toString cfg.sslPort}" + ]; + + description = '' + Specifies startup command line arguments to pass to Go.CD server + java process. + ''; + }; + + extraOptions = mkOption { + default = [ ]; + example = [ + "-X debug" + "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005" + "-verbose:gc" + "-Xloggc:go-server-gc.log" + "-XX:+PrintGCTimeStamps" + "-XX:+PrintTenuringDistribution" + "-XX:+PrintGCDetails" + "-XX:+PrintGC" + ]; + description = '' + Specifies additional command line arguments to pass to Go.CD server's + java process. Example contains debug and gcLog arguments. + ''; + }; + + environment = mkOption { + default = { }; + type = with types; attrsOf str; + description = '' + Additional environment variables to be passed to the gocd-server process. + As a base environment, gocd-server receives NIX_PATH from + <option>environment.sessionVariables</option>, NIX_REMOTE is set to + "daemon". + ''; + }; + }; + }; + + config = mkIf cfg.enable { + users.groups = optionalAttrs (cfg.group == "gocd-server") { + gocd-server.gid = config.ids.gids.gocd-server; + }; + + users.users = optionalAttrs (cfg.user == "gocd-server") { + gocd-server = { + description = "gocd-server user"; + createHome = true; + home = cfg.workDir; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + uid = config.ids.uids.gocd-server; + }; + }; + + systemd.services.gocd-server = { + description = "GoCD Server"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = + let + selectedSessionVars = + lib.filterAttrs (n: v: builtins.elem n [ "NIX_PATH" ]) + config.environment.sessionVariables; + in + selectedSessionVars // + { NIX_REMOTE = "daemon"; + } // + cfg.environment; + + path = cfg.packages; + + script = '' + ${pkgs.git}/bin/git config --global --add http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + ${pkgs.jre}/bin/java -server ${concatStringsSep " " cfg.startupOptions} \ + ${concatStringsSep " " cfg.extraOptions} \ + -jar ${pkgs.gocd-server}/go-server/go.jar + ''; + + serviceConfig = { + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.workDir; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/hail.nix b/nixpkgs/nixos/modules/services/continuous-integration/hail.nix new file mode 100644 index 000000000000..5d0c3f7b4ab3 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/hail.nix @@ -0,0 +1,61 @@ +{ config, lib, pkgs, ...}: + +with lib; + +let + cfg = config.services.hail; +in { + + + ###### interface + + options.services.hail = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enables the Hail Auto Update Service. Hail can automatically deploy artifacts + built by a Hydra Continous Integration server. A common use case is to provide + continous deployment for single services or a full NixOS configuration.''; + }; + profile = mkOption { + type = types.str; + default = "hail-profile"; + description = "The name of the Nix profile used by Hail."; + }; + hydraJobUri = mkOption { + type = types.str; + description = "The URI of the Hydra Job."; + }; + netrc = mkOption { + type = types.nullOr types.path; + description = "The netrc file to use when fetching data from Hydra."; + default = null; + }; + package = mkOption { + type = types.package; + default = pkgs.haskellPackages.hail; + defaultText = "pkgs.haskellPackages.hail"; + description = "Hail package to use."; + }; + }; + + + ###### implementation + + config = mkIf cfg.enable { + systemd.services.hail = { + description = "Hail Auto Update Service"; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + path = with pkgs; [ nix ]; + environment = { + HOME = "/var/lib/empty"; + }; + serviceConfig = { + ExecStart = "${cfg.package}/bin/hail --profile ${cfg.profile} --job-uri ${cfg.hydraJobUri}" + + lib.optionalString (cfg.netrc != null) " --netrc-file ${cfg.netrc}"; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/hydra/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/hydra/default.nix new file mode 100644 index 000000000000..502a5898a5de --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/hydra/default.nix @@ -0,0 +1,507 @@ +{ config, pkgs, lib, ... }: + +with lib; + +let + + cfg = config.services.hydra; + + baseDir = "/var/lib/hydra"; + + hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig; + + hydraEnv = + { HYDRA_DBI = cfg.dbi; + HYDRA_CONFIG = "${baseDir}/hydra.conf"; + HYDRA_DATA = "${baseDir}"; + }; + + env = + { NIX_REMOTE = "daemon"; + SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03 + PGPASSFILE = "${baseDir}/pgpass"; + NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles; + } // optionalAttrs (cfg.smtpHost != null) { + EMAIL_SENDER_TRANSPORT = "SMTP"; + EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost; + } // hydraEnv // cfg.extraEnv; + + serverEnv = env // + { HYDRA_TRACKER = cfg.tracker; + XDG_CACHE_HOME = "${baseDir}/www/.cache"; + COLUMNS = "80"; + PGPASSFILE = "${baseDir}/pgpass-www"; # grrr + } // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; }); + + localDB = "dbi:Pg:dbname=hydra;user=hydra;"; + + haveLocalDB = cfg.dbi == localDB; + + inherit (config.system) stateVersion; + + hydra-package = + let + makeWrapperArgs = concatStringsSep " " (mapAttrsToList (key: value: "--set \"${key}\" \"${value}\"") hydraEnv); + in pkgs.buildEnv rec { + name = "hydra-env"; + buildInputs = [ pkgs.makeWrapper ]; + paths = [ cfg.package ]; + + postBuild = '' + if [ -L "$out/bin" ]; then + unlink "$out/bin" + fi + mkdir -p "$out/bin" + + for path in ${concatStringsSep " " paths}; do + if [ -d "$path/bin" ]; then + cd "$path/bin" + for prg in *; do + if [ -f "$prg" ]; then + rm -f "$out/bin/$prg" + if [ -x "$prg" ]; then + makeWrapper "$path/bin/$prg" "$out/bin/$prg" ${makeWrapperArgs} + fi + fi + done + fi + done + ''; + }; + +in + +{ + ###### interface + options = { + + services.hydra = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run Hydra services. + ''; + }; + + dbi = mkOption { + type = types.str; + default = localDB; + example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;"; + description = '' + The DBI string for Hydra database connection. + ''; + }; + + package = mkOption { + type = types.package; + defaultText = "pkgs.hydra"; + description = "The Hydra package."; + }; + + hydraURL = mkOption { + type = types.str; + description = '' + The base URL for the Hydra webserver instance. Used for links in emails. + ''; + }; + + listenHost = mkOption { + type = types.str; + default = "*"; + example = "localhost"; + description = '' + The hostname or address to listen on or <literal>*</literal> to listen + on all interfaces. + ''; + }; + + port = mkOption { + type = types.int; + default = 3000; + description = '' + TCP port the web server should listen to. + ''; + }; + + minimumDiskFree = mkOption { + type = types.int; + default = 0; + description = '' + Threshold of minimum disk space (GiB) to determine if the queue runner should run or not. + ''; + }; + + minimumDiskFreeEvaluator = mkOption { + type = types.int; + default = 0; + description = '' + Threshold of minimum disk space (GiB) to determine if the evaluator should run or not. + ''; + }; + + notificationSender = mkOption { + type = types.str; + description = '' + Sender email address used for email notifications. + ''; + }; + + smtpHost = mkOption { + type = types.nullOr types.str; + default = null; + example = ["localhost"]; + description = '' + Hostname of the SMTP server to use to send email. + ''; + }; + + tracker = mkOption { + type = types.str; + default = ""; + description = '' + Piece of HTML that is included on all pages. + ''; + }; + + logo = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Path to a file containing the logo of your Hydra instance. + ''; + }; + + debugServer = mkOption { + type = types.bool; + default = false; + description = "Whether to run the server in debug mode."; + }; + + extraConfig = mkOption { + type = types.lines; + description = "Extra lines for the Hydra configuration."; + }; + + extraEnv = mkOption { + type = types.attrsOf types.str; + default = {}; + description = "Extra environment variables for Hydra."; + }; + + gcRootsDir = mkOption { + type = types.path; + default = "/nix/var/nix/gcroots/hydra"; + description = "Directory that holds Hydra garbage collector roots."; + }; + + buildMachinesFiles = mkOption { + type = types.listOf types.path; + default = optional (config.nix.buildMachines != []) "/etc/nix/machines"; + example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ]; + description = "List of files containing build machines."; + }; + + useSubstitutes = mkOption { + type = types.bool; + default = false; + description = '' + Whether to use binary caches for downloading store paths. Note that + binary substitutions trigger (a potentially large number of) additional + HTTP requests that slow down the queue monitor thread significantly. + Also, this Hydra instance will serve those downloaded store paths to + its users with its own signature attached as if it had built them + itself, so don't enable this feature unless your active binary caches + are absolute trustworthy. + ''; + }; + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + warnings = optional (cfg.package.migration or false) '' + You're currently deploying an older version of Hydra which is needed to + make some required database changes[1]. As soon as this is done, it's recommended + to run `hydra-backfill-ids` and set `services.hydra.package` to `pkgs.hydra-unstable` + after that. + + [1] https://github.com/NixOS/hydra/pull/711 + ''; + + services.hydra.package = with pkgs; + mkDefault ( + if pkgs ? hydra + then throw '' + The Hydra package doesn't exist anymore in `nixpkgs`! It probably exists + due to an overlay. To upgrade Hydra, you need to take two steps as some + bigger changes in the database schema were implemented recently[1]. You first + need to deploy `pkgs.hydra-migration`, run `hydra-backfill-ids` on the server + and then deploy `pkgs.hydra-unstable`. + + If you want to use `pkgs.hydra` from your overlay, please set `services.hydra.package` + explicitly to `pkgs.hydra` and make sure you know what you're doing. + + [1] https://github.com/NixOS/hydra/pull/711 + '' + else if versionOlder stateVersion "20.03" then hydra-migration + else hydra-unstable + ); + + users.groups.hydra = { + gid = config.ids.gids.hydra; + }; + + users.users.hydra = + { description = "Hydra"; + group = "hydra"; + createHome = true; + home = baseDir; + useDefaultShell = true; + uid = config.ids.uids.hydra; + }; + + users.users.hydra-queue-runner = + { description = "Hydra queue runner"; + group = "hydra"; + useDefaultShell = true; + home = "${baseDir}/queue-runner"; # really only to keep SSH happy + uid = config.ids.uids.hydra-queue-runner; + }; + + users.users.hydra-www = + { description = "Hydra web server"; + group = "hydra"; + useDefaultShell = true; + uid = config.ids.uids.hydra-www; + }; + + nix.trustedUsers = [ "hydra-queue-runner" ]; + + services.hydra.extraConfig = + '' + using_frontend_proxy = 1 + base_uri = ${cfg.hydraURL} + notification_sender = ${cfg.notificationSender} + max_servers = 25 + ${optionalString (cfg.logo != null) '' + hydra_logo = ${cfg.logo} + ''} + gc_roots_dir = ${cfg.gcRootsDir} + use-substitutes = ${if cfg.useSubstitutes then "1" else "0"} + ''; + + environment.systemPackages = [ hydra-package ]; + + environment.variables = hydraEnv; + + nix.extraOptions = '' + keep-outputs = true + keep-derivations = true + + # The default (`true') slows Nix down a lot since the build farm + # has so many GC roots. + gc-check-reachability = false + ''; + + systemd.services.hydra-init = + { wantedBy = [ "multi-user.target" ]; + requires = optional haveLocalDB "postgresql.service"; + after = optional haveLocalDB "postgresql.service"; + environment = env; + preStart = '' + mkdir -p ${baseDir} + chown hydra.hydra ${baseDir} + chmod 0750 ${baseDir} + + ln -sf ${hydraConf} ${baseDir}/hydra.conf + + mkdir -m 0700 -p ${baseDir}/www + chown hydra-www.hydra ${baseDir}/www + + mkdir -m 0700 -p ${baseDir}/queue-runner + mkdir -m 0750 -p ${baseDir}/build-logs + chown hydra-queue-runner.hydra ${baseDir}/queue-runner ${baseDir}/build-logs + + ${optionalString haveLocalDB '' + if ! [ -e ${baseDir}/.db-created ]; then + ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra + ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -O hydra hydra + touch ${baseDir}/.db-created + fi + echo "create extension if not exists pg_trgm" | ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra + ''} + + if [ ! -e ${cfg.gcRootsDir} ]; then + + # Move legacy roots directory. + if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then + mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir} + fi + + mkdir -p ${cfg.gcRootsDir} + fi + + # Move legacy hydra-www roots. + if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then + find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \ + | xargs -r mv -f -t ${cfg.gcRootsDir}/ + rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots + fi + + chown hydra.hydra ${cfg.gcRootsDir} + chmod 2775 ${cfg.gcRootsDir} + ''; + serviceConfig.ExecStart = "${hydra-package}/bin/hydra-init"; + serviceConfig.PermissionsStartOnly = true; + serviceConfig.User = "hydra"; + serviceConfig.Type = "oneshot"; + serviceConfig.RemainAfterExit = true; + }; + + systemd.services.hydra-server = + { wantedBy = [ "multi-user.target" ]; + requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" ]; + environment = serverEnv; + restartTriggers = [ hydraConf ]; + serviceConfig = + { ExecStart = + "@${hydra-package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' " + + "-p ${toString cfg.port} --max_spare_servers 5 --max_servers 25 " + + "--max_requests 100 ${optionalString cfg.debugServer "-d"}"; + User = "hydra-www"; + PermissionsStartOnly = true; + Restart = "always"; + }; + }; + + systemd.services.hydra-queue-runner = + { wantedBy = [ "multi-user.target" ]; + requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" "network.target" ]; + path = [ hydra-package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ]; + restartTriggers = [ hydraConf ]; + environment = env // { + PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr + IN_SYSTEMD = "1"; # to get log severity levels + }; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v"; + ExecStopPost = "${hydra-package}/bin/hydra-queue-runner --unlock"; + User = "hydra-queue-runner"; + Restart = "always"; + + # Ensure we can get core dumps. + LimitCORE = "infinity"; + WorkingDirectory = "${baseDir}/queue-runner"; + }; + }; + + systemd.services.hydra-evaluator = + { wantedBy = [ "multi-user.target" ]; + requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" "network.target" ]; + path = with pkgs; [ hydra-package nettools jq ]; + restartTriggers = [ hydraConf ]; + environment = env; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator"; + User = "hydra"; + Restart = "always"; + WorkingDirectory = baseDir; + }; + }; + + systemd.services.hydra-update-gc-roots = + { requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" ]; + environment = env; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots"; + User = "hydra"; + }; + startAt = "2,14:15"; + }; + + systemd.services.hydra-send-stats = + { wantedBy = [ "multi-user.target" ]; + after = [ "hydra-init.service" ]; + environment = env; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats"; + User = "hydra"; + }; + }; + + systemd.services.hydra-notify = + { wantedBy = [ "multi-user.target" ]; + requires = [ "hydra-init.service" ]; + after = [ "hydra-init.service" ]; + restartTriggers = [ hydraConf ]; + environment = env // { + PGPASSFILE = "${baseDir}/pgpass-queue-runner"; + }; + serviceConfig = + { ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify"; + # FIXME: run this under a less privileged user? + User = "hydra-queue-runner"; + Restart = "always"; + RestartSec = 5; + }; + }; + + # If there is less than a certain amount of free disk space, stop + # the queue/evaluator to prevent builds from failing or aborting. + systemd.services.hydra-check-space = + { script = + '' + if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then + echo "stopping Hydra queue runner due to lack of free space..." + systemctl stop hydra-queue-runner + fi + if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then + echo "stopping Hydra evaluator due to lack of free space..." + systemctl stop hydra-evaluator + fi + ''; + startAt = "*:0/5"; + }; + + # Periodically compress build logs. The queue runner compresses + # logs automatically after a step finishes, but this doesn't work + # if the queue runner is stopped prematurely. + systemd.services.hydra-compress-logs = + { path = [ pkgs.bzip2 ]; + script = + '' + find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f + ''; + startAt = "Sun 01:45"; + }; + + services.postgresql.enable = mkIf haveLocalDB true; + + services.postgresql.identMap = optionalString haveLocalDB + '' + hydra-users hydra hydra + hydra-users hydra-queue-runner hydra + hydra-users hydra-www hydra + hydra-users root hydra + # The postgres user is used to create the pg_trgm extension for the hydra database + hydra-users postgres postgres + ''; + + services.postgresql.authentication = optionalString haveLocalDB + '' + local hydra all ident map=hydra-users + ''; + + }; + +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/jenkins/default.nix b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/default.nix new file mode 100644 index 000000000000..1477c471f8ab --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/default.nix @@ -0,0 +1,228 @@ +{ config, lib, pkgs, ... }: +with lib; +let + cfg = config.services.jenkins; +in { + options = { + services.jenkins = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable the jenkins continuous integration server. + ''; + }; + + user = mkOption { + default = "jenkins"; + type = types.str; + description = '' + User the jenkins server should execute under. + ''; + }; + + group = mkOption { + default = "jenkins"; + type = types.str; + description = '' + If the default user "jenkins" is configured then this is the primary + group of that user. + ''; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "wheel" "dialout" ]; + description = '' + List of extra groups that the "jenkins" user should be a part of. + ''; + }; + + home = mkOption { + default = "/var/lib/jenkins"; + type = types.path; + description = '' + The path to use as JENKINS_HOME. If the default user "jenkins" is configured then + this is the home of the "jenkins" user. + ''; + }; + + listenAddress = mkOption { + default = "0.0.0.0"; + example = "localhost"; + type = types.str; + description = '' + Specifies the bind address on which the jenkins HTTP interface listens. + The default is the wildcard address. + ''; + }; + + port = mkOption { + default = 8080; + type = types.int; + description = '' + Specifies port number on which the jenkins HTTP interface listens. + The default is 8080. + ''; + }; + + prefix = mkOption { + default = ""; + example = "/jenkins"; + type = types.str; + description = '' + Specifies a urlPrefix to use with jenkins. + If the example /jenkins is given, the jenkins server will be + accessible using localhost:8080/jenkins. + ''; + }; + + package = mkOption { + default = pkgs.jenkins; + defaultText = "pkgs.jenkins"; + type = types.package; + description = "Jenkins package to use."; + }; + + packages = mkOption { + default = [ pkgs.stdenv pkgs.git pkgs.jdk config.programs.ssh.package pkgs.nix ]; + defaultText = "[ pkgs.stdenv pkgs.git pkgs.jdk config.programs.ssh.package pkgs.nix ]"; + type = types.listOf types.package; + description = '' + Packages to add to PATH for the jenkins process. + ''; + }; + + environment = mkOption { + default = { }; + type = with types; attrsOf str; + description = '' + Additional environment variables to be passed to the jenkins process. + As a base environment, jenkins receives NIX_PATH from + <option>environment.sessionVariables</option>, NIX_REMOTE is set to + "daemon" and JENKINS_HOME is set to the value of + <option>services.jenkins.home</option>. + This option has precedence and can be used to override those + mentioned variables. + ''; + }; + + plugins = mkOption { + default = null; + type = types.nullOr (types.attrsOf types.package); + description = '' + A set of plugins to activate. Note that this will completely + remove and replace any previously installed plugins. If you + have manually-installed plugins that you want to keep while + using this module, set this option to + <literal>null</literal>. You can generate this set with a + tool such as <literal>jenkinsPlugins2nix</literal>. + ''; + example = literalExample '' + import path/to/jenkinsPlugins2nix-generated-plugins.nix { inherit (pkgs) fetchurl stdenv; } + ''; + }; + + extraOptions = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "--debug=9" ]; + description = '' + Additional command line arguments to pass to Jenkins. + ''; + }; + + extraJavaOptions = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "-Xmx80m" ]; + description = '' + Additional command line arguments to pass to the Java run time (as opposed to Jenkins). + ''; + }; + }; + }; + + config = mkIf cfg.enable { + # server references the dejavu fonts + environment.systemPackages = [ + pkgs.dejavu_fonts + ]; + + users.groups = optionalAttrs (cfg.group == "jenkins") { + jenkins.gid = config.ids.gids.jenkins; + }; + + users.users = optionalAttrs (cfg.user == "jenkins") { + jenkins = { + description = "jenkins user"; + createHome = true; + home = cfg.home; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + uid = config.ids.uids.jenkins; + }; + }; + + systemd.services.jenkins = { + description = "Jenkins Continuous Integration Server"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = + let + selectedSessionVars = + lib.filterAttrs (n: v: builtins.elem n [ "NIX_PATH" ]) + config.environment.sessionVariables; + in + selectedSessionVars // + { JENKINS_HOME = cfg.home; + NIX_REMOTE = "daemon"; + } // + cfg.environment; + + path = cfg.packages; + + # Force .war (re)extraction, or else we might run stale Jenkins. + + preStart = + let replacePlugins = + if cfg.plugins == null + then "" + else + let pluginCmds = lib.attrsets.mapAttrsToList + (n: v: "cp ${v} ${cfg.home}/plugins/${n}.jpi") + cfg.plugins; + in '' + rm -r ${cfg.home}/plugins || true + mkdir -p ${cfg.home}/plugins + ${lib.strings.concatStringsSep "\n" pluginCmds} + ''; + in '' + rm -rf ${cfg.home}/war + ${replacePlugins} + ''; + + # For reference: https://wiki.jenkins.io/display/JENKINS/JenkinsLinuxStartupScript + script = '' + ${pkgs.jdk}/bin/java ${concatStringsSep " " cfg.extraJavaOptions} -jar ${cfg.package}/webapps/jenkins.war --httpListenAddress=${cfg.listenAddress} \ + --httpPort=${toString cfg.port} \ + --prefix=${cfg.prefix} \ + -Djava.awt.headless=true \ + ${concatStringsSep " " cfg.extraOptions} + ''; + + postStart = '' + until [[ $(${pkgs.curl.bin}/bin/curl -L -s --head -w '\n%{http_code}' http://${cfg.listenAddress}:${toString cfg.port}${cfg.prefix} | tail -n1) =~ ^(200|403)$ ]]; do + sleep 1 + done + ''; + + serviceConfig = { + User = cfg.user; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/jenkins/job-builder.nix b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/job-builder.nix new file mode 100644 index 000000000000..5d1bfe4ec407 --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/job-builder.nix @@ -0,0 +1,205 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + jenkinsCfg = config.services.jenkins; + cfg = config.services.jenkins.jobBuilder; + +in { + options = { + services.jenkins.jobBuilder = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether or not to enable the Jenkins Job Builder (JJB) service. It + allows defining jobs for Jenkins in a declarative manner. + + Jobs managed through the Jenkins WebUI (or by other means) are left + unchanged. + + Note that it really is declarative configuration; if you remove a + previously defined job, the corresponding job directory will be + deleted. + + Please see the Jenkins Job Builder documentation for more info: + <link xlink:href="http://docs.openstack.org/infra/jenkins-job-builder/"> + http://docs.openstack.org/infra/jenkins-job-builder/</link> + ''; + }; + + accessUser = mkOption { + default = ""; + type = types.str; + description = '' + User id in Jenkins used to reload config. + ''; + }; + + accessToken = mkOption { + default = ""; + type = types.str; + description = '' + User token in Jenkins used to reload config. + WARNING: This token will be world readable in the Nix store. To keep + it secret, use the <option>accessTokenFile</option> option instead. + ''; + }; + + accessTokenFile = mkOption { + default = ""; + type = types.str; + example = "/run/keys/jenkins-job-builder-access-token"; + description = '' + File containing the API token for the <option>accessUser</option> + user. + ''; + }; + + yamlJobs = mkOption { + default = ""; + type = types.lines; + example = '' + - job: + name: jenkins-job-test-1 + builders: + - shell: echo 'Hello world!' + ''; + description = '' + Job descriptions for Jenkins Job Builder in YAML format. + ''; + }; + + jsonJobs = mkOption { + default = [ ]; + type = types.listOf types.str; + example = literalExample '' + [ + ''' + [ { "job": + { "name": "jenkins-job-test-2", + "builders": [ "shell": "echo 'Hello world!'" ] + } + } + ] + ''' + ] + ''; + description = '' + Job descriptions for Jenkins Job Builder in JSON format. + ''; + }; + + nixJobs = mkOption { + default = [ ]; + type = types.listOf types.attrs; + example = literalExample '' + [ { job = + { name = "jenkins-job-test-3"; + builders = [ + { shell = "echo 'Hello world!'"; } + ]; + }; + } + ] + ''; + description = '' + Job descriptions for Jenkins Job Builder in Nix format. + + This is a trivial wrapper around jsonJobs, using builtins.toJSON + behind the scene. + ''; + }; + }; + }; + + config = mkIf (jenkinsCfg.enable && cfg.enable) { + assertions = [ + { assertion = + if cfg.accessUser != "" + then (cfg.accessToken != "" && cfg.accessTokenFile == "") || + (cfg.accessToken == "" && cfg.accessTokenFile != "") + else true; + message = '' + One of accessToken and accessTokenFile options must be non-empty + strings, but not both. Current values: + services.jenkins.jobBuilder.accessToken = "${cfg.accessToken}" + services.jenkins.jobBuilder.accessTokenFile = "${cfg.accessTokenFile}" + ''; + } + ]; + + systemd.services.jenkins-job-builder = { + description = "Jenkins Job Builder Service"; + # JJB can run either before or after jenkins. We chose after, so we can + # always use curl to notify (running) jenkins to reload its config. + after = [ "jenkins.service" ]; + wantedBy = [ "multi-user.target" ]; + + path = with pkgs; [ jenkins-job-builder curl ]; + + # Q: Why manipulate files directly instead of using "jenkins-jobs upload [...]"? + # A: Because this module is for administering a local jenkins install, + # and using local file copy allows us to not worry about + # authentication. + script = + let + yamlJobsFile = builtins.toFile "jobs.yaml" cfg.yamlJobs; + jsonJobsFiles = + map (x: (builtins.toFile "jobs.json" x)) + (cfg.jsonJobs ++ [(builtins.toJSON cfg.nixJobs)]); + jobBuilderOutputDir = "/run/jenkins-job-builder/output"; + # Stamp file is placed in $JENKINS_HOME/jobs/$JOB_NAME/ to indicate + # ownership. Enables tracking and removal of stale jobs. + ownerStamp = ".config-xml-managed-by-nixos-jenkins-job-builder"; + reloadScript = '' + echo "Asking Jenkins to reload config" + curl_opts="--silent --fail --show-error" + access_token=${if cfg.accessTokenFile != "" + then "$(cat '${cfg.accessTokenFile}')" + else cfg.accessToken} + jenkins_url="http://${cfg.accessUser}:$access_token@${jenkinsCfg.listenAddress}:${toString jenkinsCfg.port}${jenkinsCfg.prefix}" + crumb=$(curl $curl_opts "$jenkins_url"'/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)') + curl $curl_opts -X POST -H "$crumb" "$jenkins_url"/reload + ''; + in + '' + rm -rf ${jobBuilderOutputDir} + cur_decl_jobs=/run/jenkins-job-builder/declarative-jobs + rm -f "$cur_decl_jobs" + + # Create / update jobs + mkdir -p ${jobBuilderOutputDir} + for inputFile in ${yamlJobsFile} ${concatStringsSep " " jsonJobsFiles}; do + HOME="${jenkinsCfg.home}" "${pkgs.jenkins-job-builder}/bin/jenkins-jobs" --ignore-cache test -o "${jobBuilderOutputDir}" "$inputFile" + done + + for file in "${jobBuilderOutputDir}/"*; do + test -f "$file" || continue + jobname="$(basename $file)" + jobdir="${jenkinsCfg.home}/jobs/$jobname" + echo "Creating / updating job \"$jobname\"" + mkdir -p "$jobdir" + touch "$jobdir/${ownerStamp}" + cp "$file" "$jobdir/config.xml" + echo "$jobname" >> "$cur_decl_jobs" + done + + # Remove stale jobs + for file in "${jenkinsCfg.home}"/jobs/*/${ownerStamp}; do + test -f "$file" || continue + jobdir="$(dirname $file)" + jobname="$(basename "$jobdir")" + grep --quiet --line-regexp "$jobname" "$cur_decl_jobs" 2>/dev/null && continue + echo "Deleting stale job \"$jobname\"" + rm -rf "$jobdir" + done + '' + (if cfg.accessUser != "" then reloadScript else ""); + serviceConfig = { + User = jenkinsCfg.user; + RuntimeDirectory = "jenkins-job-builder"; + }; + }; + }; +} diff --git a/nixpkgs/nixos/modules/services/continuous-integration/jenkins/slave.nix b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/slave.nix new file mode 100644 index 000000000000..3c0e6f78e74c --- /dev/null +++ b/nixpkgs/nixos/modules/services/continuous-integration/jenkins/slave.nix @@ -0,0 +1,68 @@ +{ config, lib, ... }: +with lib; +let + cfg = config.services.jenkinsSlave; + masterCfg = config.services.jenkins; +in { + options = { + services.jenkinsSlave = { + # todo: + # * assure the profile of the jenkins user has a JRE and any specified packages. This would + # enable ssh slaves. + # * Optionally configure the node as a jenkins ad-hoc slave. This would imply configuration + # properties for the master node. + enable = mkOption { + type = types.bool; + default = false; + description = '' + If true the system will be configured to work as a jenkins slave. + If the system is also configured to work as a jenkins master then this has no effect. + In progress: Currently only assures the jenkins user is configured. + ''; + }; + + user = mkOption { + default = "jenkins"; + type = types.str; + description = '' + User the jenkins slave agent should execute under. + ''; + }; + + group = mkOption { + default = "jenkins"; + type = types.str; + description = '' + If the default slave agent user "jenkins" is configured then this is + the primary group of that user. + ''; + }; + + home = mkOption { + default = "/var/lib/jenkins"; + type = types.path; + description = '' + The path to use as JENKINS_HOME. If the default user "jenkins" is configured then + this is the home of the "jenkins" user. + ''; + }; + }; + }; + + config = mkIf (cfg.enable && !masterCfg.enable) { + users.groups = optionalAttrs (cfg.group == "jenkins") { + jenkins.gid = config.ids.gids.jenkins; + }; + + users.users = optionalAttrs (cfg.user == "jenkins") { + jenkins = { + description = "jenkins user"; + createHome = true; + home = cfg.home; + group = cfg.group; + useDefaultShell = true; + uid = config.ids.uids.jenkins; + }; + }; + }; +} |