diff options
author | Bas van Dijk <bas@van.dijk.ch> | 2021-10-27 14:18:00 +0200 |
---|---|---|
committer | Bas van Dijk <bas@dfinity.org> | 2021-11-04 11:15:21 +0000 |
commit | f12e976aded8113cb466efcdc2abc1932d9beb05 (patch) | |
tree | a7fad1dd1d6e4561977f691d45d2e7faf7578890 /nixos/modules/services/monitoring | |
parent | 8d5213123af4ba144de3dadbe3f9353ba9f4e8f7 (diff) | |
download | nixlib-f12e976aded8113cb466efcdc2abc1932d9beb05.tar nixlib-f12e976aded8113cb466efcdc2abc1932d9beb05.tar.gz nixlib-f12e976aded8113cb466efcdc2abc1932d9beb05.tar.bz2 nixlib-f12e976aded8113cb466efcdc2abc1932d9beb05.tar.lz nixlib-f12e976aded8113cb466efcdc2abc1932d9beb05.tar.xz nixlib-f12e976aded8113cb466efcdc2abc1932d9beb05.tar.zst nixlib-f12e976aded8113cb466efcdc2abc1932d9beb05.zip |
module/prometheus: optionally support reloading on config changes
The new option `services.prometheus.enableReload` has been introduced which, when enabled, causes the prometheus systemd service to reload when its config file changes. More specifically the following property holds: switching to a configuration (`switch-to-configuration`) that changes the prometheus configuration only finishes successully when prometheus has finished loading the new configuration. `enableReload` is `false` by default in which case the old semantics of restarting the prometheus systemd service are in effect.
Diffstat (limited to 'nixos/modules/services/monitoring')
-rw-r--r-- | nixos/modules/services/monitoring/prometheus/default.nix | 97 |
1 files changed, 94 insertions, 3 deletions
diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix index d2b37cf688bf..5f7bda1acbc6 100644 --- a/nixos/modules/services/monitoring/prometheus/default.nix +++ b/nixos/modules/services/monitoring/prometheus/default.nix @@ -7,6 +7,30 @@ let workingDir = "/var/lib/" + cfg.stateDir; + prometheusYmlOut = "${workingDir}/prometheus-substituted.yaml"; + + writeConfig = pkgs.writeShellScriptBin "write-prometheus-config" '' + PATH="${makeBinPath (with pkgs; [ coreutils envsubst ])}" + touch '${prometheusYmlOut}' + chmod 600 '${prometheusYmlOut}' + envsubst -o '${prometheusYmlOut}' -i '${prometheusYml}' + ''; + + triggerReload = pkgs.writeShellScriptBin "trigger-reload-prometheus" '' + PATH="${makeBinPath (with pkgs; [ systemd ])}" + if systemctl -q is-active prometheus.service; then + systemctl reload prometheus.service + fi + ''; + + reload = pkgs.writeShellScriptBin "reload-prometheus" '' + PATH="${makeBinPath (with pkgs; [ systemd coreutils gnugrep ])}" + cursor=$(journalctl --show-cursor -n0 | grep -oP "cursor: \K.*") + kill -HUP $MAINPID + journalctl -u prometheus.service --after-cursor="$cursor" -f \ + | grep -m 1 "Completed loading of configuration file" > /dev/null + ''; + # a wrapper that verifies that the configuration is valid promtoolCheck = what: name: file: if cfg.checkConfig then @@ -47,7 +71,11 @@ let cmdlineArgs = cfg.extraFlags ++ [ "--storage.tsdb.path=${workingDir}/data/" - "--config.file=/run/prometheus/prometheus-substituted.yaml" + "--config.file=${ + if cfg.enableReload + then prometheusYmlOut + else "/run/prometheus/prometheus-substituted.yaml" + }" "--web.listen-address=${cfg.listenAddress}:${builtins.toString cfg.port}" "--alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}" "--alertmanager.timeout=${toString cfg.alertmanagerTimeout}s" @@ -731,6 +759,25 @@ in { ''; }; + enableReload = mkOption { + default = false; + type = types.bool; + description = '' + Reload prometheus when configuration file changes (instead of restart). + + The following property holds: switching to a configuration + (<literal>switch-to-configuration</literal>) that changes the prometheus + configuration only finishes successully when prometheus has finished + loading the new configuration. + + Note that prometheus will also get reloaded when the location of the + <option>environmentFile</option> changes but not when its contents + changes. So when you change it contents make sure to reload prometheus + manually or include the hash of <option>environmentFile</option> in its + name. + ''; + }; + environmentFile = mkOption { type = types.nullOr types.path; default = null; @@ -928,7 +975,7 @@ in { systemd.services.prometheus = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; - preStart = '' + preStart = mkIf (!cfg.enableReload) '' ${lib.getBin pkgs.envsubst}/bin/envsubst -o "/run/prometheus/prometheus-substituted.yaml" \ -i "${prometheusYml}" ''; @@ -936,9 +983,10 @@ in { ExecStart = "${cfg.package}/bin/prometheus" + optionalString (length cmdlineArgs != 0) (" \\\n " + concatStringsSep " \\\n " cmdlineArgs); + ExecReload = mkIf cfg.enableReload "+${reload}/bin/reload-prometheus"; User = "prometheus"; Restart = "always"; - EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ]; + EnvironmentFile = mkIf (cfg.environmentFile != null && !cfg.enableReload) [ cfg.environmentFile ]; RuntimeDirectory = "prometheus"; RuntimeDirectoryMode = "0700"; WorkingDirectory = workingDir; @@ -946,5 +994,48 @@ in { StateDirectoryMode = "0700"; }; }; + systemd.services.prometheus-config-write = mkIf cfg.enableReload { + wantedBy = [ "prometheus.service" ]; + before = [ "prometheus.service" ]; + serviceConfig = { + Type = "oneshot"; + User = "prometheus"; + StateDirectory = cfg.stateDir; + StateDirectoryMode = "0700"; + EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ]; + ExecStart = "${writeConfig}/bin/write-prometheus-config"; + }; + }; + # prometheus-config-reload will activate after prometheus. However, what we + # don't want is that on startup it immediately reloads prometheus because + # prometheus itself might have just started. + # + # Instead we only want to reload prometheus when the config file has + # changed. So on startup prometheus-config-reload will just output a + # harmless message and then stay active (RemainAfterExit). + # + # Then, when the config file has changed, switch-to-configuration notices + # that this service has changed and needs to be reloaded + # (reloadIfChanged). The reload command then actually writes the new config + # and reloads prometheus. + systemd.services.prometheus-config-reload = mkIf cfg.enableReload { + wantedBy = [ "prometheus.service" ]; + after = [ "prometheus.service" ]; + reloadIfChanged = true; + serviceConfig = { + Type = "oneshot"; + User = "prometheus"; + StateDirectory = cfg.stateDir; + StateDirectoryMode = "0700"; + EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ]; + RemainAfterExit = true; + TimeoutSec = 60; + ExecStart = "${pkgs.logger}/bin/logger 'prometheus-config-reload will only reload prometheus when reloaded itself.'"; + ExecReload = [ + "${writeConfig}/bin/write-prometheus-config" + "+${triggerReload}/bin/trigger-reload-prometheus" + ]; + }; + }; }; } |