nixos/prometheus: check configuration before starting service

With `promtool` we can check the validity of a configuration before
deploying it. This avoids situations where you would end up with a
broken monitoring system without noticing it - since the monitoring
broke down. :-)

1 files changed, 13 insertions, 5 deletions

diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix
index 63b7d1c0bd4e..bf4dfc666bb6 100644
--- a/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixos/modules/services/monitoring/prometheus/default.nix
@@ -10,6 +10,13 @@ let
   # Get a submodule without any embedded metadata:
   _filter = x: filterAttrs (k: v: k != "_module") x;
 
+  # a wrapper that verifies that the configuration is valid
+  promtoolCheck = what: name: file: pkgs.runCommand "${name}-${what}-checked"
+    { buildInputs = [ cfg.package ]; } ''
+    ln -s ${file} $out
+    promtool ${what} $out
+  '';
+
   # Pretty-print JSON to a file
   writePrettyJSON = name: x:
     pkgs.runCommand name { } ''
@@ -19,18 +26,19 @@ let
   # This becomes the main config file
   promConfig = {
     global = cfg.globalConfig;
-    rule_files = cfg.ruleFiles ++ [
+    rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
       (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg.rules))
-    ];
+    ]);
     scrape_configs = cfg.scrapeConfigs;
   };
 
   generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
 
-  prometheusYml =
-    if cfg.configText != null then
+  prometheusYml = let
+    yml =  if cfg.configText != null then
       pkgs.writeText "prometheus.yml" cfg.configText
-    else generatedPrometheusYml;
+      else generatedPrometheusYml;
+    in promtoolCheck "check-config" "prometheus.yml" yml;
 
   cmdlineArgs = cfg.extraFlags ++ [
     "-storage.local.path=${cfg.dataDir}/metrics"