From 66fefb82e74e42cd2dd1bfbdcd32ebd0429f4565 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 21 Dec 2017 00:40:36 +0100 Subject: nixos/znapzend: stateless setup This enables znapzend users to specify its full configuration through NixOS options, without ever needing to use the stateful `znapzendzetup` command. This works by running znapzendzetup with the specified config in ExecPre, just before the znapzend daemon is started. There is also the `pure` option which will clear all previous znapzend setups, making it as stateless as can get, as only the setup declared in configuration.nix will be persisted. --- nixos/modules/services/backup/znapzend.nix | 369 ++++++++++++++++++++++++++++- 1 file changed, 360 insertions(+), 9 deletions(-) (limited to 'nixos/modules/services/backup/znapzend.nix') diff --git a/nixos/modules/services/backup/znapzend.nix b/nixos/modules/services/backup/znapzend.nix index baf99930e3eb..762bb4b38675 100644 --- a/nixos/modules/services/backup/znapzend.nix +++ b/nixos/modules/services/backup/znapzend.nix @@ -1,39 +1,372 @@ { config, lib, pkgs, ... }: with lib; +with types; let + + # Converts a plan like + # { "1d" = "1h"; "1w" = "1d"; } + # into + # "1d=>1h,1w=>1d" + attrToPlan = attrs: concatStringsSep "," (builtins.attrValues ( + mapAttrs (n: v: "${n}=>${v}") attrs)); + + planDescription = '' + The znapzend backup plan to use for the source. + + + The plan specifies how often to backup and for how long to keep the + backups. It consists of a series of retention periodes to interval + associations: + + + + retA=>intA,retB=>intB,... + + + + Both intervals and retention periods are expressed in standard units + of time or multiples of them. You can use both the full name or a + shortcut according to the following listing: + + + + second|sec|s, minute|min, hour|h, day|d, week|w, month|mon|m, year|y + + + + See znapzendzetup1 for more info. + ''; + planExample = "1h=>10min,1d=>1h,1w=>1d,1m=>1w,1y=>1m"; + + # A type for a string of the form number{b|k|M|G} + mbufferSizeType = str // { + check = x: str.check x && builtins.isList (builtins.match "^[0-9]+[bkMG]$" x); + description = "string of the form number{b|k|M|G}"; + }; + + # Type for a string that must contain certain other strings (the list parameter). + # Note that these would need regex escaping. + stringContainingStrings = list: let + matching = s: map (str: builtins.match ".*${str}.*" s) list; + in str // { + check = x: str.check x && all isList (matching x); + description = "string containing all of the characters ${concatStringsSep ", " list}"; + }; + + timestampType = stringContainingStrings [ "%Y" "%m" "%d" "%H" "%M" "%S" ]; + + destType = srcConfig: submodule ({ name, ... }: { + options = { + + label = mkOption { + type = str; + description = "Label for this destination. Defaults to the attribute name."; + }; + + plan = mkOption { + type = str; + description = planDescription; + example = planExample; + }; + + dataset = mkOption { + type = str; + description = "Dataset name to send snapshots to."; + example = "tank/main"; + }; + + host = mkOption { + type = nullOr str; + description = '' + Host to use for the destination dataset. Can be prefixed with + user@ to specify the ssh user. + ''; + default = null; + example = "john@example.com"; + }; + + presend = mkOption { + type = nullOr str; + description = '' + Command to run before sending the snapshot to the destination. + Intended to run a remote script via ssh on the + destination, e.g. to bring up a backup disk or server or to put a + zpool online/offline. See also . + ''; + default = null; + example = "ssh root@bserv zpool import -Nf tank"; + }; + + postsend = mkOption { + type = nullOr str; + description = '' + Command to run after sending the snapshot to the destination. + Intended to run a remote script via ssh on the + destination, e.g. to bring up a backup disk or server or to put a + zpool online/offline. See also . + ''; + default = null; + example = "ssh root@bserv zpool export tank"; + }; + }; + + config = { + label = mkDefault name; + plan = mkDefault srcConfig.plan; + }; + }); + + + + srcType = submodule ({ name, config, ... }: { + options = { + + enable = mkOption { + type = bool; + description = "Whether to enable this source."; + default = true; + }; + + recursive = mkOption { + type = bool; + description = "Whether to do recursive snapshots."; + default = false; + }; + + mbuffer = { + enable = mkOption { + type = bool; + description = "Whether to use mbuffer."; + default = false; + }; + + port = mkOption { + type = nullOr ints.u16; + description = '' + Port to use for mbuffer. + + + If this is null, it will run mbuffer through + ssh. + + + If this is not null, it will run mbuffer + directly through TCP, which is not encrypted but faster. In that + case the given port needs to be open on the destination host. + ''; + default = null; + }; + + size = mkOption { + type = mbufferSizeType; + description = '' + The size for mbuffer. + Supports the units b, k, M, G. + ''; + default = "1G"; + example = "128M"; + }; + }; + + presnap = mkOption { + type = nullOr str; + description = '' + Command to run before snapshots are taken on the source dataset, + e.g. for database locking/flushing. See also + . + ''; + default = null; + example = literalExample '' + ''${pkgs.mariadb}/bin/mysql -e "set autocommit=0;flush tables with read lock;\\! ''${pkgs.coreutils}/bin/sleep 600" & ''${pkgs.coreutils}/bin/echo $! > /tmp/mariadblock.pid ; sleep 10 + ''; + }; + + postsnap = mkOption { + type = nullOr str; + description = '' + Command to run after snapshots are taken on the source dataset, + e.g. for database unlocking. See also . + ''; + default = null; + example = literalExample '' + ''${pkgs.coreutils}/bin/kill `''${pkgs.coreutils}/bin/cat /tmp/mariadblock.pid`;''${pkgs.coreutils}/bin/rm /tmp/mariadblock.pid + ''; + }; + + timestampFormat = mkOption { + type = timestampType; + description = '' + The timestamp format to use for constructing snapshot names. + The syntax is strftime-like. The string must + consist of the mandatory %Y %m %d %H %M %S. + Optionally - _ . : characters as well as any + alphanumeric character are allowed. If suffixed by a + Z, times will be in UTC. + ''; + default = "%Y-%m-%d-%H%M%S"; + example = "znapzend-%m.%d.%Y-%H%M%SZ"; + }; + + sendDelay = mkOption { + type = int; + description = '' + Specify delay (in seconds) before sending snaps to the destination. + May be useful if you want to control sending time. + ''; + default = 0; + example = 60; + }; + + plan = mkOption { + type = str; + description = planDescription; + example = planExample; + }; + + dataset = mkOption { + type = str; + description = "The dataset to use for this source."; + example = "tank/home"; + }; + + destinations = mkOption { + type = loaOf (destType config); + description = "Additional destinations."; + default = {}; + example = literalExample '' + { + local = { + dataset = "btank/backup"; + presend = "zpool import -N btank"; + postsend = "zpool export btank"; + }; + remote = { + host = "john@example.com"; + dataset = "tank/john"; + }; + }; + ''; + }; + }; + + config = { + dataset = mkDefault name; + }; + + }); + + ### Generating the configuration from here + cfg = config.services.znapzend; + + onOff = b: if b then "on" else "off"; + nullOff = b: if isNull b then "off" else toString b; + stripSlashes = replaceStrings [ "/" ] [ "." ]; + + attrsToFile = config: concatStringsSep "\n" (builtins.attrValues ( + mapAttrs (n: v: "${n}=${v}") config)); + + mkDestAttrs = dst: with dst; + mapAttrs' (n: v: nameValuePair "dst_${label}${n}" v) ({ + "" = optionalString (! isNull host) "${host}:" + dataset; + _plan = plan; + } // optionalAttrs (presend != null) { + _precmd = presend; + } // optionalAttrs (postsend != null) { + _pstcmd = postsend; + }); + + mkSrcAttrs = srcCfg: with srcCfg; { + enabled = onOff enable; + mbuffer = with mbuffer; if enable then "${pkgs.mbuffer}/bin/mbuffer" + + optionalString (port != null) ":${toString port}" else "off"; + mbuffer_size = mbuffer.size; + post_znap_cmd = nullOff postsnap; + pre_znap_cmd = nullOff presnap; + recursive = onOff recursive; + src = dataset; + src_plan = plan; + tsformat = timestampFormat; + zend_delay = toString sendDelay; + } // fold (a: b: a // b) {} ( + map mkDestAttrs (builtins.attrValues destinations) + ); + + files = mapAttrs' (n: srcCfg: let + fileText = attrsToFile (mkSrcAttrs srcCfg); + in { + name = srcCfg.dataset; + value = pkgs.writeText (stripSlashes srcCfg.dataset) fileText; + }) cfg.zetup; + in { options = { services.znapzend = { - enable = mkEnableOption "ZnapZend daemon"; + enable = mkEnableOption "ZnapZend ZFS backup daemon"; logLevel = mkOption { default = "debug"; example = "warning"; - type = lib.types.enum ["debug" "info" "warning" "err" "alert"]; - description = "The log level when logging to file. Any of debug, info, warning, err, alert. Default in daemonized form is debug."; + type = enum ["debug" "info" "warning" "err" "alert"]; + description = '' + The log level when logging to file. Any of debug, info, warning, err, + alert. Default in daemonized form is debug. + ''; }; logTo = mkOption { - type = types.str; + type = str; default = "syslog::daemon"; example = "/var/log/znapzend.log"; - description = "Where to log to (syslog::<facility> or <filepath>)."; + description = '' + Where to log to (syslog::<facility> or <filepath>). + ''; }; noDestroy = mkOption { - type = types.bool; + type = bool; default = false; description = "Does all changes to the filesystem except destroy."; }; autoCreation = mkOption { - type = types.bool; + type = bool; + default = false; + description = "Automatically create the destination dataset if it does not exists."; + }; + + zetup = mkOption { + type = loaOf srcType; + description = "Znapzend configuration."; + default = {}; + example = literalExample '' + { + "tank/home" = { + # Make snapshots of tank/home every hour, keep those for 1 day, + # keep every days snapshot for 1 month, etc. + plan = "1d=>1h,1m=>1d,1y=>1m"; + recursive = true; + # Send all those snapshots to john@example.com:rtank/john as well + destinations.remote = { + host = "john@example.com"; + dataset = "rtank/john"; + }; + }; + }; + ''; + }; + + pure = mkOption { + type = bool; + description = '' + Do not persist any stateful znapzend setups. If this option is + enabled, your previously set znapzend setups will be cleared and only + the ones defined with this module will be applied. + ''; default = false; - description = "Automatically create the dataset on dest if it does not exists."; }; }; }; @@ -49,12 +382,30 @@ in path = with pkgs; [ zfs mbuffer openssh ]; + preStart = optionalString cfg.pure '' + echo Resetting znapzend zetups + ${pkgs.znapzend}/bin/znapzendzetup list \ + | grep -oP '(?<=\*\*\* backup plan: ).*(?= \*\*\*)' \ + | xargs ${pkgs.znapzend}/bin/znapzendzetup delete + '' + concatStringsSep "\n" (mapAttrsToList (dataset: config: '' + echo Importing znapzend zetup ${config} for dataset ${dataset} + ${pkgs.znapzend}/bin/znapzendzetup import --write ${dataset} ${config} + '') files); + serviceConfig = { - ExecStart = "${pkgs.znapzend}/bin/znapzend --logto=${cfg.logTo} --loglevel=${cfg.logLevel} ${optionalString cfg.noDestroy "--nodestroy"} ${optionalString cfg.autoCreation "--autoCreation"}"; + ExecStart = let + args = concatStringsSep " " [ + "--logto=${cfg.logTo}" + "--loglevel=${cfg.logLevel}" + (optionalString cfg.noDestroy "--nodestroy") + (optionalString cfg.autoCreation "--autoCreation") + ]; in "${pkgs.znapzend}/bin/znapzend ${args}"; ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; Restart = "on-failure"; }; }; }; }; + + meta.maintainers = with maintainers; [ infinisil ]; } -- cgit 1.4.1 From c2b8d14b56f70b220b4c7c253036a3b3778a6780 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 17 Feb 2018 15:50:48 +0100 Subject: nixos/znapzend: fix when no previous zetup When the znapzend module was enabled for the first time with pure = true; then the list of previous entries is empty, but xargs still tried to execute a znapzendzetup delete command with no arguments, which made it fail --- nixos/modules/services/backup/znapzend.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nixos/modules/services/backup/znapzend.nix') diff --git a/nixos/modules/services/backup/znapzend.nix b/nixos/modules/services/backup/znapzend.nix index 762bb4b38675..3d133f82d204 100644 --- a/nixos/modules/services/backup/znapzend.nix +++ b/nixos/modules/services/backup/znapzend.nix @@ -386,7 +386,7 @@ in echo Resetting znapzend zetups ${pkgs.znapzend}/bin/znapzendzetup list \ | grep -oP '(?<=\*\*\* backup plan: ).*(?= \*\*\*)' \ - | xargs ${pkgs.znapzend}/bin/znapzendzetup delete + | xargs -I{} ${pkgs.znapzend}/bin/znapzendzetup delete "{}" '' + concatStringsSep "\n" (mapAttrsToList (dataset: config: '' echo Importing znapzend zetup ${config} for dataset ${dataset} ${pkgs.znapzend}/bin/znapzendzetup import --write ${dataset} ${config} -- cgit 1.4.1