diff options
author | Matthew Justin Bauer <mjbauer95@gmail.com> | 2018-04-21 14:53:23 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-04-21 14:53:23 -0500 |
commit | 1eea73eaa8185bd6f4fe2a0facc90652277ea399 (patch) | |
tree | fa5e6aaccbdf1ef9f4e71d0f74225252a5c2168c /nixos/modules/services/backup | |
parent | e70d293b6b5f236d69d559ccccdafba19c6d29c3 (diff) | |
parent | 17b73ff90cf9f7631e543d880540441bdf56d719 (diff) | |
download | nixlib-1eea73eaa8185bd6f4fe2a0facc90652277ea399.tar nixlib-1eea73eaa8185bd6f4fe2a0facc90652277ea399.tar.gz nixlib-1eea73eaa8185bd6f4fe2a0facc90652277ea399.tar.bz2 nixlib-1eea73eaa8185bd6f4fe2a0facc90652277ea399.tar.lz nixlib-1eea73eaa8185bd6f4fe2a0facc90652277ea399.tar.xz nixlib-1eea73eaa8185bd6f4fe2a0facc90652277ea399.tar.zst nixlib-1eea73eaa8185bd6f4fe2a0facc90652277ea399.zip |
Merge branch 'master' into feat/acme/delay
Diffstat (limited to 'nixos/modules/services/backup')
-rw-r--r-- | nixos/modules/services/backup/almir.nix | 173 | ||||
-rw-r--r-- | nixos/modules/services/backup/borgbackup.nix | 580 | ||||
-rw-r--r-- | nixos/modules/services/backup/crashplan-small-business.nix | 74 | ||||
-rw-r--r-- | nixos/modules/services/backup/duplicati.nix | 40 | ||||
-rw-r--r-- | nixos/modules/services/backup/restic.nix | 150 | ||||
-rw-r--r-- | nixos/modules/services/backup/tarsnap.nix | 69 | ||||
-rw-r--r-- | nixos/modules/services/backup/znapzend.nix | 369 |
7 files changed, 1265 insertions, 190 deletions
diff --git a/nixos/modules/services/backup/almir.nix b/nixos/modules/services/backup/almir.nix deleted file mode 100644 index fbb4ff4034f1..000000000000 --- a/nixos/modules/services/backup/almir.nix +++ /dev/null @@ -1,173 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; - -let - cfg = config.services.almir; - - bconsoleconf = pkgs.writeText "bconsole.conf" - '' - Director { - Name = ${cfg.director_name} - DIRport = ${toString cfg.director_port} - address = ${cfg.director_address} - Password = "${cfg.director_password}" - } - ''; - - productionini = pkgs.writeText "production.ini" - '' -[app:main] -use = egg:almir - -pyramid.reload_templates = false -pyramid.debug_authorization = false -pyramid.debug_notfound = false -pyramid.debug_routematch = false -pyramid.debug_templates = false -pyramid.default_locale_name = en -pyramid.includes = - pyramid_exclog -exclog.extra_info = true - -sqlalchemy.url = ${cfg.sqlalchemy_engine_url} -timezone = ${cfg.timezone} -bconsole_config = ${bconsoleconf} - -[server:main] -use = egg:waitress#main -host = 127.0.0.1 -port = ${toString cfg.port} - - -# Begin logging configuration - -[loggers] -keys = root, almir, sqlalchemy, exc_logger - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console - -[logger_almir] -level = WARN -handlers = -qualname = almir - -[logger_exc_logger] -level = ERROR -handlers = -qualname = exc_logger - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine -# "level = INFO" logs SQL queries. -# "level = DEBUG" logs SQL queries and results. -# "level = WARN" logs neither. (Recommended for production systems.) - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s - ''; -in { - options = { - services.almir = { - enable = mkOption { - type = types.bool; - default = false; - description = '' - Enable Almir web server. Also configures postgresql database and installs bacula. - ''; - }; - - port = mkOption { - default = 35000; - type = types.int; - description = '' - Port for Almir web server to listen on. - ''; - }; - - timezone = mkOption { - description = '' - Timezone as specified in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - ''; - example = "Europe/Ljubljana"; - }; - - sqlalchemy_engine_url = mkOption { - default = "postgresql:///bacula"; - example = '' - postgresql://bacula:bacula@localhost:5432/bacula - mysql+mysqlconnector://<user>:<password>@<hostname>/<database>' - sqlite:////var/lib/bacula/bacula.db' - ''; - description = '' - Define SQL database connection to bacula catalog as specified in http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls - ''; - }; - - director_name = mkOption { - description = '' - Name of the Director to connect with bconsole. - ''; - }; - - director_password = mkOption { - description = '' - Password for Director to connect with bconsole. - ''; - }; - - director_port = mkOption { - default = 9101; - type = types.int; - description = '' - Port for Director to connect with bconsole. - ''; - }; - - director_address = mkOption { - default = "127.0.0.1"; - description = '' - IP/Hostname for Director to connect with bconsole. - ''; - }; - }; - }; - - config = mkIf cfg.enable { - systemd.services.almir = { - after = [ "network.target" "postgresql.service" ]; - description = "Almir web app"; - wantedBy = [ "multi-user.target" ]; - path = [ pkgs.pythonPackages.almir ]; - environment.PYTHONPATH = "${pkgs.pythonPackages.almir}/lib/${pkgs.pythonPackages.python.libPrefix}/site-packages"; - serviceConfig.ExecStart = "${pkgs.pythonPackages.pyramid}/bin/pserve ${productionini}"; - }; - - environment.systemPackages = [ pkgs.pythonPackages.almir ]; - - users.extraUsers.almir = { - group = "almir"; - uid = config.ids.uids.almir; - createHome = true; - shell = "${pkgs.bash}/bin/bash"; - }; - - users.extraGroups.almir.gid = config.ids.gids.almir; - }; -} diff --git a/nixos/modules/services/backup/borgbackup.nix b/nixos/modules/services/backup/borgbackup.nix new file mode 100644 index 000000000000..1b730e0c2b76 --- /dev/null +++ b/nixos/modules/services/backup/borgbackup.nix @@ -0,0 +1,580 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + isLocalPath = x: + builtins.substring 0 1 x == "/" # absolute path + || builtins.substring 0 1 x == "." # relative path + || builtins.match "[.*:.*]" == null; # not machine:path + + mkExcludeFile = cfg: + # Write each exclude pattern to a new line + pkgs.writeText "excludefile" (concatStringsSep "\n" cfg.exclude); + + mkKeepArgs = cfg: + # If cfg.prune.keep e.g. has a yearly attribute, + # its content is passed on as --keep-yearly + concatStringsSep " " + (mapAttrsToList (x: y: "--keep-${x}=${toString y}") cfg.prune.keep); + + mkBackupScript = cfg: '' + on_exit() + { + exitStatus=$? + # Reset the EXIT handler, or else we're called again on 'exit' below + trap - EXIT + ${cfg.postHook} + exit $exitStatus + } + trap 'on_exit' INT TERM QUIT EXIT + + archiveName="${cfg.archiveBaseName}-$(date ${cfg.dateFormat})" + archiveSuffix="${optionalString cfg.appendFailedSuffix ".failed"}" + ${cfg.preHook} + '' + optionalString cfg.doInit '' + # Run borg init if the repo doesn't exist yet + if ! borg list > /dev/null; then + borg init \ + --encryption ${cfg.encryption.mode} \ + $extraInitArgs + ${cfg.postInit} + fi + '' + '' + borg create \ + --compression ${cfg.compression} \ + --exclude-from ${mkExcludeFile cfg} \ + $extraCreateArgs \ + "::$archiveName$archiveSuffix" \ + ${escapeShellArgs cfg.paths} + '' + optionalString cfg.appendFailedSuffix '' + borg rename "::$archiveName$archiveSuffix" "$archiveName" + '' + '' + ${cfg.postCreate} + '' + optionalString (cfg.prune.keep != { }) '' + borg prune \ + ${mkKeepArgs cfg} \ + --prefix ${escapeShellArg cfg.prune.prefix} \ + $extraPruneArgs + ${cfg.postPrune} + ''; + + mkPassEnv = cfg: with cfg.encryption; + if passCommand != null then + { BORG_PASSCOMMAND = passCommand; } + else if passphrase != null then + { BORG_PASSPHRASE = passphrase; } + else { }; + + mkBackupService = name: cfg: + let + userHome = config.users.users.${cfg.user}.home; + in nameValuePair "borgbackup-job-${name}" { + description = "BorgBackup job ${name}"; + path = with pkgs; [ + borgbackup openssh + ]; + script = mkBackupScript cfg; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + # Only run when no other process is using CPU or disk + CPUSchedulingPolicy = "idle"; + IOSchedulingClass = "idle"; + ProtectSystem = "strict"; + ReadWritePaths = + [ "${userHome}/.config/borg" "${userHome}/.cache/borg" ] + # Borg needs write access to repo if it is not remote + ++ optional (isLocalPath cfg.repo) cfg.repo; + PrivateTmp = true; + }; + environment = { + BORG_REPO = cfg.repo; + inherit (cfg) extraInitArgs extraCreateArgs extraPruneArgs; + } // (mkPassEnv cfg) // cfg.environment; + inherit (cfg) startAt; + }; + + # Paths listed in ReadWritePaths must exist before service is started + mkActivationScript = name: cfg: + let + install = "install -o ${cfg.user} -g ${cfg.group}"; + in + nameValuePair "borgbackup-job-${name}" (stringAfter [ "users" ] ('' + # Eensure that the home directory already exists + # We can't assert createHome == true because that's not the case for root + cd "${config.users.users.${cfg.user}.home}" + ${install} -d .config/borg + ${install} -d .cache/borg + '' + optionalString (isLocalPath cfg.repo) '' + ${install} -d ${escapeShellArg cfg.repo} + '')); + + mkPassAssertion = name: cfg: { + assertion = with cfg.encryption; + mode != "none" -> passCommand != null || passphrase != null; + message = + "passCommand or passphrase has to be specified because" + + '' borgbackup.jobs.${name}.encryption != "none"''; + }; + + mkRepoService = name: cfg: + nameValuePair "borgbackup-repo-${name}" { + description = "Create BorgBackup repository ${name} directory"; + script = '' + mkdir -p ${escapeShellArg cfg.path} + chown ${cfg.user}:${cfg.group} ${escapeShellArg cfg.path} + ''; + serviceConfig = { + # The service's only task is to ensure that the specified path exists + Type = "oneshot"; + }; + wantedBy = [ "multi-user.target" ]; + }; + + mkAuthorizedKey = cfg: appendOnly: key: + let + # Because of the following line, clients do not need to specify an absolute repo path + cdCommand = "cd ${escapeShellArg cfg.path}"; + restrictedArg = "--restrict-to-${if cfg.allowSubRepos then "path" else "repository"} ."; + appendOnlyArg = optionalString appendOnly "--append-only"; + quotaArg = optionalString (cfg.quota != null) "--storage-quota ${cfg.quota}"; + serveCommand = "borg serve ${restrictedArg} ${appendOnlyArg} ${quotaArg}"; + in + ''command="${cdCommand} && ${serveCommand}",restrict ${key}''; + + mkUsersConfig = name: cfg: { + users.${cfg.user} = { + openssh.authorizedKeys.keys = + (map (mkAuthorizedKey cfg false) cfg.authorizedKeys + ++ map (mkAuthorizedKey cfg true) cfg.authorizedKeysAppendOnly); + useDefaultShell = true; + }; + groups.${cfg.group} = { }; + }; + + mkKeysAssertion = name: cfg: { + assertion = cfg.authorizedKeys != [ ] || cfg.authorizedKeysAppendOnly != [ ]; + message = + "borgbackup.repos.${name} does not make sense" + + " without at least one public key"; + }; + +in { + meta.maintainers = with maintainers; [ dotlambda ]; + + ###### interface + + options.services.borgbackup.jobs = mkOption { + description = "Deduplicating backups using BorgBackup."; + default = { }; + example = literalExample '' + { + rootBackup = { + paths = "/"; + exclude = [ "/nix" ]; + repo = "/path/to/local/repo"; + encryption = { + mode = "repokey"; + passphrase = "secret"; + }; + compression = "auto,lzma"; + startAt = "weekly"; + }; + } + ''; + type = types.attrsOf (types.submodule (let globalConfig = config; in + { name, config, ... }: { + options = { + + paths = mkOption { + type = with types; either path (nonEmptyListOf path); + description = "Path(s) to back up."; + example = "/home/user"; + apply = x: if isList x then x else [ x ]; + }; + + repo = mkOption { + type = types.str; + description = "Remote or local repository to back up to."; + example = "user@machine:/path/to/repo"; + }; + + archiveBaseName = mkOption { + type = types.strMatching "[^/{}]+"; + default = "${globalConfig.networking.hostName}-${name}"; + defaultText = "\${config.networking.hostName}-<name>"; + description = '' + How to name the created archives. A timestamp, whose format is + determined by <option>dateFormat</option>, will be appended. The full + name can be modified at runtime (<literal>$archiveName</literal>). + Placeholders like <literal>{hostname}</literal> must not be used. + ''; + }; + + dateFormat = mkOption { + type = types.str; + description = '' + Arguments passed to <command>date</command> + to create a timestamp suffix for the archive name. + ''; + default = "+%Y-%m-%dT%H:%M:%S"; + example = "-u +%s"; + }; + + startAt = mkOption { + type = with types; either str (listOf str); + default = "daily"; + description = '' + When or how often the backup should run. + Must be in the format described in + <citerefentry><refentrytitle>systemd.time</refentrytitle> + <manvolnum>7</manvolnum></citerefentry>. + If you do not want the backup to start + automatically, use <literal>[ ]</literal>. + ''; + }; + + user = mkOption { + type = types.str; + description = '' + The user <command>borg</command> is run as. + User or group need read permission + for the specified <option>paths</option>. + ''; + default = "root"; + }; + + group = mkOption { + type = types.str; + description = '' + The group borg is run as. User or group needs read permission + for the specified <option>paths</option>. + ''; + default = "root"; + }; + + encryption.mode = mkOption { + type = types.enum [ + "repokey" "keyfile" + "repokey-blake2" "keyfile-blake2" + "authenticated" "authenticated-blake2" + "none" + ]; + description = '' + Encryption mode to use. Setting a mode + other than <literal>"none"</literal> requires + you to specify a <option>passCommand</option> + or a <option>passphrase</option>. + ''; + }; + + encryption.passCommand = mkOption { + type = with types; nullOr str; + description = '' + A command which prints the passphrase to stdout. + Mutually exclusive with <option>passphrase</option>. + ''; + default = null; + example = "cat /path/to/passphrase_file"; + }; + + encryption.passphrase = mkOption { + type = with types; nullOr str; + description = '' + The passphrase the backups are encrypted with. + Mutually exclusive with <option>passCommand</option>. + If you do not want the passphrase to be stored in the + world-readable Nix store, use <option>passCommand</option>. + ''; + default = null; + }; + + compression = mkOption { + # "auto" is optional, + # compression mode must be given, + # compression level is optional + type = types.strMatching "none|(auto,)?(lz4|zstd|zlib|lzma)(,[[:digit:]]{1,2})?"; + description = '' + Compression method to use. Refer to + <command>borg help compression</command> + for all available options. + ''; + default = "lz4"; + example = "auto,lzma"; + }; + + exclude = mkOption { + type = with types; listOf str; + description = '' + Exclude paths matching any of the given patterns. See + <command>borg help patterns</command> for pattern syntax. + ''; + default = [ ]; + example = [ + "/home/*/.cache" + "/nix" + ]; + }; + + doInit = mkOption { + type = types.bool; + description = '' + Run <command>borg init</command> if the + specified <option>repo</option> does not exist. + You should set this to <literal>false</literal> + if the repository is located on an external drive + that might not always be mounted. + ''; + default = true; + }; + + appendFailedSuffix = mkOption { + type = types.bool; + description = '' + Append a <literal>.failed</literal> suffix + to the archive name, which is only removed if + <command>borg create</command> has a zero exit status. + ''; + default = true; + }; + + prune.keep = mkOption { + # Specifying e.g. `prune.keep.yearly = -1` + # means there is no limit of yearly archives to keep + # The regex is for use with e.g. --keep-within 1y + type = with types; attrsOf (either int (strMatching "[[:digit:]]+[Hdwmy]")); + description = '' + Prune a repository by deleting all archives not matching any of the + specified retention options. See <command>borg help prune</command> + for the available options. + ''; + default = { }; + example = literalExample '' + { + within = "1d"; # Keep all archives from the last day + daily = 7; + weekly = 4; + monthly = -1; # Keep at least one archive for each month + } + ''; + }; + + prune.prefix = mkOption { + type = types.str; + description = '' + Only consider archive names starting with this prefix for pruning. + By default, only archives created by this job are considered. + Use <literal>""</literal> to consider all archives. + ''; + default = config.archiveBaseName; + defaultText = "\${archiveBaseName}"; + }; + + environment = mkOption { + type = with types; attrsOf str; + description = '' + Environment variables passed to the backup script. + You can for example specify which SSH key to use. + ''; + default = { }; + example = { BORG_RSH = "ssh -i /path/to/key"; }; + }; + + preHook = mkOption { + type = types.lines; + description = '' + Shell commands to run before the backup. + This can for example be used to mount file systems. + ''; + default = ""; + example = '' + # To add excluded paths at runtime + extraCreateArgs="$extraCreateArgs --exclude /some/path" + ''; + }; + + postInit = mkOption { + type = types.lines; + description = '' + Shell commands to run after <command>borg init</command>. + ''; + default = ""; + }; + + postCreate = mkOption { + type = types.lines; + description = '' + Shell commands to run after <command>borg create</command>. The name + of the created archive is stored in <literal>$archiveName</literal>. + ''; + default = ""; + }; + + postPrune = mkOption { + type = types.lines; + description = '' + Shell commands to run after <command>borg prune</command>. + ''; + default = ""; + }; + + postHook = mkOption { + type = types.lines; + description = '' + Shell commands to run just before exit. They are executed + even if a previous command exits with a non-zero exit code. + The latter is available as <literal>$exitStatus</literal>. + ''; + default = ""; + }; + + extraInitArgs = mkOption { + type = types.str; + description = '' + Additional arguments for <command>borg init</command>. + Can also be set at runtime using <literal>$extraInitArgs</literal>. + ''; + default = ""; + example = "--append-only"; + }; + + extraCreateArgs = mkOption { + type = types.str; + description = '' + Additional arguments for <command>borg create</command>. + Can also be set at runtime using <literal>$extraCreateArgs</literal>. + ''; + default = ""; + example = "--stats --checkpoint-interval 600"; + }; + + extraPruneArgs = mkOption { + type = types.str; + description = '' + Additional arguments for <command>borg prune</command>. + Can also be set at runtime using <literal>$extraPruneArgs</literal>. + ''; + default = ""; + example = "--save-space"; + }; + + }; + } + )); + }; + + options.services.borgbackup.repos = mkOption { + description = '' + Serve BorgBackup repositories to given public SSH keys, + restricting their access to the repository only. + Also, clients do not need to specify the absolute path when accessing the repository, + i.e. <literal>user@machine:.</literal> is enough. (Note colon and dot.) + ''; + default = { }; + type = types.attrsOf (types.submodule ( + { name, config, ... }: { + options = { + + path = mkOption { + type = types.path; + description = '' + Where to store the backups. Note that the directory + is created automatically, with correct permissions. + ''; + default = "/var/lib/borgbackup"; + }; + + user = mkOption { + type = types.str; + description = '' + The user <command>borg serve</command> is run as. + User or group needs write permission + for the specified <option>path</option>. + ''; + default = "borg"; + }; + + group = mkOption { + type = types.str; + description = '' + The group <command>borg serve</command> is run as. + User or group needs write permission + for the specified <option>path</option>. + ''; + default = "borg"; + }; + + authorizedKeys = mkOption { + type = with types; listOf str; + description = '' + Public SSH keys that are given full write access to this repository. + You should use a different SSH key for each repository you write to, because + the specified keys are restricted to running <command>borg serve</command> + and can only access this single repository. + ''; + default = [ ]; + }; + + authorizedKeysAppendOnly = mkOption { + type = with types; listOf str; + description = '' + Public SSH keys that can only be used to append new data (archives) to the repository. + Note that archives can still be marked as deleted and are subsequently removed from disk + upon accessing the repo with full write access, e.g. when pruning. + ''; + default = [ ]; + }; + + allowSubRepos = mkOption { + type = types.bool; + description = '' + Allow clients to create repositories in subdirectories of the + specified <option>path</option>. These can be accessed using + <literal>user@machine:path/to/subrepo</literal>. Note that a + <option>quota</option> applies to repositories independently. + Therefore, if this is enabled, clients can create multiple + repositories and upload an arbitrary amount of data. + ''; + default = false; + }; + + quota = mkOption { + # See the definition of parse_file_size() in src/borg/helpers/parseformat.py + type = with types; nullOr (strMatching "[[:digit:].]+[KMGTP]?"); + description = '' + Storage quota for the repository. This quota is ensured for all + sub-repositories if <option>allowSubRepos</option> is enabled + but not for the overall storage space used. + ''; + default = null; + example = "100G"; + }; + + }; + } + )); + }; + + ###### implementation + + config = mkIf (with config.services.borgbackup; jobs != { } || repos != { }) + (with config.services.borgbackup; { + assertions = + mapAttrsToList mkPassAssertion jobs + ++ mapAttrsToList mkKeysAssertion repos; + + system.activationScripts = mapAttrs' mkActivationScript jobs; + + systemd.services = + # A job named "foo" is mapped to systemd.services.borgbackup-job-foo + mapAttrs' mkBackupService jobs + # A repo named "foo" is mapped to systemd.services.borgbackup-repo-foo + // mapAttrs' mkRepoService repos; + + users = mkMerge (mapAttrsToList mkUsersConfig repos); + + environment.systemPackages = with pkgs; [ borgbackup ]; + }); +} diff --git a/nixos/modules/services/backup/crashplan-small-business.nix b/nixos/modules/services/backup/crashplan-small-business.nix new file mode 100644 index 000000000000..9497d8c18bb7 --- /dev/null +++ b/nixos/modules/services/backup/crashplan-small-business.nix @@ -0,0 +1,74 @@ +{ config, pkgs, lib, ... }: + +let + cfg = config.services.crashplansb; + crashplansb = pkgs.crashplansb.override { maxRam = cfg.maxRam; }; + varDir = "/var/lib/crashplan"; +in + +with lib; + +{ + options = { + services.crashplansb = { + enable = mkOption { + default = false; + type = types.bool; + description = '' + Starts crashplan for small business background service. + ''; + }; + maxRam = mkOption { + default = "1024m"; + example = "2G"; + type = types.str; + description = '' + Maximum amount of ram that the crashplan engine should use. + ''; + }; + openPorts = mkOption { + description = "Open ports in the firewall for crashplan."; + default = true; + type = types.bool; + }; + ports = mkOption { + # https://support.code42.com/Administrator/6/Planning_and_installing/TCP_and_UDP_ports_used_by_the_Code42_platform + # used ports can also be checked in the desktop app console using the command connection.info + description = "which ports to open."; + default = [ 4242 4243 4244 4247 ]; + type = types.listOf types.int; + }; + }; + }; + + config = mkIf cfg.enable { + environment.systemPackages = [ crashplansb ]; + networking.firewall.allowedTCPPorts = mkIf cfg.openPorts cfg.ports; + + systemd.services.crashplansb = { + description = "CrashPlan Backup Engine"; + + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "local-fs.target" ]; + + preStart = '' + install -d -m 755 ${crashplansb.vardir} + install -d -m 700 ${crashplansb.vardir}/conf + install -d -m 700 ${crashplansb.manifestdir} + install -d -m 700 ${crashplansb.vardir}/cache + install -d -m 700 ${crashplansb.vardir}/backupArchives + install -d -m 777 ${crashplansb.vardir}/log + cp -avn ${crashplansb}/conf.template/* ${crashplansb.vardir}/conf + ''; + + serviceConfig = { + Type = "forking"; + EnvironmentFile = "${crashplansb}/bin/run.conf"; + ExecStart = "${crashplansb}/bin/CrashPlanEngine start"; + ExecStop = "${crashplansb}/bin/CrashPlanEngine stop"; + PIDFile = "${crashplansb.vardir}/CrashPlanEngine.pid"; + WorkingDirectory = crashplansb; + }; + }; + }; +} diff --git a/nixos/modules/services/backup/duplicati.nix b/nixos/modules/services/backup/duplicati.nix new file mode 100644 index 000000000000..9772ca4d20a7 --- /dev/null +++ b/nixos/modules/services/backup/duplicati.nix @@ -0,0 +1,40 @@ +{ config, pkgs, lib, ... }: + +with lib; + +let + cfg = config.services.duplicati; +in +{ + options = { + services.duplicati = { + enable = mkEnableOption "Duplicati"; + }; + }; + + config = mkIf cfg.enable { + environment.systemPackages = [ pkgs.duplicati ]; + + systemd.services.duplicati = { + description = "Duplicati backup"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + User = "duplicati"; + Group = "duplicati"; + ExecStart = "${pkgs.duplicati}/bin/duplicati-server --webservice-interface=any --webservice-port=8200 --server-datafolder=/var/lib/duplicati"; + Restart = "on-failure"; + }; + }; + + users.extraUsers.duplicati = { + uid = config.ids.uids.duplicati; + home = "/var/lib/duplicati"; + createHome = true; + group = "duplicati"; + }; + users.extraGroups.duplicati.gid = config.ids.gids.duplicati; + + }; +} + diff --git a/nixos/modules/services/backup/restic.nix b/nixos/modules/services/backup/restic.nix new file mode 100644 index 000000000000..21d82469c605 --- /dev/null +++ b/nixos/modules/services/backup/restic.nix @@ -0,0 +1,150 @@ +{ config, lib, pkgs, ... }: + +with lib; +{ + options.services.restic.backups = mkOption { + description = '' + Periodic backups to create with Restic. + ''; + type = types.attrsOf (types.submodule ({ name, config, ... }: { + options = { + passwordFile = mkOption { + type = types.str; + description = '' + Read the repository password from a file. + ''; + example = "/etc/nixos/restic-password"; + + }; + + repository = mkOption { + type = types.str; + description = '' + repository to backup to. + ''; + example = "sftp:backup@192.168.1.100:/backups/${name}"; + }; + + paths = mkOption { + type = types.listOf types.str; + default = []; + description = '' + Which paths to backup. + ''; + example = [ + "/var/lib/postgresql" + "/home/user/backup" + ]; + }; + + timerConfig = mkOption { + type = types.attrsOf types.str; + default = { + OnCalendar = "daily"; + }; + description = '' + When to run the backup. See man systemd.timer for details. + ''; + example = { + OnCalendar = "00:05"; + RandomizedDelaySec = "5h"; + }; + }; + + user = mkOption { + type = types.str; + default = "root"; + description = '' + As which user the backup should run. + ''; + example = "postgresql"; + }; + + extraBackupArgs = mkOption { + type = types.listOf types.str; + default = []; + description = '' + Extra arguments passed to restic backup. + ''; + example = [ + "--exclude-file=/etc/nixos/restic-ignore" + ]; + }; + + extraOptions = mkOption { + type = types.listOf types.str; + default = []; + description = '' + Extra extended options to be passed to the restic --option flag. + ''; + example = [ + "sftp.command='ssh backup@192.168.1.100 -i /home/user/.ssh/id_rsa -s sftp'" + ]; + }; + + initialize = mkOption { + type = types.bool; + default = false; + description = '' + Create the repository if it doesn't exist. + ''; + }; + }; + })); + default = {}; + example = { + localbackup = { + paths = [ "/home" ]; + repository = "/mnt/backup-hdd"; + passwordFile = "/etc/nixos/secrets/restic-password"; + initialize = true; + }; + remotebackup = { + paths = [ "/home" ]; + repository = "sftp:backup@host:/backups/home"; + passwordFile = "/etc/nixos/secrets/restic-password"; + extraOptions = [ + "sftp.command='ssh backup@host -i /etc/nixos/secrets/backup-private-key -s sftp'" + ]; + timerConfig = { + OnCalendar = "00:05"; + RandomizedDelaySec = "5h"; + }; + }; + }; + }; + + config = { + systemd.services = + mapAttrs' (name: backup: + let + extraOptions = concatMapStrings (arg: " -o ${arg}") backup.extraOptions; + connectTo = elemAt (splitString ":" backup.repository) 1; + resticCmd = "${pkgs.restic}/bin/restic${extraOptions}"; + in nameValuePair "restic-backups-${name}" ({ + environment = { + RESTIC_PASSWORD_FILE = backup.passwordFile; + RESTIC_REPOSITORY = backup.repository; + }; + path = with pkgs; [ + openssh + ]; + restartIfChanged = false; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${resticCmd} backup ${concatStringsSep " " backup.extraBackupArgs} ${concatStringsSep " " backup.paths}"; + User = backup.user; + }; + } // optionalAttrs backup.initialize { + preStart = '' + ${resticCmd} snapshots || ${resticCmd} init + ''; + }) + ) config.services.restic.backups; + systemd.timers = + mapAttrs' (name: backup: nameValuePair "restic-backups-${name}" { + wantedBy = [ "timers.target" ]; + timerConfig = backup.timerConfig; + }) config.services.restic.backups; + }; +} diff --git a/nixos/modules/services/backup/tarsnap.nix b/nixos/modules/services/backup/tarsnap.nix index 7c9dedb67ad2..4fc7c24813a5 100644 --- a/nixos/modules/services/backup/tarsnap.nix +++ b/nixos/modules/services/backup/tarsnap.nix @@ -115,7 +115,7 @@ in description = '' Print global archive statistics upon completion. The output is available via - <command>systemctl status tarsnap@archive-name</command>. + <command>systemctl status tarsnap-archive-name</command>. ''; }; @@ -238,6 +238,20 @@ in Whether to produce verbose logging output. ''; }; + explicitSymlinks = mkOption { + type = types.bool; + default = false; + description = '' + Whether to follow symlinks specified as archives. + ''; + }; + followSymlinks = mkOption { + type = types.bool; + default = false; + description = '' + Whether to follow all symlinks in archive trees. + ''; + }; }; } )); @@ -285,12 +299,12 @@ in }) gcfg.archives); systemd.services = - mapAttrs' (name: cfg: nameValuePair "tarsnap-${name}" { + (mapAttrs' (name: cfg: nameValuePair "tarsnap-${name}" { description = "Tarsnap archive '${name}'"; requires = [ "network-online.target" ]; after = [ "network-online.target" ]; - path = [ pkgs.iputils pkgs.tarsnap pkgs.utillinux ]; + path = with pkgs; [ iputils tarsnap utillinux ]; # In order for the persistent tarsnap timer to work reliably, we have to # make sure that the tarsnap server is reachable after systemd starts up @@ -300,10 +314,12 @@ in while ! ping -q -c 1 v1-0-0-server.tarsnap.com &> /dev/null; do sleep 3; done ''; - script = - let run = ''tarsnap --configfile "/etc/tarsnap/${name}.conf" \ - -c -f "${name}-$(date +"%Y%m%d%H%M%S")" \ + script = let + tarsnap = ''tarsnap --configfile "/etc/tarsnap/${name}.conf"''; + run = ''${tarsnap} -c -f "${name}-$(date +"%Y%m%d%H%M%S")" \ ${optionalString cfg.verbose "-v"} \ + ${optionalString cfg.explicitSymlinks "-H"} \ + ${optionalString cfg.followSymlinks "-L"} \ ${concatStringsSep " " cfg.directories}''; in if (cfg.cachedir != null) then '' mkdir -p ${cfg.cachedir} @@ -313,7 +329,7 @@ in if [ ! -e ${cfg.cachedir}/firstrun ]; then ( flock 10 flock -u 9 - tarsnap --configfile "/etc/tarsnap/${name}.conf" --fsck + ${tarsnap} --fsck flock 9 ) 10>${cfg.cachedir}/firstrun fi @@ -329,7 +345,44 @@ in CapabilityBoundingSet = [ "CAP_DAC_READ_SEARCH" ]; PermissionsStartOnly = "true"; }; - }) gcfg.archives; + }) gcfg.archives) // + + (mapAttrs' (name: cfg: nameValuePair "tarsnap-restore-${name}"{ + description = "Tarsnap restore '${name}'"; + requires = [ "network-online.target" ]; + + path = with pkgs; [ iputils tarsnap utillinux ]; + + script = let + tarsnap = ''tarsnap --configfile "/etc/tarsnap/${name}.conf"''; + lastArchive = ''$(${tarsnap} --list-archives | sort | tail -1)''; + run = ''${tarsnap} -x -f "${lastArchive}" ${optionalString cfg.verbose "-v"}''; + + in if (cfg.cachedir != null) then '' + mkdir -p ${cfg.cachedir} + chmod 0700 ${cfg.cachedir} + + ( flock 9 + if [ ! -e ${cfg.cachedir}/firstrun ]; then + ( flock 10 + flock -u 9 + ${tarsnap} --fsck + flock 9 + ) 10>${cfg.cachedir}/firstrun + fi + ) 9>${cfg.cachedir}/lockf + + exec flock ${cfg.cachedir}/firstrun ${run} + '' else "exec ${run}"; + + serviceConfig = { + Type = "oneshot"; + IOSchedulingClass = "idle"; + NoNewPrivileges = "true"; + CapabilityBoundingSet = [ "CAP_DAC_READ_SEARCH" ]; + PermissionsStartOnly = "true"; + }; + }) gcfg.archives); # Note: the timer must be Persistent=true, so that systemd will start it even # if e.g. your laptop was asleep while the latest interval occurred. diff --git a/nixos/modules/services/backup/znapzend.nix b/nixos/modules/services/backup/znapzend.nix index baf99930e3eb..3d133f82d204 100644 --- a/nixos/modules/services/backup/znapzend.nix +++ b/nixos/modules/services/backup/znapzend.nix @@ -1,39 +1,372 @@ { config, lib, pkgs, ... }: with lib; +with types; let + + # Converts a plan like + # { "1d" = "1h"; "1w" = "1d"; } + # into + # "1d=>1h,1w=>1d" + attrToPlan = attrs: concatStringsSep "," (builtins.attrValues ( + mapAttrs (n: v: "${n}=>${v}") attrs)); + + planDescription = '' + The znapzend backup plan to use for the source. + </para> + <para> + The plan specifies how often to backup and for how long to keep the + backups. It consists of a series of retention periodes to interval + associations: + </para> + <para> + <literal> + retA=>intA,retB=>intB,... + </literal> + </para> + <para> + Both intervals and retention periods are expressed in standard units + of time or multiples of them. You can use both the full name or a + shortcut according to the following listing: + </para> + <para> + <literal> + second|sec|s, minute|min, hour|h, day|d, week|w, month|mon|m, year|y + </literal> + </para> + <para> + See <citerefentry><refentrytitle>znapzendzetup</refentrytitle><manvolnum>1</manvolnum></citerefentry> for more info. + ''; + planExample = "1h=>10min,1d=>1h,1w=>1d,1m=>1w,1y=>1m"; + + # A type for a string of the form number{b|k|M|G} + mbufferSizeType = str // { + check = x: str.check x && builtins.isList (builtins.match "^[0-9]+[bkMG]$" x); + description = "string of the form number{b|k|M|G}"; + }; + + # Type for a string that must contain certain other strings (the list parameter). + # Note that these would need regex escaping. + stringContainingStrings = list: let + matching = s: map (str: builtins.match ".*${str}.*" s) list; + in str // { + check = x: str.check x && all isList (matching x); + description = "string containing all of the characters ${concatStringsSep ", " list}"; + }; + + timestampType = stringContainingStrings [ "%Y" "%m" "%d" "%H" "%M" "%S" ]; + + destType = srcConfig: submodule ({ name, ... }: { + options = { + + label = mkOption { + type = str; + description = "Label for this destination. Defaults to the attribute name."; + }; + + plan = mkOption { + type = str; + description = planDescription; + example = planExample; + }; + + dataset = mkOption { + type = str; + description = "Dataset name to send snapshots to."; + example = "tank/main"; + }; + + host = mkOption { + type = nullOr str; + description = '' + Host to use for the destination dataset. Can be prefixed with + <literal>user@</literal> to specify the ssh user. + ''; + default = null; + example = "john@example.com"; + }; + + presend = mkOption { + type = nullOr str; + description = '' + Command to run before sending the snapshot to the destination. + Intended to run a remote script via <command>ssh</command> on the + destination, e.g. to bring up a backup disk or server or to put a + zpool online/offline. See also <option>postsend</option>. + ''; + default = null; + example = "ssh root@bserv zpool import -Nf tank"; + }; + + postsend = mkOption { + type = nullOr str; + description = '' + Command to run after sending the snapshot to the destination. + Intended to run a remote script via <command>ssh</command> on the + destination, e.g. to bring up a backup disk or server or to put a + zpool online/offline. See also <option>presend</option>. + ''; + default = null; + example = "ssh root@bserv zpool export tank"; + }; + }; + + config = { + label = mkDefault name; + plan = mkDefault srcConfig.plan; + }; + }); + + + + srcType = submodule ({ name, config, ... }: { + options = { + + enable = mkOption { + type = bool; + description = "Whether to enable this source."; + default = true; + }; + + recursive = mkOption { + type = bool; + description = "Whether to do recursive snapshots."; + default = false; + }; + + mbuffer = { + enable = mkOption { + type = bool; + description = "Whether to use <command>mbuffer</command>."; + default = false; + }; + + port = mkOption { + type = nullOr ints.u16; + description = '' + Port to use for <command>mbuffer</command>. + </para> + <para> + If this is null, it will run <command>mbuffer</command> through + ssh. + </para> + <para> + If this is not null, it will run <command>mbuffer</command> + directly through TCP, which is not encrypted but faster. In that + case the given port needs to be open on the destination host. + ''; + default = null; + }; + + size = mkOption { + type = mbufferSizeType; + description = '' + The size for <command>mbuffer</command>. + Supports the units b, k, M, G. + ''; + default = "1G"; + example = "128M"; + }; + }; + + presnap = mkOption { + type = nullOr str; + description = '' + Command to run before snapshots are taken on the source dataset, + e.g. for database locking/flushing. See also + <option>postsnap</option>. + ''; + default = null; + example = literalExample '' + ''${pkgs.mariadb}/bin/mysql -e "set autocommit=0;flush tables with read lock;\\! ''${pkgs.coreutils}/bin/sleep 600" & ''${pkgs.coreutils}/bin/echo $! > /tmp/mariadblock.pid ; sleep 10 + ''; + }; + + postsnap = mkOption { + type = nullOr str; + description = '' + Command to run after snapshots are taken on the source dataset, + e.g. for database unlocking. See also <option>presnap</option>. + ''; + default = null; + example = literalExample '' + ''${pkgs.coreutils}/bin/kill `''${pkgs.coreutils}/bin/cat /tmp/mariadblock.pid`;''${pkgs.coreutils}/bin/rm /tmp/mariadblock.pid + ''; + }; + + timestampFormat = mkOption { + type = timestampType; + description = '' + The timestamp format to use for constructing snapshot names. + The syntax is <literal>strftime</literal>-like. The string must + consist of the mandatory <literal>%Y %m %d %H %M %S</literal>. + Optionally <literal>- _ . :</literal> characters as well as any + alphanumeric character are allowed. If suffixed by a + <literal>Z</literal>, times will be in UTC. + ''; + default = "%Y-%m-%d-%H%M%S"; + example = "znapzend-%m.%d.%Y-%H%M%SZ"; + }; + + sendDelay = mkOption { + type = int; + description = '' + Specify delay (in seconds) before sending snaps to the destination. + May be useful if you want to control sending time. + ''; + default = 0; + example = 60; + }; + + plan = mkOption { + type = str; + description = planDescription; + example = planExample; + }; + + dataset = mkOption { + type = str; + description = "The dataset to use for this source."; + example = "tank/home"; + }; + + destinations = mkOption { + type = loaOf (destType config); + description = "Additional destinations."; + default = {}; + example = literalExample '' + { + local = { + dataset = "btank/backup"; + presend = "zpool import -N btank"; + postsend = "zpool export btank"; + }; + remote = { + host = "john@example.com"; + dataset = "tank/john"; + }; + }; + ''; + }; + }; + + config = { + dataset = mkDefault name; + }; + + }); + + ### Generating the configuration from here + cfg = config.services.znapzend; + + onOff = b: if b then "on" else "off"; + nullOff = b: if isNull b then "off" else toString b; + stripSlashes = replaceStrings [ "/" ] [ "." ]; + + attrsToFile = config: concatStringsSep "\n" (builtins.attrValues ( + mapAttrs (n: v: "${n}=${v}") config)); + + mkDestAttrs = dst: with dst; + mapAttrs' (n: v: nameValuePair "dst_${label}${n}" v) ({ + "" = optionalString (! isNull host) "${host}:" + dataset; + _plan = plan; + } // optionalAttrs (presend != null) { + _precmd = presend; + } // optionalAttrs (postsend != null) { + _pstcmd = postsend; + }); + + mkSrcAttrs = srcCfg: with srcCfg; { + enabled = onOff enable; + mbuffer = with mbuffer; if enable then "${pkgs.mbuffer}/bin/mbuffer" + + optionalString (port != null) ":${toString port}" else "off"; + mbuffer_size = mbuffer.size; + post_znap_cmd = nullOff postsnap; + pre_znap_cmd = nullOff presnap; + recursive = onOff recursive; + src = dataset; + src_plan = plan; + tsformat = timestampFormat; + zend_delay = toString sendDelay; + } // fold (a: b: a // b) {} ( + map mkDestAttrs (builtins.attrValues destinations) + ); + + files = mapAttrs' (n: srcCfg: let + fileText = attrsToFile (mkSrcAttrs srcCfg); + in { + name = srcCfg.dataset; + value = pkgs.writeText (stripSlashes srcCfg.dataset) fileText; + }) cfg.zetup; + in { options = { services.znapzend = { - enable = mkEnableOption "ZnapZend daemon"; + enable = mkEnableOption "ZnapZend ZFS backup daemon"; logLevel = mkOption { default = "debug"; example = "warning"; - type = lib.types.enum ["debug" "info" "warning" "err" "alert"]; - description = "The log level when logging to file. Any of debug, info, warning, err, alert. Default in daemonized form is debug."; + type = enum ["debug" "info" "warning" "err" "alert"]; + description = '' + The log level when logging to file. Any of debug, info, warning, err, + alert. Default in daemonized form is debug. + ''; }; logTo = mkOption { - type = types.str; + type = str; default = "syslog::daemon"; example = "/var/log/znapzend.log"; - description = "Where to log to (syslog::<facility> or <filepath>)."; + description = '' + Where to log to (syslog::<facility> or <filepath>). + ''; }; noDestroy = mkOption { - type = types.bool; + type = bool; default = false; description = "Does all changes to the filesystem except destroy."; }; autoCreation = mkOption { - type = types.bool; + type = bool; + default = false; + description = "Automatically create the destination dataset if it does not exists."; + }; + + zetup = mkOption { + type = loaOf srcType; + description = "Znapzend configuration."; + default = {}; + example = literalExample '' + { + "tank/home" = { + # Make snapshots of tank/home every hour, keep those for 1 day, + # keep every days snapshot for 1 month, etc. + plan = "1d=>1h,1m=>1d,1y=>1m"; + recursive = true; + # Send all those snapshots to john@example.com:rtank/john as well + destinations.remote = { + host = "john@example.com"; + dataset = "rtank/john"; + }; + }; + }; + ''; + }; + + pure = mkOption { + type = bool; + description = '' + Do not persist any stateful znapzend setups. If this option is + enabled, your previously set znapzend setups will be cleared and only + the ones defined with this module will be applied. + ''; default = false; - description = "Automatically create the dataset on dest if it does not exists."; }; }; }; @@ -49,12 +382,30 @@ in path = with pkgs; [ zfs mbuffer openssh ]; + preStart = optionalString cfg.pure '' + echo Resetting znapzend zetups + ${pkgs.znapzend}/bin/znapzendzetup list \ + | grep -oP '(?<=\*\*\* backup plan: ).*(?= \*\*\*)' \ + | xargs -I{} ${pkgs.znapzend}/bin/znapzendzetup delete "{}" + '' + concatStringsSep "\n" (mapAttrsToList (dataset: config: '' + echo Importing znapzend zetup ${config} for dataset ${dataset} + ${pkgs.znapzend}/bin/znapzendzetup import --write ${dataset} ${config} + '') files); + serviceConfig = { - ExecStart = "${pkgs.znapzend}/bin/znapzend --logto=${cfg.logTo} --loglevel=${cfg.logLevel} ${optionalString cfg.noDestroy "--nodestroy"} ${optionalString cfg.autoCreation "--autoCreation"}"; + ExecStart = let + args = concatStringsSep " " [ + "--logto=${cfg.logTo}" + "--loglevel=${cfg.logLevel}" + (optionalString cfg.noDestroy "--nodestroy") + (optionalString cfg.autoCreation "--autoCreation") + ]; in "${pkgs.znapzend}/bin/znapzend ${args}"; ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; Restart = "on-failure"; }; }; }; }; + + meta.maintainers = with maintainers; [ infinisil ]; } |