From fdf0f037be55c5c14e24667b1ad7eeedf2057295 Mon Sep 17 00:00:00 2001 From: Robert Schütz Date: Mon, 12 Mar 2018 20:20:24 +0100 Subject: nixos/borgbackup: init --- nixos/modules/module-list.nix | 1 + nixos/modules/services/backup/borgbackup.nix | 580 +++++++++++++++++++++++++++ 2 files changed, 581 insertions(+) create mode 100644 nixos/modules/services/backup/borgbackup.nix (limited to 'nixos/modules') diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 3bb65c6b295a..aa7806a7a9e4 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -159,6 +159,7 @@ ./services/audio/ympd.nix ./services/backup/almir.nix ./services/backup/bacula.nix + ./services/backup/borgbackup.nix ./services/backup/crashplan.nix ./services/backup/crashplan-small-business.nix ./services/backup/mysql-backup.nix diff --git a/nixos/modules/services/backup/borgbackup.nix b/nixos/modules/services/backup/borgbackup.nix new file mode 100644 index 000000000000..1b730e0c2b76 --- /dev/null +++ b/nixos/modules/services/backup/borgbackup.nix @@ -0,0 +1,580 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + isLocalPath = x: + builtins.substring 0 1 x == "/" # absolute path + || builtins.substring 0 1 x == "." # relative path + || builtins.match "[.*:.*]" == null; # not machine:path + + mkExcludeFile = cfg: + # Write each exclude pattern to a new line + pkgs.writeText "excludefile" (concatStringsSep "\n" cfg.exclude); + + mkKeepArgs = cfg: + # If cfg.prune.keep e.g. has a yearly attribute, + # its content is passed on as --keep-yearly + concatStringsSep " " + (mapAttrsToList (x: y: "--keep-${x}=${toString y}") cfg.prune.keep); + + mkBackupScript = cfg: '' + on_exit() + { + exitStatus=$? + # Reset the EXIT handler, or else we're called again on 'exit' below + trap - EXIT + ${cfg.postHook} + exit $exitStatus + } + trap 'on_exit' INT TERM QUIT EXIT + + archiveName="${cfg.archiveBaseName}-$(date ${cfg.dateFormat})" + archiveSuffix="${optionalString cfg.appendFailedSuffix ".failed"}" + ${cfg.preHook} + '' + optionalString cfg.doInit '' + # Run borg init if the repo doesn't exist yet + if ! borg list > /dev/null; then + borg init \ + --encryption ${cfg.encryption.mode} \ + $extraInitArgs + ${cfg.postInit} + fi + '' + '' + borg create \ + --compression ${cfg.compression} \ + --exclude-from ${mkExcludeFile cfg} \ + $extraCreateArgs \ + "::$archiveName$archiveSuffix" \ + ${escapeShellArgs cfg.paths} + '' + optionalString cfg.appendFailedSuffix '' + borg rename "::$archiveName$archiveSuffix" "$archiveName" + '' + '' + ${cfg.postCreate} + '' + optionalString (cfg.prune.keep != { }) '' + borg prune \ + ${mkKeepArgs cfg} \ + --prefix ${escapeShellArg cfg.prune.prefix} \ + $extraPruneArgs + ${cfg.postPrune} + ''; + + mkPassEnv = cfg: with cfg.encryption; + if passCommand != null then + { BORG_PASSCOMMAND = passCommand; } + else if passphrase != null then + { BORG_PASSPHRASE = passphrase; } + else { }; + + mkBackupService = name: cfg: + let + userHome = config.users.users.${cfg.user}.home; + in nameValuePair "borgbackup-job-${name}" { + description = "BorgBackup job ${name}"; + path = with pkgs; [ + borgbackup openssh + ]; + script = mkBackupScript cfg; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + # Only run when no other process is using CPU or disk + CPUSchedulingPolicy = "idle"; + IOSchedulingClass = "idle"; + ProtectSystem = "strict"; + ReadWritePaths = + [ "${userHome}/.config/borg" "${userHome}/.cache/borg" ] + # Borg needs write access to repo if it is not remote + ++ optional (isLocalPath cfg.repo) cfg.repo; + PrivateTmp = true; + }; + environment = { + BORG_REPO = cfg.repo; + inherit (cfg) extraInitArgs extraCreateArgs extraPruneArgs; + } // (mkPassEnv cfg) // cfg.environment; + inherit (cfg) startAt; + }; + + # Paths listed in ReadWritePaths must exist before service is started + mkActivationScript = name: cfg: + let + install = "install -o ${cfg.user} -g ${cfg.group}"; + in + nameValuePair "borgbackup-job-${name}" (stringAfter [ "users" ] ('' + # Eensure that the home directory already exists + # We can't assert createHome == true because that's not the case for root + cd "${config.users.users.${cfg.user}.home}" + ${install} -d .config/borg + ${install} -d .cache/borg + '' + optionalString (isLocalPath cfg.repo) '' + ${install} -d ${escapeShellArg cfg.repo} + '')); + + mkPassAssertion = name: cfg: { + assertion = with cfg.encryption; + mode != "none" -> passCommand != null || passphrase != null; + message = + "passCommand or passphrase has to be specified because" + + '' borgbackup.jobs.${name}.encryption != "none"''; + }; + + mkRepoService = name: cfg: + nameValuePair "borgbackup-repo-${name}" { + description = "Create BorgBackup repository ${name} directory"; + script = '' + mkdir -p ${escapeShellArg cfg.path} + chown ${cfg.user}:${cfg.group} ${escapeShellArg cfg.path} + ''; + serviceConfig = { + # The service's only task is to ensure that the specified path exists + Type = "oneshot"; + }; + wantedBy = [ "multi-user.target" ]; + }; + + mkAuthorizedKey = cfg: appendOnly: key: + let + # Because of the following line, clients do not need to specify an absolute repo path + cdCommand = "cd ${escapeShellArg cfg.path}"; + restrictedArg = "--restrict-to-${if cfg.allowSubRepos then "path" else "repository"} ."; + appendOnlyArg = optionalString appendOnly "--append-only"; + quotaArg = optionalString (cfg.quota != null) "--storage-quota ${cfg.quota}"; + serveCommand = "borg serve ${restrictedArg} ${appendOnlyArg} ${quotaArg}"; + in + ''command="${cdCommand} && ${serveCommand}",restrict ${key}''; + + mkUsersConfig = name: cfg: { + users.${cfg.user} = { + openssh.authorizedKeys.keys = + (map (mkAuthorizedKey cfg false) cfg.authorizedKeys + ++ map (mkAuthorizedKey cfg true) cfg.authorizedKeysAppendOnly); + useDefaultShell = true; + }; + groups.${cfg.group} = { }; + }; + + mkKeysAssertion = name: cfg: { + assertion = cfg.authorizedKeys != [ ] || cfg.authorizedKeysAppendOnly != [ ]; + message = + "borgbackup.repos.${name} does not make sense" + + " without at least one public key"; + }; + +in { + meta.maintainers = with maintainers; [ dotlambda ]; + + ###### interface + + options.services.borgbackup.jobs = mkOption { + description = "Deduplicating backups using BorgBackup."; + default = { }; + example = literalExample '' + { + rootBackup = { + paths = "/"; + exclude = [ "/nix" ]; + repo = "/path/to/local/repo"; + encryption = { + mode = "repokey"; + passphrase = "secret"; + }; + compression = "auto,lzma"; + startAt = "weekly"; + }; + } + ''; + type = types.attrsOf (types.submodule (let globalConfig = config; in + { name, config, ... }: { + options = { + + paths = mkOption { + type = with types; either path (nonEmptyListOf path); + description = "Path(s) to back up."; + example = "/home/user"; + apply = x: if isList x then x else [ x ]; + }; + + repo = mkOption { + type = types.str; + description = "Remote or local repository to back up to."; + example = "user@machine:/path/to/repo"; + }; + + archiveBaseName = mkOption { + type = types.strMatching "[^/{}]+"; + default = "${globalConfig.networking.hostName}-${name}"; + defaultText = "\${config.networking.hostName}-"; + description = '' + How to name the created archives. A timestamp, whose format is + determined by , will be appended. The full + name can be modified at runtime ($archiveName). + Placeholders like {hostname} must not be used. + ''; + }; + + dateFormat = mkOption { + type = types.str; + description = '' + Arguments passed to date + to create a timestamp suffix for the archive name. + ''; + default = "+%Y-%m-%dT%H:%M:%S"; + example = "-u +%s"; + }; + + startAt = mkOption { + type = with types; either str (listOf str); + default = "daily"; + description = '' + When or how often the backup should run. + Must be in the format described in + systemd.time + 7. + If you do not want the backup to start + automatically, use [ ]. + ''; + }; + + user = mkOption { + type = types.str; + description = '' + The user borg is run as. + User or group need read permission + for the specified . + ''; + default = "root"; + }; + + group = mkOption { + type = types.str; + description = '' + The group borg is run as. User or group needs read permission + for the specified . + ''; + default = "root"; + }; + + encryption.mode = mkOption { + type = types.enum [ + "repokey" "keyfile" + "repokey-blake2" "keyfile-blake2" + "authenticated" "authenticated-blake2" + "none" + ]; + description = '' + Encryption mode to use. Setting a mode + other than "none" requires + you to specify a + or a . + ''; + }; + + encryption.passCommand = mkOption { + type = with types; nullOr str; + description = '' + A command which prints the passphrase to stdout. + Mutually exclusive with . + ''; + default = null; + example = "cat /path/to/passphrase_file"; + }; + + encryption.passphrase = mkOption { + type = with types; nullOr str; + description = '' + The passphrase the backups are encrypted with. + Mutually exclusive with . + If you do not want the passphrase to be stored in the + world-readable Nix store, use . + ''; + default = null; + }; + + compression = mkOption { + # "auto" is optional, + # compression mode must be given, + # compression level is optional + type = types.strMatching "none|(auto,)?(lz4|zstd|zlib|lzma)(,[[:digit:]]{1,2})?"; + description = '' + Compression method to use. Refer to + borg help compression + for all available options. + ''; + default = "lz4"; + example = "auto,lzma"; + }; + + exclude = mkOption { + type = with types; listOf str; + description = '' + Exclude paths matching any of the given patterns. See + borg help patterns for pattern syntax. + ''; + default = [ ]; + example = [ + "/home/*/.cache" + "/nix" + ]; + }; + + doInit = mkOption { + type = types.bool; + description = '' + Run borg init if the + specified does not exist. + You should set this to false + if the repository is located on an external drive + that might not always be mounted. + ''; + default = true; + }; + + appendFailedSuffix = mkOption { + type = types.bool; + description = '' + Append a .failed suffix + to the archive name, which is only removed if + borg create has a zero exit status. + ''; + default = true; + }; + + prune.keep = mkOption { + # Specifying e.g. `prune.keep.yearly = -1` + # means there is no limit of yearly archives to keep + # The regex is for use with e.g. --keep-within 1y + type = with types; attrsOf (either int (strMatching "[[:digit:]]+[Hdwmy]")); + description = '' + Prune a repository by deleting all archives not matching any of the + specified retention options. See borg help prune + for the available options. + ''; + default = { }; + example = literalExample '' + { + within = "1d"; # Keep all archives from the last day + daily = 7; + weekly = 4; + monthly = -1; # Keep at least one archive for each month + } + ''; + }; + + prune.prefix = mkOption { + type = types.str; + description = '' + Only consider archive names starting with this prefix for pruning. + By default, only archives created by this job are considered. + Use "" to consider all archives. + ''; + default = config.archiveBaseName; + defaultText = "\${archiveBaseName}"; + }; + + environment = mkOption { + type = with types; attrsOf str; + description = '' + Environment variables passed to the backup script. + You can for example specify which SSH key to use. + ''; + default = { }; + example = { BORG_RSH = "ssh -i /path/to/key"; }; + }; + + preHook = mkOption { + type = types.lines; + description = '' + Shell commands to run before the backup. + This can for example be used to mount file systems. + ''; + default = ""; + example = '' + # To add excluded paths at runtime + extraCreateArgs="$extraCreateArgs --exclude /some/path" + ''; + }; + + postInit = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg init. + ''; + default = ""; + }; + + postCreate = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg create. The name + of the created archive is stored in $archiveName. + ''; + default = ""; + }; + + postPrune = mkOption { + type = types.lines; + description = '' + Shell commands to run after borg prune. + ''; + default = ""; + }; + + postHook = mkOption { + type = types.lines; + description = '' + Shell commands to run just before exit. They are executed + even if a previous command exits with a non-zero exit code. + The latter is available as $exitStatus. + ''; + default = ""; + }; + + extraInitArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg init. + Can also be set at runtime using $extraInitArgs. + ''; + default = ""; + example = "--append-only"; + }; + + extraCreateArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg create. + Can also be set at runtime using $extraCreateArgs. + ''; + default = ""; + example = "--stats --checkpoint-interval 600"; + }; + + extraPruneArgs = mkOption { + type = types.str; + description = '' + Additional arguments for borg prune. + Can also be set at runtime using $extraPruneArgs. + ''; + default = ""; + example = "--save-space"; + }; + + }; + } + )); + }; + + options.services.borgbackup.repos = mkOption { + description = '' + Serve BorgBackup repositories to given public SSH keys, + restricting their access to the repository only. + Also, clients do not need to specify the absolute path when accessing the repository, + i.e. user@machine:. is enough. (Note colon and dot.) + ''; + default = { }; + type = types.attrsOf (types.submodule ( + { name, config, ... }: { + options = { + + path = mkOption { + type = types.path; + description = '' + Where to store the backups. Note that the directory + is created automatically, with correct permissions. + ''; + default = "/var/lib/borgbackup"; + }; + + user = mkOption { + type = types.str; + description = '' + The user borg serve is run as. + User or group needs write permission + for the specified . + ''; + default = "borg"; + }; + + group = mkOption { + type = types.str; + description = '' + The group borg serve is run as. + User or group needs write permission + for the specified . + ''; + default = "borg"; + }; + + authorizedKeys = mkOption { + type = with types; listOf str; + description = '' + Public SSH keys that are given full write access to this repository. + You should use a different SSH key for each repository you write to, because + the specified keys are restricted to running borg serve + and can only access this single repository. + ''; + default = [ ]; + }; + + authorizedKeysAppendOnly = mkOption { + type = with types; listOf str; + description = '' + Public SSH keys that can only be used to append new data (archives) to the repository. + Note that archives can still be marked as deleted and are subsequently removed from disk + upon accessing the repo with full write access, e.g. when pruning. + ''; + default = [ ]; + }; + + allowSubRepos = mkOption { + type = types.bool; + description = '' + Allow clients to create repositories in subdirectories of the + specified . These can be accessed using + user@machine:path/to/subrepo. Note that a + applies to repositories independently. + Therefore, if this is enabled, clients can create multiple + repositories and upload an arbitrary amount of data. + ''; + default = false; + }; + + quota = mkOption { + # See the definition of parse_file_size() in src/borg/helpers/parseformat.py + type = with types; nullOr (strMatching "[[:digit:].]+[KMGTP]?"); + description = '' + Storage quota for the repository. This quota is ensured for all + sub-repositories if is enabled + but not for the overall storage space used. + ''; + default = null; + example = "100G"; + }; + + }; + } + )); + }; + + ###### implementation + + config = mkIf (with config.services.borgbackup; jobs != { } || repos != { }) + (with config.services.borgbackup; { + assertions = + mapAttrsToList mkPassAssertion jobs + ++ mapAttrsToList mkKeysAssertion repos; + + system.activationScripts = mapAttrs' mkActivationScript jobs; + + systemd.services = + # A job named "foo" is mapped to systemd.services.borgbackup-job-foo + mapAttrs' mkBackupService jobs + # A repo named "foo" is mapped to systemd.services.borgbackup-repo-foo + // mapAttrs' mkRepoService repos; + + users = mkMerge (mapAttrsToList mkUsersConfig repos); + + environment.systemPackages = with pkgs; [ borgbackup ]; + }); +} -- cgit 1.4.1