From acd0d41b3a02f22a70aa22b9476097ece35a9eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyryl=20P=C5=82otnicki?= Date: Sat, 2 Jul 2022 09:56:25 +0100 Subject: [PATCH] Use custom restic backup module --- nixos/backups.nix | 6 + nixos/services/backup/restic.nix | 352 +++++++++++++++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 nixos/services/backup/restic.nix diff --git a/nixos/backups.nix b/nixos/backups.nix index d8bd51a5..57cbd0a4 100644 --- a/nixos/backups.nix +++ b/nixos/backups.nix @@ -9,6 +9,8 @@ let ]; in { + disabledModules = [ "services/backup/restic.nix" ]; + imports = [ services/backup/restic.nix ]; services = { restic.backups.home-to-bolty = { passwordFile = "/etc/nixos/secrets/restic-password-bolty"; @@ -16,6 +18,8 @@ in { repository = "rest:http://bolty:8000/"; timerConfig = { OnCalendar = "hourly"; }; extraBackupArgs = extraArgs; + niceness = 19; + ioSchedulingClass = "idle"; }; restic.backups.home-to-b2 = { @@ -25,6 +29,8 @@ in { timerConfig = { OnCalendar = "hourly"; }; extraBackupArgs = extraArgs; environmentFile = "/etc/nixos/secrets/b2-env"; + niceness = 19; + ioSchedulingClass = "idle"; }; }; } diff --git a/nixos/services/backup/restic.nix b/nixos/services/backup/restic.nix new file mode 100644 index 00000000..e55161eb --- /dev/null +++ b/nixos/services/backup/restic.nix @@ -0,0 +1,352 @@ +{ config, lib, pkgs, utils, ... }: + +with lib; + +let + # Type for a valid systemd unit option. Needed for correctly passing "timerConfig" to "systemd.timers" + inherit (utils.systemdUtils.unitOptions) unitOption; +in { + options.services.restic.backups = mkOption { + description = '' + Periodic backups to create with Restic. + ''; + type = types.attrsOf (types.submodule ({ config, name, ... }: { + options = { + passwordFile = mkOption { + type = types.str; + description = '' + Read the repository password from a file. + ''; + example = "/etc/nixos/restic-password"; + }; + + environmentFile = mkOption { + type = with types; nullOr str; + # added on 2021-08-28, s3CredentialsFile should + # be removed in the future (+ remember the warning) + default = config.s3CredentialsFile; + description = '' + file containing the credentials to access the repository, in the + format of an EnvironmentFile as described by systemd.exec(5) + ''; + }; + + s3CredentialsFile = mkOption { + type = with types; nullOr str; + default = null; + description = '' + file containing the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY + for an S3-hosted repository, in the format of an EnvironmentFile + as described by systemd.exec(5) + ''; + }; + + rcloneOptions = mkOption { + type = with types; nullOr (attrsOf (oneOf [ str bool ])); + default = null; + description = '' + Options to pass to rclone to control its behavior. + See for + available options. When specifying option names, strip the + leading --. To set a flag such as + --drive-use-trash, which does not take a value, + set the value to the Boolean true. + ''; + example = { + bwlimit = "10M"; + drive-use-trash = "true"; + }; + }; + + rcloneConfig = mkOption { + type = with types; nullOr (attrsOf (oneOf [ str bool ])); + default = null; + description = '' + Configuration for the rclone remote being used for backup. + See the remote's specific options under rclone's docs at + . When specifying + option names, use the "config" name specified in the docs. + For example, to set --b2-hard-delete for a B2 + remote, use hard_delete = true in the + attribute set. + Warning: Secrets set in here will be world-readable in the Nix + store! Consider using the rcloneConfigFile + option instead to specify secret values separately. Note that + options set here will override those set in the config file. + ''; + example = { + type = "b2"; + account = "xxx"; + key = "xxx"; + hard_delete = true; + }; + }; + + rcloneConfigFile = mkOption { + type = with types; nullOr path; + default = null; + description = '' + Path to the file containing rclone configuration. This file + must contain configuration for the remote specified in this backup + set and also must be readable by root. Options set in + rcloneConfig will override those set in this + file. + ''; + }; + + repository = mkOption { + type = with types; nullOr str; + default = null; + description = '' + repository to backup to. + ''; + example = "sftp:backup@192.168.1.100:/backups/${name}"; + }; + + repositoryFile = mkOption { + type = with types; nullOr path; + default = null; + description = '' + Path to the file containing the repository location to backup to. + ''; + }; + + paths = mkOption { + type = types.nullOr (types.listOf types.str); + default = null; + description = '' + Which paths to backup. If null or an empty array, no + backup command will be run. This can be used to create a + prune-only job. + ''; + example = [ "/var/lib/postgresql" "/home/user/backup" ]; + }; + + timerConfig = mkOption { + type = types.attrsOf unitOption; + default = { OnCalendar = "daily"; }; + description = '' + When to run the backup. See man systemd.timer for details. + ''; + example = { + OnCalendar = "00:05"; + RandomizedDelaySec = "5h"; + }; + }; + + user = mkOption { + type = types.str; + default = "root"; + description = '' + As which user the backup should run. + ''; + example = "postgresql"; + }; + + extraBackupArgs = mkOption { + type = types.listOf types.str; + default = [ ]; + description = '' + Extra arguments passed to restic backup. + ''; + example = [ "--exclude-file=/etc/nixos/restic-ignore" ]; + }; + + extraOptions = mkOption { + type = types.listOf types.str; + default = [ ]; + description = '' + Extra extended options to be passed to the restic --option flag. + ''; + example = [ + "sftp.command='ssh backup@192.168.1.100 -i /home/user/.ssh/id_rsa -s sftp'" + ]; + }; + + initialize = mkOption { + type = types.bool; + default = false; + description = '' + Create the repository if it doesn't exist. + ''; + }; + + pruneOpts = mkOption { + type = types.listOf types.str; + default = [ ]; + description = '' + A list of options (--keep-* et al.) for 'restic forget + --prune', to automatically prune old snapshots. The + 'forget' command is run *after* the 'backup' command, so + keep that in mind when constructing the --keep-* options. + ''; + example = [ + "--keep-daily 7" + "--keep-weekly 5" + "--keep-monthly 12" + "--keep-yearly 75" + ]; + }; + + dynamicFilesFrom = mkOption { + type = with types; nullOr str; + default = null; + description = '' + A script that produces a list of files to back up. The + results of this command are given to the '--files-from' + option. + ''; + example = "find /home/matt/git -type d -name .git"; + }; + + backupPrepareCommand = mkOption { + type = with types; nullOr str; + default = null; + description = '' + A script that must run before starting the backup process. + ''; + }; + + backupCleanupCommand = mkOption { + type = with types; nullOr str; + default = null; + description = '' + A script that must run after finishing the backup process. + ''; + }; + niceness = mkOption { + description = + "Niceness for local instances of btrbk. Also applies to remote ones connecting via ssh when positive."; + type = types.ints.between (-20) 19; + default = 10; + }; + ioSchedulingClass = mkOption { + description = + "IO scheduling class for btrbk (see ionice(1) for a quick description). Applies to local instances, and remote ones connecting by ssh if set to idle."; + type = types.enum [ "idle" "best-effort" "realtime" ]; + default = "best-effort"; + }; + }; + })); + default = { }; + example = { + localbackup = { + paths = [ "/home" ]; + repository = "/mnt/backup-hdd"; + passwordFile = "/etc/nixos/secrets/restic-password"; + initialize = true; + }; + remotebackup = { + paths = [ "/home" ]; + repository = "sftp:backup@host:/backups/home"; + passwordFile = "/etc/nixos/secrets/restic-password"; + extraOptions = [ + "sftp.command='ssh backup@host -i /etc/nixos/secrets/backup-private-key -s sftp'" + ]; + timerConfig = { + OnCalendar = "00:05"; + RandomizedDelaySec = "5h"; + }; + }; + }; + }; + + config = { + warnings = mapAttrsToList (n: v: + "services.restic.backups.${n}.s3CredentialsFile is deprecated, please use services.restic.backups.${n}.environmentFile instead.") + (filterAttrs (n: v: v.s3CredentialsFile != null) + config.services.restic.backups); + systemd.services = mapAttrs' (name: backup: + let + extraOptions = concatMapStrings (arg: " -o ${arg}") backup.extraOptions; + resticCmd = "${pkgs.restic}/bin/restic${extraOptions}"; + filesFromTmpFile = "/run/restic-backups-${name}/includes"; + backupPaths = if (backup.dynamicFilesFrom == null) then + if (backup.paths != null) then + concatStringsSep " " backup.paths + else + "" + else + "--files-from ${filesFromTmpFile}"; + pruneCmd = optionals (builtins.length backup.pruneOpts > 0) [ + (resticCmd + " forget --prune " + + (concatStringsSep " " backup.pruneOpts)) + (resticCmd + " check") + ]; + # Helper functions for rclone remotes + rcloneRemoteName = + builtins.elemAt (splitString ":" backup.repository) 1; + rcloneAttrToOpt = v: + "RCLONE_" + toUpper (builtins.replaceStrings [ "-" ] [ "_" ] v); + rcloneAttrToConf = v: + "RCLONE_CONFIG_" + toUpper (rcloneRemoteName + "_" + v); + toRcloneVal = v: if lib.isBool v then lib.boolToString v else v; + in nameValuePair "restic-backups-${name}" ({ + environment = { + RESTIC_PASSWORD_FILE = backup.passwordFile; + RESTIC_REPOSITORY = backup.repository; + RESTIC_REPOSITORY_FILE = backup.repositoryFile; + } // optionalAttrs (backup.rcloneOptions != null) (mapAttrs' + (name: value: + nameValuePair (rcloneAttrToOpt name) (toRcloneVal value)) + backup.rcloneOptions) + // optionalAttrs (backup.rcloneConfigFile != null) { + RCLONE_CONFIG = backup.rcloneConfigFile; + } // optionalAttrs (backup.rcloneConfig != null) (mapAttrs' + (name: value: + nameValuePair (rcloneAttrToConf name) (toRcloneVal value)) + backup.rcloneConfig); + path = [ pkgs.openssh ]; + restartIfChanged = false; + serviceConfig = { + Type = "oneshot"; + ExecStart = (optionals (backupPaths != "") [ + "${resticCmd} backup --cache-dir=%C/restic-backups-${name} ${ + concatStringsSep " " backup.extraBackupArgs + } ${backupPaths}" + ]) ++ pruneCmd; + User = backup.user; + RuntimeDirectory = "restic-backups-${name}"; + CacheDirectory = "restic-backups-${name}"; + CacheDirectoryMode = "0700"; + Nice = backup.niceness; + IOSchedulingClass = backup.ioSchedulingClass; + } // optionalAttrs (backup.environmentFile != null) { + EnvironmentFile = backup.environmentFile; + }; + } // optionalAttrs (backup.initialize || backup.dynamicFilesFrom != null + || backup.backupPrepareCommand != null) { + preStart = '' + ${optionalString (backup.backupPrepareCommand != null) '' + ${pkgs.writeScript "backupPrepareCommand" + backup.backupPrepareCommand} + ''} + ${optionalString (backup.initialize) '' + ${resticCmd} snapshots || ${resticCmd} init + ''} + ${optionalString (backup.dynamicFilesFrom != null) '' + ${ + pkgs.writeScript "dynamicFilesFromScript" + backup.dynamicFilesFrom + } > ${filesFromTmpFile} + ''} + ''; + } // optionalAttrs (backup.dynamicFilesFrom != null + || backup.backupCleanupCommand != null) { + postStart = '' + ${optionalString (backup.backupCleanupCommand != null) '' + ${pkgs.writeScript "backupCleanupCommand" + backup.backupCleanupCommand} + ''} + ${optionalString (backup.dynamicFilesFrom != null) '' + rm ${filesFromTmpFile} + ''} + ''; + })) config.services.restic.backups; + systemd.timers = mapAttrs' (name: backup: + nameValuePair "restic-backups-${name}" { + wantedBy = [ "timers.target" ]; + timerConfig = backup.timerConfig; + }) config.services.restic.backups; + }; +}