fix(restic): fix restic path handling

Generating a list of paths for restic to backup introduces some issues:
- restic matches incremental backups by paths, changing paths cause new
  backups
- logs and a lot of restic commands print all the paths, which makes it
  basically unusable

Thus I've reverted to using static `paths` and excluding patterns via
the `--exclude-file` argument.
To reduce files to backup from `~/dev`, a preStart job was added to the
systemd service: It clones the directory via `rsync` with `.gitignore`
files being respected.
This commit is contained in:
Felix Schröter 2022-07-04 17:12:39 +02:00
parent 3a0c9a91e8
commit 7799ef1131
Signed by: felschr
GPG key ID: 671E39E6744C807D
2 changed files with 24 additions and 21 deletions

View file

@ -14,7 +14,6 @@ in {
services.restic.backups.full = resticLib.resticConfig { services.restic.backups.full = resticLib.resticConfig {
name = "home-pc"; name = "home-pc";
ripgrep = true;
paths = [ "/etc/nixos" "/var/lib" "/home" ]; paths = [ "/etc/nixos" "/var/lib" "/home" ];
ignorePatterns = [ ignorePatterns = [
"/var/lib/systemd" "/var/lib/systemd"
@ -40,8 +39,23 @@ in {
"/home/felschr/media" "/home/felschr/media"
"/home/felschr/sync" "/home/felschr/sync"
"/home/felschr/keybase" "/home/felschr/keybase"
"/home/felschr/dev" # backup ~/dev-backup instead
]; ];
timerConfig.OnCalendar = "0/4:00:00"; timerConfig.OnCalendar = "0/4:00:00";
extraPruneOpts = [ "--keep-last 6" ]; extraPruneOpts = [ "--keep-last 6" ];
}; };
# extra handling for dev folder to respect .gitignore files:
systemd.services."restic-backups-full" = {
preStart = ''
rm -rf /home/felschr/dev-backup
${pkgs.rsync}/bin/rsync -a \
--filter=':- .gitignore' \
--exclude 'nixpkgs' \
/home/felschr/dev/* /home/felschr/dev-backup
'';
postStart = ''
rm -rf /home/felschr/dev-backup
'';
};
} }

View file

@ -1,15 +1,15 @@
{ config, lib, pkgs, ... }: { config, lib, pkgs, ... }:
# using the restic cli: # using the restic cli:
# load credentials into shell via: export $(cat /path/to/credentials/file | xargs) # load credentials into shell by adding B2 secrets to .env (see .env.example).
# useful commands for analysing restic stats [snapshot-id], restic diff [s1] [s2], # useful commands for analysing restic stats [snapshot-id], restic diff [s1] [s2],
with lib; with lib;
with builtins; with builtins;
let hasAnyAttr = flip (attrset: any (flip hasAttr attrset)); let hasAnyAttr = flip (attrset: any (flip hasAttr attrset));
in { in {
resticConfig = args@{ name, ripgrep ? false, paths ? [ ], ignorePatterns ? [ ] resticConfig = args@{ name, paths ? [ ], ignorePatterns ? [ ]
, extraPruneOpts ? [ ], ... }: , extraBackupArgs ? [ ], extraPruneOpts ? [ ], ... }:
assert !hasAnyAttr [ assert !hasAnyAttr [
"initialize" "initialize"
"repository" "repository"
@ -18,28 +18,17 @@ in {
"pruneOpts" "pruneOpts"
] args; ] args;
assert (args ? paths); assert (args ? paths);
assert (ripgrep || (!ripgrep && !(args ? ignorePatterns)));
{ {
initialize = true; initialize = true;
repository = "b2:felschr-backups:/${name}"; repository = "b2:felschr-backups:/${name}";
environmentFile = config.age.secrets.restic-b2.path; environmentFile = config.age.secrets.restic-b2.path;
passwordFile = config.age.secrets.restic-password.path; passwordFile = config.age.secrets.restic-password.path;
timerConfig.OnCalendar = "daily"; timerConfig.OnCalendar = "daily";
paths = if ripgrep then null else paths; paths = paths;
dynamicFilesFrom = if ripgrep then extraBackupArgs = let
let ignoreFile = builtins.toFile "ignore"
files = foldl (a: b: "${a} ${b}") "" paths; (foldl (a: b: a + "\n" + b) "" ignorePatterns);
ignoreFile = builtins.toFile "ignore" in [ "--exclude-file=${ignoreFile}" ] ++ extraBackupArgs;
(foldl (a: b: a + "\n" + b) "" ignorePatterns);
in ''
${pkgs.fd}/bin/fd \
--hidden \
--ignore-file ${ignoreFile} \
. ${files} \
| sed "s/\[/\\\[/" | sed "s/\]/\\\]/"
''
else
null;
pruneOpts = [ pruneOpts = [
"--keep-daily 7" "--keep-daily 7"
"--keep-weekly 4" "--keep-weekly 4"
@ -48,9 +37,9 @@ in {
] ++ extraPruneOpts; ] ++ extraPruneOpts;
} // (removeAttrs args [ } // (removeAttrs args [
"name" "name"
"ripgrep"
"paths" "paths"
"ignorePatterns" "ignorePatterns"
"extraBackupArgs"
"extraPruneOpts" "extraPruneOpts"
]); ]);
} }