From 7799ef11313afb2add27991a6ad8ba233469bf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Schr=C3=B6ter?= Date: Mon, 4 Jul 2022 17:12:39 +0200 Subject: [PATCH] fix(restic): fix restic path handling Generating a list of paths for restic to backup introduces some issues: - restic matches incremental backups by paths, changing paths cause new backups - logs and a lot of restic commands print all the paths, which makes it basically unusable Thus I've reverted to using static `paths` and excluding patterns via the `--exclude-file` argument. To reduce files to backup from `~/dev`, a preStart job was added to the systemd service: It clones the directory via `rsync` with `.gitignore` files being respected. --- services/restic/home-pc.nix | 16 +++++++++++++++- services/restic/lib.nix | 29 +++++++++-------------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/services/restic/home-pc.nix b/services/restic/home-pc.nix index 878e169..bef77ec 100644 --- a/services/restic/home-pc.nix +++ b/services/restic/home-pc.nix @@ -14,7 +14,6 @@ in { services.restic.backups.full = resticLib.resticConfig { name = "home-pc"; - ripgrep = true; paths = [ "/etc/nixos" "/var/lib" "/home" ]; ignorePatterns = [ "/var/lib/systemd" @@ -40,8 +39,23 @@ in { "/home/felschr/media" "/home/felschr/sync" "/home/felschr/keybase" + "/home/felschr/dev" # backup ~/dev-backup instead ]; timerConfig.OnCalendar = "0/4:00:00"; extraPruneOpts = [ "--keep-last 6" ]; }; + + # extra handling for dev folder to respect .gitignore files: + systemd.services."restic-backups-full" = { + preStart = '' + rm -rf /home/felschr/dev-backup + ${pkgs.rsync}/bin/rsync -a \ + --filter=':- .gitignore' \ + --exclude 'nixpkgs' \ + /home/felschr/dev/* /home/felschr/dev-backup + ''; + postStart = '' + rm -rf /home/felschr/dev-backup + ''; + }; } diff --git a/services/restic/lib.nix b/services/restic/lib.nix index 10cee79..6bf001f 100644 --- a/services/restic/lib.nix +++ b/services/restic/lib.nix @@ -1,15 +1,15 @@ { config, lib, pkgs, ... }: # using the restic cli: -# load credentials into shell via: export $(cat /path/to/credentials/file | xargs) +# load credentials into shell by adding B2 secrets to .env (see .env.example). # useful commands for analysing restic stats [snapshot-id], restic diff [s1] [s2], with lib; with builtins; let hasAnyAttr = flip (attrset: any (flip hasAttr attrset)); in { - resticConfig = args@{ name, ripgrep ? false, paths ? [ ], ignorePatterns ? [ ] - , extraPruneOpts ? [ ], ... }: + resticConfig = args@{ name, paths ? [ ], ignorePatterns ? [ ] + , extraBackupArgs ? [ ], extraPruneOpts ? [ ], ... }: assert !hasAnyAttr [ "initialize" "repository" @@ -18,28 +18,17 @@ in { "pruneOpts" ] args; assert (args ? paths); - assert (ripgrep || (!ripgrep && !(args ? ignorePatterns))); { initialize = true; repository = "b2:felschr-backups:/${name}"; environmentFile = config.age.secrets.restic-b2.path; passwordFile = config.age.secrets.restic-password.path; timerConfig.OnCalendar = "daily"; - paths = if ripgrep then null else paths; - dynamicFilesFrom = if ripgrep then - let - files = foldl (a: b: "${a} ${b}") "" paths; - ignoreFile = builtins.toFile "ignore" - (foldl (a: b: a + "\n" + b) "" ignorePatterns); - in '' - ${pkgs.fd}/bin/fd \ - --hidden \ - --ignore-file ${ignoreFile} \ - . ${files} \ - | sed "s/\[/\\\[/" | sed "s/\]/\\\]/" - '' - else - null; + paths = paths; + extraBackupArgs = let + ignoreFile = builtins.toFile "ignore" + (foldl (a: b: a + "\n" + b) "" ignorePatterns); + in [ "--exclude-file=${ignoreFile}" ] ++ extraBackupArgs; pruneOpts = [ "--keep-daily 7" "--keep-weekly 4" @@ -48,9 +37,9 @@ in { ] ++ extraPruneOpts; } // (removeAttrs args [ "name" - "ripgrep" "paths" "ignorePatterns" + "extraBackupArgs" "extraPruneOpts" ]); }