From 8d113be71e4c6e566ed8db42cdd8e44b048ac9df Mon Sep 17 00:00:00 2001 From: James Bunton Date: Sun, 7 Jan 2018 21:59:49 +1100 Subject: [PATCH] Reorganise to work with PKGBUILD --- .gitignore | 1 + PKGBUILD | 75 +++++++++++++++++++ backup | 58 ++++++++++++++ backup-mysql | 13 ---- backup.service | 6 ++ backup.timer | 10 +++ borg/backup | 40 ---------- borg/do-install | 25 ------- borg/etc-cron.d-backup | 6 -- borg/root-ssh-config | 6 -- bsnap-borg.install | 13 ++++ lvm/bsnap => bsnap-lvm | 0 bsnap-zfs | 63 ++++++++++++++++ bsnap-zfs.install | 11 +++ borg/etc-borg-env => etc-borg-root-env | 1 - .../etc-borg-exclude => etc-borg-root-exclude | 3 +- lvm/do-install | 7 -- rdiff-backup/backup-all | 17 ----- rdiff-backup/common | 18 ----- rdiff-backup/etc-cron.d-backup | 6 -- rdiff-backup/template-rdiff-backup | 22 ------ rdiff-backup/template-rsync | 8 -- setup-root-ssh | 35 +++++++++ zfs/bsnap | 31 -------- zfs/do-install | 7 -- zfs/get-custom-zfs-props.sh | 2 - zfs/znap-list | 19 ----- zfs/znap-mount | 21 ------ zfs/znap-umount-all | 18 ----- znap-daily.service | 6 ++ znap-daily.timer | 9 +++ znap-frequent.service | 6 ++ znap-frequent.timer | 9 +++ znap-hourly.service | 6 ++ znap-hourly.timer | 9 +++ znap-take | 16 ++++ 36 files changed, 334 insertions(+), 269 deletions(-) create mode 100644 .gitignore create mode 100644 PKGBUILD create mode 100755 backup delete mode 100755 backup-mysql create mode 100644 backup.service create mode 100644 backup.timer delete mode 100755 borg/backup delete mode 100755 borg/do-install delete mode 100644 borg/etc-cron.d-backup delete mode 100644 borg/root-ssh-config create mode 100644 bsnap-borg.install rename lvm/bsnap => bsnap-lvm (100%) create mode 100755 bsnap-zfs create mode 100644 bsnap-zfs.install rename borg/etc-borg-env => etc-borg-root-env (69%) rename borg/etc-borg-exclude => etc-borg-root-exclude (63%) delete mode 100755 lvm/do-install delete mode 100755 rdiff-backup/backup-all delete mode 100644 rdiff-backup/common delete mode 100644 rdiff-backup/etc-cron.d-backup delete mode 100755 rdiff-backup/template-rdiff-backup delete mode 100755 rdiff-backup/template-rsync create mode 100755 setup-root-ssh delete mode 100755 zfs/bsnap delete mode 100755 zfs/do-install delete mode 100755 zfs/get-custom-zfs-props.sh delete mode 100755 zfs/znap-list delete mode 100755 zfs/znap-mount delete mode 100755 zfs/znap-umount-all create mode 100644 znap-daily.service create mode 100644 znap-daily.timer create mode 100644 znap-frequent.service create mode 100644 znap-frequent.timer create mode 100644 znap-hourly.service create mode 100644 znap-hourly.timer create mode 100755 znap-take diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f1c33be --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pkg.* diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..de0057c --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,75 @@ +pkgbase="bsnap" +pkgname=("bsnap-borg" "bsnap-zfs" "bsnap-lvm") +pkgver=1.0.0 +pkgrel=1 +arch=("any") +source=("backup" + "backup.service" + "backup.timer" + "bsnap-lvm" + "setup-root-ssh" + "bsnap-zfs" + "etc-borg-root-env" + "etc-borg-root-exclude" + "znap-daily.service" + "znap-daily.timer" + "znap-frequent.service" + "znap-frequent.timer" + "znap-hourly.service" + "znap-hourly.timer" + "znap-take") +md5sums=('8b910ad468d754fa21f0a89dcab1715e' + 'ebc356d63196ff3ea1b3bdc51d00ae3a' + 'c0b4804dc26055ef88b3abdbba5f5b7c' + '7b7c3a2d6bff8737b9bd1e766e65818b' + '6f173fb271b64dfcc84b2536b076b36f' + 'ebf29f2525fc5ca671f4c24b90908d11' + '05897a7215d9ef75e174e3dec94ca416' + '5e2165087fa3d83c0a10f8837750bcfd' + 'b233a09cdbd997699e1ba03559ef892a' + 'eeafe687944641a401a7f7f902dac73a' + '606b981ecbde6f7c90f135e91986b678' + '916f9b7a74c4516da6aad084c0c32e3e' + '832f2cd78dc266d97d3ad37d11448462' + '0624408bb7c90a36e11f89eb64a8923e' + '4fa1ebcd4e6f4db9c92d8c1f555975fc') + +package_bsnap-borg() { + pkgdesc="Borg backup tools" + depends=("borg") + backup=("etc/borg/root/exclude" "etc/borg/root/env") + install="bsnap-borg.install" + + mkdir -p "${pkgdir}/usr/bin/" + install -m 0755 backup setup-root-ssh "${pkgdir}/usr/bin/" + + mkdir -p "${pkgdir}/etc/borg/root/" + install -m 0644 -T etc-borg-root-exclude "${pkgdir}/etc/borg/root/exclude" + install -m 0600 -T etc-borg-root-env "${pkgdir}/etc/borg/root/env" + + mkdir -p "${pkgdir}/usr/lib/systemd/system/" + install -m 0644 backup.{timer,service} "${pkgdir}/usr/lib/systemd/system/" +} + +package_bsnap-zfs() { + pkgdesc="Backup / snapshot tools for ZFS" + depends=("zfs-utils" "zfs-auto-snapshot") + conflicts=("bsnap-lvm") + install="bsnap-zfs.install" + + mkdir -p "${pkgdir}/usr/bin/" + install -m 0755 -T bsnap-zfs "${pkgdir}/usr/bin/bsnap" + install -m 0755 znap-take "${pkgdir}/usr/bin/" + + mkdir -p "${pkgdir}/usr/lib/systemd/system/" + install -m 0644 znap-*.{timer,service} "${pkgdir}/usr/lib/systemd/system/" +} + +package_bsnap-lvm() { + pkgdesc="Backup / snapshot tools for LVM" + depends=("lvm2") + conflicts=("bsnap-zfs") + + mkdir -p "${pkgdir}/usr/bin/" + install -m 0755 -T bsnap-lvm "${pkgdir}/usr/bin/bsnap" +} diff --git a/backup b/backup new file mode 100755 index 0000000..5cdab0d --- /dev/null +++ b/backup @@ -0,0 +1,58 @@ +#!/bin/bash + +set -eu + +if [ "$(id -u)" -ne 0 ]; then + echo "Must be root" + exit 1 +fi + +if pgrep borg > /dev/null; then + echo "Borg is still running!" + exit 1 +fi + +if [ -z "${1:-}" ]; then + echo "Usage: $0 root" + exit 1 +fi + +BACKUP_PROFILE="$1" +source "/etc/borg/${BACKUP_PROFILE}/env" +HOME=/root + +ionice -c 3 -p $$ +renice -n 19 -p $$ > /dev/null + +if [ "$BACKUP_SNAP" -eq 1 ]; then + echo ">>> Snapshotting" + bsnap on +fi + +cd "$BACKUP_PATH" + +echo ">>> Starting backup" +borg create \ + $([ -t 0 ] && echo --progress) \ + --info --stats \ + --compression lz4 \ + --exclude-from "/etc/borg/${BACKUP_PROFILE}/exclude" \ + "${BACKUP_REPO}::{hostname}-{now}" . + +echo ">>> Pruning old backups" +borg prune \ + --info --stats \ + --keep-daily=7 \ + --keep-weekly=4 \ + --keep-monthly=12 \ + --prefix='{hostname}-' \ + "$BACKUP_REPO" + +cd / + +if [ "$BACKUP_SNAP" -eq 1 ]; then + echo ">>> Unsnapshotting" + bsnap off +fi + +exit 0 diff --git a/backup-mysql b/backup-mysql deleted file mode 100755 index 2b66f0a..0000000 --- a/backup-mysql +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -eu - -cd ~/backup-mysql/ - -DATABASES="$(echo 'show databases' | mysql | tail -n+2)" -for db in ${DATABASES}; do - mysqldump --add-drop-table --single-transaction "${db}" | \ - gzip -9 > "${db}-$(date '+%Y%m%d').sql.gz" - - rm -f $(ls -1 "${db}-"* | sort -r | tail -n +10) -done diff --git a/backup.service b/backup.service new file mode 100644 index 0000000..5c31b45 --- /dev/null +++ b/backup.service @@ -0,0 +1,6 @@ +[Unit] +Description=Borg backup service + +[Service] +Type=oneshot +ExecStart=/usr/bin/backup root diff --git a/backup.timer b/backup.timer new file mode 100644 index 0000000..9c21fb6 --- /dev/null +++ b/backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Borg backup timer + +[Timer] +OnCalendar=daily +AccuracySec=86400 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/borg/backup b/borg/backup deleted file mode 100755 index 3147c86..0000000 --- a/borg/backup +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -eu - -if pgrep borg > /dev/null; then - echo "Borg is still running!" - exit 1 -fi - -source "$1" - -set -x - -ionice -c 3 -p $$ -renice -n 19 -p $$ > /dev/null - -[ "$BACKUP_SNAP" -eq 1 ] && bsnap on - -cd "$BACKUP_PATH" - -borg create \ - $([ -t 0 ] && echo --progress) \ - --info --stats \ - --compression lz4 \ - --exclude-from "$BACKUP_EXCLUDE_FILE" \ - "${BACKUP_REPO}::{hostname}-{now}" . - -borg prune \ - --info --stats \ - --keep-daily=7 \ - --keep-weekly=4 \ - --keep-monthly=12 \ - --prefix='{hostname}-' \ - "$BACKUP_REPO" - -cd / - -[ "$BACKUP_SNAP" -eq 1 ] && bsnap off - -exit 0 diff --git a/borg/do-install b/borg/do-install deleted file mode 100755 index bbdf7a4..0000000 --- a/borg/do-install +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -eux - -cd "$(dirname "$0")" - -install -m 0755 backup /usr/local/bin/ - -if [ "${1:-}" != "full" ]; then - exit 0 -fi - -BORG_PASSPHRASE="$(dd if=/dev/urandom of=/dev/stdout bs=1 count=15 2>/dev/null | base64)" -install -m 0644 -T etc-borg-exclude /etc/borg-exclude -install -m 0600 -T etc-borg-env /etc/borg-env -sed -i "s|SECRET|$BORG_PASSPHRASE|" /etc/borg-env - -install -m 0644 -T etc-cron.d-backup /etc/cron.d/backup -H=$((16#$(hostname|md5sum|head -c 1) % 7 + 14)) -M=$((16#$(hostname|md5sum|head -c 2) % 60)) -sed -i -e "s|\$H|$H|" -e "s|\$M|$M|" /etc/cron.d/backup - -[ -f /root/.ssh/id_rsa.pub ] || ssh-keygen -cat /root/.ssh/id_rsa.pub -install -m 0600 -T root-ssh-config /root/.ssh/config diff --git a/borg/etc-cron.d-backup b/borg/etc-cron.d-backup deleted file mode 100644 index b267e6e..0000000 --- a/borg/etc-cron.d-backup +++ /dev/null @@ -1,6 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -MAILTO=root - -# m h dom mon dow user command - $M $H * * * root chronic backup /etc/borg-env diff --git a/borg/root-ssh-config b/borg/root-ssh-config deleted file mode 100644 index 70c030f..0000000 --- a/borg/root-ssh-config +++ /dev/null @@ -1,6 +0,0 @@ -Host * - HashKnownHosts no - -Host backuphost - User borg - Hostname backup.example.com diff --git a/bsnap-borg.install b/bsnap-borg.install new file mode 100644 index 0000000..a0525ad --- /dev/null +++ b/bsnap-borg.install @@ -0,0 +1,13 @@ +#! bash + +function post_install { + if grep -q "SECRET" /etc/borg/root/env; then + echo ">>> Generating passphrase for /etc/borg/root/env" + BORG_PASSPHRASE="$(dd if=/dev/urandom of=/dev/stdout bs=1 count=15 2>/dev/null | base64)" + sed -i "s|SECRET|$BORG_PASSPHRASE|" "${pkgdir}/etc/borg/root/env" + fi + + if ! systemctl is-enabled backup.timer > /dev/null; then + echo ">>> You should run: systemctl enable backup.timer" + fi +} diff --git a/lvm/bsnap b/bsnap-lvm similarity index 100% rename from lvm/bsnap rename to bsnap-lvm diff --git a/bsnap-zfs b/bsnap-zfs new file mode 100755 index 0000000..dcd8c54 --- /dev/null +++ b/bsnap-zfs @@ -0,0 +1,63 @@ +#!/bin/bash + +set -eu + +function snap { + unsnap + + znap-take bsnap 1 + mount_snapshots + mkdir -p /a/boot + mount --bind /boot /a/boot +} + +function unsnap { + if ! [ -d /a ]; then + return + fi + + umount /a/boot || true + rmdir /a/boot || true + + if ls &> /dev/null /a/*; then + umount /a/* || true + rmdir /a/* || true + fi + rmdir /a || true +} + +function list_snapshots { + local fsname + for fsname in $(zfs list -H -o name); do + zfs list -H -o name -S creation -t snapshot -d 1 -r "$fsname" | \ + grep '@znap_....-..-..-...._bsnap' | \ + head -n1 + done +} + +function mount_snapshots { + local snapname + local fsname + local mountpoint + + for snapname in $(list_snapshots); do + fsname="${snapname%@*}" + mountpoint="/a/$(echo "$fsname" | cut -d/ -f2- | tr '/' '_')" + mkdir -p "$mountpoint" + mount -t zfs "$snapname" "$mountpoint" + done +} + +if [ "$(id -u)" -ne 0 ]; then + echo "Must be root" + exit 1 +fi + +if [ "${1:-}" = "off" ]; then + unsnap +elif [ "${1:-}" = "on" ]; then + snap +else + echo "Usage: $0 on|off" + exit 1 +fi diff --git a/bsnap-zfs.install b/bsnap-zfs.install new file mode 100644 index 0000000..5c558c4 --- /dev/null +++ b/bsnap-zfs.install @@ -0,0 +1,11 @@ +#! bash + +function post_install { + if ! zfs get all -t filesystem | grep -q "backup:"; then + echo + echo ">>> You'll need to configure ZFS properties for automatic snapshots to work" + echo ">>> zfs set backup:frequent=on backup:hourly=on backup:daily=on backup:bsnap=on tank/root" + echo + echo ">>> Then enable and start some of the znap-*.timer systemd units." + fi +} diff --git a/borg/etc-borg-env b/etc-borg-root-env similarity index 69% rename from borg/etc-borg-env rename to etc-borg-root-env index b7ed751..6627c5f 100644 --- a/borg/etc-borg-env +++ b/etc-borg-root-env @@ -1,5 +1,4 @@ BACKUP_SNAP=1 -BACKUP_EXCLUDE_FILE="/etc/borg-exclude" BACKUP_PATH="/a" BACKUP_REPO="backuphost:root" export BORG_PASSPHRASE="SECRET" diff --git a/borg/etc-borg-exclude b/etc-borg-root-exclude similarity index 63% rename from borg/etc-borg-exclude rename to etc-borg-root-exclude index 72cc2ec..e35bac9 100644 --- a/borg/etc-borg-exclude +++ b/etc-borg-root-exclude @@ -2,6 +2,5 @@ sh:**/.cache sh:**/.thumbnails sh:**/tmp -sh:var/cache/apt/* -sh:var/cache/pacman/* +sh:var/cache sh:var/lib/systemd/coredump diff --git a/lvm/do-install b/lvm/do-install deleted file mode 100755 index 1052cbc..0000000 --- a/lvm/do-install +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -eux - -cd "$(dirname "$0")" - -install -m 0755 bsnap /usr/local/bin/ diff --git a/rdiff-backup/backup-all b/rdiff-backup/backup-all deleted file mode 100755 index f11cf93..0000000 --- a/rdiff-backup/backup-all +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -for script in /backup/scripts/*; do - if ! grep -q '^function run_backup' "$script"; then - continue - fi - if [ ! -x "$script" ]; then - echo "Skipping $script because it is not executable" - continue - fi - output="$("$script" 2>&1)" - if [ "$?" -ne 0 ]; then - echo "Failed to run: $script" - echo "$output" - fi -done - diff --git a/rdiff-backup/common b/rdiff-backup/common deleted file mode 100644 index ef5972c..0000000 --- a/rdiff-backup/common +++ /dev/null @@ -1,18 +0,0 @@ -#! bash - -set -e -ionice -c 3 -p $$ -renice -n 19 -p $$ > /dev/null - -script="$(basename "${BASH_SOURCE[1]}" .sh)" - -( -if ! flock -n -x 200; then - echo "Failed to get a lock!" - exit 1 -fi - -run_backup - -) 200>"/run/lock/backup-${script}" - diff --git a/rdiff-backup/etc-cron.d-backup b/rdiff-backup/etc-cron.d-backup deleted file mode 100644 index 61838a6..0000000 --- a/rdiff-backup/etc-cron.d-backup +++ /dev/null @@ -1,6 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -MAILTO=root - -# m h dom mon dow user command - 0 2 * * * root /backup/scripts/backup-all diff --git a/rdiff-backup/template-rdiff-backup b/rdiff-backup/template-rdiff-backup deleted file mode 100755 index 1073912..0000000 --- a/rdiff-backup/template-rdiff-backup +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -function run_backup { - rdiff-backup \ - --preserve-numerical-ids \ - --exclude-sockets \ - --exclude '/home/*/tmp' \ - --include '/home' \ - --exclude '/var/cache' \ - --exclude '/var/log' \ - --exclude '/var/tmp' \ - --exclude '/tmp' \ - --include '/var' \ - --include '/boot' \ - --exclude-other-filesystems \ - my-host.example.com::/ /backup/my-host - - rdiff-backup -v1 --force --remove-older-than 365D /backup/my-host -} - -. /backup/scripts/common - diff --git a/rdiff-backup/template-rsync b/rdiff-backup/template-rsync deleted file mode 100755 index 6ba2077..0000000 --- a/rdiff-backup/template-rsync +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -function run_backup { - rsync -a --numeric-ids --delete my-host.example.com:/backup/my-host/ /backup/my-host/ -} - -. /backup/scripts/common - diff --git a/setup-root-ssh b/setup-root-ssh new file mode 100755 index 0000000..c85c8c7 --- /dev/null +++ b/setup-root-ssh @@ -0,0 +1,35 @@ +#!/bin/bash + +if [ "$(id -u)" -ne 0 ]; then + echo "Must be root" + exit 1 +fi + +set -eu + +if [ -f /root/.ssh/id_rsa.pub ]; then + echo "SSH public key already exists. Exiting!" + exit 1 +fi + +if [ -f /root/.ssh/config ]; then + echo "SSH config already exists. Exiting!" + exit 1 +fi + +umask 0077 + +echo ">>> ssh-keygen" +ssh-keygen + +cat /root/.ssh/id_rsa.pub + +echo ">>> creating .ssh/config" +cat > /root/.ssh/config < /dev/null "$PREFIX"/*; then - umount "$PREFIX"/* || true - rmdir "$PREFIX"/* || true -fi -rmdir "$PREFIX" || true diff --git a/znap-daily.service b/znap-daily.service new file mode 100644 index 0000000..332e400 --- /dev/null +++ b/znap-daily.service @@ -0,0 +1,6 @@ +[Unit] +Description=ZFS daily snapshot service + +[Service] +Type=oneshot +ExecStart=/usr/bin/znap-take daily 30 diff --git a/znap-daily.timer b/znap-daily.timer new file mode 100644 index 0000000..65dd773 --- /dev/null +++ b/znap-daily.timer @@ -0,0 +1,9 @@ +[Unit] +Description=ZFS daily snapshot timer + +[Timer] +OnCalendar=daily +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/znap-frequent.service b/znap-frequent.service new file mode 100644 index 0000000..6ee8b32 --- /dev/null +++ b/znap-frequent.service @@ -0,0 +1,6 @@ +[Unit] +Description=ZFS frequent snapshot service + +[Service] +Type=oneshot +ExecStart=/usr/bin/znap-take frequent 10 diff --git a/znap-frequent.timer b/znap-frequent.timer new file mode 100644 index 0000000..4aca231 --- /dev/null +++ b/znap-frequent.timer @@ -0,0 +1,9 @@ +[Unit] +Description=ZFS frequent snapshot timer + +[Timer] +OnCalendar=*:0/5 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/znap-hourly.service b/znap-hourly.service new file mode 100644 index 0000000..b3e5796 --- /dev/null +++ b/znap-hourly.service @@ -0,0 +1,6 @@ +[Unit] +Description=ZFS hourly snapshot service + +[Service] +Type=oneshot +ExecStart=/usr/bin/znap-take hourly 12 diff --git a/znap-hourly.timer b/znap-hourly.timer new file mode 100644 index 0000000..2943736 --- /dev/null +++ b/znap-hourly.timer @@ -0,0 +1,9 @@ +[Unit] +Description=ZFS hourly snapshot timer + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/znap-take b/znap-take new file mode 100755 index 0000000..54398f3 --- /dev/null +++ b/znap-take @@ -0,0 +1,16 @@ +#!/bin/bash + +set -eu + +if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then + echo "Usage: $0 label keep" + exit 1 +fi + +label="$1" +keep="$2" + +zfs list -H -o "backup:${label},name" | \ + grep '^on ' | \ + cut -d ' ' -f2 | \ + xargs -d '\n' zfs-auto-snapshot --prefix=znap --label="$label" --keep="$keep" -- 2.39.2