Host maintenance¶

sudo apt install unattended-upgrades needrestart

sudo unattended-upgrade --dry-run --debug 2>&1 | grep -E 'Allowed origins|Checking|pkgs that look like'

# Spot a waiting Docker update (what the report checks)
apt-cache policy docker-ce containerd.io | grep -E '^[a-z]|Installed|Candidate'

# Upgrade by hand when it suits you (restarts every container on this host)
sudo apt install --only-upgrade docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

#!/bin/bash
# Reboot for updates only when Debian says one is required.
# Waits for: SnapRAID (max 4 h), a borg backup (max 2 h, own run or another host backing up INTO this one), dpkg (max 1 h).
# DRYRUN=1: run every check and send the notification marked [dry run], but don't reboot.
set -u
DRY="${DRYRUN:-0}"
MAXWAIT_SNAPRAID=$((4*3600)); MAXWAIT_BACKUP=$((2*3600)); MAXWAIT_DPKG=3600
log(){ logger -t fleet-reboot "$*"; echo "$*"; }

[ -f /var/run/reboot-required ] || [ "$DRY" = 1 ] || { log "no reboot required"; exit 0; }
pkgs="$(sort -u /var/run/reboot-required.pkgs 2>/dev/null | tr '\n' ' ')"

waited=0
while pgrep -x snapraid >/dev/null || pgrep -f snapraid-aio-script >/dev/null; do
  [ $waited -ge $MAXWAIT_SNAPRAID ] && {
    fleet-notify hosts 4 warning,hourglass "Reboot postponed" "SnapRAID still running after 4h. Needs: ${pkgs:-unknown}"
    exit 0; }
  sleep 60; waited=$((waited+60))
done

waited=0
while pgrep -x borgmatic >/dev/null || pgrep -f 'borg serve' >/dev/null; do
  [ $waited -ge $MAXWAIT_BACKUP ] && {
    fleet-notify hosts 4 warning,hourglass "Reboot postponed" "A borg backup was still running after 2h. Needs: ${pkgs:-unknown}"
    exit 0; }
  sleep 60; waited=$((waited+60))
done

waited=0
while fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock >/dev/null 2>&1; do
  [ $waited -ge $MAXWAIT_DPKG ] && { fleet-notify hosts 4 warning "Reboot postponed" "dpkg busy for 1h. Needs: ${pkgs:-unknown}"; exit 0; }
  sleep 30; waited=$((waited+30))
done

if [ "$DRY" = 1 ]; then
  fleet-notify hosts 2 test_tube "[dry run] Reboot check passed" "Would reboot now for: ${pkgs:-nothing pending}"
  exit 0
fi
log "rebooting for updates: ${pkgs:-unknown}"
fleet-notify hosts 4 arrows_counterclockwise "Rebooting for updates" \
  "Rebooting now to finish updates (${pkgs:-packages unknown}). Expect ~3 min of downtime. A 'back up' message follows."
sleep 20
systemctl reboot

sudo apt install smartmontools

sudo systemctl enable --now smartmontools
sudo systemctl restart smartmontools
# Test the hook end to end: add "-M test" to the DEVICESCAN line, restart, then remove it again
sudo smartctl -l selftest /dev/sda        # self-test history per disk

sudo sysctl -p /etc/sysctl.d/90-panic.conf
sysctl kernel.panic
systemctl is-enabled systemd-pstore      # should be "enabled" (default on Debian 13)

#!/bin/bash
# Hourly filesystem fill check -> ntfy `alerts`, on level change only (+ daily reminder while critical).
# Default: warn 85% used / crit 95% used. Overrides in /etc/fleet/diskspace.conf, one per line, first match wins:
#   <mount|glob> <warn> <crit>     "NN%" = used percent at/above, "NNG" = free GiB at/below
#   <mount|glob> skip
# Network filesystems are skipped: the host that owns the disks checks them.
# DRYRUN=1 prints instead of notifying; CONF/STATE env vars override the paths (for testing).
set -u
CONF="${CONF:-/etc/fleet/diskspace.conf}"; STATE="${STATE:-/var/lib/fleet/diskspace.state}"
now=$(date +%s); mkdir -p "$(dirname "$STATE")"; touch "$STATE"

rule(){ # rule <mount> -> "<warn> <crit>" or "skip"
  if [ -r "$CONF" ]; then
    while read -r pat w c; do
      case "$pat" in ''|\#*) continue ;; esac
      # shellcheck disable=SC2254
      case "$1" in $pat) echo "$w ${c:-}"; return ;; esac
    done < "$CONF"
  fi
  echo "85% 95%"; }
hit(){ # hit <threshold> <used%> <freeGiB>
  case "$1" in *%) [ "$2" -ge "${1%\%}" ] ;; *G) [ "$3" -le "${1%G}" ] ;; *) return 1 ;; esac; }

new_state=""
while read -r _ type _ _ avail pct mnt; do
  set -- $(rule "$mnt"); [ "$1" = skip ] && continue
  warn="$1"; crit="${2:-$1}"; used="${pct%\%}"; free=$((avail / 1073741824))
  lvl=0; hit "$warn" "$used" "$free" && lvl=1; hit "$crit" "$used" "$free" && lvl=2
  read -r plvl plast <<<"$(awk -v m="$mnt" '$1==m{print $2, $3}' "$STATE")"
  plvl=${plvl:-0}; plast=${plast:-0}; last=$plast
  if [ "$lvl" != "$plvl" ] || { [ "$lvl" = 2 ] && [ $((now - plast)) -ge 86400 ]; }; then
    case $lvl in 2) p=5; t=rotating_light,floppy_disk; s=CRITICAL ;; 1) p=4; t=warning,floppy_disk; s=warning ;; 0) p=2; t=white_check_mark; s=ok ;; esac
    msg="$mnt ($type): ${used}% used, ${free} GiB free. Thresholds: warn $warn, crit $crit."
    if [ "${DRYRUN:-0}" = 1 ]; then echo "NOTIFY p$p [$s] $msg"
    else /usr/local/bin/fleet-notify alerts "$p" "$t" "Disk space $s: $mnt" "$msg"; fi
    last=$now
  fi
  new_state+="$mnt $lvl $last"$'\n'
done < <(df -PT -B1 -x tmpfs -x devtmpfs -x overlay -x squashfs -x efivarfs -x vfat -x cifs -x nfs -x nfs4 -x fuse.sshfs | tail -n +2)
printf '%s' "$new_state" > "$STATE"

sudo DRYRUN=1 STATE=/tmp/s /usr/local/sbin/fleet-diskspace-check   # test without sending
cat /var/lib/fleet/diskspace.state

sudo apt install bind9-dnsutils     # dig
sudo install -m 0755 wait-for-router-dns.sh /usr/local/sbin/
sudo install -m 0644 wait-for-router-dns.service /etc/systemd/system/
sudo systemctl daemon-reload && sudo systemctl enable wait-for-router-dns.service
journalctl -b -u wait-for-router-dns     # after the next boot: how long it waited

#!/bin/bash
# install-host.sh <ssh-target|local> <host> [component...]   (default: all components)
set -euo pipefail
T="$1"; H="$2"; shift 2; COMPS="${*:-notify updates smart panic diskspace}"
F="$(cd "$(dirname "$0")" && pwd)"
rx(){ if [ "$T" = local ]; then bash -c "$1"; else ssh "$T" "$1"; fi; }
put(){ rx "sudo install -D -m $3 -o root -g root /dev/stdin '$2'" < "$1"; }   # put <src> <dest> <mode>

for c in $COMPS; do case "$c" in
  notify)
    put "$F/bin/fleet-notify" /usr/local/bin/fleet-notify 755
    # Token never in git: the caller prints it from the secret store, it's piped over stdin
    [ -n "${TOKEN_FROM:-}" ] && eval "$TOKEN_FROM" | rx 'sudo install -D -m 600 /dev/stdin /etc/ntfy/host-publisher.token' ;;
  smart)
    rx 'sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -q smartmontools >/dev/null'
    put "$F/smart/fleet-smartd-notify" /usr/local/sbin/fleet-smartd-notify 755
    put "$F/smart/smartd.conf" /etc/smartd.conf 644
    rx 'sudo systemctl enable -q smartmontools && sudo systemctl restart smartmontools' ;;
  panic)
    put "$F/system/sysctl/90-panic.conf" /etc/sysctl.d/90-panic.conf 644
    rx 'sudo sysctl -q -p /etc/sysctl.d/90-panic.conf' ;;
  diskspace)
    put "$F/diskspace/fleet-diskspace-check" /usr/local/sbin/fleet-diskspace-check 755
    # per-host override if one exists, otherwise remove a stale one
    if [ -f "$F/diskspace/$H.conf" ]; then put "$F/diskspace/$H.conf" /etc/fleet/diskspace.conf 644
    else rx 'sudo rm -f /etc/fleet/diskspace.conf'; fi
    # ...units, then: daemon-reload, enable --now the timer, run once, print the state file
    ;;
  updates) : ;;   # apt config, needrestart, reboot/boot/report scripts + units, per-host timer drop-in
  *) echo "unknown component $c"; exit 1 ;;
esac; done
echo "installed [$COMPS] on $T ($H)"

hosts/_fleet/install-host.sh server2 server2 smart panic        # two components on one host
TOKEN_FROM='<command that prints the host-publisher token>' \
  hosts/_fleet/install-host.sh server1 server1                  # everything, incl. the ntfy token
hosts/_fleet/install-host.sh local workstation updates          # the workstation's own (no-reboot) variant