Add renovate.json

Exit code 1 from 7z means a non-fatal warning — the archive IS created,
just some files couldn't be compressed (e.g. broken symlinks, special
files). Previously the function required filtered output to be empty
alongside rc=1, causing any non-errno=2 stderr line to abort the entire
backup and discard the archive.

Now any rc=1 is treated as success (warning is still logged).

.gitattributes

@@ -0,0 +1,10 @@
+.gitattributes text eol=lf
+*.sh text eol=lf
+*.yaml text eol=lf
+*.yml text eol=lf
+*.md text eol=lf
+*.go text eol=lf
+go.mod text eol=lf
+go.sum text eol=lf
+Dockerfile text eol=lf
+*.tmpl text eol=lf

.gitea/workflows/haven-notify.yaml

@@ -5,91 +5,108 @@ on:
     branches:
       - main
     paths:
-      - 'haven-notify/**'
-      - '.gitea/workflows/**'
+      - "haven-notify/**"
+      - ".gitea/workflows/haven-notify.yaml"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "haven-notify/**"
+      - ".gitea/workflows/haven-notify.yaml"
   workflow_dispatch: {}
 
 env:
-  REGISTRY_HOST: git.ivanch.me
-  REGISTRY_USERNAME: ivanch
-  IMAGE_NOTIFY: ${{ env.REGISTRY_HOST }}/ivanch/haven-notify
-  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  IMAGE: git.ivanch.me/ivanch/haven-notify
 
 jobs:
-  build_haven_notify:
-    name: Build Haven Notify Image
-    runs-on: ubuntu-latest
-
+  test:
+    name: Test Haven Notify
+    runs-on: runner-full-amd64
+    timeout-minutes: 15
     steps:
       - name: Check out repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Log in to Container Registry
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: haven-notify/go.mod
+          cache-dependency-path: haven-notify/go.sum
+
+      - name: Verify formatting
+        working-directory: haven-notify
         run: |
-          echo "${{ secrets.REGISTRY_PASSWORD }}" \
-            | docker login "${{ env.REGISTRY_HOST }}" \
-                -u "${{ env.REGISTRY_USERNAME }}" \
-                --password-stdin
-
-      - name: Display Directory Structure
-        run: |
-          echo "Current directory structure:"
-          ls -lR .
-
-      - name: Build and Push Image
-        run: |
-          TAG=latest
-
-          cd haven-notify
-
-          docker buildx build \
-            -t "${{ env.IMAGE_NOTIFY }}:${TAG}" \
-            --platform linux/amd64,linux/arm64 \
-            -f Dockerfile .
-
-          docker push "${{ env.IMAGE_NOTIFY }}:${TAG}"
-
-  deploy_haven_notify:
-    name: Deploy Haven Notify (internal)
-    runs-on: ubuntu-amd64
-    needs: build_haven_notify_amd64
-    steps:
-      - name: Check KUBE_CONFIG validity
-        run: |
-          if [ -z "${KUBE_CONFIG}" ] || [ "${KUBE_CONFIG}" = "" ] || [ "${KUBE_CONFIG// }" = "" ]; then
-            echo "KUBE_CONFIG is not set or is empty."
+          unformatted="$(gofmt -l *.go)"
+          if [ -n "$unformatted" ]; then
+            echo "The following Go files need formatting:"
+            echo "$unformatted"
             exit 1
           fi
 
+      - name: Verify Docker access
+        run: docker info
+
+      - name: Run static analysis
+        working-directory: haven-notify
+        run: go vet ./...
+
+      - name: Run race tests and collect coverage
+        working-directory: haven-notify
+        run: go test -race -shuffle=on -count=1 -covermode=atomic -coverprofile=coverage.out ./...
+
+      - name: Enforce coverage threshold
+        working-directory: haven-notify
+        run: |
+          coverage="$(go tool cover -func=coverage.out | awk '/^total:/ {gsub("%", "", $3); print $3}')"
+          echo "Statement coverage: ${coverage}%"
+          awk -v coverage="$coverage" 'BEGIN { exit !(coverage + 0 >= 90) }'
+
+      - name: Run fuzz smoke test
+        working-directory: haven-notify
+        run: go test -run=^$ -fuzz=FuzzErrorNotificationHandler -fuzztime=10s .
+
+      - name: Run container end-to-end tests
+        working-directory: haven-notify
+        run: go test -tags=e2e -run '^TestContainerE2E$' -count=1 -timeout=5m -v .
+
+  build:
+    name: Build Haven Notify Image
+    runs-on: runner-slim
+    needs: test
+    if: ${{ (gitea.event_name == 'push' || gitea.event_name == 'workflow_dispatch') && gitea.ref == 'refs/heads/main' }}
+    steps:
       - name: Check out repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Download and install dependencies
-        run: |
-          apt-get update -y
-          apt-get install -y curl
-          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-          install -m 0755 kubectl /usr/local/bin/kubectl
-          kubectl version --client
+      - name: Build and Push Multi-Arch Image
+        uses: https://git.ivanch.me/ivanch/pipeline-actions/build-and-push@main
+        with:
+          image: ${{ env.IMAGE }}
+          registry_password: ${{ secrets.REGISTRY_PASSWORD }}
+          ssh_key: ${{ secrets.SSH_KEY_DOCKERBUILD }}
+          build_context: haven-notify
 
-      - name: Set up kubeconfig
-        run: |
-          cd haven-notify/deploy
-          echo "$KUBE_CONFIG" > kubeconfig.yaml
-        env:
-          KUBE_CONFIG: ${{ env.KUBE_CONFIG }}
+  build-pr:
+    name: Build Haven Notify Image (PR)
+    runs-on: runner-full-amd64
+    needs: test
+    if: ${{ gitea.event_name == 'pull_request' }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
 
-      - name: Test connection to cluster
-        run: |
-          cd haven-notify/deploy
-          kubectl --kubeconfig=kubeconfig.yaml cluster-info
+      - name: Build Docker image
+        run: docker build --pull --tag haven-notify:pr-${{ gitea.sha }} haven-notify
 
-      - name: Apply haven-notify deployment
-        run: |
-          cd haven-notify/deploy
-          kubectl --kubeconfig=kubeconfig.yaml apply -f haven-notify.yaml
-
-      - name: Rollout restart haven-notify
-        run: |
-          cd haven-notify/deploy
-          kubectl --kubeconfig=kubeconfig.yaml rollout restart deployment/haven-notify
+  deploy:
+    name: Deploy Haven Notify (internal)
+    runs-on: runner-slim
+    needs: build
+    if: ${{ (gitea.event_name == 'push' || gitea.event_name == 'workflow_dispatch') && gitea.ref == 'refs/heads/main' }}
+    steps:
+      - name: Rollout Restart Haven Notify
+        uses: https://git.ivanch.me/ivanch/pipeline-actions/deploy-restart@main
+        with:
+          kube_config: ${{ secrets.KUBE_CONFIG }}
+          deployment_name: haven-notify
+          namespace: infra

.gitea/workflows/main.yaml

@@ -3,7 +3,7 @@ on: [push]
 
 jobs:
   check-scripts-syntax:
-    runs-on: ubuntu-latest
+    runs-on: runner-slim
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
@@ -13,3 +13,4 @@ jobs:
       - name: Check scripts syntax
         run: |
           bash -n ${{ gitea.workspace }}/*.sh
+          bash -n ${{ gitea.workspace }}/k8s/*.sh

.gitea/workflows/test-automated-nfs-backup.yaml

@@ -0,0 +1,26 @@
+name: Test Kubernetes backup scripts
+
+on:
+  push:
+    paths:
+      - "k8s/**"
+      - ".gitea/workflows/**"
+  workflow_dispatch: {}
+
+jobs:
+  test-automated-nfs-backup:
+    runs-on: runner-slim
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Run automated NFS backup tests
+        run: bash k8s/test/test-automated-nfs-backup.sh
+  test-k3s-control-plane-config-backup:
+    runs-on: runner-slim
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Install GNU coreutils and findutils for control-plane tests
+        run: apk add --no-cache coreutils findutils
+      - name: Run k3s control-plane configuration backup tests
+        run: bash k8s/test/test-backup-k3s-control-plane-config.sh

README.md

@@ -1,30 +1,65 @@
 # server-scripts
 
-Useful scripts for managing servers written in Bash. Supported OS includes only Ubuntu/Debian and Alpine.
+Useful scripts for managing servers written in Bash. Supported OS includes Debian/Ubuntu, Alpine, Arch, RHEL/Fedora/CentOS, openSUSE, Void, and Gentoo.
 
-They feature auto-updating, and some error handling.
+In the past I was using it more for maintaining my home servers, bare metal with Docker, however when I changed to k3s, I started using it less. Now I'm using this as a collection of useful scripts that I can use when needed. Also `housekeeping.sh` is still useful.
 
-## `scripts-download.sh`
+### `housekeeping.sh`
+
+Unified system housekeeping script for package index updates, system cleanup, and oh-my-zsh updates.
+
+Subcommands:
+- `update` - detects the running OS and updates the package index only. It does not upgrade or install packages.
+- `clean` - runs system cleanup for Docker resources, package manager caches, temporary/cache/log directories, systemd journal, thumbnails, and memory caches.
+- `omz` - updates oh-my-zsh when `omz` is installed; otherwise it skips safely.
+- `all` - runs `update`, then `clean`, then `omz`. This is the default when no subcommand is provided.
+
+The `update` subcommand supports Alpine, Arch, Debian/Ubuntu, RHEL/Fedora/CentOS, openSUSE, Void, and Gentoo family distributions.
+
+Run with `curl | bash`:
 
-This script is used to download selected scripts in this repository to a specified directory and install them as cron jobs.
 ```bash
-curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/scripts-download.sh | bash
+curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/housekeeping.sh | bash
 ```
 
-When you run the script, you will be prompted to select which scripts you want to download and install.
+Run a specific subcommand:
 
-### `backup.sh`
+```bash
+curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/housekeeping.sh | bash -s -- clean
+```
 
-This script is used to backup a directory to a remote server using `rsync`. It is intended to be run as a cron job.
+### `windows-backup.ps1`
 
-#### `nas-gdrive-backup.sh`
+This PowerShell script is used to backup files from a Windows machine to a remote Samba server. It uses `7zip` to create the zip file, and then sends over the network using SMB protocol.
 
-This script is used to both install itself and to run a periodic backup to Google Drive for files that had changed.
+### `dns-override.sh`
 
-### `clean.sh`
+Aggressive, best-effort DNS override script for Linux hosts with support for multiple resolver stacks (`systemd-resolved`, `NetworkManager`, `resolvconf/openresolv`, `dhclient`, `systemd-networkd`, and `ifupdown`) plus direct `/etc/resolv.conf` writes.
 
-This script is used to clean some of the files, docker dangling images, and docker stopped/unused containers.
+It prompts for:
+- 2 required DNS servers
+- 1 optional third DNS server
 
-### `docker-updater.sh`
+Validation/features:
+- Accepts IPv4 and IPv6 literals
+- Rejects invalid or duplicate entries
+- Performs reachability checks (warn-only, does not block execution)
+- Can optionally lock `/etc/resolv.conf` with `chattr +i`
 
-This script is used to update all the docker containers on the server.
+Run with `curl | bash`:
+
+```bash
+curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/dns-override.sh | bash
+```
+
+Important notes:
+- Requires interactive TTY input (prompts are read from `/dev/tty`)
+- Requires root privileges (`sudo` is used/re-execed when possible)
+- Script is intentionally aggressive and may restart/reload network services
+- On heavily managed systems (custom agents, immutable images, enterprise policy), behavior is still best-effort
+
+## Haven Notify
+
+It's a small internal service designed to send notifications to a specified Discord channel via webhooks. It's written in Go and can be easily deployed.
+
+The project is defined on [/haven-notify](./haven-notify) folder.

backup.sh

@@ -1,91 +0,0 @@
-#!/bin/bash
-
-### AUTO-UPDATER ###
-# Variables
-SERVER_FILE="https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/backup.sh"
-SERVER_OK=1
-HOSTNAME=$(cat /etc/hostname)
-
-NOTIFY_URL="http://notify.haven/notify"
-
-# Function to send notification
-send_notify() {
-    local title="$1"
-    local message="$2"
-    curl -s -X POST "$NOTIFY_URL" \
-        -H "Content-Type: application/json" \
-        -d "{\"title\": \"Docker Backup - $HOSTNAME\", \"message\": \"$message\"}"
-}
-
-# Check if the server file exists
-curl -s --head $SERVER_FILE | head -n 1 | grep -E "HTTP/[12] [23].." > /dev/null
-
-if [ $? -ne 0 ]; then
-    echo "Error: $SERVER_FILE not found." >&2
-    send_notify "" "❌ Server file not found: $SERVER_FILE"
-    SERVER_OK=0
-fi
-
-if [ $SERVER_OK -eq 1 ]; then
-    echo "Running auto-update..."
-
-    # Compare the local and server files sha256sum to check if an update is needed
-    LOCAL_SHA256=$(sha256sum backup.sh | awk '{print $1}')
-    SERVER_SHA256=$(curl -s $SERVER_FILE | sha256sum | awk '{print $1}')
-
-    if [ "$LOCAL_SHA256" != "$SERVER_SHA256" ]; then
-        echo "Updating backup.sh..."
-        curl -s -o backup.sh $SERVER_FILE
-        echo "backup.sh updated."
-
-        chmod +x backup.sh
-        echo "Permissions set up."
-
-        echo "Running updated backup.sh..."
-        ./backup.sh
-        exit 0
-    else
-        echo "backup.sh is up to date.."
-    fi
-fi
-
-####################
-
-# Variables
-SOURCE_DIR="/root/docker"
-BACKUP_FILE="/tmp/docker_backup_$(date +%Y%m%d%H%M%S).tar.gz"
-REMOTE_USER="ivanch"
-REMOTE_HOST="nas.haven"
-REMOTE_DIR="/export/Backup/Docker/$(cat /etc/hostname)"
-
-# Create a compressed backup file
-zip -q -r $BACKUP_FILE $SOURCE_DIR || true
-if [ $? -ne 0 ]; then
-    send_notify "" "⚠️ Some files or folders in $SOURCE_DIR could not be backed up (possibly in use or locked). Backup archive created with available files."
-fi
-
-# Check if remote path exists
-if ! ssh $REMOTE_USER@$REMOTE_HOST "mkdir -p $REMOTE_DIR"; then
-    send_notify "" "❌ Failed to create remote directory: $REMOTE_DIR on $REMOTE_HOST"
-    exit 1
-fi
-
-# Transfer the backup file to the remote server
-if ! scp $BACKUP_FILE $REMOTE_USER@$REMOTE_HOST:$REMOTE_DIR; then
-    send_notify "" "❌ Failed to transfer backup file to remote server: $REMOTE_HOST:$REMOTE_DIR"
-    exit 1
-fi
-
-# Remove the backup file
-rm $BACKUP_FILE
-
-# Erase last 7 days backups from remote server
-if ! ssh $REMOTE_USER@$REMOTE_HOST "find $REMOTE_DIR -type f -name 'docker_backup_*' -mtime +7 -exec rm {} \;"; then
-    send_notify "" "⚠️ Failed to clean old backups on remote server: $REMOTE_HOST:$REMOTE_DIR"
-fi
-
-# Success notification
-send_notify "" "✅ Backup completed successfully for: $SOURCE_DIR to $REMOTE_HOST:$REMOTE_DIR"
-echo "Backup completed successfully"
-exit 0
-

dns-override.sh

@@ -0,0 +1,815 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly SCRIPT_NAME="dns-override.sh"
+readonly SCRIPT_VERSION="1.0.2"
+readonly IMMUTABLE_PROMPT_TIMEOUT_SEC=20
+
+readonly NC='\033[0m'
+readonly RED='\033[1;31m'
+readonly GREEN='\033[1;32m'
+readonly YELLOW='\033[1;33m'
+readonly BLUE='\033[1;34m'
+readonly GREY='\033[0;37m'
+
+readonly EMOJI_INFO='ℹ️'
+readonly EMOJI_OK='✅'
+readonly EMOJI_WARN='⚠️'
+readonly EMOJI_ERROR='❌'
+readonly EMOJI_GEAR='🛠️'
+readonly EMOJI_ROCKET='🚀'
+readonly EMOJI_NET='🌐'
+readonly EMOJI_LOCK='🔒'
+
+SUDO_BIN=""
+IS_REMOTE_SSH=0
+IMMUTABLE_LOCK_REQUESTED=0
+declare -a DNS_SERVERS=()
+declare -a DETECTED_MANAGERS=()
+declare -a ACTIONS_ATTEMPTED=()
+declare -a ACTIONS_SUCCEEDED=()
+declare -a ACTIONS_FAILED=()
+declare -a ACTIONS_SKIPPED=()
+
+log_info() { printf "%b%s %s%b\n" "$GREY" "$EMOJI_INFO" "$1" "$NC"; }
+log_success() { printf "%b%s %s%b\n" "$GREEN" "$EMOJI_OK" "$1" "$NC"; }
+log_warning() { printf "%b%s %s%b\n" "$YELLOW" "$EMOJI_WARN" "$1" "$NC"; }
+log_error() { printf "%b%s %s%b\n" "$RED" "$EMOJI_ERROR" "$1" "$NC" >&2; }
+log_step() { printf "%b%s %s%b\n" "$BLUE" "$EMOJI_GEAR" "$1" "$NC"; }
+
+die() {
+  log_error "$1"
+  exit 1
+}
+
+command_exists() {
+  command -v "$1" >/dev/null 2>&1
+}
+
+binary_or_path_exists() {
+  local bin_name="$1"
+  shift
+
+  if command_exists "$bin_name"; then
+    return 0
+  fi
+
+  local path_candidate
+  for path_candidate in "$@"; do
+    [[ -x "$path_candidate" ]] && return 0
+  done
+  return 1
+}
+
+systemd_unit_exists() {
+  local unit_name="$1"
+  command_exists systemctl || return 1
+  systemctl list-unit-files "$unit_name" >/dev/null 2>&1
+}
+
+trim() {
+  local value="$1"
+  value="${value#"${value%%[![:space:]]*}"}"
+  value="${value%"${value##*[![:space:]]}"}"
+  printf '%s' "$value"
+}
+
+contains_value() {
+  local needle="$1"
+  shift
+  local item
+  for item in "$@"; do
+    [[ "$item" == "$needle" ]] && return 0
+  done
+  return 1
+}
+
+is_valid_ipv4() {
+  local ip="$1"
+  [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || return 1
+
+  local o1 o2 o3 o4
+  IFS='.' read -r o1 o2 o3 o4 <<<"$ip"
+  local octet
+  for octet in "$o1" "$o2" "$o3" "$o4"; do
+    [[ "$octet" =~ ^[0-9]+$ ]] || return 1
+    ((octet >= 0 && octet <= 255)) || return 1
+  done
+  return 0
+}
+
+is_valid_ipv6() {
+  local ip="$1"
+  local ipv6_re='^(([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:)|([0-9A-Fa-f]{1,4}:){1,7}:|([0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|([0-9A-Fa-f]{1,4}:){1,5}(:[0-9A-Fa-f]{1,4}){1,2}|([0-9A-Fa-f]{1,4}:){1,4}(:[0-9A-Fa-f]{1,4}){1,3}|([0-9A-Fa-f]{1,4}:){1,3}(:[0-9A-Fa-f]{1,4}){1,4}|([0-9A-Fa-f]{1,4}:){1,2}(:[0-9A-Fa-f]{1,4}){1,5}|[0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4}){1,6}|:((:[0-9A-Fa-f]{1,4}){1,7}|:))(%.+)?$'
+  [[ "$ip" =~ $ipv6_re ]]
+}
+
+is_valid_ip() {
+  local ip="$1"
+  is_valid_ipv4 "$ip" || is_valid_ipv6 "$ip"
+}
+
+require_tty() {
+  [[ -r /dev/tty && -w /dev/tty ]] || die "No interactive TTY found. This script needs a terminal for prompts (even when using curl | bash)."
+}
+
+read_tty_input() {
+  local prompt="$1"
+  local value
+  printf "%b" "$prompt" >/dev/tty
+  IFS= read -r value </dev/tty || return 1
+  printf '%s' "$(trim "$value")"
+}
+
+read_tty_input_timeout() {
+  local prompt="$1"
+  local timeout_sec="$2"
+  local value
+  printf "%b" "$prompt" >/dev/tty
+  IFS= read -r -t "$timeout_sec" value </dev/tty || return $?
+  printf '%s' "$(trim "$value")"
+}
+
+run_root() {
+  if [[ -n "$SUDO_BIN" ]]; then
+    "$SUDO_BIN" "$@"
+  else
+    "$@"
+  fi
+}
+
+write_root_file() {
+  local target="$1"
+  local content="$2"
+  if [[ -n "$SUDO_BIN" ]]; then
+    printf '%s' "$content" | "$SUDO_BIN" tee "$target" >/dev/null
+  else
+    printf '%s' "$content" >"$target"
+  fi
+}
+
+append_root_file() {
+  local target="$1"
+  local content="$2"
+  if [[ -n "$SUDO_BIN" ]]; then
+    printf '%s' "$content" | "$SUDO_BIN" tee -a "$target" >/dev/null
+  else
+    printf '%s' "$content" >>"$target"
+  fi
+}
+
+ensure_root_context() {
+  if [[ "$EUID" -eq 0 ]]; then
+    return 0
+  fi
+
+  command_exists sudo || die "This script must run as root or with sudo available."
+
+  local source_path="${BASH_SOURCE[0]:-}"
+  if [[ -n "$source_path" && -f "$source_path" ]]; then
+    log_info "Re-launching with sudo for full root context..."
+    exec sudo -E bash "$source_path" "$@"
+  fi
+
+  log_warning "Running from stdin (likely curl | bash). Falling back to sudo for privileged operations."
+  sudo -v || die "Could not acquire sudo privileges."
+  SUDO_BIN="sudo"
+}
+
+detect_runtime_context() {
+  if [[ -n "${SSH_CONNECTION:-}" || -n "${SSH_TTY:-}" ]]; then
+    IS_REMOTE_SSH=1
+    log_warning "SSH session detected. Using safer apply behavior to reduce connection drops."
+  fi
+}
+
+has_systemd_resolved() {
+  binary_or_path_exists "resolvectl" "/usr/bin/resolvectl" "/bin/resolvectl" && return 0
+  systemd_unit_exists "systemd-resolved.service" && return 0
+  binary_or_path_exists "systemd-resolved" "/lib/systemd/systemd-resolved" "/usr/lib/systemd/systemd-resolved" && return 0
+  return 1
+}
+
+has_networkmanager() {
+  binary_or_path_exists "nmcli" "/usr/bin/nmcli" "/bin/nmcli" && return 0
+  binary_or_path_exists "NetworkManager" "/usr/sbin/NetworkManager" "/sbin/NetworkManager" && return 0
+  systemd_unit_exists "NetworkManager.service" && return 0
+  return 1
+}
+
+has_resolvconf_openresolv() {
+  binary_or_path_exists "resolvconf" "/sbin/resolvconf" "/usr/sbin/resolvconf" && return 0
+  return 1
+}
+
+has_dhclient() {
+  binary_or_path_exists "dhclient" "/sbin/dhclient" "/usr/sbin/dhclient" && return 0
+  return 1
+}
+
+has_systemd_networkd() {
+  systemd_unit_exists "systemd-networkd.service" && return 0
+  binary_or_path_exists "systemd-networkd" "/lib/systemd/systemd-networkd" "/usr/lib/systemd/systemd-networkd" && return 0
+  return 1
+}
+
+has_ifupdown() {
+  binary_or_path_exists "ifup" "/sbin/ifup" "/usr/sbin/ifup" && return 0
+  [[ -f /etc/network/interfaces || -d /etc/network/interfaces.d ]] && return 0
+  return 1
+}
+
+add_manager() {
+  local manager="$1"
+  if ! contains_value "$manager" "${DETECTED_MANAGERS[@]}"; then
+    DETECTED_MANAGERS+=("$manager")
+  fi
+}
+
+detect_managers() {
+  DETECTED_MANAGERS=()
+
+  if has_systemd_resolved; then
+    add_manager "systemd-resolved"
+  fi
+
+  if has_networkmanager; then
+    add_manager "NetworkManager"
+  fi
+
+  if has_resolvconf_openresolv; then
+    add_manager "resolvconf/openresolv"
+  fi
+
+  if has_dhclient; then
+    add_manager "dhclient"
+  fi
+
+  if has_systemd_networkd; then
+    add_manager "systemd-networkd"
+  fi
+
+  if has_ifupdown; then
+    add_manager "ifupdown"
+  fi
+}
+
+prompt_dns_servers() {
+  DNS_SERVERS=()
+  local input
+  local prompt
+  local index
+
+  for index in 1 2; do
+    while true; do
+      prompt="${BLUE}${EMOJI_NET} Enter DNS server ${index} (required): ${NC}"
+      input="$(read_tty_input "$prompt")" || die "Failed to read DNS input from TTY."
+
+      [[ -n "$input" ]] || {
+        log_warning "DNS server ${index} is required."
+        continue
+      }
+
+      if ! is_valid_ip "$input"; then
+        log_warning "Invalid IP literal: '$input'. Please enter a valid IPv4 or IPv6 address."
+        continue
+      fi
+
+      if contains_value "$input" "${DNS_SERVERS[@]}"; then
+        log_warning "Duplicate DNS server '$input' is not allowed."
+        continue
+      fi
+
+      DNS_SERVERS+=("$input")
+      break
+    done
+  done
+
+  while true; do
+    prompt="${BLUE}${EMOJI_NET} Enter DNS server 3 (optional, press Enter to skip): ${NC}"
+    input="$(read_tty_input "$prompt")" || die "Failed to read DNS input from TTY."
+
+    if [[ -z "$input" ]]; then
+      break
+    fi
+
+    if ! is_valid_ip "$input"; then
+      log_warning "Invalid IP literal: '$input'. Please enter a valid IPv4 or IPv6 address."
+      continue
+    fi
+
+    if contains_value "$input" "${DNS_SERVERS[@]}"; then
+      log_warning "Duplicate DNS server '$input' is not allowed."
+      continue
+    fi
+
+    DNS_SERVERS+=("$input")
+    break
+  done
+
+  log_success "DNS server list accepted: ${DNS_SERVERS[*]}"
+}
+
+check_dns_connectivity() {
+  local dns="$1"
+
+  if command_exists dig; then
+    dig @"$dns" example.com A +time=2 +tries=1 +short >/dev/null 2>&1 && return 0
+    dig @"$dns" . NS +time=2 +tries=1 +short >/dev/null 2>&1 && return 0
+    return 1
+  fi
+
+  if command_exists drill; then
+    drill @"$dns" example.com A >/dev/null 2>&1 && return 0
+    drill @"$dns" . NS >/dev/null 2>&1 && return 0
+    return 1
+  fi
+
+  if command_exists nslookup; then
+    nslookup -timeout=2 example.com "$dns" >/dev/null 2>&1 && return 0
+    nslookup example.com "$dns" >/dev/null 2>&1 && return 0
+    return 1
+  fi
+
+  if command_exists nc; then
+    nc -z -w2 "$dns" 53 >/dev/null 2>&1 && return 0
+    return 1
+  fi
+
+  if [[ "$dns" != *:* ]] && command_exists timeout; then
+    timeout 2 bash -c "exec 3<>/dev/tcp/$dns/53" >/dev/null 2>&1 && return 0
+    return 1
+  fi
+
+  return 2
+}
+
+run_connectivity_checks() {
+  log_step "Running best-effort connectivity checks for supplied DNS servers..."
+
+  local dns
+  local rc
+  for dns in "${DNS_SERVERS[@]}"; do
+    if check_dns_connectivity "$dns"; then
+      log_success "Reachability check passed for $dns"
+      continue
+    fi
+
+    rc=$?
+    if [[ "$rc" -eq 2 ]]; then
+      log_warning "No available tool for probing $dns on this host. Proceeding anyway."
+    else
+      log_warning "Could not reach DNS server $dns during probe. Proceeding as requested."
+    fi
+  done
+}
+
+dns_nameserver_lines() {
+  local lines=""
+  local dns
+  for dns in "${DNS_SERVERS[@]}"; do
+    lines+="nameserver ${dns}"$'\n'
+  done
+  printf '%s' "$lines"
+}
+
+dns_space_list() {
+  printf '%s' "${DNS_SERVERS[*]}"
+}
+
+dns_comma_list() {
+  local out=""
+  local dns
+  for dns in "${DNS_SERVERS[@]}"; do
+    if [[ -z "$out" ]]; then
+      out="$dns"
+    else
+      out+=",${dns}"
+    fi
+  done
+  printf '%s' "$out"
+}
+
+run_action() {
+  local action="$1"
+  shift
+
+  ACTIONS_ATTEMPTED+=("$action")
+  log_step "$action"
+
+  local rc=0
+  if "$@"; then
+    ACTIONS_SUCCEEDED+=("$action")
+    log_success "$action completed"
+    return 0
+  fi
+
+  rc=$?
+  if [[ "$rc" -eq 2 ]]; then
+    ACTIONS_SKIPPED+=("$action")
+    log_info "$action skipped: target manager is not present on this host"
+    return 0
+  fi
+
+  ACTIONS_FAILED+=("$action")
+  log_warning "$action failed (best-effort mode: continuing)"
+  return 0
+}
+
+ensure_resolv_conf_mutable_if_needed() {
+  if command_exists lsattr && command_exists chattr; then
+    local attr_line attr_field
+    attr_line="$(run_root lsattr /etc/resolv.conf 2>/dev/null || true)"
+    attr_field="${attr_line%% *}"
+    if [[ "$attr_field" == *i* ]]; then
+      log_warning "/etc/resolv.conf is immutable. Temporarily removing immutable flag for update."
+      run_root chattr -i /etc/resolv.conf || return 1
+    fi
+  fi
+  return 0
+}
+
+apply_static_resolv_conf() {
+  ensure_resolv_conf_mutable_if_needed || return 1
+
+  local now lines content
+  now="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+  lines="$(dns_nameserver_lines)"
+  content="# Managed by ${SCRIPT_NAME} v${SCRIPT_VERSION} at ${now}"$'\n'
+  content+="$lines"
+
+  write_root_file "/etc/resolv.conf" "$content" || return 1
+  run_root chmod 644 /etc/resolv.conf || true
+  return 0
+}
+
+apply_systemd_resolved() {
+  if ! has_systemd_resolved; then
+    log_info "Skipping systemd-resolved override: systemd-resolved is not present."
+    return 2
+  fi
+
+  local dns_line content
+  dns_line="$(dns_space_list)"
+  content="[Resolve]"$'\n'
+  content+="DNS=${dns_line}"$'\n'
+  content+="FallbackDNS="$'\n'
+  content+="Domains=~."$'\n'
+
+  run_root mkdir -p /etc/systemd/resolved.conf.d || return 1
+  write_root_file "/etc/systemd/resolved.conf.d/99-dns-override.conf" "$content" || return 1
+
+  if command_exists systemctl; then
+    run_root systemctl daemon-reload >/dev/null 2>&1 || true
+    run_root systemctl restart systemd-resolved >/dev/null 2>&1 || run_root systemctl try-restart systemd-resolved >/dev/null 2>&1 || true
+  fi
+
+  if command_exists resolvectl; then
+    if command_exists ip; then
+      local link
+      while IFS= read -r link; do
+        [[ -n "$link" && "$link" != "lo" ]] || continue
+        run_root resolvectl dns "$link" "${DNS_SERVERS[@]}" >/dev/null 2>&1 || true
+        run_root resolvectl domain "$link" "~." >/dev/null 2>&1 || true
+      done < <(ip -o link show | awk -F': ' '{print $2}' | cut -d'@' -f1)
+    fi
+    run_root resolvectl flush-caches >/dev/null 2>&1 || true
+  fi
+
+  return 0
+}
+
+apply_networkmanager() {
+  if ! has_networkmanager; then
+    log_info "Skipping NetworkManager override: NetworkManager is not present."
+    return 2
+  fi
+
+  local dns_csv nm_conf
+  dns_csv="$(dns_comma_list)"
+  nm_conf="[main]"$'\n'
+  nm_conf+="dns=default"$'\n'
+  nm_conf+="rc-manager=symlink"$'\n'
+
+  run_root mkdir -p /etc/NetworkManager/conf.d || return 1
+  write_root_file "/etc/NetworkManager/conf.d/90-dns-override.conf" "$nm_conf" || return 1
+
+  if command_exists nmcli; then
+    local conn
+    while IFS= read -r conn; do
+      [[ -n "$conn" ]] || continue
+      run_root nmcli connection modify "$conn" ipv4.ignore-auto-dns yes >/dev/null 2>&1 || true
+      run_root nmcli connection modify "$conn" ipv6.ignore-auto-dns yes >/dev/null 2>&1 || true
+      run_root nmcli connection modify "$conn" ipv4.dns "$dns_csv" >/dev/null 2>&1 || true
+      run_root nmcli connection modify "$conn" ipv6.dns "$dns_csv" >/dev/null 2>&1 || true
+    done < <(nmcli -g NAME connection show --active 2>/dev/null || true)
+
+    run_root nmcli connection reload >/dev/null 2>&1 || true
+
+    if [[ "$IS_REMOTE_SSH" -eq 1 ]]; then
+      log_warning "Skipping NetworkManager reconnect/restart operations in SSH session."
+      local dev
+      while IFS= read -r dev; do
+        [[ -n "$dev" && "$dev" != "lo" ]] || continue
+        run_root nmcli device reapply "$dev" >/dev/null 2>&1 || true
+      done < <(nmcli -t -f DEVICE,STATE device status 2>/dev/null | awk -F: '$2=="connected" || $2=="connecting" {print $1}')
+    else
+      while IFS= read -r conn; do
+        [[ -n "$conn" ]] || continue
+        run_root nmcli connection up "$conn" >/dev/null 2>&1 || true
+      done < <(nmcli -g NAME connection show --active 2>/dev/null || true)
+    fi
+
+    run_root nmcli general reload >/dev/null 2>&1 || true
+  fi
+
+  if command_exists systemctl; then
+    run_root systemctl reload NetworkManager >/dev/null 2>&1 || true
+  fi
+
+  return 0
+}
+
+apply_resolvconf_openresolv() {
+  if ! has_resolvconf_openresolv; then
+    log_info "Skipping resolvconf/openresolv override: resolvconf/openresolv is not present."
+    return 2
+  fi
+
+  local lines
+  lines="$(dns_nameserver_lines)"
+
+  run_root mkdir -p /etc/resolvconf/resolv.conf.d || true
+  write_root_file "/etc/resolvconf/resolv.conf.d/head" "$lines" || true
+  write_root_file "/etc/resolvconf/resolv.conf.d/base" "$lines" || true
+
+  if command_exists resolvconf; then
+    printf '%s' "$lines" | run_root resolvconf -a dns-override >/dev/null 2>&1 || true
+    run_root resolvconf -u >/dev/null 2>&1 || true
+  fi
+
+  return 0
+}
+
+apply_dhclient() {
+  if ! has_dhclient; then
+    log_info "Skipping dhclient override: dhclient is not present."
+    return 2
+  fi
+
+  local conf_path="/etc/dhcp/dhclient.conf"
+  local dns_csv block
+  dns_csv="$(dns_comma_list)"
+  block=$'\n'"# dns-override.sh managed begin"$'\n'
+  block+="supersede domain-name-servers ${dns_csv};"$'\n'
+  block+="prepend domain-name-servers ${dns_csv};"$'\n'
+  block+="# dns-override.sh managed end"$'\n'
+
+  run_root mkdir -p /etc/dhcp || true
+  run_root touch "$conf_path" || return 1
+  run_root sed -i '/# dns-override.sh managed begin/,/# dns-override.sh managed end/d' "$conf_path" || true
+  append_root_file "$conf_path" "$block" || return 1
+
+  return 0
+}
+
+apply_systemd_networkd() {
+  if ! has_systemd_networkd; then
+    log_info "Skipping systemd-networkd override: systemd-networkd is not present."
+    return 2
+  fi
+
+  local dns_lines dropin_content
+  dns_lines=""
+  local dns
+  for dns in "${DNS_SERVERS[@]}"; do
+    dns_lines+="DNS=${dns}"$'\n'
+  done
+
+  run_root mkdir -p /etc/systemd/network || return 1
+
+  local has_network_file=0
+  local net_file
+  for net_file in /etc/systemd/network/*.network; do
+    [[ -e "$net_file" ]] || continue
+    has_network_file=1
+    run_root mkdir -p "${net_file}.d" || true
+    dropin_content="[Network]"$'\n'"${dns_lines}Domains=~."$'\n'
+    write_root_file "${net_file}.d/99-dns-override.conf" "$dropin_content" || true
+  done
+
+  if [[ "$has_network_file" -eq 0 ]]; then
+    dropin_content="[Match]"$'\n'"Name=*"$'\n\n'"[Network]"$'\n'"${dns_lines}Domains=~."$'\n'
+    write_root_file "/etc/systemd/network/99-dns-override.network" "$dropin_content" || true
+  fi
+
+  if command_exists systemctl; then
+    run_root systemctl daemon-reload >/dev/null 2>&1 || true
+    if command_exists networkctl; then
+      run_root networkctl reload >/dev/null 2>&1 || true
+      if [[ "$IS_REMOTE_SSH" -eq 0 ]]; then
+        run_root networkctl reconfigure --all >/dev/null 2>&1 || true
+      else
+        log_warning "Skipping networkctl reconfigure --all in SSH session."
+      fi
+    elif [[ "$IS_REMOTE_SSH" -eq 0 ]]; then
+      run_root systemctl restart systemd-networkd >/dev/null 2>&1 || run_root systemctl try-restart systemd-networkd >/dev/null 2>&1 || true
+    else
+      log_warning "Skipping systemd-networkd restart in SSH session."
+    fi
+  fi
+
+  return 0
+}
+
+patch_ifupdown_file() {
+  local file_path="$1"
+  [[ -f "$file_path" ]] || return 1
+
+  local dns_line
+  dns_line="$(dns_space_list)"
+  local temp_file
+  temp_file="$(mktemp)"
+
+  set +e
+  awk -v dns="$dns_line" '
+    BEGIN { iface_count = 0 }
+    /^[[:space:]]*dns-nameservers[[:space:]]+/ { next }
+    {
+      print
+      if ($0 ~ /^[[:space:]]*iface[[:space:]]+/ && $0 !~ /[[:space:]]lo[[:space:]]/) {
+        print "    dns-nameservers " dns
+        iface_count++
+      }
+    }
+    END {
+      if (iface_count == 0) {
+        exit 2
+      }
+    }
+  ' "$file_path" >"$temp_file"
+  local rc=$?
+  set -e
+  if [[ "$rc" -eq 2 ]]; then
+    rm -f "$temp_file"
+    return 2
+  fi
+  [[ "$rc" -eq 0 ]] || {
+    rm -f "$temp_file"
+    return 1
+  }
+
+  if [[ -n "$SUDO_BIN" ]]; then
+    "$SUDO_BIN" mv "$temp_file" "$file_path"
+  else
+    mv "$temp_file" "$file_path"
+  fi
+  return 0
+}
+
+apply_ifupdown() {
+  if ! has_ifupdown; then
+    log_info "Skipping ifupdown override: ifupdown is not present."
+    return 2
+  fi
+
+  local touched=0
+  local patch_rc=0
+
+  if [[ -f /etc/network/interfaces ]]; then
+    if patch_ifupdown_file /etc/network/interfaces; then
+      touched=1
+    else
+      patch_rc=$?
+      [[ "$patch_rc" -eq 2 ]] || return 1
+    fi
+  fi
+
+  if [[ -d /etc/network/interfaces.d ]]; then
+    local iface_file rc
+    for iface_file in /etc/network/interfaces.d/*; do
+      [[ -f "$iface_file" ]] || continue
+      if patch_ifupdown_file "$iface_file"; then
+        touched=1
+      else
+        rc=$?
+        [[ "$rc" -eq 2 ]] || return 1
+      fi
+    done
+  fi
+
+  if [[ "$touched" -eq 0 ]]; then
+    log_warning "ifupdown detected, but no writable iface stanzas were found to patch automatically."
+    log_info "Actionable hint: add 'dns-nameservers $(dns_space_list)' inside each target iface block."
+  fi
+
+  return 0
+}
+
+prompt_immutable_lock() {
+  command_exists chattr || {
+    log_info "chattr not available; skipping immutable lock prompt."
+    return 0
+  }
+
+  local answer
+  if ! answer="$(read_tty_input_timeout "${YELLOW}${EMOJI_LOCK} Make /etc/resolv.conf immutable with chattr +i? [y/N] (auto No in ${IMMUTABLE_PROMPT_TIMEOUT_SEC}s): ${NC}" "$IMMUTABLE_PROMPT_TIMEOUT_SEC")"; then
+    local rc=$?
+    if [[ "$rc" -gt 128 ]]; then
+      log_warning "No response received for immutable lock prompt; defaulting to No."
+      IMMUTABLE_LOCK_REQUESTED=0
+      return 0
+    fi
+    return 1
+  fi
+
+  case "$answer" in
+    y|Y|yes|YES)
+      IMMUTABLE_LOCK_REQUESTED=1
+      log_info "Immutable lock requested. It will be applied after DNS updates."
+      ;;
+    *)
+      IMMUTABLE_LOCK_REQUESTED=0
+      log_info "Immutable lock skipped."
+      ;;
+  esac
+}
+
+apply_immutable_lock_if_requested() {
+  if [[ "$IMMUTABLE_LOCK_REQUESTED" -ne 1 ]]; then
+    return 0
+  fi
+
+  if run_root chattr +i /etc/resolv.conf; then
+    log_success "Immutable lock applied to /etc/resolv.conf."
+    log_warning "To edit DNS later, run: sudo chattr -i /etc/resolv.conf"
+  else
+    log_warning "Failed to apply immutable lock."
+  fi
+}
+
+print_summary() {
+  echo
+  log_step "Execution summary"
+  log_info "Detected managers: ${DETECTED_MANAGERS[*]:-(none detected)}"
+  log_info "Actions attempted: ${#ACTIONS_ATTEMPTED[@]}"
+  log_success "Actions succeeded: ${#ACTIONS_SUCCEEDED[@]}"
+  log_warning "Actions failed: ${#ACTIONS_FAILED[@]}"
+  log_info "Actions skipped: ${#ACTIONS_SKIPPED[@]}"
+
+  if [[ "${#ACTIONS_FAILED[@]}" -gt 0 ]]; then
+    local failed
+    for failed in "${ACTIONS_FAILED[@]}"; do
+      log_warning "Failed action: $failed"
+    done
+  fi
+
+  echo
+  log_step "Final /etc/resolv.conf"
+  run_root cat /etc/resolv.conf 2>/dev/null || cat /etc/resolv.conf 2>/dev/null || log_warning "Could not read /etc/resolv.conf"
+
+  if command_exists resolvectl; then
+    echo
+    log_step "resolvectl status snapshot"
+    run_root resolvectl status 2>/dev/null || resolvectl status 2>/dev/null || log_warning "Could not get resolvectl status."
+  fi
+
+  if command_exists nmcli; then
+    echo
+    log_step "NetworkManager DNS snapshot"
+    run_root nmcli -f GENERAL.DEVICE,IP4.DNS,IP6.DNS device show 2>/dev/null || nmcli -f GENERAL.DEVICE,IP4.DNS,IP6.DNS device show 2>/dev/null || log_warning "Could not get nmcli DNS snapshot."
+  fi
+}
+
+main() {
+  echo
+  printf "%b%s %s v%s%b\n" "$BLUE" "$EMOJI_ROCKET" "$SCRIPT_NAME" "$SCRIPT_VERSION" "$NC"
+  log_info "Aggressive best-effort DNS override for Linux hosts."
+  log_warning "This script edits multiple DNS manager configs and may restart networking services."
+  echo
+
+  require_tty
+  ensure_root_context "$@"
+  detect_runtime_context
+  prompt_dns_servers
+  prompt_immutable_lock || log_warning "Immutable lock prompt failed; defaulting to unlocked."
+  run_connectivity_checks
+  detect_managers
+
+  run_action "Apply systemd-resolved override" apply_systemd_resolved
+  run_action "Apply NetworkManager override" apply_networkmanager
+  run_action "Apply resolvconf/openresolv override" apply_resolvconf_openresolv
+  run_action "Apply dhclient override" apply_dhclient
+  run_action "Apply systemd-networkd override" apply_systemd_networkd
+  run_action "Apply ifupdown override" apply_ifupdown
+  run_action "Write /etc/resolv.conf directly" apply_static_resolv_conf
+
+  run_action "Apply immutable /etc/resolv.conf lock (if requested)" apply_immutable_lock_if_requested
+  print_summary
+
+  echo
+  log_success "DNS override flow completed."
+}
+
+main "$@"

docker-updater.sh

@@ -1,338 +0,0 @@
-#!/bin/bash
-
-# Docker Container Updater
-# 
-# Description: Automatically updates Docker containers and manages Docker images
-# Features:
-#   - Self-updating capability
-#   - Updates all Docker Compose projects in /root/docker
-#   - Skips containers with .ignore file
-#   - Removes obsolete Docker Compose version attributes
-#   - Cleans up unused Docker images
-# Author: ivanch
-# Version: 2.0
-
-set -euo pipefail  # Exit on error, undefined vars, and pipe failures
-
-#==============================================================================
-# CONFIGURATION
-#==============================================================================
-
-# Color definitions for output formatting
-readonly NC='\033[0m'
-readonly RED='\033[1;31m'
-readonly GREEN='\033[1;32m'
-readonly LIGHT_GREEN='\033[1;32m'
-readonly LIGHT_BLUE='\033[1;34m'
-readonly LIGHT_GREY='\033[0;37m'
-readonly YELLOW='\033[1;33m'
-
-# Script configuration
-readonly SCRIPT_NAME="docker-updater.sh"
-readonly SERVER_BASE_URL="https://git.ivanch.me/ivanch/server-scripts/raw/branch/main"
-readonly DOCKER_FOLDER="/root/docker"
-readonly COMPOSE_FILES=("docker-compose.yml" "docker-compose.yaml" "compose.yaml" "compose.yml")
-
-# Auto-update configuration
-readonly AUTO_UPDATE_ENABLED=true
-
-#==============================================================================
-# UTILITY FUNCTIONS
-#==============================================================================
-
-# Print formatted log messages
-log_info() { echo -e "${LIGHT_GREY}[i] $1${NC}"; }
-log_success() { echo -e "${LIGHT_GREEN}[✓] $1${NC}"; }
-log_warning() { echo -e "${YELLOW}[!] $1${NC}"; }
-log_error() { echo -e "${RED}[x] $1${NC}" >&2; }
-log_step() { echo -e "${LIGHT_BLUE}[i] $1${NC}"; }
-log_container() { echo -e "${LIGHT_BLUE}[$1] $2${NC}"; }
-
-# Exit with error message
-die() {
-    log_error "$1"
-    exit 1
-}
-
-# Check if a command exists
-command_exists() {
-    command -v "$1" >/dev/null 2>&1
-}
-
-# Check if Docker and Docker Compose are available
-check_docker_requirements() {
-    log_info "Checking Docker requirements..."
-    
-    if ! command_exists docker; then
-        die "Docker is not installed or not in PATH"
-    fi
-    
-    if ! docker compose version >/dev/null 2>&1; then
-        die "Docker Compose is not available"
-    fi
-    
-    log_success "Docker requirements satisfied"
-}
-
-# Get SHA256 hash of a file
-get_file_hash() {
-    local file="$1"
-    sha256sum "$file" 2>/dev/null | awk '{print $1}' || echo ""
-}
-
-# Get SHA256 hash from URL content
-get_url_hash() {
-    local url="$1"
-    curl -s "$url" 2>/dev/null | sha256sum | awk '{print $1}' || echo ""
-}
-
-# Check if server file is accessible
-check_server_connectivity() {
-    local url="$1"
-    curl -s --head "$url" | head -n 1 | grep -E "HTTP/[12] [23].." >/dev/null 2>&1
-}
-
-#==============================================================================
-# AUTO-UPDATE FUNCTIONALITY
-#==============================================================================
-
-# Perform self-update if newer version is available
-perform_self_update() {
-    if [[ "$AUTO_UPDATE_ENABLED" != "true" ]]; then
-        log_info "Auto-update is disabled"
-        return 0
-    fi
-    
-    local server_url="$SERVER_BASE_URL/$SCRIPT_NAME"
-    
-    log_step "Checking for script updates..."
-    
-    # Check if server file is accessible
-    if ! check_server_connectivity "$server_url"; then
-        log_warning "Cannot connect to update server, continuing with current version"
-        return 0
-    fi
-    
-    # Compare local and server file hashes
-    local local_hash
-    local server_hash
-    
-    local_hash=$(get_file_hash "$SCRIPT_NAME")
-    server_hash=$(get_url_hash "$server_url")
-    
-    if [[ -z "$local_hash" || -z "$server_hash" ]]; then
-        log_warning "Cannot determine file hashes, skipping update"
-        return 0
-    fi
-    
-    if [[ "$local_hash" != "$server_hash" ]]; then
-        log_info "Update available, downloading new version..."
-        
-        # Create backup of current script
-        local backup_file="${SCRIPT_NAME}.backup.$(date +%s)"
-        cp "$SCRIPT_NAME" "$backup_file" || die "Failed to create backup"
-        
-        # Download updated script
-        if curl -s -o "$SCRIPT_NAME" "$server_url"; then
-            chmod +x "$SCRIPT_NAME" || die "Failed to set executable permissions"
-            log_success "Script updated successfully"
-            
-            log_step "Running updated script..."
-            exec ./"$SCRIPT_NAME" "$@"
-        else
-            # Restore backup on failure
-            mv "$backup_file" "$SCRIPT_NAME"
-            die "Failed to download updated script"
-        fi
-    else
-        log_success "Script is already up to date"
-    fi
-}
-
-#==============================================================================
-# DOCKER COMPOSE MANAGEMENT
-#==============================================================================
-
-# Find the active Docker Compose file in current directory
-find_compose_file() {
-    for compose_file in "${COMPOSE_FILES[@]}"; do
-        if [[ -f "$compose_file" ]]; then
-            echo "$compose_file"
-            return 0
-        fi
-    done
-    return 1
-}
-
-# Remove obsolete version attribute from Docker Compose files
-clean_compose_files() {
-    local container_name="$1"
-    
-    for compose_file in "${COMPOSE_FILES[@]}"; do
-        if [[ -f "$compose_file" ]]; then
-            log_container "$container_name" "Cleaning obsolete version attribute from $compose_file"
-            sed -i '/^version:/d' "$compose_file" || log_warning "Failed to clean $compose_file"
-        fi
-    done
-}
-
-# Check if container should be skipped
-should_skip_container() {
-    [[ -f ".ignore" ]]
-}
-
-# Check if any containers are running in current directory
-has_running_containers() {
-    local running_containers
-    running_containers=$(docker compose ps -q 2>/dev/null || echo "")
-    [[ -n "$running_containers" ]]
-}
-
-# Update a single Docker Compose project
-update_docker_project() {
-    local project_dir="$1"
-    local container_name
-    container_name=$(basename "$project_dir")
-    
-    log_container "$container_name" "Checking for updates..."
-    
-    # Change to project directory
-    cd "$project_dir" || {
-        log_error "Cannot access directory: $project_dir"
-        return 1
-    }
-    
-    # Check if container should be skipped
-    if should_skip_container; then
-        log_container "$container_name" "Skipping (found .ignore file)"
-        return 0
-    fi
-    
-    # Verify compose file exists
-    local compose_file
-    if ! compose_file=$(find_compose_file); then
-        log_container "$container_name" "No Docker Compose file found, skipping"
-        return 0
-    fi
-    
-    # Clean compose files
-    clean_compose_files "$container_name"
-    
-    # Check if containers are running
-    if ! has_running_containers; then
-        log_container "$container_name" "No running containers, skipping update"
-        return 0
-    fi
-    
-    # Stop containers
-    log_container "$container_name" "Stopping containers..."
-    if ! docker compose down >/dev/null 2>&1; then
-        log_error "Failed to stop containers in $container_name"
-        return 1
-    fi
-    
-    # Pull updated images
-    log_container "$container_name" "Pulling updated images..."
-    if ! docker compose pull -q >/dev/null 2>&1; then
-        log_warning "Failed to pull images for $container_name, attempting to restart anyway"
-    fi
-    
-    # Start containers
-    log_container "$container_name" "Starting containers..."
-    if ! docker compose up -d >/dev/null 2>&1; then
-        log_error "Failed to start containers in $container_name"
-        return 1
-    fi
-    
-    log_container "$container_name" "Update completed successfully!"
-    return 0
-}
-
-# Update all Docker Compose projects
-update_all_docker_projects() {
-    log_step "Starting Docker container updates..."
-    
-    # Check if Docker folder exists
-    if [[ ! -d "$DOCKER_FOLDER" ]]; then
-        die "Docker folder not found: $DOCKER_FOLDER"
-    fi
-    
-    # Change to Docker folder
-    cd "$DOCKER_FOLDER" || die "Cannot access Docker folder: $DOCKER_FOLDER"
-    
-    local updated_count=0
-    local failed_count=0
-    local skipped_count=0
-    
-    # Process each subdirectory
-    for project_dir in */; do
-        if [[ -d "$project_dir" ]]; then
-            local project_path="$DOCKER_FOLDER/$project_dir"
-            
-            if update_docker_project "$project_path"; then
-                if should_skip_container; then
-                    ((skipped_count++))
-                else
-                    ((updated_count++))
-                fi
-            else
-                ((failed_count++))
-            fi
-            
-            # Return to Docker folder for next iteration
-            cd "$DOCKER_FOLDER" || die "Cannot return to Docker folder"
-        fi
-    done
-    
-    # Report results
-    log_success "Docker update summary:"
-    log_info "  Updated: $updated_count projects"
-    log_info "  Skipped: $skipped_count projects"
-    if [[ $failed_count -gt 0 ]]; then
-        log_warning "  Failed: $failed_count projects"
-    fi
-}
-
-#==============================================================================
-# DOCKER CLEANUP
-#==============================================================================
-
-# Clean up unused Docker resources
-cleanup_docker_resources() {
-    log_step "Cleaning up unused Docker resources..."
-    
-    # Remove unused images
-    log_info "Removing unused Docker images..."
-    if docker image prune -af >/dev/null 2>&1; then
-        log_success "Docker image cleanup completed"
-    else
-        log_warning "Docker image cleanup failed"
-    fi
-}
-
-#==============================================================================
-# MAIN EXECUTION
-#==============================================================================
-
-main() {
-    log_step "Starting Docker Container Updater"
-    echo
-    
-    # Check requirements
-    check_docker_requirements
-    
-    # Perform self-update if enabled
-    perform_self_update "$@"
-    
-    # Update all Docker projects
-    update_all_docker_projects
-    
-    # Clean up Docker resources
-    cleanup_docker_resources
-    
-    echo
-    log_success "Docker container update process completed!"
-}
-
-# Execute main function with all arguments
-main "$@"

haven-notify/.dockerignore

@@ -0,0 +1,6 @@
+.git
+.gitignore
+assets/
+coverage.out
+README.md
+*_test.go

haven-notify/.gitignore

@@ -0,0 +1 @@
+coverage.out

haven-notify/Dockerfile

@@ -1,16 +1,22 @@
-# Start from the official Golang image for building
-ARG TARGETARCH=amd64
-FROM golang:1.25 AS builder
-WORKDIR /app
-COPY . .
-# Build statically for Linux and selected architecture
-RUN GOOS=linux GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o haven-notify main.go
+ARG BUILDPLATFORM
+FROM --platform=$BUILDPLATFORM golang:1.25.10-alpine3.23@sha256:8d22e29d960bc50cd025d93d5b7c7d220b1ee9aa7a239b3c8f55a57e987e8d45 AS builder
 
-# Use Alpine for running, with CA certificates for TLS
-FROM alpine:latest
+ARG TARGETARCH
+ARG TARGETOS
+
+WORKDIR /src
+COPY go.mod go.sum ./
+COPY *.go ./
+COPY template/ template/
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o /out/haven-notify .
+
+FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659
 WORKDIR /app
-RUN apk --no-cache add ca-certificates
-COPY --from=builder /app/haven-notify .
+RUN apk --no-cache add ca-certificates \
+    && addgroup -S haven-notify \
+    && adduser -S -D -H -G haven-notify haven-notify
+COPY --from=builder --chown=haven-notify:haven-notify /out/haven-notify /app/haven-notify
 EXPOSE 8080
 ENV WEBHOOK_URL=""
+USER haven-notify
 ENTRYPOINT ["/app/haven-notify"]

haven-notify/README.md

@@ -1,21 +1,34 @@
-# Haven Notify
+<div align="center">
+  <img src="./assets/widelogo.png" alt="Haven Notify Logo">
+</div>
 
 ## Overview
-Haven Notify is an internal service designed to send notifications to a specified Discord channel.
 
-It's built in Go and can be deployed as a container or managed service.
+Haven Notify is an internal Go service that validates notification requests, renders Discord-compatible payloads, and delivers them to a configured Discord webhook.
+
+It runs as a standalone binary or container. Templates are embedded in the binary and validated at startup.
 
 ## Prerequisites
-- Go 1.18 or newer
-- Docker
-- A Discord Webhook URL
 
-## API Specification
+- Go 1.25.10
+- A Docker API compatible with Testcontainers for the end-to-end suite
+- A Discord webhook URL for normal service operation
+
+`WEBHOOK_URL` is required and must be an absolute HTTP or HTTPS URL. The process exits before becoming ready when the configuration or embedded templates are invalid.
+
+## API
+
+All notification endpoints require `POST` and `Content-Type: application/json`. Request bodies are limited to 64 KiB, documented fields are required, and unknown fields are accepted for forward compatibility.
+
+### V1 API
+
+The original routes, request bodies, and Discord message appearance remain unchanged.
+
+#### Send Notification
 
-### Send Notification
 - **Endpoint**: `/notify`
-- **Method**: `POST`
 - **Request Body**:
+
 ```json
 {
   "title": "Notification Title",
@@ -23,40 +36,227 @@ It's built in Go and can be deployed as a container or managed service.
 }
 ```
 
-- **Endpoint**: `/template/notify/backup_template`
-- **Method**: `POST`
+#### Send Backup Notification
+
+- **Endpoint**: `/template/notify/backup`
 - **Request Body**:
+
 ```json
 {
-  "title": "Notification Title",
-  "message": "Notification Message",
-  "backupSizeInMB": 500,
+  "title": "Nightly backup",
+  "asset": "Photos",
+  "backupSizeInMB": 1536,
   "extra": [
     {
-      "name": "Additional Info",
-      "value": "Some extra information"
+      "name": "Host",
+      "value": "nebula"
     }
   ]
 }
 ```
 
-## Setup & Usage
+#### Send Update Notification
+
+- **Endpoint**: `/template/notify/update`
+- **Request Body**:
+
+```json
+{
+  "host": "nexus",
+  "asset": "k3s",
+  "time": 42
+}
+```
+
+`time` is expressed in seconds.
+
+#### Send Error Notification
+
+- **Endpoint**: `/template/notify/error`
+- **Request Body**:
+
+```json
+{
+  "caller": "backup-job",
+  "message": "Error while moving file",
+  "critical": true,
+  "extra": [
+    {
+      "name": "Path",
+      "value": "/mnt/backups"
+    }
+  ]
+}
+```
+
+### V2 API
+
+V2 notifications use compact, consistently styled Discord embeds. Colors, icons, timestamps, field ordering, and layout are controlled by Haven Notify rather than the caller.
+
+Every v2 request may include:
+
+- `source`: a nonblank string identifying the caller.
+- `occurredAt`: an RFC3339 timestamp. When omitted, Haven Notify uses the current UTC time.
+- `extra`: an array of nonblank `name` and `value` strings. Short single-line values render inline; multiline or long values render full-width.
+
+V2 preserves caller field order. Fixed notification fields are placed before `extra` fields so that trailing extras are discarded first if Discord's 25-field or 6,000-character embed limits are reached. Oversized text is safely truncated and extra fields may be omitted from the delivered message.
+
+#### Send a Message
+
+- **Endpoint**: `/api/v2/messages`
+- **Required**: `message`
+- **Optional**: `title`, `source`, `occurredAt`, `extra`
+
+```json
+{
+  "title": "Maintenance complete",
+  "message": "The storage job completed successfully.",
+  "source": "scheduler",
+  "extra": [
+    {
+      "name": "Region",
+      "value": "home"
+    }
+  ]
+}
+```
+
+#### Send a Backup Notification
+
+- **Endpoint**: `/api/v2/notifications/backup`
+- **Required**: `asset`, nonnegative integer `sizeBytes`
+- **Optional**: `title`, `source`, nonnegative `durationSeconds`, `occurredAt`, `extra`
+
+```json
+{
+  "title": "Nightly backup",
+  "asset": "Photos",
+  "sizeBytes": 1610612736,
+  "durationSeconds": 125,
+  "source": "restic",
+  "extra": [
+    {
+      "name": "Host",
+      "value": "nebula"
+    }
+  ]
+}
+```
+
+`sizeBytes` is displayed using IEC units such as KiB, MiB, and GiB. Durations use a compact form such as `42.5s`, `2m 5s`, or `1h 2m 3s`.
+
+#### Send an Update Notification
+
+- **Endpoint**: `/api/v2/notifications/update`
+- **Required**: `host`, `asset`, nonnegative `durationSeconds`
+- **Optional**: `source`, `fromVersion`, `toVersion`, `occurredAt`, `extra`
+
+```json
+{
+  "host": "nexus",
+  "asset": "k3s",
+  "durationSeconds": 42.5,
+  "fromVersion": "1.32.5",
+  "toVersion": "1.33.1",
+  "source": "system-updater"
+}
+```
+
+When both versions are supplied, the embed displays a transition such as `1.32.5 → 1.33.1`. A single supplied version is labeled as the previous or installed version.
+
+#### Send an Error Notification
+
+- **Endpoint**: `/api/v2/notifications/error`
+- **Required**: `source`, `message`, `severity`
+- **Optional**: `errorCode`, `occurredAt`, `extra`
+
+```json
+{
+  "source": "backup-job",
+  "message": "The repository is unavailable.",
+  "severity": "critical",
+  "errorCode": "E_REPOSITORY",
+  "extra": [
+    {
+      "name": "Path",
+      "value": "/mnt/backups"
+    }
+  ]
+}
+```
+
+`severity` must be `warning`, `error`, or `critical`; it selects the embed title, icon, and color.
+
+### Health Probes
+
+- `GET` or `HEAD /ready` returns `200 Ready` after configuration and templates initialize.
+- `GET` or `HEAD /live` returns `200 Alive` while the process can serve HTTP.
+
+### Response Statuses
+
+| Status | Meaning |
+|---|---|
+| `200` | Notification delivered |
+| `400` | Invalid or missing request data |
+| `404` | Unknown route or template |
+| `405` | Unsupported method |
+| `415` | Content type is not JSON |
+| `502` | Discord rejected the request or delivery retries failed |
+| `504` | Discord delivery exceeded its timeout |
+
+Haven Notify suppresses Discord mention parsing and safely truncates content to Discord's message and embed limits. Transport errors, rate limits, and server errors receive at most two retries within a ten-second total delivery budget.
+
+## Run
+
+### Go
+
+```sh
+WEBHOOK_URL=https://discord.com/api/webhooks/... go run .
+```
 
 ### Docker
-1. Build the Docker image:
-	```sh
-	docker build -t haven-notify .
-	```
-2. Run the container:
-	```sh
-	docker run -e WEBHOOK_URL=your_webhook_url haven-notify
-	```
 
-### Kubernetes
-Deployment manifest is available at `deploy/haven-notify.yaml`.
-1. Edit the manifest to set your environment variables.
-2. Create a generic secret named `WEBHOOK_URL` with `discord-webhook=your_webhook_url`
-3. Apply deployment:
-	```sh
-	kubectl apply -f deploy/haven-notify.yaml
-	```
+```sh
+docker build -t haven-notify .
+docker run --rm -p 8080:8080 \
+  -e WEBHOOK_URL=https://discord.com/api/webhooks/... \
+  haven-notify
+```
+
+The runtime container uses a non-root `haven-notify` user and contains only the binary, Alpine runtime files, and CA certificates.
+
+## Tests
+
+Run unit and HTTP contract tests:
+
+```sh
+go test ./...
+```
+
+Run static analysis, the race detector, and coverage (the race detector requires CGO support):
+
+```sh
+go vet ./...
+go test -race -shuffle=on -count=1 -covermode=atomic -coverprofile=coverage.out ./...
+go tool cover -func=coverage.out
+```
+
+Run the seeded fuzz target:
+
+```sh
+go test -run=^$ -fuzz=FuzzErrorNotificationHandler -fuzztime=10s .
+```
+
+Run the black-box container suite:
+
+```sh
+go test -tags=e2e -run '^TestContainerE2E$' -count=1 -timeout=5m -v .
+```
+
+The end-to-end suite builds the production Dockerfile, starts Haven Notify and a disposable MockServer on an isolated network, and verifies the complete HTTP-to-Discord payload flow. It never calls a real Discord webhook. Configure `DOCKER_HOST` or the Testcontainers host settings when the Docker daemon is not available through the platform's default socket.
+
+## Deployment
+
+The Kubernetes Deployment, Service, and Ingress manifest lives in the `haven` homelab repository at `infra/haven-notify.yaml`. This repository holds the service source, Dockerfile, tests, and Gitea Actions pipeline.
+
+The production manifest maps the `HAVEN_WEBHOOK_URL` key from the `discord-webhook` secret into the container's `WEBHOOK_URL` environment variable.

haven-notify/deploy/haven-notify.yaml

@@ -1,75 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: haven-notify
-  labels:
-    app: haven-notify
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: haven-notify
-  template:
-    metadata:
-      labels:
-        app: haven-notify
-    spec:
-      containers:
-        - name: haven-notify
-          image: git.ivanch.me/ivanch/haven-notify:latest
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8080
-          env:
-          - name: WEBHOOK_URL
-            valueFrom:
-              secretKeyRef:
-                name: discord-webhook
-                key: HAVEN_WEBHOOK_URL
-          readinessProbe:
-            httpGet:
-              path: /ready
-              port: 8080
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          livenessProbe:
-            httpGet:
-              path: /live
-              port: 8080
-            initialDelaySeconds: 5
-            periodSeconds: 10
-      nodeSelector:
-        kubernetes.io/arch: amd64
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: haven-notify
-spec:
-  selector:
-    app: haven-notify
-  ports:
-    - protocol: TCP
-      port: 8080
-      targetPort: 8080
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: haven-notify
-  namespace: default
-  annotations:
-    traefik.ingress.kubernetes.io/router.entrypoints: web
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: notify.haven
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: haven-notify
-                port:
-                  number: 8080

haven-notify/discord.go

@@ -0,0 +1,282 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+const (
+	deliveryTimeout   = 10 * time.Second
+	attemptTimeout    = 5 * time.Second
+	maximumAttempts   = 3
+	maximumRetryDelay = 2 * time.Second
+	truncationMarker  = "… [truncated]"
+)
+
+var (
+	errWebhookDelivery = errors.New("webhook delivery failed")
+	errWebhookTimeout  = errors.New("webhook delivery timed out")
+)
+
+type sleepFunc func(context.Context, time.Duration) error
+
+type discordClient struct {
+	webhookURL string
+	httpClient *http.Client
+	sleep      sleepFunc
+}
+
+type discordPayload struct {
+	Content         string                 `json:"content,omitempty"`
+	Embeds          []discordEmbed         `json:"embeds,omitempty"`
+	AllowedMentions discordAllowedMentions `json:"allowed_mentions"`
+}
+
+type discordAllowedMentions struct {
+	Parse []string `json:"parse"`
+}
+
+type discordEmbed struct {
+	Title       string              `json:"title,omitempty"`
+	Description string              `json:"description,omitempty"`
+	Color       int                 `json:"color,omitempty"`
+	Timestamp   string              `json:"timestamp,omitempty"`
+	Fields      []discordEmbedField `json:"fields,omitempty"`
+	Footer      *discordEmbedFooter `json:"footer,omitempty"`
+}
+
+type discordEmbedField struct {
+	Name   string `json:"name"`
+	Value  string `json:"value"`
+	Inline bool   `json:"inline"`
+}
+
+type discordEmbedFooter struct {
+	Text string `json:"text"`
+}
+
+func newDiscordClient(webhookURL string, client *http.Client, sleep sleepFunc) (*discordClient, error) {
+	parsed, err := url.Parse(webhookURL)
+	if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
+		return nil, errors.New("WEBHOOK_URL must be an absolute HTTP(S) URL")
+	}
+	if client == nil {
+		client = &http.Client{Timeout: attemptTimeout}
+	}
+	if sleep == nil {
+		sleep = sleepContext
+	}
+	return &discordClient{webhookURL: webhookURL, httpClient: client, sleep: sleep}, nil
+}
+
+func (c *discordClient) send(ctx context.Context, payload discordPayload) error {
+	normalizeDiscordPayload(&payload)
+	body, err := json.Marshal(payload)
+	if err != nil {
+		return fmt.Errorf("%w: marshal payload", errWebhookDelivery)
+	}
+
+	deliveryCtx, cancel := context.WithTimeout(ctx, deliveryTimeout)
+	defer cancel()
+
+	var lastErr error
+	for attempt := 0; attempt < maximumAttempts; attempt++ {
+		attemptCtx, attemptCancel := context.WithTimeout(deliveryCtx, attemptTimeout)
+		request, requestErr := http.NewRequestWithContext(attemptCtx, http.MethodPost, c.webhookURL, bytes.NewReader(body))
+		if requestErr != nil {
+			attemptCancel()
+			return fmt.Errorf("%w: create request", errWebhookDelivery)
+		}
+		request.Header.Set("Content-Type", "application/json")
+
+		response, requestErr := c.httpClient.Do(request)
+		if requestErr == nil {
+			delay := retryDelay(response, attempt)
+			_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4<<10))
+			_ = response.Body.Close()
+			attemptCancel()
+
+			if response.StatusCode >= 200 && response.StatusCode < 300 {
+				return nil
+			}
+			lastErr = fmt.Errorf("%w: HTTP %d", errWebhookDelivery, response.StatusCode)
+			if !retryableStatus(response.StatusCode) || attempt == maximumAttempts-1 {
+				return lastErr
+			}
+			if err := c.sleep(deliveryCtx, delay); err != nil {
+				return errWebhookTimeout
+			}
+			continue
+		}
+
+		attemptCancel()
+		lastErr = requestErr
+		if deliveryCtx.Err() != nil {
+			return errWebhookTimeout
+		}
+		if attempt == maximumAttempts-1 {
+			if isTimeout(requestErr) {
+				return fmt.Errorf("%w: %v", errWebhookTimeout, requestErr)
+			}
+			return fmt.Errorf("%w: %v", errWebhookDelivery, requestErr)
+		}
+		if err := c.sleep(deliveryCtx, fallbackRetryDelay(attempt)); err != nil {
+			return errWebhookTimeout
+		}
+	}
+
+	return fmt.Errorf("%w: %v", errWebhookDelivery, lastErr)
+}
+
+func retryableStatus(status int) bool {
+	return status == http.StatusTooManyRequests || status >= 500
+}
+
+func retryDelay(response *http.Response, attempt int) time.Duration {
+	if response.StatusCode == http.StatusTooManyRequests {
+		for _, header := range []string{"Retry-After", "X-RateLimit-Reset-After"} {
+			if delay, ok := parseRetryDelay(response.Header.Get(header)); ok {
+				return min(delay, maximumRetryDelay)
+			}
+		}
+	}
+	return fallbackRetryDelay(attempt)
+}
+
+func parseRetryDelay(value string) (time.Duration, bool) {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return 0, false
+	}
+	if seconds, err := strconv.ParseFloat(value, 64); err == nil && seconds >= 0 {
+		return time.Duration(seconds * float64(time.Second)), true
+	}
+	if timestamp, err := http.ParseTime(value); err == nil {
+		return max(time.Until(timestamp), 0), true
+	}
+	return 0, false
+}
+
+func fallbackRetryDelay(attempt int) time.Duration {
+	return 250 * time.Millisecond * time.Duration(1<<attempt)
+}
+
+func sleepContext(ctx context.Context, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
+
+func isTimeout(err error) bool {
+	if errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	var netErr net.Error
+	return errors.As(err, &netErr) && netErr.Timeout()
+}
+
+func normalizeDiscordPayload(payload *discordPayload) {
+	payload.AllowedMentions.Parse = []string{}
+	payload.Content = truncateRunes(payload.Content, 2000)
+	if len(payload.Embeds) > 10 {
+		payload.Embeds = payload.Embeds[:10]
+	}
+
+	for embedIndex := range payload.Embeds {
+		embed := &payload.Embeds[embedIndex]
+		embed.Title = truncateRunes(embed.Title, 256)
+		embed.Description = truncateRunes(embed.Description, 4096)
+		if len(embed.Fields) > 25 {
+			embed.Fields = embed.Fields[:25]
+		}
+		for fieldIndex := range embed.Fields {
+			embed.Fields[fieldIndex].Name = truncateRunes(embed.Fields[fieldIndex].Name, 256)
+			embed.Fields[fieldIndex].Value = truncateRunes(embed.Fields[fieldIndex].Value, 1024)
+		}
+		if embed.Footer != nil {
+			embed.Footer.Text = truncateRunes(embed.Footer.Text, 2048)
+		}
+	}
+
+	for discordEmbedCharacters(payload.Embeds) > 6000 {
+		removed := false
+		for embedIndex := len(payload.Embeds) - 1; embedIndex >= 0; embedIndex-- {
+			embed := &payload.Embeds[embedIndex]
+			if len(embed.Fields) > 0 {
+				embed.Fields = embed.Fields[:len(embed.Fields)-1]
+				removed = true
+				break
+			}
+		}
+		if !removed {
+			break
+		}
+	}
+
+	remainingOverflow := discordEmbedCharacters(payload.Embeds) - 6000
+	for embedIndex := len(payload.Embeds) - 1; remainingOverflow > 0 && embedIndex >= 0; embedIndex-- {
+		embed := &payload.Embeds[embedIndex]
+		if embed.Footer != nil && embed.Footer.Text != "" {
+			newLimit := max(utf8.RuneCountInString(embed.Footer.Text)-remainingOverflow, 0)
+			embed.Footer.Text = truncateRunes(embed.Footer.Text, newLimit)
+		}
+		remainingOverflow = discordEmbedCharacters(payload.Embeds) - 6000
+		if remainingOverflow > 0 && embed.Description != "" {
+			newLimit := max(utf8.RuneCountInString(embed.Description)-remainingOverflow, 0)
+			embed.Description = truncateRunes(embed.Description, newLimit)
+		}
+		remainingOverflow = discordEmbedCharacters(payload.Embeds) - 6000
+		if remainingOverflow > 0 && embed.Title != "" {
+			newLimit := max(utf8.RuneCountInString(embed.Title)-remainingOverflow, 0)
+			embed.Title = truncateRunes(embed.Title, newLimit)
+		}
+		remainingOverflow = discordEmbedCharacters(payload.Embeds) - 6000
+	}
+}
+
+func discordEmbedCharacters(embeds []discordEmbed) int {
+	total := 0
+	for _, embed := range embeds {
+		total += utf8.RuneCountInString(embed.Title)
+		total += utf8.RuneCountInString(embed.Description)
+		if embed.Footer != nil {
+			total += utf8.RuneCountInString(embed.Footer.Text)
+		}
+		for _, field := range embed.Fields {
+			total += utf8.RuneCountInString(field.Name)
+			total += utf8.RuneCountInString(field.Value)
+		}
+	}
+	return total
+}
+
+func truncateRunes(value string, limit int) string {
+	runes := []rune(value)
+	if len(runes) <= limit {
+		return value
+	}
+	if limit <= 0 {
+		return ""
+	}
+	marker := []rune(truncationMarker)
+	if limit <= len(marker) {
+		return string(marker[:limit])
+	}
+	return string(runes[:limit-len(marker)]) + truncationMarker
+}

haven-notify/discord_test.go

@@ -0,0 +1,243 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+	"unicode/utf8"
+)
+
+func TestDiscordDeliveryRetries(t *testing.T) {
+	tests := []struct {
+		name         string
+		statuses     []int
+		headers      []http.Header
+		transportErr error
+		wantCalls    int
+		wantSleeps   []time.Duration
+		wantError    error
+	}{
+		{name: "success", statuses: []int{204}, wantCalls: 1},
+		{name: "all 2xx accepted", statuses: []int{299}, wantCalls: 1},
+		{name: "permanent 4xx", statuses: []int{400}, wantCalls: 1, wantError: errWebhookDelivery},
+		{name: "5xx recovers", statuses: []int{500, 204}, wantCalls: 2, wantSleeps: []time.Duration{250 * time.Millisecond}},
+		{name: "5xx exhausts", statuses: []int{500, 502, 503}, wantCalls: 3, wantSleeps: []time.Duration{250 * time.Millisecond, 500 * time.Millisecond}, wantError: errWebhookDelivery},
+		{name: "429 honors capped retry after", statuses: []int{429, 204}, headers: []http.Header{{"Retry-After": []string{"10"}}, nil}, wantCalls: 2, wantSleeps: []time.Duration{2 * time.Second}},
+		{name: "429 uses reset after", statuses: []int{429, 204}, headers: []http.Header{{"X-Ratelimit-Reset-After": []string{"0.125"}}, nil}, wantCalls: 2, wantSleeps: []time.Duration{125 * time.Millisecond}},
+		{name: "transport errors exhaust", transportErr: errors.New("connection reset"), wantCalls: 3, wantSleeps: []time.Duration{250 * time.Millisecond, 500 * time.Millisecond}, wantError: errWebhookDelivery},
+		{name: "timeouts exhaust", transportErr: context.DeadlineExceeded, wantCalls: 3, wantSleeps: []time.Duration{250 * time.Millisecond, 500 * time.Millisecond}, wantError: errWebhookTimeout},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var mu sync.Mutex
+			calls := 0
+			var sleeps []time.Duration
+			transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+				mu.Lock()
+				defer mu.Unlock()
+				calls++
+				if request.Header.Get("Content-Type") != "application/json" {
+					t.Errorf("Content-Type = %q", request.Header.Get("Content-Type"))
+				}
+				if test.transportErr != nil {
+					return nil, test.transportErr
+				}
+				index := min(calls-1, len(test.statuses)-1)
+				var headers http.Header
+				if index < len(test.headers) {
+					headers = test.headers[index]
+				}
+				return response(test.statuses[index], headers), nil
+			})
+			client, err := newDiscordClient("https://discord.invalid/webhook", &http.Client{Transport: transport}, func(_ context.Context, delay time.Duration) error {
+				sleeps = append(sleeps, delay)
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = client.send(context.Background(), discordPayload{Content: "hello"})
+			if !errors.Is(err, test.wantError) {
+				t.Fatalf("send() error = %v, want %v", err, test.wantError)
+			}
+			if calls != test.wantCalls {
+				t.Fatalf("calls = %d, want %d", calls, test.wantCalls)
+			}
+			if len(sleeps) != len(test.wantSleeps) {
+				t.Fatalf("sleeps = %v, want %v", sleeps, test.wantSleeps)
+			}
+			for index := range sleeps {
+				if sleeps[index] != test.wantSleeps[index] {
+					t.Fatalf("sleeps = %v, want %v", sleeps, test.wantSleeps)
+				}
+			}
+		})
+	}
+}
+
+func TestDiscordDeliveryPayloadAndCanceledSleep(t *testing.T) {
+	var sent discordPayload
+	transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+		if err := json.NewDecoder(request.Body).Decode(&sent); err != nil {
+			t.Fatal(err)
+		}
+		return response(204, nil), nil
+	})
+	client, err := newDiscordClient("http://discord.invalid/webhook", &http.Client{Transport: transport}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := client.send(context.Background(), discordPayload{Content: strings.Repeat("界", 2100)}); err != nil {
+		t.Fatal(err)
+	}
+	if utf8.RuneCountInString(sent.Content) != 2000 || !strings.HasSuffix(sent.Content, truncationMarker) {
+		t.Fatalf("content length/suffix = %d/%q", utf8.RuneCountInString(sent.Content), []rune(sent.Content)[1980:])
+	}
+	if sent.AllowedMentions.Parse == nil || len(sent.AllowedMentions.Parse) != 0 {
+		t.Fatalf("allowed mentions = %#v", sent.AllowedMentions)
+	}
+
+	client, err = newDiscordClient("http://discord.invalid/webhook", &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+		return response(500, nil), nil
+	})}, func(context.Context, time.Duration) error { return context.DeadlineExceeded })
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := client.send(context.Background(), discordPayload{Content: "x"}); !errors.Is(err, errWebhookTimeout) {
+		t.Fatalf("send() error = %v, want timeout", err)
+	}
+}
+
+func TestDiscordDeliveryRecoversFromTransportError(t *testing.T) {
+	var calls int
+	var sleeps []time.Duration
+	transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
+		calls++
+		if calls == 1 {
+			return nil, errors.New("temporary connection failure")
+		}
+		return response(http.StatusNoContent, nil), nil
+	})
+	client, err := newDiscordClient("https://discord.invalid/webhook", &http.Client{Transport: transport}, func(_ context.Context, delay time.Duration) error {
+		sleeps = append(sleeps, delay)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := client.send(context.Background(), discordPayload{Content: "hello"}); err != nil {
+		t.Fatal(err)
+	}
+	if calls != 2 || len(sleeps) != 1 || sleeps[0] != 250*time.Millisecond {
+		t.Fatalf("calls/sleeps = %d/%v", calls, sleeps)
+	}
+}
+
+func TestDiscordDeliveryHonorsCallerCancellation(t *testing.T) {
+	var calls, sleeps int
+	transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+		calls++
+		return nil, request.Context().Err()
+	})
+	client, err := newDiscordClient("https://discord.invalid/webhook", &http.Client{Transport: transport}, func(context.Context, time.Duration) error {
+		sleeps++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := client.send(ctx, discordPayload{Content: "hello"}); !errors.Is(err, errWebhookTimeout) {
+		t.Fatalf("send() error = %v, want timeout", err)
+	}
+	if calls > 1 || sleeps != 0 {
+		t.Fatalf("canceled delivery made %d calls and %d sleeps", calls, sleeps)
+	}
+}
+
+func TestRetryDelayParsing(t *testing.T) {
+	if got, ok := parseRetryDelay("1.5"); !ok || got != 1500*time.Millisecond {
+		t.Fatalf("parseRetryDelay() = %v, %v", got, ok)
+	}
+	if _, ok := parseRetryDelay("garbage"); ok {
+		t.Fatal("invalid retry delay accepted")
+	}
+	httpDate := time.Now().Add(time.Second).UTC().Format(http.TimeFormat)
+	if got, ok := parseRetryDelay(httpDate); !ok || got < 0 || got > time.Second {
+		t.Fatalf("HTTP-date retry delay = %v, %v", got, ok)
+	}
+	if got := fallbackRetryDelay(1); got != 500*time.Millisecond {
+		t.Fatalf("fallbackRetryDelay(1) = %v", got)
+	}
+	if retryableStatus(499) || !retryableStatus(500) || !retryableStatus(429) {
+		t.Fatal("retryableStatus returned unexpected result")
+	}
+
+	canceled, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := sleepContext(canceled, time.Hour); !errors.Is(err, context.Canceled) {
+		t.Fatalf("sleepContext() error = %v", err)
+	}
+}
+
+func TestNormalizeDiscordPayloadLimits(t *testing.T) {
+	fields := make([]discordEmbedField, 30)
+	for index := range fields {
+		fields[index] = discordEmbedField{
+			Name:  strings.Repeat("n", 300),
+			Value: strings.Repeat("界", 1200),
+		}
+	}
+	payload := discordPayload{
+		Content: strings.Repeat("c", 2100),
+		Embeds: []discordEmbed{{
+			Title:       strings.Repeat("t", 300),
+			Description: strings.Repeat("d", 5000),
+			Fields:      fields,
+			Footer:      &discordEmbedFooter{Text: strings.Repeat("f", 2200)},
+		}},
+	}
+	normalizeDiscordPayload(&payload)
+	if utf8.RuneCountInString(payload.Content) != 2000 || utf8.RuneCountInString(payload.Embeds[0].Title) > 256 || utf8.RuneCountInString(payload.Embeds[0].Description) > 4096 {
+		t.Fatal("individual limits were not applied")
+	}
+	if len(payload.Embeds[0].Fields) > 25 || discordEmbedCharacters(payload.Embeds) > 6000 {
+		t.Fatalf("embed limits = fields %d, chars %d", len(payload.Embeds[0].Fields), discordEmbedCharacters(payload.Embeds))
+	}
+	for _, field := range payload.Embeds[0].Fields {
+		if utf8.RuneCountInString(field.Name) > 256 || utf8.RuneCountInString(field.Value) > 1024 {
+			t.Fatal("field limit was not applied")
+		}
+	}
+
+	if got := truncateRunes("abcdef", 4); got != "… [t" {
+		t.Fatalf("small truncation = %q", got)
+	}
+	if got := truncateRunes("abc", 3); got != "abc" {
+		t.Fatalf("unneeded truncation = %q", got)
+	}
+	if got := truncateRunes("abc", 0); got != "" {
+		t.Fatalf("zero truncation = %q", got)
+	}
+}
+
+func TestDiscordClientRejectsInvalidURLs(t *testing.T) {
+	for _, value := range []string{"", "relative/path", "ftp://example.com/hook", "https:///hook"} {
+		if _, err := newDiscordClient(value, nil, nil); err == nil {
+			t.Fatalf("newDiscordClient(%q) succeeded", value)
+		}
+	}
+}
+
+func TestIsTimeout(t *testing.T) {
+	if !isTimeout(context.DeadlineExceeded) || isTimeout(errors.New("ordinary")) {
+		t.Fatal("isTimeout returned unexpected result")
+	}
+}

haven-notify/e2e_test.go

@@ -0,0 +1,322 @@
+//go:build e2e
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/log"
+	"github.com/testcontainers/testcontainers-go/modules/mockserver"
+	"github.com/testcontainers/testcontainers-go/network"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+const mockServerImage = "mockserver/mockserver:5.15.0@sha256:0f9ef78c94894ac3e70135d156193b25e23872575d58e2228344964273b4af6b"
+
+type mockRecordedRequest struct {
+	Method  string              `json:"method"`
+	Path    string              `json:"path"`
+	Headers map[string][]string `json:"headers"`
+	Body    struct {
+		RawBytes string `json:"rawBytes"`
+	} `json:"body"`
+}
+
+func TestContainerE2E(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
+	defer cancel()
+	logger := log.TestLogger(t)
+
+	testNetwork, err := network.New(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testcontainers.CleanupNetwork(t, testNetwork)
+
+	mock, err := mockserver.Run(ctx, mockServerImage,
+		network.WithNetwork([]string{"discord-mock"}, testNetwork),
+		testcontainers.WithLogger(logger),
+	)
+	testcontainers.CleanupContainer(t, mock)
+	if err != nil {
+		t.Fatalf("start MockServer: %v", err)
+	}
+	mockURL, err := mock.URL(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buildPlatform := "linux/amd64"
+	targetOS := "linux"
+	targetArch := "amd64"
+	app, err := testcontainers.Run(ctx, "",
+		testcontainers.WithDockerfile(testcontainers.FromDockerfile{
+			Context:    ".",
+			Dockerfile: "Dockerfile",
+			Repo:       "haven-notify-e2e",
+			Tag:        "test",
+			KeepImage:  false,
+			BuildArgs: map[string]*string{
+				"BUILDPLATFORM": &buildPlatform,
+				"TARGETOS":      &targetOS,
+				"TARGETARCH":    &targetArch,
+			},
+		}),
+		network.WithNetwork([]string{"haven-notify"}, testNetwork),
+		testcontainers.WithEnv(map[string]string{"WEBHOOK_URL": "http://discord-mock:1080/discord"}),
+		testcontainers.WithExposedPorts("8080/tcp"),
+		testcontainers.WithWaitStrategy(wait.ForHTTP("/ready").WithPort("8080/tcp").WithStartupTimeout(45*time.Second)),
+		testcontainers.WithLogger(logger),
+	)
+	testcontainers.CleanupContainer(t, app)
+	if err != nil {
+		t.Fatalf("start Haven Notify: %v", err)
+	}
+	appURL, err := app.PortEndpoint(ctx, "8080/tcp", "http")
+	if err != nil {
+		t.Fatal(err)
+	}
+	inspect, err := app.Inspect(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if inspect.Config.User != "haven-notify" {
+		t.Fatalf("container user = %q", inspect.Config.User)
+	}
+
+	t.Run("health probes", func(t *testing.T) {
+		for path, body := range map[string]string{"/ready": "Ready", "/live": "Alive"} {
+			response, err := http.Get(appURL + path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			contents, _ := io.ReadAll(response.Body)
+			_ = response.Body.Close()
+			if response.StatusCode != 200 || string(contents) != body {
+				t.Fatalf("%s = (%d, %q)", path, response.StatusCode, contents)
+			}
+		}
+	})
+
+	if err := configureMock(mockURL, []mockExpectation{{Status: 204}}); err != nil {
+		t.Fatal(err)
+	}
+	cases := []struct {
+		path string
+		body string
+		want discordPayload
+	}{
+		{
+			path: "/notify",
+			body: `{"title":"Título \"quoted\"","message":"line 1\nC:\\backups @everyone"}`,
+			want: discordPayload{
+				Content:         "**Título \"quoted\"**\nline 1\nC:\\backups @everyone",
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			path: "/template/notify/backup",
+			body: `{"title":"Nightly","asset":"photos","backupSizeInMB":1536,"extra":[{"name":"Host","value":"nebula"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "📦 Backup - Nightly",
+					Description: "**photos** has been backed up successfully! ✅🫡\n",
+					Color:       3066993,
+					Fields: []discordEmbedField{
+						{Name: "💾 Backup Size", Value: "1.50 GiB", Inline: true},
+						{Name: "Host", Value: "nebula", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "✨ Haven Notify ✨"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			path: "/template/notify/update",
+			body: `{"host":"nexus","asset":"k3s","time":0}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "🔄 Update - k3s",
+					Description: "**nexus** has successfully updated **k3s**! ✅",
+					Color:       3447003,
+					Fields: []discordEmbedField{
+						{Name: "⏱️ Time Taken", Value: "0 seconds", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "✨ Haven Notify ✨"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			path: "/template/notify/error",
+			body: `{"caller":"backup","message":"line 1\n\"quoted\" <tag>","critical":true}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "❌ Error",
+					Description: "**backup** encountered an error!",
+					Color:       15158332,
+					Fields: []discordEmbedField{
+						{Name: "📄 Message", Value: "line 1\n\"quoted\" <tag>", Inline: false},
+					},
+					Footer: &discordEmbedFooter{Text: "✨ Haven Notify ✨"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+	}
+	for _, test := range cases {
+		response, err := postJSON(appURL+test.path, test.body)
+		if err != nil {
+			t.Fatalf("POST %s: %v", test.path, err)
+		}
+		contents, _ := io.ReadAll(response.Body)
+		_ = response.Body.Close()
+		if response.StatusCode != 200 || string(contents) != "Notification sent" {
+			t.Fatalf("POST %s = (%d, %q)", test.path, response.StatusCode, contents)
+		}
+	}
+	recorded, err := retrieveMockRequests(mockURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(recorded) != len(cases) {
+		t.Fatalf("recorded requests = %d, want %d", len(recorded), len(cases))
+	}
+	for index, request := range recorded {
+		if request.Method != http.MethodPost || request.Path != "/discord" || len(request.Headers["Content-Type"]) == 0 || request.Headers["Content-Type"][0] != "application/json" {
+			t.Fatalf("recorded request %d = %+v", index, request)
+		}
+		body, err := base64.StdEncoding.DecodeString(request.Body.RawBytes)
+		if err != nil || !json.Valid(body) || bytes.Contains(body, []byte("&#34;")) {
+			t.Fatalf("recorded body %d = %s, %v", index, body, err)
+		}
+		want, err := json.Marshal(cases[index].want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(body, want) {
+			t.Fatalf("recorded body %d = %s, want %s", index, body, want)
+		}
+	}
+
+	t.Run("invalid request does not call webhook", func(t *testing.T) {
+		if err := configureMock(mockURL, []mockExpectation{{Status: 204}}); err != nil {
+			t.Fatal(err)
+		}
+		response, err := postJSON(appURL+"/notify", `{"title":"missing message"}`)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = response.Body.Close()
+		if response.StatusCode != 400 {
+			t.Fatalf("status = %d", response.StatusCode)
+		}
+		recorded, err := retrieveMockRequests(mockURL)
+		if err != nil || len(recorded) != 0 {
+			t.Fatalf("recorded = %d, error = %v", len(recorded), err)
+		}
+	})
+
+	t.Run("transient failure retries", func(t *testing.T) {
+		if err := configureMock(mockURL, []mockExpectation{{Status: 500, Times: 1, Priority: 10}, {Status: 204}}); err != nil {
+			t.Fatal(err)
+		}
+		response, err := postJSON(appURL+"/notify", `{"title":"retry","message":"once"}`)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = response.Body.Close()
+		if response.StatusCode != 200 {
+			t.Fatalf("status = %d", response.StatusCode)
+		}
+		recorded, err := retrieveMockRequests(mockURL)
+		if err != nil || len(recorded) != 2 {
+			t.Fatalf("recorded = %d, error = %v", len(recorded), err)
+		}
+	})
+
+	bad, err := testcontainers.Run(ctx, "haven-notify-e2e:test",
+		testcontainers.WithWaitStrategy(wait.ForExit().WithExitTimeout(15*time.Second)),
+		testcontainers.WithLogger(logger),
+	)
+	testcontainers.CleanupContainer(t, bad)
+	if err != nil {
+		t.Fatalf("start unconfigured container: %v", err)
+	}
+	state, err := bad.State(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if state.Running || state.ExitCode == 0 {
+		t.Fatalf("unconfigured container state = %+v", state)
+	}
+}
+
+type mockExpectation struct {
+	Status   int
+	Times    int
+	Priority int
+}
+
+func configureMock(baseURL string, expectations []mockExpectation) error {
+	request, _ := http.NewRequest(http.MethodPut, baseURL+"/mockserver/reset", nil)
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return err
+	}
+	_ = response.Body.Close()
+	for _, expectation := range expectations {
+		payload := map[string]any{
+			"httpRequest":  map[string]any{"method": "POST", "path": "/discord"},
+			"httpResponse": map[string]any{"statusCode": expectation.Status},
+			"priority":     expectation.Priority,
+		}
+		if expectation.Times > 0 {
+			payload["times"] = map[string]any{"remainingTimes": expectation.Times, "unlimited": false}
+		}
+		body, _ := json.Marshal(payload)
+		request, _ = http.NewRequest(http.MethodPut, baseURL+"/mockserver/expectation", bytes.NewReader(body))
+		request.Header.Set("Content-Type", "application/json")
+		response, err = http.DefaultClient.Do(request)
+		if err != nil {
+			return err
+		}
+		_ = response.Body.Close()
+		if response.StatusCode < 200 || response.StatusCode >= 300 {
+			return fmt.Errorf("MockServer expectation returned %d", response.StatusCode)
+		}
+	}
+	return nil
+}
+
+func retrieveMockRequests(baseURL string) ([]mockRecordedRequest, error) {
+	request, _ := http.NewRequest(http.MethodPut, baseURL+"/mockserver/retrieve?type=REQUESTS&format=JSON", strings.NewReader(`{"path":"/discord"}`))
+	request.Header.Set("Content-Type", "application/json")
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	var recorded []mockRecordedRequest
+	err = json.NewDecoder(response.Body).Decode(&recorded)
+	return recorded, err
+}
+
+func postJSON(url, body string) (*http.Response, error) {
+	request, err := http.NewRequest(http.MethodPost, url, strings.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("Content-Type", "application/json")
+	return http.DefaultClient.Do(request)
+}

haven-notify/go.mod

@@ -0,0 +1,61 @@
+module git.ivanch.me/ivanch/server-scripts/haven-notify
+
+go 1.25.10
+
+require (
+	github.com/testcontainers/testcontainers-go v0.42.0
+	github.com/testcontainers/testcontainers-go/modules/mockserver v0.42.0
+)
+
+require (
+	dario.cat/mergo v1.0.2 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/go-connections v0.6.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/ebitengine/purego v0.10.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.10 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/go-archive v0.2.0 // indirect
+	github.com/moby/moby/api v1.54.1 // indirect
+	github.com/moby/moby/client v0.4.0 // indirect
+	github.com/moby/patternmatcher v0.6.1 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/shirou/gopsutil/v4 v4.26.3 // indirect
+	github.com/sirupsen/logrus v1.9.4 // indirect
+	github.com/stretchr/testify v1.11.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.41.0 // indirect
+	go.opentelemetry.io/otel/metric v1.41.0 // indirect
+	go.opentelemetry.io/otel/trace v1.41.0 // indirect
+	golang.org/x/crypto v0.48.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

haven-notify/go.sum

@@ -0,0 +1,145 @@
+dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
+dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/BraspagDevelopers/mock-server-client v0.2.2 h1:Zro0OonNeaDwkkQGIxeJQfYweNKZ+m+8QIlDZAFRc/4=
+github.com/BraspagDevelopers/mock-server-client v0.2.2/go.mod h1:LHulrZSfbCNeS/CoycaWdhE495FnyeI3iXm6+4Zjz5c=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
+github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
+github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
+github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
+github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
+github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So=
+github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
+github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
+github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
+github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
+github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
+github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
+github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
+github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
+github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
+github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
+github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
+github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
+github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
+github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
+github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
+github.com/testcontainers/testcontainers-go/modules/mockserver v0.42.0 h1:UfORuhjP6+wd28O21m8cx4CLe+RbWlo31SI9ZgrqXn4=
+github.com/testcontainers/testcontainers-go/modules/mockserver v0.42.0/go.mod h1:uwubSg20I0nSLjtpVl4m2Ixin3yeXks4zkHwaCzbRvo=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
+go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
+go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
+go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
+go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
+go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
+go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
+go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
+go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
+go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
+golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
+gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
+pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
+pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=

haven-notify/main.go

@@ -1,214 +1,37 @@
 package main
 
 import (
-	"bytes"
-	"encoding/json"
+	"context"
+	"errors"
 	"fmt"
-	"html/template"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
-	"strings"
-	"time"
+	"os/signal"
+	"syscall"
 )
 
-// Notification payload
-type Notification struct {
-	Title   string `json:"title"`
-	Message string `json:"message"`
-}
+const listenAddress = ":8080"
 
 func main() {
-	http.HandleFunc("/notify", notifyHandler)
-	http.HandleFunc("/ready", readinessHandler)
-	http.HandleFunc("/live", livenessHandler)
-	http.HandleFunc("/template/notify/", templateNotifyHandler)
-	log.Println("Starting server on :8080...")
-	log.Fatal(http.ListenAndServe(":8080", nil))
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	logger := log.Default()
+	if err := run(ctx, os.Getenv("WEBHOOK_URL"), listenAddress, logger); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		logger.Fatalf("Server stopped unexpectedly: %v", err)
+	}
 }
 
-func notifyHandler(w http.ResponseWriter, r *http.Request) {
-	log.Printf("Incoming %s request from %s to %s", r.Method, r.RemoteAddr, r.URL.Path)
-	if r.Method != http.MethodPost {
-		log.Printf("Method not allowed: %s", r.Method)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		w.Write([]byte("Method not allowed"))
-		return
-	}
-
-	var notif Notification
-	if err := json.NewDecoder(r.Body).Decode(&notif); err != nil {
-		log.Printf("Invalid payload: %v", err)
-		w.WriteHeader(http.StatusBadRequest)
-		w.Write([]byte("Invalid payload"))
-		return
-	}
-	log.Printf("Received notification payload: Title='%s', Message='%s'", notif.Title, notif.Message)
-
-	// Call Discord notification function
-	if err := sendDiscordNotification(notif.Title, notif.Message); err != nil {
-		log.Printf("Failed to send Discord notification: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to send Discord notification"))
-		return
-	}
-
-	log.Printf("Notification sent successfully for Title='%s'", notif.Title)
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("Notification sent"))
-}
-
-// Readiness handler
-func readinessHandler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("Ready"))
-}
-
-// Liveness handler
-func livenessHandler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("Alive"))
-}
-
-func sendDiscordNotification(title, message string) error {
-	webhookURL := os.Getenv("WEBHOOK_URL")
-	if webhookURL == "" {
-		log.Printf("WEBHOOK_URL environment variable not set")
-		return fmt.Errorf("WEBHOOK_URL environment variable not set")
-	}
-
-	// Discord webhook payload
-	type discordPayload struct {
-		Content string `json:"content"`
-	}
-
-	content := "**" + title + "**\n" + message
-	payload := discordPayload{Content: content}
-
-	jsonData, err := json.Marshal(payload)
+func run(ctx context.Context, webhookURL, address string, logger *log.Logger) error {
+	app, err := newApplication(applicationConfig{WebhookURL: webhookURL, Logger: logger})
 	if err != nil {
-		log.Printf("Failed to marshal Discord payload: %v", err)
-		return err
+		return fmt.Errorf("configuration failed: %w", err)
 	}
-
-	log.Printf("Sending Discord notification: Title='%s', Message='%s'", title, message)
-	resp, err := http.Post(webhookURL, "application/json", bytes.NewBuffer(jsonData))
+	listener, err := netListen("tcp", address)
 	if err != nil {
-		log.Printf("Error posting to Discord webhook: %v", err)
-		return err
+		return fmt.Errorf("listen on %s: %w", address, err)
 	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		log.Printf("Discord webhook returned status: %s", resp.Status)
-		return fmt.Errorf("Discord webhook returned status: %s", resp.Status)
-	}
-
-	log.Printf("Discord notification sent successfully: Title='%s'", title)
-	return nil
-}
-
-func templateNotifyHandler(w http.ResponseWriter, r *http.Request) {
-	log.Printf("Incoming %s request from %s to %s", r.Method, r.RemoteAddr, r.URL.Path)
-	if r.Method != http.MethodPost {
-		log.Printf("Method not allowed: %s", r.Method)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		w.Write([]byte("Method not allowed"))
-		return
-	}
-
-	templateName := r.URL.Path[len("/template/notify/"):] // Extract template name
-	if templateName == "" {
-		log.Printf("Template name not provided")
-		w.WriteHeader(http.StatusBadRequest)
-		w.Write([]byte("Template name not provided"))
-		return
-	}
-
-	templatePath := "template/" + templateName + ".tmpl"
-	templateData, err := ioutil.ReadFile(templatePath)
-	if err != nil {
-		log.Printf("Failed to read template: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to read template"))
-		return
-	}
-
-	tmpl, err := template.New(templateName).Funcs(template.FuncMap{
-		"formatSize": func(size float64) string {
-			if size > 1024 {
-				return fmt.Sprintf("%.2f GiB", size/1024)
-			}
-			return fmt.Sprintf("%.2f MiB", size)
-		},
-		"upper": strings.ToUpper,
-		"lower": strings.ToLower,
-		"title": strings.Title,
-		"now": func() string {
-			return fmt.Sprintf("%d", time.Now().Unix())
-		},
-		"formatTime": func(timestamp string) string {
-			if timestamp == "" {
-				return time.Now().Format("2006-01-02T15:04:05Z")
-			}
-			return timestamp
-		},
-	}).Parse(string(templateData))
-	if err != nil {
-		log.Printf("Failed to parse template: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to parse template"))
-		return
-	}
-
-	var rawPayload map[string]interface{}
-	if err := json.NewDecoder(r.Body).Decode(&rawPayload); err != nil {
-		log.Printf("Invalid payload: %v", err)
-		w.WriteHeader(http.StatusBadRequest)
-		w.Write([]byte("Invalid payload"))
-		return
-	}
-
-	// Normalize keys to lowercase for case-insensitive parsing
-	payload := make(map[string]interface{})
-	for key, value := range rawPayload {
-		payload[strings.ToLower(key)] = value
-	}
-
-	var filledTemplate bytes.Buffer
-	if err := tmpl.Execute(&filledTemplate, payload); err != nil {
-		log.Printf("Failed to execute template: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to execute template"))
-		return
-	}
-
-	webhookURL := os.Getenv("WEBHOOK_URL")
-	if webhookURL == "" {
-		log.Printf("WEBHOOK_URL environment variable not set")
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("WEBHOOK_URL environment variable not set"))
-		return
-	}
-
-	resp, err := http.Post(webhookURL, "application/json", &filledTemplate)
-	if err != nil {
-		log.Printf("Error posting to Discord webhook: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to send notification"))
-		return
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		log.Printf("Discord webhook returned status: %s", resp.Status)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to send notification"))
-		return
-	}
-
-	log.Printf("Notification sent successfully using template '%s'", templateName)
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("Notification sent"))
+	logger.Printf("Starting server on %s", address)
+	return serve(ctx, newHTTPServer(app.handler()), listener)
 }

haven-notify/server.go

@@ -0,0 +1,402 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"mime"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+var netListen = net.Listen
+
+const (
+	maxRequestBody  = 64 << 10
+	shutdownTimeout = 10 * time.Second
+)
+
+var (
+	errInvalidPayload       = errors.New("invalid payload")
+	errUnsupportedMediaType = errors.New("unsupported media type")
+)
+
+type applicationConfig struct {
+	WebhookURL string
+	HTTPClient *http.Client
+	Sleep      sleepFunc
+	Logger     *log.Logger
+	Now        func() time.Time
+}
+
+type application struct {
+	discord  *discordClient
+	renderer *templateRenderer
+	logger   *log.Logger
+	now      func() time.Time
+}
+
+type notificationRequest struct {
+	Title   *string `json:"title"`
+	Message *string `json:"message"`
+}
+
+type extraRequest struct {
+	Name  *string `json:"name"`
+	Value *string `json:"value"`
+}
+
+type extraRequests []extraRequest
+
+type backupRequest struct {
+	Title          *string       `json:"title"`
+	Asset          *string       `json:"asset"`
+	BackupSizeInMB *float64      `json:"backupSizeInMB"`
+	Extra          extraRequests `json:"extra"`
+}
+
+type updateRequest struct {
+	Host  *string  `json:"host"`
+	Asset *string  `json:"asset"`
+	Time  *float64 `json:"time"`
+}
+
+type errorRequest struct {
+	Caller   *string       `json:"caller"`
+	Message  *string       `json:"message"`
+	Critical *bool         `json:"critical"`
+	Extra    extraRequests `json:"extra"`
+}
+
+func newApplication(cfg applicationConfig) (*application, error) {
+	logger := cfg.Logger
+	if logger == nil {
+		logger = log.New(io.Discard, "", 0)
+	}
+	now := cfg.Now
+	if now == nil {
+		now = time.Now
+	}
+
+	renderer, err := newTemplateRenderer()
+	if err != nil {
+		return nil, fmt.Errorf("load templates: %w", err)
+	}
+
+	discord, err := newDiscordClient(cfg.WebhookURL, cfg.HTTPClient, cfg.Sleep)
+	if err != nil {
+		return nil, err
+	}
+
+	return &application{discord: discord, renderer: renderer, logger: logger, now: now}, nil
+}
+
+func (a *application) handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/notify", a.requireMethod(http.MethodPost, a.notifyHandler))
+	mux.HandleFunc("/template/notify/backup", a.requireMethod(http.MethodPost, a.backupHandler))
+	mux.HandleFunc("/template/notify/update", a.requireMethod(http.MethodPost, a.updateHandler))
+	mux.HandleFunc("/template/notify/error", a.requireMethod(http.MethodPost, a.errorHandler))
+	mux.HandleFunc("/api/v2/messages", a.requireMethod(http.MethodPost, a.v2MessageHandler))
+	mux.HandleFunc("/api/v2/notifications/backup", a.requireMethod(http.MethodPost, a.v2BackupHandler))
+	mux.HandleFunc("/api/v2/notifications/update", a.requireMethod(http.MethodPost, a.v2UpdateHandler))
+	mux.HandleFunc("/api/v2/notifications/error", a.requireMethod(http.MethodPost, a.v2ErrorHandler))
+	mux.HandleFunc("/ready", a.requireProbeMethod(a.readinessHandler))
+	mux.HandleFunc("/live", a.requireProbeMethod(a.livenessHandler))
+	mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
+		writeText(w, http.StatusNotFound, "Not found")
+	})
+	return mux
+}
+
+func newHTTPServer(handler http.Handler) *http.Server {
+	return &http.Server{
+		Handler:           handler,
+		ReadHeaderTimeout: 5 * time.Second,
+		ReadTimeout:       10 * time.Second,
+		WriteTimeout:      15 * time.Second,
+		IdleTimeout:       60 * time.Second,
+	}
+}
+
+func serve(ctx context.Context, server *http.Server, listener net.Listener) error {
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- server.Serve(listener)
+	}()
+
+	select {
+	case err := <-errCh:
+		return err
+	case <-ctx.Done():
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+		defer cancel()
+		if err := server.Shutdown(shutdownCtx); err != nil {
+			return fmt.Errorf("graceful shutdown: %w", err)
+		}
+		return <-errCh
+	}
+}
+
+func (a *application) requireMethod(method string, next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != method {
+			w.Header().Set("Allow", method)
+			writeText(w, http.StatusMethodNotAllowed, "Method not allowed")
+			return
+		}
+		next(w, r)
+	}
+}
+
+func (a *application) requireProbeMethod(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet && r.Method != http.MethodHead {
+			w.Header().Set("Allow", "GET, HEAD")
+			writeText(w, http.StatusMethodNotAllowed, "Method not allowed")
+			return
+		}
+		next(w, r)
+	}
+}
+
+func (a *application) notifyHandler(w http.ResponseWriter, r *http.Request) {
+	var request notificationRequest
+	if err := decodeRequest(w, r, &request, "title", "message"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	if !requiredString(request.Title) || !requiredString(request.Message) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	payload := discordPayload{
+		Content: "**" + *request.Title + "**\n" + *request.Message,
+	}
+	if !a.send(w, r, payload) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) backupHandler(w http.ResponseWriter, r *http.Request) {
+	var request backupRequest
+	if err := decodeRequest(w, r, &request, "title", "asset", "backupsizeinmb", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	if !requiredString(request.Title) || !requiredString(request.Asset) || request.BackupSizeInMB == nil || *request.BackupSizeInMB < 0 || !validExtra(request.Extra) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	payload, err := a.renderer.renderBackup(request)
+	if err != nil {
+		a.logger.Printf("Template rendering failed for %s", r.URL.Path)
+		writeText(w, http.StatusInternalServerError, "Failed to render notification")
+		return
+	}
+	if !a.send(w, r, payload) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) updateHandler(w http.ResponseWriter, r *http.Request) {
+	var request updateRequest
+	if err := decodeRequest(w, r, &request, "host", "asset", "time"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	if !requiredString(request.Host) || !requiredString(request.Asset) || request.Time == nil || *request.Time < 0 {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	payload, err := a.renderer.renderUpdate(request)
+	if err != nil {
+		a.logger.Printf("Template rendering failed for %s", r.URL.Path)
+		writeText(w, http.StatusInternalServerError, "Failed to render notification")
+		return
+	}
+	if !a.send(w, r, payload) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) errorHandler(w http.ResponseWriter, r *http.Request) {
+	var request errorRequest
+	if err := decodeRequest(w, r, &request, "caller", "message", "critical", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	if !requiredString(request.Caller) || !requiredString(request.Message) || request.Critical == nil || !validExtra(request.Extra) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	payload, err := a.renderer.renderError(request)
+	if err != nil {
+		a.logger.Printf("Template rendering failed for %s", r.URL.Path)
+		writeText(w, http.StatusInternalServerError, "Failed to render notification")
+		return
+	}
+	if !a.send(w, r, payload) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) readinessHandler(w http.ResponseWriter, _ *http.Request) {
+	writeText(w, http.StatusOK, "Ready")
+}
+
+func (a *application) livenessHandler(w http.ResponseWriter, _ *http.Request) {
+	writeText(w, http.StatusOK, "Alive")
+}
+
+func (a *application) send(w http.ResponseWriter, r *http.Request, payload discordPayload) bool {
+	if err := a.discord.send(r.Context(), payload); err != nil {
+		if errors.Is(err, errWebhookTimeout) {
+			a.logger.Printf("Discord delivery timed out for %s", r.URL.Path)
+			writeText(w, http.StatusGatewayTimeout, "Webhook timed out")
+			return false
+		}
+		a.logger.Printf("Discord delivery failed for %s", r.URL.Path)
+		writeText(w, http.StatusBadGateway, "Failed to send notification")
+		return false
+	}
+	return true
+}
+
+func (a *application) writeDecodeError(w http.ResponseWriter, r *http.Request, err error) {
+	if errors.Is(err, errUnsupportedMediaType) {
+		a.logger.Printf("Rejected unsupported media type for %s", r.URL.Path)
+		writeText(w, http.StatusUnsupportedMediaType, "Content-Type must be application/json")
+		return
+	}
+	a.logger.Printf("Rejected invalid payload for %s", r.URL.Path)
+	writeText(w, http.StatusBadRequest, "Invalid payload")
+}
+
+func decodeRequest(w http.ResponseWriter, r *http.Request, destination any, knownKeys ...string) error {
+	mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+	if err != nil || !strings.EqualFold(mediaType, "application/json") {
+		return errUnsupportedMediaType
+	}
+
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestBody)
+	data, err := io.ReadAll(r.Body)
+	if err != nil {
+		return fmt.Errorf("read request: %w", errInvalidPayload)
+	}
+	if len(bytes.TrimSpace(data)) == 0 {
+		return errInvalidPayload
+	}
+	if err := rejectDuplicateKnownKeys(data, knownKeys); err != nil {
+		return errInvalidPayload
+	}
+	if err := json.Unmarshal(data, destination); err != nil {
+		return errInvalidPayload
+	}
+	return nil
+}
+
+func rejectDuplicateKnownKeys(data []byte, knownKeys []string) error {
+	known := make(map[string]struct{}, len(knownKeys))
+	for _, key := range knownKeys {
+		known[strings.ToLower(key)] = struct{}{}
+	}
+
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	token, err := decoder.Token()
+	if err != nil {
+		return err
+	}
+	delim, ok := token.(json.Delim)
+	if !ok || delim != '{' {
+		return errInvalidPayload
+	}
+
+	seen := make(map[string]struct{}, len(knownKeys))
+	for decoder.More() {
+		token, err = decoder.Token()
+		if err != nil {
+			return err
+		}
+		key, ok := token.(string)
+		if !ok {
+			return errInvalidPayload
+		}
+		canonical := strings.ToLower(key)
+		if _, isKnown := known[canonical]; isKnown {
+			if _, duplicate := seen[canonical]; duplicate {
+				return errInvalidPayload
+			}
+			seen[canonical] = struct{}{}
+		}
+		var value json.RawMessage
+		if err := decoder.Decode(&value); err != nil {
+			return err
+		}
+	}
+	if _, err := decoder.Token(); err != nil {
+		return err
+	}
+	if decoder.Decode(&struct{}{}) != io.EOF {
+		return errInvalidPayload
+	}
+	return nil
+}
+
+func (e *extraRequest) UnmarshalJSON(data []byte) error {
+	if err := rejectDuplicateKnownKeys(data, []string{"name", "value"}); err != nil {
+		return err
+	}
+	type rawExtra extraRequest
+	var decoded rawExtra
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return err
+	}
+	*e = extraRequest(decoded)
+	return nil
+}
+
+func (e *extraRequests) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
+		return errInvalidPayload
+	}
+	var decoded []extraRequest
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return err
+	}
+	*e = decoded
+	return nil
+}
+
+func requiredString(value *string) bool {
+	return value != nil && strings.TrimSpace(*value) != ""
+}
+
+func validExtra(extra extraRequests) bool {
+	for _, field := range extra {
+		if !requiredString(field.Name) || !requiredString(field.Value) {
+			return false
+		}
+	}
+	return true
+}
+
+func writeText(w http.ResponseWriter, status int, body string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.WriteHeader(status)
+	_, _ = io.WriteString(w, body)
+}

haven-notify/server_test.go

@@ -0,0 +1,451 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"text/template"
+	"time"
+)
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (fn roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+	return fn(request)
+}
+
+type webhookRecorder struct {
+	mu       sync.Mutex
+	requests [][]byte
+	status   int
+	err      error
+}
+
+func (r *webhookRecorder) RoundTrip(request *http.Request) (*http.Response, error) {
+	body, _ := io.ReadAll(request.Body)
+	r.mu.Lock()
+	r.requests = append(r.requests, body)
+	status, err := r.status, r.err
+	r.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	if status == 0 {
+		status = http.StatusNoContent
+	}
+	return response(status, nil), nil
+}
+
+func (r *webhookRecorder) bodies() [][]byte {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([][]byte(nil), r.requests...)
+}
+
+func testApplication(t testing.TB, transport http.RoundTripper, logger *log.Logger) *application {
+	t.Helper()
+	if transport == nil {
+		transport = &webhookRecorder{}
+	}
+	app, err := newApplication(applicationConfig{
+		WebhookURL: "http://discord.invalid/webhook",
+		HTTPClient: &http.Client{Transport: transport},
+		Sleep:      func(context.Context, time.Duration) error { return nil },
+		Logger:     logger,
+		Now:        func() time.Time { return time.Date(2026, time.July, 16, 12, 34, 56, 0, time.UTC) },
+	})
+	if err != nil {
+		t.Fatalf("newApplication() error = %v", err)
+	}
+	return app
+}
+
+func performRequest(handler http.Handler, method, path, contentType, body string) *httptest.ResponseRecorder {
+	request := httptest.NewRequest(method, path, strings.NewReader(body))
+	if contentType != "" {
+		request.Header.Set("Content-Type", contentType)
+	}
+	recorder := httptest.NewRecorder()
+	handler.ServeHTTP(recorder, request)
+	return recorder
+}
+
+func TestProbeAndRoutingContracts(t *testing.T) {
+	handler := testApplication(t, nil, nil).handler()
+	tests := []struct {
+		name       string
+		method     string
+		path       string
+		wantStatus int
+		wantBody   string
+		wantAllow  string
+	}{
+		{name: "ready get", method: http.MethodGet, path: "/ready", wantStatus: 200, wantBody: "Ready"},
+		{name: "ready head", method: http.MethodHead, path: "/ready", wantStatus: 200, wantBody: "Ready"},
+		{name: "live get", method: http.MethodGet, path: "/live", wantStatus: 200, wantBody: "Alive"},
+		{name: "probe wrong method", method: http.MethodPost, path: "/live", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "GET, HEAD"},
+		{name: "notify wrong method", method: http.MethodGet, path: "/notify", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "POST"},
+		{name: "backup wrong method", method: http.MethodGet, path: "/template/notify/backup", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "POST"},
+		{name: "update wrong method", method: http.MethodPut, path: "/template/notify/update", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "POST"},
+		{name: "error wrong method", method: http.MethodDelete, path: "/template/notify/error", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "POST"},
+		{name: "unknown template", method: http.MethodPost, path: "/template/notify/missing", wantStatus: 404, wantBody: "Not found"},
+		{name: "trailing slash is unknown", method: http.MethodPost, path: "/notify/", wantStatus: 404, wantBody: "Not found"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := performRequest(handler, test.method, test.path, "", "")
+			if got.Code != test.wantStatus || got.Body.String() != test.wantBody {
+				t.Fatalf("response = (%d, %q), want (%d, %q)", got.Code, got.Body.String(), test.wantStatus, test.wantBody)
+			}
+			if got.Header().Get("Allow") != test.wantAllow {
+				t.Fatalf("Allow = %q, want %q", got.Header().Get("Allow"), test.wantAllow)
+			}
+			if got.Header().Get("Content-Type") != "text/plain; charset=utf-8" {
+				t.Fatalf("Content-Type = %q", got.Header().Get("Content-Type"))
+			}
+		})
+	}
+}
+
+func TestRequestValidationContracts(t *testing.T) {
+	tests := []struct {
+		name        string
+		path        string
+		contentType string
+		body        string
+		wantStatus  int
+	}{
+		{name: "missing content type", path: "/notify", body: `{}`, wantStatus: 415},
+		{name: "wrong content type", path: "/notify", contentType: "text/plain", body: `{}`, wantStatus: 415},
+		{name: "media type parameters", path: "/notify", contentType: "application/json; charset=utf-8", body: `{"title":"t","message":"m"}`, wantStatus: 200},
+		{name: "empty", path: "/notify", contentType: "application/json", body: "", wantStatus: 400},
+		{name: "array", path: "/notify", contentType: "application/json", body: `[]`, wantStatus: 400},
+		{name: "malformed", path: "/notify", contentType: "application/json", body: `{`, wantStatus: 400},
+		{name: "trailing document", path: "/notify", contentType: "application/json", body: `{"title":"t","message":"m"}{}`, wantStatus: 400},
+		{name: "missing required", path: "/notify", contentType: "application/json", body: `{"title":"t"}`, wantStatus: 400},
+		{name: "missing title", path: "/notify", contentType: "application/json", body: `{"message":"m"}`, wantStatus: 400},
+		{name: "blank required", path: "/notify", contentType: "application/json", body: `{"title":" ","message":"m"}`, wantStatus: 400},
+		{name: "blank message", path: "/notify", contentType: "application/json", body: `{"title":"t","message":"\t"}`, wantStatus: 400},
+		{name: "wrong type", path: "/notify", contentType: "application/json", body: `{"title":1,"message":"m"}`, wantStatus: 400},
+		{name: "wrong message type", path: "/notify", contentType: "application/json", body: `{"title":"t","message":false}`, wantStatus: 400},
+		{name: "known duplicate", path: "/notify", contentType: "application/json", body: `{"title":"one","TITLE":"two","message":"m"}`, wantStatus: 400},
+		{name: "unknown accepted", path: "/notify", contentType: "application/json", body: `{"title":"t","message":"m","future":true}`, wantStatus: 200},
+		{name: "case insensitive", path: "/notify", contentType: "application/json", body: `{"TITLE":"t","MESSAGE":"m"}`, wantStatus: 200},
+		{name: "backup negative size", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":"a","backupSizeInMB":-1}`, wantStatus: 400},
+		{name: "backup missing size", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":"a"}`, wantStatus: 400},
+		{name: "backup wrong size type", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":"a","backupSizeInMB":"large"}`, wantStatus: 400},
+		{name: "backup missing title", path: "/template/notify/backup", contentType: "application/json", body: `{"asset":"a","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup blank title", path: "/template/notify/backup", contentType: "application/json", body: `{"title":" ","asset":"a","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup wrong title type", path: "/template/notify/backup", contentType: "application/json", body: `{"title":1,"asset":"a","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup missing asset", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup blank asset", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":" ","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup wrong asset type", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":1,"backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup wrong extra type", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":"a","backupSizeInMB":1,"extra":{}}`, wantStatus: 400},
+		{name: "update negative time", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":"a","time":-1}`, wantStatus: 400},
+		{name: "update missing host", path: "/template/notify/update", contentType: "application/json", body: `{"asset":"a","time":1}`, wantStatus: 400},
+		{name: "update blank host", path: "/template/notify/update", contentType: "application/json", body: `{"host":" ","asset":"a","time":1}`, wantStatus: 400},
+		{name: "update wrong host type", path: "/template/notify/update", contentType: "application/json", body: `{"host":1,"asset":"a","time":1}`, wantStatus: 400},
+		{name: "update missing asset", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","time":1}`, wantStatus: 400},
+		{name: "update blank asset", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":" ","time":1}`, wantStatus: 400},
+		{name: "update wrong asset type", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":false,"time":1}`, wantStatus: 400},
+		{name: "update missing time", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":"a"}`, wantStatus: 400},
+		{name: "update wrong time type", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":"a","time":"soon"}`, wantStatus: 400},
+		{name: "error missing critical", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m"}`, wantStatus: 400},
+		{name: "error wrong critical type", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":"yes"}`, wantStatus: 400},
+		{name: "error missing caller", path: "/template/notify/error", contentType: "application/json", body: `{"message":"m","critical":false}`, wantStatus: 400},
+		{name: "error blank caller", path: "/template/notify/error", contentType: "application/json", body: `{"caller":" ","message":"m","critical":false}`, wantStatus: 400},
+		{name: "error wrong caller type", path: "/template/notify/error", contentType: "application/json", body: `{"caller":1,"message":"m","critical":false}`, wantStatus: 400},
+		{name: "error missing message", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","critical":false}`, wantStatus: 400},
+		{name: "error blank message", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":" ","critical":false}`, wantStatus: 400},
+		{name: "error wrong message type", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":[],"critical":false}`, wantStatus: 400},
+		{name: "invalid extra", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":false,"extra":[{"name":"n"}]}`, wantStatus: 400},
+		{name: "null extra", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":false,"extra":null}`, wantStatus: 400},
+		{name: "duplicate nested known key", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":false,"extra":[{"name":"one","NAME":"two","value":"v"}]}`, wantStatus: 400},
+		{name: "case insensitive nested fields", path: "/template/notify/error", contentType: "application/json", body: `{"CALLER":"c","MESSAGE":"m","CRITICAL":false,"EXTRA":[{"NAME":"n","VALUE":"v"}]}`, wantStatus: 200},
+		{name: "unknown nested field accepted", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":false,"extra":[{"name":"n","value":"v","future":true}]}`, wantStatus: 200},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			got := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, test.path, test.contentType, test.body)
+			if got.Code != test.wantStatus {
+				t.Fatalf("status = %d, body = %q, want %d", got.Code, got.Body.String(), test.wantStatus)
+			}
+			wantBody := map[int]string{
+				http.StatusOK:                   "Notification sent",
+				http.StatusBadRequest:           "Invalid payload",
+				http.StatusUnsupportedMediaType: "Content-Type must be application/json",
+			}[test.wantStatus]
+			if got.Body.String() != wantBody {
+				t.Fatalf("body = %q, want %q", got.Body.String(), wantBody)
+			}
+			if got.Header().Get("Content-Type") != "text/plain; charset=utf-8" {
+				t.Fatalf("Content-Type = %q", got.Header().Get("Content-Type"))
+			}
+			wantCalls := 0
+			if test.wantStatus == http.StatusOK {
+				wantCalls = 1
+			}
+			if len(recorder.bodies()) != wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", len(recorder.bodies()), wantCalls)
+			}
+		})
+	}
+}
+
+func TestRequestBodyLimit(t *testing.T) {
+	prefix, suffix := `{"title":"t","message":"`, `"}`
+	for _, test := range []struct {
+		name      string
+		bodyBytes int
+		want      int
+		wantCalls int
+	}{
+		{name: "exactly at limit", bodyBytes: maxRequestBody, want: http.StatusOK, wantCalls: 1},
+		{name: "one byte over limit", bodyBytes: maxRequestBody + 1, want: http.StatusBadRequest},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			body := prefix + strings.Repeat("x", test.bodyBytes-len(prefix)-len(suffix)) + suffix
+			if len(body) != test.bodyBytes {
+				t.Fatalf("body length = %d, want %d", len(body), test.bodyBytes)
+			}
+			got := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, "/notify", "application/json", body)
+			if got.Code != test.want {
+				t.Fatalf("status = %d, want %d", got.Code, test.want)
+			}
+			if len(recorder.bodies()) != test.wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", len(recorder.bodies()), test.wantCalls)
+			}
+		})
+	}
+}
+
+func TestNotificationPayloads(t *testing.T) {
+	tests := []struct {
+		name  string
+		path  string
+		body  string
+		check func(*testing.T, discordPayload)
+	}{
+		{
+			name: "plain preserves unicode and quotes",
+			path: "/notify",
+			body: `{"title":"Héllo \"world\"","message":"line 1\nline 2 @everyone"}`,
+			check: func(t *testing.T, payload discordPayload) {
+				if payload.Content != "**Héllo \"world\"**\nline 1\nline 2 @everyone" {
+					t.Fatalf("content = %q", payload.Content)
+				}
+			},
+		},
+		{
+			name: "backup formats gibibytes and extras",
+			path: "/template/notify/backup",
+			body: `{"title":"Nightly","asset":"photos","backupSizeInMB":1536,"extra":[{"name":"Host","value":"nebula"}]}`,
+			check: func(t *testing.T, payload discordPayload) {
+				embed := payload.Embeds[0]
+				if embed.Title != "📦 Backup - Nightly" || embed.Fields[0].Value != "1.50 GiB" || embed.Fields[1].Value != "nebula" {
+					t.Fatalf("unexpected backup embed: %+v", embed)
+				}
+			},
+		},
+		{
+			name: "update accepts zero time",
+			path: "/template/notify/update",
+			body: `{"host":"nexus","asset":"k3s","time":0}`,
+			check: func(t *testing.T, payload discordPayload) {
+				if payload.Embeds[0].Fields[0].Value != "0 seconds" {
+					t.Fatalf("time = %q", payload.Embeds[0].Fields[0].Value)
+				}
+			},
+		},
+		{
+			name: "error does not HTML escape",
+			path: "/template/notify/error",
+			body: `{"caller":"backup","message":"line 1\n\"quoted\" <tag>","critical":true}`,
+			check: func(t *testing.T, payload discordPayload) {
+				embed := payload.Embeds[0]
+				if embed.Color != 15158332 || embed.Fields[0].Value != "line 1\n\"quoted\" <tag>" {
+					t.Fatalf("unexpected error embed: %+v", embed)
+				}
+			},
+		},
+		{
+			name: "noncritical color",
+			path: "/template/notify/error",
+			body: `{"caller":"backup","message":"failure","critical":false}`,
+			check: func(t *testing.T, payload discordPayload) {
+				if payload.Embeds[0].Color != 15844367 {
+					t.Fatalf("color = %d", payload.Embeds[0].Color)
+				}
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			got := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, test.path, "application/json", test.body)
+			if got.Code != http.StatusOK || got.Body.String() != "Notification sent" {
+				t.Fatalf("response = (%d, %q)", got.Code, got.Body.String())
+			}
+			bodies := recorder.bodies()
+			if len(bodies) != 1 || !json.Valid(bodies[0]) {
+				t.Fatalf("webhook bodies = %q", bodies)
+			}
+			var payload discordPayload
+			if err := json.Unmarshal(bodies[0], &payload); err != nil {
+				t.Fatal(err)
+			}
+			if payload.AllowedMentions.Parse == nil || len(payload.AllowedMentions.Parse) != 0 {
+				t.Fatalf("allowed mentions = %#v", payload.AllowedMentions)
+			}
+			test.check(t, payload)
+		})
+	}
+}
+
+func TestHandlerMapsWebhookErrorsAndDoesNotLogSecrets(t *testing.T) {
+	tests := []struct {
+		name       string
+		err        error
+		wantStatus int
+		wantBody   string
+	}{
+		{name: "delivery", err: errors.New("network down"), wantStatus: 502, wantBody: "Failed to send notification"},
+		{name: "timeout", err: context.DeadlineExceeded, wantStatus: 504, wantBody: "Webhook timed out"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var logs bytes.Buffer
+			transport := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, test.err })
+			app := testApplication(t, transport, log.New(&logs, "", 0))
+			got := performRequest(app.handler(), http.MethodPost, "/notify", "application/json", `{"title":"TOP-SECRET","message":"private-message"}`)
+			if got.Code != test.wantStatus || got.Body.String() != test.wantBody {
+				t.Fatalf("response = (%d, %q)", got.Code, got.Body.String())
+			}
+			for _, secret := range []string{"TOP-SECRET", "private-message", "discord.invalid"} {
+				if strings.Contains(logs.String(), secret) {
+					t.Fatalf("logs exposed %q: %s", secret, logs.String())
+				}
+			}
+		})
+	}
+}
+
+func TestConfigurationAndServerLifecycle(t *testing.T) {
+	for _, webhookURL := range []string{"", "not a URL", "ftp://example.com/hook", "https:///missing-host"} {
+		if _, err := newApplication(applicationConfig{WebhookURL: webhookURL}); err == nil {
+			t.Fatalf("newApplication(%q) succeeded", webhookURL)
+		}
+	}
+
+	app := testApplication(t, nil, nil)
+	server := newHTTPServer(app.handler())
+	if server.ReadHeaderTimeout != 5*time.Second || server.ReadTimeout != 10*time.Second || server.WriteTimeout != 15*time.Second || server.IdleTimeout != 60*time.Second {
+		t.Fatalf("unexpected server timeouts: %+v", server)
+	}
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- serve(ctx, server, listener) }()
+	cancel()
+	select {
+	case err := <-done:
+		if !errors.Is(err, http.ErrServerClosed) {
+			t.Fatalf("serve() error = %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("server did not shut down")
+	}
+}
+
+func TestRunLifecycleAndListenError(t *testing.T) {
+	if err := run(context.Background(), "", "127.0.0.1:0", log.New(io.Discard, "", 0)); err == nil {
+		t.Fatal("run() accepted missing configuration")
+	}
+
+	originalListen := netListen
+	netListen = func(string, string) (net.Listener, error) { return nil, errors.New("listen failed") }
+	if err := run(context.Background(), "http://discord.invalid/hook", "bad", log.New(io.Discard, "", 0)); err == nil || !strings.Contains(err.Error(), "listen failed") {
+		t.Fatalf("run() error = %v", err)
+	}
+	netListen = originalListen
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := run(ctx, "http://discord.invalid/hook", "127.0.0.1:0", log.New(io.Discard, "", 0)); !errors.Is(err, http.ErrServerClosed) {
+		t.Fatalf("run() error = %v", err)
+	}
+}
+
+func TestTemplateHandlerRenderFailures(t *testing.T) {
+	app := testApplication(t, nil, nil)
+	app.renderer = &templateRenderer{templates: map[string]*template.Template{}}
+	tests := []struct {
+		path string
+		body string
+	}{
+		{path: "/template/notify/backup", body: `{"title":"t","asset":"a","backupSizeInMB":1}`},
+		{path: "/template/notify/update", body: `{"host":"h","asset":"a","time":1}`},
+		{path: "/template/notify/error", body: `{"caller":"c","message":"m","critical":false}`},
+	}
+	for _, test := range tests {
+		got := performRequest(app.handler(), http.MethodPost, test.path, "application/json", test.body)
+		if got.Code != http.StatusInternalServerError || got.Body.String() != "Failed to render notification" {
+			t.Fatalf("%s response = (%d, %q)", test.path, got.Code, got.Body.String())
+		}
+	}
+}
+
+func FuzzErrorNotificationHandler(f *testing.F) {
+	seeds := []string{
+		`{"caller":"seed","message":"ok","critical":false}`,
+		`{"caller":"quotes","message":"line\n\"quoted\" <tag>","critical":true}`,
+		`{}`,
+		`not-json`,
+	}
+	for _, seed := range seeds {
+		f.Add([]byte(seed))
+	}
+
+	transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+		body, err := io.ReadAll(request.Body)
+		if err != nil || !json.Valid(body) {
+			return nil, errors.New("invalid outgoing JSON")
+		}
+		return response(http.StatusNoContent, nil), nil
+	})
+	handler := testApplication(f, transport, nil).handler()
+	f.Fuzz(func(t *testing.T, body []byte) {
+		request := httptest.NewRequest(http.MethodPost, "/template/notify/error", bytes.NewReader(body))
+		request.Header.Set("Content-Type", "application/json")
+		recorder := httptest.NewRecorder()
+		handler.ServeHTTP(recorder, request)
+		if recorder.Code < 200 || recorder.Code > 599 {
+			t.Fatalf("invalid status %d", recorder.Code)
+		}
+	})
+}
+
+func response(status int, headers http.Header) *http.Response {
+	if headers == nil {
+		headers = make(http.Header)
+	}
+	return &http.Response{StatusCode: status, Header: headers, Body: io.NopCloser(strings.NewReader(""))}
+}

haven-notify/template/backup.tmpl

@@ -0,0 +1,26 @@
+{
+  "embeds": [
+    {
+      "title": {{json (printf "📦 Backup - %s" .Title)}},
+      "description": {{json (printf "**%s** has been backed up successfully! ✅🫡\n" .Asset)}},
+      "color": 3066993,
+      "fields": [
+        {
+          "name": "💾 Backup Size",
+          "value": {{json (formatSize .BackupSizeInMB)}},
+          "inline": true
+        }
+        {{- range .Extra}},
+        {
+          "name": {{json .Name}},
+          "value": {{json .Value}},
+          "inline": true
+        }
+        {{- end}}
+      ],
+      "footer": {
+        "text": "✨ Haven Notify ✨"
+      }
+    }
+  ]
+}

haven-notify/template/backup_template.tmpl

@@ -1,40 +0,0 @@
-{{/*
-Docker Backup Notification Template
-Variables expected:
-- .title: The backup title/name
-- .asset: The asset being backed up  
-- .backupsizeinmb: The backup size in MB (will be formatted automatically)
-- .extra: Optional array of additional fields with .name and .value
-
-Template Functions Available:
-- formatSize: Formats size in MB/GiB automatically
-*/}}
-{
-  "embeds": [
-    {
-      "title": "📦 Docker Backup - {{.title}}",
-      "description": "**{{.asset}}** has been backup-ed successfully! ✅🫡\n",
-      "color": 3066993,
-      "fields": [
-        {
-          "name": "💾 Backup Size",
-          "value": "{{if .backupsizeinmb}}{{formatSize .backupsizeinmb}}{{else}}Unknown{{end}}",
-          "inline": true
-        }
-        {{- if .extra}}
-        {{- range $index, $field := .extra}},
-        {
-          "name": "{{$field.name}}",
-          "value": "{{$field.value}}",
-          "inline": true
-        }
-        {{- end}}
-        {{- end}}
-      ],
-      "footer": {
-        "text": "✨ Haven Notify ✨"
-      }{{if .timestamp}},
-      "timestamp": "{{formatTime .timestamp}}"{{end}}
-    }
-  ]
-}

haven-notify/template/error.tmpl

@@ -0,0 +1,26 @@
+{
+  "embeds": [
+    {
+      "title": "❌ Error",
+      "description": {{json (printf "**%s** encountered an error!" .Caller)}},
+      "color": {{if .Critical}}15158332{{else}}15844367{{end}},
+      "fields": [
+        {
+          "name": "📄 Message",
+          "value": {{json .Message}},
+          "inline": false
+        }
+        {{- range .Extra}},
+        {
+          "name": {{json .Name}},
+          "value": {{json .Value}},
+          "inline": true
+        }
+        {{- end}}
+      ],
+      "footer": {
+        "text": "✨ Haven Notify ✨"
+      }
+    }
+  ]
+}

haven-notify/template/update.tmpl

@@ -0,0 +1,19 @@
+{
+  "embeds": [
+    {
+      "title": {{json (printf "🔄 Update - %s" .Asset)}},
+      "description": {{json (printf "**%s** has successfully updated **%s**! ✅" .Host .Asset)}},
+      "color": 3447003,
+      "fields": [
+        {
+          "name": "⏱️ Time Taken",
+          "value": {{json (formatSeconds .Time)}},
+          "inline": true
+        }
+      ],
+      "footer": {
+        "text": "✨ Haven Notify ✨"
+      }
+    }
+  ]
+}

haven-notify/templates.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"bytes"
+	"embed"
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"strconv"
+	"text/template"
+)
+
+//go:embed template/*.tmpl
+var templatesFS embed.FS
+
+type templateRenderer struct {
+	templates map[string]*template.Template
+}
+
+type templateExtra struct {
+	Name  string
+	Value string
+}
+
+type backupTemplateData struct {
+	Title          string
+	Asset          string
+	BackupSizeInMB float64
+	Extra          []templateExtra
+}
+
+type updateTemplateData struct {
+	Host  string
+	Asset string
+	Time  float64
+}
+
+type errorTemplateData struct {
+	Caller   string
+	Message  string
+	Critical bool
+	Extra    []templateExtra
+}
+
+func newTemplateRenderer() (*templateRenderer, error) {
+	return newTemplateRendererFromFS(templatesFS)
+}
+
+func newTemplateRendererFromFS(templateFS fs.FS) (*templateRenderer, error) {
+	functions := template.FuncMap{
+		"formatSize":    formatSize,
+		"formatSeconds": formatSeconds,
+		"json":          jsonLiteral,
+	}
+	templates := make(map[string]*template.Template, 3)
+	for _, name := range []string{"backup", "update", "error"} {
+		contents, err := fs.ReadFile(templateFS, "template/"+name+".tmpl")
+		if err != nil {
+			return nil, err
+		}
+		parsed, err := template.New(name).Funcs(functions).Parse(string(contents))
+		if err != nil {
+			return nil, err
+		}
+		templates[name] = parsed
+	}
+	return &templateRenderer{templates: templates}, nil
+}
+
+func (r *templateRenderer) renderBackup(request backupRequest) (discordPayload, error) {
+	return r.render("backup", backupTemplateData{
+		Title:          *request.Title,
+		Asset:          *request.Asset,
+		BackupSizeInMB: *request.BackupSizeInMB,
+		Extra:          templateExtras(request.Extra),
+	})
+}
+
+func (r *templateRenderer) renderUpdate(request updateRequest) (discordPayload, error) {
+	return r.render("update", updateTemplateData{
+		Host:  *request.Host,
+		Asset: *request.Asset,
+		Time:  *request.Time,
+	})
+}
+
+func (r *templateRenderer) renderError(request errorRequest) (discordPayload, error) {
+	return r.render("error", errorTemplateData{
+		Caller:   *request.Caller,
+		Message:  *request.Message,
+		Critical: *request.Critical,
+		Extra:    templateExtras(request.Extra),
+	})
+}
+
+func (r *templateRenderer) render(name string, data any) (discordPayload, error) {
+	parsed, ok := r.templates[name]
+	if !ok {
+		return discordPayload{}, fmt.Errorf("unknown template %q", name)
+	}
+	var rendered bytes.Buffer
+	if err := parsed.Execute(&rendered, data); err != nil {
+		return discordPayload{}, err
+	}
+	var payload discordPayload
+	if err := json.Unmarshal(rendered.Bytes(), &payload); err != nil {
+		return discordPayload{}, fmt.Errorf("template produced invalid JSON: %w", err)
+	}
+	normalizeDiscordPayload(&payload)
+	return payload, nil
+}
+
+func templateExtras(extra extraRequests) []templateExtra {
+	result := make([]templateExtra, 0, len(extra))
+	for _, field := range extra {
+		result = append(result, templateExtra{Name: *field.Name, Value: *field.Value})
+	}
+	return result
+}
+
+func jsonLiteral(value any) (string, error) {
+	encoded, err := json.Marshal(value)
+	return string(encoded), err
+}
+
+func formatSize(size float64) string {
+	if size >= 1024 {
+		return fmt.Sprintf("%.2f GiB", size/1024)
+	}
+	return fmt.Sprintf("%.2f MiB", size)
+}
+
+func formatSeconds(seconds float64) string {
+	return strconv.FormatFloat(seconds, 'f', -1, 64) + " seconds"
+}

haven-notify/templates_test.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"testing/fstest"
+	"text/template"
+)
+
+func stringPointer(value string) *string  { return &value }
+func floatPointer(value float64) *float64 { return &value }
+func boolPointer(value bool) *bool        { return &value }
+
+func TestFormatHelpers(t *testing.T) {
+	tests := []struct {
+		value float64
+		want  string
+	}{
+		{0, "0.00 MiB"},
+		{1023, "1023.00 MiB"},
+		{1024, "1.00 GiB"},
+		{1536, "1.50 GiB"},
+	}
+	for _, test := range tests {
+		if got := formatSize(test.value); got != test.want {
+			t.Errorf("formatSize(%v) = %q, want %q", test.value, got, test.want)
+		}
+	}
+	if got := formatSeconds(42.5); got != "42.5 seconds" {
+		t.Fatalf("formatSeconds() = %q", got)
+	}
+	literal, err := jsonLiteral("line\n\"quoted\"")
+	if err != nil || literal != "\"line\\n\\\"quoted\\\"\"" {
+		t.Fatalf("jsonLiteral() = %q, %v", literal, err)
+	}
+}
+
+func TestTemplateRenderer(t *testing.T) {
+	renderer, err := newTemplateRenderer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	backup, err := renderer.renderBackup(backupRequest{
+		Title: stringPointer("Nightly"), Asset: stringPointer("photos"), BackupSizeInMB: floatPointer(1024),
+		Extra: []extraRequest{{Name: stringPointer("Host"), Value: stringPointer("nebula")}},
+	})
+	if err != nil || backup.Embeds[0].Fields[0].Value != "1.00 GiB" || len(backup.Embeds[0].Fields) != 2 {
+		t.Fatalf("backup render = %+v, %v", backup, err)
+	}
+
+	update, err := renderer.renderUpdate(updateRequest{Host: stringPointer("nexus"), Asset: stringPointer("k3s"), Time: floatPointer(0)})
+	if err != nil || update.Embeds[0].Fields[0].Value != "0 seconds" {
+		t.Fatalf("update render = %+v, %v", update, err)
+	}
+
+	errorPayload, err := renderer.renderError(errorRequest{
+		Caller: stringPointer("job"), Message: stringPointer("\"quoted\" <tag>"), Critical: boolPointer(false),
+	})
+	if err != nil || errorPayload.Embeds[0].Fields[0].Value != "\"quoted\" <tag>" || errorPayload.Embeds[0].Color != 15844367 {
+		t.Fatalf("error render = %+v, %v", errorPayload, err)
+	}
+
+	if _, err := renderer.render("missing", struct{}{}); err == nil {
+		t.Fatal("unknown template rendered")
+	}
+
+	encoded, err := json.Marshal(errorPayload)
+	if err != nil || !json.Valid(encoded) || strings.Contains(string(encoded), "&#34;") {
+		t.Fatalf("rendered JSON = %s, %v", encoded, err)
+	}
+}
+
+func TestTemplateExtraFieldCap(t *testing.T) {
+	renderer, err := newTemplateRenderer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	extra := make([]extraRequest, 40)
+	for index := range extra {
+		extra[index] = extraRequest{Name: stringPointer(fmt.Sprintf("field-%02d", index)), Value: stringPointer("value")}
+	}
+	payload, err := renderer.renderError(errorRequest{
+		Caller: stringPointer("job"), Message: stringPointer("failure"), Critical: boolPointer(true), Extra: extra,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(payload.Embeds[0].Fields) != 25 {
+		t.Fatalf("fields = %d, want 25", len(payload.Embeds[0].Fields))
+	}
+	if payload.Embeds[0].Fields[24].Name != "field-23" {
+		t.Fatalf("last retained field = %q", payload.Embeds[0].Fields[24].Name)
+	}
+}
+
+func TestTemplateRendererFailures(t *testing.T) {
+	if _, err := newTemplateRendererFromFS(fstest.MapFS{}); err == nil {
+		t.Fatal("missing templates were accepted")
+	}
+	invalidFS := fstest.MapFS{
+		"template/backup.tmpl": {Data: []byte("{{")},
+		"template/update.tmpl": {Data: []byte("{}")},
+		"template/error.tmpl":  {Data: []byte("{}")},
+	}
+	if _, err := newTemplateRendererFromFS(invalidFS); err == nil {
+		t.Fatal("invalid template was accepted")
+	}
+
+	renderer := &templateRenderer{templates: map[string]*template.Template{}}
+	renderer.templates["invalid-json"] = template.Must(template.New("invalid-json").Parse("not-json"))
+	if _, err := renderer.render("invalid-json", nil); err == nil {
+		t.Fatal("invalid JSON template rendered")
+	}
+	renderer.templates["execute-error"] = template.Must(template.New("execute-error").Parse("{{index . 0}}"))
+	if _, err := renderer.render("execute-error", nil); err == nil {
+		t.Fatal("template execution error was ignored")
+	}
+}

haven-notify/v2.go

@@ -0,0 +1,306 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+const (
+	v2ColorBrand    = 0x5865F2
+	v2ColorSuccess  = 0x57F287
+	v2ColorUpdate   = 0x3498DB
+	v2ColorWarning  = 0xFEE75C
+	v2ColorError    = 0xED4245
+	v2ColorCritical = 0x992D22
+
+	v2InlineFieldLimit = 80
+)
+
+type v2CommonRequest struct {
+	Source     *string       `json:"source"`
+	OccurredAt *string       `json:"occurredAt"`
+	Extra      extraRequests `json:"extra"`
+}
+
+type v2MessageRequest struct {
+	v2CommonRequest
+	Title   *string `json:"title"`
+	Message *string `json:"message"`
+}
+
+type v2BackupRequest struct {
+	v2CommonRequest
+	Title           *string  `json:"title"`
+	Asset           *string  `json:"asset"`
+	SizeBytes       *int64   `json:"sizeBytes"`
+	DurationSeconds *float64 `json:"durationSeconds"`
+}
+
+type v2UpdateRequest struct {
+	v2CommonRequest
+	Host            *string  `json:"host"`
+	Asset           *string  `json:"asset"`
+	DurationSeconds *float64 `json:"durationSeconds"`
+	FromVersion     *string  `json:"fromVersion"`
+	ToVersion       *string  `json:"toVersion"`
+}
+
+type v2ErrorRequest struct {
+	v2CommonRequest
+	Message   *string `json:"message"`
+	Severity  *string `json:"severity"`
+	ErrorCode *string `json:"errorCode"`
+}
+
+func (a *application) v2MessageHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2MessageRequest
+	if err := decodeRequest(w, r, &request, "title", "message", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Message) || !validOptionalString(request.Title) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2MessagePayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2BackupHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2BackupRequest
+	if err := decodeRequest(w, r, &request, "title", "asset", "sizebytes", "durationseconds", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Asset) || !validOptionalString(request.Title) || request.SizeBytes == nil || *request.SizeBytes < 0 || !validOptionalNumber(request.DurationSeconds) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2BackupPayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2UpdateHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2UpdateRequest
+	if err := decodeRequest(w, r, &request, "host", "asset", "durationseconds", "fromversion", "toversion", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Host) || !requiredString(request.Asset) || !validNumber(request.DurationSeconds) || !validOptionalString(request.FromVersion) || !validOptionalString(request.ToVersion) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2UpdatePayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2ErrorHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2ErrorRequest
+	if err := decodeRequest(w, r, &request, "source", "message", "severity", "errorcode", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Source) || !requiredString(request.Message) || !validV2Severity(request.Severity) || !validOptionalString(request.ErrorCode) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2ErrorPayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) validV2Common(request v2CommonRequest) (time.Time, bool) {
+	if !validOptionalString(request.Source) || !validExtra(request.Extra) {
+		return time.Time{}, false
+	}
+	if request.OccurredAt == nil {
+		return a.now().UTC(), true
+	}
+	if !requiredString(request.OccurredAt) {
+		return time.Time{}, false
+	}
+	when, err := time.Parse(time.RFC3339, *request.OccurredAt)
+	if err != nil {
+		return time.Time{}, false
+	}
+	return when.UTC(), true
+}
+
+func validOptionalString(value *string) bool {
+	return value == nil || requiredString(value)
+}
+
+func validNumber(value *float64) bool {
+	return value != nil && *value >= 0 && !math.IsNaN(*value) && !math.IsInf(*value, 0)
+}
+
+func validOptionalNumber(value *float64) bool {
+	return value == nil || validNumber(value)
+}
+
+func validV2Severity(value *string) bool {
+	if value == nil {
+		return false
+	}
+	switch *value {
+	case "warning", "error", "critical":
+		return true
+	default:
+		return false
+	}
+}
+
+func buildV2MessagePayload(request v2MessageRequest, when time.Time) discordPayload {
+	title := "Message"
+	if request.Title != nil {
+		title = *request.Title
+	}
+	embed := newV2Embed("💬 "+title, *request.Message, v2ColorBrand, when)
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2BackupPayload(request v2BackupRequest, when time.Time) discordPayload {
+	title := "Backup completed"
+	if request.Title != nil {
+		title = *request.Title
+	}
+	embed := newV2Embed("📦 "+title, fmt.Sprintf("**%s** was backed up successfully.", *request.Asset), v2ColorSuccess, when)
+	embed.Fields = append(embed.Fields, v2Field("Size", formatBytes(*request.SizeBytes)))
+	if request.DurationSeconds != nil {
+		embed.Fields = append(embed.Fields, v2Field("Duration", formatV2Duration(*request.DurationSeconds)))
+	}
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2UpdatePayload(request v2UpdateRequest, when time.Time) discordPayload {
+	embed := newV2Embed("🔄 Update completed", fmt.Sprintf("**%s** was updated on **%s**.", *request.Asset, *request.Host), v2ColorUpdate, when)
+	switch {
+	case request.FromVersion != nil && request.ToVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Version", *request.FromVersion+" → "+*request.ToVersion))
+	case request.FromVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Previous version", *request.FromVersion))
+	case request.ToVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Installed version", *request.ToVersion))
+	}
+	embed.Fields = append(embed.Fields, v2Field("Duration", formatV2Duration(*request.DurationSeconds)))
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2ErrorPayload(request v2ErrorRequest, when time.Time) discordPayload {
+	title, color := v2ErrorStyle(*request.Severity)
+	embed := newV2Embed(title, *request.Message, color, when)
+	embed.Fields = append(embed.Fields, v2Field("Source", *request.Source))
+	if request.ErrorCode != nil {
+		embed.Fields = append(embed.Fields, v2Field("Error code", *request.ErrorCode))
+	}
+	embed.Fields = appendV2Extras(embed.Fields, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func newV2Embed(title, description string, color int, when time.Time) discordEmbed {
+	return discordEmbed{
+		Title:       title,
+		Description: description,
+		Color:       color,
+		Timestamp:   when.UTC().Format(time.RFC3339),
+		Footer:      &discordEmbedFooter{Text: "Haven Notify"},
+	}
+}
+
+func v2ErrorStyle(severity string) (string, int) {
+	switch severity {
+	case "warning":
+		return "⚠️ Warning", v2ColorWarning
+	case "critical":
+		return "🛑 Critical error", v2ColorCritical
+	default:
+		return "❌ Error", v2ColorError
+	}
+}
+
+func appendV2SourceAndExtras(fields []discordEmbedField, source *string, extra extraRequests) []discordEmbedField {
+	if source != nil {
+		fields = append(fields, v2Field("Source", *source))
+	}
+	return appendV2Extras(fields, extra)
+}
+
+func appendV2Extras(fields []discordEmbedField, extra extraRequests) []discordEmbedField {
+	for _, field := range extra {
+		value := *field.Value
+		fields = append(fields, discordEmbedField{
+			Name:   *field.Name,
+			Value:  value,
+			Inline: !strings.ContainsAny(value, "\r\n") && utf8.RuneCountInString(value) <= v2InlineFieldLimit,
+		})
+	}
+	return fields
+}
+
+func v2Field(name, value string) discordEmbedField {
+	return discordEmbedField{Name: name, Value: value, Inline: true}
+}
+
+func formatBytes(bytes int64) string {
+	if bytes < 1024 {
+		return strconv.FormatInt(bytes, 10) + " B"
+	}
+	value := float64(bytes)
+	for _, unit := range []string{"KiB", "MiB", "GiB", "TiB", "PiB", "EiB"} {
+		value /= 1024
+		if value < 1024 || unit == "EiB" {
+			return fmt.Sprintf("%.2f %s", value, unit)
+		}
+	}
+	return strconv.FormatInt(bytes, 10) + " B"
+}
+
+func formatV2Duration(seconds float64) string {
+	if seconds < 60 {
+		return formatV2Number(seconds) + "s"
+	}
+
+	hours := math.Floor(seconds / 3600)
+	seconds -= hours * 3600
+	minutes := math.Floor(seconds / 60)
+	seconds -= minutes * 60
+	parts := make([]string, 0, 3)
+	if hours > 0 {
+		parts = append(parts, formatV2Number(hours)+"h")
+	}
+	if minutes > 0 {
+		parts = append(parts, formatV2Number(minutes)+"m")
+	}
+	if seconds > 0 {
+		parts = append(parts, formatV2Number(seconds)+"s")
+	}
+	return strings.Join(parts, " ")
+}
+
+func formatV2Number(value float64) string {
+	formatted := strconv.FormatFloat(value, 'f', 3, 64)
+	return strings.TrimRight(strings.TrimRight(formatted, "0"), ".")
+}

haven-notify/v2_test.go

@@ -0,0 +1,386 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+const v2TestTimestamp = "2026-07-16T12:34:56Z"
+
+func TestV2NotificationPayloads(t *testing.T) {
+	tests := []struct {
+		name string
+		path string
+		body string
+		want discordPayload
+	}{
+		{
+			name: "generic message",
+			path: "/api/v2/messages",
+			body: `{"title":"Maintenance","message":"The storage job completed.","source":"scheduler","occurredAt":"2026-07-16T09:34:56-03:00","extra":[{"name":"Region","value":"home"},{"name":"Details","value":"first line\nsecond line"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "💬 Maintenance",
+					Description: "The storage job completed.",
+					Color:       v2ColorBrand,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Source", Value: "scheduler", Inline: true},
+						{Name: "Region", Value: "home", Inline: true},
+						{Name: "Details", Value: "first line\nsecond line", Inline: false},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "backup",
+			path: "/api/v2/notifications/backup",
+			body: `{"title":"Nightly backup","asset":"Photos","sizeBytes":1610612736,"durationSeconds":125,"source":"restic","extra":[{"name":"Host","value":"nebula"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "📦 Nightly backup",
+					Description: "**Photos** was backed up successfully.",
+					Color:       v2ColorSuccess,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Size", Value: "1.50 GiB", Inline: true},
+						{Name: "Duration", Value: "2m 5s", Inline: true},
+						{Name: "Source", Value: "restic", Inline: true},
+						{Name: "Host", Value: "nebula", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "update",
+			path: "/api/v2/notifications/update",
+			body: `{"host":"nexus","asset":"k3s","durationSeconds":42.5,"fromVersion":"1.32.5","toVersion":"1.33.1","source":"system-updater"}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "🔄 Update completed",
+					Description: "**k3s** was updated on **nexus**.",
+					Color:       v2ColorUpdate,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Version", Value: "1.32.5 → 1.33.1", Inline: true},
+						{Name: "Duration", Value: "42.5s", Inline: true},
+						{Name: "Source", Value: "system-updater", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "critical error",
+			path: "/api/v2/notifications/error",
+			body: `{"source":"backup-job","message":"The repository is unavailable.","severity":"critical","errorCode":"E_REPOSITORY","extra":[{"name":"Path","value":"/mnt/backups"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "🛑 Critical error",
+					Description: "The repository is unavailable.",
+					Color:       v2ColorCritical,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Source", Value: "backup-job", Inline: true},
+						{Name: "Error code", Value: "E_REPOSITORY", Inline: true},
+						{Name: "Path", Value: "/mnt/backups", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := sendV2Request(t, test.path, test.body)
+			if !reflect.DeepEqual(got, test.want) {
+				gotJSON, _ := json.MarshalIndent(got, "", "  ")
+				wantJSON, _ := json.MarshalIndent(test.want, "", "  ")
+				t.Fatalf("payload = %s\nwant = %s", gotJSON, wantJSON)
+			}
+		})
+	}
+}
+
+func TestV2DefaultsAndVariants(t *testing.T) {
+	tests := []struct {
+		name       string
+		path       string
+		body       string
+		wantTitle  string
+		wantColor  int
+		wantFields []discordEmbedField
+	}{
+		{
+			name:      "message default title",
+			path:      "/api/v2/messages",
+			body:      `{"message":"hello"}`,
+			wantTitle: "💬 Message",
+			wantColor: v2ColorBrand,
+		},
+		{
+			name:      "backup defaults",
+			path:      "/api/v2/notifications/backup",
+			body:      `{"asset":"config","sizeBytes":0}`,
+			wantTitle: "📦 Backup completed",
+			wantColor: v2ColorSuccess,
+			wantFields: []discordEmbedField{
+				{Name: "Size", Value: "0 B", Inline: true},
+			},
+		},
+		{
+			name:      "update previous version",
+			path:      "/api/v2/notifications/update",
+			body:      `{"host":"nexus","asset":"k3s","durationSeconds":60,"fromVersion":"1.0"}`,
+			wantTitle: "🔄 Update completed",
+			wantColor: v2ColorUpdate,
+			wantFields: []discordEmbedField{
+				{Name: "Previous version", Value: "1.0", Inline: true},
+				{Name: "Duration", Value: "1m", Inline: true},
+			},
+		},
+		{
+			name:      "update installed version",
+			path:      "/api/v2/notifications/update",
+			body:      `{"host":"nexus","asset":"k3s","durationSeconds":0,"toVersion":"1.1"}`,
+			wantTitle: "🔄 Update completed",
+			wantColor: v2ColorUpdate,
+			wantFields: []discordEmbedField{
+				{Name: "Installed version", Value: "1.1", Inline: true},
+				{Name: "Duration", Value: "0s", Inline: true},
+			},
+		},
+		{
+			name:      "warning",
+			path:      "/api/v2/notifications/error",
+			body:      `{"source":"job","message":"slow","severity":"warning"}`,
+			wantTitle: "⚠️ Warning",
+			wantColor: v2ColorWarning,
+			wantFields: []discordEmbedField{
+				{Name: "Source", Value: "job", Inline: true},
+			},
+		},
+		{
+			name:      "error",
+			path:      "/api/v2/notifications/error",
+			body:      `{"source":"job","message":"failed","severity":"error"}`,
+			wantTitle: "❌ Error",
+			wantColor: v2ColorError,
+			wantFields: []discordEmbedField{
+				{Name: "Source", Value: "job", Inline: true},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			payload := sendV2Request(t, test.path, test.body)
+			embed := payload.Embeds[0]
+			if embed.Title != test.wantTitle || embed.Color != test.wantColor || embed.Timestamp != v2TestTimestamp || !reflect.DeepEqual(embed.Fields, test.wantFields) {
+				t.Fatalf("embed = %+v", embed)
+			}
+		})
+	}
+}
+
+func TestV2RequestValidation(t *testing.T) {
+	tests := []struct {
+		name        string
+		path        string
+		contentType string
+		body        string
+		wantStatus  int
+	}{
+		{name: "message missing", path: "/api/v2/messages", body: `{}`, wantStatus: 400},
+		{name: "message blank", path: "/api/v2/messages", body: `{"message":" "}`, wantStatus: 400},
+		{name: "blank optional title", path: "/api/v2/messages", body: `{"message":"m","title":" "}`, wantStatus: 400},
+		{name: "blank optional source", path: "/api/v2/messages", body: `{"message":"m","source":" "}`, wantStatus: 400},
+		{name: "invalid timestamp", path: "/api/v2/messages", body: `{"message":"m","occurredAt":"yesterday"}`, wantStatus: 400},
+		{name: "blank timestamp", path: "/api/v2/messages", body: `{"message":"m","occurredAt":" "}`, wantStatus: 400},
+		{name: "null extras", path: "/api/v2/messages", body: `{"message":"m","extra":null}`, wantStatus: 400},
+		{name: "invalid extra", path: "/api/v2/messages", body: `{"message":"m","extra":[{"name":"n"}]}`, wantStatus: 400},
+		{name: "duplicate common key", path: "/api/v2/messages", body: `{"message":"m","source":"one","SOURCE":"two"}`, wantStatus: 400},
+		{name: "duplicate nested key", path: "/api/v2/messages", body: `{"message":"m","extra":[{"name":"one","NAME":"two","value":"v"}]}`, wantStatus: 400},
+		{name: "backup missing asset", path: "/api/v2/notifications/backup", body: `{"sizeBytes":1}`, wantStatus: 400},
+		{name: "backup missing size", path: "/api/v2/notifications/backup", body: `{"asset":"a"}`, wantStatus: 400},
+		{name: "backup negative size", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":-1}`, wantStatus: 400},
+		{name: "backup fractional size", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":1.5}`, wantStatus: 400},
+		{name: "backup negative duration", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":1,"durationSeconds":-1}`, wantStatus: 400},
+		{name: "update missing host", path: "/api/v2/notifications/update", body: `{"asset":"a","durationSeconds":1}`, wantStatus: 400},
+		{name: "update missing asset", path: "/api/v2/notifications/update", body: `{"host":"h","durationSeconds":1}`, wantStatus: 400},
+		{name: "update missing duration", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a"}`, wantStatus: 400},
+		{name: "update negative duration", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a","durationSeconds":-1}`, wantStatus: 400},
+		{name: "update blank version", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a","durationSeconds":1,"fromVersion":" "}`, wantStatus: 400},
+		{name: "error missing source", path: "/api/v2/notifications/error", body: `{"message":"m","severity":"error"}`, wantStatus: 400},
+		{name: "error missing message", path: "/api/v2/notifications/error", body: `{"source":"s","severity":"error"}`, wantStatus: 400},
+		{name: "error missing severity", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m"}`, wantStatus: 400},
+		{name: "error invalid severity", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m","severity":"ERROR"}`, wantStatus: 400},
+		{name: "error blank code", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m","severity":"error","errorCode":" "}`, wantStatus: 400},
+		{name: "wrong media type", path: "/api/v2/messages", contentType: "text/plain", body: `{"message":"m"}`, wantStatus: 415},
+		{name: "unknown accepted", path: "/api/v2/messages", body: `{"message":"m","future":true}`, wantStatus: 200},
+		{name: "case insensitive accepted", path: "/api/v2/messages", body: `{"MESSAGE":"m","EXTRA":[{"NAME":"n","VALUE":"v"}]}`, wantStatus: 200},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			contentType := test.contentType
+			if contentType == "" {
+				contentType = "application/json"
+			}
+			recorder := &webhookRecorder{}
+			response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, test.path, contentType, test.body)
+			if response.Code != test.wantStatus {
+				t.Fatalf("response = (%d, %q), want %d", response.Code, response.Body.String(), test.wantStatus)
+			}
+			wantCalls := 0
+			if test.wantStatus == http.StatusOK {
+				wantCalls = 1
+			}
+			if calls := len(recorder.bodies()); calls != wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", calls, wantCalls)
+			}
+		})
+	}
+}
+
+func TestV2RoutingContracts(t *testing.T) {
+	handler := testApplication(t, nil, nil).handler()
+	for _, path := range []string{
+		"/api/v2/messages",
+		"/api/v2/notifications/backup",
+		"/api/v2/notifications/update",
+		"/api/v2/notifications/error",
+	} {
+		response := performRequest(handler, http.MethodGet, path, "", "")
+		if response.Code != http.StatusMethodNotAllowed || response.Header().Get("Allow") != http.MethodPost {
+			t.Fatalf("GET %s = (%d, Allow %q)", path, response.Code, response.Header().Get("Allow"))
+		}
+	}
+	response := performRequest(handler, http.MethodPost, "/api/v2/messages/", "application/json", `{"message":"m"}`)
+	if response.Code != http.StatusNotFound {
+		t.Fatalf("trailing slash status = %d", response.Code)
+	}
+}
+
+func TestV2RequestBodyLimit(t *testing.T) {
+	prefix, suffix := `{"message":"`, `"}`
+	for _, test := range []struct {
+		name      string
+		bodyBytes int
+		want      int
+		wantCalls int
+	}{
+		{name: "exactly at limit", bodyBytes: maxRequestBody, want: http.StatusOK, wantCalls: 1},
+		{name: "one byte over limit", bodyBytes: maxRequestBody + 1, want: http.StatusBadRequest},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			body := prefix + strings.Repeat("x", test.bodyBytes-len(prefix)-len(suffix)) + suffix
+			response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, "/api/v2/messages", "application/json", body)
+			if response.Code != test.want {
+				t.Fatalf("response status = %d, want %d", response.Code, test.want)
+			}
+			if calls := len(recorder.bodies()); calls != test.wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", calls, test.wantCalls)
+			}
+		})
+	}
+}
+
+func TestV2ExtrasRespectDiscordLimits(t *testing.T) {
+	extra := make([]string, 30)
+	for index := range extra {
+		extra[index] = fmt.Sprintf(`{"name":"field-%02d","value":"value"}`, index)
+	}
+	body := fmt.Sprintf(`{"host":"nexus","asset":"k3s","durationSeconds":1,"fromVersion":"1","toVersion":"2","source":"updater","extra":[%s]}`, strings.Join(extra, ","))
+	payload := sendV2Request(t, "/api/v2/notifications/update", body)
+	fields := payload.Embeds[0].Fields
+	if len(fields) != 25 {
+		t.Fatalf("fields = %d, want 25", len(fields))
+	}
+	if fields[0].Name != "Version" || fields[1].Name != "Duration" || fields[2].Name != "Source" || fields[24].Name != "field-21" {
+		t.Fatalf("field order = %+v", fields)
+	}
+
+	longValue := strings.Repeat("x", v2InlineFieldLimit+1)
+	body = fmt.Sprintf(`{"message":"m","source":"source","extra":[{"name":"long","value":%q}]}`, longValue)
+	payload = sendV2Request(t, "/api/v2/messages", body)
+	if payload.Embeds[0].Fields[0].Name != "Source" || payload.Embeds[0].Fields[1].Inline {
+		t.Fatalf("inline fields = %+v", payload.Embeds[0].Fields)
+	}
+
+	body = fmt.Sprintf(`{"message":%q,"source":"source","extra":[{"name":"large","value":%q}]}`, strings.Repeat("m", 5000), strings.Repeat("x", 2000))
+	payload = sendV2Request(t, "/api/v2/messages", body)
+	embed := payload.Embeds[0]
+	if utf8.RuneCountInString(embed.Description) > 4096 || discordEmbedCharacters(payload.Embeds) > 6000 || len(embed.Fields) == 0 || embed.Fields[0].Name != "Source" {
+		t.Fatalf("normalized embed = %+v, characters = %d", embed, discordEmbedCharacters(payload.Embeds))
+	}
+}
+
+func TestV2FormattingHelpers(t *testing.T) {
+	for _, test := range []struct {
+		bytes int64
+		want  string
+	}{
+		{0, "0 B"},
+		{1023, "1023 B"},
+		{1024, "1.00 KiB"},
+		{1536, "1.50 KiB"},
+		{1 << 30, "1.00 GiB"},
+	} {
+		if got := formatBytes(test.bytes); got != test.want {
+			t.Errorf("formatBytes(%d) = %q, want %q", test.bytes, got, test.want)
+		}
+	}
+	for _, test := range []struct {
+		seconds float64
+		want    string
+	}{
+		{0, "0s"},
+		{42.5, "42.5s"},
+		{60, "1m"},
+		{125, "2m 5s"},
+		{125.1, "2m 5.1s"},
+		{3723, "1h 2m 3s"},
+	} {
+		if got := formatV2Duration(test.seconds); got != test.want {
+			t.Errorf("formatV2Duration(%v) = %q, want %q", test.seconds, got, test.want)
+		}
+	}
+	for _, value := range []*float64{nil, floatPointer(-1), floatPointer(math.NaN()), floatPointer(math.Inf(1))} {
+		if validNumber(value) {
+			t.Fatalf("validNumber(%v) = true", value)
+		}
+	}
+}
+
+func sendV2Request(t *testing.T, path, body string) discordPayload {
+	t.Helper()
+	recorder := &webhookRecorder{}
+	response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, path, "application/json", body)
+	if response.Code != http.StatusOK || response.Body.String() != "Notification sent" {
+		t.Fatalf("response = (%d, %q)", response.Code, response.Body.String())
+	}
+	bodies := recorder.bodies()
+	if len(bodies) != 1 {
+		t.Fatalf("webhook calls = %d, want 1", len(bodies))
+	}
+	var payload discordPayload
+	if err := json.Unmarshal(bodies[0], &payload); err != nil {
+		t.Fatalf("decode webhook payload: %v", err)
+	}
+	return payload
+}

housekeeping.sh

@@ -1,21 +1,9 @@
 #!/bin/bash
 
-# System Cleanup and Maintenance Script
-# 
-# Description: Comprehensive system cleanup for Docker containers and Linux systems
-# Features:
-#   - Self-updating capability
-#   - Docker resource cleanup (images, containers, volumes, networks)
-#   - Package manager cache cleanup (APK/APT)
-#   - System cache and temporary file cleanup
-#   - Log rotation and cleanup
-#   - Memory cache optimization
-#   - Journal cleanup (systemd)
-#   - Thumbnail and user cache cleanup
-# Author: ivanch
-# Version: 2.0
+# Housekeeping Script
+# Unified system maintenance: package index update, system cleanup, and oh-my-zsh update.
 
-set -euo pipefail  # Exit on error, undefined vars, and pipe failures
+set -euo pipefail
 
 #==============================================================================
 # CONFIGURATION
@@ -30,18 +18,14 @@ readonly LIGHT_BLUE='\033[1;34m'
 readonly LIGHT_GREY='\033[0;37m'
 readonly YELLOW='\033[1;33m'
 
-# Script configuration
-readonly SCRIPT_NAME="clean.sh"
-readonly SERVER_BASE_URL="https://git.ivanch.me/ivanch/server-scripts/raw/branch/main"
-
 # Cleanup configuration
 readonly LOG_RETENTION_DAYS=30
 readonly JOURNAL_RETENTION_DAYS=7
+readonly CACHE_RETENTION_DAYS=7
 readonly TEMP_DIRS=("/tmp" "/var/tmp")
-readonly CACHE_DIRS=("/var/cache" "/root/.cache")
+readonly CACHE_DIRS=("/var/cache/apt" "/root/.cache")
 
-# Auto-update configuration
-readonly AUTO_UPDATE_ENABLED=true
+readonly SCRIPT_NAME="$(basename "$0")"
 
 #==============================================================================
 # UTILITY FUNCTIONS
@@ -76,6 +60,7 @@ get_dir_size() {
 }
 
 # Safe directory cleanup with size reporting
+# Deletes files older than CACHE_RETENTION_DAYS instead of wiping everything
 clean_directory() {
     local dir="$1"
     local description="$2"
@@ -94,16 +79,26 @@ clean_directory() {
 
     log_step "$description (was $size_before)..."
 
-    # Use find with -delete for safer cleanup
-    if find "$dir" -mindepth 1 -delete 2>/dev/null; then
-        log_success "$description: freed $size_before"
-    else
-        # Fallback to rm if find fails
-        if rm -rf "$dir"/* 2>/dev/null; then
-            log_success "$description: freed $size_before"
+    # Use find to delete files older than retention period
+    # -type f -mtime +N: only files older than N days (not directories)
+    local deleted=false
+    if find "$dir" -type f -mtime +"${CACHE_RETENTION_DAYS:-7}" -delete 2>/dev/null; then
+        deleted=true
+    fi
+
+    # Clean up empty directories left behind (but don't remove the dir itself)
+    find "$dir" -mindepth 1 -type d -empty -delete 2>/dev/null || true
+
+    if $deleted; then
+        local size_after
+        size_after=$(get_dir_size "$dir")
+        if [[ "$size_after" == "$size_before" ]]; then
+            log_info "$description: nothing old enough to clean (retention: ${CACHE_RETENTION_DAYS:-7}d)"
         else
-            log_warning "$description: partial cleanup completed"
+            log_success "$description: freed (was $size_before, now $size_after)"
         fi
+    else
+        log_warning "$description: partial cleanup completed"
     fi
 }
 
@@ -113,9 +108,9 @@ get_system_info() {
 
     # Memory info
     if [[ -f /proc/meminfo ]]; then
-        local mem_total mem_available
-        mem_total=$(grep MemTotal /proc/meminfo | awk '{print $2}')
-        mem_available=$(grep MemAvailable /proc/meminfo | awk '{print $2}')
+        local mem_total="" mem_available=""
+        mem_total=$(awk '/^MemTotal:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)
+        mem_available=$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)
         if [[ -n "$mem_total" && -n "$mem_available" ]]; then
             info+="Memory: $((mem_available/1024))MB available of $((mem_total/1024))MB total"
         fi
@@ -124,7 +119,7 @@ get_system_info() {
     # Disk space info
     if command_exists df; then
         local disk_info
-        disk_info=$(df -h / 2>/dev/null | tail -1 | awk '{print $4 " available of " $2 " total"}')
+        disk_info=$(df -h / 2>/dev/null | awk 'NR == 2 {print $4 " available of " $2 " total"}' || true)
         if [[ -n "$disk_info" ]]; then
             info+="${info:+, }Disk: $disk_info"
         fi
@@ -134,75 +129,155 @@ get_system_info() {
 }
 
 #==============================================================================
-# AUTO-UPDATE FUNCTIONALITY
+# OS DETECTION (for package index update)
 #==============================================================================
 
-# Check server connectivity
-check_server_connectivity() {
-    local url="$1"
-    curl -s --head "$url" | head -n 1 | grep -E "HTTP/[12] [23].." >/dev/null 2>&1
-}
+# Detect the running OS and its ID string.
+# Prints two values: the distro id (lowercase) and the family id (lowercase).
+detect_os() {
+    local id=""
+    local family=""
 
-# Get SHA256 hash of a file
-get_file_hash() {
-    local file="$1"
-    sha256sum "$file" 2>/dev/null | awk '{print $1}' || echo ""
-}
-
-# Get SHA256 hash from URL content
-get_url_hash() {
-    local url="$1"
-    curl -s "$url" 2>/dev/null | sha256sum | awk '{print $1}' || echo ""
-}
-
-# Perform self-update if newer version is available
-perform_self_update() {
-    if [[ "$AUTO_UPDATE_ENABLED" != "true" ]]; then
-        log_info "Auto-update is disabled"
-        return 0
-    fi
-    
-    local server_url="$SERVER_BASE_URL/$SCRIPT_NAME"
-    
-    log_step "Checking for script updates..."
-    
-    # Check if server file is accessible
-    if ! check_server_connectivity "$server_url"; then
-        log_warning "Cannot connect to update server, continuing with current version"
-        return 0
-    fi
-    
-    # Compare local and server file hashes
-    local local_hash server_hash
-    local_hash=$(get_file_hash "$SCRIPT_NAME")
-    server_hash=$(get_url_hash "$server_url")
-    
-    if [[ -z "$local_hash" || -z "$server_hash" ]]; then
-        log_warning "Cannot determine file hashes, skipping update"
-        return 0
-    fi
-    
-    if [[ "$local_hash" != "$server_hash" ]]; then
-        log_info "Update available, downloading new version..."
-        
-        # Create backup of current script
-        local backup_file="${SCRIPT_NAME}.backup.$(date +%s)"
-        cp "$SCRIPT_NAME" "$backup_file" || die "Failed to create backup"
-        
-        # Download updated script
-        if curl -s -o "$SCRIPT_NAME" "$server_url"; then
-            chmod +x "$SCRIPT_NAME" || die "Failed to set executable permissions"
-            log_success "Script updated successfully"
-            
-            log_step "Running updated script..."
-            exec ./"$SCRIPT_NAME" "$@"
+    if [[ -f /etc/os-release ]]; then
+        # shellcheck disable=SC1091
+        . /etc/os-release
+        id="${ID:-unknown}"
+        family="${ID_LIKE:-}"
+    elif [[ -f /etc/alpine-release ]]; then
+        id="alpine"
+        family="alpine"
+    else
+        # Fall back to package manager detection
+        if command_exists apk; then
+            id="alpine"; family="alpine"
+        elif command_exists pacman; then
+            id="arch"; family="arch"
+        elif command_exists apt-get; then
+            id="debian"; family="debian"
+        elif command_exists dnf; then
+            id="fedora"; family="rhel"
+        elif command_exists yum; then
+            id="centos"; family="rhel"
+        elif command_exists zypper; then
+            id="opensuse"; family="suse"
+        elif command_exists xbps-install; then
+            id="void"; family="void"
+        elif command_exists emerge; then
+            id="gentoo"; family="gentoo"
         else
-            # Restore backup on failure
-            mv "$backup_file" "$SCRIPT_NAME"
-            die "Failed to download updated script"
+            id="unknown"; family="unknown"
+        fi
+    fi
+
+    echo "${id} ${family}"
+}
+
+#==============================================================================
+# OS-SPECIFIC PACKAGE INDEX UPDATE FUNCTIONS
+#==============================================================================
+
+update_alpine() {
+    if ! command_exists apk; then
+        log_warning "apk not found on Alpine-type system"
+        return 0
+    fi
+
+    log_info "Running apk update..."
+    if apk update 2>&1; then
+        log_success "Alpine package index updated"
+    else
+        log_warning "apk update failed"
+    fi
+}
+
+update_arch() {
+    if ! command_exists pacman; then
+        log_warning "pacman not found on Arch-type system"
+        return 0
+    fi
+
+    log_info "Refreshing pacman database (pacman -Sy)..."
+    # -S = sync, -y = refresh local database only (no package installation)
+    if pacman -Sy --noconfirm 2>&1; then
+        log_success "Arch package database refreshed"
+    else
+        log_warning "pacman -Sy failed"
+    fi
+}
+
+update_debian() {
+    if ! command_exists apt-get; then
+        log_warning "apt-get not found on Debian-type system"
+        return 0
+    fi
+
+    log_info "Running apt-get update..."
+    if apt-get update 2>&1; then
+        log_success "Debian/Ubuntu package index updated"
+    else
+        log_warning "apt-get update failed"
+    fi
+}
+
+update_rhel() {
+    if command_exists dnf; then
+        log_info "Running dnf makecache..."
+        if dnf makecache 2>&1; then
+            log_success "RHEL/Fedora metadata cache refreshed (dnf)"
+        else
+            log_warning "dnf makecache failed"
+        fi
+    elif command_exists yum; then
+        log_info "Running yum makecache..."
+        if yum makecache fast 2>&1; then
+            log_success "CentOS/RHEL metadata cache refreshed (yum)"
+        else
+            log_warning "yum makecache failed"
         fi
     else
-        log_success "Script is already up to date"
+        log_warning "No DNF or YUM found on RHEL-type system"
+    fi
+}
+
+update_suse() {
+    if ! command_exists zypper; then
+        log_warning "zypper not found on openSUSE-type system"
+        return 0
+    fi
+
+    log_info "Running zypper refresh..."
+    if zypper --non-interactive refresh 2>&1; then
+        log_success "openSUSE repositories refreshed"
+    else
+        log_warning "zypper refresh failed"
+    fi
+}
+
+update_void() {
+    if ! command_exists xbps-install; then
+        log_warning "xbps-install not found on Void system"
+        return 0
+    fi
+
+    log_info "Running xbps-install -S (sync repository index)..."
+    if xbps-install -S 2>&1; then
+        log_success "Void package index synced"
+    else
+        log_warning "xbps-install -S failed"
+    fi
+}
+
+update_gentoo() {
+    if ! command_exists emerge; then
+        log_warning "emerge not found on Gentoo system"
+        return 0
+    fi
+
+    log_info "Running emaint sync -a..."
+    if emaint sync -a 2>&1; then
+        log_success "Gentoo Portage tree synced"
+    else
+        log_warning "emaint sync failed"
     fi
 }
 
@@ -225,12 +300,6 @@ cleanup_docker() {
         return 0
     fi
 
-    # Get initial Docker disk usage
-    local docker_usage_before=""
-    if docker system df >/dev/null 2>&1; then
-        docker_usage_before=$(docker system df 2>/dev/null || echo "")
-    fi
-    
     # Remove unused images
     log_info "Removing unused Docker images..."
     if docker image prune -af >/dev/null 2>&1; then
@@ -270,11 +339,6 @@ cleanup_docker() {
     else
         log_warning "Docker system cleanup failed"
     fi
-    
-    # Show space freed if possible
-    if [[ -n "$docker_usage_before" ]] && docker system df >/dev/null 2>&1; then
-        log_info "Docker cleanup completed"
-    fi
 }
 
 #==============================================================================
@@ -373,13 +437,19 @@ cleanup_cache_dirs() {
         fi
     done
 
-    # Clean additional cache locations
+    # Clean safe cache locations only
+    # NOTE: We do NOT recursively delete /var/cache or /root/.cache as a whole,
+    # as that can break package managers and remove important data.
     local additional_caches=(
-        "/var/lib/apt/lists"
+        "/var/cache/apt/archives"
+        "/var/cache/apt/archives/partial"
         "/var/cache/debconf"
+        "/var/lib/apt/lists"
         "/root/.npm"
-        "/root/.pip"
-        "/home/*/.cache"
+        "/root/.cache/pip"
+        "/root/.cache/npm"
+        "/home/*/.cache/pip"
+        "/home/*/.cache/npm"
         "/home/*/.npm"
         "/home/*/.pip"
     )
@@ -404,12 +474,16 @@ cleanup_logs() {
 
         # Find and remove old log files
         while IFS= read -r -d '' logfile; do
-            rm -f "$logfile" 2>/dev/null && ((cleaned_count++))
+            if rm -f "$logfile" 2>/dev/null; then
+                ((++cleaned_count)) || true
+            fi
         done < <(find /var/log -type f -name "*.log" -mtime +"$LOG_RETENTION_DAYS" -print0 2>/dev/null || true)
 
         # Clean compressed logs
         while IFS= read -r -d '' logfile; do
-            rm -f "$logfile" 2>/dev/null && ((cleaned_count++))
+            if rm -f "$logfile" 2>/dev/null; then
+                ((++cleaned_count)) || true
+            fi
         done < <(find /var/log -type f \( -name "*.log.gz" -o -name "*.log.bz2" -o -name "*.log.xz" \) -mtime +"$LOG_RETENTION_DAYS" -print0 2>/dev/null || true)
 
         if [[ $cleaned_count -gt 0 ]]; then
@@ -420,7 +494,6 @@ cleanup_logs() {
     fi
 
     # Truncate large active log files
-    local large_logs
     while IFS= read -r -d '' logfile; do
         if [[ -f "$logfile" && -w "$logfile" ]]; then
             truncate -s 0 "$logfile" 2>/dev/null || true
@@ -511,10 +584,87 @@ generate_summary() {
 }
 
 #==============================================================================
-# MAIN EXECUTION
+# SUBCOMMAND HANDLERS
 #==============================================================================
 
-main() {
+# update: refresh the OS package index (no upgrades)
+run_update() {
+    log_step "Starting package list update (${SCRIPT_NAME})"
+    echo
+
+    # Detect the OS
+    local distro_info distro_id family_id
+    distro_info="$(detect_os)"
+    distro_id="${distro_info%% *}"
+    family_id="${distro_info##* }"
+
+    log_info "Detected OS: ${distro_id}"
+    if [[ -n "${family_id}" && "${family_id}" != "${distro_id}" ]]; then
+        log_info "OS family: ${family_id}"
+    fi
+    echo
+
+    # Dispatch to the correct handler based on distro family
+    # Check distro_id first, then fall back to family_id for aliases
+    case "${distro_id}" in
+        alpine)
+            update_alpine
+            ;;
+        arch|artix|manjaro|garuda)
+            update_arch
+            ;;
+        debian|ubuntu|linuxmint|kali|raspbian|pop)
+            update_debian
+            ;;
+        fedora|rhel|centos|rocky|almalinux|amazon|oracle|clear-linux-os)
+            update_rhel
+            ;;
+        opensuse*|suse|sles)
+            update_suse
+            ;;
+        void)
+            update_void
+            ;;
+        gentoo|funtoo)
+            update_gentoo
+            ;;
+        *)
+            # Try matching on family ID as fallback
+            case "${family_id}" in
+                *alpine*)
+                    update_alpine
+                    ;;
+                *arch*)
+                    update_arch
+                    ;;
+                *debian*|*ubuntu*)
+                    update_debian
+                    ;;
+                *rhel*|*fedora*|*centos*)
+                    update_rhel
+                    ;;
+                *suse*|*opensuse*)
+                    update_suse
+                    ;;
+                *void*)
+                    update_void
+                    ;;
+                *gentoo*)
+                    update_gentoo
+                    ;;
+                *)
+                    die "Unsupported OS: ${distro_id} (${family_id}). Cannot update package index."
+                    ;;
+            esac
+            ;;
+    esac
+
+    echo
+    log_success "Package list update completed!"
+}
+
+# clean: full system cleanup and maintenance
+run_clean() {
     log_step "Starting System Cleanup and Maintenance"
     echo
 
@@ -526,9 +676,6 @@ main() {
         echo
     fi
 
-    # Perform self-update if enabled
-    perform_self_update "$@"
-    
     # Docker cleanup
     cleanup_docker
 
@@ -554,5 +701,80 @@ main() {
     log_success "System cleanup and maintenance completed!"
 }
 
-# Execute main function with all arguments
+# omz: update oh-my-zsh (safe fallback if not installed)
+run_omz() {
+    log_step "Starting oh-my-zsh update (${SCRIPT_NAME})"
+    echo
+
+    if ! command_exists omz; then
+        log_warning "oh-my-zsh (omz) is not installed on this system, skipping omz update"
+        return 0
+    fi
+
+    log_info "Running omz update..."
+    if omz update 2>&1; then
+        log_success "oh-my-zsh updated"
+    else
+        log_warning "omz update failed"
+    fi
+}
+
+# all: run update, clean, then omz in order
+run_all() {
+    run_update
+    echo
+    run_clean
+    echo
+    run_omz
+}
+
+# Print usage information
+print_usage() {
+    cat <<EOF
+Usage: ${SCRIPT_NAME} [subcommand]
+
+Unified system housekeeping script.
+
+Subcommands:
+  update    Update the OS package index (detects distro, updates correct package manager)
+  clean     Run system cleanup (docker, package caches, temp/cache/log dirs, journal, thumbnails, memory)
+  omz       Update oh-my-zsh (skipped safely if not installed)
+  all       Run update, then clean, then omz (default when no subcommand is given)
+
+If no subcommand is provided, 'all' is assumed.
+EOF
+}
+
+#==============================================================================
+# MAIN EXECUTION
+#==============================================================================
+
+main() {
+    local subcommand="${1:-all}"
+
+    case "$subcommand" in
+        update)
+            run_update
+            ;;
+        clean)
+            run_clean
+            ;;
+        omz)
+            run_omz
+            ;;
+        all)
+            run_all
+            ;;
+        -h|--help|help)
+            print_usage
+            ;;
+        *)
+            log_warning "Unknown subcommand: '${subcommand}'"
+            echo
+            print_usage
+            exit 1
+            ;;
+    esac
+}
+
 main "$@"

k8s/README.md

@@ -0,0 +1,120 @@
+# K8s Script Pack
+
+Kubernetes-focused shell scripts intended for cronjobs and operational utilities.
+
+## Scripts
+
+### `automated-nfs-backup.sh`
+
+Copies each top-level folder found in `NFS_SOURCE_PATH` to a per-run local staging directory, then creates one encrypted `.7z` archive from the complete staged data. Kubernetes workloads are quiesced while a folder whose name exactly matches a namespace is copied.
+
+Behavior:
+- Exact folder-to-namespace mapping only.
+- `kube-system`, `kube-public`, and `kube-node-lease` are excluded from scale actions by default.
+- Unmapped or excluded folder: copied to local staging without Kubernetes scale actions.
+- Mapped folder: saves replicas, scales selected workloads down, waits, copies the quiesced folder to local staging, restores replicas, and waits again.
+- After every folder has been processed, one `7z` invocation compresses the complete run staging directory. Workloads are therefore restored before compression starts.
+- Per-run replica state and staged data are removed on exit. The staging parent is retained.
+- Scale-down issues are warnings by policy (backup still runs).
+- Restore issues are warnings by policy (run can still complete successfully).
+- Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.
+
+### `backup-k3s-control-plane-config.sh`
+
+Creates and validates a standard `.zip` containing the iris k3s control-plane configuration and credential state, then atomically publishes it to the NAS. It never stops or restarts k3s, and deliberately excludes `/var/lib/rancher/k3s/server/db` and `kine.sock`; this is configuration/credential backup, not a live datastore snapshot.
+
+The archive includes all of `K3S_CONFIG_DIR`, any existing `K3S_SERVICE_FILES`, and existing `cred`, `etc`, `manifests`, `static`, `tls`, `token`, `agent-token`, and `node-token` entries beneath `K3S_SERVER_DIR`.
+
+Defaults mirror the existing `k8s-config-backup` schedule: `ivanch@nas.haven:/export/Backup/k8s`, archive names such as `k3s-control-plane-config_2026-07-21_12-00-00.zip`, and remote retention of the four newest matching archives. Other prefixes, including `k8s-backup_*.7z`, are never considered for retention.
+
+The script uses a non-blocking lock by default (`/var/lock/k3s-control-plane-config-backup.lock`), so overlapping cron and manual invocations fail safely. It builds in a private local temporary directory, validates with `unzip -t`, uploads to a remote temporary filename, and only then renames it into place.
+
+Useful overrides:
+
+- Source paths: `K3S_CONFIG_DIR`, `K3S_SERVER_DIR`, `K3S_SERVICE_FILES` (colon-separated service configuration files).
+- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`.
+- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion).
+- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REALPATH_BIN`, `REMOTE_BASH_BIN`. `REALPATH_BIN` must provide GNU `realpath` `-e` and `-m` options.
+- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/api/v2/notifications/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/api/v2/notifications/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_SOURCE`. `NOTIFY_CALLER` is a compatibility fallback for `NOTIFY_SOURCE`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
+
+## Environment Variables
+
+Required:
+- `NFS_SOURCE_PATH`: Root path containing folders to back up.
+- `BACKUP_OUTPUT_PATH`: Destination path for generated `.7z` archives.
+
+Optional:
+- `BACKUP_PASSWORD` (default: empty): When set, archive uses password protection; when empty, archive is not encrypted.
+- `KUBECTL_BIN` (default: `kubectl`)
+- `KUBE_CONTEXT` (default: empty)
+- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset`)
+- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions.
+- `BACKUP_STAGING_PATH` (default: `/tmp/k8s-nfs-backup-staging`): Local parent for per-run staging directories. It must be outside `NFS_SOURCE_PATH` and have enough free space for all folders included in one backup run.
+- `RSYNC_BIN` (default: `rsync`)
+- `ARCHIVE_PREFIX` (default: `nfs-backup`)
+- `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`)
+- `SEVENZ_METHOD` (default: `lzma2`)
+- `SEVENZ_LEVEL` (default: `9`)
+- `SEVENZ_HEADER_ENCRYPT` (default: `on`, only applied when `BACKUP_PASSWORD` is set)
+- `SEVENZ_THREADS` (default: `on`)
+- `SCALE_RETRY_COUNT` (default: `3`)
+- `SCALE_RETRY_DELAY_SECONDS` (default: `5`)
+- `SCALE_WAIT_SECONDS` (default: `30`): Fixed delay after scaling down and after restoring a mapped namespace.
+- `LOG_LEVEL` (default: `info`)
+- `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`): Parent directory for fresh per-run replica-state directories. A run never restores state files belonging to an earlier run.
+- `NOTIFY_SUCCESS_URL` (default: `http://notify.haven/api/v2/notifications/backup`; set to empty to disable)
+- `NOTIFY_FAILURE_URL` (default: `http://notify.haven/api/v2/notifications/error`; set to empty to disable)
+- `NOTIFY_TITLE` (default: `Kubernetes`)
+- `NOTIFY_ASSET` (default: `kube config`)
+- `NOTIFY_SOURCE` (default: `NOTIFY_CALLER`, or `Kubernetes NFS backup`): Haven Notify v2 source identifier. `NOTIFY_CALLER` remains a compatibility fallback.
+- `CLEANUP_KEEP_COUNT` (default: `4`)
+
+Notifications use the Haven Notify v2 backup and error endpoints. Success payloads include the final archive size and elapsed time:
+
+```json
+{
+  "title": "Kubernetes",
+  "asset": "kube config",
+  "sizeBytes": 128974848,
+  "durationSeconds": 125,
+  "source": "Kubernetes NFS backup"
+}
+```
+
+Failures use the v2 error contract:
+
+```json
+{
+  "source": "Kubernetes NFS backup",
+  "message": "Some NFS folders failed to back up",
+  "severity": "critical",
+  "errorCode": "BACKUP_FAILED"
+}
+```
+
+## Cronjob Notes
+
+- Script is designed to run sequentially (one folder at a time).
+- Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces.
+- Ensure `kubectl` context and credentials are present in the runtime.
+- Ensure `7z` is installed in the runtime image/host.
+- Ensure `rsync` is installed and keep `BACKUP_STAGING_PATH` on fast local storage. Capacity must accommodate one complete uncompressed backup run in addition to the output archive.
+
+## Failure Semantics
+
+- Missing required env vars, missing commands, invalid paths, or inability to list namespaces: script exits non-zero immediately.
+- Folder staging failures are counted and failed partial copies are excluded from the archive. One or more successfully staged folders continues to archive creation; zero successfully staged folders exits non-zero.
+- If the single archive operation fails, the run treats every staged folder as failed and exits non-zero.
+- Scale-down warnings/timeouts: logged and counted, backup continues.
+- Restore warnings/timeouts: logged and counted, script does not fail solely because of restore warnings.
+- If `NOTIFY_SUCCESS_URL` is set, success notification is sent at the end of a successful run.
+- If `NOTIFY_FAILURE_URL` is set, a failure notification is sent when any folder backup failure is detected, including partial failures.
+- On runs with at least one successful folder backup, cleanup retains the `CLEANUP_KEEP_COUNT` most recent matching archives and deletes older ones.
+- Final summary always logs:
+  - `processed`
+  - `mapped`
+  - `unmapped`
+  - `backup_successes`
+  - `backup_failures`
+  - `scale_warnings`
+  - `restore_warnings`

k8s/automated-nfs-backup.sh

@@ -0,0 +1,616 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME="$(basename "$0")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/common.sh
+source "${SCRIPT_DIR}/lib/common.sh"
+
+# Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
+# every scaled-down namespace is restored regardless of how we exit.
+trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
+trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
+
+# Required configuration
+require_env "NFS_SOURCE_PATH"
+require_env "BACKUP_OUTPUT_PATH"
+
+# Optional configuration
+BACKUP_PASSWORD="${BACKUP_PASSWORD:-}"
+KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
+KUBE_CONTEXT="${KUBE_CONTEXT:-}"
+WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
+EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease}"
+BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-/tmp/k8s-nfs-backup-staging}"
+RSYNC_BIN="${RSYNC_BIN:-rsync}"
+ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
+ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
+SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
+SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}"
+SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}"
+SEVENZ_THREADS="${SEVENZ_THREADS:-on}"
+SEVENZ_BIN="${SEVENZ_BIN:-7z}"
+SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
+SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
+SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
+TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}"
+NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
+NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
+NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
+NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
+NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes NFS backup}"
+NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
+CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
+RUN_STATE_DIR=""
+RUN_STAGING_DIR=""
+
+KUBECTL_ARGS=()
+if [[ -n "$KUBE_CONTEXT" ]]; then
+    KUBECTL_ARGS+=(--context "$KUBE_CONTEXT")
+fi
+
+declare -a WORKLOAD_KIND_LIST=()
+declare -A NAMESPACE_MAP=()
+declare -A EXCLUDED_NAMESPACE_MAP=()
+
+total_folders=0
+mapped_folders=0
+unmapped_folders=0
+backup_successes=0
+backup_failures=0
+scale_warnings=0
+restore_warnings=0
+total_backup_size_bytes=0
+backup_started_seconds="$SECONDS"
+
+_kubectl() {
+    "$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" </dev/null
+}
+
+parse_workload_kinds() {
+    local raw
+    local cleaned
+    IFS=',' read -r -a raw <<< "$WORKLOAD_KINDS"
+    for kind in "${raw[@]}"; do
+        cleaned="$(trim "${kind,,}")"
+        if [[ -n "$cleaned" ]]; then
+            WORKLOAD_KIND_LIST+=("$cleaned")
+        fi
+    done
+
+    if [[ "${#WORKLOAD_KIND_LIST[@]}" -eq 0 ]]; then
+        die "WORKLOAD_KINDS resolved to an empty list"
+    fi
+}
+
+parse_excluded_namespaces() {
+    local raw
+    local cleaned
+    local namespace
+    IFS=',' read -r -a raw <<< "$EXCLUDED_NAMESPACES"
+    for namespace in "${raw[@]}"; do
+        cleaned="$(trim "${namespace,,}")"
+        if [[ -n "$cleaned" ]]; then
+            EXCLUDED_NAMESPACE_MAP["$cleaned"]=1
+        fi
+    done
+}
+
+validate_inputs() {
+    require_cmd "$KUBECTL_BIN"
+    require_cmd "$SEVENZ_BIN"
+    require_cmd "mktemp"
+    require_cmd "$RSYNC_BIN"
+    if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
+        require_cmd "curl"
+    fi
+
+    if [[ ! -d "$NFS_SOURCE_PATH" ]]; then
+        die "NFS_SOURCE_PATH does not exist or is not a directory: ${NFS_SOURCE_PATH}"
+    fi
+
+    if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then
+        die "CLEANUP_KEEP_COUNT must be an integer >= 0"
+    fi
+
+    if [[ -z "$BACKUP_PASSWORD" ]]; then
+        log_warn "BACKUP_PASSWORD is empty. Archive will be created without password protection."
+    fi
+
+    mkdir -p "$BACKUP_OUTPUT_PATH"
+    mkdir -p "$TMP_STATE_DIR"
+    if ! RUN_STATE_DIR="$(mktemp -d "${TMP_STATE_DIR%/}/run-XXXXXXXXXX")"; then
+        die "Unable to create a per-run state directory under ${TMP_STATE_DIR}"
+    fi
+    log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
+
+    local source_real
+    local staging_real
+    mkdir -p "$BACKUP_STAGING_PATH"
+    source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
+    staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
+    case "${staging_real}/" in
+        "${source_real%/}/"*)
+            die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
+            ;;
+    esac
+    if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
+        die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
+    fi
+    log_info "Using local run staging directory: ${RUN_STAGING_DIR}"
+
+    if ! _kubectl get namespaces >/dev/null 2>&1; then
+        die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
+    fi
+}
+
+load_namespaces() {
+    local ns
+    while IFS= read -r ns; do
+        [[ -z "$ns" ]] && continue
+        NAMESPACE_MAP["$ns"]=1
+    done < <(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
+}
+
+namespace_for_folder() {
+    local folder_name="${1:?folder name required}"
+    if [[ -n "${EXCLUDED_NAMESPACE_MAP[$folder_name]:-}" ]]; then
+        return 2
+    fi
+    if [[ -n "${NAMESPACE_MAP[$folder_name]:-}" ]]; then
+        printf '%s' "$folder_name"
+        return 0
+    fi
+    return 1
+}
+
+state_file_for_namespace() {
+    local namespace="${1:?namespace required}"
+    printf '%s/%s.state' "$RUN_STATE_DIR" "$namespace"
+}
+
+capture_replicas_state() {
+    local namespace="${1:?namespace is required}"
+    local state_file="${2:?state file path is required}"
+    : > "$state_file"
+
+    local kind
+    local name
+    local replicas
+    local owners
+    local output
+    for kind in "${WORKLOAD_KIND_LIST[@]}"; do
+        if ! output="$(_kubectl -n "$namespace" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.replicas}{"\t"}{range .metadata.ownerReferences[*]}{.kind},{end}{"\n"}{end}' 2>/dev/null)"; then
+            log_warn "Failed to list kind '${kind}' in namespace '${namespace}' while capturing state"
+            scale_warnings=$((scale_warnings + 1))
+            continue
+        fi
+
+        while IFS=$'\t' read -r name replicas owners; do
+            [[ -z "$name" ]] && continue
+            # Skip resources managed by a higher-level controller (e.g. a
+            # ReplicaSet owned by a Deployment) — scaling them directly
+            # conflicts with the owning controller and can leave workloads
+            # in an inconsistent state after restore.
+            if [[ -n "$owners" ]]; then
+                log_debug "Skipping ${kind}/${name} in namespace '${namespace}': owned by ${owners%,}"
+                continue
+            fi
+            if [[ -z "$replicas" ]]; then
+                replicas="1"
+                log_warn "Resource ${kind}/${name} had no explicit replicas; defaulting saved value to 1"
+                scale_warnings=$((scale_warnings + 1))
+            fi
+            printf '%s\t%s\t%s\n' "$kind" "$name" "$replicas" >> "$state_file"
+        done <<< "$output"
+    done
+}
+
+scale_resource() {
+    local namespace="${1:?namespace is required}"
+    local kind="${2:?kind is required}"
+    local name="${3:?name is required}"
+    local replicas="${4:?replicas is required}"
+    retry "$SCALE_RETRY_COUNT" "$SCALE_RETRY_DELAY_SECONDS" \
+        _kubectl -n "$namespace" scale "${kind}/${name}" --replicas="$replicas"
+}
+
+scale_namespace_to_zero() {
+    local namespace="${1:?namespace is required}"
+    local state_file="${2:?state file is required}"
+    local kind
+    local name
+    local replicas
+
+    while IFS=$'\t' read -r kind name replicas; do
+        [[ -z "$kind" || -z "$name" ]] && continue
+        if scale_resource "$namespace" "$kind" "$name" "0"; then
+            log_info "Scaled down ${namespace}:${kind}/${name} to 0"
+        else
+            log_warn "Failed to scale down ${namespace}:${kind}/${name}; continuing with backup by policy"
+            scale_warnings=$((scale_warnings + 1))
+        fi
+    done < "$state_file"
+}
+
+restore_namespace_replicas() {
+    local namespace="${1:?namespace is required}"
+    local state_file="${2:?state file is required}"
+    local kind
+    local name
+    local replicas
+
+    while IFS=$'\t' read -r kind name replicas; do
+        [[ -z "$kind" || -z "$name" ]] && continue
+        if scale_resource "$namespace" "$kind" "$name" "$replicas"; then
+            log_info "Restored ${namespace}:${kind}/${name} to ${replicas}"
+        else
+            log_warn "Failed to restore ${namespace}:${kind}/${name} to ${replicas}"
+            restore_warnings=$((restore_warnings + 1))
+        fi
+    done < "$state_file"
+}
+
+restore_all_remaining() {
+    if [[ -z "${RUN_STATE_DIR:-}" ]] || [[ ! -d "${RUN_STATE_DIR:-}" ]]; then
+        return 0
+    fi
+
+    local state_file
+    local namespace
+    local kind name replicas
+    local found=0
+
+    for state_file in "${RUN_STATE_DIR}"/*.state; do
+        [[ -f "$state_file" ]] || continue
+        found=1
+        namespace="$(basename "$state_file" .state)"
+        [[ -z "$namespace" ]] && continue
+        log_info "EXIT cleanup: restoring workloads in namespace '${namespace}'"
+        while IFS=$'\t' read -r kind name replicas; do
+            [[ -z "$kind" || -z "$name" ]] && continue
+            if scale_resource "$namespace" "$kind" "$name" "$replicas"; then
+                log_info "EXIT cleanup: restored ${namespace}:${kind}/${name} to ${replicas}"
+            else
+                log_warn "EXIT cleanup: failed to restore ${namespace}:${kind}/${name} to ${replicas}"
+                restore_warnings=$((restore_warnings + 1))
+            fi
+        done < "$state_file"
+        rm -f "$state_file"
+    done
+
+    if [[ "$found" -eq 1 ]]; then
+        log_info "EXIT cleanup complete - all remaining workloads restored"
+    fi
+
+    if ! rmdir -- "$RUN_STATE_DIR" 2>/dev/null; then
+        log_warn "Run state directory is not empty; leaving it in place: ${RUN_STATE_DIR}"
+    fi
+}
+
+cleanup_run_staging() {
+    if [[ -z "${RUN_STAGING_DIR:-}" ]] || [[ ! -d "${RUN_STAGING_DIR:-}" ]]; then
+        return 0
+    fi
+
+    case "$RUN_STAGING_DIR" in
+        "${BACKUP_STAGING_PATH%/}/run-"*)
+            rm -rf -- "$RUN_STAGING_DIR"
+            ;;
+        *)
+            log_error "Refusing to remove unexpected staging directory: ${RUN_STAGING_DIR}"
+            return 1
+            ;;
+    esac
+}
+
+cleanup_staged_folder() {
+    local staging_folder="${1:?staging folder required}"
+    case "$staging_folder" in
+        "$RUN_STAGING_DIR"/*)
+            rm -rf -- "$staging_folder"
+            ;;
+        *)
+            log_error "Refusing to remove staging folder outside the current run: ${staging_folder}"
+            return 1
+            ;;
+    esac
+}
+
+sync_folder_to_staging() {
+    local source_folder="${1:?source folder required}"
+    local staging_folder="${2:?staging folder required}"
+    local sync_output
+
+    mkdir -p "$staging_folder"
+    if ! sync_output="$("$RSYNC_BIN" -a --delete -- "${source_folder%/}/" "${staging_folder%/}/" 2>&1)"; then
+        log_error "rsync failed from '${source_folder}' to '${staging_folder}'"
+        if [[ -n "$sync_output" ]]; then
+            log_error "rsync output: ${sync_output}"
+        fi
+        return 1
+    fi
+}
+
+archive_path_for_run() {
+    local ts
+    ts="$(date +"$ARCHIVE_TS_FORMAT")"
+    printf '%s/%s_%s.7z' "$BACKUP_OUTPUT_PATH" "$ARCHIVE_PREFIX" "$ts"
+}
+
+json_escape() {
+    local value="${1:-}"
+    value="${value//\\/\\\\}"
+    value="${value//\"/\\\"}"
+    value="${value//$'\n'/ }"
+    printf '%s' "$value"
+}
+
+send_backup_notification() {
+    local url="${1:-}"
+    local title="${2:-}"
+    local asset="${3:-}"
+    local size_bytes="${4:-0}"
+    local duration_seconds="${5:-0}"
+
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "title": "$(json_escape "$title")",
+  "asset": "$(json_escape "$asset")",
+  "sizeBytes": ${size_bytes},
+  "durationSeconds": ${duration_seconds},
+  "source": "$(json_escape "$NOTIFY_SOURCE")"
+}
+EOF
+)
+
+    if ! curl -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup notification to ${url}"
+        return 1
+    fi
+    return 0
+}
+
+send_backup_failure_notification() {
+    local url="${1:-}"
+    local message="${2:-}"
+    local size_bytes="${3:-0}"
+
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "source": "$(json_escape "$NOTIFY_SOURCE")",
+  "message": "$(json_escape "$message")",
+  "severity": "critical",
+  "errorCode": "BACKUP_FAILED",
+  "extra": [
+    {
+      "name": "Archive size",
+      "value": "${size_bytes} B"
+    }
+  ]
+}
+EOF
+)
+
+    if ! curl -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup failure notification to ${url}"
+        return 1
+    fi
+    return 0
+}
+
+archive_staging() {
+    local archive_path="${1:?archive path required}"
+
+    if [[ "$archive_path" != /* ]]; then
+        archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
+    fi
+
+    local -a cmd=(
+        "$SEVENZ_BIN" a -t7z
+        "$archive_path"
+        .
+        "-m0=${SEVENZ_METHOD}"
+        "-mx=${SEVENZ_LEVEL}"
+        "-mmt=${SEVENZ_THREADS}"
+    )
+
+    if [[ -n "$BACKUP_PASSWORD" ]]; then
+        cmd+=("-p${BACKUP_PASSWORD}" "-mhe=${SEVENZ_HEADER_ENCRYPT}")
+    fi
+
+    local cmd_output rc filtered_output
+    cmd_output="$(cd "$RUN_STAGING_DIR" && "${cmd[@]}" 2>&1 >/dev/null)"
+    rc=$?
+    filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
+    if [[ -n "$filtered_output" ]]; then
+        log_warn "7z output for staging directory '${RUN_STAGING_DIR}': ${filtered_output}"
+    fi
+    if (( rc == 0 )); then
+        return 0
+    elif (( rc == 1 )); then
+        return 0
+    fi
+    return 1
+}
+
+cleanup_archives() {
+    if (( CLEANUP_KEEP_COUNT == 0 )); then
+        log_info "Cleanup disabled (CLEANUP_KEEP_COUNT=0)"
+        return 0
+    fi
+
+    shopt -s nullglob
+    local archive_candidates=( "$BACKUP_OUTPUT_PATH"/"${ARCHIVE_PREFIX}"_*.7z )
+    shopt -u nullglob
+
+    local total_candidates="${#archive_candidates[@]}"
+    if (( total_candidates == 0 )); then
+        log_info "Cleanup skipped: no archives found for prefix '${ARCHIVE_PREFIX}' in ${BACKUP_OUTPUT_PATH}"
+        return 0
+    fi
+
+    local sorted_archives=()
+    mapfile -t sorted_archives < <(ls -1t -- "${archive_candidates[@]}")
+
+    local total_sorted="${#sorted_archives[@]}"
+    if (( total_sorted <= CLEANUP_KEEP_COUNT )); then
+        log_info "Cleanup skipped: ${total_sorted} archive(s) present, keeping ${CLEANUP_KEEP_COUNT}"
+        return 0
+    fi
+
+    local keep_count="$CLEANUP_KEEP_COUNT"
+    local i
+    for ((i = keep_count; i < total_sorted; i++)); do
+        local archive_path="${sorted_archives[$i]}"
+        rm -f -- "$archive_path"
+        log_info "Cleanup deleted archive: ${archive_path}"
+    done
+}
+
+process_folder() {
+    local folder_path="${1:?folder path is required}"
+    local folder_name
+    local namespace=""
+    local state_file=""
+    local has_mapping=0
+    local namespace_match_status=0
+    local staging_folder
+    local copy_succeeded=0
+
+    folder_name="$(basename "$folder_path")"
+    staging_folder="${RUN_STAGING_DIR}/${folder_name}"
+    total_folders=$((total_folders + 1))
+    log_info "Processing folder: ${folder_name}"
+
+    if namespace="$(namespace_for_folder "$folder_name")"; then
+        has_mapping=1
+        mapped_folders=$((mapped_folders + 1))
+        log_info "Exact namespace match found for folder '${folder_name}' -> namespace '${namespace}'"
+        state_file="$(state_file_for_namespace "$namespace")"
+
+        capture_replicas_state "$namespace" "$state_file"
+        scale_namespace_to_zero "$namespace" "$state_file"
+
+        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
+        sleep "$SCALE_WAIT_SECONDS"
+
+        log_info "Copying quiesced namespace folder '${folder_name}' to local staging"
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
+        fi
+    else
+        namespace_match_status=$?
+        unmapped_folders=$((unmapped_folders + 1))
+        if (( namespace_match_status == 2 )); then
+            log_warn "Folder '${folder_name}' matches an excluded namespace; skipping Kubernetes scale actions and running backup only."
+        else
+            log_warn "No namespace matched folder '${folder_name}'. Running backup only."
+        fi
+
+        log_info "Copying live folder '${folder_name}' to local staging"
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
+        fi
+    fi
+
+    if [[ "$has_mapping" -eq 1 ]]; then
+        restore_namespace_replicas "$namespace" "$state_file"
+        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
+        sleep "$SCALE_WAIT_SECONDS"
+        rm -f "$state_file"
+    fi
+
+    if (( copy_succeeded == 0 )); then
+        backup_failures=$((backup_failures + 1))
+        log_error "Backup failed for folder '${folder_name}' while copying to local staging"
+        cleanup_staged_folder "$staging_folder" || true
+        return 0
+    fi
+
+    backup_successes=$((backup_successes + 1))
+    log_info "Staged folder '${folder_name}' for the run archive"
+}
+
+print_summary() {
+    log_info "Run summary: processed=${total_folders} mapped=${mapped_folders} unmapped=${unmapped_folders} backup_successes=${backup_successes} backup_failures=${backup_failures} scale_warnings=${scale_warnings} restore_warnings=${restore_warnings}"
+}
+
+main() {
+    parse_workload_kinds
+    parse_excluded_namespaces
+    validate_inputs
+    load_namespaces
+
+    local folder_path
+    local run_archive_path
+    local found=0
+    run_archive_path="$(archive_path_for_run)"
+    log_info "Using single archive for this run: ${run_archive_path}"
+
+    for folder_path in "${NFS_SOURCE_PATH}"/*; do
+        [[ -d "$folder_path" ]] || continue
+        [[ "$(basename "$folder_path")" == .* ]] && continue
+        found=1
+        process_folder "$folder_path"
+    done
+
+    if [[ "$found" -eq 0 ]]; then
+        log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}"
+    elif (( backup_successes > 0 )); then
+        log_info "Compressing the complete local staging directory into: ${run_archive_path}"
+        if ! archive_staging "$run_archive_path"; then
+            backup_failures=$((backup_failures + backup_successes))
+            backup_successes=0
+            rm -f -- "$run_archive_path"
+            log_error "Backup failed while compressing the local staging directory"
+        fi
+    fi
+
+    if [[ -f "$run_archive_path" ]]; then
+        local file_size_bytes
+        file_size_bytes="$(wc -c < "$run_archive_path" | tr -d '[:space:]')"
+        if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then
+            total_backup_size_bytes="$file_size_bytes"
+        fi
+    fi
+
+    print_summary
+    local duration_seconds=$((SECONDS - backup_started_seconds))
+
+    if (( backup_successes == 0 )); then
+        send_backup_failure_notification \
+            "$NOTIFY_FAILURE_URL" \
+            "No NFS folders were backed up successfully" \
+            "$total_backup_size_bytes" || true
+        die "No folder backups succeeded" 1
+    fi
+
+    if (( backup_failures > 0 )); then
+        send_backup_failure_notification \
+            "$NOTIFY_FAILURE_URL" \
+            "Some NFS folders failed to back up" \
+            "$total_backup_size_bytes" || true
+        log_warn "Some folder backups failed; continuing because ${backup_successes} folder backup(s) succeeded"
+    fi
+
+    cleanup_archives || true
+
+    send_backup_notification \
+        "$NOTIFY_SUCCESS_URL" \
+        "$NOTIFY_TITLE" \
+        "${NOTIFY_ASSET}" \
+        "$total_backup_size_bytes" \
+        "$duration_seconds" || true
+}
+
+main "$@"

k8s/backup-k3s-control-plane-config.sh

@@ -0,0 +1,473 @@
+#!/usr/bin/env bash
+
+# Back up the k3s control-plane configuration and credentials, without taking
+# a live datastore snapshot or changing the running k3s service.
+
+SCRIPT_NAME="$(basename "$0")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/common.sh
+source "${SCRIPT_DIR}/lib/common.sh"
+
+# Keep the local archive and temporary files readable only by this process.
+umask 077
+
+# Source paths. K3S_CONFIG_DIR is the only required source; the service files
+# and server state entries are included only when present.
+K3S_CONFIG_DIR="${K3S_CONFIG_DIR:-/etc/rancher/k3s}"
+K3S_SERVER_DIR="${K3S_SERVER_DIR:-/var/lib/rancher/k3s/server}"
+K3S_SERVICE_FILES="${K3S_SERVICE_FILES-/etc/systemd/system/k3s.service:/etc/systemd/system/k3s.service.env:/etc/default/k3s:/etc/sysconfig/k3s}"
+
+# Archive and local working settings.
+ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-k3s-control-plane-config}"
+ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y-%m-%d_%H-%M-%S}"
+LOCAL_TMP_PARENT="${LOCAL_TMP_PARENT:-${TMPDIR:-/tmp}}"
+LOCK_PATH="${LOCK_PATH:-/var/lock/k3s-control-plane-config-backup.lock}"
+
+# Remote destination settings.
+REMOTE_USER="${REMOTE_USER:-ivanch}"
+REMOTE_HOST="${REMOTE_HOST:-nas.haven}"
+REMOTE_DIR="${REMOTE_DIR:-/export/Backup/k8s}"
+REMOTE_IDENTITY_FILE="${REMOTE_IDENTITY_FILE:-}"
+SSH_PORT="${SSH_PORT:-}"
+CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
+
+# Commands are configurable to support hosts with non-standard paths and
+# isolated tests.
+ZIP_BIN="${ZIP_BIN:-zip}"
+UNZIP_BIN="${UNZIP_BIN:-unzip}"
+SSH_BIN="${SSH_BIN:-ssh}"
+SCP_BIN="${SCP_BIN:-scp}"
+CURL_BIN="${CURL_BIN:-curl}"
+FLOCK_BIN="${FLOCK_BIN:-flock}"
+MKTEMP_BIN="${MKTEMP_BIN:-mktemp}"
+DATE_BIN="${DATE_BIN:-date}"
+REALPATH_BIN="${REALPATH_BIN:-realpath}"
+REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
+
+# Notifications use the Haven Notify v2 backup and error contracts.
+# Set either URL to an empty value to disable that notification.
+NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
+NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
+NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
+NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
+NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
+NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
+
+declare -a ARCHIVE_ENTRIES=()
+declare -a SERVICE_FILE_LIST=()
+declare -a SSH_ARGS=()
+declare -a SCP_ARGS=()
+declare -a PROTECTED_PATHS=()
+readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token)
+
+RUN_TMP_DIR=""
+LOCAL_ARCHIVE_PATH=""
+ARCHIVE_NAME=""
+ARCHIVE_SIZE_BYTES=0
+CLEANUP_KEEP_COUNT_INT=0
+LOCK_FD=""
+FAILURE_NOTIFICATION_SENT=0
+BACKUP_STARTED_SECONDS="$SECONDS"
+
+json_escape() {
+    local value="${1:-}"
+    value="${value//\\/\\\\}"
+    value="${value//\"/\\\"}"
+    value="${value//$'\n'/ }"
+    printf '%s' "$value"
+}
+
+send_backup_notification() {
+    local url="${1:-}"
+    local size_bytes="${2:-0}"
+    local duration_seconds="${3:-0}"
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "title": "$(json_escape "$NOTIFY_TITLE")",
+  "asset": "$(json_escape "$NOTIFY_ASSET")",
+  "sizeBytes": ${size_bytes},
+  "durationSeconds": ${duration_seconds},
+  "source": "$(json_escape "$NOTIFY_SOURCE")"
+}
+EOF
+)
+
+    if ! "$CURL_BIN" -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup notification"
+        return 1
+    fi
+}
+
+send_backup_failure_notification() {
+    local url="${1:-}"
+    local size_bytes="${2:-0}"
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "source": "$(json_escape "$NOTIFY_SOURCE")",
+  "message": "Control-plane configuration backup failed",
+  "severity": "critical",
+  "errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED",
+  "extra": [
+    {
+      "name": "Archive size",
+      "value": "${size_bytes} B"
+    }
+  ]
+}
+EOF
+)
+
+    if ! "$CURL_BIN" -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup failure notification"
+        return 1
+    fi
+}
+
+notify_failure_once() {
+    if (( FAILURE_NOTIFICATION_SENT == 1 )); then
+        return 0
+    fi
+    FAILURE_NOTIFICATION_SENT=1
+    send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$ARCHIVE_SIZE_BYTES" || true
+}
+
+cleanup() {
+    local exit_code=$?
+    trap - EXIT
+
+    if [[ -n "$RUN_TMP_DIR" && -d "$RUN_TMP_DIR" ]]; then
+        rm -rf -- "$RUN_TMP_DIR" || log_warn "Failed to clean local temporary files"
+    fi
+
+    if (( exit_code == 0 )); then
+        log_info "Finished successfully"
+    else
+        notify_failure_once
+        log_error "Control-plane configuration backup failed"
+    fi
+
+    exit "$exit_code"
+}
+trap cleanup EXIT
+
+shell_quote() {
+    local value="${1:-}"
+    value="${value//\'/\'\"\'\"\'}"
+    printf "'%s'" "$value"
+}
+
+require_absolute_path() {
+    local path="${1:?path is required}"
+    local description="${2:-path}"
+    if [[ "$path" != /* ]]; then
+        die "${description} must be an absolute path"
+    fi
+}
+
+path_is_ancestor_or_same() {
+    local ancestor="${1:?ancestor path is required}"
+    local path="${2:?path is required}"
+
+    [[ "$ancestor" == "/" || "$path" == "$ancestor" || "$path" == "$ancestor"/* ]]
+}
+
+paths_overlap() {
+    local first="${1:?first path is required}"
+    local second="${2:?second path is required}"
+
+    path_is_ancestor_or_same "$first" "$second" || \
+        path_is_ancestor_or_same "$second" "$first"
+}
+
+canonicalize_protected_path() {
+    local path="${1:?protected path is required}"
+    local canonical_path
+    local canonical_parent
+    local basename
+
+    if [[ -e "$path" || -L "$path" ]]; then
+        canonical_path="$("$REALPATH_BIN" -e -- "$path" 2>/dev/null)" || return 1
+        printf '%s' "$canonical_path"
+        return 0
+    fi
+
+    basename="${path##*/}"
+    canonical_parent="${path%/*}"
+    if ! canonical_parent="$("$REALPATH_BIN" -e -- "$canonical_parent" 2>/dev/null)"; then
+        canonical_parent="$("$REALPATH_BIN" -m -- "$canonical_parent" 2>/dev/null)" || return 1
+    fi
+    printf '%s/%s' "${canonical_parent%/}" "$basename"
+}
+
+determine_protected_paths() {
+    local canonical_server_dir
+    local protected_path
+
+    if ! canonical_server_dir="$("$REALPATH_BIN" -e -- "$K3S_SERVER_DIR" 2>/dev/null)"; then
+        canonical_server_dir="$("$REALPATH_BIN" -m -- "$K3S_SERVER_DIR" 2>/dev/null)" || \
+            die "Configured source path is not safe to archive"
+    fi
+
+    PROTECTED_PATHS=()
+    if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/db")"; then
+        die "Configured source path is not safe to archive"
+    fi
+    PROTECTED_PATHS+=("$protected_path")
+    if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/kine.sock")"; then
+        die "Configured source path is not safe to archive"
+    fi
+    PROTECTED_PATHS+=("$protected_path")
+}
+
+add_archive_entry_if_present() {
+    local absolute_path="${1:?source path is required}"
+    local canonical_path
+    local relative_path
+    local protected_path
+
+    if [[ "$absolute_path" =~ (^|/)(\.|\.\.)(/|$) ]]; then
+        die "Configured source path is not safe to archive"
+    fi
+
+    if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then
+        return 0
+    fi
+
+    if ! canonical_path="$("$REALPATH_BIN" -e -- "$absolute_path" 2>/dev/null)"; then
+        die "Configured source path is not safe to archive"
+    fi
+
+    for protected_path in "${PROTECTED_PATHS[@]}"; do
+        if paths_overlap "$canonical_path" "$protected_path"; then
+            die "Configured source path is not safe to archive"
+        fi
+    done
+
+    relative_path="${canonical_path#/}"
+    if [[ -z "$relative_path" ]]; then
+        die "Configured source path is not safe to archive"
+    fi
+    ARCHIVE_ENTRIES+=("$relative_path")
+}
+
+build_remote_arguments() {
+    SSH_ARGS=()
+    SCP_ARGS=()
+    if [[ -n "$REMOTE_IDENTITY_FILE" ]]; then
+        SSH_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
+        SCP_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
+    fi
+    if [[ -n "$SSH_PORT" ]]; then
+        SSH_ARGS+=(-p "$SSH_PORT")
+        SCP_ARGS+=(-P "$SSH_PORT")
+    fi
+}
+
+validate_inputs() {
+    require_cmd "$ZIP_BIN"
+    require_cmd "$UNZIP_BIN"
+    require_cmd "$SSH_BIN"
+    require_cmd "$SCP_BIN"
+    require_cmd "$FLOCK_BIN"
+    require_cmd "$MKTEMP_BIN"
+    require_cmd "$DATE_BIN"
+    require_cmd "$REALPATH_BIN"
+    if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
+        require_cmd "$CURL_BIN"
+    fi
+
+    require_absolute_path "$K3S_CONFIG_DIR" "K3S_CONFIG_DIR"
+    require_absolute_path "$K3S_SERVER_DIR" "K3S_SERVER_DIR"
+    require_absolute_path "$LOCAL_TMP_PARENT" "LOCAL_TMP_PARENT"
+    require_absolute_path "$LOCK_PATH" "LOCK_PATH"
+    require_absolute_path "$REMOTE_DIR" "REMOTE_DIR"
+
+    if [[ ! -d "$K3S_CONFIG_DIR" ]]; then
+        die "K3S_CONFIG_DIR is not an available directory"
+    fi
+    determine_protected_paths
+    if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then
+        die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens"
+    fi
+    if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then
+        die "CLEANUP_KEEP_COUNT must be an integer >= 0"
+    fi
+    CLEANUP_KEEP_COUNT_INT=$((10#$CLEANUP_KEEP_COUNT))
+    if [[ -n "$SSH_PORT" && ! "$SSH_PORT" =~ ^[0-9]+$ ]]; then
+        die "SSH_PORT must be numeric"
+    fi
+    if [[ -z "$REMOTE_USER" || -z "$REMOTE_HOST" ]]; then
+        die "REMOTE_USER and REMOTE_HOST must not be empty"
+    fi
+    if [[ ! "$REMOTE_USER" =~ ^[[:alnum:]_][[:alnum:]_.-]*$ ]]; then
+        die "REMOTE_USER may contain only letters, numbers, underscores, dots, and hyphens"
+    fi
+    if [[ ! "$REMOTE_HOST" =~ ^[[:alnum:]][[:alnum:].-]*$ ]]; then
+        die "REMOTE_HOST may contain only letters, numbers, dots, and hyphens"
+    fi
+    if [[ ! "$REMOTE_DIR" =~ ^/([[:alnum:]_.-]+/)*[[:alnum:]_.-]+/?$ || "$REMOTE_DIR" == */./* || "$REMOTE_DIR" == */../* || "$REMOTE_DIR" == */. || "$REMOTE_DIR" == */.. ]]; then
+        die "REMOTE_DIR must be an absolute path containing only letters, numbers, dots, underscores, hyphens, and slashes without dot segments"
+    fi
+
+    mkdir -p -- "$LOCAL_TMP_PARENT"
+    mkdir -p -- "$(dirname -- "$LOCK_PATH")"
+    build_remote_arguments
+}
+
+collect_sources() {
+    local service_file
+    local server_entry
+
+    IFS=':' read -r -a SERVICE_FILE_LIST <<< "$K3S_SERVICE_FILES"
+
+    # These are the only server entries ever included. In particular, server/db
+    # and server/kine.sock are intentionally not archive inputs.
+    add_archive_entry_if_present "$K3S_CONFIG_DIR"
+    for service_file in "${SERVICE_FILE_LIST[@]}"; do
+        [[ -z "$service_file" ]] && continue
+        require_absolute_path "$service_file" "K3S_SERVICE_FILES entry"
+        add_archive_entry_if_present "$service_file"
+    done
+    for server_entry in "${K3S_SERVER_ENTRIES[@]}"; do
+        add_archive_entry_if_present "${K3S_SERVER_DIR%/}/${server_entry}"
+    done
+
+    if (( ${#ARCHIVE_ENTRIES[@]} == 0 )); then
+        die "No k3s configuration sources are available"
+    fi
+}
+
+acquire_lock() {
+    exec {LOCK_FD}>"$LOCK_PATH"
+    if ! "$FLOCK_BIN" -n "$LOCK_FD"; then
+        die "Another k3s control-plane configuration backup is already running"
+    fi
+}
+
+make_archive_name() {
+    local timestamp
+    timestamp="$("$DATE_BIN" +"$ARCHIVE_TS_FORMAT")"
+    if [[ ! "$timestamp" =~ ^[[:alnum:]_.-]+$ ]]; then
+        die "ARCHIVE_TS_FORMAT produced an unsafe archive timestamp"
+    fi
+    ARCHIVE_NAME="${ARCHIVE_PREFIX}_${timestamp}.zip"
+}
+
+create_and_validate_archive() {
+    if ! RUN_TMP_DIR="$("$MKTEMP_BIN" -d "${LOCAL_TMP_PARENT%/}/k3s-control-plane-config.XXXXXX")"; then
+        die "Unable to create local temporary directory"
+    fi
+    LOCAL_ARCHIVE_PATH="${RUN_TMP_DIR}/${ARCHIVE_NAME}"
+
+    log_info "Creating k3s control-plane configuration archive"
+    (
+        cd /
+        "$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" -- "${ARCHIVE_ENTRIES[@]}" >/dev/null
+    )
+    "$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null
+
+    if [[ ! -f "$LOCAL_ARCHIVE_PATH" ]]; then
+        die "Archive creation did not produce an archive"
+    fi
+    ARCHIVE_SIZE_BYTES="$(wc -c < "$LOCAL_ARCHIVE_PATH" | tr -d '[:space:]')"
+    if [[ ! "$ARCHIVE_SIZE_BYTES" =~ ^[0-9]+$ ]]; then
+        die "Unable to determine archive size"
+    fi
+    log_info "Validated local archive"
+}
+
+run_ssh() {
+    "$SSH_BIN" "${SSH_ARGS[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "$@"
+}
+
+remove_remote_temporary_archive() {
+    local remote_temp_path="${1:?temporary remote path is required}"
+    run_ssh "rm -f -- $(shell_quote "$remote_temp_path")" >/dev/null 2>&1 || true
+}
+
+publish_archive() {
+    local remote_final_path="${REMOTE_DIR%/}/${ARCHIVE_NAME}"
+    local remote_temp_path="${REMOTE_DIR%/}/.${ARCHIVE_NAME}.upload-$$.tmp"
+    local remote_spec="${REMOTE_USER}@${REMOTE_HOST}:${remote_temp_path}"
+    local move_command
+
+    run_ssh "mkdir -p -- $(shell_quote "$REMOTE_DIR")"
+
+    if ! "$SCP_BIN" "${SCP_ARGS[@]}" -- "$LOCAL_ARCHIVE_PATH" "$remote_spec"; then
+        log_error "Archive upload failed"
+        remove_remote_temporary_archive "$remote_temp_path"
+        return 1
+    fi
+
+    move_command="if [ -e $(shell_quote "$remote_final_path") ]; then exit 17; fi; mv -- $(shell_quote "$remote_temp_path") $(shell_quote "$remote_final_path")"
+    if ! run_ssh "$move_command"; then
+        log_error "Remote archive publish failed"
+        remove_remote_temporary_archive "$remote_temp_path"
+        return 1
+    fi
+
+    log_info "Published archive successfully"
+}
+
+cleanup_remote_archives() {
+    if (( CLEANUP_KEEP_COUNT_INT == 0 )); then
+        log_info "Remote retention disabled"
+        return 0
+    fi
+
+    local remote_cleanup_command
+    remote_cleanup_command="$(shell_quote "$REMOTE_BASH_BIN") -s -- $(shell_quote "$REMOTE_DIR") $(shell_quote "$ARCHIVE_PREFIX") $(shell_quote "$CLEANUP_KEEP_COUNT_INT")"
+
+    run_ssh "$remote_cleanup_command" <<'REMOTE_CLEANUP'
+set -Eeuo pipefail
+
+remote_dir="$1"
+archive_prefix="$2"
+keep_count="$3"
+
+mapfile -d '' -t sorted_archives < <(
+    find -P "$remote_dir" -maxdepth 1 -type f \
+        -name "${archive_prefix}_*.zip" -printf '%T@\t%p\0' |
+        sort -z -t $'\t' -k1,1nr
+)
+
+if (( ${#sorted_archives[@]} <= keep_count )); then
+    exit 0
+fi
+
+for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do
+    archive_record="${sorted_archives[$index]}"
+    archive_path="${archive_record#*$'\t'}"
+    rm -f -- "$archive_path"
+done
+REMOTE_CLEANUP
+
+    log_info "Remote archive retention completed"
+}
+
+main() {
+    validate_inputs
+    acquire_lock
+    collect_sources
+    make_archive_name
+    create_and_validate_archive
+    publish_archive
+    cleanup_remote_archives
+
+    # Notifications are informational: an unavailable notification endpoint
+    # must not change the result of a completed backup.
+    local duration_seconds=$((SECONDS - BACKUP_STARTED_SECONDS))
+    send_backup_notification "$NOTIFY_SUCCESS_URL" "$ARCHIVE_SIZE_BYTES" "$duration_seconds" || true
+}
+
+main "$@"

k8s/examples/automated-nfs-backup.env.example

@@ -0,0 +1,26 @@
+NFS_SOURCE_PATH=/path/to/nfs/source
+BACKUP_OUTPUT_PATH=/path/to/backup/output
+BACKUP_PASSWORD=
+KUBECTL_BIN=kubectl
+KUBE_CONTEXT=
+WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller
+EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default
+BACKUP_STAGING_PATH=/tmp/k8s-nfs-backup-staging
+RSYNC_BIN=rsync
+ARCHIVE_PREFIX=nfs-backup
+ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S
+SEVENZ_METHOD=lzma2
+SEVENZ_LEVEL=9
+SEVENZ_HEADER_ENCRYPT=on
+SEVENZ_THREADS=on
+SCALE_RETRY_COUNT=3
+SCALE_RETRY_DELAY_SECONDS=5
+SCALE_WAIT_SECONDS=30
+LOG_LEVEL=info
+TMP_STATE_DIR=/tmp/k8s-nfs-backup
+NOTIFY_SUCCESS_URL=http://notify.haven/api/v2/notifications/backup
+NOTIFY_FAILURE_URL=http://notify.haven/api/v2/notifications/error
+NOTIFY_TITLE=Kubernetes
+NOTIFY_ASSET=kube config
+NOTIFY_SOURCE=Kubernetes NFS backup
+CLEANUP_KEEP_COUNT=5

k8s/lib/common.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+# Shared helpers for Kubernetes-focused cron/utility scripts.
+
+set -Eeuo pipefail
+
+if [[ -n "${COMMON_SH_LOADED:-}" ]]; then
+    return 0
+fi
+readonly COMMON_SH_LOADED=1
+
+if [[ -z "${SCRIPT_NAME:-}" ]]; then
+    _source_last_index=$(( ${#BASH_SOURCE[@]} - 1 ))
+    SCRIPT_NAME="$(basename "${BASH_SOURCE[$_source_last_index]}")"
+fi
+LOG_LEVEL="${LOG_LEVEL:-info}"
+
+_log_level_to_int() {
+    case "${1,,}" in
+        debug) echo 10 ;;
+        info) echo 20 ;;
+        warn|warning) echo 30 ;;
+        error) echo 40 ;;
+        *) echo 20 ;;
+    esac
+}
+
+readonly _CURRENT_LOG_LEVEL_INT="$(_log_level_to_int "$LOG_LEVEL")"
+
+_should_log() {
+    local level="${1:-info}"
+    local requested
+    requested="$(_log_level_to_int "$level")"
+    [[ "$requested" -ge "$_CURRENT_LOG_LEVEL_INT" ]]
+}
+
+_timestamp() {
+    date '+%Y-%m-%d %H:%M:%S'
+}
+
+_log() {
+    local level="${1:-info}"
+    shift || true
+    local message="${*:-}"
+    if _should_log "$level"; then
+        printf '%s [%s] [%s] %s\n' "$(_timestamp)" "${level^^}" "$SCRIPT_NAME" "$message"
+    fi
+}
+
+log_debug() { _log "debug" "$@"; }
+log_info() { _log "info" "$@"; }
+log_warn() { _log "warn" "$@"; }
+log_error() { _log "error" "$@"; }
+
+die() {
+    local message="${1:-Unspecified error}"
+    local code="${2:-1}"
+    log_error "$message"
+    exit "$code"
+}
+
+trim() {
+    local value="${1:-}"
+    value="${value#"${value%%[![:space:]]*}"}"
+    value="${value%"${value##*[![:space:]]}"}"
+    printf '%s' "$value"
+}
+
+require_env() {
+    local var_name="${1:?variable name is required}"
+    if [[ -z "${!var_name:-}" ]]; then
+        die "Missing required environment variable: ${var_name}"
+    fi
+}
+
+require_cmd() {
+    local cmd="${1:?command is required}"
+    if ! command -v "$cmd" >/dev/null 2>&1; then
+        die "Required command not found: ${cmd}"
+    fi
+}
+
+retry() {
+    local attempts="${1:?attempt count required}"
+    local delay_seconds="${2:?delay seconds required}"
+    shift 2
+
+    if (( attempts < 1 )); then
+        die "retry attempts must be >= 1"
+    fi
+
+    local attempt=1
+    local exit_code=0
+    while true; do
+        if "$@"; then
+            return 0
+        fi
+        exit_code=$?
+        if (( attempt >= attempts )); then
+            return "$exit_code"
+        fi
+        log_warn "Command failed (attempt ${attempt}/${attempts}): $*; retrying in ${delay_seconds}s"
+        sleep "$delay_seconds"
+        ((attempt++))
+    done
+}
+
+on_error() {
+    local line="${1:-unknown}"
+    local command="${2:-unknown}"
+    log_error "Unhandled error at line ${line}: ${command}"
+}
+
+on_exit() {
+    local code="${1:-0}"
+    if [[ "$code" -eq 0 ]]; then
+        log_info "Finished successfully"
+    else
+        log_error "Finished with errors (exit code ${code})"
+    fi
+}
+
+setup_exit_handling() {
+    trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
+    trap 'on_exit "$?"' EXIT
+}

k8s/test/test-automated-nfs-backup.sh

@@ -0,0 +1,619 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/automated-nfs-backup.sh"
+TMPD="$(mktemp -d "${TMPDIR:-/tmp}/automated-nfs-backup-test.XXXXXX")"
+
+tests_total=0
+tests_passed=0
+tests_failed=0
+
+cleanup() {
+    rm -rf "${TMPD}"
+}
+trap cleanup EXIT
+
+assert() {
+    local message="$1"
+    shift
+
+    tests_total=$((tests_total + 1))
+    if "$@"; then
+        printf 'PASS: %s\n' "$message"
+        tests_passed=$((tests_passed + 1))
+    else
+        printf 'FAIL: %s\n' "$message"
+        tests_failed=$((tests_failed + 1))
+    fi
+}
+
+is_zero() {
+    [[ "$1" -eq 0 ]]
+}
+
+is_nonzero() {
+    [[ "$1" -ne 0 ]]
+}
+
+file_contains() {
+    local file="$1"
+    local value="$2"
+    grep -Fq -- "$value" "$file"
+}
+
+file_lacks() {
+    local file="$1"
+    local value="$2"
+    ! grep -Fq -- "$value" "$file"
+}
+
+file_matches() {
+    local file="$1"
+    local regex="$2"
+    grep -Eq -- "$regex" "$file"
+}
+
+archive_exists() {
+    local output_dir="$1"
+    local -a archives=()
+
+    shopt -s nullglob
+    archives=("${output_dir}"/test_*.7z)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -gt 0 ]]
+}
+
+archive_count_is() {
+    local output_dir="$1"
+    local expected_count="$2"
+    local -a archives=()
+
+    shopt -s nullglob
+    archives=("${output_dir}"/test_*.7z)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -eq "$expected_count" ]]
+}
+
+path_does_not_exist() {
+    [[ ! -e "$1" ]]
+}
+
+directory_is_empty() {
+    local directory="$1"
+    local -a entries=()
+
+    shopt -s nullglob dotglob
+    entries=("${directory}"/*)
+    shopt -u nullglob dotglob
+    [[ "${#entries[@]}" -eq 0 ]]
+}
+
+events_are_ordered() {
+    local trace_file="$1"
+    shift
+    local event
+    local line_number
+    local previous_line=0
+
+    for event in "$@"; do
+        line_number="$(grep -n -m1 -F -- "$event" "$trace_file" | cut -d: -f1)"
+        [[ -n "$line_number" ]] || return 1
+        (( line_number > previous_line )) || return 1
+        previous_line="$line_number"
+    done
+}
+
+trace_has_scale() {
+    local trace_file="$1"
+    local namespace="$2"
+    local replicas="$3"
+
+    grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file"
+}
+
+# Asserts the real script invoked 7z exactly once for the complete contents of
+# the run staging directory.
+sevenz_called_once_for_staging() {
+    [[ "$(wc -l < "${TMPD}/7z_invocations.log" | tr -d '[:space:]')" -eq 1 ]] &&
+        grep -Eq -- '^a -t7z [^ ]+\.7z \. ' "${TMPD}/7z_invocations.log"
+}
+
+write_fake_script() {
+    local path="$1"
+
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        printf '%s\n' "$line"
+    done > "$path"
+    chmod +x "$path"
+}
+
+write_fake_binaries() {
+    mkdir -p "${TMPD}/bin"
+
+    write_fake_script "${TMPD}/bin/kubectl" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+namespace=""
+get_kind=""
+has_output_format=0
+
+for ((i = 1; i <= $#; i++)); do
+    arg="${!i}"
+    case "$arg" in
+        -n)
+            next=$((i + 1))
+            namespace="${!next}"
+            ;;
+        get)
+            next=$((i + 1))
+            get_kind="${!next}"
+            ;;
+        -o)
+            has_output_format=1
+            ;;
+    esac
+done
+
+if [[ "$get_kind" == "namespaces" ]]; then
+    if [[ "$has_output_format" -eq 1 ]]; then
+        while IFS= read -r configured_namespace; do
+            [[ -n "$configured_namespace" ]] && printf '%s\n' "$configured_namespace"
+        done < "${TMPD}/namespaces.txt"
+    fi
+    exit 0
+fi
+
+if [[ -n "$get_kind" && -n "$namespace" ]]; then
+    replicas=""
+    while IFS=: read -r configured_namespace configured_replicas; do
+        if [[ "$configured_namespace" == "$namespace" ]]; then
+            replicas="$configured_replicas"
+            break
+        fi
+    done < "${TMPD}/replicas.txt"
+
+    [[ -n "$replicas" ]] || replicas=1
+    printf 'web\t%s\t\n' "$replicas"
+    exit 0
+fi
+
+for ((i = 1; i <= $#; i++)); do
+    if [[ "${!i}" == "scale" ]]; then
+        resource_index=$((i + 1))
+        resource="${!resource_index}"
+        kind="${resource%%/*}"
+        name="${resource#*/}"
+        replicas_arg=""
+        for arg in "$@"; do
+            [[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
+        done
+        printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
+        printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log"
+        exit 0
+    fi
+done
+
+exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/7z" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+# Record the full invocation so the harness can assert argument order.
+printf '%s\n' "$*" >> "${TMPD}/7z_invocations.log"
+
+if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
+    archive="${3:?archive path required}"
+    folder="${4:?folder path required}"
+    folder="${folder%/}"
+    printf '7z staging\n' >> "${TMPD}/event_trace.log"
+
+    if [[ -s "${TMPD}/fail_archive.txt" ]]; then
+        touch "$archive"
+        exit 2
+    fi
+
+    for staged_folder in ./*; do
+        [[ -d "$staged_folder" ]] || continue
+        printf '%s\n' "${staged_folder##*/}" >> "${TMPD}/7z_trace.log"
+    done
+    touch "$archive"
+fi
+
+exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/rsync" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+source_folder="${@: -2:1}"
+staging_folder="${@: -1}"
+source_folder="${source_folder%/}"
+staging_folder="${staging_folder%/}"
+folder_name="${source_folder##*/}"
+
+mkdir -p "${TMPD}/rsync_counts"
+count_file="${TMPD}/rsync_counts/${folder_name}"
+count=0
+if [[ -f "$count_file" ]]; then
+    count="$(<"$count_file")"
+fi
+count=$((count + 1))
+printf '%s\n' "$count" > "$count_file"
+printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
+
+if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
+    printf 'simulated rsync failure for %s\n' "$folder_name" >&2
+    exit 23
+fi
+
+mkdir -p "$staging_folder"
+cp -a "${source_folder}/." "$staging_folder/"
+EOF
+
+    write_fake_script "${TMPD}/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+url=""
+body=""
+for ((i = 1; i <= $#; i++)); do
+    arg="${!i}"
+    case "$arg" in
+        POST)
+            next=$((i + 1))
+            url="${!next}"
+            ;;
+        -d)
+            next=$((i + 1))
+            body="${!next}"
+            ;;
+    esac
+done
+
+body="${body//$'\n'/ }"
+printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
+exit 0
+EOF
+}
+
+run_backup() {
+    local source_dir="$1"
+    local output_dir="$2"
+    local state_dir="$3"
+    local log_file="$4"
+    shift 4
+    local -a env_overrides=("$@")
+
+    env BACKUP_STAGING_PATH="${state_dir}/staging" \
+        "${env_overrides[@]}" \
+        NFS_SOURCE_PATH="$source_dir" \
+        BACKUP_OUTPUT_PATH="$output_dir" \
+        BACKUP_PASSWORD="" \
+        ARCHIVE_PREFIX="test" \
+        NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
+        NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
+        KUBECTL_BIN="${TMPD}/bin/kubectl" \
+        SEVENZ_BIN="${TMPD}/bin/7z" \
+        RSYNC_BIN="${TMPD}/bin/rsync" \
+        SCALE_WAIT_SECONDS=0 \
+        TMP_STATE_DIR="$state_dir" \
+        PATH="${TMPD}/bin:${PATH}" \
+        TMPD="$TMPD" \
+        bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
+}
+
+write_fake_binaries
+export TMPD
+: > "${TMPD}/fail_archive.txt"
+
+# Scenario A: a mapped folder is scaled down, copied, and restored, while
+# unmapped and excluded folders are copied live. One 7z call archives them all.
+success_dir="${TMPD}/success"
+success_source="${success_dir}/source"
+success_output="${success_dir}/out"
+success_state="${success_dir}/state"
+mkdir -p "${success_source}/app-ns" "${success_source}/kube-system" "${success_source}/loose-data" "$success_output" "${success_state}/run-stale"
+printf 'data\n' > "${success_source}/app-ns/file.txt"
+printf 'data\n' > "${success_source}/kube-system/file.txt"
+printf 'data\n' > "${success_source}/loose-data/file.txt"
+printf 'deployment\tstale\t9\n' > "${success_state}/run-stale/stale-ns.state"
+printf 'app-ns\nkube-system\nstale-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+
+set +e
+run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log"
+success_exit_code=$?
+set -e
+
+assert "success scenario exits with code 0" is_zero "$success_exit_code"
+assert "success scenario creates a test archive" archive_exists "$success_output"
+assert "success scenario reports three backup successes" file_contains "${success_dir}/success.log" "backup_successes=3"
+assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
+assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
+assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
+assert "success scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "success scenario sends the archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes": 0'
+assert "success scenario sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "Kubernetes NFS backup"'
+assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
+assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system"
+assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace"
+assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns"
+assert "success scenario invokes 7z exactly once for staging" sevenz_called_once_for_staging
+assert "success scenario archives mapped app-ns" file_contains "${TMPD}/7z_trace.log" "app-ns"
+assert "success scenario archives excluded kube-system" file_contains "${TMPD}/7z_trace.log" "kube-system"
+assert "success scenario archives unmapped loose-data" file_contains "${TMPD}/7z_trace.log" "loose-data"
+
+# Scenario A2: the exclusion list can be replaced through the environment.
+custom_exclude_dir="${TMPD}/custom-exclude"
+custom_exclude_source="${custom_exclude_dir}/source"
+custom_exclude_output="${custom_exclude_dir}/out"
+custom_exclude_state="${custom_exclude_dir}/state"
+mkdir -p "${custom_exclude_source}/app-ns" "$custom_exclude_output" "$custom_exclude_state"
+printf 'data\n' > "${custom_exclude_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "EXCLUDED_NAMESPACES=app-ns"
+custom_exclude_exit_code=$?
+set -e
+
+assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_exit_code"
+assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
+assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
+
+# Scenario A3: the mapped namespace is scaled down before its single local copy,
+# restored immediately afterward, and compression starts only after restoration.
+staging_dir="${TMPD}/staging"
+staging_source="${staging_dir}/source"
+staging_output="${staging_dir}/out"
+staging_state="${staging_dir}/state"
+staging_work="${staging_dir}/work"
+mkdir -p "${staging_source}/app-ns" "$staging_output" "$staging_state" "$staging_work"
+printf 'data\n' > "${staging_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+rm -rf "${TMPD}/rsync_counts"
+
+set +e
+run_backup "$staging_source" "$staging_output" "$staging_state" "${staging_dir}/staging.log" "BACKUP_STAGING_PATH=${staging_work}"
+staging_exit_code=$?
+set -e
+
+assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
+assert "staging scenario creates an archive" archive_exists "$staging_output"
+assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
+    "scale app-ns --replicas=0" \
+    "rsync app-ns 1" \
+    "scale app-ns --replicas=3" \
+    "7z staging"
+assert "staging scenario copies app-ns only once" file_contains "${TMPD}/rsync_counts/app-ns" "1"
+assert "staging scenario invokes 7z exactly once" sevenz_called_once_for_staging
+assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
+assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
+
+# Scenario A4: a failed local copy never archives the incomplete staging copy,
+# but the namespace is still restored immediately.
+rsync_failure_dir="${TMPD}/rsync-failure"
+rsync_failure_source="${rsync_failure_dir}/source"
+rsync_failure_output="${rsync_failure_dir}/out"
+rsync_failure_state="${rsync_failure_dir}/state"
+rsync_failure_work="${rsync_failure_dir}/work"
+mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failure_state" "$rsync_failure_work"
+printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+printf 'app-ns\n' > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+rm -rf "${TMPD}/rsync_counts"
+
+set +e
+run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state" "${rsync_failure_dir}/rsync-failure.log" "BACKUP_STAGING_PATH=${rsync_failure_work}"
+rsync_failure_exit_code=$?
+set -e
+
+assert "rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
+assert "rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
+assert "rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z staging"
+assert "rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
+
+# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
+# are restored, retention still runs, and the script reports success.
+failure_dir="${TMPD}/failure"
+failure_source="${failure_dir}/source"
+failure_output="${failure_dir}/out"
+failure_state="${failure_dir}/state"
+mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state"
+printf 'data\n' > "${failure_source}/good-ns/file.txt"
+printf 'data\n' > "${failure_source}/fail-ns/file.txt"
+for i in 1 2 3 4; do
+    touch -t "2020010${i}0000" "${failure_output}/test_2020010${i}_000000.7z"
+done
+printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt"
+printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
+printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+
+set +e
+run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log"
+failure_exit_code=$?
+set -e
+
+assert "partial failure scenario exits with code 0" is_zero "$failure_exit_code"
+assert "partial failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
+assert "partial failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
+assert "partial failure scenario logs that it is continuing" file_contains "${failure_dir}/failure.log" "Some folder backups failed; continuing"
+assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
+assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
+assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
+assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
+assert "partial failure scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "partial failure scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "partial failure scenario marks the error as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
+assert "partial failure scenario includes the backup error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "BACKUP_FAILED"'
+assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
+assert "partial failure scenario archives the successfully staged folder" file_contains "${TMPD}/7z_trace.log" "good-ns"
+assert "partial failure scenario invokes 7z exactly once" sevenz_called_once_for_staging
+assert "partial failure scenario compresses after all namespaces are restored" events_are_ordered "${TMPD}/event_trace.log" \
+    "scale fail-ns --replicas=4" \
+    "scale good-ns --replicas=2" \
+    "7z staging"
+assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4
+assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z"
+
+# Scenario C: every attempted folder backup fails, so the script must still
+# return non-zero and must not send a success notification.
+all_failed_dir="${TMPD}/all-failed"
+all_failed_source="${all_failed_dir}/source"
+all_failed_output="${all_failed_dir}/out"
+all_failed_state="${all_failed_dir}/state"
+mkdir -p "${all_failed_source}/only-ns" "$all_failed_output" "$all_failed_state"
+printf 'data\n' > "${all_failed_source}/only-ns/file.txt"
+printf 'only-ns\n' > "${TMPD}/namespaces.txt"
+printf 'only-ns:2\n' > "${TMPD}/replicas.txt"
+printf 'only-ns\n' > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$all_failed_source" "$all_failed_output" "$all_failed_state" "${all_failed_dir}/all-failed.log"
+all_failed_exit_code=$?
+set -e
+
+assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code"
+assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0"
+assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1"
+assert "all-failed scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "all-failed scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+
+# Scenario C2: a failed single archive operation invalidates all staged folders
+# and removes the partial archive left by 7z.
+archive_failure_dir="${TMPD}/archive-failure"
+archive_failure_source="${archive_failure_dir}/source"
+archive_failure_output="${archive_failure_dir}/out"
+archive_failure_state="${archive_failure_dir}/state"
+mkdir -p "${archive_failure_source}/archive-ns" "$archive_failure_output" "$archive_failure_state"
+printf 'data\n' > "${archive_failure_source}/archive-ns/file.txt"
+printf 'archive-ns\n' > "${TMPD}/namespaces.txt"
+printf 'archive-ns:2\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+printf 'fail\n' > "${TMPD}/fail_archive.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$archive_failure_source" "$archive_failure_output" "$archive_failure_state" "${archive_failure_dir}/archive-failure.log"
+archive_failure_exit_code=$?
+set -e
+
+assert "archive failure exits non-zero" is_nonzero "$archive_failure_exit_code"
+assert "archive failure restores archive-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "archive-ns" "2"
+assert "archive failure reports zero backup successes" file_contains "${archive_failure_dir}/archive-failure.log" "backup_successes=0"
+assert "archive failure removes the partial archive" archive_count_is "$archive_failure_output" 0
+assert "archive failure does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+: > "${TMPD}/fail_archive.txt"
+
+# Scenario D: no source folders means zero successful backups, which must also
+# return non-zero rather than reporting an empty run as successful.
+empty_dir="${TMPD}/empty"
+empty_source="${empty_dir}/source"
+empty_output="${empty_dir}/out"
+empty_state="${empty_dir}/state"
+mkdir -p "$empty_source" "$empty_output" "$empty_state"
+: > "${TMPD}/namespaces.txt"
+: > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$empty_source" "$empty_output" "$empty_state" "${empty_dir}/empty.log"
+empty_exit_code=$?
+set -e
+
+assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code"
+assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0"
+assert "empty scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "empty scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+
+# Scenario E: abnormal abort while a namespace is still scaled to zero.
+# We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
+# errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its
+# .state file is written, but BEFORE the in-line restore. The EXIT trap must then
+# drive restore_all_remaining() and scale the namespace back up. This exercises
+# the real safety-net path that scenario B leaves unproven.
+abort_dir="${TMPD}/abort"
+abort_source="${abort_dir}/source"
+abort_output="${abort_dir}/out"
+abort_state="${abort_dir}/state"
+abort_work="${abort_dir}/work"
+mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" "${abort_work}"
+printf 'data\n' > "${abort_source}/crash-ns/file.txt"
+printf 'crash-ns\n' > "${TMPD}/namespaces.txt"
+printf 'crash-ns:5\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/7z_invocations.log"
+
+set +e
+NFS_SOURCE_PATH="${abort_source}" \
+BACKUP_OUTPUT_PATH="${abort_output}" \
+BACKUP_PASSWORD="" \
+ARCHIVE_PREFIX="test" \
+NOTIFY_SUCCESS_URL="" \
+NOTIFY_FAILURE_URL="" \
+KUBECTL_BIN="${TMPD}/bin/kubectl" \
+SEVENZ_BIN="${TMPD}/bin/7z" \
+BACKUP_STAGING_PATH="${abort_work}" \
+SCALE_WAIT_SECONDS="not-a-number" \
+TMP_STATE_DIR="${abort_state}" \
+PATH="${TMPD}/bin:${PATH}" \
+TMPD="${TMPD}" \
+bash "${SCRIPT_UNDER_TEST}" >"${abort_dir}/abort.log" 2>&1
+abort_exit_code=$?
+set -e
+
+assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}"
+assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0"
+assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5"
+assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state"
+
+printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
+if [[ "$tests_failed" -gt 0 ]]; then
+    exit 1
+fi

k8s/test/test-backup-k3s-control-plane-config.sh

@@ -0,0 +1,439 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/backup-k3s-control-plane-config.sh"
+TMPD="$(mktemp -d "${TMPDIR:-/tmp}/k3s-control-plane-config-test.XXXXXX")"
+
+tests_total=0
+tests_passed=0
+tests_failed=0
+
+cleanup() {
+    rm -rf -- "$TMPD"
+}
+trap cleanup EXIT
+
+assert() {
+    local message="$1"
+    shift
+    tests_total=$((tests_total + 1))
+    if "$@"; then
+        printf 'PASS: %s\n' "$message"
+        tests_passed=$((tests_passed + 1))
+    else
+        printf 'FAIL: %s\n' "$message"
+        tests_failed=$((tests_failed + 1))
+    fi
+}
+
+is_zero() { [[ "$1" -eq 0 ]]; }
+is_nonzero() { [[ "$1" -ne 0 ]]; }
+path_exists() { [[ -e "$1" ]]; }
+path_does_not_exist() { [[ ! -e "$1" ]]; }
+file_contains() { grep -Fq -- "$2" "$1"; }
+file_lacks() { [[ ! -e "$1" ]] || ! grep -Fq -- "$2" "$1"; }
+file_is_empty() { [[ ! -s "$1" ]]; }
+file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; }
+
+matching_archive_count_is() {
+    local directory="$1"
+    local expected="$2"
+    local -a archives=()
+    shopt -s nullglob
+    archives=("${directory}"/k3s-control-plane-config_*.zip)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -eq "$expected" ]]
+}
+
+directory_is_empty() {
+    local directory="$1"
+    local -a entries=()
+    shopt -s nullglob dotglob
+    entries=("${directory}"/*)
+    shopt -u nullglob dotglob
+    [[ "${#entries[@]}" -eq 0 ]]
+}
+
+write_fake_script() {
+    local path="$1"
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        printf '%s\n' "$line"
+    done > "$path"
+    chmod +x "$path"
+}
+
+write_fake_binaries() {
+    mkdir -p "${TMPD}/bin"
+
+    write_fake_script "${TMPD}/bin/zip" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[[ "${1:-}" == "-r" ]]
+archive="${2:?archive path required}"
+shift 2
+[[ "${1:-}" == "--" ]]
+shift
+: > "$archive"
+for entry in "$@"; do
+    source_path="/$entry"
+    if [[ -d "$source_path" ]]; then
+        while IFS= read -r path; do
+            printf '%s\n' "${path#/}" >> "$archive"
+        done < <(find "$source_path" -print | sort)
+    else
+        printf '%s\n' "$entry" >> "$archive"
+    fi
+done
+printf '%s\n' "$*" >> "${TMPD}/zip_invocations.log"
+EOF
+
+    write_fake_script "${TMPD}/bin/unzip" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[[ "${1:-}" == "-t" ]]
+archive="${2:?archive path required}"
+[[ -f "$archive" && -s "$archive" ]]
+printf '%s\n' "$archive" >> "${TMPD}/unzip_trace.log"
+EOF
+
+    write_fake_script "${TMPD}/bin/scp" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+source_path="${@: -2:1}"
+destination="${@: -1}"
+remote_path="${destination#*:}"
+if [[ "$remote_path" == *"'"* ]]; then
+    printf 'scp destination contains literal single quotes: %s\n' "$destination" >&2
+    exit 64
+fi
+remote_name="${remote_path##*/}"
+target_path="${FAKE_REMOTE_ROOT}/${remote_name}"
+
+cp -- "$source_path" "$target_path"
+printf '%s\n' "$remote_name" >> "${TMPD}/scp_trace.log"
+if [[ -f "${TMPD}/fail_scp" ]]; then
+    exit 42
+fi
+EOF
+
+    write_fake_script "${TMPD}/bin/ssh" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+command="${@: -1}"
+printf '%s\n' "$command" >> "${TMPD}/ssh_trace.log"
+
+quoted_values=()
+while IFS= read -r value; do
+    quoted_values+=("${value#\'}")
+    quoted_values[${#quoted_values[@]}-1]="${quoted_values[${#quoted_values[@]}-1]%\'}"
+done < <(printf '%s\n' "$command" | grep -o "'[^']*'" || true)
+
+if [[ "$command" == mkdir\ -p* ]]; then
+    mkdir -p -- "$FAKE_REMOTE_ROOT"
+    exit 0
+fi
+
+if [[ "$command" == rm\ -f* ]]; then
+    temp_name="${quoted_values[0]##*/}"
+    rm -f -- "${FAKE_REMOTE_ROOT}/${temp_name}"
+    exit 0
+fi
+
+if [[ "$command" == *" mv -- "* ]]; then
+    value_count="${#quoted_values[@]}"
+    temp_name="${quoted_values[value_count-2]##*/}"
+    final_name="${quoted_values[value_count-1]##*/}"
+    [[ ! -e "${FAKE_REMOTE_ROOT}/${final_name}" ]]
+    mv -- "${FAKE_REMOTE_ROOT}/${temp_name}" "${FAKE_REMOTE_ROOT}/${final_name}"
+    exit 0
+fi
+
+if [[ "$command" == *" -s -- "* ]]; then
+    value_count="${#quoted_values[@]}"
+    (
+        cd -- "$FAKE_REMOTE_ROOT"
+        bash -s -- "$FAKE_REMOTE_ROOT" "${quoted_values[value_count-2]}" "${quoted_values[value_count-1]}"
+    )
+    exit 0
+fi
+
+exit 1
+EOF
+
+    write_fake_script "${TMPD}/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+url=""
+body=""
+for ((i = 1; i <= $#; i++)); do
+    case "${!i}" in
+        POST|-d)
+            next=$((i + 1))
+            if [[ "${!i}" == "POST" ]]; then
+                url="${!next}"
+            else
+                body="${!next}"
+            fi
+            ;;
+    esac
+done
+body="${body//$'\n'/ }"
+printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
+[[ ! -f "${TMPD}/fail_curl" ]]
+EOF
+
+    write_fake_script "${TMPD}/bin/flock" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/date" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+printf '%s\n' '2026-07-21_12-00-00'
+EOF
+}
+
+run_backup() {
+    local fixture_root="$1"
+    local remote_root="$2"
+    local local_temp_parent="$3"
+    local log_file="$4"
+    shift 4
+    local -a overrides=("$@")
+
+    env \
+        K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \
+        K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \
+        K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \
+        LOCAL_TMP_PARENT="$local_temp_parent" \
+        LOCK_PATH="${fixture_root}/lock/k3s-control-plane-config.lock" \
+        REMOTE_USER="test-user" \
+        REMOTE_HOST="test-host" \
+        REMOTE_DIR="/backup" \
+        ZIP_BIN="${TMPD}/bin/zip" \
+        UNZIP_BIN="${TMPD}/bin/unzip" \
+        SSH_BIN="${TMPD}/bin/ssh" \
+        SCP_BIN="${TMPD}/bin/scp" \
+        CURL_BIN="${TMPD}/bin/curl" \
+        FLOCK_BIN="${TMPD}/bin/flock" \
+        DATE_BIN="${TMPD}/bin/date" \
+        NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
+        NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
+        PATH="${TMPD}/bin:${PATH}" \
+        TMPD="$TMPD" \
+        FAKE_REMOTE_ROOT="$remote_root" \
+        "${overrides[@]}" \
+        bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
+}
+
+create_source_fixture() {
+    local fixture_root="$1"
+    mkdir -p \
+        "${fixture_root}/etc/rancher/k3s" \
+        "${fixture_root}/etc/systemd/system" \
+        "${fixture_root}/etc/default" \
+        "${fixture_root}/etc/sysconfig" \
+        "${fixture_root}/var/lib/rancher/k3s/server/cred" \
+        "${fixture_root}/var/lib/rancher/k3s/server/etc" \
+        "${fixture_root}/var/lib/rancher/k3s/server/manifests" \
+        "${fixture_root}/var/lib/rancher/k3s/server/static" \
+        "${fixture_root}/var/lib/rancher/k3s/server/tls" \
+        "${fixture_root}/var/lib/rancher/k3s/server/db" \
+        "${fixture_root}/lock"
+    printf 'config\n' > "${fixture_root}/etc/rancher/k3s/config.yaml"
+    printf 'service\n' > "${fixture_root}/etc/systemd/system/k3s.service"
+    printf 'environment\n' > "${fixture_root}/etc/systemd/system/k3s.service.env"
+    printf 'default\n' > "${fixture_root}/etc/default/k3s"
+    printf 'sysconfig\n' > "${fixture_root}/etc/sysconfig/k3s"
+    printf 'credential\n' > "${fixture_root}/var/lib/rancher/k3s/server/cred/passwd"
+    printf 'state\n' > "${fixture_root}/var/lib/rancher/k3s/server/etc/config"
+    printf 'manifest\n' > "${fixture_root}/var/lib/rancher/k3s/server/manifests/custom.yaml"
+    printf 'static\n' > "${fixture_root}/var/lib/rancher/k3s/server/static/pod.yaml"
+    printf 'tls\n' > "${fixture_root}/var/lib/rancher/k3s/server/tls/server-ca.crt"
+    printf 'token\n' > "${fixture_root}/var/lib/rancher/k3s/server/token"
+    printf 'agent-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/agent-token"
+    printf 'node-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/node-token"
+    printf 'database must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/db/state.db"
+    printf 'socket must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/kine.sock"
+}
+
+write_fake_binaries
+export TMPD
+
+# Successful publication: source layout is preserved relative to /, unzip is
+# called, and retention deletes only old archives with this archive prefix.
+success_dir="${TMPD}/success"
+success_fixture="${success_dir}/fixture"
+success_remote="${success_dir}/remote"
+success_local_tmp="${success_dir}/local-tmp"
+mkdir -p "$success_remote" "$success_local_tmp"
+create_source_fixture "$success_fixture"
+for day in 1 2 3 4 5; do
+    touch -t "2026070${day}1200" "${success_remote}/k3s-control-plane-config_2026-07-0${day}_12-00-00.zip"
+done
+printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z"
+printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip"
+newline_archive="${success_remote}/k3s-control-plane-config_2026-06-01_12-00-00.zip"$'\nunrelated.zip'
+touch -t "202606011200" "$newline_archive"
+: > "$newline_archive"
+touch -t "202606011200" "$newline_archive"
+tab_archive="${success_remote}/k3s-control-plane-config_2026-05-01_12-00-00.zip"$'\tlegacy.zip'
+: > "$tab_archive"
+touch -t "202605011200" "$tab_archive"
+: > "${success_remote}/unrelated.zip"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/unzip_trace.log"
+: > "${TMPD}/zip_invocations.log"
+: > "${TMPD}/scp_trace.log"
+: > "${TMPD}/ssh_trace.log"
+
+set +e
+run_backup "$success_fixture" "$success_remote" "$success_local_tmp" "${success_dir}/run.log"
+success_exit_code=$?
+set -e
+
+fixture_relative="${success_fixture#/}"
+assert "successful backup exits with code 0" is_zero "$success_exit_code"
+assert "successful backup publishes the final zip archive" path_exists "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "successful backup validates its archive with unzip" file_contains "${TMPD}/unzip_trace.log" "k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "archive contains the k3s configuration path" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/rancher/k3s/config.yaml"
+assert "archive contains a service configuration file" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/systemd/system/k3s.service"
+assert "archive contains control-plane credentials" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/var/lib/rancher/k3s/server/cred/passwd"
+assert "archive excludes server database content" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "server/db/state.db"
+assert "archive excludes kine socket" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "kine.sock"
+assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4
+assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip"
+assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip"
+assert "retention removes matching old archives with newlines in their filenames" path_does_not_exist "$newline_archive"
+assert "retention removes matching old archives with tabs in their filenames" path_does_not_exist "$tab_archive"
+assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${success_remote}/unrelated.zip"
+assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
+assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
+assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
+assert "successful backup sends a v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "successful backup sends its archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes":'
+assert "successful backup sends its elapsed duration" file_contains "${TMPD}/curl_trace.log" '"durationSeconds":'
+assert "successful backup sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "K3s control-plane config backup"'
+
+# A configured source path ending in server/db/.. must be rejected before zip is
+# invoked, rather than recursively archiving server/db and its sibling socket.
+source_scope_dir="${TMPD}/source-scope"
+source_scope_fixture="${source_scope_dir}/fixture"
+source_scope_remote="${source_scope_dir}/remote"
+source_scope_local_tmp="${source_scope_dir}/local-tmp"
+source_scope_archive="${source_scope_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+mkdir -p "$source_scope_remote" "$source_scope_local_tmp"
+create_source_fixture "$source_scope_fixture"
+: > "${TMPD}/zip_invocations.log"
+
+set +e
+run_backup \
+    "$source_scope_fixture" \
+    "$source_scope_remote" \
+    "$source_scope_local_tmp" \
+    "${source_scope_dir}/run.log" \
+    "K3S_CONFIG_DIR=${source_scope_fixture}/var/lib/rancher/k3s/server/db/.."
+source_scope_exit_code=$?
+set -e
+
+assert "unsafe source override exits non-zero" is_nonzero "$source_scope_exit_code"
+assert "unsafe source override reports the source-scope error" file_contains "${source_scope_dir}/run.log" "Configured source path is not safe to archive"
+assert "unsafe source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log"
+assert "unsafe source override archive excludes server database content" file_lacks "$source_scope_archive" "state.db"
+assert "unsafe source override archive excludes kine socket" file_lacks "$source_scope_archive" "kine.sock"
+assert "unsafe source override does not publish an archive" directory_is_empty "$source_scope_remote"
+assert "unsafe source override removes local temporary state" directory_is_empty "$source_scope_local_tmp"
+
+# The source path and the server directory can be overridden independently;
+# protected database paths must still be derived from the configured source.
+source_scope_mismatch_dir="${TMPD}/source-scope-mismatch"
+source_scope_mismatch_fixture="${source_scope_mismatch_dir}/fixture"
+source_scope_mismatch_remote="${source_scope_mismatch_dir}/remote"
+source_scope_mismatch_local_tmp="${source_scope_mismatch_dir}/local-tmp"
+source_scope_mismatch_unrelated_server="${source_scope_mismatch_fixture}/unrelated-server"
+source_scope_mismatch_archive="${source_scope_mismatch_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+mkdir -p "$source_scope_mismatch_remote" "$source_scope_mismatch_local_tmp"
+create_source_fixture "$source_scope_mismatch_fixture"
+mkdir -p "$source_scope_mismatch_unrelated_server"
+: > "${TMPD}/zip_invocations.log"
+
+set +e
+run_backup \
+    "$source_scope_mismatch_fixture" \
+    "$source_scope_mismatch_remote" \
+    "$source_scope_mismatch_local_tmp" \
+    "${source_scope_mismatch_dir}/run.log" \
+    "K3S_CONFIG_DIR=${source_scope_mismatch_fixture}/var/lib/rancher/k3s/server/db/.." \
+    "K3S_SERVER_DIR=${source_scope_mismatch_unrelated_server}"
+source_scope_mismatch_exit_code=$?
+set -e
+
+assert "mismatched source override exits non-zero" is_nonzero "$source_scope_mismatch_exit_code"
+assert "mismatched source override reports the source-scope error" file_contains "${source_scope_mismatch_dir}/run.log" "Configured source path is not safe to archive"
+assert "mismatched source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log"
+assert "mismatched source override archive excludes server database content" file_lacks "$source_scope_mismatch_archive" "state.db"
+assert "mismatched source override archive excludes kine socket" file_lacks "$source_scope_mismatch_archive" "kine.sock"
+assert "mismatched source override does not publish an archive" path_does_not_exist "$source_scope_mismatch_archive"
+assert "mismatched source override leaves the remote directory empty" directory_is_empty "$source_scope_mismatch_remote"
+assert "mismatched source override removes local temporary state" directory_is_empty "$source_scope_mismatch_local_tmp"
+
+# Upload failure: the partial remote temporary file is removed, no final archive
+# is published, local temporary data is removed, and one safe error notification
+# is attempted.
+failure_dir="${TMPD}/upload-failure"
+failure_fixture="${failure_dir}/fixture"
+failure_remote="${failure_dir}/remote"
+failure_local_tmp="${failure_dir}/local-tmp"
+mkdir -p "$failure_remote" "$failure_local_tmp"
+create_source_fixture "$failure_fixture"
+: > "${TMPD}/fail_scp"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$failure_fixture" "$failure_remote" "$failure_local_tmp" "${failure_dir}/run.log"
+failure_exit_code=$?
+set -e
+rm -f -- "${TMPD}/fail_scp"
+
+assert "upload failure exits non-zero" is_nonzero "$failure_exit_code"
+assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
+assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
+assert "upload failure attempts exactly one v2 error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" 1
+assert "upload failure marks the notification as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
+assert "upload failure includes the v2 error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED"'
+assert "upload failure does not send a v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
+
+# A notification error is non-fatal after a completed backup.
+notify_dir="${TMPD}/notify-failure"
+notify_fixture="${notify_dir}/fixture"
+notify_remote="${notify_dir}/remote"
+notify_local_tmp="${notify_dir}/local-tmp"
+mkdir -p "$notify_remote" "$notify_local_tmp"
+create_source_fixture "$notify_fixture"
+: > "${TMPD}/fail_curl"
+
+set +e
+run_backup "$notify_fixture" "$notify_remote" "$notify_local_tmp" "${notify_dir}/run.log" "CLEANUP_KEEP_COUNT=0"
+notify_exit_code=$?
+set -e
+rm -f -- "${TMPD}/fail_curl"
+
+assert "failed success notification does not fail a completed backup" is_zero "$notify_exit_code"
+assert "completed backup remains published after notification failure" path_exists "${notify_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+
+printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
+if (( tests_failed > 0 )); then
+    exit 1
+fi

nas-gdrive-backup.sh

@@ -1,292 +0,0 @@
-#!/bin/bash
-
-# NAS Backup Script to Google Drive using rclone and 7zip
-# For each folder on BACKUP_SOURCE, it gets the sha256 checksum of it, checks the checksum against the previous backup, and if it has changed, it creates a 7zip archive of the folder with encryption.
-# It then uploads the archive to Google Drive using rclone.
-
-# Install: curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/nas-gdrive-backup.sh | bash -s -- --install
-# Run manually: /usr/local/bin/nas-gdrive-backup.sh
-
-# Configuration
-BACKUP_SOURCE="/export/Backup"
-META_DIR="/export/Backup/.gdrive"
-TMP_DIR="/export/Backup/.gdrive/tmp"
-ZIP_PASSWORD="password"
-GDRIVE_REMOTE="gdrive"
-GDRIVE_PATH="/NAS-Backups"
-ARCHIVE_NAME="backup.7z"
-LOG_FILE="/var/log/nas-gdrive-backup.log"
-
-# Function for logging
-log() {
-    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
-}
-
-clean_up() {
-    log "Cleaning up temporary files"
-    rm -rf "$TMP_DIR"
-}
-
-trap clean_up EXIT
-
-create_7z() {
-    local folder="$1"
-    local archive_name="$2"
-    log "Creating 7zip archive of $folder"
-    7z a -p"$ZIP_PASSWORD" -mhe=on -mx=3 "$archive_name" "$folder"
-    if [ $? -ne 0 ]; then
-        log "ERROR: Failed to create 7zip archive of $folder"
-    fi
-}
-
-upload_to_gdrive() {
-    local archive_name="$1"
-    log "Uploading $archive_name to Google Drive"
-
-    # Should replace existing file if it exists
-    rclone copy "$archive_name" "$GDRIVE_REMOTE:$GDRIVE_PATH" \
-        --progress \
-        --check-first \
-        --transfers 1 \
-        --checkers 1 \
-        --retries 1 \
-        --low-level-retries 10
-    if [ $? -ne 0 ]; then
-        log "ERROR: Failed to upload $archive_name to Google Drive"
-    fi
-}
-
-main() {
-    # Check if 7z is installed
-    if ! which 7z > /dev/null; then
-        log "ERROR: 7z is not installed"
-        exit 1
-    fi
-
-    # Check if rclone is installed
-    if ! which rclone > /dev/null; then
-        log "ERROR: rclone is not installed"
-        exit 1
-    fi
-
-    # Create meta directory if it doesn't exist
-    if [ ! -d "$META_DIR" ]; then
-        log "Creating meta directory: $META_DIR"
-        mkdir -p "$META_DIR"
-    fi
-
-    # Fix permissions for the meta directory (777 recursively)
-    chmod -R 777 "$META_DIR"
-
-    # Loop through each folder in the backup source
-    for folder in "$BACKUP_SOURCE"/*; do
-        if [ -d "$folder" ]; then
-            log "Processing folder: $folder"
-            # Get the sha256 checksum of the folder
-            CHECKSUM=$(find "$folder" -type f -exec sha256sum {} + | sha256sum | awk '{print $1}')
-            META_FILE="$META_DIR/$(basename "$folder").sha256"
-            # Check if the checksum file exists
-            if [ -f "$META_FILE" ]; then
-                # Read the previous checksum from the file
-                PREV_CHECKSUM=$(cat "$META_FILE")
-                # Compare the checksums
-                if [ "$CHECKSUM" != "$PREV_CHECKSUM" ]; then
-                    log "Changes detected in $folder - creating new archive"
-
-                    create_7z "$folder" "$TMP_DIR/$(basename "$folder")_$ARCHIVE_NAME"
-                    upload_to_gdrive "$TMP_DIR/$(basename "$folder")_$ARCHIVE_NAME"
-
-                    # Update the checksum file
-                    echo "$CHECKSUM" > "$META_FILE"
-                    
-                    # Remove the temporary archive file
-                    log "Removing temporary archive file"
-                    rm "$TMP_DIR/$(basename "$folder")_$ARCHIVE_NAME"
-                else
-                    log "No changes detected in $folder"
-                fi
-            else
-                log "No previous checksum found for $folder - creating new archive"
-
-                create_7z "$folder" "$TMP_DIR/$(basename "$folder")_$ARCHIVE_NAME"
-                upload_to_gdrive "$TMP_DIR/$(basename "$folder")_$ARCHIVE_NAME"
-                
-                # Create a checksum file for the folder
-                echo "$CHECKSUM" > "$META_FILE"
-
-                # Remove the temporary archive file
-                log "Removing temporary archive file"
-                rm "$TMP_DIR/$(basename "$folder")_$ARCHIVE_NAME"
-            fi
-        else
-            log "Skipping $folder, not a directory"
-        fi
-        log ""
-    done
-
-    # Fix permissions for the meta directory (777 recursively)
-    chmod -R 777 "$META_DIR"
-
-    log "Backup process completed successfully"
-
-    # Exit with success
-    exit 0
-}
-
-###########################
-### Installation script ###
-###########################
-# Function to install a dependency if not already installed
-install_dependency() {
-    local package="$1"
-    if ! dpkg -l | grep -q "$package"; then
-        install_log_info "Installing $package"
-        apt-get update && apt-get install -y "$package"
-        if [ $? -ne 0 ]; then
-            install_log_error "ERROR: Failed to install $package"
-            exit 1
-        fi
-        install_log_ok "$package installed successfully"
-    else
-        install_log_ok "$package is already installed"
-    fi
-}
-
-install_log_ok() {
-    echo -e "\e[32m[✓]\e[0m $1"
-}
-install_log_error() {
-    echo -e "\e[31m[✗]\e[0m $1"
-}
-install_log_info() {
-    echo -e "\e[34m[!]\e[0m $1"
-}
-install_log_separator() {
-    echo -e "\e[36m========================================\e[0m"
-}
-
-install_script() {
-    echo -e ""
-    install_log_separator
-    install_log_info "Starting installation of NAS to Google Drive backup script"
-    install_log_separator
-    echo -e ""
-    
-    install_log_separator
-    # Check if running as root
-    install_log_info "Checking if the script is running as root"
-    if [ "$(id -u)" -ne 0 ]; then
-        install_log_error "ERROR: This script must be run as root"
-        exit 1
-    else
-        install_log_ok "Running as root"
-    fi
-    install_log_separator
-    
-    # Check for dependencies
-    install_log_info "Checking for required dependencies"
-    install_dependency "rclone"
-    install_dependency "p7zip-full"
-    install_log_separator
-
-    # Check if crontab exists
-    install_log_info "Checking if crontab is installed"
-    if ! command -v crontab &>/dev/null; then
-        install_log_error "crontab is not installed"
-        exit 1
-    else
-        install_log_ok "crontab is installed"
-    fi
-    install_log_separator
-
-    install_log_info "Installing script to /usr/local/bin/nas-gdrive-backup.sh"
-    curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/nas-gdrive-backup.sh -o /usr/local/bin/nas-gdrive-backup.sh
-    chmod +x /usr/local/bin/nas-gdrive-backup.sh
-
-    install_log_info "Setting up ZIP_PASSWORD in $0"
-    if [ -z "$ZIP_PASSWORD" ]; then
-        install_log_error "ERROR: ZIP_PASSWORD is not set"
-        exit 1
-    fi
-    read -p "Enter ZIP_PASSWORD: " ZIP_PASSWORD </dev/tty
-    if [ -z "$ZIP_PASSWORD" ]; then
-        install_log_error "ERROR: ZIP_PASSWORD cannot be empty"
-        exit 1
-    fi
-    # Update the ZIP_PASSWORD in the script
-    sed -i "s/^ZIP_PASSWORD=.*/ZIP_PASSWORD=\"$ZIP_PASSWORD\"/" /usr/local/bin/nas-gdrive-backup.sh
-    log "ZIP_PASSWORD updated in /usr/local/bin/nas-gdrive-backup.sh"
-    install_log_separator
-
-    # Check for existence of source directories
-    install_log_info "Checking if BACKUP_SOURCE, META_DIR, and TMP_DIR exists"
-    if ! [ -d "$BACKUP_SOURCE" ]; then
-        install_log_error "ERROR: BACKUP_SOURCE directory does not exist"
-        exit 1
-    else
-        install_log_ok "BACKUP_SOURCE directory exists: $BACKUP_SOURCE"
-    fi
-    if ! [ -d "$META_DIR" ]; then
-        install_log_info "Creating META_DIR: $META_DIR"
-        mkdir -p "$META_DIR"
-    fi
-    if ! [ -d "$TMP_DIR" ]; then
-        install_log_info "Creating TMP_DIR: $TMP_DIR"
-        mkdir -p "$TMP_DIR"
-    fi
-    install_log_info "Setting permissions for $META_DIR and $TMP_DIR to 777"
-    chmod -R 777 "$META_DIR" "$TMP_DIR"
-    install_log_ok "Directories checked and are ok"
-
-    # Check for existing .sha256 files, if there are any, prompt to remove them
-    install_log_info "Verifying existing .sha256 files in $META_DIR"
-    for file in "$META_DIR"/*; do
-        if [ -f "$file" ] && [[ "$file" == *.sha256 ]]; then
-            install_log_info "Found .sha256 file: \e[96m\e[4m$file\e[0m"
-            read -p "Do you want to remove this file? [y/N]: " choice </dev/tty
-            if [[ "$choice" == "y" || "$choice" == "Y" ]]; then
-                install_log_info "Removing $file"
-                rm "$file"
-            else
-                install_log_info "Skipping $file"
-            fi
-        fi
-    done
-    install_log_ok "Existing .sha256 files checked"
-
-    install_log_separator
-    
-    install_log_info "Setting up rclone configuration"
-    if ! rclone config show gdrive &>/dev/null; then
-        install_log_error "ERROR: rclone gdrive remote is not configured"
-        install_log_error "Please run 'rclone config' to set up your Google Drive remote"
-        exit 1
-    fi
-    install_log_ok "rclone gdrive remote is configured"
-    install_log_separator
-
-    install_log_info "Setting up cron job for backup script"
-    (crontab -l 2>/dev/null; echo "55 23 * * 1 /usr/local/bin/nas-gdrive-backup.sh > /tmp/nas-gdrive-backup.log") | crontab -
-    install_log_ok "Cron job set up to run /usr/local/bin/nas-gdrive-backup.sh every Monday at 23:55"
-    install_log_separator
-    
-    echo -e ""
-    install_log_separator
-    install_log_ok "Installation completed successfully!"
-    install_log_separator
-    echo -e ""
-    echo -e "You can now run the script manually with: \e[32mnas-gdrive-backup.sh\e[0m"
-    echo -e "Or it will run automatically according to the cron schedule."
-
-    # Exit with success
-    exit 0
-}
-
-# Check for install flag
-if [[ "$1" == "--install" ]]; then
-    install_script
-    exit 0
-fi
-
-main "$@"
-exit 0

renovate.json

@@ -0,0 +1,3 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json"
+}

scripts-download.sh

@@ -1,330 +0,0 @@
-#!/bin/bash
-
-# Usage: curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/scripts-download.sh | bash
-
-set -euo pipefail
-
-#==============================================================================
-# CONFIGURATION
-#==============================================================================
-
-# Color definitions for output formatting
-readonly RED='\033[1;31m'
-readonly GREEN='\033[1;32m'
-readonly NC='\033[0m'
-readonly LIGHT_BLUE='\033[1;34m'
-readonly LIGHT_RED='\033[1;31m'
-readonly LIGHT_GREEN='\033[1;32m'
-readonly GREY='\033[1;30m'
-readonly YELLOW='\033[1;33m'
-
-# Configuration
-readonly FILES_URL="https://git.ivanch.me/ivanch/server-scripts/raw/branch/main"
-readonly REQUIRED_PACKAGES=("zip" "unzip" "curl")
-readonly REQUIRED_COMMANDS=("zip" "unzip" "sha256sum" "curl" "crontab")
-readonly AVAILABLE_SCRIPTS=("clean.sh" "backup.sh" "docker-updater.sh")
-
-# Format: [script_name]="cron_schedule"
-declare -A CRONTAB_SCHEDULES=(
-    ["clean.sh"]="0 23 * * *"              # Daily at 11 PM
-    ["backup.sh"]="30 23 * * 1,5"          # Monday and Friday at 11:30 PM
-    ["docker-updater.sh"]="0 3 */4 * *"    # Every 4 days at 3 AM
-)
-
-#==============================================================================
-# UTILITY FUNCTIONS
-#==============================================================================
-
-# Print formatted log messages
-log_info() { echo -e "${GREY}[i] $1${NC}"; }
-log_success() { echo -e "${GREEN}[✓] $1${NC}"; }
-log_warning() { echo -e "${YELLOW}[!] $1${NC}"; }
-log_error() { echo -e "${RED}[x] $1${NC}" >&2; }
-log_step() { echo -e "${LIGHT_BLUE}[i] $1${NC}"; }
-
-# Exit with error message
-die() {
-    log_error "$1"
-    exit 1
-}
-
-# Check if a command exists
-command_exists() {
-    command -v "$1" >/dev/null 2>&1
-}
-
-# Check if a process is running
-process_running() {
-    pgrep "$1" >/dev/null 2>&1
-}
-
-#==============================================================================
-# MAIN FUNCTIONS
-#==============================================================================
-
-# Detect the operating system
-detect_operating_system() {
-    if command_exists apk; then
-        echo "Alpine"
-    elif command_exists apt; then
-        echo "Debian"
-    else
-        die "Unsupported operating system. This script supports Alpine and Debian-based systems only."
-    fi
-}
-
-# Check for missing packages
-get_missing_packages() {
-    local missing=()
-    
-    # Check each required command and map to package names
-    if ! command_exists "zip"; then
-        missing+=("zip")
-    fi
-    if ! command_exists "unzip"; then
-        missing+=("unzip")
-    fi
-    if ! command_exists "curl"; then
-        missing+=("curl")
-    fi
-    # sha256sum is part of coreutils (usually pre-installed)
-    # crontab is part of cron package, but we'll check for cron service later
-    
-    # Only print if there are missing packages
-    if [[ ${#missing[@]} -gt 0 ]]; then
-        printf '%s\n' "${missing[@]}"
-    fi
-}
-
-# Install packages based on the detected OS
-install_packages() {
-    local os="$1"
-    shift
-    local packages=("$@")
-    
-    if [[ ${#packages[@]} -eq 0 ]]; then
-        log_info "No packages to install"
-        return 0
-    fi
-    
-    log_info "Installing required packages: ${packages[*]}"
-    log_info "Debug: Installing ${#packages[@]} packages on $os"
-    
-    case "$os" in
-        "Alpine")
-            log_info "Updating APK package index..."
-            apk update >/dev/null || die "Failed to update APK package index"
-            
-            log_info "Installing packages via APK..."
-            apk add --no-cache "${packages[@]}" >/dev/null || die "Failed to install packages via APK"
-            ;;
-        "Debian")
-            log_info "Ensuring /var/cache/apt/archives/partial exists..."
-            mkdir -p /var/cache/apt/archives/partial || die "Failed to create /var/cache/apt/archives/partial"
-            log_info "Updating APT package index..."
-            apt-get update -y >/dev/null || die "Failed to update APT package index"
-            
-            log_info "Installing packages via APT..."
-            apt-get install -y "${packages[@]}" >/dev/null || die "Failed to install packages via APT"
-            ;;
-        *)
-            log_error "Debug info - OS variable content: '$os'"
-            log_error "Debug info - OS variable length: ${#os}"
-            die "Unknown operating system: '$os'"
-            ;;
-    esac
-}
-
-# Verify all required packages are available
-verify_packages() {
-    log_info "Verifying package installation..."
-    
-    local missing_packages
-    readarray -t missing_packages < <(get_missing_packages)
-    
-    if [[ ${#missing_packages[@]} -gt 0 ]]; then
-        log_error "Failed to install required packages: ${missing_packages[*]}"
-        die "Please install the missing packages manually and try again"
-    fi
-    
-    log_success "All required packages are available"
-}
-
-# Check if crontab service is running
-check_crontab_service() {
-    log_info "Checking crontab service status..."
-    
-    if ! process_running "cron"; then
-        die "Crontab service is not running. Please start the cron service first."
-    fi
-    
-    log_success "Crontab service is running"
-}
-
-# Prompt user to select scripts for installation
-select_scripts() {
-    local selected=()
-    
-    echo >&2  # Send to stderr so it doesn't get captured
-    echo -e "${GREY}[i] Available scripts for download and installation:${NC}" >&2
-    echo >&2
-    
-    for script in "${AVAILABLE_SCRIPTS[@]}"; do
-        local schedule="${CRONTAB_SCHEDULES[$script]:-"0 0 * * *"}"
-        echo -e "  ${LIGHT_BLUE}$script${NC} - Schedule: ${GREY}$schedule${NC}" >&2
-    done
-    
-    echo >&2
-    echo -e "${GREY}[i] Select scripts to download and install:${NC}" >&2
-    
-    for script in "${AVAILABLE_SCRIPTS[@]}"; do
-        read -p "Install $script? [Y/n]: " choice </dev/tty
-        if [[ "$choice" =~ ^[Yy]?$ ]]; then
-            selected+=("$script")
-        fi
-    done
-    
-    if [[ ${#selected[@]} -eq 0 ]]; then
-        echo -e "${RED}[x] No scripts selected. Exiting...${NC}" >&2
-        exit 1
-    fi
-    
-    # Only output the selected scripts to stdout
-    printf '%s\n' "${selected[@]}"
-}
-
-# Verify server connectivity for selected scripts
-verify_server_connectivity() {
-    local scripts=("$@")
-    
-    log_info "Verifying server connectivity..."
-    
-    for script in "${scripts[@]}"; do
-        local url="$FILES_URL/$script"
-        if ! curl -s --head "$url" | head -n 1 | grep -E "HTTP/[12] [23].." >/dev/null; then
-            die "Script '$script' not found on server: $url"
-        fi
-    done
-    
-    log_success "Server connectivity verified"
-}
-
-# Download selected scripts
-download_scripts() {
-    local scripts=("$@")
-    
-    log_info "Downloading ${#scripts[@]} script(s)..."
-    
-    for script in "${scripts[@]}"; do
-        local url="$FILES_URL/$script"
-        log_step "Downloading $script..."
-        
-        if ! curl -s -o "./$script" "$url"; then
-            die "Failed to download $script from $url"
-        fi
-        
-        # Set executable permissions
-        chmod +x "./$script" || die "Failed to set executable permissions for $script"
-    done
-    
-    log_success "All scripts downloaded and configured"
-}
-
-# Setup crontab entries for selected scripts
-setup_crontab() {
-    local scripts=("$@")
-    local current_workdir
-    current_workdir=$(pwd)
-    
-    log_info "Setting up crontab entries..."
-    
-    for script in "${scripts[@]}"; do
-        local schedule="${CRONTAB_SCHEDULES[$script]:-"0 0 * * *"}"
-        local log_file="/tmp/${script%.*}.log"
-        local cron_entry="$schedule $current_workdir/$script > $log_file 2>&1"
-        
-        log_step "Configuring crontab for $script (Schedule: $schedule)..."
-        
-        # Remove existing crontab entry for this script
-        if crontab -l 2>/dev/null | grep -q "$script"; then
-            log_step "Removing existing crontab entry for $script..."
-            crontab -l 2>/dev/null | grep -v "$script" | crontab - || die "Failed to remove existing crontab entry"
-        fi
-        
-        # Add new crontab entry
-        (crontab -l 2>/dev/null; echo "$cron_entry") | crontab - || die "Failed to add crontab entry for $script"
-        
-        # Verify the entry was added
-        if ! crontab -l 2>/dev/null | grep -q "$script"; then
-            die "Failed to verify crontab entry for $script"
-        fi
-        
-        log_success "Crontab configured for $script"
-    done
-    
-    log_success "All crontab entries configured successfully"
-}
-
-#==============================================================================
-# MAIN EXECUTION
-#==============================================================================
-
-main() {
-    log_step "Starting Server Scripts Downloader"
-    echo
-    
-    # System detection and validation
-    log_info "Detecting operating system..."
-    local detected_os
-    detected_os=$(detect_operating_system)
-    detected_os=$(echo "$detected_os" | tr -d '\n\r' | xargs)  # Clean any whitespace/newlines
-    log_success "Detected $detected_os Linux"
-    
-    # Package management
-    local missing_packages
-    readarray -t missing_packages < <(get_missing_packages)
-    
-    # Filter out empty strings
-    local filtered_packages=()
-    for pkg in "${missing_packages[@]}"; do
-        if [[ -n "$pkg" ]]; then
-            filtered_packages+=("$pkg")
-        fi
-    done
-    missing_packages=("${filtered_packages[@]}")
-    
-    # Debug output
-    log_info "Debug: Found ${#missing_packages[@]} missing packages"
-    for i in "${!missing_packages[@]}"; do
-        log_info "Debug: Missing package $((i+1)): '${missing_packages[i]}'"
-    done
-    
-    if [[ ${#missing_packages[@]} -gt 0 ]]; then
-        log_warning "Missing packages detected: ${missing_packages[*]}"
-        install_packages "$detected_os" "${missing_packages[@]}"
-    else
-        log_success "All required packages are already installed"
-    fi
-    
-    verify_packages
-    check_crontab_service
-    
-    # Script selection and installation
-    local selected_scripts
-    readarray -t selected_scripts < <(select_scripts)
-    
-    log_info "Selected scripts: ${selected_scripts[*]}"
-    
-    verify_server_connectivity "${selected_scripts[@]}"
-    download_scripts "${selected_scripts[@]}"
-    setup_crontab "${selected_scripts[@]}"
-    
-    echo
-    log_success "Installation completed successfully!"
-    log_info "Scripts have been downloaded to: $(pwd)"
-    log_info "Crontab entries have been configured. Use 'crontab -l' to view them."
-    log_info "Log files will be created in /tmp/ directory."
-}
-
-# Execute main function
-main "$@"