feat(k8s): back up control plane configuration

2026-07-21 15:52:43 -03:00
parent 8db1db888f
commit dcbb8eddc2
4 changed files with 785 additions and 1 deletions
--- a/.gitea/workflows/test-automated-nfs-backup.yaml
+++ b/.gitea/workflows/test-automated-nfs-backup.yaml
@@ -1,4 +1,4 @@
-name: Test automated-nfs-backup
+name: Test Kubernetes backup scripts

 on:
  push:
@@ -15,3 +15,10 @@ jobs:
        uses: actions/checkout@v4
      - name: Run automated NFS backup tests
        run: bash k8s/test/test-automated-nfs-backup.sh
+  test-k3s-control-plane-config-backup:
+    runs-on: runner-slim
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Run k3s control-plane configuration backup tests
+        run: bash k8s/test/test-backup-k3s-control-plane-config.sh
--- a/k8s/README.md
+++ b/k8s/README.md
@@ -18,6 +18,24 @@ Behavior:
 - Restore issues are warnings by policy (run can still complete successfully).
 - Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.

+### `backup-k3s-control-plane-config.sh`
+
+Creates and validates a standard `.zip` containing the iris k3s control-plane configuration and credential state, then atomically publishes it to the NAS. It never stops or restarts k3s, and deliberately excludes `/var/lib/rancher/k3s/server/db` and `kine.sock`; this is configuration/credential backup, not a live datastore snapshot.
+
+The archive includes all of `K3S_CONFIG_DIR`, any existing `K3S_SERVICE_FILES`, and existing `cred`, `etc`, `manifests`, `static`, `tls`, `token`, `agent-token`, and `node-token` entries beneath `K3S_SERVER_DIR`.
+
+Defaults mirror the existing `k8s-config-backup` schedule: `ivanch@nas.haven:/export/Backup/k8s`, archive names such as `k3s-control-plane-config_2026-07-21_12-00-00.zip`, and remote retention of the four newest matching archives. Other prefixes, including `k8s-backup_*.7z`, are never considered for retention.
+
+The script uses a non-blocking lock by default (`/var/lock/k3s-control-plane-config-backup.lock`), so overlapping cron and manual invocations fail safely. It builds in a private local temporary directory, validates with `unzip -t`, uploads to a remote temporary filename, and only then renames it into place.
+
+Useful overrides:
+
+- Source paths: `K3S_CONFIG_DIR`, `K3S_SERVER_DIR`, `K3S_SERVICE_FILES` (colon-separated service configuration files).
+- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`.
+- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion).
+- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REMOTE_BASH_BIN`.
+- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/template/notify/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/template/notify/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_CALLER`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
+
 ## Environment Variables

 Required:
--- a/k8s/backup-k3s-control-plane-config.sh
+++ b/k8s/backup-k3s-control-plane-config.sh
@@ -0,0 +1,402 @@
+#!/usr/bin/env bash
+
+# Back up the k3s control-plane configuration and credentials, without taking
+# a live datastore snapshot or changing the running k3s service.
+
+SCRIPT_NAME="$(basename "$0")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/common.sh
+source "${SCRIPT_DIR}/lib/common.sh"
+
+# Keep the local archive and temporary files readable only by this process.
+umask 077
+
+# Source paths. K3S_CONFIG_DIR is the only required source; the service files
+# and server state entries are included only when present.
+K3S_CONFIG_DIR="${K3S_CONFIG_DIR:-/etc/rancher/k3s}"
+K3S_SERVER_DIR="${K3S_SERVER_DIR:-/var/lib/rancher/k3s/server}"
+K3S_SERVICE_FILES="${K3S_SERVICE_FILES-/etc/systemd/system/k3s.service:/etc/systemd/system/k3s.service.env:/etc/default/k3s:/etc/sysconfig/k3s}"
+
+# Archive and local working settings.
+ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-k3s-control-plane-config}"
+ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y-%m-%d_%H-%M-%S}"
+LOCAL_TMP_PARENT="${LOCAL_TMP_PARENT:-${TMPDIR:-/tmp}}"
+LOCK_PATH="${LOCK_PATH:-/var/lock/k3s-control-plane-config-backup.lock}"
+
+# Remote destination settings.
+REMOTE_USER="${REMOTE_USER:-ivanch}"
+REMOTE_HOST="${REMOTE_HOST:-nas.haven}"
+REMOTE_DIR="${REMOTE_DIR:-/export/Backup/k8s}"
+REMOTE_IDENTITY_FILE="${REMOTE_IDENTITY_FILE:-}"
+SSH_PORT="${SSH_PORT:-}"
+CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
+
+# Commands are configurable to support hosts with non-standard paths and
+# isolated tests.
+ZIP_BIN="${ZIP_BIN:-zip}"
+UNZIP_BIN="${UNZIP_BIN:-unzip}"
+SSH_BIN="${SSH_BIN:-ssh}"
+SCP_BIN="${SCP_BIN:-scp}"
+CURL_BIN="${CURL_BIN:-curl}"
+FLOCK_BIN="${FLOCK_BIN:-flock}"
+MKTEMP_BIN="${MKTEMP_BIN:-mktemp}"
+DATE_BIN="${DATE_BIN:-date}"
+REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
+
+# Notifications follow the JSON formats used by automated-nfs-backup.sh.
+# Set either URL to an empty value to disable that notification.
+NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/template/notify/backup}"
+NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/template/notify/error}"
+NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
+NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
+NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
+
+declare -a ARCHIVE_ENTRIES=()
+declare -a SERVICE_FILE_LIST=()
+declare -a SSH_ARGS=()
+declare -a SCP_ARGS=()
+readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token)
+
+RUN_TMP_DIR=""
+LOCAL_ARCHIVE_PATH=""
+ARCHIVE_NAME=""
+ARCHIVE_SIZE_BYTES=0
+CLEANUP_KEEP_COUNT_INT=0
+LOCK_FD=""
+FAILURE_NOTIFICATION_SENT=0
+
+json_escape() {
+    local value="${1:-}"
+    value="${value//\\/\\\\}"
+    value="${value//\"/\\\"}"
+    value="${value//$'\n'/ }"
+    printf '%s' "$value"
+}
+
+backup_size_mb() {
+    local bytes="${1:-0}"
+    if (( bytes <= 0 )); then
+        printf '0'
+        return 0
+    fi
+    printf '%s' "$(( (bytes + 1048575) / 1048576 ))"
+}
+
+send_backup_notification() {
+    local url="${1:-}"
+    local size_mb="${2:-0}"
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "title": "$(json_escape "$NOTIFY_TITLE")",
+  "asset": "$(json_escape "$NOTIFY_ASSET")",
+  "backupSizeInMB": ${size_mb}
+}
+EOF
+)
+
+    if ! "$CURL_BIN" -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup notification"
+        return 1
+    fi
+}
+
+send_backup_failure_notification() {
+    local url="${1:-}"
+    local size_mb="${2:-0}"
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "caller": "$(json_escape "$NOTIFY_CALLER")",
+  "message": "Control-plane configuration backup failed",
+  "critical": true,
+  "extra": [
+    {
+      "name": "Backup Size (MB)",
+      "value": "${size_mb}"
+    }
+  ]
+}
+EOF
+)
+
+    if ! "$CURL_BIN" -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup failure notification"
+        return 1
+    fi
+}
+
+notify_failure_once() {
+    if (( FAILURE_NOTIFICATION_SENT == 1 )); then
+        return 0
+    fi
+    FAILURE_NOTIFICATION_SENT=1
+    send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
+}
+
+cleanup() {
+    local exit_code=$?
+    trap - EXIT
+
+    if [[ -n "$RUN_TMP_DIR" && -d "$RUN_TMP_DIR" ]]; then
+        rm -rf -- "$RUN_TMP_DIR" || log_warn "Failed to clean local temporary files"
+    fi
+
+    if (( exit_code == 0 )); then
+        log_info "Finished successfully"
+    else
+        notify_failure_once
+        log_error "Control-plane configuration backup failed"
+    fi
+
+    exit "$exit_code"
+}
+trap cleanup EXIT
+
+shell_quote() {
+    local value="${1:-}"
+    value="${value//\'/\'\"\'\"\'}"
+    printf "'%s'" "$value"
+}
+
+require_absolute_path() {
+    local path="${1:?path is required}"
+    local description="${2:-path}"
+    if [[ "$path" != /* ]]; then
+        die "${description} must be an absolute path"
+    fi
+}
+
+add_archive_entry_if_present() {
+    local absolute_path="${1:?source path is required}"
+    local relative_path
+
+    if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then
+        return 0
+    fi
+
+    relative_path="${absolute_path#/}"
+    if [[ -z "$relative_path" || "$relative_path" == *"/../"* || "$relative_path" == ../* ]]; then
+        die "Configured source path is not safe to archive"
+    fi
+    ARCHIVE_ENTRIES+=("$relative_path")
+}
+
+build_remote_arguments() {
+    SSH_ARGS=()
+    SCP_ARGS=()
+    if [[ -n "$REMOTE_IDENTITY_FILE" ]]; then
+        SSH_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
+        SCP_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
+    fi
+    if [[ -n "$SSH_PORT" ]]; then
+        SSH_ARGS+=(-p "$SSH_PORT")
+        SCP_ARGS+=(-P "$SSH_PORT")
+    fi
+}
+
+validate_inputs() {
+    require_cmd "$ZIP_BIN"
+    require_cmd "$UNZIP_BIN"
+    require_cmd "$SSH_BIN"
+    require_cmd "$SCP_BIN"
+    require_cmd "$FLOCK_BIN"
+    require_cmd "$MKTEMP_BIN"
+    require_cmd "$DATE_BIN"
+    if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
+        require_cmd "$CURL_BIN"
+    fi
+
+    require_absolute_path "$K3S_CONFIG_DIR" "K3S_CONFIG_DIR"
+    require_absolute_path "$K3S_SERVER_DIR" "K3S_SERVER_DIR"
+    require_absolute_path "$LOCAL_TMP_PARENT" "LOCAL_TMP_PARENT"
+    require_absolute_path "$LOCK_PATH" "LOCK_PATH"
+    require_absolute_path "$REMOTE_DIR" "REMOTE_DIR"
+
+    if [[ ! -d "$K3S_CONFIG_DIR" ]]; then
+        die "K3S_CONFIG_DIR is not an available directory"
+    fi
+    if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then
+        die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens"
+    fi
+    if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then
+        die "CLEANUP_KEEP_COUNT must be an integer >= 0"
+    fi
+    CLEANUP_KEEP_COUNT_INT=$((10#$CLEANUP_KEEP_COUNT))
+    if [[ -n "$SSH_PORT" && ! "$SSH_PORT" =~ ^[0-9]+$ ]]; then
+        die "SSH_PORT must be numeric"
+    fi
+    if [[ -z "$REMOTE_USER" || -z "$REMOTE_HOST" ]]; then
+        die "REMOTE_USER and REMOTE_HOST must not be empty"
+    fi
+    if [[ ! "$REMOTE_USER" =~ ^[[:alnum:]_][[:alnum:]_.-]*$ ]]; then
+        die "REMOTE_USER may contain only letters, numbers, underscores, dots, and hyphens"
+    fi
+    if [[ ! "$REMOTE_HOST" =~ ^[[:alnum:]][[:alnum:].-]*$ ]]; then
+        die "REMOTE_HOST may contain only letters, numbers, dots, and hyphens"
+    fi
+    if [[ ! "$REMOTE_DIR" =~ ^/([[:alnum:]_.-]+/)*[[:alnum:]_.-]+/?$ || "$REMOTE_DIR" == */./* || "$REMOTE_DIR" == */../* || "$REMOTE_DIR" == */. || "$REMOTE_DIR" == */.. ]]; then
+        die "REMOTE_DIR must be an absolute path containing only letters, numbers, dots, underscores, hyphens, and slashes without dot segments"
+    fi
+
+    mkdir -p -- "$LOCAL_TMP_PARENT"
+    mkdir -p -- "$(dirname -- "$LOCK_PATH")"
+    build_remote_arguments
+}
+
+collect_sources() {
+    local service_file
+    local server_entry
+
+    IFS=':' read -r -a SERVICE_FILE_LIST <<< "$K3S_SERVICE_FILES"
+
+    # These are the only server entries ever included. In particular, server/db
+    # and server/kine.sock are intentionally not archive inputs.
+    add_archive_entry_if_present "$K3S_CONFIG_DIR"
+    for service_file in "${SERVICE_FILE_LIST[@]}"; do
+        [[ -z "$service_file" ]] && continue
+        require_absolute_path "$service_file" "K3S_SERVICE_FILES entry"
+        add_archive_entry_if_present "$service_file"
+    done
+    for server_entry in "${K3S_SERVER_ENTRIES[@]}"; do
+        add_archive_entry_if_present "${K3S_SERVER_DIR%/}/${server_entry}"
+    done
+
+    if (( ${#ARCHIVE_ENTRIES[@]} == 0 )); then
+        die "No k3s configuration sources are available"
+    fi
+}
+
+acquire_lock() {
+    exec {LOCK_FD}>"$LOCK_PATH"
+    if ! "$FLOCK_BIN" -n "$LOCK_FD"; then
+        die "Another k3s control-plane configuration backup is already running"
+    fi
+}
+
+make_archive_name() {
+    local timestamp
+    timestamp="$("$DATE_BIN" +"$ARCHIVE_TS_FORMAT")"
+    if [[ ! "$timestamp" =~ ^[[:alnum:]_.-]+$ ]]; then
+        die "ARCHIVE_TS_FORMAT produced an unsafe archive timestamp"
+    fi
+    ARCHIVE_NAME="${ARCHIVE_PREFIX}_${timestamp}.zip"
+}
+
+create_and_validate_archive() {
+    if ! RUN_TMP_DIR="$("$MKTEMP_BIN" -d "${LOCAL_TMP_PARENT%/}/k3s-control-plane-config.XXXXXX")"; then
+        die "Unable to create local temporary directory"
+    fi
+    LOCAL_ARCHIVE_PATH="${RUN_TMP_DIR}/${ARCHIVE_NAME}"
+
+    log_info "Creating k3s control-plane configuration archive"
+    (
+        cd /
+        "$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" "${ARCHIVE_ENTRIES[@]}" >/dev/null
+    )
+    "$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null
+
+    if [[ ! -f "$LOCAL_ARCHIVE_PATH" ]]; then
+        die "Archive creation did not produce an archive"
+    fi
+    ARCHIVE_SIZE_BYTES="$(wc -c < "$LOCAL_ARCHIVE_PATH" | tr -d '[:space:]')"
+    if [[ ! "$ARCHIVE_SIZE_BYTES" =~ ^[0-9]+$ ]]; then
+        die "Unable to determine archive size"
+    fi
+    log_info "Validated local archive"
+}
+
+run_ssh() {
+    "$SSH_BIN" "${SSH_ARGS[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "$@"
+}
+
+remove_remote_temporary_archive() {
+    local remote_temp_path="${1:?temporary remote path is required}"
+    run_ssh "rm -f -- $(shell_quote "$remote_temp_path")" >/dev/null 2>&1 || true
+}
+
+publish_archive() {
+    local remote_final_path="${REMOTE_DIR%/}/${ARCHIVE_NAME}"
+    local remote_temp_path="${REMOTE_DIR%/}/.${ARCHIVE_NAME}.upload-$$.tmp"
+    local remote_spec="${REMOTE_USER}@${REMOTE_HOST}:${remote_temp_path}"
+    local move_command
+
+    run_ssh "mkdir -p -- $(shell_quote "$REMOTE_DIR")"
+
+    if ! "$SCP_BIN" "${SCP_ARGS[@]}" -- "$LOCAL_ARCHIVE_PATH" "$remote_spec"; then
+        log_error "Archive upload failed"
+        remove_remote_temporary_archive "$remote_temp_path"
+        return 1
+    fi
+
+    move_command="if [ -e $(shell_quote "$remote_final_path") ]; then exit 17; fi; mv -- $(shell_quote "$remote_temp_path") $(shell_quote "$remote_final_path")"
+    if ! run_ssh "$move_command"; then
+        log_error "Remote archive publish failed"
+        remove_remote_temporary_archive "$remote_temp_path"
+        return 1
+    fi
+
+    log_info "Published archive successfully"
+}
+
+cleanup_remote_archives() {
+    if (( CLEANUP_KEEP_COUNT_INT == 0 )); then
+        log_info "Remote retention disabled"
+        return 0
+    fi
+
+    local remote_cleanup_command
+    remote_cleanup_command="$(shell_quote "$REMOTE_BASH_BIN") -s -- $(shell_quote "$REMOTE_DIR") $(shell_quote "$ARCHIVE_PREFIX") $(shell_quote "$CLEANUP_KEEP_COUNT_INT")"
+
+    run_ssh "$remote_cleanup_command" <<'REMOTE_CLEANUP'
+set -Eeuo pipefail
+
+remote_dir="$1"
+archive_prefix="$2"
+keep_count="$3"
+
+shopt -s nullglob
+candidates=("${remote_dir%/}/${archive_prefix}_"*.zip)
+shopt -u nullglob
+
+matching_archives=()
+for candidate in "${candidates[@]}"; do
+    [[ -f "$candidate" ]] && matching_archives+=("$candidate")
+done
+
+if (( ${#matching_archives[@]} <= keep_count )); then
+    exit 0
+fi
+
+mapfile -t sorted_archives < <(ls -1t -- "${matching_archives[@]}")
+for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do
+    rm -f -- "${sorted_archives[$index]}"
+done
+REMOTE_CLEANUP
+
+    log_info "Remote archive retention completed"
+}
+
+main() {
+    validate_inputs
+    acquire_lock
+    collect_sources
+    make_archive_name
+    create_and_validate_archive
+    publish_archive
+    cleanup_remote_archives
+
+    # Notifications are informational: an unavailable notification endpoint
+    # must not change the result of a completed backup.
+    send_backup_notification "$NOTIFY_SUCCESS_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
+}
+
+main "$@"
--- a/k8s/test/test-backup-k3s-control-plane-config.sh
+++ b/k8s/test/test-backup-k3s-control-plane-config.sh
@@ -0,0 +1,357 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/backup-k3s-control-plane-config.sh"
+TMPD="$(mktemp -d "${TMPDIR:-/tmp}/k3s-control-plane-config-test.XXXXXX")"
+
+tests_total=0
+tests_passed=0
+tests_failed=0
+
+cleanup() {
+    rm -rf -- "$TMPD"
+}
+trap cleanup EXIT
+
+assert() {
+    local message="$1"
+    shift
+    tests_total=$((tests_total + 1))
+    if "$@"; then
+        printf 'PASS: %s\n' "$message"
+        tests_passed=$((tests_passed + 1))
+    else
+        printf 'FAIL: %s\n' "$message"
+        tests_failed=$((tests_failed + 1))
+    fi
+}
+
+is_zero() { [[ "$1" -eq 0 ]]; }
+is_nonzero() { [[ "$1" -ne 0 ]]; }
+path_exists() { [[ -e "$1" ]]; }
+path_does_not_exist() { [[ ! -e "$1" ]]; }
+file_contains() { grep -Fq -- "$2" "$1"; }
+file_lacks() { ! grep -Fq -- "$2" "$1"; }
+file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; }
+
+matching_archive_count_is() {
+    local directory="$1"
+    local expected="$2"
+    local -a archives=()
+    shopt -s nullglob
+    archives=("${directory}"/k3s-control-plane-config_*.zip)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -eq "$expected" ]]
+}
+
+directory_is_empty() {
+    local directory="$1"
+    local -a entries=()
+    shopt -s nullglob dotglob
+    entries=("${directory}"/*)
+    shopt -u nullglob dotglob
+    [[ "${#entries[@]}" -eq 0 ]]
+}
+
+write_fake_script() {
+    local path="$1"
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        printf '%s\n' "$line"
+    done > "$path"
+    chmod +x "$path"
+}
+
+write_fake_binaries() {
+    mkdir -p "${TMPD}/bin"
+
+    write_fake_script "${TMPD}/bin/zip" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[[ "${1:-}" == "-r" ]]
+archive="${2:?archive path required}"
+shift 2
+: > "$archive"
+for entry in "$@"; do
+    source_path="/$entry"
+    if [[ -d "$source_path" ]]; then
+        while IFS= read -r path; do
+            printf '%s\n' "${path#/}" >> "$archive"
+        done < <(find "$source_path" -print | sort)
+    else
+        printf '%s\n' "$entry" >> "$archive"
+    fi
+done
+printf '%s\n' "$*" >> "${TMPD}/zip_invocations.log"
+EOF
+
+    write_fake_script "${TMPD}/bin/unzip" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[[ "${1:-}" == "-t" ]]
+archive="${2:?archive path required}"
+[[ -f "$archive" && -s "$archive" ]]
+printf '%s\n' "$archive" >> "${TMPD}/unzip_trace.log"
+EOF
+
+    write_fake_script "${TMPD}/bin/scp" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+source_path="${@: -2:1}"
+destination="${@: -1}"
+remote_path="${destination#*:}"
+if [[ "$remote_path" == *"'"* ]]; then
+    printf 'scp destination contains literal single quotes: %s\n' "$destination" >&2
+    exit 64
+fi
+remote_name="${remote_path##*/}"
+target_path="${FAKE_REMOTE_ROOT}/${remote_name}"
+
+cp -- "$source_path" "$target_path"
+printf '%s\n' "$remote_name" >> "${TMPD}/scp_trace.log"
+if [[ -f "${TMPD}/fail_scp" ]]; then
+    exit 42
+fi
+EOF
+
+    write_fake_script "${TMPD}/bin/ssh" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+command="${@: -1}"
+printf '%s\n' "$command" >> "${TMPD}/ssh_trace.log"
+
+quoted_values=()
+while IFS= read -r value; do
+    quoted_values+=("${value#\'}")
+    quoted_values[${#quoted_values[@]}-1]="${quoted_values[${#quoted_values[@]}-1]%\'}"
+done < <(printf '%s\n' "$command" | grep -o "'[^']*'" || true)
+
+if [[ "$command" == mkdir\ -p* ]]; then
+    mkdir -p -- "$FAKE_REMOTE_ROOT"
+    exit 0
+fi
+
+if [[ "$command" == rm\ -f* ]]; then
+    temp_name="${quoted_values[0]##*/}"
+    rm -f -- "${FAKE_REMOTE_ROOT}/${temp_name}"
+    exit 0
+fi
+
+if [[ "$command" == *" mv -- "* ]]; then
+    value_count="${#quoted_values[@]}"
+    temp_name="${quoted_values[value_count-2]##*/}"
+    final_name="${quoted_values[value_count-1]##*/}"
+    [[ ! -e "${FAKE_REMOTE_ROOT}/${final_name}" ]]
+    mv -- "${FAKE_REMOTE_ROOT}/${temp_name}" "${FAKE_REMOTE_ROOT}/${final_name}"
+    exit 0
+fi
+
+if [[ "$command" == *" -s -- "* ]]; then
+    value_count="${#quoted_values[@]}"
+    prefix="${quoted_values[value_count-2]}"
+    keep_count="${quoted_values[value_count-1]}"
+    shopt -s nullglob
+    archives=("${FAKE_REMOTE_ROOT}/${prefix}_"*.zip)
+    shopt -u nullglob
+    if (( ${#archives[@]} > keep_count )); then
+        mapfile -t sorted_archives < <(ls -1t -- "${archives[@]}")
+        for ((i = keep_count; i < ${#sorted_archives[@]}; i++)); do
+            rm -f -- "${sorted_archives[$i]}"
+        done
+    fi
+    cat >/dev/null
+    exit 0
+fi
+
+exit 1
+EOF
+
+    write_fake_script "${TMPD}/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+url=""
+for ((i = 1; i <= $#; i++)); do
+    if [[ "${!i}" == "POST" ]]; then
+        next=$((i + 1))
+        url="${!next}"
+    fi
+done
+printf '%s\n' "$url" >> "${TMPD}/curl_trace.log"
+[[ ! -f "${TMPD}/fail_curl" ]]
+EOF
+
+    write_fake_script "${TMPD}/bin/flock" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/date" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+printf '%s\n' '2026-07-21_12-00-00'
+EOF
+}
+
+run_backup() {
+    local fixture_root="$1"
+    local remote_root="$2"
+    local local_temp_parent="$3"
+    local log_file="$4"
+    shift 4
+    local -a overrides=("$@")
+
+    env "${overrides[@]}" \
+        K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \
+        K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \
+        K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \
+        LOCAL_TMP_PARENT="$local_temp_parent" \
+        LOCK_PATH="${fixture_root}/lock/k3s-control-plane-config.lock" \
+        REMOTE_USER="test-user" \
+        REMOTE_HOST="test-host" \
+        REMOTE_DIR="/backup" \
+        ZIP_BIN="${TMPD}/bin/zip" \
+        UNZIP_BIN="${TMPD}/bin/unzip" \
+        SSH_BIN="${TMPD}/bin/ssh" \
+        SCP_BIN="${TMPD}/bin/scp" \
+        CURL_BIN="${TMPD}/bin/curl" \
+        FLOCK_BIN="${TMPD}/bin/flock" \
+        DATE_BIN="${TMPD}/bin/date" \
+        NOTIFY_SUCCESS_URL="http://example.invalid/success" \
+        NOTIFY_FAILURE_URL="http://example.invalid/failure" \
+        PATH="${TMPD}/bin:${PATH}" \
+        TMPD="$TMPD" \
+        FAKE_REMOTE_ROOT="$remote_root" \
+        bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
+}
+
+create_source_fixture() {
+    local fixture_root="$1"
+    mkdir -p \
+        "${fixture_root}/etc/rancher/k3s" \
+        "${fixture_root}/etc/systemd/system" \
+        "${fixture_root}/etc/default" \
+        "${fixture_root}/etc/sysconfig" \
+        "${fixture_root}/var/lib/rancher/k3s/server/cred" \
+        "${fixture_root}/var/lib/rancher/k3s/server/etc" \
+        "${fixture_root}/var/lib/rancher/k3s/server/manifests" \
+        "${fixture_root}/var/lib/rancher/k3s/server/static" \
+        "${fixture_root}/var/lib/rancher/k3s/server/tls" \
+        "${fixture_root}/var/lib/rancher/k3s/server/db" \
+        "${fixture_root}/lock"
+    printf 'config\n' > "${fixture_root}/etc/rancher/k3s/config.yaml"
+    printf 'service\n' > "${fixture_root}/etc/systemd/system/k3s.service"
+    printf 'environment\n' > "${fixture_root}/etc/systemd/system/k3s.service.env"
+    printf 'default\n' > "${fixture_root}/etc/default/k3s"
+    printf 'sysconfig\n' > "${fixture_root}/etc/sysconfig/k3s"
+    printf 'credential\n' > "${fixture_root}/var/lib/rancher/k3s/server/cred/passwd"
+    printf 'state\n' > "${fixture_root}/var/lib/rancher/k3s/server/etc/config"
+    printf 'manifest\n' > "${fixture_root}/var/lib/rancher/k3s/server/manifests/custom.yaml"
+    printf 'static\n' > "${fixture_root}/var/lib/rancher/k3s/server/static/pod.yaml"
+    printf 'tls\n' > "${fixture_root}/var/lib/rancher/k3s/server/tls/server-ca.crt"
+    printf 'token\n' > "${fixture_root}/var/lib/rancher/k3s/server/token"
+    printf 'agent-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/agent-token"
+    printf 'node-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/node-token"
+    printf 'database must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/db/state.db"
+    printf 'socket must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/kine.sock"
+}
+
+write_fake_binaries
+export TMPD
+
+# Successful publication: source layout is preserved relative to /, unzip is
+# called, and retention deletes only old archives with this archive prefix.
+success_dir="${TMPD}/success"
+success_fixture="${success_dir}/fixture"
+success_remote="${success_dir}/remote"
+success_local_tmp="${success_dir}/local-tmp"
+mkdir -p "$success_remote" "$success_local_tmp"
+create_source_fixture "$success_fixture"
+for day in 1 2 3 4 5; do
+    touch -t "2026070${day}1200" "${success_remote}/k3s-control-plane-config_2026-07-0${day}_12-00-00.zip"
+done
+printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z"
+printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/unzip_trace.log"
+: > "${TMPD}/zip_invocations.log"
+: > "${TMPD}/scp_trace.log"
+: > "${TMPD}/ssh_trace.log"
+
+set +e
+run_backup "$success_fixture" "$success_remote" "$success_local_tmp" "${success_dir}/run.log"
+success_exit_code=$?
+set -e
+
+fixture_relative="${success_fixture#/}"
+assert "successful backup exits with code 0" is_zero "$success_exit_code"
+assert "successful backup publishes the final zip archive" path_exists "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "successful backup validates its archive with unzip" file_contains "${TMPD}/unzip_trace.log" "k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "archive contains the k3s configuration path" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/rancher/k3s/config.yaml"
+assert "archive contains a service configuration file" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/systemd/system/k3s.service"
+assert "archive contains control-plane credentials" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/var/lib/rancher/k3s/server/cred/passwd"
+assert "archive excludes server database content" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "server/db/state.db"
+assert "archive excludes kine socket" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "kine.sock"
+assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4
+assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip"
+assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip"
+assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
+assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
+assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
+assert "successful backup sends a success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
+
+# Upload failure: the partial remote temporary file is removed, no final archive
+# is published, local temporary data is removed, and one safe error notification
+# is attempted.
+failure_dir="${TMPD}/upload-failure"
+failure_fixture="${failure_dir}/fixture"
+failure_remote="${failure_dir}/remote"
+failure_local_tmp="${failure_dir}/local-tmp"
+mkdir -p "$failure_remote" "$failure_local_tmp"
+create_source_fixture "$failure_fixture"
+: > "${TMPD}/fail_scp"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$failure_fixture" "$failure_remote" "$failure_local_tmp" "${failure_dir}/run.log"
+failure_exit_code=$?
+set -e
+rm -f -- "${TMPD}/fail_scp"
+
+assert "upload failure exits non-zero" is_nonzero "$failure_exit_code"
+assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
+assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
+assert "upload failure attempts exactly one error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/failure" 1
+assert "upload failure does not send a success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
+assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
+
+# A notification error is non-fatal after a completed backup.
+notify_dir="${TMPD}/notify-failure"
+notify_fixture="${notify_dir}/fixture"
+notify_remote="${notify_dir}/remote"
+notify_local_tmp="${notify_dir}/local-tmp"
+mkdir -p "$notify_remote" "$notify_local_tmp"
+create_source_fixture "$notify_fixture"
+: > "${TMPD}/fail_curl"
+
+set +e
+run_backup "$notify_fixture" "$notify_remote" "$notify_local_tmp" "${notify_dir}/run.log" "CLEANUP_KEEP_COUNT=0"
+notify_exit_code=$?
+set -e
+rm -f -- "${TMPD}/fail_curl"
+
+assert "failed success notification does not fail a completed backup" is_zero "$notify_exit_code"
+assert "completed backup remains published after notification failure" path_exists "${notify_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+
+printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
+if (( tests_failed > 0 )); then
+    exit 1
+fi