From dcbb8eddc2f67407d6a8465bbda83cfb1dc6a965 Mon Sep 17 00:00:00 2001 From: Jose Henrique Date: Tue, 21 Jul 2026 15:52:43 -0300 Subject: [PATCH] feat(k8s): back up control plane configuration --- .../workflows/test-automated-nfs-backup.yaml | 9 +- k8s/README.md | 18 + k8s/backup-k3s-control-plane-config.sh | 402 ++++++++++++++++++ .../test-backup-k3s-control-plane-config.sh | 357 ++++++++++++++++ 4 files changed, 785 insertions(+), 1 deletion(-) create mode 100755 k8s/backup-k3s-control-plane-config.sh create mode 100755 k8s/test/test-backup-k3s-control-plane-config.sh diff --git a/.gitea/workflows/test-automated-nfs-backup.yaml b/.gitea/workflows/test-automated-nfs-backup.yaml index ce2aa48..1f3b14a 100644 --- a/.gitea/workflows/test-automated-nfs-backup.yaml +++ b/.gitea/workflows/test-automated-nfs-backup.yaml @@ -1,4 +1,4 @@ -name: Test automated-nfs-backup +name: Test Kubernetes backup scripts on: push: @@ -15,3 +15,10 @@ jobs: uses: actions/checkout@v4 - name: Run automated NFS backup tests run: bash k8s/test/test-automated-nfs-backup.sh + test-k3s-control-plane-config-backup: + runs-on: runner-slim + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Run k3s control-plane configuration backup tests + run: bash k8s/test/test-backup-k3s-control-plane-config.sh diff --git a/k8s/README.md b/k8s/README.md index 30817b1..17a1c64 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -18,6 +18,24 @@ Behavior: - Restore issues are warnings by policy (run can still complete successfully). - Cleanup retains the `N` most recent matching archives by modification time and deletes older ones. +### `backup-k3s-control-plane-config.sh` + +Creates and validates a standard `.zip` containing the iris k3s control-plane configuration and credential state, then atomically publishes it to the NAS. It never stops or restarts k3s, and deliberately excludes `/var/lib/rancher/k3s/server/db` and `kine.sock`; this is configuration/credential backup, not a live datastore snapshot. + +The archive includes all of `K3S_CONFIG_DIR`, any existing `K3S_SERVICE_FILES`, and existing `cred`, `etc`, `manifests`, `static`, `tls`, `token`, `agent-token`, and `node-token` entries beneath `K3S_SERVER_DIR`. + +Defaults mirror the existing `k8s-config-backup` schedule: `ivanch@nas.haven:/export/Backup/k8s`, archive names such as `k3s-control-plane-config_2026-07-21_12-00-00.zip`, and remote retention of the four newest matching archives. Other prefixes, including `k8s-backup_*.7z`, are never considered for retention. + +The script uses a non-blocking lock by default (`/var/lock/k3s-control-plane-config-backup.lock`), so overlapping cron and manual invocations fail safely. It builds in a private local temporary directory, validates with `unzip -t`, uploads to a remote temporary filename, and only then renames it into place. + +Useful overrides: + +- Source paths: `K3S_CONFIG_DIR`, `K3S_SERVER_DIR`, `K3S_SERVICE_FILES` (colon-separated service configuration files). +- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`. +- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion). +- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REMOTE_BASH_BIN`. +- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/template/notify/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/template/notify/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_CALLER`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result. + ## Environment Variables Required: diff --git a/k8s/backup-k3s-control-plane-config.sh b/k8s/backup-k3s-control-plane-config.sh new file mode 100755 index 0000000..d3f7890 --- /dev/null +++ b/k8s/backup-k3s-control-plane-config.sh @@ -0,0 +1,402 @@ +#!/usr/bin/env bash + +# Back up the k3s control-plane configuration and credentials, without taking +# a live datastore snapshot or changing the running k3s service. + +SCRIPT_NAME="$(basename "$0")" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" + +# Keep the local archive and temporary files readable only by this process. +umask 077 + +# Source paths. K3S_CONFIG_DIR is the only required source; the service files +# and server state entries are included only when present. +K3S_CONFIG_DIR="${K3S_CONFIG_DIR:-/etc/rancher/k3s}" +K3S_SERVER_DIR="${K3S_SERVER_DIR:-/var/lib/rancher/k3s/server}" +K3S_SERVICE_FILES="${K3S_SERVICE_FILES-/etc/systemd/system/k3s.service:/etc/systemd/system/k3s.service.env:/etc/default/k3s:/etc/sysconfig/k3s}" + +# Archive and local working settings. +ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-k3s-control-plane-config}" +ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y-%m-%d_%H-%M-%S}" +LOCAL_TMP_PARENT="${LOCAL_TMP_PARENT:-${TMPDIR:-/tmp}}" +LOCK_PATH="${LOCK_PATH:-/var/lock/k3s-control-plane-config-backup.lock}" + +# Remote destination settings. +REMOTE_USER="${REMOTE_USER:-ivanch}" +REMOTE_HOST="${REMOTE_HOST:-nas.haven}" +REMOTE_DIR="${REMOTE_DIR:-/export/Backup/k8s}" +REMOTE_IDENTITY_FILE="${REMOTE_IDENTITY_FILE:-}" +SSH_PORT="${SSH_PORT:-}" +CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}" + +# Commands are configurable to support hosts with non-standard paths and +# isolated tests. +ZIP_BIN="${ZIP_BIN:-zip}" +UNZIP_BIN="${UNZIP_BIN:-unzip}" +SSH_BIN="${SSH_BIN:-ssh}" +SCP_BIN="${SCP_BIN:-scp}" +CURL_BIN="${CURL_BIN:-curl}" +FLOCK_BIN="${FLOCK_BIN:-flock}" +MKTEMP_BIN="${MKTEMP_BIN:-mktemp}" +DATE_BIN="${DATE_BIN:-date}" +REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}" + +# Notifications follow the JSON formats used by automated-nfs-backup.sh. +# Set either URL to an empty value to disable that notification. +NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/template/notify/backup}" +NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/template/notify/error}" +NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}" +NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}" +NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}" + +declare -a ARCHIVE_ENTRIES=() +declare -a SERVICE_FILE_LIST=() +declare -a SSH_ARGS=() +declare -a SCP_ARGS=() +readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token) + +RUN_TMP_DIR="" +LOCAL_ARCHIVE_PATH="" +ARCHIVE_NAME="" +ARCHIVE_SIZE_BYTES=0 +CLEANUP_KEEP_COUNT_INT=0 +LOCK_FD="" +FAILURE_NOTIFICATION_SENT=0 + +json_escape() { + local value="${1:-}" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + value="${value//$'\n'/ }" + printf '%s' "$value" +} + +backup_size_mb() { + local bytes="${1:-0}" + if (( bytes <= 0 )); then + printf '0' + return 0 + fi + printf '%s' "$(( (bytes + 1048575) / 1048576 ))" +} + +send_backup_notification() { + local url="${1:-}" + local size_mb="${2:-0}" + [[ -z "$url" ]] && return 0 + + local payload + payload=$(cat </dev/null; then + log_warn "Failed to send backup notification" + return 1 + fi +} + +send_backup_failure_notification() { + local url="${1:-}" + local size_mb="${2:-0}" + [[ -z "$url" ]] && return 0 + + local payload + payload=$(cat </dev/null; then + log_warn "Failed to send backup failure notification" + return 1 + fi +} + +notify_failure_once() { + if (( FAILURE_NOTIFICATION_SENT == 1 )); then + return 0 + fi + FAILURE_NOTIFICATION_SENT=1 + send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true +} + +cleanup() { + local exit_code=$? + trap - EXIT + + if [[ -n "$RUN_TMP_DIR" && -d "$RUN_TMP_DIR" ]]; then + rm -rf -- "$RUN_TMP_DIR" || log_warn "Failed to clean local temporary files" + fi + + if (( exit_code == 0 )); then + log_info "Finished successfully" + else + notify_failure_once + log_error "Control-plane configuration backup failed" + fi + + exit "$exit_code" +} +trap cleanup EXIT + +shell_quote() { + local value="${1:-}" + value="${value//\'/\'\"\'\"\'}" + printf "'%s'" "$value" +} + +require_absolute_path() { + local path="${1:?path is required}" + local description="${2:-path}" + if [[ "$path" != /* ]]; then + die "${description} must be an absolute path" + fi +} + +add_archive_entry_if_present() { + local absolute_path="${1:?source path is required}" + local relative_path + + if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then + return 0 + fi + + relative_path="${absolute_path#/}" + if [[ -z "$relative_path" || "$relative_path" == *"/../"* || "$relative_path" == ../* ]]; then + die "Configured source path is not safe to archive" + fi + ARCHIVE_ENTRIES+=("$relative_path") +} + +build_remote_arguments() { + SSH_ARGS=() + SCP_ARGS=() + if [[ -n "$REMOTE_IDENTITY_FILE" ]]; then + SSH_ARGS+=(-i "$REMOTE_IDENTITY_FILE") + SCP_ARGS+=(-i "$REMOTE_IDENTITY_FILE") + fi + if [[ -n "$SSH_PORT" ]]; then + SSH_ARGS+=(-p "$SSH_PORT") + SCP_ARGS+=(-P "$SSH_PORT") + fi +} + +validate_inputs() { + require_cmd "$ZIP_BIN" + require_cmd "$UNZIP_BIN" + require_cmd "$SSH_BIN" + require_cmd "$SCP_BIN" + require_cmd "$FLOCK_BIN" + require_cmd "$MKTEMP_BIN" + require_cmd "$DATE_BIN" + if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then + require_cmd "$CURL_BIN" + fi + + require_absolute_path "$K3S_CONFIG_DIR" "K3S_CONFIG_DIR" + require_absolute_path "$K3S_SERVER_DIR" "K3S_SERVER_DIR" + require_absolute_path "$LOCAL_TMP_PARENT" "LOCAL_TMP_PARENT" + require_absolute_path "$LOCK_PATH" "LOCK_PATH" + require_absolute_path "$REMOTE_DIR" "REMOTE_DIR" + + if [[ ! -d "$K3S_CONFIG_DIR" ]]; then + die "K3S_CONFIG_DIR is not an available directory" + fi + if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then + die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens" + fi + if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then + die "CLEANUP_KEEP_COUNT must be an integer >= 0" + fi + CLEANUP_KEEP_COUNT_INT=$((10#$CLEANUP_KEEP_COUNT)) + if [[ -n "$SSH_PORT" && ! "$SSH_PORT" =~ ^[0-9]+$ ]]; then + die "SSH_PORT must be numeric" + fi + if [[ -z "$REMOTE_USER" || -z "$REMOTE_HOST" ]]; then + die "REMOTE_USER and REMOTE_HOST must not be empty" + fi + if [[ ! "$REMOTE_USER" =~ ^[[:alnum:]_][[:alnum:]_.-]*$ ]]; then + die "REMOTE_USER may contain only letters, numbers, underscores, dots, and hyphens" + fi + if [[ ! "$REMOTE_HOST" =~ ^[[:alnum:]][[:alnum:].-]*$ ]]; then + die "REMOTE_HOST may contain only letters, numbers, dots, and hyphens" + fi + if [[ ! "$REMOTE_DIR" =~ ^/([[:alnum:]_.-]+/)*[[:alnum:]_.-]+/?$ || "$REMOTE_DIR" == */./* || "$REMOTE_DIR" == */../* || "$REMOTE_DIR" == */. || "$REMOTE_DIR" == */.. ]]; then + die "REMOTE_DIR must be an absolute path containing only letters, numbers, dots, underscores, hyphens, and slashes without dot segments" + fi + + mkdir -p -- "$LOCAL_TMP_PARENT" + mkdir -p -- "$(dirname -- "$LOCK_PATH")" + build_remote_arguments +} + +collect_sources() { + local service_file + local server_entry + + IFS=':' read -r -a SERVICE_FILE_LIST <<< "$K3S_SERVICE_FILES" + + # These are the only server entries ever included. In particular, server/db + # and server/kine.sock are intentionally not archive inputs. + add_archive_entry_if_present "$K3S_CONFIG_DIR" + for service_file in "${SERVICE_FILE_LIST[@]}"; do + [[ -z "$service_file" ]] && continue + require_absolute_path "$service_file" "K3S_SERVICE_FILES entry" + add_archive_entry_if_present "$service_file" + done + for server_entry in "${K3S_SERVER_ENTRIES[@]}"; do + add_archive_entry_if_present "${K3S_SERVER_DIR%/}/${server_entry}" + done + + if (( ${#ARCHIVE_ENTRIES[@]} == 0 )); then + die "No k3s configuration sources are available" + fi +} + +acquire_lock() { + exec {LOCK_FD}>"$LOCK_PATH" + if ! "$FLOCK_BIN" -n "$LOCK_FD"; then + die "Another k3s control-plane configuration backup is already running" + fi +} + +make_archive_name() { + local timestamp + timestamp="$("$DATE_BIN" +"$ARCHIVE_TS_FORMAT")" + if [[ ! "$timestamp" =~ ^[[:alnum:]_.-]+$ ]]; then + die "ARCHIVE_TS_FORMAT produced an unsafe archive timestamp" + fi + ARCHIVE_NAME="${ARCHIVE_PREFIX}_${timestamp}.zip" +} + +create_and_validate_archive() { + if ! RUN_TMP_DIR="$("$MKTEMP_BIN" -d "${LOCAL_TMP_PARENT%/}/k3s-control-plane-config.XXXXXX")"; then + die "Unable to create local temporary directory" + fi + LOCAL_ARCHIVE_PATH="${RUN_TMP_DIR}/${ARCHIVE_NAME}" + + log_info "Creating k3s control-plane configuration archive" + ( + cd / + "$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" "${ARCHIVE_ENTRIES[@]}" >/dev/null + ) + "$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null + + if [[ ! -f "$LOCAL_ARCHIVE_PATH" ]]; then + die "Archive creation did not produce an archive" + fi + ARCHIVE_SIZE_BYTES="$(wc -c < "$LOCAL_ARCHIVE_PATH" | tr -d '[:space:]')" + if [[ ! "$ARCHIVE_SIZE_BYTES" =~ ^[0-9]+$ ]]; then + die "Unable to determine archive size" + fi + log_info "Validated local archive" +} + +run_ssh() { + "$SSH_BIN" "${SSH_ARGS[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "$@" +} + +remove_remote_temporary_archive() { + local remote_temp_path="${1:?temporary remote path is required}" + run_ssh "rm -f -- $(shell_quote "$remote_temp_path")" >/dev/null 2>&1 || true +} + +publish_archive() { + local remote_final_path="${REMOTE_DIR%/}/${ARCHIVE_NAME}" + local remote_temp_path="${REMOTE_DIR%/}/.${ARCHIVE_NAME}.upload-$$.tmp" + local remote_spec="${REMOTE_USER}@${REMOTE_HOST}:${remote_temp_path}" + local move_command + + run_ssh "mkdir -p -- $(shell_quote "$REMOTE_DIR")" + + if ! "$SCP_BIN" "${SCP_ARGS[@]}" -- "$LOCAL_ARCHIVE_PATH" "$remote_spec"; then + log_error "Archive upload failed" + remove_remote_temporary_archive "$remote_temp_path" + return 1 + fi + + move_command="if [ -e $(shell_quote "$remote_final_path") ]; then exit 17; fi; mv -- $(shell_quote "$remote_temp_path") $(shell_quote "$remote_final_path")" + if ! run_ssh "$move_command"; then + log_error "Remote archive publish failed" + remove_remote_temporary_archive "$remote_temp_path" + return 1 + fi + + log_info "Published archive successfully" +} + +cleanup_remote_archives() { + if (( CLEANUP_KEEP_COUNT_INT == 0 )); then + log_info "Remote retention disabled" + return 0 + fi + + local remote_cleanup_command + remote_cleanup_command="$(shell_quote "$REMOTE_BASH_BIN") -s -- $(shell_quote "$REMOTE_DIR") $(shell_quote "$ARCHIVE_PREFIX") $(shell_quote "$CLEANUP_KEEP_COUNT_INT")" + + run_ssh "$remote_cleanup_command" <<'REMOTE_CLEANUP' +set -Eeuo pipefail + +remote_dir="$1" +archive_prefix="$2" +keep_count="$3" + +shopt -s nullglob +candidates=("${remote_dir%/}/${archive_prefix}_"*.zip) +shopt -u nullglob + +matching_archives=() +for candidate in "${candidates[@]}"; do + [[ -f "$candidate" ]] && matching_archives+=("$candidate") +done + +if (( ${#matching_archives[@]} <= keep_count )); then + exit 0 +fi + +mapfile -t sorted_archives < <(ls -1t -- "${matching_archives[@]}") +for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do + rm -f -- "${sorted_archives[$index]}" +done +REMOTE_CLEANUP + + log_info "Remote archive retention completed" +} + +main() { + validate_inputs + acquire_lock + collect_sources + make_archive_name + create_and_validate_archive + publish_archive + cleanup_remote_archives + + # Notifications are informational: an unavailable notification endpoint + # must not change the result of a completed backup. + send_backup_notification "$NOTIFY_SUCCESS_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true +} + +main "$@" diff --git a/k8s/test/test-backup-k3s-control-plane-config.sh b/k8s/test/test-backup-k3s-control-plane-config.sh new file mode 100755 index 0000000..2a355c2 --- /dev/null +++ b/k8s/test/test-backup-k3s-control-plane-config.sh @@ -0,0 +1,357 @@ +#!/usr/bin/env bash + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/backup-k3s-control-plane-config.sh" +TMPD="$(mktemp -d "${TMPDIR:-/tmp}/k3s-control-plane-config-test.XXXXXX")" + +tests_total=0 +tests_passed=0 +tests_failed=0 + +cleanup() { + rm -rf -- "$TMPD" +} +trap cleanup EXIT + +assert() { + local message="$1" + shift + tests_total=$((tests_total + 1)) + if "$@"; then + printf 'PASS: %s\n' "$message" + tests_passed=$((tests_passed + 1)) + else + printf 'FAIL: %s\n' "$message" + tests_failed=$((tests_failed + 1)) + fi +} + +is_zero() { [[ "$1" -eq 0 ]]; } +is_nonzero() { [[ "$1" -ne 0 ]]; } +path_exists() { [[ -e "$1" ]]; } +path_does_not_exist() { [[ ! -e "$1" ]]; } +file_contains() { grep -Fq -- "$2" "$1"; } +file_lacks() { ! grep -Fq -- "$2" "$1"; } +file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; } + +matching_archive_count_is() { + local directory="$1" + local expected="$2" + local -a archives=() + shopt -s nullglob + archives=("${directory}"/k3s-control-plane-config_*.zip) + shopt -u nullglob + [[ "${#archives[@]}" -eq "$expected" ]] +} + +directory_is_empty() { + local directory="$1" + local -a entries=() + shopt -s nullglob dotglob + entries=("${directory}"/*) + shopt -u nullglob dotglob + [[ "${#entries[@]}" -eq 0 ]] +} + +write_fake_script() { + local path="$1" + while IFS= read -r line || [[ -n "$line" ]]; do + printf '%s\n' "$line" + done > "$path" + chmod +x "$path" +} + +write_fake_binaries() { + mkdir -p "${TMPD}/bin" + + write_fake_script "${TMPD}/bin/zip" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail + +[[ "${1:-}" == "-r" ]] +archive="${2:?archive path required}" +shift 2 +: > "$archive" +for entry in "$@"; do + source_path="/$entry" + if [[ -d "$source_path" ]]; then + while IFS= read -r path; do + printf '%s\n' "${path#/}" >> "$archive" + done < <(find "$source_path" -print | sort) + else + printf '%s\n' "$entry" >> "$archive" + fi +done +printf '%s\n' "$*" >> "${TMPD}/zip_invocations.log" +EOF + + write_fake_script "${TMPD}/bin/unzip" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail + +[[ "${1:-}" == "-t" ]] +archive="${2:?archive path required}" +[[ -f "$archive" && -s "$archive" ]] +printf '%s\n' "$archive" >> "${TMPD}/unzip_trace.log" +EOF + + write_fake_script "${TMPD}/bin/scp" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail + +source_path="${@: -2:1}" +destination="${@: -1}" +remote_path="${destination#*:}" +if [[ "$remote_path" == *"'"* ]]; then + printf 'scp destination contains literal single quotes: %s\n' "$destination" >&2 + exit 64 +fi +remote_name="${remote_path##*/}" +target_path="${FAKE_REMOTE_ROOT}/${remote_name}" + +cp -- "$source_path" "$target_path" +printf '%s\n' "$remote_name" >> "${TMPD}/scp_trace.log" +if [[ -f "${TMPD}/fail_scp" ]]; then + exit 42 +fi +EOF + + write_fake_script "${TMPD}/bin/ssh" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail + +command="${@: -1}" +printf '%s\n' "$command" >> "${TMPD}/ssh_trace.log" + +quoted_values=() +while IFS= read -r value; do + quoted_values+=("${value#\'}") + quoted_values[${#quoted_values[@]}-1]="${quoted_values[${#quoted_values[@]}-1]%\'}" +done < <(printf '%s\n' "$command" | grep -o "'[^']*'" || true) + +if [[ "$command" == mkdir\ -p* ]]; then + mkdir -p -- "$FAKE_REMOTE_ROOT" + exit 0 +fi + +if [[ "$command" == rm\ -f* ]]; then + temp_name="${quoted_values[0]##*/}" + rm -f -- "${FAKE_REMOTE_ROOT}/${temp_name}" + exit 0 +fi + +if [[ "$command" == *" mv -- "* ]]; then + value_count="${#quoted_values[@]}" + temp_name="${quoted_values[value_count-2]##*/}" + final_name="${quoted_values[value_count-1]##*/}" + [[ ! -e "${FAKE_REMOTE_ROOT}/${final_name}" ]] + mv -- "${FAKE_REMOTE_ROOT}/${temp_name}" "${FAKE_REMOTE_ROOT}/${final_name}" + exit 0 +fi + +if [[ "$command" == *" -s -- "* ]]; then + value_count="${#quoted_values[@]}" + prefix="${quoted_values[value_count-2]}" + keep_count="${quoted_values[value_count-1]}" + shopt -s nullglob + archives=("${FAKE_REMOTE_ROOT}/${prefix}_"*.zip) + shopt -u nullglob + if (( ${#archives[@]} > keep_count )); then + mapfile -t sorted_archives < <(ls -1t -- "${archives[@]}") + for ((i = keep_count; i < ${#sorted_archives[@]}; i++)); do + rm -f -- "${sorted_archives[$i]}" + done + fi + cat >/dev/null + exit 0 +fi + +exit 1 +EOF + + write_fake_script "${TMPD}/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail + +url="" +for ((i = 1; i <= $#; i++)); do + if [[ "${!i}" == "POST" ]]; then + next=$((i + 1)) + url="${!next}" + fi +done +printf '%s\n' "$url" >> "${TMPD}/curl_trace.log" +[[ ! -f "${TMPD}/fail_curl" ]] +EOF + + write_fake_script "${TMPD}/bin/flock" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +exit 0 +EOF + + write_fake_script "${TMPD}/bin/date" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +printf '%s\n' '2026-07-21_12-00-00' +EOF +} + +run_backup() { + local fixture_root="$1" + local remote_root="$2" + local local_temp_parent="$3" + local log_file="$4" + shift 4 + local -a overrides=("$@") + + env "${overrides[@]}" \ + K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \ + K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \ + K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \ + LOCAL_TMP_PARENT="$local_temp_parent" \ + LOCK_PATH="${fixture_root}/lock/k3s-control-plane-config.lock" \ + REMOTE_USER="test-user" \ + REMOTE_HOST="test-host" \ + REMOTE_DIR="/backup" \ + ZIP_BIN="${TMPD}/bin/zip" \ + UNZIP_BIN="${TMPD}/bin/unzip" \ + SSH_BIN="${TMPD}/bin/ssh" \ + SCP_BIN="${TMPD}/bin/scp" \ + CURL_BIN="${TMPD}/bin/curl" \ + FLOCK_BIN="${TMPD}/bin/flock" \ + DATE_BIN="${TMPD}/bin/date" \ + NOTIFY_SUCCESS_URL="http://example.invalid/success" \ + NOTIFY_FAILURE_URL="http://example.invalid/failure" \ + PATH="${TMPD}/bin:${PATH}" \ + TMPD="$TMPD" \ + FAKE_REMOTE_ROOT="$remote_root" \ + bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1 +} + +create_source_fixture() { + local fixture_root="$1" + mkdir -p \ + "${fixture_root}/etc/rancher/k3s" \ + "${fixture_root}/etc/systemd/system" \ + "${fixture_root}/etc/default" \ + "${fixture_root}/etc/sysconfig" \ + "${fixture_root}/var/lib/rancher/k3s/server/cred" \ + "${fixture_root}/var/lib/rancher/k3s/server/etc" \ + "${fixture_root}/var/lib/rancher/k3s/server/manifests" \ + "${fixture_root}/var/lib/rancher/k3s/server/static" \ + "${fixture_root}/var/lib/rancher/k3s/server/tls" \ + "${fixture_root}/var/lib/rancher/k3s/server/db" \ + "${fixture_root}/lock" + printf 'config\n' > "${fixture_root}/etc/rancher/k3s/config.yaml" + printf 'service\n' > "${fixture_root}/etc/systemd/system/k3s.service" + printf 'environment\n' > "${fixture_root}/etc/systemd/system/k3s.service.env" + printf 'default\n' > "${fixture_root}/etc/default/k3s" + printf 'sysconfig\n' > "${fixture_root}/etc/sysconfig/k3s" + printf 'credential\n' > "${fixture_root}/var/lib/rancher/k3s/server/cred/passwd" + printf 'state\n' > "${fixture_root}/var/lib/rancher/k3s/server/etc/config" + printf 'manifest\n' > "${fixture_root}/var/lib/rancher/k3s/server/manifests/custom.yaml" + printf 'static\n' > "${fixture_root}/var/lib/rancher/k3s/server/static/pod.yaml" + printf 'tls\n' > "${fixture_root}/var/lib/rancher/k3s/server/tls/server-ca.crt" + printf 'token\n' > "${fixture_root}/var/lib/rancher/k3s/server/token" + printf 'agent-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/agent-token" + printf 'node-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/node-token" + printf 'database must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/db/state.db" + printf 'socket must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/kine.sock" +} + +write_fake_binaries +export TMPD + +# Successful publication: source layout is preserved relative to /, unzip is +# called, and retention deletes only old archives with this archive prefix. +success_dir="${TMPD}/success" +success_fixture="${success_dir}/fixture" +success_remote="${success_dir}/remote" +success_local_tmp="${success_dir}/local-tmp" +mkdir -p "$success_remote" "$success_local_tmp" +create_source_fixture "$success_fixture" +for day in 1 2 3 4 5; do + touch -t "2026070${day}1200" "${success_remote}/k3s-control-plane-config_2026-07-0${day}_12-00-00.zip" +done +printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z" +printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip" +: > "${TMPD}/curl_trace.log" +: > "${TMPD}/unzip_trace.log" +: > "${TMPD}/zip_invocations.log" +: > "${TMPD}/scp_trace.log" +: > "${TMPD}/ssh_trace.log" + +set +e +run_backup "$success_fixture" "$success_remote" "$success_local_tmp" "${success_dir}/run.log" +success_exit_code=$? +set -e + +fixture_relative="${success_fixture#/}" +assert "successful backup exits with code 0" is_zero "$success_exit_code" +assert "successful backup publishes the final zip archive" path_exists "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" +assert "successful backup validates its archive with unzip" file_contains "${TMPD}/unzip_trace.log" "k3s-control-plane-config_2026-07-21_12-00-00.zip" +assert "archive contains the k3s configuration path" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/rancher/k3s/config.yaml" +assert "archive contains a service configuration file" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/systemd/system/k3s.service" +assert "archive contains control-plane credentials" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/var/lib/rancher/k3s/server/cred/passwd" +assert "archive excludes server database content" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "server/db/state.db" +assert "archive excludes kine socket" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "kine.sock" +assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4 +assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip" +assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip" +assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z" +assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip" +assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp" +assert "successful backup sends a success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success" + +# Upload failure: the partial remote temporary file is removed, no final archive +# is published, local temporary data is removed, and one safe error notification +# is attempted. +failure_dir="${TMPD}/upload-failure" +failure_fixture="${failure_dir}/fixture" +failure_remote="${failure_dir}/remote" +failure_local_tmp="${failure_dir}/local-tmp" +mkdir -p "$failure_remote" "$failure_local_tmp" +create_source_fixture "$failure_fixture" +: > "${TMPD}/fail_scp" +: > "${TMPD}/curl_trace.log" + +set +e +run_backup "$failure_fixture" "$failure_remote" "$failure_local_tmp" "${failure_dir}/run.log" +failure_exit_code=$? +set -e +rm -f -- "${TMPD}/fail_scp" + +assert "upload failure exits non-zero" is_nonzero "$failure_exit_code" +assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" +assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote" +assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp" +assert "upload failure attempts exactly one error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/failure" 1 +assert "upload failure does not send a success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success" +assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture" + +# A notification error is non-fatal after a completed backup. +notify_dir="${TMPD}/notify-failure" +notify_fixture="${notify_dir}/fixture" +notify_remote="${notify_dir}/remote" +notify_local_tmp="${notify_dir}/local-tmp" +mkdir -p "$notify_remote" "$notify_local_tmp" +create_source_fixture "$notify_fixture" +: > "${TMPD}/fail_curl" + +set +e +run_backup "$notify_fixture" "$notify_remote" "$notify_local_tmp" "${notify_dir}/run.log" "CLEANUP_KEEP_COUNT=0" +notify_exit_code=$? +set -e +rm -f -- "${TMPD}/fail_curl" + +assert "failed success notification does not fail a completed backup" is_zero "$notify_exit_code" +assert "completed backup remains published after notification failure" path_exists "${notify_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" + +printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total" +if (( tests_failed > 0 )); then + exit 1 +fi