#!/usr/bin/env bash set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/backup-k3s-control-plane-config.sh" TMPD="$(mktemp -d "${TMPDIR:-/tmp}/k3s-control-plane-config-test.XXXXXX")" tests_total=0 tests_passed=0 tests_failed=0 cleanup() { rm -rf -- "$TMPD" } trap cleanup EXIT assert() { local message="$1" shift tests_total=$((tests_total + 1)) if "$@"; then printf 'PASS: %s\n' "$message" tests_passed=$((tests_passed + 1)) else printf 'FAIL: %s\n' "$message" tests_failed=$((tests_failed + 1)) fi } is_zero() { [[ "$1" -eq 0 ]]; } is_nonzero() { [[ "$1" -ne 0 ]]; } path_exists() { [[ -e "$1" ]]; } path_does_not_exist() { [[ ! -e "$1" ]]; } file_contains() { grep -Fq -- "$2" "$1"; } file_lacks() { [[ ! -e "$1" ]] || ! grep -Fq -- "$2" "$1"; } file_is_empty() { [[ ! -s "$1" ]]; } file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; } matching_archive_count_is() { local directory="$1" local expected="$2" local -a archives=() shopt -s nullglob archives=("${directory}"/k3s-control-plane-config_*.zip) shopt -u nullglob [[ "${#archives[@]}" -eq "$expected" ]] } directory_is_empty() { local directory="$1" local -a entries=() shopt -s nullglob dotglob entries=("${directory}"/*) shopt -u nullglob dotglob [[ "${#entries[@]}" -eq 0 ]] } write_fake_script() { local path="$1" while IFS= read -r line || [[ -n "$line" ]]; do printf '%s\n' "$line" done > "$path" chmod +x "$path" } write_fake_binaries() { mkdir -p "${TMPD}/bin" write_fake_script "${TMPD}/bin/zip" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail [[ "${1:-}" == "-r" ]] archive="${2:?archive path required}" shift 2 [[ "${1:-}" == "--" ]] shift : > "$archive" for entry in "$@"; do source_path="/$entry" if [[ -d "$source_path" ]]; then while IFS= read -r path; do printf '%s\n' "${path#/}" >> "$archive" done < <(find "$source_path" -print | sort) else printf '%s\n' "$entry" >> "$archive" fi done printf '%s\n' "$*" >> "${TMPD}/zip_invocations.log" EOF write_fake_script "${TMPD}/bin/unzip" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail [[ "${1:-}" == "-t" ]] archive="${2:?archive path required}" [[ -f "$archive" && -s "$archive" ]] printf '%s\n' "$archive" >> "${TMPD}/unzip_trace.log" EOF write_fake_script "${TMPD}/bin/scp" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail source_path="${@: -2:1}" destination="${@: -1}" remote_path="${destination#*:}" if [[ "$remote_path" == *"'"* ]]; then printf 'scp destination contains literal single quotes: %s\n' "$destination" >&2 exit 64 fi remote_name="${remote_path##*/}" target_path="${FAKE_REMOTE_ROOT}/${remote_name}" cp -- "$source_path" "$target_path" printf '%s\n' "$remote_name" >> "${TMPD}/scp_trace.log" if [[ -f "${TMPD}/fail_scp" ]]; then exit 42 fi EOF write_fake_script "${TMPD}/bin/ssh" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail command="${@: -1}" printf '%s\n' "$command" >> "${TMPD}/ssh_trace.log" quoted_values=() while IFS= read -r value; do quoted_values+=("${value#\'}") quoted_values[${#quoted_values[@]}-1]="${quoted_values[${#quoted_values[@]}-1]%\'}" done < <(printf '%s\n' "$command" | grep -o "'[^']*'" || true) if [[ "$command" == mkdir\ -p* ]]; then mkdir -p -- "$FAKE_REMOTE_ROOT" exit 0 fi if [[ "$command" == rm\ -f* ]]; then temp_name="${quoted_values[0]##*/}" rm -f -- "${FAKE_REMOTE_ROOT}/${temp_name}" exit 0 fi if [[ "$command" == *" mv -- "* ]]; then value_count="${#quoted_values[@]}" temp_name="${quoted_values[value_count-2]##*/}" final_name="${quoted_values[value_count-1]##*/}" [[ ! -e "${FAKE_REMOTE_ROOT}/${final_name}" ]] mv -- "${FAKE_REMOTE_ROOT}/${temp_name}" "${FAKE_REMOTE_ROOT}/${final_name}" exit 0 fi if [[ "$command" == *" -s -- "* ]]; then value_count="${#quoted_values[@]}" ( cd -- "$FAKE_REMOTE_ROOT" bash -s -- "$FAKE_REMOTE_ROOT" "${quoted_values[value_count-2]}" "${quoted_values[value_count-1]}" ) exit 0 fi exit 1 EOF write_fake_script "${TMPD}/bin/curl" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail url="" body="" for ((i = 1; i <= $#; i++)); do case "${!i}" in POST|-d) next=$((i + 1)) if [[ "${!i}" == "POST" ]]; then url="${!next}" else body="${!next}" fi ;; esac done body="${body//$'\n'/ }" printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log" [[ ! -f "${TMPD}/fail_curl" ]] EOF write_fake_script "${TMPD}/bin/flock" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail exit 0 EOF write_fake_script "${TMPD}/bin/date" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail printf '%s\n' '2026-07-21_12-00-00' EOF } run_backup() { local fixture_root="$1" local remote_root="$2" local local_temp_parent="$3" local log_file="$4" shift 4 local -a overrides=("$@") env \ K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \ K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \ K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \ LOCAL_TMP_PARENT="$local_temp_parent" \ LOCK_PATH="${fixture_root}/lock/k3s-control-plane-config.lock" \ REMOTE_USER="test-user" \ REMOTE_HOST="test-host" \ REMOTE_DIR="/backup" \ ZIP_BIN="${TMPD}/bin/zip" \ UNZIP_BIN="${TMPD}/bin/unzip" \ SSH_BIN="${TMPD}/bin/ssh" \ SCP_BIN="${TMPD}/bin/scp" \ CURL_BIN="${TMPD}/bin/curl" \ FLOCK_BIN="${TMPD}/bin/flock" \ DATE_BIN="${TMPD}/bin/date" \ NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \ NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \ PATH="${TMPD}/bin:${PATH}" \ TMPD="$TMPD" \ FAKE_REMOTE_ROOT="$remote_root" \ "${overrides[@]}" \ bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1 } create_source_fixture() { local fixture_root="$1" mkdir -p \ "${fixture_root}/etc/rancher/k3s" \ "${fixture_root}/etc/systemd/system" \ "${fixture_root}/etc/default" \ "${fixture_root}/etc/sysconfig" \ "${fixture_root}/var/lib/rancher/k3s/server/cred" \ "${fixture_root}/var/lib/rancher/k3s/server/etc" \ "${fixture_root}/var/lib/rancher/k3s/server/manifests" \ "${fixture_root}/var/lib/rancher/k3s/server/static" \ "${fixture_root}/var/lib/rancher/k3s/server/tls" \ "${fixture_root}/var/lib/rancher/k3s/server/db" \ "${fixture_root}/lock" printf 'config\n' > "${fixture_root}/etc/rancher/k3s/config.yaml" printf 'service\n' > "${fixture_root}/etc/systemd/system/k3s.service" printf 'environment\n' > "${fixture_root}/etc/systemd/system/k3s.service.env" printf 'default\n' > "${fixture_root}/etc/default/k3s" printf 'sysconfig\n' > "${fixture_root}/etc/sysconfig/k3s" printf 'credential\n' > "${fixture_root}/var/lib/rancher/k3s/server/cred/passwd" printf 'state\n' > "${fixture_root}/var/lib/rancher/k3s/server/etc/config" printf 'manifest\n' > "${fixture_root}/var/lib/rancher/k3s/server/manifests/custom.yaml" printf 'static\n' > "${fixture_root}/var/lib/rancher/k3s/server/static/pod.yaml" printf 'tls\n' > "${fixture_root}/var/lib/rancher/k3s/server/tls/server-ca.crt" printf 'token\n' > "${fixture_root}/var/lib/rancher/k3s/server/token" printf 'agent-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/agent-token" printf 'node-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/node-token" printf 'database must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/db/state.db" printf 'socket must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/kine.sock" } write_fake_binaries export TMPD # Successful publication: source layout is preserved relative to /, unzip is # called, and retention deletes only old archives with this archive prefix. success_dir="${TMPD}/success" success_fixture="${success_dir}/fixture" success_remote="${success_dir}/remote" success_local_tmp="${success_dir}/local-tmp" mkdir -p "$success_remote" "$success_local_tmp" create_source_fixture "$success_fixture" for day in 1 2 3 4 5; do touch -t "2026070${day}1200" "${success_remote}/k3s-control-plane-config_2026-07-0${day}_12-00-00.zip" done printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z" printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip" newline_archive="${success_remote}/k3s-control-plane-config_2026-06-01_12-00-00.zip"$'\nunrelated.zip' touch -t "202606011200" "$newline_archive" : > "$newline_archive" touch -t "202606011200" "$newline_archive" tab_archive="${success_remote}/k3s-control-plane-config_2026-05-01_12-00-00.zip"$'\tlegacy.zip' : > "$tab_archive" touch -t "202605011200" "$tab_archive" : > "${success_remote}/unrelated.zip" : > "${TMPD}/curl_trace.log" : > "${TMPD}/unzip_trace.log" : > "${TMPD}/zip_invocations.log" : > "${TMPD}/scp_trace.log" : > "${TMPD}/ssh_trace.log" set +e run_backup "$success_fixture" "$success_remote" "$success_local_tmp" "${success_dir}/run.log" success_exit_code=$? set -e fixture_relative="${success_fixture#/}" assert "successful backup exits with code 0" is_zero "$success_exit_code" assert "successful backup publishes the final zip archive" path_exists "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" assert "successful backup validates its archive with unzip" file_contains "${TMPD}/unzip_trace.log" "k3s-control-plane-config_2026-07-21_12-00-00.zip" assert "archive contains the k3s configuration path" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/rancher/k3s/config.yaml" assert "archive contains a service configuration file" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/systemd/system/k3s.service" assert "archive contains control-plane credentials" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/var/lib/rancher/k3s/server/cred/passwd" assert "archive excludes server database content" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "server/db/state.db" assert "archive excludes kine socket" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "kine.sock" assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4 assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip" assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip" assert "retention removes matching old archives with newlines in their filenames" path_does_not_exist "$newline_archive" assert "retention removes matching old archives with tabs in their filenames" path_does_not_exist "$tab_archive" assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${success_remote}/unrelated.zip" assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z" assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip" assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp" assert "successful backup sends a v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup" assert "successful backup sends its archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes":' assert "successful backup sends its elapsed duration" file_contains "${TMPD}/curl_trace.log" '"durationSeconds":' assert "successful backup sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "K3s control-plane config backup"' # A configured source path ending in server/db/.. must be rejected before zip is # invoked, rather than recursively archiving server/db and its sibling socket. source_scope_dir="${TMPD}/source-scope" source_scope_fixture="${source_scope_dir}/fixture" source_scope_remote="${source_scope_dir}/remote" source_scope_local_tmp="${source_scope_dir}/local-tmp" source_scope_archive="${source_scope_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" mkdir -p "$source_scope_remote" "$source_scope_local_tmp" create_source_fixture "$source_scope_fixture" : > "${TMPD}/zip_invocations.log" set +e run_backup \ "$source_scope_fixture" \ "$source_scope_remote" \ "$source_scope_local_tmp" \ "${source_scope_dir}/run.log" \ "K3S_CONFIG_DIR=${source_scope_fixture}/var/lib/rancher/k3s/server/db/.." source_scope_exit_code=$? set -e assert "unsafe source override exits non-zero" is_nonzero "$source_scope_exit_code" assert "unsafe source override reports the source-scope error" file_contains "${source_scope_dir}/run.log" "Configured source path is not safe to archive" assert "unsafe source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log" assert "unsafe source override archive excludes server database content" file_lacks "$source_scope_archive" "state.db" assert "unsafe source override archive excludes kine socket" file_lacks "$source_scope_archive" "kine.sock" assert "unsafe source override does not publish an archive" directory_is_empty "$source_scope_remote" assert "unsafe source override removes local temporary state" directory_is_empty "$source_scope_local_tmp" # The source path and the server directory can be overridden independently; # protected database paths must still be derived from the configured source. source_scope_mismatch_dir="${TMPD}/source-scope-mismatch" source_scope_mismatch_fixture="${source_scope_mismatch_dir}/fixture" source_scope_mismatch_remote="${source_scope_mismatch_dir}/remote" source_scope_mismatch_local_tmp="${source_scope_mismatch_dir}/local-tmp" source_scope_mismatch_unrelated_server="${source_scope_mismatch_fixture}/unrelated-server" source_scope_mismatch_archive="${source_scope_mismatch_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" mkdir -p "$source_scope_mismatch_remote" "$source_scope_mismatch_local_tmp" create_source_fixture "$source_scope_mismatch_fixture" mkdir -p "$source_scope_mismatch_unrelated_server" : > "${TMPD}/zip_invocations.log" set +e run_backup \ "$source_scope_mismatch_fixture" \ "$source_scope_mismatch_remote" \ "$source_scope_mismatch_local_tmp" \ "${source_scope_mismatch_dir}/run.log" \ "K3S_CONFIG_DIR=${source_scope_mismatch_fixture}/var/lib/rancher/k3s/server/db/.." \ "K3S_SERVER_DIR=${source_scope_mismatch_unrelated_server}" source_scope_mismatch_exit_code=$? set -e assert "mismatched source override exits non-zero" is_nonzero "$source_scope_mismatch_exit_code" assert "mismatched source override reports the source-scope error" file_contains "${source_scope_mismatch_dir}/run.log" "Configured source path is not safe to archive" assert "mismatched source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log" assert "mismatched source override archive excludes server database content" file_lacks "$source_scope_mismatch_archive" "state.db" assert "mismatched source override archive excludes kine socket" file_lacks "$source_scope_mismatch_archive" "kine.sock" assert "mismatched source override does not publish an archive" path_does_not_exist "$source_scope_mismatch_archive" assert "mismatched source override leaves the remote directory empty" directory_is_empty "$source_scope_mismatch_remote" assert "mismatched source override removes local temporary state" directory_is_empty "$source_scope_mismatch_local_tmp" # Upload failure: the partial remote temporary file is removed, no final archive # is published, local temporary data is removed, and one safe error notification # is attempted. failure_dir="${TMPD}/upload-failure" failure_fixture="${failure_dir}/fixture" failure_remote="${failure_dir}/remote" failure_local_tmp="${failure_dir}/local-tmp" mkdir -p "$failure_remote" "$failure_local_tmp" create_source_fixture "$failure_fixture" : > "${TMPD}/fail_scp" : > "${TMPD}/curl_trace.log" set +e run_backup "$failure_fixture" "$failure_remote" "$failure_local_tmp" "${failure_dir}/run.log" failure_exit_code=$? set -e rm -f -- "${TMPD}/fail_scp" assert "upload failure exits non-zero" is_nonzero "$failure_exit_code" assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote" assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp" assert "upload failure attempts exactly one v2 error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" 1 assert "upload failure marks the notification as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"' assert "upload failure includes the v2 error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED"' assert "upload failure does not send a v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup" assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture" # A notification error is non-fatal after a completed backup. notify_dir="${TMPD}/notify-failure" notify_fixture="${notify_dir}/fixture" notify_remote="${notify_dir}/remote" notify_local_tmp="${notify_dir}/local-tmp" mkdir -p "$notify_remote" "$notify_local_tmp" create_source_fixture "$notify_fixture" : > "${TMPD}/fail_curl" set +e run_backup "$notify_fixture" "$notify_remote" "$notify_local_tmp" "${notify_dir}/run.log" "CLEANUP_KEEP_COUNT=0" notify_exit_code=$? set -e rm -f -- "${TMPD}/fail_curl" assert "failed success notification does not fail a completed backup" is_zero "$notify_exit_code" assert "completed backup remains published after notification failure" path_exists "${notify_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total" if (( tests_failed > 0 )); then exit 1 fi