k8s nfs backup: exclude namespaces from scaling, isolate per-run state, harden failure handling
- Add EXCLUDED_NAMESPACES (default kube-system,kube-public,kube-node-lease,default); excluded folders are backed up but never scaled - Use a per-run state dir via mktemp -d so stale .state files from an earlier run can't leak into restores - Partial folder-backup failure now exits 0 (only zero successes fails); retention runs even on partial failure - Remove dead SCALE_TIMEOUT_SECONDS config; document SCALE_WAIT_SECONDS - Add .gitattributes to keep *.sh at LF - Extend test harness to 44 passing assertions covering exclusions, per-run state, partial/all-failed/empty scenarios
This commit is contained in:
@@ -66,6 +66,31 @@ archive_exists() {
|
||||
[[ "${#archives[@]}" -gt 0 ]]
|
||||
}
|
||||
|
||||
archive_count_is() {
|
||||
local output_dir="$1"
|
||||
local expected_count="$2"
|
||||
local -a archives=()
|
||||
|
||||
shopt -s nullglob
|
||||
archives=("${output_dir}"/test_*.7z)
|
||||
shopt -u nullglob
|
||||
[[ "${#archives[@]}" -eq "$expected_count" ]]
|
||||
}
|
||||
|
||||
path_does_not_exist() {
|
||||
[[ ! -e "$1" ]]
|
||||
}
|
||||
|
||||
directory_is_empty() {
|
||||
local directory="$1"
|
||||
local -a entries=()
|
||||
|
||||
shopt -s nullglob dotglob
|
||||
entries=("${directory}"/*)
|
||||
shopt -u nullglob dotglob
|
||||
[[ "${#entries[@]}" -eq 0 ]]
|
||||
}
|
||||
|
||||
trace_has_scale() {
|
||||
local trace_file="$1"
|
||||
local namespace="$2"
|
||||
@@ -218,20 +243,26 @@ run_backup() {
|
||||
local output_dir="$2"
|
||||
local state_dir="$3"
|
||||
local log_file="$4"
|
||||
local -a env_overrides=()
|
||||
|
||||
NFS_SOURCE_PATH="$source_dir" \
|
||||
BACKUP_OUTPUT_PATH="$output_dir" \
|
||||
BACKUP_PASSWORD="" \
|
||||
ARCHIVE_PREFIX="test" \
|
||||
NOTIFY_SUCCESS_URL="http://example.invalid/success" \
|
||||
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
|
||||
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
||||
SEVENZ_BIN="${TMPD}/bin/7z" \
|
||||
SCALE_WAIT_SECONDS=0 \
|
||||
TMP_STATE_DIR="$state_dir" \
|
||||
PATH="${TMPD}/bin:${PATH}" \
|
||||
TMPD="$TMPD" \
|
||||
bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
|
||||
if (( $# >= 5 )); then
|
||||
env_overrides+=("EXCLUDED_NAMESPACES=$5")
|
||||
fi
|
||||
|
||||
env "${env_overrides[@]}" \
|
||||
NFS_SOURCE_PATH="$source_dir" \
|
||||
BACKUP_OUTPUT_PATH="$output_dir" \
|
||||
BACKUP_PASSWORD="" \
|
||||
ARCHIVE_PREFIX="test" \
|
||||
NOTIFY_SUCCESS_URL="http://example.invalid/success" \
|
||||
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
|
||||
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
||||
SEVENZ_BIN="${TMPD}/bin/7z" \
|
||||
SCALE_WAIT_SECONDS=0 \
|
||||
TMP_STATE_DIR="$state_dir" \
|
||||
PATH="${TMPD}/bin:${PATH}" \
|
||||
TMPD="$TMPD" \
|
||||
bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
|
||||
}
|
||||
|
||||
write_fake_binaries
|
||||
@@ -243,11 +274,13 @@ success_dir="${TMPD}/success"
|
||||
success_source="${success_dir}/source"
|
||||
success_output="${success_dir}/out"
|
||||
success_state="${success_dir}/state"
|
||||
mkdir -p "${success_source}/app-ns" "${success_source}/loose-data" "$success_output" "$success_state"
|
||||
mkdir -p "${success_source}/app-ns" "${success_source}/kube-system" "${success_source}/loose-data" "$success_output" "${success_state}/run-stale"
|
||||
printf 'data\n' > "${success_source}/app-ns/file.txt"
|
||||
printf 'data\n' > "${success_source}/kube-system/file.txt"
|
||||
printf 'data\n' > "${success_source}/loose-data/file.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
||||
printf 'deployment\tstale\t9\n' > "${success_state}/run-stale/stale-ns.state"
|
||||
printf 'app-ns\nkube-system\nstale-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
@@ -260,17 +293,44 @@ set -e
|
||||
|
||||
assert "success scenario exits with code 0" is_zero "$success_exit_code"
|
||||
assert "success scenario creates a test archive" archive_exists "$success_output"
|
||||
assert "success scenario reports two backup successes" file_contains "${success_dir}/success.log" "backup_successes=2"
|
||||
assert "success scenario reports three backup successes" file_contains "${success_dir}/success.log" "backup_successes=3"
|
||||
assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
|
||||
assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
|
||||
assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
|
||||
assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
|
||||
assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system"
|
||||
assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace"
|
||||
assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns"
|
||||
assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns"
|
||||
assert "success scenario invokes 7z for excluded kube-system" sevenz_called_for "kube-system"
|
||||
assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data"
|
||||
|
||||
# Scenario B: one namespace backup fails, but both namespaces are restored and
|
||||
# the script exits through its documented handled-failure path.
|
||||
# Scenario A2: the exclusion list can be replaced through the environment.
|
||||
custom_exclude_dir="${TMPD}/custom-exclude"
|
||||
custom_exclude_source="${custom_exclude_dir}/source"
|
||||
custom_exclude_output="${custom_exclude_dir}/out"
|
||||
custom_exclude_state="${custom_exclude_dir}/state"
|
||||
mkdir -p "${custom_exclude_source}/app-ns" "$custom_exclude_output" "$custom_exclude_state"
|
||||
printf 'data\n' > "${custom_exclude_source}/app-ns/file.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
|
||||
set +e
|
||||
run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns"
|
||||
custom_exclude_exit_code=$?
|
||||
set -e
|
||||
|
||||
assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_exit_code"
|
||||
assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
|
||||
assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
|
||||
|
||||
# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
|
||||
# are restored, retention still runs, and the script reports success.
|
||||
failure_dir="${TMPD}/failure"
|
||||
failure_source="${failure_dir}/source"
|
||||
failure_output="${failure_dir}/out"
|
||||
@@ -278,6 +338,9 @@ failure_state="${failure_dir}/state"
|
||||
mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state"
|
||||
printf 'data\n' > "${failure_source}/good-ns/file.txt"
|
||||
printf 'data\n' > "${failure_source}/fail-ns/file.txt"
|
||||
for i in 1 2 3 4; do
|
||||
touch -t "2020010${i}0000" "${failure_output}/test_2020010${i}_000000.7z"
|
||||
done
|
||||
printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
|
||||
printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
|
||||
@@ -290,18 +353,72 @@ run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/
|
||||
failure_exit_code=$?
|
||||
set -e
|
||||
|
||||
assert "failure scenario exits non-zero" is_nonzero "$failure_exit_code"
|
||||
assert "failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
|
||||
assert "failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
|
||||
assert "failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
|
||||
assert "failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
|
||||
assert "failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
|
||||
assert "failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
|
||||
assert "failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
||||
assert "failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
|
||||
assert "failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
|
||||
assert "partial failure scenario exits with code 0" is_zero "$failure_exit_code"
|
||||
assert "partial failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
|
||||
assert "partial failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
|
||||
assert "partial failure scenario logs that it is continuing" file_contains "${failure_dir}/failure.log" "Some folder backups failed; continuing"
|
||||
assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
|
||||
assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
|
||||
assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
|
||||
assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
|
||||
assert "partial failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
||||
assert "partial failure scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
|
||||
assert "partial failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
|
||||
assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4
|
||||
assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z"
|
||||
|
||||
# Scenario C: abnormal abort while a namespace is still scaled to zero.
|
||||
# Scenario C: every attempted folder backup fails, so the script must still
|
||||
# return non-zero and must not send a success notification.
|
||||
all_failed_dir="${TMPD}/all-failed"
|
||||
all_failed_source="${all_failed_dir}/source"
|
||||
all_failed_output="${all_failed_dir}/out"
|
||||
all_failed_state="${all_failed_dir}/state"
|
||||
mkdir -p "${all_failed_source}/only-ns" "$all_failed_output" "$all_failed_state"
|
||||
printf 'data\n' > "${all_failed_source}/only-ns/file.txt"
|
||||
printf 'only-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'only-ns:2\n' > "${TMPD}/replicas.txt"
|
||||
printf 'only-ns\n' > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
|
||||
set +e
|
||||
run_backup "$all_failed_source" "$all_failed_output" "$all_failed_state" "${all_failed_dir}/all-failed.log"
|
||||
all_failed_exit_code=$?
|
||||
set -e
|
||||
|
||||
assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code"
|
||||
assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0"
|
||||
assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1"
|
||||
assert "all-failed scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
||||
assert "all-failed scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
|
||||
# Scenario D: no source folders means zero successful backups, which must also
|
||||
# return non-zero rather than reporting an empty run as successful.
|
||||
empty_dir="${TMPD}/empty"
|
||||
empty_source="${empty_dir}/source"
|
||||
empty_output="${empty_dir}/out"
|
||||
empty_state="${empty_dir}/state"
|
||||
mkdir -p "$empty_source" "$empty_output" "$empty_state"
|
||||
: > "${TMPD}/namespaces.txt"
|
||||
: > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
|
||||
set +e
|
||||
run_backup "$empty_source" "$empty_output" "$empty_state" "${empty_dir}/empty.log"
|
||||
empty_exit_code=$?
|
||||
set -e
|
||||
|
||||
assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code"
|
||||
assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0"
|
||||
assert "empty scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
||||
assert "empty scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
|
||||
# Scenario E: abnormal abort while a namespace is still scaled to zero.
|
||||
# We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
|
||||
# errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its
|
||||
# .state file is written, but BEFORE the in-line restore. The EXIT trap must then
|
||||
@@ -341,7 +458,7 @@ set -e
|
||||
assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}"
|
||||
assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0"
|
||||
assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5"
|
||||
assert "abort scenario left no stale .state file" file_lacks "${abort_dir}/abort.log" ".state"
|
||||
assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state"
|
||||
|
||||
printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
|
||||
if [[ "$tests_failed" -gt 0 ]]; then
|
||||
|
||||
Reference in New Issue
Block a user