Jose Henrique
2bd9e90f00
- Add EXCLUDED_NAMESPACES (default kube-system,kube-public,kube-node-lease,default); excluded folders are backed up but never scaled - Use a per-run state dir via mktemp -d so stale .state files from an earlier run can't leak into restores - Partial folder-backup failure now exits 0 (only zero successes fails); retention runs even on partial failure - Remove dead SCALE_TIMEOUT_SECONDS config; document SCALE_WAIT_SECONDS - Add .gitattributes to keep *.sh at LF - Extend test harness to 44 passing assertions covering exclusions, per-run state, partial/all-failed/empty scenarios
467 lines
16 KiB
Bash
467 lines
16 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/automated-nfs-backup.sh"
|
|
TMPD="$(mktemp -d "${TMPDIR:-/tmp}/automated-nfs-backup-test.XXXXXX")"
|
|
|
|
tests_total=0
|
|
tests_passed=0
|
|
tests_failed=0
|
|
|
|
cleanup() {
|
|
rm -rf "${TMPD}"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
assert() {
|
|
local message="$1"
|
|
shift
|
|
|
|
tests_total=$((tests_total + 1))
|
|
if "$@"; then
|
|
printf 'PASS: %s\n' "$message"
|
|
tests_passed=$((tests_passed + 1))
|
|
else
|
|
printf 'FAIL: %s\n' "$message"
|
|
tests_failed=$((tests_failed + 1))
|
|
fi
|
|
}
|
|
|
|
is_zero() {
|
|
[[ "$1" -eq 0 ]]
|
|
}
|
|
|
|
is_nonzero() {
|
|
[[ "$1" -ne 0 ]]
|
|
}
|
|
|
|
file_contains() {
|
|
local file="$1"
|
|
local value="$2"
|
|
grep -Fq -- "$value" "$file"
|
|
}
|
|
|
|
file_lacks() {
|
|
local file="$1"
|
|
local value="$2"
|
|
! grep -Fq -- "$value" "$file"
|
|
}
|
|
|
|
file_matches() {
|
|
local file="$1"
|
|
local regex="$2"
|
|
grep -Eq -- "$regex" "$file"
|
|
}
|
|
|
|
archive_exists() {
|
|
local output_dir="$1"
|
|
local -a archives=()
|
|
|
|
shopt -s nullglob
|
|
archives=("${output_dir}"/test_*.7z)
|
|
shopt -u nullglob
|
|
[[ "${#archives[@]}" -gt 0 ]]
|
|
}
|
|
|
|
archive_count_is() {
|
|
local output_dir="$1"
|
|
local expected_count="$2"
|
|
local -a archives=()
|
|
|
|
shopt -s nullglob
|
|
archives=("${output_dir}"/test_*.7z)
|
|
shopt -u nullglob
|
|
[[ "${#archives[@]}" -eq "$expected_count" ]]
|
|
}
|
|
|
|
path_does_not_exist() {
|
|
[[ ! -e "$1" ]]
|
|
}
|
|
|
|
directory_is_empty() {
|
|
local directory="$1"
|
|
local -a entries=()
|
|
|
|
shopt -s nullglob dotglob
|
|
entries=("${directory}"/*)
|
|
shopt -u nullglob dotglob
|
|
[[ "${#entries[@]}" -eq 0 ]]
|
|
}
|
|
|
|
trace_has_scale() {
|
|
local trace_file="$1"
|
|
local namespace="$2"
|
|
local replicas="$3"
|
|
|
|
grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file"
|
|
}
|
|
|
|
# Asserts the real script invoked 7z as `7z a -t7z <archive.7z> <folder> ...`
|
|
# for the given folder. The archive is the 3rd token (ends in .7z) and the
|
|
# folder path is the 4th token, so a swap would fail to match.
|
|
sevenz_called_for() {
|
|
local folder_name="$1"
|
|
|
|
grep -Eq -- "a -t7z [^ ]+\\.7z .*/${folder_name}( |$)" "${TMPD}/7z_invocations.log"
|
|
}
|
|
|
|
write_fake_script() {
|
|
local path="$1"
|
|
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
printf '%s\n' "$line"
|
|
done > "$path"
|
|
chmod +x "$path"
|
|
}
|
|
|
|
write_fake_binaries() {
|
|
mkdir -p "${TMPD}/bin"
|
|
|
|
write_fake_script "${TMPD}/bin/kubectl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
namespace=""
|
|
get_kind=""
|
|
has_output_format=0
|
|
|
|
for ((i = 1; i <= $#; i++)); do
|
|
arg="${!i}"
|
|
case "$arg" in
|
|
-n)
|
|
next=$((i + 1))
|
|
namespace="${!next}"
|
|
;;
|
|
get)
|
|
next=$((i + 1))
|
|
get_kind="${!next}"
|
|
;;
|
|
-o)
|
|
has_output_format=1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$get_kind" == "namespaces" ]]; then
|
|
if [[ "$has_output_format" -eq 1 ]]; then
|
|
while IFS= read -r configured_namespace; do
|
|
[[ -n "$configured_namespace" ]] && printf '%s\n' "$configured_namespace"
|
|
done < "${TMPD}/namespaces.txt"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -n "$get_kind" && -n "$namespace" ]]; then
|
|
replicas=""
|
|
while IFS=: read -r configured_namespace configured_replicas; do
|
|
if [[ "$configured_namespace" == "$namespace" ]]; then
|
|
replicas="$configured_replicas"
|
|
break
|
|
fi
|
|
done < "${TMPD}/replicas.txt"
|
|
|
|
[[ -n "$replicas" ]] || replicas=1
|
|
printf 'web\t%s\t\n' "$replicas"
|
|
exit 0
|
|
fi
|
|
|
|
for ((i = 1; i <= $#; i++)); do
|
|
if [[ "${!i}" == "scale" ]]; then
|
|
resource_index=$((i + 1))
|
|
resource="${!resource_index}"
|
|
kind="${resource%%/*}"
|
|
name="${resource#*/}"
|
|
replicas_arg=""
|
|
for arg in "$@"; do
|
|
[[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
|
|
done
|
|
printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
|
|
exit 0
|
|
fi
|
|
done
|
|
|
|
exit 0
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/7z" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# Record the full invocation so the harness can assert argument order.
|
|
printf '%s\n' "$*" >> "${TMPD}/7z_invocations.log"
|
|
|
|
if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
|
|
archive="${3:?archive path required}"
|
|
folder="${4:?folder path required}"
|
|
folder="${folder%/}"
|
|
folder_name="${folder##*/}"
|
|
|
|
# Simulate a backup failure for configured folders BEFORE recording a
|
|
# successful archive, so 7z_trace.log only lists folders actually archived.
|
|
if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
|
|
exit 2
|
|
fi
|
|
|
|
printf '%s\n' "$folder_name" >> "${TMPD}/7z_trace.log"
|
|
touch "$archive"
|
|
fi
|
|
|
|
exit 0
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/curl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
url=""
|
|
body=""
|
|
for ((i = 1; i <= $#; i++)); do
|
|
arg="${!i}"
|
|
case "$arg" in
|
|
POST)
|
|
next=$((i + 1))
|
|
url="${!next}"
|
|
;;
|
|
-d)
|
|
next=$((i + 1))
|
|
body="${!next}"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
body="${body//$'\n'/ }"
|
|
printf '%s %s\n' "$url" "${body:0:120}" >> "${TMPD}/curl_trace.log"
|
|
exit 0
|
|
EOF
|
|
}
|
|
|
|
run_backup() {
|
|
local source_dir="$1"
|
|
local output_dir="$2"
|
|
local state_dir="$3"
|
|
local log_file="$4"
|
|
local -a env_overrides=()
|
|
|
|
if (( $# >= 5 )); then
|
|
env_overrides+=("EXCLUDED_NAMESPACES=$5")
|
|
fi
|
|
|
|
env "${env_overrides[@]}" \
|
|
NFS_SOURCE_PATH="$source_dir" \
|
|
BACKUP_OUTPUT_PATH="$output_dir" \
|
|
BACKUP_PASSWORD="" \
|
|
ARCHIVE_PREFIX="test" \
|
|
NOTIFY_SUCCESS_URL="http://example.invalid/success" \
|
|
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
|
|
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
|
SEVENZ_BIN="${TMPD}/bin/7z" \
|
|
SCALE_WAIT_SECONDS=0 \
|
|
TMP_STATE_DIR="$state_dir" \
|
|
PATH="${TMPD}/bin:${PATH}" \
|
|
TMPD="$TMPD" \
|
|
bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
|
|
}
|
|
|
|
write_fake_binaries
|
|
export TMPD
|
|
|
|
# Scenario A: a mapped folder is scaled down and restored, while an unmapped
|
|
# folder is backed up directly. Both folders share one archive.
|
|
success_dir="${TMPD}/success"
|
|
success_source="${success_dir}/source"
|
|
success_output="${success_dir}/out"
|
|
success_state="${success_dir}/state"
|
|
mkdir -p "${success_source}/app-ns" "${success_source}/kube-system" "${success_source}/loose-data" "$success_output" "${success_state}/run-stale"
|
|
printf 'data\n' > "${success_source}/app-ns/file.txt"
|
|
printf 'data\n' > "${success_source}/kube-system/file.txt"
|
|
printf 'data\n' > "${success_source}/loose-data/file.txt"
|
|
printf 'deployment\tstale\t9\n' > "${success_state}/run-stale/stale-ns.state"
|
|
printf 'app-ns\nkube-system\nstale-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log"
|
|
success_exit_code=$?
|
|
set -e
|
|
|
|
assert "success scenario exits with code 0" is_zero "$success_exit_code"
|
|
assert "success scenario creates a test archive" archive_exists "$success_output"
|
|
assert "success scenario reports three backup successes" file_contains "${success_dir}/success.log" "backup_successes=3"
|
|
assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
|
|
assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
|
|
assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
|
|
assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
|
assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
|
|
assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system"
|
|
assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace"
|
|
assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns"
|
|
assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns"
|
|
assert "success scenario invokes 7z for excluded kube-system" sevenz_called_for "kube-system"
|
|
assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data"
|
|
|
|
# Scenario A2: the exclusion list can be replaced through the environment.
|
|
custom_exclude_dir="${TMPD}/custom-exclude"
|
|
custom_exclude_source="${custom_exclude_dir}/source"
|
|
custom_exclude_output="${custom_exclude_dir}/out"
|
|
custom_exclude_state="${custom_exclude_dir}/state"
|
|
mkdir -p "${custom_exclude_source}/app-ns" "$custom_exclude_output" "$custom_exclude_state"
|
|
printf 'data\n' > "${custom_exclude_source}/app-ns/file.txt"
|
|
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns"
|
|
custom_exclude_exit_code=$?
|
|
set -e
|
|
|
|
assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_exit_code"
|
|
assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
|
|
assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
|
|
|
|
# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
|
|
# are restored, retention still runs, and the script reports success.
|
|
failure_dir="${TMPD}/failure"
|
|
failure_source="${failure_dir}/source"
|
|
failure_output="${failure_dir}/out"
|
|
failure_state="${failure_dir}/state"
|
|
mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state"
|
|
printf 'data\n' > "${failure_source}/good-ns/file.txt"
|
|
printf 'data\n' > "${failure_source}/fail-ns/file.txt"
|
|
for i in 1 2 3 4; do
|
|
touch -t "2020010${i}0000" "${failure_output}/test_2020010${i}_000000.7z"
|
|
done
|
|
printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
|
|
printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log"
|
|
failure_exit_code=$?
|
|
set -e
|
|
|
|
assert "partial failure scenario exits with code 0" is_zero "$failure_exit_code"
|
|
assert "partial failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
|
|
assert "partial failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
|
|
assert "partial failure scenario logs that it is continuing" file_contains "${failure_dir}/failure.log" "Some folder backups failed; continuing"
|
|
assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
|
|
assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
|
|
assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
|
|
assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
|
|
assert "partial failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
|
assert "partial failure scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
|
assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
|
|
assert "partial failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
|
|
assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4
|
|
assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z"
|
|
|
|
# Scenario C: every attempted folder backup fails, so the script must still
|
|
# return non-zero and must not send a success notification.
|
|
all_failed_dir="${TMPD}/all-failed"
|
|
all_failed_source="${all_failed_dir}/source"
|
|
all_failed_output="${all_failed_dir}/out"
|
|
all_failed_state="${all_failed_dir}/state"
|
|
mkdir -p "${all_failed_source}/only-ns" "$all_failed_output" "$all_failed_state"
|
|
printf 'data\n' > "${all_failed_source}/only-ns/file.txt"
|
|
printf 'only-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'only-ns:2\n' > "${TMPD}/replicas.txt"
|
|
printf 'only-ns\n' > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$all_failed_source" "$all_failed_output" "$all_failed_state" "${all_failed_dir}/all-failed.log"
|
|
all_failed_exit_code=$?
|
|
set -e
|
|
|
|
assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code"
|
|
assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0"
|
|
assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1"
|
|
assert "all-failed scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
|
assert "all-failed scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
|
|
|
# Scenario D: no source folders means zero successful backups, which must also
|
|
# return non-zero rather than reporting an empty run as successful.
|
|
empty_dir="${TMPD}/empty"
|
|
empty_source="${empty_dir}/source"
|
|
empty_output="${empty_dir}/out"
|
|
empty_state="${empty_dir}/state"
|
|
mkdir -p "$empty_source" "$empty_output" "$empty_state"
|
|
: > "${TMPD}/namespaces.txt"
|
|
: > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$empty_source" "$empty_output" "$empty_state" "${empty_dir}/empty.log"
|
|
empty_exit_code=$?
|
|
set -e
|
|
|
|
assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code"
|
|
assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0"
|
|
assert "empty scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
|
assert "empty scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
|
|
|
# Scenario E: abnormal abort while a namespace is still scaled to zero.
|
|
# We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
|
|
# errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its
|
|
# .state file is written, but BEFORE the in-line restore. The EXIT trap must then
|
|
# drive restore_all_remaining() and scale the namespace back up. This exercises
|
|
# the real safety-net path that scenario B leaves unproven.
|
|
abort_dir="${TMPD}/abort"
|
|
abort_source="${abort_dir}/source"
|
|
abort_output="${abort_dir}/out"
|
|
abort_state="${abort_dir}/state"
|
|
mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}"
|
|
printf 'data\n' > "${abort_source}/crash-ns/file.txt"
|
|
printf 'crash-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'crash-ns:5\n' > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
: > "${TMPD}/7z_invocations.log"
|
|
|
|
set +e
|
|
NFS_SOURCE_PATH="${abort_source}" \
|
|
BACKUP_OUTPUT_PATH="${abort_output}" \
|
|
BACKUP_PASSWORD="" \
|
|
ARCHIVE_PREFIX="test" \
|
|
NOTIFY_SUCCESS_URL="" \
|
|
NOTIFY_FAILURE_URL="" \
|
|
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
|
SEVENZ_BIN="${TMPD}/bin/7z" \
|
|
SCALE_WAIT_SECONDS="not-a-number" \
|
|
TMP_STATE_DIR="${abort_state}" \
|
|
PATH="${TMPD}/bin:${PATH}" \
|
|
TMPD="${TMPD}" \
|
|
bash "${SCRIPT_UNDER_TEST}" >"${abort_dir}/abort.log" 2>&1
|
|
abort_exit_code=$?
|
|
set -e
|
|
|
|
assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}"
|
|
assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0"
|
|
assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5"
|
|
assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state"
|
|
|
|
printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
|
|
if [[ "$tests_failed" -gt 0 ]]; then
|
|
exit 1
|
|
fi
|