620 lines
24 KiB
Bash
620 lines
24 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/automated-nfs-backup.sh"
|
|
TMPD="$(mktemp -d "${TMPDIR:-/tmp}/automated-nfs-backup-test.XXXXXX")"
|
|
|
|
tests_total=0
|
|
tests_passed=0
|
|
tests_failed=0
|
|
|
|
cleanup() {
|
|
rm -rf "${TMPD}"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
assert() {
|
|
local message="$1"
|
|
shift
|
|
|
|
tests_total=$((tests_total + 1))
|
|
if "$@"; then
|
|
printf 'PASS: %s\n' "$message"
|
|
tests_passed=$((tests_passed + 1))
|
|
else
|
|
printf 'FAIL: %s\n' "$message"
|
|
tests_failed=$((tests_failed + 1))
|
|
fi
|
|
}
|
|
|
|
is_zero() {
|
|
[[ "$1" -eq 0 ]]
|
|
}
|
|
|
|
is_nonzero() {
|
|
[[ "$1" -ne 0 ]]
|
|
}
|
|
|
|
file_contains() {
|
|
local file="$1"
|
|
local value="$2"
|
|
grep -Fq -- "$value" "$file"
|
|
}
|
|
|
|
file_lacks() {
|
|
local file="$1"
|
|
local value="$2"
|
|
! grep -Fq -- "$value" "$file"
|
|
}
|
|
|
|
file_matches() {
|
|
local file="$1"
|
|
local regex="$2"
|
|
grep -Eq -- "$regex" "$file"
|
|
}
|
|
|
|
archive_exists() {
|
|
local output_dir="$1"
|
|
local -a archives=()
|
|
|
|
shopt -s nullglob
|
|
archives=("${output_dir}"/test_*.7z)
|
|
shopt -u nullglob
|
|
[[ "${#archives[@]}" -gt 0 ]]
|
|
}
|
|
|
|
archive_count_is() {
|
|
local output_dir="$1"
|
|
local expected_count="$2"
|
|
local -a archives=()
|
|
|
|
shopt -s nullglob
|
|
archives=("${output_dir}"/test_*.7z)
|
|
shopt -u nullglob
|
|
[[ "${#archives[@]}" -eq "$expected_count" ]]
|
|
}
|
|
|
|
path_does_not_exist() {
|
|
[[ ! -e "$1" ]]
|
|
}
|
|
|
|
directory_is_empty() {
|
|
local directory="$1"
|
|
local -a entries=()
|
|
|
|
shopt -s nullglob dotglob
|
|
entries=("${directory}"/*)
|
|
shopt -u nullglob dotglob
|
|
[[ "${#entries[@]}" -eq 0 ]]
|
|
}
|
|
|
|
events_are_ordered() {
|
|
local trace_file="$1"
|
|
shift
|
|
local event
|
|
local line_number
|
|
local previous_line=0
|
|
|
|
for event in "$@"; do
|
|
line_number="$(grep -n -m1 -F -- "$event" "$trace_file" | cut -d: -f1)"
|
|
[[ -n "$line_number" ]] || return 1
|
|
(( line_number > previous_line )) || return 1
|
|
previous_line="$line_number"
|
|
done
|
|
}
|
|
|
|
trace_has_scale() {
|
|
local trace_file="$1"
|
|
local namespace="$2"
|
|
local replicas="$3"
|
|
|
|
grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file"
|
|
}
|
|
|
|
# Asserts the real script invoked 7z exactly once for the complete contents of
|
|
# the run staging directory.
|
|
sevenz_called_once_for_staging() {
|
|
[[ "$(wc -l < "${TMPD}/7z_invocations.log" | tr -d '[:space:]')" -eq 1 ]] &&
|
|
grep -Eq -- '^a -t7z [^ ]+\.7z \. ' "${TMPD}/7z_invocations.log"
|
|
}
|
|
|
|
write_fake_script() {
|
|
local path="$1"
|
|
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
printf '%s\n' "$line"
|
|
done > "$path"
|
|
chmod +x "$path"
|
|
}
|
|
|
|
write_fake_binaries() {
|
|
mkdir -p "${TMPD}/bin"
|
|
|
|
write_fake_script "${TMPD}/bin/kubectl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
namespace=""
|
|
get_kind=""
|
|
has_output_format=0
|
|
|
|
for ((i = 1; i <= $#; i++)); do
|
|
arg="${!i}"
|
|
case "$arg" in
|
|
-n)
|
|
next=$((i + 1))
|
|
namespace="${!next}"
|
|
;;
|
|
get)
|
|
next=$((i + 1))
|
|
get_kind="${!next}"
|
|
;;
|
|
-o)
|
|
has_output_format=1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$get_kind" == "namespaces" ]]; then
|
|
if [[ "$has_output_format" -eq 1 ]]; then
|
|
while IFS= read -r configured_namespace; do
|
|
[[ -n "$configured_namespace" ]] && printf '%s\n' "$configured_namespace"
|
|
done < "${TMPD}/namespaces.txt"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -n "$get_kind" && -n "$namespace" ]]; then
|
|
replicas=""
|
|
while IFS=: read -r configured_namespace configured_replicas; do
|
|
if [[ "$configured_namespace" == "$namespace" ]]; then
|
|
replicas="$configured_replicas"
|
|
break
|
|
fi
|
|
done < "${TMPD}/replicas.txt"
|
|
|
|
[[ -n "$replicas" ]] || replicas=1
|
|
printf 'web\t%s\t\n' "$replicas"
|
|
exit 0
|
|
fi
|
|
|
|
for ((i = 1; i <= $#; i++)); do
|
|
if [[ "${!i}" == "scale" ]]; then
|
|
resource_index=$((i + 1))
|
|
resource="${!resource_index}"
|
|
kind="${resource%%/*}"
|
|
name="${resource#*/}"
|
|
replicas_arg=""
|
|
for arg in "$@"; do
|
|
[[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
|
|
done
|
|
printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
|
|
printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log"
|
|
exit 0
|
|
fi
|
|
done
|
|
|
|
exit 0
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/7z" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# Record the full invocation so the harness can assert argument order.
|
|
printf '%s\n' "$*" >> "${TMPD}/7z_invocations.log"
|
|
|
|
if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
|
|
archive="${3:?archive path required}"
|
|
folder="${4:?folder path required}"
|
|
folder="${folder%/}"
|
|
printf '7z staging\n' >> "${TMPD}/event_trace.log"
|
|
|
|
if [[ -s "${TMPD}/fail_archive.txt" ]]; then
|
|
touch "$archive"
|
|
exit 2
|
|
fi
|
|
|
|
for staged_folder in ./*; do
|
|
[[ -d "$staged_folder" ]] || continue
|
|
printf '%s\n' "${staged_folder##*/}" >> "${TMPD}/7z_trace.log"
|
|
done
|
|
touch "$archive"
|
|
fi
|
|
|
|
exit 0
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/rsync" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
source_folder="${@: -2:1}"
|
|
staging_folder="${@: -1}"
|
|
source_folder="${source_folder%/}"
|
|
staging_folder="${staging_folder%/}"
|
|
folder_name="${source_folder##*/}"
|
|
|
|
mkdir -p "${TMPD}/rsync_counts"
|
|
count_file="${TMPD}/rsync_counts/${folder_name}"
|
|
count=0
|
|
if [[ -f "$count_file" ]]; then
|
|
count="$(<"$count_file")"
|
|
fi
|
|
count=$((count + 1))
|
|
printf '%s\n' "$count" > "$count_file"
|
|
printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
|
|
|
|
if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
|
|
printf 'simulated rsync failure for %s\n' "$folder_name" >&2
|
|
exit 23
|
|
fi
|
|
|
|
mkdir -p "$staging_folder"
|
|
cp -a "${source_folder}/." "$staging_folder/"
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/curl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
url=""
|
|
body=""
|
|
for ((i = 1; i <= $#; i++)); do
|
|
arg="${!i}"
|
|
case "$arg" in
|
|
POST)
|
|
next=$((i + 1))
|
|
url="${!next}"
|
|
;;
|
|
-d)
|
|
next=$((i + 1))
|
|
body="${!next}"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
body="${body//$'\n'/ }"
|
|
printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
|
|
exit 0
|
|
EOF
|
|
}
|
|
|
|
run_backup() {
|
|
local source_dir="$1"
|
|
local output_dir="$2"
|
|
local state_dir="$3"
|
|
local log_file="$4"
|
|
shift 4
|
|
local -a env_overrides=("$@")
|
|
|
|
env BACKUP_STAGING_PATH="${state_dir}/staging" \
|
|
"${env_overrides[@]}" \
|
|
NFS_SOURCE_PATH="$source_dir" \
|
|
BACKUP_OUTPUT_PATH="$output_dir" \
|
|
BACKUP_PASSWORD="" \
|
|
ARCHIVE_PREFIX="test" \
|
|
NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
|
|
NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
|
|
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
|
SEVENZ_BIN="${TMPD}/bin/7z" \
|
|
RSYNC_BIN="${TMPD}/bin/rsync" \
|
|
SCALE_WAIT_SECONDS=0 \
|
|
TMP_STATE_DIR="$state_dir" \
|
|
PATH="${TMPD}/bin:${PATH}" \
|
|
TMPD="$TMPD" \
|
|
bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
|
|
}
|
|
|
|
write_fake_binaries
|
|
export TMPD
|
|
: > "${TMPD}/fail_archive.txt"
|
|
|
|
# Scenario A: a mapped folder is scaled down, copied, and restored, while
|
|
# unmapped and excluded folders are copied live. One 7z call archives them all.
|
|
success_dir="${TMPD}/success"
|
|
success_source="${success_dir}/source"
|
|
success_output="${success_dir}/out"
|
|
success_state="${success_dir}/state"
|
|
mkdir -p "${success_source}/app-ns" "${success_source}/kube-system" "${success_source}/loose-data" "$success_output" "${success_state}/run-stale"
|
|
printf 'data\n' > "${success_source}/app-ns/file.txt"
|
|
printf 'data\n' > "${success_source}/kube-system/file.txt"
|
|
printf 'data\n' > "${success_source}/loose-data/file.txt"
|
|
printf 'deployment\tstale\t9\n' > "${success_state}/run-stale/stale-ns.state"
|
|
printf 'app-ns\nkube-system\nstale-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/7z_invocations.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
: > "${TMPD}/event_trace.log"
|
|
|
|
set +e
|
|
run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log"
|
|
success_exit_code=$?
|
|
set -e
|
|
|
|
assert "success scenario exits with code 0" is_zero "$success_exit_code"
|
|
assert "success scenario creates a test archive" archive_exists "$success_output"
|
|
assert "success scenario reports three backup successes" file_contains "${success_dir}/success.log" "backup_successes=3"
|
|
assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
|
|
assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
|
|
assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
|
|
assert "success scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
|
assert "success scenario sends the archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes": 0'
|
|
assert "success scenario sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "Kubernetes NFS backup"'
|
|
assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
|
|
assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system"
|
|
assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace"
|
|
assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns"
|
|
assert "success scenario invokes 7z exactly once for staging" sevenz_called_once_for_staging
|
|
assert "success scenario archives mapped app-ns" file_contains "${TMPD}/7z_trace.log" "app-ns"
|
|
assert "success scenario archives excluded kube-system" file_contains "${TMPD}/7z_trace.log" "kube-system"
|
|
assert "success scenario archives unmapped loose-data" file_contains "${TMPD}/7z_trace.log" "loose-data"
|
|
|
|
# Scenario A2: the exclusion list can be replaced through the environment.
|
|
custom_exclude_dir="${TMPD}/custom-exclude"
|
|
custom_exclude_source="${custom_exclude_dir}/source"
|
|
custom_exclude_output="${custom_exclude_dir}/out"
|
|
custom_exclude_state="${custom_exclude_dir}/state"
|
|
mkdir -p "${custom_exclude_source}/app-ns" "$custom_exclude_output" "$custom_exclude_state"
|
|
printf 'data\n' > "${custom_exclude_source}/app-ns/file.txt"
|
|
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/7z_invocations.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "EXCLUDED_NAMESPACES=app-ns"
|
|
custom_exclude_exit_code=$?
|
|
set -e
|
|
|
|
assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_exit_code"
|
|
assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
|
|
assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
|
|
|
|
# Scenario A3: the mapped namespace is scaled down before its single local copy,
|
|
# restored immediately afterward, and compression starts only after restoration.
|
|
staging_dir="${TMPD}/staging"
|
|
staging_source="${staging_dir}/source"
|
|
staging_output="${staging_dir}/out"
|
|
staging_state="${staging_dir}/state"
|
|
staging_work="${staging_dir}/work"
|
|
mkdir -p "${staging_source}/app-ns" "$staging_output" "$staging_state" "$staging_work"
|
|
printf 'data\n' > "${staging_source}/app-ns/file.txt"
|
|
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/7z_invocations.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
: > "${TMPD}/event_trace.log"
|
|
rm -rf "${TMPD}/rsync_counts"
|
|
|
|
set +e
|
|
run_backup "$staging_source" "$staging_output" "$staging_state" "${staging_dir}/staging.log" "BACKUP_STAGING_PATH=${staging_work}"
|
|
staging_exit_code=$?
|
|
set -e
|
|
|
|
assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
|
|
assert "staging scenario creates an archive" archive_exists "$staging_output"
|
|
assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
|
|
"scale app-ns --replicas=0" \
|
|
"rsync app-ns 1" \
|
|
"scale app-ns --replicas=3" \
|
|
"7z staging"
|
|
assert "staging scenario copies app-ns only once" file_contains "${TMPD}/rsync_counts/app-ns" "1"
|
|
assert "staging scenario invokes 7z exactly once" sevenz_called_once_for_staging
|
|
assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
|
|
assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
|
|
|
|
# Scenario A4: a failed local copy never archives the incomplete staging copy,
|
|
# but the namespace is still restored immediately.
|
|
rsync_failure_dir="${TMPD}/rsync-failure"
|
|
rsync_failure_source="${rsync_failure_dir}/source"
|
|
rsync_failure_output="${rsync_failure_dir}/out"
|
|
rsync_failure_state="${rsync_failure_dir}/state"
|
|
rsync_failure_work="${rsync_failure_dir}/work"
|
|
mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failure_state" "$rsync_failure_work"
|
|
printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
|
|
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
|
printf 'app-ns\n' > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
: > "${TMPD}/event_trace.log"
|
|
rm -rf "${TMPD}/rsync_counts"
|
|
|
|
set +e
|
|
run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state" "${rsync_failure_dir}/rsync-failure.log" "BACKUP_STAGING_PATH=${rsync_failure_work}"
|
|
rsync_failure_exit_code=$?
|
|
set -e
|
|
|
|
assert "rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
|
|
assert "rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
|
|
assert "rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z staging"
|
|
assert "rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
|
|
|
|
# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
|
|
# are restored, retention still runs, and the script reports success.
|
|
failure_dir="${TMPD}/failure"
|
|
failure_source="${failure_dir}/source"
|
|
failure_output="${failure_dir}/out"
|
|
failure_state="${failure_dir}/state"
|
|
mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state"
|
|
printf 'data\n' > "${failure_source}/good-ns/file.txt"
|
|
printf 'data\n' > "${failure_source}/fail-ns/file.txt"
|
|
for i in 1 2 3 4; do
|
|
touch -t "2020010${i}0000" "${failure_output}/test_2020010${i}_000000.7z"
|
|
done
|
|
printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
|
|
printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/7z_invocations.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
: > "${TMPD}/event_trace.log"
|
|
|
|
set +e
|
|
run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log"
|
|
failure_exit_code=$?
|
|
set -e
|
|
|
|
assert "partial failure scenario exits with code 0" is_zero "$failure_exit_code"
|
|
assert "partial failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
|
|
assert "partial failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
|
|
assert "partial failure scenario logs that it is continuing" file_contains "${failure_dir}/failure.log" "Some folder backups failed; continuing"
|
|
assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
|
|
assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
|
|
assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
|
|
assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
|
|
assert "partial failure scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
|
|
assert "partial failure scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
|
assert "partial failure scenario marks the error as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
|
|
assert "partial failure scenario includes the backup error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "BACKUP_FAILED"'
|
|
assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
|
|
assert "partial failure scenario archives the successfully staged folder" file_contains "${TMPD}/7z_trace.log" "good-ns"
|
|
assert "partial failure scenario invokes 7z exactly once" sevenz_called_once_for_staging
|
|
assert "partial failure scenario compresses after all namespaces are restored" events_are_ordered "${TMPD}/event_trace.log" \
|
|
"scale fail-ns --replicas=4" \
|
|
"scale good-ns --replicas=2" \
|
|
"7z staging"
|
|
assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4
|
|
assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z"
|
|
|
|
# Scenario C: every attempted folder backup fails, so the script must still
|
|
# return non-zero and must not send a success notification.
|
|
all_failed_dir="${TMPD}/all-failed"
|
|
all_failed_source="${all_failed_dir}/source"
|
|
all_failed_output="${all_failed_dir}/out"
|
|
all_failed_state="${all_failed_dir}/state"
|
|
mkdir -p "${all_failed_source}/only-ns" "$all_failed_output" "$all_failed_state"
|
|
printf 'data\n' > "${all_failed_source}/only-ns/file.txt"
|
|
printf 'only-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'only-ns:2\n' > "${TMPD}/replicas.txt"
|
|
printf 'only-ns\n' > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$all_failed_source" "$all_failed_output" "$all_failed_state" "${all_failed_dir}/all-failed.log"
|
|
all_failed_exit_code=$?
|
|
set -e
|
|
|
|
assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code"
|
|
assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0"
|
|
assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1"
|
|
assert "all-failed scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
|
|
assert "all-failed scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
|
|
|
# Scenario C2: a failed single archive operation invalidates all staged folders
|
|
# and removes the partial archive left by 7z.
|
|
archive_failure_dir="${TMPD}/archive-failure"
|
|
archive_failure_source="${archive_failure_dir}/source"
|
|
archive_failure_output="${archive_failure_dir}/out"
|
|
archive_failure_state="${archive_failure_dir}/state"
|
|
mkdir -p "${archive_failure_source}/archive-ns" "$archive_failure_output" "$archive_failure_state"
|
|
printf 'data\n' > "${archive_failure_source}/archive-ns/file.txt"
|
|
printf 'archive-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'archive-ns:2\n' > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
printf 'fail\n' > "${TMPD}/fail_archive.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$archive_failure_source" "$archive_failure_output" "$archive_failure_state" "${archive_failure_dir}/archive-failure.log"
|
|
archive_failure_exit_code=$?
|
|
set -e
|
|
|
|
assert "archive failure exits non-zero" is_nonzero "$archive_failure_exit_code"
|
|
assert "archive failure restores archive-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "archive-ns" "2"
|
|
assert "archive failure reports zero backup successes" file_contains "${archive_failure_dir}/archive-failure.log" "backup_successes=0"
|
|
assert "archive failure removes the partial archive" archive_count_is "$archive_failure_output" 0
|
|
assert "archive failure does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
|
: > "${TMPD}/fail_archive.txt"
|
|
|
|
# Scenario D: no source folders means zero successful backups, which must also
|
|
# return non-zero rather than reporting an empty run as successful.
|
|
empty_dir="${TMPD}/empty"
|
|
empty_source="${empty_dir}/source"
|
|
empty_output="${empty_dir}/out"
|
|
empty_state="${empty_dir}/state"
|
|
mkdir -p "$empty_source" "$empty_output" "$empty_state"
|
|
: > "${TMPD}/namespaces.txt"
|
|
: > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$empty_source" "$empty_output" "$empty_state" "${empty_dir}/empty.log"
|
|
empty_exit_code=$?
|
|
set -e
|
|
|
|
assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code"
|
|
assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0"
|
|
assert "empty scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
|
|
assert "empty scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
|
|
|
# Scenario E: abnormal abort while a namespace is still scaled to zero.
|
|
# We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
|
|
# errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its
|
|
# .state file is written, but BEFORE the in-line restore. The EXIT trap must then
|
|
# drive restore_all_remaining() and scale the namespace back up. This exercises
|
|
# the real safety-net path that scenario B leaves unproven.
|
|
abort_dir="${TMPD}/abort"
|
|
abort_source="${abort_dir}/source"
|
|
abort_output="${abort_dir}/out"
|
|
abort_state="${abort_dir}/state"
|
|
abort_work="${abort_dir}/work"
|
|
mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" "${abort_work}"
|
|
printf 'data\n' > "${abort_source}/crash-ns/file.txt"
|
|
printf 'crash-ns\n' > "${TMPD}/namespaces.txt"
|
|
printf 'crash-ns:5\n' > "${TMPD}/replicas.txt"
|
|
: > "${TMPD}/fail_folders.txt"
|
|
: > "${TMPD}/scale_trace.log"
|
|
: > "${TMPD}/7z_trace.log"
|
|
: > "${TMPD}/curl_trace.log"
|
|
: > "${TMPD}/7z_invocations.log"
|
|
|
|
set +e
|
|
NFS_SOURCE_PATH="${abort_source}" \
|
|
BACKUP_OUTPUT_PATH="${abort_output}" \
|
|
BACKUP_PASSWORD="" \
|
|
ARCHIVE_PREFIX="test" \
|
|
NOTIFY_SUCCESS_URL="" \
|
|
NOTIFY_FAILURE_URL="" \
|
|
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
|
SEVENZ_BIN="${TMPD}/bin/7z" \
|
|
BACKUP_STAGING_PATH="${abort_work}" \
|
|
SCALE_WAIT_SECONDS="not-a-number" \
|
|
TMP_STATE_DIR="${abort_state}" \
|
|
PATH="${TMPD}/bin:${PATH}" \
|
|
TMPD="${TMPD}" \
|
|
bash "${SCRIPT_UNDER_TEST}" >"${abort_dir}/abort.log" 2>&1
|
|
abort_exit_code=$?
|
|
set -e
|
|
|
|
assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}"
|
|
assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0"
|
|
assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5"
|
|
assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state"
|
|
|
|
printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
|
|
if [[ "$tests_failed" -gt 0 ]]; then
|
|
exit 1
|
|
fi
|