updating flow and notify endpoint
This commit is contained in:
@@ -114,13 +114,11 @@ trace_has_scale() {
|
||||
grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file"
|
||||
}
|
||||
|
||||
# Asserts the real script invoked 7z as `7z a -t7z <archive.7z> <folder> ...`
|
||||
# for the given folder. The archive is the 3rd token (ends in .7z) and the
|
||||
# folder path is the 4th token, so a swap would fail to match.
|
||||
sevenz_called_for() {
|
||||
local folder_name="$1"
|
||||
|
||||
grep -Eq -- "a -t7z [^ ]+\\.7z .*/${folder_name}( |$)" "${TMPD}/7z_invocations.log"
|
||||
# Asserts the real script invoked 7z exactly once for the complete contents of
|
||||
# the run staging directory.
|
||||
sevenz_called_once_for_staging() {
|
||||
[[ "$(wc -l < "${TMPD}/7z_invocations.log" | tr -d '[:space:]')" -eq 1 ]] &&
|
||||
grep -Eq -- '^a -t7z [^ ]+\.7z \. ' "${TMPD}/7z_invocations.log"
|
||||
}
|
||||
|
||||
write_fake_script() {
|
||||
@@ -213,16 +211,17 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
|
||||
archive="${3:?archive path required}"
|
||||
folder="${4:?folder path required}"
|
||||
folder="${folder%/}"
|
||||
folder_name="${folder##*/}"
|
||||
printf '7z %s\n' "$folder_name" >> "${TMPD}/event_trace.log"
|
||||
printf '7z staging\n' >> "${TMPD}/event_trace.log"
|
||||
|
||||
# Simulate a backup failure for configured folders BEFORE recording a
|
||||
# successful archive, so 7z_trace.log only lists folders actually archived.
|
||||
if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
|
||||
if [[ -s "${TMPD}/fail_archive.txt" ]]; then
|
||||
touch "$archive"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
printf '%s\n' "$folder_name" >> "${TMPD}/7z_trace.log"
|
||||
for staged_folder in ./*; do
|
||||
[[ -d "$staged_folder" ]] || continue
|
||||
printf '%s\n' "${staged_folder##*/}" >> "${TMPD}/7z_trace.log"
|
||||
done
|
||||
touch "$archive"
|
||||
fi
|
||||
|
||||
@@ -249,8 +248,8 @@ count=$((count + 1))
|
||||
printf '%s\n' "$count" > "$count_file"
|
||||
printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
|
||||
|
||||
if (( count == 2 )) && [[ -f "${TMPD}/fail_rsync_second_pass.txt" ]] && grep -Fxq -- "$folder_name" "${TMPD}/fail_rsync_second_pass.txt"; then
|
||||
printf 'simulated final rsync failure for %s\n' "$folder_name" >&2
|
||||
if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
|
||||
printf 'simulated rsync failure for %s\n' "$folder_name" >&2
|
||||
exit 23
|
||||
fi
|
||||
|
||||
@@ -279,7 +278,7 @@ for ((i = 1; i <= $#; i++)); do
|
||||
done
|
||||
|
||||
body="${body//$'\n'/ }"
|
||||
printf '%s %s\n' "$url" "${body:0:120}" >> "${TMPD}/curl_trace.log"
|
||||
printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
|
||||
exit 0
|
||||
EOF
|
||||
}
|
||||
@@ -292,13 +291,14 @@ run_backup() {
|
||||
shift 4
|
||||
local -a env_overrides=("$@")
|
||||
|
||||
env "${env_overrides[@]}" \
|
||||
env BACKUP_STAGING_PATH="${state_dir}/staging" \
|
||||
"${env_overrides[@]}" \
|
||||
NFS_SOURCE_PATH="$source_dir" \
|
||||
BACKUP_OUTPUT_PATH="$output_dir" \
|
||||
BACKUP_PASSWORD="" \
|
||||
ARCHIVE_PREFIX="test" \
|
||||
NOTIFY_SUCCESS_URL="http://example.invalid/success" \
|
||||
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
|
||||
NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
|
||||
NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
|
||||
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
||||
SEVENZ_BIN="${TMPD}/bin/7z" \
|
||||
RSYNC_BIN="${TMPD}/bin/rsync" \
|
||||
@@ -311,9 +311,10 @@ run_backup() {
|
||||
|
||||
write_fake_binaries
|
||||
export TMPD
|
||||
: > "${TMPD}/fail_archive.txt"
|
||||
|
||||
# Scenario A: a mapped folder is scaled down and restored, while an unmapped
|
||||
# folder is backed up directly. Both folders share one archive.
|
||||
# Scenario A: a mapped folder is scaled down, copied, and restored, while
|
||||
# unmapped and excluded folders are copied live. One 7z call archives them all.
|
||||
success_dir="${TMPD}/success"
|
||||
success_source="${success_dir}/source"
|
||||
success_output="${success_dir}/out"
|
||||
@@ -330,6 +331,7 @@ printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/7z_invocations.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
: > "${TMPD}/event_trace.log"
|
||||
|
||||
set +e
|
||||
run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log"
|
||||
@@ -342,14 +344,17 @@ assert "success scenario reports three backup successes" file_contains "${succes
|
||||
assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
|
||||
assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
|
||||
assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
|
||||
assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
assert "success scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||
assert "success scenario sends the archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes": 0'
|
||||
assert "success scenario sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "Kubernetes NFS backup"'
|
||||
assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
|
||||
assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system"
|
||||
assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace"
|
||||
assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns"
|
||||
assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns"
|
||||
assert "success scenario invokes 7z for excluded kube-system" sevenz_called_for "kube-system"
|
||||
assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data"
|
||||
assert "success scenario invokes 7z exactly once for staging" sevenz_called_once_for_staging
|
||||
assert "success scenario archives mapped app-ns" file_contains "${TMPD}/7z_trace.log" "app-ns"
|
||||
assert "success scenario archives excluded kube-system" file_contains "${TMPD}/7z_trace.log" "kube-system"
|
||||
assert "success scenario archives unmapped loose-data" file_contains "${TMPD}/7z_trace.log" "loose-data"
|
||||
|
||||
# Scenario A2: the exclusion list can be replaced through the environment.
|
||||
custom_exclude_dir="${TMPD}/custom-exclude"
|
||||
@@ -363,6 +368,7 @@ printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/7z_invocations.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
|
||||
set +e
|
||||
@@ -374,8 +380,8 @@ assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_ex
|
||||
assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
|
||||
assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
|
||||
|
||||
# Scenario A3: two-pass rsync copies data while live, captures a small delta
|
||||
# while offline, restores the namespace, and only then starts compression.
|
||||
# Scenario A3: the mapped namespace is scaled down before its single local copy,
|
||||
# restored immediately afterward, and compression starts only after restoration.
|
||||
staging_dir="${TMPD}/staging"
|
||||
staging_source="${staging_dir}/source"
|
||||
staging_output="${staging_dir}/out"
|
||||
@@ -386,7 +392,6 @@ printf 'data\n' > "${staging_source}/app-ns/file.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/fail_rsync_second_pass.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/7z_invocations.log"
|
||||
@@ -402,15 +407,16 @@ set -e
|
||||
assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
|
||||
assert "staging scenario creates an archive" archive_exists "$staging_output"
|
||||
assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
|
||||
"rsync app-ns 1" \
|
||||
"scale app-ns --replicas=0" \
|
||||
"rsync app-ns 2" \
|
||||
"rsync app-ns 1" \
|
||||
"scale app-ns --replicas=3" \
|
||||
"7z app-ns"
|
||||
"7z staging"
|
||||
assert "staging scenario copies app-ns only once" file_contains "${TMPD}/rsync_counts/app-ns" "1"
|
||||
assert "staging scenario invokes 7z exactly once" sevenz_called_once_for_staging
|
||||
assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
|
||||
assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
|
||||
|
||||
# Scenario A4: a failed final delta never archives the incomplete staging copy,
|
||||
# Scenario A4: a failed local copy never archives the incomplete staging copy,
|
||||
# but the namespace is still restored immediately.
|
||||
rsync_failure_dir="${TMPD}/rsync-failure"
|
||||
rsync_failure_source="${rsync_failure_dir}/source"
|
||||
@@ -421,8 +427,7 @@ mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failur
|
||||
printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/fail_rsync_second_pass.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
@@ -434,11 +439,10 @@ run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state
|
||||
rsync_failure_exit_code=$?
|
||||
set -e
|
||||
|
||||
assert "final rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
|
||||
assert "final rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
|
||||
assert "final rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z app-ns"
|
||||
assert "final rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
|
||||
: > "${TMPD}/fail_rsync_second_pass.txt"
|
||||
assert "rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
|
||||
assert "rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
|
||||
assert "rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z staging"
|
||||
assert "rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
|
||||
|
||||
# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
|
||||
# are restored, retention still runs, and the script reports success.
|
||||
@@ -457,7 +461,9 @@ printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
|
||||
printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/7z_invocations.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
: > "${TMPD}/event_trace.log"
|
||||
|
||||
set +e
|
||||
run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log"
|
||||
@@ -472,10 +478,17 @@ assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/s
|
||||
assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
|
||||
assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
|
||||
assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
|
||||
assert "partial failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
||||
assert "partial failure scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
assert "partial failure scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
|
||||
assert "partial failure scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||
assert "partial failure scenario marks the error as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
|
||||
assert "partial failure scenario includes the backup error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "BACKUP_FAILED"'
|
||||
assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
|
||||
assert "partial failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
|
||||
assert "partial failure scenario archives the successfully staged folder" file_contains "${TMPD}/7z_trace.log" "good-ns"
|
||||
assert "partial failure scenario invokes 7z exactly once" sevenz_called_once_for_staging
|
||||
assert "partial failure scenario compresses after all namespaces are restored" events_are_ordered "${TMPD}/event_trace.log" \
|
||||
"scale fail-ns --replicas=4" \
|
||||
"scale good-ns --replicas=2" \
|
||||
"7z staging"
|
||||
assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4
|
||||
assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z"
|
||||
|
||||
@@ -502,8 +515,35 @@ set -e
|
||||
assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code"
|
||||
assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0"
|
||||
assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1"
|
||||
assert "all-failed scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
||||
assert "all-failed scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
assert "all-failed scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
|
||||
assert "all-failed scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||
|
||||
# Scenario C2: a failed single archive operation invalidates all staged folders
|
||||
# and removes the partial archive left by 7z.
|
||||
archive_failure_dir="${TMPD}/archive-failure"
|
||||
archive_failure_source="${archive_failure_dir}/source"
|
||||
archive_failure_output="${archive_failure_dir}/out"
|
||||
archive_failure_state="${archive_failure_dir}/state"
|
||||
mkdir -p "${archive_failure_source}/archive-ns" "$archive_failure_output" "$archive_failure_state"
|
||||
printf 'data\n' > "${archive_failure_source}/archive-ns/file.txt"
|
||||
printf 'archive-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'archive-ns:2\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
printf 'fail\n' > "${TMPD}/fail_archive.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
|
||||
set +e
|
||||
run_backup "$archive_failure_source" "$archive_failure_output" "$archive_failure_state" "${archive_failure_dir}/archive-failure.log"
|
||||
archive_failure_exit_code=$?
|
||||
set -e
|
||||
|
||||
assert "archive failure exits non-zero" is_nonzero "$archive_failure_exit_code"
|
||||
assert "archive failure restores archive-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "archive-ns" "2"
|
||||
assert "archive failure reports zero backup successes" file_contains "${archive_failure_dir}/archive-failure.log" "backup_successes=0"
|
||||
assert "archive failure removes the partial archive" archive_count_is "$archive_failure_output" 0
|
||||
assert "archive failure does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||
: > "${TMPD}/fail_archive.txt"
|
||||
|
||||
# Scenario D: no source folders means zero successful backups, which must also
|
||||
# return non-zero rather than reporting an empty run as successful.
|
||||
@@ -526,8 +566,8 @@ set -e
|
||||
|
||||
assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code"
|
||||
assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0"
|
||||
assert "empty scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
|
||||
assert "empty scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
assert "empty scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
|
||||
assert "empty scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||
|
||||
# Scenario E: abnormal abort while a namespace is still scaled to zero.
|
||||
# We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
|
||||
@@ -539,7 +579,8 @@ abort_dir="${TMPD}/abort"
|
||||
abort_source="${abort_dir}/source"
|
||||
abort_output="${abort_dir}/out"
|
||||
abort_state="${abort_dir}/state"
|
||||
mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}"
|
||||
abort_work="${abort_dir}/work"
|
||||
mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" "${abort_work}"
|
||||
printf 'data\n' > "${abort_source}/crash-ns/file.txt"
|
||||
printf 'crash-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'crash-ns:5\n' > "${TMPD}/replicas.txt"
|
||||
@@ -558,6 +599,7 @@ NOTIFY_SUCCESS_URL="" \
|
||||
NOTIFY_FAILURE_URL="" \
|
||||
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
||||
SEVENZ_BIN="${TMPD}/bin/7z" \
|
||||
BACKUP_STAGING_PATH="${abort_work}" \
|
||||
SCALE_WAIT_SECONDS="not-a-number" \
|
||||
TMP_STATE_DIR="${abort_state}" \
|
||||
PATH="${TMPD}/bin:${PATH}" \
|
||||
|
||||
Reference in New Issue
Block a user