diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfdb8b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/k8s/README.md b/k8s/README.md index d68e0b5..a1d8301 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -10,11 +10,12 @@ Backs up each top-level folder found in `NFS_SOURCE_PATH` into a single encrypte Behavior: - Exact folder-to-namespace mapping only. +- `kube-system`, `kube-public`, `kube-node-lease`, and `default` are excluded from scale actions by default. - Unmapped folder: backup still runs, Kubernetes scale actions are skipped. - Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again. - Scale-down issues are warnings by policy (backup still runs). - Restore issues are warnings by policy (run can still complete successfully). -- Cleanup can delete the last `N` archives ordered by date (oldest side), while keeping at least one archive. +- Cleanup retains the `N` most recent matching archives by modification time and deletes older ones. ## Environment Variables @@ -27,17 +28,18 @@ Optional: - `KUBECTL_BIN` (default: `kubectl`) - `KUBE_CONTEXT` (default: empty) - `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`) +- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease,default`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions. - `ARCHIVE_PREFIX` (default: `nfs-backup`) - `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`) - `SEVENZ_METHOD` (default: `lzma2`) - `SEVENZ_LEVEL` (default: `9`) - `SEVENZ_HEADER_ENCRYPT` (default: `on`, only applied when `BACKUP_PASSWORD` is set) - `SEVENZ_THREADS` (default: `on`) -- `SCALE_TIMEOUT_SECONDS` (default: `600`) - `SCALE_RETRY_COUNT` (default: `3`) - `SCALE_RETRY_DELAY_SECONDS` (default: `5`) +- `SCALE_WAIT_SECONDS` (default: `30`): Fixed delay after scaling down and after restoring a mapped namespace. - `LOG_LEVEL` (default: `info`) -- `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`) +- `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`): Parent directory for fresh per-run replica-state directories. A run never restores state files belonging to an earlier run. - `NOTIFY_SUCCESS_URL` (default: empty, disabled) - `NOTIFY_FAILURE_URL` (default: empty, disabled) - `NOTIFY_TITLE` (default: `Kubernetes`) @@ -64,12 +66,12 @@ Notification payload (success and failure): ## Failure Semantics - Missing required env vars, missing commands, invalid paths, or inability to list namespaces: script exits non-zero immediately. -- Folder backup failures: counted and script exits non-zero at end. +- Folder backup failures are counted. One or more successful folder backups continues with cleanup and exits zero; zero successful folder backups exits non-zero. - Scale-down warnings/timeouts: logged and counted, backup continues. - Restore warnings/timeouts: logged and counted, script does not fail solely because of restore warnings. - If `NOTIFY_SUCCESS_URL` is set, success notification is sent at the end of a successful run. -- If `NOTIFY_FAILURE_URL` is set, failure notification is sent when backup failures are detected. -- On successful runs, cleanup removes the last `CLEANUP_KEEP_COUNT` archives by date ordering (oldest side), without deleting the final remaining archive. +- If `NOTIFY_FAILURE_URL` is set, a failure notification is sent when any folder backup failure is detected, including partial failures. +- On runs with at least one successful folder backup, cleanup retains the `CLEANUP_KEEP_COUNT` most recent matching archives and deletes older ones. - Final summary always logs: - `processed` - `mapped` diff --git a/k8s/automated-nfs-backup.sh b/k8s/automated-nfs-backup.sh index 3b027cc..00b3032 100644 --- a/k8s/automated-nfs-backup.sh +++ b/k8s/automated-nfs-backup.sh @@ -19,6 +19,7 @@ BACKUP_PASSWORD="${BACKUP_PASSWORD:-}" KUBECTL_BIN="${KUBECTL_BIN:-kubectl}" KUBE_CONTEXT="${KUBE_CONTEXT:-}" WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}" +EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease,default}" ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}" ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}" SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}" @@ -26,7 +27,6 @@ SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}" SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}" SEVENZ_THREADS="${SEVENZ_THREADS:-on}" SEVENZ_BIN="${SEVENZ_BIN:-7z}" -SCALE_TIMEOUT_SECONDS="${SCALE_TIMEOUT_SECONDS:-600}" SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}" SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}" SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}" @@ -37,6 +37,7 @@ NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}" NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}" NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}" CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}" +RUN_STATE_DIR="" KUBECTL_ARGS=() if [[ -n "$KUBE_CONTEXT" ]]; then @@ -45,6 +46,7 @@ fi declare -a WORKLOAD_KIND_LIST=() declare -A NAMESPACE_MAP=() +declare -A EXCLUDED_NAMESPACE_MAP=() total_folders=0 mapped_folders=0 @@ -75,9 +77,23 @@ parse_workload_kinds() { fi } +parse_excluded_namespaces() { + local raw + local cleaned + local namespace + IFS=',' read -r -a raw <<< "$EXCLUDED_NAMESPACES" + for namespace in "${raw[@]}"; do + cleaned="$(trim "${namespace,,}")" + if [[ -n "$cleaned" ]]; then + EXCLUDED_NAMESPACE_MAP["$cleaned"]=1 + fi + done +} + validate_inputs() { require_cmd "$KUBECTL_BIN" require_cmd "$SEVENZ_BIN" + require_cmd "mktemp" if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then require_cmd "curl" fi @@ -96,6 +112,10 @@ validate_inputs() { mkdir -p "$BACKUP_OUTPUT_PATH" mkdir -p "$TMP_STATE_DIR" + if ! RUN_STATE_DIR="$(mktemp -d "${TMP_STATE_DIR%/}/run-XXXXXXXXXX")"; then + die "Unable to create a per-run state directory under ${TMP_STATE_DIR}" + fi + log_debug "Using isolated run state directory: ${RUN_STATE_DIR}" if ! _kubectl get namespaces >/dev/null 2>&1; then die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}" @@ -112,6 +132,9 @@ load_namespaces() { namespace_for_folder() { local folder_name="${1:?folder name required}" + if [[ -n "${EXCLUDED_NAMESPACE_MAP[$folder_name]:-}" ]]; then + return 2 + fi if [[ -n "${NAMESPACE_MAP[$folder_name]:-}" ]]; then printf '%s' "$folder_name" return 0 @@ -121,7 +144,7 @@ namespace_for_folder() { state_file_for_namespace() { local namespace="${1:?namespace required}" - printf '%s/%s.state' "$TMP_STATE_DIR" "$namespace" + printf '%s/%s.state' "$RUN_STATE_DIR" "$namespace" } capture_replicas_state() { @@ -207,7 +230,7 @@ restore_namespace_replicas() { } restore_all_remaining() { - if [[ -z "${TMP_STATE_DIR:-}" ]] || [[ ! -d "${TMP_STATE_DIR:-}" ]]; then + if [[ -z "${RUN_STATE_DIR:-}" ]] || [[ ! -d "${RUN_STATE_DIR:-}" ]]; then return 0 fi @@ -216,7 +239,7 @@ restore_all_remaining() { local kind name replicas local found=0 - for state_file in "${TMP_STATE_DIR}"/*.state; do + for state_file in "${RUN_STATE_DIR}"/*.state; do [[ -f "$state_file" ]] || continue found=1 namespace="$(basename "$state_file" .state)" @@ -237,6 +260,10 @@ restore_all_remaining() { if [[ "$found" -eq 1 ]]; then log_info "EXIT cleanup complete - all remaining workloads restored" fi + + if ! rmdir -- "$RUN_STATE_DIR" 2>/dev/null; then + log_warn "Run state directory is not empty; leaving it in place: ${RUN_STATE_DIR}" + fi } archive_path_for_run() { @@ -393,6 +420,7 @@ process_folder() { local namespace="" local state_file="" local has_mapping=0 + local namespace_match_status=0 folder_name="$(basename "$folder_path")" total_folders=$((total_folders + 1)) @@ -410,8 +438,13 @@ process_folder() { log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..." sleep "$SCALE_WAIT_SECONDS" else + namespace_match_status=$? unmapped_folders=$((unmapped_folders + 1)) - log_warn "No namespace matched folder '${folder_name}'. Running backup only." + if (( namespace_match_status == 2 )); then + log_warn "Folder '${folder_name}' matches an excluded namespace; skipping Kubernetes scale actions and running backup only." + else + log_warn "No namespace matched folder '${folder_name}'. Running backup only." + fi fi if backup_folder "$folder_path" "$archive_path"; then @@ -436,6 +469,7 @@ print_summary() { main() { parse_workload_kinds + parse_excluded_namespaces validate_inputs load_namespaces @@ -467,12 +501,20 @@ main() { local size_mb size_mb="$(backup_size_mb "$total_backup_size_bytes")" + if (( backup_successes == 0 )); then + send_backup_failure_notification \ + "$NOTIFY_FAILURE_URL" \ + "No NFS folders were backed up successfully" \ + "$size_mb" || true + die "No folder backups succeeded" 1 + fi + if (( backup_failures > 0 )); then send_backup_failure_notification \ "$NOTIFY_FAILURE_URL" \ - "Error while backing up NFS folders" \ + "Some NFS folders failed to back up" \ "$size_mb" || true - die "One or more folder backups failed" 1 + log_warn "Some folder backups failed; continuing because ${backup_successes} folder backup(s) succeeded" fi cleanup_archives || true diff --git a/k8s/examples/automated-nfs-backup.env.example b/k8s/examples/automated-nfs-backup.env.example index 78f3741..c6b33f9 100644 --- a/k8s/examples/automated-nfs-backup.env.example +++ b/k8s/examples/automated-nfs-backup.env.example @@ -4,15 +4,16 @@ BACKUP_PASSWORD= KUBECTL_BIN=kubectl KUBE_CONTEXT= WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller +EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default ARCHIVE_PREFIX=nfs-backup ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S SEVENZ_METHOD=lzma2 SEVENZ_LEVEL=9 SEVENZ_HEADER_ENCRYPT=on SEVENZ_THREADS=on -SCALE_TIMEOUT_SECONDS=600 SCALE_RETRY_COUNT=3 SCALE_RETRY_DELAY_SECONDS=5 +SCALE_WAIT_SECONDS=30 LOG_LEVEL=info TMP_STATE_DIR=/tmp/k8s-nfs-backup NOTIFY_SUCCESS_URL= diff --git a/k8s/test/test-automated-nfs-backup.sh b/k8s/test/test-automated-nfs-backup.sh index d0110aa..1afc188 100644 --- a/k8s/test/test-automated-nfs-backup.sh +++ b/k8s/test/test-automated-nfs-backup.sh @@ -66,6 +66,31 @@ archive_exists() { [[ "${#archives[@]}" -gt 0 ]] } +archive_count_is() { + local output_dir="$1" + local expected_count="$2" + local -a archives=() + + shopt -s nullglob + archives=("${output_dir}"/test_*.7z) + shopt -u nullglob + [[ "${#archives[@]}" -eq "$expected_count" ]] +} + +path_does_not_exist() { + [[ ! -e "$1" ]] +} + +directory_is_empty() { + local directory="$1" + local -a entries=() + + shopt -s nullglob dotglob + entries=("${directory}"/*) + shopt -u nullglob dotglob + [[ "${#entries[@]}" -eq 0 ]] +} + trace_has_scale() { local trace_file="$1" local namespace="$2" @@ -218,20 +243,26 @@ run_backup() { local output_dir="$2" local state_dir="$3" local log_file="$4" + local -a env_overrides=() - NFS_SOURCE_PATH="$source_dir" \ - BACKUP_OUTPUT_PATH="$output_dir" \ - BACKUP_PASSWORD="" \ - ARCHIVE_PREFIX="test" \ - NOTIFY_SUCCESS_URL="http://example.invalid/success" \ - NOTIFY_FAILURE_URL="http://example.invalid/failure" \ - KUBECTL_BIN="${TMPD}/bin/kubectl" \ - SEVENZ_BIN="${TMPD}/bin/7z" \ - SCALE_WAIT_SECONDS=0 \ - TMP_STATE_DIR="$state_dir" \ - PATH="${TMPD}/bin:${PATH}" \ - TMPD="$TMPD" \ - bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1 + if (( $# >= 5 )); then + env_overrides+=("EXCLUDED_NAMESPACES=$5") + fi + + env "${env_overrides[@]}" \ + NFS_SOURCE_PATH="$source_dir" \ + BACKUP_OUTPUT_PATH="$output_dir" \ + BACKUP_PASSWORD="" \ + ARCHIVE_PREFIX="test" \ + NOTIFY_SUCCESS_URL="http://example.invalid/success" \ + NOTIFY_FAILURE_URL="http://example.invalid/failure" \ + KUBECTL_BIN="${TMPD}/bin/kubectl" \ + SEVENZ_BIN="${TMPD}/bin/7z" \ + SCALE_WAIT_SECONDS=0 \ + TMP_STATE_DIR="$state_dir" \ + PATH="${TMPD}/bin:${PATH}" \ + TMPD="$TMPD" \ + bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1 } write_fake_binaries @@ -243,11 +274,13 @@ success_dir="${TMPD}/success" success_source="${success_dir}/source" success_output="${success_dir}/out" success_state="${success_dir}/state" -mkdir -p "${success_source}/app-ns" "${success_source}/loose-data" "$success_output" "$success_state" +mkdir -p "${success_source}/app-ns" "${success_source}/kube-system" "${success_source}/loose-data" "$success_output" "${success_state}/run-stale" printf 'data\n' > "${success_source}/app-ns/file.txt" +printf 'data\n' > "${success_source}/kube-system/file.txt" printf 'data\n' > "${success_source}/loose-data/file.txt" -printf 'app-ns\n' > "${TMPD}/namespaces.txt" -printf 'app-ns:3\n' > "${TMPD}/replicas.txt" +printf 'deployment\tstale\t9\n' > "${success_state}/run-stale/stale-ns.state" +printf 'app-ns\nkube-system\nstale-ns\n' > "${TMPD}/namespaces.txt" +printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt" : > "${TMPD}/fail_folders.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" @@ -260,17 +293,44 @@ set -e assert "success scenario exits with code 0" is_zero "$success_exit_code" assert "success scenario creates a test archive" archive_exists "$success_output" -assert "success scenario reports two backup successes" file_contains "${success_dir}/success.log" "backup_successes=2" +assert "success scenario reports three backup successes" file_contains "${success_dir}/success.log" "backup_successes=3" assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0" assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0" assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3" assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success" assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data" +assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system" +assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace" +assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns" assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns" +assert "success scenario invokes 7z for excluded kube-system" sevenz_called_for "kube-system" assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data" -# Scenario B: one namespace backup fails, but both namespaces are restored and -# the script exits through its documented handled-failure path. +# Scenario A2: the exclusion list can be replaced through the environment. +custom_exclude_dir="${TMPD}/custom-exclude" +custom_exclude_source="${custom_exclude_dir}/source" +custom_exclude_output="${custom_exclude_dir}/out" +custom_exclude_state="${custom_exclude_dir}/state" +mkdir -p "${custom_exclude_source}/app-ns" "$custom_exclude_output" "$custom_exclude_state" +printf 'data\n' > "${custom_exclude_source}/app-ns/file.txt" +printf 'app-ns\n' > "${TMPD}/namespaces.txt" +printf 'app-ns:3\n' > "${TMPD}/replicas.txt" +: > "${TMPD}/fail_folders.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/7z_trace.log" +: > "${TMPD}/curl_trace.log" + +set +e +run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns" +custom_exclude_exit_code=$? +set -e + +assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_exit_code" +assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns" +assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace" + +# Scenario B: one namespace backup fails, but another succeeds. Both namespaces +# are restored, retention still runs, and the script reports success. failure_dir="${TMPD}/failure" failure_source="${failure_dir}/source" failure_output="${failure_dir}/out" @@ -278,6 +338,9 @@ failure_state="${failure_dir}/state" mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state" printf 'data\n' > "${failure_source}/good-ns/file.txt" printf 'data\n' > "${failure_source}/fail-ns/file.txt" +for i in 1 2 3 4; do + touch -t "2020010${i}0000" "${failure_output}/test_2020010${i}_000000.7z" +done printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt" printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt" printf 'fail-ns\n' > "${TMPD}/fail_folders.txt" @@ -290,18 +353,72 @@ run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/ failure_exit_code=$? set -e -assert "failure scenario exits non-zero" is_nonzero "$failure_exit_code" -assert "failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1" -assert "failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1" -assert "failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0" -assert "failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2" -assert "failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0" -assert "failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4" -assert "failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" -assert "failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns" -assert "failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns" +assert "partial failure scenario exits with code 0" is_zero "$failure_exit_code" +assert "partial failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1" +assert "partial failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1" +assert "partial failure scenario logs that it is continuing" file_contains "${failure_dir}/failure.log" "Some folder backups failed; continuing" +assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0" +assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2" +assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0" +assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4" +assert "partial failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" +assert "partial failure scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success" +assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns" +assert "partial failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns" +assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4 +assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z" -# Scenario C: abnormal abort while a namespace is still scaled to zero. +# Scenario C: every attempted folder backup fails, so the script must still +# return non-zero and must not send a success notification. +all_failed_dir="${TMPD}/all-failed" +all_failed_source="${all_failed_dir}/source" +all_failed_output="${all_failed_dir}/out" +all_failed_state="${all_failed_dir}/state" +mkdir -p "${all_failed_source}/only-ns" "$all_failed_output" "$all_failed_state" +printf 'data\n' > "${all_failed_source}/only-ns/file.txt" +printf 'only-ns\n' > "${TMPD}/namespaces.txt" +printf 'only-ns:2\n' > "${TMPD}/replicas.txt" +printf 'only-ns\n' > "${TMPD}/fail_folders.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/7z_trace.log" +: > "${TMPD}/curl_trace.log" + +set +e +run_backup "$all_failed_source" "$all_failed_output" "$all_failed_state" "${all_failed_dir}/all-failed.log" +all_failed_exit_code=$? +set -e + +assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code" +assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0" +assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1" +assert "all-failed scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" +assert "all-failed scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success" + +# Scenario D: no source folders means zero successful backups, which must also +# return non-zero rather than reporting an empty run as successful. +empty_dir="${TMPD}/empty" +empty_source="${empty_dir}/source" +empty_output="${empty_dir}/out" +empty_state="${empty_dir}/state" +mkdir -p "$empty_source" "$empty_output" "$empty_state" +: > "${TMPD}/namespaces.txt" +: > "${TMPD}/replicas.txt" +: > "${TMPD}/fail_folders.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/7z_trace.log" +: > "${TMPD}/curl_trace.log" + +set +e +run_backup "$empty_source" "$empty_output" "$empty_state" "${empty_dir}/empty.log" +empty_exit_code=$? +set -e + +assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code" +assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0" +assert "empty scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" +assert "empty scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success" + +# Scenario E: abnormal abort while a namespace is still scaled to zero. # We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep` # errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its # .state file is written, but BEFORE the in-line restore. The EXIT trap must then @@ -341,7 +458,7 @@ set -e assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}" assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0" assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5" -assert "abort scenario left no stale .state file" file_lacks "${abort_dir}/abort.log" ".state" +assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state" printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total" if [[ "$tests_failed" -gt 0 ]]; then