k8s nfs backup: exclude namespaces from scaling, isolate per-run state, harden failure handling

- Add EXCLUDED_NAMESPACES (default kube-system,kube-public,kube-node-lease,default); excluded folders are backed up but never scaled - Use a per-run state dir via mktemp -d so stale .state files from an earlier run can't leak into restores - Partial folder-backup failure now exits 0 (only zero successes fails); retention runs even on partial failure - Remove dead SCALE_TIMEOUT_SECONDS config; document SCALE_WAIT_SECONDS - Add .gitattributes to keep *.sh at LF - Extend test harness to 44 passing assertions covering exclusions, per-run state, partial/all-failed/empty scenarios
2026-07-14 09:02:06 -03:00
parent 91572e4819
commit 2bd9e90f00
5 changed files with 208 additions and 45 deletions
--- a/k8s/README.md
+++ b/k8s/README.md
@@ -10,11 +10,12 @@ Backs up each top-level folder found in `NFS_SOURCE_PATH` into a single encrypte

 Behavior:
 - Exact folder-to-namespace mapping only.
+- `kube-system`, `kube-public`, `kube-node-lease`, and `default` are excluded from scale actions by default.
 - Unmapped folder: backup still runs, Kubernetes scale actions are skipped.
 - Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again.
 - Scale-down issues are warnings by policy (backup still runs).
 - Restore issues are warnings by policy (run can still complete successfully).
- Cleanup can delete the last `N` archives ordered by date (oldest side), while keeping at least one archive.
+- Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.

 ## Environment Variables

@@ -27,17 +28,18 @@ Optional:
 - `KUBECTL_BIN` (default: `kubectl`)
 - `KUBE_CONTEXT` (default: empty)
 - `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`)
+- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease,default`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions.
 - `ARCHIVE_PREFIX` (default: `nfs-backup`)
 - `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`)
 - `SEVENZ_METHOD` (default: `lzma2`)
 - `SEVENZ_LEVEL` (default: `9`)
 - `SEVENZ_HEADER_ENCRYPT` (default: `on`, only applied when `BACKUP_PASSWORD` is set)
 - `SEVENZ_THREADS` (default: `on`)
- `SCALE_TIMEOUT_SECONDS` (default: `600`)
 - `SCALE_RETRY_COUNT` (default: `3`)
 - `SCALE_RETRY_DELAY_SECONDS` (default: `5`)
+- `SCALE_WAIT_SECONDS` (default: `30`): Fixed delay after scaling down and after restoring a mapped namespace.
 - `LOG_LEVEL` (default: `info`)
- `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`)
+- `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`): Parent directory for fresh per-run replica-state directories. A run never restores state files belonging to an earlier run.
 - `NOTIFY_SUCCESS_URL` (default: empty, disabled)
 - `NOTIFY_FAILURE_URL` (default: empty, disabled)
 - `NOTIFY_TITLE` (default: `Kubernetes`)
@@ -64,12 +66,12 @@ Notification payload (success and failure):
 ## Failure Semantics

 - Missing required env vars, missing commands, invalid paths, or inability to list namespaces: script exits non-zero immediately.
- Folder backup failures: counted and script exits non-zero at end.
+- Folder backup failures are counted. One or more successful folder backups continues with cleanup and exits zero; zero successful folder backups exits non-zero.
 - Scale-down warnings/timeouts: logged and counted, backup continues.
 - Restore warnings/timeouts: logged and counted, script does not fail solely because of restore warnings.
 - If `NOTIFY_SUCCESS_URL` is set, success notification is sent at the end of a successful run.
- If `NOTIFY_FAILURE_URL` is set, failure notification is sent when backup failures are detected.
- On successful runs, cleanup removes the last `CLEANUP_KEEP_COUNT` archives by date ordering (oldest side), without deleting the final remaining archive.
+- If `NOTIFY_FAILURE_URL` is set, a failure notification is sent when any folder backup failure is detected, including partial failures.
+- On runs with at least one successful folder backup, cleanup retains the `CLEANUP_KEEP_COUNT` most recent matching archives and deletes older ones.
 - Final summary always logs:
  - `processed`
  - `mapped`
--- a/k8s/automated-nfs-backup.sh
+++ b/k8s/automated-nfs-backup.sh
@@ -19,6 +19,7 @@ BACKUP_PASSWORD="${BACKUP_PASSWORD:-}"
 KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 KUBE_CONTEXT="${KUBE_CONTEXT:-}"
 WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
+EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease,default}"
 ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
 ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
 SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
@@ -26,7 +27,6 @@ SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}"
 SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}"
 SEVENZ_THREADS="${SEVENZ_THREADS:-on}"
 SEVENZ_BIN="${SEVENZ_BIN:-7z}"
-SCALE_TIMEOUT_SECONDS="${SCALE_TIMEOUT_SECONDS:-600}"
 SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
 SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
 SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
@@ -37,6 +37,7 @@ NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
 NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
 NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}"
 CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
+RUN_STATE_DIR=""

 KUBECTL_ARGS=()
 if [[ -n "$KUBE_CONTEXT" ]]; then
@@ -45,6 +46,7 @@ fi

 declare -a WORKLOAD_KIND_LIST=()
 declare -A NAMESPACE_MAP=()
+declare -A EXCLUDED_NAMESPACE_MAP=()

 total_folders=0
 mapped_folders=0
@@ -75,9 +77,23 @@ parse_workload_kinds() {
    fi
 }

+parse_excluded_namespaces() {
+    local raw
+    local cleaned
+    local namespace
+    IFS=',' read -r -a raw <<< "$EXCLUDED_NAMESPACES"
+    for namespace in "${raw[@]}"; do
+        cleaned="$(trim "${namespace,,}")"
+        if [[ -n "$cleaned" ]]; then
+            EXCLUDED_NAMESPACE_MAP["$cleaned"]=1
+        fi
+    done
+}
+
 validate_inputs() {
    require_cmd "$KUBECTL_BIN"
    require_cmd "$SEVENZ_BIN"
+    require_cmd "mktemp"
    if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
        require_cmd "curl"
    fi
@@ -96,6 +112,10 @@ validate_inputs() {

    mkdir -p "$BACKUP_OUTPUT_PATH"
    mkdir -p "$TMP_STATE_DIR"
+    if ! RUN_STATE_DIR="$(mktemp -d "${TMP_STATE_DIR%/}/run-XXXXXXXXXX")"; then
+        die "Unable to create a per-run state directory under ${TMP_STATE_DIR}"
+    fi
+    log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"

    if ! _kubectl get namespaces >/dev/null 2>&1; then
        die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
@@ -112,6 +132,9 @@ load_namespaces() {

 namespace_for_folder() {
    local folder_name="${1:?folder name required}"
+    if [[ -n "${EXCLUDED_NAMESPACE_MAP[$folder_name]:-}" ]]; then
+        return 2
+    fi
    if [[ -n "${NAMESPACE_MAP[$folder_name]:-}" ]]; then
        printf '%s' "$folder_name"
        return 0
@@ -121,7 +144,7 @@ namespace_for_folder() {

 state_file_for_namespace() {
    local namespace="${1:?namespace required}"
-    printf '%s/%s.state' "$TMP_STATE_DIR" "$namespace"
+    printf '%s/%s.state' "$RUN_STATE_DIR" "$namespace"
 }

 capture_replicas_state() {
@@ -207,7 +230,7 @@ restore_namespace_replicas() {
 }

 restore_all_remaining() {
-    if [[ -z "${TMP_STATE_DIR:-}" ]] || [[ ! -d "${TMP_STATE_DIR:-}" ]]; then
+    if [[ -z "${RUN_STATE_DIR:-}" ]] || [[ ! -d "${RUN_STATE_DIR:-}" ]]; then
        return 0
    fi

@@ -216,7 +239,7 @@ restore_all_remaining() {
    local kind name replicas
    local found=0

-    for state_file in "${TMP_STATE_DIR}"/*.state; do
+    for state_file in "${RUN_STATE_DIR}"/*.state; do
        [[ -f "$state_file" ]] || continue
        found=1
        namespace="$(basename "$state_file" .state)"
@@ -237,6 +260,10 @@ restore_all_remaining() {
    if [[ "$found" -eq 1 ]]; then
        log_info "EXIT cleanup complete - all remaining workloads restored"
    fi
+
+    if ! rmdir -- "$RUN_STATE_DIR" 2>/dev/null; then
+        log_warn "Run state directory is not empty; leaving it in place: ${RUN_STATE_DIR}"
+    fi
 }

 archive_path_for_run() {
@@ -393,6 +420,7 @@ process_folder() {
    local namespace=""
    local state_file=""
    local has_mapping=0
+    local namespace_match_status=0

    folder_name="$(basename "$folder_path")"
    total_folders=$((total_folders + 1))
@@ -410,8 +438,13 @@ process_folder() {
        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
        sleep "$SCALE_WAIT_SECONDS"
    else
+        namespace_match_status=$?
        unmapped_folders=$((unmapped_folders + 1))
-        log_warn "No namespace matched folder '${folder_name}'. Running backup only."
+        if (( namespace_match_status == 2 )); then
+            log_warn "Folder '${folder_name}' matches an excluded namespace; skipping Kubernetes scale actions and running backup only."
+        else
+            log_warn "No namespace matched folder '${folder_name}'. Running backup only."
+        fi
    fi

    if backup_folder "$folder_path" "$archive_path"; then
@@ -436,6 +469,7 @@ print_summary() {

 main() {
    parse_workload_kinds
+    parse_excluded_namespaces
    validate_inputs
    load_namespaces

@@ -467,12 +501,20 @@ main() {
    local size_mb
    size_mb="$(backup_size_mb "$total_backup_size_bytes")"

+    if (( backup_successes == 0 )); then
+        send_backup_failure_notification \
+            "$NOTIFY_FAILURE_URL" \
+            "No NFS folders were backed up successfully" \
+            "$size_mb" || true
+        die "No folder backups succeeded" 1
+    fi
+
    if (( backup_failures > 0 )); then
        send_backup_failure_notification \
            "$NOTIFY_FAILURE_URL" \
-            "Error while backing up NFS folders" \
+            "Some NFS folders failed to back up" \
            "$size_mb" || true
-        die "One or more folder backups failed" 1
+        log_warn "Some folder backups failed; continuing because ${backup_successes} folder backup(s) succeeded"
    fi

    cleanup_archives || true
--- a/k8s/examples/automated-nfs-backup.env.example
+++ b/k8s/examples/automated-nfs-backup.env.example
@@ -4,15 +4,16 @@ BACKUP_PASSWORD=
 KUBECTL_BIN=kubectl
 KUBE_CONTEXT=
 WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller
+EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default
 ARCHIVE_PREFIX=nfs-backup
 ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S
 SEVENZ_METHOD=lzma2
 SEVENZ_LEVEL=9
 SEVENZ_HEADER_ENCRYPT=on
 SEVENZ_THREADS=on
-SCALE_TIMEOUT_SECONDS=600
 SCALE_RETRY_COUNT=3
 SCALE_RETRY_DELAY_SECONDS=5
+SCALE_WAIT_SECONDS=30
 LOG_LEVEL=info
 TMP_STATE_DIR=/tmp/k8s-nfs-backup
 NOTIFY_SUCCESS_URL=
--- a/k8s/test/test-automated-nfs-backup.sh
+++ b/k8s/test/test-automated-nfs-backup.sh
@@ -66,6 +66,31 @@ archive_exists() {
    [[ "${#archives[@]}" -gt 0 ]]
 }

+archive_count_is() {
+    local output_dir="$1"
+    local expected_count="$2"
+    local -a archives=()
+
+    shopt -s nullglob
+    archives=("${output_dir}"/test_*.7z)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -eq "$expected_count" ]]
+}
+
+path_does_not_exist() {
+    [[ ! -e "$1" ]]
+}
+
+directory_is_empty() {
+    local directory="$1"
+    local -a entries=()
+
+    shopt -s nullglob dotglob
+    entries=("${directory}"/*)
+    shopt -u nullglob dotglob
+    [[ "${#entries[@]}" -eq 0 ]]
+}
+
 trace_has_scale() {
    local trace_file="$1"
    local namespace="$2"
@@ -218,20 +243,26 @@ run_backup() {
    local output_dir="$2"
    local state_dir="$3"
    local log_file="$4"
+    local -a env_overrides=()

-    NFS_SOURCE_PATH="$source_dir" \
-    BACKUP_OUTPUT_PATH="$output_dir" \
-    BACKUP_PASSWORD="" \
-    ARCHIVE_PREFIX="test" \
-    NOTIFY_SUCCESS_URL="http://example.invalid/success" \
-    NOTIFY_FAILURE_URL="http://example.invalid/failure" \
-    KUBECTL_BIN="${TMPD}/bin/kubectl" \
-    SEVENZ_BIN="${TMPD}/bin/7z" \
-    SCALE_WAIT_SECONDS=0 \
-    TMP_STATE_DIR="$state_dir" \
-    PATH="${TMPD}/bin:${PATH}" \
-    TMPD="$TMPD" \
-    bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
+    if (( $# >= 5 )); then
+        env_overrides+=("EXCLUDED_NAMESPACES=$5")
+    fi
+
+    env "${env_overrides[@]}" \
+        NFS_SOURCE_PATH="$source_dir" \
+        BACKUP_OUTPUT_PATH="$output_dir" \
+        BACKUP_PASSWORD="" \
+        ARCHIVE_PREFIX="test" \
+        NOTIFY_SUCCESS_URL="http://example.invalid/success" \
+        NOTIFY_FAILURE_URL="http://example.invalid/failure" \
+        KUBECTL_BIN="${TMPD}/bin/kubectl" \
+        SEVENZ_BIN="${TMPD}/bin/7z" \
+        SCALE_WAIT_SECONDS=0 \
+        TMP_STATE_DIR="$state_dir" \
+        PATH="${TMPD}/bin:${PATH}" \
+        TMPD="$TMPD" \
+        bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
 }

 write_fake_binaries
@@ -243,11 +274,13 @@ success_dir="${TMPD}/success"
 success_source="${success_dir}/source"
 success_output="${success_dir}/out"
 success_state="${success_dir}/state"
-mkdir -p "${success_source}/app-ns" "${success_source}/loose-data" "$success_output" "$success_state"
+mkdir -p "${success_source}/app-ns" "${success_source}/kube-system" "${success_source}/loose-data" "$success_output" "${success_state}/run-stale"
 printf 'data\n' > "${success_source}/app-ns/file.txt"
+printf 'data\n' > "${success_source}/kube-system/file.txt"
 printf 'data\n' > "${success_source}/loose-data/file.txt"
-printf 'app-ns\n' > "${TMPD}/namespaces.txt"
-printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+printf 'deployment\tstale\t9\n' > "${success_state}/run-stale/stale-ns.state"
+printf 'app-ns\nkube-system\nstale-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/fail_folders.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
@@ -260,17 +293,44 @@ set -e

 assert "success scenario exits with code 0" is_zero "$success_exit_code"
 assert "success scenario creates a test archive" archive_exists "$success_output"
-assert "success scenario reports two backup successes" file_contains "${success_dir}/success.log" "backup_successes=2"
+assert "success scenario reports three backup successes" file_contains "${success_dir}/success.log" "backup_successes=3"
 assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
 assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
 assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
 assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
 assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
+assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system"
+assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace"
+assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns"
 assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns"
+assert "success scenario invokes 7z for excluded kube-system" sevenz_called_for "kube-system"
 assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data"

-# Scenario B: one namespace backup fails, but both namespaces are restored and
-# the script exits through its documented handled-failure path.
+# Scenario A2: the exclusion list can be replaced through the environment.
+custom_exclude_dir="${TMPD}/custom-exclude"
+custom_exclude_source="${custom_exclude_dir}/source"
+custom_exclude_output="${custom_exclude_dir}/out"
+custom_exclude_state="${custom_exclude_dir}/state"
+mkdir -p "${custom_exclude_source}/app-ns" "$custom_exclude_output" "$custom_exclude_state"
+printf 'data\n' > "${custom_exclude_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns"
+custom_exclude_exit_code=$?
+set -e
+
+assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_exit_code"
+assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
+assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
+
+# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
+# are restored, retention still runs, and the script reports success.
 failure_dir="${TMPD}/failure"
 failure_source="${failure_dir}/source"
 failure_output="${failure_dir}/out"
@@ -278,6 +338,9 @@ failure_state="${failure_dir}/state"
 mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state"
 printf 'data\n' > "${failure_source}/good-ns/file.txt"
 printf 'data\n' > "${failure_source}/fail-ns/file.txt"
+for i in 1 2 3 4; do
+    touch -t "2020010${i}0000" "${failure_output}/test_2020010${i}_000000.7z"
+done
 printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt"
 printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
 printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
@@ -290,18 +353,72 @@ run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/
 failure_exit_code=$?
 set -e

-assert "failure scenario exits non-zero" is_nonzero "$failure_exit_code"
-assert "failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
-assert "failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
-assert "failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
-assert "failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
-assert "failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
-assert "failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
-assert "failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
-assert "failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
-assert "failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
+assert "partial failure scenario exits with code 0" is_zero "$failure_exit_code"
+assert "partial failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
+assert "partial failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
+assert "partial failure scenario logs that it is continuing" file_contains "${failure_dir}/failure.log" "Some folder backups failed; continuing"
+assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
+assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
+assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
+assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
+assert "partial failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
+assert "partial failure scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
+assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
+assert "partial failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
+assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4
+assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z"

-# Scenario C: abnormal abort while a namespace is still scaled to zero.
+# Scenario C: every attempted folder backup fails, so the script must still
+# return non-zero and must not send a success notification.
+all_failed_dir="${TMPD}/all-failed"
+all_failed_source="${all_failed_dir}/source"
+all_failed_output="${all_failed_dir}/out"
+all_failed_state="${all_failed_dir}/state"
+mkdir -p "${all_failed_source}/only-ns" "$all_failed_output" "$all_failed_state"
+printf 'data\n' > "${all_failed_source}/only-ns/file.txt"
+printf 'only-ns\n' > "${TMPD}/namespaces.txt"
+printf 'only-ns:2\n' > "${TMPD}/replicas.txt"
+printf 'only-ns\n' > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$all_failed_source" "$all_failed_output" "$all_failed_state" "${all_failed_dir}/all-failed.log"
+all_failed_exit_code=$?
+set -e
+
+assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code"
+assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0"
+assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1"
+assert "all-failed scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
+assert "all-failed scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
+
+# Scenario D: no source folders means zero successful backups, which must also
+# return non-zero rather than reporting an empty run as successful.
+empty_dir="${TMPD}/empty"
+empty_source="${empty_dir}/source"
+empty_output="${empty_dir}/out"
+empty_state="${empty_dir}/state"
+mkdir -p "$empty_source" "$empty_output" "$empty_state"
+: > "${TMPD}/namespaces.txt"
+: > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$empty_source" "$empty_output" "$empty_state" "${empty_dir}/empty.log"
+empty_exit_code=$?
+set -e
+
+assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code"
+assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0"
+assert "empty scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
+assert "empty scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
+
+# Scenario E: abnormal abort while a namespace is still scaled to zero.
 # We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
 # errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its
 # .state file is written, but BEFORE the in-line restore. The EXIT trap must then
@@ -341,7 +458,7 @@ set -e
 assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}"
 assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0"
 assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5"
-assert "abort scenario left no stale .state file" file_lacks "${abort_dir}/abort.log" ".state"
+assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state"

 printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
 if [[ "$tests_failed" -gt 0 ]]; then