k8s nfs backup: add rsync staging path and per-run staging cleanup

- Add BACKUP_STAGING_PATH + RSYNC_BIN for staging before archive
- Add cleanup_run_staging to EXIT trap to remove staging dir
- Extend test harness for staging scenarios

k8s/test/test-automated-nfs-backup.sh

@@ -91,6 +91,21 @@ directory_is_empty() {
     [[ "${#entries[@]}" -eq 0 ]]
 }
 
+events_are_ordered() {
+    local trace_file="$1"
+    shift
+    local event
+    local line_number
+    local previous_line=0
+
+    for event in "$@"; do
+        line_number="$(grep -n -m1 -F -- "$event" "$trace_file" | cut -d: -f1)"
+        [[ -n "$line_number" ]] || return 1
+        (( line_number > previous_line )) || return 1
+        previous_line="$line_number"
+    done
+}
+
 trace_has_scale() {
     local trace_file="$1"
     local namespace="$2"
@@ -179,6 +194,7 @@ for ((i = 1; i <= $#; i++)); do
             [[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
         done
         printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
+        printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log"
         exit 0
     fi
 done
@@ -198,6 +214,7 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
     folder="${4:?folder path required}"
     folder="${folder%/}"
     folder_name="${folder##*/}"
+    printf '7z %s\n' "$folder_name" >> "${TMPD}/event_trace.log"
 
     # Simulate a backup failure for configured folders BEFORE recording a
     # successful archive, so 7z_trace.log only lists folders actually archived.
@@ -210,6 +227,35 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
 fi
 
 exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/rsync" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+source_folder="${@: -2:1}"
+staging_folder="${@: -1}"
+source_folder="${source_folder%/}"
+staging_folder="${staging_folder%/}"
+folder_name="${source_folder##*/}"
+
+mkdir -p "${TMPD}/rsync_counts"
+count_file="${TMPD}/rsync_counts/${folder_name}"
+count=0
+if [[ -f "$count_file" ]]; then
+    count="$(<"$count_file")"
+fi
+count=$((count + 1))
+printf '%s\n' "$count" > "$count_file"
+printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
+
+if (( count == 2 )) && [[ -f "${TMPD}/fail_rsync_second_pass.txt" ]] && grep -Fxq -- "$folder_name" "${TMPD}/fail_rsync_second_pass.txt"; then
+    printf 'simulated final rsync failure for %s\n' "$folder_name" >&2
+    exit 23
+fi
+
+mkdir -p "$staging_folder"
+cp -a "${source_folder}/." "$staging_folder/"
 EOF
 
     write_fake_script "${TMPD}/bin/curl" <<'EOF'
@@ -243,11 +289,8 @@ run_backup() {
     local output_dir="$2"
     local state_dir="$3"
     local log_file="$4"
-    local -a env_overrides=()
-
-    if (( $# >= 5 )); then
-        env_overrides+=("EXCLUDED_NAMESPACES=$5")
-    fi
+    shift 4
+    local -a env_overrides=("$@")
 
     env "${env_overrides[@]}" \
         NFS_SOURCE_PATH="$source_dir" \
@@ -258,6 +301,7 @@ run_backup() {
         NOTIFY_FAILURE_URL="http://example.invalid/failure" \
         KUBECTL_BIN="${TMPD}/bin/kubectl" \
         SEVENZ_BIN="${TMPD}/bin/7z" \
+        RSYNC_BIN="${TMPD}/bin/rsync" \
         SCALE_WAIT_SECONDS=0 \
         TMP_STATE_DIR="$state_dir" \
         PATH="${TMPD}/bin:${PATH}" \
@@ -284,6 +328,7 @@ printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/fail_folders.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
 : > "${TMPD}/curl_trace.log"
 
 set +e
@@ -321,7 +366,7 @@ printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/curl_trace.log"
 
 set +e
-run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns"
+run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "EXCLUDED_NAMESPACES=app-ns"
 custom_exclude_exit_code=$?
 set -e
 
@@ -329,6 +374,72 @@ assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_ex
 assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
 assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
 
+# Scenario A3: two-pass rsync copies data while live, captures a small delta
+# while offline, restores the namespace, and only then starts compression.
+staging_dir="${TMPD}/staging"
+staging_source="${staging_dir}/source"
+staging_output="${staging_dir}/out"
+staging_state="${staging_dir}/state"
+staging_work="${staging_dir}/work"
+mkdir -p "${staging_source}/app-ns" "$staging_output" "$staging_state" "$staging_work"
+printf 'data\n' > "${staging_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/fail_rsync_second_pass.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+rm -rf "${TMPD}/rsync_counts"
+
+set +e
+run_backup "$staging_source" "$staging_output" "$staging_state" "${staging_dir}/staging.log" "BACKUP_STAGING_PATH=${staging_work}"
+staging_exit_code=$?
+set -e
+
+assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
+assert "staging scenario creates an archive" archive_exists "$staging_output"
+assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
+    "rsync app-ns 1" \
+    "scale app-ns --replicas=0" \
+    "rsync app-ns 2" \
+    "scale app-ns --replicas=3" \
+    "7z app-ns"
+assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
+assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
+
+# Scenario A4: a failed final delta never archives the incomplete staging copy,
+# but the namespace is still restored immediately.
+rsync_failure_dir="${TMPD}/rsync-failure"
+rsync_failure_source="${rsync_failure_dir}/source"
+rsync_failure_output="${rsync_failure_dir}/out"
+rsync_failure_state="${rsync_failure_dir}/state"
+rsync_failure_work="${rsync_failure_dir}/work"
+mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failure_state" "$rsync_failure_work"
+printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+printf 'app-ns\n' > "${TMPD}/fail_rsync_second_pass.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+rm -rf "${TMPD}/rsync_counts"
+
+set +e
+run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state" "${rsync_failure_dir}/rsync-failure.log" "BACKUP_STAGING_PATH=${rsync_failure_work}"
+rsync_failure_exit_code=$?
+set -e
+
+assert "final rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
+assert "final rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
+assert "final rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z app-ns"
+assert "final rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
+: > "${TMPD}/fail_rsync_second_pass.txt"
+
 # Scenario B: one namespace backup fails, but another succeeds. Both namespaces
 # are restored, retention still runs, and the script reports success.
 failure_dir="${TMPD}/failure"