k8s nfs backup: add rsync staging path and per-run staging cleanup

- Add BACKUP_STAGING_PATH + RSYNC_BIN for staging before archive
- Add cleanup_run_staging to EXIT trap to remove staging dir
- Extend test harness for staging scenarios

k8s/README.md

@@ -13,6 +13,7 @@ Behavior:
 - `kube-system`, `kube-public`, `kube-node-lease`, and `default` are excluded from scale actions by default.
 - Unmapped folder: backup still runs, Kubernetes scale actions are skipped.
 - Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again.
+- When `BACKUP_STAGING_PATH` is configured, a live rsync pre-copy is followed by a final offline delta; the namespace is restored before the staged copy is compressed.
 - Scale-down issues are warnings by policy (backup still runs).
 - Restore issues are warnings by policy (run can still complete successfully).
 - Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.
@@ -29,6 +30,8 @@ Optional:
 - `KUBE_CONTEXT` (default: empty)
 - `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`)
 - `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease,default`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions.
+- `BACKUP_STAGING_PATH` (default: empty, disabled): Local staging parent for two-pass rsync. It must be outside `NFS_SOURCE_PATH` and have enough free space for the largest folder being backed up.
+- `RSYNC_BIN` (default: `rsync`)
 - `ARCHIVE_PREFIX` (default: `nfs-backup`)
 - `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`)
 - `SEVENZ_METHOD` (default: `lzma2`)
@@ -62,6 +65,7 @@ Notification payload (success and failure):
 - Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces.
 - Ensure `kubectl` context and credentials are present in the runtime.
 - Ensure `7z` is installed in the runtime image/host.
+- When `BACKUP_STAGING_PATH` is enabled, ensure `rsync` is installed and place staging on fast local storage for the shortest offline delta.
 
 ## Failure Semantics
 

k8s/automated-nfs-backup.sh

@@ -8,7 +8,7 @@ source "${SCRIPT_DIR}/lib/common.sh"
 # Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
 # every scaled-down namespace is restored regardless of how we exit.
 trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
-trap '_exit_code=$?; restore_all_remaining; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
+trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
 
 # Required configuration
 require_env "NFS_SOURCE_PATH"
@@ -20,6 +20,8 @@ KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 KUBE_CONTEXT="${KUBE_CONTEXT:-}"
 WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
 EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease,default}"
+BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-}"
+RSYNC_BIN="${RSYNC_BIN:-rsync}"
 ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
 ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
 SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
@@ -38,6 +40,7 @@ NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
 NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}"
 CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
 RUN_STATE_DIR=""
+RUN_STAGING_DIR=""
 
 KUBECTL_ARGS=()
 if [[ -n "$KUBE_CONTEXT" ]]; then
@@ -94,6 +97,9 @@ validate_inputs() {
     require_cmd "$KUBECTL_BIN"
     require_cmd "$SEVENZ_BIN"
     require_cmd "mktemp"
+    if [[ -n "$BACKUP_STAGING_PATH" ]]; then
+        require_cmd "$RSYNC_BIN"
+    fi
     if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
         require_cmd "curl"
     fi
@@ -117,6 +123,23 @@ validate_inputs() {
     fi
     log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
 
+    if [[ -n "$BACKUP_STAGING_PATH" ]]; then
+        local source_real
+        local staging_real
+        mkdir -p "$BACKUP_STAGING_PATH"
+        source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
+        staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
+        case "${staging_real}/" in
+            "${source_real%/}/"*)
+                die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
+                ;;
+        esac
+        if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
+            die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
+        fi
+        log_info "Two-pass rsync staging enabled: ${RUN_STAGING_DIR}"
+    fi
+
     if ! _kubectl get namespaces >/dev/null 2>&1; then
         die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
     fi
@@ -266,6 +289,50 @@ restore_all_remaining() {
     fi
 }
 
+cleanup_run_staging() {
+    if [[ -z "${RUN_STAGING_DIR:-}" ]] || [[ ! -d "${RUN_STAGING_DIR:-}" ]]; then
+        return 0
+    fi
+
+    case "$RUN_STAGING_DIR" in
+        "${BACKUP_STAGING_PATH%/}/run-"*)
+            rm -rf -- "$RUN_STAGING_DIR"
+            ;;
+        *)
+            log_error "Refusing to remove unexpected staging directory: ${RUN_STAGING_DIR}"
+            return 1
+            ;;
+    esac
+}
+
+cleanup_staged_folder() {
+    local staging_folder="${1:?staging folder required}"
+    case "$staging_folder" in
+        "$RUN_STAGING_DIR"/*)
+            rm -rf -- "$staging_folder"
+            ;;
+        *)
+            log_error "Refusing to remove staging folder outside the current run: ${staging_folder}"
+            return 1
+            ;;
+    esac
+}
+
+sync_folder_to_staging() {
+    local source_folder="${1:?source folder required}"
+    local staging_folder="${2:?staging folder required}"
+    local sync_output
+
+    mkdir -p "$staging_folder"
+    if ! sync_output="$("$RSYNC_BIN" -a --delete -- "${source_folder%/}/" "${staging_folder%/}/" 2>&1)"; then
+        log_error "rsync failed from '${source_folder}' to '${staging_folder}'"
+        if [[ -n "$sync_output" ]]; then
+            log_error "rsync output: ${sync_output}"
+        fi
+        return 1
+    fi
+}
+
 archive_path_for_run() {
     local ts
     ts="$(date +"$ARCHIVE_TS_FORMAT")"
@@ -351,10 +418,20 @@ EOF
 backup_folder() {
     local folder_path="${1:?folder path required}"
     local archive_path="${2:?archive path required}"
+    local working_directory="${3:-}"
+    local archive_input="$folder_path"
+
+    if [[ -n "$working_directory" ]]; then
+        archive_input="$(basename "$folder_path")"
+        if [[ "$archive_path" != /* ]]; then
+            archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
+        fi
+    fi
+
     local -a cmd=(
         "$SEVENZ_BIN" a -t7z
         "$archive_path"
-        "$folder_path"
+        "$archive_input"
         "-m0=${SEVENZ_METHOD}"
         "-mx=${SEVENZ_LEVEL}"
         "-mmt=${SEVENZ_THREADS}"
@@ -365,7 +442,11 @@ backup_folder() {
     fi
 
     local cmd_output rc filtered_output
+    if [[ -n "$working_directory" ]]; then
+        cmd_output="$(cd "$working_directory" && "${cmd[@]}" 2>&1 >/dev/null)"
+    else
         cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)"
+    fi
     rc=$?
     filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
     if [[ -n "$filtered_output" ]]; then
@@ -421,11 +502,26 @@ process_folder() {
     local state_file=""
     local has_mapping=0
     local namespace_match_status=0
+    local backup_source_path="$folder_path"
+    local staging_folder=""
+    local final_sync_failed=0
 
     folder_name="$(basename "$folder_path")"
     total_folders=$((total_folders + 1))
     log_info "Processing folder: ${folder_name}"
 
+    if [[ -n "$RUN_STAGING_DIR" ]]; then
+        staging_folder="${RUN_STAGING_DIR}/${folder_name}"
+        log_info "Starting live pre-sync for folder '${folder_name}'"
+        if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            backup_failures=$((backup_failures + 1))
+            log_error "Backup failed for folder '${folder_name}' during live pre-sync"
+            cleanup_staged_folder "$staging_folder" || true
+            return 0
+        fi
+        backup_source_path="$staging_folder"
+    fi
+
     if namespace="$(namespace_for_folder "$folder_name")"; then
         has_mapping=1
         mapped_folders=$((mapped_folders + 1))
@@ -437,6 +533,13 @@ process_folder() {
 
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
         sleep "$SCALE_WAIT_SECONDS"
+
+        if [[ -n "$staging_folder" ]]; then
+            log_info "Starting final offline rsync delta for namespace '${namespace}'"
+            if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
+                final_sync_failed=1
+            fi
+        fi
     else
         namespace_match_status=$?
         unmapped_folders=$((unmapped_folders + 1))
@@ -447,7 +550,21 @@ process_folder() {
         fi
     fi
 
-    if backup_folder "$folder_path" "$archive_path"; then
+    if [[ "$has_mapping" -eq 1 && -n "$staging_folder" ]]; then
+        restore_namespace_replicas "$namespace" "$state_file"
+        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
+        sleep "$SCALE_WAIT_SECONDS"
+        rm -f "$state_file"
+    fi
+
+    if (( final_sync_failed == 1 )); then
+        backup_failures=$((backup_failures + 1))
+        log_error "Backup failed for folder '${folder_name}' during final offline rsync"
+        cleanup_staged_folder "$staging_folder" || true
+        return 0
+    fi
+
+    if backup_folder "$backup_source_path" "$archive_path" "${RUN_STAGING_DIR:-}"; then
         backup_successes=$((backup_successes + 1))
         log_info "Added folder '${folder_name}' to archive: ${archive_path}"
     else
@@ -455,12 +572,16 @@ process_folder() {
         log_error "Backup failed for folder '${folder_name}'"
     fi
 
-    if [[ "$has_mapping" -eq 1 ]]; then
+    if [[ "$has_mapping" -eq 1 && -z "$staging_folder" ]]; then
         restore_namespace_replicas "$namespace" "$state_file"
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
         sleep "$SCALE_WAIT_SECONDS"
         rm -f "$state_file"
     fi
+
+    if [[ -n "$staging_folder" ]]; then
+        cleanup_staged_folder "$staging_folder" || true
+    fi
 }
 
 print_summary() {

k8s/examples/automated-nfs-backup.env.example

@@ -5,6 +5,8 @@ KUBECTL_BIN=kubectl
 KUBE_CONTEXT=
 WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller
 EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default
+BACKUP_STAGING_PATH=
+RSYNC_BIN=rsync
 ARCHIVE_PREFIX=nfs-backup
 ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S
 SEVENZ_METHOD=lzma2

k8s/test/test-automated-nfs-backup.sh

@@ -91,6 +91,21 @@ directory_is_empty() {
     [[ "${#entries[@]}" -eq 0 ]]
 }
 
+events_are_ordered() {
+    local trace_file="$1"
+    shift
+    local event
+    local line_number
+    local previous_line=0
+
+    for event in "$@"; do
+        line_number="$(grep -n -m1 -F -- "$event" "$trace_file" | cut -d: -f1)"
+        [[ -n "$line_number" ]] || return 1
+        (( line_number > previous_line )) || return 1
+        previous_line="$line_number"
+    done
+}
+
 trace_has_scale() {
     local trace_file="$1"
     local namespace="$2"
@@ -179,6 +194,7 @@ for ((i = 1; i <= $#; i++)); do
             [[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
         done
         printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
+        printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log"
         exit 0
     fi
 done
@@ -198,6 +214,7 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
     folder="${4:?folder path required}"
     folder="${folder%/}"
     folder_name="${folder##*/}"
+    printf '7z %s\n' "$folder_name" >> "${TMPD}/event_trace.log"
 
     # Simulate a backup failure for configured folders BEFORE recording a
     # successful archive, so 7z_trace.log only lists folders actually archived.
@@ -210,6 +227,35 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
 fi
 
 exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/rsync" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+source_folder="${@: -2:1}"
+staging_folder="${@: -1}"
+source_folder="${source_folder%/}"
+staging_folder="${staging_folder%/}"
+folder_name="${source_folder##*/}"
+
+mkdir -p "${TMPD}/rsync_counts"
+count_file="${TMPD}/rsync_counts/${folder_name}"
+count=0
+if [[ -f "$count_file" ]]; then
+    count="$(<"$count_file")"
+fi
+count=$((count + 1))
+printf '%s\n' "$count" > "$count_file"
+printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
+
+if (( count == 2 )) && [[ -f "${TMPD}/fail_rsync_second_pass.txt" ]] && grep -Fxq -- "$folder_name" "${TMPD}/fail_rsync_second_pass.txt"; then
+    printf 'simulated final rsync failure for %s\n' "$folder_name" >&2
+    exit 23
+fi
+
+mkdir -p "$staging_folder"
+cp -a "${source_folder}/." "$staging_folder/"
 EOF
 
     write_fake_script "${TMPD}/bin/curl" <<'EOF'
@@ -243,11 +289,8 @@ run_backup() {
     local output_dir="$2"
     local state_dir="$3"
     local log_file="$4"
-    local -a env_overrides=()
+    shift 4
-
+    local -a env_overrides=("$@")
-    if (( $# >= 5 )); then
-        env_overrides+=("EXCLUDED_NAMESPACES=$5")
-    fi
 
     env "${env_overrides[@]}" \
         NFS_SOURCE_PATH="$source_dir" \
@@ -258,6 +301,7 @@ run_backup() {
         NOTIFY_FAILURE_URL="http://example.invalid/failure" \
         KUBECTL_BIN="${TMPD}/bin/kubectl" \
         SEVENZ_BIN="${TMPD}/bin/7z" \
+        RSYNC_BIN="${TMPD}/bin/rsync" \
         SCALE_WAIT_SECONDS=0 \
         TMP_STATE_DIR="$state_dir" \
         PATH="${TMPD}/bin:${PATH}" \
@@ -284,6 +328,7 @@ printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/fail_folders.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
 : > "${TMPD}/curl_trace.log"
 
 set +e
@@ -321,7 +366,7 @@ printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/curl_trace.log"
 
 set +e
-run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns"
+run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "EXCLUDED_NAMESPACES=app-ns"
 custom_exclude_exit_code=$?
 set -e
 
@@ -329,6 +374,72 @@ assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_ex
 assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
 assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
 
+# Scenario A3: two-pass rsync copies data while live, captures a small delta
+# while offline, restores the namespace, and only then starts compression.
+staging_dir="${TMPD}/staging"
+staging_source="${staging_dir}/source"
+staging_output="${staging_dir}/out"
+staging_state="${staging_dir}/state"
+staging_work="${staging_dir}/work"
+mkdir -p "${staging_source}/app-ns" "$staging_output" "$staging_state" "$staging_work"
+printf 'data\n' > "${staging_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/fail_rsync_second_pass.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+rm -rf "${TMPD}/rsync_counts"
+
+set +e
+run_backup "$staging_source" "$staging_output" "$staging_state" "${staging_dir}/staging.log" "BACKUP_STAGING_PATH=${staging_work}"
+staging_exit_code=$?
+set -e
+
+assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
+assert "staging scenario creates an archive" archive_exists "$staging_output"
+assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
+    "rsync app-ns 1" \
+    "scale app-ns --replicas=0" \
+    "rsync app-ns 2" \
+    "scale app-ns --replicas=3" \
+    "7z app-ns"
+assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
+assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
+
+# Scenario A4: a failed final delta never archives the incomplete staging copy,
+# but the namespace is still restored immediately.
+rsync_failure_dir="${TMPD}/rsync-failure"
+rsync_failure_source="${rsync_failure_dir}/source"
+rsync_failure_output="${rsync_failure_dir}/out"
+rsync_failure_state="${rsync_failure_dir}/state"
+rsync_failure_work="${rsync_failure_dir}/work"
+mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failure_state" "$rsync_failure_work"
+printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+printf 'app-ns\n' > "${TMPD}/fail_rsync_second_pass.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+rm -rf "${TMPD}/rsync_counts"
+
+set +e
+run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state" "${rsync_failure_dir}/rsync-failure.log" "BACKUP_STAGING_PATH=${rsync_failure_work}"
+rsync_failure_exit_code=$?
+set -e
+
+assert "final rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
+assert "final rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
+assert "final rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z app-ns"
+assert "final rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
+: > "${TMPD}/fail_rsync_second_pass.txt"
+
 # Scenario B: one namespace backup fails, but another succeeds. Both namespaces
 # are restored, retention still runs, and the script reports success.
 failure_dir="${TMPD}/failure"