k8s nfs backup: add rsync staging path and per-run staging cleanup

- Add BACKUP_STAGING_PATH + RSYNC_BIN for staging before archive
- Add cleanup_run_staging to EXIT trap to remove staging dir
- Extend test harness for staging scenarios

k8s/automated-nfs-backup.sh

@@ -8,7 +8,7 @@ source "${SCRIPT_DIR}/lib/common.sh"
 # Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
 # every scaled-down namespace is restored regardless of how we exit.
 trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
-trap '_exit_code=$?; restore_all_remaining; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
+trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
 
 # Required configuration
 require_env "NFS_SOURCE_PATH"
@@ -20,6 +20,8 @@ KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 KUBE_CONTEXT="${KUBE_CONTEXT:-}"
 WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
 EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease,default}"
+BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-}"
+RSYNC_BIN="${RSYNC_BIN:-rsync}"
 ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
 ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
 SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
@@ -38,6 +40,7 @@ NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
 NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}"
 CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
 RUN_STATE_DIR=""
+RUN_STAGING_DIR=""
 
 KUBECTL_ARGS=()
 if [[ -n "$KUBE_CONTEXT" ]]; then
@@ -94,6 +97,9 @@ validate_inputs() {
     require_cmd "$KUBECTL_BIN"
     require_cmd "$SEVENZ_BIN"
     require_cmd "mktemp"
+    if [[ -n "$BACKUP_STAGING_PATH" ]]; then
+        require_cmd "$RSYNC_BIN"
+    fi
     if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
         require_cmd "curl"
     fi
@@ -117,6 +123,23 @@ validate_inputs() {
     fi
     log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
 
+    if [[ -n "$BACKUP_STAGING_PATH" ]]; then
+        local source_real
+        local staging_real
+        mkdir -p "$BACKUP_STAGING_PATH"
+        source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
+        staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
+        case "${staging_real}/" in
+            "${source_real%/}/"*)
+                die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
+                ;;
+        esac
+        if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
+            die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
+        fi
+        log_info "Two-pass rsync staging enabled: ${RUN_STAGING_DIR}"
+    fi
+
     if ! _kubectl get namespaces >/dev/null 2>&1; then
         die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
     fi
@@ -266,6 +289,50 @@ restore_all_remaining() {
     fi
 }
 
+cleanup_run_staging() {
+    if [[ -z "${RUN_STAGING_DIR:-}" ]] || [[ ! -d "${RUN_STAGING_DIR:-}" ]]; then
+        return 0
+    fi
+
+    case "$RUN_STAGING_DIR" in
+        "${BACKUP_STAGING_PATH%/}/run-"*)
+            rm -rf -- "$RUN_STAGING_DIR"
+            ;;
+        *)
+            log_error "Refusing to remove unexpected staging directory: ${RUN_STAGING_DIR}"
+            return 1
+            ;;
+    esac
+}
+
+cleanup_staged_folder() {
+    local staging_folder="${1:?staging folder required}"
+    case "$staging_folder" in
+        "$RUN_STAGING_DIR"/*)
+            rm -rf -- "$staging_folder"
+            ;;
+        *)
+            log_error "Refusing to remove staging folder outside the current run: ${staging_folder}"
+            return 1
+            ;;
+    esac
+}
+
+sync_folder_to_staging() {
+    local source_folder="${1:?source folder required}"
+    local staging_folder="${2:?staging folder required}"
+    local sync_output
+
+    mkdir -p "$staging_folder"
+    if ! sync_output="$("$RSYNC_BIN" -a --delete -- "${source_folder%/}/" "${staging_folder%/}/" 2>&1)"; then
+        log_error "rsync failed from '${source_folder}' to '${staging_folder}'"
+        if [[ -n "$sync_output" ]]; then
+            log_error "rsync output: ${sync_output}"
+        fi
+        return 1
+    fi
+}
+
 archive_path_for_run() {
     local ts
     ts="$(date +"$ARCHIVE_TS_FORMAT")"
@@ -351,10 +418,20 @@ EOF
 backup_folder() {
     local folder_path="${1:?folder path required}"
     local archive_path="${2:?archive path required}"
+    local working_directory="${3:-}"
+    local archive_input="$folder_path"
+
+    if [[ -n "$working_directory" ]]; then
+        archive_input="$(basename "$folder_path")"
+        if [[ "$archive_path" != /* ]]; then
+            archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
+        fi
+    fi
+
     local -a cmd=(
         "$SEVENZ_BIN" a -t7z
         "$archive_path"
-        "$folder_path"
+        "$archive_input"
         "-m0=${SEVENZ_METHOD}"
         "-mx=${SEVENZ_LEVEL}"
         "-mmt=${SEVENZ_THREADS}"
@@ -365,7 +442,11 @@ backup_folder() {
     fi
 
     local cmd_output rc filtered_output
-    cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)"
+    if [[ -n "$working_directory" ]]; then
+        cmd_output="$(cd "$working_directory" && "${cmd[@]}" 2>&1 >/dev/null)"
+    else
+        cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)"
+    fi
     rc=$?
     filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
     if [[ -n "$filtered_output" ]]; then
@@ -421,11 +502,26 @@ process_folder() {
     local state_file=""
     local has_mapping=0
     local namespace_match_status=0
+    local backup_source_path="$folder_path"
+    local staging_folder=""
+    local final_sync_failed=0
 
     folder_name="$(basename "$folder_path")"
     total_folders=$((total_folders + 1))
     log_info "Processing folder: ${folder_name}"
 
+    if [[ -n "$RUN_STAGING_DIR" ]]; then
+        staging_folder="${RUN_STAGING_DIR}/${folder_name}"
+        log_info "Starting live pre-sync for folder '${folder_name}'"
+        if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            backup_failures=$((backup_failures + 1))
+            log_error "Backup failed for folder '${folder_name}' during live pre-sync"
+            cleanup_staged_folder "$staging_folder" || true
+            return 0
+        fi
+        backup_source_path="$staging_folder"
+    fi
+
     if namespace="$(namespace_for_folder "$folder_name")"; then
         has_mapping=1
         mapped_folders=$((mapped_folders + 1))
@@ -437,6 +533,13 @@ process_folder() {
 
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
         sleep "$SCALE_WAIT_SECONDS"
+
+        if [[ -n "$staging_folder" ]]; then
+            log_info "Starting final offline rsync delta for namespace '${namespace}'"
+            if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
+                final_sync_failed=1
+            fi
+        fi
     else
         namespace_match_status=$?
         unmapped_folders=$((unmapped_folders + 1))
@@ -447,7 +550,21 @@ process_folder() {
         fi
     fi
 
-    if backup_folder "$folder_path" "$archive_path"; then
+    if [[ "$has_mapping" -eq 1 && -n "$staging_folder" ]]; then
+        restore_namespace_replicas "$namespace" "$state_file"
+        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
+        sleep "$SCALE_WAIT_SECONDS"
+        rm -f "$state_file"
+    fi
+
+    if (( final_sync_failed == 1 )); then
+        backup_failures=$((backup_failures + 1))
+        log_error "Backup failed for folder '${folder_name}' during final offline rsync"
+        cleanup_staged_folder "$staging_folder" || true
+        return 0
+    fi
+
+    if backup_folder "$backup_source_path" "$archive_path" "${RUN_STAGING_DIR:-}"; then
         backup_successes=$((backup_successes + 1))
         log_info "Added folder '${folder_name}' to archive: ${archive_path}"
     else
@@ -455,12 +572,16 @@ process_folder() {
         log_error "Backup failed for folder '${folder_name}'"
     fi
 
-    if [[ "$has_mapping" -eq 1 ]]; then
+    if [[ "$has_mapping" -eq 1 && -z "$staging_folder" ]]; then
         restore_namespace_replicas "$namespace" "$state_file"
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
         sleep "$SCALE_WAIT_SECONDS"
         rm -f "$state_file"
     fi
+
+    if [[ -n "$staging_folder" ]]; then
+        cleanup_staged_folder "$staging_folder" || true
+    fi
 }
 
 print_summary() {