From 43351abb11155de11701858782a370ab869c9a72 Mon Sep 17 00:00:00 2001 From: Jose Henrique Date: Tue, 14 Jul 2026 09:36:39 -0300 Subject: [PATCH] k8s nfs backup: add rsync staging path and per-run staging cleanup - Add BACKUP_STAGING_PATH + RSYNC_BIN for staging before archive - Add cleanup_run_staging to EXIT trap to remove staging dir - Extend test harness for staging scenarios --- k8s/README.md | 4 + k8s/automated-nfs-backup.sh | 131 +++++++++++++++++- k8s/examples/automated-nfs-backup.env.example | 2 + k8s/test/test-automated-nfs-backup.sh | 123 +++++++++++++++- 4 files changed, 249 insertions(+), 11 deletions(-) diff --git a/k8s/README.md b/k8s/README.md index a1d8301..30817b1 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,6 +13,7 @@ Behavior: - `kube-system`, `kube-public`, `kube-node-lease`, and `default` are excluded from scale actions by default. - Unmapped folder: backup still runs, Kubernetes scale actions are skipped. - Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again. +- When `BACKUP_STAGING_PATH` is configured, a live rsync pre-copy is followed by a final offline delta; the namespace is restored before the staged copy is compressed. - Scale-down issues are warnings by policy (backup still runs). - Restore issues are warnings by policy (run can still complete successfully). - Cleanup retains the `N` most recent matching archives by modification time and deletes older ones. @@ -29,6 +30,8 @@ Optional: - `KUBE_CONTEXT` (default: empty) - `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`) - `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease,default`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions. +- `BACKUP_STAGING_PATH` (default: empty, disabled): Local staging parent for two-pass rsync. It must be outside `NFS_SOURCE_PATH` and have enough free space for the largest folder being backed up. +- `RSYNC_BIN` (default: `rsync`) - `ARCHIVE_PREFIX` (default: `nfs-backup`) - `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`) - `SEVENZ_METHOD` (default: `lzma2`) @@ -62,6 +65,7 @@ Notification payload (success and failure): - Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces. - Ensure `kubectl` context and credentials are present in the runtime. - Ensure `7z` is installed in the runtime image/host. +- When `BACKUP_STAGING_PATH` is enabled, ensure `rsync` is installed and place staging on fast local storage for the shortest offline delta. ## Failure Semantics diff --git a/k8s/automated-nfs-backup.sh b/k8s/automated-nfs-backup.sh index 00b3032..89ea93c 100644 --- a/k8s/automated-nfs-backup.sh +++ b/k8s/automated-nfs-backup.sh @@ -8,7 +8,7 @@ source "${SCRIPT_DIR}/lib/common.sh" # Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees # every scaled-down namespace is restored regardless of how we exit. trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR -trap '_exit_code=$?; restore_all_remaining; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT +trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT # Required configuration require_env "NFS_SOURCE_PATH" @@ -20,6 +20,8 @@ KUBECTL_BIN="${KUBECTL_BIN:-kubectl}" KUBE_CONTEXT="${KUBE_CONTEXT:-}" WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}" EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease,default}" +BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-}" +RSYNC_BIN="${RSYNC_BIN:-rsync}" ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}" ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}" SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}" @@ -38,6 +40,7 @@ NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}" NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}" CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}" RUN_STATE_DIR="" +RUN_STAGING_DIR="" KUBECTL_ARGS=() if [[ -n "$KUBE_CONTEXT" ]]; then @@ -94,6 +97,9 @@ validate_inputs() { require_cmd "$KUBECTL_BIN" require_cmd "$SEVENZ_BIN" require_cmd "mktemp" + if [[ -n "$BACKUP_STAGING_PATH" ]]; then + require_cmd "$RSYNC_BIN" + fi if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then require_cmd "curl" fi @@ -117,6 +123,23 @@ validate_inputs() { fi log_debug "Using isolated run state directory: ${RUN_STATE_DIR}" + if [[ -n "$BACKUP_STAGING_PATH" ]]; then + local source_real + local staging_real + mkdir -p "$BACKUP_STAGING_PATH" + source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)" + staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)" + case "${staging_real}/" in + "${source_real%/}/"*) + die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH" + ;; + esac + if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then + die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}" + fi + log_info "Two-pass rsync staging enabled: ${RUN_STAGING_DIR}" + fi + if ! _kubectl get namespaces >/dev/null 2>&1; then die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}" fi @@ -266,6 +289,50 @@ restore_all_remaining() { fi } +cleanup_run_staging() { + if [[ -z "${RUN_STAGING_DIR:-}" ]] || [[ ! -d "${RUN_STAGING_DIR:-}" ]]; then + return 0 + fi + + case "$RUN_STAGING_DIR" in + "${BACKUP_STAGING_PATH%/}/run-"*) + rm -rf -- "$RUN_STAGING_DIR" + ;; + *) + log_error "Refusing to remove unexpected staging directory: ${RUN_STAGING_DIR}" + return 1 + ;; + esac +} + +cleanup_staged_folder() { + local staging_folder="${1:?staging folder required}" + case "$staging_folder" in + "$RUN_STAGING_DIR"/*) + rm -rf -- "$staging_folder" + ;; + *) + log_error "Refusing to remove staging folder outside the current run: ${staging_folder}" + return 1 + ;; + esac +} + +sync_folder_to_staging() { + local source_folder="${1:?source folder required}" + local staging_folder="${2:?staging folder required}" + local sync_output + + mkdir -p "$staging_folder" + if ! sync_output="$("$RSYNC_BIN" -a --delete -- "${source_folder%/}/" "${staging_folder%/}/" 2>&1)"; then + log_error "rsync failed from '${source_folder}' to '${staging_folder}'" + if [[ -n "$sync_output" ]]; then + log_error "rsync output: ${sync_output}" + fi + return 1 + fi +} + archive_path_for_run() { local ts ts="$(date +"$ARCHIVE_TS_FORMAT")" @@ -351,10 +418,20 @@ EOF backup_folder() { local folder_path="${1:?folder path required}" local archive_path="${2:?archive path required}" + local working_directory="${3:-}" + local archive_input="$folder_path" + + if [[ -n "$working_directory" ]]; then + archive_input="$(basename "$folder_path")" + if [[ "$archive_path" != /* ]]; then + archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")" + fi + fi + local -a cmd=( "$SEVENZ_BIN" a -t7z "$archive_path" - "$folder_path" + "$archive_input" "-m0=${SEVENZ_METHOD}" "-mx=${SEVENZ_LEVEL}" "-mmt=${SEVENZ_THREADS}" @@ -365,7 +442,11 @@ backup_folder() { fi local cmd_output rc filtered_output - cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)" + if [[ -n "$working_directory" ]]; then + cmd_output="$(cd "$working_directory" && "${cmd[@]}" 2>&1 >/dev/null)" + else + cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)" + fi rc=$? filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)" if [[ -n "$filtered_output" ]]; then @@ -421,11 +502,26 @@ process_folder() { local state_file="" local has_mapping=0 local namespace_match_status=0 + local backup_source_path="$folder_path" + local staging_folder="" + local final_sync_failed=0 folder_name="$(basename "$folder_path")" total_folders=$((total_folders + 1)) log_info "Processing folder: ${folder_name}" + if [[ -n "$RUN_STAGING_DIR" ]]; then + staging_folder="${RUN_STAGING_DIR}/${folder_name}" + log_info "Starting live pre-sync for folder '${folder_name}'" + if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then + backup_failures=$((backup_failures + 1)) + log_error "Backup failed for folder '${folder_name}' during live pre-sync" + cleanup_staged_folder "$staging_folder" || true + return 0 + fi + backup_source_path="$staging_folder" + fi + if namespace="$(namespace_for_folder "$folder_name")"; then has_mapping=1 mapped_folders=$((mapped_folders + 1)) @@ -437,6 +533,13 @@ process_folder() { log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..." sleep "$SCALE_WAIT_SECONDS" + + if [[ -n "$staging_folder" ]]; then + log_info "Starting final offline rsync delta for namespace '${namespace}'" + if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then + final_sync_failed=1 + fi + fi else namespace_match_status=$? unmapped_folders=$((unmapped_folders + 1)) @@ -447,7 +550,21 @@ process_folder() { fi fi - if backup_folder "$folder_path" "$archive_path"; then + if [[ "$has_mapping" -eq 1 && -n "$staging_folder" ]]; then + restore_namespace_replicas "$namespace" "$state_file" + log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..." + sleep "$SCALE_WAIT_SECONDS" + rm -f "$state_file" + fi + + if (( final_sync_failed == 1 )); then + backup_failures=$((backup_failures + 1)) + log_error "Backup failed for folder '${folder_name}' during final offline rsync" + cleanup_staged_folder "$staging_folder" || true + return 0 + fi + + if backup_folder "$backup_source_path" "$archive_path" "${RUN_STAGING_DIR:-}"; then backup_successes=$((backup_successes + 1)) log_info "Added folder '${folder_name}' to archive: ${archive_path}" else @@ -455,12 +572,16 @@ process_folder() { log_error "Backup failed for folder '${folder_name}'" fi - if [[ "$has_mapping" -eq 1 ]]; then + if [[ "$has_mapping" -eq 1 && -z "$staging_folder" ]]; then restore_namespace_replicas "$namespace" "$state_file" log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..." sleep "$SCALE_WAIT_SECONDS" rm -f "$state_file" fi + + if [[ -n "$staging_folder" ]]; then + cleanup_staged_folder "$staging_folder" || true + fi } print_summary() { diff --git a/k8s/examples/automated-nfs-backup.env.example b/k8s/examples/automated-nfs-backup.env.example index c6b33f9..85b5cc1 100644 --- a/k8s/examples/automated-nfs-backup.env.example +++ b/k8s/examples/automated-nfs-backup.env.example @@ -5,6 +5,8 @@ KUBECTL_BIN=kubectl KUBE_CONTEXT= WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default +BACKUP_STAGING_PATH= +RSYNC_BIN=rsync ARCHIVE_PREFIX=nfs-backup ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S SEVENZ_METHOD=lzma2 diff --git a/k8s/test/test-automated-nfs-backup.sh b/k8s/test/test-automated-nfs-backup.sh index 1afc188..e5f52cd 100644 --- a/k8s/test/test-automated-nfs-backup.sh +++ b/k8s/test/test-automated-nfs-backup.sh @@ -91,6 +91,21 @@ directory_is_empty() { [[ "${#entries[@]}" -eq 0 ]] } +events_are_ordered() { + local trace_file="$1" + shift + local event + local line_number + local previous_line=0 + + for event in "$@"; do + line_number="$(grep -n -m1 -F -- "$event" "$trace_file" | cut -d: -f1)" + [[ -n "$line_number" ]] || return 1 + (( line_number > previous_line )) || return 1 + previous_line="$line_number" + done +} + trace_has_scale() { local trace_file="$1" local namespace="$2" @@ -179,6 +194,7 @@ for ((i = 1; i <= $#; i++)); do [[ "$arg" == --replicas=* ]] && replicas_arg="$arg" done printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log" + printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log" exit 0 fi done @@ -198,6 +214,7 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then folder="${4:?folder path required}" folder="${folder%/}" folder_name="${folder##*/}" + printf '7z %s\n' "$folder_name" >> "${TMPD}/event_trace.log" # Simulate a backup failure for configured folders BEFORE recording a # successful archive, so 7z_trace.log only lists folders actually archived. @@ -210,6 +227,35 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then fi exit 0 +EOF + + write_fake_script "${TMPD}/bin/rsync" <<'EOF' +#!/usr/bin/env bash +set -uo pipefail + +source_folder="${@: -2:1}" +staging_folder="${@: -1}" +source_folder="${source_folder%/}" +staging_folder="${staging_folder%/}" +folder_name="${source_folder##*/}" + +mkdir -p "${TMPD}/rsync_counts" +count_file="${TMPD}/rsync_counts/${folder_name}" +count=0 +if [[ -f "$count_file" ]]; then + count="$(<"$count_file")" +fi +count=$((count + 1)) +printf '%s\n' "$count" > "$count_file" +printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log" + +if (( count == 2 )) && [[ -f "${TMPD}/fail_rsync_second_pass.txt" ]] && grep -Fxq -- "$folder_name" "${TMPD}/fail_rsync_second_pass.txt"; then + printf 'simulated final rsync failure for %s\n' "$folder_name" >&2 + exit 23 +fi + +mkdir -p "$staging_folder" +cp -a "${source_folder}/." "$staging_folder/" EOF write_fake_script "${TMPD}/bin/curl" <<'EOF' @@ -243,11 +289,8 @@ run_backup() { local output_dir="$2" local state_dir="$3" local log_file="$4" - local -a env_overrides=() - - if (( $# >= 5 )); then - env_overrides+=("EXCLUDED_NAMESPACES=$5") - fi + shift 4 + local -a env_overrides=("$@") env "${env_overrides[@]}" \ NFS_SOURCE_PATH="$source_dir" \ @@ -258,6 +301,7 @@ run_backup() { NOTIFY_FAILURE_URL="http://example.invalid/failure" \ KUBECTL_BIN="${TMPD}/bin/kubectl" \ SEVENZ_BIN="${TMPD}/bin/7z" \ + RSYNC_BIN="${TMPD}/bin/rsync" \ SCALE_WAIT_SECONDS=0 \ TMP_STATE_DIR="$state_dir" \ PATH="${TMPD}/bin:${PATH}" \ @@ -284,6 +328,7 @@ printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt" : > "${TMPD}/fail_folders.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" +: > "${TMPD}/7z_invocations.log" : > "${TMPD}/curl_trace.log" set +e @@ -321,7 +366,7 @@ printf 'app-ns:3\n' > "${TMPD}/replicas.txt" : > "${TMPD}/curl_trace.log" set +e -run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns" +run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "EXCLUDED_NAMESPACES=app-ns" custom_exclude_exit_code=$? set -e @@ -329,6 +374,72 @@ assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_ex assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns" assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace" +# Scenario A3: two-pass rsync copies data while live, captures a small delta +# while offline, restores the namespace, and only then starts compression. +staging_dir="${TMPD}/staging" +staging_source="${staging_dir}/source" +staging_output="${staging_dir}/out" +staging_state="${staging_dir}/state" +staging_work="${staging_dir}/work" +mkdir -p "${staging_source}/app-ns" "$staging_output" "$staging_state" "$staging_work" +printf 'data\n' > "${staging_source}/app-ns/file.txt" +printf 'app-ns\n' > "${TMPD}/namespaces.txt" +printf 'app-ns:3\n' > "${TMPD}/replicas.txt" +: > "${TMPD}/fail_folders.txt" +: > "${TMPD}/fail_rsync_second_pass.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/7z_trace.log" +: > "${TMPD}/7z_invocations.log" +: > "${TMPD}/curl_trace.log" +: > "${TMPD}/event_trace.log" +rm -rf "${TMPD}/rsync_counts" + +set +e +run_backup "$staging_source" "$staging_output" "$staging_state" "${staging_dir}/staging.log" "BACKUP_STAGING_PATH=${staging_work}" +staging_exit_code=$? +set -e + +assert "staging scenario exits with code 0" is_zero "$staging_exit_code" +assert "staging scenario creates an archive" archive_exists "$staging_output" +assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \ + "rsync app-ns 1" \ + "scale app-ns --replicas=0" \ + "rsync app-ns 2" \ + "scale app-ns --replicas=3" \ + "7z app-ns" +assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-" +assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work" + +# Scenario A4: a failed final delta never archives the incomplete staging copy, +# but the namespace is still restored immediately. +rsync_failure_dir="${TMPD}/rsync-failure" +rsync_failure_source="${rsync_failure_dir}/source" +rsync_failure_output="${rsync_failure_dir}/out" +rsync_failure_state="${rsync_failure_dir}/state" +rsync_failure_work="${rsync_failure_dir}/work" +mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failure_state" "$rsync_failure_work" +printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt" +printf 'app-ns\n' > "${TMPD}/namespaces.txt" +printf 'app-ns:3\n' > "${TMPD}/replicas.txt" +: > "${TMPD}/fail_folders.txt" +printf 'app-ns\n' > "${TMPD}/fail_rsync_second_pass.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/7z_trace.log" +: > "${TMPD}/curl_trace.log" +: > "${TMPD}/event_trace.log" +rm -rf "${TMPD}/rsync_counts" + +set +e +run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state" "${rsync_failure_dir}/rsync-failure.log" "BACKUP_STAGING_PATH=${rsync_failure_work}" +rsync_failure_exit_code=$? +set -e + +assert "final rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code" +assert "final rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3" +assert "final rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z app-ns" +assert "final rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work" +: > "${TMPD}/fail_rsync_second_pass.txt" + # Scenario B: one namespace backup fails, but another succeeds. Both namespaces # are restored, retention still runs, and the script reports success. failure_dir="${TMPD}/failure"