k8s nfs backup: add rsync staging path and per-run staging cleanup
All checks were successful
Check scripts syntax / check-scripts-syntax (push) Successful in 7s
Test automated-nfs-backup / test-automated-nfs-backup (push) Successful in 8s

- Add BACKUP_STAGING_PATH + RSYNC_BIN for staging before archive
- Add cleanup_run_staging to EXIT trap to remove staging dir
- Extend test harness for staging scenarios
This commit is contained in:
2026-07-14 09:36:39 -03:00
parent 2bd9e90f00
commit 43351abb11
4 changed files with 249 additions and 11 deletions

View File

@@ -13,6 +13,7 @@ Behavior:
- `kube-system`, `kube-public`, `kube-node-lease`, and `default` are excluded from scale actions by default.
- Unmapped folder: backup still runs, Kubernetes scale actions are skipped.
- Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again.
- When `BACKUP_STAGING_PATH` is configured, a live rsync pre-copy is followed by a final offline delta; the namespace is restored before the staged copy is compressed.
- Scale-down issues are warnings by policy (backup still runs).
- Restore issues are warnings by policy (run can still complete successfully).
- Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.
@@ -29,6 +30,8 @@ Optional:
- `KUBE_CONTEXT` (default: empty)
- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`)
- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease,default`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions.
- `BACKUP_STAGING_PATH` (default: empty, disabled): Local staging parent for two-pass rsync. It must be outside `NFS_SOURCE_PATH` and have enough free space for the largest folder being backed up.
- `RSYNC_BIN` (default: `rsync`)
- `ARCHIVE_PREFIX` (default: `nfs-backup`)
- `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`)
- `SEVENZ_METHOD` (default: `lzma2`)
@@ -62,6 +65,7 @@ Notification payload (success and failure):
- Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces.
- Ensure `kubectl` context and credentials are present in the runtime.
- Ensure `7z` is installed in the runtime image/host.
- When `BACKUP_STAGING_PATH` is enabled, ensure `rsync` is installed and place staging on fast local storage for the shortest offline delta.
## Failure Semantics

View File

@@ -8,7 +8,7 @@ source "${SCRIPT_DIR}/lib/common.sh"
# Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
# every scaled-down namespace is restored regardless of how we exit.
trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
trap '_exit_code=$?; restore_all_remaining; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
# Required configuration
require_env "NFS_SOURCE_PATH"
@@ -20,6 +20,8 @@ KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
KUBE_CONTEXT="${KUBE_CONTEXT:-}"
WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease,default}"
BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-}"
RSYNC_BIN="${RSYNC_BIN:-rsync}"
ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
@@ -38,6 +40,7 @@ NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}"
CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
RUN_STATE_DIR=""
RUN_STAGING_DIR=""
KUBECTL_ARGS=()
if [[ -n "$KUBE_CONTEXT" ]]; then
@@ -94,6 +97,9 @@ validate_inputs() {
require_cmd "$KUBECTL_BIN"
require_cmd "$SEVENZ_BIN"
require_cmd "mktemp"
if [[ -n "$BACKUP_STAGING_PATH" ]]; then
require_cmd "$RSYNC_BIN"
fi
if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
require_cmd "curl"
fi
@@ -117,6 +123,23 @@ validate_inputs() {
fi
log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
if [[ -n "$BACKUP_STAGING_PATH" ]]; then
local source_real
local staging_real
mkdir -p "$BACKUP_STAGING_PATH"
source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
case "${staging_real}/" in
"${source_real%/}/"*)
die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
;;
esac
if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
fi
log_info "Two-pass rsync staging enabled: ${RUN_STAGING_DIR}"
fi
if ! _kubectl get namespaces >/dev/null 2>&1; then
die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
fi
@@ -266,6 +289,50 @@ restore_all_remaining() {
fi
}
cleanup_run_staging() {
if [[ -z "${RUN_STAGING_DIR:-}" ]] || [[ ! -d "${RUN_STAGING_DIR:-}" ]]; then
return 0
fi
case "$RUN_STAGING_DIR" in
"${BACKUP_STAGING_PATH%/}/run-"*)
rm -rf -- "$RUN_STAGING_DIR"
;;
*)
log_error "Refusing to remove unexpected staging directory: ${RUN_STAGING_DIR}"
return 1
;;
esac
}
cleanup_staged_folder() {
local staging_folder="${1:?staging folder required}"
case "$staging_folder" in
"$RUN_STAGING_DIR"/*)
rm -rf -- "$staging_folder"
;;
*)
log_error "Refusing to remove staging folder outside the current run: ${staging_folder}"
return 1
;;
esac
}
sync_folder_to_staging() {
local source_folder="${1:?source folder required}"
local staging_folder="${2:?staging folder required}"
local sync_output
mkdir -p "$staging_folder"
if ! sync_output="$("$RSYNC_BIN" -a --delete -- "${source_folder%/}/" "${staging_folder%/}/" 2>&1)"; then
log_error "rsync failed from '${source_folder}' to '${staging_folder}'"
if [[ -n "$sync_output" ]]; then
log_error "rsync output: ${sync_output}"
fi
return 1
fi
}
archive_path_for_run() {
local ts
ts="$(date +"$ARCHIVE_TS_FORMAT")"
@@ -351,10 +418,20 @@ EOF
backup_folder() {
local folder_path="${1:?folder path required}"
local archive_path="${2:?archive path required}"
local working_directory="${3:-}"
local archive_input="$folder_path"
if [[ -n "$working_directory" ]]; then
archive_input="$(basename "$folder_path")"
if [[ "$archive_path" != /* ]]; then
archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
fi
fi
local -a cmd=(
"$SEVENZ_BIN" a -t7z
"$archive_path"
"$folder_path"
"$archive_input"
"-m0=${SEVENZ_METHOD}"
"-mx=${SEVENZ_LEVEL}"
"-mmt=${SEVENZ_THREADS}"
@@ -365,7 +442,11 @@ backup_folder() {
fi
local cmd_output rc filtered_output
if [[ -n "$working_directory" ]]; then
cmd_output="$(cd "$working_directory" && "${cmd[@]}" 2>&1 >/dev/null)"
else
cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)"
fi
rc=$?
filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
if [[ -n "$filtered_output" ]]; then
@@ -421,11 +502,26 @@ process_folder() {
local state_file=""
local has_mapping=0
local namespace_match_status=0
local backup_source_path="$folder_path"
local staging_folder=""
local final_sync_failed=0
folder_name="$(basename "$folder_path")"
total_folders=$((total_folders + 1))
log_info "Processing folder: ${folder_name}"
if [[ -n "$RUN_STAGING_DIR" ]]; then
staging_folder="${RUN_STAGING_DIR}/${folder_name}"
log_info "Starting live pre-sync for folder '${folder_name}'"
if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
backup_failures=$((backup_failures + 1))
log_error "Backup failed for folder '${folder_name}' during live pre-sync"
cleanup_staged_folder "$staging_folder" || true
return 0
fi
backup_source_path="$staging_folder"
fi
if namespace="$(namespace_for_folder "$folder_name")"; then
has_mapping=1
mapped_folders=$((mapped_folders + 1))
@@ -437,6 +533,13 @@ process_folder() {
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
sleep "$SCALE_WAIT_SECONDS"
if [[ -n "$staging_folder" ]]; then
log_info "Starting final offline rsync delta for namespace '${namespace}'"
if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
final_sync_failed=1
fi
fi
else
namespace_match_status=$?
unmapped_folders=$((unmapped_folders + 1))
@@ -447,7 +550,21 @@ process_folder() {
fi
fi
if backup_folder "$folder_path" "$archive_path"; then
if [[ "$has_mapping" -eq 1 && -n "$staging_folder" ]]; then
restore_namespace_replicas "$namespace" "$state_file"
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
sleep "$SCALE_WAIT_SECONDS"
rm -f "$state_file"
fi
if (( final_sync_failed == 1 )); then
backup_failures=$((backup_failures + 1))
log_error "Backup failed for folder '${folder_name}' during final offline rsync"
cleanup_staged_folder "$staging_folder" || true
return 0
fi
if backup_folder "$backup_source_path" "$archive_path" "${RUN_STAGING_DIR:-}"; then
backup_successes=$((backup_successes + 1))
log_info "Added folder '${folder_name}' to archive: ${archive_path}"
else
@@ -455,12 +572,16 @@ process_folder() {
log_error "Backup failed for folder '${folder_name}'"
fi
if [[ "$has_mapping" -eq 1 ]]; then
if [[ "$has_mapping" -eq 1 && -z "$staging_folder" ]]; then
restore_namespace_replicas "$namespace" "$state_file"
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
sleep "$SCALE_WAIT_SECONDS"
rm -f "$state_file"
fi
if [[ -n "$staging_folder" ]]; then
cleanup_staged_folder "$staging_folder" || true
fi
}
print_summary() {

View File

@@ -5,6 +5,8 @@ KUBECTL_BIN=kubectl
KUBE_CONTEXT=
WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller
EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default
BACKUP_STAGING_PATH=
RSYNC_BIN=rsync
ARCHIVE_PREFIX=nfs-backup
ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S
SEVENZ_METHOD=lzma2

View File

@@ -91,6 +91,21 @@ directory_is_empty() {
[[ "${#entries[@]}" -eq 0 ]]
}
events_are_ordered() {
local trace_file="$1"
shift
local event
local line_number
local previous_line=0
for event in "$@"; do
line_number="$(grep -n -m1 -F -- "$event" "$trace_file" | cut -d: -f1)"
[[ -n "$line_number" ]] || return 1
(( line_number > previous_line )) || return 1
previous_line="$line_number"
done
}
trace_has_scale() {
local trace_file="$1"
local namespace="$2"
@@ -179,6 +194,7 @@ for ((i = 1; i <= $#; i++)); do
[[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
done
printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log"
exit 0
fi
done
@@ -198,6 +214,7 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
folder="${4:?folder path required}"
folder="${folder%/}"
folder_name="${folder##*/}"
printf '7z %s\n' "$folder_name" >> "${TMPD}/event_trace.log"
# Simulate a backup failure for configured folders BEFORE recording a
# successful archive, so 7z_trace.log only lists folders actually archived.
@@ -210,6 +227,35 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
fi
exit 0
EOF
write_fake_script "${TMPD}/bin/rsync" <<'EOF'
#!/usr/bin/env bash
set -uo pipefail
source_folder="${@: -2:1}"
staging_folder="${@: -1}"
source_folder="${source_folder%/}"
staging_folder="${staging_folder%/}"
folder_name="${source_folder##*/}"
mkdir -p "${TMPD}/rsync_counts"
count_file="${TMPD}/rsync_counts/${folder_name}"
count=0
if [[ -f "$count_file" ]]; then
count="$(<"$count_file")"
fi
count=$((count + 1))
printf '%s\n' "$count" > "$count_file"
printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
if (( count == 2 )) && [[ -f "${TMPD}/fail_rsync_second_pass.txt" ]] && grep -Fxq -- "$folder_name" "${TMPD}/fail_rsync_second_pass.txt"; then
printf 'simulated final rsync failure for %s\n' "$folder_name" >&2
exit 23
fi
mkdir -p "$staging_folder"
cp -a "${source_folder}/." "$staging_folder/"
EOF
write_fake_script "${TMPD}/bin/curl" <<'EOF'
@@ -243,11 +289,8 @@ run_backup() {
local output_dir="$2"
local state_dir="$3"
local log_file="$4"
local -a env_overrides=()
if (( $# >= 5 )); then
env_overrides+=("EXCLUDED_NAMESPACES=$5")
fi
shift 4
local -a env_overrides=("$@")
env "${env_overrides[@]}" \
NFS_SOURCE_PATH="$source_dir" \
@@ -258,6 +301,7 @@ run_backup() {
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
KUBECTL_BIN="${TMPD}/bin/kubectl" \
SEVENZ_BIN="${TMPD}/bin/7z" \
RSYNC_BIN="${TMPD}/bin/rsync" \
SCALE_WAIT_SECONDS=0 \
TMP_STATE_DIR="$state_dir" \
PATH="${TMPD}/bin:${PATH}" \
@@ -284,6 +328,7 @@ printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
: > "${TMPD}/fail_folders.txt"
: > "${TMPD}/scale_trace.log"
: > "${TMPD}/7z_trace.log"
: > "${TMPD}/7z_invocations.log"
: > "${TMPD}/curl_trace.log"
set +e
@@ -321,7 +366,7 @@ printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
: > "${TMPD}/curl_trace.log"
set +e
run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns"
run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "EXCLUDED_NAMESPACES=app-ns"
custom_exclude_exit_code=$?
set -e
@@ -329,6 +374,72 @@ assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_ex
assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
# Scenario A3: two-pass rsync copies data while live, captures a small delta
# while offline, restores the namespace, and only then starts compression.
staging_dir="${TMPD}/staging"
staging_source="${staging_dir}/source"
staging_output="${staging_dir}/out"
staging_state="${staging_dir}/state"
staging_work="${staging_dir}/work"
mkdir -p "${staging_source}/app-ns" "$staging_output" "$staging_state" "$staging_work"
printf 'data\n' > "${staging_source}/app-ns/file.txt"
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
: > "${TMPD}/fail_folders.txt"
: > "${TMPD}/fail_rsync_second_pass.txt"
: > "${TMPD}/scale_trace.log"
: > "${TMPD}/7z_trace.log"
: > "${TMPD}/7z_invocations.log"
: > "${TMPD}/curl_trace.log"
: > "${TMPD}/event_trace.log"
rm -rf "${TMPD}/rsync_counts"
set +e
run_backup "$staging_source" "$staging_output" "$staging_state" "${staging_dir}/staging.log" "BACKUP_STAGING_PATH=${staging_work}"
staging_exit_code=$?
set -e
assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
assert "staging scenario creates an archive" archive_exists "$staging_output"
assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
"rsync app-ns 1" \
"scale app-ns --replicas=0" \
"rsync app-ns 2" \
"scale app-ns --replicas=3" \
"7z app-ns"
assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
# Scenario A4: a failed final delta never archives the incomplete staging copy,
# but the namespace is still restored immediately.
rsync_failure_dir="${TMPD}/rsync-failure"
rsync_failure_source="${rsync_failure_dir}/source"
rsync_failure_output="${rsync_failure_dir}/out"
rsync_failure_state="${rsync_failure_dir}/state"
rsync_failure_work="${rsync_failure_dir}/work"
mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failure_state" "$rsync_failure_work"
printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
: > "${TMPD}/fail_folders.txt"
printf 'app-ns\n' > "${TMPD}/fail_rsync_second_pass.txt"
: > "${TMPD}/scale_trace.log"
: > "${TMPD}/7z_trace.log"
: > "${TMPD}/curl_trace.log"
: > "${TMPD}/event_trace.log"
rm -rf "${TMPD}/rsync_counts"
set +e
run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state" "${rsync_failure_dir}/rsync-failure.log" "BACKUP_STAGING_PATH=${rsync_failure_work}"
rsync_failure_exit_code=$?
set -e
assert "final rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
assert "final rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
assert "final rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z app-ns"
assert "final rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
: > "${TMPD}/fail_rsync_second_pass.txt"
# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
# are restored, retention still runs, and the script reports success.
failure_dir="${TMPD}/failure"