k8s nfs backup: add rsync staging path and per-run staging cleanup
- Add BACKUP_STAGING_PATH + RSYNC_BIN for staging before archive - Add cleanup_run_staging to EXIT trap to remove staging dir - Extend test harness for staging scenarios
This commit is contained in:
@@ -13,6 +13,7 @@ Behavior:
|
||||
- `kube-system`, `kube-public`, `kube-node-lease`, and `default` are excluded from scale actions by default.
|
||||
- Unmapped folder: backup still runs, Kubernetes scale actions are skipped.
|
||||
- Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again.
|
||||
- When `BACKUP_STAGING_PATH` is configured, a live rsync pre-copy is followed by a final offline delta; the namespace is restored before the staged copy is compressed.
|
||||
- Scale-down issues are warnings by policy (backup still runs).
|
||||
- Restore issues are warnings by policy (run can still complete successfully).
|
||||
- Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.
|
||||
@@ -29,6 +30,8 @@ Optional:
|
||||
- `KUBE_CONTEXT` (default: empty)
|
||||
- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`)
|
||||
- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease,default`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions.
|
||||
- `BACKUP_STAGING_PATH` (default: empty, disabled): Local staging parent for two-pass rsync. It must be outside `NFS_SOURCE_PATH` and have enough free space for the largest folder being backed up.
|
||||
- `RSYNC_BIN` (default: `rsync`)
|
||||
- `ARCHIVE_PREFIX` (default: `nfs-backup`)
|
||||
- `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`)
|
||||
- `SEVENZ_METHOD` (default: `lzma2`)
|
||||
@@ -62,6 +65,7 @@ Notification payload (success and failure):
|
||||
- Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces.
|
||||
- Ensure `kubectl` context and credentials are present in the runtime.
|
||||
- Ensure `7z` is installed in the runtime image/host.
|
||||
- When `BACKUP_STAGING_PATH` is enabled, ensure `rsync` is installed and place staging on fast local storage for the shortest offline delta.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ source "${SCRIPT_DIR}/lib/common.sh"
|
||||
# Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
|
||||
# every scaled-down namespace is restored regardless of how we exit.
|
||||
trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
|
||||
trap '_exit_code=$?; restore_all_remaining; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
|
||||
trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
|
||||
|
||||
# Required configuration
|
||||
require_env "NFS_SOURCE_PATH"
|
||||
@@ -20,6 +20,8 @@ KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
|
||||
KUBE_CONTEXT="${KUBE_CONTEXT:-}"
|
||||
WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
|
||||
EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease,default}"
|
||||
BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-}"
|
||||
RSYNC_BIN="${RSYNC_BIN:-rsync}"
|
||||
ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
|
||||
ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
|
||||
SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
|
||||
@@ -38,6 +40,7 @@ NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
|
||||
NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}"
|
||||
CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
|
||||
RUN_STATE_DIR=""
|
||||
RUN_STAGING_DIR=""
|
||||
|
||||
KUBECTL_ARGS=()
|
||||
if [[ -n "$KUBE_CONTEXT" ]]; then
|
||||
@@ -94,6 +97,9 @@ validate_inputs() {
|
||||
require_cmd "$KUBECTL_BIN"
|
||||
require_cmd "$SEVENZ_BIN"
|
||||
require_cmd "mktemp"
|
||||
if [[ -n "$BACKUP_STAGING_PATH" ]]; then
|
||||
require_cmd "$RSYNC_BIN"
|
||||
fi
|
||||
if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
|
||||
require_cmd "curl"
|
||||
fi
|
||||
@@ -117,6 +123,23 @@ validate_inputs() {
|
||||
fi
|
||||
log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
|
||||
|
||||
if [[ -n "$BACKUP_STAGING_PATH" ]]; then
|
||||
local source_real
|
||||
local staging_real
|
||||
mkdir -p "$BACKUP_STAGING_PATH"
|
||||
source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
|
||||
staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
|
||||
case "${staging_real}/" in
|
||||
"${source_real%/}/"*)
|
||||
die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
|
||||
;;
|
||||
esac
|
||||
if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
|
||||
die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
|
||||
fi
|
||||
log_info "Two-pass rsync staging enabled: ${RUN_STAGING_DIR}"
|
||||
fi
|
||||
|
||||
if ! _kubectl get namespaces >/dev/null 2>&1; then
|
||||
die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
|
||||
fi
|
||||
@@ -266,6 +289,50 @@ restore_all_remaining() {
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_run_staging() {
|
||||
if [[ -z "${RUN_STAGING_DIR:-}" ]] || [[ ! -d "${RUN_STAGING_DIR:-}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
case "$RUN_STAGING_DIR" in
|
||||
"${BACKUP_STAGING_PATH%/}/run-"*)
|
||||
rm -rf -- "$RUN_STAGING_DIR"
|
||||
;;
|
||||
*)
|
||||
log_error "Refusing to remove unexpected staging directory: ${RUN_STAGING_DIR}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
cleanup_staged_folder() {
|
||||
local staging_folder="${1:?staging folder required}"
|
||||
case "$staging_folder" in
|
||||
"$RUN_STAGING_DIR"/*)
|
||||
rm -rf -- "$staging_folder"
|
||||
;;
|
||||
*)
|
||||
log_error "Refusing to remove staging folder outside the current run: ${staging_folder}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
sync_folder_to_staging() {
|
||||
local source_folder="${1:?source folder required}"
|
||||
local staging_folder="${2:?staging folder required}"
|
||||
local sync_output
|
||||
|
||||
mkdir -p "$staging_folder"
|
||||
if ! sync_output="$("$RSYNC_BIN" -a --delete -- "${source_folder%/}/" "${staging_folder%/}/" 2>&1)"; then
|
||||
log_error "rsync failed from '${source_folder}' to '${staging_folder}'"
|
||||
if [[ -n "$sync_output" ]]; then
|
||||
log_error "rsync output: ${sync_output}"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
archive_path_for_run() {
|
||||
local ts
|
||||
ts="$(date +"$ARCHIVE_TS_FORMAT")"
|
||||
@@ -351,10 +418,20 @@ EOF
|
||||
backup_folder() {
|
||||
local folder_path="${1:?folder path required}"
|
||||
local archive_path="${2:?archive path required}"
|
||||
local working_directory="${3:-}"
|
||||
local archive_input="$folder_path"
|
||||
|
||||
if [[ -n "$working_directory" ]]; then
|
||||
archive_input="$(basename "$folder_path")"
|
||||
if [[ "$archive_path" != /* ]]; then
|
||||
archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
|
||||
fi
|
||||
fi
|
||||
|
||||
local -a cmd=(
|
||||
"$SEVENZ_BIN" a -t7z
|
||||
"$archive_path"
|
||||
"$folder_path"
|
||||
"$archive_input"
|
||||
"-m0=${SEVENZ_METHOD}"
|
||||
"-mx=${SEVENZ_LEVEL}"
|
||||
"-mmt=${SEVENZ_THREADS}"
|
||||
@@ -365,7 +442,11 @@ backup_folder() {
|
||||
fi
|
||||
|
||||
local cmd_output rc filtered_output
|
||||
if [[ -n "$working_directory" ]]; then
|
||||
cmd_output="$(cd "$working_directory" && "${cmd[@]}" 2>&1 >/dev/null)"
|
||||
else
|
||||
cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)"
|
||||
fi
|
||||
rc=$?
|
||||
filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
|
||||
if [[ -n "$filtered_output" ]]; then
|
||||
@@ -421,11 +502,26 @@ process_folder() {
|
||||
local state_file=""
|
||||
local has_mapping=0
|
||||
local namespace_match_status=0
|
||||
local backup_source_path="$folder_path"
|
||||
local staging_folder=""
|
||||
local final_sync_failed=0
|
||||
|
||||
folder_name="$(basename "$folder_path")"
|
||||
total_folders=$((total_folders + 1))
|
||||
log_info "Processing folder: ${folder_name}"
|
||||
|
||||
if [[ -n "$RUN_STAGING_DIR" ]]; then
|
||||
staging_folder="${RUN_STAGING_DIR}/${folder_name}"
|
||||
log_info "Starting live pre-sync for folder '${folder_name}'"
|
||||
if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
|
||||
backup_failures=$((backup_failures + 1))
|
||||
log_error "Backup failed for folder '${folder_name}' during live pre-sync"
|
||||
cleanup_staged_folder "$staging_folder" || true
|
||||
return 0
|
||||
fi
|
||||
backup_source_path="$staging_folder"
|
||||
fi
|
||||
|
||||
if namespace="$(namespace_for_folder "$folder_name")"; then
|
||||
has_mapping=1
|
||||
mapped_folders=$((mapped_folders + 1))
|
||||
@@ -437,6 +533,13 @@ process_folder() {
|
||||
|
||||
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
|
||||
sleep "$SCALE_WAIT_SECONDS"
|
||||
|
||||
if [[ -n "$staging_folder" ]]; then
|
||||
log_info "Starting final offline rsync delta for namespace '${namespace}'"
|
||||
if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
|
||||
final_sync_failed=1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
namespace_match_status=$?
|
||||
unmapped_folders=$((unmapped_folders + 1))
|
||||
@@ -447,7 +550,21 @@ process_folder() {
|
||||
fi
|
||||
fi
|
||||
|
||||
if backup_folder "$folder_path" "$archive_path"; then
|
||||
if [[ "$has_mapping" -eq 1 && -n "$staging_folder" ]]; then
|
||||
restore_namespace_replicas "$namespace" "$state_file"
|
||||
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
|
||||
sleep "$SCALE_WAIT_SECONDS"
|
||||
rm -f "$state_file"
|
||||
fi
|
||||
|
||||
if (( final_sync_failed == 1 )); then
|
||||
backup_failures=$((backup_failures + 1))
|
||||
log_error "Backup failed for folder '${folder_name}' during final offline rsync"
|
||||
cleanup_staged_folder "$staging_folder" || true
|
||||
return 0
|
||||
fi
|
||||
|
||||
if backup_folder "$backup_source_path" "$archive_path" "${RUN_STAGING_DIR:-}"; then
|
||||
backup_successes=$((backup_successes + 1))
|
||||
log_info "Added folder '${folder_name}' to archive: ${archive_path}"
|
||||
else
|
||||
@@ -455,12 +572,16 @@ process_folder() {
|
||||
log_error "Backup failed for folder '${folder_name}'"
|
||||
fi
|
||||
|
||||
if [[ "$has_mapping" -eq 1 ]]; then
|
||||
if [[ "$has_mapping" -eq 1 && -z "$staging_folder" ]]; then
|
||||
restore_namespace_replicas "$namespace" "$state_file"
|
||||
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
|
||||
sleep "$SCALE_WAIT_SECONDS"
|
||||
rm -f "$state_file"
|
||||
fi
|
||||
|
||||
if [[ -n "$staging_folder" ]]; then
|
||||
cleanup_staged_folder "$staging_folder" || true
|
||||
fi
|
||||
}
|
||||
|
||||
print_summary() {
|
||||
|
||||
@@ -5,6 +5,8 @@ KUBECTL_BIN=kubectl
|
||||
KUBE_CONTEXT=
|
||||
WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller
|
||||
EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default
|
||||
BACKUP_STAGING_PATH=
|
||||
RSYNC_BIN=rsync
|
||||
ARCHIVE_PREFIX=nfs-backup
|
||||
ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S
|
||||
SEVENZ_METHOD=lzma2
|
||||
|
||||
@@ -91,6 +91,21 @@ directory_is_empty() {
|
||||
[[ "${#entries[@]}" -eq 0 ]]
|
||||
}
|
||||
|
||||
events_are_ordered() {
|
||||
local trace_file="$1"
|
||||
shift
|
||||
local event
|
||||
local line_number
|
||||
local previous_line=0
|
||||
|
||||
for event in "$@"; do
|
||||
line_number="$(grep -n -m1 -F -- "$event" "$trace_file" | cut -d: -f1)"
|
||||
[[ -n "$line_number" ]] || return 1
|
||||
(( line_number > previous_line )) || return 1
|
||||
previous_line="$line_number"
|
||||
done
|
||||
}
|
||||
|
||||
trace_has_scale() {
|
||||
local trace_file="$1"
|
||||
local namespace="$2"
|
||||
@@ -179,6 +194,7 @@ for ((i = 1; i <= $#; i++)); do
|
||||
[[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
|
||||
done
|
||||
printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
|
||||
printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
@@ -198,6 +214,7 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
|
||||
folder="${4:?folder path required}"
|
||||
folder="${folder%/}"
|
||||
folder_name="${folder##*/}"
|
||||
printf '7z %s\n' "$folder_name" >> "${TMPD}/event_trace.log"
|
||||
|
||||
# Simulate a backup failure for configured folders BEFORE recording a
|
||||
# successful archive, so 7z_trace.log only lists folders actually archived.
|
||||
@@ -210,6 +227,35 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
|
||||
fi
|
||||
|
||||
exit 0
|
||||
EOF
|
||||
|
||||
write_fake_script "${TMPD}/bin/rsync" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
source_folder="${@: -2:1}"
|
||||
staging_folder="${@: -1}"
|
||||
source_folder="${source_folder%/}"
|
||||
staging_folder="${staging_folder%/}"
|
||||
folder_name="${source_folder##*/}"
|
||||
|
||||
mkdir -p "${TMPD}/rsync_counts"
|
||||
count_file="${TMPD}/rsync_counts/${folder_name}"
|
||||
count=0
|
||||
if [[ -f "$count_file" ]]; then
|
||||
count="$(<"$count_file")"
|
||||
fi
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "$count" > "$count_file"
|
||||
printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
|
||||
|
||||
if (( count == 2 )) && [[ -f "${TMPD}/fail_rsync_second_pass.txt" ]] && grep -Fxq -- "$folder_name" "${TMPD}/fail_rsync_second_pass.txt"; then
|
||||
printf 'simulated final rsync failure for %s\n' "$folder_name" >&2
|
||||
exit 23
|
||||
fi
|
||||
|
||||
mkdir -p "$staging_folder"
|
||||
cp -a "${source_folder}/." "$staging_folder/"
|
||||
EOF
|
||||
|
||||
write_fake_script "${TMPD}/bin/curl" <<'EOF'
|
||||
@@ -243,11 +289,8 @@ run_backup() {
|
||||
local output_dir="$2"
|
||||
local state_dir="$3"
|
||||
local log_file="$4"
|
||||
local -a env_overrides=()
|
||||
|
||||
if (( $# >= 5 )); then
|
||||
env_overrides+=("EXCLUDED_NAMESPACES=$5")
|
||||
fi
|
||||
shift 4
|
||||
local -a env_overrides=("$@")
|
||||
|
||||
env "${env_overrides[@]}" \
|
||||
NFS_SOURCE_PATH="$source_dir" \
|
||||
@@ -258,6 +301,7 @@ run_backup() {
|
||||
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
|
||||
KUBECTL_BIN="${TMPD}/bin/kubectl" \
|
||||
SEVENZ_BIN="${TMPD}/bin/7z" \
|
||||
RSYNC_BIN="${TMPD}/bin/rsync" \
|
||||
SCALE_WAIT_SECONDS=0 \
|
||||
TMP_STATE_DIR="$state_dir" \
|
||||
PATH="${TMPD}/bin:${PATH}" \
|
||||
@@ -284,6 +328,7 @@ printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/7z_invocations.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
|
||||
set +e
|
||||
@@ -321,7 +366,7 @@ printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
|
||||
set +e
|
||||
run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "app-ns"
|
||||
run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "EXCLUDED_NAMESPACES=app-ns"
|
||||
custom_exclude_exit_code=$?
|
||||
set -e
|
||||
|
||||
@@ -329,6 +374,72 @@ assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_ex
|
||||
assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
|
||||
assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
|
||||
|
||||
# Scenario A3: two-pass rsync copies data while live, captures a small delta
|
||||
# while offline, restores the namespace, and only then starts compression.
|
||||
staging_dir="${TMPD}/staging"
|
||||
staging_source="${staging_dir}/source"
|
||||
staging_output="${staging_dir}/out"
|
||||
staging_state="${staging_dir}/state"
|
||||
staging_work="${staging_dir}/work"
|
||||
mkdir -p "${staging_source}/app-ns" "$staging_output" "$staging_state" "$staging_work"
|
||||
printf 'data\n' > "${staging_source}/app-ns/file.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
: > "${TMPD}/fail_rsync_second_pass.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/7z_invocations.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
: > "${TMPD}/event_trace.log"
|
||||
rm -rf "${TMPD}/rsync_counts"
|
||||
|
||||
set +e
|
||||
run_backup "$staging_source" "$staging_output" "$staging_state" "${staging_dir}/staging.log" "BACKUP_STAGING_PATH=${staging_work}"
|
||||
staging_exit_code=$?
|
||||
set -e
|
||||
|
||||
assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
|
||||
assert "staging scenario creates an archive" archive_exists "$staging_output"
|
||||
assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
|
||||
"rsync app-ns 1" \
|
||||
"scale app-ns --replicas=0" \
|
||||
"rsync app-ns 2" \
|
||||
"scale app-ns --replicas=3" \
|
||||
"7z app-ns"
|
||||
assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
|
||||
assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
|
||||
|
||||
# Scenario A4: a failed final delta never archives the incomplete staging copy,
|
||||
# but the namespace is still restored immediately.
|
||||
rsync_failure_dir="${TMPD}/rsync-failure"
|
||||
rsync_failure_source="${rsync_failure_dir}/source"
|
||||
rsync_failure_output="${rsync_failure_dir}/out"
|
||||
rsync_failure_state="${rsync_failure_dir}/state"
|
||||
rsync_failure_work="${rsync_failure_dir}/work"
|
||||
mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failure_state" "$rsync_failure_work"
|
||||
printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/namespaces.txt"
|
||||
printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
|
||||
: > "${TMPD}/fail_folders.txt"
|
||||
printf 'app-ns\n' > "${TMPD}/fail_rsync_second_pass.txt"
|
||||
: > "${TMPD}/scale_trace.log"
|
||||
: > "${TMPD}/7z_trace.log"
|
||||
: > "${TMPD}/curl_trace.log"
|
||||
: > "${TMPD}/event_trace.log"
|
||||
rm -rf "${TMPD}/rsync_counts"
|
||||
|
||||
set +e
|
||||
run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state" "${rsync_failure_dir}/rsync-failure.log" "BACKUP_STAGING_PATH=${rsync_failure_work}"
|
||||
rsync_failure_exit_code=$?
|
||||
set -e
|
||||
|
||||
assert "final rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
|
||||
assert "final rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
|
||||
assert "final rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z app-ns"
|
||||
assert "final rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
|
||||
: > "${TMPD}/fail_rsync_second_pass.txt"
|
||||
|
||||
# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
|
||||
# are restored, retention still runs, and the script reports success.
|
||||
failure_dir="${TMPD}/failure"
|
||||
|
||||
Reference in New Issue
Block a user