diff --git a/k8s/README.md b/k8s/README.md index 5333165..249a6cc 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,10 +13,12 @@ Behavior: - `kube-system`, `kube-public`, and `kube-node-lease` are excluded from scale actions by default. - Unmapped or excluded folder: copied to local staging without Kubernetes scale actions. - Mapped folder: saves replicas, scales selected workloads down, waits, copies the quiesced folder to local staging, restores replicas, and waits again. -- After every folder has been processed, one `7z` invocation compresses the complete run staging directory. Workloads are therefore restored before compression starts. -- Per-run replica state and staged data are removed on exit. The staging parent is retained. -- Scale-down issues are warnings by policy (backup still runs). -- Restore issues are warnings by policy (run can still complete successfully). +- When `argocd` exists, its Deployments and StatefulSets are saved and scaled to zero first, even if excluded or lacking a source folder. The script waits for its pods to disappear and restores ArgoCD last, after compression or during exit cleanup. +- After every folder has been processed, one `7z` invocation compresses the complete run staging directory. Other workloads are restored before compression starts. +- Failed state capture or scaling aborts the run. Failed restoration is retried at exit; unrecovered replica state is retained and the run exits nonzero. INT and TERM also trigger restoration; forced termination or host failure requires manual recovery from saved state. +- Staged data is removed on exit. The staging parent and lock file are retained. Runs sharing `TMP_STATE_DIR` cannot overlap (`flock` required); use the same state directory for all invocations targeting the cluster. +- Existing archive paths are rejected; 7z warnings count as failures. Output, state and staging directories must be outside the source tree. +- Other reconcilers (such as operators and HPAs) and writers outside selected workload kinds are not suspended. Ordinary namespaces use a fixed shutdown delay; choose it to cover their termination time. - Cleanup retains the `N` most recent matching archives by modification time and deletes older ones. ### `backup-k3s-control-plane-config.sh` @@ -60,6 +62,7 @@ Optional: - `SCALE_RETRY_COUNT` (default: `3`) - `SCALE_RETRY_DELAY_SECONDS` (default: `5`) - `SCALE_WAIT_SECONDS` (default: `30`): Fixed delay after scaling down and after restoring a mapped namespace. +- `ARGOCD_WAIT_TIMEOUT` (default: `300s`): Maximum wait for ArgoCD pods to disappear before processing folders. - `LOG_LEVEL` (default: `info`) - `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`): Parent directory for fresh per-run replica-state directories. A run never restores state files belonging to an earlier run. - `NOTIFY_SUCCESS_URL` (default: `http://notify.haven/api/v2/notifications/backup`; set to empty to disable) diff --git a/k8s/automated-nfs-backup.sh b/k8s/automated-nfs-backup.sh index ded04ca..bb1845f 100644 --- a/k8s/automated-nfs-backup.sh +++ b/k8s/automated-nfs-backup.sh @@ -8,7 +8,6 @@ source "${SCRIPT_DIR}/lib/common.sh" # Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees # every scaled-down namespace is restored regardless of how we exit. trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR -trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT # Required configuration require_env "NFS_SOURCE_PATH" @@ -32,6 +31,7 @@ SEVENZ_BIN="${SEVENZ_BIN:-7z}" SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}" SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}" SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}" +ARGOCD_WAIT_TIMEOUT="${ARGOCD_WAIT_TIMEOUT:-300s}" TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}" NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}" NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}" @@ -99,6 +99,7 @@ validate_inputs() { require_cmd "$KUBECTL_BIN" require_cmd "$SEVENZ_BIN" require_cmd "mktemp" + require_cmd "flock" require_cmd "$RSYNC_BIN" if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then require_cmd "curl" @@ -118,6 +119,8 @@ validate_inputs() { mkdir -p "$BACKUP_OUTPUT_PATH" mkdir -p "$TMP_STATE_DIR" + exec {BACKUP_LOCK_FD}>"${TMP_STATE_DIR%/}/backup.lock" + flock -n "$BACKUP_LOCK_FD" || die "Another NFS backup is running (lock: ${TMP_STATE_DIR}/backup.lock)" if ! RUN_STATE_DIR="$(mktemp -d "${TMP_STATE_DIR%/}/run-XXXXXXXXXX")"; then die "Unable to create a per-run state directory under ${TMP_STATE_DIR}" fi @@ -125,8 +128,15 @@ validate_inputs() { local source_real local staging_real + local destination_real mkdir -p "$BACKUP_STAGING_PATH" source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)" + for destination_real in "$BACKUP_OUTPUT_PATH" "$TMP_STATE_DIR"; do + destination_real="$(cd "$destination_real" && pwd -P)" + case "${destination_real}/" in + "${source_real%/}/"*) die "Backup output and state directories must be outside NFS_SOURCE_PATH" ;; + esac + done staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)" case "${staging_real}/" in "${source_real%/}/"*) @@ -144,11 +154,12 @@ validate_inputs() { } load_namespaces() { - local ns + local ns output + output="$(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')" || die "Unable to load Kubernetes namespaces" while IFS= read -r ns; do [[ -z "$ns" ]] && continue NAMESPACE_MAP["$ns"]=1 - done < <(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + done <<< "$output" } namespace_for_folder() { @@ -182,7 +193,7 @@ capture_replicas_state() { if ! output="$(_kubectl -n "$namespace" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.replicas}{"\t"}{range .metadata.ownerReferences[*]}{.kind},{end}{"\n"}{end}' 2>/dev/null)"; then log_warn "Failed to list kind '${kind}' in namespace '${namespace}' while capturing state" scale_warnings=$((scale_warnings + 1)) - continue + return 1 fi while IFS=$'\t' read -r name replicas owners; do @@ -191,7 +202,7 @@ capture_replicas_state() { # ReplicaSet owned by a Deployment) — scaling them directly # conflicts with the owning controller and can leave workloads # in an inconsistent state after restore. - if [[ -n "$owners" ]]; then + if [[ -n "$owners" && "$namespace" != argocd ]]; then log_debug "Skipping ${kind}/${name} in namespace '${namespace}': owned by ${owners%,}" continue fi @@ -226,8 +237,9 @@ scale_namespace_to_zero() { if scale_resource "$namespace" "$kind" "$name" "0"; then log_info "Scaled down ${namespace}:${kind}/${name} to 0" else - log_warn "Failed to scale down ${namespace}:${kind}/${name}; continuing with backup by policy" + log_warn "Failed to scale down ${namespace}:${kind}/${name}" scale_warnings=$((scale_warnings + 1)) + return 1 fi done < "$state_file" } @@ -238,6 +250,7 @@ restore_namespace_replicas() { local kind local name local replicas + local failed=0 while IFS=$'\t' read -r kind name replicas; do [[ -z "$kind" || -z "$name" ]] && continue @@ -246,8 +259,10 @@ restore_namespace_replicas() { else log_warn "Failed to restore ${namespace}:${kind}/${name} to ${replicas}" restore_warnings=$((restore_warnings + 1)) + failed=1 fi done < "$state_file" + return "$failed" } restore_all_remaining() { @@ -257,34 +272,54 @@ restore_all_remaining() { local state_file local namespace - local kind name replicas - local found=0 + local failed=0 + local argocd_seen=0 - for state_file in "${RUN_STATE_DIR}"/*.state; do + # ArgoCD must be restored last, including on abnormal exits. + for state_file in "${RUN_STATE_DIR}"/*.state "${RUN_STATE_DIR}/argocd.state"; do [[ -f "$state_file" ]] || continue - found=1 namespace="$(basename "$state_file" .state)" + if [[ "$namespace" == argocd && "${argocd_seen:-0}" == 0 ]]; then + argocd_seen=1 + continue + fi [[ -z "$namespace" ]] && continue log_info "EXIT cleanup: restoring workloads in namespace '${namespace}'" - while IFS=$'\t' read -r kind name replicas; do - [[ -z "$kind" || -z "$name" ]] && continue - if scale_resource "$namespace" "$kind" "$name" "$replicas"; then - log_info "EXIT cleanup: restored ${namespace}:${kind}/${name} to ${replicas}" - else - log_warn "EXIT cleanup: failed to restore ${namespace}:${kind}/${name} to ${replicas}" - restore_warnings=$((restore_warnings + 1)) - fi - done < "$state_file" - rm -f "$state_file" + if restore_namespace_replicas "$namespace" "$state_file"; then + rm -f "$state_file" + else + failed=1 + log_error "Replica restoration failed; saved state retained at ${state_file}" + fi done - if [[ "$found" -eq 1 ]]; then - log_info "EXIT cleanup complete - all remaining workloads restored" - fi - if ! rmdir -- "$RUN_STATE_DIR" 2>/dev/null; then log_warn "Run state directory is not empty; leaving it in place: ${RUN_STATE_DIR}" fi + return "$failed" +} + +finish_run() { + local code="$1" + trap - ERR INT TERM + restore_all_remaining || code=1 + cleanup_run_staging || code=1 + on_exit "$code" + exit "$code" +} + +pause_argocd() { + [[ -n "${NAMESPACE_MAP[argocd]:-}" ]] || return 0 + local state_file + state_file="$(state_file_for_namespace argocd)" + # Independent of exclusions and WORKLOAD_KINDS: the application controller + # can be a Deployment or StatefulSet, and may be operator-owned. + local -a WORKLOAD_KIND_LIST=(deployment statefulset) + capture_replicas_state argocd "$state_file" || die "Unable to capture ArgoCD replicas" + scale_namespace_to_zero argocd "$state_file" || die "Unable to stop ArgoCD" + # A successful scale only changes desired replicas. Wait for the actual + # controllers to exit before allowing any other namespace to scale down. + _kubectl -n argocd wait --for=delete pod --all --timeout="$ARGOCD_WAIT_TIMEOUT" || die "ArgoCD pods did not stop" } cleanup_run_staging() { @@ -437,8 +472,6 @@ archive_staging() { fi if (( rc == 0 )); then return 0 - elif (( rc == 1 )); then - return 0 fi return 1 } @@ -492,14 +525,19 @@ process_folder() { total_folders=$((total_folders + 1)) log_info "Processing folder: ${folder_name}" - if namespace="$(namespace_for_folder "$folder_name")"; then + if [[ "$folder_name" == argocd && -n "${NAMESPACE_MAP[argocd]:-}" ]]; then + mapped_folders=$((mapped_folders + 1)) + if sync_folder_to_staging "$folder_path" "$staging_folder"; then + copy_succeeded=1 + fi + elif namespace="$(namespace_for_folder "$folder_name")"; then has_mapping=1 mapped_folders=$((mapped_folders + 1)) log_info "Exact namespace match found for folder '${folder_name}' -> namespace '${namespace}'" state_file="$(state_file_for_namespace "$namespace")" - capture_replicas_state "$namespace" "$state_file" - scale_namespace_to_zero "$namespace" "$state_file" + capture_replicas_state "$namespace" "$state_file" || die "Unable to capture replicas for ${namespace}" + scale_namespace_to_zero "$namespace" "$state_file" || die "Unable to stop ${namespace}" log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..." sleep "$SCALE_WAIT_SECONDS" @@ -524,7 +562,7 @@ process_folder() { fi if [[ "$has_mapping" -eq 1 ]]; then - restore_namespace_replicas "$namespace" "$state_file" + restore_namespace_replicas "$namespace" "$state_file" || die "Unable to restore ${namespace}; EXIT cleanup will retry" log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..." sleep "$SCALE_WAIT_SECONDS" rm -f "$state_file" @@ -550,11 +588,13 @@ main() { parse_excluded_namespaces validate_inputs load_namespaces + pause_argocd local folder_path local run_archive_path local found=0 run_archive_path="$(archive_path_for_run)" + [[ ! -e "$run_archive_path" ]] || die "Archive already exists: ${run_archive_path}" log_info "Using single archive for this run: ${run_archive_path}" for folder_path in "${NFS_SOURCE_PATH}"/*; do @@ -584,6 +624,7 @@ main() { fi fi + restore_all_remaining || die "Unable to restore workloads; replica state retained" print_summary local duration_seconds=$((SECONDS - backup_started_seconds)) @@ -613,4 +654,7 @@ main() { "$duration_seconds" || true } +trap 'finish_run "$?"' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM main "$@" diff --git a/k8s/examples/automated-nfs-backup.env.example b/k8s/examples/automated-nfs-backup.env.example index aa2eb50..9fbbf4a 100644 --- a/k8s/examples/automated-nfs-backup.env.example +++ b/k8s/examples/automated-nfs-backup.env.example @@ -16,6 +16,7 @@ SEVENZ_THREADS=on SCALE_RETRY_COUNT=3 SCALE_RETRY_DELAY_SECONDS=5 SCALE_WAIT_SECONDS=30 +ARGOCD_WAIT_TIMEOUT=300s LOG_LEVEL=info TMP_STATE_DIR=/tmp/k8s-nfs-backup NOTIFY_SUCCESS_URL=http://notify.haven/api/v2/notifications/backup diff --git a/k8s/lib/common.sh b/k8s/lib/common.sh index 53535af..a1e3005 100644 --- a/k8s/lib/common.sh +++ b/k8s/lib/common.sh @@ -94,8 +94,9 @@ retry() { while true; do if "$@"; then return 0 + else + exit_code=$? fi - exit_code=$? if (( attempt >= attempts )); then return "$exit_code" fi diff --git a/k8s/test/test-automated-nfs-backup.sh b/k8s/test/test-automated-nfs-backup.sh index 71de377..3e39455 100644 --- a/k8s/test/test-automated-nfs-backup.sh +++ b/k8s/test/test-automated-nfs-backup.sh @@ -168,6 +168,7 @@ if [[ "$get_kind" == "namespaces" ]]; then fi if [[ -n "$get_kind" && -n "$namespace" ]]; then + [[ "${FAIL_CAPTURE_NS:-}" != "$namespace" ]] || exit 1 replicas="" while IFS=: read -r configured_namespace configured_replicas; do if [[ "$configured_namespace" == "$namespace" ]]; then @@ -193,10 +194,15 @@ for ((i = 1; i <= $#; i++)); do done printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log" printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log" + [[ "${FAIL_SCALE:-}" != "$namespace:$replicas_arg" ]] || exit 1 exit 0 fi done +if [[ " $* " == *" wait "* ]]; then + printf 'wait %s\n' "$namespace" >> "${TMPD}/event_trace.log" + exit "${FAIL_WAIT:-0}" +fi exit 0 EOF @@ -215,7 +221,7 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then if [[ -s "${TMPD}/fail_archive.txt" ]]; then touch "$archive" - exit 2 + exit "${ARCHIVE_FAILURE_CODE:-2}" fi for staged_folder in ./*; do @@ -303,6 +309,8 @@ run_backup() { SEVENZ_BIN="${TMPD}/bin/7z" \ RSYNC_BIN="${TMPD}/bin/rsync" \ SCALE_WAIT_SECONDS=0 \ + SCALE_RETRY_COUNT=2 \ + SCALE_RETRY_DELAY_SECONDS=0 \ TMP_STATE_DIR="$state_dir" \ PATH="${TMPD}/bin:${PATH}" \ TMPD="$TMPD" \ @@ -611,7 +619,77 @@ set -e assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}" assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0" assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5" -assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state" +assert "abort scenario removed its replica state" file_lacks "${abort_dir}/abort.log" "saved state retained" + +# ArgoCD is global, irrespective of folders, exclusions or workload overrides. +for scenario in normal no-folder scale-failure wait-failure capture-failure restore-failure warning; do + case_dir="${TMPD}/argocd-${scenario}" + mkdir -p "$case_dir/source/app-ns" "$case_dir/out" + [[ "$scenario" == no-folder ]] || mkdir -p "$case_dir/source/argocd" + printf 'argocd\napp-ns\n' > "${TMPD}/namespaces.txt" + printf 'argocd:2\napp-ns:3\n' > "${TMPD}/replicas.txt" + : > "${TMPD}/fail_folders.txt" + : > "${TMPD}/fail_archive.txt" + : > "${TMPD}/event_trace.log" + : > "${TMPD}/scale_trace.log" + overrides=(WORKLOAD_KINDS=deployment EXCLUDED_NAMESPACES=argocd) + case "$scenario" in + scale-failure) overrides+=(FAIL_SCALE=argocd:--replicas=0) ;; + wait-failure) overrides+=(FAIL_WAIT=1) ;; + capture-failure) overrides+=(FAIL_CAPTURE_NS=argocd) ;; + restore-failure) overrides+=(FAIL_SCALE=app-ns:--replicas=3) ;; + warning) + printf 'fail\n' > "${TMPD}/fail_archive.txt" + overrides+=(ARCHIVE_FAILURE_CODE=1) + ;; + esac + set +e + run_backup "$case_dir/source" "$case_dir/out" "$case_dir/state" "$case_dir/run.log" "${overrides[@]}" + result=$? + set -e + case "$scenario" in + normal|no-folder) + assert "$scenario succeeds with ArgoCD present" is_zero "$result" + assert "$scenario stops ArgoCD before app and restores it after compression" events_are_ordered "${TMPD}/event_trace.log" \ + 'scale argocd --replicas=0' 'wait argocd' 'scale app-ns --replicas=0' 'scale app-ns --replicas=3' '7z staging' 'scale argocd --replicas=2' + assert "$scenario also stops ArgoCD StatefulSets" file_contains "${TMPD}/scale_trace.log" 'argocd statefulset web --replicas=0' + ;; + scale-failure|wait-failure|capture-failure) + assert "$scenario aborts" is_nonzero "$result" + assert "$scenario never stops app workloads" file_lacks "${TMPD}/scale_trace.log" 'app-ns' + ;; + restore-failure) + assert "restore failure exits nonzero" is_nonzero "$result" + assert "restore failure retains recovery state" file_contains "$case_dir/run.log" 'saved state retained' + assert "restore failure still restores ArgoCD last" events_are_ordered "${TMPD}/event_trace.log" 'scale app-ns --replicas=3' 'scale argocd --replicas=2' + ;; + warning) + assert "7z warning fails backup" is_nonzero "$result" + assert "7z warning removes incomplete archive" archive_count_is "$case_dir/out" 0 + ;; + esac +done + +# Collision and overlap guards must preserve existing backups and avoid scaling. +guard_dir="${TMPD}/guards" +mkdir -p "$guard_dir/source/app-ns" "$guard_dir/out" "$guard_dir/state" +: > "${TMPD}/namespaces.txt" +: > "${TMPD}/fail_archive.txt" +printf 'existing archive\n' > "$guard_dir/out/test_fixed.7z" +set +e +run_backup "$guard_dir/source" "$guard_dir/out" "$guard_dir/state" "$guard_dir/collision.log" ARCHIVE_TS_FORMAT=fixed +result=$? +set -e +assert "archive collision is rejected" is_nonzero "$result" +assert "archive collision preserves original content" file_contains "$guard_dir/out/test_fixed.7z" 'existing archive' + +( + flock -n 9 || exit 1 + set +e + run_backup "$guard_dir/source" "$guard_dir/out" "$guard_dir/state" "$guard_dir/locked.log" + [[ $? -ne 0 ]] +) 9>"$guard_dir/state/backup.lock" +assert "overlapping run is rejected by lock" file_contains "$guard_dir/locked.log" 'Another NFS backup is running' printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total" if [[ "$tests_failed" -gt 0 ]]; then