#!/usr/bin/env bash SCRIPT_NAME="$(basename "$0")" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=lib/common.sh source "${SCRIPT_DIR}/lib/common.sh" # Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees # every scaled-down namespace is restored regardless of how we exit. trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR trap '_exit_code=$?; restore_all_remaining; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT # Required configuration require_env "NFS_SOURCE_PATH" require_env "BACKUP_OUTPUT_PATH" # Optional configuration BACKUP_PASSWORD="${BACKUP_PASSWORD:-}" KUBECTL_BIN="${KUBECTL_BIN:-kubectl}" KUBE_CONTEXT="${KUBE_CONTEXT:-}" WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}" ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}" ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}" SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}" SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}" SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}" SEVENZ_THREADS="${SEVENZ_THREADS:-on}" SCALE_TIMEOUT_SECONDS="${SCALE_TIMEOUT_SECONDS:-600}" SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}" SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}" TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}" NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}" NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}" NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}" NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}" NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}" CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}" KUBECTL_ARGS=() if [[ -n "$KUBE_CONTEXT" ]]; then KUBECTL_ARGS+=(--context "$KUBE_CONTEXT") fi declare -a WORKLOAD_KIND_LIST=() declare -A NAMESPACE_MAP=() total_folders=0 mapped_folders=0 unmapped_folders=0 backup_successes=0 backup_failures=0 scale_warnings=0 restore_warnings=0 total_backup_size_bytes=0 _kubectl() { "$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" = 0" fi if [[ -z "$BACKUP_PASSWORD" ]]; then log_warn "BACKUP_PASSWORD is empty. Archive will be created without password protection." fi mkdir -p "$BACKUP_OUTPUT_PATH" mkdir -p "$TMP_STATE_DIR" if ! _kubectl get namespaces >/dev/null 2>&1; then die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}" fi } load_namespaces() { local ns while IFS= read -r ns; do [[ -z "$ns" ]] && continue NAMESPACE_MAP["$ns"]=1 done < <(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') } namespace_for_folder() { local folder_name="${1:?folder name required}" if [[ -n "${NAMESPACE_MAP[$folder_name]:-}" ]]; then printf '%s' "$folder_name" return 0 fi return 1 } state_file_for_namespace() { local namespace="${1:?namespace required}" printf '%s/%s.state' "$TMP_STATE_DIR" "$namespace" } capture_replicas_state() { local namespace="${1:?namespace is required}" local state_file="${2:?state file path is required}" : > "$state_file" local kind local name local replicas local owners local output for kind in "${WORKLOAD_KIND_LIST[@]}"; do if ! output="$(_kubectl -n "$namespace" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.replicas}{"\t"}{range .metadata.ownerReferences[*]}{.kind},{end}{"\n"}{end}' 2>/dev/null)"; then log_warn "Failed to list kind '${kind}' in namespace '${namespace}' while capturing state" scale_warnings=$((scale_warnings + 1)) continue fi while IFS=$'\t' read -r name replicas owners; do [[ -z "$name" ]] && continue # Skip resources managed by a higher-level controller (e.g. a # ReplicaSet owned by a Deployment) — scaling them directly # conflicts with the owning controller and can leave workloads # in an inconsistent state after restore. if [[ -n "$owners" ]]; then log_debug "Skipping ${kind}/${name} in namespace '${namespace}': owned by ${owners%,}" continue fi if [[ -z "$replicas" ]]; then replicas="1" log_warn "Resource ${kind}/${name} had no explicit replicas; defaulting saved value to 1" scale_warnings=$((scale_warnings + 1)) fi printf '%s\t%s\t%s\n' "$kind" "$name" "$replicas" >> "$state_file" done <<< "$output" done } scale_resource() { local namespace="${1:?namespace is required}" local kind="${2:?kind is required}" local name="${3:?name is required}" local replicas="${4:?replicas is required}" retry "$SCALE_RETRY_COUNT" "$SCALE_RETRY_DELAY_SECONDS" \ _kubectl -n "$namespace" scale "${kind}/${name}" --replicas="$replicas" } scale_namespace_to_zero() { local namespace="${1:?namespace is required}" local state_file="${2:?state file is required}" local kind local name local replicas while IFS=$'\t' read -r kind name replicas; do [[ -z "$kind" || -z "$name" ]] && continue if scale_resource "$namespace" "$kind" "$name" "0"; then log_info "Scaled down ${namespace}:${kind}/${name} to 0" else log_warn "Failed to scale down ${namespace}:${kind}/${name}; continuing with backup by policy" scale_warnings=$((scale_warnings + 1)) fi done < "$state_file" } restore_namespace_replicas() { local namespace="${1:?namespace is required}" local state_file="${2:?state file is required}" local kind local name local replicas while IFS=$'\t' read -r kind name replicas; do [[ -z "$kind" || -z "$name" ]] && continue if scale_resource "$namespace" "$kind" "$name" "$replicas"; then log_info "Restored ${namespace}:${kind}/${name} to ${replicas}" else log_warn "Failed to restore ${namespace}:${kind}/${name} to ${replicas}" restore_warnings=$((restore_warnings + 1)) fi done < "$state_file" } restore_all_remaining() { if [[ -z "${TMP_STATE_DIR:-}" ]] || [[ ! -d "${TMP_STATE_DIR:-}" ]]; then return 0 fi local state_file local namespace local kind name replicas local found=0 for state_file in "${TMP_STATE_DIR}"/*.state; do [[ -f "$state_file" ]] || continue found=1 namespace="$(basename "$state_file" .state)" [[ -z "$namespace" ]] && continue log_info "EXIT cleanup: restoring workloads in namespace '${namespace}'" while IFS=$'\t' read -r kind name replicas; do [[ -z "$kind" || -z "$name" ]] && continue if scale_resource "$namespace" "$kind" "$name" "$replicas"; then log_info "EXIT cleanup: restored ${namespace}:${kind}/${name} to ${replicas}" else log_warn "EXIT cleanup: failed to restore ${namespace}:${kind}/${name} to ${replicas}" restore_warnings=$((restore_warnings + 1)) fi done < "$state_file" rm -f "$state_file" done if [[ "$found" -eq 1 ]]; then log_info "EXIT cleanup complete - all remaining workloads restored" fi } archive_path_for_run() { local ts ts="$(date +"$ARCHIVE_TS_FORMAT")" printf '%s/%s_%s.7z' "$BACKUP_OUTPUT_PATH" "$ARCHIVE_PREFIX" "$ts" } json_escape() { local value="${1:-}" value="${value//\\/\\\\}" value="${value//\"/\\\"}" value="${value//$'\n'/ }" printf '%s' "$value" } backup_size_mb() { local bytes="$1" if (( bytes <= 0 )); then echo 0 return 0 fi echo $(( (bytes + 1048575) / 1048576 )) } send_backup_notification() { local url="${1:-}" local title="${2:-}" local asset="${3:-}" local size_mb="${4:-0}" [[ -z "$url" ]] && return 0 local payload payload=$(cat </dev/null; then log_warn "Failed to send backup notification to ${url}" return 1 fi return 0 } send_backup_failure_notification() { local url="${1:-}" local message="${2:-}" local size_mb="${3:-0}" [[ -z "$url" ]] && return 0 local payload payload=$(cat </dev/null; then log_warn "Failed to send backup failure notification to ${url}" return 1 fi return 0 } backup_folder() { local folder_path="${1:?folder path required}" local archive_path="${2:?archive path required}" local -a cmd=( 7z a -t7z "$archive_path" "$folder_path" "-m0=${SEVENZ_METHOD}" "-mx=${SEVENZ_LEVEL}" "-mmt=${SEVENZ_THREADS}" ) if [[ -n "$BACKUP_PASSWORD" ]]; then cmd+=("-p${BACKUP_PASSWORD}" "-mhe=${SEVENZ_HEADER_ENCRYPT}") fi local cmd_output rc filtered_output cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)" rc=$? filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)" if [[ -n "$filtered_output" ]]; then log_warn "7z output for '${folder_path}': ${filtered_output}" fi if (( rc == 0 )); then return 0 elif (( rc == 1 )) && [[ -z "$filtered_output" ]]; then return 0 fi return 1 } cleanup_archives() { if (( CLEANUP_KEEP_COUNT == 0 )); then log_info "Cleanup disabled (CLEANUP_KEEP_COUNT=0)" return 0 fi shopt -s nullglob local archive_candidates=( "$BACKUP_OUTPUT_PATH"/"${ARCHIVE_PREFIX}"_*.7z ) shopt -u nullglob local total_candidates="${#archive_candidates[@]}" if (( total_candidates == 0 )); then log_info "Cleanup skipped: no archives found for prefix '${ARCHIVE_PREFIX}' in ${BACKUP_OUTPUT_PATH}" return 0 fi local sorted_archives=() mapfile -t sorted_archives < <(ls -1t -- "${archive_candidates[@]}") local total_sorted="${#sorted_archives[@]}" if (( total_sorted <= CLEANUP_KEEP_COUNT )); then log_info "Cleanup skipped: ${total_sorted} archive(s) present, keeping ${CLEANUP_KEEP_COUNT}" return 0 fi local keep_count="$CLEANUP_KEEP_COUNT" local i for ((i = keep_count; i < total_sorted; i++)); do local archive_path="${sorted_archives[$i]}" rm -f -- "$archive_path" log_info "Cleanup deleted archive: ${archive_path}" done } process_folder() { local folder_path="${1:?folder path is required}" local archive_path="${2:?archive path is required}" local folder_name local namespace="" local state_file="" local has_mapping=0 folder_name="$(basename "$folder_path")" total_folders=$((total_folders + 1)) log_info "Processing folder: ${folder_name}" if namespace="$(namespace_for_folder "$folder_name")"; then has_mapping=1 mapped_folders=$((mapped_folders + 1)) log_info "Exact namespace match found for folder '${folder_name}' -> namespace '${namespace}'" state_file="$(state_file_for_namespace "$namespace")" capture_replicas_state "$namespace" "$state_file" scale_namespace_to_zero "$namespace" "$state_file" log_info "Waiting 30 seconds for namespace '${namespace}' to scale down..." sleep 30 else unmapped_folders=$((unmapped_folders + 1)) log_warn "No namespace matched folder '${folder_name}'. Running backup only." fi if backup_folder "$folder_path" "$archive_path"; then backup_successes=$((backup_successes + 1)) log_info "Added folder '${folder_name}' to archive: ${archive_path}" else backup_failures=$((backup_failures + 1)) log_error "Backup failed for folder '${folder_name}'" fi if [[ "$has_mapping" -eq 1 ]]; then restore_namespace_replicas "$namespace" "$state_file" log_info "Waiting 30 seconds for namespace '${namespace}' to restore replicas..." sleep 30 rm -f "$state_file" fi } print_summary() { log_info "Run summary: processed=${total_folders} mapped=${mapped_folders} unmapped=${unmapped_folders} backup_successes=${backup_successes} backup_failures=${backup_failures} scale_warnings=${scale_warnings} restore_warnings=${restore_warnings}" } main() { parse_workload_kinds validate_inputs load_namespaces local folder_path local run_archive_path="" local found=0 for folder_path in "${NFS_SOURCE_PATH}"/*; do [[ -d "$folder_path" ]] || continue [[ "$(basename "$folder_path")" == .* ]] && continue if [[ -z "$run_archive_path" ]]; then run_archive_path="$(archive_path_for_run)" log_info "Using single archive for this run: ${run_archive_path}" fi found=1 process_folder "$folder_path" "$run_archive_path" done if [[ "$found" -eq 0 ]]; then log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}" elif [[ -f "$run_archive_path" ]]; then local file_size_bytes file_size_bytes="$(wc -c < "$run_archive_path" | tr -d '[:space:]')" if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then total_backup_size_bytes="$file_size_bytes" fi fi print_summary local size_mb size_mb="$(backup_size_mb "$total_backup_size_bytes")" if (( backup_failures > 0 )); then send_backup_failure_notification \ "$NOTIFY_FAILURE_URL" \ "Error while backing up NFS folders" \ "$size_mb" || true die "One or more folder backups failed" 1 fi cleanup_archives || true send_backup_notification \ "$NOTIFY_SUCCESS_URL" \ "$NOTIFY_TITLE" \ "${NOTIFY_ASSET}" \ "$size_mb" || true } main "$@"