#!/usr/bin/env bash SCRIPT_NAME="$(basename "$0")" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=lib/common.sh source "${SCRIPT_DIR}/lib/common.sh" setup_exit_handling # Required configuration require_env "NFS_SOURCE_PATH" require_env "BACKUP_OUTPUT_PATH" require_env "BACKUP_PASSWORD" # Optional configuration KUBECTL_BIN="${KUBECTL_BIN:-kubectl}" KUBE_CONTEXT="${KUBE_CONTEXT:-}" WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset,replicationcontroller}" ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}" ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}" SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}" SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}" SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}" SEVENZ_THREADS="${SEVENZ_THREADS:-on}" SCALE_TIMEOUT_SECONDS="${SCALE_TIMEOUT_SECONDS:-600}" SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}" SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}" TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}" NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}" NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}" NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}" NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}" KUBECTL_ARGS=() if [[ -n "$KUBE_CONTEXT" ]]; then KUBECTL_ARGS+=(--context "$KUBE_CONTEXT") fi declare -a WORKLOAD_KIND_LIST=() declare -A NAMESPACE_MAP=() total_folders=0 mapped_folders=0 unmapped_folders=0 backup_successes=0 backup_failures=0 scale_warnings=0 restore_warnings=0 total_backup_size_bytes=0 _kubectl() { "$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" } parse_workload_kinds() { local raw local cleaned IFS=',' read -r -a raw <<< "$WORKLOAD_KINDS" for kind in "${raw[@]}"; do cleaned="$(trim "${kind,,}")" if [[ -n "$cleaned" ]]; then WORKLOAD_KIND_LIST+=("$cleaned") fi done if [[ "${#WORKLOAD_KIND_LIST[@]}" -eq 0 ]]; then die "WORKLOAD_KINDS resolved to an empty list" fi } validate_inputs() { require_cmd "$KUBECTL_BIN" require_cmd "7z" if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then require_cmd "curl" fi if [[ ! -d "$NFS_SOURCE_PATH" ]]; then die "NFS_SOURCE_PATH does not exist or is not a directory: ${NFS_SOURCE_PATH}" fi mkdir -p "$BACKUP_OUTPUT_PATH" mkdir -p "$TMP_STATE_DIR" if ! _kubectl get namespaces >/dev/null 2>&1; then die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}" fi } load_namespaces() { local ns while IFS= read -r ns; do [[ -z "$ns" ]] && continue NAMESPACE_MAP["$ns"]=1 done < <(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') } namespace_for_folder() { local folder_name="${1:?folder name required}" if [[ -n "${NAMESPACE_MAP[$folder_name]:-}" ]]; then printf '%s' "$folder_name" return 0 fi return 1 } state_file_for_folder() { local folder_name="${1:?folder name required}" local safe_name safe_name="$(printf '%s' "$folder_name" | tr -c 'A-Za-z0-9._-' '_')" printf '%s/%s.state' "$TMP_STATE_DIR" "$safe_name" } capture_replicas_state() { local namespace="${1:?namespace is required}" local state_file="${2:?state file path is required}" : > "$state_file" local kind local name local replicas local output for kind in "${WORKLOAD_KIND_LIST[@]}"; do if ! output="$(_kubectl -n "$namespace" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.replicas}{"\n"}{end}' 2>/dev/null)"; then log_warn "Failed to list kind '${kind}' in namespace '${namespace}' while capturing state" ((scale_warnings++)) continue fi while IFS=$'\t' read -r name replicas; do [[ -z "$name" ]] && continue if [[ -z "$replicas" ]]; then replicas="1" log_warn "Resource ${kind}/${name} had no explicit replicas; defaulting saved value to 1" ((scale_warnings++)) fi printf '%s\t%s\t%s\n' "$kind" "$name" "$replicas" >> "$state_file" done <<< "$output" done } scale_resource() { local namespace="${1:?namespace is required}" local kind="${2:?kind is required}" local name="${3:?name is required}" local replicas="${4:?replicas is required}" retry "$SCALE_RETRY_COUNT" "$SCALE_RETRY_DELAY_SECONDS" \ _kubectl -n "$namespace" scale "${kind}/${name}" --replicas="$replicas" >/dev/null } scale_namespace_to_zero() { local namespace="${1:?namespace is required}" local state_file="${2:?state file is required}" local kind local name local replicas while IFS=$'\t' read -r kind name replicas; do [[ -z "$kind" || -z "$name" ]] && continue if scale_resource "$namespace" "$kind" "$name" "0"; then log_info "Scaled down ${namespace}:${kind}/${name} to 0" else log_warn "Failed to scale down ${namespace}:${kind}/${name}; continuing with backup by policy" ((scale_warnings++)) fi done < "$state_file" } restore_namespace_replicas() { local namespace="${1:?namespace is required}" local state_file="${2:?state file is required}" local kind local name local replicas while IFS=$'\t' read -r kind name replicas; do [[ -z "$kind" || -z "$name" ]] && continue if scale_resource "$namespace" "$kind" "$name" "$replicas"; then log_info "Restored ${namespace}:${kind}/${name} to ${replicas}" else log_warn "Failed to restore ${namespace}:${kind}/${name} to ${replicas}" ((restore_warnings++)) fi done < "$state_file" } resource_replicas() { local namespace="${1:?namespace required}" local kind="${2:?kind required}" local name="${3:?name required}" local json_path="${4:?jsonpath required}" _kubectl -n "$namespace" get "${kind}/${name}" -o "jsonpath=${json_path}" 2>/dev/null || true } wait_for_scale_state() { local namespace="${1:?namespace is required}" local state_file="${2:?state file is required}" local direction="${3:?direction is required}" # down|up local start_ts local now_ts local elapsed local unresolved local kind local name local target local spec local status local ready start_ts="$(date +%s)" while true; do unresolved=0 while IFS=$'\t' read -r kind name target; do [[ -z "$kind" || -z "$name" ]] && continue spec="$(resource_replicas "$namespace" "$kind" "$name" '{.spec.replicas}')" status="$(resource_replicas "$namespace" "$kind" "$name" '{.status.replicas}')" ready="$(resource_replicas "$namespace" "$kind" "$name" '{.status.readyReplicas}')" [[ -z "$spec" ]] && spec="1" [[ -z "$status" ]] && status="0" [[ -z "$ready" ]] && ready="0" if [[ "$direction" == "down" ]]; then if [[ "$spec" != "0" || "$status" != "0" ]]; then ((unresolved++)) fi else if [[ "$target" == "0" ]]; then if [[ "$spec" != "0" || "$status" != "0" ]]; then ((unresolved++)) fi else if [[ "$spec" != "$target" || "$ready" != "$target" ]]; then ((unresolved++)) fi fi fi done < "$state_file" if [[ "$unresolved" -eq 0 ]]; then return 0 fi now_ts="$(date +%s)" elapsed=$((now_ts - start_ts)) if (( elapsed >= SCALE_TIMEOUT_SECONDS )); then return 1 fi sleep 3 done } archive_path_for_folder() { local folder_name="${1:?folder name required}" local ts ts="$(date +"$ARCHIVE_TS_FORMAT")" printf '%s/%s_%s_%s.7z' "$BACKUP_OUTPUT_PATH" "$ARCHIVE_PREFIX" "$folder_name" "$ts" } json_escape() { local value="${1:-}" value="${value//\\/\\\\}" value="${value//\"/\\\"}" value="${value//$'\n'/ }" printf '%s' "$value" } backup_size_mb() { local bytes="$1" if (( bytes <= 0 )); then echo 0 return 0 fi echo $(( (bytes + 1048575) / 1048576 )) } send_backup_notification() { local url="${1:-}" local title="${2:-}" local asset="${3:-}" local size_mb="${4:-0}" [[ -z "$url" ]] && return 0 local payload payload=$(cat </dev/null; then log_warn "Failed to send backup notification to ${url}" return 1 fi return 0 } backup_folder() { local folder_path="${1:?folder path required}" local archive_path="${2:?archive path required}" local password_arg="-p${BACKUP_PASSWORD}" 7z a -t7z \ "$archive_path" \ "$folder_path" \ -m0="$SEVENZ_METHOD" \ -mx="$SEVENZ_LEVEL" \ -mhe="$SEVENZ_HEADER_ENCRYPT" \ -mmt="$SEVENZ_THREADS" \ "$password_arg" \ >/dev/null } process_folder() { local folder_path="${1:?folder path is required}" local folder_name local namespace="" local state_file="" local archive_path="" local has_mapping=0 folder_name="$(basename "$folder_path")" ((total_folders++)) log_info "Processing folder: ${folder_name}" if namespace="$(namespace_for_folder "$folder_name")"; then has_mapping=1 ((mapped_folders++)) log_info "Exact namespace match found for folder '${folder_name}' -> namespace '${namespace}'" state_file="$(state_file_for_folder "$folder_name")" capture_replicas_state "$namespace" "$state_file" scale_namespace_to_zero "$namespace" "$state_file" if wait_for_scale_state "$namespace" "$state_file" "down"; then log_info "Namespace '${namespace}' reached scale-down target" else log_warn "Timeout waiting for namespace '${namespace}' to fully scale down; continuing by policy" ((scale_warnings++)) fi else ((unmapped_folders++)) log_warn "No namespace matched folder '${folder_name}'. Running backup only." fi archive_path="$(archive_path_for_folder "$folder_name")" if backup_folder "$folder_path" "$archive_path"; then ((backup_successes++)) local file_size_bytes file_size_bytes="$(wc -c < "$archive_path" | tr -d '[:space:]')" if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then ((total_backup_size_bytes += file_size_bytes)) fi log_info "Backup created: ${archive_path}" else ((backup_failures++)) log_error "Backup failed for folder '${folder_name}'" fi if [[ "$has_mapping" -eq 1 ]]; then restore_namespace_replicas "$namespace" "$state_file" if wait_for_scale_state "$namespace" "$state_file" "up"; then log_info "Namespace '${namespace}' reached restore target" else log_warn "Timeout waiting for namespace '${namespace}' to fully restore replicas" ((restore_warnings++)) fi rm -f "$state_file" fi } print_summary() { log_info "Run summary: processed=${total_folders} mapped=${mapped_folders} unmapped=${unmapped_folders} backup_successes=${backup_successes} backup_failures=${backup_failures} scale_warnings=${scale_warnings} restore_warnings=${restore_warnings}" } main() { parse_workload_kinds validate_inputs load_namespaces local folder_path local found=0 for folder_path in "${NFS_SOURCE_PATH}"/*; do [[ -d "$folder_path" ]] || continue [[ "$(basename "$folder_path")" == .* ]] && continue found=1 process_folder "$folder_path" done if [[ "$found" -eq 0 ]]; then log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}" fi print_summary local size_mb size_mb="$(backup_size_mb "$total_backup_size_bytes")" if (( backup_failures > 0 )); then send_backup_notification \ "$NOTIFY_FAILURE_URL" \ "${NOTIFY_TITLE} - FAILED" \ "${NOTIFY_ASSET}" \ "$size_mb" || true die "One or more folder backups failed" 1 fi send_backup_notification \ "$NOTIFY_SUCCESS_URL" \ "$NOTIFY_TITLE" \ "${NOTIFY_ASSET}" \ "$size_mb" || true } main "$@"