Jose Henrique
ceee89f86b
All checks were successful
Check scripts syntax / check-scripts-syntax (push) Successful in 14s
Test Kubernetes backup scripts / test-automated-nfs-backup (push) Successful in 15s
Test Kubernetes backup scripts / test-k3s-control-plane-config-backup (push) Successful in 15s
Haven Notify Build and Deploy / Test Haven Notify (push) Successful in 2m8s
Haven Notify Build and Deploy / Build Haven Notify Image (PR) (push) Has been skipped
Haven Notify Build and Deploy / Build Haven Notify Image (push) Successful in 29s
Haven Notify Build and Deploy / Deploy Haven Notify (internal) (push) Successful in 7s
Exit code 1 from 7z means a non-fatal warning — the archive IS created, just some files couldn't be compressed (e.g. broken symlinks, special files). Previously the function required filtered output to be empty alongside rc=1, causing any non-errno=2 stderr line to abort the entire backup and discard the archive. Now any rc=1 is treated as success (warning is still logged).
617 lines
20 KiB
Bash
617 lines
20 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
SCRIPT_NAME="$(basename "$0")"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=lib/common.sh
|
|
source "${SCRIPT_DIR}/lib/common.sh"
|
|
|
|
# Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
|
|
# every scaled-down namespace is restored regardless of how we exit.
|
|
trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
|
|
trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
|
|
|
|
# Required configuration
|
|
require_env "NFS_SOURCE_PATH"
|
|
require_env "BACKUP_OUTPUT_PATH"
|
|
|
|
# Optional configuration
|
|
BACKUP_PASSWORD="${BACKUP_PASSWORD:-}"
|
|
KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
|
|
KUBE_CONTEXT="${KUBE_CONTEXT:-}"
|
|
WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
|
|
EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease}"
|
|
BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-/tmp/k8s-nfs-backup-staging}"
|
|
RSYNC_BIN="${RSYNC_BIN:-rsync}"
|
|
ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
|
|
ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
|
|
SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
|
|
SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}"
|
|
SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}"
|
|
SEVENZ_THREADS="${SEVENZ_THREADS:-on}"
|
|
SEVENZ_BIN="${SEVENZ_BIN:-7z}"
|
|
SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
|
|
SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
|
|
SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
|
|
TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}"
|
|
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
|
|
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
|
|
NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
|
|
NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
|
|
NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes NFS backup}"
|
|
NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
|
|
CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
|
|
RUN_STATE_DIR=""
|
|
RUN_STAGING_DIR=""
|
|
|
|
KUBECTL_ARGS=()
|
|
if [[ -n "$KUBE_CONTEXT" ]]; then
|
|
KUBECTL_ARGS+=(--context "$KUBE_CONTEXT")
|
|
fi
|
|
|
|
declare -a WORKLOAD_KIND_LIST=()
|
|
declare -A NAMESPACE_MAP=()
|
|
declare -A EXCLUDED_NAMESPACE_MAP=()
|
|
|
|
total_folders=0
|
|
mapped_folders=0
|
|
unmapped_folders=0
|
|
backup_successes=0
|
|
backup_failures=0
|
|
scale_warnings=0
|
|
restore_warnings=0
|
|
total_backup_size_bytes=0
|
|
backup_started_seconds="$SECONDS"
|
|
|
|
_kubectl() {
|
|
"$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" </dev/null
|
|
}
|
|
|
|
parse_workload_kinds() {
|
|
local raw
|
|
local cleaned
|
|
IFS=',' read -r -a raw <<< "$WORKLOAD_KINDS"
|
|
for kind in "${raw[@]}"; do
|
|
cleaned="$(trim "${kind,,}")"
|
|
if [[ -n "$cleaned" ]]; then
|
|
WORKLOAD_KIND_LIST+=("$cleaned")
|
|
fi
|
|
done
|
|
|
|
if [[ "${#WORKLOAD_KIND_LIST[@]}" -eq 0 ]]; then
|
|
die "WORKLOAD_KINDS resolved to an empty list"
|
|
fi
|
|
}
|
|
|
|
parse_excluded_namespaces() {
|
|
local raw
|
|
local cleaned
|
|
local namespace
|
|
IFS=',' read -r -a raw <<< "$EXCLUDED_NAMESPACES"
|
|
for namespace in "${raw[@]}"; do
|
|
cleaned="$(trim "${namespace,,}")"
|
|
if [[ -n "$cleaned" ]]; then
|
|
EXCLUDED_NAMESPACE_MAP["$cleaned"]=1
|
|
fi
|
|
done
|
|
}
|
|
|
|
validate_inputs() {
|
|
require_cmd "$KUBECTL_BIN"
|
|
require_cmd "$SEVENZ_BIN"
|
|
require_cmd "mktemp"
|
|
require_cmd "$RSYNC_BIN"
|
|
if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
|
|
require_cmd "curl"
|
|
fi
|
|
|
|
if [[ ! -d "$NFS_SOURCE_PATH" ]]; then
|
|
die "NFS_SOURCE_PATH does not exist or is not a directory: ${NFS_SOURCE_PATH}"
|
|
fi
|
|
|
|
if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then
|
|
die "CLEANUP_KEEP_COUNT must be an integer >= 0"
|
|
fi
|
|
|
|
if [[ -z "$BACKUP_PASSWORD" ]]; then
|
|
log_warn "BACKUP_PASSWORD is empty. Archive will be created without password protection."
|
|
fi
|
|
|
|
mkdir -p "$BACKUP_OUTPUT_PATH"
|
|
mkdir -p "$TMP_STATE_DIR"
|
|
if ! RUN_STATE_DIR="$(mktemp -d "${TMP_STATE_DIR%/}/run-XXXXXXXXXX")"; then
|
|
die "Unable to create a per-run state directory under ${TMP_STATE_DIR}"
|
|
fi
|
|
log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
|
|
|
|
local source_real
|
|
local staging_real
|
|
mkdir -p "$BACKUP_STAGING_PATH"
|
|
source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
|
|
staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
|
|
case "${staging_real}/" in
|
|
"${source_real%/}/"*)
|
|
die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
|
|
;;
|
|
esac
|
|
if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
|
|
die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
|
|
fi
|
|
log_info "Using local run staging directory: ${RUN_STAGING_DIR}"
|
|
|
|
if ! _kubectl get namespaces >/dev/null 2>&1; then
|
|
die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
|
|
fi
|
|
}
|
|
|
|
load_namespaces() {
|
|
local ns
|
|
while IFS= read -r ns; do
|
|
[[ -z "$ns" ]] && continue
|
|
NAMESPACE_MAP["$ns"]=1
|
|
done < <(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
|
}
|
|
|
|
namespace_for_folder() {
|
|
local folder_name="${1:?folder name required}"
|
|
if [[ -n "${EXCLUDED_NAMESPACE_MAP[$folder_name]:-}" ]]; then
|
|
return 2
|
|
fi
|
|
if [[ -n "${NAMESPACE_MAP[$folder_name]:-}" ]]; then
|
|
printf '%s' "$folder_name"
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
state_file_for_namespace() {
|
|
local namespace="${1:?namespace required}"
|
|
printf '%s/%s.state' "$RUN_STATE_DIR" "$namespace"
|
|
}
|
|
|
|
capture_replicas_state() {
|
|
local namespace="${1:?namespace is required}"
|
|
local state_file="${2:?state file path is required}"
|
|
: > "$state_file"
|
|
|
|
local kind
|
|
local name
|
|
local replicas
|
|
local owners
|
|
local output
|
|
for kind in "${WORKLOAD_KIND_LIST[@]}"; do
|
|
if ! output="$(_kubectl -n "$namespace" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.replicas}{"\t"}{range .metadata.ownerReferences[*]}{.kind},{end}{"\n"}{end}' 2>/dev/null)"; then
|
|
log_warn "Failed to list kind '${kind}' in namespace '${namespace}' while capturing state"
|
|
scale_warnings=$((scale_warnings + 1))
|
|
continue
|
|
fi
|
|
|
|
while IFS=$'\t' read -r name replicas owners; do
|
|
[[ -z "$name" ]] && continue
|
|
# Skip resources managed by a higher-level controller (e.g. a
|
|
# ReplicaSet owned by a Deployment) — scaling them directly
|
|
# conflicts with the owning controller and can leave workloads
|
|
# in an inconsistent state after restore.
|
|
if [[ -n "$owners" ]]; then
|
|
log_debug "Skipping ${kind}/${name} in namespace '${namespace}': owned by ${owners%,}"
|
|
continue
|
|
fi
|
|
if [[ -z "$replicas" ]]; then
|
|
replicas="1"
|
|
log_warn "Resource ${kind}/${name} had no explicit replicas; defaulting saved value to 1"
|
|
scale_warnings=$((scale_warnings + 1))
|
|
fi
|
|
printf '%s\t%s\t%s\n' "$kind" "$name" "$replicas" >> "$state_file"
|
|
done <<< "$output"
|
|
done
|
|
}
|
|
|
|
scale_resource() {
|
|
local namespace="${1:?namespace is required}"
|
|
local kind="${2:?kind is required}"
|
|
local name="${3:?name is required}"
|
|
local replicas="${4:?replicas is required}"
|
|
retry "$SCALE_RETRY_COUNT" "$SCALE_RETRY_DELAY_SECONDS" \
|
|
_kubectl -n "$namespace" scale "${kind}/${name}" --replicas="$replicas"
|
|
}
|
|
|
|
scale_namespace_to_zero() {
|
|
local namespace="${1:?namespace is required}"
|
|
local state_file="${2:?state file is required}"
|
|
local kind
|
|
local name
|
|
local replicas
|
|
|
|
while IFS=$'\t' read -r kind name replicas; do
|
|
[[ -z "$kind" || -z "$name" ]] && continue
|
|
if scale_resource "$namespace" "$kind" "$name" "0"; then
|
|
log_info "Scaled down ${namespace}:${kind}/${name} to 0"
|
|
else
|
|
log_warn "Failed to scale down ${namespace}:${kind}/${name}; continuing with backup by policy"
|
|
scale_warnings=$((scale_warnings + 1))
|
|
fi
|
|
done < "$state_file"
|
|
}
|
|
|
|
restore_namespace_replicas() {
|
|
local namespace="${1:?namespace is required}"
|
|
local state_file="${2:?state file is required}"
|
|
local kind
|
|
local name
|
|
local replicas
|
|
|
|
while IFS=$'\t' read -r kind name replicas; do
|
|
[[ -z "$kind" || -z "$name" ]] && continue
|
|
if scale_resource "$namespace" "$kind" "$name" "$replicas"; then
|
|
log_info "Restored ${namespace}:${kind}/${name} to ${replicas}"
|
|
else
|
|
log_warn "Failed to restore ${namespace}:${kind}/${name} to ${replicas}"
|
|
restore_warnings=$((restore_warnings + 1))
|
|
fi
|
|
done < "$state_file"
|
|
}
|
|
|
|
restore_all_remaining() {
|
|
if [[ -z "${RUN_STATE_DIR:-}" ]] || [[ ! -d "${RUN_STATE_DIR:-}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local state_file
|
|
local namespace
|
|
local kind name replicas
|
|
local found=0
|
|
|
|
for state_file in "${RUN_STATE_DIR}"/*.state; do
|
|
[[ -f "$state_file" ]] || continue
|
|
found=1
|
|
namespace="$(basename "$state_file" .state)"
|
|
[[ -z "$namespace" ]] && continue
|
|
log_info "EXIT cleanup: restoring workloads in namespace '${namespace}'"
|
|
while IFS=$'\t' read -r kind name replicas; do
|
|
[[ -z "$kind" || -z "$name" ]] && continue
|
|
if scale_resource "$namespace" "$kind" "$name" "$replicas"; then
|
|
log_info "EXIT cleanup: restored ${namespace}:${kind}/${name} to ${replicas}"
|
|
else
|
|
log_warn "EXIT cleanup: failed to restore ${namespace}:${kind}/${name} to ${replicas}"
|
|
restore_warnings=$((restore_warnings + 1))
|
|
fi
|
|
done < "$state_file"
|
|
rm -f "$state_file"
|
|
done
|
|
|
|
if [[ "$found" -eq 1 ]]; then
|
|
log_info "EXIT cleanup complete - all remaining workloads restored"
|
|
fi
|
|
|
|
if ! rmdir -- "$RUN_STATE_DIR" 2>/dev/null; then
|
|
log_warn "Run state directory is not empty; leaving it in place: ${RUN_STATE_DIR}"
|
|
fi
|
|
}
|
|
|
|
cleanup_run_staging() {
|
|
if [[ -z "${RUN_STAGING_DIR:-}" ]] || [[ ! -d "${RUN_STAGING_DIR:-}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
case "$RUN_STAGING_DIR" in
|
|
"${BACKUP_STAGING_PATH%/}/run-"*)
|
|
rm -rf -- "$RUN_STAGING_DIR"
|
|
;;
|
|
*)
|
|
log_error "Refusing to remove unexpected staging directory: ${RUN_STAGING_DIR}"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
cleanup_staged_folder() {
|
|
local staging_folder="${1:?staging folder required}"
|
|
case "$staging_folder" in
|
|
"$RUN_STAGING_DIR"/*)
|
|
rm -rf -- "$staging_folder"
|
|
;;
|
|
*)
|
|
log_error "Refusing to remove staging folder outside the current run: ${staging_folder}"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
sync_folder_to_staging() {
|
|
local source_folder="${1:?source folder required}"
|
|
local staging_folder="${2:?staging folder required}"
|
|
local sync_output
|
|
|
|
mkdir -p "$staging_folder"
|
|
if ! sync_output="$("$RSYNC_BIN" -a --delete -- "${source_folder%/}/" "${staging_folder%/}/" 2>&1)"; then
|
|
log_error "rsync failed from '${source_folder}' to '${staging_folder}'"
|
|
if [[ -n "$sync_output" ]]; then
|
|
log_error "rsync output: ${sync_output}"
|
|
fi
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
archive_path_for_run() {
|
|
local ts
|
|
ts="$(date +"$ARCHIVE_TS_FORMAT")"
|
|
printf '%s/%s_%s.7z' "$BACKUP_OUTPUT_PATH" "$ARCHIVE_PREFIX" "$ts"
|
|
}
|
|
|
|
json_escape() {
|
|
local value="${1:-}"
|
|
value="${value//\\/\\\\}"
|
|
value="${value//\"/\\\"}"
|
|
value="${value//$'\n'/ }"
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
send_backup_notification() {
|
|
local url="${1:-}"
|
|
local title="${2:-}"
|
|
local asset="${3:-}"
|
|
local size_bytes="${4:-0}"
|
|
local duration_seconds="${5:-0}"
|
|
|
|
[[ -z "$url" ]] && return 0
|
|
|
|
local payload
|
|
payload=$(cat <<EOF
|
|
{
|
|
"title": "$(json_escape "$title")",
|
|
"asset": "$(json_escape "$asset")",
|
|
"sizeBytes": ${size_bytes},
|
|
"durationSeconds": ${duration_seconds},
|
|
"source": "$(json_escape "$NOTIFY_SOURCE")"
|
|
}
|
|
EOF
|
|
)
|
|
|
|
if ! curl -fsS -X POST "$url" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" >/dev/null; then
|
|
log_warn "Failed to send backup notification to ${url}"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
send_backup_failure_notification() {
|
|
local url="${1:-}"
|
|
local message="${2:-}"
|
|
local size_bytes="${3:-0}"
|
|
|
|
[[ -z "$url" ]] && return 0
|
|
|
|
local payload
|
|
payload=$(cat <<EOF
|
|
{
|
|
"source": "$(json_escape "$NOTIFY_SOURCE")",
|
|
"message": "$(json_escape "$message")",
|
|
"severity": "critical",
|
|
"errorCode": "BACKUP_FAILED",
|
|
"extra": [
|
|
{
|
|
"name": "Archive size",
|
|
"value": "${size_bytes} B"
|
|
}
|
|
]
|
|
}
|
|
EOF
|
|
)
|
|
|
|
if ! curl -fsS -X POST "$url" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" >/dev/null; then
|
|
log_warn "Failed to send backup failure notification to ${url}"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
archive_staging() {
|
|
local archive_path="${1:?archive path required}"
|
|
|
|
if [[ "$archive_path" != /* ]]; then
|
|
archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
|
|
fi
|
|
|
|
local -a cmd=(
|
|
"$SEVENZ_BIN" a -t7z
|
|
"$archive_path"
|
|
.
|
|
"-m0=${SEVENZ_METHOD}"
|
|
"-mx=${SEVENZ_LEVEL}"
|
|
"-mmt=${SEVENZ_THREADS}"
|
|
)
|
|
|
|
if [[ -n "$BACKUP_PASSWORD" ]]; then
|
|
cmd+=("-p${BACKUP_PASSWORD}" "-mhe=${SEVENZ_HEADER_ENCRYPT}")
|
|
fi
|
|
|
|
local cmd_output rc filtered_output
|
|
cmd_output="$(cd "$RUN_STAGING_DIR" && "${cmd[@]}" 2>&1 >/dev/null)"
|
|
rc=$?
|
|
filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
|
|
if [[ -n "$filtered_output" ]]; then
|
|
log_warn "7z output for staging directory '${RUN_STAGING_DIR}': ${filtered_output}"
|
|
fi
|
|
if (( rc == 0 )); then
|
|
return 0
|
|
elif (( rc == 1 )); then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
cleanup_archives() {
|
|
if (( CLEANUP_KEEP_COUNT == 0 )); then
|
|
log_info "Cleanup disabled (CLEANUP_KEEP_COUNT=0)"
|
|
return 0
|
|
fi
|
|
|
|
shopt -s nullglob
|
|
local archive_candidates=( "$BACKUP_OUTPUT_PATH"/"${ARCHIVE_PREFIX}"_*.7z )
|
|
shopt -u nullglob
|
|
|
|
local total_candidates="${#archive_candidates[@]}"
|
|
if (( total_candidates == 0 )); then
|
|
log_info "Cleanup skipped: no archives found for prefix '${ARCHIVE_PREFIX}' in ${BACKUP_OUTPUT_PATH}"
|
|
return 0
|
|
fi
|
|
|
|
local sorted_archives=()
|
|
mapfile -t sorted_archives < <(ls -1t -- "${archive_candidates[@]}")
|
|
|
|
local total_sorted="${#sorted_archives[@]}"
|
|
if (( total_sorted <= CLEANUP_KEEP_COUNT )); then
|
|
log_info "Cleanup skipped: ${total_sorted} archive(s) present, keeping ${CLEANUP_KEEP_COUNT}"
|
|
return 0
|
|
fi
|
|
|
|
local keep_count="$CLEANUP_KEEP_COUNT"
|
|
local i
|
|
for ((i = keep_count; i < total_sorted; i++)); do
|
|
local archive_path="${sorted_archives[$i]}"
|
|
rm -f -- "$archive_path"
|
|
log_info "Cleanup deleted archive: ${archive_path}"
|
|
done
|
|
}
|
|
|
|
process_folder() {
|
|
local folder_path="${1:?folder path is required}"
|
|
local folder_name
|
|
local namespace=""
|
|
local state_file=""
|
|
local has_mapping=0
|
|
local namespace_match_status=0
|
|
local staging_folder
|
|
local copy_succeeded=0
|
|
|
|
folder_name="$(basename "$folder_path")"
|
|
staging_folder="${RUN_STAGING_DIR}/${folder_name}"
|
|
total_folders=$((total_folders + 1))
|
|
log_info "Processing folder: ${folder_name}"
|
|
|
|
if namespace="$(namespace_for_folder "$folder_name")"; then
|
|
has_mapping=1
|
|
mapped_folders=$((mapped_folders + 1))
|
|
log_info "Exact namespace match found for folder '${folder_name}' -> namespace '${namespace}'"
|
|
state_file="$(state_file_for_namespace "$namespace")"
|
|
|
|
capture_replicas_state "$namespace" "$state_file"
|
|
scale_namespace_to_zero "$namespace" "$state_file"
|
|
|
|
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
|
|
sleep "$SCALE_WAIT_SECONDS"
|
|
|
|
log_info "Copying quiesced namespace folder '${folder_name}' to local staging"
|
|
if sync_folder_to_staging "$folder_path" "$staging_folder"; then
|
|
copy_succeeded=1
|
|
fi
|
|
else
|
|
namespace_match_status=$?
|
|
unmapped_folders=$((unmapped_folders + 1))
|
|
if (( namespace_match_status == 2 )); then
|
|
log_warn "Folder '${folder_name}' matches an excluded namespace; skipping Kubernetes scale actions and running backup only."
|
|
else
|
|
log_warn "No namespace matched folder '${folder_name}'. Running backup only."
|
|
fi
|
|
|
|
log_info "Copying live folder '${folder_name}' to local staging"
|
|
if sync_folder_to_staging "$folder_path" "$staging_folder"; then
|
|
copy_succeeded=1
|
|
fi
|
|
fi
|
|
|
|
if [[ "$has_mapping" -eq 1 ]]; then
|
|
restore_namespace_replicas "$namespace" "$state_file"
|
|
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
|
|
sleep "$SCALE_WAIT_SECONDS"
|
|
rm -f "$state_file"
|
|
fi
|
|
|
|
if (( copy_succeeded == 0 )); then
|
|
backup_failures=$((backup_failures + 1))
|
|
log_error "Backup failed for folder '${folder_name}' while copying to local staging"
|
|
cleanup_staged_folder "$staging_folder" || true
|
|
return 0
|
|
fi
|
|
|
|
backup_successes=$((backup_successes + 1))
|
|
log_info "Staged folder '${folder_name}' for the run archive"
|
|
}
|
|
|
|
print_summary() {
|
|
log_info "Run summary: processed=${total_folders} mapped=${mapped_folders} unmapped=${unmapped_folders} backup_successes=${backup_successes} backup_failures=${backup_failures} scale_warnings=${scale_warnings} restore_warnings=${restore_warnings}"
|
|
}
|
|
|
|
main() {
|
|
parse_workload_kinds
|
|
parse_excluded_namespaces
|
|
validate_inputs
|
|
load_namespaces
|
|
|
|
local folder_path
|
|
local run_archive_path
|
|
local found=0
|
|
run_archive_path="$(archive_path_for_run)"
|
|
log_info "Using single archive for this run: ${run_archive_path}"
|
|
|
|
for folder_path in "${NFS_SOURCE_PATH}"/*; do
|
|
[[ -d "$folder_path" ]] || continue
|
|
[[ "$(basename "$folder_path")" == .* ]] && continue
|
|
found=1
|
|
process_folder "$folder_path"
|
|
done
|
|
|
|
if [[ "$found" -eq 0 ]]; then
|
|
log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}"
|
|
elif (( backup_successes > 0 )); then
|
|
log_info "Compressing the complete local staging directory into: ${run_archive_path}"
|
|
if ! archive_staging "$run_archive_path"; then
|
|
backup_failures=$((backup_failures + backup_successes))
|
|
backup_successes=0
|
|
rm -f -- "$run_archive_path"
|
|
log_error "Backup failed while compressing the local staging directory"
|
|
fi
|
|
fi
|
|
|
|
if [[ -f "$run_archive_path" ]]; then
|
|
local file_size_bytes
|
|
file_size_bytes="$(wc -c < "$run_archive_path" | tr -d '[:space:]')"
|
|
if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then
|
|
total_backup_size_bytes="$file_size_bytes"
|
|
fi
|
|
fi
|
|
|
|
print_summary
|
|
local duration_seconds=$((SECONDS - backup_started_seconds))
|
|
|
|
if (( backup_successes == 0 )); then
|
|
send_backup_failure_notification \
|
|
"$NOTIFY_FAILURE_URL" \
|
|
"No NFS folders were backed up successfully" \
|
|
"$total_backup_size_bytes" || true
|
|
die "No folder backups succeeded" 1
|
|
fi
|
|
|
|
if (( backup_failures > 0 )); then
|
|
send_backup_failure_notification \
|
|
"$NOTIFY_FAILURE_URL" \
|
|
"Some NFS folders failed to back up" \
|
|
"$total_backup_size_bytes" || true
|
|
log_warn "Some folder backups failed; continuing because ${backup_successes} folder backup(s) succeeded"
|
|
fi
|
|
|
|
cleanup_archives || true
|
|
|
|
send_backup_notification \
|
|
"$NOTIFY_SUCCESS_URL" \
|
|
"$NOTIFY_TITLE" \
|
|
"${NOTIFY_ASSET}" \
|
|
"$total_backup_size_bytes" \
|
|
"$duration_seconds" || true
|
|
}
|
|
|
|
main "$@"
|