considering argocd for backups
This commit is contained in:
+74
-30
@@ -8,7 +8,6 @@ source "${SCRIPT_DIR}/lib/common.sh"
|
||||
# Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
|
||||
# every scaled-down namespace is restored regardless of how we exit.
|
||||
trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
|
||||
trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
|
||||
|
||||
# Required configuration
|
||||
require_env "NFS_SOURCE_PATH"
|
||||
@@ -32,6 +31,7 @@ SEVENZ_BIN="${SEVENZ_BIN:-7z}"
|
||||
SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
|
||||
SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
|
||||
SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
|
||||
ARGOCD_WAIT_TIMEOUT="${ARGOCD_WAIT_TIMEOUT:-300s}"
|
||||
TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}"
|
||||
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
|
||||
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
|
||||
@@ -99,6 +99,7 @@ validate_inputs() {
|
||||
require_cmd "$KUBECTL_BIN"
|
||||
require_cmd "$SEVENZ_BIN"
|
||||
require_cmd "mktemp"
|
||||
require_cmd "flock"
|
||||
require_cmd "$RSYNC_BIN"
|
||||
if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
|
||||
require_cmd "curl"
|
||||
@@ -118,6 +119,8 @@ validate_inputs() {
|
||||
|
||||
mkdir -p "$BACKUP_OUTPUT_PATH"
|
||||
mkdir -p "$TMP_STATE_DIR"
|
||||
exec {BACKUP_LOCK_FD}>"${TMP_STATE_DIR%/}/backup.lock"
|
||||
flock -n "$BACKUP_LOCK_FD" || die "Another NFS backup is running (lock: ${TMP_STATE_DIR}/backup.lock)"
|
||||
if ! RUN_STATE_DIR="$(mktemp -d "${TMP_STATE_DIR%/}/run-XXXXXXXXXX")"; then
|
||||
die "Unable to create a per-run state directory under ${TMP_STATE_DIR}"
|
||||
fi
|
||||
@@ -125,8 +128,15 @@ validate_inputs() {
|
||||
|
||||
local source_real
|
||||
local staging_real
|
||||
local destination_real
|
||||
mkdir -p "$BACKUP_STAGING_PATH"
|
||||
source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
|
||||
for destination_real in "$BACKUP_OUTPUT_PATH" "$TMP_STATE_DIR"; do
|
||||
destination_real="$(cd "$destination_real" && pwd -P)"
|
||||
case "${destination_real}/" in
|
||||
"${source_real%/}/"*) die "Backup output and state directories must be outside NFS_SOURCE_PATH" ;;
|
||||
esac
|
||||
done
|
||||
staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
|
||||
case "${staging_real}/" in
|
||||
"${source_real%/}/"*)
|
||||
@@ -144,11 +154,12 @@ validate_inputs() {
|
||||
}
|
||||
|
||||
load_namespaces() {
|
||||
local ns
|
||||
local ns output
|
||||
output="$(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')" || die "Unable to load Kubernetes namespaces"
|
||||
while IFS= read -r ns; do
|
||||
[[ -z "$ns" ]] && continue
|
||||
NAMESPACE_MAP["$ns"]=1
|
||||
done < <(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
done <<< "$output"
|
||||
}
|
||||
|
||||
namespace_for_folder() {
|
||||
@@ -182,7 +193,7 @@ capture_replicas_state() {
|
||||
if ! output="$(_kubectl -n "$namespace" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.replicas}{"\t"}{range .metadata.ownerReferences[*]}{.kind},{end}{"\n"}{end}' 2>/dev/null)"; then
|
||||
log_warn "Failed to list kind '${kind}' in namespace '${namespace}' while capturing state"
|
||||
scale_warnings=$((scale_warnings + 1))
|
||||
continue
|
||||
return 1
|
||||
fi
|
||||
|
||||
while IFS=$'\t' read -r name replicas owners; do
|
||||
@@ -191,7 +202,7 @@ capture_replicas_state() {
|
||||
# ReplicaSet owned by a Deployment) — scaling them directly
|
||||
# conflicts with the owning controller and can leave workloads
|
||||
# in an inconsistent state after restore.
|
||||
if [[ -n "$owners" ]]; then
|
||||
if [[ -n "$owners" && "$namespace" != argocd ]]; then
|
||||
log_debug "Skipping ${kind}/${name} in namespace '${namespace}': owned by ${owners%,}"
|
||||
continue
|
||||
fi
|
||||
@@ -226,8 +237,9 @@ scale_namespace_to_zero() {
|
||||
if scale_resource "$namespace" "$kind" "$name" "0"; then
|
||||
log_info "Scaled down ${namespace}:${kind}/${name} to 0"
|
||||
else
|
||||
log_warn "Failed to scale down ${namespace}:${kind}/${name}; continuing with backup by policy"
|
||||
log_warn "Failed to scale down ${namespace}:${kind}/${name}"
|
||||
scale_warnings=$((scale_warnings + 1))
|
||||
return 1
|
||||
fi
|
||||
done < "$state_file"
|
||||
}
|
||||
@@ -238,6 +250,7 @@ restore_namespace_replicas() {
|
||||
local kind
|
||||
local name
|
||||
local replicas
|
||||
local failed=0
|
||||
|
||||
while IFS=$'\t' read -r kind name replicas; do
|
||||
[[ -z "$kind" || -z "$name" ]] && continue
|
||||
@@ -246,8 +259,10 @@ restore_namespace_replicas() {
|
||||
else
|
||||
log_warn "Failed to restore ${namespace}:${kind}/${name} to ${replicas}"
|
||||
restore_warnings=$((restore_warnings + 1))
|
||||
failed=1
|
||||
fi
|
||||
done < "$state_file"
|
||||
return "$failed"
|
||||
}
|
||||
|
||||
restore_all_remaining() {
|
||||
@@ -257,34 +272,54 @@ restore_all_remaining() {
|
||||
|
||||
local state_file
|
||||
local namespace
|
||||
local kind name replicas
|
||||
local found=0
|
||||
local failed=0
|
||||
local argocd_seen=0
|
||||
|
||||
for state_file in "${RUN_STATE_DIR}"/*.state; do
|
||||
# ArgoCD must be restored last, including on abnormal exits.
|
||||
for state_file in "${RUN_STATE_DIR}"/*.state "${RUN_STATE_DIR}/argocd.state"; do
|
||||
[[ -f "$state_file" ]] || continue
|
||||
found=1
|
||||
namespace="$(basename "$state_file" .state)"
|
||||
if [[ "$namespace" == argocd && "${argocd_seen:-0}" == 0 ]]; then
|
||||
argocd_seen=1
|
||||
continue
|
||||
fi
|
||||
[[ -z "$namespace" ]] && continue
|
||||
log_info "EXIT cleanup: restoring workloads in namespace '${namespace}'"
|
||||
while IFS=$'\t' read -r kind name replicas; do
|
||||
[[ -z "$kind" || -z "$name" ]] && continue
|
||||
if scale_resource "$namespace" "$kind" "$name" "$replicas"; then
|
||||
log_info "EXIT cleanup: restored ${namespace}:${kind}/${name} to ${replicas}"
|
||||
else
|
||||
log_warn "EXIT cleanup: failed to restore ${namespace}:${kind}/${name} to ${replicas}"
|
||||
restore_warnings=$((restore_warnings + 1))
|
||||
fi
|
||||
done < "$state_file"
|
||||
rm -f "$state_file"
|
||||
if restore_namespace_replicas "$namespace" "$state_file"; then
|
||||
rm -f "$state_file"
|
||||
else
|
||||
failed=1
|
||||
log_error "Replica restoration failed; saved state retained at ${state_file}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$found" -eq 1 ]]; then
|
||||
log_info "EXIT cleanup complete - all remaining workloads restored"
|
||||
fi
|
||||
|
||||
if ! rmdir -- "$RUN_STATE_DIR" 2>/dev/null; then
|
||||
log_warn "Run state directory is not empty; leaving it in place: ${RUN_STATE_DIR}"
|
||||
fi
|
||||
return "$failed"
|
||||
}
|
||||
|
||||
finish_run() {
|
||||
local code="$1"
|
||||
trap - ERR INT TERM
|
||||
restore_all_remaining || code=1
|
||||
cleanup_run_staging || code=1
|
||||
on_exit "$code"
|
||||
exit "$code"
|
||||
}
|
||||
|
||||
pause_argocd() {
|
||||
[[ -n "${NAMESPACE_MAP[argocd]:-}" ]] || return 0
|
||||
local state_file
|
||||
state_file="$(state_file_for_namespace argocd)"
|
||||
# Independent of exclusions and WORKLOAD_KINDS: the application controller
|
||||
# can be a Deployment or StatefulSet, and may be operator-owned.
|
||||
local -a WORKLOAD_KIND_LIST=(deployment statefulset)
|
||||
capture_replicas_state argocd "$state_file" || die "Unable to capture ArgoCD replicas"
|
||||
scale_namespace_to_zero argocd "$state_file" || die "Unable to stop ArgoCD"
|
||||
# A successful scale only changes desired replicas. Wait for the actual
|
||||
# controllers to exit before allowing any other namespace to scale down.
|
||||
_kubectl -n argocd wait --for=delete pod --all --timeout="$ARGOCD_WAIT_TIMEOUT" || die "ArgoCD pods did not stop"
|
||||
}
|
||||
|
||||
cleanup_run_staging() {
|
||||
@@ -437,8 +472,6 @@ archive_staging() {
|
||||
fi
|
||||
if (( rc == 0 )); then
|
||||
return 0
|
||||
elif (( rc == 1 )); then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
@@ -492,14 +525,19 @@ process_folder() {
|
||||
total_folders=$((total_folders + 1))
|
||||
log_info "Processing folder: ${folder_name}"
|
||||
|
||||
if namespace="$(namespace_for_folder "$folder_name")"; then
|
||||
if [[ "$folder_name" == argocd && -n "${NAMESPACE_MAP[argocd]:-}" ]]; then
|
||||
mapped_folders=$((mapped_folders + 1))
|
||||
if sync_folder_to_staging "$folder_path" "$staging_folder"; then
|
||||
copy_succeeded=1
|
||||
fi
|
||||
elif namespace="$(namespace_for_folder "$folder_name")"; then
|
||||
has_mapping=1
|
||||
mapped_folders=$((mapped_folders + 1))
|
||||
log_info "Exact namespace match found for folder '${folder_name}' -> namespace '${namespace}'"
|
||||
state_file="$(state_file_for_namespace "$namespace")"
|
||||
|
||||
capture_replicas_state "$namespace" "$state_file"
|
||||
scale_namespace_to_zero "$namespace" "$state_file"
|
||||
capture_replicas_state "$namespace" "$state_file" || die "Unable to capture replicas for ${namespace}"
|
||||
scale_namespace_to_zero "$namespace" "$state_file" || die "Unable to stop ${namespace}"
|
||||
|
||||
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
|
||||
sleep "$SCALE_WAIT_SECONDS"
|
||||
@@ -524,7 +562,7 @@ process_folder() {
|
||||
fi
|
||||
|
||||
if [[ "$has_mapping" -eq 1 ]]; then
|
||||
restore_namespace_replicas "$namespace" "$state_file"
|
||||
restore_namespace_replicas "$namespace" "$state_file" || die "Unable to restore ${namespace}; EXIT cleanup will retry"
|
||||
log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
|
||||
sleep "$SCALE_WAIT_SECONDS"
|
||||
rm -f "$state_file"
|
||||
@@ -550,11 +588,13 @@ main() {
|
||||
parse_excluded_namespaces
|
||||
validate_inputs
|
||||
load_namespaces
|
||||
pause_argocd
|
||||
|
||||
local folder_path
|
||||
local run_archive_path
|
||||
local found=0
|
||||
run_archive_path="$(archive_path_for_run)"
|
||||
[[ ! -e "$run_archive_path" ]] || die "Archive already exists: ${run_archive_path}"
|
||||
log_info "Using single archive for this run: ${run_archive_path}"
|
||||
|
||||
for folder_path in "${NFS_SOURCE_PATH}"/*; do
|
||||
@@ -584,6 +624,7 @@ main() {
|
||||
fi
|
||||
fi
|
||||
|
||||
restore_all_remaining || die "Unable to restore workloads; replica state retained"
|
||||
print_summary
|
||||
local duration_seconds=$((SECONDS - backup_started_seconds))
|
||||
|
||||
@@ -613,4 +654,7 @@ main() {
|
||||
"$duration_seconds" || true
|
||||
}
|
||||
|
||||
trap 'finish_run "$?"' EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user