considering argocd for backups

k8s/README.md

@@ -13,10 +13,12 @@ Behavior:
 - `kube-system`, `kube-public`, and `kube-node-lease` are excluded from scale actions by default.
 - Unmapped or excluded folder: copied to local staging without Kubernetes scale actions.
 - Mapped folder: saves replicas, scales selected workloads down, waits, copies the quiesced folder to local staging, restores replicas, and waits again.
-- After every folder has been processed, one `7z` invocation compresses the complete run staging directory. Workloads are therefore restored before compression starts.
-- Per-run replica state and staged data are removed on exit. The staging parent is retained.
-- Scale-down issues are warnings by policy (backup still runs).
-- Restore issues are warnings by policy (run can still complete successfully).
+- When `argocd` exists, its Deployments and StatefulSets are saved and scaled to zero first, even if excluded or lacking a source folder. The script waits for its pods to disappear and restores ArgoCD last, after compression or during exit cleanup.
+- After every folder has been processed, one `7z` invocation compresses the complete run staging directory. Other workloads are restored before compression starts.
+- Failed state capture or scaling aborts the run. Failed restoration is retried at exit; unrecovered replica state is retained and the run exits nonzero. INT and TERM also trigger restoration; forced termination or host failure requires manual recovery from saved state.
+- Staged data is removed on exit. The staging parent and lock file are retained. Runs sharing `TMP_STATE_DIR` cannot overlap (`flock` required); use the same state directory for all invocations targeting the cluster.
+- Existing archive paths are rejected; 7z warnings count as failures. Output, state and staging directories must be outside the source tree.
+- Other reconcilers (such as operators and HPAs) and writers outside selected workload kinds are not suspended. Ordinary namespaces use a fixed shutdown delay; choose it to cover their termination time.
 - Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.
 
 ### `backup-k3s-control-plane-config.sh`
@@ -60,6 +62,7 @@ Optional:
 - `SCALE_RETRY_COUNT` (default: `3`)
 - `SCALE_RETRY_DELAY_SECONDS` (default: `5`)
 - `SCALE_WAIT_SECONDS` (default: `30`): Fixed delay after scaling down and after restoring a mapped namespace.
+- `ARGOCD_WAIT_TIMEOUT` (default: `300s`): Maximum wait for ArgoCD pods to disappear before processing folders.
 - `LOG_LEVEL` (default: `info`)
 - `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`): Parent directory for fresh per-run replica-state directories. A run never restores state files belonging to an earlier run.
 - `NOTIFY_SUCCESS_URL` (default: `http://notify.haven/api/v2/notifications/backup`; set to empty to disable)

k8s/automated-nfs-backup.sh

@@ -8,7 +8,6 @@ source "${SCRIPT_DIR}/lib/common.sh"
 # Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
 # every scaled-down namespace is restored regardless of how we exit.
 trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
-trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
 
 # Required configuration
 require_env "NFS_SOURCE_PATH"
@@ -32,6 +31,7 @@ SEVENZ_BIN="${SEVENZ_BIN:-7z}"
 SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
 SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
 SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
+ARGOCD_WAIT_TIMEOUT="${ARGOCD_WAIT_TIMEOUT:-300s}"
 TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}"
 NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
 NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
@@ -99,6 +99,7 @@ validate_inputs() {
     require_cmd "$KUBECTL_BIN"
     require_cmd "$SEVENZ_BIN"
     require_cmd "mktemp"
+    require_cmd "flock"
     require_cmd "$RSYNC_BIN"
     if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
         require_cmd "curl"
@@ -118,6 +119,8 @@ validate_inputs() {
 
     mkdir -p "$BACKUP_OUTPUT_PATH"
     mkdir -p "$TMP_STATE_DIR"
+    exec {BACKUP_LOCK_FD}>"${TMP_STATE_DIR%/}/backup.lock"
+    flock -n "$BACKUP_LOCK_FD" || die "Another NFS backup is running (lock: ${TMP_STATE_DIR}/backup.lock)"
     if ! RUN_STATE_DIR="$(mktemp -d "${TMP_STATE_DIR%/}/run-XXXXXXXXXX")"; then
         die "Unable to create a per-run state directory under ${TMP_STATE_DIR}"
     fi
@@ -125,8 +128,15 @@ validate_inputs() {
 
     local source_real
     local staging_real
+    local destination_real
     mkdir -p "$BACKUP_STAGING_PATH"
     source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
+    for destination_real in "$BACKUP_OUTPUT_PATH" "$TMP_STATE_DIR"; do
+        destination_real="$(cd "$destination_real" && pwd -P)"
+        case "${destination_real}/" in
+            "${source_real%/}/"*) die "Backup output and state directories must be outside NFS_SOURCE_PATH" ;;
+        esac
+    done
     staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
     case "${staging_real}/" in
         "${source_real%/}/"*)
@@ -144,11 +154,12 @@ validate_inputs() {
 }
 
 load_namespaces() {
-    local ns
+    local ns output
+    output="$(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')" || die "Unable to load Kubernetes namespaces"
     while IFS= read -r ns; do
         [[ -z "$ns" ]] && continue
         NAMESPACE_MAP["$ns"]=1
-    done < <(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
+    done <<< "$output"
 }
 
 namespace_for_folder() {
@@ -182,7 +193,7 @@ capture_replicas_state() {
         if ! output="$(_kubectl -n "$namespace" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.replicas}{"\t"}{range .metadata.ownerReferences[*]}{.kind},{end}{"\n"}{end}' 2>/dev/null)"; then
             log_warn "Failed to list kind '${kind}' in namespace '${namespace}' while capturing state"
             scale_warnings=$((scale_warnings + 1))
-            continue
+            return 1
         fi
 
         while IFS=$'\t' read -r name replicas owners; do
@@ -191,7 +202,7 @@ capture_replicas_state() {
             # ReplicaSet owned by a Deployment) — scaling them directly
             # conflicts with the owning controller and can leave workloads
             # in an inconsistent state after restore.
-            if [[ -n "$owners" ]]; then
+            if [[ -n "$owners" && "$namespace" != argocd ]]; then
                 log_debug "Skipping ${kind}/${name} in namespace '${namespace}': owned by ${owners%,}"
                 continue
             fi
@@ -226,8 +237,9 @@ scale_namespace_to_zero() {
         if scale_resource "$namespace" "$kind" "$name" "0"; then
             log_info "Scaled down ${namespace}:${kind}/${name} to 0"
         else
-            log_warn "Failed to scale down ${namespace}:${kind}/${name}; continuing with backup by policy"
+            log_warn "Failed to scale down ${namespace}:${kind}/${name}"
             scale_warnings=$((scale_warnings + 1))
+            return 1
         fi
     done < "$state_file"
 }
@@ -238,6 +250,7 @@ restore_namespace_replicas() {
     local kind
     local name
     local replicas
+    local failed=0
 
     while IFS=$'\t' read -r kind name replicas; do
         [[ -z "$kind" || -z "$name" ]] && continue
@@ -246,8 +259,10 @@ restore_namespace_replicas() {
         else
             log_warn "Failed to restore ${namespace}:${kind}/${name} to ${replicas}"
             restore_warnings=$((restore_warnings + 1))
+            failed=1
         fi
     done < "$state_file"
+    return "$failed"
 }
 
 restore_all_remaining() {
@@ -257,34 +272,54 @@ restore_all_remaining() {
 
     local state_file
     local namespace
-    local kind name replicas
-    local found=0
+    local failed=0
+    local argocd_seen=0
 
-    for state_file in "${RUN_STATE_DIR}"/*.state; do
+    # ArgoCD must be restored last, including on abnormal exits.
+    for state_file in "${RUN_STATE_DIR}"/*.state "${RUN_STATE_DIR}/argocd.state"; do
         [[ -f "$state_file" ]] || continue
-        found=1
         namespace="$(basename "$state_file" .state)"
+        if [[ "$namespace" == argocd && "${argocd_seen:-0}" == 0 ]]; then
+            argocd_seen=1
+            continue
+        fi
         [[ -z "$namespace" ]] && continue
         log_info "EXIT cleanup: restoring workloads in namespace '${namespace}'"
-        while IFS=$'\t' read -r kind name replicas; do
-            [[ -z "$kind" || -z "$name" ]] && continue
-            if scale_resource "$namespace" "$kind" "$name" "$replicas"; then
-                log_info "EXIT cleanup: restored ${namespace}:${kind}/${name} to ${replicas}"
-            else
-                log_warn "EXIT cleanup: failed to restore ${namespace}:${kind}/${name} to ${replicas}"
-                restore_warnings=$((restore_warnings + 1))
-            fi
-        done < "$state_file"
-        rm -f "$state_file"
+        if restore_namespace_replicas "$namespace" "$state_file"; then
+            rm -f "$state_file"
+        else
+            failed=1
+            log_error "Replica restoration failed; saved state retained at ${state_file}"
+        fi
     done
 
-    if [[ "$found" -eq 1 ]]; then
-        log_info "EXIT cleanup complete - all remaining workloads restored"
-    fi
-
     if ! rmdir -- "$RUN_STATE_DIR" 2>/dev/null; then
         log_warn "Run state directory is not empty; leaving it in place: ${RUN_STATE_DIR}"
     fi
+    return "$failed"
+}
+
+finish_run() {
+    local code="$1"
+    trap - ERR INT TERM
+    restore_all_remaining || code=1
+    cleanup_run_staging || code=1
+    on_exit "$code"
+    exit "$code"
+}
+
+pause_argocd() {
+    [[ -n "${NAMESPACE_MAP[argocd]:-}" ]] || return 0
+    local state_file
+    state_file="$(state_file_for_namespace argocd)"
+    # Independent of exclusions and WORKLOAD_KINDS: the application controller
+    # can be a Deployment or StatefulSet, and may be operator-owned.
+    local -a WORKLOAD_KIND_LIST=(deployment statefulset)
+    capture_replicas_state argocd "$state_file" || die "Unable to capture ArgoCD replicas"
+    scale_namespace_to_zero argocd "$state_file" || die "Unable to stop ArgoCD"
+    # A successful scale only changes desired replicas. Wait for the actual
+    # controllers to exit before allowing any other namespace to scale down.
+    _kubectl -n argocd wait --for=delete pod --all --timeout="$ARGOCD_WAIT_TIMEOUT" || die "ArgoCD pods did not stop"
 }
 
 cleanup_run_staging() {
@@ -437,8 +472,6 @@ archive_staging() {
     fi
     if (( rc == 0 )); then
         return 0
-    elif (( rc == 1 )); then
-        return 0
     fi
     return 1
 }
@@ -492,14 +525,19 @@ process_folder() {
     total_folders=$((total_folders + 1))
     log_info "Processing folder: ${folder_name}"
 
-    if namespace="$(namespace_for_folder "$folder_name")"; then
+    if [[ "$folder_name" == argocd && -n "${NAMESPACE_MAP[argocd]:-}" ]]; then
+        mapped_folders=$((mapped_folders + 1))
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
+        fi
+    elif namespace="$(namespace_for_folder "$folder_name")"; then
         has_mapping=1
         mapped_folders=$((mapped_folders + 1))
         log_info "Exact namespace match found for folder '${folder_name}' -> namespace '${namespace}'"
         state_file="$(state_file_for_namespace "$namespace")"
 
-        capture_replicas_state "$namespace" "$state_file"
-        scale_namespace_to_zero "$namespace" "$state_file"
+        capture_replicas_state "$namespace" "$state_file" || die "Unable to capture replicas for ${namespace}"
+        scale_namespace_to_zero "$namespace" "$state_file" || die "Unable to stop ${namespace}"
 
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
         sleep "$SCALE_WAIT_SECONDS"
@@ -524,7 +562,7 @@ process_folder() {
     fi
 
     if [[ "$has_mapping" -eq 1 ]]; then
-        restore_namespace_replicas "$namespace" "$state_file"
+        restore_namespace_replicas "$namespace" "$state_file" || die "Unable to restore ${namespace}; EXIT cleanup will retry"
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
         sleep "$SCALE_WAIT_SECONDS"
         rm -f "$state_file"
@@ -550,11 +588,13 @@ main() {
     parse_excluded_namespaces
     validate_inputs
     load_namespaces
+    pause_argocd
 
     local folder_path
     local run_archive_path
     local found=0
     run_archive_path="$(archive_path_for_run)"
+    [[ ! -e "$run_archive_path" ]] || die "Archive already exists: ${run_archive_path}"
     log_info "Using single archive for this run: ${run_archive_path}"
 
     for folder_path in "${NFS_SOURCE_PATH}"/*; do
@@ -584,6 +624,7 @@ main() {
         fi
     fi
 
+    restore_all_remaining || die "Unable to restore workloads; replica state retained"
     print_summary
     local duration_seconds=$((SECONDS - backup_started_seconds))
 
@@ -613,4 +654,7 @@ main() {
         "$duration_seconds" || true
 }
 
+trap 'finish_run "$?"' EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
 main "$@"

k8s/examples/automated-nfs-backup.env.example

@@ -16,6 +16,7 @@ SEVENZ_THREADS=on
 SCALE_RETRY_COUNT=3
 SCALE_RETRY_DELAY_SECONDS=5
 SCALE_WAIT_SECONDS=30
+ARGOCD_WAIT_TIMEOUT=300s
 LOG_LEVEL=info
 TMP_STATE_DIR=/tmp/k8s-nfs-backup
 NOTIFY_SUCCESS_URL=http://notify.haven/api/v2/notifications/backup

k8s/lib/common.sh

@@ -94,8 +94,9 @@ retry() {
     while true; do
         if "$@"; then
             return 0
+        else
+            exit_code=$?
         fi
-        exit_code=$?
         if (( attempt >= attempts )); then
             return "$exit_code"
         fi

k8s/test/test-automated-nfs-backup.sh

@@ -168,6 +168,7 @@ if [[ "$get_kind" == "namespaces" ]]; then
 fi
 
 if [[ -n "$get_kind" && -n "$namespace" ]]; then
+    [[ "${FAIL_CAPTURE_NS:-}" != "$namespace" ]] || exit 1
     replicas=""
     while IFS=: read -r configured_namespace configured_replicas; do
         if [[ "$configured_namespace" == "$namespace" ]]; then
@@ -193,10 +194,15 @@ for ((i = 1; i <= $#; i++)); do
         done
         printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
         printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log"
+        [[ "${FAIL_SCALE:-}" != "$namespace:$replicas_arg" ]] || exit 1
         exit 0
     fi
 done
 
+if [[ " $* " == *" wait "* ]]; then
+    printf 'wait %s\n' "$namespace" >> "${TMPD}/event_trace.log"
+    exit "${FAIL_WAIT:-0}"
+fi
 exit 0
 EOF
 
@@ -215,7 +221,7 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
 
     if [[ -s "${TMPD}/fail_archive.txt" ]]; then
         touch "$archive"
-        exit 2
+        exit "${ARCHIVE_FAILURE_CODE:-2}"
     fi
 
     for staged_folder in ./*; do
@@ -303,6 +309,8 @@ run_backup() {
         SEVENZ_BIN="${TMPD}/bin/7z" \
         RSYNC_BIN="${TMPD}/bin/rsync" \
         SCALE_WAIT_SECONDS=0 \
+        SCALE_RETRY_COUNT=2 \
+        SCALE_RETRY_DELAY_SECONDS=0 \
         TMP_STATE_DIR="$state_dir" \
         PATH="${TMPD}/bin:${PATH}" \
         TMPD="$TMPD" \
@@ -611,7 +619,77 @@ set -e
 assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}"
 assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0"
 assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5"
-assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state"
+assert "abort scenario removed its replica state" file_lacks "${abort_dir}/abort.log" "saved state retained"
+
+# ArgoCD is global, irrespective of folders, exclusions or workload overrides.
+for scenario in normal no-folder scale-failure wait-failure capture-failure restore-failure warning; do
+    case_dir="${TMPD}/argocd-${scenario}"
+    mkdir -p "$case_dir/source/app-ns" "$case_dir/out"
+    [[ "$scenario" == no-folder ]] || mkdir -p "$case_dir/source/argocd"
+    printf 'argocd\napp-ns\n' > "${TMPD}/namespaces.txt"
+    printf 'argocd:2\napp-ns:3\n' > "${TMPD}/replicas.txt"
+    : > "${TMPD}/fail_folders.txt"
+    : > "${TMPD}/fail_archive.txt"
+    : > "${TMPD}/event_trace.log"
+    : > "${TMPD}/scale_trace.log"
+    overrides=(WORKLOAD_KINDS=deployment EXCLUDED_NAMESPACES=argocd)
+    case "$scenario" in
+        scale-failure) overrides+=(FAIL_SCALE=argocd:--replicas=0) ;;
+        wait-failure) overrides+=(FAIL_WAIT=1) ;;
+        capture-failure) overrides+=(FAIL_CAPTURE_NS=argocd) ;;
+        restore-failure) overrides+=(FAIL_SCALE=app-ns:--replicas=3) ;;
+        warning)
+            printf 'fail\n' > "${TMPD}/fail_archive.txt"
+            overrides+=(ARCHIVE_FAILURE_CODE=1)
+            ;;
+    esac
+    set +e
+    run_backup "$case_dir/source" "$case_dir/out" "$case_dir/state" "$case_dir/run.log" "${overrides[@]}"
+    result=$?
+    set -e
+    case "$scenario" in
+        normal|no-folder)
+            assert "$scenario succeeds with ArgoCD present" is_zero "$result"
+            assert "$scenario stops ArgoCD before app and restores it after compression" events_are_ordered "${TMPD}/event_trace.log" \
+                'scale argocd --replicas=0' 'wait argocd' 'scale app-ns --replicas=0' 'scale app-ns --replicas=3' '7z staging' 'scale argocd --replicas=2'
+            assert "$scenario also stops ArgoCD StatefulSets" file_contains "${TMPD}/scale_trace.log" 'argocd statefulset web --replicas=0'
+            ;;
+        scale-failure|wait-failure|capture-failure)
+            assert "$scenario aborts" is_nonzero "$result"
+            assert "$scenario never stops app workloads" file_lacks "${TMPD}/scale_trace.log" 'app-ns'
+            ;;
+        restore-failure)
+            assert "restore failure exits nonzero" is_nonzero "$result"
+            assert "restore failure retains recovery state" file_contains "$case_dir/run.log" 'saved state retained'
+            assert "restore failure still restores ArgoCD last" events_are_ordered "${TMPD}/event_trace.log" 'scale app-ns --replicas=3' 'scale argocd --replicas=2'
+            ;;
+        warning)
+            assert "7z warning fails backup" is_nonzero "$result"
+            assert "7z warning removes incomplete archive" archive_count_is "$case_dir/out" 0
+            ;;
+    esac
+done
+
+# Collision and overlap guards must preserve existing backups and avoid scaling.
+guard_dir="${TMPD}/guards"
+mkdir -p "$guard_dir/source/app-ns" "$guard_dir/out" "$guard_dir/state"
+: > "${TMPD}/namespaces.txt"
+: > "${TMPD}/fail_archive.txt"
+printf 'existing archive\n' > "$guard_dir/out/test_fixed.7z"
+set +e
+run_backup "$guard_dir/source" "$guard_dir/out" "$guard_dir/state" "$guard_dir/collision.log" ARCHIVE_TS_FORMAT=fixed
+result=$?
+set -e
+assert "archive collision is rejected" is_nonzero "$result"
+assert "archive collision preserves original content" file_contains "$guard_dir/out/test_fixed.7z" 'existing archive'
+
+(
+    flock -n 9 || exit 1
+    set +e
+    run_backup "$guard_dir/source" "$guard_dir/out" "$guard_dir/state" "$guard_dir/locked.log"
+    [[ $? -ne 0 ]]
+) 9>"$guard_dir/state/backup.lock"
+assert "overlapping run is rejected by lock" file_contains "$guard_dir/locked.log" 'Another NFS backup is running'
 
 printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
 if [[ "$tests_failed" -gt 0 ]]; then