updating flow and notify endpoint

k8s/automated-nfs-backup.sh

@@ -20,7 +20,7 @@ KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 KUBE_CONTEXT="${KUBE_CONTEXT:-}"
 WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
 EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease}"
-BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-}"
+BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-/tmp/k8s-nfs-backup-staging}"
 RSYNC_BIN="${RSYNC_BIN:-rsync}"
 ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
 ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
@@ -33,11 +33,12 @@ SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
 SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
 SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
 TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}"
-NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}"
-NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}"
+NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
+NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
 NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
 NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
-NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}"
+NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes NFS backup}"
+NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
 CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
 RUN_STATE_DIR=""
 RUN_STAGING_DIR=""
@@ -59,6 +60,7 @@ backup_failures=0
 scale_warnings=0
 restore_warnings=0
 total_backup_size_bytes=0
+backup_started_seconds="$SECONDS"
 
 _kubectl() {
     "$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" </dev/null
@@ -97,9 +99,7 @@ validate_inputs() {
     require_cmd "$KUBECTL_BIN"
     require_cmd "$SEVENZ_BIN"
     require_cmd "mktemp"
-    if [[ -n "$BACKUP_STAGING_PATH" ]]; then
-        require_cmd "$RSYNC_BIN"
-    fi
+    require_cmd "$RSYNC_BIN"
     if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
         require_cmd "curl"
     fi
@@ -123,22 +123,20 @@ validate_inputs() {
     fi
     log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
 
-    if [[ -n "$BACKUP_STAGING_PATH" ]]; then
-        local source_real
-        local staging_real
-        mkdir -p "$BACKUP_STAGING_PATH"
-        source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
-        staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
-        case "${staging_real}/" in
-            "${source_real%/}/"*)
-                die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
-                ;;
-        esac
-        if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
-            die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
-        fi
-        log_info "Two-pass rsync staging enabled: ${RUN_STAGING_DIR}"
+    local source_real
+    local staging_real
+    mkdir -p "$BACKUP_STAGING_PATH"
+    source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
+    staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
+    case "${staging_real}/" in
+        "${source_real%/}/"*)
+            die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
+            ;;
+    esac
+    if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
+        die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
     fi
+    log_info "Using local run staging directory: ${RUN_STAGING_DIR}"
 
     if ! _kubectl get namespaces >/dev/null 2>&1; then
         die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
@@ -347,20 +345,12 @@ json_escape() {
     printf '%s' "$value"
 }
 
-backup_size_mb() {
-    local bytes="$1"
-    if (( bytes <= 0 )); then
-        echo 0
-        return 0
-    fi
-    echo $(( (bytes + 1048575) / 1048576 ))
-}
-
 send_backup_notification() {
     local url="${1:-}"
     local title="${2:-}"
     local asset="${3:-}"
-    local size_mb="${4:-0}"
+    local size_bytes="${4:-0}"
+    local duration_seconds="${5:-0}"
 
     [[ -z "$url" ]] && return 0
 
@@ -369,7 +359,9 @@ send_backup_notification() {
 {
   "title": "$(json_escape "$title")",
   "asset": "$(json_escape "$asset")",
-  "backupSizeInMB": ${size_mb}
+  "sizeBytes": ${size_bytes},
+  "durationSeconds": ${duration_seconds},
+  "source": "$(json_escape "$NOTIFY_SOURCE")"
 }
 EOF
 )
@@ -386,20 +378,21 @@ EOF
 send_backup_failure_notification() {
     local url="${1:-}"
     local message="${2:-}"
-    local size_mb="${3:-0}"
+    local size_bytes="${3:-0}"
 
     [[ -z "$url" ]] && return 0
 
     local payload
     payload=$(cat <<EOF
 {
-  "caller": "$(json_escape "$NOTIFY_CALLER")",
+  "source": "$(json_escape "$NOTIFY_SOURCE")",
   "message": "$(json_escape "$message")",
-  "critical": true,
+  "severity": "critical",
+  "errorCode": "BACKUP_FAILED",
   "extra": [
     {
-      "name": "Backup Size (MB)",
-      "value": "${size_mb}"
+      "name": "Archive size",
+      "value": "${size_bytes} B"
     }
   ]
 }
@@ -415,23 +408,17 @@ EOF
     return 0
 }
 
-backup_folder() {
-    local folder_path="${1:?folder path required}"
-    local archive_path="${2:?archive path required}"
-    local working_directory="${3:-}"
-    local archive_input="$folder_path"
+archive_staging() {
+    local archive_path="${1:?archive path required}"
 
-    if [[ -n "$working_directory" ]]; then
-        archive_input="$(basename "$folder_path")"
-        if [[ "$archive_path" != /* ]]; then
-            archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
-        fi
+    if [[ "$archive_path" != /* ]]; then
+        archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
     fi
 
     local -a cmd=(
         "$SEVENZ_BIN" a -t7z
         "$archive_path"
-        "$archive_input"
+        .
         "-m0=${SEVENZ_METHOD}"
         "-mx=${SEVENZ_LEVEL}"
         "-mmt=${SEVENZ_THREADS}"
@@ -442,15 +429,11 @@ backup_folder() {
     fi
 
     local cmd_output rc filtered_output
-    if [[ -n "$working_directory" ]]; then
-        cmd_output="$(cd "$working_directory" && "${cmd[@]}" 2>&1 >/dev/null)"
-    else
-        cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)"
-    fi
+    cmd_output="$(cd "$RUN_STAGING_DIR" && "${cmd[@]}" 2>&1 >/dev/null)"
     rc=$?
     filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
     if [[ -n "$filtered_output" ]]; then
-        log_warn "7z output for '${folder_path}': ${filtered_output}"
+        log_warn "7z output for staging directory '${RUN_STAGING_DIR}': ${filtered_output}"
     fi
     if (( rc == 0 )); then
         return 0
@@ -496,32 +479,19 @@ cleanup_archives() {
 
 process_folder() {
     local folder_path="${1:?folder path is required}"
-    local archive_path="${2:?archive path is required}"
     local folder_name
     local namespace=""
     local state_file=""
     local has_mapping=0
     local namespace_match_status=0
-    local backup_source_path="$folder_path"
-    local staging_folder=""
-    local final_sync_failed=0
+    local staging_folder
+    local copy_succeeded=0
 
     folder_name="$(basename "$folder_path")"
+    staging_folder="${RUN_STAGING_DIR}/${folder_name}"
     total_folders=$((total_folders + 1))
     log_info "Processing folder: ${folder_name}"
 
-    if [[ -n "$RUN_STAGING_DIR" ]]; then
-        staging_folder="${RUN_STAGING_DIR}/${folder_name}"
-        log_info "Starting live pre-sync for folder '${folder_name}'"
-        if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
-            backup_failures=$((backup_failures + 1))
-            log_error "Backup failed for folder '${folder_name}' during live pre-sync"
-            cleanup_staged_folder "$staging_folder" || true
-            return 0
-        fi
-        backup_source_path="$staging_folder"
-    fi
-
     if namespace="$(namespace_for_folder "$folder_name")"; then
         has_mapping=1
         mapped_folders=$((mapped_folders + 1))
@@ -534,11 +504,9 @@ process_folder() {
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
         sleep "$SCALE_WAIT_SECONDS"
 
-        if [[ -n "$staging_folder" ]]; then
-            log_info "Starting final offline rsync delta for namespace '${namespace}'"
-            if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
-                final_sync_failed=1
-            fi
+        log_info "Copying quiesced namespace folder '${folder_name}' to local staging"
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
         fi
     else
         namespace_match_status=$?
@@ -548,40 +516,29 @@ process_folder() {
         else
             log_warn "No namespace matched folder '${folder_name}'. Running backup only."
         fi
+
+        log_info "Copying live folder '${folder_name}' to local staging"
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
+        fi
     fi
 
-    if [[ "$has_mapping" -eq 1 && -n "$staging_folder" ]]; then
+    if [[ "$has_mapping" -eq 1 ]]; then
         restore_namespace_replicas "$namespace" "$state_file"
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
         sleep "$SCALE_WAIT_SECONDS"
         rm -f "$state_file"
     fi
 
-    if (( final_sync_failed == 1 )); then
+    if (( copy_succeeded == 0 )); then
         backup_failures=$((backup_failures + 1))
-        log_error "Backup failed for folder '${folder_name}' during final offline rsync"
+        log_error "Backup failed for folder '${folder_name}' while copying to local staging"
         cleanup_staged_folder "$staging_folder" || true
         return 0
     fi
 
-    if backup_folder "$backup_source_path" "$archive_path" "${RUN_STAGING_DIR:-}"; then
-        backup_successes=$((backup_successes + 1))
-        log_info "Added folder '${folder_name}' to archive: ${archive_path}"
-    else
-        backup_failures=$((backup_failures + 1))
-        log_error "Backup failed for folder '${folder_name}'"
-    fi
-
-    if [[ "$has_mapping" -eq 1 && -z "$staging_folder" ]]; then
-        restore_namespace_replicas "$namespace" "$state_file"
-        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
-        sleep "$SCALE_WAIT_SECONDS"
-        rm -f "$state_file"
-    fi
-
-    if [[ -n "$staging_folder" ]]; then
-        cleanup_staged_folder "$staging_folder" || true
-    fi
+    backup_successes=$((backup_successes + 1))
+    log_info "Staged folder '${folder_name}' for the run archive"
 }
 
 print_summary() {
@@ -595,22 +552,31 @@ main() {
     load_namespaces
 
     local folder_path
-    local run_archive_path=""
+    local run_archive_path
     local found=0
+    run_archive_path="$(archive_path_for_run)"
+    log_info "Using single archive for this run: ${run_archive_path}"
+
     for folder_path in "${NFS_SOURCE_PATH}"/*; do
         [[ -d "$folder_path" ]] || continue
         [[ "$(basename "$folder_path")" == .* ]] && continue
-        if [[ -z "$run_archive_path" ]]; then
-            run_archive_path="$(archive_path_for_run)"
-            log_info "Using single archive for this run: ${run_archive_path}"
-        fi
         found=1
-        process_folder "$folder_path" "$run_archive_path"
+        process_folder "$folder_path"
     done
 
     if [[ "$found" -eq 0 ]]; then
         log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}"
-    elif [[ -f "$run_archive_path" ]]; then
+    elif (( backup_successes > 0 )); then
+        log_info "Compressing the complete local staging directory into: ${run_archive_path}"
+        if ! archive_staging "$run_archive_path"; then
+            backup_failures=$((backup_failures + backup_successes))
+            backup_successes=0
+            rm -f -- "$run_archive_path"
+            log_error "Backup failed while compressing the local staging directory"
+        fi
+    fi
+
+    if [[ -f "$run_archive_path" ]]; then
         local file_size_bytes
         file_size_bytes="$(wc -c < "$run_archive_path" | tr -d '[:space:]')"
         if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then
@@ -619,14 +585,13 @@ main() {
     fi
 
     print_summary
-    local size_mb
-    size_mb="$(backup_size_mb "$total_backup_size_bytes")"
+    local duration_seconds=$((SECONDS - backup_started_seconds))
 
     if (( backup_successes == 0 )); then
         send_backup_failure_notification \
             "$NOTIFY_FAILURE_URL" \
             "No NFS folders were backed up successfully" \
-            "$size_mb" || true
+            "$total_backup_size_bytes" || true
         die "No folder backups succeeded" 1
     fi
 
@@ -634,7 +599,7 @@ main() {
         send_backup_failure_notification \
             "$NOTIFY_FAILURE_URL" \
             "Some NFS folders failed to back up" \
-            "$size_mb" || true
+            "$total_backup_size_bytes" || true
         log_warn "Some folder backups failed; continuing because ${backup_successes} folder backup(s) succeeded"
     fi
 
@@ -644,7 +609,8 @@ main() {
         "$NOTIFY_SUCCESS_URL" \
         "$NOTIFY_TITLE" \
         "${NOTIFY_ASSET}" \
-        "$size_mb" || true
+        "$total_backup_size_bytes" \
+        "$duration_seconds" || true
 }
 
 main "$@"