From cafbb01200293e3e1e05bb4e5474deee4d8ad6e7 Mon Sep 17 00:00:00 2001 From: Jose Henrique Date: Wed, 22 Jul 2026 08:32:55 -0300 Subject: [PATCH] updating flow and notify endpoint --- k8s/README.md | 44 +++-- k8s/automated-nfs-backup.sh | 184 +++++++----------- k8s/examples/automated-nfs-backup.env.example | 7 +- k8s/test/test-automated-nfs-backup.sh | 136 ++++++++----- 4 files changed, 198 insertions(+), 173 deletions(-) diff --git a/k8s/README.md b/k8s/README.md index 63d9865..2364a5f 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -6,14 +6,15 @@ Kubernetes-focused shell scripts intended for cronjobs and operational utilities ### `automated-nfs-backup.sh` -Backs up each top-level folder found in `NFS_SOURCE_PATH` into a single encrypted `.7z` archive per run, with optional Kubernetes workload quiescing when a folder name exactly matches a namespace name. +Copies each top-level folder found in `NFS_SOURCE_PATH` to a per-run local staging directory, then creates one encrypted `.7z` archive from the complete staged data. Kubernetes workloads are quiesced while a folder whose name exactly matches a namespace is copied. Behavior: - Exact folder-to-namespace mapping only. -- `kube-system`, `kube-public`, `kube-node-lease`, and `default` are excluded from scale actions by default. -- Unmapped folder: backup still runs, Kubernetes scale actions are skipped. -- Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again. -- When `BACKUP_STAGING_PATH` is configured, a live rsync pre-copy is followed by a final offline delta; the namespace is restored before the staged copy is compressed. +- `kube-system`, `kube-public`, and `kube-node-lease` are excluded from scale actions by default. +- Unmapped or excluded folder: copied to local staging without Kubernetes scale actions. +- Mapped folder: saves replicas, scales selected workloads down, waits, copies the quiesced folder to local staging, restores replicas, and waits again. +- After every folder has been processed, one `7z` invocation compresses the complete run staging directory. Workloads are therefore restored before compression starts. +- Per-run replica state and staged data are removed on exit. The staging parent is retained. - Scale-down issues are warnings by policy (backup still runs). - Restore issues are warnings by policy (run can still complete successfully). - Cleanup retains the `N` most recent matching archives by modification time and deletes older ones. @@ -46,9 +47,9 @@ Optional: - `BACKUP_PASSWORD` (default: empty): When set, archive uses password protection; when empty, archive is not encrypted. - `KUBECTL_BIN` (default: `kubectl`) - `KUBE_CONTEXT` (default: empty) -- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`) -- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease,default`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions. -- `BACKUP_STAGING_PATH` (default: empty, disabled): Local staging parent for two-pass rsync. It must be outside `NFS_SOURCE_PATH` and have enough free space for the largest folder being backed up. +- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset`) +- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions. +- `BACKUP_STAGING_PATH` (default: `/tmp/k8s-nfs-backup-staging`): Local parent for per-run staging directories. It must be outside `NFS_SOURCE_PATH` and have enough free space for all folders included in one backup run. - `RSYNC_BIN` (default: `rsync`) - `ARCHIVE_PREFIX` (default: `nfs-backup`) - `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`) @@ -61,19 +62,33 @@ Optional: - `SCALE_WAIT_SECONDS` (default: `30`): Fixed delay after scaling down and after restoring a mapped namespace. - `LOG_LEVEL` (default: `info`) - `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`): Parent directory for fresh per-run replica-state directories. A run never restores state files belonging to an earlier run. -- `NOTIFY_SUCCESS_URL` (default: empty, disabled) -- `NOTIFY_FAILURE_URL` (default: empty, disabled) +- `NOTIFY_SUCCESS_URL` (default: `http://notify.haven/api/v2/notifications/backup`; set to empty to disable) +- `NOTIFY_FAILURE_URL` (default: `http://notify.haven/api/v2/notifications/error`; set to empty to disable) - `NOTIFY_TITLE` (default: `Kubernetes`) - `NOTIFY_ASSET` (default: `kube config`) +- `NOTIFY_SOURCE` (default: `NOTIFY_CALLER`, or `Kubernetes NFS backup`): Haven Notify v2 source identifier. `NOTIFY_CALLER` remains a compatibility fallback. - `CLEANUP_KEEP_COUNT` (default: `4`) -Notification payload (success and failure): +Notifications use the Haven Notify v2 backup and error endpoints. Success payloads include the final archive size and elapsed time: ```json { "title": "Kubernetes", "asset": "kube config", - "backupSizeInMB": 123 + "sizeBytes": 128974848, + "durationSeconds": 125, + "source": "Kubernetes NFS backup" +} +``` + +Failures use the v2 error contract: + +```json +{ + "source": "Kubernetes NFS backup", + "message": "Some NFS folders failed to back up", + "severity": "critical", + "errorCode": "BACKUP_FAILED" } ``` @@ -83,12 +98,13 @@ Notification payload (success and failure): - Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces. - Ensure `kubectl` context and credentials are present in the runtime. - Ensure `7z` is installed in the runtime image/host. -- When `BACKUP_STAGING_PATH` is enabled, ensure `rsync` is installed and place staging on fast local storage for the shortest offline delta. +- Ensure `rsync` is installed and keep `BACKUP_STAGING_PATH` on fast local storage. Capacity must accommodate one complete uncompressed backup run in addition to the output archive. ## Failure Semantics - Missing required env vars, missing commands, invalid paths, or inability to list namespaces: script exits non-zero immediately. -- Folder backup failures are counted. One or more successful folder backups continues with cleanup and exits zero; zero successful folder backups exits non-zero. +- Folder staging failures are counted and failed partial copies are excluded from the archive. One or more successfully staged folders continues to archive creation; zero successfully staged folders exits non-zero. +- If the single archive operation fails, the run treats every staged folder as failed and exits non-zero. - Scale-down warnings/timeouts: logged and counted, backup continues. - Restore warnings/timeouts: logged and counted, script does not fail solely because of restore warnings. - If `NOTIFY_SUCCESS_URL` is set, success notification is sent at the end of a successful run. diff --git a/k8s/automated-nfs-backup.sh b/k8s/automated-nfs-backup.sh index 3eb4ba1..1ed6763 100644 --- a/k8s/automated-nfs-backup.sh +++ b/k8s/automated-nfs-backup.sh @@ -20,7 +20,7 @@ KUBECTL_BIN="${KUBECTL_BIN:-kubectl}" KUBE_CONTEXT="${KUBE_CONTEXT:-}" WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}" EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease}" -BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-}" +BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-/tmp/k8s-nfs-backup-staging}" RSYNC_BIN="${RSYNC_BIN:-rsync}" ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}" ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}" @@ -33,11 +33,12 @@ SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}" SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}" SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}" TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}" -NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}" -NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}" +NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}" +NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}" NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}" NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}" -NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}" +NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes NFS backup}" +NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}" CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}" RUN_STATE_DIR="" RUN_STAGING_DIR="" @@ -59,6 +60,7 @@ backup_failures=0 scale_warnings=0 restore_warnings=0 total_backup_size_bytes=0 +backup_started_seconds="$SECONDS" _kubectl() { "$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" /dev/null 2>&1; then die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}" @@ -347,20 +345,12 @@ json_escape() { printf '%s' "$value" } -backup_size_mb() { - local bytes="$1" - if (( bytes <= 0 )); then - echo 0 - return 0 - fi - echo $(( (bytes + 1048575) / 1048576 )) -} - send_backup_notification() { local url="${1:-}" local title="${2:-}" local asset="${3:-}" - local size_mb="${4:-0}" + local size_bytes="${4:-0}" + local duration_seconds="${5:-0}" [[ -z "$url" ]] && return 0 @@ -369,7 +359,9 @@ send_backup_notification() { { "title": "$(json_escape "$title")", "asset": "$(json_escape "$asset")", - "backupSizeInMB": ${size_mb} + "sizeBytes": ${size_bytes}, + "durationSeconds": ${duration_seconds}, + "source": "$(json_escape "$NOTIFY_SOURCE")" } EOF ) @@ -386,20 +378,21 @@ EOF send_backup_failure_notification() { local url="${1:-}" local message="${2:-}" - local size_mb="${3:-0}" + local size_bytes="${3:-0}" [[ -z "$url" ]] && return 0 local payload payload=$(cat <&1 >/dev/null)" - else - cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)" - fi + cmd_output="$(cd "$RUN_STAGING_DIR" && "${cmd[@]}" 2>&1 >/dev/null)" rc=$? filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)" if [[ -n "$filtered_output" ]]; then - log_warn "7z output for '${folder_path}': ${filtered_output}" + log_warn "7z output for staging directory '${RUN_STAGING_DIR}': ${filtered_output}" fi if (( rc == 0 )); then return 0 @@ -496,32 +479,19 @@ cleanup_archives() { process_folder() { local folder_path="${1:?folder path is required}" - local archive_path="${2:?archive path is required}" local folder_name local namespace="" local state_file="" local has_mapping=0 local namespace_match_status=0 - local backup_source_path="$folder_path" - local staging_folder="" - local final_sync_failed=0 + local staging_folder + local copy_succeeded=0 folder_name="$(basename "$folder_path")" + staging_folder="${RUN_STAGING_DIR}/${folder_name}" total_folders=$((total_folders + 1)) log_info "Processing folder: ${folder_name}" - if [[ -n "$RUN_STAGING_DIR" ]]; then - staging_folder="${RUN_STAGING_DIR}/${folder_name}" - log_info "Starting live pre-sync for folder '${folder_name}'" - if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then - backup_failures=$((backup_failures + 1)) - log_error "Backup failed for folder '${folder_name}' during live pre-sync" - cleanup_staged_folder "$staging_folder" || true - return 0 - fi - backup_source_path="$staging_folder" - fi - if namespace="$(namespace_for_folder "$folder_name")"; then has_mapping=1 mapped_folders=$((mapped_folders + 1)) @@ -534,11 +504,9 @@ process_folder() { log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..." sleep "$SCALE_WAIT_SECONDS" - if [[ -n "$staging_folder" ]]; then - log_info "Starting final offline rsync delta for namespace '${namespace}'" - if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then - final_sync_failed=1 - fi + log_info "Copying quiesced namespace folder '${folder_name}' to local staging" + if sync_folder_to_staging "$folder_path" "$staging_folder"; then + copy_succeeded=1 fi else namespace_match_status=$? @@ -548,40 +516,29 @@ process_folder() { else log_warn "No namespace matched folder '${folder_name}'. Running backup only." fi + + log_info "Copying live folder '${folder_name}' to local staging" + if sync_folder_to_staging "$folder_path" "$staging_folder"; then + copy_succeeded=1 + fi fi - if [[ "$has_mapping" -eq 1 && -n "$staging_folder" ]]; then + if [[ "$has_mapping" -eq 1 ]]; then restore_namespace_replicas "$namespace" "$state_file" log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..." sleep "$SCALE_WAIT_SECONDS" rm -f "$state_file" fi - if (( final_sync_failed == 1 )); then + if (( copy_succeeded == 0 )); then backup_failures=$((backup_failures + 1)) - log_error "Backup failed for folder '${folder_name}' during final offline rsync" + log_error "Backup failed for folder '${folder_name}' while copying to local staging" cleanup_staged_folder "$staging_folder" || true return 0 fi - if backup_folder "$backup_source_path" "$archive_path" "${RUN_STAGING_DIR:-}"; then - backup_successes=$((backup_successes + 1)) - log_info "Added folder '${folder_name}' to archive: ${archive_path}" - else - backup_failures=$((backup_failures + 1)) - log_error "Backup failed for folder '${folder_name}'" - fi - - if [[ "$has_mapping" -eq 1 && -z "$staging_folder" ]]; then - restore_namespace_replicas "$namespace" "$state_file" - log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..." - sleep "$SCALE_WAIT_SECONDS" - rm -f "$state_file" - fi - - if [[ -n "$staging_folder" ]]; then - cleanup_staged_folder "$staging_folder" || true - fi + backup_successes=$((backup_successes + 1)) + log_info "Staged folder '${folder_name}' for the run archive" } print_summary() { @@ -595,22 +552,31 @@ main() { load_namespaces local folder_path - local run_archive_path="" + local run_archive_path local found=0 + run_archive_path="$(archive_path_for_run)" + log_info "Using single archive for this run: ${run_archive_path}" + for folder_path in "${NFS_SOURCE_PATH}"/*; do [[ -d "$folder_path" ]] || continue [[ "$(basename "$folder_path")" == .* ]] && continue - if [[ -z "$run_archive_path" ]]; then - run_archive_path="$(archive_path_for_run)" - log_info "Using single archive for this run: ${run_archive_path}" - fi found=1 - process_folder "$folder_path" "$run_archive_path" + process_folder "$folder_path" done if [[ "$found" -eq 0 ]]; then log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}" - elif [[ -f "$run_archive_path" ]]; then + elif (( backup_successes > 0 )); then + log_info "Compressing the complete local staging directory into: ${run_archive_path}" + if ! archive_staging "$run_archive_path"; then + backup_failures=$((backup_failures + backup_successes)) + backup_successes=0 + rm -f -- "$run_archive_path" + log_error "Backup failed while compressing the local staging directory" + fi + fi + + if [[ -f "$run_archive_path" ]]; then local file_size_bytes file_size_bytes="$(wc -c < "$run_archive_path" | tr -d '[:space:]')" if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then @@ -619,14 +585,13 @@ main() { fi print_summary - local size_mb - size_mb="$(backup_size_mb "$total_backup_size_bytes")" + local duration_seconds=$((SECONDS - backup_started_seconds)) if (( backup_successes == 0 )); then send_backup_failure_notification \ "$NOTIFY_FAILURE_URL" \ "No NFS folders were backed up successfully" \ - "$size_mb" || true + "$total_backup_size_bytes" || true die "No folder backups succeeded" 1 fi @@ -634,7 +599,7 @@ main() { send_backup_failure_notification \ "$NOTIFY_FAILURE_URL" \ "Some NFS folders failed to back up" \ - "$size_mb" || true + "$total_backup_size_bytes" || true log_warn "Some folder backups failed; continuing because ${backup_successes} folder backup(s) succeeded" fi @@ -644,7 +609,8 @@ main() { "$NOTIFY_SUCCESS_URL" \ "$NOTIFY_TITLE" \ "${NOTIFY_ASSET}" \ - "$size_mb" || true + "$total_backup_size_bytes" \ + "$duration_seconds" || true } main "$@" diff --git a/k8s/examples/automated-nfs-backup.env.example b/k8s/examples/automated-nfs-backup.env.example index 85b5cc1..aa2eb50 100644 --- a/k8s/examples/automated-nfs-backup.env.example +++ b/k8s/examples/automated-nfs-backup.env.example @@ -5,7 +5,7 @@ KUBECTL_BIN=kubectl KUBE_CONTEXT= WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default -BACKUP_STAGING_PATH= +BACKUP_STAGING_PATH=/tmp/k8s-nfs-backup-staging RSYNC_BIN=rsync ARCHIVE_PREFIX=nfs-backup ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S @@ -18,8 +18,9 @@ SCALE_RETRY_DELAY_SECONDS=5 SCALE_WAIT_SECONDS=30 LOG_LEVEL=info TMP_STATE_DIR=/tmp/k8s-nfs-backup -NOTIFY_SUCCESS_URL= -NOTIFY_FAILURE_URL= +NOTIFY_SUCCESS_URL=http://notify.haven/api/v2/notifications/backup +NOTIFY_FAILURE_URL=http://notify.haven/api/v2/notifications/error NOTIFY_TITLE=Kubernetes NOTIFY_ASSET=kube config +NOTIFY_SOURCE=Kubernetes NFS backup CLEANUP_KEEP_COUNT=5 diff --git a/k8s/test/test-automated-nfs-backup.sh b/k8s/test/test-automated-nfs-backup.sh index e5f52cd..71de377 100644 --- a/k8s/test/test-automated-nfs-backup.sh +++ b/k8s/test/test-automated-nfs-backup.sh @@ -114,13 +114,11 @@ trace_has_scale() { grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file" } -# Asserts the real script invoked 7z as `7z a -t7z ...` -# for the given folder. The archive is the 3rd token (ends in .7z) and the -# folder path is the 4th token, so a swap would fail to match. -sevenz_called_for() { - local folder_name="$1" - - grep -Eq -- "a -t7z [^ ]+\\.7z .*/${folder_name}( |$)" "${TMPD}/7z_invocations.log" +# Asserts the real script invoked 7z exactly once for the complete contents of +# the run staging directory. +sevenz_called_once_for_staging() { + [[ "$(wc -l < "${TMPD}/7z_invocations.log" | tr -d '[:space:]')" -eq 1 ]] && + grep -Eq -- '^a -t7z [^ ]+\.7z \. ' "${TMPD}/7z_invocations.log" } write_fake_script() { @@ -213,16 +211,17 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then archive="${3:?archive path required}" folder="${4:?folder path required}" folder="${folder%/}" - folder_name="${folder##*/}" - printf '7z %s\n' "$folder_name" >> "${TMPD}/event_trace.log" + printf '7z staging\n' >> "${TMPD}/event_trace.log" - # Simulate a backup failure for configured folders BEFORE recording a - # successful archive, so 7z_trace.log only lists folders actually archived. - if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then + if [[ -s "${TMPD}/fail_archive.txt" ]]; then + touch "$archive" exit 2 fi - printf '%s\n' "$folder_name" >> "${TMPD}/7z_trace.log" + for staged_folder in ./*; do + [[ -d "$staged_folder" ]] || continue + printf '%s\n' "${staged_folder##*/}" >> "${TMPD}/7z_trace.log" + done touch "$archive" fi @@ -249,8 +248,8 @@ count=$((count + 1)) printf '%s\n' "$count" > "$count_file" printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log" -if (( count == 2 )) && [[ -f "${TMPD}/fail_rsync_second_pass.txt" ]] && grep -Fxq -- "$folder_name" "${TMPD}/fail_rsync_second_pass.txt"; then - printf 'simulated final rsync failure for %s\n' "$folder_name" >&2 +if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then + printf 'simulated rsync failure for %s\n' "$folder_name" >&2 exit 23 fi @@ -279,7 +278,7 @@ for ((i = 1; i <= $#; i++)); do done body="${body//$'\n'/ }" -printf '%s %s\n' "$url" "${body:0:120}" >> "${TMPD}/curl_trace.log" +printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log" exit 0 EOF } @@ -292,13 +291,14 @@ run_backup() { shift 4 local -a env_overrides=("$@") - env "${env_overrides[@]}" \ + env BACKUP_STAGING_PATH="${state_dir}/staging" \ + "${env_overrides[@]}" \ NFS_SOURCE_PATH="$source_dir" \ BACKUP_OUTPUT_PATH="$output_dir" \ BACKUP_PASSWORD="" \ ARCHIVE_PREFIX="test" \ - NOTIFY_SUCCESS_URL="http://example.invalid/success" \ - NOTIFY_FAILURE_URL="http://example.invalid/failure" \ + NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \ + NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \ KUBECTL_BIN="${TMPD}/bin/kubectl" \ SEVENZ_BIN="${TMPD}/bin/7z" \ RSYNC_BIN="${TMPD}/bin/rsync" \ @@ -311,9 +311,10 @@ run_backup() { write_fake_binaries export TMPD +: > "${TMPD}/fail_archive.txt" -# Scenario A: a mapped folder is scaled down and restored, while an unmapped -# folder is backed up directly. Both folders share one archive. +# Scenario A: a mapped folder is scaled down, copied, and restored, while +# unmapped and excluded folders are copied live. One 7z call archives them all. success_dir="${TMPD}/success" success_source="${success_dir}/source" success_output="${success_dir}/out" @@ -330,6 +331,7 @@ printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt" : > "${TMPD}/7z_trace.log" : > "${TMPD}/7z_invocations.log" : > "${TMPD}/curl_trace.log" +: > "${TMPD}/event_trace.log" set +e run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log" @@ -342,14 +344,17 @@ assert "success scenario reports three backup successes" file_contains "${succes assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0" assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0" assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3" -assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success" +assert "success scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup" +assert "success scenario sends the archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes": 0' +assert "success scenario sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "Kubernetes NFS backup"' assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data" assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system" assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace" assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns" -assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns" -assert "success scenario invokes 7z for excluded kube-system" sevenz_called_for "kube-system" -assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data" +assert "success scenario invokes 7z exactly once for staging" sevenz_called_once_for_staging +assert "success scenario archives mapped app-ns" file_contains "${TMPD}/7z_trace.log" "app-ns" +assert "success scenario archives excluded kube-system" file_contains "${TMPD}/7z_trace.log" "kube-system" +assert "success scenario archives unmapped loose-data" file_contains "${TMPD}/7z_trace.log" "loose-data" # Scenario A2: the exclusion list can be replaced through the environment. custom_exclude_dir="${TMPD}/custom-exclude" @@ -363,6 +368,7 @@ printf 'app-ns:3\n' > "${TMPD}/replicas.txt" : > "${TMPD}/fail_folders.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" +: > "${TMPD}/7z_invocations.log" : > "${TMPD}/curl_trace.log" set +e @@ -374,8 +380,8 @@ assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_ex assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns" assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace" -# Scenario A3: two-pass rsync copies data while live, captures a small delta -# while offline, restores the namespace, and only then starts compression. +# Scenario A3: the mapped namespace is scaled down before its single local copy, +# restored immediately afterward, and compression starts only after restoration. staging_dir="${TMPD}/staging" staging_source="${staging_dir}/source" staging_output="${staging_dir}/out" @@ -386,7 +392,6 @@ printf 'data\n' > "${staging_source}/app-ns/file.txt" printf 'app-ns\n' > "${TMPD}/namespaces.txt" printf 'app-ns:3\n' > "${TMPD}/replicas.txt" : > "${TMPD}/fail_folders.txt" -: > "${TMPD}/fail_rsync_second_pass.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" : > "${TMPD}/7z_invocations.log" @@ -402,15 +407,16 @@ set -e assert "staging scenario exits with code 0" is_zero "$staging_exit_code" assert "staging scenario creates an archive" archive_exists "$staging_output" assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \ - "rsync app-ns 1" \ "scale app-ns --replicas=0" \ - "rsync app-ns 2" \ + "rsync app-ns 1" \ "scale app-ns --replicas=3" \ - "7z app-ns" + "7z staging" +assert "staging scenario copies app-ns only once" file_contains "${TMPD}/rsync_counts/app-ns" "1" +assert "staging scenario invokes 7z exactly once" sevenz_called_once_for_staging assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-" assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work" -# Scenario A4: a failed final delta never archives the incomplete staging copy, +# Scenario A4: a failed local copy never archives the incomplete staging copy, # but the namespace is still restored immediately. rsync_failure_dir="${TMPD}/rsync-failure" rsync_failure_source="${rsync_failure_dir}/source" @@ -421,8 +427,7 @@ mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failur printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt" printf 'app-ns\n' > "${TMPD}/namespaces.txt" printf 'app-ns:3\n' > "${TMPD}/replicas.txt" -: > "${TMPD}/fail_folders.txt" -printf 'app-ns\n' > "${TMPD}/fail_rsync_second_pass.txt" +printf 'app-ns\n' > "${TMPD}/fail_folders.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" : > "${TMPD}/curl_trace.log" @@ -434,11 +439,10 @@ run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state rsync_failure_exit_code=$? set -e -assert "final rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code" -assert "final rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3" -assert "final rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z app-ns" -assert "final rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work" -: > "${TMPD}/fail_rsync_second_pass.txt" +assert "rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code" +assert "rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3" +assert "rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z staging" +assert "rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work" # Scenario B: one namespace backup fails, but another succeeds. Both namespaces # are restored, retention still runs, and the script reports success. @@ -457,7 +461,9 @@ printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt" printf 'fail-ns\n' > "${TMPD}/fail_folders.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" +: > "${TMPD}/7z_invocations.log" : > "${TMPD}/curl_trace.log" +: > "${TMPD}/event_trace.log" set +e run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log" @@ -472,10 +478,17 @@ assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/s assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2" assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0" assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4" -assert "partial failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" -assert "partial failure scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success" +assert "partial failure scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" +assert "partial failure scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup" +assert "partial failure scenario marks the error as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"' +assert "partial failure scenario includes the backup error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "BACKUP_FAILED"' assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns" -assert "partial failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns" +assert "partial failure scenario archives the successfully staged folder" file_contains "${TMPD}/7z_trace.log" "good-ns" +assert "partial failure scenario invokes 7z exactly once" sevenz_called_once_for_staging +assert "partial failure scenario compresses after all namespaces are restored" events_are_ordered "${TMPD}/event_trace.log" \ + "scale fail-ns --replicas=4" \ + "scale good-ns --replicas=2" \ + "7z staging" assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4 assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z" @@ -502,8 +515,35 @@ set -e assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code" assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0" assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1" -assert "all-failed scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" -assert "all-failed scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success" +assert "all-failed scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" +assert "all-failed scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup" + +# Scenario C2: a failed single archive operation invalidates all staged folders +# and removes the partial archive left by 7z. +archive_failure_dir="${TMPD}/archive-failure" +archive_failure_source="${archive_failure_dir}/source" +archive_failure_output="${archive_failure_dir}/out" +archive_failure_state="${archive_failure_dir}/state" +mkdir -p "${archive_failure_source}/archive-ns" "$archive_failure_output" "$archive_failure_state" +printf 'data\n' > "${archive_failure_source}/archive-ns/file.txt" +printf 'archive-ns\n' > "${TMPD}/namespaces.txt" +printf 'archive-ns:2\n' > "${TMPD}/replicas.txt" +: > "${TMPD}/fail_folders.txt" +printf 'fail\n' > "${TMPD}/fail_archive.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/curl_trace.log" + +set +e +run_backup "$archive_failure_source" "$archive_failure_output" "$archive_failure_state" "${archive_failure_dir}/archive-failure.log" +archive_failure_exit_code=$? +set -e + +assert "archive failure exits non-zero" is_nonzero "$archive_failure_exit_code" +assert "archive failure restores archive-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "archive-ns" "2" +assert "archive failure reports zero backup successes" file_contains "${archive_failure_dir}/archive-failure.log" "backup_successes=0" +assert "archive failure removes the partial archive" archive_count_is "$archive_failure_output" 0 +assert "archive failure does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup" +: > "${TMPD}/fail_archive.txt" # Scenario D: no source folders means zero successful backups, which must also # return non-zero rather than reporting an empty run as successful. @@ -526,8 +566,8 @@ set -e assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code" assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0" -assert "empty scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" -assert "empty scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success" +assert "empty scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" +assert "empty scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup" # Scenario E: abnormal abort while a namespace is still scaled to zero. # We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep` @@ -539,7 +579,8 @@ abort_dir="${TMPD}/abort" abort_source="${abort_dir}/source" abort_output="${abort_dir}/out" abort_state="${abort_dir}/state" -mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" +abort_work="${abort_dir}/work" +mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" "${abort_work}" printf 'data\n' > "${abort_source}/crash-ns/file.txt" printf 'crash-ns\n' > "${TMPD}/namespaces.txt" printf 'crash-ns:5\n' > "${TMPD}/replicas.txt" @@ -558,6 +599,7 @@ NOTIFY_SUCCESS_URL="" \ NOTIFY_FAILURE_URL="" \ KUBECTL_BIN="${TMPD}/bin/kubectl" \ SEVENZ_BIN="${TMPD}/bin/7z" \ +BACKUP_STAGING_PATH="${abort_work}" \ SCALE_WAIT_SECONDS="not-a-number" \ TMP_STATE_DIR="${abort_state}" \ PATH="${TMPD}/bin:${PATH}" \