fixing control plane backup too
This commit is contained in:
@@ -35,7 +35,7 @@ Useful overrides:
|
||||
- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`.
|
||||
- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion).
|
||||
- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REALPATH_BIN`, `REMOTE_BASH_BIN`. `REALPATH_BIN` must provide GNU `realpath` `-e` and `-m` options.
|
||||
- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/template/notify/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/template/notify/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_CALLER`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
|
||||
- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/api/v2/notifications/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/api/v2/notifications/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_SOURCE`. `NOTIFY_CALLER` is a compatibility fallback for `NOTIFY_SOURCE`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
|
||||
@@ -44,13 +44,14 @@ DATE_BIN="${DATE_BIN:-date}"
|
||||
REALPATH_BIN="${REALPATH_BIN:-realpath}"
|
||||
REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
|
||||
|
||||
# Notifications follow the JSON formats used by automated-nfs-backup.sh.
|
||||
# Notifications use the Haven Notify v2 backup and error contracts.
|
||||
# Set either URL to an empty value to disable that notification.
|
||||
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/template/notify/backup}"
|
||||
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/template/notify/error}"
|
||||
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
|
||||
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
|
||||
NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
|
||||
NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
|
||||
NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
|
||||
NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
|
||||
|
||||
declare -a ARCHIVE_ENTRIES=()
|
||||
declare -a SERVICE_FILE_LIST=()
|
||||
@@ -66,6 +67,7 @@ ARCHIVE_SIZE_BYTES=0
|
||||
CLEANUP_KEEP_COUNT_INT=0
|
||||
LOCK_FD=""
|
||||
FAILURE_NOTIFICATION_SENT=0
|
||||
BACKUP_STARTED_SECONDS="$SECONDS"
|
||||
|
||||
json_escape() {
|
||||
local value="${1:-}"
|
||||
@@ -75,18 +77,10 @@ json_escape() {
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
backup_size_mb() {
|
||||
local bytes="${1:-0}"
|
||||
if (( bytes <= 0 )); then
|
||||
printf '0'
|
||||
return 0
|
||||
fi
|
||||
printf '%s' "$(( (bytes + 1048575) / 1048576 ))"
|
||||
}
|
||||
|
||||
send_backup_notification() {
|
||||
local url="${1:-}"
|
||||
local size_mb="${2:-0}"
|
||||
local size_bytes="${2:-0}"
|
||||
local duration_seconds="${3:-0}"
|
||||
[[ -z "$url" ]] && return 0
|
||||
|
||||
local payload
|
||||
@@ -94,7 +88,9 @@ send_backup_notification() {
|
||||
{
|
||||
"title": "$(json_escape "$NOTIFY_TITLE")",
|
||||
"asset": "$(json_escape "$NOTIFY_ASSET")",
|
||||
"backupSizeInMB": ${size_mb}
|
||||
"sizeBytes": ${size_bytes},
|
||||
"durationSeconds": ${duration_seconds},
|
||||
"source": "$(json_escape "$NOTIFY_SOURCE")"
|
||||
}
|
||||
EOF
|
||||
)
|
||||
@@ -109,19 +105,20 @@ EOF
|
||||
|
||||
send_backup_failure_notification() {
|
||||
local url="${1:-}"
|
||||
local size_mb="${2:-0}"
|
||||
local size_bytes="${2:-0}"
|
||||
[[ -z "$url" ]] && return 0
|
||||
|
||||
local payload
|
||||
payload=$(cat <<EOF
|
||||
{
|
||||
"caller": "$(json_escape "$NOTIFY_CALLER")",
|
||||
"source": "$(json_escape "$NOTIFY_SOURCE")",
|
||||
"message": "Control-plane configuration backup failed",
|
||||
"critical": true,
|
||||
"severity": "critical",
|
||||
"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED",
|
||||
"extra": [
|
||||
{
|
||||
"name": "Backup Size (MB)",
|
||||
"value": "${size_mb}"
|
||||
"name": "Archive size",
|
||||
"value": "${size_bytes} B"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -141,7 +138,7 @@ notify_failure_once() {
|
||||
return 0
|
||||
fi
|
||||
FAILURE_NOTIFICATION_SENT=1
|
||||
send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
|
||||
send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$ARCHIVE_SIZE_BYTES" || true
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
@@ -469,7 +466,8 @@ main() {
|
||||
|
||||
# Notifications are informational: an unavailable notification endpoint
|
||||
# must not change the result of a completed backup.
|
||||
send_backup_notification "$NOTIFY_SUCCESS_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
|
||||
local duration_seconds=$((SECONDS - BACKUP_STARTED_SECONDS))
|
||||
send_backup_notification "$NOTIFY_SUCCESS_URL" "$ARCHIVE_SIZE_BYTES" "$duration_seconds" || true
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
@@ -172,13 +172,21 @@ EOF
|
||||
set -Eeuo pipefail
|
||||
|
||||
url=""
|
||||
body=""
|
||||
for ((i = 1; i <= $#; i++)); do
|
||||
if [[ "${!i}" == "POST" ]]; then
|
||||
case "${!i}" in
|
||||
POST|-d)
|
||||
next=$((i + 1))
|
||||
if [[ "${!i}" == "POST" ]]; then
|
||||
url="${!next}"
|
||||
else
|
||||
body="${!next}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
printf '%s\n' "$url" >> "${TMPD}/curl_trace.log"
|
||||
body="${body//$'\n'/ }"
|
||||
printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
|
||||
[[ ! -f "${TMPD}/fail_curl" ]]
|
||||
EOF
|
||||
|
||||
@@ -219,8 +227,8 @@ run_backup() {
|
||||
CURL_BIN="${TMPD}/bin/curl" \
|
||||
FLOCK_BIN="${TMPD}/bin/flock" \
|
||||
DATE_BIN="${TMPD}/bin/date" \
|
||||
NOTIFY_SUCCESS_URL="http://example.invalid/success" \
|
||||
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
|
||||
NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
|
||||
NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
|
||||
PATH="${TMPD}/bin:${PATH}" \
|
||||
TMPD="$TMPD" \
|
||||
FAKE_REMOTE_ROOT="$remote_root" \
|
||||
@@ -312,7 +320,10 @@ assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${su
|
||||
assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
|
||||
assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
|
||||
assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
|
||||
assert "successful backup sends a success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
assert "successful backup sends a v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||
assert "successful backup sends its archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes":'
|
||||
assert "successful backup sends its elapsed duration" file_contains "${TMPD}/curl_trace.log" '"durationSeconds":'
|
||||
assert "successful backup sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "K3s control-plane config backup"'
|
||||
|
||||
# A configured source path ending in server/db/.. must be rejected before zip is
|
||||
# invoked, rather than recursively archiving server/db and its sibling socket.
|
||||
@@ -398,8 +409,10 @@ assert "upload failure exits non-zero" is_nonzero "$failure_exit_code"
|
||||
assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
||||
assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
|
||||
assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
|
||||
assert "upload failure attempts exactly one error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/failure" 1
|
||||
assert "upload failure does not send a success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||
assert "upload failure attempts exactly one v2 error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" 1
|
||||
assert "upload failure marks the notification as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
|
||||
assert "upload failure includes the v2 error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED"'
|
||||
assert "upload failure does not send a v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||
assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
|
||||
|
||||
# A notification error is non-fatal after a completed backup.
|
||||
|
||||
Reference in New Issue
Block a user