fixing control plane backup too
All checks were successful
Check scripts syntax / check-scripts-syntax (push) Successful in 2s
Test Kubernetes backup scripts / test-automated-nfs-backup (push) Successful in 3s
Test Kubernetes backup scripts / test-k3s-control-plane-config-backup (push) Successful in 4s

This commit is contained in:
2026-07-22 08:38:47 -03:00
parent cafbb01200
commit 0f7c56d7eb
3 changed files with 43 additions and 32 deletions

View File

@@ -35,7 +35,7 @@ Useful overrides:
- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`.
- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion).
- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REALPATH_BIN`, `REMOTE_BASH_BIN`. `REALPATH_BIN` must provide GNU `realpath` `-e` and `-m` options.
- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/template/notify/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/template/notify/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_CALLER`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/api/v2/notifications/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/api/v2/notifications/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_SOURCE`. `NOTIFY_CALLER` is a compatibility fallback for `NOTIFY_SOURCE`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
## Environment Variables

View File

@@ -44,13 +44,14 @@ DATE_BIN="${DATE_BIN:-date}"
REALPATH_BIN="${REALPATH_BIN:-realpath}"
REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
# Notifications follow the JSON formats used by automated-nfs-backup.sh.
# Notifications use the Haven Notify v2 backup and error contracts.
# Set either URL to an empty value to disable that notification.
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/template/notify/backup}"
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/template/notify/error}"
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
declare -a ARCHIVE_ENTRIES=()
declare -a SERVICE_FILE_LIST=()
@@ -66,6 +67,7 @@ ARCHIVE_SIZE_BYTES=0
CLEANUP_KEEP_COUNT_INT=0
LOCK_FD=""
FAILURE_NOTIFICATION_SENT=0
BACKUP_STARTED_SECONDS="$SECONDS"
json_escape() {
local value="${1:-}"
@@ -75,18 +77,10 @@ json_escape() {
printf '%s' "$value"
}
backup_size_mb() {
local bytes="${1:-0}"
if (( bytes <= 0 )); then
printf '0'
return 0
fi
printf '%s' "$(( (bytes + 1048575) / 1048576 ))"
}
send_backup_notification() {
local url="${1:-}"
local size_mb="${2:-0}"
local size_bytes="${2:-0}"
local duration_seconds="${3:-0}"
[[ -z "$url" ]] && return 0
local payload
@@ -94,7 +88,9 @@ send_backup_notification() {
{
"title": "$(json_escape "$NOTIFY_TITLE")",
"asset": "$(json_escape "$NOTIFY_ASSET")",
"backupSizeInMB": ${size_mb}
"sizeBytes": ${size_bytes},
"durationSeconds": ${duration_seconds},
"source": "$(json_escape "$NOTIFY_SOURCE")"
}
EOF
)
@@ -109,19 +105,20 @@ EOF
send_backup_failure_notification() {
local url="${1:-}"
local size_mb="${2:-0}"
local size_bytes="${2:-0}"
[[ -z "$url" ]] && return 0
local payload
payload=$(cat <<EOF
{
"caller": "$(json_escape "$NOTIFY_CALLER")",
"source": "$(json_escape "$NOTIFY_SOURCE")",
"message": "Control-plane configuration backup failed",
"critical": true,
"severity": "critical",
"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED",
"extra": [
{
"name": "Backup Size (MB)",
"value": "${size_mb}"
"name": "Archive size",
"value": "${size_bytes} B"
}
]
}
@@ -141,7 +138,7 @@ notify_failure_once() {
return 0
fi
FAILURE_NOTIFICATION_SENT=1
send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$ARCHIVE_SIZE_BYTES" || true
}
cleanup() {
@@ -469,7 +466,8 @@ main() {
# Notifications are informational: an unavailable notification endpoint
# must not change the result of a completed backup.
send_backup_notification "$NOTIFY_SUCCESS_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
local duration_seconds=$((SECONDS - BACKUP_STARTED_SECONDS))
send_backup_notification "$NOTIFY_SUCCESS_URL" "$ARCHIVE_SIZE_BYTES" "$duration_seconds" || true
}
main "$@"

View File

@@ -172,13 +172,21 @@ EOF
set -Eeuo pipefail
url=""
body=""
for ((i = 1; i <= $#; i++)); do
if [[ "${!i}" == "POST" ]]; then
case "${!i}" in
POST|-d)
next=$((i + 1))
if [[ "${!i}" == "POST" ]]; then
url="${!next}"
else
body="${!next}"
fi
;;
esac
done
printf '%s\n' "$url" >> "${TMPD}/curl_trace.log"
body="${body//$'\n'/ }"
printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
[[ ! -f "${TMPD}/fail_curl" ]]
EOF
@@ -219,8 +227,8 @@ run_backup() {
CURL_BIN="${TMPD}/bin/curl" \
FLOCK_BIN="${TMPD}/bin/flock" \
DATE_BIN="${TMPD}/bin/date" \
NOTIFY_SUCCESS_URL="http://example.invalid/success" \
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
PATH="${TMPD}/bin:${PATH}" \
TMPD="$TMPD" \
FAKE_REMOTE_ROOT="$remote_root" \
@@ -312,7 +320,10 @@ assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${su
assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
assert "successful backup sends a success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
assert "successful backup sends a v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
assert "successful backup sends its archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes":'
assert "successful backup sends its elapsed duration" file_contains "${TMPD}/curl_trace.log" '"durationSeconds":'
assert "successful backup sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "K3s control-plane config backup"'
# A configured source path ending in server/db/.. must be rejected before zip is
# invoked, rather than recursively archiving server/db and its sibling socket.
@@ -398,8 +409,10 @@ assert "upload failure exits non-zero" is_nonzero "$failure_exit_code"
assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
assert "upload failure attempts exactly one error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/failure" 1
assert "upload failure does not send a success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
assert "upload failure attempts exactly one v2 error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" 1
assert "upload failure marks the notification as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
assert "upload failure includes the v2 error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED"'
assert "upload failure does not send a v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
# A notification error is non-fatal after a completed backup.