fixing control plane backup too
This commit is contained in:
@@ -35,7 +35,7 @@ Useful overrides:
|
|||||||
- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`.
|
- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`.
|
||||||
- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion).
|
- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion).
|
||||||
- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REALPATH_BIN`, `REMOTE_BASH_BIN`. `REALPATH_BIN` must provide GNU `realpath` `-e` and `-m` options.
|
- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REALPATH_BIN`, `REMOTE_BASH_BIN`. `REALPATH_BIN` must provide GNU `realpath` `-e` and `-m` options.
|
||||||
- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/template/notify/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/template/notify/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_CALLER`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
|
- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/api/v2/notifications/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/api/v2/notifications/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_SOURCE`. `NOTIFY_CALLER` is a compatibility fallback for `NOTIFY_SOURCE`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
|
||||||
|
|
||||||
## Environment Variables
|
## Environment Variables
|
||||||
|
|
||||||
|
|||||||
@@ -44,13 +44,14 @@ DATE_BIN="${DATE_BIN:-date}"
|
|||||||
REALPATH_BIN="${REALPATH_BIN:-realpath}"
|
REALPATH_BIN="${REALPATH_BIN:-realpath}"
|
||||||
REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
|
REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
|
||||||
|
|
||||||
# Notifications follow the JSON formats used by automated-nfs-backup.sh.
|
# Notifications use the Haven Notify v2 backup and error contracts.
|
||||||
# Set either URL to an empty value to disable that notification.
|
# Set either URL to an empty value to disable that notification.
|
||||||
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/template/notify/backup}"
|
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
|
||||||
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/template/notify/error}"
|
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
|
||||||
NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
|
NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
|
||||||
NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
|
NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
|
||||||
NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
|
NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
|
||||||
|
NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
|
||||||
|
|
||||||
declare -a ARCHIVE_ENTRIES=()
|
declare -a ARCHIVE_ENTRIES=()
|
||||||
declare -a SERVICE_FILE_LIST=()
|
declare -a SERVICE_FILE_LIST=()
|
||||||
@@ -66,6 +67,7 @@ ARCHIVE_SIZE_BYTES=0
|
|||||||
CLEANUP_KEEP_COUNT_INT=0
|
CLEANUP_KEEP_COUNT_INT=0
|
||||||
LOCK_FD=""
|
LOCK_FD=""
|
||||||
FAILURE_NOTIFICATION_SENT=0
|
FAILURE_NOTIFICATION_SENT=0
|
||||||
|
BACKUP_STARTED_SECONDS="$SECONDS"
|
||||||
|
|
||||||
json_escape() {
|
json_escape() {
|
||||||
local value="${1:-}"
|
local value="${1:-}"
|
||||||
@@ -75,18 +77,10 @@ json_escape() {
|
|||||||
printf '%s' "$value"
|
printf '%s' "$value"
|
||||||
}
|
}
|
||||||
|
|
||||||
backup_size_mb() {
|
|
||||||
local bytes="${1:-0}"
|
|
||||||
if (( bytes <= 0 )); then
|
|
||||||
printf '0'
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
printf '%s' "$(( (bytes + 1048575) / 1048576 ))"
|
|
||||||
}
|
|
||||||
|
|
||||||
send_backup_notification() {
|
send_backup_notification() {
|
||||||
local url="${1:-}"
|
local url="${1:-}"
|
||||||
local size_mb="${2:-0}"
|
local size_bytes="${2:-0}"
|
||||||
|
local duration_seconds="${3:-0}"
|
||||||
[[ -z "$url" ]] && return 0
|
[[ -z "$url" ]] && return 0
|
||||||
|
|
||||||
local payload
|
local payload
|
||||||
@@ -94,7 +88,9 @@ send_backup_notification() {
|
|||||||
{
|
{
|
||||||
"title": "$(json_escape "$NOTIFY_TITLE")",
|
"title": "$(json_escape "$NOTIFY_TITLE")",
|
||||||
"asset": "$(json_escape "$NOTIFY_ASSET")",
|
"asset": "$(json_escape "$NOTIFY_ASSET")",
|
||||||
"backupSizeInMB": ${size_mb}
|
"sizeBytes": ${size_bytes},
|
||||||
|
"durationSeconds": ${duration_seconds},
|
||||||
|
"source": "$(json_escape "$NOTIFY_SOURCE")"
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
)
|
)
|
||||||
@@ -109,19 +105,20 @@ EOF
|
|||||||
|
|
||||||
send_backup_failure_notification() {
|
send_backup_failure_notification() {
|
||||||
local url="${1:-}"
|
local url="${1:-}"
|
||||||
local size_mb="${2:-0}"
|
local size_bytes="${2:-0}"
|
||||||
[[ -z "$url" ]] && return 0
|
[[ -z "$url" ]] && return 0
|
||||||
|
|
||||||
local payload
|
local payload
|
||||||
payload=$(cat <<EOF
|
payload=$(cat <<EOF
|
||||||
{
|
{
|
||||||
"caller": "$(json_escape "$NOTIFY_CALLER")",
|
"source": "$(json_escape "$NOTIFY_SOURCE")",
|
||||||
"message": "Control-plane configuration backup failed",
|
"message": "Control-plane configuration backup failed",
|
||||||
"critical": true,
|
"severity": "critical",
|
||||||
|
"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED",
|
||||||
"extra": [
|
"extra": [
|
||||||
{
|
{
|
||||||
"name": "Backup Size (MB)",
|
"name": "Archive size",
|
||||||
"value": "${size_mb}"
|
"value": "${size_bytes} B"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -141,7 +138,7 @@ notify_failure_once() {
|
|||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
FAILURE_NOTIFICATION_SENT=1
|
FAILURE_NOTIFICATION_SENT=1
|
||||||
send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
|
send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$ARCHIVE_SIZE_BYTES" || true
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
@@ -469,7 +466,8 @@ main() {
|
|||||||
|
|
||||||
# Notifications are informational: an unavailable notification endpoint
|
# Notifications are informational: an unavailable notification endpoint
|
||||||
# must not change the result of a completed backup.
|
# must not change the result of a completed backup.
|
||||||
send_backup_notification "$NOTIFY_SUCCESS_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
|
local duration_seconds=$((SECONDS - BACKUP_STARTED_SECONDS))
|
||||||
|
send_backup_notification "$NOTIFY_SUCCESS_URL" "$ARCHIVE_SIZE_BYTES" "$duration_seconds" || true
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -172,13 +172,21 @@ EOF
|
|||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
|
||||||
url=""
|
url=""
|
||||||
|
body=""
|
||||||
for ((i = 1; i <= $#; i++)); do
|
for ((i = 1; i <= $#; i++)); do
|
||||||
if [[ "${!i}" == "POST" ]]; then
|
case "${!i}" in
|
||||||
|
POST|-d)
|
||||||
next=$((i + 1))
|
next=$((i + 1))
|
||||||
|
if [[ "${!i}" == "POST" ]]; then
|
||||||
url="${!next}"
|
url="${!next}"
|
||||||
|
else
|
||||||
|
body="${!next}"
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
done
|
done
|
||||||
printf '%s\n' "$url" >> "${TMPD}/curl_trace.log"
|
body="${body//$'\n'/ }"
|
||||||
|
printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
|
||||||
[[ ! -f "${TMPD}/fail_curl" ]]
|
[[ ! -f "${TMPD}/fail_curl" ]]
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
@@ -219,8 +227,8 @@ run_backup() {
|
|||||||
CURL_BIN="${TMPD}/bin/curl" \
|
CURL_BIN="${TMPD}/bin/curl" \
|
||||||
FLOCK_BIN="${TMPD}/bin/flock" \
|
FLOCK_BIN="${TMPD}/bin/flock" \
|
||||||
DATE_BIN="${TMPD}/bin/date" \
|
DATE_BIN="${TMPD}/bin/date" \
|
||||||
NOTIFY_SUCCESS_URL="http://example.invalid/success" \
|
NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
|
||||||
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
|
NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
|
||||||
PATH="${TMPD}/bin:${PATH}" \
|
PATH="${TMPD}/bin:${PATH}" \
|
||||||
TMPD="$TMPD" \
|
TMPD="$TMPD" \
|
||||||
FAKE_REMOTE_ROOT="$remote_root" \
|
FAKE_REMOTE_ROOT="$remote_root" \
|
||||||
@@ -312,7 +320,10 @@ assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${su
|
|||||||
assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
|
assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
|
||||||
assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
|
assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
|
||||||
assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
|
assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
|
||||||
assert "successful backup sends a success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
assert "successful backup sends a v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||||
|
assert "successful backup sends its archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes":'
|
||||||
|
assert "successful backup sends its elapsed duration" file_contains "${TMPD}/curl_trace.log" '"durationSeconds":'
|
||||||
|
assert "successful backup sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "K3s control-plane config backup"'
|
||||||
|
|
||||||
# A configured source path ending in server/db/.. must be rejected before zip is
|
# A configured source path ending in server/db/.. must be rejected before zip is
|
||||||
# invoked, rather than recursively archiving server/db and its sibling socket.
|
# invoked, rather than recursively archiving server/db and its sibling socket.
|
||||||
@@ -398,8 +409,10 @@ assert "upload failure exits non-zero" is_nonzero "$failure_exit_code"
|
|||||||
assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
||||||
assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
|
assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
|
||||||
assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
|
assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
|
||||||
assert "upload failure attempts exactly one error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/failure" 1
|
assert "upload failure attempts exactly one v2 error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" 1
|
||||||
assert "upload failure does not send a success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
assert "upload failure marks the notification as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
|
||||||
|
assert "upload failure includes the v2 error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED"'
|
||||||
|
assert "upload failure does not send a v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
|
||||||
assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
|
assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
|
||||||
|
|
||||||
# A notification error is non-fatal after a completed backup.
|
# A notification error is non-fatal after a completed backup.
|
||||||
|
|||||||
Reference in New Issue
Block a user