474 lines
15 KiB
Bash
Executable File
474 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Back up the k3s control-plane configuration and credentials, without taking
|
|
# a live datastore snapshot or changing the running k3s service.
|
|
|
|
SCRIPT_NAME="$(basename "$0")"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=lib/common.sh
|
|
source "${SCRIPT_DIR}/lib/common.sh"
|
|
|
|
# Keep the local archive and temporary files readable only by this process.
|
|
umask 077
|
|
|
|
# Source paths. K3S_CONFIG_DIR is the only required source; the service files
|
|
# and server state entries are included only when present.
|
|
K3S_CONFIG_DIR="${K3S_CONFIG_DIR:-/etc/rancher/k3s}"
|
|
K3S_SERVER_DIR="${K3S_SERVER_DIR:-/var/lib/rancher/k3s/server}"
|
|
K3S_SERVICE_FILES="${K3S_SERVICE_FILES-/etc/systemd/system/k3s.service:/etc/systemd/system/k3s.service.env:/etc/default/k3s:/etc/sysconfig/k3s}"
|
|
|
|
# Archive and local working settings.
|
|
ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-k3s-control-plane-config}"
|
|
ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y-%m-%d_%H-%M-%S}"
|
|
LOCAL_TMP_PARENT="${LOCAL_TMP_PARENT:-${TMPDIR:-/tmp}}"
|
|
LOCK_PATH="${LOCK_PATH:-/var/lock/k3s-control-plane-config-backup.lock}"
|
|
|
|
# Remote destination settings.
|
|
REMOTE_USER="${REMOTE_USER:-ivanch}"
|
|
REMOTE_HOST="${REMOTE_HOST:-nas.haven}"
|
|
REMOTE_DIR="${REMOTE_DIR:-/export/Backup/k8s}"
|
|
REMOTE_IDENTITY_FILE="${REMOTE_IDENTITY_FILE:-}"
|
|
SSH_PORT="${SSH_PORT:-}"
|
|
CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
|
|
|
|
# Commands are configurable to support hosts with non-standard paths and
|
|
# isolated tests.
|
|
ZIP_BIN="${ZIP_BIN:-zip}"
|
|
UNZIP_BIN="${UNZIP_BIN:-unzip}"
|
|
SSH_BIN="${SSH_BIN:-ssh}"
|
|
SCP_BIN="${SCP_BIN:-scp}"
|
|
CURL_BIN="${CURL_BIN:-curl}"
|
|
FLOCK_BIN="${FLOCK_BIN:-flock}"
|
|
MKTEMP_BIN="${MKTEMP_BIN:-mktemp}"
|
|
DATE_BIN="${DATE_BIN:-date}"
|
|
REALPATH_BIN="${REALPATH_BIN:-realpath}"
|
|
REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
|
|
|
|
# Notifications use the Haven Notify v2 backup and error contracts.
|
|
# Set either URL to an empty value to disable that notification.
|
|
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
|
|
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
|
|
NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
|
|
NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
|
|
NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
|
|
NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
|
|
|
|
declare -a ARCHIVE_ENTRIES=()
|
|
declare -a SERVICE_FILE_LIST=()
|
|
declare -a SSH_ARGS=()
|
|
declare -a SCP_ARGS=()
|
|
declare -a PROTECTED_PATHS=()
|
|
readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token)
|
|
|
|
RUN_TMP_DIR=""
|
|
LOCAL_ARCHIVE_PATH=""
|
|
ARCHIVE_NAME=""
|
|
ARCHIVE_SIZE_BYTES=0
|
|
CLEANUP_KEEP_COUNT_INT=0
|
|
LOCK_FD=""
|
|
FAILURE_NOTIFICATION_SENT=0
|
|
BACKUP_STARTED_SECONDS="$SECONDS"
|
|
|
|
json_escape() {
|
|
local value="${1:-}"
|
|
value="${value//\\/\\\\}"
|
|
value="${value//\"/\\\"}"
|
|
value="${value//$'\n'/ }"
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
send_backup_notification() {
|
|
local url="${1:-}"
|
|
local size_bytes="${2:-0}"
|
|
local duration_seconds="${3:-0}"
|
|
[[ -z "$url" ]] && return 0
|
|
|
|
local payload
|
|
payload=$(cat <<EOF
|
|
{
|
|
"title": "$(json_escape "$NOTIFY_TITLE")",
|
|
"asset": "$(json_escape "$NOTIFY_ASSET")",
|
|
"sizeBytes": ${size_bytes},
|
|
"durationSeconds": ${duration_seconds},
|
|
"source": "$(json_escape "$NOTIFY_SOURCE")"
|
|
}
|
|
EOF
|
|
)
|
|
|
|
if ! "$CURL_BIN" -fsS -X POST "$url" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" >/dev/null; then
|
|
log_warn "Failed to send backup notification"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
send_backup_failure_notification() {
|
|
local url="${1:-}"
|
|
local size_bytes="${2:-0}"
|
|
[[ -z "$url" ]] && return 0
|
|
|
|
local payload
|
|
payload=$(cat <<EOF
|
|
{
|
|
"source": "$(json_escape "$NOTIFY_SOURCE")",
|
|
"message": "Control-plane configuration backup failed",
|
|
"severity": "critical",
|
|
"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED",
|
|
"extra": [
|
|
{
|
|
"name": "Archive size",
|
|
"value": "${size_bytes} B"
|
|
}
|
|
]
|
|
}
|
|
EOF
|
|
)
|
|
|
|
if ! "$CURL_BIN" -fsS -X POST "$url" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" >/dev/null; then
|
|
log_warn "Failed to send backup failure notification"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
notify_failure_once() {
|
|
if (( FAILURE_NOTIFICATION_SENT == 1 )); then
|
|
return 0
|
|
fi
|
|
FAILURE_NOTIFICATION_SENT=1
|
|
send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$ARCHIVE_SIZE_BYTES" || true
|
|
}
|
|
|
|
cleanup() {
|
|
local exit_code=$?
|
|
trap - EXIT
|
|
|
|
if [[ -n "$RUN_TMP_DIR" && -d "$RUN_TMP_DIR" ]]; then
|
|
rm -rf -- "$RUN_TMP_DIR" || log_warn "Failed to clean local temporary files"
|
|
fi
|
|
|
|
if (( exit_code == 0 )); then
|
|
log_info "Finished successfully"
|
|
else
|
|
notify_failure_once
|
|
log_error "Control-plane configuration backup failed"
|
|
fi
|
|
|
|
exit "$exit_code"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
shell_quote() {
|
|
local value="${1:-}"
|
|
value="${value//\'/\'\"\'\"\'}"
|
|
printf "'%s'" "$value"
|
|
}
|
|
|
|
require_absolute_path() {
|
|
local path="${1:?path is required}"
|
|
local description="${2:-path}"
|
|
if [[ "$path" != /* ]]; then
|
|
die "${description} must be an absolute path"
|
|
fi
|
|
}
|
|
|
|
path_is_ancestor_or_same() {
|
|
local ancestor="${1:?ancestor path is required}"
|
|
local path="${2:?path is required}"
|
|
|
|
[[ "$ancestor" == "/" || "$path" == "$ancestor" || "$path" == "$ancestor"/* ]]
|
|
}
|
|
|
|
paths_overlap() {
|
|
local first="${1:?first path is required}"
|
|
local second="${2:?second path is required}"
|
|
|
|
path_is_ancestor_or_same "$first" "$second" || \
|
|
path_is_ancestor_or_same "$second" "$first"
|
|
}
|
|
|
|
canonicalize_protected_path() {
|
|
local path="${1:?protected path is required}"
|
|
local canonical_path
|
|
local canonical_parent
|
|
local basename
|
|
|
|
if [[ -e "$path" || -L "$path" ]]; then
|
|
canonical_path="$("$REALPATH_BIN" -e -- "$path" 2>/dev/null)" || return 1
|
|
printf '%s' "$canonical_path"
|
|
return 0
|
|
fi
|
|
|
|
basename="${path##*/}"
|
|
canonical_parent="${path%/*}"
|
|
if ! canonical_parent="$("$REALPATH_BIN" -e -- "$canonical_parent" 2>/dev/null)"; then
|
|
canonical_parent="$("$REALPATH_BIN" -m -- "$canonical_parent" 2>/dev/null)" || return 1
|
|
fi
|
|
printf '%s/%s' "${canonical_parent%/}" "$basename"
|
|
}
|
|
|
|
determine_protected_paths() {
|
|
local canonical_server_dir
|
|
local protected_path
|
|
|
|
if ! canonical_server_dir="$("$REALPATH_BIN" -e -- "$K3S_SERVER_DIR" 2>/dev/null)"; then
|
|
canonical_server_dir="$("$REALPATH_BIN" -m -- "$K3S_SERVER_DIR" 2>/dev/null)" || \
|
|
die "Configured source path is not safe to archive"
|
|
fi
|
|
|
|
PROTECTED_PATHS=()
|
|
if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/db")"; then
|
|
die "Configured source path is not safe to archive"
|
|
fi
|
|
PROTECTED_PATHS+=("$protected_path")
|
|
if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/kine.sock")"; then
|
|
die "Configured source path is not safe to archive"
|
|
fi
|
|
PROTECTED_PATHS+=("$protected_path")
|
|
}
|
|
|
|
add_archive_entry_if_present() {
|
|
local absolute_path="${1:?source path is required}"
|
|
local canonical_path
|
|
local relative_path
|
|
local protected_path
|
|
|
|
if [[ "$absolute_path" =~ (^|/)(\.|\.\.)(/|$) ]]; then
|
|
die "Configured source path is not safe to archive"
|
|
fi
|
|
|
|
if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if ! canonical_path="$("$REALPATH_BIN" -e -- "$absolute_path" 2>/dev/null)"; then
|
|
die "Configured source path is not safe to archive"
|
|
fi
|
|
|
|
for protected_path in "${PROTECTED_PATHS[@]}"; do
|
|
if paths_overlap "$canonical_path" "$protected_path"; then
|
|
die "Configured source path is not safe to archive"
|
|
fi
|
|
done
|
|
|
|
relative_path="${canonical_path#/}"
|
|
if [[ -z "$relative_path" ]]; then
|
|
die "Configured source path is not safe to archive"
|
|
fi
|
|
ARCHIVE_ENTRIES+=("$relative_path")
|
|
}
|
|
|
|
build_remote_arguments() {
|
|
SSH_ARGS=()
|
|
SCP_ARGS=()
|
|
if [[ -n "$REMOTE_IDENTITY_FILE" ]]; then
|
|
SSH_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
|
|
SCP_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
|
|
fi
|
|
if [[ -n "$SSH_PORT" ]]; then
|
|
SSH_ARGS+=(-p "$SSH_PORT")
|
|
SCP_ARGS+=(-P "$SSH_PORT")
|
|
fi
|
|
}
|
|
|
|
validate_inputs() {
|
|
require_cmd "$ZIP_BIN"
|
|
require_cmd "$UNZIP_BIN"
|
|
require_cmd "$SSH_BIN"
|
|
require_cmd "$SCP_BIN"
|
|
require_cmd "$FLOCK_BIN"
|
|
require_cmd "$MKTEMP_BIN"
|
|
require_cmd "$DATE_BIN"
|
|
require_cmd "$REALPATH_BIN"
|
|
if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
|
|
require_cmd "$CURL_BIN"
|
|
fi
|
|
|
|
require_absolute_path "$K3S_CONFIG_DIR" "K3S_CONFIG_DIR"
|
|
require_absolute_path "$K3S_SERVER_DIR" "K3S_SERVER_DIR"
|
|
require_absolute_path "$LOCAL_TMP_PARENT" "LOCAL_TMP_PARENT"
|
|
require_absolute_path "$LOCK_PATH" "LOCK_PATH"
|
|
require_absolute_path "$REMOTE_DIR" "REMOTE_DIR"
|
|
|
|
if [[ ! -d "$K3S_CONFIG_DIR" ]]; then
|
|
die "K3S_CONFIG_DIR is not an available directory"
|
|
fi
|
|
determine_protected_paths
|
|
if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then
|
|
die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens"
|
|
fi
|
|
if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then
|
|
die "CLEANUP_KEEP_COUNT must be an integer >= 0"
|
|
fi
|
|
CLEANUP_KEEP_COUNT_INT=$((10#$CLEANUP_KEEP_COUNT))
|
|
if [[ -n "$SSH_PORT" && ! "$SSH_PORT" =~ ^[0-9]+$ ]]; then
|
|
die "SSH_PORT must be numeric"
|
|
fi
|
|
if [[ -z "$REMOTE_USER" || -z "$REMOTE_HOST" ]]; then
|
|
die "REMOTE_USER and REMOTE_HOST must not be empty"
|
|
fi
|
|
if [[ ! "$REMOTE_USER" =~ ^[[:alnum:]_][[:alnum:]_.-]*$ ]]; then
|
|
die "REMOTE_USER may contain only letters, numbers, underscores, dots, and hyphens"
|
|
fi
|
|
if [[ ! "$REMOTE_HOST" =~ ^[[:alnum:]][[:alnum:].-]*$ ]]; then
|
|
die "REMOTE_HOST may contain only letters, numbers, dots, and hyphens"
|
|
fi
|
|
if [[ ! "$REMOTE_DIR" =~ ^/([[:alnum:]_.-]+/)*[[:alnum:]_.-]+/?$ || "$REMOTE_DIR" == */./* || "$REMOTE_DIR" == */../* || "$REMOTE_DIR" == */. || "$REMOTE_DIR" == */.. ]]; then
|
|
die "REMOTE_DIR must be an absolute path containing only letters, numbers, dots, underscores, hyphens, and slashes without dot segments"
|
|
fi
|
|
|
|
mkdir -p -- "$LOCAL_TMP_PARENT"
|
|
mkdir -p -- "$(dirname -- "$LOCK_PATH")"
|
|
build_remote_arguments
|
|
}
|
|
|
|
collect_sources() {
|
|
local service_file
|
|
local server_entry
|
|
|
|
IFS=':' read -r -a SERVICE_FILE_LIST <<< "$K3S_SERVICE_FILES"
|
|
|
|
# These are the only server entries ever included. In particular, server/db
|
|
# and server/kine.sock are intentionally not archive inputs.
|
|
add_archive_entry_if_present "$K3S_CONFIG_DIR"
|
|
for service_file in "${SERVICE_FILE_LIST[@]}"; do
|
|
[[ -z "$service_file" ]] && continue
|
|
require_absolute_path "$service_file" "K3S_SERVICE_FILES entry"
|
|
add_archive_entry_if_present "$service_file"
|
|
done
|
|
for server_entry in "${K3S_SERVER_ENTRIES[@]}"; do
|
|
add_archive_entry_if_present "${K3S_SERVER_DIR%/}/${server_entry}"
|
|
done
|
|
|
|
if (( ${#ARCHIVE_ENTRIES[@]} == 0 )); then
|
|
die "No k3s configuration sources are available"
|
|
fi
|
|
}
|
|
|
|
acquire_lock() {
|
|
exec {LOCK_FD}>"$LOCK_PATH"
|
|
if ! "$FLOCK_BIN" -n "$LOCK_FD"; then
|
|
die "Another k3s control-plane configuration backup is already running"
|
|
fi
|
|
}
|
|
|
|
make_archive_name() {
|
|
local timestamp
|
|
timestamp="$("$DATE_BIN" +"$ARCHIVE_TS_FORMAT")"
|
|
if [[ ! "$timestamp" =~ ^[[:alnum:]_.-]+$ ]]; then
|
|
die "ARCHIVE_TS_FORMAT produced an unsafe archive timestamp"
|
|
fi
|
|
ARCHIVE_NAME="${ARCHIVE_PREFIX}_${timestamp}.zip"
|
|
}
|
|
|
|
create_and_validate_archive() {
|
|
if ! RUN_TMP_DIR="$("$MKTEMP_BIN" -d "${LOCAL_TMP_PARENT%/}/k3s-control-plane-config.XXXXXX")"; then
|
|
die "Unable to create local temporary directory"
|
|
fi
|
|
LOCAL_ARCHIVE_PATH="${RUN_TMP_DIR}/${ARCHIVE_NAME}"
|
|
|
|
log_info "Creating k3s control-plane configuration archive"
|
|
(
|
|
cd /
|
|
"$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" -- "${ARCHIVE_ENTRIES[@]}" >/dev/null
|
|
)
|
|
"$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null
|
|
|
|
if [[ ! -f "$LOCAL_ARCHIVE_PATH" ]]; then
|
|
die "Archive creation did not produce an archive"
|
|
fi
|
|
ARCHIVE_SIZE_BYTES="$(wc -c < "$LOCAL_ARCHIVE_PATH" | tr -d '[:space:]')"
|
|
if [[ ! "$ARCHIVE_SIZE_BYTES" =~ ^[0-9]+$ ]]; then
|
|
die "Unable to determine archive size"
|
|
fi
|
|
log_info "Validated local archive"
|
|
}
|
|
|
|
run_ssh() {
|
|
"$SSH_BIN" "${SSH_ARGS[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "$@"
|
|
}
|
|
|
|
remove_remote_temporary_archive() {
|
|
local remote_temp_path="${1:?temporary remote path is required}"
|
|
run_ssh "rm -f -- $(shell_quote "$remote_temp_path")" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
publish_archive() {
|
|
local remote_final_path="${REMOTE_DIR%/}/${ARCHIVE_NAME}"
|
|
local remote_temp_path="${REMOTE_DIR%/}/.${ARCHIVE_NAME}.upload-$$.tmp"
|
|
local remote_spec="${REMOTE_USER}@${REMOTE_HOST}:${remote_temp_path}"
|
|
local move_command
|
|
|
|
run_ssh "mkdir -p -- $(shell_quote "$REMOTE_DIR")"
|
|
|
|
if ! "$SCP_BIN" "${SCP_ARGS[@]}" -- "$LOCAL_ARCHIVE_PATH" "$remote_spec"; then
|
|
log_error "Archive upload failed"
|
|
remove_remote_temporary_archive "$remote_temp_path"
|
|
return 1
|
|
fi
|
|
|
|
move_command="if [ -e $(shell_quote "$remote_final_path") ]; then exit 17; fi; mv -- $(shell_quote "$remote_temp_path") $(shell_quote "$remote_final_path")"
|
|
if ! run_ssh "$move_command"; then
|
|
log_error "Remote archive publish failed"
|
|
remove_remote_temporary_archive "$remote_temp_path"
|
|
return 1
|
|
fi
|
|
|
|
log_info "Published archive successfully"
|
|
}
|
|
|
|
cleanup_remote_archives() {
|
|
if (( CLEANUP_KEEP_COUNT_INT == 0 )); then
|
|
log_info "Remote retention disabled"
|
|
return 0
|
|
fi
|
|
|
|
local remote_cleanup_command
|
|
remote_cleanup_command="$(shell_quote "$REMOTE_BASH_BIN") -s -- $(shell_quote "$REMOTE_DIR") $(shell_quote "$ARCHIVE_PREFIX") $(shell_quote "$CLEANUP_KEEP_COUNT_INT")"
|
|
|
|
run_ssh "$remote_cleanup_command" <<'REMOTE_CLEANUP'
|
|
set -Eeuo pipefail
|
|
|
|
remote_dir="$1"
|
|
archive_prefix="$2"
|
|
keep_count="$3"
|
|
|
|
mapfile -d '' -t sorted_archives < <(
|
|
find -P "$remote_dir" -maxdepth 1 -type f \
|
|
-name "${archive_prefix}_*.zip" -printf '%T@\t%p\0' |
|
|
sort -z -t $'\t' -k1,1nr
|
|
)
|
|
|
|
if (( ${#sorted_archives[@]} <= keep_count )); then
|
|
exit 0
|
|
fi
|
|
|
|
for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do
|
|
archive_record="${sorted_archives[$index]}"
|
|
archive_path="${archive_record#*$'\t'}"
|
|
rm -f -- "$archive_path"
|
|
done
|
|
REMOTE_CLEANUP
|
|
|
|
log_info "Remote archive retention completed"
|
|
}
|
|
|
|
main() {
|
|
validate_inputs
|
|
acquire_lock
|
|
collect_sources
|
|
make_archive_name
|
|
create_and_validate_archive
|
|
publish_archive
|
|
cleanup_remote_archives
|
|
|
|
# Notifications are informational: an unavailable notification endpoint
|
|
# must not change the result of a completed backup.
|
|
local duration_seconds=$((SECONDS - BACKUP_STARTED_SECONDS))
|
|
send_backup_notification "$NOTIFY_SUCCESS_URL" "$ARCHIVE_SIZE_BYTES" "$duration_seconds" || true
|
|
}
|
|
|
|
main "$@"
|