server-scripts/k8s/backup-k3s-control-plane-config.sh

#!/usr/bin/env bash

# Back up the k3s control-plane configuration and credentials, without taking
# a live datastore snapshot or changing the running k3s service.

SCRIPT_NAME="$(basename "$0")"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/common.sh
source "${SCRIPT_DIR}/lib/common.sh"

# Keep the local archive and temporary files readable only by this process.
umask 077

# Source paths. K3S_CONFIG_DIR is the only required source; the service files
# and server state entries are included only when present.
K3S_CONFIG_DIR="${K3S_CONFIG_DIR:-/etc/rancher/k3s}"
K3S_SERVER_DIR="${K3S_SERVER_DIR:-/var/lib/rancher/k3s/server}"
K3S_SERVICE_FILES="${K3S_SERVICE_FILES-/etc/systemd/system/k3s.service:/etc/systemd/system/k3s.service.env:/etc/default/k3s:/etc/sysconfig/k3s}"

# Archive and local working settings.
ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-k3s-control-plane-config}"
ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y-%m-%d_%H-%M-%S}"
LOCAL_TMP_PARENT="${LOCAL_TMP_PARENT:-${TMPDIR:-/tmp}}"
LOCK_PATH="${LOCK_PATH:-/var/lock/k3s-control-plane-config-backup.lock}"

# Remote destination settings.
REMOTE_USER="${REMOTE_USER:-ivanch}"
REMOTE_HOST="${REMOTE_HOST:-nas.haven}"
REMOTE_DIR="${REMOTE_DIR:-/export/Backup/k8s}"
REMOTE_IDENTITY_FILE="${REMOTE_IDENTITY_FILE:-}"
SSH_PORT="${SSH_PORT:-}"
CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"

# Commands are configurable to support hosts with non-standard paths and
# isolated tests.
ZIP_BIN="${ZIP_BIN:-zip}"
UNZIP_BIN="${UNZIP_BIN:-unzip}"
SSH_BIN="${SSH_BIN:-ssh}"
SCP_BIN="${SCP_BIN:-scp}"
CURL_BIN="${CURL_BIN:-curl}"
FLOCK_BIN="${FLOCK_BIN:-flock}"
MKTEMP_BIN="${MKTEMP_BIN:-mktemp}"
DATE_BIN="${DATE_BIN:-date}"
REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"

# Notifications follow the JSON formats used by automated-nfs-backup.sh.
# Set either URL to an empty value to disable that notification.
NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/template/notify/backup}"
NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/template/notify/error}"
NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"

declare -a ARCHIVE_ENTRIES=()
declare -a SERVICE_FILE_LIST=()
declare -a SSH_ARGS=()
declare -a SCP_ARGS=()
readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token)

RUN_TMP_DIR=""
LOCAL_ARCHIVE_PATH=""
ARCHIVE_NAME=""
ARCHIVE_SIZE_BYTES=0
CLEANUP_KEEP_COUNT_INT=0
LOCK_FD=""
FAILURE_NOTIFICATION_SENT=0

json_escape() {
    local value="${1:-}"
    value="${value//\\/\\\\}"
    value="${value//\"/\\\"}"
    value="${value//$'\n'/ }"
    printf '%s' "$value"
}

backup_size_mb() {
    local bytes="${1:-0}"
    if (( bytes <= 0 )); then
        printf '0'
        return 0
    fi
    printf '%s' "$(( (bytes + 1048575) / 1048576 ))"
}

send_backup_notification() {
    local url="${1:-}"
    local size_mb="${2:-0}"
    [[ -z "$url" ]] && return 0

    local payload
    payload=$(cat <<EOF
{
  "title": "$(json_escape "$NOTIFY_TITLE")",
  "asset": "$(json_escape "$NOTIFY_ASSET")",
  "backupSizeInMB": ${size_mb}
}
EOF
)

    if ! "$CURL_BIN" -fsS -X POST "$url" \
        -H "Content-Type: application/json" \
        -d "$payload" >/dev/null; then
        log_warn "Failed to send backup notification"
        return 1
    fi
}

send_backup_failure_notification() {
    local url="${1:-}"
    local size_mb="${2:-0}"
    [[ -z "$url" ]] && return 0

    local payload
    payload=$(cat <<EOF
{
  "caller": "$(json_escape "$NOTIFY_CALLER")",
  "message": "Control-plane configuration backup failed",
  "critical": true,
  "extra": [
    {
      "name": "Backup Size (MB)",
      "value": "${size_mb}"
    }
  ]
}
EOF
)

    if ! "$CURL_BIN" -fsS -X POST "$url" \
        -H "Content-Type: application/json" \
        -d "$payload" >/dev/null; then
        log_warn "Failed to send backup failure notification"
        return 1
    fi
}

notify_failure_once() {
    if (( FAILURE_NOTIFICATION_SENT == 1 )); then
        return 0
    fi
    FAILURE_NOTIFICATION_SENT=1
    send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
}

cleanup() {
    local exit_code=$?
    trap - EXIT

    if [[ -n "$RUN_TMP_DIR" && -d "$RUN_TMP_DIR" ]]; then
        rm -rf -- "$RUN_TMP_DIR" || log_warn "Failed to clean local temporary files"
    fi

    if (( exit_code == 0 )); then
        log_info "Finished successfully"
    else
        notify_failure_once
        log_error "Control-plane configuration backup failed"
    fi

    exit "$exit_code"
}
trap cleanup EXIT

shell_quote() {
    local value="${1:-}"
    value="${value//\'/\'\"\'\"\'}"
    printf "'%s'" "$value"
}

require_absolute_path() {
    local path="${1:?path is required}"
    local description="${2:-path}"
    if [[ "$path" != /* ]]; then
        die "${description} must be an absolute path"
    fi
}

add_archive_entry_if_present() {
    local absolute_path="${1:?source path is required}"
    local relative_path

    if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then
        return 0
    fi

    relative_path="${absolute_path#/}"
    if [[ -z "$relative_path" || "$relative_path" == *"/../"* || "$relative_path" == ../* ]]; then
        die "Configured source path is not safe to archive"
    fi
    ARCHIVE_ENTRIES+=("$relative_path")
}

build_remote_arguments() {
    SSH_ARGS=()
    SCP_ARGS=()
    if [[ -n "$REMOTE_IDENTITY_FILE" ]]; then
        SSH_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
        SCP_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
    fi
    if [[ -n "$SSH_PORT" ]]; then
        SSH_ARGS+=(-p "$SSH_PORT")
        SCP_ARGS+=(-P "$SSH_PORT")
    fi
}

validate_inputs() {
    require_cmd "$ZIP_BIN"
    require_cmd "$UNZIP_BIN"
    require_cmd "$SSH_BIN"
    require_cmd "$SCP_BIN"
    require_cmd "$FLOCK_BIN"
    require_cmd "$MKTEMP_BIN"
    require_cmd "$DATE_BIN"
    if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
        require_cmd "$CURL_BIN"
    fi

    require_absolute_path "$K3S_CONFIG_DIR" "K3S_CONFIG_DIR"
    require_absolute_path "$K3S_SERVER_DIR" "K3S_SERVER_DIR"
    require_absolute_path "$LOCAL_TMP_PARENT" "LOCAL_TMP_PARENT"
    require_absolute_path "$LOCK_PATH" "LOCK_PATH"
    require_absolute_path "$REMOTE_DIR" "REMOTE_DIR"

    if [[ ! -d "$K3S_CONFIG_DIR" ]]; then
        die "K3S_CONFIG_DIR is not an available directory"
    fi
    if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then
        die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens"
    fi
    if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then
        die "CLEANUP_KEEP_COUNT must be an integer >= 0"
    fi
    CLEANUP_KEEP_COUNT_INT=$((10#$CLEANUP_KEEP_COUNT))
    if [[ -n "$SSH_PORT" && ! "$SSH_PORT" =~ ^[0-9]+$ ]]; then
        die "SSH_PORT must be numeric"
    fi
    if [[ -z "$REMOTE_USER" || -z "$REMOTE_HOST" ]]; then
        die "REMOTE_USER and REMOTE_HOST must not be empty"
    fi
    if [[ ! "$REMOTE_USER" =~ ^[[:alnum:]_][[:alnum:]_.-]*$ ]]; then
        die "REMOTE_USER may contain only letters, numbers, underscores, dots, and hyphens"
    fi
    if [[ ! "$REMOTE_HOST" =~ ^[[:alnum:]][[:alnum:].-]*$ ]]; then
        die "REMOTE_HOST may contain only letters, numbers, dots, and hyphens"
    fi
    if [[ ! "$REMOTE_DIR" =~ ^/([[:alnum:]_.-]+/)*[[:alnum:]_.-]+/?$ || "$REMOTE_DIR" == */./* || "$REMOTE_DIR" == */../* || "$REMOTE_DIR" == */. || "$REMOTE_DIR" == */.. ]]; then
        die "REMOTE_DIR must be an absolute path containing only letters, numbers, dots, underscores, hyphens, and slashes without dot segments"
    fi

    mkdir -p -- "$LOCAL_TMP_PARENT"
    mkdir -p -- "$(dirname -- "$LOCK_PATH")"
    build_remote_arguments
}

collect_sources() {
    local service_file
    local server_entry

    IFS=':' read -r -a SERVICE_FILE_LIST <<< "$K3S_SERVICE_FILES"

    # These are the only server entries ever included. In particular, server/db
    # and server/kine.sock are intentionally not archive inputs.
    add_archive_entry_if_present "$K3S_CONFIG_DIR"
    for service_file in "${SERVICE_FILE_LIST[@]}"; do
        [[ -z "$service_file" ]] && continue
        require_absolute_path "$service_file" "K3S_SERVICE_FILES entry"
        add_archive_entry_if_present "$service_file"
    done
    for server_entry in "${K3S_SERVER_ENTRIES[@]}"; do
        add_archive_entry_if_present "${K3S_SERVER_DIR%/}/${server_entry}"
    done

    if (( ${#ARCHIVE_ENTRIES[@]} == 0 )); then
        die "No k3s configuration sources are available"
    fi
}

acquire_lock() {
    exec {LOCK_FD}>"$LOCK_PATH"
    if ! "$FLOCK_BIN" -n "$LOCK_FD"; then
        die "Another k3s control-plane configuration backup is already running"
    fi
}

make_archive_name() {
    local timestamp
    timestamp="$("$DATE_BIN" +"$ARCHIVE_TS_FORMAT")"
    if [[ ! "$timestamp" =~ ^[[:alnum:]_.-]+$ ]]; then
        die "ARCHIVE_TS_FORMAT produced an unsafe archive timestamp"
    fi
    ARCHIVE_NAME="${ARCHIVE_PREFIX}_${timestamp}.zip"
}

create_and_validate_archive() {
    if ! RUN_TMP_DIR="$("$MKTEMP_BIN" -d "${LOCAL_TMP_PARENT%/}/k3s-control-plane-config.XXXXXX")"; then
        die "Unable to create local temporary directory"
    fi
    LOCAL_ARCHIVE_PATH="${RUN_TMP_DIR}/${ARCHIVE_NAME}"

    log_info "Creating k3s control-plane configuration archive"
    (
        cd /
        "$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" "${ARCHIVE_ENTRIES[@]}" >/dev/null
    )
    "$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null

    if [[ ! -f "$LOCAL_ARCHIVE_PATH" ]]; then
        die "Archive creation did not produce an archive"
    fi
    ARCHIVE_SIZE_BYTES="$(wc -c < "$LOCAL_ARCHIVE_PATH" | tr -d '[:space:]')"
    if [[ ! "$ARCHIVE_SIZE_BYTES" =~ ^[0-9]+$ ]]; then
        die "Unable to determine archive size"
    fi
    log_info "Validated local archive"
}

run_ssh() {
    "$SSH_BIN" "${SSH_ARGS[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "$@"
}

remove_remote_temporary_archive() {
    local remote_temp_path="${1:?temporary remote path is required}"
    run_ssh "rm -f -- $(shell_quote "$remote_temp_path")" >/dev/null 2>&1 || true
}

publish_archive() {
    local remote_final_path="${REMOTE_DIR%/}/${ARCHIVE_NAME}"
    local remote_temp_path="${REMOTE_DIR%/}/.${ARCHIVE_NAME}.upload-$$.tmp"
    local remote_spec="${REMOTE_USER}@${REMOTE_HOST}:${remote_temp_path}"
    local move_command

    run_ssh "mkdir -p -- $(shell_quote "$REMOTE_DIR")"

    if ! "$SCP_BIN" "${SCP_ARGS[@]}" -- "$LOCAL_ARCHIVE_PATH" "$remote_spec"; then
        log_error "Archive upload failed"
        remove_remote_temporary_archive "$remote_temp_path"
        return 1
    fi

    move_command="if [ -e $(shell_quote "$remote_final_path") ]; then exit 17; fi; mv -- $(shell_quote "$remote_temp_path") $(shell_quote "$remote_final_path")"
    if ! run_ssh "$move_command"; then
        log_error "Remote archive publish failed"
        remove_remote_temporary_archive "$remote_temp_path"
        return 1
    fi

    log_info "Published archive successfully"
}

cleanup_remote_archives() {
    if (( CLEANUP_KEEP_COUNT_INT == 0 )); then
        log_info "Remote retention disabled"
        return 0
    fi

    local remote_cleanup_command
    remote_cleanup_command="$(shell_quote "$REMOTE_BASH_BIN") -s -- $(shell_quote "$REMOTE_DIR") $(shell_quote "$ARCHIVE_PREFIX") $(shell_quote "$CLEANUP_KEEP_COUNT_INT")"

    run_ssh "$remote_cleanup_command" <<'REMOTE_CLEANUP'
set -Eeuo pipefail

remote_dir="$1"
archive_prefix="$2"
keep_count="$3"

shopt -s nullglob
candidates=("${remote_dir%/}/${archive_prefix}_"*.zip)
shopt -u nullglob

matching_archives=()
for candidate in "${candidates[@]}"; do
    [[ -f "$candidate" ]] && matching_archives+=("$candidate")
done

if (( ${#matching_archives[@]} <= keep_count )); then
    exit 0
fi

mapfile -t sorted_archives < <(ls -1t -- "${matching_archives[@]}")
for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do
    rm -f -- "${sorted_archives[$index]}"
done
REMOTE_CLEANUP

    log_info "Remote archive retention completed"
}

main() {
    validate_inputs
    acquire_lock
    collect_sources
    make_archive_name
    create_and_validate_archive
    publish_archive
    cleanup_remote_archives

    # Notifications are informational: an unavailable notification endpoint
    # must not change the result of a completed backup.
    send_backup_notification "$NOTIFY_SUCCESS_URL" "$(backup_size_mb "$ARCHIVE_SIZE_BYTES")" || true
}

main "$@"