fix(k8s): harden control-plane backup safety
This commit is contained in:
@@ -41,6 +41,7 @@ CURL_BIN="${CURL_BIN:-curl}"
|
|||||||
FLOCK_BIN="${FLOCK_BIN:-flock}"
|
FLOCK_BIN="${FLOCK_BIN:-flock}"
|
||||||
MKTEMP_BIN="${MKTEMP_BIN:-mktemp}"
|
MKTEMP_BIN="${MKTEMP_BIN:-mktemp}"
|
||||||
DATE_BIN="${DATE_BIN:-date}"
|
DATE_BIN="${DATE_BIN:-date}"
|
||||||
|
REALPATH_BIN="${REALPATH_BIN:-realpath}"
|
||||||
REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
|
REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
|
||||||
|
|
||||||
# Notifications follow the JSON formats used by automated-nfs-backup.sh.
|
# Notifications follow the JSON formats used by automated-nfs-backup.sh.
|
||||||
@@ -55,6 +56,7 @@ declare -a ARCHIVE_ENTRIES=()
|
|||||||
declare -a SERVICE_FILE_LIST=()
|
declare -a SERVICE_FILE_LIST=()
|
||||||
declare -a SSH_ARGS=()
|
declare -a SSH_ARGS=()
|
||||||
declare -a SCP_ARGS=()
|
declare -a SCP_ARGS=()
|
||||||
|
declare -a PROTECTED_PATHS=()
|
||||||
readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token)
|
readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token)
|
||||||
|
|
||||||
RUN_TMP_DIR=""
|
RUN_TMP_DIR=""
|
||||||
@@ -175,16 +177,83 @@ require_absolute_path() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
path_is_ancestor_or_same() {
|
||||||
|
local ancestor="${1:?ancestor path is required}"
|
||||||
|
local path="${2:?path is required}"
|
||||||
|
|
||||||
|
[[ "$ancestor" == "/" || "$path" == "$ancestor" || "$path" == "$ancestor"/* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
paths_overlap() {
|
||||||
|
local first="${1:?first path is required}"
|
||||||
|
local second="${2:?second path is required}"
|
||||||
|
|
||||||
|
path_is_ancestor_or_same "$first" "$second" || \
|
||||||
|
path_is_ancestor_or_same "$second" "$first"
|
||||||
|
}
|
||||||
|
|
||||||
|
canonicalize_protected_path() {
|
||||||
|
local path="${1:?protected path is required}"
|
||||||
|
local canonical_path
|
||||||
|
local canonical_parent
|
||||||
|
local basename
|
||||||
|
|
||||||
|
if [[ -e "$path" || -L "$path" ]]; then
|
||||||
|
canonical_path="$("$REALPATH_BIN" -e -- "$path" 2>/dev/null)" || return 1
|
||||||
|
printf '%s' "$canonical_path"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
basename="${path##*/}"
|
||||||
|
canonical_parent="${path%/*}"
|
||||||
|
if ! canonical_parent="$("$REALPATH_BIN" -e -- "$canonical_parent" 2>/dev/null)"; then
|
||||||
|
canonical_parent="$("$REALPATH_BIN" -m -- "$canonical_parent" 2>/dev/null)" || return 1
|
||||||
|
fi
|
||||||
|
printf '%s/%s' "${canonical_parent%/}" "$basename"
|
||||||
|
}
|
||||||
|
|
||||||
|
determine_protected_paths() {
|
||||||
|
local canonical_server_dir
|
||||||
|
local protected_path
|
||||||
|
|
||||||
|
if ! canonical_server_dir="$("$REALPATH_BIN" -e -- "$K3S_SERVER_DIR" 2>/dev/null)"; then
|
||||||
|
canonical_server_dir="$("$REALPATH_BIN" -m -- "$K3S_SERVER_DIR" 2>/dev/null)" || \
|
||||||
|
die "Configured source path is not safe to archive"
|
||||||
|
fi
|
||||||
|
|
||||||
|
PROTECTED_PATHS=()
|
||||||
|
if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/db")"; then
|
||||||
|
die "Configured source path is not safe to archive"
|
||||||
|
fi
|
||||||
|
PROTECTED_PATHS+=("$protected_path")
|
||||||
|
if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/kine.sock")"; then
|
||||||
|
die "Configured source path is not safe to archive"
|
||||||
|
fi
|
||||||
|
PROTECTED_PATHS+=("$protected_path")
|
||||||
|
}
|
||||||
|
|
||||||
add_archive_entry_if_present() {
|
add_archive_entry_if_present() {
|
||||||
local absolute_path="${1:?source path is required}"
|
local absolute_path="${1:?source path is required}"
|
||||||
|
local canonical_path
|
||||||
local relative_path
|
local relative_path
|
||||||
|
local protected_path
|
||||||
|
|
||||||
if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then
|
if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
relative_path="${absolute_path#/}"
|
if ! canonical_path="$("$REALPATH_BIN" -e -- "$absolute_path" 2>/dev/null)"; then
|
||||||
if [[ -z "$relative_path" || "$relative_path" == *"/../"* || "$relative_path" == ../* ]]; then
|
die "Configured source path is not safe to archive"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for protected_path in "${PROTECTED_PATHS[@]}"; do
|
||||||
|
if paths_overlap "$canonical_path" "$protected_path"; then
|
||||||
|
die "Configured source path is not safe to archive"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
relative_path="${canonical_path#/}"
|
||||||
|
if [[ -z "$relative_path" ]]; then
|
||||||
die "Configured source path is not safe to archive"
|
die "Configured source path is not safe to archive"
|
||||||
fi
|
fi
|
||||||
ARCHIVE_ENTRIES+=("$relative_path")
|
ARCHIVE_ENTRIES+=("$relative_path")
|
||||||
@@ -211,6 +280,7 @@ validate_inputs() {
|
|||||||
require_cmd "$FLOCK_BIN"
|
require_cmd "$FLOCK_BIN"
|
||||||
require_cmd "$MKTEMP_BIN"
|
require_cmd "$MKTEMP_BIN"
|
||||||
require_cmd "$DATE_BIN"
|
require_cmd "$DATE_BIN"
|
||||||
|
require_cmd "$REALPATH_BIN"
|
||||||
if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
|
if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
|
||||||
require_cmd "$CURL_BIN"
|
require_cmd "$CURL_BIN"
|
||||||
fi
|
fi
|
||||||
@@ -224,6 +294,7 @@ validate_inputs() {
|
|||||||
if [[ ! -d "$K3S_CONFIG_DIR" ]]; then
|
if [[ ! -d "$K3S_CONFIG_DIR" ]]; then
|
||||||
die "K3S_CONFIG_DIR is not an available directory"
|
die "K3S_CONFIG_DIR is not an available directory"
|
||||||
fi
|
fi
|
||||||
|
determine_protected_paths
|
||||||
if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then
|
if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then
|
||||||
die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens"
|
die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens"
|
||||||
fi
|
fi
|
||||||
@@ -300,7 +371,7 @@ create_and_validate_archive() {
|
|||||||
log_info "Creating k3s control-plane configuration archive"
|
log_info "Creating k3s control-plane configuration archive"
|
||||||
(
|
(
|
||||||
cd /
|
cd /
|
||||||
"$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" "${ARCHIVE_ENTRIES[@]}" >/dev/null
|
"$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" -- "${ARCHIVE_ENTRIES[@]}" >/dev/null
|
||||||
)
|
)
|
||||||
"$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null
|
"$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null
|
||||||
|
|
||||||
@@ -363,22 +434,20 @@ remote_dir="$1"
|
|||||||
archive_prefix="$2"
|
archive_prefix="$2"
|
||||||
keep_count="$3"
|
keep_count="$3"
|
||||||
|
|
||||||
shopt -s nullglob
|
mapfile -d '' -t sorted_archives < <(
|
||||||
candidates=("${remote_dir%/}/${archive_prefix}_"*.zip)
|
find -P "$remote_dir" -maxdepth 1 -type f \
|
||||||
shopt -u nullglob
|
-name "${archive_prefix}_*.zip" -printf '%T@\t%p\0' |
|
||||||
|
sort -z -t $'\t' -k1,1nr
|
||||||
|
)
|
||||||
|
|
||||||
matching_archives=()
|
if (( ${#sorted_archives[@]} <= keep_count )); then
|
||||||
for candidate in "${candidates[@]}"; do
|
|
||||||
[[ -f "$candidate" ]] && matching_archives+=("$candidate")
|
|
||||||
done
|
|
||||||
|
|
||||||
if (( ${#matching_archives[@]} <= keep_count )); then
|
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
mapfile -t sorted_archives < <(ls -1t -- "${matching_archives[@]}")
|
|
||||||
for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do
|
for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do
|
||||||
rm -f -- "${sorted_archives[$index]}"
|
archive_record="${sorted_archives[$index]}"
|
||||||
|
archive_path="${archive_record#*$'\t'}"
|
||||||
|
rm -f -- "$archive_path"
|
||||||
done
|
done
|
||||||
REMOTE_CLEANUP
|
REMOTE_CLEANUP
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ is_nonzero() { [[ "$1" -ne 0 ]]; }
|
|||||||
path_exists() { [[ -e "$1" ]]; }
|
path_exists() { [[ -e "$1" ]]; }
|
||||||
path_does_not_exist() { [[ ! -e "$1" ]]; }
|
path_does_not_exist() { [[ ! -e "$1" ]]; }
|
||||||
file_contains() { grep -Fq -- "$2" "$1"; }
|
file_contains() { grep -Fq -- "$2" "$1"; }
|
||||||
file_lacks() { ! grep -Fq -- "$2" "$1"; }
|
file_lacks() { [[ ! -e "$1" ]] || ! grep -Fq -- "$2" "$1"; }
|
||||||
file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; }
|
file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; }
|
||||||
|
|
||||||
matching_archive_count_is() {
|
matching_archive_count_is() {
|
||||||
@@ -74,6 +74,8 @@ set -Eeuo pipefail
|
|||||||
[[ "${1:-}" == "-r" ]]
|
[[ "${1:-}" == "-r" ]]
|
||||||
archive="${2:?archive path required}"
|
archive="${2:?archive path required}"
|
||||||
shift 2
|
shift 2
|
||||||
|
[[ "${1:-}" == "--" ]]
|
||||||
|
shift
|
||||||
: > "$archive"
|
: > "$archive"
|
||||||
for entry in "$@"; do
|
for entry in "$@"; do
|
||||||
source_path="/$entry"
|
source_path="/$entry"
|
||||||
@@ -154,18 +156,10 @@ fi
|
|||||||
|
|
||||||
if [[ "$command" == *" -s -- "* ]]; then
|
if [[ "$command" == *" -s -- "* ]]; then
|
||||||
value_count="${#quoted_values[@]}"
|
value_count="${#quoted_values[@]}"
|
||||||
prefix="${quoted_values[value_count-2]}"
|
(
|
||||||
keep_count="${quoted_values[value_count-1]}"
|
cd -- "$FAKE_REMOTE_ROOT"
|
||||||
shopt -s nullglob
|
bash -s -- "$FAKE_REMOTE_ROOT" "${quoted_values[value_count-2]}" "${quoted_values[value_count-1]}"
|
||||||
archives=("${FAKE_REMOTE_ROOT}/${prefix}_"*.zip)
|
)
|
||||||
shopt -u nullglob
|
|
||||||
if (( ${#archives[@]} > keep_count )); then
|
|
||||||
mapfile -t sorted_archives < <(ls -1t -- "${archives[@]}")
|
|
||||||
for ((i = keep_count; i < ${#sorted_archives[@]}; i++)); do
|
|
||||||
rm -f -- "${sorted_archives[$i]}"
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
cat >/dev/null
|
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -208,7 +202,7 @@ run_backup() {
|
|||||||
shift 4
|
shift 4
|
||||||
local -a overrides=("$@")
|
local -a overrides=("$@")
|
||||||
|
|
||||||
env "${overrides[@]}" \
|
env \
|
||||||
K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \
|
K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \
|
||||||
K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \
|
K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \
|
||||||
K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \
|
K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \
|
||||||
@@ -229,6 +223,7 @@ run_backup() {
|
|||||||
PATH="${TMPD}/bin:${PATH}" \
|
PATH="${TMPD}/bin:${PATH}" \
|
||||||
TMPD="$TMPD" \
|
TMPD="$TMPD" \
|
||||||
FAKE_REMOTE_ROOT="$remote_root" \
|
FAKE_REMOTE_ROOT="$remote_root" \
|
||||||
|
"${overrides[@]}" \
|
||||||
bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
|
bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -279,6 +274,11 @@ for day in 1 2 3 4 5; do
|
|||||||
done
|
done
|
||||||
printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z"
|
printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z"
|
||||||
printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip"
|
printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip"
|
||||||
|
newline_archive="${success_remote}/k3s-control-plane-config_2026-06-01_12-00-00.zip"$'\nunrelated.zip'
|
||||||
|
touch -t "202606011200" "$newline_archive"
|
||||||
|
: > "$newline_archive"
|
||||||
|
touch -t "202606011200" "$newline_archive"
|
||||||
|
: > "${success_remote}/unrelated.zip"
|
||||||
: > "${TMPD}/curl_trace.log"
|
: > "${TMPD}/curl_trace.log"
|
||||||
: > "${TMPD}/unzip_trace.log"
|
: > "${TMPD}/unzip_trace.log"
|
||||||
: > "${TMPD}/zip_invocations.log"
|
: > "${TMPD}/zip_invocations.log"
|
||||||
@@ -302,11 +302,42 @@ assert "archive excludes kine socket" file_lacks "${success_remote}/k3s-control-
|
|||||||
assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4
|
assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4
|
||||||
assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip"
|
assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip"
|
||||||
assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip"
|
assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip"
|
||||||
|
assert "retention removes matching old archives with newlines in their filenames" path_does_not_exist "$newline_archive"
|
||||||
|
assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${success_remote}/unrelated.zip"
|
||||||
assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
|
assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
|
||||||
assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
|
assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
|
||||||
assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
|
assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
|
||||||
assert "successful backup sends a success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
assert "successful backup sends a success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
||||||
|
|
||||||
|
# A configured source path ending in server/db/.. must be rejected before zip is
|
||||||
|
# invoked, rather than recursively archiving server/db and its sibling socket.
|
||||||
|
source_scope_dir="${TMPD}/source-scope"
|
||||||
|
source_scope_fixture="${source_scope_dir}/fixture"
|
||||||
|
source_scope_remote="${source_scope_dir}/remote"
|
||||||
|
source_scope_local_tmp="${source_scope_dir}/local-tmp"
|
||||||
|
source_scope_archive="${source_scope_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
||||||
|
mkdir -p "$source_scope_remote" "$source_scope_local_tmp"
|
||||||
|
create_source_fixture "$source_scope_fixture"
|
||||||
|
: > "${TMPD}/zip_invocations.log"
|
||||||
|
|
||||||
|
set +e
|
||||||
|
run_backup \
|
||||||
|
"$source_scope_fixture" \
|
||||||
|
"$source_scope_remote" \
|
||||||
|
"$source_scope_local_tmp" \
|
||||||
|
"${source_scope_dir}/run.log" \
|
||||||
|
"K3S_CONFIG_DIR=${source_scope_fixture}/var/lib/rancher/k3s/server/db/.."
|
||||||
|
source_scope_exit_code=$?
|
||||||
|
set -e
|
||||||
|
|
||||||
|
assert "unsafe source override exits non-zero" is_nonzero "$source_scope_exit_code"
|
||||||
|
assert "unsafe source override reports the source-scope error" file_contains "${source_scope_dir}/run.log" "Configured source path is not safe to archive"
|
||||||
|
assert "unsafe source override is rejected before zip runs" file_lacks "${TMPD}/zip_invocations.log" "server/db/.."
|
||||||
|
assert "unsafe source override archive excludes server database content" file_lacks "$source_scope_archive" "state.db"
|
||||||
|
assert "unsafe source override archive excludes kine socket" file_lacks "$source_scope_archive" "kine.sock"
|
||||||
|
assert "unsafe source override does not publish an archive" directory_is_empty "$source_scope_remote"
|
||||||
|
assert "unsafe source override removes local temporary state" directory_is_empty "$source_scope_local_tmp"
|
||||||
|
|
||||||
# Upload failure: the partial remote temporary file is removed, no final archive
|
# Upload failure: the partial remote temporary file is removed, no final archive
|
||||||
# is published, local temporary data is removed, and one safe error notification
|
# is published, local temporary data is removed, and one safe error notification
|
||||||
# is attempted.
|
# is attempted.
|
||||||
|
|||||||
Reference in New Issue
Block a user