diff --git a/.gitea/workflows/test-automated-nfs-backup.yaml b/.gitea/workflows/test-automated-nfs-backup.yaml new file mode 100644 index 0000000..9bebfeb --- /dev/null +++ b/.gitea/workflows/test-automated-nfs-backup.yaml @@ -0,0 +1,17 @@ +name: Test automated-nfs-backup + +on: + push: + paths: + - 'k8s/**' + - '.gitea/workflows/**' + workflow_dispatch: {} + +jobs: + test-automated-nfs-backup: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Run automated NFS backup tests + run: bash k8s/test/test-automated-nfs-backup.sh diff --git a/k8s/automated-nfs-backup.sh b/k8s/automated-nfs-backup.sh index 21bf38e..3b027cc 100644 --- a/k8s/automated-nfs-backup.sh +++ b/k8s/automated-nfs-backup.sh @@ -25,9 +25,11 @@ SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}" SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}" SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}" SEVENZ_THREADS="${SEVENZ_THREADS:-on}" +SEVENZ_BIN="${SEVENZ_BIN:-7z}" SCALE_TIMEOUT_SECONDS="${SCALE_TIMEOUT_SECONDS:-600}" SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}" SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}" +SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}" TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}" NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}" NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}" @@ -75,7 +77,7 @@ parse_workload_kinds() { validate_inputs() { require_cmd "$KUBECTL_BIN" - require_cmd "7z" + require_cmd "$SEVENZ_BIN" if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then require_cmd "curl" fi @@ -323,7 +325,7 @@ backup_folder() { local folder_path="${1:?folder path required}" local archive_path="${2:?archive path required}" local -a cmd=( - 7z a -t7z + "$SEVENZ_BIN" a -t7z "$archive_path" "$folder_path" "-m0=${SEVENZ_METHOD}" @@ -405,8 +407,8 @@ process_folder() { capture_replicas_state "$namespace" "$state_file" scale_namespace_to_zero "$namespace" "$state_file" - log_info "Waiting 30 seconds for namespace '${namespace}' to scale down..." - sleep 30 + log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..." + sleep "$SCALE_WAIT_SECONDS" else unmapped_folders=$((unmapped_folders + 1)) log_warn "No namespace matched folder '${folder_name}'. Running backup only." @@ -422,8 +424,8 @@ process_folder() { if [[ "$has_mapping" -eq 1 ]]; then restore_namespace_replicas "$namespace" "$state_file" - log_info "Waiting 30 seconds for namespace '${namespace}' to restore replicas..." - sleep 30 + log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..." + sleep "$SCALE_WAIT_SECONDS" rm -f "$state_file" fi } diff --git a/k8s/test/test-automated-nfs-backup.sh b/k8s/test/test-automated-nfs-backup.sh new file mode 100644 index 0000000..d0110aa --- /dev/null +++ b/k8s/test/test-automated-nfs-backup.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/automated-nfs-backup.sh" +TMPD="$(mktemp -d "${TMPDIR:-/tmp}/automated-nfs-backup-test.XXXXXX")" + +tests_total=0 +tests_passed=0 +tests_failed=0 + +cleanup() { + rm -rf "${TMPD}" +} +trap cleanup EXIT + +assert() { + local message="$1" + shift + + tests_total=$((tests_total + 1)) + if "$@"; then + printf 'PASS: %s\n' "$message" + tests_passed=$((tests_passed + 1)) + else + printf 'FAIL: %s\n' "$message" + tests_failed=$((tests_failed + 1)) + fi +} + +is_zero() { + [[ "$1" -eq 0 ]] +} + +is_nonzero() { + [[ "$1" -ne 0 ]] +} + +file_contains() { + local file="$1" + local value="$2" + grep -Fq -- "$value" "$file" +} + +file_lacks() { + local file="$1" + local value="$2" + ! grep -Fq -- "$value" "$file" +} + +file_matches() { + local file="$1" + local regex="$2" + grep -Eq -- "$regex" "$file" +} + +archive_exists() { + local output_dir="$1" + local -a archives=() + + shopt -s nullglob + archives=("${output_dir}"/test_*.7z) + shopt -u nullglob + [[ "${#archives[@]}" -gt 0 ]] +} + +trace_has_scale() { + local trace_file="$1" + local namespace="$2" + local replicas="$3" + + grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file" +} + +# Asserts the real script invoked 7z as `7z a -t7z ...` +# for the given folder. The archive is the 3rd token (ends in .7z) and the +# folder path is the 4th token, so a swap would fail to match. +sevenz_called_for() { + local folder_name="$1" + + grep -Eq -- "a -t7z [^ ]+\\.7z .*/${folder_name}( |$)" "${TMPD}/7z_invocations.log" +} + +write_fake_script() { + local path="$1" + + while IFS= read -r line || [[ -n "$line" ]]; do + printf '%s\n' "$line" + done > "$path" + chmod +x "$path" +} + +write_fake_binaries() { + mkdir -p "${TMPD}/bin" + + write_fake_script "${TMPD}/bin/kubectl" <<'EOF' +#!/usr/bin/env bash +set -uo pipefail + +namespace="" +get_kind="" +has_output_format=0 + +for ((i = 1; i <= $#; i++)); do + arg="${!i}" + case "$arg" in + -n) + next=$((i + 1)) + namespace="${!next}" + ;; + get) + next=$((i + 1)) + get_kind="${!next}" + ;; + -o) + has_output_format=1 + ;; + esac +done + +if [[ "$get_kind" == "namespaces" ]]; then + if [[ "$has_output_format" -eq 1 ]]; then + while IFS= read -r configured_namespace; do + [[ -n "$configured_namespace" ]] && printf '%s\n' "$configured_namespace" + done < "${TMPD}/namespaces.txt" + fi + exit 0 +fi + +if [[ -n "$get_kind" && -n "$namespace" ]]; then + replicas="" + while IFS=: read -r configured_namespace configured_replicas; do + if [[ "$configured_namespace" == "$namespace" ]]; then + replicas="$configured_replicas" + break + fi + done < "${TMPD}/replicas.txt" + + [[ -n "$replicas" ]] || replicas=1 + printf 'web\t%s\t\n' "$replicas" + exit 0 +fi + +for ((i = 1; i <= $#; i++)); do + if [[ "${!i}" == "scale" ]]; then + resource_index=$((i + 1)) + resource="${!resource_index}" + kind="${resource%%/*}" + name="${resource#*/}" + replicas_arg="" + for arg in "$@"; do + [[ "$arg" == --replicas=* ]] && replicas_arg="$arg" + done + printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log" + exit 0 + fi +done + +exit 0 +EOF + + write_fake_script "${TMPD}/bin/7z" <<'EOF' +#!/usr/bin/env bash +set -uo pipefail + +# Record the full invocation so the harness can assert argument order. +printf '%s\n' "$*" >> "${TMPD}/7z_invocations.log" + +if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then + archive="${3:?archive path required}" + folder="${4:?folder path required}" + folder="${folder%/}" + folder_name="${folder##*/}" + + # Simulate a backup failure for configured folders BEFORE recording a + # successful archive, so 7z_trace.log only lists folders actually archived. + if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then + exit 2 + fi + + printf '%s\n' "$folder_name" >> "${TMPD}/7z_trace.log" + touch "$archive" +fi + +exit 0 +EOF + + write_fake_script "${TMPD}/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -uo pipefail + +url="" +body="" +for ((i = 1; i <= $#; i++)); do + arg="${!i}" + case "$arg" in + POST) + next=$((i + 1)) + url="${!next}" + ;; + -d) + next=$((i + 1)) + body="${!next}" + ;; + esac +done + +body="${body//$'\n'/ }" +printf '%s %s\n' "$url" "${body:0:120}" >> "${TMPD}/curl_trace.log" +exit 0 +EOF +} + +run_backup() { + local source_dir="$1" + local output_dir="$2" + local state_dir="$3" + local log_file="$4" + + NFS_SOURCE_PATH="$source_dir" \ + BACKUP_OUTPUT_PATH="$output_dir" \ + BACKUP_PASSWORD="" \ + ARCHIVE_PREFIX="test" \ + NOTIFY_SUCCESS_URL="http://example.invalid/success" \ + NOTIFY_FAILURE_URL="http://example.invalid/failure" \ + KUBECTL_BIN="${TMPD}/bin/kubectl" \ + SEVENZ_BIN="${TMPD}/bin/7z" \ + SCALE_WAIT_SECONDS=0 \ + TMP_STATE_DIR="$state_dir" \ + PATH="${TMPD}/bin:${PATH}" \ + TMPD="$TMPD" \ + bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1 +} + +write_fake_binaries +export TMPD + +# Scenario A: a mapped folder is scaled down and restored, while an unmapped +# folder is backed up directly. Both folders share one archive. +success_dir="${TMPD}/success" +success_source="${success_dir}/source" +success_output="${success_dir}/out" +success_state="${success_dir}/state" +mkdir -p "${success_source}/app-ns" "${success_source}/loose-data" "$success_output" "$success_state" +printf 'data\n' > "${success_source}/app-ns/file.txt" +printf 'data\n' > "${success_source}/loose-data/file.txt" +printf 'app-ns\n' > "${TMPD}/namespaces.txt" +printf 'app-ns:3\n' > "${TMPD}/replicas.txt" +: > "${TMPD}/fail_folders.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/7z_trace.log" +: > "${TMPD}/curl_trace.log" + +set +e +run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log" +success_exit_code=$? +set -e + +assert "success scenario exits with code 0" is_zero "$success_exit_code" +assert "success scenario creates a test archive" archive_exists "$success_output" +assert "success scenario reports two backup successes" file_contains "${success_dir}/success.log" "backup_successes=2" +assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0" +assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0" +assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3" +assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success" +assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data" +assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns" +assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data" + +# Scenario B: one namespace backup fails, but both namespaces are restored and +# the script exits through its documented handled-failure path. +failure_dir="${TMPD}/failure" +failure_source="${failure_dir}/source" +failure_output="${failure_dir}/out" +failure_state="${failure_dir}/state" +mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state" +printf 'data\n' > "${failure_source}/good-ns/file.txt" +printf 'data\n' > "${failure_source}/fail-ns/file.txt" +printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt" +printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt" +printf 'fail-ns\n' > "${TMPD}/fail_folders.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/7z_trace.log" +: > "${TMPD}/curl_trace.log" + +set +e +run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log" +failure_exit_code=$? +set -e + +assert "failure scenario exits non-zero" is_nonzero "$failure_exit_code" +assert "failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1" +assert "failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1" +assert "failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0" +assert "failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2" +assert "failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0" +assert "failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4" +assert "failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" +assert "failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns" +assert "failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns" + +# Scenario C: abnormal abort while a namespace is still scaled to zero. +# We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep` +# errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its +# .state file is written, but BEFORE the in-line restore. The EXIT trap must then +# drive restore_all_remaining() and scale the namespace back up. This exercises +# the real safety-net path that scenario B leaves unproven. +abort_dir="${TMPD}/abort" +abort_source="${abort_dir}/source" +abort_output="${abort_dir}/out" +abort_state="${abort_dir}/state" +mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" +printf 'data\n' > "${abort_source}/crash-ns/file.txt" +printf 'crash-ns\n' > "${TMPD}/namespaces.txt" +printf 'crash-ns:5\n' > "${TMPD}/replicas.txt" +: > "${TMPD}/fail_folders.txt" +: > "${TMPD}/scale_trace.log" +: > "${TMPD}/7z_trace.log" +: > "${TMPD}/curl_trace.log" +: > "${TMPD}/7z_invocations.log" + +set +e +NFS_SOURCE_PATH="${abort_source}" \ +BACKUP_OUTPUT_PATH="${abort_output}" \ +BACKUP_PASSWORD="" \ +ARCHIVE_PREFIX="test" \ +NOTIFY_SUCCESS_URL="" \ +NOTIFY_FAILURE_URL="" \ +KUBECTL_BIN="${TMPD}/bin/kubectl" \ +SEVENZ_BIN="${TMPD}/bin/7z" \ +SCALE_WAIT_SECONDS="not-a-number" \ +TMP_STATE_DIR="${abort_state}" \ +PATH="${TMPD}/bin:${PATH}" \ +TMPD="${TMPD}" \ +bash "${SCRIPT_UNDER_TEST}" >"${abort_dir}/abort.log" 2>&1 +abort_exit_code=$? +set -e + +assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}" +assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0" +assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5" +assert "abort scenario left no stale .state file" file_lacks "${abort_dir}/abort.log" ".state" + +printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total" +if [[ "$tests_failed" -gt 0 ]]; then + exit 1 +fi