#!/usr/bin/env bash set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/automated-nfs-backup.sh" TMPD="$(mktemp -d "${TMPDIR:-/tmp}/automated-nfs-backup-test.XXXXXX")" tests_total=0 tests_passed=0 tests_failed=0 cleanup() { rm -rf "${TMPD}" } trap cleanup EXIT assert() { local message="$1" shift tests_total=$((tests_total + 1)) if "$@"; then printf 'PASS: %s\n' "$message" tests_passed=$((tests_passed + 1)) else printf 'FAIL: %s\n' "$message" tests_failed=$((tests_failed + 1)) fi } is_zero() { [[ "$1" -eq 0 ]] } is_nonzero() { [[ "$1" -ne 0 ]] } file_contains() { local file="$1" local value="$2" grep -Fq -- "$value" "$file" } file_lacks() { local file="$1" local value="$2" ! grep -Fq -- "$value" "$file" } file_matches() { local file="$1" local regex="$2" grep -Eq -- "$regex" "$file" } archive_exists() { local output_dir="$1" local -a archives=() shopt -s nullglob archives=("${output_dir}"/test_*.7z) shopt -u nullglob [[ "${#archives[@]}" -gt 0 ]] } trace_has_scale() { local trace_file="$1" local namespace="$2" local replicas="$3" grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file" } # Asserts the real script invoked 7z as `7z a -t7z ...` # for the given folder. The archive is the 3rd token (ends in .7z) and the # folder path is the 4th token, so a swap would fail to match. sevenz_called_for() { local folder_name="$1" grep -Eq -- "a -t7z [^ ]+\\.7z .*/${folder_name}( |$)" "${TMPD}/7z_invocations.log" } write_fake_script() { local path="$1" while IFS= read -r line || [[ -n "$line" ]]; do printf '%s\n' "$line" done > "$path" chmod +x "$path" } write_fake_binaries() { mkdir -p "${TMPD}/bin" write_fake_script "${TMPD}/bin/kubectl" <<'EOF' #!/usr/bin/env bash set -uo pipefail namespace="" get_kind="" has_output_format=0 for ((i = 1; i <= $#; i++)); do arg="${!i}" case "$arg" in -n) next=$((i + 1)) namespace="${!next}" ;; get) next=$((i + 1)) get_kind="${!next}" ;; -o) has_output_format=1 ;; esac done if [[ "$get_kind" == "namespaces" ]]; then if [[ "$has_output_format" -eq 1 ]]; then while IFS= read -r configured_namespace; do [[ -n "$configured_namespace" ]] && printf '%s\n' "$configured_namespace" done < "${TMPD}/namespaces.txt" fi exit 0 fi if [[ -n "$get_kind" && -n "$namespace" ]]; then replicas="" while IFS=: read -r configured_namespace configured_replicas; do if [[ "$configured_namespace" == "$namespace" ]]; then replicas="$configured_replicas" break fi done < "${TMPD}/replicas.txt" [[ -n "$replicas" ]] || replicas=1 printf 'web\t%s\t\n' "$replicas" exit 0 fi for ((i = 1; i <= $#; i++)); do if [[ "${!i}" == "scale" ]]; then resource_index=$((i + 1)) resource="${!resource_index}" kind="${resource%%/*}" name="${resource#*/}" replicas_arg="" for arg in "$@"; do [[ "$arg" == --replicas=* ]] && replicas_arg="$arg" done printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log" exit 0 fi done exit 0 EOF write_fake_script "${TMPD}/bin/7z" <<'EOF' #!/usr/bin/env bash set -uo pipefail # Record the full invocation so the harness can assert argument order. printf '%s\n' "$*" >> "${TMPD}/7z_invocations.log" if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then archive="${3:?archive path required}" folder="${4:?folder path required}" folder="${folder%/}" folder_name="${folder##*/}" # Simulate a backup failure for configured folders BEFORE recording a # successful archive, so 7z_trace.log only lists folders actually archived. if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then exit 2 fi printf '%s\n' "$folder_name" >> "${TMPD}/7z_trace.log" touch "$archive" fi exit 0 EOF write_fake_script "${TMPD}/bin/curl" <<'EOF' #!/usr/bin/env bash set -uo pipefail url="" body="" for ((i = 1; i <= $#; i++)); do arg="${!i}" case "$arg" in POST) next=$((i + 1)) url="${!next}" ;; -d) next=$((i + 1)) body="${!next}" ;; esac done body="${body//$'\n'/ }" printf '%s %s\n' "$url" "${body:0:120}" >> "${TMPD}/curl_trace.log" exit 0 EOF } run_backup() { local source_dir="$1" local output_dir="$2" local state_dir="$3" local log_file="$4" NFS_SOURCE_PATH="$source_dir" \ BACKUP_OUTPUT_PATH="$output_dir" \ BACKUP_PASSWORD="" \ ARCHIVE_PREFIX="test" \ NOTIFY_SUCCESS_URL="http://example.invalid/success" \ NOTIFY_FAILURE_URL="http://example.invalid/failure" \ KUBECTL_BIN="${TMPD}/bin/kubectl" \ SEVENZ_BIN="${TMPD}/bin/7z" \ SCALE_WAIT_SECONDS=0 \ TMP_STATE_DIR="$state_dir" \ PATH="${TMPD}/bin:${PATH}" \ TMPD="$TMPD" \ bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1 } write_fake_binaries export TMPD # Scenario A: a mapped folder is scaled down and restored, while an unmapped # folder is backed up directly. Both folders share one archive. success_dir="${TMPD}/success" success_source="${success_dir}/source" success_output="${success_dir}/out" success_state="${success_dir}/state" mkdir -p "${success_source}/app-ns" "${success_source}/loose-data" "$success_output" "$success_state" printf 'data\n' > "${success_source}/app-ns/file.txt" printf 'data\n' > "${success_source}/loose-data/file.txt" printf 'app-ns\n' > "${TMPD}/namespaces.txt" printf 'app-ns:3\n' > "${TMPD}/replicas.txt" : > "${TMPD}/fail_folders.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" : > "${TMPD}/curl_trace.log" set +e run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log" success_exit_code=$? set -e assert "success scenario exits with code 0" is_zero "$success_exit_code" assert "success scenario creates a test archive" archive_exists "$success_output" assert "success scenario reports two backup successes" file_contains "${success_dir}/success.log" "backup_successes=2" assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0" assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0" assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3" assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success" assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data" assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns" assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data" # Scenario B: one namespace backup fails, but both namespaces are restored and # the script exits through its documented handled-failure path. failure_dir="${TMPD}/failure" failure_source="${failure_dir}/source" failure_output="${failure_dir}/out" failure_state="${failure_dir}/state" mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state" printf 'data\n' > "${failure_source}/good-ns/file.txt" printf 'data\n' > "${failure_source}/fail-ns/file.txt" printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt" printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt" printf 'fail-ns\n' > "${TMPD}/fail_folders.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" : > "${TMPD}/curl_trace.log" set +e run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log" failure_exit_code=$? set -e assert "failure scenario exits non-zero" is_nonzero "$failure_exit_code" assert "failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1" assert "failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1" assert "failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0" assert "failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2" assert "failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0" assert "failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4" assert "failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure" assert "failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns" assert "failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns" # Scenario C: abnormal abort while a namespace is still scaled to zero. # We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep` # errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its # .state file is written, but BEFORE the in-line restore. The EXIT trap must then # drive restore_all_remaining() and scale the namespace back up. This exercises # the real safety-net path that scenario B leaves unproven. abort_dir="${TMPD}/abort" abort_source="${abort_dir}/source" abort_output="${abort_dir}/out" abort_state="${abort_dir}/state" mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" printf 'data\n' > "${abort_source}/crash-ns/file.txt" printf 'crash-ns\n' > "${TMPD}/namespaces.txt" printf 'crash-ns:5\n' > "${TMPD}/replicas.txt" : > "${TMPD}/fail_folders.txt" : > "${TMPD}/scale_trace.log" : > "${TMPD}/7z_trace.log" : > "${TMPD}/curl_trace.log" : > "${TMPD}/7z_invocations.log" set +e NFS_SOURCE_PATH="${abort_source}" \ BACKUP_OUTPUT_PATH="${abort_output}" \ BACKUP_PASSWORD="" \ ARCHIVE_PREFIX="test" \ NOTIFY_SUCCESS_URL="" \ NOTIFY_FAILURE_URL="" \ KUBECTL_BIN="${TMPD}/bin/kubectl" \ SEVENZ_BIN="${TMPD}/bin/7z" \ SCALE_WAIT_SECONDS="not-a-number" \ TMP_STATE_DIR="${abort_state}" \ PATH="${TMPD}/bin:${PATH}" \ TMPD="${TMPD}" \ bash "${SCRIPT_UNDER_TEST}" >"${abort_dir}/abort.log" 2>&1 abort_exit_code=$? set -e assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}" assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0" assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5" assert "abort scenario left no stale .state file" file_lacks "${abort_dir}/abort.log" ".state" printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total" if [[ "$tests_failed" -gt 0 ]]; then exit 1 fi