427 lines
18 KiB
Bash
Executable File
427 lines
18 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/backup-k3s-control-plane-config.sh"
|
|
TMPD="$(mktemp -d "${TMPDIR:-/tmp}/k3s-control-plane-config-test.XXXXXX")"
|
|
|
|
tests_total=0
|
|
tests_passed=0
|
|
tests_failed=0
|
|
|
|
cleanup() {
|
|
rm -rf -- "$TMPD"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
assert() {
|
|
local message="$1"
|
|
shift
|
|
tests_total=$((tests_total + 1))
|
|
if "$@"; then
|
|
printf 'PASS: %s\n' "$message"
|
|
tests_passed=$((tests_passed + 1))
|
|
else
|
|
printf 'FAIL: %s\n' "$message"
|
|
tests_failed=$((tests_failed + 1))
|
|
fi
|
|
}
|
|
|
|
is_zero() { [[ "$1" -eq 0 ]]; }
|
|
is_nonzero() { [[ "$1" -ne 0 ]]; }
|
|
path_exists() { [[ -e "$1" ]]; }
|
|
path_does_not_exist() { [[ ! -e "$1" ]]; }
|
|
file_contains() { grep -Fq -- "$2" "$1"; }
|
|
file_lacks() { [[ ! -e "$1" ]] || ! grep -Fq -- "$2" "$1"; }
|
|
file_is_empty() { [[ ! -s "$1" ]]; }
|
|
file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; }
|
|
|
|
matching_archive_count_is() {
|
|
local directory="$1"
|
|
local expected="$2"
|
|
local -a archives=()
|
|
shopt -s nullglob
|
|
archives=("${directory}"/k3s-control-plane-config_*.zip)
|
|
shopt -u nullglob
|
|
[[ "${#archives[@]}" -eq "$expected" ]]
|
|
}
|
|
|
|
directory_is_empty() {
|
|
local directory="$1"
|
|
local -a entries=()
|
|
shopt -s nullglob dotglob
|
|
entries=("${directory}"/*)
|
|
shopt -u nullglob dotglob
|
|
[[ "${#entries[@]}" -eq 0 ]]
|
|
}
|
|
|
|
write_fake_script() {
|
|
local path="$1"
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
printf '%s\n' "$line"
|
|
done > "$path"
|
|
chmod +x "$path"
|
|
}
|
|
|
|
write_fake_binaries() {
|
|
mkdir -p "${TMPD}/bin"
|
|
|
|
write_fake_script "${TMPD}/bin/zip" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
[[ "${1:-}" == "-r" ]]
|
|
archive="${2:?archive path required}"
|
|
shift 2
|
|
[[ "${1:-}" == "--" ]]
|
|
shift
|
|
: > "$archive"
|
|
for entry in "$@"; do
|
|
source_path="/$entry"
|
|
if [[ -d "$source_path" ]]; then
|
|
while IFS= read -r path; do
|
|
printf '%s\n' "${path#/}" >> "$archive"
|
|
done < <(find "$source_path" -print | sort)
|
|
else
|
|
printf '%s\n' "$entry" >> "$archive"
|
|
fi
|
|
done
|
|
printf '%s\n' "$*" >> "${TMPD}/zip_invocations.log"
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/unzip" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
[[ "${1:-}" == "-t" ]]
|
|
archive="${2:?archive path required}"
|
|
[[ -f "$archive" && -s "$archive" ]]
|
|
printf '%s\n' "$archive" >> "${TMPD}/unzip_trace.log"
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/scp" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
source_path="${@: -2:1}"
|
|
destination="${@: -1}"
|
|
remote_path="${destination#*:}"
|
|
if [[ "$remote_path" == *"'"* ]]; then
|
|
printf 'scp destination contains literal single quotes: %s\n' "$destination" >&2
|
|
exit 64
|
|
fi
|
|
remote_name="${remote_path##*/}"
|
|
target_path="${FAKE_REMOTE_ROOT}/${remote_name}"
|
|
|
|
cp -- "$source_path" "$target_path"
|
|
printf '%s\n' "$remote_name" >> "${TMPD}/scp_trace.log"
|
|
if [[ -f "${TMPD}/fail_scp" ]]; then
|
|
exit 42
|
|
fi
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/ssh" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
command="${@: -1}"
|
|
printf '%s\n' "$command" >> "${TMPD}/ssh_trace.log"
|
|
|
|
quoted_values=()
|
|
while IFS= read -r value; do
|
|
quoted_values+=("${value#\'}")
|
|
quoted_values[${#quoted_values[@]}-1]="${quoted_values[${#quoted_values[@]}-1]%\'}"
|
|
done < <(printf '%s\n' "$command" | grep -o "'[^']*'" || true)
|
|
|
|
if [[ "$command" == mkdir\ -p* ]]; then
|
|
mkdir -p -- "$FAKE_REMOTE_ROOT"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$command" == rm\ -f* ]]; then
|
|
temp_name="${quoted_values[0]##*/}"
|
|
rm -f -- "${FAKE_REMOTE_ROOT}/${temp_name}"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$command" == *" mv -- "* ]]; then
|
|
value_count="${#quoted_values[@]}"
|
|
temp_name="${quoted_values[value_count-2]##*/}"
|
|
final_name="${quoted_values[value_count-1]##*/}"
|
|
[[ ! -e "${FAKE_REMOTE_ROOT}/${final_name}" ]]
|
|
mv -- "${FAKE_REMOTE_ROOT}/${temp_name}" "${FAKE_REMOTE_ROOT}/${final_name}"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$command" == *" -s -- "* ]]; then
|
|
value_count="${#quoted_values[@]}"
|
|
(
|
|
cd -- "$FAKE_REMOTE_ROOT"
|
|
bash -s -- "$FAKE_REMOTE_ROOT" "${quoted_values[value_count-2]}" "${quoted_values[value_count-1]}"
|
|
)
|
|
exit 0
|
|
fi
|
|
|
|
exit 1
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/curl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
url=""
|
|
for ((i = 1; i <= $#; i++)); do
|
|
if [[ "${!i}" == "POST" ]]; then
|
|
next=$((i + 1))
|
|
url="${!next}"
|
|
fi
|
|
done
|
|
printf '%s\n' "$url" >> "${TMPD}/curl_trace.log"
|
|
[[ ! -f "${TMPD}/fail_curl" ]]
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/flock" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
exit 0
|
|
EOF
|
|
|
|
write_fake_script "${TMPD}/bin/date" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
printf '%s\n' '2026-07-21_12-00-00'
|
|
EOF
|
|
}
|
|
|
|
run_backup() {
|
|
local fixture_root="$1"
|
|
local remote_root="$2"
|
|
local local_temp_parent="$3"
|
|
local log_file="$4"
|
|
shift 4
|
|
local -a overrides=("$@")
|
|
|
|
env \
|
|
K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \
|
|
K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \
|
|
K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \
|
|
LOCAL_TMP_PARENT="$local_temp_parent" \
|
|
LOCK_PATH="${fixture_root}/lock/k3s-control-plane-config.lock" \
|
|
REMOTE_USER="test-user" \
|
|
REMOTE_HOST="test-host" \
|
|
REMOTE_DIR="/backup" \
|
|
ZIP_BIN="${TMPD}/bin/zip" \
|
|
UNZIP_BIN="${TMPD}/bin/unzip" \
|
|
SSH_BIN="${TMPD}/bin/ssh" \
|
|
SCP_BIN="${TMPD}/bin/scp" \
|
|
CURL_BIN="${TMPD}/bin/curl" \
|
|
FLOCK_BIN="${TMPD}/bin/flock" \
|
|
DATE_BIN="${TMPD}/bin/date" \
|
|
NOTIFY_SUCCESS_URL="http://example.invalid/success" \
|
|
NOTIFY_FAILURE_URL="http://example.invalid/failure" \
|
|
PATH="${TMPD}/bin:${PATH}" \
|
|
TMPD="$TMPD" \
|
|
FAKE_REMOTE_ROOT="$remote_root" \
|
|
"${overrides[@]}" \
|
|
bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
|
|
}
|
|
|
|
create_source_fixture() {
|
|
local fixture_root="$1"
|
|
mkdir -p \
|
|
"${fixture_root}/etc/rancher/k3s" \
|
|
"${fixture_root}/etc/systemd/system" \
|
|
"${fixture_root}/etc/default" \
|
|
"${fixture_root}/etc/sysconfig" \
|
|
"${fixture_root}/var/lib/rancher/k3s/server/cred" \
|
|
"${fixture_root}/var/lib/rancher/k3s/server/etc" \
|
|
"${fixture_root}/var/lib/rancher/k3s/server/manifests" \
|
|
"${fixture_root}/var/lib/rancher/k3s/server/static" \
|
|
"${fixture_root}/var/lib/rancher/k3s/server/tls" \
|
|
"${fixture_root}/var/lib/rancher/k3s/server/db" \
|
|
"${fixture_root}/lock"
|
|
printf 'config\n' > "${fixture_root}/etc/rancher/k3s/config.yaml"
|
|
printf 'service\n' > "${fixture_root}/etc/systemd/system/k3s.service"
|
|
printf 'environment\n' > "${fixture_root}/etc/systemd/system/k3s.service.env"
|
|
printf 'default\n' > "${fixture_root}/etc/default/k3s"
|
|
printf 'sysconfig\n' > "${fixture_root}/etc/sysconfig/k3s"
|
|
printf 'credential\n' > "${fixture_root}/var/lib/rancher/k3s/server/cred/passwd"
|
|
printf 'state\n' > "${fixture_root}/var/lib/rancher/k3s/server/etc/config"
|
|
printf 'manifest\n' > "${fixture_root}/var/lib/rancher/k3s/server/manifests/custom.yaml"
|
|
printf 'static\n' > "${fixture_root}/var/lib/rancher/k3s/server/static/pod.yaml"
|
|
printf 'tls\n' > "${fixture_root}/var/lib/rancher/k3s/server/tls/server-ca.crt"
|
|
printf 'token\n' > "${fixture_root}/var/lib/rancher/k3s/server/token"
|
|
printf 'agent-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/agent-token"
|
|
printf 'node-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/node-token"
|
|
printf 'database must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/db/state.db"
|
|
printf 'socket must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/kine.sock"
|
|
}
|
|
|
|
write_fake_binaries
|
|
export TMPD
|
|
|
|
# Successful publication: source layout is preserved relative to /, unzip is
|
|
# called, and retention deletes only old archives with this archive prefix.
|
|
success_dir="${TMPD}/success"
|
|
success_fixture="${success_dir}/fixture"
|
|
success_remote="${success_dir}/remote"
|
|
success_local_tmp="${success_dir}/local-tmp"
|
|
mkdir -p "$success_remote" "$success_local_tmp"
|
|
create_source_fixture "$success_fixture"
|
|
for day in 1 2 3 4 5; do
|
|
touch -t "2026070${day}1200" "${success_remote}/k3s-control-plane-config_2026-07-0${day}_12-00-00.zip"
|
|
done
|
|
printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z"
|
|
printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip"
|
|
newline_archive="${success_remote}/k3s-control-plane-config_2026-06-01_12-00-00.zip"$'\nunrelated.zip'
|
|
touch -t "202606011200" "$newline_archive"
|
|
: > "$newline_archive"
|
|
touch -t "202606011200" "$newline_archive"
|
|
tab_archive="${success_remote}/k3s-control-plane-config_2026-05-01_12-00-00.zip"$'\tlegacy.zip'
|
|
: > "$tab_archive"
|
|
touch -t "202605011200" "$tab_archive"
|
|
: > "${success_remote}/unrelated.zip"
|
|
: > "${TMPD}/curl_trace.log"
|
|
: > "${TMPD}/unzip_trace.log"
|
|
: > "${TMPD}/zip_invocations.log"
|
|
: > "${TMPD}/scp_trace.log"
|
|
: > "${TMPD}/ssh_trace.log"
|
|
|
|
set +e
|
|
run_backup "$success_fixture" "$success_remote" "$success_local_tmp" "${success_dir}/run.log"
|
|
success_exit_code=$?
|
|
set -e
|
|
|
|
fixture_relative="${success_fixture#/}"
|
|
assert "successful backup exits with code 0" is_zero "$success_exit_code"
|
|
assert "successful backup publishes the final zip archive" path_exists "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
|
assert "successful backup validates its archive with unzip" file_contains "${TMPD}/unzip_trace.log" "k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
|
assert "archive contains the k3s configuration path" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/rancher/k3s/config.yaml"
|
|
assert "archive contains a service configuration file" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/systemd/system/k3s.service"
|
|
assert "archive contains control-plane credentials" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/var/lib/rancher/k3s/server/cred/passwd"
|
|
assert "archive excludes server database content" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "server/db/state.db"
|
|
assert "archive excludes kine socket" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "kine.sock"
|
|
assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4
|
|
assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip"
|
|
assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip"
|
|
assert "retention removes matching old archives with newlines in their filenames" path_does_not_exist "$newline_archive"
|
|
assert "retention removes matching old archives with tabs in their filenames" path_does_not_exist "$tab_archive"
|
|
assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${success_remote}/unrelated.zip"
|
|
assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
|
|
assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
|
|
assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
|
|
assert "successful backup sends a success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
|
|
|
# A configured source path ending in server/db/.. must be rejected before zip is
|
|
# invoked, rather than recursively archiving server/db and its sibling socket.
|
|
source_scope_dir="${TMPD}/source-scope"
|
|
source_scope_fixture="${source_scope_dir}/fixture"
|
|
source_scope_remote="${source_scope_dir}/remote"
|
|
source_scope_local_tmp="${source_scope_dir}/local-tmp"
|
|
source_scope_archive="${source_scope_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
|
mkdir -p "$source_scope_remote" "$source_scope_local_tmp"
|
|
create_source_fixture "$source_scope_fixture"
|
|
: > "${TMPD}/zip_invocations.log"
|
|
|
|
set +e
|
|
run_backup \
|
|
"$source_scope_fixture" \
|
|
"$source_scope_remote" \
|
|
"$source_scope_local_tmp" \
|
|
"${source_scope_dir}/run.log" \
|
|
"K3S_CONFIG_DIR=${source_scope_fixture}/var/lib/rancher/k3s/server/db/.."
|
|
source_scope_exit_code=$?
|
|
set -e
|
|
|
|
assert "unsafe source override exits non-zero" is_nonzero "$source_scope_exit_code"
|
|
assert "unsafe source override reports the source-scope error" file_contains "${source_scope_dir}/run.log" "Configured source path is not safe to archive"
|
|
assert "unsafe source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log"
|
|
assert "unsafe source override archive excludes server database content" file_lacks "$source_scope_archive" "state.db"
|
|
assert "unsafe source override archive excludes kine socket" file_lacks "$source_scope_archive" "kine.sock"
|
|
assert "unsafe source override does not publish an archive" directory_is_empty "$source_scope_remote"
|
|
assert "unsafe source override removes local temporary state" directory_is_empty "$source_scope_local_tmp"
|
|
|
|
# The source path and the server directory can be overridden independently;
|
|
# protected database paths must still be derived from the configured source.
|
|
source_scope_mismatch_dir="${TMPD}/source-scope-mismatch"
|
|
source_scope_mismatch_fixture="${source_scope_mismatch_dir}/fixture"
|
|
source_scope_mismatch_remote="${source_scope_mismatch_dir}/remote"
|
|
source_scope_mismatch_local_tmp="${source_scope_mismatch_dir}/local-tmp"
|
|
source_scope_mismatch_unrelated_server="${source_scope_mismatch_fixture}/unrelated-server"
|
|
source_scope_mismatch_archive="${source_scope_mismatch_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
|
mkdir -p "$source_scope_mismatch_remote" "$source_scope_mismatch_local_tmp"
|
|
create_source_fixture "$source_scope_mismatch_fixture"
|
|
mkdir -p "$source_scope_mismatch_unrelated_server"
|
|
: > "${TMPD}/zip_invocations.log"
|
|
|
|
set +e
|
|
run_backup \
|
|
"$source_scope_mismatch_fixture" \
|
|
"$source_scope_mismatch_remote" \
|
|
"$source_scope_mismatch_local_tmp" \
|
|
"${source_scope_mismatch_dir}/run.log" \
|
|
"K3S_CONFIG_DIR=${source_scope_mismatch_fixture}/var/lib/rancher/k3s/server/db/.." \
|
|
"K3S_SERVER_DIR=${source_scope_mismatch_unrelated_server}"
|
|
source_scope_mismatch_exit_code=$?
|
|
set -e
|
|
|
|
assert "mismatched source override exits non-zero" is_nonzero "$source_scope_mismatch_exit_code"
|
|
assert "mismatched source override reports the source-scope error" file_contains "${source_scope_mismatch_dir}/run.log" "Configured source path is not safe to archive"
|
|
assert "mismatched source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log"
|
|
assert "mismatched source override archive excludes server database content" file_lacks "$source_scope_mismatch_archive" "state.db"
|
|
assert "mismatched source override archive excludes kine socket" file_lacks "$source_scope_mismatch_archive" "kine.sock"
|
|
assert "mismatched source override does not publish an archive" path_does_not_exist "$source_scope_mismatch_archive"
|
|
assert "mismatched source override leaves the remote directory empty" directory_is_empty "$source_scope_mismatch_remote"
|
|
assert "mismatched source override removes local temporary state" directory_is_empty "$source_scope_mismatch_local_tmp"
|
|
|
|
# Upload failure: the partial remote temporary file is removed, no final archive
|
|
# is published, local temporary data is removed, and one safe error notification
|
|
# is attempted.
|
|
failure_dir="${TMPD}/upload-failure"
|
|
failure_fixture="${failure_dir}/fixture"
|
|
failure_remote="${failure_dir}/remote"
|
|
failure_local_tmp="${failure_dir}/local-tmp"
|
|
mkdir -p "$failure_remote" "$failure_local_tmp"
|
|
create_source_fixture "$failure_fixture"
|
|
: > "${TMPD}/fail_scp"
|
|
: > "${TMPD}/curl_trace.log"
|
|
|
|
set +e
|
|
run_backup "$failure_fixture" "$failure_remote" "$failure_local_tmp" "${failure_dir}/run.log"
|
|
failure_exit_code=$?
|
|
set -e
|
|
rm -f -- "${TMPD}/fail_scp"
|
|
|
|
assert "upload failure exits non-zero" is_nonzero "$failure_exit_code"
|
|
assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
|
assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
|
|
assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
|
|
assert "upload failure attempts exactly one error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/failure" 1
|
|
assert "upload failure does not send a success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
|
|
assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
|
|
|
|
# A notification error is non-fatal after a completed backup.
|
|
notify_dir="${TMPD}/notify-failure"
|
|
notify_fixture="${notify_dir}/fixture"
|
|
notify_remote="${notify_dir}/remote"
|
|
notify_local_tmp="${notify_dir}/local-tmp"
|
|
mkdir -p "$notify_remote" "$notify_local_tmp"
|
|
create_source_fixture "$notify_fixture"
|
|
: > "${TMPD}/fail_curl"
|
|
|
|
set +e
|
|
run_backup "$notify_fixture" "$notify_remote" "$notify_local_tmp" "${notify_dir}/run.log" "CLEANUP_KEEP_COUNT=0"
|
|
notify_exit_code=$?
|
|
set -e
|
|
rm -f -- "${TMPD}/fail_curl"
|
|
|
|
assert "failed success notification does not fail a completed backup" is_zero "$notify_exit_code"
|
|
assert "completed backup remains published after notification failure" path_exists "${notify_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
|
|
|
|
printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
|
|
if (( tests_failed > 0 )); then
|
|
exit 1
|
|
fi
|