refactor: make NFS backup script testable with minimal config vars, add test harness + Gitea workflow, address review nits

.gitea/workflows/test-automated-nfs-backup.yaml

@@ -0,0 +1,17 @@
+name: Test automated-nfs-backup
+
+on:
+  push:
+    paths:
+      - 'k8s/**'
+      - '.gitea/workflows/**'
+  workflow_dispatch: {}
+
+jobs:
+  test-automated-nfs-backup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Run automated NFS backup tests
+        run: bash k8s/test/test-automated-nfs-backup.sh

k8s/automated-nfs-backup.sh

@@ -25,9 +25,11 @@ SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
 SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}"
 SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}"
 SEVENZ_THREADS="${SEVENZ_THREADS:-on}"
+SEVENZ_BIN="${SEVENZ_BIN:-7z}"
 SCALE_TIMEOUT_SECONDS="${SCALE_TIMEOUT_SECONDS:-600}"
 SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
 SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
+SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
 TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}"
 NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}"
 NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}"
@@ -75,7 +77,7 @@ parse_workload_kinds() {
 
 validate_inputs() {
     require_cmd "$KUBECTL_BIN"
-    require_cmd "7z"
+    require_cmd "$SEVENZ_BIN"
     if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
         require_cmd "curl"
     fi
@@ -323,7 +325,7 @@ backup_folder() {
     local folder_path="${1:?folder path required}"
     local archive_path="${2:?archive path required}"
     local -a cmd=(
-        7z a -t7z
+        "$SEVENZ_BIN" a -t7z
         "$archive_path"
         "$folder_path"
         "-m0=${SEVENZ_METHOD}"
@@ -405,8 +407,8 @@ process_folder() {
         capture_replicas_state "$namespace" "$state_file"
         scale_namespace_to_zero "$namespace" "$state_file"
 
-        log_info "Waiting 30 seconds for namespace '${namespace}' to scale down..."
-        sleep 30
+        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
+        sleep "$SCALE_WAIT_SECONDS"
     else
         unmapped_folders=$((unmapped_folders + 1))
         log_warn "No namespace matched folder '${folder_name}'. Running backup only."
@@ -422,8 +424,8 @@ process_folder() {
 
     if [[ "$has_mapping" -eq 1 ]]; then
         restore_namespace_replicas "$namespace" "$state_file"
-        log_info "Waiting 30 seconds for namespace '${namespace}' to restore replicas..."
-        sleep 30
+        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
+        sleep "$SCALE_WAIT_SECONDS"
         rm -f "$state_file"
     fi
 }

k8s/test/test-automated-nfs-backup.sh

@@ -0,0 +1,349 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/automated-nfs-backup.sh"
+TMPD="$(mktemp -d "${TMPDIR:-/tmp}/automated-nfs-backup-test.XXXXXX")"
+
+tests_total=0
+tests_passed=0
+tests_failed=0
+
+cleanup() {
+    rm -rf "${TMPD}"
+}
+trap cleanup EXIT
+
+assert() {
+    local message="$1"
+    shift
+
+    tests_total=$((tests_total + 1))
+    if "$@"; then
+        printf 'PASS: %s\n' "$message"
+        tests_passed=$((tests_passed + 1))
+    else
+        printf 'FAIL: %s\n' "$message"
+        tests_failed=$((tests_failed + 1))
+    fi
+}
+
+is_zero() {
+    [[ "$1" -eq 0 ]]
+}
+
+is_nonzero() {
+    [[ "$1" -ne 0 ]]
+}
+
+file_contains() {
+    local file="$1"
+    local value="$2"
+    grep -Fq -- "$value" "$file"
+}
+
+file_lacks() {
+    local file="$1"
+    local value="$2"
+    ! grep -Fq -- "$value" "$file"
+}
+
+file_matches() {
+    local file="$1"
+    local regex="$2"
+    grep -Eq -- "$regex" "$file"
+}
+
+archive_exists() {
+    local output_dir="$1"
+    local -a archives=()
+
+    shopt -s nullglob
+    archives=("${output_dir}"/test_*.7z)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -gt 0 ]]
+}
+
+trace_has_scale() {
+    local trace_file="$1"
+    local namespace="$2"
+    local replicas="$3"
+
+    grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file"
+}
+
+# Asserts the real script invoked 7z as `7z a -t7z <archive.7z> <folder> ...`
+# for the given folder. The archive is the 3rd token (ends in .7z) and the
+# folder path is the 4th token, so a swap would fail to match.
+sevenz_called_for() {
+    local folder_name="$1"
+
+    grep -Eq -- "a -t7z [^ ]+\\.7z .*/${folder_name}( |$)" "${TMPD}/7z_invocations.log"
+}
+
+write_fake_script() {
+    local path="$1"
+
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        printf '%s\n' "$line"
+    done > "$path"
+    chmod +x "$path"
+}
+
+write_fake_binaries() {
+    mkdir -p "${TMPD}/bin"
+
+    write_fake_script "${TMPD}/bin/kubectl" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+namespace=""
+get_kind=""
+has_output_format=0
+
+for ((i = 1; i <= $#; i++)); do
+    arg="${!i}"
+    case "$arg" in
+        -n)
+            next=$((i + 1))
+            namespace="${!next}"
+            ;;
+        get)
+            next=$((i + 1))
+            get_kind="${!next}"
+            ;;
+        -o)
+            has_output_format=1
+            ;;
+    esac
+done
+
+if [[ "$get_kind" == "namespaces" ]]; then
+    if [[ "$has_output_format" -eq 1 ]]; then
+        while IFS= read -r configured_namespace; do
+            [[ -n "$configured_namespace" ]] && printf '%s\n' "$configured_namespace"
+        done < "${TMPD}/namespaces.txt"
+    fi
+    exit 0
+fi
+
+if [[ -n "$get_kind" && -n "$namespace" ]]; then
+    replicas=""
+    while IFS=: read -r configured_namespace configured_replicas; do
+        if [[ "$configured_namespace" == "$namespace" ]]; then
+            replicas="$configured_replicas"
+            break
+        fi
+    done < "${TMPD}/replicas.txt"
+
+    [[ -n "$replicas" ]] || replicas=1
+    printf 'web\t%s\t\n' "$replicas"
+    exit 0
+fi
+
+for ((i = 1; i <= $#; i++)); do
+    if [[ "${!i}" == "scale" ]]; then
+        resource_index=$((i + 1))
+        resource="${!resource_index}"
+        kind="${resource%%/*}"
+        name="${resource#*/}"
+        replicas_arg=""
+        for arg in "$@"; do
+            [[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
+        done
+        printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
+        exit 0
+    fi
+done
+
+exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/7z" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+# Record the full invocation so the harness can assert argument order.
+printf '%s\n' "$*" >> "${TMPD}/7z_invocations.log"
+
+if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
+    archive="${3:?archive path required}"
+    folder="${4:?folder path required}"
+    folder="${folder%/}"
+    folder_name="${folder##*/}"
+
+    # Simulate a backup failure for configured folders BEFORE recording a
+    # successful archive, so 7z_trace.log only lists folders actually archived.
+    if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
+        exit 2
+    fi
+
+    printf '%s\n' "$folder_name" >> "${TMPD}/7z_trace.log"
+    touch "$archive"
+fi
+
+exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+url=""
+body=""
+for ((i = 1; i <= $#; i++)); do
+    arg="${!i}"
+    case "$arg" in
+        POST)
+            next=$((i + 1))
+            url="${!next}"
+            ;;
+        -d)
+            next=$((i + 1))
+            body="${!next}"
+            ;;
+    esac
+done
+
+body="${body//$'\n'/ }"
+printf '%s %s\n' "$url" "${body:0:120}" >> "${TMPD}/curl_trace.log"
+exit 0
+EOF
+}
+
+run_backup() {
+    local source_dir="$1"
+    local output_dir="$2"
+    local state_dir="$3"
+    local log_file="$4"
+
+    NFS_SOURCE_PATH="$source_dir" \
+    BACKUP_OUTPUT_PATH="$output_dir" \
+    BACKUP_PASSWORD="" \
+    ARCHIVE_PREFIX="test" \
+    NOTIFY_SUCCESS_URL="http://example.invalid/success" \
+    NOTIFY_FAILURE_URL="http://example.invalid/failure" \
+    KUBECTL_BIN="${TMPD}/bin/kubectl" \
+    SEVENZ_BIN="${TMPD}/bin/7z" \
+    SCALE_WAIT_SECONDS=0 \
+    TMP_STATE_DIR="$state_dir" \
+    PATH="${TMPD}/bin:${PATH}" \
+    TMPD="$TMPD" \
+    bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
+}
+
+write_fake_binaries
+export TMPD
+
+# Scenario A: a mapped folder is scaled down and restored, while an unmapped
+# folder is backed up directly. Both folders share one archive.
+success_dir="${TMPD}/success"
+success_source="${success_dir}/source"
+success_output="${success_dir}/out"
+success_state="${success_dir}/state"
+mkdir -p "${success_source}/app-ns" "${success_source}/loose-data" "$success_output" "$success_state"
+printf 'data\n' > "${success_source}/app-ns/file.txt"
+printf 'data\n' > "${success_source}/loose-data/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log"
+success_exit_code=$?
+set -e
+
+assert "success scenario exits with code 0" is_zero "$success_exit_code"
+assert "success scenario creates a test archive" archive_exists "$success_output"
+assert "success scenario reports two backup successes" file_contains "${success_dir}/success.log" "backup_successes=2"
+assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
+assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
+assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
+assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
+assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
+assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns"
+assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data"
+
+# Scenario B: one namespace backup fails, but both namespaces are restored and
+# the script exits through its documented handled-failure path.
+failure_dir="${TMPD}/failure"
+failure_source="${failure_dir}/source"
+failure_output="${failure_dir}/out"
+failure_state="${failure_dir}/state"
+mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state"
+printf 'data\n' > "${failure_source}/good-ns/file.txt"
+printf 'data\n' > "${failure_source}/fail-ns/file.txt"
+printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt"
+printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
+printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log"
+failure_exit_code=$?
+set -e
+
+assert "failure scenario exits non-zero" is_nonzero "$failure_exit_code"
+assert "failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
+assert "failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
+assert "failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
+assert "failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
+assert "failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
+assert "failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
+assert "failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
+assert "failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
+assert "failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
+
+# Scenario C: abnormal abort while a namespace is still scaled to zero.
+# We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
+# errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its
+# .state file is written, but BEFORE the in-line restore. The EXIT trap must then
+# drive restore_all_remaining() and scale the namespace back up. This exercises
+# the real safety-net path that scenario B leaves unproven.
+abort_dir="${TMPD}/abort"
+abort_source="${abort_dir}/source"
+abort_output="${abort_dir}/out"
+abort_state="${abort_dir}/state"
+mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}"
+printf 'data\n' > "${abort_source}/crash-ns/file.txt"
+printf 'crash-ns\n' > "${TMPD}/namespaces.txt"
+printf 'crash-ns:5\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/7z_invocations.log"
+
+set +e
+NFS_SOURCE_PATH="${abort_source}" \
+BACKUP_OUTPUT_PATH="${abort_output}" \
+BACKUP_PASSWORD="" \
+ARCHIVE_PREFIX="test" \
+NOTIFY_SUCCESS_URL="" \
+NOTIFY_FAILURE_URL="" \
+KUBECTL_BIN="${TMPD}/bin/kubectl" \
+SEVENZ_BIN="${TMPD}/bin/7z" \
+SCALE_WAIT_SECONDS="not-a-number" \
+TMP_STATE_DIR="${abort_state}" \
+PATH="${TMPD}/bin:${PATH}" \
+TMPD="${TMPD}" \
+bash "${SCRIPT_UNDER_TEST}" >"${abort_dir}/abort.log" 2>&1
+abort_exit_code=$?
+set -e
+
+assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}"
+assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0"
+assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5"
+assert "abort scenario left no stale .state file" file_lacks "${abort_dir}/abort.log" ".state"
+
+printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
+if [[ "$tests_failed" -gt 0 ]]; then
+    exit 1
+fi