diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..dcb1f5c --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,77 @@ +# K8s Script Pack + +Kubernetes-focused shell scripts intended for cronjobs and operational utilities. + +## Scripts + +### `automated-nfs-backup.sh` + +Backs up each top-level folder found in `NFS_SOURCE_PATH` into an encrypted `.7z` archive, with optional Kubernetes workload quiescing when a folder name exactly matches a namespace name. + +Behavior: +- Exact folder-to-namespace mapping only. +- Unmapped folder: backup still runs, Kubernetes scale actions are skipped. +- Mapped folder: saves replicas, scales selected workloads down, waits, runs backup, restores replicas, waits again. +- Scale-down issues are warnings by policy (backup still runs). +- Restore issues are warnings by policy (run can still complete successfully). + +## Environment Variables + +Required: +- `NFS_SOURCE_PATH`: Root path containing folders to back up. +- `BACKUP_OUTPUT_PATH`: Destination path for generated `.7z` archives. +- `BACKUP_PASSWORD`: Password used for 7z encryption. + +Optional: +- `KUBECTL_BIN` (default: `kubectl`) +- `KUBE_CONTEXT` (default: empty) +- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`) +- `ARCHIVE_PREFIX` (default: `nfs-backup`) +- `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`) +- `SEVENZ_METHOD` (default: `lzma2`) +- `SEVENZ_LEVEL` (default: `9`) +- `SEVENZ_HEADER_ENCRYPT` (default: `on`) +- `SEVENZ_THREADS` (default: `on`) +- `SCALE_TIMEOUT_SECONDS` (default: `600`) +- `SCALE_RETRY_COUNT` (default: `3`) +- `SCALE_RETRY_DELAY_SECONDS` (default: `5`) +- `LOG_LEVEL` (default: `info`) +- `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`) +- `NOTIFY_SUCCESS_URL` (default: empty, disabled) +- `NOTIFY_FAILURE_URL` (default: empty, disabled) +- `NOTIFY_TITLE` (default: `Kubernetes`) +- `NOTIFY_ASSET` (default: `kube config`) + +Notification payload (success and failure): + +```json +{ + "title": "Kubernetes", + "asset": "kube config", + "backupSizeInMB": 123 +} +``` + +## Cronjob Notes + +- Script is designed to run sequentially (one folder at a time). +- Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces. +- Ensure `kubectl` context and credentials are present in the runtime. +- Ensure `7z` is installed in the runtime image/host. + +## Failure Semantics + +- Missing required env vars, missing commands, invalid paths, or inability to list namespaces: script exits non-zero immediately. +- Folder backup failures: counted and script exits non-zero at end. +- Scale-down warnings/timeouts: logged and counted, backup continues. +- Restore warnings/timeouts: logged and counted, script does not fail solely because of restore warnings. +- If `NOTIFY_SUCCESS_URL` is set, success notification is sent at the end of a successful run. +- If `NOTIFY_FAILURE_URL` is set, failure notification is sent when backup failures are detected. +- Final summary always logs: + - `processed` + - `mapped` + - `unmapped` + - `backup_successes` + - `backup_failures` + - `scale_warnings` + - `restore_warnings` diff --git a/k8s/automated-nfs-backup.sh b/k8s/automated-nfs-backup.sh new file mode 100644 index 0000000..5b9f72c --- /dev/null +++ b/k8s/automated-nfs-backup.sh @@ -0,0 +1,420 @@ +#!/usr/bin/env bash + +SCRIPT_NAME="$(basename "$0")" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" +setup_exit_handling + +# Required configuration +require_env "NFS_SOURCE_PATH" +require_env "BACKUP_OUTPUT_PATH" +require_env "BACKUP_PASSWORD" + +# Optional configuration +KUBECTL_BIN="${KUBECTL_BIN:-kubectl}" +KUBE_CONTEXT="${KUBE_CONTEXT:-}" +WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset,replicationcontroller}" +ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}" +ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}" +SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}" +SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}" +SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}" +SEVENZ_THREADS="${SEVENZ_THREADS:-on}" +SCALE_TIMEOUT_SECONDS="${SCALE_TIMEOUT_SECONDS:-600}" +SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}" +SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}" +TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}" +NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}" +NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}" +NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}" +NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}" + +KUBECTL_ARGS=() +if [[ -n "$KUBE_CONTEXT" ]]; then + KUBECTL_ARGS+=(--context "$KUBE_CONTEXT") +fi + +declare -a WORKLOAD_KIND_LIST=() +declare -A NAMESPACE_MAP=() + +total_folders=0 +mapped_folders=0 +unmapped_folders=0 +backup_successes=0 +backup_failures=0 +scale_warnings=0 +restore_warnings=0 +total_backup_size_bytes=0 + +_kubectl() { + "$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" +} + +parse_workload_kinds() { + local raw + local cleaned + IFS=',' read -r -a raw <<< "$WORKLOAD_KINDS" + for kind in "${raw[@]}"; do + cleaned="$(trim "${kind,,}")" + if [[ -n "$cleaned" ]]; then + WORKLOAD_KIND_LIST+=("$cleaned") + fi + done + + if [[ "${#WORKLOAD_KIND_LIST[@]}" -eq 0 ]]; then + die "WORKLOAD_KINDS resolved to an empty list" + fi +} + +validate_inputs() { + require_cmd "$KUBECTL_BIN" + require_cmd "7z" + if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then + require_cmd "curl" + fi + + if [[ ! -d "$NFS_SOURCE_PATH" ]]; then + die "NFS_SOURCE_PATH does not exist or is not a directory: ${NFS_SOURCE_PATH}" + fi + + mkdir -p "$BACKUP_OUTPUT_PATH" + mkdir -p "$TMP_STATE_DIR" + + if ! _kubectl get namespaces >/dev/null 2>&1; then + die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}" + fi +} + +load_namespaces() { + local ns + while IFS= read -r ns; do + [[ -z "$ns" ]] && continue + NAMESPACE_MAP["$ns"]=1 + done < <(_kubectl get namespaces -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') +} + +namespace_for_folder() { + local folder_name="${1:?folder name required}" + if [[ -n "${NAMESPACE_MAP[$folder_name]:-}" ]]; then + printf '%s' "$folder_name" + return 0 + fi + return 1 +} + +state_file_for_folder() { + local folder_name="${1:?folder name required}" + local safe_name + safe_name="$(printf '%s' "$folder_name" | tr -c 'A-Za-z0-9._-' '_')" + printf '%s/%s.state' "$TMP_STATE_DIR" "$safe_name" +} + +capture_replicas_state() { + local namespace="${1:?namespace is required}" + local state_file="${2:?state file path is required}" + : > "$state_file" + + local kind + local name + local replicas + local output + for kind in "${WORKLOAD_KIND_LIST[@]}"; do + if ! output="$(_kubectl -n "$namespace" get "$kind" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.replicas}{"\n"}{end}' 2>/dev/null)"; then + log_warn "Failed to list kind '${kind}' in namespace '${namespace}' while capturing state" + ((scale_warnings++)) + continue + fi + + while IFS=$'\t' read -r name replicas; do + [[ -z "$name" ]] && continue + if [[ -z "$replicas" ]]; then + replicas="1" + log_warn "Resource ${kind}/${name} had no explicit replicas; defaulting saved value to 1" + ((scale_warnings++)) + fi + printf '%s\t%s\t%s\n' "$kind" "$name" "$replicas" >> "$state_file" + done <<< "$output" + done +} + +scale_resource() { + local namespace="${1:?namespace is required}" + local kind="${2:?kind is required}" + local name="${3:?name is required}" + local replicas="${4:?replicas is required}" + retry "$SCALE_RETRY_COUNT" "$SCALE_RETRY_DELAY_SECONDS" \ + _kubectl -n "$namespace" scale "${kind}/${name}" --replicas="$replicas" >/dev/null +} + +scale_namespace_to_zero() { + local namespace="${1:?namespace is required}" + local state_file="${2:?state file is required}" + local kind + local name + local replicas + + while IFS=$'\t' read -r kind name replicas; do + [[ -z "$kind" || -z "$name" ]] && continue + if scale_resource "$namespace" "$kind" "$name" "0"; then + log_info "Scaled down ${namespace}:${kind}/${name} to 0" + else + log_warn "Failed to scale down ${namespace}:${kind}/${name}; continuing with backup by policy" + ((scale_warnings++)) + fi + done < "$state_file" +} + +restore_namespace_replicas() { + local namespace="${1:?namespace is required}" + local state_file="${2:?state file is required}" + local kind + local name + local replicas + + while IFS=$'\t' read -r kind name replicas; do + [[ -z "$kind" || -z "$name" ]] && continue + if scale_resource "$namespace" "$kind" "$name" "$replicas"; then + log_info "Restored ${namespace}:${kind}/${name} to ${replicas}" + else + log_warn "Failed to restore ${namespace}:${kind}/${name} to ${replicas}" + ((restore_warnings++)) + fi + done < "$state_file" +} + +resource_replicas() { + local namespace="${1:?namespace required}" + local kind="${2:?kind required}" + local name="${3:?name required}" + local json_path="${4:?jsonpath required}" + _kubectl -n "$namespace" get "${kind}/${name}" -o "jsonpath=${json_path}" 2>/dev/null || true +} + +wait_for_scale_state() { + local namespace="${1:?namespace is required}" + local state_file="${2:?state file is required}" + local direction="${3:?direction is required}" # down|up + local start_ts + local now_ts + local elapsed + local unresolved + local kind + local name + local target + local spec + local status + local ready + + start_ts="$(date +%s)" + while true; do + unresolved=0 + while IFS=$'\t' read -r kind name target; do + [[ -z "$kind" || -z "$name" ]] && continue + + spec="$(resource_replicas "$namespace" "$kind" "$name" '{.spec.replicas}')" + status="$(resource_replicas "$namespace" "$kind" "$name" '{.status.replicas}')" + ready="$(resource_replicas "$namespace" "$kind" "$name" '{.status.readyReplicas}')" + + [[ -z "$spec" ]] && spec="1" + [[ -z "$status" ]] && status="0" + [[ -z "$ready" ]] && ready="0" + + if [[ "$direction" == "down" ]]; then + if [[ "$spec" != "0" || "$status" != "0" ]]; then + ((unresolved++)) + fi + else + if [[ "$target" == "0" ]]; then + if [[ "$spec" != "0" || "$status" != "0" ]]; then + ((unresolved++)) + fi + else + if [[ "$spec" != "$target" || "$ready" != "$target" ]]; then + ((unresolved++)) + fi + fi + fi + done < "$state_file" + + if [[ "$unresolved" -eq 0 ]]; then + return 0 + fi + + now_ts="$(date +%s)" + elapsed=$((now_ts - start_ts)) + if (( elapsed >= SCALE_TIMEOUT_SECONDS )); then + return 1 + fi + sleep 3 + done +} + +archive_path_for_folder() { + local folder_name="${1:?folder name required}" + local ts + ts="$(date +"$ARCHIVE_TS_FORMAT")" + printf '%s/%s_%s_%s.7z' "$BACKUP_OUTPUT_PATH" "$ARCHIVE_PREFIX" "$folder_name" "$ts" +} + +json_escape() { + local value="${1:-}" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + value="${value//$'\n'/ }" + printf '%s' "$value" +} + +backup_size_mb() { + local bytes="$1" + if (( bytes <= 0 )); then + echo 0 + return 0 + fi + echo $(( (bytes + 1048575) / 1048576 )) +} + +send_backup_notification() { + local url="${1:-}" + local title="${2:-}" + local asset="${3:-}" + local size_mb="${4:-0}" + + [[ -z "$url" ]] && return 0 + + local payload + payload=$(cat </dev/null; then + log_warn "Failed to send backup notification to ${url}" + return 1 + fi + return 0 +} + +backup_folder() { + local folder_path="${1:?folder path required}" + local archive_path="${2:?archive path required}" + local password_arg="-p${BACKUP_PASSWORD}" + + 7z a -t7z \ + "$archive_path" \ + "$folder_path" \ + -m0="$SEVENZ_METHOD" \ + -mx="$SEVENZ_LEVEL" \ + -mhe="$SEVENZ_HEADER_ENCRYPT" \ + -mmt="$SEVENZ_THREADS" \ + "$password_arg" \ + >/dev/null +} + +process_folder() { + local folder_path="${1:?folder path is required}" + local folder_name + local namespace="" + local state_file="" + local archive_path="" + local has_mapping=0 + + folder_name="$(basename "$folder_path")" + ((total_folders++)) + log_info "Processing folder: ${folder_name}" + + if namespace="$(namespace_for_folder "$folder_name")"; then + has_mapping=1 + ((mapped_folders++)) + log_info "Exact namespace match found for folder '${folder_name}' -> namespace '${namespace}'" + state_file="$(state_file_for_folder "$folder_name")" + + capture_replicas_state "$namespace" "$state_file" + scale_namespace_to_zero "$namespace" "$state_file" + + if wait_for_scale_state "$namespace" "$state_file" "down"; then + log_info "Namespace '${namespace}' reached scale-down target" + else + log_warn "Timeout waiting for namespace '${namespace}' to fully scale down; continuing by policy" + ((scale_warnings++)) + fi + else + ((unmapped_folders++)) + log_warn "No namespace matched folder '${folder_name}'. Running backup only." + fi + + archive_path="$(archive_path_for_folder "$folder_name")" + if backup_folder "$folder_path" "$archive_path"; then + ((backup_successes++)) + local file_size_bytes + file_size_bytes="$(wc -c < "$archive_path" | tr -d '[:space:]')" + if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then + ((total_backup_size_bytes += file_size_bytes)) + fi + log_info "Backup created: ${archive_path}" + else + ((backup_failures++)) + log_error "Backup failed for folder '${folder_name}'" + fi + + if [[ "$has_mapping" -eq 1 ]]; then + restore_namespace_replicas "$namespace" "$state_file" + if wait_for_scale_state "$namespace" "$state_file" "up"; then + log_info "Namespace '${namespace}' reached restore target" + else + log_warn "Timeout waiting for namespace '${namespace}' to fully restore replicas" + ((restore_warnings++)) + fi + rm -f "$state_file" + fi +} + +print_summary() { + log_info "Run summary: processed=${total_folders} mapped=${mapped_folders} unmapped=${unmapped_folders} backup_successes=${backup_successes} backup_failures=${backup_failures} scale_warnings=${scale_warnings} restore_warnings=${restore_warnings}" +} + +main() { + parse_workload_kinds + validate_inputs + load_namespaces + + local folder_path + local found=0 + for folder_path in "${NFS_SOURCE_PATH}"/*; do + [[ -d "$folder_path" ]] || continue + [[ "$(basename "$folder_path")" == .* ]] && continue + found=1 + process_folder "$folder_path" + done + + if [[ "$found" -eq 0 ]]; then + log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}" + fi + + print_summary + local size_mb + size_mb="$(backup_size_mb "$total_backup_size_bytes")" + + if (( backup_failures > 0 )); then + send_backup_notification \ + "$NOTIFY_FAILURE_URL" \ + "${NOTIFY_TITLE} - FAILED" \ + "${NOTIFY_ASSET}" \ + "$size_mb" || true + die "One or more folder backups failed" 1 + fi + + send_backup_notification \ + "$NOTIFY_SUCCESS_URL" \ + "$NOTIFY_TITLE" \ + "${NOTIFY_ASSET}" \ + "$size_mb" || true +} + +main "$@" diff --git a/k8s/examples/automated-nfs-backup.env.example b/k8s/examples/automated-nfs-backup.env.example new file mode 100644 index 0000000..951d627 --- /dev/null +++ b/k8s/examples/automated-nfs-backup.env.example @@ -0,0 +1,21 @@ +NFS_SOURCE_PATH=/path/to/nfs/source +BACKUP_OUTPUT_PATH=/path/to/backup/output +BACKUP_PASSWORD=replace-with-strong-password +KUBECTL_BIN=kubectl +KUBE_CONTEXT= +WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller +ARCHIVE_PREFIX=nfs-backup +ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S +SEVENZ_METHOD=lzma2 +SEVENZ_LEVEL=9 +SEVENZ_HEADER_ENCRYPT=on +SEVENZ_THREADS=on +SCALE_TIMEOUT_SECONDS=600 +SCALE_RETRY_COUNT=3 +SCALE_RETRY_DELAY_SECONDS=5 +LOG_LEVEL=info +TMP_STATE_DIR=/tmp/k8s-nfs-backup +NOTIFY_SUCCESS_URL= +NOTIFY_FAILURE_URL= +NOTIFY_TITLE=Kubernetes +NOTIFY_ASSET=kube config diff --git a/k8s/lib/common.sh b/k8s/lib/common.sh new file mode 100644 index 0000000..53535af --- /dev/null +++ b/k8s/lib/common.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +# Shared helpers for Kubernetes-focused cron/utility scripts. + +set -Eeuo pipefail + +if [[ -n "${COMMON_SH_LOADED:-}" ]]; then + return 0 +fi +readonly COMMON_SH_LOADED=1 + +if [[ -z "${SCRIPT_NAME:-}" ]]; then + _source_last_index=$(( ${#BASH_SOURCE[@]} - 1 )) + SCRIPT_NAME="$(basename "${BASH_SOURCE[$_source_last_index]}")" +fi +LOG_LEVEL="${LOG_LEVEL:-info}" + +_log_level_to_int() { + case "${1,,}" in + debug) echo 10 ;; + info) echo 20 ;; + warn|warning) echo 30 ;; + error) echo 40 ;; + *) echo 20 ;; + esac +} + +readonly _CURRENT_LOG_LEVEL_INT="$(_log_level_to_int "$LOG_LEVEL")" + +_should_log() { + local level="${1:-info}" + local requested + requested="$(_log_level_to_int "$level")" + [[ "$requested" -ge "$_CURRENT_LOG_LEVEL_INT" ]] +} + +_timestamp() { + date '+%Y-%m-%d %H:%M:%S' +} + +_log() { + local level="${1:-info}" + shift || true + local message="${*:-}" + if _should_log "$level"; then + printf '%s [%s] [%s] %s\n' "$(_timestamp)" "${level^^}" "$SCRIPT_NAME" "$message" + fi +} + +log_debug() { _log "debug" "$@"; } +log_info() { _log "info" "$@"; } +log_warn() { _log "warn" "$@"; } +log_error() { _log "error" "$@"; } + +die() { + local message="${1:-Unspecified error}" + local code="${2:-1}" + log_error "$message" + exit "$code" +} + +trim() { + local value="${1:-}" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + printf '%s' "$value" +} + +require_env() { + local var_name="${1:?variable name is required}" + if [[ -z "${!var_name:-}" ]]; then + die "Missing required environment variable: ${var_name}" + fi +} + +require_cmd() { + local cmd="${1:?command is required}" + if ! command -v "$cmd" >/dev/null 2>&1; then + die "Required command not found: ${cmd}" + fi +} + +retry() { + local attempts="${1:?attempt count required}" + local delay_seconds="${2:?delay seconds required}" + shift 2 + + if (( attempts < 1 )); then + die "retry attempts must be >= 1" + fi + + local attempt=1 + local exit_code=0 + while true; do + if "$@"; then + return 0 + fi + exit_code=$? + if (( attempt >= attempts )); then + return "$exit_code" + fi + log_warn "Command failed (attempt ${attempt}/${attempts}): $*; retrying in ${delay_seconds}s" + sleep "$delay_seconds" + ((attempt++)) + done +} + +on_error() { + local line="${1:-unknown}" + local command="${2:-unknown}" + log_error "Unhandled error at line ${line}: ${command}" +} + +on_exit() { + local code="${1:-0}" + if [[ "$code" -eq 0 ]]; then + log_info "Finished successfully" + else + log_error "Finished with errors (exit code ${code})" + fi +} + +setup_exit_handling() { + trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR + trap 'on_exit "$?"' EXIT +}