fix(backup): treat 7z exit code 1 (warning) as success in archive_staging

Exit code 1 from 7z means a non-fatal warning — the archive IS created,
just some files couldn't be compressed (e.g. broken symlinks, special
files). Previously the function required filtered output to be empty
alongside rc=1, causing any non-errno=2 stderr line to abort the entire
backup and discard the archive.

Now any rc=1 is treated as success (warning is still logged).

.gitea/workflows/haven-notify.yaml

@@ -109,4 +109,4 @@ jobs:
         with:
           kube_config: ${{ secrets.KUBE_CONFIG }}
           deployment_name: haven-notify
-          namespace: default
+          namespace: infra

.gitea/workflows/test-automated-nfs-backup.yaml

@@ -1,4 +1,4 @@
-name: Test automated-nfs-backup
+name: Test Kubernetes backup scripts
 
 on:
   push:
@@ -15,3 +15,12 @@ jobs:
         uses: actions/checkout@v4
       - name: Run automated NFS backup tests
         run: bash k8s/test/test-automated-nfs-backup.sh
+  test-k3s-control-plane-config-backup:
+    runs-on: runner-slim
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Install GNU coreutils and findutils for control-plane tests
+        run: apk add --no-cache coreutils findutils
+      - name: Run k3s control-plane configuration backup tests
+        run: bash k8s/test/test-backup-k3s-control-plane-config.sh

haven-notify/README.md

@@ -20,7 +20,11 @@ It runs as a standalone binary or container. Templates are embedded in the binar
 
 All notification endpoints require `POST` and `Content-Type: application/json`. Request bodies are limited to 64 KiB, documented fields are required, and unknown fields are accepted for forward compatibility.
 
-### Send Notification
+### V1 API
+
+The original routes, request bodies, and Discord message appearance remain unchanged.
+
+#### Send Notification
 
 - **Endpoint**: `/notify`
 - **Request Body**:
@@ -32,7 +36,7 @@ All notification endpoints require `POST` and `Content-Type: application/json`.
 }
 ```
 
-### Send Backup Notification
+#### Send Backup Notification
 
 - **Endpoint**: `/template/notify/backup`
 - **Request Body**:
@@ -51,7 +55,7 @@ All notification endpoints require `POST` and `Content-Type: application/json`.
 }
 ```
 
-### Send Update Notification
+#### Send Update Notification
 
 - **Endpoint**: `/template/notify/update`
 - **Request Body**:
@@ -66,7 +70,7 @@ All notification endpoints require `POST` and `Content-Type: application/json`.
 
 `time` is expressed in seconds.
 
-### Send Error Notification
+#### Send Error Notification
 
 - **Endpoint**: `/template/notify/error`
 - **Request Body**:
@@ -85,6 +89,104 @@ All notification endpoints require `POST` and `Content-Type: application/json`.
 }
 ```
 
+### V2 API
+
+V2 notifications use compact, consistently styled Discord embeds. Colors, icons, timestamps, field ordering, and layout are controlled by Haven Notify rather than the caller.
+
+Every v2 request may include:
+
+- `source`: a nonblank string identifying the caller.
+- `occurredAt`: an RFC3339 timestamp. When omitted, Haven Notify uses the current UTC time.
+- `extra`: an array of nonblank `name` and `value` strings. Short single-line values render inline; multiline or long values render full-width.
+
+V2 preserves caller field order. Fixed notification fields are placed before `extra` fields so that trailing extras are discarded first if Discord's 25-field or 6,000-character embed limits are reached. Oversized text is safely truncated and extra fields may be omitted from the delivered message.
+
+#### Send a Message
+
+- **Endpoint**: `/api/v2/messages`
+- **Required**: `message`
+- **Optional**: `title`, `source`, `occurredAt`, `extra`
+
+```json
+{
+  "title": "Maintenance complete",
+  "message": "The storage job completed successfully.",
+  "source": "scheduler",
+  "extra": [
+    {
+      "name": "Region",
+      "value": "home"
+    }
+  ]
+}
+```
+
+#### Send a Backup Notification
+
+- **Endpoint**: `/api/v2/notifications/backup`
+- **Required**: `asset`, nonnegative integer `sizeBytes`
+- **Optional**: `title`, `source`, nonnegative `durationSeconds`, `occurredAt`, `extra`
+
+```json
+{
+  "title": "Nightly backup",
+  "asset": "Photos",
+  "sizeBytes": 1610612736,
+  "durationSeconds": 125,
+  "source": "restic",
+  "extra": [
+    {
+      "name": "Host",
+      "value": "nebula"
+    }
+  ]
+}
+```
+
+`sizeBytes` is displayed using IEC units such as KiB, MiB, and GiB. Durations use a compact form such as `42.5s`, `2m 5s`, or `1h 2m 3s`.
+
+#### Send an Update Notification
+
+- **Endpoint**: `/api/v2/notifications/update`
+- **Required**: `host`, `asset`, nonnegative `durationSeconds`
+- **Optional**: `source`, `fromVersion`, `toVersion`, `occurredAt`, `extra`
+
+```json
+{
+  "host": "nexus",
+  "asset": "k3s",
+  "durationSeconds": 42.5,
+  "fromVersion": "1.32.5",
+  "toVersion": "1.33.1",
+  "source": "system-updater"
+}
+```
+
+When both versions are supplied, the embed displays a transition such as `1.32.5 → 1.33.1`. A single supplied version is labeled as the previous or installed version.
+
+#### Send an Error Notification
+
+- **Endpoint**: `/api/v2/notifications/error`
+- **Required**: `source`, `message`, `severity`
+- **Optional**: `errorCode`, `occurredAt`, `extra`
+
+```json
+{
+  "source": "backup-job",
+  "message": "The repository is unavailable.",
+  "severity": "critical",
+  "errorCode": "E_REPOSITORY",
+  "extra": [
+    {
+      "name": "Path",
+      "value": "/mnt/backups"
+    }
+  ]
+}
+```
+
+`severity` must be `warning`, `error`, or `critical`; it selects the embed title, icon, and color.
+
 ### Health Probes
 
 - `GET` or `HEAD /ready` returns `200 Ready` after configuration and templates initialize.

haven-notify/discord.go

@@ -51,6 +51,7 @@ type discordEmbed struct {
 	Title       string              `json:"title,omitempty"`
 	Description string              `json:"description,omitempty"`
 	Color       int                 `json:"color,omitempty"`
+	Timestamp   string              `json:"timestamp,omitempty"`
 	Fields      []discordEmbedField `json:"fields,omitempty"`
 	Footer      *discordEmbedFooter `json:"footer,omitempty"`
 }

haven-notify/server.go

@@ -32,12 +32,14 @@ type applicationConfig struct {
 	HTTPClient *http.Client
 	Sleep      sleepFunc
 	Logger     *log.Logger
+	Now        func() time.Time
 }
 
 type application struct {
 	discord  *discordClient
 	renderer *templateRenderer
 	logger   *log.Logger
+	now      func() time.Time
 }
 
 type notificationRequest struct {
@@ -77,6 +79,10 @@ func newApplication(cfg applicationConfig) (*application, error) {
 	if logger == nil {
 		logger = log.New(io.Discard, "", 0)
 	}
+	now := cfg.Now
+	if now == nil {
+		now = time.Now
+	}
 
 	renderer, err := newTemplateRenderer()
 	if err != nil {
@@ -88,7 +94,7 @@ func newApplication(cfg applicationConfig) (*application, error) {
 		return nil, err
 	}
 
-	return &application{discord: discord, renderer: renderer, logger: logger}, nil
+	return &application{discord: discord, renderer: renderer, logger: logger, now: now}, nil
 }
 
 func (a *application) handler() http.Handler {
@@ -97,6 +103,10 @@ func (a *application) handler() http.Handler {
 	mux.HandleFunc("/template/notify/backup", a.requireMethod(http.MethodPost, a.backupHandler))
 	mux.HandleFunc("/template/notify/update", a.requireMethod(http.MethodPost, a.updateHandler))
 	mux.HandleFunc("/template/notify/error", a.requireMethod(http.MethodPost, a.errorHandler))
+	mux.HandleFunc("/api/v2/messages", a.requireMethod(http.MethodPost, a.v2MessageHandler))
+	mux.HandleFunc("/api/v2/notifications/backup", a.requireMethod(http.MethodPost, a.v2BackupHandler))
+	mux.HandleFunc("/api/v2/notifications/update", a.requireMethod(http.MethodPost, a.v2UpdateHandler))
+	mux.HandleFunc("/api/v2/notifications/error", a.requireMethod(http.MethodPost, a.v2ErrorHandler))
 	mux.HandleFunc("/ready", a.requireProbeMethod(a.readinessHandler))
 	mux.HandleFunc("/live", a.requireProbeMethod(a.livenessHandler))
 	mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {

haven-notify/server_test.go

@@ -61,6 +61,7 @@ func testApplication(t testing.TB, transport http.RoundTripper, logger *log.Logg
 		HTTPClient: &http.Client{Transport: transport},
 		Sleep:      func(context.Context, time.Duration) error { return nil },
 		Logger:     logger,
+		Now:        func() time.Time { return time.Date(2026, time.July, 16, 12, 34, 56, 0, time.UTC) },
 	})
 	if err != nil {
 		t.Fatalf("newApplication() error = %v", err)

haven-notify/v2.go

@@ -0,0 +1,306 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+const (
+	v2ColorBrand    = 0x5865F2
+	v2ColorSuccess  = 0x57F287
+	v2ColorUpdate   = 0x3498DB
+	v2ColorWarning  = 0xFEE75C
+	v2ColorError    = 0xED4245
+	v2ColorCritical = 0x992D22
+
+	v2InlineFieldLimit = 80
+)
+
+type v2CommonRequest struct {
+	Source     *string       `json:"source"`
+	OccurredAt *string       `json:"occurredAt"`
+	Extra      extraRequests `json:"extra"`
+}
+
+type v2MessageRequest struct {
+	v2CommonRequest
+	Title   *string `json:"title"`
+	Message *string `json:"message"`
+}
+
+type v2BackupRequest struct {
+	v2CommonRequest
+	Title           *string  `json:"title"`
+	Asset           *string  `json:"asset"`
+	SizeBytes       *int64   `json:"sizeBytes"`
+	DurationSeconds *float64 `json:"durationSeconds"`
+}
+
+type v2UpdateRequest struct {
+	v2CommonRequest
+	Host            *string  `json:"host"`
+	Asset           *string  `json:"asset"`
+	DurationSeconds *float64 `json:"durationSeconds"`
+	FromVersion     *string  `json:"fromVersion"`
+	ToVersion       *string  `json:"toVersion"`
+}
+
+type v2ErrorRequest struct {
+	v2CommonRequest
+	Message   *string `json:"message"`
+	Severity  *string `json:"severity"`
+	ErrorCode *string `json:"errorCode"`
+}
+
+func (a *application) v2MessageHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2MessageRequest
+	if err := decodeRequest(w, r, &request, "title", "message", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Message) || !validOptionalString(request.Title) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2MessagePayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2BackupHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2BackupRequest
+	if err := decodeRequest(w, r, &request, "title", "asset", "sizebytes", "durationseconds", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Asset) || !validOptionalString(request.Title) || request.SizeBytes == nil || *request.SizeBytes < 0 || !validOptionalNumber(request.DurationSeconds) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2BackupPayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2UpdateHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2UpdateRequest
+	if err := decodeRequest(w, r, &request, "host", "asset", "durationseconds", "fromversion", "toversion", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Host) || !requiredString(request.Asset) || !validNumber(request.DurationSeconds) || !validOptionalString(request.FromVersion) || !validOptionalString(request.ToVersion) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2UpdatePayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2ErrorHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2ErrorRequest
+	if err := decodeRequest(w, r, &request, "source", "message", "severity", "errorcode", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Source) || !requiredString(request.Message) || !validV2Severity(request.Severity) || !validOptionalString(request.ErrorCode) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2ErrorPayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) validV2Common(request v2CommonRequest) (time.Time, bool) {
+	if !validOptionalString(request.Source) || !validExtra(request.Extra) {
+		return time.Time{}, false
+	}
+	if request.OccurredAt == nil {
+		return a.now().UTC(), true
+	}
+	if !requiredString(request.OccurredAt) {
+		return time.Time{}, false
+	}
+	when, err := time.Parse(time.RFC3339, *request.OccurredAt)
+	if err != nil {
+		return time.Time{}, false
+	}
+	return when.UTC(), true
+}
+
+func validOptionalString(value *string) bool {
+	return value == nil || requiredString(value)
+}
+
+func validNumber(value *float64) bool {
+	return value != nil && *value >= 0 && !math.IsNaN(*value) && !math.IsInf(*value, 0)
+}
+
+func validOptionalNumber(value *float64) bool {
+	return value == nil || validNumber(value)
+}
+
+func validV2Severity(value *string) bool {
+	if value == nil {
+		return false
+	}
+	switch *value {
+	case "warning", "error", "critical":
+		return true
+	default:
+		return false
+	}
+}
+
+func buildV2MessagePayload(request v2MessageRequest, when time.Time) discordPayload {
+	title := "Message"
+	if request.Title != nil {
+		title = *request.Title
+	}
+	embed := newV2Embed("💬 "+title, *request.Message, v2ColorBrand, when)
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2BackupPayload(request v2BackupRequest, when time.Time) discordPayload {
+	title := "Backup completed"
+	if request.Title != nil {
+		title = *request.Title
+	}
+	embed := newV2Embed("📦 "+title, fmt.Sprintf("**%s** was backed up successfully.", *request.Asset), v2ColorSuccess, when)
+	embed.Fields = append(embed.Fields, v2Field("Size", formatBytes(*request.SizeBytes)))
+	if request.DurationSeconds != nil {
+		embed.Fields = append(embed.Fields, v2Field("Duration", formatV2Duration(*request.DurationSeconds)))
+	}
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2UpdatePayload(request v2UpdateRequest, when time.Time) discordPayload {
+	embed := newV2Embed("🔄 Update completed", fmt.Sprintf("**%s** was updated on **%s**.", *request.Asset, *request.Host), v2ColorUpdate, when)
+	switch {
+	case request.FromVersion != nil && request.ToVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Version", *request.FromVersion+" → "+*request.ToVersion))
+	case request.FromVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Previous version", *request.FromVersion))
+	case request.ToVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Installed version", *request.ToVersion))
+	}
+	embed.Fields = append(embed.Fields, v2Field("Duration", formatV2Duration(*request.DurationSeconds)))
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2ErrorPayload(request v2ErrorRequest, when time.Time) discordPayload {
+	title, color := v2ErrorStyle(*request.Severity)
+	embed := newV2Embed(title, *request.Message, color, when)
+	embed.Fields = append(embed.Fields, v2Field("Source", *request.Source))
+	if request.ErrorCode != nil {
+		embed.Fields = append(embed.Fields, v2Field("Error code", *request.ErrorCode))
+	}
+	embed.Fields = appendV2Extras(embed.Fields, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func newV2Embed(title, description string, color int, when time.Time) discordEmbed {
+	return discordEmbed{
+		Title:       title,
+		Description: description,
+		Color:       color,
+		Timestamp:   when.UTC().Format(time.RFC3339),
+		Footer:      &discordEmbedFooter{Text: "Haven Notify"},
+	}
+}
+
+func v2ErrorStyle(severity string) (string, int) {
+	switch severity {
+	case "warning":
+		return "⚠️ Warning", v2ColorWarning
+	case "critical":
+		return "🛑 Critical error", v2ColorCritical
+	default:
+		return "❌ Error", v2ColorError
+	}
+}
+
+func appendV2SourceAndExtras(fields []discordEmbedField, source *string, extra extraRequests) []discordEmbedField {
+	if source != nil {
+		fields = append(fields, v2Field("Source", *source))
+	}
+	return appendV2Extras(fields, extra)
+}
+
+func appendV2Extras(fields []discordEmbedField, extra extraRequests) []discordEmbedField {
+	for _, field := range extra {
+		value := *field.Value
+		fields = append(fields, discordEmbedField{
+			Name:   *field.Name,
+			Value:  value,
+			Inline: !strings.ContainsAny(value, "\r\n") && utf8.RuneCountInString(value) <= v2InlineFieldLimit,
+		})
+	}
+	return fields
+}
+
+func v2Field(name, value string) discordEmbedField {
+	return discordEmbedField{Name: name, Value: value, Inline: true}
+}
+
+func formatBytes(bytes int64) string {
+	if bytes < 1024 {
+		return strconv.FormatInt(bytes, 10) + " B"
+	}
+	value := float64(bytes)
+	for _, unit := range []string{"KiB", "MiB", "GiB", "TiB", "PiB", "EiB"} {
+		value /= 1024
+		if value < 1024 || unit == "EiB" {
+			return fmt.Sprintf("%.2f %s", value, unit)
+		}
+	}
+	return strconv.FormatInt(bytes, 10) + " B"
+}
+
+func formatV2Duration(seconds float64) string {
+	if seconds < 60 {
+		return formatV2Number(seconds) + "s"
+	}
+
+	hours := math.Floor(seconds / 3600)
+	seconds -= hours * 3600
+	minutes := math.Floor(seconds / 60)
+	seconds -= minutes * 60
+	parts := make([]string, 0, 3)
+	if hours > 0 {
+		parts = append(parts, formatV2Number(hours)+"h")
+	}
+	if minutes > 0 {
+		parts = append(parts, formatV2Number(minutes)+"m")
+	}
+	if seconds > 0 {
+		parts = append(parts, formatV2Number(seconds)+"s")
+	}
+	return strings.Join(parts, " ")
+}
+
+func formatV2Number(value float64) string {
+	formatted := strconv.FormatFloat(value, 'f', 3, 64)
+	return strings.TrimRight(strings.TrimRight(formatted, "0"), ".")
+}

haven-notify/v2_test.go

@@ -0,0 +1,386 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+const v2TestTimestamp = "2026-07-16T12:34:56Z"
+
+func TestV2NotificationPayloads(t *testing.T) {
+	tests := []struct {
+		name string
+		path string
+		body string
+		want discordPayload
+	}{
+		{
+			name: "generic message",
+			path: "/api/v2/messages",
+			body: `{"title":"Maintenance","message":"The storage job completed.","source":"scheduler","occurredAt":"2026-07-16T09:34:56-03:00","extra":[{"name":"Region","value":"home"},{"name":"Details","value":"first line\nsecond line"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "💬 Maintenance",
+					Description: "The storage job completed.",
+					Color:       v2ColorBrand,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Source", Value: "scheduler", Inline: true},
+						{Name: "Region", Value: "home", Inline: true},
+						{Name: "Details", Value: "first line\nsecond line", Inline: false},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "backup",
+			path: "/api/v2/notifications/backup",
+			body: `{"title":"Nightly backup","asset":"Photos","sizeBytes":1610612736,"durationSeconds":125,"source":"restic","extra":[{"name":"Host","value":"nebula"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "📦 Nightly backup",
+					Description: "**Photos** was backed up successfully.",
+					Color:       v2ColorSuccess,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Size", Value: "1.50 GiB", Inline: true},
+						{Name: "Duration", Value: "2m 5s", Inline: true},
+						{Name: "Source", Value: "restic", Inline: true},
+						{Name: "Host", Value: "nebula", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "update",
+			path: "/api/v2/notifications/update",
+			body: `{"host":"nexus","asset":"k3s","durationSeconds":42.5,"fromVersion":"1.32.5","toVersion":"1.33.1","source":"system-updater"}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "🔄 Update completed",
+					Description: "**k3s** was updated on **nexus**.",
+					Color:       v2ColorUpdate,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Version", Value: "1.32.5 → 1.33.1", Inline: true},
+						{Name: "Duration", Value: "42.5s", Inline: true},
+						{Name: "Source", Value: "system-updater", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "critical error",
+			path: "/api/v2/notifications/error",
+			body: `{"source":"backup-job","message":"The repository is unavailable.","severity":"critical","errorCode":"E_REPOSITORY","extra":[{"name":"Path","value":"/mnt/backups"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "🛑 Critical error",
+					Description: "The repository is unavailable.",
+					Color:       v2ColorCritical,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Source", Value: "backup-job", Inline: true},
+						{Name: "Error code", Value: "E_REPOSITORY", Inline: true},
+						{Name: "Path", Value: "/mnt/backups", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := sendV2Request(t, test.path, test.body)
+			if !reflect.DeepEqual(got, test.want) {
+				gotJSON, _ := json.MarshalIndent(got, "", "  ")
+				wantJSON, _ := json.MarshalIndent(test.want, "", "  ")
+				t.Fatalf("payload = %s\nwant = %s", gotJSON, wantJSON)
+			}
+		})
+	}
+}
+
+func TestV2DefaultsAndVariants(t *testing.T) {
+	tests := []struct {
+		name       string
+		path       string
+		body       string
+		wantTitle  string
+		wantColor  int
+		wantFields []discordEmbedField
+	}{
+		{
+			name:      "message default title",
+			path:      "/api/v2/messages",
+			body:      `{"message":"hello"}`,
+			wantTitle: "💬 Message",
+			wantColor: v2ColorBrand,
+		},
+		{
+			name:      "backup defaults",
+			path:      "/api/v2/notifications/backup",
+			body:      `{"asset":"config","sizeBytes":0}`,
+			wantTitle: "📦 Backup completed",
+			wantColor: v2ColorSuccess,
+			wantFields: []discordEmbedField{
+				{Name: "Size", Value: "0 B", Inline: true},
+			},
+		},
+		{
+			name:      "update previous version",
+			path:      "/api/v2/notifications/update",
+			body:      `{"host":"nexus","asset":"k3s","durationSeconds":60,"fromVersion":"1.0"}`,
+			wantTitle: "🔄 Update completed",
+			wantColor: v2ColorUpdate,
+			wantFields: []discordEmbedField{
+				{Name: "Previous version", Value: "1.0", Inline: true},
+				{Name: "Duration", Value: "1m", Inline: true},
+			},
+		},
+		{
+			name:      "update installed version",
+			path:      "/api/v2/notifications/update",
+			body:      `{"host":"nexus","asset":"k3s","durationSeconds":0,"toVersion":"1.1"}`,
+			wantTitle: "🔄 Update completed",
+			wantColor: v2ColorUpdate,
+			wantFields: []discordEmbedField{
+				{Name: "Installed version", Value: "1.1", Inline: true},
+				{Name: "Duration", Value: "0s", Inline: true},
+			},
+		},
+		{
+			name:      "warning",
+			path:      "/api/v2/notifications/error",
+			body:      `{"source":"job","message":"slow","severity":"warning"}`,
+			wantTitle: "⚠️ Warning",
+			wantColor: v2ColorWarning,
+			wantFields: []discordEmbedField{
+				{Name: "Source", Value: "job", Inline: true},
+			},
+		},
+		{
+			name:      "error",
+			path:      "/api/v2/notifications/error",
+			body:      `{"source":"job","message":"failed","severity":"error"}`,
+			wantTitle: "❌ Error",
+			wantColor: v2ColorError,
+			wantFields: []discordEmbedField{
+				{Name: "Source", Value: "job", Inline: true},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			payload := sendV2Request(t, test.path, test.body)
+			embed := payload.Embeds[0]
+			if embed.Title != test.wantTitle || embed.Color != test.wantColor || embed.Timestamp != v2TestTimestamp || !reflect.DeepEqual(embed.Fields, test.wantFields) {
+				t.Fatalf("embed = %+v", embed)
+			}
+		})
+	}
+}
+
+func TestV2RequestValidation(t *testing.T) {
+	tests := []struct {
+		name        string
+		path        string
+		contentType string
+		body        string
+		wantStatus  int
+	}{
+		{name: "message missing", path: "/api/v2/messages", body: `{}`, wantStatus: 400},
+		{name: "message blank", path: "/api/v2/messages", body: `{"message":" "}`, wantStatus: 400},
+		{name: "blank optional title", path: "/api/v2/messages", body: `{"message":"m","title":" "}`, wantStatus: 400},
+		{name: "blank optional source", path: "/api/v2/messages", body: `{"message":"m","source":" "}`, wantStatus: 400},
+		{name: "invalid timestamp", path: "/api/v2/messages", body: `{"message":"m","occurredAt":"yesterday"}`, wantStatus: 400},
+		{name: "blank timestamp", path: "/api/v2/messages", body: `{"message":"m","occurredAt":" "}`, wantStatus: 400},
+		{name: "null extras", path: "/api/v2/messages", body: `{"message":"m","extra":null}`, wantStatus: 400},
+		{name: "invalid extra", path: "/api/v2/messages", body: `{"message":"m","extra":[{"name":"n"}]}`, wantStatus: 400},
+		{name: "duplicate common key", path: "/api/v2/messages", body: `{"message":"m","source":"one","SOURCE":"two"}`, wantStatus: 400},
+		{name: "duplicate nested key", path: "/api/v2/messages", body: `{"message":"m","extra":[{"name":"one","NAME":"two","value":"v"}]}`, wantStatus: 400},
+		{name: "backup missing asset", path: "/api/v2/notifications/backup", body: `{"sizeBytes":1}`, wantStatus: 400},
+		{name: "backup missing size", path: "/api/v2/notifications/backup", body: `{"asset":"a"}`, wantStatus: 400},
+		{name: "backup negative size", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":-1}`, wantStatus: 400},
+		{name: "backup fractional size", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":1.5}`, wantStatus: 400},
+		{name: "backup negative duration", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":1,"durationSeconds":-1}`, wantStatus: 400},
+		{name: "update missing host", path: "/api/v2/notifications/update", body: `{"asset":"a","durationSeconds":1}`, wantStatus: 400},
+		{name: "update missing asset", path: "/api/v2/notifications/update", body: `{"host":"h","durationSeconds":1}`, wantStatus: 400},
+		{name: "update missing duration", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a"}`, wantStatus: 400},
+		{name: "update negative duration", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a","durationSeconds":-1}`, wantStatus: 400},
+		{name: "update blank version", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a","durationSeconds":1,"fromVersion":" "}`, wantStatus: 400},
+		{name: "error missing source", path: "/api/v2/notifications/error", body: `{"message":"m","severity":"error"}`, wantStatus: 400},
+		{name: "error missing message", path: "/api/v2/notifications/error", body: `{"source":"s","severity":"error"}`, wantStatus: 400},
+		{name: "error missing severity", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m"}`, wantStatus: 400},
+		{name: "error invalid severity", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m","severity":"ERROR"}`, wantStatus: 400},
+		{name: "error blank code", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m","severity":"error","errorCode":" "}`, wantStatus: 400},
+		{name: "wrong media type", path: "/api/v2/messages", contentType: "text/plain", body: `{"message":"m"}`, wantStatus: 415},
+		{name: "unknown accepted", path: "/api/v2/messages", body: `{"message":"m","future":true}`, wantStatus: 200},
+		{name: "case insensitive accepted", path: "/api/v2/messages", body: `{"MESSAGE":"m","EXTRA":[{"NAME":"n","VALUE":"v"}]}`, wantStatus: 200},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			contentType := test.contentType
+			if contentType == "" {
+				contentType = "application/json"
+			}
+			recorder := &webhookRecorder{}
+			response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, test.path, contentType, test.body)
+			if response.Code != test.wantStatus {
+				t.Fatalf("response = (%d, %q), want %d", response.Code, response.Body.String(), test.wantStatus)
+			}
+			wantCalls := 0
+			if test.wantStatus == http.StatusOK {
+				wantCalls = 1
+			}
+			if calls := len(recorder.bodies()); calls != wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", calls, wantCalls)
+			}
+		})
+	}
+}
+
+func TestV2RoutingContracts(t *testing.T) {
+	handler := testApplication(t, nil, nil).handler()
+	for _, path := range []string{
+		"/api/v2/messages",
+		"/api/v2/notifications/backup",
+		"/api/v2/notifications/update",
+		"/api/v2/notifications/error",
+	} {
+		response := performRequest(handler, http.MethodGet, path, "", "")
+		if response.Code != http.StatusMethodNotAllowed || response.Header().Get("Allow") != http.MethodPost {
+			t.Fatalf("GET %s = (%d, Allow %q)", path, response.Code, response.Header().Get("Allow"))
+		}
+	}
+	response := performRequest(handler, http.MethodPost, "/api/v2/messages/", "application/json", `{"message":"m"}`)
+	if response.Code != http.StatusNotFound {
+		t.Fatalf("trailing slash status = %d", response.Code)
+	}
+}
+
+func TestV2RequestBodyLimit(t *testing.T) {
+	prefix, suffix := `{"message":"`, `"}`
+	for _, test := range []struct {
+		name      string
+		bodyBytes int
+		want      int
+		wantCalls int
+	}{
+		{name: "exactly at limit", bodyBytes: maxRequestBody, want: http.StatusOK, wantCalls: 1},
+		{name: "one byte over limit", bodyBytes: maxRequestBody + 1, want: http.StatusBadRequest},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			body := prefix + strings.Repeat("x", test.bodyBytes-len(prefix)-len(suffix)) + suffix
+			response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, "/api/v2/messages", "application/json", body)
+			if response.Code != test.want {
+				t.Fatalf("response status = %d, want %d", response.Code, test.want)
+			}
+			if calls := len(recorder.bodies()); calls != test.wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", calls, test.wantCalls)
+			}
+		})
+	}
+}
+
+func TestV2ExtrasRespectDiscordLimits(t *testing.T) {
+	extra := make([]string, 30)
+	for index := range extra {
+		extra[index] = fmt.Sprintf(`{"name":"field-%02d","value":"value"}`, index)
+	}
+	body := fmt.Sprintf(`{"host":"nexus","asset":"k3s","durationSeconds":1,"fromVersion":"1","toVersion":"2","source":"updater","extra":[%s]}`, strings.Join(extra, ","))
+	payload := sendV2Request(t, "/api/v2/notifications/update", body)
+	fields := payload.Embeds[0].Fields
+	if len(fields) != 25 {
+		t.Fatalf("fields = %d, want 25", len(fields))
+	}
+	if fields[0].Name != "Version" || fields[1].Name != "Duration" || fields[2].Name != "Source" || fields[24].Name != "field-21" {
+		t.Fatalf("field order = %+v", fields)
+	}
+
+	longValue := strings.Repeat("x", v2InlineFieldLimit+1)
+	body = fmt.Sprintf(`{"message":"m","source":"source","extra":[{"name":"long","value":%q}]}`, longValue)
+	payload = sendV2Request(t, "/api/v2/messages", body)
+	if payload.Embeds[0].Fields[0].Name != "Source" || payload.Embeds[0].Fields[1].Inline {
+		t.Fatalf("inline fields = %+v", payload.Embeds[0].Fields)
+	}
+
+	body = fmt.Sprintf(`{"message":%q,"source":"source","extra":[{"name":"large","value":%q}]}`, strings.Repeat("m", 5000), strings.Repeat("x", 2000))
+	payload = sendV2Request(t, "/api/v2/messages", body)
+	embed := payload.Embeds[0]
+	if utf8.RuneCountInString(embed.Description) > 4096 || discordEmbedCharacters(payload.Embeds) > 6000 || len(embed.Fields) == 0 || embed.Fields[0].Name != "Source" {
+		t.Fatalf("normalized embed = %+v, characters = %d", embed, discordEmbedCharacters(payload.Embeds))
+	}
+}
+
+func TestV2FormattingHelpers(t *testing.T) {
+	for _, test := range []struct {
+		bytes int64
+		want  string
+	}{
+		{0, "0 B"},
+		{1023, "1023 B"},
+		{1024, "1.00 KiB"},
+		{1536, "1.50 KiB"},
+		{1 << 30, "1.00 GiB"},
+	} {
+		if got := formatBytes(test.bytes); got != test.want {
+			t.Errorf("formatBytes(%d) = %q, want %q", test.bytes, got, test.want)
+		}
+	}
+	for _, test := range []struct {
+		seconds float64
+		want    string
+	}{
+		{0, "0s"},
+		{42.5, "42.5s"},
+		{60, "1m"},
+		{125, "2m 5s"},
+		{125.1, "2m 5.1s"},
+		{3723, "1h 2m 3s"},
+	} {
+		if got := formatV2Duration(test.seconds); got != test.want {
+			t.Errorf("formatV2Duration(%v) = %q, want %q", test.seconds, got, test.want)
+		}
+	}
+	for _, value := range []*float64{nil, floatPointer(-1), floatPointer(math.NaN()), floatPointer(math.Inf(1))} {
+		if validNumber(value) {
+			t.Fatalf("validNumber(%v) = true", value)
+		}
+	}
+}
+
+func sendV2Request(t *testing.T, path, body string) discordPayload {
+	t.Helper()
+	recorder := &webhookRecorder{}
+	response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, path, "application/json", body)
+	if response.Code != http.StatusOK || response.Body.String() != "Notification sent" {
+		t.Fatalf("response = (%d, %q)", response.Code, response.Body.String())
+	}
+	bodies := recorder.bodies()
+	if len(bodies) != 1 {
+		t.Fatalf("webhook calls = %d, want 1", len(bodies))
+	}
+	var payload discordPayload
+	if err := json.Unmarshal(bodies[0], &payload); err != nil {
+		t.Fatalf("decode webhook payload: %v", err)
+	}
+	return payload
+}

k8s/README.md

@@ -6,18 +6,37 @@ Kubernetes-focused shell scripts intended for cronjobs and operational utilities
 
 ### `automated-nfs-backup.sh`
 
-Backs up each top-level folder found in `NFS_SOURCE_PATH` into a single encrypted `.7z` archive per run, with optional Kubernetes workload quiescing when a folder name exactly matches a namespace name.
+Copies each top-level folder found in `NFS_SOURCE_PATH` to a per-run local staging directory, then creates one encrypted `.7z` archive from the complete staged data. Kubernetes workloads are quiesced while a folder whose name exactly matches a namespace is copied.
 
 Behavior:
 - Exact folder-to-namespace mapping only.
-- `kube-system`, `kube-public`, `kube-node-lease`, and `default` are excluded from scale actions by default.
-- Unmapped folder: backup still runs, Kubernetes scale actions are skipped.
-- Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again.
-- When `BACKUP_STAGING_PATH` is configured, a live rsync pre-copy is followed by a final offline delta; the namespace is restored before the staged copy is compressed.
+- `kube-system`, `kube-public`, and `kube-node-lease` are excluded from scale actions by default.
+- Unmapped or excluded folder: copied to local staging without Kubernetes scale actions.
+- Mapped folder: saves replicas, scales selected workloads down, waits, copies the quiesced folder to local staging, restores replicas, and waits again.
+- After every folder has been processed, one `7z` invocation compresses the complete run staging directory. Workloads are therefore restored before compression starts.
+- Per-run replica state and staged data are removed on exit. The staging parent is retained.
 - Scale-down issues are warnings by policy (backup still runs).
 - Restore issues are warnings by policy (run can still complete successfully).
 - Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.
 
+### `backup-k3s-control-plane-config.sh`
+
+Creates and validates a standard `.zip` containing the iris k3s control-plane configuration and credential state, then atomically publishes it to the NAS. It never stops or restarts k3s, and deliberately excludes `/var/lib/rancher/k3s/server/db` and `kine.sock`; this is configuration/credential backup, not a live datastore snapshot.
+
+The archive includes all of `K3S_CONFIG_DIR`, any existing `K3S_SERVICE_FILES`, and existing `cred`, `etc`, `manifests`, `static`, `tls`, `token`, `agent-token`, and `node-token` entries beneath `K3S_SERVER_DIR`.
+
+Defaults mirror the existing `k8s-config-backup` schedule: `ivanch@nas.haven:/export/Backup/k8s`, archive names such as `k3s-control-plane-config_2026-07-21_12-00-00.zip`, and remote retention of the four newest matching archives. Other prefixes, including `k8s-backup_*.7z`, are never considered for retention.
+
+The script uses a non-blocking lock by default (`/var/lock/k3s-control-plane-config-backup.lock`), so overlapping cron and manual invocations fail safely. It builds in a private local temporary directory, validates with `unzip -t`, uploads to a remote temporary filename, and only then renames it into place.
+
+Useful overrides:
+
+- Source paths: `K3S_CONFIG_DIR`, `K3S_SERVER_DIR`, `K3S_SERVICE_FILES` (colon-separated service configuration files).
+- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`.
+- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion).
+- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REALPATH_BIN`, `REMOTE_BASH_BIN`. `REALPATH_BIN` must provide GNU `realpath` `-e` and `-m` options.
+- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/api/v2/notifications/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/api/v2/notifications/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_SOURCE`. `NOTIFY_CALLER` is a compatibility fallback for `NOTIFY_SOURCE`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
+
 ## Environment Variables
 
 Required:
@@ -28,9 +47,9 @@ Optional:
 - `BACKUP_PASSWORD` (default: empty): When set, archive uses password protection; when empty, archive is not encrypted.
 - `KUBECTL_BIN` (default: `kubectl`)
 - `KUBE_CONTEXT` (default: empty)
-- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`)
-- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease,default`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions.
-- `BACKUP_STAGING_PATH` (default: empty, disabled): Local staging parent for two-pass rsync. It must be outside `NFS_SOURCE_PATH` and have enough free space for the largest folder being backed up.
+- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset`)
+- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions.
+- `BACKUP_STAGING_PATH` (default: `/tmp/k8s-nfs-backup-staging`): Local parent for per-run staging directories. It must be outside `NFS_SOURCE_PATH` and have enough free space for all folders included in one backup run.
 - `RSYNC_BIN` (default: `rsync`)
 - `ARCHIVE_PREFIX` (default: `nfs-backup`)
 - `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`)
@@ -43,19 +62,33 @@ Optional:
 - `SCALE_WAIT_SECONDS` (default: `30`): Fixed delay after scaling down and after restoring a mapped namespace.
 - `LOG_LEVEL` (default: `info`)
 - `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`): Parent directory for fresh per-run replica-state directories. A run never restores state files belonging to an earlier run.
-- `NOTIFY_SUCCESS_URL` (default: empty, disabled)
-- `NOTIFY_FAILURE_URL` (default: empty, disabled)
+- `NOTIFY_SUCCESS_URL` (default: `http://notify.haven/api/v2/notifications/backup`; set to empty to disable)
+- `NOTIFY_FAILURE_URL` (default: `http://notify.haven/api/v2/notifications/error`; set to empty to disable)
 - `NOTIFY_TITLE` (default: `Kubernetes`)
 - `NOTIFY_ASSET` (default: `kube config`)
+- `NOTIFY_SOURCE` (default: `NOTIFY_CALLER`, or `Kubernetes NFS backup`): Haven Notify v2 source identifier. `NOTIFY_CALLER` remains a compatibility fallback.
 - `CLEANUP_KEEP_COUNT` (default: `4`)
 
-Notification payload (success and failure):
+Notifications use the Haven Notify v2 backup and error endpoints. Success payloads include the final archive size and elapsed time:
 
 ```json
 {
   "title": "Kubernetes",
   "asset": "kube config",
-  "backupSizeInMB": 123
+  "sizeBytes": 128974848,
+  "durationSeconds": 125,
+  "source": "Kubernetes NFS backup"
+}
+```
+
+Failures use the v2 error contract:
+
+```json
+{
+  "source": "Kubernetes NFS backup",
+  "message": "Some NFS folders failed to back up",
+  "severity": "critical",
+  "errorCode": "BACKUP_FAILED"
 }
 ```
 
@@ -65,12 +98,13 @@ Notification payload (success and failure):
 - Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces.
 - Ensure `kubectl` context and credentials are present in the runtime.
 - Ensure `7z` is installed in the runtime image/host.
-- When `BACKUP_STAGING_PATH` is enabled, ensure `rsync` is installed and place staging on fast local storage for the shortest offline delta.
+- Ensure `rsync` is installed and keep `BACKUP_STAGING_PATH` on fast local storage. Capacity must accommodate one complete uncompressed backup run in addition to the output archive.
 
 ## Failure Semantics
 
 - Missing required env vars, missing commands, invalid paths, or inability to list namespaces: script exits non-zero immediately.
-- Folder backup failures are counted. One or more successful folder backups continues with cleanup and exits zero; zero successful folder backups exits non-zero.
+- Folder staging failures are counted and failed partial copies are excluded from the archive. One or more successfully staged folders continues to archive creation; zero successfully staged folders exits non-zero.
+- If the single archive operation fails, the run treats every staged folder as failed and exits non-zero.
 - Scale-down warnings/timeouts: logged and counted, backup continues.
 - Restore warnings/timeouts: logged and counted, script does not fail solely because of restore warnings.
 - If `NOTIFY_SUCCESS_URL` is set, success notification is sent at the end of a successful run.

k8s/automated-nfs-backup.sh

@@ -20,7 +20,7 @@ KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 KUBE_CONTEXT="${KUBE_CONTEXT:-}"
 WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
 EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease}"
-BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-}"
+BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-/tmp/k8s-nfs-backup-staging}"
 RSYNC_BIN="${RSYNC_BIN:-rsync}"
 ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
 ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
@@ -33,11 +33,12 @@ SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
 SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
 SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
 TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}"
-NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}"
-NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}"
+NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
+NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
 NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
 NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
-NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}"
+NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes NFS backup}"
+NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
 CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
 RUN_STATE_DIR=""
 RUN_STAGING_DIR=""
@@ -59,6 +60,7 @@ backup_failures=0
 scale_warnings=0
 restore_warnings=0
 total_backup_size_bytes=0
+backup_started_seconds="$SECONDS"
 
 _kubectl() {
     "$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" </dev/null
@@ -97,9 +99,7 @@ validate_inputs() {
     require_cmd "$KUBECTL_BIN"
     require_cmd "$SEVENZ_BIN"
     require_cmd "mktemp"
-    if [[ -n "$BACKUP_STAGING_PATH" ]]; then
-        require_cmd "$RSYNC_BIN"
-    fi
+    require_cmd "$RSYNC_BIN"
     if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
         require_cmd "curl"
     fi
@@ -123,22 +123,20 @@ validate_inputs() {
     fi
     log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
 
-    if [[ -n "$BACKUP_STAGING_PATH" ]]; then
-        local source_real
-        local staging_real
-        mkdir -p "$BACKUP_STAGING_PATH"
-        source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
-        staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
-        case "${staging_real}/" in
-            "${source_real%/}/"*)
-                die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
-                ;;
-        esac
-        if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
-            die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
-        fi
-        log_info "Two-pass rsync staging enabled: ${RUN_STAGING_DIR}"
+    local source_real
+    local staging_real
+    mkdir -p "$BACKUP_STAGING_PATH"
+    source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
+    staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
+    case "${staging_real}/" in
+        "${source_real%/}/"*)
+            die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
+            ;;
+    esac
+    if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
+        die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
     fi
+    log_info "Using local run staging directory: ${RUN_STAGING_DIR}"
 
     if ! _kubectl get namespaces >/dev/null 2>&1; then
         die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
@@ -347,20 +345,12 @@ json_escape() {
     printf '%s' "$value"
 }
 
-backup_size_mb() {
-    local bytes="$1"
-    if (( bytes <= 0 )); then
-        echo 0
-        return 0
-    fi
-    echo $(( (bytes + 1048575) / 1048576 ))
-}
-
 send_backup_notification() {
     local url="${1:-}"
     local title="${2:-}"
     local asset="${3:-}"
-    local size_mb="${4:-0}"
+    local size_bytes="${4:-0}"
+    local duration_seconds="${5:-0}"
 
     [[ -z "$url" ]] && return 0
 
@@ -369,7 +359,9 @@ send_backup_notification() {
 {
   "title": "$(json_escape "$title")",
   "asset": "$(json_escape "$asset")",
-  "backupSizeInMB": ${size_mb}
+  "sizeBytes": ${size_bytes},
+  "durationSeconds": ${duration_seconds},
+  "source": "$(json_escape "$NOTIFY_SOURCE")"
 }
 EOF
 )
@@ -386,20 +378,21 @@ EOF
 send_backup_failure_notification() {
     local url="${1:-}"
     local message="${2:-}"
-    local size_mb="${3:-0}"
+    local size_bytes="${3:-0}"
 
     [[ -z "$url" ]] && return 0
 
     local payload
     payload=$(cat <<EOF
 {
-  "caller": "$(json_escape "$NOTIFY_CALLER")",
+  "source": "$(json_escape "$NOTIFY_SOURCE")",
   "message": "$(json_escape "$message")",
-  "critical": true,
+  "severity": "critical",
+  "errorCode": "BACKUP_FAILED",
   "extra": [
     {
-      "name": "Backup Size (MB)",
-      "value": "${size_mb}"
+      "name": "Archive size",
+      "value": "${size_bytes} B"
     }
   ]
 }
@@ -415,23 +408,17 @@ EOF
     return 0
 }
 
-backup_folder() {
-    local folder_path="${1:?folder path required}"
-    local archive_path="${2:?archive path required}"
-    local working_directory="${3:-}"
-    local archive_input="$folder_path"
+archive_staging() {
+    local archive_path="${1:?archive path required}"
 
-    if [[ -n "$working_directory" ]]; then
-        archive_input="$(basename "$folder_path")"
-        if [[ "$archive_path" != /* ]]; then
-            archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
-        fi
+    if [[ "$archive_path" != /* ]]; then
+        archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
     fi
 
     local -a cmd=(
         "$SEVENZ_BIN" a -t7z
         "$archive_path"
-        "$archive_input"
+        .
         "-m0=${SEVENZ_METHOD}"
         "-mx=${SEVENZ_LEVEL}"
         "-mmt=${SEVENZ_THREADS}"
@@ -442,19 +429,15 @@ backup_folder() {
     fi
 
     local cmd_output rc filtered_output
-    if [[ -n "$working_directory" ]]; then
-        cmd_output="$(cd "$working_directory" && "${cmd[@]}" 2>&1 >/dev/null)"
-    else
-        cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)"
-    fi
+    cmd_output="$(cd "$RUN_STAGING_DIR" && "${cmd[@]}" 2>&1 >/dev/null)"
     rc=$?
     filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
     if [[ -n "$filtered_output" ]]; then
-        log_warn "7z output for '${folder_path}': ${filtered_output}"
+        log_warn "7z output for staging directory '${RUN_STAGING_DIR}': ${filtered_output}"
     fi
     if (( rc == 0 )); then
         return 0
-    elif (( rc == 1 )) && [[ -z "$filtered_output" ]]; then
+    elif (( rc == 1 )); then
         return 0
     fi
     return 1
@@ -496,32 +479,19 @@ cleanup_archives() {
 
 process_folder() {
     local folder_path="${1:?folder path is required}"
-    local archive_path="${2:?archive path is required}"
     local folder_name
     local namespace=""
     local state_file=""
     local has_mapping=0
     local namespace_match_status=0
-    local backup_source_path="$folder_path"
-    local staging_folder=""
-    local final_sync_failed=0
+    local staging_folder
+    local copy_succeeded=0
 
     folder_name="$(basename "$folder_path")"
+    staging_folder="${RUN_STAGING_DIR}/${folder_name}"
     total_folders=$((total_folders + 1))
     log_info "Processing folder: ${folder_name}"
 
-    if [[ -n "$RUN_STAGING_DIR" ]]; then
-        staging_folder="${RUN_STAGING_DIR}/${folder_name}"
-        log_info "Starting live pre-sync for folder '${folder_name}'"
-        if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
-            backup_failures=$((backup_failures + 1))
-            log_error "Backup failed for folder '${folder_name}' during live pre-sync"
-            cleanup_staged_folder "$staging_folder" || true
-            return 0
-        fi
-        backup_source_path="$staging_folder"
-    fi
-
     if namespace="$(namespace_for_folder "$folder_name")"; then
         has_mapping=1
         mapped_folders=$((mapped_folders + 1))
@@ -534,11 +504,9 @@ process_folder() {
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
         sleep "$SCALE_WAIT_SECONDS"
 
-        if [[ -n "$staging_folder" ]]; then
-            log_info "Starting final offline rsync delta for namespace '${namespace}'"
-            if ! sync_folder_to_staging "$folder_path" "$staging_folder"; then
-                final_sync_failed=1
-            fi
+        log_info "Copying quiesced namespace folder '${folder_name}' to local staging"
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
         fi
     else
         namespace_match_status=$?
@@ -548,40 +516,29 @@ process_folder() {
         else
             log_warn "No namespace matched folder '${folder_name}'. Running backup only."
         fi
+
+        log_info "Copying live folder '${folder_name}' to local staging"
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
+        fi
     fi
 
-    if [[ "$has_mapping" -eq 1 && -n "$staging_folder" ]]; then
+    if [[ "$has_mapping" -eq 1 ]]; then
         restore_namespace_replicas "$namespace" "$state_file"
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
         sleep "$SCALE_WAIT_SECONDS"
         rm -f "$state_file"
     fi
 
-    if (( final_sync_failed == 1 )); then
+    if (( copy_succeeded == 0 )); then
         backup_failures=$((backup_failures + 1))
-        log_error "Backup failed for folder '${folder_name}' during final offline rsync"
+        log_error "Backup failed for folder '${folder_name}' while copying to local staging"
         cleanup_staged_folder "$staging_folder" || true
         return 0
     fi
 
-    if backup_folder "$backup_source_path" "$archive_path" "${RUN_STAGING_DIR:-}"; then
-        backup_successes=$((backup_successes + 1))
-        log_info "Added folder '${folder_name}' to archive: ${archive_path}"
-    else
-        backup_failures=$((backup_failures + 1))
-        log_error "Backup failed for folder '${folder_name}'"
-    fi
-
-    if [[ "$has_mapping" -eq 1 && -z "$staging_folder" ]]; then
-        restore_namespace_replicas "$namespace" "$state_file"
-        log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to restore replicas..."
-        sleep "$SCALE_WAIT_SECONDS"
-        rm -f "$state_file"
-    fi
-
-    if [[ -n "$staging_folder" ]]; then
-        cleanup_staged_folder "$staging_folder" || true
-    fi
+    backup_successes=$((backup_successes + 1))
+    log_info "Staged folder '${folder_name}' for the run archive"
 }
 
 print_summary() {
@@ -595,22 +552,31 @@ main() {
     load_namespaces
 
     local folder_path
-    local run_archive_path=""
+    local run_archive_path
     local found=0
+    run_archive_path="$(archive_path_for_run)"
+    log_info "Using single archive for this run: ${run_archive_path}"
+
     for folder_path in "${NFS_SOURCE_PATH}"/*; do
         [[ -d "$folder_path" ]] || continue
         [[ "$(basename "$folder_path")" == .* ]] && continue
-        if [[ -z "$run_archive_path" ]]; then
-            run_archive_path="$(archive_path_for_run)"
-            log_info "Using single archive for this run: ${run_archive_path}"
-        fi
         found=1
-        process_folder "$folder_path" "$run_archive_path"
+        process_folder "$folder_path"
     done
 
     if [[ "$found" -eq 0 ]]; then
         log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}"
-    elif [[ -f "$run_archive_path" ]]; then
+    elif (( backup_successes > 0 )); then
+        log_info "Compressing the complete local staging directory into: ${run_archive_path}"
+        if ! archive_staging "$run_archive_path"; then
+            backup_failures=$((backup_failures + backup_successes))
+            backup_successes=0
+            rm -f -- "$run_archive_path"
+            log_error "Backup failed while compressing the local staging directory"
+        fi
+    fi
+
+    if [[ -f "$run_archive_path" ]]; then
         local file_size_bytes
         file_size_bytes="$(wc -c < "$run_archive_path" | tr -d '[:space:]')"
         if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then
@@ -619,14 +585,13 @@ main() {
     fi
 
     print_summary
-    local size_mb
-    size_mb="$(backup_size_mb "$total_backup_size_bytes")"
+    local duration_seconds=$((SECONDS - backup_started_seconds))
 
     if (( backup_successes == 0 )); then
         send_backup_failure_notification \
             "$NOTIFY_FAILURE_URL" \
             "No NFS folders were backed up successfully" \
-            "$size_mb" || true
+            "$total_backup_size_bytes" || true
         die "No folder backups succeeded" 1
     fi
 
@@ -634,7 +599,7 @@ main() {
         send_backup_failure_notification \
             "$NOTIFY_FAILURE_URL" \
             "Some NFS folders failed to back up" \
-            "$size_mb" || true
+            "$total_backup_size_bytes" || true
         log_warn "Some folder backups failed; continuing because ${backup_successes} folder backup(s) succeeded"
     fi
 
@@ -644,7 +609,8 @@ main() {
         "$NOTIFY_SUCCESS_URL" \
         "$NOTIFY_TITLE" \
         "${NOTIFY_ASSET}" \
-        "$size_mb" || true
+        "$total_backup_size_bytes" \
+        "$duration_seconds" || true
 }
 
 main "$@"

k8s/backup-k3s-control-plane-config.sh

@@ -0,0 +1,473 @@
+#!/usr/bin/env bash
+
+# Back up the k3s control-plane configuration and credentials, without taking
+# a live datastore snapshot or changing the running k3s service.
+
+SCRIPT_NAME="$(basename "$0")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/common.sh
+source "${SCRIPT_DIR}/lib/common.sh"
+
+# Keep the local archive and temporary files readable only by this process.
+umask 077
+
+# Source paths. K3S_CONFIG_DIR is the only required source; the service files
+# and server state entries are included only when present.
+K3S_CONFIG_DIR="${K3S_CONFIG_DIR:-/etc/rancher/k3s}"
+K3S_SERVER_DIR="${K3S_SERVER_DIR:-/var/lib/rancher/k3s/server}"
+K3S_SERVICE_FILES="${K3S_SERVICE_FILES-/etc/systemd/system/k3s.service:/etc/systemd/system/k3s.service.env:/etc/default/k3s:/etc/sysconfig/k3s}"
+
+# Archive and local working settings.
+ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-k3s-control-plane-config}"
+ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y-%m-%d_%H-%M-%S}"
+LOCAL_TMP_PARENT="${LOCAL_TMP_PARENT:-${TMPDIR:-/tmp}}"
+LOCK_PATH="${LOCK_PATH:-/var/lock/k3s-control-plane-config-backup.lock}"
+
+# Remote destination settings.
+REMOTE_USER="${REMOTE_USER:-ivanch}"
+REMOTE_HOST="${REMOTE_HOST:-nas.haven}"
+REMOTE_DIR="${REMOTE_DIR:-/export/Backup/k8s}"
+REMOTE_IDENTITY_FILE="${REMOTE_IDENTITY_FILE:-}"
+SSH_PORT="${SSH_PORT:-}"
+CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
+
+# Commands are configurable to support hosts with non-standard paths and
+# isolated tests.
+ZIP_BIN="${ZIP_BIN:-zip}"
+UNZIP_BIN="${UNZIP_BIN:-unzip}"
+SSH_BIN="${SSH_BIN:-ssh}"
+SCP_BIN="${SCP_BIN:-scp}"
+CURL_BIN="${CURL_BIN:-curl}"
+FLOCK_BIN="${FLOCK_BIN:-flock}"
+MKTEMP_BIN="${MKTEMP_BIN:-mktemp}"
+DATE_BIN="${DATE_BIN:-date}"
+REALPATH_BIN="${REALPATH_BIN:-realpath}"
+REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
+
+# Notifications use the Haven Notify v2 backup and error contracts.
+# Set either URL to an empty value to disable that notification.
+NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
+NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
+NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
+NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
+NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
+NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
+
+declare -a ARCHIVE_ENTRIES=()
+declare -a SERVICE_FILE_LIST=()
+declare -a SSH_ARGS=()
+declare -a SCP_ARGS=()
+declare -a PROTECTED_PATHS=()
+readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token)
+
+RUN_TMP_DIR=""
+LOCAL_ARCHIVE_PATH=""
+ARCHIVE_NAME=""
+ARCHIVE_SIZE_BYTES=0
+CLEANUP_KEEP_COUNT_INT=0
+LOCK_FD=""
+FAILURE_NOTIFICATION_SENT=0
+BACKUP_STARTED_SECONDS="$SECONDS"
+
+json_escape() {
+    local value="${1:-}"
+    value="${value//\\/\\\\}"
+    value="${value//\"/\\\"}"
+    value="${value//$'\n'/ }"
+    printf '%s' "$value"
+}
+
+send_backup_notification() {
+    local url="${1:-}"
+    local size_bytes="${2:-0}"
+    local duration_seconds="${3:-0}"
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "title": "$(json_escape "$NOTIFY_TITLE")",
+  "asset": "$(json_escape "$NOTIFY_ASSET")",
+  "sizeBytes": ${size_bytes},
+  "durationSeconds": ${duration_seconds},
+  "source": "$(json_escape "$NOTIFY_SOURCE")"
+}
+EOF
+)
+
+    if ! "$CURL_BIN" -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup notification"
+        return 1
+    fi
+}
+
+send_backup_failure_notification() {
+    local url="${1:-}"
+    local size_bytes="${2:-0}"
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "source": "$(json_escape "$NOTIFY_SOURCE")",
+  "message": "Control-plane configuration backup failed",
+  "severity": "critical",
+  "errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED",
+  "extra": [
+    {
+      "name": "Archive size",
+      "value": "${size_bytes} B"
+    }
+  ]
+}
+EOF
+)
+
+    if ! "$CURL_BIN" -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup failure notification"
+        return 1
+    fi
+}
+
+notify_failure_once() {
+    if (( FAILURE_NOTIFICATION_SENT == 1 )); then
+        return 0
+    fi
+    FAILURE_NOTIFICATION_SENT=1
+    send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$ARCHIVE_SIZE_BYTES" || true
+}
+
+cleanup() {
+    local exit_code=$?
+    trap - EXIT
+
+    if [[ -n "$RUN_TMP_DIR" && -d "$RUN_TMP_DIR" ]]; then
+        rm -rf -- "$RUN_TMP_DIR" || log_warn "Failed to clean local temporary files"
+    fi
+
+    if (( exit_code == 0 )); then
+        log_info "Finished successfully"
+    else
+        notify_failure_once
+        log_error "Control-plane configuration backup failed"
+    fi
+
+    exit "$exit_code"
+}
+trap cleanup EXIT
+
+shell_quote() {
+    local value="${1:-}"
+    value="${value//\'/\'\"\'\"\'}"
+    printf "'%s'" "$value"
+}
+
+require_absolute_path() {
+    local path="${1:?path is required}"
+    local description="${2:-path}"
+    if [[ "$path" != /* ]]; then
+        die "${description} must be an absolute path"
+    fi
+}
+
+path_is_ancestor_or_same() {
+    local ancestor="${1:?ancestor path is required}"
+    local path="${2:?path is required}"
+
+    [[ "$ancestor" == "/" || "$path" == "$ancestor" || "$path" == "$ancestor"/* ]]
+}
+
+paths_overlap() {
+    local first="${1:?first path is required}"
+    local second="${2:?second path is required}"
+
+    path_is_ancestor_or_same "$first" "$second" || \
+        path_is_ancestor_or_same "$second" "$first"
+}
+
+canonicalize_protected_path() {
+    local path="${1:?protected path is required}"
+    local canonical_path
+    local canonical_parent
+    local basename
+
+    if [[ -e "$path" || -L "$path" ]]; then
+        canonical_path="$("$REALPATH_BIN" -e -- "$path" 2>/dev/null)" || return 1
+        printf '%s' "$canonical_path"
+        return 0
+    fi
+
+    basename="${path##*/}"
+    canonical_parent="${path%/*}"
+    if ! canonical_parent="$("$REALPATH_BIN" -e -- "$canonical_parent" 2>/dev/null)"; then
+        canonical_parent="$("$REALPATH_BIN" -m -- "$canonical_parent" 2>/dev/null)" || return 1
+    fi
+    printf '%s/%s' "${canonical_parent%/}" "$basename"
+}
+
+determine_protected_paths() {
+    local canonical_server_dir
+    local protected_path
+
+    if ! canonical_server_dir="$("$REALPATH_BIN" -e -- "$K3S_SERVER_DIR" 2>/dev/null)"; then
+        canonical_server_dir="$("$REALPATH_BIN" -m -- "$K3S_SERVER_DIR" 2>/dev/null)" || \
+            die "Configured source path is not safe to archive"
+    fi
+
+    PROTECTED_PATHS=()
+    if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/db")"; then
+        die "Configured source path is not safe to archive"
+    fi
+    PROTECTED_PATHS+=("$protected_path")
+    if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/kine.sock")"; then
+        die "Configured source path is not safe to archive"
+    fi
+    PROTECTED_PATHS+=("$protected_path")
+}
+
+add_archive_entry_if_present() {
+    local absolute_path="${1:?source path is required}"
+    local canonical_path
+    local relative_path
+    local protected_path
+
+    if [[ "$absolute_path" =~ (^|/)(\.|\.\.)(/|$) ]]; then
+        die "Configured source path is not safe to archive"
+    fi
+
+    if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then
+        return 0
+    fi
+
+    if ! canonical_path="$("$REALPATH_BIN" -e -- "$absolute_path" 2>/dev/null)"; then
+        die "Configured source path is not safe to archive"
+    fi
+
+    for protected_path in "${PROTECTED_PATHS[@]}"; do
+        if paths_overlap "$canonical_path" "$protected_path"; then
+            die "Configured source path is not safe to archive"
+        fi
+    done
+
+    relative_path="${canonical_path#/}"
+    if [[ -z "$relative_path" ]]; then
+        die "Configured source path is not safe to archive"
+    fi
+    ARCHIVE_ENTRIES+=("$relative_path")
+}
+
+build_remote_arguments() {
+    SSH_ARGS=()
+    SCP_ARGS=()
+    if [[ -n "$REMOTE_IDENTITY_FILE" ]]; then
+        SSH_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
+        SCP_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
+    fi
+    if [[ -n "$SSH_PORT" ]]; then
+        SSH_ARGS+=(-p "$SSH_PORT")
+        SCP_ARGS+=(-P "$SSH_PORT")
+    fi
+}
+
+validate_inputs() {
+    require_cmd "$ZIP_BIN"
+    require_cmd "$UNZIP_BIN"
+    require_cmd "$SSH_BIN"
+    require_cmd "$SCP_BIN"
+    require_cmd "$FLOCK_BIN"
+    require_cmd "$MKTEMP_BIN"
+    require_cmd "$DATE_BIN"
+    require_cmd "$REALPATH_BIN"
+    if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
+        require_cmd "$CURL_BIN"
+    fi
+
+    require_absolute_path "$K3S_CONFIG_DIR" "K3S_CONFIG_DIR"
+    require_absolute_path "$K3S_SERVER_DIR" "K3S_SERVER_DIR"
+    require_absolute_path "$LOCAL_TMP_PARENT" "LOCAL_TMP_PARENT"
+    require_absolute_path "$LOCK_PATH" "LOCK_PATH"
+    require_absolute_path "$REMOTE_DIR" "REMOTE_DIR"
+
+    if [[ ! -d "$K3S_CONFIG_DIR" ]]; then
+        die "K3S_CONFIG_DIR is not an available directory"
+    fi
+    determine_protected_paths
+    if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then
+        die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens"
+    fi
+    if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then
+        die "CLEANUP_KEEP_COUNT must be an integer >= 0"
+    fi
+    CLEANUP_KEEP_COUNT_INT=$((10#$CLEANUP_KEEP_COUNT))
+    if [[ -n "$SSH_PORT" && ! "$SSH_PORT" =~ ^[0-9]+$ ]]; then
+        die "SSH_PORT must be numeric"
+    fi
+    if [[ -z "$REMOTE_USER" || -z "$REMOTE_HOST" ]]; then
+        die "REMOTE_USER and REMOTE_HOST must not be empty"
+    fi
+    if [[ ! "$REMOTE_USER" =~ ^[[:alnum:]_][[:alnum:]_.-]*$ ]]; then
+        die "REMOTE_USER may contain only letters, numbers, underscores, dots, and hyphens"
+    fi
+    if [[ ! "$REMOTE_HOST" =~ ^[[:alnum:]][[:alnum:].-]*$ ]]; then
+        die "REMOTE_HOST may contain only letters, numbers, dots, and hyphens"
+    fi
+    if [[ ! "$REMOTE_DIR" =~ ^/([[:alnum:]_.-]+/)*[[:alnum:]_.-]+/?$ || "$REMOTE_DIR" == */./* || "$REMOTE_DIR" == */../* || "$REMOTE_DIR" == */. || "$REMOTE_DIR" == */.. ]]; then
+        die "REMOTE_DIR must be an absolute path containing only letters, numbers, dots, underscores, hyphens, and slashes without dot segments"
+    fi
+
+    mkdir -p -- "$LOCAL_TMP_PARENT"
+    mkdir -p -- "$(dirname -- "$LOCK_PATH")"
+    build_remote_arguments
+}
+
+collect_sources() {
+    local service_file
+    local server_entry
+
+    IFS=':' read -r -a SERVICE_FILE_LIST <<< "$K3S_SERVICE_FILES"
+
+    # These are the only server entries ever included. In particular, server/db
+    # and server/kine.sock are intentionally not archive inputs.
+    add_archive_entry_if_present "$K3S_CONFIG_DIR"
+    for service_file in "${SERVICE_FILE_LIST[@]}"; do
+        [[ -z "$service_file" ]] && continue
+        require_absolute_path "$service_file" "K3S_SERVICE_FILES entry"
+        add_archive_entry_if_present "$service_file"
+    done
+    for server_entry in "${K3S_SERVER_ENTRIES[@]}"; do
+        add_archive_entry_if_present "${K3S_SERVER_DIR%/}/${server_entry}"
+    done
+
+    if (( ${#ARCHIVE_ENTRIES[@]} == 0 )); then
+        die "No k3s configuration sources are available"
+    fi
+}
+
+acquire_lock() {
+    exec {LOCK_FD}>"$LOCK_PATH"
+    if ! "$FLOCK_BIN" -n "$LOCK_FD"; then
+        die "Another k3s control-plane configuration backup is already running"
+    fi
+}
+
+make_archive_name() {
+    local timestamp
+    timestamp="$("$DATE_BIN" +"$ARCHIVE_TS_FORMAT")"
+    if [[ ! "$timestamp" =~ ^[[:alnum:]_.-]+$ ]]; then
+        die "ARCHIVE_TS_FORMAT produced an unsafe archive timestamp"
+    fi
+    ARCHIVE_NAME="${ARCHIVE_PREFIX}_${timestamp}.zip"
+}
+
+create_and_validate_archive() {
+    if ! RUN_TMP_DIR="$("$MKTEMP_BIN" -d "${LOCAL_TMP_PARENT%/}/k3s-control-plane-config.XXXXXX")"; then
+        die "Unable to create local temporary directory"
+    fi
+    LOCAL_ARCHIVE_PATH="${RUN_TMP_DIR}/${ARCHIVE_NAME}"
+
+    log_info "Creating k3s control-plane configuration archive"
+    (
+        cd /
+        "$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" -- "${ARCHIVE_ENTRIES[@]}" >/dev/null
+    )
+    "$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null
+
+    if [[ ! -f "$LOCAL_ARCHIVE_PATH" ]]; then
+        die "Archive creation did not produce an archive"
+    fi
+    ARCHIVE_SIZE_BYTES="$(wc -c < "$LOCAL_ARCHIVE_PATH" | tr -d '[:space:]')"
+    if [[ ! "$ARCHIVE_SIZE_BYTES" =~ ^[0-9]+$ ]]; then
+        die "Unable to determine archive size"
+    fi
+    log_info "Validated local archive"
+}
+
+run_ssh() {
+    "$SSH_BIN" "${SSH_ARGS[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "$@"
+}
+
+remove_remote_temporary_archive() {
+    local remote_temp_path="${1:?temporary remote path is required}"
+    run_ssh "rm -f -- $(shell_quote "$remote_temp_path")" >/dev/null 2>&1 || true
+}
+
+publish_archive() {
+    local remote_final_path="${REMOTE_DIR%/}/${ARCHIVE_NAME}"
+    local remote_temp_path="${REMOTE_DIR%/}/.${ARCHIVE_NAME}.upload-$$.tmp"
+    local remote_spec="${REMOTE_USER}@${REMOTE_HOST}:${remote_temp_path}"
+    local move_command
+
+    run_ssh "mkdir -p -- $(shell_quote "$REMOTE_DIR")"
+
+    if ! "$SCP_BIN" "${SCP_ARGS[@]}" -- "$LOCAL_ARCHIVE_PATH" "$remote_spec"; then
+        log_error "Archive upload failed"
+        remove_remote_temporary_archive "$remote_temp_path"
+        return 1
+    fi
+
+    move_command="if [ -e $(shell_quote "$remote_final_path") ]; then exit 17; fi; mv -- $(shell_quote "$remote_temp_path") $(shell_quote "$remote_final_path")"
+    if ! run_ssh "$move_command"; then
+        log_error "Remote archive publish failed"
+        remove_remote_temporary_archive "$remote_temp_path"
+        return 1
+    fi
+
+    log_info "Published archive successfully"
+}
+
+cleanup_remote_archives() {
+    if (( CLEANUP_KEEP_COUNT_INT == 0 )); then
+        log_info "Remote retention disabled"
+        return 0
+    fi
+
+    local remote_cleanup_command
+    remote_cleanup_command="$(shell_quote "$REMOTE_BASH_BIN") -s -- $(shell_quote "$REMOTE_DIR") $(shell_quote "$ARCHIVE_PREFIX") $(shell_quote "$CLEANUP_KEEP_COUNT_INT")"
+
+    run_ssh "$remote_cleanup_command" <<'REMOTE_CLEANUP'
+set -Eeuo pipefail
+
+remote_dir="$1"
+archive_prefix="$2"
+keep_count="$3"
+
+mapfile -d '' -t sorted_archives < <(
+    find -P "$remote_dir" -maxdepth 1 -type f \
+        -name "${archive_prefix}_*.zip" -printf '%T@\t%p\0' |
+        sort -z -t $'\t' -k1,1nr
+)
+
+if (( ${#sorted_archives[@]} <= keep_count )); then
+    exit 0
+fi
+
+for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do
+    archive_record="${sorted_archives[$index]}"
+    archive_path="${archive_record#*$'\t'}"
+    rm -f -- "$archive_path"
+done
+REMOTE_CLEANUP
+
+    log_info "Remote archive retention completed"
+}
+
+main() {
+    validate_inputs
+    acquire_lock
+    collect_sources
+    make_archive_name
+    create_and_validate_archive
+    publish_archive
+    cleanup_remote_archives
+
+    # Notifications are informational: an unavailable notification endpoint
+    # must not change the result of a completed backup.
+    local duration_seconds=$((SECONDS - BACKUP_STARTED_SECONDS))
+    send_backup_notification "$NOTIFY_SUCCESS_URL" "$ARCHIVE_SIZE_BYTES" "$duration_seconds" || true
+}
+
+main "$@"

k8s/examples/automated-nfs-backup.env.example

@@ -5,7 +5,7 @@ KUBECTL_BIN=kubectl
 KUBE_CONTEXT=
 WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller
 EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default
-BACKUP_STAGING_PATH=
+BACKUP_STAGING_PATH=/tmp/k8s-nfs-backup-staging
 RSYNC_BIN=rsync
 ARCHIVE_PREFIX=nfs-backup
 ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S
@@ -18,8 +18,9 @@ SCALE_RETRY_DELAY_SECONDS=5
 SCALE_WAIT_SECONDS=30
 LOG_LEVEL=info
 TMP_STATE_DIR=/tmp/k8s-nfs-backup
-NOTIFY_SUCCESS_URL=
-NOTIFY_FAILURE_URL=
+NOTIFY_SUCCESS_URL=http://notify.haven/api/v2/notifications/backup
+NOTIFY_FAILURE_URL=http://notify.haven/api/v2/notifications/error
 NOTIFY_TITLE=Kubernetes
 NOTIFY_ASSET=kube config
+NOTIFY_SOURCE=Kubernetes NFS backup
 CLEANUP_KEEP_COUNT=5

k8s/test/test-automated-nfs-backup.sh

@@ -114,13 +114,11 @@ trace_has_scale() {
     grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file"
 }
 
-# Asserts the real script invoked 7z as `7z a -t7z <archive.7z> <folder> ...`
-# for the given folder. The archive is the 3rd token (ends in .7z) and the
-# folder path is the 4th token, so a swap would fail to match.
-sevenz_called_for() {
-    local folder_name="$1"
-
-    grep -Eq -- "a -t7z [^ ]+\\.7z .*/${folder_name}( |$)" "${TMPD}/7z_invocations.log"
+# Asserts the real script invoked 7z exactly once for the complete contents of
+# the run staging directory.
+sevenz_called_once_for_staging() {
+    [[ "$(wc -l < "${TMPD}/7z_invocations.log" | tr -d '[:space:]')" -eq 1 ]] &&
+        grep -Eq -- '^a -t7z [^ ]+\.7z \. ' "${TMPD}/7z_invocations.log"
 }
 
 write_fake_script() {
@@ -213,16 +211,17 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
     archive="${3:?archive path required}"
     folder="${4:?folder path required}"
     folder="${folder%/}"
-    folder_name="${folder##*/}"
-    printf '7z %s\n' "$folder_name" >> "${TMPD}/event_trace.log"
+    printf '7z staging\n' >> "${TMPD}/event_trace.log"
 
-    # Simulate a backup failure for configured folders BEFORE recording a
-    # successful archive, so 7z_trace.log only lists folders actually archived.
-    if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
+    if [[ -s "${TMPD}/fail_archive.txt" ]]; then
+        touch "$archive"
         exit 2
     fi
 
-    printf '%s\n' "$folder_name" >> "${TMPD}/7z_trace.log"
+    for staged_folder in ./*; do
+        [[ -d "$staged_folder" ]] || continue
+        printf '%s\n' "${staged_folder##*/}" >> "${TMPD}/7z_trace.log"
+    done
     touch "$archive"
 fi
 
@@ -249,8 +248,8 @@ count=$((count + 1))
 printf '%s\n' "$count" > "$count_file"
 printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
 
-if (( count == 2 )) && [[ -f "${TMPD}/fail_rsync_second_pass.txt" ]] && grep -Fxq -- "$folder_name" "${TMPD}/fail_rsync_second_pass.txt"; then
-    printf 'simulated final rsync failure for %s\n' "$folder_name" >&2
+if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
+    printf 'simulated rsync failure for %s\n' "$folder_name" >&2
     exit 23
 fi
 
@@ -279,7 +278,7 @@ for ((i = 1; i <= $#; i++)); do
 done
 
 body="${body//$'\n'/ }"
-printf '%s %s\n' "$url" "${body:0:120}" >> "${TMPD}/curl_trace.log"
+printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
 exit 0
 EOF
 }
@@ -292,13 +291,14 @@ run_backup() {
     shift 4
     local -a env_overrides=("$@")
 
-    env "${env_overrides[@]}" \
+    env BACKUP_STAGING_PATH="${state_dir}/staging" \
+        "${env_overrides[@]}" \
         NFS_SOURCE_PATH="$source_dir" \
         BACKUP_OUTPUT_PATH="$output_dir" \
         BACKUP_PASSWORD="" \
         ARCHIVE_PREFIX="test" \
-        NOTIFY_SUCCESS_URL="http://example.invalid/success" \
-        NOTIFY_FAILURE_URL="http://example.invalid/failure" \
+        NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
+        NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
         KUBECTL_BIN="${TMPD}/bin/kubectl" \
         SEVENZ_BIN="${TMPD}/bin/7z" \
         RSYNC_BIN="${TMPD}/bin/rsync" \
@@ -311,9 +311,10 @@ run_backup() {
 
 write_fake_binaries
 export TMPD
+: > "${TMPD}/fail_archive.txt"
 
-# Scenario A: a mapped folder is scaled down and restored, while an unmapped
-# folder is backed up directly. Both folders share one archive.
+# Scenario A: a mapped folder is scaled down, copied, and restored, while
+# unmapped and excluded folders are copied live. One 7z call archives them all.
 success_dir="${TMPD}/success"
 success_source="${success_dir}/source"
 success_output="${success_dir}/out"
@@ -330,6 +331,7 @@ printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/7z_trace.log"
 : > "${TMPD}/7z_invocations.log"
 : > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
 
 set +e
 run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log"
@@ -342,14 +344,17 @@ assert "success scenario reports three backup successes" file_contains "${succes
 assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
 assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
 assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
-assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
+assert "success scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "success scenario sends the archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes": 0'
+assert "success scenario sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "Kubernetes NFS backup"'
 assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
 assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system"
 assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace"
 assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns"
-assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns"
-assert "success scenario invokes 7z for excluded kube-system" sevenz_called_for "kube-system"
-assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data"
+assert "success scenario invokes 7z exactly once for staging" sevenz_called_once_for_staging
+assert "success scenario archives mapped app-ns" file_contains "${TMPD}/7z_trace.log" "app-ns"
+assert "success scenario archives excluded kube-system" file_contains "${TMPD}/7z_trace.log" "kube-system"
+assert "success scenario archives unmapped loose-data" file_contains "${TMPD}/7z_trace.log" "loose-data"
 
 # Scenario A2: the exclusion list can be replaced through the environment.
 custom_exclude_dir="${TMPD}/custom-exclude"
@@ -363,6 +368,7 @@ printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/fail_folders.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
 : > "${TMPD}/curl_trace.log"
 
 set +e
@@ -374,8 +380,8 @@ assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_ex
 assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
 assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
 
-# Scenario A3: two-pass rsync copies data while live, captures a small delta
-# while offline, restores the namespace, and only then starts compression.
+# Scenario A3: the mapped namespace is scaled down before its single local copy,
+# restored immediately afterward, and compression starts only after restoration.
 staging_dir="${TMPD}/staging"
 staging_source="${staging_dir}/source"
 staging_output="${staging_dir}/out"
@@ -386,7 +392,6 @@ printf 'data\n' > "${staging_source}/app-ns/file.txt"
 printf 'app-ns\n' > "${TMPD}/namespaces.txt"
 printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/fail_folders.txt"
-: > "${TMPD}/fail_rsync_second_pass.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
 : > "${TMPD}/7z_invocations.log"
@@ -402,15 +407,16 @@ set -e
 assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
 assert "staging scenario creates an archive" archive_exists "$staging_output"
 assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
-    "rsync app-ns 1" \
     "scale app-ns --replicas=0" \
-    "rsync app-ns 2" \
+    "rsync app-ns 1" \
     "scale app-ns --replicas=3" \
-    "7z app-ns"
+    "7z staging"
+assert "staging scenario copies app-ns only once" file_contains "${TMPD}/rsync_counts/app-ns" "1"
+assert "staging scenario invokes 7z exactly once" sevenz_called_once_for_staging
 assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
 assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
 
-# Scenario A4: a failed final delta never archives the incomplete staging copy,
+# Scenario A4: a failed local copy never archives the incomplete staging copy,
 # but the namespace is still restored immediately.
 rsync_failure_dir="${TMPD}/rsync-failure"
 rsync_failure_source="${rsync_failure_dir}/source"
@@ -421,8 +427,7 @@ mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failur
 printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
 printf 'app-ns\n' > "${TMPD}/namespaces.txt"
 printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
-: > "${TMPD}/fail_folders.txt"
-printf 'app-ns\n' > "${TMPD}/fail_rsync_second_pass.txt"
+printf 'app-ns\n' > "${TMPD}/fail_folders.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
 : > "${TMPD}/curl_trace.log"
@@ -434,11 +439,10 @@ run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state
 rsync_failure_exit_code=$?
 set -e
 
-assert "final rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
-assert "final rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
-assert "final rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z app-ns"
-assert "final rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
-: > "${TMPD}/fail_rsync_second_pass.txt"
+assert "rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
+assert "rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
+assert "rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z staging"
+assert "rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
 
 # Scenario B: one namespace backup fails, but another succeeds. Both namespaces
 # are restored, retention still runs, and the script reports success.
@@ -457,7 +461,9 @@ printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
 printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
 : > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
 
 set +e
 run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log"
@@ -472,10 +478,17 @@ assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/s
 assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
 assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
 assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
-assert "partial failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
-assert "partial failure scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
+assert "partial failure scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "partial failure scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "partial failure scenario marks the error as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
+assert "partial failure scenario includes the backup error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "BACKUP_FAILED"'
 assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
-assert "partial failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
+assert "partial failure scenario archives the successfully staged folder" file_contains "${TMPD}/7z_trace.log" "good-ns"
+assert "partial failure scenario invokes 7z exactly once" sevenz_called_once_for_staging
+assert "partial failure scenario compresses after all namespaces are restored" events_are_ordered "${TMPD}/event_trace.log" \
+    "scale fail-ns --replicas=4" \
+    "scale good-ns --replicas=2" \
+    "7z staging"
 assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4
 assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z"
 
@@ -502,8 +515,35 @@ set -e
 assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code"
 assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0"
 assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1"
-assert "all-failed scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
-assert "all-failed scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
+assert "all-failed scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "all-failed scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+
+# Scenario C2: a failed single archive operation invalidates all staged folders
+# and removes the partial archive left by 7z.
+archive_failure_dir="${TMPD}/archive-failure"
+archive_failure_source="${archive_failure_dir}/source"
+archive_failure_output="${archive_failure_dir}/out"
+archive_failure_state="${archive_failure_dir}/state"
+mkdir -p "${archive_failure_source}/archive-ns" "$archive_failure_output" "$archive_failure_state"
+printf 'data\n' > "${archive_failure_source}/archive-ns/file.txt"
+printf 'archive-ns\n' > "${TMPD}/namespaces.txt"
+printf 'archive-ns:2\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+printf 'fail\n' > "${TMPD}/fail_archive.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$archive_failure_source" "$archive_failure_output" "$archive_failure_state" "${archive_failure_dir}/archive-failure.log"
+archive_failure_exit_code=$?
+set -e
+
+assert "archive failure exits non-zero" is_nonzero "$archive_failure_exit_code"
+assert "archive failure restores archive-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "archive-ns" "2"
+assert "archive failure reports zero backup successes" file_contains "${archive_failure_dir}/archive-failure.log" "backup_successes=0"
+assert "archive failure removes the partial archive" archive_count_is "$archive_failure_output" 0
+assert "archive failure does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+: > "${TMPD}/fail_archive.txt"
 
 # Scenario D: no source folders means zero successful backups, which must also
 # return non-zero rather than reporting an empty run as successful.
@@ -526,8 +566,8 @@ set -e
 
 assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code"
 assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0"
-assert "empty scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
-assert "empty scenario does not send success notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/success"
+assert "empty scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "empty scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
 
 # Scenario E: abnormal abort while a namespace is still scaled to zero.
 # We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
@@ -539,7 +579,8 @@ abort_dir="${TMPD}/abort"
 abort_source="${abort_dir}/source"
 abort_output="${abort_dir}/out"
 abort_state="${abort_dir}/state"
-mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}"
+abort_work="${abort_dir}/work"
+mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" "${abort_work}"
 printf 'data\n' > "${abort_source}/crash-ns/file.txt"
 printf 'crash-ns\n' > "${TMPD}/namespaces.txt"
 printf 'crash-ns:5\n' > "${TMPD}/replicas.txt"
@@ -558,6 +599,7 @@ NOTIFY_SUCCESS_URL="" \
 NOTIFY_FAILURE_URL="" \
 KUBECTL_BIN="${TMPD}/bin/kubectl" \
 SEVENZ_BIN="${TMPD}/bin/7z" \
+BACKUP_STAGING_PATH="${abort_work}" \
 SCALE_WAIT_SECONDS="not-a-number" \
 TMP_STATE_DIR="${abort_state}" \
 PATH="${TMPD}/bin:${PATH}" \

k8s/test/test-backup-k3s-control-plane-config.sh

@@ -0,0 +1,439 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/backup-k3s-control-plane-config.sh"
+TMPD="$(mktemp -d "${TMPDIR:-/tmp}/k3s-control-plane-config-test.XXXXXX")"
+
+tests_total=0
+tests_passed=0
+tests_failed=0
+
+cleanup() {
+    rm -rf -- "$TMPD"
+}
+trap cleanup EXIT
+
+assert() {
+    local message="$1"
+    shift
+    tests_total=$((tests_total + 1))
+    if "$@"; then
+        printf 'PASS: %s\n' "$message"
+        tests_passed=$((tests_passed + 1))
+    else
+        printf 'FAIL: %s\n' "$message"
+        tests_failed=$((tests_failed + 1))
+    fi
+}
+
+is_zero() { [[ "$1" -eq 0 ]]; }
+is_nonzero() { [[ "$1" -ne 0 ]]; }
+path_exists() { [[ -e "$1" ]]; }
+path_does_not_exist() { [[ ! -e "$1" ]]; }
+file_contains() { grep -Fq -- "$2" "$1"; }
+file_lacks() { [[ ! -e "$1" ]] || ! grep -Fq -- "$2" "$1"; }
+file_is_empty() { [[ ! -s "$1" ]]; }
+file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; }
+
+matching_archive_count_is() {
+    local directory="$1"
+    local expected="$2"
+    local -a archives=()
+    shopt -s nullglob
+    archives=("${directory}"/k3s-control-plane-config_*.zip)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -eq "$expected" ]]
+}
+
+directory_is_empty() {
+    local directory="$1"
+    local -a entries=()
+    shopt -s nullglob dotglob
+    entries=("${directory}"/*)
+    shopt -u nullglob dotglob
+    [[ "${#entries[@]}" -eq 0 ]]
+}
+
+write_fake_script() {
+    local path="$1"
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        printf '%s\n' "$line"
+    done > "$path"
+    chmod +x "$path"
+}
+
+write_fake_binaries() {
+    mkdir -p "${TMPD}/bin"
+
+    write_fake_script "${TMPD}/bin/zip" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[[ "${1:-}" == "-r" ]]
+archive="${2:?archive path required}"
+shift 2
+[[ "${1:-}" == "--" ]]
+shift
+: > "$archive"
+for entry in "$@"; do
+    source_path="/$entry"
+    if [[ -d "$source_path" ]]; then
+        while IFS= read -r path; do
+            printf '%s\n' "${path#/}" >> "$archive"
+        done < <(find "$source_path" -print | sort)
+    else
+        printf '%s\n' "$entry" >> "$archive"
+    fi
+done
+printf '%s\n' "$*" >> "${TMPD}/zip_invocations.log"
+EOF
+
+    write_fake_script "${TMPD}/bin/unzip" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[[ "${1:-}" == "-t" ]]
+archive="${2:?archive path required}"
+[[ -f "$archive" && -s "$archive" ]]
+printf '%s\n' "$archive" >> "${TMPD}/unzip_trace.log"
+EOF
+
+    write_fake_script "${TMPD}/bin/scp" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+source_path="${@: -2:1}"
+destination="${@: -1}"
+remote_path="${destination#*:}"
+if [[ "$remote_path" == *"'"* ]]; then
+    printf 'scp destination contains literal single quotes: %s\n' "$destination" >&2
+    exit 64
+fi
+remote_name="${remote_path##*/}"
+target_path="${FAKE_REMOTE_ROOT}/${remote_name}"
+
+cp -- "$source_path" "$target_path"
+printf '%s\n' "$remote_name" >> "${TMPD}/scp_trace.log"
+if [[ -f "${TMPD}/fail_scp" ]]; then
+    exit 42
+fi
+EOF
+
+    write_fake_script "${TMPD}/bin/ssh" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+command="${@: -1}"
+printf '%s\n' "$command" >> "${TMPD}/ssh_trace.log"
+
+quoted_values=()
+while IFS= read -r value; do
+    quoted_values+=("${value#\'}")
+    quoted_values[${#quoted_values[@]}-1]="${quoted_values[${#quoted_values[@]}-1]%\'}"
+done < <(printf '%s\n' "$command" | grep -o "'[^']*'" || true)
+
+if [[ "$command" == mkdir\ -p* ]]; then
+    mkdir -p -- "$FAKE_REMOTE_ROOT"
+    exit 0
+fi
+
+if [[ "$command" == rm\ -f* ]]; then
+    temp_name="${quoted_values[0]##*/}"
+    rm -f -- "${FAKE_REMOTE_ROOT}/${temp_name}"
+    exit 0
+fi
+
+if [[ "$command" == *" mv -- "* ]]; then
+    value_count="${#quoted_values[@]}"
+    temp_name="${quoted_values[value_count-2]##*/}"
+    final_name="${quoted_values[value_count-1]##*/}"
+    [[ ! -e "${FAKE_REMOTE_ROOT}/${final_name}" ]]
+    mv -- "${FAKE_REMOTE_ROOT}/${temp_name}" "${FAKE_REMOTE_ROOT}/${final_name}"
+    exit 0
+fi
+
+if [[ "$command" == *" -s -- "* ]]; then
+    value_count="${#quoted_values[@]}"
+    (
+        cd -- "$FAKE_REMOTE_ROOT"
+        bash -s -- "$FAKE_REMOTE_ROOT" "${quoted_values[value_count-2]}" "${quoted_values[value_count-1]}"
+    )
+    exit 0
+fi
+
+exit 1
+EOF
+
+    write_fake_script "${TMPD}/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+url=""
+body=""
+for ((i = 1; i <= $#; i++)); do
+    case "${!i}" in
+        POST|-d)
+            next=$((i + 1))
+            if [[ "${!i}" == "POST" ]]; then
+                url="${!next}"
+            else
+                body="${!next}"
+            fi
+            ;;
+    esac
+done
+body="${body//$'\n'/ }"
+printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
+[[ ! -f "${TMPD}/fail_curl" ]]
+EOF
+
+    write_fake_script "${TMPD}/bin/flock" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/date" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+printf '%s\n' '2026-07-21_12-00-00'
+EOF
+}
+
+run_backup() {
+    local fixture_root="$1"
+    local remote_root="$2"
+    local local_temp_parent="$3"
+    local log_file="$4"
+    shift 4
+    local -a overrides=("$@")
+
+    env \
+        K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \
+        K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \
+        K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \
+        LOCAL_TMP_PARENT="$local_temp_parent" \
+        LOCK_PATH="${fixture_root}/lock/k3s-control-plane-config.lock" \
+        REMOTE_USER="test-user" \
+        REMOTE_HOST="test-host" \
+        REMOTE_DIR="/backup" \
+        ZIP_BIN="${TMPD}/bin/zip" \
+        UNZIP_BIN="${TMPD}/bin/unzip" \
+        SSH_BIN="${TMPD}/bin/ssh" \
+        SCP_BIN="${TMPD}/bin/scp" \
+        CURL_BIN="${TMPD}/bin/curl" \
+        FLOCK_BIN="${TMPD}/bin/flock" \
+        DATE_BIN="${TMPD}/bin/date" \
+        NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
+        NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
+        PATH="${TMPD}/bin:${PATH}" \
+        TMPD="$TMPD" \
+        FAKE_REMOTE_ROOT="$remote_root" \
+        "${overrides[@]}" \
+        bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
+}
+
+create_source_fixture() {
+    local fixture_root="$1"
+    mkdir -p \
+        "${fixture_root}/etc/rancher/k3s" \
+        "${fixture_root}/etc/systemd/system" \
+        "${fixture_root}/etc/default" \
+        "${fixture_root}/etc/sysconfig" \
+        "${fixture_root}/var/lib/rancher/k3s/server/cred" \
+        "${fixture_root}/var/lib/rancher/k3s/server/etc" \
+        "${fixture_root}/var/lib/rancher/k3s/server/manifests" \
+        "${fixture_root}/var/lib/rancher/k3s/server/static" \
+        "${fixture_root}/var/lib/rancher/k3s/server/tls" \
+        "${fixture_root}/var/lib/rancher/k3s/server/db" \
+        "${fixture_root}/lock"
+    printf 'config\n' > "${fixture_root}/etc/rancher/k3s/config.yaml"
+    printf 'service\n' > "${fixture_root}/etc/systemd/system/k3s.service"
+    printf 'environment\n' > "${fixture_root}/etc/systemd/system/k3s.service.env"
+    printf 'default\n' > "${fixture_root}/etc/default/k3s"
+    printf 'sysconfig\n' > "${fixture_root}/etc/sysconfig/k3s"
+    printf 'credential\n' > "${fixture_root}/var/lib/rancher/k3s/server/cred/passwd"
+    printf 'state\n' > "${fixture_root}/var/lib/rancher/k3s/server/etc/config"
+    printf 'manifest\n' > "${fixture_root}/var/lib/rancher/k3s/server/manifests/custom.yaml"
+    printf 'static\n' > "${fixture_root}/var/lib/rancher/k3s/server/static/pod.yaml"
+    printf 'tls\n' > "${fixture_root}/var/lib/rancher/k3s/server/tls/server-ca.crt"
+    printf 'token\n' > "${fixture_root}/var/lib/rancher/k3s/server/token"
+    printf 'agent-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/agent-token"
+    printf 'node-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/node-token"
+    printf 'database must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/db/state.db"
+    printf 'socket must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/kine.sock"
+}
+
+write_fake_binaries
+export TMPD
+
+# Successful publication: source layout is preserved relative to /, unzip is
+# called, and retention deletes only old archives with this archive prefix.
+success_dir="${TMPD}/success"
+success_fixture="${success_dir}/fixture"
+success_remote="${success_dir}/remote"
+success_local_tmp="${success_dir}/local-tmp"
+mkdir -p "$success_remote" "$success_local_tmp"
+create_source_fixture "$success_fixture"
+for day in 1 2 3 4 5; do
+    touch -t "2026070${day}1200" "${success_remote}/k3s-control-plane-config_2026-07-0${day}_12-00-00.zip"
+done
+printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z"
+printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip"
+newline_archive="${success_remote}/k3s-control-plane-config_2026-06-01_12-00-00.zip"$'\nunrelated.zip'
+touch -t "202606011200" "$newline_archive"
+: > "$newline_archive"
+touch -t "202606011200" "$newline_archive"
+tab_archive="${success_remote}/k3s-control-plane-config_2026-05-01_12-00-00.zip"$'\tlegacy.zip'
+: > "$tab_archive"
+touch -t "202605011200" "$tab_archive"
+: > "${success_remote}/unrelated.zip"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/unzip_trace.log"
+: > "${TMPD}/zip_invocations.log"
+: > "${TMPD}/scp_trace.log"
+: > "${TMPD}/ssh_trace.log"
+
+set +e
+run_backup "$success_fixture" "$success_remote" "$success_local_tmp" "${success_dir}/run.log"
+success_exit_code=$?
+set -e
+
+fixture_relative="${success_fixture#/}"
+assert "successful backup exits with code 0" is_zero "$success_exit_code"
+assert "successful backup publishes the final zip archive" path_exists "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "successful backup validates its archive with unzip" file_contains "${TMPD}/unzip_trace.log" "k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "archive contains the k3s configuration path" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/rancher/k3s/config.yaml"
+assert "archive contains a service configuration file" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/systemd/system/k3s.service"
+assert "archive contains control-plane credentials" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/var/lib/rancher/k3s/server/cred/passwd"
+assert "archive excludes server database content" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "server/db/state.db"
+assert "archive excludes kine socket" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "kine.sock"
+assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4
+assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip"
+assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip"
+assert "retention removes matching old archives with newlines in their filenames" path_does_not_exist "$newline_archive"
+assert "retention removes matching old archives with tabs in their filenames" path_does_not_exist "$tab_archive"
+assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${success_remote}/unrelated.zip"
+assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
+assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
+assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
+assert "successful backup sends a v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "successful backup sends its archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes":'
+assert "successful backup sends its elapsed duration" file_contains "${TMPD}/curl_trace.log" '"durationSeconds":'
+assert "successful backup sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "K3s control-plane config backup"'
+
+# A configured source path ending in server/db/.. must be rejected before zip is
+# invoked, rather than recursively archiving server/db and its sibling socket.
+source_scope_dir="${TMPD}/source-scope"
+source_scope_fixture="${source_scope_dir}/fixture"
+source_scope_remote="${source_scope_dir}/remote"
+source_scope_local_tmp="${source_scope_dir}/local-tmp"
+source_scope_archive="${source_scope_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+mkdir -p "$source_scope_remote" "$source_scope_local_tmp"
+create_source_fixture "$source_scope_fixture"
+: > "${TMPD}/zip_invocations.log"
+
+set +e
+run_backup \
+    "$source_scope_fixture" \
+    "$source_scope_remote" \
+    "$source_scope_local_tmp" \
+    "${source_scope_dir}/run.log" \
+    "K3S_CONFIG_DIR=${source_scope_fixture}/var/lib/rancher/k3s/server/db/.."
+source_scope_exit_code=$?
+set -e
+
+assert "unsafe source override exits non-zero" is_nonzero "$source_scope_exit_code"
+assert "unsafe source override reports the source-scope error" file_contains "${source_scope_dir}/run.log" "Configured source path is not safe to archive"
+assert "unsafe source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log"
+assert "unsafe source override archive excludes server database content" file_lacks "$source_scope_archive" "state.db"
+assert "unsafe source override archive excludes kine socket" file_lacks "$source_scope_archive" "kine.sock"
+assert "unsafe source override does not publish an archive" directory_is_empty "$source_scope_remote"
+assert "unsafe source override removes local temporary state" directory_is_empty "$source_scope_local_tmp"
+
+# The source path and the server directory can be overridden independently;
+# protected database paths must still be derived from the configured source.
+source_scope_mismatch_dir="${TMPD}/source-scope-mismatch"
+source_scope_mismatch_fixture="${source_scope_mismatch_dir}/fixture"
+source_scope_mismatch_remote="${source_scope_mismatch_dir}/remote"
+source_scope_mismatch_local_tmp="${source_scope_mismatch_dir}/local-tmp"
+source_scope_mismatch_unrelated_server="${source_scope_mismatch_fixture}/unrelated-server"
+source_scope_mismatch_archive="${source_scope_mismatch_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+mkdir -p "$source_scope_mismatch_remote" "$source_scope_mismatch_local_tmp"
+create_source_fixture "$source_scope_mismatch_fixture"
+mkdir -p "$source_scope_mismatch_unrelated_server"
+: > "${TMPD}/zip_invocations.log"
+
+set +e
+run_backup \
+    "$source_scope_mismatch_fixture" \
+    "$source_scope_mismatch_remote" \
+    "$source_scope_mismatch_local_tmp" \
+    "${source_scope_mismatch_dir}/run.log" \
+    "K3S_CONFIG_DIR=${source_scope_mismatch_fixture}/var/lib/rancher/k3s/server/db/.." \
+    "K3S_SERVER_DIR=${source_scope_mismatch_unrelated_server}"
+source_scope_mismatch_exit_code=$?
+set -e
+
+assert "mismatched source override exits non-zero" is_nonzero "$source_scope_mismatch_exit_code"
+assert "mismatched source override reports the source-scope error" file_contains "${source_scope_mismatch_dir}/run.log" "Configured source path is not safe to archive"
+assert "mismatched source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log"
+assert "mismatched source override archive excludes server database content" file_lacks "$source_scope_mismatch_archive" "state.db"
+assert "mismatched source override archive excludes kine socket" file_lacks "$source_scope_mismatch_archive" "kine.sock"
+assert "mismatched source override does not publish an archive" path_does_not_exist "$source_scope_mismatch_archive"
+assert "mismatched source override leaves the remote directory empty" directory_is_empty "$source_scope_mismatch_remote"
+assert "mismatched source override removes local temporary state" directory_is_empty "$source_scope_mismatch_local_tmp"
+
+# Upload failure: the partial remote temporary file is removed, no final archive
+# is published, local temporary data is removed, and one safe error notification
+# is attempted.
+failure_dir="${TMPD}/upload-failure"
+failure_fixture="${failure_dir}/fixture"
+failure_remote="${failure_dir}/remote"
+failure_local_tmp="${failure_dir}/local-tmp"
+mkdir -p "$failure_remote" "$failure_local_tmp"
+create_source_fixture "$failure_fixture"
+: > "${TMPD}/fail_scp"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$failure_fixture" "$failure_remote" "$failure_local_tmp" "${failure_dir}/run.log"
+failure_exit_code=$?
+set -e
+rm -f -- "${TMPD}/fail_scp"
+
+assert "upload failure exits non-zero" is_nonzero "$failure_exit_code"
+assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
+assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
+assert "upload failure attempts exactly one v2 error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" 1
+assert "upload failure marks the notification as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
+assert "upload failure includes the v2 error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED"'
+assert "upload failure does not send a v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
+
+# A notification error is non-fatal after a completed backup.
+notify_dir="${TMPD}/notify-failure"
+notify_fixture="${notify_dir}/fixture"
+notify_remote="${notify_dir}/remote"
+notify_local_tmp="${notify_dir}/local-tmp"
+mkdir -p "$notify_remote" "$notify_local_tmp"
+create_source_fixture "$notify_fixture"
+: > "${TMPD}/fail_curl"
+
+set +e
+run_backup "$notify_fixture" "$notify_remote" "$notify_local_tmp" "${notify_dir}/run.log" "CLEANUP_KEEP_COUNT=0"
+notify_exit_code=$?
+set -e
+rm -f -- "${TMPD}/fail_curl"
+
+assert "failed success notification does not fail a completed backup" is_zero "$notify_exit_code"
+assert "completed backup remains published after notification failure" path_exists "${notify_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+
+printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
+if (( tests_failed > 0 )); then
+    exit 1
+fi