fix(backup): treat 7z exit code 1 (warning) as success in archive_staging

Exit code 1 from 7z means a non-fatal warning — the archive IS created,
just some files couldn't be compressed (e.g. broken symlinks, special
files). Previously the function required filtered output to be empty
alongside rc=1, causing any non-errno=2 stderr line to abort the entire
backup and discard the archive.

Now any rc=1 is treated as success (warning is still logged).

.gitattributes

@@ -0,0 +1,10 @@
+.gitattributes text eol=lf
+*.sh text eol=lf
+*.yaml text eol=lf
+*.yml text eol=lf
+*.md text eol=lf
+*.go text eol=lf
+go.mod text eol=lf
+go.sum text eol=lf
+Dockerfile text eol=lf
+*.tmpl text eol=lf

.gitea/workflows/haven-notify.yaml

@@ -5,85 +5,108 @@ on:
     branches:
       - main
     paths:
-      - 'haven-notify/**'
-      - '.gitea/workflows/**'
+      - "haven-notify/**"
+      - ".gitea/workflows/haven-notify.yaml"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "haven-notify/**"
+      - ".gitea/workflows/haven-notify.yaml"
   workflow_dispatch: {}
 
 env:
-  REGISTRY_HOST: git.ivanch.me
-  REGISTRY_USERNAME: ivanch
-  IMAGE_NOTIFY: ${{ env.REGISTRY_HOST }}/ivanch/haven-notify
-  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  IMAGE: git.ivanch.me/ivanch/haven-notify
 
 jobs:
-  build_haven_notify:
-    name: Build Haven Notify Image
-    runs-on: ubuntu-22.04
-
+  test:
+    name: Test Haven Notify
+    runs-on: runner-full-amd64
+    timeout-minutes: 15
     steps:
       - name: Check out repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Log in to Container Registry
-        run: |
-          echo "${{ secrets.REGISTRY_PASSWORD }}" \
-            | docker login "${{ env.REGISTRY_HOST }}" \
-                -u "${{ env.REGISTRY_USERNAME }}" \
-                --password-stdin
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and Push Multi-Arch Image
-        uses: docker/build-push-action@v6
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
-          push: true
-          context: haven-notify
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            ${{ env.IMAGE_NOTIFY }}:latest
+          go-version-file: haven-notify/go.mod
+          cache-dependency-path: haven-notify/go.sum
 
-  deploy_haven_notify:
-    name: Deploy Haven Notify (internal)
-    runs-on: ubuntu-amd64
-    needs: build_haven_notify
-    steps:
-      - name: Check KUBE_CONFIG validity
+      - name: Verify formatting
+        working-directory: haven-notify
         run: |
-          if [ -z "${KUBE_CONFIG}" ] || [ "${KUBE_CONFIG}" = "" ] || [ "${KUBE_CONFIG// }" = "" ]; then
-            echo "KUBE_CONFIG is not set or is empty."
+          unformatted="$(gofmt -l *.go)"
+          if [ -n "$unformatted" ]; then
+            echo "The following Go files need formatting:"
+            echo "$unformatted"
             exit 1
           fi
 
+      - name: Verify Docker access
+        run: docker info
+
+      - name: Run static analysis
+        working-directory: haven-notify
+        run: go vet ./...
+
+      - name: Run race tests and collect coverage
+        working-directory: haven-notify
+        run: go test -race -shuffle=on -count=1 -covermode=atomic -coverprofile=coverage.out ./...
+
+      - name: Enforce coverage threshold
+        working-directory: haven-notify
+        run: |
+          coverage="$(go tool cover -func=coverage.out | awk '/^total:/ {gsub("%", "", $3); print $3}')"
+          echo "Statement coverage: ${coverage}%"
+          awk -v coverage="$coverage" 'BEGIN { exit !(coverage + 0 >= 90) }'
+
+      - name: Run fuzz smoke test
+        working-directory: haven-notify
+        run: go test -run=^$ -fuzz=FuzzErrorNotificationHandler -fuzztime=10s .
+
+      - name: Run container end-to-end tests
+        working-directory: haven-notify
+        run: go test -tags=e2e -run '^TestContainerE2E$' -count=1 -timeout=5m -v .
+
+  build:
+    name: Build Haven Notify Image
+    runs-on: runner-slim
+    needs: test
+    if: ${{ (gitea.event_name == 'push' || gitea.event_name == 'workflow_dispatch') && gitea.ref == 'refs/heads/main' }}
+    steps:
       - name: Check out repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Download and install dependencies
-        run: |
-          apt-get update -y
-          apt-get install -y curl
-          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-          install -m 0755 kubectl /usr/local/bin/kubectl
-          kubectl version --client
+      - name: Build and Push Multi-Arch Image
+        uses: https://git.ivanch.me/ivanch/pipeline-actions/build-and-push@main
+        with:
+          image: ${{ env.IMAGE }}
+          registry_password: ${{ secrets.REGISTRY_PASSWORD }}
+          ssh_key: ${{ secrets.SSH_KEY_DOCKERBUILD }}
+          build_context: haven-notify
 
-      - name: Set up kubeconfig
-        run: |
-          cd haven-notify/deploy
-          echo "$KUBE_CONFIG" > kubeconfig.yaml
-        env:
-          KUBE_CONFIG: ${{ env.KUBE_CONFIG }}
+  build-pr:
+    name: Build Haven Notify Image (PR)
+    runs-on: runner-full-amd64
+    needs: test
+    if: ${{ gitea.event_name == 'pull_request' }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
 
-      - name: Check connection to cluster
-        run: |
-          cd haven-notify/deploy
-          kubectl --kubeconfig=kubeconfig.yaml cluster-info
+      - name: Build Docker image
+        run: docker build --pull --tag haven-notify:pr-${{ gitea.sha }} haven-notify
 
-      - name: Apply haven-notify deployment
-        run: |
-          cd haven-notify/deploy
-          kubectl --kubeconfig=kubeconfig.yaml apply -f haven-notify.yaml
-
-      - name: Rollout restart haven-notify
-        run: |
-          cd haven-notify/deploy
-          kubectl --kubeconfig=kubeconfig.yaml rollout restart deployment/haven-notify
+  deploy:
+    name: Deploy Haven Notify (internal)
+    runs-on: runner-slim
+    needs: build
+    if: ${{ (gitea.event_name == 'push' || gitea.event_name == 'workflow_dispatch') && gitea.ref == 'refs/heads/main' }}
+    steps:
+      - name: Rollout Restart Haven Notify
+        uses: https://git.ivanch.me/ivanch/pipeline-actions/deploy-restart@main
+        with:
+          kube_config: ${{ secrets.KUBE_CONFIG }}
+          deployment_name: haven-notify
+          namespace: infra

.gitea/workflows/main.yaml

@@ -3,7 +3,7 @@ on: [push]
 
 jobs:
   check-scripts-syntax:
-    runs-on: ubuntu-latest
+    runs-on: runner-slim
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
@@ -13,3 +13,4 @@ jobs:
       - name: Check scripts syntax
         run: |
           bash -n ${{ gitea.workspace }}/*.sh
+          bash -n ${{ gitea.workspace }}/k8s/*.sh

.gitea/workflows/test-automated-nfs-backup.yaml

@@ -1,17 +1,26 @@
-name: Test automated-nfs-backup
+name: Test Kubernetes backup scripts
 
 on:
   push:
     paths:
-      - 'k8s/**'
-      - '.gitea/workflows/**'
+      - "k8s/**"
+      - ".gitea/workflows/**"
   workflow_dispatch: {}
 
 jobs:
   test-automated-nfs-backup:
-    runs-on: ubuntu-latest
+    runs-on: runner-slim
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Run automated NFS backup tests
         run: bash k8s/test/test-automated-nfs-backup.sh
+  test-k3s-control-plane-config-backup:
+    runs-on: runner-slim
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Install GNU coreutils and findutils for control-plane tests
+        run: apk add --no-cache coreutils findutils
+      - name: Run k3s control-plane configuration backup tests
+        run: bash k8s/test/test-backup-k3s-control-plane-config.sh

README.md

@@ -2,20 +2,30 @@
 
 Useful scripts for managing servers written in Bash. Supported OS includes Debian/Ubuntu, Alpine, Arch, RHEL/Fedora/CentOS, openSUSE, Void, and Gentoo.
 
-In the past I was using it more for maintaining my home servers, bare metal with Docker, however when I changed to k3s, I started using it less. Now I'm using this as a collection of useful scripts that I can use when needed. Also `clean.sh` is still useful.
+In the past I was using it more for maintaining my home servers, bare metal with Docker, however when I changed to k3s, I started using it less. Now I'm using this as a collection of useful scripts that I can use when needed. Also `housekeeping.sh` is still useful.
 
-### `clean.sh`
+### `housekeeping.sh`
 
-This script is used to clean some of the files, docker dangling images, and docker stopped/unused containers.
+Unified system housekeeping script for package index updates, system cleanup, and oh-my-zsh updates.
 
-### `update.sh`
+Subcommands:
+- `update` - detects the running OS and updates the package index only. It does not upgrade or install packages.
+- `clean` - runs system cleanup for Docker resources, package manager caches, temporary/cache/log directories, systemd journal, thumbnails, and memory caches.
+- `omz` - updates oh-my-zsh when `omz` is installed; otherwise it skips safely.
+- `all` - runs `update`, then `clean`, then `omz`. This is the default when no subcommand is provided.
 
-Detects the running OS (Alpine, Arch, Debian/Ubuntu, RHEL/Fedora/CentOS, openSUSE, Void, Gentoo) and updates the **package list** only — it does not upgrade or install any packages. Useful for keeping package indexes fresh so that `upgrade` commands run faster later.
+The `update` subcommand supports Alpine, Arch, Debian/Ubuntu, RHEL/Fedora/CentOS, openSUSE, Void, and Gentoo family distributions.
 
 Run with `curl | bash`:
 
 ```bash
-curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/update.sh | bash
+curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/housekeeping.sh | bash
+```
+
+Run a specific subcommand:
+
+```bash
+curl -sSL https://git.ivanch.me/ivanch/server-scripts/raw/branch/main/housekeeping.sh | bash -s -- clean
 ```
 
 ### `windows-backup.ps1`

haven-notify/.dockerignore

@@ -0,0 +1,6 @@
+.git
+.gitignore
+assets/
+coverage.out
+README.md
+*_test.go

haven-notify/.gitignore

@@ -0,0 +1 @@
+coverage.out

haven-notify/Dockerfile

@@ -1,20 +1,22 @@
-# Start from the official Golang image for building
-FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder
+ARG BUILDPLATFORM
+FROM --platform=$BUILDPLATFORM golang:1.25.10-alpine3.23@sha256:8d22e29d960bc50cd025d93d5b7c7d220b1ee9aa7a239b3c8f55a57e987e8d45 AS builder
 
 ARG TARGETARCH
 ARG TARGETOS
 
-WORKDIR /app
-COPY . .
-# Build statically for Linux
-RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o haven-notify main.go
-
-# Use Alpine for running, with CA certificates for TLS
-FROM alpine:latest
-WORKDIR /app
-RUN apk --no-cache add ca-certificates
+WORKDIR /src
+COPY go.mod go.sum ./
+COPY *.go ./
 COPY template/ template/
-COPY --from=builder /app/haven-notify .
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o /out/haven-notify .
+
+FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659
+WORKDIR /app
+RUN apk --no-cache add ca-certificates \
+    && addgroup -S haven-notify \
+    && adduser -S -D -H -G haven-notify haven-notify
+COPY --from=builder --chown=haven-notify:haven-notify /out/haven-notify /app/haven-notify
 EXPOSE 8080
 ENV WEBHOOK_URL=""
+USER haven-notify
 ENTRYPOINT ["/app/haven-notify"]

haven-notify/README.md

@@ -3,21 +3,32 @@
 </div>
 
 ## Overview
-Haven Notify is an internal service designed to send notifications to a specified Discord channel.
 
-It's built in Go and can be deployed as a container or managed service.
+Haven Notify is an internal Go service that validates notification requests, renders Discord-compatible payloads, and delivers them to a configured Discord webhook.
+
+It runs as a standalone binary or container. Templates are embedded in the binary and validated at startup.
 
 ## Prerequisites
-- Go 1.18 or newer
-- Docker
-- A Discord Webhook URL
 
-## API Specification
+- Go 1.25.10
+- A Docker API compatible with Testcontainers for the end-to-end suite
+- A Discord webhook URL for normal service operation
+
+`WEBHOOK_URL` is required and must be an absolute HTTP or HTTPS URL. The process exits before becoming ready when the configuration or embedded templates are invalid.
+
+## API
+
+All notification endpoints require `POST` and `Content-Type: application/json`. Request bodies are limited to 64 KiB, documented fields are required, and unknown fields are accepted for forward compatibility.
+
+### V1 API
+
+The original routes, request bodies, and Discord message appearance remain unchanged.
+
+#### Send Notification
 
-### Send Notification
 - **Endpoint**: `/notify`
-- **Method**: `POST`
 - **Request Body**:
+
 ```json
 {
   "title": "Notification Title",
@@ -25,72 +36,227 @@ It's built in Go and can be deployed as a container or managed service.
 }
 ```
 
-### Send Backup Notification
+#### Send Backup Notification
+
 - **Endpoint**: `/template/notify/backup`
-- **Method**: `POST`
 - **Request Body**:
+
 ```json
 {
-  "title": "Notification Title",
-  "asset": "Notification Asset Name",
-  "backupSizeInMB": 500,
+  "title": "Nightly backup",
+  "asset": "Photos",
+  "backupSizeInMB": 1536,
   "extra": [
     {
-      "name": "Additional Info",
-      "value": "Some extra information"
+      "name": "Host",
+      "value": "nebula"
     }
   ]
 }
 ```
 
-### Send Update Notification
+#### Send Update Notification
+
 - **Endpoint**: `/template/notify/update`
-- **Method**: `POST`
 - **Request Body**:
+
 ```json
 {
-  "host": "Notification Title",
-  "asset": "Notification Message",
-  "time": 500 // in seconds
+  "host": "nexus",
+  "asset": "k3s",
+  "time": 42
 }
 ```
 
-### Send Error Notification
+`time` is expressed in seconds.
+
+#### Send Error Notification
+
 - **Endpoint**: `/template/notify/error`
-- **Method**: `POST`
 - **Request Body**:
+
 ```json
 {
-  "caller": "Who triggered the error",
+  "caller": "backup-job",
   "message": "Error while moving file",
   "critical": true,
   "extra": [
     {
-      "name": "Additional Info",
-      "value": "Some extra information"
+      "name": "Path",
+      "value": "/mnt/backups"
     }
   ]
 }
 ```
 
-## Setup & Usage
+### V2 API
+
+V2 notifications use compact, consistently styled Discord embeds. Colors, icons, timestamps, field ordering, and layout are controlled by Haven Notify rather than the caller.
+
+Every v2 request may include:
+
+- `source`: a nonblank string identifying the caller.
+- `occurredAt`: an RFC3339 timestamp. When omitted, Haven Notify uses the current UTC time.
+- `extra`: an array of nonblank `name` and `value` strings. Short single-line values render inline; multiline or long values render full-width.
+
+V2 preserves caller field order. Fixed notification fields are placed before `extra` fields so that trailing extras are discarded first if Discord's 25-field or 6,000-character embed limits are reached. Oversized text is safely truncated and extra fields may be omitted from the delivered message.
+
+#### Send a Message
+
+- **Endpoint**: `/api/v2/messages`
+- **Required**: `message`
+- **Optional**: `title`, `source`, `occurredAt`, `extra`
+
+```json
+{
+  "title": "Maintenance complete",
+  "message": "The storage job completed successfully.",
+  "source": "scheduler",
+  "extra": [
+    {
+      "name": "Region",
+      "value": "home"
+    }
+  ]
+}
+```
+
+#### Send a Backup Notification
+
+- **Endpoint**: `/api/v2/notifications/backup`
+- **Required**: `asset`, nonnegative integer `sizeBytes`
+- **Optional**: `title`, `source`, nonnegative `durationSeconds`, `occurredAt`, `extra`
+
+```json
+{
+  "title": "Nightly backup",
+  "asset": "Photos",
+  "sizeBytes": 1610612736,
+  "durationSeconds": 125,
+  "source": "restic",
+  "extra": [
+    {
+      "name": "Host",
+      "value": "nebula"
+    }
+  ]
+}
+```
+
+`sizeBytes` is displayed using IEC units such as KiB, MiB, and GiB. Durations use a compact form such as `42.5s`, `2m 5s`, or `1h 2m 3s`.
+
+#### Send an Update Notification
+
+- **Endpoint**: `/api/v2/notifications/update`
+- **Required**: `host`, `asset`, nonnegative `durationSeconds`
+- **Optional**: `source`, `fromVersion`, `toVersion`, `occurredAt`, `extra`
+
+```json
+{
+  "host": "nexus",
+  "asset": "k3s",
+  "durationSeconds": 42.5,
+  "fromVersion": "1.32.5",
+  "toVersion": "1.33.1",
+  "source": "system-updater"
+}
+```
+
+When both versions are supplied, the embed displays a transition such as `1.32.5 → 1.33.1`. A single supplied version is labeled as the previous or installed version.
+
+#### Send an Error Notification
+
+- **Endpoint**: `/api/v2/notifications/error`
+- **Required**: `source`, `message`, `severity`
+- **Optional**: `errorCode`, `occurredAt`, `extra`
+
+```json
+{
+  "source": "backup-job",
+  "message": "The repository is unavailable.",
+  "severity": "critical",
+  "errorCode": "E_REPOSITORY",
+  "extra": [
+    {
+      "name": "Path",
+      "value": "/mnt/backups"
+    }
+  ]
+}
+```
+
+`severity` must be `warning`, `error`, or `critical`; it selects the embed title, icon, and color.
+
+### Health Probes
+
+- `GET` or `HEAD /ready` returns `200 Ready` after configuration and templates initialize.
+- `GET` or `HEAD /live` returns `200 Alive` while the process can serve HTTP.
+
+### Response Statuses
+
+| Status | Meaning |
+|---|---|
+| `200` | Notification delivered |
+| `400` | Invalid or missing request data |
+| `404` | Unknown route or template |
+| `405` | Unsupported method |
+| `415` | Content type is not JSON |
+| `502` | Discord rejected the request or delivery retries failed |
+| `504` | Discord delivery exceeded its timeout |
+
+Haven Notify suppresses Discord mention parsing and safely truncates content to Discord's message and embed limits. Transport errors, rate limits, and server errors receive at most two retries within a ten-second total delivery budget.
+
+## Run
+
+### Go
+
+```sh
+WEBHOOK_URL=https://discord.com/api/webhooks/... go run .
+```
 
 ### Docker
-1. Build the Docker image:
-	```sh
-	docker build -t haven-notify .
-	```
-2. Run the container:
-	```sh
-	docker run -e WEBHOOK_URL=your_webhook_url haven-notify
-	```
 
-### Kubernetes
-Deployment manifest is available at `deploy/haven-notify.yaml`.
-1. Edit the manifest to set your environment variables.
-2. Create a generic secret named `WEBHOOK_URL` with `discord-webhook=your_webhook_url`:
-  - `kubectl create secret generic WEBHOOK_URL --from-literal=discord-webhook=<your_webhook_url> -n <namespace>`
-3. Apply deployment:
-	```sh
-	kubectl apply -f deploy/haven-notify.yaml
-	```
+```sh
+docker build -t haven-notify .
+docker run --rm -p 8080:8080 \
+  -e WEBHOOK_URL=https://discord.com/api/webhooks/... \
+  haven-notify
+```
+
+The runtime container uses a non-root `haven-notify` user and contains only the binary, Alpine runtime files, and CA certificates.
+
+## Tests
+
+Run unit and HTTP contract tests:
+
+```sh
+go test ./...
+```
+
+Run static analysis, the race detector, and coverage (the race detector requires CGO support):
+
+```sh
+go vet ./...
+go test -race -shuffle=on -count=1 -covermode=atomic -coverprofile=coverage.out ./...
+go tool cover -func=coverage.out
+```
+
+Run the seeded fuzz target:
+
+```sh
+go test -run=^$ -fuzz=FuzzErrorNotificationHandler -fuzztime=10s .
+```
+
+Run the black-box container suite:
+
+```sh
+go test -tags=e2e -run '^TestContainerE2E$' -count=1 -timeout=5m -v .
+```
+
+The end-to-end suite builds the production Dockerfile, starts Haven Notify and a disposable MockServer on an isolated network, and verifies the complete HTTP-to-Discord payload flow. It never calls a real Discord webhook. Configure `DOCKER_HOST` or the Testcontainers host settings when the Docker daemon is not available through the platform's default socket.
+
+## Deployment
+
+The Kubernetes Deployment, Service, and Ingress manifest lives in the `haven` homelab repository at `infra/haven-notify.yaml`. This repository holds the service source, Dockerfile, tests, and Gitea Actions pipeline.
+
+The production manifest maps the `HAVEN_WEBHOOK_URL` key from the `discord-webhook` secret into the container's `WEBHOOK_URL` environment variable.

haven-notify/deploy/haven-notify.yaml

@@ -1,71 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: haven-notify
-  labels:
-    app: haven-notify
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: haven-notify
-  template:
-    metadata:
-      labels:
-        app: haven-notify
-    spec:
-      containers:
-        - name: haven-notify
-          image: git.ivanch.me/ivanch/haven-notify:latest
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8080
-          env:
-          - name: WEBHOOK_URL
-            valueFrom:
-              secretKeyRef:
-                name: discord-webhook
-                key: HAVEN_WEBHOOK_URL
-          readinessProbe:
-            httpGet:
-              path: /ready
-              port: 8080
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          livenessProbe:
-            httpGet:
-              path: /live
-              port: 8080
-            initialDelaySeconds: 5
-            periodSeconds: 10
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: haven-notify
-spec:
-  selector:
-    app: haven-notify
-  ports:
-    - protocol: TCP
-      port: 8080
-      targetPort: 8080
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: haven-notify
-  namespace: default
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: notify.haven
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: haven-notify
-                port:
-                  number: 8080

haven-notify/discord.go

@@ -0,0 +1,282 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+const (
+	deliveryTimeout   = 10 * time.Second
+	attemptTimeout    = 5 * time.Second
+	maximumAttempts   = 3
+	maximumRetryDelay = 2 * time.Second
+	truncationMarker  = "… [truncated]"
+)
+
+var (
+	errWebhookDelivery = errors.New("webhook delivery failed")
+	errWebhookTimeout  = errors.New("webhook delivery timed out")
+)
+
+type sleepFunc func(context.Context, time.Duration) error
+
+type discordClient struct {
+	webhookURL string
+	httpClient *http.Client
+	sleep      sleepFunc
+}
+
+type discordPayload struct {
+	Content         string                 `json:"content,omitempty"`
+	Embeds          []discordEmbed         `json:"embeds,omitempty"`
+	AllowedMentions discordAllowedMentions `json:"allowed_mentions"`
+}
+
+type discordAllowedMentions struct {
+	Parse []string `json:"parse"`
+}
+
+type discordEmbed struct {
+	Title       string              `json:"title,omitempty"`
+	Description string              `json:"description,omitempty"`
+	Color       int                 `json:"color,omitempty"`
+	Timestamp   string              `json:"timestamp,omitempty"`
+	Fields      []discordEmbedField `json:"fields,omitempty"`
+	Footer      *discordEmbedFooter `json:"footer,omitempty"`
+}
+
+type discordEmbedField struct {
+	Name   string `json:"name"`
+	Value  string `json:"value"`
+	Inline bool   `json:"inline"`
+}
+
+type discordEmbedFooter struct {
+	Text string `json:"text"`
+}
+
+func newDiscordClient(webhookURL string, client *http.Client, sleep sleepFunc) (*discordClient, error) {
+	parsed, err := url.Parse(webhookURL)
+	if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
+		return nil, errors.New("WEBHOOK_URL must be an absolute HTTP(S) URL")
+	}
+	if client == nil {
+		client = &http.Client{Timeout: attemptTimeout}
+	}
+	if sleep == nil {
+		sleep = sleepContext
+	}
+	return &discordClient{webhookURL: webhookURL, httpClient: client, sleep: sleep}, nil
+}
+
+func (c *discordClient) send(ctx context.Context, payload discordPayload) error {
+	normalizeDiscordPayload(&payload)
+	body, err := json.Marshal(payload)
+	if err != nil {
+		return fmt.Errorf("%w: marshal payload", errWebhookDelivery)
+	}
+
+	deliveryCtx, cancel := context.WithTimeout(ctx, deliveryTimeout)
+	defer cancel()
+
+	var lastErr error
+	for attempt := 0; attempt < maximumAttempts; attempt++ {
+		attemptCtx, attemptCancel := context.WithTimeout(deliveryCtx, attemptTimeout)
+		request, requestErr := http.NewRequestWithContext(attemptCtx, http.MethodPost, c.webhookURL, bytes.NewReader(body))
+		if requestErr != nil {
+			attemptCancel()
+			return fmt.Errorf("%w: create request", errWebhookDelivery)
+		}
+		request.Header.Set("Content-Type", "application/json")
+
+		response, requestErr := c.httpClient.Do(request)
+		if requestErr == nil {
+			delay := retryDelay(response, attempt)
+			_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4<<10))
+			_ = response.Body.Close()
+			attemptCancel()
+
+			if response.StatusCode >= 200 && response.StatusCode < 300 {
+				return nil
+			}
+			lastErr = fmt.Errorf("%w: HTTP %d", errWebhookDelivery, response.StatusCode)
+			if !retryableStatus(response.StatusCode) || attempt == maximumAttempts-1 {
+				return lastErr
+			}
+			if err := c.sleep(deliveryCtx, delay); err != nil {
+				return errWebhookTimeout
+			}
+			continue
+		}
+
+		attemptCancel()
+		lastErr = requestErr
+		if deliveryCtx.Err() != nil {
+			return errWebhookTimeout
+		}
+		if attempt == maximumAttempts-1 {
+			if isTimeout(requestErr) {
+				return fmt.Errorf("%w: %v", errWebhookTimeout, requestErr)
+			}
+			return fmt.Errorf("%w: %v", errWebhookDelivery, requestErr)
+		}
+		if err := c.sleep(deliveryCtx, fallbackRetryDelay(attempt)); err != nil {
+			return errWebhookTimeout
+		}
+	}
+
+	return fmt.Errorf("%w: %v", errWebhookDelivery, lastErr)
+}
+
+func retryableStatus(status int) bool {
+	return status == http.StatusTooManyRequests || status >= 500
+}
+
+func retryDelay(response *http.Response, attempt int) time.Duration {
+	if response.StatusCode == http.StatusTooManyRequests {
+		for _, header := range []string{"Retry-After", "X-RateLimit-Reset-After"} {
+			if delay, ok := parseRetryDelay(response.Header.Get(header)); ok {
+				return min(delay, maximumRetryDelay)
+			}
+		}
+	}
+	return fallbackRetryDelay(attempt)
+}
+
+func parseRetryDelay(value string) (time.Duration, bool) {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return 0, false
+	}
+	if seconds, err := strconv.ParseFloat(value, 64); err == nil && seconds >= 0 {
+		return time.Duration(seconds * float64(time.Second)), true
+	}
+	if timestamp, err := http.ParseTime(value); err == nil {
+		return max(time.Until(timestamp), 0), true
+	}
+	return 0, false
+}
+
+func fallbackRetryDelay(attempt int) time.Duration {
+	return 250 * time.Millisecond * time.Duration(1<<attempt)
+}
+
+func sleepContext(ctx context.Context, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
+
+func isTimeout(err error) bool {
+	if errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	var netErr net.Error
+	return errors.As(err, &netErr) && netErr.Timeout()
+}
+
+func normalizeDiscordPayload(payload *discordPayload) {
+	payload.AllowedMentions.Parse = []string{}
+	payload.Content = truncateRunes(payload.Content, 2000)
+	if len(payload.Embeds) > 10 {
+		payload.Embeds = payload.Embeds[:10]
+	}
+
+	for embedIndex := range payload.Embeds {
+		embed := &payload.Embeds[embedIndex]
+		embed.Title = truncateRunes(embed.Title, 256)
+		embed.Description = truncateRunes(embed.Description, 4096)
+		if len(embed.Fields) > 25 {
+			embed.Fields = embed.Fields[:25]
+		}
+		for fieldIndex := range embed.Fields {
+			embed.Fields[fieldIndex].Name = truncateRunes(embed.Fields[fieldIndex].Name, 256)
+			embed.Fields[fieldIndex].Value = truncateRunes(embed.Fields[fieldIndex].Value, 1024)
+		}
+		if embed.Footer != nil {
+			embed.Footer.Text = truncateRunes(embed.Footer.Text, 2048)
+		}
+	}
+
+	for discordEmbedCharacters(payload.Embeds) > 6000 {
+		removed := false
+		for embedIndex := len(payload.Embeds) - 1; embedIndex >= 0; embedIndex-- {
+			embed := &payload.Embeds[embedIndex]
+			if len(embed.Fields) > 0 {
+				embed.Fields = embed.Fields[:len(embed.Fields)-1]
+				removed = true
+				break
+			}
+		}
+		if !removed {
+			break
+		}
+	}
+
+	remainingOverflow := discordEmbedCharacters(payload.Embeds) - 6000
+	for embedIndex := len(payload.Embeds) - 1; remainingOverflow > 0 && embedIndex >= 0; embedIndex-- {
+		embed := &payload.Embeds[embedIndex]
+		if embed.Footer != nil && embed.Footer.Text != "" {
+			newLimit := max(utf8.RuneCountInString(embed.Footer.Text)-remainingOverflow, 0)
+			embed.Footer.Text = truncateRunes(embed.Footer.Text, newLimit)
+		}
+		remainingOverflow = discordEmbedCharacters(payload.Embeds) - 6000
+		if remainingOverflow > 0 && embed.Description != "" {
+			newLimit := max(utf8.RuneCountInString(embed.Description)-remainingOverflow, 0)
+			embed.Description = truncateRunes(embed.Description, newLimit)
+		}
+		remainingOverflow = discordEmbedCharacters(payload.Embeds) - 6000
+		if remainingOverflow > 0 && embed.Title != "" {
+			newLimit := max(utf8.RuneCountInString(embed.Title)-remainingOverflow, 0)
+			embed.Title = truncateRunes(embed.Title, newLimit)
+		}
+		remainingOverflow = discordEmbedCharacters(payload.Embeds) - 6000
+	}
+}
+
+func discordEmbedCharacters(embeds []discordEmbed) int {
+	total := 0
+	for _, embed := range embeds {
+		total += utf8.RuneCountInString(embed.Title)
+		total += utf8.RuneCountInString(embed.Description)
+		if embed.Footer != nil {
+			total += utf8.RuneCountInString(embed.Footer.Text)
+		}
+		for _, field := range embed.Fields {
+			total += utf8.RuneCountInString(field.Name)
+			total += utf8.RuneCountInString(field.Value)
+		}
+	}
+	return total
+}
+
+func truncateRunes(value string, limit int) string {
+	runes := []rune(value)
+	if len(runes) <= limit {
+		return value
+	}
+	if limit <= 0 {
+		return ""
+	}
+	marker := []rune(truncationMarker)
+	if limit <= len(marker) {
+		return string(marker[:limit])
+	}
+	return string(runes[:limit-len(marker)]) + truncationMarker
+}

haven-notify/discord_test.go

@@ -0,0 +1,243 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+	"unicode/utf8"
+)
+
+func TestDiscordDeliveryRetries(t *testing.T) {
+	tests := []struct {
+		name         string
+		statuses     []int
+		headers      []http.Header
+		transportErr error
+		wantCalls    int
+		wantSleeps   []time.Duration
+		wantError    error
+	}{
+		{name: "success", statuses: []int{204}, wantCalls: 1},
+		{name: "all 2xx accepted", statuses: []int{299}, wantCalls: 1},
+		{name: "permanent 4xx", statuses: []int{400}, wantCalls: 1, wantError: errWebhookDelivery},
+		{name: "5xx recovers", statuses: []int{500, 204}, wantCalls: 2, wantSleeps: []time.Duration{250 * time.Millisecond}},
+		{name: "5xx exhausts", statuses: []int{500, 502, 503}, wantCalls: 3, wantSleeps: []time.Duration{250 * time.Millisecond, 500 * time.Millisecond}, wantError: errWebhookDelivery},
+		{name: "429 honors capped retry after", statuses: []int{429, 204}, headers: []http.Header{{"Retry-After": []string{"10"}}, nil}, wantCalls: 2, wantSleeps: []time.Duration{2 * time.Second}},
+		{name: "429 uses reset after", statuses: []int{429, 204}, headers: []http.Header{{"X-Ratelimit-Reset-After": []string{"0.125"}}, nil}, wantCalls: 2, wantSleeps: []time.Duration{125 * time.Millisecond}},
+		{name: "transport errors exhaust", transportErr: errors.New("connection reset"), wantCalls: 3, wantSleeps: []time.Duration{250 * time.Millisecond, 500 * time.Millisecond}, wantError: errWebhookDelivery},
+		{name: "timeouts exhaust", transportErr: context.DeadlineExceeded, wantCalls: 3, wantSleeps: []time.Duration{250 * time.Millisecond, 500 * time.Millisecond}, wantError: errWebhookTimeout},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var mu sync.Mutex
+			calls := 0
+			var sleeps []time.Duration
+			transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+				mu.Lock()
+				defer mu.Unlock()
+				calls++
+				if request.Header.Get("Content-Type") != "application/json" {
+					t.Errorf("Content-Type = %q", request.Header.Get("Content-Type"))
+				}
+				if test.transportErr != nil {
+					return nil, test.transportErr
+				}
+				index := min(calls-1, len(test.statuses)-1)
+				var headers http.Header
+				if index < len(test.headers) {
+					headers = test.headers[index]
+				}
+				return response(test.statuses[index], headers), nil
+			})
+			client, err := newDiscordClient("https://discord.invalid/webhook", &http.Client{Transport: transport}, func(_ context.Context, delay time.Duration) error {
+				sleeps = append(sleeps, delay)
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = client.send(context.Background(), discordPayload{Content: "hello"})
+			if !errors.Is(err, test.wantError) {
+				t.Fatalf("send() error = %v, want %v", err, test.wantError)
+			}
+			if calls != test.wantCalls {
+				t.Fatalf("calls = %d, want %d", calls, test.wantCalls)
+			}
+			if len(sleeps) != len(test.wantSleeps) {
+				t.Fatalf("sleeps = %v, want %v", sleeps, test.wantSleeps)
+			}
+			for index := range sleeps {
+				if sleeps[index] != test.wantSleeps[index] {
+					t.Fatalf("sleeps = %v, want %v", sleeps, test.wantSleeps)
+				}
+			}
+		})
+	}
+}
+
+func TestDiscordDeliveryPayloadAndCanceledSleep(t *testing.T) {
+	var sent discordPayload
+	transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+		if err := json.NewDecoder(request.Body).Decode(&sent); err != nil {
+			t.Fatal(err)
+		}
+		return response(204, nil), nil
+	})
+	client, err := newDiscordClient("http://discord.invalid/webhook", &http.Client{Transport: transport}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := client.send(context.Background(), discordPayload{Content: strings.Repeat("界", 2100)}); err != nil {
+		t.Fatal(err)
+	}
+	if utf8.RuneCountInString(sent.Content) != 2000 || !strings.HasSuffix(sent.Content, truncationMarker) {
+		t.Fatalf("content length/suffix = %d/%q", utf8.RuneCountInString(sent.Content), []rune(sent.Content)[1980:])
+	}
+	if sent.AllowedMentions.Parse == nil || len(sent.AllowedMentions.Parse) != 0 {
+		t.Fatalf("allowed mentions = %#v", sent.AllowedMentions)
+	}
+
+	client, err = newDiscordClient("http://discord.invalid/webhook", &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+		return response(500, nil), nil
+	})}, func(context.Context, time.Duration) error { return context.DeadlineExceeded })
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := client.send(context.Background(), discordPayload{Content: "x"}); !errors.Is(err, errWebhookTimeout) {
+		t.Fatalf("send() error = %v, want timeout", err)
+	}
+}
+
+func TestDiscordDeliveryRecoversFromTransportError(t *testing.T) {
+	var calls int
+	var sleeps []time.Duration
+	transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
+		calls++
+		if calls == 1 {
+			return nil, errors.New("temporary connection failure")
+		}
+		return response(http.StatusNoContent, nil), nil
+	})
+	client, err := newDiscordClient("https://discord.invalid/webhook", &http.Client{Transport: transport}, func(_ context.Context, delay time.Duration) error {
+		sleeps = append(sleeps, delay)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := client.send(context.Background(), discordPayload{Content: "hello"}); err != nil {
+		t.Fatal(err)
+	}
+	if calls != 2 || len(sleeps) != 1 || sleeps[0] != 250*time.Millisecond {
+		t.Fatalf("calls/sleeps = %d/%v", calls, sleeps)
+	}
+}
+
+func TestDiscordDeliveryHonorsCallerCancellation(t *testing.T) {
+	var calls, sleeps int
+	transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+		calls++
+		return nil, request.Context().Err()
+	})
+	client, err := newDiscordClient("https://discord.invalid/webhook", &http.Client{Transport: transport}, func(context.Context, time.Duration) error {
+		sleeps++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := client.send(ctx, discordPayload{Content: "hello"}); !errors.Is(err, errWebhookTimeout) {
+		t.Fatalf("send() error = %v, want timeout", err)
+	}
+	if calls > 1 || sleeps != 0 {
+		t.Fatalf("canceled delivery made %d calls and %d sleeps", calls, sleeps)
+	}
+}
+
+func TestRetryDelayParsing(t *testing.T) {
+	if got, ok := parseRetryDelay("1.5"); !ok || got != 1500*time.Millisecond {
+		t.Fatalf("parseRetryDelay() = %v, %v", got, ok)
+	}
+	if _, ok := parseRetryDelay("garbage"); ok {
+		t.Fatal("invalid retry delay accepted")
+	}
+	httpDate := time.Now().Add(time.Second).UTC().Format(http.TimeFormat)
+	if got, ok := parseRetryDelay(httpDate); !ok || got < 0 || got > time.Second {
+		t.Fatalf("HTTP-date retry delay = %v, %v", got, ok)
+	}
+	if got := fallbackRetryDelay(1); got != 500*time.Millisecond {
+		t.Fatalf("fallbackRetryDelay(1) = %v", got)
+	}
+	if retryableStatus(499) || !retryableStatus(500) || !retryableStatus(429) {
+		t.Fatal("retryableStatus returned unexpected result")
+	}
+
+	canceled, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := sleepContext(canceled, time.Hour); !errors.Is(err, context.Canceled) {
+		t.Fatalf("sleepContext() error = %v", err)
+	}
+}
+
+func TestNormalizeDiscordPayloadLimits(t *testing.T) {
+	fields := make([]discordEmbedField, 30)
+	for index := range fields {
+		fields[index] = discordEmbedField{
+			Name:  strings.Repeat("n", 300),
+			Value: strings.Repeat("界", 1200),
+		}
+	}
+	payload := discordPayload{
+		Content: strings.Repeat("c", 2100),
+		Embeds: []discordEmbed{{
+			Title:       strings.Repeat("t", 300),
+			Description: strings.Repeat("d", 5000),
+			Fields:      fields,
+			Footer:      &discordEmbedFooter{Text: strings.Repeat("f", 2200)},
+		}},
+	}
+	normalizeDiscordPayload(&payload)
+	if utf8.RuneCountInString(payload.Content) != 2000 || utf8.RuneCountInString(payload.Embeds[0].Title) > 256 || utf8.RuneCountInString(payload.Embeds[0].Description) > 4096 {
+		t.Fatal("individual limits were not applied")
+	}
+	if len(payload.Embeds[0].Fields) > 25 || discordEmbedCharacters(payload.Embeds) > 6000 {
+		t.Fatalf("embed limits = fields %d, chars %d", len(payload.Embeds[0].Fields), discordEmbedCharacters(payload.Embeds))
+	}
+	for _, field := range payload.Embeds[0].Fields {
+		if utf8.RuneCountInString(field.Name) > 256 || utf8.RuneCountInString(field.Value) > 1024 {
+			t.Fatal("field limit was not applied")
+		}
+	}
+
+	if got := truncateRunes("abcdef", 4); got != "… [t" {
+		t.Fatalf("small truncation = %q", got)
+	}
+	if got := truncateRunes("abc", 3); got != "abc" {
+		t.Fatalf("unneeded truncation = %q", got)
+	}
+	if got := truncateRunes("abc", 0); got != "" {
+		t.Fatalf("zero truncation = %q", got)
+	}
+}
+
+func TestDiscordClientRejectsInvalidURLs(t *testing.T) {
+	for _, value := range []string{"", "relative/path", "ftp://example.com/hook", "https:///hook"} {
+		if _, err := newDiscordClient(value, nil, nil); err == nil {
+			t.Fatalf("newDiscordClient(%q) succeeded", value)
+		}
+	}
+}
+
+func TestIsTimeout(t *testing.T) {
+	if !isTimeout(context.DeadlineExceeded) || isTimeout(errors.New("ordinary")) {
+		t.Fatal("isTimeout returned unexpected result")
+	}
+}

haven-notify/e2e_test.go

@@ -0,0 +1,322 @@
+//go:build e2e
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/log"
+	"github.com/testcontainers/testcontainers-go/modules/mockserver"
+	"github.com/testcontainers/testcontainers-go/network"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+const mockServerImage = "mockserver/mockserver:5.15.0@sha256:0f9ef78c94894ac3e70135d156193b25e23872575d58e2228344964273b4af6b"
+
+type mockRecordedRequest struct {
+	Method  string              `json:"method"`
+	Path    string              `json:"path"`
+	Headers map[string][]string `json:"headers"`
+	Body    struct {
+		RawBytes string `json:"rawBytes"`
+	} `json:"body"`
+}
+
+func TestContainerE2E(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
+	defer cancel()
+	logger := log.TestLogger(t)
+
+	testNetwork, err := network.New(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testcontainers.CleanupNetwork(t, testNetwork)
+
+	mock, err := mockserver.Run(ctx, mockServerImage,
+		network.WithNetwork([]string{"discord-mock"}, testNetwork),
+		testcontainers.WithLogger(logger),
+	)
+	testcontainers.CleanupContainer(t, mock)
+	if err != nil {
+		t.Fatalf("start MockServer: %v", err)
+	}
+	mockURL, err := mock.URL(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buildPlatform := "linux/amd64"
+	targetOS := "linux"
+	targetArch := "amd64"
+	app, err := testcontainers.Run(ctx, "",
+		testcontainers.WithDockerfile(testcontainers.FromDockerfile{
+			Context:    ".",
+			Dockerfile: "Dockerfile",
+			Repo:       "haven-notify-e2e",
+			Tag:        "test",
+			KeepImage:  false,
+			BuildArgs: map[string]*string{
+				"BUILDPLATFORM": &buildPlatform,
+				"TARGETOS":      &targetOS,
+				"TARGETARCH":    &targetArch,
+			},
+		}),
+		network.WithNetwork([]string{"haven-notify"}, testNetwork),
+		testcontainers.WithEnv(map[string]string{"WEBHOOK_URL": "http://discord-mock:1080/discord"}),
+		testcontainers.WithExposedPorts("8080/tcp"),
+		testcontainers.WithWaitStrategy(wait.ForHTTP("/ready").WithPort("8080/tcp").WithStartupTimeout(45*time.Second)),
+		testcontainers.WithLogger(logger),
+	)
+	testcontainers.CleanupContainer(t, app)
+	if err != nil {
+		t.Fatalf("start Haven Notify: %v", err)
+	}
+	appURL, err := app.PortEndpoint(ctx, "8080/tcp", "http")
+	if err != nil {
+		t.Fatal(err)
+	}
+	inspect, err := app.Inspect(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if inspect.Config.User != "haven-notify" {
+		t.Fatalf("container user = %q", inspect.Config.User)
+	}
+
+	t.Run("health probes", func(t *testing.T) {
+		for path, body := range map[string]string{"/ready": "Ready", "/live": "Alive"} {
+			response, err := http.Get(appURL + path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			contents, _ := io.ReadAll(response.Body)
+			_ = response.Body.Close()
+			if response.StatusCode != 200 || string(contents) != body {
+				t.Fatalf("%s = (%d, %q)", path, response.StatusCode, contents)
+			}
+		}
+	})
+
+	if err := configureMock(mockURL, []mockExpectation{{Status: 204}}); err != nil {
+		t.Fatal(err)
+	}
+	cases := []struct {
+		path string
+		body string
+		want discordPayload
+	}{
+		{
+			path: "/notify",
+			body: `{"title":"Título \"quoted\"","message":"line 1\nC:\\backups @everyone"}`,
+			want: discordPayload{
+				Content:         "**Título \"quoted\"**\nline 1\nC:\\backups @everyone",
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			path: "/template/notify/backup",
+			body: `{"title":"Nightly","asset":"photos","backupSizeInMB":1536,"extra":[{"name":"Host","value":"nebula"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "📦 Backup - Nightly",
+					Description: "**photos** has been backed up successfully! ✅🫡\n",
+					Color:       3066993,
+					Fields: []discordEmbedField{
+						{Name: "💾 Backup Size", Value: "1.50 GiB", Inline: true},
+						{Name: "Host", Value: "nebula", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "✨ Haven Notify ✨"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			path: "/template/notify/update",
+			body: `{"host":"nexus","asset":"k3s","time":0}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "🔄 Update - k3s",
+					Description: "**nexus** has successfully updated **k3s**! ✅",
+					Color:       3447003,
+					Fields: []discordEmbedField{
+						{Name: "⏱️ Time Taken", Value: "0 seconds", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "✨ Haven Notify ✨"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			path: "/template/notify/error",
+			body: `{"caller":"backup","message":"line 1\n\"quoted\" <tag>","critical":true}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "❌ Error",
+					Description: "**backup** encountered an error!",
+					Color:       15158332,
+					Fields: []discordEmbedField{
+						{Name: "📄 Message", Value: "line 1\n\"quoted\" <tag>", Inline: false},
+					},
+					Footer: &discordEmbedFooter{Text: "✨ Haven Notify ✨"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+	}
+	for _, test := range cases {
+		response, err := postJSON(appURL+test.path, test.body)
+		if err != nil {
+			t.Fatalf("POST %s: %v", test.path, err)
+		}
+		contents, _ := io.ReadAll(response.Body)
+		_ = response.Body.Close()
+		if response.StatusCode != 200 || string(contents) != "Notification sent" {
+			t.Fatalf("POST %s = (%d, %q)", test.path, response.StatusCode, contents)
+		}
+	}
+	recorded, err := retrieveMockRequests(mockURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(recorded) != len(cases) {
+		t.Fatalf("recorded requests = %d, want %d", len(recorded), len(cases))
+	}
+	for index, request := range recorded {
+		if request.Method != http.MethodPost || request.Path != "/discord" || len(request.Headers["Content-Type"]) == 0 || request.Headers["Content-Type"][0] != "application/json" {
+			t.Fatalf("recorded request %d = %+v", index, request)
+		}
+		body, err := base64.StdEncoding.DecodeString(request.Body.RawBytes)
+		if err != nil || !json.Valid(body) || bytes.Contains(body, []byte("&#34;")) {
+			t.Fatalf("recorded body %d = %s, %v", index, body, err)
+		}
+		want, err := json.Marshal(cases[index].want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(body, want) {
+			t.Fatalf("recorded body %d = %s, want %s", index, body, want)
+		}
+	}
+
+	t.Run("invalid request does not call webhook", func(t *testing.T) {
+		if err := configureMock(mockURL, []mockExpectation{{Status: 204}}); err != nil {
+			t.Fatal(err)
+		}
+		response, err := postJSON(appURL+"/notify", `{"title":"missing message"}`)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = response.Body.Close()
+		if response.StatusCode != 400 {
+			t.Fatalf("status = %d", response.StatusCode)
+		}
+		recorded, err := retrieveMockRequests(mockURL)
+		if err != nil || len(recorded) != 0 {
+			t.Fatalf("recorded = %d, error = %v", len(recorded), err)
+		}
+	})
+
+	t.Run("transient failure retries", func(t *testing.T) {
+		if err := configureMock(mockURL, []mockExpectation{{Status: 500, Times: 1, Priority: 10}, {Status: 204}}); err != nil {
+			t.Fatal(err)
+		}
+		response, err := postJSON(appURL+"/notify", `{"title":"retry","message":"once"}`)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = response.Body.Close()
+		if response.StatusCode != 200 {
+			t.Fatalf("status = %d", response.StatusCode)
+		}
+		recorded, err := retrieveMockRequests(mockURL)
+		if err != nil || len(recorded) != 2 {
+			t.Fatalf("recorded = %d, error = %v", len(recorded), err)
+		}
+	})
+
+	bad, err := testcontainers.Run(ctx, "haven-notify-e2e:test",
+		testcontainers.WithWaitStrategy(wait.ForExit().WithExitTimeout(15*time.Second)),
+		testcontainers.WithLogger(logger),
+	)
+	testcontainers.CleanupContainer(t, bad)
+	if err != nil {
+		t.Fatalf("start unconfigured container: %v", err)
+	}
+	state, err := bad.State(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if state.Running || state.ExitCode == 0 {
+		t.Fatalf("unconfigured container state = %+v", state)
+	}
+}
+
+type mockExpectation struct {
+	Status   int
+	Times    int
+	Priority int
+}
+
+func configureMock(baseURL string, expectations []mockExpectation) error {
+	request, _ := http.NewRequest(http.MethodPut, baseURL+"/mockserver/reset", nil)
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return err
+	}
+	_ = response.Body.Close()
+	for _, expectation := range expectations {
+		payload := map[string]any{
+			"httpRequest":  map[string]any{"method": "POST", "path": "/discord"},
+			"httpResponse": map[string]any{"statusCode": expectation.Status},
+			"priority":     expectation.Priority,
+		}
+		if expectation.Times > 0 {
+			payload["times"] = map[string]any{"remainingTimes": expectation.Times, "unlimited": false}
+		}
+		body, _ := json.Marshal(payload)
+		request, _ = http.NewRequest(http.MethodPut, baseURL+"/mockserver/expectation", bytes.NewReader(body))
+		request.Header.Set("Content-Type", "application/json")
+		response, err = http.DefaultClient.Do(request)
+		if err != nil {
+			return err
+		}
+		_ = response.Body.Close()
+		if response.StatusCode < 200 || response.StatusCode >= 300 {
+			return fmt.Errorf("MockServer expectation returned %d", response.StatusCode)
+		}
+	}
+	return nil
+}
+
+func retrieveMockRequests(baseURL string) ([]mockRecordedRequest, error) {
+	request, _ := http.NewRequest(http.MethodPut, baseURL+"/mockserver/retrieve?type=REQUESTS&format=JSON", strings.NewReader(`{"path":"/discord"}`))
+	request.Header.Set("Content-Type", "application/json")
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	var recorded []mockRecordedRequest
+	err = json.NewDecoder(response.Body).Decode(&recorded)
+	return recorded, err
+}
+
+func postJSON(url, body string) (*http.Response, error) {
+	request, err := http.NewRequest(http.MethodPost, url, strings.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("Content-Type", "application/json")
+	return http.DefaultClient.Do(request)
+}

haven-notify/go.mod

@@ -0,0 +1,61 @@
+module git.ivanch.me/ivanch/server-scripts/haven-notify
+
+go 1.25.10
+
+require (
+	github.com/testcontainers/testcontainers-go v0.42.0
+	github.com/testcontainers/testcontainers-go/modules/mockserver v0.42.0
+)
+
+require (
+	dario.cat/mergo v1.0.2 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/go-connections v0.6.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/ebitengine/purego v0.10.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.10 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/go-archive v0.2.0 // indirect
+	github.com/moby/moby/api v1.54.1 // indirect
+	github.com/moby/moby/client v0.4.0 // indirect
+	github.com/moby/patternmatcher v0.6.1 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/shirou/gopsutil/v4 v4.26.3 // indirect
+	github.com/sirupsen/logrus v1.9.4 // indirect
+	github.com/stretchr/testify v1.11.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.41.0 // indirect
+	go.opentelemetry.io/otel/metric v1.41.0 // indirect
+	go.opentelemetry.io/otel/trace v1.41.0 // indirect
+	golang.org/x/crypto v0.48.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

haven-notify/go.sum

@@ -0,0 +1,145 @@
+dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
+dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/BraspagDevelopers/mock-server-client v0.2.2 h1:Zro0OonNeaDwkkQGIxeJQfYweNKZ+m+8QIlDZAFRc/4=
+github.com/BraspagDevelopers/mock-server-client v0.2.2/go.mod h1:LHulrZSfbCNeS/CoycaWdhE495FnyeI3iXm6+4Zjz5c=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
+github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
+github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
+github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
+github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
+github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So=
+github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
+github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
+github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
+github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
+github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
+github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
+github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
+github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
+github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
+github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
+github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
+github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
+github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
+github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
+github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
+github.com/testcontainers/testcontainers-go/modules/mockserver v0.42.0 h1:UfORuhjP6+wd28O21m8cx4CLe+RbWlo31SI9ZgrqXn4=
+github.com/testcontainers/testcontainers-go/modules/mockserver v0.42.0/go.mod h1:uwubSg20I0nSLjtpVl4m2Ixin3yeXks4zkHwaCzbRvo=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
+go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
+go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
+go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
+go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
+go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
+go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
+go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
+go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
+go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
+golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
+gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
+pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
+pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=

haven-notify/main.go

@@ -1,214 +1,37 @@
 package main
 
 import (
-	"bytes"
-	"encoding/json"
+	"context"
+	"errors"
 	"fmt"
-	"html/template"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
-	"strings"
-	"time"
+	"os/signal"
+	"syscall"
 )
 
-// Notification payload
-type Notification struct {
-	Title   string `json:"title"`
-	Message string `json:"message"`
-}
+const listenAddress = ":8080"
 
 func main() {
-	http.HandleFunc("/notify", notifyHandler)
-	http.HandleFunc("/ready", readinessHandler)
-	http.HandleFunc("/live", livenessHandler)
-	http.HandleFunc("/template/notify/", templateNotifyHandler)
-	log.Println("Starting server on :8080...")
-	log.Fatal(http.ListenAndServe(":8080", nil))
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	logger := log.Default()
+	if err := run(ctx, os.Getenv("WEBHOOK_URL"), listenAddress, logger); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		logger.Fatalf("Server stopped unexpectedly: %v", err)
+	}
 }
 
-func notifyHandler(w http.ResponseWriter, r *http.Request) {
-	log.Printf("Incoming %s request from %s to %s", r.Method, r.RemoteAddr, r.URL.Path)
-	if r.Method != http.MethodPost {
-		log.Printf("Method not allowed: %s", r.Method)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		w.Write([]byte("Method not allowed"))
-		return
-	}
-
-	var notif Notification
-	if err := json.NewDecoder(r.Body).Decode(&notif); err != nil {
-		log.Printf("Invalid payload: %v", err)
-		w.WriteHeader(http.StatusBadRequest)
-		w.Write([]byte("Invalid payload"))
-		return
-	}
-	log.Printf("Received notification payload: Title='%s', Message='%s'", notif.Title, notif.Message)
-
-	// Call Discord notification function
-	if err := sendDiscordNotification(notif.Title, notif.Message); err != nil {
-		log.Printf("Failed to send Discord notification: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to send Discord notification"))
-		return
-	}
-
-	log.Printf("Notification sent successfully for Title='%s'", notif.Title)
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("Notification sent"))
-}
-
-// Readiness handler
-func readinessHandler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("Ready"))
-}
-
-// Liveness handler
-func livenessHandler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("Alive"))
-}
-
-func sendDiscordNotification(title, message string) error {
-	webhookURL := os.Getenv("WEBHOOK_URL")
-	if webhookURL == "" {
-		log.Printf("WEBHOOK_URL environment variable not set")
-		return fmt.Errorf("WEBHOOK_URL environment variable not set")
-	}
-
-	// Discord webhook payload
-	type discordPayload struct {
-		Content string `json:"content"`
-	}
-
-	content := "**" + title + "**\n" + message
-	payload := discordPayload{Content: content}
-
-	jsonData, err := json.Marshal(payload)
+func run(ctx context.Context, webhookURL, address string, logger *log.Logger) error {
+	app, err := newApplication(applicationConfig{WebhookURL: webhookURL, Logger: logger})
 	if err != nil {
-		log.Printf("Failed to marshal Discord payload: %v", err)
-		return err
+		return fmt.Errorf("configuration failed: %w", err)
 	}
-
-	log.Printf("Sending Discord notification: Title='%s', Message='%s'", title, message)
-	resp, err := http.Post(webhookURL, "application/json", bytes.NewBuffer(jsonData))
+	listener, err := netListen("tcp", address)
 	if err != nil {
-		log.Printf("Error posting to Discord webhook: %v", err)
-		return err
+		return fmt.Errorf("listen on %s: %w", address, err)
 	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		log.Printf("Discord webhook returned status: %s", resp.Status)
-		return fmt.Errorf("Discord webhook returned status: %s", resp.Status)
-	}
-
-	log.Printf("Discord notification sent successfully: Title='%s'", title)
-	return nil
-}
-
-func templateNotifyHandler(w http.ResponseWriter, r *http.Request) {
-	log.Printf("Incoming %s request from %s to %s", r.Method, r.RemoteAddr, r.URL.Path)
-	if r.Method != http.MethodPost {
-		log.Printf("Method not allowed: %s", r.Method)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		w.Write([]byte("Method not allowed"))
-		return
-	}
-
-	templateName := r.URL.Path[len("/template/notify/"):] // Extract template name
-	if templateName == "" {
-		log.Printf("Template name not provided")
-		w.WriteHeader(http.StatusBadRequest)
-		w.Write([]byte("Template name not provided"))
-		return
-	}
-
-	templatePath := "template/" + templateName + ".tmpl"
-	templateData, err := ioutil.ReadFile(templatePath)
-	if err != nil {
-		log.Printf("Failed to read template: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to read template"))
-		return
-	}
-
-	tmpl, err := template.New(templateName).Funcs(template.FuncMap{
-		"formatSize": func(size float64) string {
-			if size > 1024 {
-				return fmt.Sprintf("%.2f GiB", size/1024)
-			}
-			return fmt.Sprintf("%.2f MiB", size)
-		},
-		"upper": strings.ToUpper,
-		"lower": strings.ToLower,
-		"title": strings.Title,
-		"now": func() string {
-			return fmt.Sprintf("%d", time.Now().Unix())
-		},
-		"formatTime": func(timestamp string) string {
-			if timestamp == "" {
-				return time.Now().Format("2006-01-02T15:04:05Z")
-			}
-			return timestamp
-		},
-	}).Parse(string(templateData))
-	if err != nil {
-		log.Printf("Failed to parse template: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to parse template"))
-		return
-	}
-
-	var rawPayload map[string]interface{}
-	if err := json.NewDecoder(r.Body).Decode(&rawPayload); err != nil {
-		log.Printf("Invalid payload: %v", err)
-		w.WriteHeader(http.StatusBadRequest)
-		w.Write([]byte("Invalid payload"))
-		return
-	}
-
-	// Normalize keys to lowercase for case-insensitive parsing
-	payload := make(map[string]interface{})
-	for key, value := range rawPayload {
-		payload[strings.ToLower(key)] = value
-	}
-
-	var filledTemplate bytes.Buffer
-	if err := tmpl.Execute(&filledTemplate, payload); err != nil {
-		log.Printf("Failed to execute template: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to execute template"))
-		return
-	}
-
-	webhookURL := os.Getenv("WEBHOOK_URL")
-	if webhookURL == "" {
-		log.Printf("WEBHOOK_URL environment variable not set")
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("WEBHOOK_URL environment variable not set"))
-		return
-	}
-
-	resp, err := http.Post(webhookURL, "application/json", &filledTemplate)
-	if err != nil {
-		log.Printf("Error posting to Discord webhook: %v", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to send notification"))
-		return
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		log.Printf("Discord webhook returned status: %s", resp.Status)
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Failed to send notification"))
-		return
-	}
-
-	log.Printf("Notification sent successfully using template '%s'", templateName)
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("Notification sent"))
+	logger.Printf("Starting server on %s", address)
+	return serve(ctx, newHTTPServer(app.handler()), listener)
 }

haven-notify/server.go

@@ -0,0 +1,402 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"mime"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+var netListen = net.Listen
+
+const (
+	maxRequestBody  = 64 << 10
+	shutdownTimeout = 10 * time.Second
+)
+
+var (
+	errInvalidPayload       = errors.New("invalid payload")
+	errUnsupportedMediaType = errors.New("unsupported media type")
+)
+
+type applicationConfig struct {
+	WebhookURL string
+	HTTPClient *http.Client
+	Sleep      sleepFunc
+	Logger     *log.Logger
+	Now        func() time.Time
+}
+
+type application struct {
+	discord  *discordClient
+	renderer *templateRenderer
+	logger   *log.Logger
+	now      func() time.Time
+}
+
+type notificationRequest struct {
+	Title   *string `json:"title"`
+	Message *string `json:"message"`
+}
+
+type extraRequest struct {
+	Name  *string `json:"name"`
+	Value *string `json:"value"`
+}
+
+type extraRequests []extraRequest
+
+type backupRequest struct {
+	Title          *string       `json:"title"`
+	Asset          *string       `json:"asset"`
+	BackupSizeInMB *float64      `json:"backupSizeInMB"`
+	Extra          extraRequests `json:"extra"`
+}
+
+type updateRequest struct {
+	Host  *string  `json:"host"`
+	Asset *string  `json:"asset"`
+	Time  *float64 `json:"time"`
+}
+
+type errorRequest struct {
+	Caller   *string       `json:"caller"`
+	Message  *string       `json:"message"`
+	Critical *bool         `json:"critical"`
+	Extra    extraRequests `json:"extra"`
+}
+
+func newApplication(cfg applicationConfig) (*application, error) {
+	logger := cfg.Logger
+	if logger == nil {
+		logger = log.New(io.Discard, "", 0)
+	}
+	now := cfg.Now
+	if now == nil {
+		now = time.Now
+	}
+
+	renderer, err := newTemplateRenderer()
+	if err != nil {
+		return nil, fmt.Errorf("load templates: %w", err)
+	}
+
+	discord, err := newDiscordClient(cfg.WebhookURL, cfg.HTTPClient, cfg.Sleep)
+	if err != nil {
+		return nil, err
+	}
+
+	return &application{discord: discord, renderer: renderer, logger: logger, now: now}, nil
+}
+
+func (a *application) handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/notify", a.requireMethod(http.MethodPost, a.notifyHandler))
+	mux.HandleFunc("/template/notify/backup", a.requireMethod(http.MethodPost, a.backupHandler))
+	mux.HandleFunc("/template/notify/update", a.requireMethod(http.MethodPost, a.updateHandler))
+	mux.HandleFunc("/template/notify/error", a.requireMethod(http.MethodPost, a.errorHandler))
+	mux.HandleFunc("/api/v2/messages", a.requireMethod(http.MethodPost, a.v2MessageHandler))
+	mux.HandleFunc("/api/v2/notifications/backup", a.requireMethod(http.MethodPost, a.v2BackupHandler))
+	mux.HandleFunc("/api/v2/notifications/update", a.requireMethod(http.MethodPost, a.v2UpdateHandler))
+	mux.HandleFunc("/api/v2/notifications/error", a.requireMethod(http.MethodPost, a.v2ErrorHandler))
+	mux.HandleFunc("/ready", a.requireProbeMethod(a.readinessHandler))
+	mux.HandleFunc("/live", a.requireProbeMethod(a.livenessHandler))
+	mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
+		writeText(w, http.StatusNotFound, "Not found")
+	})
+	return mux
+}
+
+func newHTTPServer(handler http.Handler) *http.Server {
+	return &http.Server{
+		Handler:           handler,
+		ReadHeaderTimeout: 5 * time.Second,
+		ReadTimeout:       10 * time.Second,
+		WriteTimeout:      15 * time.Second,
+		IdleTimeout:       60 * time.Second,
+	}
+}
+
+func serve(ctx context.Context, server *http.Server, listener net.Listener) error {
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- server.Serve(listener)
+	}()
+
+	select {
+	case err := <-errCh:
+		return err
+	case <-ctx.Done():
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+		defer cancel()
+		if err := server.Shutdown(shutdownCtx); err != nil {
+			return fmt.Errorf("graceful shutdown: %w", err)
+		}
+		return <-errCh
+	}
+}
+
+func (a *application) requireMethod(method string, next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != method {
+			w.Header().Set("Allow", method)
+			writeText(w, http.StatusMethodNotAllowed, "Method not allowed")
+			return
+		}
+		next(w, r)
+	}
+}
+
+func (a *application) requireProbeMethod(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet && r.Method != http.MethodHead {
+			w.Header().Set("Allow", "GET, HEAD")
+			writeText(w, http.StatusMethodNotAllowed, "Method not allowed")
+			return
+		}
+		next(w, r)
+	}
+}
+
+func (a *application) notifyHandler(w http.ResponseWriter, r *http.Request) {
+	var request notificationRequest
+	if err := decodeRequest(w, r, &request, "title", "message"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	if !requiredString(request.Title) || !requiredString(request.Message) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	payload := discordPayload{
+		Content: "**" + *request.Title + "**\n" + *request.Message,
+	}
+	if !a.send(w, r, payload) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) backupHandler(w http.ResponseWriter, r *http.Request) {
+	var request backupRequest
+	if err := decodeRequest(w, r, &request, "title", "asset", "backupsizeinmb", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	if !requiredString(request.Title) || !requiredString(request.Asset) || request.BackupSizeInMB == nil || *request.BackupSizeInMB < 0 || !validExtra(request.Extra) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	payload, err := a.renderer.renderBackup(request)
+	if err != nil {
+		a.logger.Printf("Template rendering failed for %s", r.URL.Path)
+		writeText(w, http.StatusInternalServerError, "Failed to render notification")
+		return
+	}
+	if !a.send(w, r, payload) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) updateHandler(w http.ResponseWriter, r *http.Request) {
+	var request updateRequest
+	if err := decodeRequest(w, r, &request, "host", "asset", "time"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	if !requiredString(request.Host) || !requiredString(request.Asset) || request.Time == nil || *request.Time < 0 {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	payload, err := a.renderer.renderUpdate(request)
+	if err != nil {
+		a.logger.Printf("Template rendering failed for %s", r.URL.Path)
+		writeText(w, http.StatusInternalServerError, "Failed to render notification")
+		return
+	}
+	if !a.send(w, r, payload) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) errorHandler(w http.ResponseWriter, r *http.Request) {
+	var request errorRequest
+	if err := decodeRequest(w, r, &request, "caller", "message", "critical", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	if !requiredString(request.Caller) || !requiredString(request.Message) || request.Critical == nil || !validExtra(request.Extra) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	payload, err := a.renderer.renderError(request)
+	if err != nil {
+		a.logger.Printf("Template rendering failed for %s", r.URL.Path)
+		writeText(w, http.StatusInternalServerError, "Failed to render notification")
+		return
+	}
+	if !a.send(w, r, payload) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) readinessHandler(w http.ResponseWriter, _ *http.Request) {
+	writeText(w, http.StatusOK, "Ready")
+}
+
+func (a *application) livenessHandler(w http.ResponseWriter, _ *http.Request) {
+	writeText(w, http.StatusOK, "Alive")
+}
+
+func (a *application) send(w http.ResponseWriter, r *http.Request, payload discordPayload) bool {
+	if err := a.discord.send(r.Context(), payload); err != nil {
+		if errors.Is(err, errWebhookTimeout) {
+			a.logger.Printf("Discord delivery timed out for %s", r.URL.Path)
+			writeText(w, http.StatusGatewayTimeout, "Webhook timed out")
+			return false
+		}
+		a.logger.Printf("Discord delivery failed for %s", r.URL.Path)
+		writeText(w, http.StatusBadGateway, "Failed to send notification")
+		return false
+	}
+	return true
+}
+
+func (a *application) writeDecodeError(w http.ResponseWriter, r *http.Request, err error) {
+	if errors.Is(err, errUnsupportedMediaType) {
+		a.logger.Printf("Rejected unsupported media type for %s", r.URL.Path)
+		writeText(w, http.StatusUnsupportedMediaType, "Content-Type must be application/json")
+		return
+	}
+	a.logger.Printf("Rejected invalid payload for %s", r.URL.Path)
+	writeText(w, http.StatusBadRequest, "Invalid payload")
+}
+
+func decodeRequest(w http.ResponseWriter, r *http.Request, destination any, knownKeys ...string) error {
+	mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+	if err != nil || !strings.EqualFold(mediaType, "application/json") {
+		return errUnsupportedMediaType
+	}
+
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestBody)
+	data, err := io.ReadAll(r.Body)
+	if err != nil {
+		return fmt.Errorf("read request: %w", errInvalidPayload)
+	}
+	if len(bytes.TrimSpace(data)) == 0 {
+		return errInvalidPayload
+	}
+	if err := rejectDuplicateKnownKeys(data, knownKeys); err != nil {
+		return errInvalidPayload
+	}
+	if err := json.Unmarshal(data, destination); err != nil {
+		return errInvalidPayload
+	}
+	return nil
+}
+
+func rejectDuplicateKnownKeys(data []byte, knownKeys []string) error {
+	known := make(map[string]struct{}, len(knownKeys))
+	for _, key := range knownKeys {
+		known[strings.ToLower(key)] = struct{}{}
+	}
+
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	token, err := decoder.Token()
+	if err != nil {
+		return err
+	}
+	delim, ok := token.(json.Delim)
+	if !ok || delim != '{' {
+		return errInvalidPayload
+	}
+
+	seen := make(map[string]struct{}, len(knownKeys))
+	for decoder.More() {
+		token, err = decoder.Token()
+		if err != nil {
+			return err
+		}
+		key, ok := token.(string)
+		if !ok {
+			return errInvalidPayload
+		}
+		canonical := strings.ToLower(key)
+		if _, isKnown := known[canonical]; isKnown {
+			if _, duplicate := seen[canonical]; duplicate {
+				return errInvalidPayload
+			}
+			seen[canonical] = struct{}{}
+		}
+		var value json.RawMessage
+		if err := decoder.Decode(&value); err != nil {
+			return err
+		}
+	}
+	if _, err := decoder.Token(); err != nil {
+		return err
+	}
+	if decoder.Decode(&struct{}{}) != io.EOF {
+		return errInvalidPayload
+	}
+	return nil
+}
+
+func (e *extraRequest) UnmarshalJSON(data []byte) error {
+	if err := rejectDuplicateKnownKeys(data, []string{"name", "value"}); err != nil {
+		return err
+	}
+	type rawExtra extraRequest
+	var decoded rawExtra
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return err
+	}
+	*e = extraRequest(decoded)
+	return nil
+}
+
+func (e *extraRequests) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
+		return errInvalidPayload
+	}
+	var decoded []extraRequest
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return err
+	}
+	*e = decoded
+	return nil
+}
+
+func requiredString(value *string) bool {
+	return value != nil && strings.TrimSpace(*value) != ""
+}
+
+func validExtra(extra extraRequests) bool {
+	for _, field := range extra {
+		if !requiredString(field.Name) || !requiredString(field.Value) {
+			return false
+		}
+	}
+	return true
+}
+
+func writeText(w http.ResponseWriter, status int, body string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.WriteHeader(status)
+	_, _ = io.WriteString(w, body)
+}

haven-notify/server_test.go

@@ -0,0 +1,451 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"text/template"
+	"time"
+)
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (fn roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+	return fn(request)
+}
+
+type webhookRecorder struct {
+	mu       sync.Mutex
+	requests [][]byte
+	status   int
+	err      error
+}
+
+func (r *webhookRecorder) RoundTrip(request *http.Request) (*http.Response, error) {
+	body, _ := io.ReadAll(request.Body)
+	r.mu.Lock()
+	r.requests = append(r.requests, body)
+	status, err := r.status, r.err
+	r.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	if status == 0 {
+		status = http.StatusNoContent
+	}
+	return response(status, nil), nil
+}
+
+func (r *webhookRecorder) bodies() [][]byte {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([][]byte(nil), r.requests...)
+}
+
+func testApplication(t testing.TB, transport http.RoundTripper, logger *log.Logger) *application {
+	t.Helper()
+	if transport == nil {
+		transport = &webhookRecorder{}
+	}
+	app, err := newApplication(applicationConfig{
+		WebhookURL: "http://discord.invalid/webhook",
+		HTTPClient: &http.Client{Transport: transport},
+		Sleep:      func(context.Context, time.Duration) error { return nil },
+		Logger:     logger,
+		Now:        func() time.Time { return time.Date(2026, time.July, 16, 12, 34, 56, 0, time.UTC) },
+	})
+	if err != nil {
+		t.Fatalf("newApplication() error = %v", err)
+	}
+	return app
+}
+
+func performRequest(handler http.Handler, method, path, contentType, body string) *httptest.ResponseRecorder {
+	request := httptest.NewRequest(method, path, strings.NewReader(body))
+	if contentType != "" {
+		request.Header.Set("Content-Type", contentType)
+	}
+	recorder := httptest.NewRecorder()
+	handler.ServeHTTP(recorder, request)
+	return recorder
+}
+
+func TestProbeAndRoutingContracts(t *testing.T) {
+	handler := testApplication(t, nil, nil).handler()
+	tests := []struct {
+		name       string
+		method     string
+		path       string
+		wantStatus int
+		wantBody   string
+		wantAllow  string
+	}{
+		{name: "ready get", method: http.MethodGet, path: "/ready", wantStatus: 200, wantBody: "Ready"},
+		{name: "ready head", method: http.MethodHead, path: "/ready", wantStatus: 200, wantBody: "Ready"},
+		{name: "live get", method: http.MethodGet, path: "/live", wantStatus: 200, wantBody: "Alive"},
+		{name: "probe wrong method", method: http.MethodPost, path: "/live", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "GET, HEAD"},
+		{name: "notify wrong method", method: http.MethodGet, path: "/notify", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "POST"},
+		{name: "backup wrong method", method: http.MethodGet, path: "/template/notify/backup", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "POST"},
+		{name: "update wrong method", method: http.MethodPut, path: "/template/notify/update", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "POST"},
+		{name: "error wrong method", method: http.MethodDelete, path: "/template/notify/error", wantStatus: 405, wantBody: "Method not allowed", wantAllow: "POST"},
+		{name: "unknown template", method: http.MethodPost, path: "/template/notify/missing", wantStatus: 404, wantBody: "Not found"},
+		{name: "trailing slash is unknown", method: http.MethodPost, path: "/notify/", wantStatus: 404, wantBody: "Not found"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := performRequest(handler, test.method, test.path, "", "")
+			if got.Code != test.wantStatus || got.Body.String() != test.wantBody {
+				t.Fatalf("response = (%d, %q), want (%d, %q)", got.Code, got.Body.String(), test.wantStatus, test.wantBody)
+			}
+			if got.Header().Get("Allow") != test.wantAllow {
+				t.Fatalf("Allow = %q, want %q", got.Header().Get("Allow"), test.wantAllow)
+			}
+			if got.Header().Get("Content-Type") != "text/plain; charset=utf-8" {
+				t.Fatalf("Content-Type = %q", got.Header().Get("Content-Type"))
+			}
+		})
+	}
+}
+
+func TestRequestValidationContracts(t *testing.T) {
+	tests := []struct {
+		name        string
+		path        string
+		contentType string
+		body        string
+		wantStatus  int
+	}{
+		{name: "missing content type", path: "/notify", body: `{}`, wantStatus: 415},
+		{name: "wrong content type", path: "/notify", contentType: "text/plain", body: `{}`, wantStatus: 415},
+		{name: "media type parameters", path: "/notify", contentType: "application/json; charset=utf-8", body: `{"title":"t","message":"m"}`, wantStatus: 200},
+		{name: "empty", path: "/notify", contentType: "application/json", body: "", wantStatus: 400},
+		{name: "array", path: "/notify", contentType: "application/json", body: `[]`, wantStatus: 400},
+		{name: "malformed", path: "/notify", contentType: "application/json", body: `{`, wantStatus: 400},
+		{name: "trailing document", path: "/notify", contentType: "application/json", body: `{"title":"t","message":"m"}{}`, wantStatus: 400},
+		{name: "missing required", path: "/notify", contentType: "application/json", body: `{"title":"t"}`, wantStatus: 400},
+		{name: "missing title", path: "/notify", contentType: "application/json", body: `{"message":"m"}`, wantStatus: 400},
+		{name: "blank required", path: "/notify", contentType: "application/json", body: `{"title":" ","message":"m"}`, wantStatus: 400},
+		{name: "blank message", path: "/notify", contentType: "application/json", body: `{"title":"t","message":"\t"}`, wantStatus: 400},
+		{name: "wrong type", path: "/notify", contentType: "application/json", body: `{"title":1,"message":"m"}`, wantStatus: 400},
+		{name: "wrong message type", path: "/notify", contentType: "application/json", body: `{"title":"t","message":false}`, wantStatus: 400},
+		{name: "known duplicate", path: "/notify", contentType: "application/json", body: `{"title":"one","TITLE":"two","message":"m"}`, wantStatus: 400},
+		{name: "unknown accepted", path: "/notify", contentType: "application/json", body: `{"title":"t","message":"m","future":true}`, wantStatus: 200},
+		{name: "case insensitive", path: "/notify", contentType: "application/json", body: `{"TITLE":"t","MESSAGE":"m"}`, wantStatus: 200},
+		{name: "backup negative size", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":"a","backupSizeInMB":-1}`, wantStatus: 400},
+		{name: "backup missing size", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":"a"}`, wantStatus: 400},
+		{name: "backup wrong size type", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":"a","backupSizeInMB":"large"}`, wantStatus: 400},
+		{name: "backup missing title", path: "/template/notify/backup", contentType: "application/json", body: `{"asset":"a","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup blank title", path: "/template/notify/backup", contentType: "application/json", body: `{"title":" ","asset":"a","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup wrong title type", path: "/template/notify/backup", contentType: "application/json", body: `{"title":1,"asset":"a","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup missing asset", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup blank asset", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":" ","backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup wrong asset type", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":1,"backupSizeInMB":1}`, wantStatus: 400},
+		{name: "backup wrong extra type", path: "/template/notify/backup", contentType: "application/json", body: `{"title":"t","asset":"a","backupSizeInMB":1,"extra":{}}`, wantStatus: 400},
+		{name: "update negative time", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":"a","time":-1}`, wantStatus: 400},
+		{name: "update missing host", path: "/template/notify/update", contentType: "application/json", body: `{"asset":"a","time":1}`, wantStatus: 400},
+		{name: "update blank host", path: "/template/notify/update", contentType: "application/json", body: `{"host":" ","asset":"a","time":1}`, wantStatus: 400},
+		{name: "update wrong host type", path: "/template/notify/update", contentType: "application/json", body: `{"host":1,"asset":"a","time":1}`, wantStatus: 400},
+		{name: "update missing asset", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","time":1}`, wantStatus: 400},
+		{name: "update blank asset", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":" ","time":1}`, wantStatus: 400},
+		{name: "update wrong asset type", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":false,"time":1}`, wantStatus: 400},
+		{name: "update missing time", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":"a"}`, wantStatus: 400},
+		{name: "update wrong time type", path: "/template/notify/update", contentType: "application/json", body: `{"host":"h","asset":"a","time":"soon"}`, wantStatus: 400},
+		{name: "error missing critical", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m"}`, wantStatus: 400},
+		{name: "error wrong critical type", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":"yes"}`, wantStatus: 400},
+		{name: "error missing caller", path: "/template/notify/error", contentType: "application/json", body: `{"message":"m","critical":false}`, wantStatus: 400},
+		{name: "error blank caller", path: "/template/notify/error", contentType: "application/json", body: `{"caller":" ","message":"m","critical":false}`, wantStatus: 400},
+		{name: "error wrong caller type", path: "/template/notify/error", contentType: "application/json", body: `{"caller":1,"message":"m","critical":false}`, wantStatus: 400},
+		{name: "error missing message", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","critical":false}`, wantStatus: 400},
+		{name: "error blank message", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":" ","critical":false}`, wantStatus: 400},
+		{name: "error wrong message type", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":[],"critical":false}`, wantStatus: 400},
+		{name: "invalid extra", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":false,"extra":[{"name":"n"}]}`, wantStatus: 400},
+		{name: "null extra", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":false,"extra":null}`, wantStatus: 400},
+		{name: "duplicate nested known key", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":false,"extra":[{"name":"one","NAME":"two","value":"v"}]}`, wantStatus: 400},
+		{name: "case insensitive nested fields", path: "/template/notify/error", contentType: "application/json", body: `{"CALLER":"c","MESSAGE":"m","CRITICAL":false,"EXTRA":[{"NAME":"n","VALUE":"v"}]}`, wantStatus: 200},
+		{name: "unknown nested field accepted", path: "/template/notify/error", contentType: "application/json", body: `{"caller":"c","message":"m","critical":false,"extra":[{"name":"n","value":"v","future":true}]}`, wantStatus: 200},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			got := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, test.path, test.contentType, test.body)
+			if got.Code != test.wantStatus {
+				t.Fatalf("status = %d, body = %q, want %d", got.Code, got.Body.String(), test.wantStatus)
+			}
+			wantBody := map[int]string{
+				http.StatusOK:                   "Notification sent",
+				http.StatusBadRequest:           "Invalid payload",
+				http.StatusUnsupportedMediaType: "Content-Type must be application/json",
+			}[test.wantStatus]
+			if got.Body.String() != wantBody {
+				t.Fatalf("body = %q, want %q", got.Body.String(), wantBody)
+			}
+			if got.Header().Get("Content-Type") != "text/plain; charset=utf-8" {
+				t.Fatalf("Content-Type = %q", got.Header().Get("Content-Type"))
+			}
+			wantCalls := 0
+			if test.wantStatus == http.StatusOK {
+				wantCalls = 1
+			}
+			if len(recorder.bodies()) != wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", len(recorder.bodies()), wantCalls)
+			}
+		})
+	}
+}
+
+func TestRequestBodyLimit(t *testing.T) {
+	prefix, suffix := `{"title":"t","message":"`, `"}`
+	for _, test := range []struct {
+		name      string
+		bodyBytes int
+		want      int
+		wantCalls int
+	}{
+		{name: "exactly at limit", bodyBytes: maxRequestBody, want: http.StatusOK, wantCalls: 1},
+		{name: "one byte over limit", bodyBytes: maxRequestBody + 1, want: http.StatusBadRequest},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			body := prefix + strings.Repeat("x", test.bodyBytes-len(prefix)-len(suffix)) + suffix
+			if len(body) != test.bodyBytes {
+				t.Fatalf("body length = %d, want %d", len(body), test.bodyBytes)
+			}
+			got := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, "/notify", "application/json", body)
+			if got.Code != test.want {
+				t.Fatalf("status = %d, want %d", got.Code, test.want)
+			}
+			if len(recorder.bodies()) != test.wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", len(recorder.bodies()), test.wantCalls)
+			}
+		})
+	}
+}
+
+func TestNotificationPayloads(t *testing.T) {
+	tests := []struct {
+		name  string
+		path  string
+		body  string
+		check func(*testing.T, discordPayload)
+	}{
+		{
+			name: "plain preserves unicode and quotes",
+			path: "/notify",
+			body: `{"title":"Héllo \"world\"","message":"line 1\nline 2 @everyone"}`,
+			check: func(t *testing.T, payload discordPayload) {
+				if payload.Content != "**Héllo \"world\"**\nline 1\nline 2 @everyone" {
+					t.Fatalf("content = %q", payload.Content)
+				}
+			},
+		},
+		{
+			name: "backup formats gibibytes and extras",
+			path: "/template/notify/backup",
+			body: `{"title":"Nightly","asset":"photos","backupSizeInMB":1536,"extra":[{"name":"Host","value":"nebula"}]}`,
+			check: func(t *testing.T, payload discordPayload) {
+				embed := payload.Embeds[0]
+				if embed.Title != "📦 Backup - Nightly" || embed.Fields[0].Value != "1.50 GiB" || embed.Fields[1].Value != "nebula" {
+					t.Fatalf("unexpected backup embed: %+v", embed)
+				}
+			},
+		},
+		{
+			name: "update accepts zero time",
+			path: "/template/notify/update",
+			body: `{"host":"nexus","asset":"k3s","time":0}`,
+			check: func(t *testing.T, payload discordPayload) {
+				if payload.Embeds[0].Fields[0].Value != "0 seconds" {
+					t.Fatalf("time = %q", payload.Embeds[0].Fields[0].Value)
+				}
+			},
+		},
+		{
+			name: "error does not HTML escape",
+			path: "/template/notify/error",
+			body: `{"caller":"backup","message":"line 1\n\"quoted\" <tag>","critical":true}`,
+			check: func(t *testing.T, payload discordPayload) {
+				embed := payload.Embeds[0]
+				if embed.Color != 15158332 || embed.Fields[0].Value != "line 1\n\"quoted\" <tag>" {
+					t.Fatalf("unexpected error embed: %+v", embed)
+				}
+			},
+		},
+		{
+			name: "noncritical color",
+			path: "/template/notify/error",
+			body: `{"caller":"backup","message":"failure","critical":false}`,
+			check: func(t *testing.T, payload discordPayload) {
+				if payload.Embeds[0].Color != 15844367 {
+					t.Fatalf("color = %d", payload.Embeds[0].Color)
+				}
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			got := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, test.path, "application/json", test.body)
+			if got.Code != http.StatusOK || got.Body.String() != "Notification sent" {
+				t.Fatalf("response = (%d, %q)", got.Code, got.Body.String())
+			}
+			bodies := recorder.bodies()
+			if len(bodies) != 1 || !json.Valid(bodies[0]) {
+				t.Fatalf("webhook bodies = %q", bodies)
+			}
+			var payload discordPayload
+			if err := json.Unmarshal(bodies[0], &payload); err != nil {
+				t.Fatal(err)
+			}
+			if payload.AllowedMentions.Parse == nil || len(payload.AllowedMentions.Parse) != 0 {
+				t.Fatalf("allowed mentions = %#v", payload.AllowedMentions)
+			}
+			test.check(t, payload)
+		})
+	}
+}
+
+func TestHandlerMapsWebhookErrorsAndDoesNotLogSecrets(t *testing.T) {
+	tests := []struct {
+		name       string
+		err        error
+		wantStatus int
+		wantBody   string
+	}{
+		{name: "delivery", err: errors.New("network down"), wantStatus: 502, wantBody: "Failed to send notification"},
+		{name: "timeout", err: context.DeadlineExceeded, wantStatus: 504, wantBody: "Webhook timed out"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var logs bytes.Buffer
+			transport := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, test.err })
+			app := testApplication(t, transport, log.New(&logs, "", 0))
+			got := performRequest(app.handler(), http.MethodPost, "/notify", "application/json", `{"title":"TOP-SECRET","message":"private-message"}`)
+			if got.Code != test.wantStatus || got.Body.String() != test.wantBody {
+				t.Fatalf("response = (%d, %q)", got.Code, got.Body.String())
+			}
+			for _, secret := range []string{"TOP-SECRET", "private-message", "discord.invalid"} {
+				if strings.Contains(logs.String(), secret) {
+					t.Fatalf("logs exposed %q: %s", secret, logs.String())
+				}
+			}
+		})
+	}
+}
+
+func TestConfigurationAndServerLifecycle(t *testing.T) {
+	for _, webhookURL := range []string{"", "not a URL", "ftp://example.com/hook", "https:///missing-host"} {
+		if _, err := newApplication(applicationConfig{WebhookURL: webhookURL}); err == nil {
+			t.Fatalf("newApplication(%q) succeeded", webhookURL)
+		}
+	}
+
+	app := testApplication(t, nil, nil)
+	server := newHTTPServer(app.handler())
+	if server.ReadHeaderTimeout != 5*time.Second || server.ReadTimeout != 10*time.Second || server.WriteTimeout != 15*time.Second || server.IdleTimeout != 60*time.Second {
+		t.Fatalf("unexpected server timeouts: %+v", server)
+	}
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- serve(ctx, server, listener) }()
+	cancel()
+	select {
+	case err := <-done:
+		if !errors.Is(err, http.ErrServerClosed) {
+			t.Fatalf("serve() error = %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("server did not shut down")
+	}
+}
+
+func TestRunLifecycleAndListenError(t *testing.T) {
+	if err := run(context.Background(), "", "127.0.0.1:0", log.New(io.Discard, "", 0)); err == nil {
+		t.Fatal("run() accepted missing configuration")
+	}
+
+	originalListen := netListen
+	netListen = func(string, string) (net.Listener, error) { return nil, errors.New("listen failed") }
+	if err := run(context.Background(), "http://discord.invalid/hook", "bad", log.New(io.Discard, "", 0)); err == nil || !strings.Contains(err.Error(), "listen failed") {
+		t.Fatalf("run() error = %v", err)
+	}
+	netListen = originalListen
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := run(ctx, "http://discord.invalid/hook", "127.0.0.1:0", log.New(io.Discard, "", 0)); !errors.Is(err, http.ErrServerClosed) {
+		t.Fatalf("run() error = %v", err)
+	}
+}
+
+func TestTemplateHandlerRenderFailures(t *testing.T) {
+	app := testApplication(t, nil, nil)
+	app.renderer = &templateRenderer{templates: map[string]*template.Template{}}
+	tests := []struct {
+		path string
+		body string
+	}{
+		{path: "/template/notify/backup", body: `{"title":"t","asset":"a","backupSizeInMB":1}`},
+		{path: "/template/notify/update", body: `{"host":"h","asset":"a","time":1}`},
+		{path: "/template/notify/error", body: `{"caller":"c","message":"m","critical":false}`},
+	}
+	for _, test := range tests {
+		got := performRequest(app.handler(), http.MethodPost, test.path, "application/json", test.body)
+		if got.Code != http.StatusInternalServerError || got.Body.String() != "Failed to render notification" {
+			t.Fatalf("%s response = (%d, %q)", test.path, got.Code, got.Body.String())
+		}
+	}
+}
+
+func FuzzErrorNotificationHandler(f *testing.F) {
+	seeds := []string{
+		`{"caller":"seed","message":"ok","critical":false}`,
+		`{"caller":"quotes","message":"line\n\"quoted\" <tag>","critical":true}`,
+		`{}`,
+		`not-json`,
+	}
+	for _, seed := range seeds {
+		f.Add([]byte(seed))
+	}
+
+	transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+		body, err := io.ReadAll(request.Body)
+		if err != nil || !json.Valid(body) {
+			return nil, errors.New("invalid outgoing JSON")
+		}
+		return response(http.StatusNoContent, nil), nil
+	})
+	handler := testApplication(f, transport, nil).handler()
+	f.Fuzz(func(t *testing.T, body []byte) {
+		request := httptest.NewRequest(http.MethodPost, "/template/notify/error", bytes.NewReader(body))
+		request.Header.Set("Content-Type", "application/json")
+		recorder := httptest.NewRecorder()
+		handler.ServeHTTP(recorder, request)
+		if recorder.Code < 200 || recorder.Code > 599 {
+			t.Fatalf("invalid status %d", recorder.Code)
+		}
+	})
+}
+
+func response(status int, headers http.Header) *http.Response {
+	if headers == nil {
+		headers = make(http.Header)
+	}
+	return &http.Response{StatusCode: status, Header: headers, Body: io.NopCloser(strings.NewReader(""))}
+}

haven-notify/template/backup.tmpl

@@ -1,35 +1,22 @@
-{{/*
-Docker Backup Notification Template
-Variables expected:
-- .title: The backup title/name
-- .asset: The asset being backed up  
-- .backupsizeinmb: The backup size in MB (will be formatted automatically)
-- .extra: Optional array of additional fields with .name and .value
-
-Template Functions Available:
-- formatSize: Formats size in MB/GB automatically
-*/}}
 {
   "embeds": [
     {
-      "title": "📦 Backup - {{.title}}",
-      "description": "**{{.asset}}** has been backup-ed successfully! ✅🫡\n",
+      "title": {{json (printf "📦 Backup - %s" .Title)}},
+      "description": {{json (printf "**%s** has been backed up successfully! ✅🫡\n" .Asset)}},
       "color": 3066993,
       "fields": [
         {
           "name": "💾 Backup Size",
-          "value": "{{if .backupsizeinmb}}{{formatSize .backupsizeinmb}}{{else}}Unknown{{end}}",
+          "value": {{json (formatSize .BackupSizeInMB)}},
           "inline": true
         }
-        {{- if .extra}}
-        {{- range $index, $field := .extra}},
+        {{- range .Extra}},
         {
-          "name": "{{$field.name}}",
-          "value": "{{$field.value}}",
+          "name": {{json .Name}},
+          "value": {{json .Value}},
           "inline": true
         }
         {{- end}}
-        {{- end}}
       ],
       "footer": {
         "text": "✨ Haven Notify ✨"

haven-notify/template/error.tmpl

@@ -1,36 +1,26 @@
-{{/*
-Error Notification Template
-Variables expected:
-- .caller: The caller of the error
-- .message: The error message
-- .critical: Boolean indicating if the error is critical
-- .extra: Optional array of additional fields with .name and .value
-*/}}
 {
   "embeds": [
     {
       "title": "❌ Error",
-      "description": "**{{.caller}}** encountered an error!",
-      "color": {{if .critical}}15158332{{else}}15844367{{end}},
+      "description": {{json (printf "**%s** encountered an error!" .Caller)}},
+      "color": {{if .Critical}}15158332{{else}}15844367{{end}},
       "fields": [
         {
           "name": "📄 Message",
-          "value": "{{.message}}",
+          "value": {{json .Message}},
           "inline": false
         }
-        {{- if .extra}}
-        {{- range $index, $field := .extra}},
+        {{- range .Extra}},
         {
-          "name": "{{$field.name}}",
-          "value": "{{$field.value}}",
+          "name": {{json .Name}},
+          "value": {{json .Value}},
           "inline": true
         }
         {{- end}}
-        {{- end}}
       ],
       "footer": {
         "text": "✨ Haven Notify ✨"
       }
     }
   ]
-}
+}

haven-notify/template/update.tmpl

@@ -1,23 +1,13 @@
-{{/*
-Update Notification Template
-Variables expected:
-- .host: The host where the update occurred
-- .asset: The asset being updated (Docker or k8s)
-- .time: The time in seconds that the script took to run
-
-Template Functions Available:
-- formatTime: Formats time in seconds to a human-readable format
-*/}}
 {
   "embeds": [
     {
-      "title": "🔄 Update - {{.asset}}",
-      "description": "**{{.host}}** has successfully updated **{{.asset}}**! ✅",
+      "title": {{json (printf "🔄 Update - %s" .Asset)}},
+      "description": {{json (printf "**%s** has successfully updated **%s**! ✅" .Host .Asset)}},
       "color": 3447003,
       "fields": [
         {
           "name": "⏱️ Time Taken",
-          "value": "{{if .time}}{{.time}}{{else}}Unknown{{end}} seconds",
+          "value": {{json (formatSeconds .Time)}},
           "inline": true
         }
       ],
@@ -26,4 +16,4 @@ Template Functions Available:
       }
     }
   ]
-}
+}

haven-notify/templates.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"bytes"
+	"embed"
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"strconv"
+	"text/template"
+)
+
+//go:embed template/*.tmpl
+var templatesFS embed.FS
+
+type templateRenderer struct {
+	templates map[string]*template.Template
+}
+
+type templateExtra struct {
+	Name  string
+	Value string
+}
+
+type backupTemplateData struct {
+	Title          string
+	Asset          string
+	BackupSizeInMB float64
+	Extra          []templateExtra
+}
+
+type updateTemplateData struct {
+	Host  string
+	Asset string
+	Time  float64
+}
+
+type errorTemplateData struct {
+	Caller   string
+	Message  string
+	Critical bool
+	Extra    []templateExtra
+}
+
+func newTemplateRenderer() (*templateRenderer, error) {
+	return newTemplateRendererFromFS(templatesFS)
+}
+
+func newTemplateRendererFromFS(templateFS fs.FS) (*templateRenderer, error) {
+	functions := template.FuncMap{
+		"formatSize":    formatSize,
+		"formatSeconds": formatSeconds,
+		"json":          jsonLiteral,
+	}
+	templates := make(map[string]*template.Template, 3)
+	for _, name := range []string{"backup", "update", "error"} {
+		contents, err := fs.ReadFile(templateFS, "template/"+name+".tmpl")
+		if err != nil {
+			return nil, err
+		}
+		parsed, err := template.New(name).Funcs(functions).Parse(string(contents))
+		if err != nil {
+			return nil, err
+		}
+		templates[name] = parsed
+	}
+	return &templateRenderer{templates: templates}, nil
+}
+
+func (r *templateRenderer) renderBackup(request backupRequest) (discordPayload, error) {
+	return r.render("backup", backupTemplateData{
+		Title:          *request.Title,
+		Asset:          *request.Asset,
+		BackupSizeInMB: *request.BackupSizeInMB,
+		Extra:          templateExtras(request.Extra),
+	})
+}
+
+func (r *templateRenderer) renderUpdate(request updateRequest) (discordPayload, error) {
+	return r.render("update", updateTemplateData{
+		Host:  *request.Host,
+		Asset: *request.Asset,
+		Time:  *request.Time,
+	})
+}
+
+func (r *templateRenderer) renderError(request errorRequest) (discordPayload, error) {
+	return r.render("error", errorTemplateData{
+		Caller:   *request.Caller,
+		Message:  *request.Message,
+		Critical: *request.Critical,
+		Extra:    templateExtras(request.Extra),
+	})
+}
+
+func (r *templateRenderer) render(name string, data any) (discordPayload, error) {
+	parsed, ok := r.templates[name]
+	if !ok {
+		return discordPayload{}, fmt.Errorf("unknown template %q", name)
+	}
+	var rendered bytes.Buffer
+	if err := parsed.Execute(&rendered, data); err != nil {
+		return discordPayload{}, err
+	}
+	var payload discordPayload
+	if err := json.Unmarshal(rendered.Bytes(), &payload); err != nil {
+		return discordPayload{}, fmt.Errorf("template produced invalid JSON: %w", err)
+	}
+	normalizeDiscordPayload(&payload)
+	return payload, nil
+}
+
+func templateExtras(extra extraRequests) []templateExtra {
+	result := make([]templateExtra, 0, len(extra))
+	for _, field := range extra {
+		result = append(result, templateExtra{Name: *field.Name, Value: *field.Value})
+	}
+	return result
+}
+
+func jsonLiteral(value any) (string, error) {
+	encoded, err := json.Marshal(value)
+	return string(encoded), err
+}
+
+func formatSize(size float64) string {
+	if size >= 1024 {
+		return fmt.Sprintf("%.2f GiB", size/1024)
+	}
+	return fmt.Sprintf("%.2f MiB", size)
+}
+
+func formatSeconds(seconds float64) string {
+	return strconv.FormatFloat(seconds, 'f', -1, 64) + " seconds"
+}

haven-notify/templates_test.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"testing/fstest"
+	"text/template"
+)
+
+func stringPointer(value string) *string  { return &value }
+func floatPointer(value float64) *float64 { return &value }
+func boolPointer(value bool) *bool        { return &value }
+
+func TestFormatHelpers(t *testing.T) {
+	tests := []struct {
+		value float64
+		want  string
+	}{
+		{0, "0.00 MiB"},
+		{1023, "1023.00 MiB"},
+		{1024, "1.00 GiB"},
+		{1536, "1.50 GiB"},
+	}
+	for _, test := range tests {
+		if got := formatSize(test.value); got != test.want {
+			t.Errorf("formatSize(%v) = %q, want %q", test.value, got, test.want)
+		}
+	}
+	if got := formatSeconds(42.5); got != "42.5 seconds" {
+		t.Fatalf("formatSeconds() = %q", got)
+	}
+	literal, err := jsonLiteral("line\n\"quoted\"")
+	if err != nil || literal != "\"line\\n\\\"quoted\\\"\"" {
+		t.Fatalf("jsonLiteral() = %q, %v", literal, err)
+	}
+}
+
+func TestTemplateRenderer(t *testing.T) {
+	renderer, err := newTemplateRenderer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	backup, err := renderer.renderBackup(backupRequest{
+		Title: stringPointer("Nightly"), Asset: stringPointer("photos"), BackupSizeInMB: floatPointer(1024),
+		Extra: []extraRequest{{Name: stringPointer("Host"), Value: stringPointer("nebula")}},
+	})
+	if err != nil || backup.Embeds[0].Fields[0].Value != "1.00 GiB" || len(backup.Embeds[0].Fields) != 2 {
+		t.Fatalf("backup render = %+v, %v", backup, err)
+	}
+
+	update, err := renderer.renderUpdate(updateRequest{Host: stringPointer("nexus"), Asset: stringPointer("k3s"), Time: floatPointer(0)})
+	if err != nil || update.Embeds[0].Fields[0].Value != "0 seconds" {
+		t.Fatalf("update render = %+v, %v", update, err)
+	}
+
+	errorPayload, err := renderer.renderError(errorRequest{
+		Caller: stringPointer("job"), Message: stringPointer("\"quoted\" <tag>"), Critical: boolPointer(false),
+	})
+	if err != nil || errorPayload.Embeds[0].Fields[0].Value != "\"quoted\" <tag>" || errorPayload.Embeds[0].Color != 15844367 {
+		t.Fatalf("error render = %+v, %v", errorPayload, err)
+	}
+
+	if _, err := renderer.render("missing", struct{}{}); err == nil {
+		t.Fatal("unknown template rendered")
+	}
+
+	encoded, err := json.Marshal(errorPayload)
+	if err != nil || !json.Valid(encoded) || strings.Contains(string(encoded), "&#34;") {
+		t.Fatalf("rendered JSON = %s, %v", encoded, err)
+	}
+}
+
+func TestTemplateExtraFieldCap(t *testing.T) {
+	renderer, err := newTemplateRenderer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	extra := make([]extraRequest, 40)
+	for index := range extra {
+		extra[index] = extraRequest{Name: stringPointer(fmt.Sprintf("field-%02d", index)), Value: stringPointer("value")}
+	}
+	payload, err := renderer.renderError(errorRequest{
+		Caller: stringPointer("job"), Message: stringPointer("failure"), Critical: boolPointer(true), Extra: extra,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(payload.Embeds[0].Fields) != 25 {
+		t.Fatalf("fields = %d, want 25", len(payload.Embeds[0].Fields))
+	}
+	if payload.Embeds[0].Fields[24].Name != "field-23" {
+		t.Fatalf("last retained field = %q", payload.Embeds[0].Fields[24].Name)
+	}
+}
+
+func TestTemplateRendererFailures(t *testing.T) {
+	if _, err := newTemplateRendererFromFS(fstest.MapFS{}); err == nil {
+		t.Fatal("missing templates were accepted")
+	}
+	invalidFS := fstest.MapFS{
+		"template/backup.tmpl": {Data: []byte("{{")},
+		"template/update.tmpl": {Data: []byte("{}")},
+		"template/error.tmpl":  {Data: []byte("{}")},
+	}
+	if _, err := newTemplateRendererFromFS(invalidFS); err == nil {
+		t.Fatal("invalid template was accepted")
+	}
+
+	renderer := &templateRenderer{templates: map[string]*template.Template{}}
+	renderer.templates["invalid-json"] = template.Must(template.New("invalid-json").Parse("not-json"))
+	if _, err := renderer.render("invalid-json", nil); err == nil {
+		t.Fatal("invalid JSON template rendered")
+	}
+	renderer.templates["execute-error"] = template.Must(template.New("execute-error").Parse("{{index . 0}}"))
+	if _, err := renderer.render("execute-error", nil); err == nil {
+		t.Fatal("template execution error was ignored")
+	}
+}

haven-notify/v2.go

@@ -0,0 +1,306 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+const (
+	v2ColorBrand    = 0x5865F2
+	v2ColorSuccess  = 0x57F287
+	v2ColorUpdate   = 0x3498DB
+	v2ColorWarning  = 0xFEE75C
+	v2ColorError    = 0xED4245
+	v2ColorCritical = 0x992D22
+
+	v2InlineFieldLimit = 80
+)
+
+type v2CommonRequest struct {
+	Source     *string       `json:"source"`
+	OccurredAt *string       `json:"occurredAt"`
+	Extra      extraRequests `json:"extra"`
+}
+
+type v2MessageRequest struct {
+	v2CommonRequest
+	Title   *string `json:"title"`
+	Message *string `json:"message"`
+}
+
+type v2BackupRequest struct {
+	v2CommonRequest
+	Title           *string  `json:"title"`
+	Asset           *string  `json:"asset"`
+	SizeBytes       *int64   `json:"sizeBytes"`
+	DurationSeconds *float64 `json:"durationSeconds"`
+}
+
+type v2UpdateRequest struct {
+	v2CommonRequest
+	Host            *string  `json:"host"`
+	Asset           *string  `json:"asset"`
+	DurationSeconds *float64 `json:"durationSeconds"`
+	FromVersion     *string  `json:"fromVersion"`
+	ToVersion       *string  `json:"toVersion"`
+}
+
+type v2ErrorRequest struct {
+	v2CommonRequest
+	Message   *string `json:"message"`
+	Severity  *string `json:"severity"`
+	ErrorCode *string `json:"errorCode"`
+}
+
+func (a *application) v2MessageHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2MessageRequest
+	if err := decodeRequest(w, r, &request, "title", "message", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Message) || !validOptionalString(request.Title) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2MessagePayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2BackupHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2BackupRequest
+	if err := decodeRequest(w, r, &request, "title", "asset", "sizebytes", "durationseconds", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Asset) || !validOptionalString(request.Title) || request.SizeBytes == nil || *request.SizeBytes < 0 || !validOptionalNumber(request.DurationSeconds) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2BackupPayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2UpdateHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2UpdateRequest
+	if err := decodeRequest(w, r, &request, "host", "asset", "durationseconds", "fromversion", "toversion", "source", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Host) || !requiredString(request.Asset) || !validNumber(request.DurationSeconds) || !validOptionalString(request.FromVersion) || !validOptionalString(request.ToVersion) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2UpdatePayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) v2ErrorHandler(w http.ResponseWriter, r *http.Request) {
+	var request v2ErrorRequest
+	if err := decodeRequest(w, r, &request, "source", "message", "severity", "errorcode", "occurredat", "extra"); err != nil {
+		a.writeDecodeError(w, r, err)
+		return
+	}
+	when, valid := a.validV2Common(request.v2CommonRequest)
+	if !valid || !requiredString(request.Source) || !requiredString(request.Message) || !validV2Severity(request.Severity) || !validOptionalString(request.ErrorCode) {
+		a.writeDecodeError(w, r, errInvalidPayload)
+		return
+	}
+
+	if !a.send(w, r, buildV2ErrorPayload(request, when)) {
+		return
+	}
+	writeText(w, http.StatusOK, "Notification sent")
+}
+
+func (a *application) validV2Common(request v2CommonRequest) (time.Time, bool) {
+	if !validOptionalString(request.Source) || !validExtra(request.Extra) {
+		return time.Time{}, false
+	}
+	if request.OccurredAt == nil {
+		return a.now().UTC(), true
+	}
+	if !requiredString(request.OccurredAt) {
+		return time.Time{}, false
+	}
+	when, err := time.Parse(time.RFC3339, *request.OccurredAt)
+	if err != nil {
+		return time.Time{}, false
+	}
+	return when.UTC(), true
+}
+
+func validOptionalString(value *string) bool {
+	return value == nil || requiredString(value)
+}
+
+func validNumber(value *float64) bool {
+	return value != nil && *value >= 0 && !math.IsNaN(*value) && !math.IsInf(*value, 0)
+}
+
+func validOptionalNumber(value *float64) bool {
+	return value == nil || validNumber(value)
+}
+
+func validV2Severity(value *string) bool {
+	if value == nil {
+		return false
+	}
+	switch *value {
+	case "warning", "error", "critical":
+		return true
+	default:
+		return false
+	}
+}
+
+func buildV2MessagePayload(request v2MessageRequest, when time.Time) discordPayload {
+	title := "Message"
+	if request.Title != nil {
+		title = *request.Title
+	}
+	embed := newV2Embed("💬 "+title, *request.Message, v2ColorBrand, when)
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2BackupPayload(request v2BackupRequest, when time.Time) discordPayload {
+	title := "Backup completed"
+	if request.Title != nil {
+		title = *request.Title
+	}
+	embed := newV2Embed("📦 "+title, fmt.Sprintf("**%s** was backed up successfully.", *request.Asset), v2ColorSuccess, when)
+	embed.Fields = append(embed.Fields, v2Field("Size", formatBytes(*request.SizeBytes)))
+	if request.DurationSeconds != nil {
+		embed.Fields = append(embed.Fields, v2Field("Duration", formatV2Duration(*request.DurationSeconds)))
+	}
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2UpdatePayload(request v2UpdateRequest, when time.Time) discordPayload {
+	embed := newV2Embed("🔄 Update completed", fmt.Sprintf("**%s** was updated on **%s**.", *request.Asset, *request.Host), v2ColorUpdate, when)
+	switch {
+	case request.FromVersion != nil && request.ToVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Version", *request.FromVersion+" → "+*request.ToVersion))
+	case request.FromVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Previous version", *request.FromVersion))
+	case request.ToVersion != nil:
+		embed.Fields = append(embed.Fields, v2Field("Installed version", *request.ToVersion))
+	}
+	embed.Fields = append(embed.Fields, v2Field("Duration", formatV2Duration(*request.DurationSeconds)))
+	embed.Fields = appendV2SourceAndExtras(embed.Fields, request.Source, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func buildV2ErrorPayload(request v2ErrorRequest, when time.Time) discordPayload {
+	title, color := v2ErrorStyle(*request.Severity)
+	embed := newV2Embed(title, *request.Message, color, when)
+	embed.Fields = append(embed.Fields, v2Field("Source", *request.Source))
+	if request.ErrorCode != nil {
+		embed.Fields = append(embed.Fields, v2Field("Error code", *request.ErrorCode))
+	}
+	embed.Fields = appendV2Extras(embed.Fields, request.Extra)
+	return discordPayload{Embeds: []discordEmbed{embed}}
+}
+
+func newV2Embed(title, description string, color int, when time.Time) discordEmbed {
+	return discordEmbed{
+		Title:       title,
+		Description: description,
+		Color:       color,
+		Timestamp:   when.UTC().Format(time.RFC3339),
+		Footer:      &discordEmbedFooter{Text: "Haven Notify"},
+	}
+}
+
+func v2ErrorStyle(severity string) (string, int) {
+	switch severity {
+	case "warning":
+		return "⚠️ Warning", v2ColorWarning
+	case "critical":
+		return "🛑 Critical error", v2ColorCritical
+	default:
+		return "❌ Error", v2ColorError
+	}
+}
+
+func appendV2SourceAndExtras(fields []discordEmbedField, source *string, extra extraRequests) []discordEmbedField {
+	if source != nil {
+		fields = append(fields, v2Field("Source", *source))
+	}
+	return appendV2Extras(fields, extra)
+}
+
+func appendV2Extras(fields []discordEmbedField, extra extraRequests) []discordEmbedField {
+	for _, field := range extra {
+		value := *field.Value
+		fields = append(fields, discordEmbedField{
+			Name:   *field.Name,
+			Value:  value,
+			Inline: !strings.ContainsAny(value, "\r\n") && utf8.RuneCountInString(value) <= v2InlineFieldLimit,
+		})
+	}
+	return fields
+}
+
+func v2Field(name, value string) discordEmbedField {
+	return discordEmbedField{Name: name, Value: value, Inline: true}
+}
+
+func formatBytes(bytes int64) string {
+	if bytes < 1024 {
+		return strconv.FormatInt(bytes, 10) + " B"
+	}
+	value := float64(bytes)
+	for _, unit := range []string{"KiB", "MiB", "GiB", "TiB", "PiB", "EiB"} {
+		value /= 1024
+		if value < 1024 || unit == "EiB" {
+			return fmt.Sprintf("%.2f %s", value, unit)
+		}
+	}
+	return strconv.FormatInt(bytes, 10) + " B"
+}
+
+func formatV2Duration(seconds float64) string {
+	if seconds < 60 {
+		return formatV2Number(seconds) + "s"
+	}
+
+	hours := math.Floor(seconds / 3600)
+	seconds -= hours * 3600
+	minutes := math.Floor(seconds / 60)
+	seconds -= minutes * 60
+	parts := make([]string, 0, 3)
+	if hours > 0 {
+		parts = append(parts, formatV2Number(hours)+"h")
+	}
+	if minutes > 0 {
+		parts = append(parts, formatV2Number(minutes)+"m")
+	}
+	if seconds > 0 {
+		parts = append(parts, formatV2Number(seconds)+"s")
+	}
+	return strings.Join(parts, " ")
+}
+
+func formatV2Number(value float64) string {
+	formatted := strconv.FormatFloat(value, 'f', 3, 64)
+	return strings.TrimRight(strings.TrimRight(formatted, "0"), ".")
+}

haven-notify/v2_test.go

@@ -0,0 +1,386 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+const v2TestTimestamp = "2026-07-16T12:34:56Z"
+
+func TestV2NotificationPayloads(t *testing.T) {
+	tests := []struct {
+		name string
+		path string
+		body string
+		want discordPayload
+	}{
+		{
+			name: "generic message",
+			path: "/api/v2/messages",
+			body: `{"title":"Maintenance","message":"The storage job completed.","source":"scheduler","occurredAt":"2026-07-16T09:34:56-03:00","extra":[{"name":"Region","value":"home"},{"name":"Details","value":"first line\nsecond line"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "💬 Maintenance",
+					Description: "The storage job completed.",
+					Color:       v2ColorBrand,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Source", Value: "scheduler", Inline: true},
+						{Name: "Region", Value: "home", Inline: true},
+						{Name: "Details", Value: "first line\nsecond line", Inline: false},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "backup",
+			path: "/api/v2/notifications/backup",
+			body: `{"title":"Nightly backup","asset":"Photos","sizeBytes":1610612736,"durationSeconds":125,"source":"restic","extra":[{"name":"Host","value":"nebula"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "📦 Nightly backup",
+					Description: "**Photos** was backed up successfully.",
+					Color:       v2ColorSuccess,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Size", Value: "1.50 GiB", Inline: true},
+						{Name: "Duration", Value: "2m 5s", Inline: true},
+						{Name: "Source", Value: "restic", Inline: true},
+						{Name: "Host", Value: "nebula", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "update",
+			path: "/api/v2/notifications/update",
+			body: `{"host":"nexus","asset":"k3s","durationSeconds":42.5,"fromVersion":"1.32.5","toVersion":"1.33.1","source":"system-updater"}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "🔄 Update completed",
+					Description: "**k3s** was updated on **nexus**.",
+					Color:       v2ColorUpdate,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Version", Value: "1.32.5 → 1.33.1", Inline: true},
+						{Name: "Duration", Value: "42.5s", Inline: true},
+						{Name: "Source", Value: "system-updater", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+		{
+			name: "critical error",
+			path: "/api/v2/notifications/error",
+			body: `{"source":"backup-job","message":"The repository is unavailable.","severity":"critical","errorCode":"E_REPOSITORY","extra":[{"name":"Path","value":"/mnt/backups"}]}`,
+			want: discordPayload{
+				Embeds: []discordEmbed{{
+					Title:       "🛑 Critical error",
+					Description: "The repository is unavailable.",
+					Color:       v2ColorCritical,
+					Timestamp:   v2TestTimestamp,
+					Fields: []discordEmbedField{
+						{Name: "Source", Value: "backup-job", Inline: true},
+						{Name: "Error code", Value: "E_REPOSITORY", Inline: true},
+						{Name: "Path", Value: "/mnt/backups", Inline: true},
+					},
+					Footer: &discordEmbedFooter{Text: "Haven Notify"},
+				}},
+				AllowedMentions: discordAllowedMentions{Parse: []string{}},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := sendV2Request(t, test.path, test.body)
+			if !reflect.DeepEqual(got, test.want) {
+				gotJSON, _ := json.MarshalIndent(got, "", "  ")
+				wantJSON, _ := json.MarshalIndent(test.want, "", "  ")
+				t.Fatalf("payload = %s\nwant = %s", gotJSON, wantJSON)
+			}
+		})
+	}
+}
+
+func TestV2DefaultsAndVariants(t *testing.T) {
+	tests := []struct {
+		name       string
+		path       string
+		body       string
+		wantTitle  string
+		wantColor  int
+		wantFields []discordEmbedField
+	}{
+		{
+			name:      "message default title",
+			path:      "/api/v2/messages",
+			body:      `{"message":"hello"}`,
+			wantTitle: "💬 Message",
+			wantColor: v2ColorBrand,
+		},
+		{
+			name:      "backup defaults",
+			path:      "/api/v2/notifications/backup",
+			body:      `{"asset":"config","sizeBytes":0}`,
+			wantTitle: "📦 Backup completed",
+			wantColor: v2ColorSuccess,
+			wantFields: []discordEmbedField{
+				{Name: "Size", Value: "0 B", Inline: true},
+			},
+		},
+		{
+			name:      "update previous version",
+			path:      "/api/v2/notifications/update",
+			body:      `{"host":"nexus","asset":"k3s","durationSeconds":60,"fromVersion":"1.0"}`,
+			wantTitle: "🔄 Update completed",
+			wantColor: v2ColorUpdate,
+			wantFields: []discordEmbedField{
+				{Name: "Previous version", Value: "1.0", Inline: true},
+				{Name: "Duration", Value: "1m", Inline: true},
+			},
+		},
+		{
+			name:      "update installed version",
+			path:      "/api/v2/notifications/update",
+			body:      `{"host":"nexus","asset":"k3s","durationSeconds":0,"toVersion":"1.1"}`,
+			wantTitle: "🔄 Update completed",
+			wantColor: v2ColorUpdate,
+			wantFields: []discordEmbedField{
+				{Name: "Installed version", Value: "1.1", Inline: true},
+				{Name: "Duration", Value: "0s", Inline: true},
+			},
+		},
+		{
+			name:      "warning",
+			path:      "/api/v2/notifications/error",
+			body:      `{"source":"job","message":"slow","severity":"warning"}`,
+			wantTitle: "⚠️ Warning",
+			wantColor: v2ColorWarning,
+			wantFields: []discordEmbedField{
+				{Name: "Source", Value: "job", Inline: true},
+			},
+		},
+		{
+			name:      "error",
+			path:      "/api/v2/notifications/error",
+			body:      `{"source":"job","message":"failed","severity":"error"}`,
+			wantTitle: "❌ Error",
+			wantColor: v2ColorError,
+			wantFields: []discordEmbedField{
+				{Name: "Source", Value: "job", Inline: true},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			payload := sendV2Request(t, test.path, test.body)
+			embed := payload.Embeds[0]
+			if embed.Title != test.wantTitle || embed.Color != test.wantColor || embed.Timestamp != v2TestTimestamp || !reflect.DeepEqual(embed.Fields, test.wantFields) {
+				t.Fatalf("embed = %+v", embed)
+			}
+		})
+	}
+}
+
+func TestV2RequestValidation(t *testing.T) {
+	tests := []struct {
+		name        string
+		path        string
+		contentType string
+		body        string
+		wantStatus  int
+	}{
+		{name: "message missing", path: "/api/v2/messages", body: `{}`, wantStatus: 400},
+		{name: "message blank", path: "/api/v2/messages", body: `{"message":" "}`, wantStatus: 400},
+		{name: "blank optional title", path: "/api/v2/messages", body: `{"message":"m","title":" "}`, wantStatus: 400},
+		{name: "blank optional source", path: "/api/v2/messages", body: `{"message":"m","source":" "}`, wantStatus: 400},
+		{name: "invalid timestamp", path: "/api/v2/messages", body: `{"message":"m","occurredAt":"yesterday"}`, wantStatus: 400},
+		{name: "blank timestamp", path: "/api/v2/messages", body: `{"message":"m","occurredAt":" "}`, wantStatus: 400},
+		{name: "null extras", path: "/api/v2/messages", body: `{"message":"m","extra":null}`, wantStatus: 400},
+		{name: "invalid extra", path: "/api/v2/messages", body: `{"message":"m","extra":[{"name":"n"}]}`, wantStatus: 400},
+		{name: "duplicate common key", path: "/api/v2/messages", body: `{"message":"m","source":"one","SOURCE":"two"}`, wantStatus: 400},
+		{name: "duplicate nested key", path: "/api/v2/messages", body: `{"message":"m","extra":[{"name":"one","NAME":"two","value":"v"}]}`, wantStatus: 400},
+		{name: "backup missing asset", path: "/api/v2/notifications/backup", body: `{"sizeBytes":1}`, wantStatus: 400},
+		{name: "backup missing size", path: "/api/v2/notifications/backup", body: `{"asset":"a"}`, wantStatus: 400},
+		{name: "backup negative size", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":-1}`, wantStatus: 400},
+		{name: "backup fractional size", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":1.5}`, wantStatus: 400},
+		{name: "backup negative duration", path: "/api/v2/notifications/backup", body: `{"asset":"a","sizeBytes":1,"durationSeconds":-1}`, wantStatus: 400},
+		{name: "update missing host", path: "/api/v2/notifications/update", body: `{"asset":"a","durationSeconds":1}`, wantStatus: 400},
+		{name: "update missing asset", path: "/api/v2/notifications/update", body: `{"host":"h","durationSeconds":1}`, wantStatus: 400},
+		{name: "update missing duration", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a"}`, wantStatus: 400},
+		{name: "update negative duration", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a","durationSeconds":-1}`, wantStatus: 400},
+		{name: "update blank version", path: "/api/v2/notifications/update", body: `{"host":"h","asset":"a","durationSeconds":1,"fromVersion":" "}`, wantStatus: 400},
+		{name: "error missing source", path: "/api/v2/notifications/error", body: `{"message":"m","severity":"error"}`, wantStatus: 400},
+		{name: "error missing message", path: "/api/v2/notifications/error", body: `{"source":"s","severity":"error"}`, wantStatus: 400},
+		{name: "error missing severity", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m"}`, wantStatus: 400},
+		{name: "error invalid severity", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m","severity":"ERROR"}`, wantStatus: 400},
+		{name: "error blank code", path: "/api/v2/notifications/error", body: `{"source":"s","message":"m","severity":"error","errorCode":" "}`, wantStatus: 400},
+		{name: "wrong media type", path: "/api/v2/messages", contentType: "text/plain", body: `{"message":"m"}`, wantStatus: 415},
+		{name: "unknown accepted", path: "/api/v2/messages", body: `{"message":"m","future":true}`, wantStatus: 200},
+		{name: "case insensitive accepted", path: "/api/v2/messages", body: `{"MESSAGE":"m","EXTRA":[{"NAME":"n","VALUE":"v"}]}`, wantStatus: 200},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			contentType := test.contentType
+			if contentType == "" {
+				contentType = "application/json"
+			}
+			recorder := &webhookRecorder{}
+			response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, test.path, contentType, test.body)
+			if response.Code != test.wantStatus {
+				t.Fatalf("response = (%d, %q), want %d", response.Code, response.Body.String(), test.wantStatus)
+			}
+			wantCalls := 0
+			if test.wantStatus == http.StatusOK {
+				wantCalls = 1
+			}
+			if calls := len(recorder.bodies()); calls != wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", calls, wantCalls)
+			}
+		})
+	}
+}
+
+func TestV2RoutingContracts(t *testing.T) {
+	handler := testApplication(t, nil, nil).handler()
+	for _, path := range []string{
+		"/api/v2/messages",
+		"/api/v2/notifications/backup",
+		"/api/v2/notifications/update",
+		"/api/v2/notifications/error",
+	} {
+		response := performRequest(handler, http.MethodGet, path, "", "")
+		if response.Code != http.StatusMethodNotAllowed || response.Header().Get("Allow") != http.MethodPost {
+			t.Fatalf("GET %s = (%d, Allow %q)", path, response.Code, response.Header().Get("Allow"))
+		}
+	}
+	response := performRequest(handler, http.MethodPost, "/api/v2/messages/", "application/json", `{"message":"m"}`)
+	if response.Code != http.StatusNotFound {
+		t.Fatalf("trailing slash status = %d", response.Code)
+	}
+}
+
+func TestV2RequestBodyLimit(t *testing.T) {
+	prefix, suffix := `{"message":"`, `"}`
+	for _, test := range []struct {
+		name      string
+		bodyBytes int
+		want      int
+		wantCalls int
+	}{
+		{name: "exactly at limit", bodyBytes: maxRequestBody, want: http.StatusOK, wantCalls: 1},
+		{name: "one byte over limit", bodyBytes: maxRequestBody + 1, want: http.StatusBadRequest},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			recorder := &webhookRecorder{}
+			body := prefix + strings.Repeat("x", test.bodyBytes-len(prefix)-len(suffix)) + suffix
+			response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, "/api/v2/messages", "application/json", body)
+			if response.Code != test.want {
+				t.Fatalf("response status = %d, want %d", response.Code, test.want)
+			}
+			if calls := len(recorder.bodies()); calls != test.wantCalls {
+				t.Fatalf("webhook calls = %d, want %d", calls, test.wantCalls)
+			}
+		})
+	}
+}
+
+func TestV2ExtrasRespectDiscordLimits(t *testing.T) {
+	extra := make([]string, 30)
+	for index := range extra {
+		extra[index] = fmt.Sprintf(`{"name":"field-%02d","value":"value"}`, index)
+	}
+	body := fmt.Sprintf(`{"host":"nexus","asset":"k3s","durationSeconds":1,"fromVersion":"1","toVersion":"2","source":"updater","extra":[%s]}`, strings.Join(extra, ","))
+	payload := sendV2Request(t, "/api/v2/notifications/update", body)
+	fields := payload.Embeds[0].Fields
+	if len(fields) != 25 {
+		t.Fatalf("fields = %d, want 25", len(fields))
+	}
+	if fields[0].Name != "Version" || fields[1].Name != "Duration" || fields[2].Name != "Source" || fields[24].Name != "field-21" {
+		t.Fatalf("field order = %+v", fields)
+	}
+
+	longValue := strings.Repeat("x", v2InlineFieldLimit+1)
+	body = fmt.Sprintf(`{"message":"m","source":"source","extra":[{"name":"long","value":%q}]}`, longValue)
+	payload = sendV2Request(t, "/api/v2/messages", body)
+	if payload.Embeds[0].Fields[0].Name != "Source" || payload.Embeds[0].Fields[1].Inline {
+		t.Fatalf("inline fields = %+v", payload.Embeds[0].Fields)
+	}
+
+	body = fmt.Sprintf(`{"message":%q,"source":"source","extra":[{"name":"large","value":%q}]}`, strings.Repeat("m", 5000), strings.Repeat("x", 2000))
+	payload = sendV2Request(t, "/api/v2/messages", body)
+	embed := payload.Embeds[0]
+	if utf8.RuneCountInString(embed.Description) > 4096 || discordEmbedCharacters(payload.Embeds) > 6000 || len(embed.Fields) == 0 || embed.Fields[0].Name != "Source" {
+		t.Fatalf("normalized embed = %+v, characters = %d", embed, discordEmbedCharacters(payload.Embeds))
+	}
+}
+
+func TestV2FormattingHelpers(t *testing.T) {
+	for _, test := range []struct {
+		bytes int64
+		want  string
+	}{
+		{0, "0 B"},
+		{1023, "1023 B"},
+		{1024, "1.00 KiB"},
+		{1536, "1.50 KiB"},
+		{1 << 30, "1.00 GiB"},
+	} {
+		if got := formatBytes(test.bytes); got != test.want {
+			t.Errorf("formatBytes(%d) = %q, want %q", test.bytes, got, test.want)
+		}
+	}
+	for _, test := range []struct {
+		seconds float64
+		want    string
+	}{
+		{0, "0s"},
+		{42.5, "42.5s"},
+		{60, "1m"},
+		{125, "2m 5s"},
+		{125.1, "2m 5.1s"},
+		{3723, "1h 2m 3s"},
+	} {
+		if got := formatV2Duration(test.seconds); got != test.want {
+			t.Errorf("formatV2Duration(%v) = %q, want %q", test.seconds, got, test.want)
+		}
+	}
+	for _, value := range []*float64{nil, floatPointer(-1), floatPointer(math.NaN()), floatPointer(math.Inf(1))} {
+		if validNumber(value) {
+			t.Fatalf("validNumber(%v) = true", value)
+		}
+	}
+}
+
+func sendV2Request(t *testing.T, path, body string) discordPayload {
+	t.Helper()
+	recorder := &webhookRecorder{}
+	response := performRequest(testApplication(t, recorder, nil).handler(), http.MethodPost, path, "application/json", body)
+	if response.Code != http.StatusOK || response.Body.String() != "Notification sent" {
+		t.Fatalf("response = (%d, %q)", response.Code, response.Body.String())
+	}
+	bodies := recorder.bodies()
+	if len(bodies) != 1 {
+		t.Fatalf("webhook calls = %d, want 1", len(bodies))
+	}
+	var payload discordPayload
+	if err := json.Unmarshal(bodies[0], &payload); err != nil {
+		t.Fatalf("decode webhook payload: %v", err)
+	}
+	return payload
+}

housekeeping.sh

@@ -2,7 +2,6 @@
 
 # Housekeeping Script
 # Unified system maintenance: package index update, system cleanup, and oh-my-zsh update.
-# Combines the former update.sh and clean.sh into a single script with subcommand dispatch.
 
 set -euo pipefail
 
@@ -109,9 +108,9 @@ get_system_info() {
 
     # Memory info
     if [[ -f /proc/meminfo ]]; then
-        local mem_total mem_available
-        mem_total=$(grep MemTotal /proc/meminfo | awk '{print $2}')
-        mem_available=$(grep MemAvailable /proc/meminfo | awk '{print $2}')
+        local mem_total="" mem_available=""
+        mem_total=$(awk '/^MemTotal:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)
+        mem_available=$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)
         if [[ -n "$mem_total" && -n "$mem_available" ]]; then
             info+="Memory: $((mem_available/1024))MB available of $((mem_total/1024))MB total"
         fi
@@ -120,7 +119,7 @@ get_system_info() {
     # Disk space info
     if command_exists df; then
         local disk_info
-        disk_info=$(df -h / 2>/dev/null | tail -1 | awk '{print $4 " available of " $2 " total"}')
+        disk_info=$(df -h / 2>/dev/null | awk 'NR == 2 {print $4 " available of " $2 " total"}' || true)
         if [[ -n "$disk_info" ]]; then
             info+="${info:+, }Disk: $disk_info"
         fi
@@ -495,7 +494,6 @@ cleanup_logs() {
     fi
 
     # Truncate large active log files
-    local large_logs
     while IFS= read -r -d '' logfile; do
         if [[ -f "$logfile" && -w "$logfile" ]]; then
             truncate -s 0 "$logfile" 2>/dev/null || true
@@ -774,7 +772,7 @@ main() {
             log_warning "Unknown subcommand: '${subcommand}'"
             echo
             print_usage
-            exit 0
+            exit 1
             ;;
     esac
 }

k8s/README.md

@@ -6,15 +6,36 @@ Kubernetes-focused shell scripts intended for cronjobs and operational utilities
 
 ### `automated-nfs-backup.sh`
 
-Backs up each top-level folder found in `NFS_SOURCE_PATH` into a single encrypted `.7z` archive per run, with optional Kubernetes workload quiescing when a folder name exactly matches a namespace name.
+Copies each top-level folder found in `NFS_SOURCE_PATH` to a per-run local staging directory, then creates one encrypted `.7z` archive from the complete staged data. Kubernetes workloads are quiesced while a folder whose name exactly matches a namespace is copied.
 
 Behavior:
 - Exact folder-to-namespace mapping only.
-- Unmapped folder: backup still runs, Kubernetes scale actions are skipped.
-- Mapped folder: saves replicas, scales selected workloads down, waits, adds that folder to the shared run archive, restores replicas, waits again.
+- `kube-system`, `kube-public`, and `kube-node-lease` are excluded from scale actions by default.
+- Unmapped or excluded folder: copied to local staging without Kubernetes scale actions.
+- Mapped folder: saves replicas, scales selected workloads down, waits, copies the quiesced folder to local staging, restores replicas, and waits again.
+- After every folder has been processed, one `7z` invocation compresses the complete run staging directory. Workloads are therefore restored before compression starts.
+- Per-run replica state and staged data are removed on exit. The staging parent is retained.
 - Scale-down issues are warnings by policy (backup still runs).
 - Restore issues are warnings by policy (run can still complete successfully).
-- Cleanup can delete the last `N` archives ordered by date (oldest side), while keeping at least one archive.
+- Cleanup retains the `N` most recent matching archives by modification time and deletes older ones.
+
+### `backup-k3s-control-plane-config.sh`
+
+Creates and validates a standard `.zip` containing the iris k3s control-plane configuration and credential state, then atomically publishes it to the NAS. It never stops or restarts k3s, and deliberately excludes `/var/lib/rancher/k3s/server/db` and `kine.sock`; this is configuration/credential backup, not a live datastore snapshot.
+
+The archive includes all of `K3S_CONFIG_DIR`, any existing `K3S_SERVICE_FILES`, and existing `cred`, `etc`, `manifests`, `static`, `tls`, `token`, `agent-token`, and `node-token` entries beneath `K3S_SERVER_DIR`.
+
+Defaults mirror the existing `k8s-config-backup` schedule: `ivanch@nas.haven:/export/Backup/k8s`, archive names such as `k3s-control-plane-config_2026-07-21_12-00-00.zip`, and remote retention of the four newest matching archives. Other prefixes, including `k8s-backup_*.7z`, are never considered for retention.
+
+The script uses a non-blocking lock by default (`/var/lock/k3s-control-plane-config-backup.lock`), so overlapping cron and manual invocations fail safely. It builds in a private local temporary directory, validates with `unzip -t`, uploads to a remote temporary filename, and only then renames it into place.
+
+Useful overrides:
+
+- Source paths: `K3S_CONFIG_DIR`, `K3S_SERVER_DIR`, `K3S_SERVICE_FILES` (colon-separated service configuration files).
+- Destination and identity: `REMOTE_USER`, `REMOTE_HOST`, `REMOTE_DIR`, `REMOTE_IDENTITY_FILE`, `SSH_PORT`.
+- Archive and locking: `ARCHIVE_PREFIX`, `ARCHIVE_TS_FORMAT`, `LOCAL_TMP_PARENT`, `LOCK_PATH`, `CLEANUP_KEEP_COUNT` (default `4`; `0` disables deletion).
+- Binaries: `ZIP_BIN`, `UNZIP_BIN`, `SSH_BIN`, `SCP_BIN`, `CURL_BIN`, `FLOCK_BIN`, `MKTEMP_BIN`, `DATE_BIN`, `REALPATH_BIN`, `REMOTE_BASH_BIN`. `REALPATH_BIN` must provide GNU `realpath` `-e` and `-m` options.
+- Notifications: `NOTIFY_SUCCESS_URL` (default `http://notify.haven/api/v2/notifications/backup`), `NOTIFY_FAILURE_URL` (default `http://notify.haven/api/v2/notifications/error`), `NOTIFY_TITLE`, `NOTIFY_ASSET`, and `NOTIFY_SOURCE`. `NOTIFY_CALLER` is a compatibility fallback for `NOTIFY_SOURCE`. Set either URL to an empty value to disable it. Failed notifications only generate a warning; they do not change an otherwise successful backup result.
 
 ## Environment Variables
 
@@ -26,31 +47,48 @@ Optional:
 - `BACKUP_PASSWORD` (default: empty): When set, archive uses password protection; when empty, archive is not encrypted.
 - `KUBECTL_BIN` (default: `kubectl`)
 - `KUBE_CONTEXT` (default: empty)
-- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset,replicationcontroller`)
+- `WORKLOAD_KINDS` (default: `deployment,statefulset,replicaset`)
+- `EXCLUDED_NAMESPACES` (default: `kube-system,kube-public,kube-node-lease`): Comma-separated namespace names that are always backed up without scale actions. Set an explicit empty value to disable the default exclusions.
+- `BACKUP_STAGING_PATH` (default: `/tmp/k8s-nfs-backup-staging`): Local parent for per-run staging directories. It must be outside `NFS_SOURCE_PATH` and have enough free space for all folders included in one backup run.
+- `RSYNC_BIN` (default: `rsync`)
 - `ARCHIVE_PREFIX` (default: `nfs-backup`)
 - `ARCHIVE_TS_FORMAT` (default: `%Y%m%d_%H%M%S`)
 - `SEVENZ_METHOD` (default: `lzma2`)
 - `SEVENZ_LEVEL` (default: `9`)
 - `SEVENZ_HEADER_ENCRYPT` (default: `on`, only applied when `BACKUP_PASSWORD` is set)
 - `SEVENZ_THREADS` (default: `on`)
-- `SCALE_TIMEOUT_SECONDS` (default: `600`)
 - `SCALE_RETRY_COUNT` (default: `3`)
 - `SCALE_RETRY_DELAY_SECONDS` (default: `5`)
+- `SCALE_WAIT_SECONDS` (default: `30`): Fixed delay after scaling down and after restoring a mapped namespace.
 - `LOG_LEVEL` (default: `info`)
-- `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`)
-- `NOTIFY_SUCCESS_URL` (default: empty, disabled)
-- `NOTIFY_FAILURE_URL` (default: empty, disabled)
+- `TMP_STATE_DIR` (default: `/tmp/k8s-nfs-backup`): Parent directory for fresh per-run replica-state directories. A run never restores state files belonging to an earlier run.
+- `NOTIFY_SUCCESS_URL` (default: `http://notify.haven/api/v2/notifications/backup`; set to empty to disable)
+- `NOTIFY_FAILURE_URL` (default: `http://notify.haven/api/v2/notifications/error`; set to empty to disable)
 - `NOTIFY_TITLE` (default: `Kubernetes`)
 - `NOTIFY_ASSET` (default: `kube config`)
+- `NOTIFY_SOURCE` (default: `NOTIFY_CALLER`, or `Kubernetes NFS backup`): Haven Notify v2 source identifier. `NOTIFY_CALLER` remains a compatibility fallback.
 - `CLEANUP_KEEP_COUNT` (default: `4`)
 
-Notification payload (success and failure):
+Notifications use the Haven Notify v2 backup and error endpoints. Success payloads include the final archive size and elapsed time:
 
 ```json
 {
   "title": "Kubernetes",
   "asset": "kube config",
-  "backupSizeInMB": 123
+  "sizeBytes": 128974848,
+  "durationSeconds": 125,
+  "source": "Kubernetes NFS backup"
+}
+```
+
+Failures use the v2 error contract:
+
+```json
+{
+  "source": "Kubernetes NFS backup",
+  "message": "Some NFS folders failed to back up",
+  "severity": "critical",
+  "errorCode": "BACKUP_FAILED"
 }
 ```
 
@@ -60,16 +98,18 @@ Notification payload (success and failure):
 - Provide Kubernetes RBAC allowing `get`, `list`, and `scale` on configured workload kinds in target namespaces.
 - Ensure `kubectl` context and credentials are present in the runtime.
 - Ensure `7z` is installed in the runtime image/host.
+- Ensure `rsync` is installed and keep `BACKUP_STAGING_PATH` on fast local storage. Capacity must accommodate one complete uncompressed backup run in addition to the output archive.
 
 ## Failure Semantics
 
 - Missing required env vars, missing commands, invalid paths, or inability to list namespaces: script exits non-zero immediately.
-- Folder backup failures: counted and script exits non-zero at end.
+- Folder staging failures are counted and failed partial copies are excluded from the archive. One or more successfully staged folders continues to archive creation; zero successfully staged folders exits non-zero.
+- If the single archive operation fails, the run treats every staged folder as failed and exits non-zero.
 - Scale-down warnings/timeouts: logged and counted, backup continues.
 - Restore warnings/timeouts: logged and counted, script does not fail solely because of restore warnings.
 - If `NOTIFY_SUCCESS_URL` is set, success notification is sent at the end of a successful run.
-- If `NOTIFY_FAILURE_URL` is set, failure notification is sent when backup failures are detected.
-- On successful runs, cleanup removes the last `CLEANUP_KEEP_COUNT` archives by date ordering (oldest side), without deleting the final remaining archive.
+- If `NOTIFY_FAILURE_URL` is set, a failure notification is sent when any folder backup failure is detected, including partial failures.
+- On runs with at least one successful folder backup, cleanup retains the `CLEANUP_KEEP_COUNT` most recent matching archives and deletes older ones.
 - Final summary always logs:
   - `processed`
   - `mapped`

k8s/automated-nfs-backup.sh

@@ -8,7 +8,7 @@ source "${SCRIPT_DIR}/lib/common.sh"
 # Core error/exit traps — ERR trap logs the failure, EXIT trap guarantees
 # every scaled-down namespace is restored regardless of how we exit.
 trap 'on_error "${LINENO}" "${BASH_COMMAND:-unknown}"' ERR
-trap '_exit_code=$?; restore_all_remaining; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
+trap '_exit_code=$?; restore_all_remaining; cleanup_run_staging; if [[ "$_exit_code" -eq 0 ]]; then log_info "Finished successfully"; else log_error "Finished with errors (exit code $_exit_code)"; fi' EXIT
 
 # Required configuration
 require_env "NFS_SOURCE_PATH"
@@ -19,6 +19,9 @@ BACKUP_PASSWORD="${BACKUP_PASSWORD:-}"
 KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 KUBE_CONTEXT="${KUBE_CONTEXT:-}"
 WORKLOAD_KINDS="${WORKLOAD_KINDS:-deployment,statefulset,replicaset}"
+EXCLUDED_NAMESPACES="${EXCLUDED_NAMESPACES-kube-system,kube-public,kube-node-lease}"
+BACKUP_STAGING_PATH="${BACKUP_STAGING_PATH:-/tmp/k8s-nfs-backup-staging}"
+RSYNC_BIN="${RSYNC_BIN:-rsync}"
 ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-nfs-backup}"
 ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y%m%d_%H%M%S}"
 SEVENZ_METHOD="${SEVENZ_METHOD:-lzma2}"
@@ -26,17 +29,19 @@ SEVENZ_LEVEL="${SEVENZ_LEVEL:-9}"
 SEVENZ_HEADER_ENCRYPT="${SEVENZ_HEADER_ENCRYPT:-on}"
 SEVENZ_THREADS="${SEVENZ_THREADS:-on}"
 SEVENZ_BIN="${SEVENZ_BIN:-7z}"
-SCALE_TIMEOUT_SECONDS="${SCALE_TIMEOUT_SECONDS:-600}"
 SCALE_RETRY_COUNT="${SCALE_RETRY_COUNT:-3}"
 SCALE_RETRY_DELAY_SECONDS="${SCALE_RETRY_DELAY_SECONDS:-5}"
 SCALE_WAIT_SECONDS="${SCALE_WAIT_SECONDS:-30}"
 TMP_STATE_DIR="${TMP_STATE_DIR:-/tmp/k8s-nfs-backup}"
-NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL:-}"
-NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL:-}"
+NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
+NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
 NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
 NOTIFY_ASSET="${NOTIFY_ASSET:-kube config}"
-NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes config backup}"
+NOTIFY_CALLER="${NOTIFY_CALLER:-Kubernetes NFS backup}"
+NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
 CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
+RUN_STATE_DIR=""
+RUN_STAGING_DIR=""
 
 KUBECTL_ARGS=()
 if [[ -n "$KUBE_CONTEXT" ]]; then
@@ -45,6 +50,7 @@ fi
 
 declare -a WORKLOAD_KIND_LIST=()
 declare -A NAMESPACE_MAP=()
+declare -A EXCLUDED_NAMESPACE_MAP=()
 
 total_folders=0
 mapped_folders=0
@@ -54,6 +60,7 @@ backup_failures=0
 scale_warnings=0
 restore_warnings=0
 total_backup_size_bytes=0
+backup_started_seconds="$SECONDS"
 
 _kubectl() {
     "$KUBECTL_BIN" "${KUBECTL_ARGS[@]}" "$@" </dev/null
@@ -75,9 +82,24 @@ parse_workload_kinds() {
     fi
 }
 
+parse_excluded_namespaces() {
+    local raw
+    local cleaned
+    local namespace
+    IFS=',' read -r -a raw <<< "$EXCLUDED_NAMESPACES"
+    for namespace in "${raw[@]}"; do
+        cleaned="$(trim "${namespace,,}")"
+        if [[ -n "$cleaned" ]]; then
+            EXCLUDED_NAMESPACE_MAP["$cleaned"]=1
+        fi
+    done
+}
+
 validate_inputs() {
     require_cmd "$KUBECTL_BIN"
     require_cmd "$SEVENZ_BIN"
+    require_cmd "mktemp"
+    require_cmd "$RSYNC_BIN"
     if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
         require_cmd "curl"
     fi
@@ -96,6 +118,25 @@ validate_inputs() {
 
     mkdir -p "$BACKUP_OUTPUT_PATH"
     mkdir -p "$TMP_STATE_DIR"
+    if ! RUN_STATE_DIR="$(mktemp -d "${TMP_STATE_DIR%/}/run-XXXXXXXXXX")"; then
+        die "Unable to create a per-run state directory under ${TMP_STATE_DIR}"
+    fi
+    log_debug "Using isolated run state directory: ${RUN_STATE_DIR}"
+
+    local source_real
+    local staging_real
+    mkdir -p "$BACKUP_STAGING_PATH"
+    source_real="$(cd "$NFS_SOURCE_PATH" && pwd -P)"
+    staging_real="$(cd "$BACKUP_STAGING_PATH" && pwd -P)"
+    case "${staging_real}/" in
+        "${source_real%/}/"*)
+            die "BACKUP_STAGING_PATH must not be inside NFS_SOURCE_PATH"
+            ;;
+    esac
+    if ! RUN_STAGING_DIR="$(mktemp -d "${BACKUP_STAGING_PATH%/}/run-XXXXXXXXXX")"; then
+        die "Unable to create a per-run staging directory under ${BACKUP_STAGING_PATH}"
+    fi
+    log_info "Using local run staging directory: ${RUN_STAGING_DIR}"
 
     if ! _kubectl get namespaces >/dev/null 2>&1; then
         die "Unable to list Kubernetes namespaces with ${KUBECTL_BIN}"
@@ -112,6 +153,9 @@ load_namespaces() {
 
 namespace_for_folder() {
     local folder_name="${1:?folder name required}"
+    if [[ -n "${EXCLUDED_NAMESPACE_MAP[$folder_name]:-}" ]]; then
+        return 2
+    fi
     if [[ -n "${NAMESPACE_MAP[$folder_name]:-}" ]]; then
         printf '%s' "$folder_name"
         return 0
@@ -121,7 +165,7 @@ namespace_for_folder() {
 
 state_file_for_namespace() {
     local namespace="${1:?namespace required}"
-    printf '%s/%s.state' "$TMP_STATE_DIR" "$namespace"
+    printf '%s/%s.state' "$RUN_STATE_DIR" "$namespace"
 }
 
 capture_replicas_state() {
@@ -207,7 +251,7 @@ restore_namespace_replicas() {
 }
 
 restore_all_remaining() {
-    if [[ -z "${TMP_STATE_DIR:-}" ]] || [[ ! -d "${TMP_STATE_DIR:-}" ]]; then
+    if [[ -z "${RUN_STATE_DIR:-}" ]] || [[ ! -d "${RUN_STATE_DIR:-}" ]]; then
         return 0
     fi
 
@@ -216,7 +260,7 @@ restore_all_remaining() {
     local kind name replicas
     local found=0
 
-    for state_file in "${TMP_STATE_DIR}"/*.state; do
+    for state_file in "${RUN_STATE_DIR}"/*.state; do
         [[ -f "$state_file" ]] || continue
         found=1
         namespace="$(basename "$state_file" .state)"
@@ -237,6 +281,54 @@ restore_all_remaining() {
     if [[ "$found" -eq 1 ]]; then
         log_info "EXIT cleanup complete - all remaining workloads restored"
     fi
+
+    if ! rmdir -- "$RUN_STATE_DIR" 2>/dev/null; then
+        log_warn "Run state directory is not empty; leaving it in place: ${RUN_STATE_DIR}"
+    fi
+}
+
+cleanup_run_staging() {
+    if [[ -z "${RUN_STAGING_DIR:-}" ]] || [[ ! -d "${RUN_STAGING_DIR:-}" ]]; then
+        return 0
+    fi
+
+    case "$RUN_STAGING_DIR" in
+        "${BACKUP_STAGING_PATH%/}/run-"*)
+            rm -rf -- "$RUN_STAGING_DIR"
+            ;;
+        *)
+            log_error "Refusing to remove unexpected staging directory: ${RUN_STAGING_DIR}"
+            return 1
+            ;;
+    esac
+}
+
+cleanup_staged_folder() {
+    local staging_folder="${1:?staging folder required}"
+    case "$staging_folder" in
+        "$RUN_STAGING_DIR"/*)
+            rm -rf -- "$staging_folder"
+            ;;
+        *)
+            log_error "Refusing to remove staging folder outside the current run: ${staging_folder}"
+            return 1
+            ;;
+    esac
+}
+
+sync_folder_to_staging() {
+    local source_folder="${1:?source folder required}"
+    local staging_folder="${2:?staging folder required}"
+    local sync_output
+
+    mkdir -p "$staging_folder"
+    if ! sync_output="$("$RSYNC_BIN" -a --delete -- "${source_folder%/}/" "${staging_folder%/}/" 2>&1)"; then
+        log_error "rsync failed from '${source_folder}' to '${staging_folder}'"
+        if [[ -n "$sync_output" ]]; then
+            log_error "rsync output: ${sync_output}"
+        fi
+        return 1
+    fi
 }
 
 archive_path_for_run() {
@@ -253,20 +345,12 @@ json_escape() {
     printf '%s' "$value"
 }
 
-backup_size_mb() {
-    local bytes="$1"
-    if (( bytes <= 0 )); then
-        echo 0
-        return 0
-    fi
-    echo $(( (bytes + 1048575) / 1048576 ))
-}
-
 send_backup_notification() {
     local url="${1:-}"
     local title="${2:-}"
     local asset="${3:-}"
-    local size_mb="${4:-0}"
+    local size_bytes="${4:-0}"
+    local duration_seconds="${5:-0}"
 
     [[ -z "$url" ]] && return 0
 
@@ -275,7 +359,9 @@ send_backup_notification() {
 {
   "title": "$(json_escape "$title")",
   "asset": "$(json_escape "$asset")",
-  "backupSizeInMB": ${size_mb}
+  "sizeBytes": ${size_bytes},
+  "durationSeconds": ${duration_seconds},
+  "source": "$(json_escape "$NOTIFY_SOURCE")"
 }
 EOF
 )
@@ -292,20 +378,21 @@ EOF
 send_backup_failure_notification() {
     local url="${1:-}"
     local message="${2:-}"
-    local size_mb="${3:-0}"
+    local size_bytes="${3:-0}"
 
     [[ -z "$url" ]] && return 0
 
     local payload
     payload=$(cat <<EOF
 {
-  "caller": "$(json_escape "$NOTIFY_CALLER")",
+  "source": "$(json_escape "$NOTIFY_SOURCE")",
   "message": "$(json_escape "$message")",
-  "critical": true,
+  "severity": "critical",
+  "errorCode": "BACKUP_FAILED",
   "extra": [
     {
-      "name": "Backup Size (MB)",
-      "value": "${size_mb}"
+      "name": "Archive size",
+      "value": "${size_bytes} B"
     }
   ]
 }
@@ -321,13 +408,17 @@ EOF
     return 0
 }
 
-backup_folder() {
-    local folder_path="${1:?folder path required}"
-    local archive_path="${2:?archive path required}"
+archive_staging() {
+    local archive_path="${1:?archive path required}"
+
+    if [[ "$archive_path" != /* ]]; then
+        archive_path="$(cd "$(dirname "$archive_path")" && pwd -P)/$(basename "$archive_path")"
+    fi
+
     local -a cmd=(
         "$SEVENZ_BIN" a -t7z
         "$archive_path"
-        "$folder_path"
+        .
         "-m0=${SEVENZ_METHOD}"
         "-mx=${SEVENZ_LEVEL}"
         "-mmt=${SEVENZ_THREADS}"
@@ -338,15 +429,15 @@ backup_folder() {
     fi
 
     local cmd_output rc filtered_output
-    cmd_output="$("${cmd[@]}" 2>&1 >/dev/null)"
+    cmd_output="$(cd "$RUN_STAGING_DIR" && "${cmd[@]}" 2>&1 >/dev/null)"
     rc=$?
     filtered_output="$(printf '%s\n' "$cmd_output" | grep -v -F 'WARNING: errno=2 : No such file or directory' || true)"
     if [[ -n "$filtered_output" ]]; then
-        log_warn "7z output for '${folder_path}': ${filtered_output}"
+        log_warn "7z output for staging directory '${RUN_STAGING_DIR}': ${filtered_output}"
     fi
     if (( rc == 0 )); then
         return 0
-    elif (( rc == 1 )) && [[ -z "$filtered_output" ]]; then
+    elif (( rc == 1 )); then
         return 0
     fi
     return 1
@@ -388,13 +479,16 @@ cleanup_archives() {
 
 process_folder() {
     local folder_path="${1:?folder path is required}"
-    local archive_path="${2:?archive path is required}"
     local folder_name
     local namespace=""
     local state_file=""
     local has_mapping=0
+    local namespace_match_status=0
+    local staging_folder
+    local copy_succeeded=0
 
     folder_name="$(basename "$folder_path")"
+    staging_folder="${RUN_STAGING_DIR}/${folder_name}"
     total_folders=$((total_folders + 1))
     log_info "Processing folder: ${folder_name}"
 
@@ -409,17 +503,24 @@ process_folder() {
 
         log_info "Waiting ${SCALE_WAIT_SECONDS} seconds for namespace '${namespace}' to scale down..."
         sleep "$SCALE_WAIT_SECONDS"
-    else
-        unmapped_folders=$((unmapped_folders + 1))
-        log_warn "No namespace matched folder '${folder_name}'. Running backup only."
-    fi
 
-    if backup_folder "$folder_path" "$archive_path"; then
-        backup_successes=$((backup_successes + 1))
-        log_info "Added folder '${folder_name}' to archive: ${archive_path}"
+        log_info "Copying quiesced namespace folder '${folder_name}' to local staging"
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
+        fi
     else
-        backup_failures=$((backup_failures + 1))
-        log_error "Backup failed for folder '${folder_name}'"
+        namespace_match_status=$?
+        unmapped_folders=$((unmapped_folders + 1))
+        if (( namespace_match_status == 2 )); then
+            log_warn "Folder '${folder_name}' matches an excluded namespace; skipping Kubernetes scale actions and running backup only."
+        else
+            log_warn "No namespace matched folder '${folder_name}'. Running backup only."
+        fi
+
+        log_info "Copying live folder '${folder_name}' to local staging"
+        if sync_folder_to_staging "$folder_path" "$staging_folder"; then
+            copy_succeeded=1
+        fi
     fi
 
     if [[ "$has_mapping" -eq 1 ]]; then
@@ -428,6 +529,16 @@ process_folder() {
         sleep "$SCALE_WAIT_SECONDS"
         rm -f "$state_file"
     fi
+
+    if (( copy_succeeded == 0 )); then
+        backup_failures=$((backup_failures + 1))
+        log_error "Backup failed for folder '${folder_name}' while copying to local staging"
+        cleanup_staged_folder "$staging_folder" || true
+        return 0
+    fi
+
+    backup_successes=$((backup_successes + 1))
+    log_info "Staged folder '${folder_name}' for the run archive"
 }
 
 print_summary() {
@@ -436,26 +547,36 @@ print_summary() {
 
 main() {
     parse_workload_kinds
+    parse_excluded_namespaces
     validate_inputs
     load_namespaces
 
     local folder_path
-    local run_archive_path=""
+    local run_archive_path
     local found=0
+    run_archive_path="$(archive_path_for_run)"
+    log_info "Using single archive for this run: ${run_archive_path}"
+
     for folder_path in "${NFS_SOURCE_PATH}"/*; do
         [[ -d "$folder_path" ]] || continue
         [[ "$(basename "$folder_path")" == .* ]] && continue
-        if [[ -z "$run_archive_path" ]]; then
-            run_archive_path="$(archive_path_for_run)"
-            log_info "Using single archive for this run: ${run_archive_path}"
-        fi
         found=1
-        process_folder "$folder_path" "$run_archive_path"
+        process_folder "$folder_path"
     done
 
     if [[ "$found" -eq 0 ]]; then
         log_warn "No folders found in NFS_SOURCE_PATH=${NFS_SOURCE_PATH}"
-    elif [[ -f "$run_archive_path" ]]; then
+    elif (( backup_successes > 0 )); then
+        log_info "Compressing the complete local staging directory into: ${run_archive_path}"
+        if ! archive_staging "$run_archive_path"; then
+            backup_failures=$((backup_failures + backup_successes))
+            backup_successes=0
+            rm -f -- "$run_archive_path"
+            log_error "Backup failed while compressing the local staging directory"
+        fi
+    fi
+
+    if [[ -f "$run_archive_path" ]]; then
         local file_size_bytes
         file_size_bytes="$(wc -c < "$run_archive_path" | tr -d '[:space:]')"
         if [[ "$file_size_bytes" =~ ^[0-9]+$ ]]; then
@@ -464,15 +585,22 @@ main() {
     fi
 
     print_summary
-    local size_mb
-    size_mb="$(backup_size_mb "$total_backup_size_bytes")"
+    local duration_seconds=$((SECONDS - backup_started_seconds))
+
+    if (( backup_successes == 0 )); then
+        send_backup_failure_notification \
+            "$NOTIFY_FAILURE_URL" \
+            "No NFS folders were backed up successfully" \
+            "$total_backup_size_bytes" || true
+        die "No folder backups succeeded" 1
+    fi
 
     if (( backup_failures > 0 )); then
         send_backup_failure_notification \
             "$NOTIFY_FAILURE_URL" \
-            "Error while backing up NFS folders" \
-            "$size_mb" || true
-        die "One or more folder backups failed" 1
+            "Some NFS folders failed to back up" \
+            "$total_backup_size_bytes" || true
+        log_warn "Some folder backups failed; continuing because ${backup_successes} folder backup(s) succeeded"
     fi
 
     cleanup_archives || true
@@ -481,7 +609,8 @@ main() {
         "$NOTIFY_SUCCESS_URL" \
         "$NOTIFY_TITLE" \
         "${NOTIFY_ASSET}" \
-        "$size_mb" || true
+        "$total_backup_size_bytes" \
+        "$duration_seconds" || true
 }
 
 main "$@"

k8s/backup-k3s-control-plane-config.sh

@@ -0,0 +1,473 @@
+#!/usr/bin/env bash
+
+# Back up the k3s control-plane configuration and credentials, without taking
+# a live datastore snapshot or changing the running k3s service.
+
+SCRIPT_NAME="$(basename "$0")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/common.sh
+source "${SCRIPT_DIR}/lib/common.sh"
+
+# Keep the local archive and temporary files readable only by this process.
+umask 077
+
+# Source paths. K3S_CONFIG_DIR is the only required source; the service files
+# and server state entries are included only when present.
+K3S_CONFIG_DIR="${K3S_CONFIG_DIR:-/etc/rancher/k3s}"
+K3S_SERVER_DIR="${K3S_SERVER_DIR:-/var/lib/rancher/k3s/server}"
+K3S_SERVICE_FILES="${K3S_SERVICE_FILES-/etc/systemd/system/k3s.service:/etc/systemd/system/k3s.service.env:/etc/default/k3s:/etc/sysconfig/k3s}"
+
+# Archive and local working settings.
+ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-k3s-control-plane-config}"
+ARCHIVE_TS_FORMAT="${ARCHIVE_TS_FORMAT:-%Y-%m-%d_%H-%M-%S}"
+LOCAL_TMP_PARENT="${LOCAL_TMP_PARENT:-${TMPDIR:-/tmp}}"
+LOCK_PATH="${LOCK_PATH:-/var/lock/k3s-control-plane-config-backup.lock}"
+
+# Remote destination settings.
+REMOTE_USER="${REMOTE_USER:-ivanch}"
+REMOTE_HOST="${REMOTE_HOST:-nas.haven}"
+REMOTE_DIR="${REMOTE_DIR:-/export/Backup/k8s}"
+REMOTE_IDENTITY_FILE="${REMOTE_IDENTITY_FILE:-}"
+SSH_PORT="${SSH_PORT:-}"
+CLEANUP_KEEP_COUNT="${CLEANUP_KEEP_COUNT:-4}"
+
+# Commands are configurable to support hosts with non-standard paths and
+# isolated tests.
+ZIP_BIN="${ZIP_BIN:-zip}"
+UNZIP_BIN="${UNZIP_BIN:-unzip}"
+SSH_BIN="${SSH_BIN:-ssh}"
+SCP_BIN="${SCP_BIN:-scp}"
+CURL_BIN="${CURL_BIN:-curl}"
+FLOCK_BIN="${FLOCK_BIN:-flock}"
+MKTEMP_BIN="${MKTEMP_BIN:-mktemp}"
+DATE_BIN="${DATE_BIN:-date}"
+REALPATH_BIN="${REALPATH_BIN:-realpath}"
+REMOTE_BASH_BIN="${REMOTE_BASH_BIN:-bash}"
+
+# Notifications use the Haven Notify v2 backup and error contracts.
+# Set either URL to an empty value to disable that notification.
+NOTIFY_SUCCESS_URL="${NOTIFY_SUCCESS_URL-http://notify.haven/api/v2/notifications/backup}"
+NOTIFY_FAILURE_URL="${NOTIFY_FAILURE_URL-http://notify.haven/api/v2/notifications/error}"
+NOTIFY_TITLE="${NOTIFY_TITLE:-Kubernetes}"
+NOTIFY_ASSET="${NOTIFY_ASSET:-k3s control-plane config}"
+NOTIFY_CALLER="${NOTIFY_CALLER:-K3s control-plane config backup}"
+NOTIFY_SOURCE="${NOTIFY_SOURCE:-$NOTIFY_CALLER}"
+
+declare -a ARCHIVE_ENTRIES=()
+declare -a SERVICE_FILE_LIST=()
+declare -a SSH_ARGS=()
+declare -a SCP_ARGS=()
+declare -a PROTECTED_PATHS=()
+readonly -a K3S_SERVER_ENTRIES=(cred etc manifests static tls token agent-token node-token)
+
+RUN_TMP_DIR=""
+LOCAL_ARCHIVE_PATH=""
+ARCHIVE_NAME=""
+ARCHIVE_SIZE_BYTES=0
+CLEANUP_KEEP_COUNT_INT=0
+LOCK_FD=""
+FAILURE_NOTIFICATION_SENT=0
+BACKUP_STARTED_SECONDS="$SECONDS"
+
+json_escape() {
+    local value="${1:-}"
+    value="${value//\\/\\\\}"
+    value="${value//\"/\\\"}"
+    value="${value//$'\n'/ }"
+    printf '%s' "$value"
+}
+
+send_backup_notification() {
+    local url="${1:-}"
+    local size_bytes="${2:-0}"
+    local duration_seconds="${3:-0}"
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "title": "$(json_escape "$NOTIFY_TITLE")",
+  "asset": "$(json_escape "$NOTIFY_ASSET")",
+  "sizeBytes": ${size_bytes},
+  "durationSeconds": ${duration_seconds},
+  "source": "$(json_escape "$NOTIFY_SOURCE")"
+}
+EOF
+)
+
+    if ! "$CURL_BIN" -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup notification"
+        return 1
+    fi
+}
+
+send_backup_failure_notification() {
+    local url="${1:-}"
+    local size_bytes="${2:-0}"
+    [[ -z "$url" ]] && return 0
+
+    local payload
+    payload=$(cat <<EOF
+{
+  "source": "$(json_escape "$NOTIFY_SOURCE")",
+  "message": "Control-plane configuration backup failed",
+  "severity": "critical",
+  "errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED",
+  "extra": [
+    {
+      "name": "Archive size",
+      "value": "${size_bytes} B"
+    }
+  ]
+}
+EOF
+)
+
+    if ! "$CURL_BIN" -fsS -X POST "$url" \
+        -H "Content-Type: application/json" \
+        -d "$payload" >/dev/null; then
+        log_warn "Failed to send backup failure notification"
+        return 1
+    fi
+}
+
+notify_failure_once() {
+    if (( FAILURE_NOTIFICATION_SENT == 1 )); then
+        return 0
+    fi
+    FAILURE_NOTIFICATION_SENT=1
+    send_backup_failure_notification "$NOTIFY_FAILURE_URL" "$ARCHIVE_SIZE_BYTES" || true
+}
+
+cleanup() {
+    local exit_code=$?
+    trap - EXIT
+
+    if [[ -n "$RUN_TMP_DIR" && -d "$RUN_TMP_DIR" ]]; then
+        rm -rf -- "$RUN_TMP_DIR" || log_warn "Failed to clean local temporary files"
+    fi
+
+    if (( exit_code == 0 )); then
+        log_info "Finished successfully"
+    else
+        notify_failure_once
+        log_error "Control-plane configuration backup failed"
+    fi
+
+    exit "$exit_code"
+}
+trap cleanup EXIT
+
+shell_quote() {
+    local value="${1:-}"
+    value="${value//\'/\'\"\'\"\'}"
+    printf "'%s'" "$value"
+}
+
+require_absolute_path() {
+    local path="${1:?path is required}"
+    local description="${2:-path}"
+    if [[ "$path" != /* ]]; then
+        die "${description} must be an absolute path"
+    fi
+}
+
+path_is_ancestor_or_same() {
+    local ancestor="${1:?ancestor path is required}"
+    local path="${2:?path is required}"
+
+    [[ "$ancestor" == "/" || "$path" == "$ancestor" || "$path" == "$ancestor"/* ]]
+}
+
+paths_overlap() {
+    local first="${1:?first path is required}"
+    local second="${2:?second path is required}"
+
+    path_is_ancestor_or_same "$first" "$second" || \
+        path_is_ancestor_or_same "$second" "$first"
+}
+
+canonicalize_protected_path() {
+    local path="${1:?protected path is required}"
+    local canonical_path
+    local canonical_parent
+    local basename
+
+    if [[ -e "$path" || -L "$path" ]]; then
+        canonical_path="$("$REALPATH_BIN" -e -- "$path" 2>/dev/null)" || return 1
+        printf '%s' "$canonical_path"
+        return 0
+    fi
+
+    basename="${path##*/}"
+    canonical_parent="${path%/*}"
+    if ! canonical_parent="$("$REALPATH_BIN" -e -- "$canonical_parent" 2>/dev/null)"; then
+        canonical_parent="$("$REALPATH_BIN" -m -- "$canonical_parent" 2>/dev/null)" || return 1
+    fi
+    printf '%s/%s' "${canonical_parent%/}" "$basename"
+}
+
+determine_protected_paths() {
+    local canonical_server_dir
+    local protected_path
+
+    if ! canonical_server_dir="$("$REALPATH_BIN" -e -- "$K3S_SERVER_DIR" 2>/dev/null)"; then
+        canonical_server_dir="$("$REALPATH_BIN" -m -- "$K3S_SERVER_DIR" 2>/dev/null)" || \
+            die "Configured source path is not safe to archive"
+    fi
+
+    PROTECTED_PATHS=()
+    if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/db")"; then
+        die "Configured source path is not safe to archive"
+    fi
+    PROTECTED_PATHS+=("$protected_path")
+    if ! protected_path="$(canonicalize_protected_path "${canonical_server_dir%/}/kine.sock")"; then
+        die "Configured source path is not safe to archive"
+    fi
+    PROTECTED_PATHS+=("$protected_path")
+}
+
+add_archive_entry_if_present() {
+    local absolute_path="${1:?source path is required}"
+    local canonical_path
+    local relative_path
+    local protected_path
+
+    if [[ "$absolute_path" =~ (^|/)(\.|\.\.)(/|$) ]]; then
+        die "Configured source path is not safe to archive"
+    fi
+
+    if [[ ! -e "$absolute_path" && ! -L "$absolute_path" ]]; then
+        return 0
+    fi
+
+    if ! canonical_path="$("$REALPATH_BIN" -e -- "$absolute_path" 2>/dev/null)"; then
+        die "Configured source path is not safe to archive"
+    fi
+
+    for protected_path in "${PROTECTED_PATHS[@]}"; do
+        if paths_overlap "$canonical_path" "$protected_path"; then
+            die "Configured source path is not safe to archive"
+        fi
+    done
+
+    relative_path="${canonical_path#/}"
+    if [[ -z "$relative_path" ]]; then
+        die "Configured source path is not safe to archive"
+    fi
+    ARCHIVE_ENTRIES+=("$relative_path")
+}
+
+build_remote_arguments() {
+    SSH_ARGS=()
+    SCP_ARGS=()
+    if [[ -n "$REMOTE_IDENTITY_FILE" ]]; then
+        SSH_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
+        SCP_ARGS+=(-i "$REMOTE_IDENTITY_FILE")
+    fi
+    if [[ -n "$SSH_PORT" ]]; then
+        SSH_ARGS+=(-p "$SSH_PORT")
+        SCP_ARGS+=(-P "$SSH_PORT")
+    fi
+}
+
+validate_inputs() {
+    require_cmd "$ZIP_BIN"
+    require_cmd "$UNZIP_BIN"
+    require_cmd "$SSH_BIN"
+    require_cmd "$SCP_BIN"
+    require_cmd "$FLOCK_BIN"
+    require_cmd "$MKTEMP_BIN"
+    require_cmd "$DATE_BIN"
+    require_cmd "$REALPATH_BIN"
+    if [[ -n "$NOTIFY_SUCCESS_URL" || -n "$NOTIFY_FAILURE_URL" ]]; then
+        require_cmd "$CURL_BIN"
+    fi
+
+    require_absolute_path "$K3S_CONFIG_DIR" "K3S_CONFIG_DIR"
+    require_absolute_path "$K3S_SERVER_DIR" "K3S_SERVER_DIR"
+    require_absolute_path "$LOCAL_TMP_PARENT" "LOCAL_TMP_PARENT"
+    require_absolute_path "$LOCK_PATH" "LOCK_PATH"
+    require_absolute_path "$REMOTE_DIR" "REMOTE_DIR"
+
+    if [[ ! -d "$K3S_CONFIG_DIR" ]]; then
+        die "K3S_CONFIG_DIR is not an available directory"
+    fi
+    determine_protected_paths
+    if [[ ! "$ARCHIVE_PREFIX" =~ ^[[:alnum:]][[:alnum:]._-]*$ ]]; then
+        die "ARCHIVE_PREFIX may contain only letters, numbers, dots, underscores, and hyphens"
+    fi
+    if [[ ! "$CLEANUP_KEEP_COUNT" =~ ^[0-9]+$ ]]; then
+        die "CLEANUP_KEEP_COUNT must be an integer >= 0"
+    fi
+    CLEANUP_KEEP_COUNT_INT=$((10#$CLEANUP_KEEP_COUNT))
+    if [[ -n "$SSH_PORT" && ! "$SSH_PORT" =~ ^[0-9]+$ ]]; then
+        die "SSH_PORT must be numeric"
+    fi
+    if [[ -z "$REMOTE_USER" || -z "$REMOTE_HOST" ]]; then
+        die "REMOTE_USER and REMOTE_HOST must not be empty"
+    fi
+    if [[ ! "$REMOTE_USER" =~ ^[[:alnum:]_][[:alnum:]_.-]*$ ]]; then
+        die "REMOTE_USER may contain only letters, numbers, underscores, dots, and hyphens"
+    fi
+    if [[ ! "$REMOTE_HOST" =~ ^[[:alnum:]][[:alnum:].-]*$ ]]; then
+        die "REMOTE_HOST may contain only letters, numbers, dots, and hyphens"
+    fi
+    if [[ ! "$REMOTE_DIR" =~ ^/([[:alnum:]_.-]+/)*[[:alnum:]_.-]+/?$ || "$REMOTE_DIR" == */./* || "$REMOTE_DIR" == */../* || "$REMOTE_DIR" == */. || "$REMOTE_DIR" == */.. ]]; then
+        die "REMOTE_DIR must be an absolute path containing only letters, numbers, dots, underscores, hyphens, and slashes without dot segments"
+    fi
+
+    mkdir -p -- "$LOCAL_TMP_PARENT"
+    mkdir -p -- "$(dirname -- "$LOCK_PATH")"
+    build_remote_arguments
+}
+
+collect_sources() {
+    local service_file
+    local server_entry
+
+    IFS=':' read -r -a SERVICE_FILE_LIST <<< "$K3S_SERVICE_FILES"
+
+    # These are the only server entries ever included. In particular, server/db
+    # and server/kine.sock are intentionally not archive inputs.
+    add_archive_entry_if_present "$K3S_CONFIG_DIR"
+    for service_file in "${SERVICE_FILE_LIST[@]}"; do
+        [[ -z "$service_file" ]] && continue
+        require_absolute_path "$service_file" "K3S_SERVICE_FILES entry"
+        add_archive_entry_if_present "$service_file"
+    done
+    for server_entry in "${K3S_SERVER_ENTRIES[@]}"; do
+        add_archive_entry_if_present "${K3S_SERVER_DIR%/}/${server_entry}"
+    done
+
+    if (( ${#ARCHIVE_ENTRIES[@]} == 0 )); then
+        die "No k3s configuration sources are available"
+    fi
+}
+
+acquire_lock() {
+    exec {LOCK_FD}>"$LOCK_PATH"
+    if ! "$FLOCK_BIN" -n "$LOCK_FD"; then
+        die "Another k3s control-plane configuration backup is already running"
+    fi
+}
+
+make_archive_name() {
+    local timestamp
+    timestamp="$("$DATE_BIN" +"$ARCHIVE_TS_FORMAT")"
+    if [[ ! "$timestamp" =~ ^[[:alnum:]_.-]+$ ]]; then
+        die "ARCHIVE_TS_FORMAT produced an unsafe archive timestamp"
+    fi
+    ARCHIVE_NAME="${ARCHIVE_PREFIX}_${timestamp}.zip"
+}
+
+create_and_validate_archive() {
+    if ! RUN_TMP_DIR="$("$MKTEMP_BIN" -d "${LOCAL_TMP_PARENT%/}/k3s-control-plane-config.XXXXXX")"; then
+        die "Unable to create local temporary directory"
+    fi
+    LOCAL_ARCHIVE_PATH="${RUN_TMP_DIR}/${ARCHIVE_NAME}"
+
+    log_info "Creating k3s control-plane configuration archive"
+    (
+        cd /
+        "$ZIP_BIN" -r "$LOCAL_ARCHIVE_PATH" -- "${ARCHIVE_ENTRIES[@]}" >/dev/null
+    )
+    "$UNZIP_BIN" -t "$LOCAL_ARCHIVE_PATH" >/dev/null
+
+    if [[ ! -f "$LOCAL_ARCHIVE_PATH" ]]; then
+        die "Archive creation did not produce an archive"
+    fi
+    ARCHIVE_SIZE_BYTES="$(wc -c < "$LOCAL_ARCHIVE_PATH" | tr -d '[:space:]')"
+    if [[ ! "$ARCHIVE_SIZE_BYTES" =~ ^[0-9]+$ ]]; then
+        die "Unable to determine archive size"
+    fi
+    log_info "Validated local archive"
+}
+
+run_ssh() {
+    "$SSH_BIN" "${SSH_ARGS[@]}" "${REMOTE_USER}@${REMOTE_HOST}" "$@"
+}
+
+remove_remote_temporary_archive() {
+    local remote_temp_path="${1:?temporary remote path is required}"
+    run_ssh "rm -f -- $(shell_quote "$remote_temp_path")" >/dev/null 2>&1 || true
+}
+
+publish_archive() {
+    local remote_final_path="${REMOTE_DIR%/}/${ARCHIVE_NAME}"
+    local remote_temp_path="${REMOTE_DIR%/}/.${ARCHIVE_NAME}.upload-$$.tmp"
+    local remote_spec="${REMOTE_USER}@${REMOTE_HOST}:${remote_temp_path}"
+    local move_command
+
+    run_ssh "mkdir -p -- $(shell_quote "$REMOTE_DIR")"
+
+    if ! "$SCP_BIN" "${SCP_ARGS[@]}" -- "$LOCAL_ARCHIVE_PATH" "$remote_spec"; then
+        log_error "Archive upload failed"
+        remove_remote_temporary_archive "$remote_temp_path"
+        return 1
+    fi
+
+    move_command="if [ -e $(shell_quote "$remote_final_path") ]; then exit 17; fi; mv -- $(shell_quote "$remote_temp_path") $(shell_quote "$remote_final_path")"
+    if ! run_ssh "$move_command"; then
+        log_error "Remote archive publish failed"
+        remove_remote_temporary_archive "$remote_temp_path"
+        return 1
+    fi
+
+    log_info "Published archive successfully"
+}
+
+cleanup_remote_archives() {
+    if (( CLEANUP_KEEP_COUNT_INT == 0 )); then
+        log_info "Remote retention disabled"
+        return 0
+    fi
+
+    local remote_cleanup_command
+    remote_cleanup_command="$(shell_quote "$REMOTE_BASH_BIN") -s -- $(shell_quote "$REMOTE_DIR") $(shell_quote "$ARCHIVE_PREFIX") $(shell_quote "$CLEANUP_KEEP_COUNT_INT")"
+
+    run_ssh "$remote_cleanup_command" <<'REMOTE_CLEANUP'
+set -Eeuo pipefail
+
+remote_dir="$1"
+archive_prefix="$2"
+keep_count="$3"
+
+mapfile -d '' -t sorted_archives < <(
+    find -P "$remote_dir" -maxdepth 1 -type f \
+        -name "${archive_prefix}_*.zip" -printf '%T@\t%p\0' |
+        sort -z -t $'\t' -k1,1nr
+)
+
+if (( ${#sorted_archives[@]} <= keep_count )); then
+    exit 0
+fi
+
+for ((index = keep_count; index < ${#sorted_archives[@]}; index++)); do
+    archive_record="${sorted_archives[$index]}"
+    archive_path="${archive_record#*$'\t'}"
+    rm -f -- "$archive_path"
+done
+REMOTE_CLEANUP
+
+    log_info "Remote archive retention completed"
+}
+
+main() {
+    validate_inputs
+    acquire_lock
+    collect_sources
+    make_archive_name
+    create_and_validate_archive
+    publish_archive
+    cleanup_remote_archives
+
+    # Notifications are informational: an unavailable notification endpoint
+    # must not change the result of a completed backup.
+    local duration_seconds=$((SECONDS - BACKUP_STARTED_SECONDS))
+    send_backup_notification "$NOTIFY_SUCCESS_URL" "$ARCHIVE_SIZE_BYTES" "$duration_seconds" || true
+}
+
+main "$@"

k8s/examples/automated-nfs-backup.env.example

@@ -4,19 +4,23 @@ BACKUP_PASSWORD=
 KUBECTL_BIN=kubectl
 KUBE_CONTEXT=
 WORKLOAD_KINDS=deployment,statefulset,replicaset,replicationcontroller
+EXCLUDED_NAMESPACES=kube-system,kube-public,kube-node-lease,default
+BACKUP_STAGING_PATH=/tmp/k8s-nfs-backup-staging
+RSYNC_BIN=rsync
 ARCHIVE_PREFIX=nfs-backup
 ARCHIVE_TS_FORMAT=%Y%m%d_%H%M%S
 SEVENZ_METHOD=lzma2
 SEVENZ_LEVEL=9
 SEVENZ_HEADER_ENCRYPT=on
 SEVENZ_THREADS=on
-SCALE_TIMEOUT_SECONDS=600
 SCALE_RETRY_COUNT=3
 SCALE_RETRY_DELAY_SECONDS=5
+SCALE_WAIT_SECONDS=30
 LOG_LEVEL=info
 TMP_STATE_DIR=/tmp/k8s-nfs-backup
-NOTIFY_SUCCESS_URL=
-NOTIFY_FAILURE_URL=
+NOTIFY_SUCCESS_URL=http://notify.haven/api/v2/notifications/backup
+NOTIFY_FAILURE_URL=http://notify.haven/api/v2/notifications/error
 NOTIFY_TITLE=Kubernetes
 NOTIFY_ASSET=kube config
+NOTIFY_SOURCE=Kubernetes NFS backup
 CLEANUP_KEEP_COUNT=5

k8s/test/test-automated-nfs-backup.sh

@@ -66,6 +66,46 @@ archive_exists() {
     [[ "${#archives[@]}" -gt 0 ]]
 }
 
+archive_count_is() {
+    local output_dir="$1"
+    local expected_count="$2"
+    local -a archives=()
+
+    shopt -s nullglob
+    archives=("${output_dir}"/test_*.7z)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -eq "$expected_count" ]]
+}
+
+path_does_not_exist() {
+    [[ ! -e "$1" ]]
+}
+
+directory_is_empty() {
+    local directory="$1"
+    local -a entries=()
+
+    shopt -s nullglob dotglob
+    entries=("${directory}"/*)
+    shopt -u nullglob dotglob
+    [[ "${#entries[@]}" -eq 0 ]]
+}
+
+events_are_ordered() {
+    local trace_file="$1"
+    shift
+    local event
+    local line_number
+    local previous_line=0
+
+    for event in "$@"; do
+        line_number="$(grep -n -m1 -F -- "$event" "$trace_file" | cut -d: -f1)"
+        [[ -n "$line_number" ]] || return 1
+        (( line_number > previous_line )) || return 1
+        previous_line="$line_number"
+    done
+}
+
 trace_has_scale() {
     local trace_file="$1"
     local namespace="$2"
@@ -74,13 +114,11 @@ trace_has_scale() {
     grep -Eq -- "^${namespace} [^ ]+ [^ ]+ --replicas=${replicas}$" "$trace_file"
 }
 
-# Asserts the real script invoked 7z as `7z a -t7z <archive.7z> <folder> ...`
-# for the given folder. The archive is the 3rd token (ends in .7z) and the
-# folder path is the 4th token, so a swap would fail to match.
-sevenz_called_for() {
-    local folder_name="$1"
-
-    grep -Eq -- "a -t7z [^ ]+\\.7z .*/${folder_name}( |$)" "${TMPD}/7z_invocations.log"
+# Asserts the real script invoked 7z exactly once for the complete contents of
+# the run staging directory.
+sevenz_called_once_for_staging() {
+    [[ "$(wc -l < "${TMPD}/7z_invocations.log" | tr -d '[:space:]')" -eq 1 ]] &&
+        grep -Eq -- '^a -t7z [^ ]+\.7z \. ' "${TMPD}/7z_invocations.log"
 }
 
 write_fake_script() {
@@ -154,6 +192,7 @@ for ((i = 1; i <= $#; i++)); do
             [[ "$arg" == --replicas=* ]] && replicas_arg="$arg"
         done
         printf '%s %s %s %s\n' "$namespace" "$kind" "$name" "$replicas_arg" >> "${TMPD}/scale_trace.log"
+        printf 'scale %s %s\n' "$namespace" "$replicas_arg" >> "${TMPD}/event_trace.log"
         exit 0
     fi
 done
@@ -172,19 +211,50 @@ if [[ "${1:-}" == "a" && "${2:-}" == "-t7z" ]]; then
     archive="${3:?archive path required}"
     folder="${4:?folder path required}"
     folder="${folder%/}"
-    folder_name="${folder##*/}"
+    printf '7z staging\n' >> "${TMPD}/event_trace.log"
 
-    # Simulate a backup failure for configured folders BEFORE recording a
-    # successful archive, so 7z_trace.log only lists folders actually archived.
-    if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
+    if [[ -s "${TMPD}/fail_archive.txt" ]]; then
+        touch "$archive"
         exit 2
     fi
 
-    printf '%s\n' "$folder_name" >> "${TMPD}/7z_trace.log"
+    for staged_folder in ./*; do
+        [[ -d "$staged_folder" ]] || continue
+        printf '%s\n' "${staged_folder##*/}" >> "${TMPD}/7z_trace.log"
+    done
     touch "$archive"
 fi
 
 exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/rsync" <<'EOF'
+#!/usr/bin/env bash
+set -uo pipefail
+
+source_folder="${@: -2:1}"
+staging_folder="${@: -1}"
+source_folder="${source_folder%/}"
+staging_folder="${staging_folder%/}"
+folder_name="${source_folder##*/}"
+
+mkdir -p "${TMPD}/rsync_counts"
+count_file="${TMPD}/rsync_counts/${folder_name}"
+count=0
+if [[ -f "$count_file" ]]; then
+    count="$(<"$count_file")"
+fi
+count=$((count + 1))
+printf '%s\n' "$count" > "$count_file"
+printf 'rsync %s %s\n' "$folder_name" "$count" >> "${TMPD}/event_trace.log"
+
+if grep -Fxq -- "$folder_name" "${TMPD}/fail_folders.txt"; then
+    printf 'simulated rsync failure for %s\n' "$folder_name" >&2
+    exit 23
+fi
+
+mkdir -p "$staging_folder"
+cp -a "${source_folder}/." "$staging_folder/"
 EOF
 
     write_fake_script "${TMPD}/bin/curl" <<'EOF'
@@ -208,7 +278,7 @@ for ((i = 1; i <= $#; i++)); do
 done
 
 body="${body//$'\n'/ }"
-printf '%s %s\n' "$url" "${body:0:120}" >> "${TMPD}/curl_trace.log"
+printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
 exit 0
 EOF
 }
@@ -218,40 +288,50 @@ run_backup() {
     local output_dir="$2"
     local state_dir="$3"
     local log_file="$4"
+    shift 4
+    local -a env_overrides=("$@")
 
-    NFS_SOURCE_PATH="$source_dir" \
-    BACKUP_OUTPUT_PATH="$output_dir" \
-    BACKUP_PASSWORD="" \
-    ARCHIVE_PREFIX="test" \
-    NOTIFY_SUCCESS_URL="http://example.invalid/success" \
-    NOTIFY_FAILURE_URL="http://example.invalid/failure" \
-    KUBECTL_BIN="${TMPD}/bin/kubectl" \
-    SEVENZ_BIN="${TMPD}/bin/7z" \
-    SCALE_WAIT_SECONDS=0 \
-    TMP_STATE_DIR="$state_dir" \
-    PATH="${TMPD}/bin:${PATH}" \
-    TMPD="$TMPD" \
-    bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
+    env BACKUP_STAGING_PATH="${state_dir}/staging" \
+        "${env_overrides[@]}" \
+        NFS_SOURCE_PATH="$source_dir" \
+        BACKUP_OUTPUT_PATH="$output_dir" \
+        BACKUP_PASSWORD="" \
+        ARCHIVE_PREFIX="test" \
+        NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
+        NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
+        KUBECTL_BIN="${TMPD}/bin/kubectl" \
+        SEVENZ_BIN="${TMPD}/bin/7z" \
+        RSYNC_BIN="${TMPD}/bin/rsync" \
+        SCALE_WAIT_SECONDS=0 \
+        TMP_STATE_DIR="$state_dir" \
+        PATH="${TMPD}/bin:${PATH}" \
+        TMPD="$TMPD" \
+        bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
 }
 
 write_fake_binaries
 export TMPD
+: > "${TMPD}/fail_archive.txt"
 
-# Scenario A: a mapped folder is scaled down and restored, while an unmapped
-# folder is backed up directly. Both folders share one archive.
+# Scenario A: a mapped folder is scaled down, copied, and restored, while
+# unmapped and excluded folders are copied live. One 7z call archives them all.
 success_dir="${TMPD}/success"
 success_source="${success_dir}/source"
 success_output="${success_dir}/out"
 success_state="${success_dir}/state"
-mkdir -p "${success_source}/app-ns" "${success_source}/loose-data" "$success_output" "$success_state"
+mkdir -p "${success_source}/app-ns" "${success_source}/kube-system" "${success_source}/loose-data" "$success_output" "${success_state}/run-stale"
 printf 'data\n' > "${success_source}/app-ns/file.txt"
+printf 'data\n' > "${success_source}/kube-system/file.txt"
 printf 'data\n' > "${success_source}/loose-data/file.txt"
-printf 'app-ns\n' > "${TMPD}/namespaces.txt"
-printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+printf 'deployment\tstale\t9\n' > "${success_state}/run-stale/stale-ns.state"
+printf 'app-ns\nkube-system\nstale-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\nkube-system:2\nstale-ns:9\n' > "${TMPD}/replicas.txt"
 : > "${TMPD}/fail_folders.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
 : > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
 
 set +e
 run_backup "$success_source" "$success_output" "$success_state" "${success_dir}/success.log"
@@ -260,17 +340,112 @@ set -e
 
 assert "success scenario exits with code 0" is_zero "$success_exit_code"
 assert "success scenario creates a test archive" archive_exists "$success_output"
-assert "success scenario reports two backup successes" file_contains "${success_dir}/success.log" "backup_successes=2"
+assert "success scenario reports three backup successes" file_contains "${success_dir}/success.log" "backup_successes=3"
 assert "success scenario reports zero backup failures" file_contains "${success_dir}/success.log" "backup_failures=0"
 assert "success scenario scales app-ns down" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "0"
 assert "success scenario restores app-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
-assert "success scenario sends success notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/success"
+assert "success scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "success scenario sends the archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes": 0'
+assert "success scenario sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "Kubernetes NFS backup"'
 assert "success scenario does NOT scale unmapped loose-data" file_lacks "${TMPD}/scale_trace.log" "loose-data"
-assert "success scenario invokes 7z with archive then folder (app-ns)" sevenz_called_for "app-ns"
-assert "success scenario invokes 7z with archive then folder (loose-data)" sevenz_called_for "loose-data"
+assert "success scenario does NOT scale excluded kube-system" file_lacks "${TMPD}/scale_trace.log" "kube-system"
+assert "success scenario logs excluded namespace" file_contains "${success_dir}/success.log" "Folder 'kube-system' matches an excluded namespace"
+assert "success scenario ignores stale state from an earlier run" file_lacks "${TMPD}/scale_trace.log" "stale-ns"
+assert "success scenario invokes 7z exactly once for staging" sevenz_called_once_for_staging
+assert "success scenario archives mapped app-ns" file_contains "${TMPD}/7z_trace.log" "app-ns"
+assert "success scenario archives excluded kube-system" file_contains "${TMPD}/7z_trace.log" "kube-system"
+assert "success scenario archives unmapped loose-data" file_contains "${TMPD}/7z_trace.log" "loose-data"
 
-# Scenario B: one namespace backup fails, but both namespaces are restored and
-# the script exits through its documented handled-failure path.
+# Scenario A2: the exclusion list can be replaced through the environment.
+custom_exclude_dir="${TMPD}/custom-exclude"
+custom_exclude_source="${custom_exclude_dir}/source"
+custom_exclude_output="${custom_exclude_dir}/out"
+custom_exclude_state="${custom_exclude_dir}/state"
+mkdir -p "${custom_exclude_source}/app-ns" "$custom_exclude_output" "$custom_exclude_state"
+printf 'data\n' > "${custom_exclude_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$custom_exclude_source" "$custom_exclude_output" "$custom_exclude_state" "${custom_exclude_dir}/custom-exclude.log" "EXCLUDED_NAMESPACES=app-ns"
+custom_exclude_exit_code=$?
+set -e
+
+assert "custom exclusion scenario exits with code 0" is_zero "$custom_exclude_exit_code"
+assert "custom exclusion scenario does NOT scale app-ns" file_lacks "${TMPD}/scale_trace.log" "app-ns"
+assert "custom exclusion scenario logs excluded app-ns" file_contains "${custom_exclude_dir}/custom-exclude.log" "Folder 'app-ns' matches an excluded namespace"
+
+# Scenario A3: the mapped namespace is scaled down before its single local copy,
+# restored immediately afterward, and compression starts only after restoration.
+staging_dir="${TMPD}/staging"
+staging_source="${staging_dir}/source"
+staging_output="${staging_dir}/out"
+staging_state="${staging_dir}/state"
+staging_work="${staging_dir}/work"
+mkdir -p "${staging_source}/app-ns" "$staging_output" "$staging_state" "$staging_work"
+printf 'data\n' > "${staging_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+rm -rf "${TMPD}/rsync_counts"
+
+set +e
+run_backup "$staging_source" "$staging_output" "$staging_state" "${staging_dir}/staging.log" "BACKUP_STAGING_PATH=${staging_work}"
+staging_exit_code=$?
+set -e
+
+assert "staging scenario exits with code 0" is_zero "$staging_exit_code"
+assert "staging scenario creates an archive" archive_exists "$staging_output"
+assert "staging scenario restores before compression" events_are_ordered "${TMPD}/event_trace.log" \
+    "scale app-ns --replicas=0" \
+    "rsync app-ns 1" \
+    "scale app-ns --replicas=3" \
+    "7z staging"
+assert "staging scenario copies app-ns only once" file_contains "${TMPD}/rsync_counts/app-ns" "1"
+assert "staging scenario invokes 7z exactly once" sevenz_called_once_for_staging
+assert "staging scenario does not archive the temporary run path" file_lacks "${TMPD}/7z_invocations.log" "run-"
+assert "staging scenario cleans its staged copy" directory_is_empty "$staging_work"
+
+# Scenario A4: a failed local copy never archives the incomplete staging copy,
+# but the namespace is still restored immediately.
+rsync_failure_dir="${TMPD}/rsync-failure"
+rsync_failure_source="${rsync_failure_dir}/source"
+rsync_failure_output="${rsync_failure_dir}/out"
+rsync_failure_state="${rsync_failure_dir}/state"
+rsync_failure_work="${rsync_failure_dir}/work"
+mkdir -p "${rsync_failure_source}/app-ns" "$rsync_failure_output" "$rsync_failure_state" "$rsync_failure_work"
+printf 'data\n' > "${rsync_failure_source}/app-ns/file.txt"
+printf 'app-ns\n' > "${TMPD}/namespaces.txt"
+printf 'app-ns:3\n' > "${TMPD}/replicas.txt"
+printf 'app-ns\n' > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
+rm -rf "${TMPD}/rsync_counts"
+
+set +e
+run_backup "$rsync_failure_source" "$rsync_failure_output" "$rsync_failure_state" "${rsync_failure_dir}/rsync-failure.log" "BACKUP_STAGING_PATH=${rsync_failure_work}"
+rsync_failure_exit_code=$?
+set -e
+
+assert "rsync failure exits non-zero when nothing was backed up" is_nonzero "$rsync_failure_exit_code"
+assert "rsync failure restores app-ns" trace_has_scale "${TMPD}/scale_trace.log" "app-ns" "3"
+assert "rsync failure does not invoke 7z" file_lacks "${TMPD}/event_trace.log" "7z staging"
+assert "rsync failure cleans its staged copy" directory_is_empty "$rsync_failure_work"
+
+# Scenario B: one namespace backup fails, but another succeeds. Both namespaces
+# are restored, retention still runs, and the script reports success.
 failure_dir="${TMPD}/failure"
 failure_source="${failure_dir}/source"
 failure_output="${failure_dir}/out"
@@ -278,30 +453,123 @@ failure_state="${failure_dir}/state"
 mkdir -p "${failure_source}/good-ns" "${failure_source}/fail-ns" "$failure_output" "$failure_state"
 printf 'data\n' > "${failure_source}/good-ns/file.txt"
 printf 'data\n' > "${failure_source}/fail-ns/file.txt"
+for i in 1 2 3 4; do
+    touch -t "2020010${i}0000" "${failure_output}/test_2020010${i}_000000.7z"
+done
 printf 'good-ns\nfail-ns\n' > "${TMPD}/namespaces.txt"
 printf 'good-ns:2\nfail-ns:4\n' > "${TMPD}/replicas.txt"
 printf 'fail-ns\n' > "${TMPD}/fail_folders.txt"
 : > "${TMPD}/scale_trace.log"
 : > "${TMPD}/7z_trace.log"
+: > "${TMPD}/7z_invocations.log"
 : > "${TMPD}/curl_trace.log"
+: > "${TMPD}/event_trace.log"
 
 set +e
 run_backup "$failure_source" "$failure_output" "$failure_state" "${failure_dir}/failure.log"
 failure_exit_code=$?
 set -e
 
-assert "failure scenario exits non-zero" is_nonzero "$failure_exit_code"
-assert "failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
-assert "failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
-assert "failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
-assert "failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
-assert "failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
-assert "failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
-assert "failure scenario sends failure notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/failure"
-assert "failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
-assert "failure scenario invokes 7z with archive then folder (good-ns)" sevenz_called_for "good-ns"
+assert "partial failure scenario exits with code 0" is_zero "$failure_exit_code"
+assert "partial failure scenario reports one backup success" file_contains "${failure_dir}/failure.log" "backup_successes=1"
+assert "partial failure scenario reports one backup failure" file_contains "${failure_dir}/failure.log" "backup_failures=1"
+assert "partial failure scenario logs that it is continuing" file_contains "${failure_dir}/failure.log" "Some folder backups failed; continuing"
+assert "partial failure scenario scales good-ns down" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "0"
+assert "partial failure scenario restores good-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "good-ns" "2"
+assert "partial failure scenario scales fail-ns down" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "0"
+assert "partial failure scenario restores fail-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "fail-ns" "4"
+assert "partial failure scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "partial failure scenario sends v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "partial failure scenario marks the error as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
+assert "partial failure scenario includes the backup error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "BACKUP_FAILED"'
+assert "partial failure scenario does NOT archive the failed folder" file_lacks "${TMPD}/7z_trace.log" "fail-ns"
+assert "partial failure scenario archives the successfully staged folder" file_contains "${TMPD}/7z_trace.log" "good-ns"
+assert "partial failure scenario invokes 7z exactly once" sevenz_called_once_for_staging
+assert "partial failure scenario compresses after all namespaces are restored" events_are_ordered "${TMPD}/event_trace.log" \
+    "scale fail-ns --replicas=4" \
+    "scale good-ns --replicas=2" \
+    "7z staging"
+assert "partial failure scenario retains only four archives" archive_count_is "$failure_output" 4
+assert "partial failure scenario deletes the oldest archive" path_does_not_exist "${failure_output}/test_20200101_000000.7z"
 
-# Scenario C: abnormal abort while a namespace is still scaled to zero.
+# Scenario C: every attempted folder backup fails, so the script must still
+# return non-zero and must not send a success notification.
+all_failed_dir="${TMPD}/all-failed"
+all_failed_source="${all_failed_dir}/source"
+all_failed_output="${all_failed_dir}/out"
+all_failed_state="${all_failed_dir}/state"
+mkdir -p "${all_failed_source}/only-ns" "$all_failed_output" "$all_failed_state"
+printf 'data\n' > "${all_failed_source}/only-ns/file.txt"
+printf 'only-ns\n' > "${TMPD}/namespaces.txt"
+printf 'only-ns:2\n' > "${TMPD}/replicas.txt"
+printf 'only-ns\n' > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$all_failed_source" "$all_failed_output" "$all_failed_state" "${all_failed_dir}/all-failed.log"
+all_failed_exit_code=$?
+set -e
+
+assert "all-failed scenario exits non-zero" is_nonzero "$all_failed_exit_code"
+assert "all-failed scenario reports zero backup successes" file_contains "${all_failed_dir}/all-failed.log" "backup_successes=0"
+assert "all-failed scenario reports one backup failure" file_contains "${all_failed_dir}/all-failed.log" "backup_failures=1"
+assert "all-failed scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "all-failed scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+
+# Scenario C2: a failed single archive operation invalidates all staged folders
+# and removes the partial archive left by 7z.
+archive_failure_dir="${TMPD}/archive-failure"
+archive_failure_source="${archive_failure_dir}/source"
+archive_failure_output="${archive_failure_dir}/out"
+archive_failure_state="${archive_failure_dir}/state"
+mkdir -p "${archive_failure_source}/archive-ns" "$archive_failure_output" "$archive_failure_state"
+printf 'data\n' > "${archive_failure_source}/archive-ns/file.txt"
+printf 'archive-ns\n' > "${TMPD}/namespaces.txt"
+printf 'archive-ns:2\n' > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+printf 'fail\n' > "${TMPD}/fail_archive.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$archive_failure_source" "$archive_failure_output" "$archive_failure_state" "${archive_failure_dir}/archive-failure.log"
+archive_failure_exit_code=$?
+set -e
+
+assert "archive failure exits non-zero" is_nonzero "$archive_failure_exit_code"
+assert "archive failure restores archive-ns replicas" trace_has_scale "${TMPD}/scale_trace.log" "archive-ns" "2"
+assert "archive failure reports zero backup successes" file_contains "${archive_failure_dir}/archive-failure.log" "backup_successes=0"
+assert "archive failure removes the partial archive" archive_count_is "$archive_failure_output" 0
+assert "archive failure does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+: > "${TMPD}/fail_archive.txt"
+
+# Scenario D: no source folders means zero successful backups, which must also
+# return non-zero rather than reporting an empty run as successful.
+empty_dir="${TMPD}/empty"
+empty_source="${empty_dir}/source"
+empty_output="${empty_dir}/out"
+empty_state="${empty_dir}/state"
+mkdir -p "$empty_source" "$empty_output" "$empty_state"
+: > "${TMPD}/namespaces.txt"
+: > "${TMPD}/replicas.txt"
+: > "${TMPD}/fail_folders.txt"
+: > "${TMPD}/scale_trace.log"
+: > "${TMPD}/7z_trace.log"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$empty_source" "$empty_output" "$empty_state" "${empty_dir}/empty.log"
+empty_exit_code=$?
+set -e
+
+assert "empty scenario exits non-zero" is_nonzero "$empty_exit_code"
+assert "empty scenario reports zero backup successes" file_contains "${empty_dir}/empty.log" "backup_successes=0"
+assert "empty scenario sends v2 error notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error"
+assert "empty scenario does not send v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+
+# Scenario E: abnormal abort while a namespace is still scaled to zero.
 # We make the second `sleep "$SCALE_WAIT_SECONDS"` fail (invalid value -> `sleep`
 # errors, and `set -e` aborts the run) AFTER the namespace is scaled to 0 and its
 # .state file is written, but BEFORE the in-line restore. The EXIT trap must then
@@ -311,7 +579,8 @@ abort_dir="${TMPD}/abort"
 abort_source="${abort_dir}/source"
 abort_output="${abort_dir}/out"
 abort_state="${abort_dir}/state"
-mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}"
+abort_work="${abort_dir}/work"
+mkdir -p "${abort_source}/crash-ns" "${abort_output}" "${abort_state}" "${abort_work}"
 printf 'data\n' > "${abort_source}/crash-ns/file.txt"
 printf 'crash-ns\n' > "${TMPD}/namespaces.txt"
 printf 'crash-ns:5\n' > "${TMPD}/replicas.txt"
@@ -330,6 +599,7 @@ NOTIFY_SUCCESS_URL="" \
 NOTIFY_FAILURE_URL="" \
 KUBECTL_BIN="${TMPD}/bin/kubectl" \
 SEVENZ_BIN="${TMPD}/bin/7z" \
+BACKUP_STAGING_PATH="${abort_work}" \
 SCALE_WAIT_SECONDS="not-a-number" \
 TMP_STATE_DIR="${abort_state}" \
 PATH="${TMPD}/bin:${PATH}" \
@@ -341,7 +611,7 @@ set -e
 assert "abort scenario exits non-zero" is_nonzero "${abort_exit_code}"
 assert "abort scenario scaled crash-ns down to 0" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "0"
 assert "abort scenario EXIT trap restored crash-ns to 5" trace_has_scale "${TMPD}/scale_trace.log" "crash-ns" "5"
-assert "abort scenario left no stale .state file" file_lacks "${abort_dir}/abort.log" ".state"
+assert "abort scenario cleaned its per-run state directory" directory_is_empty "$abort_state"
 
 printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
 if [[ "$tests_failed" -gt 0 ]]; then

k8s/test/test-backup-k3s-control-plane-config.sh

@@ -0,0 +1,439 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+SCRIPT_UNDER_TEST="${REPO_ROOT}/k8s/backup-k3s-control-plane-config.sh"
+TMPD="$(mktemp -d "${TMPDIR:-/tmp}/k3s-control-plane-config-test.XXXXXX")"
+
+tests_total=0
+tests_passed=0
+tests_failed=0
+
+cleanup() {
+    rm -rf -- "$TMPD"
+}
+trap cleanup EXIT
+
+assert() {
+    local message="$1"
+    shift
+    tests_total=$((tests_total + 1))
+    if "$@"; then
+        printf 'PASS: %s\n' "$message"
+        tests_passed=$((tests_passed + 1))
+    else
+        printf 'FAIL: %s\n' "$message"
+        tests_failed=$((tests_failed + 1))
+    fi
+}
+
+is_zero() { [[ "$1" -eq 0 ]]; }
+is_nonzero() { [[ "$1" -ne 0 ]]; }
+path_exists() { [[ -e "$1" ]]; }
+path_does_not_exist() { [[ ! -e "$1" ]]; }
+file_contains() { grep -Fq -- "$2" "$1"; }
+file_lacks() { [[ ! -e "$1" ]] || ! grep -Fq -- "$2" "$1"; }
+file_is_empty() { [[ ! -s "$1" ]]; }
+file_line_count_is() { [[ "$(grep -Fc -- "$2" "$1")" -eq "$3" ]]; }
+
+matching_archive_count_is() {
+    local directory="$1"
+    local expected="$2"
+    local -a archives=()
+    shopt -s nullglob
+    archives=("${directory}"/k3s-control-plane-config_*.zip)
+    shopt -u nullglob
+    [[ "${#archives[@]}" -eq "$expected" ]]
+}
+
+directory_is_empty() {
+    local directory="$1"
+    local -a entries=()
+    shopt -s nullglob dotglob
+    entries=("${directory}"/*)
+    shopt -u nullglob dotglob
+    [[ "${#entries[@]}" -eq 0 ]]
+}
+
+write_fake_script() {
+    local path="$1"
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        printf '%s\n' "$line"
+    done > "$path"
+    chmod +x "$path"
+}
+
+write_fake_binaries() {
+    mkdir -p "${TMPD}/bin"
+
+    write_fake_script "${TMPD}/bin/zip" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[[ "${1:-}" == "-r" ]]
+archive="${2:?archive path required}"
+shift 2
+[[ "${1:-}" == "--" ]]
+shift
+: > "$archive"
+for entry in "$@"; do
+    source_path="/$entry"
+    if [[ -d "$source_path" ]]; then
+        while IFS= read -r path; do
+            printf '%s\n' "${path#/}" >> "$archive"
+        done < <(find "$source_path" -print | sort)
+    else
+        printf '%s\n' "$entry" >> "$archive"
+    fi
+done
+printf '%s\n' "$*" >> "${TMPD}/zip_invocations.log"
+EOF
+
+    write_fake_script "${TMPD}/bin/unzip" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[[ "${1:-}" == "-t" ]]
+archive="${2:?archive path required}"
+[[ -f "$archive" && -s "$archive" ]]
+printf '%s\n' "$archive" >> "${TMPD}/unzip_trace.log"
+EOF
+
+    write_fake_script "${TMPD}/bin/scp" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+source_path="${@: -2:1}"
+destination="${@: -1}"
+remote_path="${destination#*:}"
+if [[ "$remote_path" == *"'"* ]]; then
+    printf 'scp destination contains literal single quotes: %s\n' "$destination" >&2
+    exit 64
+fi
+remote_name="${remote_path##*/}"
+target_path="${FAKE_REMOTE_ROOT}/${remote_name}"
+
+cp -- "$source_path" "$target_path"
+printf '%s\n' "$remote_name" >> "${TMPD}/scp_trace.log"
+if [[ -f "${TMPD}/fail_scp" ]]; then
+    exit 42
+fi
+EOF
+
+    write_fake_script "${TMPD}/bin/ssh" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+command="${@: -1}"
+printf '%s\n' "$command" >> "${TMPD}/ssh_trace.log"
+
+quoted_values=()
+while IFS= read -r value; do
+    quoted_values+=("${value#\'}")
+    quoted_values[${#quoted_values[@]}-1]="${quoted_values[${#quoted_values[@]}-1]%\'}"
+done < <(printf '%s\n' "$command" | grep -o "'[^']*'" || true)
+
+if [[ "$command" == mkdir\ -p* ]]; then
+    mkdir -p -- "$FAKE_REMOTE_ROOT"
+    exit 0
+fi
+
+if [[ "$command" == rm\ -f* ]]; then
+    temp_name="${quoted_values[0]##*/}"
+    rm -f -- "${FAKE_REMOTE_ROOT}/${temp_name}"
+    exit 0
+fi
+
+if [[ "$command" == *" mv -- "* ]]; then
+    value_count="${#quoted_values[@]}"
+    temp_name="${quoted_values[value_count-2]##*/}"
+    final_name="${quoted_values[value_count-1]##*/}"
+    [[ ! -e "${FAKE_REMOTE_ROOT}/${final_name}" ]]
+    mv -- "${FAKE_REMOTE_ROOT}/${temp_name}" "${FAKE_REMOTE_ROOT}/${final_name}"
+    exit 0
+fi
+
+if [[ "$command" == *" -s -- "* ]]; then
+    value_count="${#quoted_values[@]}"
+    (
+        cd -- "$FAKE_REMOTE_ROOT"
+        bash -s -- "$FAKE_REMOTE_ROOT" "${quoted_values[value_count-2]}" "${quoted_values[value_count-1]}"
+    )
+    exit 0
+fi
+
+exit 1
+EOF
+
+    write_fake_script "${TMPD}/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+url=""
+body=""
+for ((i = 1; i <= $#; i++)); do
+    case "${!i}" in
+        POST|-d)
+            next=$((i + 1))
+            if [[ "${!i}" == "POST" ]]; then
+                url="${!next}"
+            else
+                body="${!next}"
+            fi
+            ;;
+    esac
+done
+body="${body//$'\n'/ }"
+printf '%s %s\n' "$url" "$body" >> "${TMPD}/curl_trace.log"
+[[ ! -f "${TMPD}/fail_curl" ]]
+EOF
+
+    write_fake_script "${TMPD}/bin/flock" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+exit 0
+EOF
+
+    write_fake_script "${TMPD}/bin/date" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+printf '%s\n' '2026-07-21_12-00-00'
+EOF
+}
+
+run_backup() {
+    local fixture_root="$1"
+    local remote_root="$2"
+    local local_temp_parent="$3"
+    local log_file="$4"
+    shift 4
+    local -a overrides=("$@")
+
+    env \
+        K3S_CONFIG_DIR="${fixture_root}/etc/rancher/k3s" \
+        K3S_SERVER_DIR="${fixture_root}/var/lib/rancher/k3s/server" \
+        K3S_SERVICE_FILES="${fixture_root}/etc/systemd/system/k3s.service:${fixture_root}/etc/systemd/system/k3s.service.env:${fixture_root}/etc/default/k3s:${fixture_root}/etc/sysconfig/k3s" \
+        LOCAL_TMP_PARENT="$local_temp_parent" \
+        LOCK_PATH="${fixture_root}/lock/k3s-control-plane-config.lock" \
+        REMOTE_USER="test-user" \
+        REMOTE_HOST="test-host" \
+        REMOTE_DIR="/backup" \
+        ZIP_BIN="${TMPD}/bin/zip" \
+        UNZIP_BIN="${TMPD}/bin/unzip" \
+        SSH_BIN="${TMPD}/bin/ssh" \
+        SCP_BIN="${TMPD}/bin/scp" \
+        CURL_BIN="${TMPD}/bin/curl" \
+        FLOCK_BIN="${TMPD}/bin/flock" \
+        DATE_BIN="${TMPD}/bin/date" \
+        NOTIFY_SUCCESS_URL="http://example.invalid/api/v2/notifications/backup" \
+        NOTIFY_FAILURE_URL="http://example.invalid/api/v2/notifications/error" \
+        PATH="${TMPD}/bin:${PATH}" \
+        TMPD="$TMPD" \
+        FAKE_REMOTE_ROOT="$remote_root" \
+        "${overrides[@]}" \
+        bash "$SCRIPT_UNDER_TEST" >"$log_file" 2>&1
+}
+
+create_source_fixture() {
+    local fixture_root="$1"
+    mkdir -p \
+        "${fixture_root}/etc/rancher/k3s" \
+        "${fixture_root}/etc/systemd/system" \
+        "${fixture_root}/etc/default" \
+        "${fixture_root}/etc/sysconfig" \
+        "${fixture_root}/var/lib/rancher/k3s/server/cred" \
+        "${fixture_root}/var/lib/rancher/k3s/server/etc" \
+        "${fixture_root}/var/lib/rancher/k3s/server/manifests" \
+        "${fixture_root}/var/lib/rancher/k3s/server/static" \
+        "${fixture_root}/var/lib/rancher/k3s/server/tls" \
+        "${fixture_root}/var/lib/rancher/k3s/server/db" \
+        "${fixture_root}/lock"
+    printf 'config\n' > "${fixture_root}/etc/rancher/k3s/config.yaml"
+    printf 'service\n' > "${fixture_root}/etc/systemd/system/k3s.service"
+    printf 'environment\n' > "${fixture_root}/etc/systemd/system/k3s.service.env"
+    printf 'default\n' > "${fixture_root}/etc/default/k3s"
+    printf 'sysconfig\n' > "${fixture_root}/etc/sysconfig/k3s"
+    printf 'credential\n' > "${fixture_root}/var/lib/rancher/k3s/server/cred/passwd"
+    printf 'state\n' > "${fixture_root}/var/lib/rancher/k3s/server/etc/config"
+    printf 'manifest\n' > "${fixture_root}/var/lib/rancher/k3s/server/manifests/custom.yaml"
+    printf 'static\n' > "${fixture_root}/var/lib/rancher/k3s/server/static/pod.yaml"
+    printf 'tls\n' > "${fixture_root}/var/lib/rancher/k3s/server/tls/server-ca.crt"
+    printf 'token\n' > "${fixture_root}/var/lib/rancher/k3s/server/token"
+    printf 'agent-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/agent-token"
+    printf 'node-token\n' > "${fixture_root}/var/lib/rancher/k3s/server/node-token"
+    printf 'database must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/db/state.db"
+    printf 'socket must not be archived\n' > "${fixture_root}/var/lib/rancher/k3s/server/kine.sock"
+}
+
+write_fake_binaries
+export TMPD
+
+# Successful publication: source layout is preserved relative to /, unzip is
+# called, and retention deletes only old archives with this archive prefix.
+success_dir="${TMPD}/success"
+success_fixture="${success_dir}/fixture"
+success_remote="${success_dir}/remote"
+success_local_tmp="${success_dir}/local-tmp"
+mkdir -p "$success_remote" "$success_local_tmp"
+create_source_fixture "$success_fixture"
+for day in 1 2 3 4 5; do
+    touch -t "2026070${day}1200" "${success_remote}/k3s-control-plane-config_2026-07-0${day}_12-00-00.zip"
+done
+printf 'legacy backup\n' > "${success_remote}/k8s-backup_2026-07-01.7z"
+printf 'other zip\n' > "${success_remote}/other-prefix_2026-07-01.zip"
+newline_archive="${success_remote}/k3s-control-plane-config_2026-06-01_12-00-00.zip"$'\nunrelated.zip'
+touch -t "202606011200" "$newline_archive"
+: > "$newline_archive"
+touch -t "202606011200" "$newline_archive"
+tab_archive="${success_remote}/k3s-control-plane-config_2026-05-01_12-00-00.zip"$'\tlegacy.zip'
+: > "$tab_archive"
+touch -t "202605011200" "$tab_archive"
+: > "${success_remote}/unrelated.zip"
+: > "${TMPD}/curl_trace.log"
+: > "${TMPD}/unzip_trace.log"
+: > "${TMPD}/zip_invocations.log"
+: > "${TMPD}/scp_trace.log"
+: > "${TMPD}/ssh_trace.log"
+
+set +e
+run_backup "$success_fixture" "$success_remote" "$success_local_tmp" "${success_dir}/run.log"
+success_exit_code=$?
+set -e
+
+fixture_relative="${success_fixture#/}"
+assert "successful backup exits with code 0" is_zero "$success_exit_code"
+assert "successful backup publishes the final zip archive" path_exists "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "successful backup validates its archive with unzip" file_contains "${TMPD}/unzip_trace.log" "k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "archive contains the k3s configuration path" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/rancher/k3s/config.yaml"
+assert "archive contains a service configuration file" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/etc/systemd/system/k3s.service"
+assert "archive contains control-plane credentials" file_contains "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "${fixture_relative}/var/lib/rancher/k3s/server/cred/passwd"
+assert "archive excludes server database content" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "server/db/state.db"
+assert "archive excludes kine socket" file_lacks "${success_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip" "kine.sock"
+assert "retention leaves exactly four newest matching zip archives" matching_archive_count_is "$success_remote" 4
+assert "retention removes the two oldest matching zip archives" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-01_12-00-00.zip"
+assert "retention removes the next-oldest matching zip archive" path_does_not_exist "${success_remote}/k3s-control-plane-config_2026-07-02_12-00-00.zip"
+assert "retention removes matching old archives with newlines in their filenames" path_does_not_exist "$newline_archive"
+assert "retention removes matching old archives with tabs in their filenames" path_does_not_exist "$tab_archive"
+assert "retention keeps the standalone unrelated.zip sentinel" path_exists "${success_remote}/unrelated.zip"
+assert "retention keeps unrelated legacy .7z backups" path_exists "${success_remote}/k8s-backup_2026-07-01.7z"
+assert "retention keeps unrelated zip backups" path_exists "${success_remote}/other-prefix_2026-07-01.zip"
+assert "local temporary archive state is removed" directory_is_empty "$success_local_tmp"
+assert "successful backup sends a v2 backup notification" file_contains "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "successful backup sends its archive size in bytes" file_contains "${TMPD}/curl_trace.log" '"sizeBytes":'
+assert "successful backup sends its elapsed duration" file_contains "${TMPD}/curl_trace.log" '"durationSeconds":'
+assert "successful backup sends the v2 notification source" file_contains "${TMPD}/curl_trace.log" '"source": "K3s control-plane config backup"'
+
+# A configured source path ending in server/db/.. must be rejected before zip is
+# invoked, rather than recursively archiving server/db and its sibling socket.
+source_scope_dir="${TMPD}/source-scope"
+source_scope_fixture="${source_scope_dir}/fixture"
+source_scope_remote="${source_scope_dir}/remote"
+source_scope_local_tmp="${source_scope_dir}/local-tmp"
+source_scope_archive="${source_scope_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+mkdir -p "$source_scope_remote" "$source_scope_local_tmp"
+create_source_fixture "$source_scope_fixture"
+: > "${TMPD}/zip_invocations.log"
+
+set +e
+run_backup \
+    "$source_scope_fixture" \
+    "$source_scope_remote" \
+    "$source_scope_local_tmp" \
+    "${source_scope_dir}/run.log" \
+    "K3S_CONFIG_DIR=${source_scope_fixture}/var/lib/rancher/k3s/server/db/.."
+source_scope_exit_code=$?
+set -e
+
+assert "unsafe source override exits non-zero" is_nonzero "$source_scope_exit_code"
+assert "unsafe source override reports the source-scope error" file_contains "${source_scope_dir}/run.log" "Configured source path is not safe to archive"
+assert "unsafe source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log"
+assert "unsafe source override archive excludes server database content" file_lacks "$source_scope_archive" "state.db"
+assert "unsafe source override archive excludes kine socket" file_lacks "$source_scope_archive" "kine.sock"
+assert "unsafe source override does not publish an archive" directory_is_empty "$source_scope_remote"
+assert "unsafe source override removes local temporary state" directory_is_empty "$source_scope_local_tmp"
+
+# The source path and the server directory can be overridden independently;
+# protected database paths must still be derived from the configured source.
+source_scope_mismatch_dir="${TMPD}/source-scope-mismatch"
+source_scope_mismatch_fixture="${source_scope_mismatch_dir}/fixture"
+source_scope_mismatch_remote="${source_scope_mismatch_dir}/remote"
+source_scope_mismatch_local_tmp="${source_scope_mismatch_dir}/local-tmp"
+source_scope_mismatch_unrelated_server="${source_scope_mismatch_fixture}/unrelated-server"
+source_scope_mismatch_archive="${source_scope_mismatch_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+mkdir -p "$source_scope_mismatch_remote" "$source_scope_mismatch_local_tmp"
+create_source_fixture "$source_scope_mismatch_fixture"
+mkdir -p "$source_scope_mismatch_unrelated_server"
+: > "${TMPD}/zip_invocations.log"
+
+set +e
+run_backup \
+    "$source_scope_mismatch_fixture" \
+    "$source_scope_mismatch_remote" \
+    "$source_scope_mismatch_local_tmp" \
+    "${source_scope_mismatch_dir}/run.log" \
+    "K3S_CONFIG_DIR=${source_scope_mismatch_fixture}/var/lib/rancher/k3s/server/db/.." \
+    "K3S_SERVER_DIR=${source_scope_mismatch_unrelated_server}"
+source_scope_mismatch_exit_code=$?
+set -e
+
+assert "mismatched source override exits non-zero" is_nonzero "$source_scope_mismatch_exit_code"
+assert "mismatched source override reports the source-scope error" file_contains "${source_scope_mismatch_dir}/run.log" "Configured source path is not safe to archive"
+assert "mismatched source override is rejected before zip runs" file_is_empty "${TMPD}/zip_invocations.log"
+assert "mismatched source override archive excludes server database content" file_lacks "$source_scope_mismatch_archive" "state.db"
+assert "mismatched source override archive excludes kine socket" file_lacks "$source_scope_mismatch_archive" "kine.sock"
+assert "mismatched source override does not publish an archive" path_does_not_exist "$source_scope_mismatch_archive"
+assert "mismatched source override leaves the remote directory empty" directory_is_empty "$source_scope_mismatch_remote"
+assert "mismatched source override removes local temporary state" directory_is_empty "$source_scope_mismatch_local_tmp"
+
+# Upload failure: the partial remote temporary file is removed, no final archive
+# is published, local temporary data is removed, and one safe error notification
+# is attempted.
+failure_dir="${TMPD}/upload-failure"
+failure_fixture="${failure_dir}/fixture"
+failure_remote="${failure_dir}/remote"
+failure_local_tmp="${failure_dir}/local-tmp"
+mkdir -p "$failure_remote" "$failure_local_tmp"
+create_source_fixture "$failure_fixture"
+: > "${TMPD}/fail_scp"
+: > "${TMPD}/curl_trace.log"
+
+set +e
+run_backup "$failure_fixture" "$failure_remote" "$failure_local_tmp" "${failure_dir}/run.log"
+failure_exit_code=$?
+set -e
+rm -f -- "${TMPD}/fail_scp"
+
+assert "upload failure exits non-zero" is_nonzero "$failure_exit_code"
+assert "upload failure does not publish a final archive" path_does_not_exist "${failure_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+assert "upload failure removes the remote temporary archive" directory_is_empty "$failure_remote"
+assert "upload failure removes local temporary state" directory_is_empty "$failure_local_tmp"
+assert "upload failure attempts exactly one v2 error notification" file_line_count_is "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/error" 1
+assert "upload failure marks the notification as critical" file_contains "${TMPD}/curl_trace.log" '"severity": "critical"'
+assert "upload failure includes the v2 error code" file_contains "${TMPD}/curl_trace.log" '"errorCode": "K3S_CONTROL_PLANE_BACKUP_FAILED"'
+assert "upload failure does not send a v2 backup notification" file_lacks "${TMPD}/curl_trace.log" "http://example.invalid/api/v2/notifications/backup"
+assert "failure log does not disclose fixture source paths" file_lacks "${failure_dir}/run.log" "$failure_fixture"
+
+# A notification error is non-fatal after a completed backup.
+notify_dir="${TMPD}/notify-failure"
+notify_fixture="${notify_dir}/fixture"
+notify_remote="${notify_dir}/remote"
+notify_local_tmp="${notify_dir}/local-tmp"
+mkdir -p "$notify_remote" "$notify_local_tmp"
+create_source_fixture "$notify_fixture"
+: > "${TMPD}/fail_curl"
+
+set +e
+run_backup "$notify_fixture" "$notify_remote" "$notify_local_tmp" "${notify_dir}/run.log" "CLEANUP_KEEP_COUNT=0"
+notify_exit_code=$?
+set -e
+rm -f -- "${TMPD}/fail_curl"
+
+assert "failed success notification does not fail a completed backup" is_zero "$notify_exit_code"
+assert "completed backup remains published after notification failure" path_exists "${notify_remote}/k3s-control-plane-config_2026-07-21_12-00-00.zip"
+
+printf 'TESTS PASSED: %s / %s\n' "$tests_passed" "$tests_total"
+if (( tests_failed > 0 )); then
+    exit 1
+fi