Configure bounded monitoring storage on nexus sandbox

apps/monitoring/loki.yaml

@@ -5,6 +5,8 @@ metadata:
   namespace: monitoring
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: loki
@@ -36,8 +38,9 @@ spec:
           configMap:
             name: loki-config
         - name: loki-storage
-          emptyDir:
-            medium: Memory
+          nfs:
+            server: nexus.haven
+            path: /export/sandbox/loki
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -79,7 +82,12 @@ data:
 
     limits_config:
       allow_structured_metadata: true
-      retention_period: 0
+      retention_period: 72h
+      max_query_lookback: 72h
+      # Global per-tenant cap: <= 0.1 MB/s sustained keeps this sandbox disk bounded.
+      ingestion_rate_strategy: global
+      ingestion_rate_mb: 0.1
+      ingestion_burst_size_mb: 0.5
 
     ingester:
       lifecycler:
@@ -92,7 +100,12 @@ data:
       chunk_target_size: 1536000
 
     compactor:
-      retention_enabled: false
+      working_directory: /tmp/loki/compactor
+      compaction_interval: 10m
+      retention_enabled: true
+      retention_delete_delay: 2h
+      retention_delete_worker_count: 10
+      delete_request_store: filesystem
 ---
 apiVersion: v1
 kind: Service

apps/monitoring/prometheus.yaml

@@ -7,6 +7,8 @@ metadata:
     app: prometheus
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: prometheus
@@ -22,7 +24,9 @@ spec:
           args:
             - "--config.file=/etc/prometheus/prometheus.yml"
             - "--storage.tsdb.path=/prometheus"
-            - "--storage.tsdb.retention.time=1d"
+            - "--storage.tsdb.retention.time=2d"
+            # Keep a generous buffer on the 117 GiB sandbox filesystem; WAL may exceed this briefly.
+            - "--storage.tsdb.retention.size=8GB"
             - "--web.enable-lifecycle"
           ports:
             - containerPort: 9090
@@ -44,8 +48,9 @@ spec:
           persistentVolumeClaim:
             claimName: prometheus-pvc
         - name: prometheus-storage
-          persistentVolumeClaim:
-            claimName: prometheus-storage-pvc
+          nfs:
+            server: nexus.haven
+            path: /export/sandbox/prometheus
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim