From af7106546ea4dec73e78c5e50765955318a0f45f Mon Sep 17 00:00:00 2001
From: Jose Henrique <jose.henrique.ivan@gmail.com>
Date: Thu, 17 Sep 2026 09:32:45 -0300
Subject: [PATCH] Configure bounded monitoring storage on nexus sandbox

---
 apps/monitoring/loki.yaml       | 21 +++++++++++++++++----
 apps/monitoring/prometheus.yaml | 11 ++++++++---
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/apps/monitoring/loki.yaml b/apps/monitoring/loki.yaml
index 952a937..ad0eb34 100644
--- a/apps/monitoring/loki.yaml
+++ b/apps/monitoring/loki.yaml
@@ -5,6 +5,8 @@ metadata:
   namespace: monitoring
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: loki
@@ -36,8 +38,9 @@ spec:
           configMap:
             name: loki-config
         - name: loki-storage
-          emptyDir:
-            medium: Memory
+          nfs:
+            server: nexus.haven
+            path: /export/sandbox/loki
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -79,7 +82,12 @@ data:
 
     limits_config:
       allow_structured_metadata: true
-      retention_period: 0
+      retention_period: 72h
+      max_query_lookback: 72h
+      # Global per-tenant cap: <= 0.1 MB/s sustained keeps this sandbox disk bounded.
+      ingestion_rate_strategy: global
+      ingestion_rate_mb: 0.1
+      ingestion_burst_size_mb: 0.5
 
     ingester:
       lifecycler:
@@ -92,7 +100,12 @@ data:
       chunk_target_size: 1536000
 
     compactor:
-      retention_enabled: false
+      working_directory: /tmp/loki/compactor
+      compaction_interval: 10m
+      retention_enabled: true
+      retention_delete_delay: 2h
+      retention_delete_worker_count: 10
+      delete_request_store: filesystem
 ---
 apiVersion: v1
 kind: Service
diff --git a/apps/monitoring/prometheus.yaml b/apps/monitoring/prometheus.yaml
index 17d76b4..5a470ff 100644
--- a/apps/monitoring/prometheus.yaml
+++ b/apps/monitoring/prometheus.yaml
@@ -7,6 +7,8 @@ metadata:
     app: prometheus
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: prometheus
@@ -22,7 +24,9 @@ spec:
           args:
             - "--config.file=/etc/prometheus/prometheus.yml"
             - "--storage.tsdb.path=/prometheus"
-            - "--storage.tsdb.retention.time=1d"
+            - "--storage.tsdb.retention.time=2d"
+            # Keep a generous buffer on the 117 GiB sandbox filesystem; WAL may exceed this briefly.
+            - "--storage.tsdb.retention.size=8GB"
             - "--web.enable-lifecycle"
           ports:
             - containerPort: 9090
@@ -44,8 +48,9 @@ spec:
           persistentVolumeClaim:
             claimName: prometheus-pvc
         - name: prometheus-storage
-          persistentVolumeClaim:
-            claimName: prometheus-storage-pvc
+          nfs:
+            server: nexus.haven
+            path: /export/sandbox/prometheus
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim