Merge branch 'main' of git.ivanch.me:ivanch/opencand.infra

.gitignore

@@ -1,3 +0,0 @@
-**/etl-data
-**/fotos_cand
-*.zip

backup-db.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# OpenCand Database Backup Script
+# This script creates a backup of the PostgreSQL database (excluding materialized views)
+
+set -e  # Exit on any error
+
+# Configuration
+DB_CONTAINER="opencand_db"
+DB_NAME="opencand"
+DB_USER="root"
+BACKUP_FILE="backup_$(date +%Y%m%d_%H%M%S).sql"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+echo -e "${YELLOW}💾 OpenCand Database Backup Script${NC}"
+echo "=================================="
+
+# Check if Docker is running
+if ! docker info >/dev/null 2>&1; then
+    echo -e "${RED}❌ Error: Docker is not running or not accessible${NC}"
+    exit 1
+fi
+
+# Check if the database container is running
+if ! docker ps --format "table {{.Names}}" | grep -q "^${DB_CONTAINER}$"; then
+    echo -e "${RED}❌ Error: Database container '${DB_CONTAINER}' is not running${NC}"
+    echo "Please start the services with: docker-compose up -d"
+    exit 1
+fi
+
+echo -e "${YELLOW}📋 Creating backup (excluding materialized views)...${NC}"
+
+# Get list of materialized views
+MATVIEWS=$(docker exec $DB_CONTAINER psql -U $DB_USER -d $DB_NAME -Atc "SELECT schemaname || '.' || matviewname FROM pg_matviews;")
+
+# Build exclude flags for pg_dump
+EXCLUDE_FLAGS=""
+for mv in $MATVIEWS; do
+    EXCLUDE_FLAGS+=" --exclude-table=$mv"
+done
+
+# Run pg_dump excluding materialized views
+if docker exec $DB_CONTAINER pg_dump -U $DB_USER -d $DB_NAME $EXCLUDE_FLAGS > "$BACKUP_FILE"; then
+    echo -e "${GREEN}✅ Backup created: $BACKUP_FILE${NC}"
+else
+    echo -e "${RED}❌ Error: Failed to create backup${NC}"
+    exit 1
+fi
+
+echo -e "${GREEN}🎉 Backup completed successfully!${NC}"
+echo "=================================="

db/db.sql

@@ -1,136 +0,0 @@
-DROP TABLE IF EXISTS bem_candidato CASCADE;
-DROP TABLE IF EXISTS candidato_mapping CASCADE;
-DROP TABLE IF EXISTS rede_social CASCADE;
-DROP TABLE IF EXISTS candidato CASCADE;
-DROP TABLE IF EXISTS partido CASCADE;
-DROP TABLE IF EXISTS despesas_candidato CASCADE;
-DROP TABLE IF EXISTS receitas_candidato CASCADE;
-
-CREATE TABLE candidato (
-    idcandidato UUID NOT NULL PRIMARY KEY,
-    cpf VARCHAR(11),
-    nome VARCHAR(255) NOT NULL,
-    apelido VARCHAR(255),
-    datanascimento TIMESTAMPTZ,
-    email TEXT,
-    sexo CHAR(15),
-    estadocivil VARCHAR(50),
-    escolaridade VARCHAR(50),
-    ocupacao VARCHAR(150)
-);
-CREATE INDEX idx_candidato_nome ON candidato (nome);
-CREATE INDEX idx_candidato_apelido ON candidato (apelido);
-
--- Each candidato (idcandidato, cpf, nome) will be mapped to a (sqcandidato, ano, tipo_eleicao, sg_uf, cargo, resultado) 
-CREATE TABLE candidato_mapping (
-    idcandidato UUID NOT NULL,
-    cpf VARCHAR(11),
-    nome VARCHAR(255) NOT NULL,
-    apelido VARCHAR(255),
-    sqcandidato VARCHAR(50) NOT NULL,
-    turno VARCHAR(2) NOT NULL,
-    ano INT NOT NULL,
-    tipoeleicao VARCHAR(50),
-    siglauf VARCHAR(2),
-    nomeue VARCHAR(100),
-    cargo VARCHAR(50),
-    sgpartido VARCHAR(50),
-    nrcandidato VARCHAR(20),
-    resultado VARCHAR(50),
-    CONSTRAINT pk_candidato_mapping PRIMARY KEY (idcandidato, ano, siglauf, nomeue, cargo, nrcandidato, resultado),
-    CONSTRAINT fk_candidato_mapping_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
-);
-CREATE INDEX idx_candidato_mapping_cpf ON candidato_mapping (cpf);
-CREATE INDEX idx_candidato_mapping_nome ON candidato_mapping (nome);
-CREATE INDEX idx_candidato_mapping_apelido ON candidato_mapping (apelido);
-CREATE INDEX idx_candidato_mapping_ano ON candidato_mapping (ano);
-CREATE INDEX idx_candidato_mapping_sqcandidato ON candidato_mapping (sqcandidato);
-
----- Table for storing assets of candidates
-CREATE TABLE bem_candidato (
-    idcandidato UUID NOT NULL,
-    ano INT NOT NULL,
-    ordembem INT,
-    tipobem VARCHAR(150),
-    descricao VARCHAR(500),
-    valor NUMERIC(20, 2),
-    CONSTRAINT fk_bem_candidato_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
-);
-ALTER TABLE bem_candidato ADD CONSTRAINT pk_bem_candidato PRIMARY KEY (idcandidato, ano, ordembem);
-CREATE INDEX idx_bem_candidato_idcandidato ON bem_candidato (idcandidato);
-CREATE INDEX idx_bem_candidato_valor ON bem_candidato (valor);
-
----- Table for storing social media links of candidates
-CREATE TABLE rede_social (
-    idcandidato UUID NOT NULL,
-    rede VARCHAR(50) NOT NULL,
-    siglauf VARCHAR(2),
-    ano INT NOT NULL,
-    link TEXT NOT NULL,
-    CONSTRAINT pk_rede_social PRIMARY KEY (idcandidato, rede, siglauf, ano),
-    CONSTRAINT fk_rede_social_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
-);
-CREATE INDEX idx_rede_social_idcandidato ON rede_social (idcandidato);
-
----- Table for storing party information
-CREATE TABLE partido (
-    sigla VARCHAR(50) NOT NULL PRIMARY KEY,
-    nome VARCHAR(255) NOT NULL,
-    numero INT NOT NULL
-);
-CREATE INDEX idx_partido_nome ON partido (nome);
-CREATE INDEX idx_partido_numero ON partido (numero);
-
----- Tables for storing despesas e receitas of candidacies
-CREATE TABLE despesas_candidato (
-    idreceita UUID NOT NULL DEFAULT gen_random_uuid(),
-    idcandidato UUID NOT NULL,
-    ano INT NOT NULL,
-    turno VARCHAR(2) NOT NULL,
-    sqcandidato VARCHAR(50) NOT NULL,
-    sgpartido VARCHAR(50) NOT NULL,
-    tipofornecedor VARCHAR(150),
-    cnpjfornecedor VARCHAR(14),
-    cpffornecedor VARCHAR(11),
-    nomefornecedor VARCHAR(255),
-    nomefornecedorrfb VARCHAR(255),
-    municipiofornecedor VARCHAR(100),
-    tipodocumento VARCHAR(50),
-    datadespesa TIMESTAMPTZ,
-    descricao TEXT,
-    origemdespesa TEXT,
-    valor NUMERIC(20, 2),
-    CONSTRAINT pk_despesas_candidato PRIMARY KEY (idreceita),
-    CONSTRAINT fk_despesas_candidato_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
-);
-CREATE INDEX idx_despesas_candidato_idcandidato ON despesas_candidato (idcandidato);
-CREATE INDEX idx_despesas_candidato_ano ON despesas_candidato (ano);
-CREATE INDEX idx_despesas_candidato_sqcandidato ON despesas_candidato (sqcandidato);
-CREATE INDEX idx_despesas_candidato_sgpartido ON despesas_candidato (sgpartido);
-
-CREATE TABLE receitas_candidato (
-    idreceita UUID NOT NULL DEFAULT gen_random_uuid(),
-    idcandidato UUID NOT NULL,
-    ano INT NOT NULL,
-    turno VARCHAR(2) NOT NULL,
-    sqcandidato VARCHAR(50) NOT NULL,
-    sgpartido VARCHAR(50) NOT NULL,
-    fontereceita VARCHAR(150),
-    origemreceita VARCHAR(250),
-    naturezareceita VARCHAR(250),
-    especiereceita VARCHAR(250),
-    cnpjdoador VARCHAR(14),
-    cpfdoador VARCHAR(11),
-    nomedoador VARCHAR(255),
-    nomedoadorrfb VARCHAR(255),
-    municipiodoador VARCHAR(100),
-    datareceita TIMESTAMPTZ,
-    descricao TEXT,
-    valor NUMERIC(20, 2),
-    CONSTRAINT pk_receitas_candidato PRIMARY KEY (idreceita),
-    CONSTRAINT fk_receitas_candidato_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
-);
-CREATE INDEX idx_receitas_candidato_idcandidato ON receitas_candidato (idcandidato);
-CREATE INDEX idx_receitas_candidato_ano ON receitas_candidato (ano);
-CREATE INDEX idx_receitas_candidato_sqcandidato ON receitas_candidato (sqcandidato);
-CREATE INDEX idx_receitas_candidato_sgpartido ON receitas_candidato (sgpartido);

deploy-db.ps1

@@ -1,91 +0,0 @@
-# OpenCand Database Deployment Script (PowerShell)
-# This script deploys the database schema changes to the PostgreSQL container
-
-param(
-    [string]$ContainerName = "opencand_db",
-    [string]$DatabaseName = "opencand", 
-    [string]$DatabaseUser = "root",
-    [string]$SqlFile = ".\db\db.sql"
-)
-
-# Configuration
-$ErrorActionPreference = "Stop"
-
-Write-Host "🚀 OpenCand Database Deployment Script" -ForegroundColor Yellow
-Write-Host "==================================" -ForegroundColor Yellow
-
-try {
-    # Check if Docker is running
-    Write-Host "📋 Pre-deployment checks:" -ForegroundColor Yellow
-    docker info | Out-Null
-    Write-Host "✅ Docker is running" -ForegroundColor Green
-
-    # Check if the database container is running
-    $runningContainers = docker ps --format "{{.Names}}"
-    if ($runningContainers -notcontains $ContainerName) {
-        throw "Database container '$ContainerName' is not running. Please start with: docker-compose up -d"
-    }
-    Write-Host "✅ Database container is running" -ForegroundColor Green
-
-    # Check if SQL file exists
-    if (-not (Test-Path $SqlFile)) {
-        throw "SQL file '$SqlFile' not found"
-    }
-    Write-Host "✅ SQL file exists" -ForegroundColor Green
-    Write-Host ""
-
-    # Wait for database to be ready
-    Write-Host "⏳ Waiting for database to be ready..." -ForegroundColor Yellow
-    $timeout = 30
-    $counter = 0
-    do {
-        $ready = $false
-        try {
-            docker exec $ContainerName pg_isready -U $DatabaseUser -d $DatabaseName | Out-Null
-            $ready = $true
-        }
-        catch {
-            Start-Sleep -Seconds 1
-            $counter++
-            if ($counter -ge $timeout) {
-                throw "Database failed to become ready within $timeout seconds"
-            }
-        }
-    } while (-not $ready)
-    
-    Write-Host "✅ Database is ready" -ForegroundColor Green
-    Write-Host ""
-
-    # Create backup before deployment
-    Write-Host "💾 Creating database backup..." -ForegroundColor Yellow
-    $backupFile = "backup_$(Get-Date -Format 'yyyyMMdd_HHmmss').sql"
-    docker exec $ContainerName pg_dump -U $DatabaseUser -d $DatabaseName | Out-File -FilePath $backupFile -Encoding UTF8
-    Write-Host "✅ Backup created: $backupFile" -ForegroundColor Green
-    Write-Host ""
-
-    # Deploy the SQL file
-    Write-Host "🔧 Deploying database changes..." -ForegroundColor Yellow
-    Write-Host "Executing: $SqlFile"
-
-    # Execute the SQL file
-    Get-Content $SqlFile | docker exec -i $ContainerName psql -U $DatabaseUser -d $DatabaseName
-    
-    Write-Host "✅ Database deployment completed successfully!" -ForegroundColor Green
-    Write-Host ""
-    
-    # Show table information
-    Write-Host "📊 Current database tables:" -ForegroundColor Yellow
-    docker exec $ContainerName psql -U $DatabaseUser -d $DatabaseName -c "\dt"
-    
-    Write-Host ""
-    Write-Host "🎉 Deployment completed successfully!" -ForegroundColor Green
-    Write-Host "==================================" -ForegroundColor Green
-
-} catch {
-    Write-Host "❌ Error: $($_.Exception.Message)" -ForegroundColor Red
-    if (Test-Path $backupFile) {
-        Write-Host "💡 You can restore from backup using:" -ForegroundColor Yellow
-        Write-Host "Get-Content $backupFile | docker exec -i $ContainerName psql -U $DatabaseUser -d $DatabaseName" -ForegroundColor Yellow
-    }
-    exit 1
-}

deploy-db.sh

@@ -9,7 +9,10 @@ set -e  # Exit on any error
 DB_CONTAINER="opencand_db"
 DB_NAME="opencand"
 DB_USER="root"
-SQL_FILE="./db/db.sql"
+SQL_URL="https://git.ivanch.me/ivanch/opencand/raw/branch/main/db/db.sql"
+SQL_FILE="./db.sql"
+MV_URL="https://git.ivanch.me/ivanch/opencand/raw/branch/main/db/mv.sql"
+MV_FILE="./mv.sql"
 
 # Colors for output
 RED='\033[0;31m'
@@ -33,16 +36,54 @@ if ! docker ps --format "table {{.Names}}" | grep -q "^${DB_CONTAINER}$"; then
     exit 1
 fi
 
-# Check if SQL file exists
+# Download the SQL files
+echo -e "${YELLOW}📥 Downloading SQL files...${NC}"
+if command -v curl &> /dev/null; then
+    if curl -L -o "$SQL_FILE" "$SQL_URL"; then
+        echo "✅ db.sql downloaded successfully"
+    else
+        echo -e "${RED}❌ Error: Failed to download db.sql from ${SQL_URL}${NC}"
+        exit 1
+    fi
+    if curl -L -o "$MV_FILE" "$MV_URL"; then
+        echo "✅ mv.sql downloaded successfully"
+    else
+        echo -e "${RED}❌ Error: Failed to download mv.sql from ${MV_URL}${NC}"
+        exit 1
+    fi
+elif command -v wget &> /dev/null; then
+    if wget -O "$SQL_FILE" "$SQL_URL"; then
+        echo "✅ db.sql downloaded successfully"
+    else
+        echo -e "${RED}❌ Error: Failed to download db.sql from ${SQL_URL}${NC}"
+        exit 1
+    fi
+    if wget -O "$MV_FILE" "$MV_URL"; then
+        echo "✅ mv.sql downloaded successfully"
+    else
+        echo -e "${RED}❌ Error: Failed to download mv.sql from ${MV_URL}${NC}"
+        exit 1
+    fi
+else
+    echo -e "${RED}❌ Error: Neither curl nor wget is available for downloading${NC}"
+    exit 1
+fi
+
+# Check if SQL files exist (after download)
 if [ ! -f "$SQL_FILE" ]; then
-    echo -e "${RED}❌ Error: SQL file '${SQL_FILE}' not found${NC}"
+    echo -e "${RED}❌ Error: SQL file '${SQL_FILE}' not found after download${NC}"
+    exit 1
+fi
+if [ ! -f "$MV_FILE" ]; then
+    echo -e "${RED}❌ Error: SQL file '${MV_FILE}' not found after download${NC}"
     exit 1
 fi
 
 echo -e "${YELLOW}📋 Pre-deployment checks:${NC}"
 echo "✅ Docker is running"
 echo "✅ Database container is running"
-echo "✅ SQL file exists"
+echo "✅ db.sql downloaded"
+echo "✅ mv.sql downloaded"
 echo ""
 
 # Wait for database to be ready
@@ -60,32 +101,49 @@ done
 echo "✅ Database is ready"
 echo ""
 
-# Create backup before deployment
-echo -e "${YELLOW}💾 Creating database backup...${NC}"
-BACKUP_FILE="backup_$(date +%Y%m%d_%H%M%S).sql"
-docker exec $DB_CONTAINER pg_dump -U $DB_USER -d $DB_NAME > $BACKUP_FILE
-echo "✅ Backup created: $BACKUP_FILE"
-echo ""
 
-# Deploy the SQL file
+# Deploy the db.sql file
-echo -e "${YELLOW}🔧 Deploying database changes...${NC}"
+echo -e "${YELLOW}🔧 Deploying database changes (db.sql)...${NC}"
 echo "Executing: $SQL_FILE"
 
 if docker exec -i $DB_CONTAINER psql -U $DB_USER -d $DB_NAME < $SQL_FILE; then
-    echo -e "${GREEN}✅ Database deployment completed successfully!${NC}"
+    echo -e "${GREEN}✅ db.sql deployment completed successfully!${NC}"
-    echo ""
-    
-    # Show table information
-    echo -e "${YELLOW}📊 Current database tables:${NC}"
-    docker exec $DB_CONTAINER psql -U $DB_USER -d $DB_NAME -c "\dt"
-    
 else
-    echo -e "${RED}❌ Error: Database deployment failed${NC}"
+    echo -e "${RED}❌ Error: db.sql deployment failed${NC}"
     echo -e "${YELLOW}💡 You can restore from backup using:${NC}"
     echo "docker exec -i $DB_CONTAINER psql -U $DB_USER -d $DB_NAME < $BACKUP_FILE"
+    # Clean up before exit
+    rm -f "$SQL_FILE" "$MV_FILE"
     exit 1
 fi
 
+# Deploy the mv.sql file
+echo -e "${YELLOW}<EFBFBD> Deploying database changes (mv.sql)...${NC}"
+echo "Executing: $MV_FILE"
+
+if docker exec -i $DB_CONTAINER psql -U $DB_USER -d $DB_NAME < $MV_FILE; then
+    echo -e "${GREEN}✅ mv.sql deployment completed successfully!${NC}"
+else
+    echo -e "${RED}❌ Error: mv.sql deployment failed${NC}"
+    echo -e "${YELLOW}💡 You can restore from backup using:${NC}"
+    echo "docker exec -i $DB_CONTAINER psql -U $DB_USER -d $DB_NAME < $BACKUP_FILE"
+    # Clean up before exit
+    rm -f "$SQL_FILE" "$MV_FILE"
+    exit 1
+fi
+
+
+# Show table and materialized view information
+echo -e "${YELLOW}📊 Current database tables:${NC}"
+docker exec $DB_CONTAINER psql -U $DB_USER -d $DB_NAME -c "\dt"
+echo -e "${YELLOW}📄 Current materialized views:${NC}"
+docker exec $DB_CONTAINER psql -U $DB_USER -d $DB_NAME -c "\dm"
+
+# Clean up downloaded SQL files
+echo -e "${YELLOW}🧹 Cleaning up...${NC}"
+rm -f "$SQL_FILE" "$MV_FILE"
+echo "✅ Temporary SQL files removed"
+
 echo ""
 echo -e "${GREEN}🎉 Deployment completed successfully!${NC}"
 echo "=================================="

docker-compose.yaml

@@ -1,11 +1,77 @@
-services:  # ───────────────────────────────────────────────────────────────────────────
+services:
-  # 1. PostgreSQL Database
   # ───────────────────────────────────────────────────────────────────────────
+  # 1. .NET API
+  # ───────────────────────────────────────────────────────────────────────────
+  api:
+    container_name: opencand_api
+    restart: unless-stopped
+    image: git.ivanch.me/ivanch/opencand.api:latest
+    ports:
+      - "5100:8080"
+    environment:
+      ASPNETCORE_ENVIRONMENT: "Production"
+      Logging__LogLevel__Default: "Information"
+      DatabaseSettings__ConnectionString: "Host=db;Port=5432;Database=opencand;Username=root;Password=root;Pooling=true;Minimum Pool Size=1;Maximum Pool Size=20;Connection Lifetime=300;Command Timeout=30;Application Name=OpenCand.API;Include Error Detail=true"
+      FotosSettings__ApiBasePath: "https://api.opencand.ivanch.me/assets/fotos"
+    volumes:
+      - ./fotos_cand:/app/fotos_cand
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+        labels: "service=api"
+    labels:
+      - "promtail.enable=true"
+      - "promtail.job=opencand-api"
+
+  # ───────────────────────────────────────────────────────────────────────────
+  # 2. React + Vite Front‐end
+  # ───────────────────────────────────────────────────────────────────────────
+  frontend:
+    container_name: opencand_frontend
+    restart: unless-stopped
+    image: git.ivanch.me/ivanch/opencand.ui:latest
+    ports:
+      - "5110:80"
+    depends_on:
+      - api
+  # ───────────────────────────────────────────────────────────────────────────
+  # 3. ETL (Optional: runs once at startup)
+  #
+  #    If you want the ETL to run on every compose up, give it restart: "no" or 
+  #    some other policy. It will run, then exit.
+  #
+  #    If you instead prefer to run ETL manually or via host cron, you can omit 
+  #    this service and just `docker run myorg/etl:latest ...` on demand.
+  # ───────────────────────────────────────────────────────────────────────────
+  etl:
+    image: git.ivanch.me/ivanch/opencand.etl:latest
+    container_name: opencand_etl
+    restart: "no"
+    environment:
+      DatabaseSettings__ConnectionString: "Host=db;Port=5432;Database=opencand;Username=root;Password=root"
+      ParserSettings_CandidatoCSVThreads: "40"
+      BasePath: "etl-data"
+    volumes:
+      - ./etl-data:/app/etl-data
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+        labels: "service=etl"
+    labels:
+      - "promtail.enable=true"
+      - "promtail.job=opencand-etl"
+
   db:
     image: postgres:14-alpine
     container_name: opencand_db
+    profiles: ["infra"]
     restart: unless-stopped
     hostname: db
+    shm_size: 4g
     environment:
       POSTGRES_USER: root
       POSTGRES_PASSWORD: root
@@ -27,85 +93,13 @@ services:  # ──────────────────────
       - "promtail.enable=true"
       - "promtail.job=opencand-db"
   # ───────────────────────────────────────────────────────────────────────────
-  # 2. .NET API
-  # ───────────────────────────────────────────────────────────────────────────
-  api:
-    container_name: opencand_api
-    restart: unless-stopped
-    image: git.ivanch.me/ivanch/opencand.api:latest
-    ports:
-      - "5100:8080" 
-    depends_on:
-      db:
-        condition: service_healthy
-    environment:
-      ASPNETCORE_ENVIRONMENT: "Production"
-      DatabaseSettings__ConnectionString: "Host=db;Port=5432;Database=opencand;Username=root;Password=root"
-      FotosSettings__ApiBasePath: "https://api.opencand.ivanch.me/assets/fotos"
-    volumes:
-      - ./fotos_cand:/app/fotos_cand
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "10m"
-        max-file: "3"
-        labels: "service=api"
-    labels:
-      - "promtail.enable=true"
-      - "promtail.job=opencand-api"
-
-  # ───────────────────────────────────────────────────────────────────────────
-  # 3. React + Vite Front‐end
-  # ───────────────────────────────────────────────────────────────────────────
-  frontend:
-    container_name: opencand_frontend
-    restart: unless-stopped
-    image: git.ivanch.me/ivanch/opencand.ui:latest
-    ports:
-      - "5110:80"
-    depends_on:
-      - api
-  # ───────────────────────────────────────────────────────────────────────────
-  # 4. ETL (Optional: runs once at startup)
-  #
-  #    If you want the ETL to run on every compose up, give it restart: "no" or 
-  #    some other policy. It will run, then exit.
-  #
-  #    If you instead prefer to run ETL manually or via host cron, you can omit 
-  #    this service and just `docker run myorg/etl:latest ...` on demand.
-  # ───────────────────────────────────────────────────────────────────────────
-  etl:
-    image: git.ivanch.me/ivanch/opencand.etl:latest
-    container_name: opencand_etl
-    restart: "no"
-    depends_on:
-      db:
-        condition: service_healthy
-      loki:
-        condition: service_started
-    environment:
-      DatabaseSettings__ConnectionString: "Host=db;Port=5432;Database=opencand;Username=root;Password=root"
-      ParserSettings_CandidatoCSVThreads: "40"
-      BasePath: "etl-data"
-    volumes:
-      - ./etl-data:/app/etl-data
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "10m"
-        max-file: "3"
-        labels: "service=etl"
-    labels:
-      - "promtail.enable=true"
-      - "promtail.job=opencand-etl"
-
-  # ───────────────────────────────────────────────────────────────────────────
   # 5. Log Monitoring Stack
   # ───────────────────────────────────────────────────────────────────────────
     # Loki - Log aggregation system
   loki:
     image: grafana/loki:2.9.0
     container_name: opencand_loki
+    profiles: ["infra"]
     restart: unless-stopped
     ports:
       - "127.0.0.1:6110:3100"
@@ -123,6 +117,7 @@ services:  # ──────────────────────
   promtail:
     image: grafana/promtail:2.9.0
     container_name: opencand_promtail
+    profiles: ["infra"]
     restart: unless-stopped
     volumes:
       - ./monitoring/promtail-config.yaml:/etc/promtail/config.yml
@@ -137,6 +132,7 @@ services:  # ──────────────────────
   prometheus:
     image: prom/prometheus:v2.45.0
     container_name: opencand_prometheus
+    profiles: ["infra"]
     restart: unless-stopped
     ports:
       - "127.0.0.1:6090:9090"
@@ -155,6 +151,7 @@ services:  # ──────────────────────
   node-exporter:
     image: prom/node-exporter:v1.6.0
     container_name: opencand_node_exporter
+    profiles: ["infra"]
     restart: unless-stopped
     ports:
       - "127.0.0.1:6100:9100"
@@ -171,6 +168,7 @@ services:  # ──────────────────────
   grafana:
     image: grafana/grafana:10.0.0
     container_name: opencand_grafana
+    profiles: ["infra"]
     restart: unless-stopped
     ports:
       - "6000:3000"

monitoring/README.md

@@ -1,149 +0,0 @@
-# OpenCand Monitoring Setup
-
-This monitoring stack provides comprehensive log aggregation and visualization for the OpenCand project, with special focus on the ETL service.
-
-## Services Overview
-
-### 🔍 **Grafana Loki** (Port 3100)
-- **Purpose**: Log aggregation and storage
-- **Access**: http://localhost:3100
-- **Description**: Collects and stores all container logs in a structured format
-
-### 📊 **Grafana** (Port 3000)
-- **Purpose**: Log visualization and dashboards
-- **Access**: http://localhost:3000
-- **Credentials**: 
-  - Username: `admin`
-  - Password: `admin`
-- **Pre-configured Dashboards**: OpenCand ETL Monitoring dashboard
-
-### 📈 **Prometheus** (Port 9090)
-- **Purpose**: Metrics collection and storage
-- **Access**: http://localhost:9090
-- **Description**: Collects system and application metrics
-
-### 🖥️ **Node Exporter** (Port 9100)
-- **Purpose**: System metrics collection
-- **Access**: http://localhost:9100/metrics
-- **Description**: Provides host system metrics (CPU, memory, disk, etc.)
-
-### 🚚 **Promtail**
-- **Purpose**: Log collection agent
-- **Description**: Automatically discovers and ships Docker container logs to Loki
-
-## Key Features
-
-### ETL-Specific Monitoring
-- ✅ Real-time ETL process logs
-- ✅ Error tracking and alerting capabilities
-- ✅ Performance metrics monitoring
-- ✅ Data processing progress tracking
-
-### Container Log Management
-- ✅ Automatic log rotation (10MB max size, 3 files)
-- ✅ Structured log labeling
-- ✅ Multi-service log aggregation
-
-### Pre-built Dashboards
-- ✅ OpenCand ETL Logs viewer
-- ✅ API logs monitoring
-- ✅ Database logs tracking
-- ✅ Container resource usage
-
-## Getting Started
-
-1. **Start the monitoring stack**:
-   ```bash
-   docker-compose up -d
-   ```
-
-2. **Access Grafana**:
-   - Open http://localhost:3000
-   - Login with admin/admin
-   - Navigate to "Dashboards" → "OpenCand ETL Monitoring"
-
-3. **View ETL Logs in Real-time**:
-   - In Grafana, go to "Explore"
-   - Select "Loki" as datasource
-   - Use query: `{container_name="opencand_etl"}`
-
-4. **Monitor System Metrics**:
-   - Access Prometheus at http://localhost:9090
-   - View system metrics from Node Exporter
-
-## Log Queries Examples
-
-### ETL Service Logs
-```logql
-{container_name="opencand_etl"}
-```
-
-### Error Logs Only
-```logql
-{container_name="opencand_etl"} |= "ERROR"
-```
-
-### API Logs with Filtering
-```logql
-{container_name="opencand_api"} |= "Microsoft.AspNetCore"
-```
-
-### Database Connection Logs
-```logql
-{container_name="opencand_db"} |= "connection"
-```
-
-## Configuration Files
-
-- **Loki**: `./monitoring/loki-config.yaml`
-- **Promtail**: `./monitoring/promtail-config.yaml`
-- **Prometheus**: `./monitoring/prometheus.yml`
-- **Grafana Datasources**: `./monitoring/grafana/provisioning/datasources/`
-- **Grafana Dashboards**: `./monitoring/grafana/provisioning/dashboards/`
-
-## Data Persistence
-
-The following volumes are created for data persistence:
-- `loki-data`: Loki log storage
-- `prometheus-data`: Prometheus metrics storage
-- `grafana-data`: Grafana dashboards and settings
-
-## Troubleshooting
-
-### ETL Logs Not Appearing
-1. Check if ETL container is running: `docker ps`
-2. Verify Promtail is collecting logs: `docker logs opencand_promtail`
-3. Check Loki status: `curl http://localhost:3100/ready`
-
-### Grafana Dashboard Issues
-1. Verify datasources are configured correctly
-2. Check if Loki is accessible from Grafana container
-3. Restart Grafana container: `docker-compose restart grafana`
-
-### Performance Issues
-1. Monitor disk usage for log storage
-2. Adjust log retention in `loki-config.yaml`
-3. Increase resource limits if needed
-
-## Customization
-
-### Adding More Dashboards
-1. Create JSON dashboard files in `./monitoring/grafana/provisioning/dashboards/`
-2. Restart Grafana container
-
-### Log Retention Configuration
-Edit `./monitoring/loki-config.yaml` to adjust retention policies:
-```yaml
-limits_config:
-  retention_period: 168h  # 7 days
-```
-
-### Alert Configuration
-Add alerting rules to Prometheus configuration for ETL failure notifications.
-
-## Security Notes
-
-- Change default Grafana admin password in production
-- Restrict network access to monitoring ports
-- Consider using authentication for external access
-- Regularly update monitoring stack images