Compare commits
11 Commits
506debfcab
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 75fab63809 | |||
| e88a5a4cf4 | |||
| 3721e1fd30 | |||
| d9fa233c6c | |||
| 4b2e2a764b | |||
| e62713b377 | |||
| 8aba3107cf | |||
| 7fe8d21d38 | |||
| ed950811b6 | |||
| 214528ed76 | |||
| 18934e8862 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -1,3 +0,0 @@
|
||||
**/etl-data
|
||||
**/fotos_cand
|
||||
*.zip
|
||||
56
backup-db.sh
Executable file
56
backup-db.sh
Executable file
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
|
||||
# OpenCand Database Backup Script
|
||||
# This script creates a backup of the PostgreSQL database (excluding materialized views)
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
# Configuration
|
||||
DB_CONTAINER="opencand_db"
|
||||
DB_NAME="opencand"
|
||||
DB_USER="root"
|
||||
BACKUP_FILE="backup_$(date +%Y%m%d_%H%M%S).sql"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${YELLOW}💾 OpenCand Database Backup Script${NC}"
|
||||
echo "=================================="
|
||||
|
||||
# Check if Docker is running
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
echo -e "${RED}❌ Error: Docker is not running or not accessible${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if the database container is running
|
||||
if ! docker ps --format "table {{.Names}}" | grep -q "^${DB_CONTAINER}$"; then
|
||||
echo -e "${RED}❌ Error: Database container '${DB_CONTAINER}' is not running${NC}"
|
||||
echo "Please start the services with: docker-compose up -d"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}📋 Creating backup (excluding materialized views)...${NC}"
|
||||
|
||||
# Get list of materialized views
|
||||
MATVIEWS=$(docker exec $DB_CONTAINER psql -U $DB_USER -d $DB_NAME -Atc "SELECT schemaname || '.' || matviewname FROM pg_matviews;")
|
||||
|
||||
# Build exclude flags for pg_dump
|
||||
EXCLUDE_FLAGS=""
|
||||
for mv in $MATVIEWS; do
|
||||
EXCLUDE_FLAGS+=" --exclude-table=$mv"
|
||||
done
|
||||
|
||||
# Run pg_dump excluding materialized views
|
||||
if docker exec $DB_CONTAINER pg_dump -U $DB_USER -d $DB_NAME $EXCLUDE_FLAGS > "$BACKUP_FILE"; then
|
||||
echo -e "${GREEN}✅ Backup created: $BACKUP_FILE${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ Error: Failed to create backup${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}🎉 Backup completed successfully!${NC}"
|
||||
echo "=================================="
|
||||
136
db/db.sql
136
db/db.sql
@@ -1,136 +0,0 @@
|
||||
DROP TABLE IF EXISTS bem_candidato CASCADE;
|
||||
DROP TABLE IF EXISTS candidato_mapping CASCADE;
|
||||
DROP TABLE IF EXISTS rede_social CASCADE;
|
||||
DROP TABLE IF EXISTS candidato CASCADE;
|
||||
DROP TABLE IF EXISTS partido CASCADE;
|
||||
DROP TABLE IF EXISTS despesas_candidato CASCADE;
|
||||
DROP TABLE IF EXISTS receitas_candidato CASCADE;
|
||||
|
||||
CREATE TABLE candidato (
|
||||
idcandidato UUID NOT NULL PRIMARY KEY,
|
||||
cpf VARCHAR(11),
|
||||
nome VARCHAR(255) NOT NULL,
|
||||
apelido VARCHAR(255),
|
||||
datanascimento TIMESTAMPTZ,
|
||||
email TEXT,
|
||||
sexo CHAR(15),
|
||||
estadocivil VARCHAR(50),
|
||||
escolaridade VARCHAR(50),
|
||||
ocupacao VARCHAR(150)
|
||||
);
|
||||
CREATE INDEX idx_candidato_nome ON candidato (nome);
|
||||
CREATE INDEX idx_candidato_apelido ON candidato (apelido);
|
||||
|
||||
-- Each candidato (idcandidato, cpf, nome) will be mapped to a (sqcandidato, ano, tipo_eleicao, sg_uf, cargo, resultado)
|
||||
CREATE TABLE candidato_mapping (
|
||||
idcandidato UUID NOT NULL,
|
||||
cpf VARCHAR(11),
|
||||
nome VARCHAR(255) NOT NULL,
|
||||
apelido VARCHAR(255),
|
||||
sqcandidato VARCHAR(50) NOT NULL,
|
||||
turno VARCHAR(2) NOT NULL,
|
||||
ano INT NOT NULL,
|
||||
tipoeleicao VARCHAR(50),
|
||||
siglauf VARCHAR(2),
|
||||
nomeue VARCHAR(100),
|
||||
cargo VARCHAR(50),
|
||||
sgpartido VARCHAR(50),
|
||||
nrcandidato VARCHAR(20),
|
||||
resultado VARCHAR(50),
|
||||
CONSTRAINT pk_candidato_mapping PRIMARY KEY (idcandidato, ano, siglauf, nomeue, cargo, nrcandidato, resultado),
|
||||
CONSTRAINT fk_candidato_mapping_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
CREATE INDEX idx_candidato_mapping_cpf ON candidato_mapping (cpf);
|
||||
CREATE INDEX idx_candidato_mapping_nome ON candidato_mapping (nome);
|
||||
CREATE INDEX idx_candidato_mapping_apelido ON candidato_mapping (apelido);
|
||||
CREATE INDEX idx_candidato_mapping_ano ON candidato_mapping (ano);
|
||||
CREATE INDEX idx_candidato_mapping_sqcandidato ON candidato_mapping (sqcandidato);
|
||||
|
||||
---- Table for storing assets of candidates
|
||||
CREATE TABLE bem_candidato (
|
||||
idcandidato UUID NOT NULL,
|
||||
ano INT NOT NULL,
|
||||
ordembem INT,
|
||||
tipobem VARCHAR(150),
|
||||
descricao VARCHAR(500),
|
||||
valor NUMERIC(20, 2),
|
||||
CONSTRAINT fk_bem_candidato_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
ALTER TABLE bem_candidato ADD CONSTRAINT pk_bem_candidato PRIMARY KEY (idcandidato, ano, ordembem);
|
||||
CREATE INDEX idx_bem_candidato_idcandidato ON bem_candidato (idcandidato);
|
||||
CREATE INDEX idx_bem_candidato_valor ON bem_candidato (valor);
|
||||
|
||||
---- Table for storing social media links of candidates
|
||||
CREATE TABLE rede_social (
|
||||
idcandidato UUID NOT NULL,
|
||||
rede VARCHAR(50) NOT NULL,
|
||||
siglauf VARCHAR(2),
|
||||
ano INT NOT NULL,
|
||||
link TEXT NOT NULL,
|
||||
CONSTRAINT pk_rede_social PRIMARY KEY (idcandidato, rede, siglauf, ano),
|
||||
CONSTRAINT fk_rede_social_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
CREATE INDEX idx_rede_social_idcandidato ON rede_social (idcandidato);
|
||||
|
||||
---- Table for storing party information
|
||||
CREATE TABLE partido (
|
||||
sigla VARCHAR(50) NOT NULL PRIMARY KEY,
|
||||
nome VARCHAR(255) NOT NULL,
|
||||
numero INT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_partido_nome ON partido (nome);
|
||||
CREATE INDEX idx_partido_numero ON partido (numero);
|
||||
|
||||
---- Tables for storing despesas e receitas of candidacies
|
||||
CREATE TABLE despesas_candidato (
|
||||
idreceita UUID NOT NULL DEFAULT gen_random_uuid(),
|
||||
idcandidato UUID NOT NULL,
|
||||
ano INT NOT NULL,
|
||||
turno VARCHAR(2) NOT NULL,
|
||||
sqcandidato VARCHAR(50) NOT NULL,
|
||||
sgpartido VARCHAR(50) NOT NULL,
|
||||
tipofornecedor VARCHAR(150),
|
||||
cnpjfornecedor VARCHAR(14),
|
||||
cpffornecedor VARCHAR(11),
|
||||
nomefornecedor VARCHAR(255),
|
||||
nomefornecedorrfb VARCHAR(255),
|
||||
municipiofornecedor VARCHAR(100),
|
||||
tipodocumento VARCHAR(50),
|
||||
datadespesa TIMESTAMPTZ,
|
||||
descricao TEXT,
|
||||
origemdespesa TEXT,
|
||||
valor NUMERIC(20, 2),
|
||||
CONSTRAINT pk_despesas_candidato PRIMARY KEY (idreceita),
|
||||
CONSTRAINT fk_despesas_candidato_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
CREATE INDEX idx_despesas_candidato_idcandidato ON despesas_candidato (idcandidato);
|
||||
CREATE INDEX idx_despesas_candidato_ano ON despesas_candidato (ano);
|
||||
CREATE INDEX idx_despesas_candidato_sqcandidato ON despesas_candidato (sqcandidato);
|
||||
CREATE INDEX idx_despesas_candidato_sgpartido ON despesas_candidato (sgpartido);
|
||||
|
||||
CREATE TABLE receitas_candidato (
|
||||
idreceita UUID NOT NULL DEFAULT gen_random_uuid(),
|
||||
idcandidato UUID NOT NULL,
|
||||
ano INT NOT NULL,
|
||||
turno VARCHAR(2) NOT NULL,
|
||||
sqcandidato VARCHAR(50) NOT NULL,
|
||||
sgpartido VARCHAR(50) NOT NULL,
|
||||
fontereceita VARCHAR(150),
|
||||
origemreceita VARCHAR(250),
|
||||
naturezareceita VARCHAR(250),
|
||||
especiereceita VARCHAR(250),
|
||||
cnpjdoador VARCHAR(14),
|
||||
cpfdoador VARCHAR(11),
|
||||
nomedoador VARCHAR(255),
|
||||
nomedoadorrfb VARCHAR(255),
|
||||
municipiodoador VARCHAR(100),
|
||||
datareceita TIMESTAMPTZ,
|
||||
descricao TEXT,
|
||||
valor NUMERIC(20, 2),
|
||||
CONSTRAINT pk_receitas_candidato PRIMARY KEY (idreceita),
|
||||
CONSTRAINT fk_receitas_candidato_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
CREATE INDEX idx_receitas_candidato_idcandidato ON receitas_candidato (idcandidato);
|
||||
CREATE INDEX idx_receitas_candidato_ano ON receitas_candidato (ano);
|
||||
CREATE INDEX idx_receitas_candidato_sqcandidato ON receitas_candidato (sqcandidato);
|
||||
CREATE INDEX idx_receitas_candidato_sgpartido ON receitas_candidato (sgpartido);
|
||||
@@ -1,91 +0,0 @@
|
||||
# OpenCand Database Deployment Script (PowerShell)
|
||||
# This script deploys the database schema changes to the PostgreSQL container
|
||||
|
||||
param(
|
||||
[string]$ContainerName = "opencand_db",
|
||||
[string]$DatabaseName = "opencand",
|
||||
[string]$DatabaseUser = "root",
|
||||
[string]$SqlFile = ".\db\db.sql"
|
||||
)
|
||||
|
||||
# Configuration
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
Write-Host "🚀 OpenCand Database Deployment Script" -ForegroundColor Yellow
|
||||
Write-Host "==================================" -ForegroundColor Yellow
|
||||
|
||||
try {
|
||||
# Check if Docker is running
|
||||
Write-Host "📋 Pre-deployment checks:" -ForegroundColor Yellow
|
||||
docker info | Out-Null
|
||||
Write-Host "✅ Docker is running" -ForegroundColor Green
|
||||
|
||||
# Check if the database container is running
|
||||
$runningContainers = docker ps --format "{{.Names}}"
|
||||
if ($runningContainers -notcontains $ContainerName) {
|
||||
throw "Database container '$ContainerName' is not running. Please start with: docker-compose up -d"
|
||||
}
|
||||
Write-Host "✅ Database container is running" -ForegroundColor Green
|
||||
|
||||
# Check if SQL file exists
|
||||
if (-not (Test-Path $SqlFile)) {
|
||||
throw "SQL file '$SqlFile' not found"
|
||||
}
|
||||
Write-Host "✅ SQL file exists" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
|
||||
# Wait for database to be ready
|
||||
Write-Host "⏳ Waiting for database to be ready..." -ForegroundColor Yellow
|
||||
$timeout = 30
|
||||
$counter = 0
|
||||
do {
|
||||
$ready = $false
|
||||
try {
|
||||
docker exec $ContainerName pg_isready -U $DatabaseUser -d $DatabaseName | Out-Null
|
||||
$ready = $true
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
$counter++
|
||||
if ($counter -ge $timeout) {
|
||||
throw "Database failed to become ready within $timeout seconds"
|
||||
}
|
||||
}
|
||||
} while (-not $ready)
|
||||
|
||||
Write-Host "✅ Database is ready" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
|
||||
# Create backup before deployment
|
||||
Write-Host "💾 Creating database backup..." -ForegroundColor Yellow
|
||||
$backupFile = "backup_$(Get-Date -Format 'yyyyMMdd_HHmmss').sql"
|
||||
docker exec $ContainerName pg_dump -U $DatabaseUser -d $DatabaseName | Out-File -FilePath $backupFile -Encoding UTF8
|
||||
Write-Host "✅ Backup created: $backupFile" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
|
||||
# Deploy the SQL file
|
||||
Write-Host "🔧 Deploying database changes..." -ForegroundColor Yellow
|
||||
Write-Host "Executing: $SqlFile"
|
||||
|
||||
# Execute the SQL file
|
||||
Get-Content $SqlFile | docker exec -i $ContainerName psql -U $DatabaseUser -d $DatabaseName
|
||||
|
||||
Write-Host "✅ Database deployment completed successfully!" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
|
||||
# Show table information
|
||||
Write-Host "📊 Current database tables:" -ForegroundColor Yellow
|
||||
docker exec $ContainerName psql -U $DatabaseUser -d $DatabaseName -c "\dt"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "🎉 Deployment completed successfully!" -ForegroundColor Green
|
||||
Write-Host "==================================" -ForegroundColor Green
|
||||
|
||||
} catch {
|
||||
Write-Host "❌ Error: $($_.Exception.Message)" -ForegroundColor Red
|
||||
if (Test-Path $backupFile) {
|
||||
Write-Host "💡 You can restore from backup using:" -ForegroundColor Yellow
|
||||
Write-Host "Get-Content $backupFile | docker exec -i $ContainerName psql -U $DatabaseUser -d $DatabaseName" -ForegroundColor Yellow
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
98
deploy-db.sh
98
deploy-db.sh
@@ -9,7 +9,10 @@ set -e # Exit on any error
|
||||
DB_CONTAINER="opencand_db"
|
||||
DB_NAME="opencand"
|
||||
DB_USER="root"
|
||||
SQL_FILE="./db/db.sql"
|
||||
SQL_URL="https://git.ivanch.me/ivanch/opencand/raw/branch/main/db/db.sql"
|
||||
SQL_FILE="./db.sql"
|
||||
MV_URL="https://git.ivanch.me/ivanch/opencand/raw/branch/main/db/mv.sql"
|
||||
MV_FILE="./mv.sql"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
@@ -33,16 +36,54 @@ if ! docker ps --format "table {{.Names}}" | grep -q "^${DB_CONTAINER}$"; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if SQL file exists
|
||||
# Download the SQL files
|
||||
echo -e "${YELLOW}📥 Downloading SQL files...${NC}"
|
||||
if command -v curl &> /dev/null; then
|
||||
if curl -L -o "$SQL_FILE" "$SQL_URL"; then
|
||||
echo "✅ db.sql downloaded successfully"
|
||||
else
|
||||
echo -e "${RED}❌ Error: Failed to download db.sql from ${SQL_URL}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if curl -L -o "$MV_FILE" "$MV_URL"; then
|
||||
echo "✅ mv.sql downloaded successfully"
|
||||
else
|
||||
echo -e "${RED}❌ Error: Failed to download mv.sql from ${MV_URL}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
elif command -v wget &> /dev/null; then
|
||||
if wget -O "$SQL_FILE" "$SQL_URL"; then
|
||||
echo "✅ db.sql downloaded successfully"
|
||||
else
|
||||
echo -e "${RED}❌ Error: Failed to download db.sql from ${SQL_URL}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if wget -O "$MV_FILE" "$MV_URL"; then
|
||||
echo "✅ mv.sql downloaded successfully"
|
||||
else
|
||||
echo -e "${RED}❌ Error: Failed to download mv.sql from ${MV_URL}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}❌ Error: Neither curl nor wget is available for downloading${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if SQL files exist (after download)
|
||||
if [ ! -f "$SQL_FILE" ]; then
|
||||
echo -e "${RED}❌ Error: SQL file '${SQL_FILE}' not found${NC}"
|
||||
echo -e "${RED}❌ Error: SQL file '${SQL_FILE}' not found after download${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$MV_FILE" ]; then
|
||||
echo -e "${RED}❌ Error: SQL file '${MV_FILE}' not found after download${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}📋 Pre-deployment checks:${NC}"
|
||||
echo "✅ Docker is running"
|
||||
echo "✅ Database container is running"
|
||||
echo "✅ SQL file exists"
|
||||
echo "✅ db.sql downloaded"
|
||||
echo "✅ mv.sql downloaded"
|
||||
echo ""
|
||||
|
||||
# Wait for database to be ready
|
||||
@@ -60,32 +101,49 @@ done
|
||||
echo "✅ Database is ready"
|
||||
echo ""
|
||||
|
||||
# Create backup before deployment
|
||||
echo -e "${YELLOW}💾 Creating database backup...${NC}"
|
||||
BACKUP_FILE="backup_$(date +%Y%m%d_%H%M%S).sql"
|
||||
docker exec $DB_CONTAINER pg_dump -U $DB_USER -d $DB_NAME > $BACKUP_FILE
|
||||
echo "✅ Backup created: $BACKUP_FILE"
|
||||
echo ""
|
||||
|
||||
# Deploy the SQL file
|
||||
echo -e "${YELLOW}🔧 Deploying database changes...${NC}"
|
||||
# Deploy the db.sql file
|
||||
echo -e "${YELLOW}🔧 Deploying database changes (db.sql)...${NC}"
|
||||
echo "Executing: $SQL_FILE"
|
||||
|
||||
if docker exec -i $DB_CONTAINER psql -U $DB_USER -d $DB_NAME < $SQL_FILE; then
|
||||
echo -e "${GREEN}✅ Database deployment completed successfully!${NC}"
|
||||
echo ""
|
||||
|
||||
# Show table information
|
||||
echo -e "${YELLOW}📊 Current database tables:${NC}"
|
||||
docker exec $DB_CONTAINER psql -U $DB_USER -d $DB_NAME -c "\dt"
|
||||
|
||||
echo -e "${GREEN}✅ db.sql deployment completed successfully!${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ Error: Database deployment failed${NC}"
|
||||
echo -e "${RED}❌ Error: db.sql deployment failed${NC}"
|
||||
echo -e "${YELLOW}💡 You can restore from backup using:${NC}"
|
||||
echo "docker exec -i $DB_CONTAINER psql -U $DB_USER -d $DB_NAME < $BACKUP_FILE"
|
||||
# Clean up before exit
|
||||
rm -f "$SQL_FILE" "$MV_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Deploy the mv.sql file
|
||||
echo -e "${YELLOW}<EFBFBD> Deploying database changes (mv.sql)...${NC}"
|
||||
echo "Executing: $MV_FILE"
|
||||
|
||||
if docker exec -i $DB_CONTAINER psql -U $DB_USER -d $DB_NAME < $MV_FILE; then
|
||||
echo -e "${GREEN}✅ mv.sql deployment completed successfully!${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ Error: mv.sql deployment failed${NC}"
|
||||
echo -e "${YELLOW}💡 You can restore from backup using:${NC}"
|
||||
echo "docker exec -i $DB_CONTAINER psql -U $DB_USER -d $DB_NAME < $BACKUP_FILE"
|
||||
# Clean up before exit
|
||||
rm -f "$SQL_FILE" "$MV_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# Show table and materialized view information
|
||||
echo -e "${YELLOW}📊 Current database tables:${NC}"
|
||||
docker exec $DB_CONTAINER psql -U $DB_USER -d $DB_NAME -c "\dt"
|
||||
echo -e "${YELLOW}📄 Current materialized views:${NC}"
|
||||
docker exec $DB_CONTAINER psql -U $DB_USER -d $DB_NAME -c "\dm"
|
||||
|
||||
# Clean up downloaded SQL files
|
||||
echo -e "${YELLOW}🧹 Cleaning up...${NC}"
|
||||
rm -f "$SQL_FILE" "$MV_FILE"
|
||||
echo "✅ Temporary SQL files removed"
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}🎉 Deployment completed successfully!${NC}"
|
||||
echo "=================================="
|
||||
|
||||
@@ -1,11 +1,77 @@
|
||||
services: # ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. PostgreSQL Database
|
||||
services:
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. .NET API
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
api:
|
||||
container_name: opencand_api
|
||||
restart: unless-stopped
|
||||
image: git.ivanch.me/ivanch/opencand.api:latest
|
||||
ports:
|
||||
- "5100:8080"
|
||||
environment:
|
||||
ASPNETCORE_ENVIRONMENT: "Production"
|
||||
Logging__LogLevel__Default: "Information"
|
||||
DatabaseSettings__ConnectionString: "Host=db;Port=5432;Database=opencand;Username=root;Password=root;Pooling=true;Minimum Pool Size=1;Maximum Pool Size=20;Connection Lifetime=300;Command Timeout=30;Application Name=OpenCand.API;Include Error Detail=true"
|
||||
FotosSettings__ApiBasePath: "https://api.opencand.ivanch.me/assets/fotos"
|
||||
volumes:
|
||||
- ./fotos_cand:/app/fotos_cand
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
labels: "service=api"
|
||||
labels:
|
||||
- "promtail.enable=true"
|
||||
- "promtail.job=opencand-api"
|
||||
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# 2. React + Vite Front‐end
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
frontend:
|
||||
container_name: opencand_frontend
|
||||
restart: unless-stopped
|
||||
image: git.ivanch.me/ivanch/opencand.ui:latest
|
||||
ports:
|
||||
- "5110:80"
|
||||
depends_on:
|
||||
- api
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# 3. ETL (Optional: runs once at startup)
|
||||
#
|
||||
# If you want the ETL to run on every compose up, give it restart: "no" or
|
||||
# some other policy. It will run, then exit.
|
||||
#
|
||||
# If you instead prefer to run ETL manually or via host cron, you can omit
|
||||
# this service and just `docker run myorg/etl:latest ...` on demand.
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
etl:
|
||||
image: git.ivanch.me/ivanch/opencand.etl:latest
|
||||
container_name: opencand_etl
|
||||
restart: "no"
|
||||
environment:
|
||||
DatabaseSettings__ConnectionString: "Host=db;Port=5432;Database=opencand;Username=root;Password=root"
|
||||
ParserSettings_CandidatoCSVThreads: "40"
|
||||
BasePath: "etl-data"
|
||||
volumes:
|
||||
- ./etl-data:/app/etl-data
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
labels: "service=etl"
|
||||
labels:
|
||||
- "promtail.enable=true"
|
||||
- "promtail.job=opencand-etl"
|
||||
|
||||
db:
|
||||
image: postgres:14-alpine
|
||||
container_name: opencand_db
|
||||
profiles: ["infra"]
|
||||
restart: unless-stopped
|
||||
hostname: db
|
||||
shm_size: 4g
|
||||
environment:
|
||||
POSTGRES_USER: root
|
||||
POSTGRES_PASSWORD: root
|
||||
@@ -27,85 +93,13 @@ services: # ──────────────────────
|
||||
- "promtail.enable=true"
|
||||
- "promtail.job=opencand-db"
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# 2. .NET API
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
api:
|
||||
container_name: opencand_api
|
||||
restart: unless-stopped
|
||||
image: git.ivanch.me/ivanch/opencand.api:latest
|
||||
ports:
|
||||
- "5100:8080"
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
ASPNETCORE_ENVIRONMENT: "Production"
|
||||
DatabaseSettings__ConnectionString: "Host=db;Port=5432;Database=opencand;Username=root;Password=root"
|
||||
FotosSettings__ApiBasePath: "https://api.opencand.ivanch.me/assets/fotos"
|
||||
volumes:
|
||||
- ./fotos_cand:/app/fotos_cand
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
labels: "service=api"
|
||||
labels:
|
||||
- "promtail.enable=true"
|
||||
- "promtail.job=opencand-api"
|
||||
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# 3. React + Vite Front‐end
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
frontend:
|
||||
container_name: opencand_frontend
|
||||
restart: unless-stopped
|
||||
image: git.ivanch.me/ivanch/opencand.ui:latest
|
||||
ports:
|
||||
- "5110:80"
|
||||
depends_on:
|
||||
- api
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# 4. ETL (Optional: runs once at startup)
|
||||
#
|
||||
# If you want the ETL to run on every compose up, give it restart: "no" or
|
||||
# some other policy. It will run, then exit.
|
||||
#
|
||||
# If you instead prefer to run ETL manually or via host cron, you can omit
|
||||
# this service and just `docker run myorg/etl:latest ...` on demand.
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
etl:
|
||||
image: git.ivanch.me/ivanch/opencand.etl:latest
|
||||
container_name: opencand_etl
|
||||
restart: "no"
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_started
|
||||
environment:
|
||||
DatabaseSettings__ConnectionString: "Host=db;Port=5432;Database=opencand;Username=root;Password=root"
|
||||
ParserSettings_CandidatoCSVThreads: "40"
|
||||
BasePath: "etl-data"
|
||||
volumes:
|
||||
- ./etl-data:/app/etl-data
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
labels: "service=etl"
|
||||
labels:
|
||||
- "promtail.enable=true"
|
||||
- "promtail.job=opencand-etl"
|
||||
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# 5. Log Monitoring Stack
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# Loki - Log aggregation system
|
||||
loki:
|
||||
image: grafana/loki:2.9.0
|
||||
container_name: opencand_loki
|
||||
profiles: ["infra"]
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:6110:3100"
|
||||
@@ -123,6 +117,7 @@ services: # ──────────────────────
|
||||
promtail:
|
||||
image: grafana/promtail:2.9.0
|
||||
container_name: opencand_promtail
|
||||
profiles: ["infra"]
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./monitoring/promtail-config.yaml:/etc/promtail/config.yml
|
||||
@@ -137,6 +132,7 @@ services: # ──────────────────────
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.45.0
|
||||
container_name: opencand_prometheus
|
||||
profiles: ["infra"]
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:6090:9090"
|
||||
@@ -155,6 +151,7 @@ services: # ──────────────────────
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.6.0
|
||||
container_name: opencand_node_exporter
|
||||
profiles: ["infra"]
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:6100:9100"
|
||||
@@ -171,6 +168,7 @@ services: # ──────────────────────
|
||||
grafana:
|
||||
image: grafana/grafana:10.0.0
|
||||
container_name: opencand_grafana
|
||||
profiles: ["infra"]
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "6000:3000"
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
# OpenCand Monitoring Setup
|
||||
|
||||
This monitoring stack provides comprehensive log aggregation and visualization for the OpenCand project, with special focus on the ETL service.
|
||||
|
||||
## Services Overview
|
||||
|
||||
### 🔍 **Grafana Loki** (Port 3100)
|
||||
- **Purpose**: Log aggregation and storage
|
||||
- **Access**: http://localhost:3100
|
||||
- **Description**: Collects and stores all container logs in a structured format
|
||||
|
||||
### 📊 **Grafana** (Port 3000)
|
||||
- **Purpose**: Log visualization and dashboards
|
||||
- **Access**: http://localhost:3000
|
||||
- **Credentials**:
|
||||
- Username: `admin`
|
||||
- Password: `admin`
|
||||
- **Pre-configured Dashboards**: OpenCand ETL Monitoring dashboard
|
||||
|
||||
### 📈 **Prometheus** (Port 9090)
|
||||
- **Purpose**: Metrics collection and storage
|
||||
- **Access**: http://localhost:9090
|
||||
- **Description**: Collects system and application metrics
|
||||
|
||||
### 🖥️ **Node Exporter** (Port 9100)
|
||||
- **Purpose**: System metrics collection
|
||||
- **Access**: http://localhost:9100/metrics
|
||||
- **Description**: Provides host system metrics (CPU, memory, disk, etc.)
|
||||
|
||||
### 🚚 **Promtail**
|
||||
- **Purpose**: Log collection agent
|
||||
- **Description**: Automatically discovers and ships Docker container logs to Loki
|
||||
|
||||
## Key Features
|
||||
|
||||
### ETL-Specific Monitoring
|
||||
- ✅ Real-time ETL process logs
|
||||
- ✅ Error tracking and alerting capabilities
|
||||
- ✅ Performance metrics monitoring
|
||||
- ✅ Data processing progress tracking
|
||||
|
||||
### Container Log Management
|
||||
- ✅ Automatic log rotation (10MB max size, 3 files)
|
||||
- ✅ Structured log labeling
|
||||
- ✅ Multi-service log aggregation
|
||||
|
||||
### Pre-built Dashboards
|
||||
- ✅ OpenCand ETL Logs viewer
|
||||
- ✅ API logs monitoring
|
||||
- ✅ Database logs tracking
|
||||
- ✅ Container resource usage
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. **Start the monitoring stack**:
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
2. **Access Grafana**:
|
||||
- Open http://localhost:3000
|
||||
- Login with admin/admin
|
||||
- Navigate to "Dashboards" → "OpenCand ETL Monitoring"
|
||||
|
||||
3. **View ETL Logs in Real-time**:
|
||||
- In Grafana, go to "Explore"
|
||||
- Select "Loki" as datasource
|
||||
- Use query: `{container_name="opencand_etl"}`
|
||||
|
||||
4. **Monitor System Metrics**:
|
||||
- Access Prometheus at http://localhost:9090
|
||||
- View system metrics from Node Exporter
|
||||
|
||||
## Log Queries Examples
|
||||
|
||||
### ETL Service Logs
|
||||
```logql
|
||||
{container_name="opencand_etl"}
|
||||
```
|
||||
|
||||
### Error Logs Only
|
||||
```logql
|
||||
{container_name="opencand_etl"} |= "ERROR"
|
||||
```
|
||||
|
||||
### API Logs with Filtering
|
||||
```logql
|
||||
{container_name="opencand_api"} |= "Microsoft.AspNetCore"
|
||||
```
|
||||
|
||||
### Database Connection Logs
|
||||
```logql
|
||||
{container_name="opencand_db"} |= "connection"
|
||||
```
|
||||
|
||||
## Configuration Files
|
||||
|
||||
- **Loki**: `./monitoring/loki-config.yaml`
|
||||
- **Promtail**: `./monitoring/promtail-config.yaml`
|
||||
- **Prometheus**: `./monitoring/prometheus.yml`
|
||||
- **Grafana Datasources**: `./monitoring/grafana/provisioning/datasources/`
|
||||
- **Grafana Dashboards**: `./monitoring/grafana/provisioning/dashboards/`
|
||||
|
||||
## Data Persistence
|
||||
|
||||
The following volumes are created for data persistence:
|
||||
- `loki-data`: Loki log storage
|
||||
- `prometheus-data`: Prometheus metrics storage
|
||||
- `grafana-data`: Grafana dashboards and settings
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### ETL Logs Not Appearing
|
||||
1. Check if ETL container is running: `docker ps`
|
||||
2. Verify Promtail is collecting logs: `docker logs opencand_promtail`
|
||||
3. Check Loki status: `curl http://localhost:3100/ready`
|
||||
|
||||
### Grafana Dashboard Issues
|
||||
1. Verify datasources are configured correctly
|
||||
2. Check if Loki is accessible from Grafana container
|
||||
3. Restart Grafana container: `docker-compose restart grafana`
|
||||
|
||||
### Performance Issues
|
||||
1. Monitor disk usage for log storage
|
||||
2. Adjust log retention in `loki-config.yaml`
|
||||
3. Increase resource limits if needed
|
||||
|
||||
## Customization
|
||||
|
||||
### Adding More Dashboards
|
||||
1. Create JSON dashboard files in `./monitoring/grafana/provisioning/dashboards/`
|
||||
2. Restart Grafana container
|
||||
|
||||
### Log Retention Configuration
|
||||
Edit `./monitoring/loki-config.yaml` to adjust retention policies:
|
||||
```yaml
|
||||
limits_config:
|
||||
retention_period: 168h # 7 days
|
||||
```
|
||||
|
||||
### Alert Configuration
|
||||
Add alerting rules to Prometheus configuration for ETL failure notifications.
|
||||
|
||||
## Security Notes
|
||||
|
||||
- Change default Grafana admin password in production
|
||||
- Restrict network access to monitoring ports
|
||||
- Consider using authentication for external access
|
||||
- Regularly update monitoring stack images
|
||||
Reference in New Issue
Block a user