commit 1cb7645910b0145c0b519aa80fe276a4539de2e8 Author: Jose Henrique Date: Sat May 31 10:58:30 2025 -0300 init diff --git a/.gitea/workflows/main-api.yaml b/.gitea/workflows/main-api.yaml new file mode 100644 index 0000000..764e031 --- /dev/null +++ b/.gitea/workflows/main-api.yaml @@ -0,0 +1,64 @@ +name: API Build and Deploy + +on: + workflow_dispatch: {} + +env: + REGISTRY_HOST: git.ivanch.me + IMAGE_API: ${{ env.REGISTRY_HOST }}/ivanch/opencand.api + # ─────────────────────────────────────────────────────────────────────────── + DEPLOY_USER: ${{ secrets.LIVE_USERNAME }} + DEPLOY_HOST: ${{ secrets.LIVE_HOST }} + DEPLOY_PATH: ${{ secrets.LIVE_PROJECT_DIR }} + +jobs: + build_and_deploy_api: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v2 + + - name: Log in to Container Registry + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" \ + | docker login "${{ env.REGISTRY_HOST }}" \ + -u "${{ secrets.REGISTRY_USERNAME }}" \ + --password-stdin + + - name: Build and Push API Image + run: | + TAG=latest + + docker build \ + -t "${{ env.IMAGE_API }}:${TAG}" \ + -f OpenCand.API.dockerfile \ + . + + docker push "${{ env.IMAGE_API }}:${TAG}" + + - name: Set up SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.LIVE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + # Add the host to known_hosts so SSH does not prompt + ssh-keyscan -H "${{ env.DEPLOY_HOST }}" >> ~/.ssh/known_hosts + + - name: Deploy to Production Server + run: | + TAG=latest + + ssh "${{ env.DEPLOY_USER }}@${{ env.DEPLOY_HOST }}" << 'EOF' + cd "${{ env.DEPLOY_PATH }}" + + # Replace the “image:” line for the frontend service + # sed -i \ + # "s|image: .*/frontend:.*|image: ${{ env.IMAGE_FRONTEND }}:${TAG}|g" \ + # docker-compose.yml + + # Pull only the new frontend image, then restart that service + docker compose pull api + docker compose up api -d + EOF diff --git a/.gitea/workflows/main-etl.yaml b/.gitea/workflows/main-etl.yaml new file mode 100644 index 0000000..fd16755 --- /dev/null +++ b/.gitea/workflows/main-etl.yaml @@ -0,0 +1,64 @@ +name: ETL Build and Deploy + +on: + workflow_dispatch: {} + +env: + REGISTRY_HOST: git.ivanch.me + IMAGE_ETL: ${{ env.REGISTRY_HOST }}/ivanch/opencand.etl + # ─────────────────────────────────────────────────────────────────────────── + DEPLOY_USER: ${{ secrets.LIVE_USERNAME }} + DEPLOY_HOST: ${{ secrets.LIVE_HOST }} + DEPLOY_PATH: ${{ secrets.LIVE_PROJECT_DIR }} + +jobs: + build_and_deploy_etl: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v2 + + - name: Log in to Container Registry + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" \ + | docker login "${{ env.REGISTRY_HOST }}" \ + -u "${{ secrets.REGISTRY_USERNAME }}" \ + --password-stdin + + - name: Build and Push ETL Image + run: | + TAG=latest + + docker build \ + -t "${{ env.IMAGE_ETL }}:${TAG}" \ + -f OpenCand.ETL.dockerfile \ + . + + docker push "${{ env.IMAGE_ETL }}:${TAG}" + + - name: Set up SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.LIVE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + # Add the host to known_hosts so SSH does not prompt + ssh-keyscan -H "${{ env.DEPLOY_HOST }}" >> ~/.ssh/known_hosts + + - name: Deploy to Production Server + run: | + TAG=latest + + ssh "${{ env.DEPLOY_USER }}@${{ env.DEPLOY_HOST }}" << 'EOF' + cd "${{ env.DEPLOY_PATH }}" + + # Replace the “image:” line for the frontend service + # sed -i \ + # "s|image: .*/frontend:.*|image: ${{ env.IMAGE_FRONTEND }}:${TAG}|g" \ + # docker-compose.yml + + # Pull only the new frontend image, then restart that service + docker compose pull etl + docker compose up etl -d + EOF diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..46a7e35 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +sample* +.vs +.vscode +**/bin +**/obj +*.user +*.suo +*.userosscache +*.sln.docstates +*.sln.ide +*.sln.ide.* +data +appsettings.Development.json + +docker-compose.yaml \ No newline at end of file diff --git a/API.md b/API.md new file mode 100644 index 0000000..1172d2e --- /dev/null +++ b/API.md @@ -0,0 +1,80 @@ +# OpenCand API + +## specs +### GET /v1/stats +Returns statistics about the OpenCand platform. +```json +{ + "totalCandidatos": 1000, + "totalBemCandidatos": 2500, + "totalEleicoes": 500 +} +``` + +### GET /v1/candidato/search?q={query} +Search for candidates by name or other attributes. +```json +{ + "candidatos": [ + { + "id": "6c2be869-339c-47d0-aeb6-77c686e528b5", + "nome": "João Silva", + "cpf": "123.***.789-10", + "dataNascimento": "1990-01-01", + "email": "email@test.com", + "estadoCivil": "Solteiro", + "sexo": "Masculino", + "ocupacao": "1234-5678", + } + ] +} +``` + +### GET /v1/candidato/{id} +Get detailed information about a specific candidate by ID. +```json +{ + "id": "6c2be869-339c-47d0-aeb6-77c686e528b5", + "nome": "João Silva", + "cpf": "123.***.789-10", + "dataNascimento": "1990-01-01", + "email": "email@test.com", + "estadoCivil": "Solteiro", + "sexo": "Masculino", + "ocupacao": "1234-5678", + "eleicoes": [ + { + "sqid": "160002325330", + "tipoeleicao": "ESTADUAL", + "siglaUf": "SP", + "nomeue": "São Paulo", + "nrCandidato": "123456", + "nomeCandidato": "João Silva", + "resultado": "ELEITO" + } + ] +} +``` + +### GET /v1/candidato/{id}/bens +Get the assets of a specific candidate by ID. +```json +{ + "bens": [ + { + "idCandidato": "6c2be869-339c-47d0-aeb6-77c686e528b5", + "ano": 2020, + "tipoBem": "Apartamento", + "descricao": "Apartamento", + "valor": 250000.00, + }, + { + "idCandidato": "6c2be869-339c-47d0-aeb6-77c686e528b5", + "ano": 2020, + "tipoBem": "Veículo automotor terrestre: caminhão, automóvel, moto, etc.", + "descricao": "Veículo VolksWagem POLO mca", + "valor": 40000.00, + } + ] +} +``` \ No newline at end of file diff --git a/OpenCand.API.dockerfile b/OpenCand.API.dockerfile new file mode 100644 index 0000000..5138d13 --- /dev/null +++ b/OpenCand.API.dockerfile @@ -0,0 +1,27 @@ +# ─── Stage 1: Build ─────────────────────────────────────────────────────── +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +WORKDIR /src + +# Copy only the solution and project files first, restore, then copy the rest +COPY ./OpenCand.sln ./ +COPY ./OpenCand.ETL/OpenCand.API.csproj ./OpenCand.API/ +COPY ./OpenCand.Core/OpenCand.Core.csproj ./OpenCand.Core/ + +RUN dotnet restore ./OpenCand.API/OpenCand.API.csproj + +# Now copy the rest of the API source code +COPY ./OpenCand.API/. ./OpenCand.API/ +COPY ./OpenCand.Core/. ./OpenCand.Core/ + +WORKDIR /src/OpenCand.API +RUN dotnet publish -c Release -o /app/publish + +# ─── Stage 2: Runtime ───────────────────────────────────────────────────── +FROM mcr.microsoft.com/dotnet/runtime:8.0 AS runtime +WORKDIR /app +COPY --from=build /app/publish . + +ENV ConnectionStrings__DefaultConnection="Host=db;Port=5432;Database=opencand;Username=root;Password=root" +EXPOSE 8080 + +ENTRYPOINT ["dotnet", "OpenCand.API.dll"] diff --git a/OpenCand.API/Config/FotosSettings.cs b/OpenCand.API/Config/FotosSettings.cs new file mode 100644 index 0000000..3324199 --- /dev/null +++ b/OpenCand.API/Config/FotosSettings.cs @@ -0,0 +1,8 @@ +namespace OpenCand.API.Config +{ + public class FotosSettings + { + public string Path { get; set; } = string.Empty; + public string ApiBasePath { get; set; } = string.Empty; + } +} diff --git a/OpenCand.API/Controllers/BaseController.cs b/OpenCand.API/Controllers/BaseController.cs new file mode 100644 index 0000000..e4da2a3 --- /dev/null +++ b/OpenCand.API/Controllers/BaseController.cs @@ -0,0 +1,12 @@ +using Microsoft.AspNetCore.Mvc; + +namespace OpenCand.API.Controllers +{ + [ApiController] + [Route("v1/[controller]")] + [Produces("application/json")] + public class BaseController : Controller + { + + } +} diff --git a/OpenCand.API/Controllers/CandidatoController.cs b/OpenCand.API/Controllers/CandidatoController.cs new file mode 100644 index 0000000..ed07952 --- /dev/null +++ b/OpenCand.API/Controllers/CandidatoController.cs @@ -0,0 +1,41 @@ +using Microsoft.AspNetCore.Mvc; +using OpenCand.API.Model; +using OpenCand.API.Services; +using OpenCand.Core.Models; + +namespace OpenCand.API.Controllers +{ + public class CandidatoController : BaseController + { + private readonly OpenCandService openCandService; + + public CandidatoController(OpenCandService openCandService) + { + this.openCandService = openCandService; + } + + [HttpGet("search")] + public async Task CandidatoSearch([FromQuery] string q) + { + return await openCandService.SearchCandidatosAsync(q); + } + + [HttpGet("{id}")] + public async Task GetCandidatoById([FromRoute] Guid id) + { + return await openCandService.GetCandidatoAsync(id); + } + + [HttpGet("{id}/bens")] + public async Task GetBensCandidatoById([FromRoute] Guid id) + { + return await openCandService.GetBemCandidatoById(id); + } + + [HttpGet("{id}/rede-social")] + public async Task GetCandidatoRedeSocialById([FromRoute] Guid id) + { + return await openCandService.GetCandidatoRedeSocialById(id); + } + } +} diff --git a/OpenCand.API/Controllers/StatsController.cs b/OpenCand.API/Controllers/StatsController.cs new file mode 100644 index 0000000..5516ac7 --- /dev/null +++ b/OpenCand.API/Controllers/StatsController.cs @@ -0,0 +1,22 @@ +using Microsoft.AspNetCore.Mvc; +using OpenCand.API.Services; +using OpenCand.Core.Models; + +namespace OpenCand.API.Controllers +{ + public class StatsController : BaseController + { + private readonly OpenCandService openCandService; + + public StatsController(OpenCandService openCandService) + { + this.openCandService = openCandService; + } + + [HttpGet()] + public async Task GetStats() + { + return await openCandService.GetOpenCandStatsAsync(); + } + } +} diff --git a/OpenCand.API/Model/ListSearchResult.cs b/OpenCand.API/Model/ListSearchResult.cs new file mode 100644 index 0000000..f680a9d --- /dev/null +++ b/OpenCand.API/Model/ListSearchResult.cs @@ -0,0 +1,19 @@ +using OpenCand.Core.Models; + +namespace OpenCand.API.Model +{ + public class CandidatoSearchResult + { + public List Candidatos { get; set; } + } + + public class BemCandidatoResult + { + public List Bens { get; set; } + } + + public class RedeSocialResult + { + public List RedesSociais { get; set; } + } +} diff --git a/OpenCand.API/OpenCand.API.csproj b/OpenCand.API/OpenCand.API.csproj new file mode 100644 index 0000000..82db15b --- /dev/null +++ b/OpenCand.API/OpenCand.API.csproj @@ -0,0 +1,28 @@ + + + + net8.0 + enable + enable + + + + + + + + + + + + + + + Always + + + Always + + + + diff --git a/OpenCand.API/Program.cs b/OpenCand.API/Program.cs new file mode 100644 index 0000000..9c6a3fd --- /dev/null +++ b/OpenCand.API/Program.cs @@ -0,0 +1,68 @@ +using Microsoft.Extensions.FileProviders; +using OpenCand.API.Config; +using OpenCand.API.Repository; +using OpenCand.API.Services; +using OpenCand.Repository; + +namespace OpenCand.API +{ + public class Program + { + public static void Main(string[] args) + { + var builder = WebApplication.CreateBuilder(args); + + if (string.IsNullOrEmpty(Environment.ProcessPath)) + { + throw new InvalidOperationException("Environment.ProcessPath is not set. Ensure the application is running in a valid environment."); + } + + // Add services to the container. + builder.Services.AddControllers(); + + SetupServices(builder); + + builder.Services.AddEndpointsApiExplorer(); + builder.Services.AddSwaggerGen(); + + var app = builder.Build(); + + // Configure the HTTP request pipeline. + if (app.Environment.IsDevelopment()) + { + app.UseSwagger(); + app.UseSwaggerUI(); + } + + var workingDir = Path.GetDirectoryName(Environment.ProcessPath); + app.UseStaticFiles(new StaticFileOptions + { + FileProvider = new PhysicalFileProvider(Path.Combine(workingDir, "fotos_cand")), + RequestPath = "/assets/fotos" + }); + + app.UseHttpsRedirection(); + + app.UseAuthorization(); + + app.MapControllers(); + + app.UseCors(x => x.AllowAnyMethod() + .AllowAnyHeader() + .SetIsOriginAllowed(origin => true) // allow any origin + .AllowCredentials()); // allow credentials + + + app.Run(); + } + + private static void SetupServices(WebApplicationBuilder builder) + { + builder.Services.Configure(builder.Configuration.GetSection("FotosSettings")); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + } + } +} diff --git a/OpenCand.API/Properties/launchSettings.json b/OpenCand.API/Properties/launchSettings.json new file mode 100644 index 0000000..5cda2cd --- /dev/null +++ b/OpenCand.API/Properties/launchSettings.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:54223", + "sslPort": 44383 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "http://localhost:5299", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7051;http://localhost:5299", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/OpenCand.API/Repository/BaseRepository.cs b/OpenCand.API/Repository/BaseRepository.cs new file mode 100644 index 0000000..8575077 --- /dev/null +++ b/OpenCand.API/Repository/BaseRepository.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.Configuration; +using Npgsql; + +namespace OpenCand.Repository +{ + public abstract class BaseRepository + { + protected string ConnectionString { get; private set; } + protected NpgsqlConnection? Connection { get; private set; } + + public BaseRepository(IConfiguration configuration) + { + ConnectionString = configuration["DatabaseSettings:ConnectionString"] ?? + throw new ArgumentNullException("Connection string not found in configuration"); + } + } +} diff --git a/OpenCand.API/Repository/BemCandidatoRepository.cs b/OpenCand.API/Repository/BemCandidatoRepository.cs new file mode 100644 index 0000000..08dfb53 --- /dev/null +++ b/OpenCand.API/Repository/BemCandidatoRepository.cs @@ -0,0 +1,26 @@ +using Dapper; +using Npgsql; +using OpenCand.Core.Models; + +namespace OpenCand.Repository +{ + public class BemCandidatoRepository : BaseRepository + { + public BemCandidatoRepository(IConfiguration configuration) : base(configuration) + { + } + + public async Task?> GetBemCandidatoAsync(Guid idcandidato) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var query = @" + SELECT * FROM bem_candidato + WHERE idcandidato = @idcandidato + ORDER BY ano DESC, ordembem ASC;"; + + return (await connection.QueryAsync(query, new { idcandidato })).AsList(); + } + } + } +} diff --git a/OpenCand.API/Repository/CandidatoRepository.cs b/OpenCand.API/Repository/CandidatoRepository.cs new file mode 100644 index 0000000..1f5efea --- /dev/null +++ b/OpenCand.API/Repository/CandidatoRepository.cs @@ -0,0 +1,61 @@ +using Dapper; +using Npgsql; +using OpenCand.Core.Models; + +namespace OpenCand.Repository +{ + public class CandidatoRepository : BaseRepository + { + public CandidatoRepository(IConfiguration configuration) : base(configuration) + { + } + + public async Task> SearchCandidatosAsync(string query) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + return (await connection.QueryAsync(@" + SELECT idcandidato, cpf, nome, datanascimento, email, sexo, estadocivil, escolaridade, ocupacao + FROM candidato + WHERE nome ILIKE '%' || @query || '%' OR + cpf ILIKE '%' || @query || '%' OR + email ILIKE '%' || @query || '%' + LIMIT 10;", + new { query })).AsList(); + } + } + + public async Task GetCandidatoAsync(Guid idcandidato) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + return await connection.QueryFirstOrDefaultAsync(@" + SELECT * FROM candidato + WHERE idcandidato = @idcandidato;", + new { idcandidato }); + } + } + + public async Task?> GetCandidatoMappingById(Guid idcandidato) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var query = @" + SELECT * FROM candidato_mapping + WHERE idcandidato = @idcandidato"; + return (await connection.QueryAsync(query, new { idcandidato })).AsList(); + } + } + + public async Task?> GetCandidatoRedeSocialById(Guid idcandidato) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var query = @" + SELECT * FROM rede_social + WHERE idcandidato = @idcandidato"; + return (await connection.QueryAsync(query, new { idcandidato })).AsList(); + } + } + } +} diff --git a/OpenCand.API/Repository/OpenCandRepository.cs b/OpenCand.API/Repository/OpenCandRepository.cs new file mode 100644 index 0000000..3076352 --- /dev/null +++ b/OpenCand.API/Repository/OpenCandRepository.cs @@ -0,0 +1,29 @@ +using Dapper; +using Npgsql; +using OpenCand.Core.Models; +using OpenCand.Repository; + +namespace OpenCand.API.Repository +{ + public class OpenCandRepository : BaseRepository + { + public OpenCandRepository(IConfiguration configuration) : base(configuration) + { + } + + public async Task GetOpenCandStatsAsync() + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var stats = await connection.QueryFirstOrDefaultAsync(@" + SELECT + (SELECT COUNT(idcandidato) FROM candidato) AS TotalCandidatos, + (SELECT COUNT(*) FROM bem_candidato) AS TotalBemCandidatos, + (SELECT SUM(valor) FROM bem_candidato) AS TotalValorBemCandidatos, + (SELECT COUNT(*) FROM rede_social) AS TotalRedesSociais, + (SELECT COUNT(DISTINCT ano) FROM bem_candidato) AS TotalEleicoes;"); + return stats ?? new OpenCandStats(); + } + } + } +} diff --git a/OpenCand.API/Services/OpenCandService.cs b/OpenCand.API/Services/OpenCandService.cs new file mode 100644 index 0000000..cb86fa7 --- /dev/null +++ b/OpenCand.API/Services/OpenCandService.cs @@ -0,0 +1,98 @@ +using Microsoft.Extensions.Options; +using OpenCand.API.Config; +using OpenCand.API.Model; +using OpenCand.API.Repository; +using OpenCand.Core.Models; +using OpenCand.Repository; + +namespace OpenCand.API.Services +{ + public class OpenCandService + { + private readonly OpenCandRepository openCandRepository; + private readonly CandidatoRepository candidatoRepository; + private readonly BemCandidatoRepository bemCandidatoRepository; + private readonly IConfiguration configuration; + private readonly FotosSettings fotoSettings; + private readonly ILogger logger; + + public OpenCandService( + OpenCandRepository openCandRepository, + CandidatoRepository candidatoRepository, + BemCandidatoRepository bemCandidatoRepository, + IOptions fotoSettings, + IConfiguration configuration, + ILogger logger) + { + this.openCandRepository = openCandRepository; + this.candidatoRepository = candidatoRepository; + this.bemCandidatoRepository = bemCandidatoRepository; + this.fotoSettings = fotoSettings.Value; + this.configuration = configuration; + this.logger = logger; + } + + public async Task GetOpenCandStatsAsync() + { + return await openCandRepository.GetOpenCandStatsAsync(); + } + + public async Task SearchCandidatosAsync(string query) + { + return new CandidatoSearchResult() + { + Candidatos = await candidatoRepository.SearchCandidatosAsync(query) + }; + } + + public async Task GetCandidatoAsync(Guid idcandidato) + { + var result = await candidatoRepository.GetCandidatoAsync(idcandidato); + var eleicoes = await candidatoRepository.GetCandidatoMappingById(idcandidato); + + if (result == null) + { + throw new KeyNotFoundException($"Candidato with ID {idcandidato} not found."); + } + if (eleicoes == null || eleicoes.Count == 0) + { + throw new KeyNotFoundException($"CandidatoMapping for ID {idcandidato} not found."); + } + + var lastEleicao = eleicoes.OrderByDescending(e => e.Ano).First(); + + result.FotoUrl = $"{fotoSettings.ApiBasePath}/foto_cand{lastEleicao.Ano}_{lastEleicao.SiglaUF}_div/F{lastEleicao.SiglaUF}{lastEleicao.SqCandidato}_div.jpg"; + result.Eleicoes = eleicoes.OrderByDescending(e => e.Ano).ToList(); + + return result; + } + + public async Task GetBemCandidatoById(Guid idcandidato) + { + var result = await bemCandidatoRepository.GetBemCandidatoAsync(idcandidato); + if (result == null) + { + result = new List(); + } + + return new BemCandidatoResult() + { + Bens = result.OrderByDescending(r => r.TipoBem).ThenByDescending(r => r.Valor).ToList() + }; + } + + public async Task GetCandidatoRedeSocialById(Guid idcandidato) + { + var result = await candidatoRepository.GetCandidatoRedeSocialById(idcandidato); + if (result == null) + { + result = new List(); + } + + return new RedeSocialResult() + { + RedesSociais = result.OrderByDescending(r => r.Ano).ToList() + }; + } + } +} diff --git a/OpenCand.API/appsettings.json b/OpenCand.API/appsettings.json new file mode 100644 index 0000000..4b45929 --- /dev/null +++ b/OpenCand.API/appsettings.json @@ -0,0 +1,16 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "DatabaseSettings": { + "ConnectionString": "Host=localhost;Database=opencand;Username=root;Password=root;Include Error Detail=true;CommandTimeout=300" + }, + "FotosSettings": { + "Path": "./foto_cand", + "ApiBasePath": "http://localhost:5299/assets/fotos" + }, + "AllowedHosts": "*" +} diff --git a/OpenCand.Core/Models/BemCandidato.cs b/OpenCand.Core/Models/BemCandidato.cs new file mode 100644 index 0000000..1f94c03 --- /dev/null +++ b/OpenCand.Core/Models/BemCandidato.cs @@ -0,0 +1,24 @@ +namespace OpenCand.Core.Models +{ + public class BemCandidato + { + public Guid IdCandidato { get; set; } + + // Only for fetching purposes + public string? SqCandidato { get; set; } + + public int Ano { get; set; } + + public string SiglaUF { get; set; } + + public string NomeUE { get; set; } + + public int OrdemBem { get; set; } + + public string? TipoBem { get; set; } + + public string? Descricao { get; set; } + + public decimal? Valor { get; set; } + } +} diff --git a/OpenCand.Core/Models/Candidato.cs b/OpenCand.Core/Models/Candidato.cs new file mode 100644 index 0000000..d4af8ba --- /dev/null +++ b/OpenCand.Core/Models/Candidato.cs @@ -0,0 +1,56 @@ +namespace OpenCand.Core.Models +{ + public class Candidato + { + public Guid IdCandidato { get; set; } + + public string Cpf { get; set; } + + public string SqCandidato { get; set; } + + public string Nome { get; set; } + + public DateTime? DataNascimento { get; set; } + + public string Email { get; set; } + + public string Sexo { get; set; } + + public string EstadoCivil { get; set; } + + public string Escolaridade { get; set; } + + public string Ocupacao { get; set; } + + public List Eleicoes { get; set; } + + // API ONLY + public string FotoUrl { get; set; } + } + + public class CandidatoMapping + { + public Guid IdCandidato { get; set; } + public string Cpf { get; set; } + public string Nome { get; set; } + public string SqCandidato { get; set; } + public int Ano { get; set; } + public string TipoEleicao { get; set; } + public string SiglaUF { get; set; } + public string NomeUE { get; set; } + public string Cargo { get; set; } + public string NrCandidato { get; set; } + public string Resultado { get; set; } + } + + public class RedeSocial + { + public Guid IdCandidato { get; set; } + public string SqCandidato { get; set; } + + public string Rede { get; set; } + public string SiglaUF { get; set; } + public int Ano { get; set; } + public string Link { get; set; } + } +} diff --git a/OpenCand.Core/Models/OpenCandStats.cs b/OpenCand.Core/Models/OpenCandStats.cs new file mode 100644 index 0000000..bb64f7c --- /dev/null +++ b/OpenCand.Core/Models/OpenCandStats.cs @@ -0,0 +1,11 @@ +namespace OpenCand.Core.Models +{ + public class OpenCandStats + { + public long TotalCandidatos { get; set; } + public long TotalBemCandidatos { get; set; } + public long TotalValorBemCandidatos { get; set; } + public long TotalRedesSociais { get; set; } + public long TotalEleicoes { get; set; } + } +} diff --git a/OpenCand.Core/OpenCand.Core.csproj b/OpenCand.Core/OpenCand.Core.csproj new file mode 100644 index 0000000..fa71b7a --- /dev/null +++ b/OpenCand.Core/OpenCand.Core.csproj @@ -0,0 +1,9 @@ + + + + net8.0 + enable + enable + + + diff --git a/OpenCand.ETL.dockerfile b/OpenCand.ETL.dockerfile new file mode 100644 index 0000000..3c28bde --- /dev/null +++ b/OpenCand.ETL.dockerfile @@ -0,0 +1,26 @@ +# ─── Stage 1: Build ─────────────────────────────────────────────────────── +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +WORKDIR /src + +# Copy only the solution and project files first, restore, then copy the rest +COPY ./OpenCand.sln ./ +COPY ./OpenCand.ETL/OpenCand.ETL.csproj ./OpenCand.ETL/ +COPY ./OpenCand.Core/OpenCand.Core.csproj ./OpenCand.Core/ + +RUN dotnet restore ./OpenCand.ETL/OpenCand.ETL.csproj + +# Now copy the rest of the ETL source code +COPY ./OpenCand.ETL/. ./OpenCand.ETL/ +COPY ./OpenCand.Core/. ./OpenCand.Core/ + +WORKDIR /src/OpenCand.ETL +RUN dotnet publish -c Release -o /app/publish + +# ─── Stage 2: Runtime ───────────────────────────────────────────────────── +FROM mcr.microsoft.com/dotnet/runtime:8.0 AS runtime +WORKDIR /app +COPY --from=build /app/publish . + +ENV ConnectionStrings__DefaultConnection="Host=db;Port=5432;Database=opencand;Username=root;Password=root" + +ENTRYPOINT ["dotnet", "OpenCand.ETL.dll"] diff --git a/OpenCand.ETL/Config/CsvSettings.cs b/OpenCand.ETL/Config/CsvSettings.cs new file mode 100644 index 0000000..24394ff --- /dev/null +++ b/OpenCand.ETL/Config/CsvSettings.cs @@ -0,0 +1,9 @@ +namespace OpenCand.Config +{ + public class CsvSettings + { + public string CandidatosFolder { get; set; } = string.Empty; + public string BensCandidatosFolder { get; set; } = string.Empty; + public string RedesSociaisFolder { get; set; } = string.Empty; + } +} diff --git a/OpenCand.ETL/OpenCand.ETL.csproj b/OpenCand.ETL/OpenCand.ETL.csproj new file mode 100644 index 0000000..18d19bf --- /dev/null +++ b/OpenCand.ETL/OpenCand.ETL.csproj @@ -0,0 +1,29 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + + + Always + + + Always + + + + diff --git a/OpenCand.ETL/Parser/CsvMappers/BemCandidatoMap.cs b/OpenCand.ETL/Parser/CsvMappers/BemCandidatoMap.cs new file mode 100644 index 0000000..ceb731b --- /dev/null +++ b/OpenCand.ETL/Parser/CsvMappers/BemCandidatoMap.cs @@ -0,0 +1,15 @@ +using CsvHelper.Configuration; +using OpenCand.Parser.Models; +using System.Globalization; + +namespace OpenCand.Parser.CsvMappers +{ + public class BemCandidatoMap : ClassMap + { + public BemCandidatoMap() + { + AutoMap(CultureInfo.InvariantCulture); + // Explicitly handle any special mappings if needed + } + } +} diff --git a/OpenCand.ETL/Parser/CsvMappers/CandidatoMap.cs b/OpenCand.ETL/Parser/CsvMappers/CandidatoMap.cs new file mode 100644 index 0000000..72d4556 --- /dev/null +++ b/OpenCand.ETL/Parser/CsvMappers/CandidatoMap.cs @@ -0,0 +1,15 @@ +using CsvHelper.Configuration; +using OpenCand.Parser.Models; +using System.Globalization; + +namespace OpenCand.Parser.CsvMappers +{ + public class CandidatoMap : ClassMap + { + public CandidatoMap() + { + AutoMap(CultureInfo.InvariantCulture); + // Explicitly handle any special mappings if needed + } + } +} diff --git a/OpenCand.ETL/Parser/CsvMappers/RedeSocialMap.cs b/OpenCand.ETL/Parser/CsvMappers/RedeSocialMap.cs new file mode 100644 index 0000000..779ef61 --- /dev/null +++ b/OpenCand.ETL/Parser/CsvMappers/RedeSocialMap.cs @@ -0,0 +1,14 @@ +using System.Globalization; +using CsvHelper.Configuration; +using OpenCand.Parser.Models; + +namespace OpenCand.ETL.Parser.CsvMappers +{ + public class RedeSocialMap : ClassMap + { + public RedeSocialMap() + { + AutoMap(CultureInfo.InvariantCulture); + } + } +} diff --git a/OpenCand.ETL/Parser/Models/BemCandidatoCSV.cs b/OpenCand.ETL/Parser/Models/BemCandidatoCSV.cs new file mode 100644 index 0000000..8e60d7c --- /dev/null +++ b/OpenCand.ETL/Parser/Models/BemCandidatoCSV.cs @@ -0,0 +1,58 @@ +using CsvHelper.Configuration.Attributes; + +namespace OpenCand.Parser.Models +{ + public class BemCandidatoCSV + { + [Name("DT_GERACAO")] + public string DataGeracao { get; set; } + + [Name("HH_GERACAO")] + public string HoraGeracao { get; set; } + + [Name("ANO_ELEICAO")] + public int AnoEleicao { get; set; } + + [Name("CD_TIPO_ELEICAO")] + public int CodigoTipoEleicao { get; set; } + + [Name("NM_TIPO_ELEICAO")] + public string NomeTipoEleicao { get; set; } + + [Name("CD_ELEICAO")] + public int CodigoEleicao { get; set; } + + [Name("DS_ELEICAO")] + public string DescricaoEleicao { get; set; } + + [Name("DT_ELEICAO")] + public string DataEleicao { get; set; } + + [Name("SG_UF")] + public string SiglaUF { get; set; } + + [Name("SG_UE")] + public string SiglaUE { get; set; } + + [Name("NM_UE")] + public string NomeUE { get; set; } + + [Name("SQ_CANDIDATO")] + public string SequencialCandidato { get; set; } + + [Name("NR_ORDEM_BEM_CANDIDATO")] + public int NumeroOrdemBemCandidato { get; set; } + + [Name("CD_TIPO_BEM_CANDIDATO")] + public int CodigoTipoBemCandidato { get; set; } + + [Name("DS_TIPO_BEM_CANDIDATO")] + public string DescricaoTipoBemCandidato { get; set; } + + [Name("DS_BEM_CANDIDATO")] + public string DescricaoBemCandidato { get; set; } + + [Name("VR_BEM_CANDIDATO")] + public string ValorBemCandidato { get; set; } + } +} diff --git a/OpenCand.ETL/Parser/Models/CandidatoCSV.cs b/OpenCand.ETL/Parser/Models/CandidatoCSV.cs new file mode 100644 index 0000000..6a2acd4 --- /dev/null +++ b/OpenCand.ETL/Parser/Models/CandidatoCSV.cs @@ -0,0 +1,95 @@ +using System; +using CsvHelper.Configuration.Attributes; + +namespace OpenCand.Parser.Models +{ + public class CandidatoCSV + { + [Name("DT_GERACAO")] + public string DataGeracao { get; set; } + + [Name("HH_GERACAO")] + public string HoraGeracao { get; set; } + + [Name("ANO_ELEICAO")] + public int AnoEleicao { get; set; } + + [Name("CD_TIPO_ELEICAO")] + public int CodigoTipoEleicao { get; set; } + + [Name("NM_TIPO_ELEICAO")] + public string NomeTipoEleicao { get; set; } + + [Name("NR_TURNO")] + public int NumeroTurno { get; set; } + + [Name("CD_ELEICAO")] + public int CodigoEleicao { get; set; } + + [Name("DS_ELEICAO")] + public string DescricaoEleicao { get; set; } + + [Name("DT_ELEICAO")] + public string DataEleicao { get; set; } + + [Name("TP_ABRANGENCIA")] + public string TipoAbrangencia { get; set; } + + [Name("SG_UF")] + public string SiglaUF { get; set; } + + [Name("SG_UE")] + public string SiglaUE { get; set; } + + [Name("NM_UE")] + public string NomeUE { get; set; } + + [Name("CD_CARGO")] + public int CodigoCargo { get; set; } + + [Name("DS_CARGO")] + public string DescricaoCargo { get; set; } + + [Name("SQ_CANDIDATO")] + public string SequencialCandidato { get; set; } + + [Name("NR_CANDIDATO")] + public string NumeroCandidato { get; set; } + + [Name("NM_CANDIDATO")] + public string NomeCandidato { get; set; } + + [Name("NM_URNA_CANDIDATO")] + public string NomeUrnaCandidato { get; set; } + + [Name("NM_SOCIAL_CANDIDATO")] + public string NomeSocialCandidato { get; set; } + + [Name("NR_CPF_CANDIDATO")] + public string CPFCandidato { get; set; } + + [Name("DS_EMAIL", "NM_EMAIL")] + public string Email { get; set; } + + [Name("SG_UF_NASCIMENTO")] + public string SiglaUFNascimento { get; set; } + + [Name("DT_NASCIMENTO")] + public string DataNascimento { get; set; } + + [Name("DS_GENERO")] + public string Genero { get; set; } + + [Name("DS_OCUPACAO")] + public string Ocupacao { get; set; } + + [Name("DS_ESTADO_CIVIL")] + public string EstadoCivil { get; set; } + + [Name("DS_GRAU_INSTRUCAO")] + public string GrauInstrucao { get; set; } + + [Name("DS_SIT_TOT_TURNO")] + public string SituacaoTurno { get; set; } + } +} diff --git a/OpenCand.ETL/Parser/Models/RedeSocialCSV.cs b/OpenCand.ETL/Parser/Models/RedeSocialCSV.cs new file mode 100644 index 0000000..ab4615e --- /dev/null +++ b/OpenCand.ETL/Parser/Models/RedeSocialCSV.cs @@ -0,0 +1,19 @@ +using CsvHelper.Configuration.Attributes; + +namespace OpenCand.Parser.Models +{ + public class RedeSocialCSV + { + [Name("AA_ELEICAO")] + public int DataEleicao { get; set; } + + [Name("SG_UF")] + public string SiglaUF { get; set; } + + [Name("SQ_CANDIDATO")] + public string SequencialCandidato { get; set; } + + [Name("DS_URL")] + public string Url { get; set; } + } +} diff --git a/OpenCand.ETL/Parser/ParserManager.cs b/OpenCand.ETL/Parser/ParserManager.cs new file mode 100644 index 0000000..839166a --- /dev/null +++ b/OpenCand.ETL/Parser/ParserManager.cs @@ -0,0 +1,118 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using OpenCand.Config; +using OpenCand.Parser.Services; + +namespace OpenCand.Parser +{ + public class ParserManager + { + private readonly CsvParserService csvParserService; + private readonly ILogger logger; + private readonly CsvSettings csvSettings; + private readonly IConfiguration configuration; + + public ParserManager( + CsvParserService csvParserService, + IOptions csvSettings, + ILogger logger, + IConfiguration configuration) + { + this.csvParserService = csvParserService; + this.logger = logger; + this.csvSettings = csvSettings.Value; + this.configuration = configuration; + } + + public async Task ParseFullDataAsync() + { + logger.LogInformation("ParseFullDataAsync - Starting parsing"); + + // Get the base path from either SampleFolder in csvSettings or the BasePath in configuration + var basePath = configuration.GetValue("BasePath"); + + if (string.IsNullOrEmpty(basePath)) + { + logger.LogError("ParseFullDataAsync - BasePath is not configured in appsettings.json or CsvSettings.SampleFolder"); + return; + } + + logger.LogInformation("ParseFullDataAsync - Processing will happen with BasePath: {BasePath}", basePath); + + try + { + var candidatosDirectory = Path.Combine(basePath, csvSettings.CandidatosFolder); + var bensCandidatosDirectory = Path.Combine(basePath, csvSettings.BensCandidatosFolder); + var redesSociaisDirectory = Path.Combine(basePath, csvSettings.RedesSociaisFolder); + + if (Directory.Exists(candidatosDirectory)) + { + foreach (var filePath in Directory.GetFiles(candidatosDirectory, "*.csv")) + { + // Check if filePath contains "fix_" prefix + if (filePath.Contains("fix_")) + { + logger.LogInformation("ParseFullDataAsync - Skipping already fixed file: {FilePath}", filePath); + continue; + } + + logger.LogInformation("ParseFullDataAsync - Parsing candidatos data from {FilePath}", filePath); + await csvParserService.ParseCandidatosAsync(filePath); + } + } + else + { + logger.LogWarning("ParseFullDataAsync - 'Candidatos' directory not found at {Directory}", candidatosDirectory); + } + + if (Directory.Exists(bensCandidatosDirectory)) + { + foreach (var filePath in Directory.GetFiles(bensCandidatosDirectory, "*.csv")) + { + // Check if filePath contains "fix_" prefix + if (filePath.Contains("fix_")) + { + logger.LogInformation("ParseFullDataAsync - Skipping already fixed file: {FilePath}", filePath); + continue; + } + + logger.LogInformation("ParseFullDataAsync - Parsing bens candidatos data from {FilePath}", filePath); + await csvParserService.ParseBensCandidatosAsync(filePath); + } + } + else + { + logger.LogWarning("ParseFullDataAsync - 'Bens candidatos' directory not found at {Directory}", bensCandidatosDirectory); + } + + if (Directory.Exists(redesSociaisDirectory)) + { + foreach (var filePath in Directory.GetFiles(redesSociaisDirectory, "*.csv")) + { + // Check if filePath contains "fix_" prefix + if (filePath.Contains("fix_")) + { + logger.LogInformation("ParseFullDataAsync - Skipping already fixed file: {FilePath}", filePath); + continue; + } + + logger.LogInformation("ParseFullDataAsync - Parsing redes sociais data from {FilePath}", filePath); + await csvParserService.ParseRedeSocialAsync(filePath); + } + } + else + { + logger.LogWarning("ParseFullDataAsync - 'Redes sociais' directory not found at {Directory}", redesSociaisDirectory); + } + + logger.LogInformation("ParseFullDataAsync - Full data parsing completed!"); + } + catch (Exception ex) + { + logger.LogError(ex, "ParseFullDataAsync - Error parsing full data set"); + throw; + } + } + } +} diff --git a/OpenCand.ETL/Parser/Services/CsvFixerService.cs b/OpenCand.ETL/Parser/Services/CsvFixerService.cs new file mode 100644 index 0000000..a932aef --- /dev/null +++ b/OpenCand.ETL/Parser/Services/CsvFixerService.cs @@ -0,0 +1,119 @@ +using System.Text; +using Microsoft.Extensions.Logging; +using OpenCand.Repository; + +namespace OpenCand.Parser.Services +{ + public class CsvFixerService + { + private readonly ILogger logger; + + public CsvFixerService( + ILogger logger) + { + this.logger = logger; + } + + public string FixCsvFile(string filePath) + { + var filename = Path.GetFileName(filePath); + var path = Path.GetDirectoryName(filePath); + + // Check if the file exists + if (!File.Exists(filePath)) + { + logger.LogError($"FixCsvFile - The file at '{filePath}' does not exist"); + return string.Empty; + } + + if (string.IsNullOrEmpty(filename) || string.IsNullOrEmpty(path)) + { + logger.LogError($"FixCsvFile - The file path '{filePath}' is invalid"); + return string.Empty; + } + + // Fixed file will have the same name but with "fix_" prefix + var newFilePath = Path.Combine(path, $"fix_{filename}"); + if (File.Exists(newFilePath)) + { + logger.LogWarning($"FixCsvFile - A fixed file already exists at '{newFilePath}'. It will be overwritten."); + } + + logger.LogInformation($"FixCsvFile - Starting to fix CSV file at '{filePath}'"); + + try + { + // Read the file + var lines = File.ReadAllLines(filePath, encoding: Encoding.GetEncoding(1252)); + + if (lines.Length == 0) + { + logger.LogError($"FixCsvFile - The file at '{filePath}' is empty"); + return string.Empty; + } + + var newLines = new List(); + + var headerCount = lines[0].Split(';').Length; + + if (headerCount == 0) + { + logger.LogError($"FixCsvFile - The first line of the file at '{filePath}' does not contain any headers"); + return string.Empty; + } + + logger.LogInformation($"FixCsvFile - Detected {headerCount} headers in the CSV file"); + + for (int i = 0; i < lines.Length;) + { + var line = lines[i]; + var columns = line.Split(';'); + var lineJump = 1; + + while (columns.Length != headerCount) + { + if (columns.Length > headerCount) + { + logger.LogCritical($"FixCsvFile - Line {i + 1} has {columns.Length} columns, expected {headerCount}. Halting process."); + return string.Empty; // Critical error, cannot fix this line => needs manual intervention + } + + logger.LogWarning($"FixCsvFile - Line {i + 1} has {columns.Length} columns, expected {headerCount}. Attempting to fix [i = {lineJump}]..."); + + // Likely the "original line" had some \n that were processed incorrectly + // Append lines[i + 1] to the current line and re-do the check + + if (i + lineJump >= lines.Length) + { + logger.LogCritical($"FixCsvFile - Reached the end of the file while trying to fix line {i + 1}. Cannot continue."); + return string.Empty; // Cannot fix this line, reached the end of the file + } + + // Append the next line to the current line + line += lines[i + lineJump]; + + // Re-split the line to check the number of columns again + columns = line.Split(';'); + + // increment lineJump + lineJump++; + } + + newLines.Add(line); + i += lineJump; + } + + // Write the fixed lines to the new filepath + File.WriteAllLines(newFilePath, newLines, Encoding.UTF8); + + logger.LogInformation($"FixCsvFile - Successfully fixed CSV file at {newFilePath}"); + return newFilePath; + } + catch (Exception ex) + { + logger.LogError(ex, $"FixCsvFile - Error fixing CSV file at {filePath}"); + return string.Empty; + } + } + } +} diff --git a/OpenCand.ETL/Parser/Services/CsvParserService.cs b/OpenCand.ETL/Parser/Services/CsvParserService.cs new file mode 100644 index 0000000..47ddd8b --- /dev/null +++ b/OpenCand.ETL/Parser/Services/CsvParserService.cs @@ -0,0 +1,268 @@ +using System.Globalization; +using CsvHelper; +using CsvHelper.Configuration; +using Microsoft.Extensions.Logging; +using OpenCand.Core.Models; +using OpenCand.ETL.Parser.CsvMappers; +using OpenCand.Parser.CsvMappers; +using OpenCand.Parser.Models; +using OpenCand.Services; + +namespace OpenCand.Parser.Services +{ + public class CsvParserService + { + private readonly ILogger logger; + private readonly CandidatoService candidatoService; + private readonly BemCandidatoService bemCandidatoService; + private readonly RedeSocialService redeSocialService; + private readonly CsvFixerService csvFixerService; + + private readonly CsvConfiguration parserConfig; + + public CsvParserService( + ILogger logger, + CandidatoService candidatoService, + BemCandidatoService bemCandidatoService, + RedeSocialService redeSocialService, + CsvFixerService csvFixerService) + { + this.logger = logger; + this.candidatoService = candidatoService; + this.bemCandidatoService = bemCandidatoService; + this.redeSocialService = redeSocialService; + this.csvFixerService = csvFixerService; + + parserConfig = new CsvConfiguration(CultureInfo.InvariantCulture) + { + Delimiter = ";", + HasHeaderRecord = true, + PrepareHeaderForMatch = args => args.Header.ToLower(), + MissingFieldFound = null, + TrimOptions = TrimOptions.Trim, + Encoding = System.Text.Encoding.UTF8 + }; + } + + public async Task ParseCandidatosAsync(string filePath) + { + logger.LogInformation($"ParseCandidatosAsync - Starting to parse 'candidatos' from '{filePath}'"); + + filePath = csvFixerService.FixCsvFile(filePath); + + // Fix the CSV file if necessary + if (string.IsNullOrEmpty(filePath)) + { + logger.LogError($"ParseCandidatosAsync - Failed to fix CSV file at '{filePath}'"); + throw new InvalidOperationException($"Failed to fix CSV file at '{filePath}'"); + } + + try + { + using var reader = new StreamReader(filePath); + using var csv = new CsvReader(reader, parserConfig); + var po = new ParallelOptions + { + MaxDegreeOfParallelism = 100 + }; + + csv.Context.RegisterClassMap(); + + var records = csv.GetRecords(); + + await Parallel.ForEachAsync(records, po, async (record, ct) => + { + try + { + if (string.IsNullOrWhiteSpace(record.CPFCandidato) || record.CPFCandidato.Length <= 3) + { + record.CPFCandidato = null; // Handle null/empty/whitespace CPF + } + + if (record.NomeCandidato == "NÃO DIVULGÁVEL" || + string.IsNullOrEmpty(record.NomeCandidato) || + record.NomeCandidato == "#NULO") + { + logger.LogCritical($"ParseCandidatosAsync - Candidate with id {record.SequencialCandidato} with invalid name, skipping..."); + return; // Skip candidates with invalid name + } + + var candidato = new Candidato + { + Cpf = record.CPFCandidato, + SqCandidato = record.SequencialCandidato, + Nome = record.NomeCandidato, + Email = record.Email.Contains("@") ? record.Email : null, + Sexo = record.Genero, + EstadoCivil = record.EstadoCivil, + Escolaridade = record.GrauInstrucao, + Ocupacao = record.Ocupacao, + Eleicoes = new List() + { + new CandidatoMapping + { + Cpf = record.CPFCandidato, + Nome = record.NomeCandidato, + SqCandidato = record.SequencialCandidato, + Ano = record.AnoEleicao, + TipoEleicao = record.TipoAbrangencia, + NomeUE = record.NomeUE, + SiglaUF = record.SiglaUF, + Cargo = record.DescricaoCargo, + NrCandidato = record.NumeroCandidato, + Resultado = record.SituacaoTurno, + } + } + }; + + if (!string.IsNullOrEmpty(record.DataNascimento) && + record.DataNascimento != "#NULO") + { + if (DateTime.TryParseExact(record.DataNascimento, "dd/MM/yyyy", + CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var dataNascimento)) + { + // Convert to UTC DateTime to work with PostgreSQL timestamp with time zone + candidato.DataNascimento = DateTime.SpecifyKind(dataNascimento, DateTimeKind.Utc); + } + } + else + { + candidato.DataNascimento = null; // Handle null/empty/whitespace date + } + + await candidatoService.AddCandidatoAsync(candidato); + } + catch (Exception ex) + { + logger.LogError(ex, "ParseCandidatosAsync - Error processing candidate with id {CandidatoId}", record.SequencialCandidato); + } + }); + + logger.LogInformation("ParseCandidatosAsync - Finished parsing candidatos from {FilePath}", filePath); + } + catch (Exception ex) + { + logger.LogError(ex, "ParseCandidatosAsync - Error parsing candidatos file {FilePath}", filePath); + throw; + } + } + + public async Task ParseBensCandidatosAsync(string filePath) + { + logger.LogInformation($"ParseBensCandidatosAsync - Starting to parse bens candidatos from '{filePath}'"); + + filePath = csvFixerService.FixCsvFile(filePath); + + // Fix the CSV file if necessary + if (string.IsNullOrEmpty(filePath)) + { + logger.LogError($"ParseBensCandidatosAsync - Failed to fix CSV file at '{filePath}'"); + throw new InvalidOperationException($"Failed to fix CSV file at '{filePath}'"); + } + + try + { + using var reader = new StreamReader(filePath); + using var csv = new CsvReader(reader, parserConfig); + csv.Context.RegisterClassMap(); + + var records = csv.GetRecords(); + + foreach (var record in records) + { + try + { + // Parse decimal value + decimal? valor = null; + if (!string.IsNullOrEmpty(record.ValorBemCandidato)) + { + string normalizedValue = record.ValorBemCandidato.Replace(".", "").Replace(",", "."); + if (decimal.TryParse(normalizedValue, NumberStyles.Any, CultureInfo.InvariantCulture, out var parsedValue)) + { + valor = parsedValue; + } + } + + var bemCandidato = new BemCandidato + { + SqCandidato = record.SequencialCandidato, + Ano = record.AnoEleicao, + SiglaUF = record.SiglaUF, + NomeUE = record.NomeUE, + OrdemBem = record.NumeroOrdemBemCandidato, + TipoBem = record.DescricaoTipoBemCandidato, + Descricao = record.DescricaoBemCandidato, + Valor = valor + }; + + await bemCandidatoService.AddBemCandidatoAsync(bemCandidato); + } + catch (Exception ex) + { + logger.LogError(ex, "ParseBensCandidatosAsync - Error processing bem candidato with id {CandidatoId} and ordem {OrdemBem}", + record.SequencialCandidato, record.NumeroOrdemBemCandidato); + } + } + + logger.LogInformation("ParseBensCandidatosAsync - Finished parsing bens candidatos from {FilePath}", filePath); + } + catch (Exception ex) + { + logger.LogError(ex, "ParseBensCandidatosAsync - Error parsing bens candidatos file {FilePath}", filePath); + throw; + } + } + + public async Task ParseRedeSocialAsync(string filePath) + { + logger.LogInformation($"ParseRedeSocialAsync - Starting to parse redes sociais from '{filePath}'"); + + filePath = csvFixerService.FixCsvFile(filePath); + + // Fix the CSV file if necessary + if (string.IsNullOrEmpty(filePath)) + { + logger.LogError($"ParseRedeSocialAsync - Failed to fix CSV file at '{filePath}'"); + throw new InvalidOperationException($"Failed to fix CSV file at '{filePath}'"); + } + + try + { + using var reader = new StreamReader(filePath); + using var csv = new CsvReader(reader, parserConfig); + csv.Context.RegisterClassMap(); + + var records = csv.GetRecords(); + + foreach (var record in records) + { + try + { + var redeSocial = new RedeSocial + { + SqCandidato = record.SequencialCandidato, + Ano = record.DataEleicao, + SiglaUF = record.SiglaUF, + Link = record.Url, + Rede = string.Empty + }; + + await redeSocialService.AddRedeSocialAsync(redeSocial); + } + catch (Exception ex) + { + logger.LogError(ex, "ParseRedeSocialAsync - Error processing redes sociais with id {SequencialCandidato} and link {Url}", + record.SequencialCandidato, record.Url); + } + } + + logger.LogInformation("ParseRedeSocialAsync - Finished parsing redes sociais from {FilePath}", filePath); + } + catch (Exception ex) + { + logger.LogError(ex, "ParseRedeSocialAsync - Error parsing redes sociais file {FilePath}", filePath); + throw; + } + } + } +} diff --git a/OpenCand.ETL/Program.cs b/OpenCand.ETL/Program.cs new file mode 100644 index 0000000..33397c3 --- /dev/null +++ b/OpenCand.ETL/Program.cs @@ -0,0 +1,70 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using OpenCand.Config; +using OpenCand.Parser; +using OpenCand.Parser.Services; +using OpenCand.Repository; +using OpenCand.Services; + +namespace OpenCand +{ + public class Program + { + static async Task Main(string[] args) + { + var host = CreateHostBuilder(args).Build(); + + using (var scope = host.Services.CreateScope()) + { + var services = scope.ServiceProvider; + var logger = services.GetRequiredService>(); + + try + { + logger.LogInformation("Initializing database"); + // make a test connection to the database + + logger.LogInformation("Starting data parsing"); + var parserManager = services.GetRequiredService(); + await parserManager.ParseFullDataAsync(); + + logger.LogInformation("Data parsing completed successfully!"); + } + catch (Exception ex) + { + logger.LogError(ex, "An error occurred during application startup"); + } + } + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration((hostingContext, config) => + { + config.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true); + config.AddJsonFile($"appsettings.{hostingContext.HostingEnvironment.EnvironmentName}.json", optional: true); + config.AddEnvironmentVariables(); + config.AddCommandLine(args); + + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + }) + .ConfigureServices((hostContext, services) => + { + // Configuration + services.Configure(hostContext.Configuration.GetSection("CsvSettings")); + + // Services + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + }); + } +} diff --git a/OpenCand.ETL/Properties/launchSettings.json b/OpenCand.ETL/Properties/launchSettings.json new file mode 100644 index 0000000..6cba6e6 --- /dev/null +++ b/OpenCand.ETL/Properties/launchSettings.json @@ -0,0 +1,10 @@ +{ + "profiles": { + "OpenCand": { + "commandName": "Project", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} \ No newline at end of file diff --git a/OpenCand.ETL/Repository/BaseRepository.cs b/OpenCand.ETL/Repository/BaseRepository.cs new file mode 100644 index 0000000..8575077 --- /dev/null +++ b/OpenCand.ETL/Repository/BaseRepository.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.Configuration; +using Npgsql; + +namespace OpenCand.Repository +{ + public abstract class BaseRepository + { + protected string ConnectionString { get; private set; } + protected NpgsqlConnection? Connection { get; private set; } + + public BaseRepository(IConfiguration configuration) + { + ConnectionString = configuration["DatabaseSettings:ConnectionString"] ?? + throw new ArgumentNullException("Connection string not found in configuration"); + } + } +} diff --git a/OpenCand.ETL/Repository/BemCandidatoRepository.cs b/OpenCand.ETL/Repository/BemCandidatoRepository.cs new file mode 100644 index 0000000..123b68d --- /dev/null +++ b/OpenCand.ETL/Repository/BemCandidatoRepository.cs @@ -0,0 +1,37 @@ +using Dapper; +using Microsoft.Extensions.Configuration; +using Npgsql; +using OpenCand.Core.Models; + +namespace OpenCand.Repository +{ + public class BemCandidatoRepository : BaseRepository + { + public BemCandidatoRepository(IConfiguration configuration) : base(configuration) + { + } + + public async Task AddBemCandidatoAsync(BemCandidato bemCandidato) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + await connection.ExecuteAsync(@" + INSERT INTO bem_candidato (idcandidato, ano, ordembem, tipobem, descricao, valor) + VALUES (@idcandidato, @ano, @ordembem, @tipobem, @descricao, @valor) + ON CONFLICT (idcandidato, ano, ordembem) DO UPDATE SET + tipobem = EXCLUDED.tipobem, + descricao = EXCLUDED.descricao, + valor = EXCLUDED.valor;", + new + { + idcandidato = bemCandidato.IdCandidato, + ano = bemCandidato.Ano, + ordembem = bemCandidato.OrdemBem, + tipobem = bemCandidato.TipoBem, + descricao = bemCandidato.Descricao, + valor = bemCandidato.Valor + }); + } + } + } +} diff --git a/OpenCand.ETL/Repository/CandidatoRepository.cs b/OpenCand.ETL/Repository/CandidatoRepository.cs new file mode 100644 index 0000000..d573a4f --- /dev/null +++ b/OpenCand.ETL/Repository/CandidatoRepository.cs @@ -0,0 +1,122 @@ +using Dapper; +using Microsoft.Extensions.Configuration; +using Npgsql; +using OpenCand.Core.Models; + +namespace OpenCand.Repository +{ + public class CandidatoRepository : BaseRepository + { + public CandidatoRepository(IConfiguration configuration) : base(configuration) + { + } + + public async Task AddCandidatoAsync(Candidato candidato) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + await connection.ExecuteAsync(@" + INSERT INTO candidato (idcandidato, cpf, nome, datanascimento, email, sexo, estadocivil, escolaridade, ocupacao) + VALUES (@idcandidato, @cpf, @nome, @datanascimento, @email, @sexo, @estadocivil, @escolaridade, @ocupacao) + ON CONFLICT (idcandidato) DO UPDATE SET + cpf = EXCLUDED.cpf, + nome = EXCLUDED.nome, + datanascimento = EXCLUDED.datanascimento, + email = EXCLUDED.email, + sexo = EXCLUDED.sexo, + estadocivil = EXCLUDED.estadocivil, + escolaridade = EXCLUDED.escolaridade, + ocupacao = EXCLUDED.ocupacao;", + new + { + idcandidato = candidato.IdCandidato, + cpf = candidato.Cpf, + nome = candidato.Nome, + datanascimento = candidato.DataNascimento, + email = candidato.Email, + sexo = candidato.Sexo, + estadocivil = candidato.EstadoCivil, + escolaridade = candidato.Escolaridade, + ocupacao = candidato.Ocupacao + }); + } + } + + public async Task AddCandidatoMappingAsync(CandidatoMapping candidatoMapping) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + await connection.ExecuteAsync(@" + INSERT INTO candidato_mapping (idcandidato, cpf, nome, sqcandidato, ano, tipoeleicao, siglauf, nomeue, cargo, nrcandidato, resultado) + VALUES (@idcandidato, @cpf, @nome, @sqcandidato, @ano, @tipoeleicao, @siglauf, @nomeue, @cargo, @nrcandidato, @resultado);", + new + { + idcandidato = candidatoMapping.IdCandidato, + cpf = candidatoMapping.Cpf, + nome = candidatoMapping.Nome, + sqcandidato = candidatoMapping.SqCandidato, + ano = candidatoMapping.Ano, + tipoeleicao = candidatoMapping.TipoEleicao, + siglauf = candidatoMapping.SiglaUF, + nomeue = candidatoMapping.NomeUE, + nrcandidato = candidatoMapping.NrCandidato, + cargo = candidatoMapping.Cargo, + resultado = candidatoMapping.Resultado + }); + } + } + + public async Task?> GetCandidatoMappingByCpf(string cpf) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var query = @" + SELECT idcandidato, cpf, nome, sqcandidato, ano, tipoeleicao, siglauf, nomeue, cargo, nrcandidato, resultado + FROM candidato_mapping + WHERE cpf = @cpf"; + return (await connection.QueryAsync(query, new { cpf })).AsList(); + } + } + + public async Task?> GetCandidatoMappingByNome(string nome) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var query = @" + SELECT idcandidato, cpf, nome, sqcandidato, ano, tipoeleicao, siglauf, nomeue, cargo, nrcandidato, resultado + FROM candidato_mapping + WHERE nome = @nome"; + return (await connection.QueryAsync(query, new { nome })).AsList(); + } + } + + public async Task GetIdCandidatoBySqCandidato(string sqCandidato, int ano, string siglauf, string nomeue) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var query = @" + SELECT idcandidato + FROM candidato_mapping + WHERE sqcandidato = @sqCandidato AND + ano = @ano AND + siglauf = @siglauf AND + nomeue = @nomeue"; + return await connection.QueryFirstOrDefaultAsync(query, new { sqCandidato, ano, siglauf, nomeue }); + } + } + + public async Task GetIdCandidatoBySqCandidato(string sqCandidato, int ano, string siglauf) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var query = @" + SELECT idcandidato + FROM candidato_mapping + WHERE sqcandidato = @sqCandidato AND + ano = @ano AND + siglauf = @siglauf"; + return await connection.QueryFirstOrDefaultAsync(query, new { sqCandidato, ano, siglauf }); + } + } + } +} diff --git a/OpenCand.ETL/Repository/RedeSocialRepository.cs b/OpenCand.ETL/Repository/RedeSocialRepository.cs new file mode 100644 index 0000000..77ebfdc --- /dev/null +++ b/OpenCand.ETL/Repository/RedeSocialRepository.cs @@ -0,0 +1,34 @@ +using Dapper; +using Microsoft.Extensions.Configuration; +using Npgsql; +using OpenCand.Core.Models; + +namespace OpenCand.Repository +{ + public class RedeSocialRepository : BaseRepository + { + public RedeSocialRepository(IConfiguration configuration) : base(configuration) + { + } + + public async Task AddRedeSocialAsync(RedeSocial redeSocial) + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + await connection.ExecuteAsync(@" + INSERT INTO rede_social (idcandidato, rede, siglauf, ano, link) + VALUES (@idcandidato, @rede, @siglauf, @ano, @link) + ON CONFLICT (idcandidato, rede, siglauf, ano) DO UPDATE SET + link = EXCLUDED.link;", + new + { + idcandidato = redeSocial.IdCandidato, + rede = redeSocial.Rede, + siglauf = redeSocial.SiglaUF, + ano = redeSocial.Ano, + link = redeSocial.Link + }); + } + } + } +} diff --git a/OpenCand.ETL/Services/BemCandidatoService.cs b/OpenCand.ETL/Services/BemCandidatoService.cs new file mode 100644 index 0000000..42c7fd3 --- /dev/null +++ b/OpenCand.ETL/Services/BemCandidatoService.cs @@ -0,0 +1,37 @@ +using OpenCand.Core.Models; +using OpenCand.Repository; + +namespace OpenCand.Services +{ + public class BemCandidatoService + { + private readonly CandidatoRepository candidatoRepository; + private readonly BemCandidatoRepository bemCandidatoRepository; + + public BemCandidatoService(CandidatoRepository candidatoRepository, BemCandidatoRepository bemCandidatoRepository) + { + this.candidatoRepository = candidatoRepository; + this.bemCandidatoRepository = bemCandidatoRepository; + } + + public async Task AddBemCandidatoAsync(BemCandidato bemCandidato) + { + if (bemCandidato == null || string.IsNullOrWhiteSpace(bemCandidato.SqCandidato)) + { + throw new ArgumentNullException(nameof(bemCandidato), "BemCandidato cannot be null"); + } + + // Get idCandidato from CandidatoRepository + var candidato = await candidatoRepository.GetIdCandidatoBySqCandidato(bemCandidato.SqCandidato, bemCandidato.Ano, bemCandidato.SiglaUF, bemCandidato.NomeUE); + + if (candidato == null || candidato.IdCandidato == Guid.Empty) + { + throw new InvalidOperationException($"AddBemCandidatoAsync - Candidato '{bemCandidato.SqCandidato}'/{bemCandidato.Ano}/'{bemCandidato.SiglaUF}'/'{bemCandidato.NomeUE}' not found."); + } + + bemCandidato.IdCandidato = candidato.IdCandidato; + + await bemCandidatoRepository.AddBemCandidatoAsync(bemCandidato); + } + } +} diff --git a/OpenCand.ETL/Services/CandidatoService.cs b/OpenCand.ETL/Services/CandidatoService.cs new file mode 100644 index 0000000..f23ec39 --- /dev/null +++ b/OpenCand.ETL/Services/CandidatoService.cs @@ -0,0 +1,85 @@ +using OpenCand.Core.Models; +using OpenCand.Repository; + +namespace OpenCand.Services +{ + public class CandidatoService + { + private readonly CandidatoRepository candidatoRepository; + + public CandidatoService(CandidatoRepository candidatoRepository) + { + this.candidatoRepository = candidatoRepository; + } + + public async Task AddCandidatoAsync(Candidato candidato) + { + if (candidato == null) + { + throw new ArgumentNullException(nameof(candidato), "Candidato cannot be null"); + } + + if (candidato.Eleicoes == null || candidato.Eleicoes.Count == 0) + { + throw new ArgumentException("Candidato must have at least one mapping", nameof(candidato)); + } + + var candidatoMapping = candidato.Eleicoes.First(); + + List? mappings = null; + CandidatoMapping? existingMapping = null; + if (candidato.Cpf == null || candidato.Cpf.Length != 11) + { + mappings = await candidatoRepository.GetCandidatoMappingByNome(candidato.Nome); + } + else + { + mappings = await candidatoRepository.GetCandidatoMappingByCpf(candidato.Cpf); + } + + // Check if exists + if (mappings != null && mappings.Count > 0) + { + existingMapping = mappings.FirstOrDefault(m => m.Ano == candidatoMapping.Ano && + m.Cargo == candidatoMapping.Cargo && + m.SiglaUF == candidatoMapping.SiglaUF && + m.NomeUE == candidatoMapping.NomeUE && + m.NrCandidato == candidatoMapping.NrCandidato && + m.Resultado == candidatoMapping.Resultado); + + // Already exists one for the current election + if (existingMapping != null) + { + candidato.IdCandidato = existingMapping.IdCandidato; + candidato.Cpf = existingMapping.Cpf; + + await candidatoRepository.AddCandidatoAsync(candidato); + return; + } + // If exists (but not for the current election), we take the existing idcandidato + // and create a new mapping for the current election + else + { + existingMapping = mappings.First(); + candidato.IdCandidato = existingMapping.IdCandidato; + candidato.Cpf = existingMapping.Cpf; + } + } + else + { + // No current mapping, we create a new one + // and create a new mapping for the current election + candidato.IdCandidato = Guid.NewGuid(); + } + + // Set the mapping properties + candidatoMapping.IdCandidato = candidato.IdCandidato; + candidatoMapping.Cpf = candidato.Cpf; + candidatoMapping.Nome = candidato.Nome; + + await candidatoRepository.AddCandidatoAsync(candidato); + await candidatoRepository.AddCandidatoMappingAsync(candidatoMapping); + } + + } +} diff --git a/OpenCand.ETL/Services/RedeSocialService.cs b/OpenCand.ETL/Services/RedeSocialService.cs new file mode 100644 index 0000000..d18cd55 --- /dev/null +++ b/OpenCand.ETL/Services/RedeSocialService.cs @@ -0,0 +1,74 @@ +using OpenCand.Core.Models; +using OpenCand.Repository; + +namespace OpenCand.Services +{ + public class RedeSocialService + { + private readonly CandidatoRepository candidatoRepository; + private readonly RedeSocialRepository redeSocialRepository; + + public RedeSocialService(CandidatoRepository candidatoRepository, RedeSocialRepository redeSocialRepository) + { + this.candidatoRepository = candidatoRepository; + this.redeSocialRepository = redeSocialRepository; + } + + public async Task AddRedeSocialAsync(RedeSocial redeSocial) + { + if (redeSocial == null || string.IsNullOrWhiteSpace(redeSocial.SqCandidato)) + { + throw new ArgumentNullException(nameof(redeSocial), "RedeSocial cannot be null"); + } + + // Get idCandidato from CandidatoRepository + var candidato = await candidatoRepository.GetIdCandidatoBySqCandidato(redeSocial.SqCandidato, redeSocial.Ano, redeSocial.SiglaUF); + + if (candidato == null || candidato.IdCandidato == Guid.Empty) + { + throw new InvalidOperationException($"AddRedeSocialAsync - Candidato '{redeSocial.SqCandidato}'/{redeSocial.Ano}/'{redeSocial.SiglaUF}' not found."); + } + + redeSocial.IdCandidato = candidato.IdCandidato; + redeSocial.Rede = GetRedeSocialType(redeSocial.Link); + + await redeSocialRepository.AddRedeSocialAsync(redeSocial); + } + + private string GetRedeSocialType(string url) + { + switch (url.ToLower()) + { + case var s when s.Contains("facebook.com"): + return "Facebook"; + case var s when s.Contains("twitter.com"): + case var ss when ss.Contains("x.com"): + return "X/Twitter"; + case var s when s.Contains("instagram.com"): + return "Instagram"; + case var s when s.Contains("youtube.com"): + return "YouTube"; + case var s when s.Contains("linkedin.com"): + return "LinkedIn"; + case var s when s.Contains("spotify.com"): + return "Spotify"; + case var s when s.Contains("kwai.com"): + return "Kwai"; + case var s when s.Contains("tiktok.com"): + return "TikTok"; + case var s when s.Contains("threads.com"): + case var ss when ss.Contains("threads.net"): + return "Threads"; + case var s when s.Contains("t.me"): + case var ss when ss.Contains("telegram.com"): + return "Telegram"; + case var s when s.Contains("api.whatsapp"): + case var ss when ss.Contains("whatsapp.com"): + case var sss when sss.Contains("wa.me"): + return "WhatsApp"; + default: + return "Outros"; + } + } + } +} diff --git a/OpenCand.ETL/appsettings.json b/OpenCand.ETL/appsettings.json new file mode 100644 index 0000000..fb386ca --- /dev/null +++ b/OpenCand.ETL/appsettings.json @@ -0,0 +1,18 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "DatabaseSettings": { + "ConnectionString": "Host=localhost;Database=opencand;Username=root;Password=root;Include Error Detail=true;CommandTimeout=300" + }, + "CsvSettings": { + "CandidatosFolder": "data\\consulta_cand", + "BensCandidatosFolder": "data\\bem_candidato", + "RedesSociaisFolder": "data\\rede_social" + }, + "BasePath": "sample" +} diff --git a/OpenCand.sln b/OpenCand.sln new file mode 100644 index 0000000..98aead9 --- /dev/null +++ b/OpenCand.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.13.36105.23 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenCand.ETL", "OpenCand.ETL\OpenCand.ETL.csproj", "{92737233-15B6-4258-AB93-55DF13FA9998}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenCand.API", "OpenCand.API\OpenCand.API.csproj", "{70D6DCD0-19EB-4ADB-9183-DA553441BFD7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenCand.Core", "OpenCand.Core\OpenCand.Core.csproj", "{EB4A73F8-CA75-407A-AC48-1DC8F8CE27EF}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {92737233-15B6-4258-AB93-55DF13FA9998}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {92737233-15B6-4258-AB93-55DF13FA9998}.Debug|Any CPU.Build.0 = Debug|Any CPU + {92737233-15B6-4258-AB93-55DF13FA9998}.Release|Any CPU.ActiveCfg = Release|Any CPU + {92737233-15B6-4258-AB93-55DF13FA9998}.Release|Any CPU.Build.0 = Release|Any CPU + {70D6DCD0-19EB-4ADB-9183-DA553441BFD7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {70D6DCD0-19EB-4ADB-9183-DA553441BFD7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {70D6DCD0-19EB-4ADB-9183-DA553441BFD7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {70D6DCD0-19EB-4ADB-9183-DA553441BFD7}.Release|Any CPU.Build.0 = Release|Any CPU + {EB4A73F8-CA75-407A-AC48-1DC8F8CE27EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EB4A73F8-CA75-407A-AC48-1DC8F8CE27EF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EB4A73F8-CA75-407A-AC48-1DC8F8CE27EF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EB4A73F8-CA75-407A-AC48-1DC8F8CE27EF}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {64ADC98D-6111-4542-9515-22445D65F004} + EndGlobalSection +EndGlobal diff --git a/README.md b/README.md new file mode 100644 index 0000000..16bee55 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# OpenCand + +Open Candidate is a project that aims to provide, in a simpler way, informations about candidates in Brazilian elections. It is a web application that allows users to search for candidates by name, state, and position. The application retrieves data from the TSE (Tribunal Superior Eleitoral) API and displays it in a user-friendly format. + +## Architecture + +OpenCand is built using: +* .NET 8 - for parsing initial information from CSV files to the PostgreSQL database using Entity Framework. +* .NET Core 8 - for the API +* PostgreSQL - for the database +* React - for the front-end \ No newline at end of file diff --git a/db/db.sql b/db/db.sql new file mode 100644 index 0000000..02d063b --- /dev/null +++ b/db/db.sql @@ -0,0 +1,62 @@ +DROP TABLE IF EXISTS bem_candidato CASCADE; +DROP TABLE IF EXISTS candidato_mapping CASCADE; +DROP TABLE IF EXISTS rede_social CASCADE; +DROP TABLE IF EXISTS candidato CASCADE; + +CREATE TABLE candidato ( + idcandidato UUID NOT NULL PRIMARY KEY, + cpf VARCHAR(11), + nome VARCHAR(255) NOT NULL, + datanascimento TIMESTAMPTZ, + email TEXT, + sexo CHAR(15), + estadocivil VARCHAR(50), + escolaridade VARCHAR(50), + ocupacao VARCHAR(150) +); +CREATE INDEX idx_candidato_nome ON candidato (nome); + +-- Each candidato (idcandidato, cpf, nome) will be mapped to a (sqcandidato, ano, tipo_eleicao, sg_uf, cargo, resultado) +CREATE TABLE candidato_mapping ( + idcandidato UUID NOT NULL, + cpf VARCHAR(11), + nome VARCHAR(255) NOT NULL, + sqcandidato TEXT, + ano INT NOT NULL, + tipoeleicao VARCHAR(50), + siglauf VARCHAR(2), + nomeue VARCHAR(100), + cargo VARCHAR(50), + nrcandidato VARCHAR(20), + resultado VARCHAR(50), + CONSTRAINT pk_candidato_mapping PRIMARY KEY (idcandidato, ano, siglauf, nomeue, cargo, nrcandidato, resultado), + CONSTRAINT fk_candidato_mapping_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE +); +CREATE INDEX idx_candidato_mapping_cpf ON candidato_mapping (cpf); +CREATE INDEX idx_candidato_mapping_nome ON candidato_mapping (nome); +CREATE INDEX idx_candidato_mapping_ano ON candidato_mapping (ano); +CREATE INDEX idx_candidato_mapping_sqcandidato ON candidato_mapping (sqcandidato); + +CREATE TABLE bem_candidato ( + idcandidato UUID NOT NULL, + ano INT NOT NULL, + ordembem INT, + tipobem VARCHAR(150), + descricao VARCHAR(500), + valor NUMERIC(20, 2), + CONSTRAINT fk_bem_candidato_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE +); +ALTER TABLE bem_candidato ADD CONSTRAINT pk_bem_candidato PRIMARY KEY (idcandidato, ano, ordembem); +CREATE INDEX idx_bem_candidato_idcandidato ON bem_candidato (idcandidato); +CREATE INDEX idx_bem_candidato_valor ON bem_candidato (valor); + +CREATE TABLE rede_social ( + idcandidato UUID NOT NULL, + rede VARCHAR(50) NOT NULL, + siglauf VARCHAR(2), + ano INT NOT NULL, + link TEXT NOT NULL, + CONSTRAINT pk_rede_social PRIMARY KEY (idcandidato, rede, siglauf, ano), + CONSTRAINT fk_rede_social_candidato FOREIGN KEY (idcandidato) REFERENCES candidato(idcandidato) ON DELETE CASCADE ON UPDATE CASCADE +); +CREATE INDEX idx_rede_social_idcandidato ON rede_social (idcandidato); \ No newline at end of file