From 684a2c0630650e866d11804b3d7ed8059c712b4c Mon Sep 17 00:00:00 2001 From: Jose Henrique Date: Tue, 10 Jun 2025 13:26:58 -0300 Subject: [PATCH] adding disponibilidade de dados --- OpenCand.API/Controllers/StatsController.cs | 7 +++ OpenCand.API/Repository/OpenCandRepository.cs | 51 ++++++++++++++++--- OpenCand.API/Services/OpenCandService.cs | 7 +++ OpenCand.Core/Models/OpenCandStats.cs | 11 ++++ OpenCand.ETL/Parser/ParserManager.cs | 10 +++- .../Repository/DespesaReceitaRepository.cs | 16 ++++++ .../Services/DespesaReceitaService.cs | 10 ++++ OpenCand.ETL/appsettings.json | 2 +- 8 files changed, 105 insertions(+), 9 deletions(-) diff --git a/OpenCand.API/Controllers/StatsController.cs b/OpenCand.API/Controllers/StatsController.cs index daacf4b..876b8d4 100644 --- a/OpenCand.API/Controllers/StatsController.cs +++ b/OpenCand.API/Controllers/StatsController.cs @@ -21,5 +21,12 @@ namespace OpenCand.API.Controllers { return await openCandService.GetOpenCandStatsAsync(); } + + [HttpGet("data-availability")] + public async Task GetDataAvailabilityStats() + { + return await openCandService.GetDataAvailabilityStatsAsync(); + } } } + diff --git a/OpenCand.API/Repository/OpenCandRepository.cs b/OpenCand.API/Repository/OpenCandRepository.cs index 7176b06..a1c6057 100644 --- a/OpenCand.API/Repository/OpenCandRepository.cs +++ b/OpenCand.API/Repository/OpenCandRepository.cs @@ -7,8 +7,11 @@ using OpenCand.Repository; namespace OpenCand.API.Repository { public class OpenCandRepository : BaseRepository { + private readonly IConfiguration configuration; + public OpenCandRepository(IConfiguration configuration, IMemoryCache? cache = null) : base(configuration, cache) { + this.configuration = configuration; } public async Task GetOpenCandStatsAsync() @@ -32,14 +35,48 @@ namespace OpenCand.API.Repository }); return result ?? new OpenCandStats(); - } - - /// - /// Clears the cached OpenCand statistics, forcing a fresh fetch on the next call - /// - public void ClearStatsCache() + } public async Task GetDataAvailabilityAsync() { - ClearCache(GenerateCacheKey("OpenCandStats")); + string cacheKey = GenerateCacheKey("DataAvailabilityStats"); + + var result = await GetOrSetCacheAsync(cacheKey, async () => + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + var stats = new DataAvailabilityStats(); + + // Get years for each data type separately + var candidatosYears = await connection.QueryAsync("SELECT DISTINCT ano FROM candidato_mapping ORDER BY ano DESC"); + var bemCandidatosYears = await connection.QueryAsync("SELECT DISTINCT ano FROM bem_candidato ORDER BY ano DESC"); + var despesaCandidatosYears = await connection.QueryAsync("SELECT DISTINCT ano FROM despesas_candidato ORDER BY ano DESC"); + var receitaCandidatosYears = await connection.QueryAsync("SELECT DISTINCT ano FROM receitas_candidato ORDER BY ano DESC"); + var redeSocialCandidatosYears = await connection.QueryAsync("SELECT DISTINCT ano FROM rede_social ORDER BY ano DESC"); + + stats.Candidatos = candidatosYears.ToList(); + stats.BemCandidatos = bemCandidatosYears.ToList(); + stats.DespesaCandidatos = despesaCandidatosYears.ToList(); + stats.ReceitaCandidatos = receitaCandidatosYears.ToList(); + stats.RedeSocialCandidatos = redeSocialCandidatosYears.ToList(); + + // Get all folders from appsetting `FotosSettings__BasePath` + string basePath = configuration["FotosSettings:Path"] ?? string.Empty; + if (string.IsNullOrEmpty(basePath)) + throw new InvalidOperationException("Base path for photos is not configured."); + + var directories = Directory.GetDirectories(basePath); + if (directories.Any()) + stats.FotosCandidatos = directories + .Select(dir => dir.Split(Path.DirectorySeparatorChar).Last().Split("_")[1].Replace("cand", "")) + .Select(ano => Convert.ToInt32(ano)) + .Distinct() + .OrderByDescending(ano => ano) + .ToList(); + + return stats; + } + }); + + return result ?? new DataAvailabilityStats(); } } } diff --git a/OpenCand.API/Services/OpenCandService.cs b/OpenCand.API/Services/OpenCandService.cs index af1cd0e..eb3966c 100644 --- a/OpenCand.API/Services/OpenCandService.cs +++ b/OpenCand.API/Services/OpenCandService.cs @@ -40,6 +40,13 @@ namespace OpenCand.API.Services return await openCandRepository.GetOpenCandStatsAsync(); } + public async Task GetDataAvailabilityStatsAsync() + { + var stats = await openCandRepository.GetDataAvailabilityAsync(); + + return stats; + } + public async Task SearchCandidatosAsync(string query) { return new CandidatoSearchResult() diff --git a/OpenCand.Core/Models/OpenCandStats.cs b/OpenCand.Core/Models/OpenCandStats.cs index bb64f7c..6ffe41d 100644 --- a/OpenCand.Core/Models/OpenCandStats.cs +++ b/OpenCand.Core/Models/OpenCandStats.cs @@ -8,4 +8,15 @@ public long TotalRedesSociais { get; set; } public long TotalEleicoes { get; set; } } + + public class DataAvailabilityStats + { + public List Candidatos { get; set; } + public List BemCandidatos { get; set; } + public List DespesaCandidatos { get; set; } + public List ReceitaCandidatos { get; set; } + public List RedeSocialCandidatos { get; set; } + + public List FotosCandidatos { get; set; } + } } diff --git a/OpenCand.ETL/Parser/ParserManager.cs b/OpenCand.ETL/Parser/ParserManager.cs index 602b2a0..467bbcb 100644 --- a/OpenCand.ETL/Parser/ParserManager.cs +++ b/OpenCand.ETL/Parser/ParserManager.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using OpenCand.Config; +using OpenCand.ETL.Services; using OpenCand.Parser.Models; using OpenCand.Parser.Services; @@ -18,6 +19,8 @@ namespace OpenCand.Parser private readonly CsvParserService despesaParserService; private readonly CsvParserService receitaParserService; + private readonly DespesaReceitaService despesaReceitaService; + private readonly string BasePath; public ParserManager( @@ -28,7 +31,8 @@ namespace OpenCand.Parser CsvParserService bemCandidatoParserService, CsvParserService redeSocialParserService, CsvParserService despesaParserService, - CsvParserService receitaParserService) + CsvParserService receitaParserService, + DespesaReceitaService despesaReceitaService) { this.logger = logger; this.csvSettings = csvSettings.Value; @@ -38,6 +42,7 @@ namespace OpenCand.Parser this.redeSocialParserService = redeSocialParserService; this.despesaParserService = despesaParserService; this.receitaParserService = receitaParserService; + this.despesaReceitaService = despesaReceitaService; // Get the base path from either SampleFolder in csvSettings or the BasePath in configuration BasePath = configuration.GetValue("BasePath") ?? string.Empty; @@ -61,7 +66,10 @@ namespace OpenCand.Parser await ParseFolder(candidatosDirectory, candidatoParserService); await ParseFolder(bensCandidatosDirectory, bemCandidatoParserService); await ParseFolder(redesSociaisDirectory, redeSocialParserService); + + await despesaReceitaService.DeleteDespesaAsync(); await ParseFolder(despesasDirectory, despesaParserService); + await despesaReceitaService.DeleteReceitaAsync(); await ParseFolder(receitasDirectory, receitaParserService); logger.LogInformation("ParseFullDataAsync - Full data parsing completed!"); diff --git a/OpenCand.ETL/Repository/DespesaReceitaRepository.cs b/OpenCand.ETL/Repository/DespesaReceitaRepository.cs index b07aa82..0c14ae1 100644 --- a/OpenCand.ETL/Repository/DespesaReceitaRepository.cs +++ b/OpenCand.ETL/Repository/DespesaReceitaRepository.cs @@ -135,5 +135,21 @@ namespace OpenCand.Repository }); } } + + public async Task DeleteDespesaAsync() + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + await connection.ExecuteAsync("DELETE FROM despesas_candidato;"); + } + } + + public async Task DeleteReceitaAsync() + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + await connection.ExecuteAsync("DELETE FROM receitas_candidato;"); + } + } } } diff --git a/OpenCand.ETL/Services/DespesaReceitaService.cs b/OpenCand.ETL/Services/DespesaReceitaService.cs index 34f5f27..f62b203 100644 --- a/OpenCand.ETL/Services/DespesaReceitaService.cs +++ b/OpenCand.ETL/Services/DespesaReceitaService.cs @@ -50,5 +50,15 @@ namespace OpenCand.ETL.Services await despesaReceitaRepository.AddReceitaAsync(receita); } + public async Task DeleteDespesaAsync() + { + await despesaReceitaRepository.DeleteDespesaAsync(); + } + + public async Task DeleteReceitaAsync() + { + await despesaReceitaRepository.DeleteReceitaAsync(); + } + } } diff --git a/OpenCand.ETL/appsettings.json b/OpenCand.ETL/appsettings.json index 4626753..cc6710a 100644 --- a/OpenCand.ETL/appsettings.json +++ b/OpenCand.ETL/appsettings.json @@ -18,7 +18,7 @@ }, "ParserSettings": { "DefaultThreads": 40, - "CandidatoCSVThreads": 5, + "CandidatoCSVThreads": 40, "DepesasCSVThreads": 50, "ReceitasCSVThreads": 50 },