adding disponibilidade de dados

OpenCand.API/Controllers/StatsController.cs

@@ -21,5 +21,12 @@ namespace OpenCand.API.Controllers
         {
             return await openCandService.GetOpenCandStatsAsync();
         }
+
+        [HttpGet("data-availability")]
+        public async Task<DataAvailabilityStats> GetDataAvailabilityStats()
+        {
+            return await openCandService.GetDataAvailabilityStatsAsync();
         }
     }
+}
+

OpenCand.API/Repository/OpenCandRepository.cs

@@ -7,8 +7,11 @@ using OpenCand.Repository;
 namespace OpenCand.API.Repository
 {    public class OpenCandRepository : BaseRepository
     {
+        private readonly IConfiguration configuration;
+
         public OpenCandRepository(IConfiguration configuration, IMemoryCache? cache = null) : base(configuration, cache)
         {
+            this.configuration = configuration;
         }
 
         public async Task<OpenCandStats> GetOpenCandStatsAsync()
@@ -32,14 +35,48 @@ namespace OpenCand.API.Repository
             });
             
             return result ?? new OpenCandStats();
-        }
+        }        public async Task<DataAvailabilityStats> GetDataAvailabilityAsync()
-
-        /// <summary>
-        /// Clears the cached OpenCand statistics, forcing a fresh fetch on the next call
-        /// </summary>
-        public void ClearStatsCache()
         {
-            ClearCache(GenerateCacheKey("OpenCandStats"));
+            string cacheKey = GenerateCacheKey("DataAvailabilityStats");
+
+            var result = await GetOrSetCacheAsync(cacheKey, async () =>
+            {
+                using (var connection = new NpgsqlConnection(ConnectionString))
+                {
+                    var stats = new DataAvailabilityStats();
+
+                    // Get years for each data type separately
+                    var candidatosYears = await connection.QueryAsync<int>("SELECT DISTINCT ano FROM candidato_mapping ORDER BY ano DESC");
+                    var bemCandidatosYears = await connection.QueryAsync<int>("SELECT DISTINCT ano FROM bem_candidato ORDER BY ano DESC");
+                    var despesaCandidatosYears = await connection.QueryAsync<int>("SELECT DISTINCT ano FROM despesas_candidato ORDER BY ano DESC");
+                    var receitaCandidatosYears = await connection.QueryAsync<int>("SELECT DISTINCT ano FROM receitas_candidato ORDER BY ano DESC");
+                    var redeSocialCandidatosYears = await connection.QueryAsync<int>("SELECT DISTINCT ano FROM rede_social ORDER BY ano DESC");
+
+                    stats.Candidatos = candidatosYears.ToList();
+                    stats.BemCandidatos = bemCandidatosYears.ToList();
+                    stats.DespesaCandidatos = despesaCandidatosYears.ToList();
+                    stats.ReceitaCandidatos = receitaCandidatosYears.ToList();
+                    stats.RedeSocialCandidatos = redeSocialCandidatosYears.ToList();
+
+                    // Get all folders from appsetting `FotosSettings__BasePath`
+                    string basePath = configuration["FotosSettings:Path"] ?? string.Empty;
+                    if (string.IsNullOrEmpty(basePath))
+                        throw new InvalidOperationException("Base path for photos is not configured.");
+
+                    var directories = Directory.GetDirectories(basePath);
+                    if (directories.Any())
+                        stats.FotosCandidatos = directories
+                            .Select(dir => dir.Split(Path.DirectorySeparatorChar).Last().Split("_")[1].Replace("cand", ""))
+                            .Select(ano => Convert.ToInt32(ano))
+                            .Distinct()
+                            .OrderByDescending(ano => ano)
+                            .ToList();
+
+                    return stats;
+                }
+            });
+
+            return result ?? new DataAvailabilityStats();
         }
     }
 }

OpenCand.API/Services/OpenCandService.cs

@@ -40,6 +40,13 @@ namespace OpenCand.API.Services
             return await openCandRepository.GetOpenCandStatsAsync();
         }
 
+        public async Task<DataAvailabilityStats> GetDataAvailabilityStatsAsync()
+        {
+            var stats = await openCandRepository.GetDataAvailabilityAsync();
+
+            return stats;
+        }
+
         public async Task<CandidatoSearchResult> SearchCandidatosAsync(string query)
         {
             return new CandidatoSearchResult()

OpenCand.Core/Models/OpenCandStats.cs

@@ -8,4 +8,15 @@
         public long TotalRedesSociais { get; set; }
         public long TotalEleicoes { get; set; }
     }
+
+    public class DataAvailabilityStats
+    {
+        public List<int> Candidatos { get; set; }
+        public List<int> BemCandidatos { get; set; }
+        public List<int> DespesaCandidatos { get; set; }
+        public List<int> ReceitaCandidatos { get; set; }
+        public List<int> RedeSocialCandidatos { get; set; }
+        
+        public List<int> FotosCandidatos { get; set; }
+    }
 }

OpenCand.ETL/Parser/ParserManager.cs

@@ -2,6 +2,7 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using OpenCand.Config;
+using OpenCand.ETL.Services;
 using OpenCand.Parser.Models;
 using OpenCand.Parser.Services;
 
@@ -18,6 +19,8 @@ namespace OpenCand.Parser
         private readonly CsvParserService<DespesasCSV> despesaParserService;
         private readonly CsvParserService<ReceitasCSV> receitaParserService;
 
+        private readonly DespesaReceitaService despesaReceitaService;
+
         private readonly string BasePath;
 
         public ParserManager(
@@ -28,7 +31,8 @@ namespace OpenCand.Parser
             CsvParserService<BemCandidatoCSV> bemCandidatoParserService,
             CsvParserService<RedeSocialCSV> redeSocialParserService,
             CsvParserService<DespesasCSV> despesaParserService,
-            CsvParserService<ReceitasCSV> receitaParserService)
+            CsvParserService<ReceitasCSV> receitaParserService,
+            DespesaReceitaService despesaReceitaService)
         {
             this.logger = logger;
             this.csvSettings = csvSettings.Value;
@@ -38,6 +42,7 @@ namespace OpenCand.Parser
             this.redeSocialParserService = redeSocialParserService;
             this.despesaParserService = despesaParserService;
             this.receitaParserService = receitaParserService;
+            this.despesaReceitaService = despesaReceitaService;
 
             // Get the base path from either SampleFolder in csvSettings or the BasePath in configuration
             BasePath = configuration.GetValue<string>("BasePath") ?? string.Empty;
@@ -61,7 +66,10 @@ namespace OpenCand.Parser
             await ParseFolder(candidatosDirectory, candidatoParserService);
             await ParseFolder(bensCandidatosDirectory, bemCandidatoParserService);
             await ParseFolder(redesSociaisDirectory, redeSocialParserService);
+
+            await despesaReceitaService.DeleteDespesaAsync();
             await ParseFolder(despesasDirectory, despesaParserService);
+            await despesaReceitaService.DeleteReceitaAsync();
             await ParseFolder(receitasDirectory, receitaParserService);
 
             logger.LogInformation("ParseFullDataAsync - Full data parsing completed!");

OpenCand.ETL/Repository/DespesaReceitaRepository.cs

@@ -135,5 +135,21 @@ namespace OpenCand.Repository
                 });
             }
         }
+
+        public async Task DeleteDespesaAsync()
+        {
+            using (var connection = new NpgsqlConnection(ConnectionString))
+            {
+                await connection.ExecuteAsync("DELETE FROM despesas_candidato;");
+            }
+        }
+
+        public async Task DeleteReceitaAsync()
+        {
+            using (var connection = new NpgsqlConnection(ConnectionString))
+            {
+                await connection.ExecuteAsync("DELETE FROM receitas_candidato;");
+            }
+        }
     }
 }

OpenCand.ETL/Services/DespesaReceitaService.cs

@@ -50,5 +50,15 @@ namespace OpenCand.ETL.Services
             await despesaReceitaRepository.AddReceitaAsync(receita);
         }
 
+        public async Task DeleteDespesaAsync()
+        {
+            await despesaReceitaRepository.DeleteDespesaAsync();
+        }
+
+        public async Task DeleteReceitaAsync()
+        {
+            await despesaReceitaRepository.DeleteReceitaAsync();
+        }
+
     }
 }

OpenCand.ETL/appsettings.json

@@ -18,7 +18,7 @@
     },
     "ParserSettings": {
         "DefaultThreads": 40,
-        "CandidatoCSVThreads": 5,
+        "CandidatoCSVThreads": 40,
         "DepesasCSVThreads": 50,
         "ReceitasCSVThreads": 50
     },