improving db connection

OpenCand.API/Config/RateLimitingConfig.cs

@@ -27,13 +27,12 @@ namespace OpenCand.API.Config
                 // Default policy: 200 requests per minute with burst of 100
                 options.AddFixedWindowLimiter(policyName: DefaultPolicy, options =>
                 {
-                    options.PermitLimit = 200;
+                    options.PermitLimit = 400;
                     options.Window = TimeSpan.FromMinutes(1);
                     options.QueueProcessingOrder = QueueProcessingOrder.OldestFirst;
                     options.QueueLimit = 100; // Burst capacity
                 });
 
-                // Candidato Search policy: 300 requests per minute with burst of 200
                 options.AddFixedWindowLimiter(policyName: CandidatoSearchPolicy, options =>
                 {
                     options.PermitLimit = 300;
@@ -42,32 +41,27 @@ namespace OpenCand.API.Config
                     options.QueueLimit = 200; // Burst capacity
                 });
 
-                // CPF Reveal policy: 15 requests per minute without burst
                 options.AddFixedWindowLimiter(policyName: CpfRevealPolicy, options =>
                 {
-                    options.PermitLimit = 15;
+                    options.PermitLimit = 20;
                     options.Window = TimeSpan.FromMinutes(1);
                     options.QueueProcessingOrder = QueueProcessingOrder.OldestFirst;
-                    options.QueueLimit = 0; // No burst
-                });
-
-                // CPF Reveal policy: 25 requests per minute with 10 burst
-                options.AddFixedWindowLimiter(policyName: EstatisticaPolicy, options =>
-                {
-                    options.PermitLimit = 25;
-                    options.Window = TimeSpan.FromMinutes(1);
-                    options.QueueProcessingOrder = QueueProcessingOrder.OldestFirst;
-                    options.QueueLimit = 10; // No burst
+                    options.QueueLimit = 5; // Burst capacity
                 });
 
                 options.OnRejected = async (context, token) =>
                 {
+                    var loggerFactory = context.HttpContext.RequestServices.GetRequiredService<ILoggerFactory>();
+                    var logger = loggerFactory.CreateLogger("RateLimitingConfig");
+                    var clientIdentifier = GetClientIdentifier(context.HttpContext);
+                    logger.LogWarning("Rate limit exceeded for client {ClientIdentifier}", clientIdentifier);
+
                     context.HttpContext.Response.StatusCode = 429;
                     
                     var retryAfter = GetRetryAfter(context);
                     if (retryAfter.HasValue)
                     {
-                        context.HttpContext.Response.Headers.Add("Retry-After", retryAfter.Value.ToString());
+                        context.HttpContext.Response.Headers.Append("Retry-After", retryAfter.Value.ToString());
                     }
 
                     await context.HttpContext.Response.WriteAsync(

OpenCand.API/Controllers/BaseController.cs

@@ -1,10 +1,13 @@
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.RateLimiting;
+using OpenCand.API.Config;
 
 namespace OpenCand.API.Controllers
 {
     [ApiController]
     [Route("v1/[controller]")]
     [Produces("application/json")]
+    [EnableRateLimiting(RateLimitingConfig.DefaultPolicy)]
     public class BaseController : Controller
     {
 

OpenCand.API/Controllers/CandidatoController.cs

@@ -1,6 +1,5 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.RateLimiting;
-using Microsoft.VisualBasic;
 using OpenCand.API.Config;
 using OpenCand.API.Model;
 using OpenCand.API.Services;
@@ -9,7 +8,6 @@ using OpenCand.Core.Utils;
 
 namespace OpenCand.API.Controllers
 {
-    [EnableRateLimiting(RateLimitingConfig.DefaultPolicy)]
     public class CandidatoController : BaseController
     {
         private readonly OpenCandService openCandService;
@@ -23,7 +21,7 @@ namespace OpenCand.API.Controllers
         [EnableRateLimiting(RateLimitingConfig.CandidatoSearchPolicy)]
         public async Task<CandidatoSearchResult> CandidatoSearch([FromQuery] string q)
         {
-            if (string.IsNullOrEmpty(q) || q.Length == 1)
+            if (string.IsNullOrEmpty(q) || q.Length < 3)
             {
                 throw new ArgumentException("Query parameter 'q' cannot be null/empty.", nameof(q));
             }

OpenCand.API/Controllers/EstatisticaController.cs

@@ -1,12 +1,10 @@
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.RateLimiting;
-using OpenCand.API.Config;
 using OpenCand.API.Model;
 using OpenCand.API.Services;
+using static OpenCand.API.Model.GetValueSumRequest;
 
 namespace OpenCand.API.Controllers
 {
-    [EnableRateLimiting(RateLimitingConfig.EstatisticaPolicy)]
     public class EstatisticaController : BaseController
     {
         private readonly EstatisticaService estatisticaService;
@@ -23,9 +21,9 @@ namespace OpenCand.API.Controllers
         }
 
         [HttpGet("enriquecimento")]
-        public async Task<List<MaioresEnriquecimento>> GetMaioresEnriquecimentos()
+        public async Task<List<MaioresEnriquecimento>> GetMaioresEnriquecimentos([FromQuery] GetValueSumRequestFilter requestFilter)
         {
-            return await estatisticaService.GetMaioresEnriquecimentos();
+            return await estatisticaService.GetMaioresEnriquecimentos(requestFilter);
         }
 
         [HttpPost("values-sum")]

OpenCand.API/Program.cs

@@ -36,7 +36,8 @@ namespace OpenCand.API
             {
                 FileProvider = new PhysicalFileProvider(Path.Combine(workingDir, "fotos_cand")),
                 RequestPath = "/assets/fotos"
-            });            app.UseHttpsRedirection();
+            });
+            app.UseHttpsRedirection();
 
             // Use rate limiting middleware
             app.UseRateLimiter();
@@ -53,10 +54,17 @@ namespace OpenCand.API
 
             app.Run();
         }
-          private static void SetupServices(WebApplicationBuilder builder)
+
+        private static void SetupServices(WebApplicationBuilder builder)
         {
             builder.Services.Configure<FotosSettings>(builder.Configuration.GetSection("FotosSettings"));
-            builder.Services.AddMemoryCache();
+
+            // Configure memory cache with size limit from appsettings
+            var sizeLimitMB = builder.Configuration.GetValue<int>("CacheSettings:SizeLimitMB", 15);
+            builder.Services.AddMemoryCache(options =>
+            {
+                options.SizeLimit = sizeLimitMB * 1024L * 1024L; // Convert MB to bytes
+            });
 
             builder.Services.AddScoped(provider => new OpenCandRepository(provider.GetRequiredService<IConfiguration>(), provider.GetService<IMemoryCache>()));
             builder.Services.AddScoped(provider => new CandidatoRepository(provider.GetRequiredService<IConfiguration>(), provider.GetService<IMemoryCache>()));

OpenCand.API/Repository/BaseRepository.cs

@@ -1,5 +1,7 @@
 using Microsoft.Extensions.Caching.Memory;
 using Npgsql;
+using System.Text.Json;
+using System.Text;
 
 namespace OpenCand.Repository
 {
@@ -56,7 +58,8 @@ namespace OpenCand.Repository
                 _cache.Set(cacheKey, result, new MemoryCacheEntryOptions
                 {
                     SlidingExpiration = expiration ?? DefaultCacheExpiration,
-                    Priority = priority ?? DefaultCachePriority
+                    Priority = priority ?? DefaultCachePriority,
+                    Size = EstimateSize(result)
                 });
             }
 
@@ -99,7 +102,8 @@ namespace OpenCand.Repository
                 _cache.Set(cacheKey, result, new MemoryCacheEntryOptions
                 {
                     SlidingExpiration = expiration ?? DefaultCacheExpiration,
-                    Priority = priority ?? DefaultCachePriority
+                    Priority = priority ?? DefaultCachePriority,
+                    Size = EstimateSize(result)
                 });
             }
 
@@ -133,5 +137,25 @@ namespace OpenCand.Repository
         {
             return identifier != null ? $"{entityName}_{identifier}" : entityName;
         }
+
+        /// <summary>
+        /// Estimates the memory size of an object by serializing it to JSON
+        /// </summary>
+        /// <typeparam name="T">Type of the object</typeparam>
+        /// <param name="obj">The object to estimate size for</param>
+        /// <returns>Estimated size in bytes</returns>
+        private static long EstimateSize<T>(T obj)
+        {
+            if (obj == null) return 0;
+            try
+            {
+                var json = JsonSerializer.Serialize(obj);
+                return Encoding.UTF8.GetByteCount(json);
+            }
+            catch
+            {
+                return 1024; // Default estimate if serialization fails
+            }
+        }
     }
 }

OpenCand.API/Repository/CandidatoRepository.cs

@@ -102,6 +102,16 @@ namespace OpenCand.Repository
             }
         }
 
+        public async Task IncreaseCandidatoPopularity(Guid idcandidato)
+        {
+            using var connection = new NpgsqlConnection(ConnectionString);
+            await connection.ExecuteAsync(@"
+                UPDATE candidato
+                SET popularidade = popularidade + 1
+                WHERE idcandidato = @idcandidato;
+            ", new { idcandidato });
+        }
+
         public async Task<Guid?> GetRandomCandidatoIdAsync()
         {
             using var connection = new NpgsqlConnection(ConnectionString);

OpenCand.API/Repository/EstatisticaRepository.cs

@@ -2,8 +2,9 @@
 using Microsoft.Extensions.Caching.Memory;
 using Npgsql;
 using OpenCand.API.Model;
-using OpenCand.Core.Models;
 using OpenCand.Repository;
+using System.Text.Json;
+using static OpenCand.API.Model.GetValueSumRequest;
 
 namespace OpenCand.API.Repository
 {
@@ -16,9 +17,39 @@ namespace OpenCand.API.Repository
             this.configuration = configuration;
         }
 
-        public async Task<List<MaioresEnriquecimento>> GetMaioresEnriquecimentos()
+        public async Task<List<MaioresEnriquecimento>> GetMaioresEnriquecimentos(GetValueSumRequestFilter? requestFilter = null)
         {
-            string cacheKey = GenerateCacheKey("GetMaioresEnriquecimento");
+            var joinBase = string.Empty;
+            if (requestFilter == null) requestFilter = new GetValueSumRequestFilter();
+            else
+            {
+                joinBase = " JOIN candidato_mapping cm ON ed.idcandidato = cm.idcandidato ";
+                var whereConditions = new List<string>();
+                if (!string.IsNullOrWhiteSpace(requestFilter.Partido))
+                {
+                    whereConditions.Add($"cm.sgpartido = '{requestFilter.Partido}'");
+                }
+                if (!string.IsNullOrWhiteSpace(requestFilter.Uf))
+                {
+                    whereConditions.Add($"cm.siglauf = '{requestFilter.Uf.ToUpper()}'");
+                }
+                if (requestFilter.Ano != null)
+                {
+                    whereConditions.Add($"cm.ano = '{requestFilter.Ano}'");
+                }
+                if (!string.IsNullOrEmpty(requestFilter.Cargo))
+                {
+                    whereConditions.Add($"cm.cargo = '{requestFilter.Cargo}'");
+                }
+                if (whereConditions.Count > 0)
+                {
+                    joinBase += " WHERE " + string.Join(" AND ", whereConditions);
+                }
+            }
+
+            var requestJson = JsonSerializer.Serialize(requestFilter);
+
+            string cacheKey = GenerateCacheKey("GetMaioresEnriquecimento", requestJson);
 
             var result = await GetOrSetCacheAsync(cacheKey, async () =>
             {
@@ -46,6 +77,7 @@ namespace OpenCand.API.Repository
                         JOIN candidato c ON ed.idcandidato = c.idcandidato
                         JOIN mv_bem_candidato pi ON ed.idcandidato = pi.idcandidato AND ed.anoInicial = pi.ano
                         JOIN mv_bem_candidato pf ON ed.idcandidato = pf.idcandidato AND ed.anoFinal = pf.ano
+                        " + joinBase + @"
                         ORDER BY
                             enriquecimento DESC
                         LIMIT 25;")

OpenCand.API/Services/CachePreloadService.cs

@@ -58,13 +58,14 @@ namespace OpenCand.API.Services
                     await PerformPreLoad("GetValueSum", () => estatisticaService.GetValueSum(request));
                 }
             }
+
+            await PerformPreLoad("GetMaioresEnriquecimentos", () => estatisticaService.GetMaioresEnriquecimentos(null));
         }
 
         private async Task PreloadSingleEndpoints(EstatisticaService estatisticaService, OpenCandService openCandService)
         {
             logger.LogInformation("Preloading single-call endpoints...");
 
-            await PerformPreLoad("GetOpenCandStatsAsync", estatisticaService.GetMaioresEnriquecimentos);
             await PerformPreLoad("GetOpenCandStatsAsync", openCandService.GetOpenCandStatsAsync);
             await PerformPreLoad("GetDatabaseTechStatsAsync", openCandService.GetDatabaseTechStatsAsync);
             await PerformPreLoad("GetDataAvailabilityStatsAsync", openCandService.GetDataAvailabilityStatsAsync);

OpenCand.API/Services/EstatisticaService.cs

@@ -1,56 +1,54 @@
 using OpenCand.API.Model;
 using OpenCand.API.Repository;
-using OpenCand.Repository;
+using static OpenCand.API.Model.GetValueSumRequest;
 
 namespace OpenCand.API.Services
 {
     public class EstatisticaService
     {
-        private readonly OpenCandRepository openCandRepository;
-        private readonly CandidatoRepository candidatoRepository;
-        private readonly BemCandidatoRepository bemCandidatoRepository;
-        private readonly DespesaReceitaRepository despesaReceitaRepository;
         private readonly EstatisticaRepository estatisticaRepository;
-        private readonly IConfiguration configuration;
         private readonly ILogger<OpenCandService> logger;
 
         public EstatisticaService(
-            OpenCandRepository openCandRepository,
-            CandidatoRepository candidatoRepository,
-            BemCandidatoRepository bemCandidatoRepository,
-            DespesaReceitaRepository despesaReceitaRepository,
             EstatisticaRepository estatisticaRepository,
-            IConfiguration configuration,
             ILogger<OpenCandService> logger)
         {
-            this.openCandRepository = openCandRepository;
-            this.candidatoRepository = candidatoRepository;
-            this.bemCandidatoRepository = bemCandidatoRepository;
-            this.despesaReceitaRepository = despesaReceitaRepository;
             this.estatisticaRepository = estatisticaRepository;
-            this.configuration = configuration;
             this.logger = logger;
         }
 
         public async Task<ConfigurationModel> GetConfigurationModel()
         {
+            logger.LogInformation("Getting configuration model");
+
             return await estatisticaRepository.GetConfiguration();
         }
 
-        public async Task<List<MaioresEnriquecimento>> GetMaioresEnriquecimentos()
+        public async Task<List<MaioresEnriquecimento>> GetMaioresEnriquecimentos(GetValueSumRequestFilter? requestFilter = null)
         {
-            return await estatisticaRepository.GetMaioresEnriquecimentos();
+            logger.LogInformation($"Getting maiores enriquecimentos. Filters: Partido={requestFilter?.Partido}, Uf={requestFilter?.Uf}, Ano={requestFilter?.Ano}, Cargo={requestFilter?.Cargo}");
+
+            return await estatisticaRepository.GetMaioresEnriquecimentos(requestFilter);
         }
 
         public async Task<List<GetValueSumResponse>> GetValueSum(GetValueSumRequest request)
         {
+            logger.LogInformation($"Getting value sum for {request.Type} grouped by {request.GroupBy}. Filters: Partido={request.Filter?.Partido}, Uf={request.Filter?.Uf}, Ano={request.Filter?.Ano}, Cargo={request.Filter?.Cargo}");
+            // count exec time
+            var stopwatch = System.Diagnostics.Stopwatch.StartNew();
+
             ValidateRequest(request);
 
             var sqlBuilder = new SqlQueryBuilder();
             var query = sqlBuilder.BuildQuery(request);
             var parameters = sqlBuilder.GetParameters();
 
-            return await estatisticaRepository.GetValueSum(query, parameters);
+            var result = await estatisticaRepository.GetValueSum(query, parameters);
+
+            stopwatch.Stop();
+            logger.LogInformation($"GetValueSum - Execution time: {stopwatch.ElapsedMilliseconds} ms");
+
+            return result;
         }
 
         private void ValidateRequest(GetValueSumRequest request)

OpenCand.API/Services/OpenCandService.cs

@@ -37,11 +37,15 @@ namespace OpenCand.API.Services
 
         public async Task<OpenCandStats> GetOpenCandStatsAsync()
         {
+            logger.LogInformation("Getting OpenCand stats");
+            
             return await openCandRepository.GetOpenCandStatsAsync();
         }
 
         public async Task<DataAvailabilityStats> GetDataAvailabilityStatsAsync()
         {
+            logger.LogInformation("Getting data availability stats");
+
             var stats = await openCandRepository.GetDataAvailabilityAsync();
 
             return stats;
@@ -49,6 +53,8 @@ namespace OpenCand.API.Services
 
         public async Task<DatabaseTechStats> GetDatabaseTechStatsAsync()
         {
+            logger.LogInformation("Getting database tech stats");
+
             var stats = await openCandRepository.GetDatabaseTechStatsAsync();
 
             stats.Tables = stats.Tables.OrderBy(t => t.Name).ToList();
@@ -59,14 +65,18 @@ namespace OpenCand.API.Services
 
         public async Task<CandidatoSearchResult> SearchCandidatosAsync(string query)
         {
+            logger.LogInformation($"Searching candidatos with query: {query}");
+
             return new CandidatoSearchResult()
             {
-                Candidatos = await candidatoRepository.SearchCandidatosAsync(query)
+                Candidatos = await candidatoRepository.SearchCandidatosAsync(query) ?? new List<Candidato>()
             };
         }
 
         public async Task<Guid?> GetRandomCandidato()
         {
+            logger.LogInformation("Getting random candidato");
+
             return await candidatoRepository.GetRandomCandidatoIdAsync();
         }
 
@@ -76,6 +86,15 @@ namespace OpenCand.API.Services
             var eleicoes = await candidatoRepository.GetCandidatoMappingById(idcandidato);
             var candidatoExt = await candidatoRepository.GetCandidatoExtById(idcandidato);
 
+            try
+            {
+                await candidatoRepository.IncreaseCandidatoPopularity(idcandidato);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, $"Error increasing popularity for Candidato ID {idcandidato}");
+            }
+
             if (result == null)
             {
                 throw new KeyNotFoundException($"Candidato with ID {idcandidato} not found.");
@@ -84,6 +103,10 @@ namespace OpenCand.API.Services
             {
                 throw new KeyNotFoundException($"CandidatoMapping for ID {idcandidato} not found.");
             }
+            if (candidatoExt == null)
+            {
+                throw new KeyNotFoundException($"CandidatoExt for ID {idcandidato} not found.");
+            }
 
             foreach (var eleicao in eleicoes)
             {
@@ -96,6 +119,8 @@ namespace OpenCand.API.Services
             result.Eleicoes = eleicoes.OrderByDescending(e => e.Ano).ToList();
             result.CandidatoExt = candidatoExt.OrderByDescending(ce => ce.Ano).ToList();
 
+            logger.LogDebug($"Found Candidato: {result.Nome}, ID: {result.IdCandidato}, CPF: {result.Cpf}, FotoUrl: {result.FotoUrl}");
+
             return result;
         }
 
@@ -107,6 +132,8 @@ namespace OpenCand.API.Services
                 result = new List<BemCandidato>();
             }
 
+            logger.LogInformation($"Found {result.Count} bens for Candidato ID {idcandidato}");
+
             return new BemCandidatoResult()
             {
                 Bens = result.OrderByDescending(r => r.TipoBem).ThenByDescending(r => r.Valor).ToList()
@@ -115,12 +142,16 @@ namespace OpenCand.API.Services
 
         public async Task<RedeSocialResult> GetCandidatoRedeSocialById(Guid idcandidato)
         {
+            logger.LogInformation($"Getting redes sociais for Candidato ID {idcandidato}");
+
             var result = await candidatoRepository.GetCandidatoRedeSocialById(idcandidato);
             if (result == null)
             {
                 result = new List<RedeSocial>();
             }
 
+            logger.LogDebug($"Found {result.Count} redes sociais for Candidato ID {idcandidato}");
+
             return new RedeSocialResult()
             {
                 RedesSociais = result.OrderByDescending(r => r.Ano).ToList()
@@ -129,12 +160,16 @@ namespace OpenCand.API.Services
 
         public async Task<CpfRevealResult> GetCandidatoCpfById(Guid idcandidato)
         {
+            logger.LogInformation($"Getting CPF for Candidato ID {idcandidato}");
+
             var result = await candidatoRepository.GetCandidatoCpfAsync(idcandidato);
             if (result == null)
             {
                 return new CpfRevealResult();
             }
 
+            logger.LogDebug($"Found CPF {result} for Candidato ID {idcandidato}");
+
             return new CpfRevealResult()
             {
                 Cpf = result
@@ -143,12 +178,16 @@ namespace OpenCand.API.Services
 
         public async Task<DespesasResult> GetDespesasByIdAndYear(Guid idcandidato)
         {
+            logger.LogInformation($"Getting despesas for Candidato ID {idcandidato}");
+
             var result = await despesaReceitaRepository.GetDespesasByCandidatoIdYearAsync(idcandidato);
             if (result == null)
             {
                 return new DespesasResult();
             }
 
+            logger.LogDebug($"Found {result.Count} despesas for Candidato ID {idcandidato}");
+
             return new DespesasResult()
             {
                 Despesas = result.OrderByDescending(d => d.Ano).ThenByDescending(d => d.Valor).ToList()
@@ -157,12 +196,16 @@ namespace OpenCand.API.Services
 
         public async Task<ReceitaResult> GetReceitasByIdAndYear(Guid idcandidato)
         {
+            logger.LogInformation($"Getting receitas for Candidato ID {idcandidato}");
+
             var result = await despesaReceitaRepository.GetReceitasByCandidatoIdYearAsync(idcandidato);
             if (result == null)
             {
                 return new ReceitaResult();
             }
 
+            logger.LogDebug($"Found {result.Count} receitas for Candidato ID {idcandidato}");
+
             return new ReceitaResult()
             {
                 Receitas = result.OrderByDescending(d => d.Ano).ThenByDescending(d => d.Valor).ToList()

OpenCand.API/appsettings.json

@@ -1,16 +1,19 @@
 {
     "Logging": {
         "LogLevel": {
-            "Default": "Information",
-            "Microsoft.AspNetCore": "Warning"
+            "Default": "Debug",
+            "Microsoft.AspNetCore": "Information"
         }
     },
     "DatabaseSettings": {
-        "ConnectionString": "Host=localhost;Database=opencand;Username=root;Password=root;Include Error Detail=true;CommandTimeout=300"
+        "ConnectionString": "Host=localhost;Database=opencand;Username=root;Password=root;Pooling=true;Minimum Pool Size=1;Maximum Pool Size=20;Connection Lifetime=300;Command Timeout=30;Application Name=OpenCand.API;Include Error Detail=true"
     },
     "FotosSettings": {
         "Path": "./fotos_cand",
         "ApiBasePath": "http://localhost:5299/assets/fotos"
     },
+    "CacheSettings": {
+        "SizeLimitMB": 15
+    },
     "AllowedHosts": "*"
 }