diff --git a/OpenCand.API/Program.cs b/OpenCand.API/Program.cs index e08575d..1b726f3 100644 --- a/OpenCand.API/Program.cs +++ b/OpenCand.API/Program.cs @@ -53,8 +53,7 @@ namespace OpenCand.API app.Run(); } - - private static void SetupServices(WebApplicationBuilder builder) + private static void SetupServices(WebApplicationBuilder builder) { builder.Services.Configure(builder.Configuration.GetSection("FotosSettings")); builder.Services.AddMemoryCache(); @@ -67,6 +66,9 @@ namespace OpenCand.API builder.Services.AddScoped(); builder.Services.AddScoped(); + + // Add cache preload background service + builder.Services.AddHostedService(); } } } diff --git a/OpenCand.API/Services/CachePreloadService.cs b/OpenCand.API/Services/CachePreloadService.cs new file mode 100644 index 0000000..4cb3263 --- /dev/null +++ b/OpenCand.API/Services/CachePreloadService.cs @@ -0,0 +1,96 @@ +using System; +using Microsoft.Extensions.Logging; +using OpenCand.API.Model; + +namespace OpenCand.API.Services +{ + public class CachePreloadService : BackgroundService + { + private readonly IServiceProvider serviceProvider; + private readonly ILogger logger; + + public CachePreloadService(IServiceProvider serviceProvider, ILogger logger) + { + this.serviceProvider = serviceProvider; + this.logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + // Wait a bit for the application to fully start up + await Task.Delay(5000, stoppingToken); + logger.LogInformation("Starting cache preload process..."); + + using var scope = serviceProvider.CreateScope(); + var estatisticaService = scope.ServiceProvider.GetRequiredService(); + var openCandService = scope.ServiceProvider.GetRequiredService(); + + // First, preload single-call endpoints + await PreloadSingleEndpoints(estatisticaService, openCandService); + + await PerformPreLoadEstatistica(estatisticaService); + + logger.LogInformation("Cache preload process completed."); + } + + private async Task PerformPreLoadEstatistica(EstatisticaService estatisticaService) + { + await PerformPreLoad("GetConfigurationModel", estatisticaService.GetConfigurationModel); + + var types = new[] { "bem", "despesa", "receita" }; + var groupByValues = new[] { "candidato", "partido", "uf", "cargo" }; + + logger.LogInformation($"Preloading cache with GetValueSum requests (3 types * 4 groupBy combinations)"); + + foreach (var type in types) + { + foreach (var groupBy in groupByValues) + { + var request = new GetValueSumRequest + { + Type = type, + GroupBy = groupBy, + Filter = null // No filters as requested + }; + + logger.LogDebug($"Executing cache preload request: Type={type}, GroupBy={groupBy}"); + + await PerformPreLoad("GetValueSum", () => estatisticaService.GetValueSum(request)); + } + } + } + + private async Task PreloadSingleEndpoints(EstatisticaService estatisticaService, OpenCandService openCandService) + { + logger.LogInformation("Preloading single-call endpoints..."); + + await PerformPreLoad("GetOpenCandStatsAsync", estatisticaService.GetMaioresEnriquecimentos); + await PerformPreLoad("GetOpenCandStatsAsync", openCandService.GetOpenCandStatsAsync); + await PerformPreLoad("GetDataAvailabilityStatsAsync", openCandService.GetDataAvailabilityStatsAsync); + + logger.LogInformation("Single-call endpoints preload completed"); + } + + private async Task PerformPreLoad(string name, Func action) + { + try + { + logger.LogDebug($"Executing cache preload for {name}"); + + var startTime = DateTime.UtcNow; + await action(); + var duration = DateTime.UtcNow - startTime; + + logger.LogInformation($"Cache preload completed for {name}: Duration={duration.TotalMilliseconds}ms"); + } + catch (Exception ex) + { + logger.LogError(ex, $"Failed to perform preload action for {name}"); + } + finally + { + await Task.Delay(100); // Small delay to avoid overwhelming the database + } + } + } +} diff --git a/OpenCand.ETL/Parser/ParserManager.cs b/OpenCand.ETL/Parser/ParserManager.cs index 467bbcb..f47779c 100644 --- a/OpenCand.ETL/Parser/ParserManager.cs +++ b/OpenCand.ETL/Parser/ParserManager.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using OpenCand.Config; +using OpenCand.ETL.Repository; using OpenCand.ETL.Services; using OpenCand.Parser.Models; using OpenCand.Parser.Services; @@ -21,6 +22,8 @@ namespace OpenCand.Parser private readonly DespesaReceitaService despesaReceitaService; + private readonly ViewRepository viewRepository; + private readonly string BasePath; public ParserManager( @@ -32,7 +35,8 @@ namespace OpenCand.Parser CsvParserService redeSocialParserService, CsvParserService despesaParserService, CsvParserService receitaParserService, - DespesaReceitaService despesaReceitaService) + DespesaReceitaService despesaReceitaService, + ViewRepository viewRepository) { this.logger = logger; this.csvSettings = csvSettings.Value; @@ -43,6 +47,7 @@ namespace OpenCand.Parser this.despesaParserService = despesaParserService; this.receitaParserService = receitaParserService; this.despesaReceitaService = despesaReceitaService; + this.viewRepository = viewRepository; // Get the base path from either SampleFolder in csvSettings or the BasePath in configuration BasePath = configuration.GetValue("BasePath") ?? string.Empty; @@ -73,6 +78,12 @@ namespace OpenCand.Parser await ParseFolder(receitasDirectory, receitaParserService); logger.LogInformation("ParseFullDataAsync - Full data parsing completed!"); + + logger.LogInformation("ParseFullDataAsync - Will refresh materialized views and re-run the analyzes."); + + await viewRepository.RefreshMaterializedViews(); + + logger.LogInformation("ParseFullDataAsync - Materialized views refreshed successfully!"); } private async Task ParseFolder(string csvDirectory, CsvParserService csvParserService) diff --git a/OpenCand.ETL/Program.cs b/OpenCand.ETL/Program.cs index a294514..7f1a04a 100644 --- a/OpenCand.ETL/Program.cs +++ b/OpenCand.ETL/Program.cs @@ -79,6 +79,7 @@ namespace OpenCand services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); }); } } diff --git a/OpenCand.ETL/Repository/ViewRepository.cs b/OpenCand.ETL/Repository/ViewRepository.cs new file mode 100644 index 0000000..3a3833f --- /dev/null +++ b/OpenCand.ETL/Repository/ViewRepository.cs @@ -0,0 +1,36 @@ +using Dapper; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Configuration; +using Npgsql; +using OpenCand.Core.Models; +using OpenCand.Repository; + +namespace OpenCand.ETL.Repository +{ + public class ViewRepository : BaseRepository + { + public ViewRepository(IConfiguration configuration) : base(configuration) + { } + + public async Task RefreshMaterializedViews() + { + using (var connection = new NpgsqlConnection(ConnectionString)) + { + // Get all materialized view names + var materializedViews = await connection.QueryAsync( + @"SELECT schemaname || '.' || matviewname as full_name + FROM pg_matviews + ORDER BY schemaname, matviewname"); + + foreach (var viewName in materializedViews) + { + // Refresh the materialized view + await connection.ExecuteAsync($"REFRESH MATERIALIZED VIEW {viewName}"); + + // Analyze the materialized view to update statistics + await connection.ExecuteAsync($"ANALYZE {viewName}"); + } + } + } + } +}