mais otimizações 3.0

OpenCand.API/Program.cs

@@ -53,8 +53,7 @@ namespace OpenCand.API
 
             app.Run();
         }
-        
-        private static void SetupServices(WebApplicationBuilder builder)
+          private static void SetupServices(WebApplicationBuilder builder)
         {
             builder.Services.Configure<FotosSettings>(builder.Configuration.GetSection("FotosSettings"));
             builder.Services.AddMemoryCache();
@@ -67,6 +66,9 @@ namespace OpenCand.API
 
             builder.Services.AddScoped<OpenCandService>();
             builder.Services.AddScoped<EstatisticaService>();
+            
+            // Add cache preload background service
+            builder.Services.AddHostedService<CachePreloadService>();
         }
     }
 }

OpenCand.API/Services/CachePreloadService.cs

@@ -0,0 +1,96 @@
+using System;
+using Microsoft.Extensions.Logging;
+using OpenCand.API.Model;
+
+namespace OpenCand.API.Services
+{
+    public class CachePreloadService : BackgroundService
+    {
+        private readonly IServiceProvider serviceProvider;
+        private readonly ILogger<CachePreloadService> logger;
+
+        public CachePreloadService(IServiceProvider serviceProvider, ILogger<CachePreloadService> logger)
+        {
+            this.serviceProvider = serviceProvider;
+            this.logger = logger;
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            // Wait a bit for the application to fully start up
+            await Task.Delay(5000, stoppingToken);
+            logger.LogInformation("Starting cache preload process...");
+
+            using var scope = serviceProvider.CreateScope();
+            var estatisticaService = scope.ServiceProvider.GetRequiredService<EstatisticaService>();
+            var openCandService = scope.ServiceProvider.GetRequiredService<OpenCandService>();
+
+            // First, preload single-call endpoints
+            await PreloadSingleEndpoints(estatisticaService, openCandService);
+
+            await PerformPreLoadEstatistica(estatisticaService);
+
+            logger.LogInformation("Cache preload process completed.");
+        }
+
+        private async Task PerformPreLoadEstatistica(EstatisticaService estatisticaService)
+        {
+            await PerformPreLoad("GetConfigurationModel", estatisticaService.GetConfigurationModel);
+
+            var types = new[] { "bem", "despesa", "receita" };
+            var groupByValues = new[] { "candidato", "partido", "uf", "cargo" };
+
+            logger.LogInformation($"Preloading cache with GetValueSum requests (3 types * 4 groupBy combinations)");
+
+            foreach (var type in types)
+            {
+                foreach (var groupBy in groupByValues)
+                {
+                    var request = new GetValueSumRequest
+                    {
+                        Type = type,
+                        GroupBy = groupBy,
+                        Filter = null // No filters as requested
+                    };
+
+                    logger.LogDebug($"Executing cache preload request: Type={type}, GroupBy={groupBy}");
+
+                    await PerformPreLoad("GetValueSum", () => estatisticaService.GetValueSum(request));
+                }
+            }
+        }
+
+        private async Task PreloadSingleEndpoints(EstatisticaService estatisticaService, OpenCandService openCandService)
+        {
+            logger.LogInformation("Preloading single-call endpoints...");
+
+            await PerformPreLoad("GetOpenCandStatsAsync", estatisticaService.GetMaioresEnriquecimentos);
+            await PerformPreLoad("GetOpenCandStatsAsync", openCandService.GetOpenCandStatsAsync);
+            await PerformPreLoad("GetDataAvailabilityStatsAsync", openCandService.GetDataAvailabilityStatsAsync);
+
+            logger.LogInformation("Single-call endpoints preload completed");
+        }
+
+        private async Task PerformPreLoad(string name, Func<Task> action)
+        {
+            try
+            {
+                logger.LogDebug($"Executing cache preload for {name}");
+
+                var startTime = DateTime.UtcNow;
+                await action();
+                var duration = DateTime.UtcNow - startTime;
+
+                logger.LogInformation($"Cache preload completed for {name}: Duration={duration.TotalMilliseconds}ms");
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, $"Failed to perform preload action for {name}");
+            }
+            finally
+            {
+                await Task.Delay(100); // Small delay to avoid overwhelming the database
+            }
+        }
+    }
+}

OpenCand.ETL/Parser/ParserManager.cs

@@ -2,6 +2,7 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using OpenCand.Config;
+using OpenCand.ETL.Repository;
 using OpenCand.ETL.Services;
 using OpenCand.Parser.Models;
 using OpenCand.Parser.Services;
@@ -21,6 +22,8 @@ namespace OpenCand.Parser
 
         private readonly DespesaReceitaService despesaReceitaService;
 
+        private readonly ViewRepository viewRepository;
+
         private readonly string BasePath;
 
         public ParserManager(
@@ -32,7 +35,8 @@ namespace OpenCand.Parser
             CsvParserService<RedeSocialCSV> redeSocialParserService,
             CsvParserService<DespesasCSV> despesaParserService,
             CsvParserService<ReceitasCSV> receitaParserService,
-            DespesaReceitaService despesaReceitaService)
+            DespesaReceitaService despesaReceitaService,
+            ViewRepository viewRepository)
         {
             this.logger = logger;
             this.csvSettings = csvSettings.Value;
@@ -43,6 +47,7 @@ namespace OpenCand.Parser
             this.despesaParserService = despesaParserService;
             this.receitaParserService = receitaParserService;
             this.despesaReceitaService = despesaReceitaService;
+            this.viewRepository = viewRepository;
 
             // Get the base path from either SampleFolder in csvSettings or the BasePath in configuration
             BasePath = configuration.GetValue<string>("BasePath") ?? string.Empty;
@@ -73,6 +78,12 @@ namespace OpenCand.Parser
             await ParseFolder(receitasDirectory, receitaParserService);
 
             logger.LogInformation("ParseFullDataAsync - Full data parsing completed!");
+
+            logger.LogInformation("ParseFullDataAsync - Will refresh materialized views and re-run the analyzes.");
+            
+            await viewRepository.RefreshMaterializedViews();
+            
+            logger.LogInformation("ParseFullDataAsync - Materialized views refreshed successfully!");
         }
 
         private async Task ParseFolder<CsvObj>(string csvDirectory, CsvParserService<CsvObj> csvParserService)

OpenCand.ETL/Program.cs

@@ -79,6 +79,7 @@ namespace OpenCand
                     services.AddTransient<RedeSocialRepository>();
                     services.AddTransient<PartidoRepository>();
                     services.AddTransient<CsvFixerService>();
+                    services.AddTransient<ViewRepository>();
                 });
     }
 }

OpenCand.ETL/Repository/ViewRepository.cs

@@ -0,0 +1,36 @@
+using Dapper;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Configuration;
+using Npgsql;
+using OpenCand.Core.Models;
+using OpenCand.Repository;
+
+namespace OpenCand.ETL.Repository
+{
+    public class ViewRepository : BaseRepository
+    {
+        public ViewRepository(IConfiguration configuration) : base(configuration)
+        { }
+
+        public async Task RefreshMaterializedViews()
+        {
+            using (var connection = new NpgsqlConnection(ConnectionString))
+            {
+                // Get all materialized view names
+                var materializedViews = await connection.QueryAsync<string>(
+                    @"SELECT schemaname || '.' || matviewname as full_name 
+                      FROM pg_matviews 
+                      ORDER BY schemaname, matviewname");
+
+                foreach (var viewName in materializedViews)
+                {
+                    // Refresh the materialized view
+                    await connection.ExecuteAsync($"REFRESH MATERIALIZED VIEW {viewName}");
+
+                    // Analyze the materialized view to update statistics
+                    await connection.ExecuteAsync($"ANALYZE {viewName}");
+                }
+            }
+        }
+    }
+}