diff --git a/.github/otel-collector-config.yaml b/.github/otel-collector-config.yaml new file mode 100644 index 000000000..25c01c5ab --- /dev/null +++ b/.github/otel-collector-config.yaml @@ -0,0 +1,41 @@ +receivers: + otlp: + protocols: + grpc: + http: + include_metadata: true + cors: + max_age: 7200 + +exporters: + logging: + + otlp/elastic: + endpoint: https://a8e27ab098f54752b227af78fc609169.es.ops.vl.be:9243 + headers: + Authorization: "Bearer iHyyarDx8Bvn9WYXDv" + +extensions: + health_check: + path: "/health" + +processors: + batch: + +service: + extensions: + - health_check + + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, otlp/elastic] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [logging, otlp/elastic] + logs: + receivers: [otlp] + processors: [batch] + exporters: [logging, otlp/elastic] diff --git a/src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/AppSettings.cs b/src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/AppSettings.cs index ce4e4a20f..59aee565b 100644 --- a/src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/AppSettings.cs +++ b/src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/AppSettings.cs @@ -3,7 +3,22 @@ public class AppSettings { private string? _baseUrl; + private string? _beheerApiBaseUrl; + private string? _beheerProjectionHostBaseUrl; private string? _publicApiBaseUrl; + private string? _publicProjectionHostBaseUrl; + + public string BeheerApiBaseUrl + { + get => _beheerApiBaseUrl?.TrimEnd(trimChar: '/') ?? string.Empty; + set => _beheerApiBaseUrl = value; + } + + public string BeheerProjectionHostBaseUrl + { + get => _beheerProjectionHostBaseUrl?.TrimEnd(trimChar: '/') ?? string.Empty; + set => _beheerProjectionHostBaseUrl = value; + } public string PublicApiBaseUrl { @@ -11,6 +26,12 @@ public string PublicApiBaseUrl set => _publicApiBaseUrl = value; } + public string PublicProjectionHostBaseUrl + { + get => _publicProjectionHostBaseUrl?.TrimEnd(trimChar: '/') ?? string.Empty; + set => _publicProjectionHostBaseUrl = value; + } + public string BaseUrl { get => _baseUrl?.TrimEnd(trimChar: '/') ?? string.Empty; diff --git a/src/AssociationRegistry.Admin.Api/Infrastructure/HttpClients/AdminProjectionHostHttpClient.cs b/src/AssociationRegistry.Admin.Api/Infrastructure/HttpClients/AdminProjectionHostHttpClient.cs new file mode 100644 index 000000000..a1ff6f146 --- /dev/null +++ b/src/AssociationRegistry.Admin.Api/Infrastructure/HttpClients/AdminProjectionHostHttpClient.cs @@ -0,0 +1,41 @@ +namespace AssociationRegistry.Admin.Api.Infrastructure.HttpClients; + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +public class AdminProjectionHostHttpClient : IDisposable +{ + private readonly HttpClient _httpClient; + + public AdminProjectionHostHttpClient(HttpClient httpClient) + { + _httpClient = httpClient; + } + + public async Task RebuildAllProjections(CancellationToken cancellationToken) + => await _httpClient.PostAsync(requestUri: "/projections/all/rebuild", content: null, cancellationToken); + + public async Task RebuildDetailProjection(CancellationToken cancellationToken) + => await _httpClient.PostAsync(requestUri: "/projections/detail/rebuild", content: null, cancellationToken); + + public async Task RebuildHistoriekProjection(CancellationToken cancellationToken) + => await _httpClient.PostAsync(requestUri: "/projections/historiek/rebuild", content: null, cancellationToken); + + public async Task RebuildZoekenProjection(CancellationToken cancellationToken) + => await _httpClient.PostAsync(requestUri: "/projections/search/rebuild", content: null, cancellationToken); + + public async Task GetStatus(CancellationToken cancellationToken) + { + var request = new HttpRequestMessage(HttpMethod.Get, requestUri: "/projections/status"); + request.Headers.Add(name: "X-Correlation-Id", Guid.NewGuid().ToString()); + + return await _httpClient.SendAsync(request, cancellationToken); + } + + public void Dispose() + { + _httpClient.Dispose(); + } +} diff --git a/src/AssociationRegistry.Admin.Api/Infrastructure/HttpClients/PublicProjectionHostHttpClient.cs b/src/AssociationRegistry.Admin.Api/Infrastructure/HttpClients/PublicProjectionHostHttpClient.cs new file mode 100644 index 000000000..f8911ce79 --- /dev/null +++ b/src/AssociationRegistry.Admin.Api/Infrastructure/HttpClients/PublicProjectionHostHttpClient.cs @@ -0,0 +1,30 @@ +namespace AssociationRegistry.Admin.Api.Infrastructure.HttpClients; + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +public class PublicProjectionHostHttpClient : IDisposable +{ + private readonly HttpClient _httpClient; + + public PublicProjectionHostHttpClient(HttpClient httpClient) + { + _httpClient = httpClient; + } + + public async Task RebuildDetailProjection(CancellationToken cancellationToken) + => await _httpClient.PostAsync(requestUri: "/projections/detail/rebuild", content: null, cancellationToken); + + public async Task RebuildZoekenProjection(CancellationToken cancellationToken) + => await _httpClient.PostAsync(requestUri: "/projections/search/rebuild", content: null, cancellationToken); + + public async Task GetStatus(CancellationToken cancellationToken) + => await _httpClient.GetAsync(requestUri: "/projections/status", cancellationToken); + + public void Dispose() + { + _httpClient.Dispose(); + } +} diff --git a/src/AssociationRegistry.Admin.Api/Infrastructure/ProjectionHostController.cs b/src/AssociationRegistry.Admin.Api/Infrastructure/ProjectionHostController.cs new file mode 100644 index 000000000..a8ec7969e --- /dev/null +++ b/src/AssociationRegistry.Admin.Api/Infrastructure/ProjectionHostController.cs @@ -0,0 +1,115 @@ +namespace AssociationRegistry.Admin.Api.Infrastructure; + +using Be.Vlaanderen.Basisregisters.Api; +using HttpClients; +using Microsoft.AspNetCore.Mvc; +using System.Net.Http.Json; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +[ApiVersion("1.0")] +[AdvertiseApiVersions("1.0")] +[ApiRoute("projections")] +[ApiExplorerSettings(IgnoreApi = true)] +public class ProjectionHostController : ApiController +{ + private readonly AdminProjectionHostHttpClient _adminHttpClient; + private readonly PublicProjectionHostHttpClient _publicHttpClient; + private readonly JsonSerializerOptions _jsonSerializerOptions; + + public ProjectionHostController(AdminProjectionHostHttpClient adminHttpClient, PublicProjectionHostHttpClient publicHttpClient) + { + _adminHttpClient = adminHttpClient; + _publicHttpClient = publicHttpClient; + + _jsonSerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }; + } + + [HttpPost("admin/all/rebuild")] + public async Task RebuildAdminProjectionAll(CancellationToken cancellationToken) + { + var response = await _adminHttpClient.RebuildAllProjections(cancellationToken); + + return response.IsSuccessStatusCode + ? Ok() + : UnprocessableEntity(); + } + + [HttpPost("admin/detail/rebuild")] + public async Task RebuildAdminProjectionDetail(CancellationToken cancellationToken) + { + var response = await _adminHttpClient.RebuildDetailProjection(cancellationToken); + + return response.IsSuccessStatusCode + ? Ok() + : UnprocessableEntity(); + } + + [HttpPost("admin/historiek/rebuild")] + public async Task RebuildAdminProjectionHistoriek(CancellationToken cancellationToken) + { + var response = await _adminHttpClient.RebuildHistoriekProjection(cancellationToken); + + return response.IsSuccessStatusCode + ? Ok() + : UnprocessableEntity(); + } + + [HttpPost("admin/search/rebuild")] + public async Task RebuildAdminProjectionZoeken(CancellationToken cancellationToken) + { + var response = await _adminHttpClient.RebuildZoekenProjection(cancellationToken); + + return response.IsSuccessStatusCode + ? Ok() + : UnprocessableEntity(); + } + + [HttpGet("admin/status")] + public async Task GetAdminProjectionStatus(CancellationToken cancellationToken) + { + var response = await _adminHttpClient.GetStatus(cancellationToken); + + if (!response.IsSuccessStatusCode) return BadRequest(); + + var projectionProgress = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, cancellationToken); + + return new OkObjectResult(projectionProgress); + } + + [HttpPost("public/detail/rebuild")] + public async Task RebuildPublicProjectionDetail(CancellationToken cancellationToken) + { + var response = await _publicHttpClient.RebuildDetailProjection(cancellationToken); + + return response.IsSuccessStatusCode + ? Ok() + : UnprocessableEntity(); + } + + [HttpPost("public/search/rebuild")] + public async Task RebuildPublicProjectionZoeken(CancellationToken cancellationToken) + { + var response = await _publicHttpClient.RebuildZoekenProjection(cancellationToken); + + return response.IsSuccessStatusCode + ? Ok() + : UnprocessableEntity(); + } + + [HttpGet("public/status")] + public async Task GetPublicProjectionStatus(CancellationToken cancellationToken) + { + var response = await _publicHttpClient.GetStatus(cancellationToken); + + if (!response.IsSuccessStatusCode) return BadRequest(); + + var projectionProgress = await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, cancellationToken); + + return new OkObjectResult(projectionProgress); + } +} diff --git a/src/AssociationRegistry.Admin.Api/Program.cs b/src/AssociationRegistry.Admin.Api/Program.cs index 612c91fc5..d1f65098d 100755 --- a/src/AssociationRegistry.Admin.Api/Program.cs +++ b/src/AssociationRegistry.Admin.Api/Program.cs @@ -18,15 +18,14 @@ namespace AssociationRegistry.Admin.Api; using FluentValidation; using Framework; using IdentityModel.AspNetCore.OAuth2Introspection; -using Infrastructure; using Infrastructure.Configuration; using Infrastructure.ConfigurationBindings; using Infrastructure.ExceptionHandlers; using Infrastructure.Extensions; +using Infrastructure.HttpClients; using Infrastructure.Json; using Infrastructure.Middleware; using JasperFx.CodeGeneration; -using JasperFx.Core; using Kbo; using Lamar.Microsoft.DependencyInjection; using Magda; @@ -308,6 +307,14 @@ private static void ConfigureServices(WebApplicationBuilder builder) .AddHttpContextAccessor() .AddControllers(options => options.Filters.Add()); + builder.Services + .AddHttpClient() + .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(appSettings.BeheerProjectionHostBaseUrl)); + + builder.Services + .AddHttpClient() + .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(appSettings.PublicProjectionHostBaseUrl)); + builder.Services.TryAddEnumerable(ServiceDescriptor.Transient()); builder.Services diff --git a/src/AssociationRegistry.Admin.Api/appsettings.development.json b/src/AssociationRegistry.Admin.Api/appsettings.development.json index 2d654b4df..56ee42878 100644 --- a/src/AssociationRegistry.Admin.Api/appsettings.development.json +++ b/src/AssociationRegistry.Admin.Api/appsettings.development.json @@ -32,7 +32,14 @@ "Authority": "http://127.0.0.1:5051", "IntrospectionEndpoint": "http://127.0.0.1:5051/connect/introspect" }, + "BaseUrl": "http://127.0.0.1:11004/", + + "BeheerApiBaseUrl": "http://127.0.0.1:11004/", + "BeheerProjectionHostBaseUrl": "http://127.0.0.1:11006/", + "PublicApiBaseUrl": "http://127.0.0.1:11003/", + "PublicProjectionHostBaseUrl": "http://127.0.0.1:11005/", + "MagdaOptions": { "Afzender": "1234", "Hoedanigheid": "1234", diff --git a/src/AssociationRegistry.Admin.ProjectionHost/AssociationRegistry.Admin.ProjectionHost.csproj b/src/AssociationRegistry.Admin.ProjectionHost/AssociationRegistry.Admin.ProjectionHost.csproj index ecb23c0b1..81258c7a4 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/AssociationRegistry.Admin.ProjectionHost.csproj +++ b/src/AssociationRegistry.Admin.ProjectionHost/AssociationRegistry.Admin.ProjectionHost.csproj @@ -41,6 +41,7 @@ + diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs index a2bda978a..c2b63869e 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs @@ -12,6 +12,12 @@ public static void CreateVerenigingIndex(this IndicesNamespace indicesNamespace, selector: descriptor => descriptor.Map(VerenigingZoekDocumentMapping.Get)); + public static Task CreateVerenigingIndexAsync(this IndicesNamespace indicesNamespace, IndexName index) + => indicesNamespace.CreateAsync( + index, + selector: descriptor => + descriptor.Map(VerenigingZoekDocumentMapping.Get)); + public static void CreateDuplicateDetectionIndex(this IndicesNamespace indicesNamespace, IndexName index) { var createIndexResponse = indicesNamespace.Create( @@ -33,6 +39,24 @@ public static void CreateDuplicateDetectionIndex(this IndicesNamespace indicesNa throw createIndexResponse.OriginalException; } + public static async Task CreateDuplicateDetectionIndexAsync( + this IndicesNamespace indicesNamespace, + IndexName index) + => await indicesNamespace.CreateAsync( + index, + selector: c => c + .Settings(s => s + .Analysis(a => a + .CharFilters(cf => cf.PatternReplace(name: "dot_replace", + selector: prcf + => prcf.Pattern("\\.").Replacement("")) + .PatternReplace(name: "underscore_replace", + selector: prcf + => prcf.Pattern("_").Replacement(" "))) + .Analyzers(AddDuplicateDetectionAnalyzer) + .TokenFilters(AddDutchStopWordsFilter))) + .Map(DuplicateDetectionDocumentMapping.Get)); + private static TokenFiltersDescriptor AddDutchStopWordsFilter(TokenFiltersDescriptor tf) => tf.Stop(name: "dutch_stop", selector: st => st .StopWords("_dutch_") // Or provide your custom list diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplication/PrepareElasticSearch.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplication/PrepareElasticSearch.cs new file mode 100644 index 000000000..a408bcdeb --- /dev/null +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplication/PrepareElasticSearch.cs @@ -0,0 +1,33 @@ +namespace AssociationRegistry.Admin.ProjectionHost.Infrastructure.Program.WebApplication; + +using ConfigurationBindings; +using Extensions; +using Hosts; +using Nest; +using Program = ProjectionHost.Program; +using WebApplication = Microsoft.AspNetCore.Builder.WebApplication; + +public static class PrepareElasticSearch +{ + public static async Task EnsureElasticSearchIsInitialized(this WebApplication source) + { + var elasticClient = source.Services.GetRequiredService(); + var elasticSearchOptions = source.Services.GetRequiredService(); + + await WaitFor.ElasticSearchToBecomeAvailable( + elasticClient, source.Services.GetRequiredService>(), CancellationToken.None); + + await EnsureIndexExists(elasticClient, elasticSearchOptions.Indices!.Verenigingen!); + } + + private static async Task EnsureIndexExists(IElasticClient elasticClient, string verenigingenIndexName) + { + if (!(await elasticClient.Indices.ExistsAsync(verenigingenIndexName)).Exists) + { + var response = await elasticClient.Indices.CreateVerenigingIndexAsync(verenigingenIndexName); + + if (!response.IsValid) + throw response.OriginalException; + } + } +} diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs index ab31d4f1c..49f13cdc5 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs @@ -15,6 +15,7 @@ public static IServiceCollection ConfigureElasticSearch( ElasticSearchExtensions.EnsureIndexExists(elasticClient, elasticSearchOptions.Indices!.Verenigingen!, elasticSearchOptions.Indices!.DuplicateDetection!); + services.AddSingleton(elasticSearchOptions); services.AddSingleton(_ => elasticClient); services.AddSingleton(_ => elasticClient); diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs index 581287fa5..d11d0a957 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs @@ -10,6 +10,7 @@ namespace AssociationRegistry.Admin.ProjectionHost.Infrastructure.Program.WebApp using Marten.Events.Projections; using Marten.Services; using Newtonsoft.Json; +using Projections; using Projections.Detail; using Projections.Historiek; using Projections.Search; @@ -90,7 +91,7 @@ static JsonNetSerializer CreateCustomMartenSerializer() ) ), ProjectionLifecycle.Async, - projectionName: "BeheerVerenigingZoekenDocument"); + projectionName: ProjectionNames.VerenigingZoeken); opts.Serializer(CreateCustomMartenSerializer()); diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Program.cs b/src/AssociationRegistry.Admin.ProjectionHost/Program.cs index 465ec8981..0d041c08a 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Program.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Program.cs @@ -1,22 +1,29 @@ namespace AssociationRegistry.Admin.ProjectionHost; using Be.Vlaanderen.Basisregisters.Aws.DistributedMutex; +using Infrastructure.ConfigurationBindings; +using Infrastructure.Extensions; using Infrastructure.Json; using Infrastructure.Program; using Infrastructure.Program.WebApplication; using Infrastructure.Program.WebApplicationBuilder; using JasperFx.CodeGeneration; using Marten; +using Marten.Events.Daemon; using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.Mvc.ApplicationModels; using Microsoft.AspNetCore.Mvc.NewtonsoftJson; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Nest; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using NodaTime; using Oakton; using OpenTelemetry.Extensions; +using Projections; using Projections.Detail; +using Projections.Historiek; using Serilog; using Serilog.Debugging; using System.Net; @@ -31,7 +38,8 @@ public static async Task Main(string[] args) builder.Configuration .AddJsonFile("appsettings.json") - .AddJsonFile($"appsettings.{builder.Environment.EnvironmentName.ToLowerInvariant()}.json", optional: true, reloadOnChange: false) + .AddJsonFile($"appsettings.{builder.Environment.EnvironmentName.ToLowerInvariant()}.json", optional: true, + reloadOnChange: false) .AddJsonFile($"appsettings.{Environment.MachineName.ToLowerInvariant()}.json", optional: true, reloadOnChange: false) .AddEnvironmentVariables() .AddCommandLine(args); @@ -76,7 +84,27 @@ public static async Task Main(string[] args) var app = builder.Build(); app.MapPost( - pattern: "/rebuild", + pattern: "projections/all/rebuild", + handler: async ( + IDocumentStore store, + IElasticClient elasticClient, + ElasticSearchOptionsSection options, + ILogger logger, + CancellationToken cancellationToken) => + { + var projectionDaemon = await store.BuildProjectionDaemonAsync(); + await projectionDaemon.RebuildProjection(cancellationToken); + logger.LogInformation("Rebuild BeheerVerenigingDetailProjection complete"); + + await projectionDaemon.RebuildProjection(cancellationToken); + logger.LogInformation("Rebuild BeheerVerenigingHistoriekProjection complete"); + + await RebuildElasticProjections(projectionDaemon, elasticClient, options, cancellationToken); + logger.LogInformation("Rebuild ElasticSearch complete"); + }); + + app.MapPost( + pattern: "projections/detail/rebuild", handler: async (IDocumentStore store, ILogger logger, CancellationToken cancellationToken) => { var projectionDaemon = await store.BuildProjectionDaemonAsync(); @@ -84,6 +112,38 @@ public static async Task Main(string[] args) logger.LogInformation("Rebuild complete"); }); + app.MapPost( + pattern: "projections/historiek/rebuild", + handler: async (IDocumentStore store, ILogger logger, CancellationToken cancellationToken) => + { + var projectionDaemon = await store.BuildProjectionDaemonAsync(); + await projectionDaemon.RebuildProjection(cancellationToken); + logger.LogInformation("Rebuild complete"); + }); + + app.MapPost( + pattern: "projections/search/rebuild", + handler: async ( + IDocumentStore store, + IElasticClient elasticClient, + ElasticSearchOptionsSection options, + ILogger logger, + CancellationToken cancellationToken) => + { + var projectionDaemon = await store.BuildProjectionDaemonAsync(); + await RebuildElasticProjections(projectionDaemon, elasticClient, options, cancellationToken); + logger.LogInformation("Rebuild complete"); + }); + + app.MapGet( + pattern: "projections/status", + handler: async (IDocumentStore store, ILogger logger, CancellationToken cancellationToken) => + { + var projectionProgress = await store.Advanced.AllProjectionProgress(token: cancellationToken); + + return projectionProgress; + }); + app.SetUpSwagger(); ConfigureHealtChecks(app); @@ -93,6 +153,29 @@ await DistributedLock.RunAsync( app.Services.GetRequiredService>()); } + private static async Task RebuildElasticProjections( + IProjectionDaemon projectionDaemon, + IElasticClient elasticClient, + ElasticSearchOptionsSection options, + CancellationToken cancellationToken) + { + await projectionDaemon.StopShard($"{ProjectionNames.VerenigingZoeken}:All"); + var oldVerenigingenIndices = await elasticClient.GetIndicesPointingToAliasAsync(options.Indices.Verenigingen); + var newIndicesVerenigingen = options.Indices.Verenigingen + "-" + SystemClock.Instance.GetCurrentInstant().ToZuluTime(); + await elasticClient.Indices.CreateVerenigingIndexAsync(newIndicesVerenigingen).ThrowIfInvalidAsync(); + + await elasticClient.Indices.DeleteAsync(options.Indices.DuplicateDetection, ct: cancellationToken).ThrowIfInvalidAsync(); + await elasticClient.Indices.CreateDuplicateDetectionIndexAsync(options.Indices.DuplicateDetection).ThrowIfInvalidAsync(); + await projectionDaemon.RebuildProjection(ProjectionNames.VerenigingZoeken, cancellationToken); + + await elasticClient.Indices.PutAliasAsync(newIndicesVerenigingen, options.Indices.Verenigingen, ct: cancellationToken); + + foreach (var indeces in oldVerenigingenIndices) + { + await elasticClient.Indices.DeleteAsync(indeces, ct: cancellationToken).ThrowIfInvalidAsync(); + } + } + private static void ConfigureEncoding() { Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); @@ -163,3 +246,14 @@ private static void ConfigureHealtChecks(WebApplication app) app.UseHealthChecks(path: "/health", healthCheckOptions); } } + +public static class ResponseExtensions +{ + public static async Task ThrowIfInvalidAsync(this Task response) + where TResponse : AcknowledgedResponseBase + { + var acknowledgedResponseBase = await response; + + return acknowledgedResponseBase.IsValid ? acknowledgedResponseBase : throw acknowledgedResponseBase.OriginalException; + } +} diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Projections/ProjectionNames.cs b/src/AssociationRegistry.Admin.ProjectionHost/Projections/ProjectionNames.cs new file mode 100644 index 000000000..6559cba24 --- /dev/null +++ b/src/AssociationRegistry.Admin.ProjectionHost/Projections/ProjectionNames.cs @@ -0,0 +1,6 @@ +namespace AssociationRegistry.Admin.ProjectionHost.Projections; + +public class ProjectionNames +{ + public const string VerenigingZoeken = "BeheerVerenigingZoekenDocument"; +} diff --git a/src/AssociationRegistry.Public.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs b/src/AssociationRegistry.Public.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs index dd4b71cc8..b2a60428a 100644 --- a/src/AssociationRegistry.Public.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs +++ b/src/AssociationRegistry.Public.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs @@ -12,8 +12,9 @@ public static IServiceCollection ConfigureElasticSearch( { var elasticClient = CreateElasticClient(elasticSearchOptions); - services.AddSingleton(elasticClient); services.AddSingleton(elasticSearchOptions); + + services.AddSingleton(elasticClient); services.AddSingleton(provider => provider.GetRequiredService()); return services; diff --git a/src/AssociationRegistry.Public.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs b/src/AssociationRegistry.Public.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs index b530d04d3..b628d523b 100644 --- a/src/AssociationRegistry.Public.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs +++ b/src/AssociationRegistry.Public.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs @@ -10,6 +10,7 @@ namespace AssociationRegistry.Public.ProjectionHost.Infrastructure.Program.WebAp using Marten.Events.Projections; using Marten.Services; using Newtonsoft.Json; +using Projections; using Projections.Detail; using Projections.Search; using Schema.Detail; @@ -84,7 +85,7 @@ static JsonNetSerializer CreateCustomMartenSerializer() ) ), ProjectionLifecycle.Async, - projectionName: "PubliekVerenigingZoekenDocument"); + projectionName: ProjectionNames.VerenigingZoeken); opts.Serializer(CreateCustomMartenSerializer()); diff --git a/src/AssociationRegistry.Public.ProjectionHost/Program.cs b/src/AssociationRegistry.Public.ProjectionHost/Program.cs index c7986fba7..68babc618 100644 --- a/src/AssociationRegistry.Public.ProjectionHost/Program.cs +++ b/src/AssociationRegistry.Public.ProjectionHost/Program.cs @@ -1,6 +1,8 @@ namespace AssociationRegistry.Public.ProjectionHost; using Be.Vlaanderen.Basisregisters.Aws.DistributedMutex; +using Infrastructure.ConfigurationBindings; +using Infrastructure.Extensions; using Infrastructure.Json; using Infrastructure.Program; using Infrastructure.Program.WebApplication; @@ -12,10 +14,12 @@ namespace AssociationRegistry.Public.ProjectionHost; using Microsoft.AspNetCore.Mvc.NewtonsoftJson; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Nest; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Oakton; using OpenTelemetry.Extensions; +using Projections; using Projections.Detail; using Projections.Search; using Serilog; @@ -79,7 +83,7 @@ public static async Task Main(string[] args) var app = builder.Build(); app.MapPost( - pattern: "/rebuild", + pattern: "projections/detail/rebuild", handler: async (IDocumentStore store, ILogger logger, CancellationToken cancellationToken) => { var projectionDaemon = await store.BuildProjectionDaemonAsync(); @@ -87,6 +91,32 @@ public static async Task Main(string[] args) logger.LogInformation("Rebuild complete"); }); + app.MapPost( + pattern: "projections/search/rebuild", + handler: async ( + IDocumentStore store, + IElasticClient elasticClient, + ElasticSearchOptionsSection options, + ILogger logger, + CancellationToken cancellationToken) => + { + var projectionDaemon = await store.BuildProjectionDaemonAsync(); + await projectionDaemon.StopShard($"{ProjectionNames.VerenigingZoeken}:All"); + + await elasticClient.Indices.DeleteAsync(options.Indices.Verenigingen, ct: cancellationToken); + await elasticClient.Indices.CreateVerenigingIndex(options.Indices.Verenigingen); + + await projectionDaemon.RebuildProjection(ProjectionNames.VerenigingZoeken, cancellationToken); + logger.LogInformation("Rebuild complete"); + }); + + app.MapGet( + "projections/status", + async (IDocumentStore store, ILogger logger, CancellationToken cancellationToken) => + { + return await store.Advanced.AllProjectionProgress(token: cancellationToken); + }); + app.SetUpSwagger(); await app.EnsureElasticSearchIsInitialized(); ConfigureHealtChecks(app); diff --git a/src/AssociationRegistry.Public.ProjectionHost/Projections/ProjectionNames.cs b/src/AssociationRegistry.Public.ProjectionHost/Projections/ProjectionNames.cs new file mode 100644 index 000000000..15b66eec8 --- /dev/null +++ b/src/AssociationRegistry.Public.ProjectionHost/Projections/ProjectionNames.cs @@ -0,0 +1,6 @@ +namespace AssociationRegistry.Public.ProjectionHost.Projections; + +public class ProjectionNames +{ + public const string VerenigingZoeken = "PubliekVerenigingZoekenDocument"; +} diff --git a/src/AssociationRegistry/ProjectionStatus.cs b/src/AssociationRegistry/ProjectionStatus.cs new file mode 100644 index 000000000..3d26e03ef --- /dev/null +++ b/src/AssociationRegistry/ProjectionStatus.cs @@ -0,0 +1,9 @@ +namespace AssociationRegistry; + +public class ProjectionStatus +{ + public DateTimeOffset Timestamp { get; set; } + public string ShardName { get; set; } + public int Sequence { get; set; } + public string Exception { get; set; } +}