feat: or-1349 add rebuild en status endpoints for projections

.github/otel-collector-config.yaml

@@ -0,0 +1,41 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+        include_metadata: true
+        cors:
+          max_age: 7200
+
+exporters:
+  logging:
+
+  otlp/elastic:
+    endpoint: https://a8e27ab098f54752b227af78fc609169.es.ops.vl.be:9243
+    headers:
+      Authorization: "Bearer iHyyarDx8Bvn9WYXDv"
+
+extensions:
+  health_check:
+    path: "/health"
+
+processors:
+  batch:
+
+service:
+  extensions:
+    - health_check
+
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging, otlp/elastic]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging, otlp/elastic]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging, otlp/elastic]

src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/AppSettings.cs

@@ -3,14 +3,35 @@
 public class AppSettings
 {
     private string? _baseUrl;
+    private string? _beheerApiBaseUrl;
+    private string? _beheerProjectionHostBaseUrl;
     private string? _publicApiBaseUrl;
+    private string? _publicProjectionHostBaseUrl;
+
+    public string BeheerApiBaseUrl
+    {
+        get => _beheerApiBaseUrl?.TrimEnd(trimChar: '/') ?? string.Empty;
+        set => _beheerApiBaseUrl = value;
+    }
+
+    public string BeheerProjectionHostBaseUrl
+    {
+        get => _beheerProjectionHostBaseUrl?.TrimEnd(trimChar: '/') ?? string.Empty;
+        set => _beheerProjectionHostBaseUrl = value;
+    }
 
     public string PublicApiBaseUrl
     {
         get => _publicApiBaseUrl?.TrimEnd(trimChar: '/') ?? string.Empty;
         set => _publicApiBaseUrl = value;
     }
 
+    public string PublicProjectionHostBaseUrl
+    {
+        get => _publicProjectionHostBaseUrl?.TrimEnd(trimChar: '/') ?? string.Empty;
+        set => _publicProjectionHostBaseUrl = value;
+    }
+
     public string BaseUrl
     {
         get => _baseUrl?.TrimEnd(trimChar: '/') ?? string.Empty;

src/AssociationRegistry.Admin.Api/Infrastructure/HttpClients/AdminProjectionHostHttpClient.cs

@@ -0,0 +1,41 @@
+namespace AssociationRegistry.Admin.Api.Infrastructure.HttpClients;
+
+using System;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class AdminProjectionHostHttpClient : IDisposable
+{
+    private readonly HttpClient _httpClient;
+
+    public AdminProjectionHostHttpClient(HttpClient httpClient)
+    {
+        _httpClient = httpClient;
+    }
+
+    public async Task<HttpResponseMessage> RebuildAllProjections(CancellationToken cancellationToken)
+        => await _httpClient.PostAsync(requestUri: "/projections/all/rebuild", content: null, cancellationToken);
+
+    public async Task<HttpResponseMessage> RebuildDetailProjection(CancellationToken cancellationToken)
+        => await _httpClient.PostAsync(requestUri: "/projections/detail/rebuild", content: null, cancellationToken);
+
+    public async Task<HttpResponseMessage> RebuildHistoriekProjection(CancellationToken cancellationToken)
+        => await _httpClient.PostAsync(requestUri: "/projections/historiek/rebuild", content: null, cancellationToken);
+
+    public async Task<HttpResponseMessage> RebuildZoekenProjection(CancellationToken cancellationToken)
+        => await _httpClient.PostAsync(requestUri: "/projections/search/rebuild", content: null, cancellationToken);
+
+    public async Task<HttpResponseMessage> GetStatus(CancellationToken cancellationToken)
+    {
+        var request = new HttpRequestMessage(HttpMethod.Get, requestUri: "/projections/status");
+        request.Headers.Add(name: "X-Correlation-Id", Guid.NewGuid().ToString());
+
+        return await _httpClient.SendAsync(request, cancellationToken);
+    }
+
+    public void Dispose()
+    {
+        _httpClient.Dispose();
+    }
+}

src/AssociationRegistry.Admin.Api/Infrastructure/HttpClients/PublicProjectionHostHttpClient.cs

@@ -0,0 +1,30 @@
+namespace AssociationRegistry.Admin.Api.Infrastructure.HttpClients;
+
+using System;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class PublicProjectionHostHttpClient : IDisposable
+{
+    private readonly HttpClient _httpClient;
+
+    public PublicProjectionHostHttpClient(HttpClient httpClient)
+    {
+        _httpClient = httpClient;
+    }
+
+    public async Task<HttpResponseMessage> RebuildDetailProjection(CancellationToken cancellationToken)
+        => await _httpClient.PostAsync(requestUri: "/projections/detail/rebuild", content: null, cancellationToken);
+
+    public async Task<HttpResponseMessage> RebuildZoekenProjection(CancellationToken cancellationToken)
+        => await _httpClient.PostAsync(requestUri: "/projections/search/rebuild", content: null, cancellationToken);
+
+    public async Task<HttpResponseMessage> GetStatus(CancellationToken cancellationToken)
+        => await _httpClient.GetAsync(requestUri: "/projections/status", cancellationToken);
+
+    public void Dispose()
+    {
+        _httpClient.Dispose();
+    }
+}

src/AssociationRegistry.Admin.Api/Infrastructure/ProjectionHostController.cs

@@ -0,0 +1,115 @@
+namespace AssociationRegistry.Admin.Api.Infrastructure;
+
+using Be.Vlaanderen.Basisregisters.Api;
+using HttpClients;
+using Microsoft.AspNetCore.Mvc;
+using System.Net.Http.Json;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+[ApiVersion("1.0")]
+[AdvertiseApiVersions("1.0")]
+[ApiRoute("projections")]
+[ApiExplorerSettings(IgnoreApi = true)]
+public class ProjectionHostController : ApiController
+{
+    private readonly AdminProjectionHostHttpClient _adminHttpClient;
+    private readonly PublicProjectionHostHttpClient _publicHttpClient;
+    private readonly JsonSerializerOptions _jsonSerializerOptions;
+
+    public ProjectionHostController(AdminProjectionHostHttpClient adminHttpClient, PublicProjectionHostHttpClient publicHttpClient)
+    {
+        _adminHttpClient = adminHttpClient;
+        _publicHttpClient = publicHttpClient;
+
+        _jsonSerializerOptions = new JsonSerializerOptions
+        {
+            PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        };
+    }
+
+    [HttpPost("admin/all/rebuild")]
+    public async Task<IActionResult> RebuildAdminProjectionAll(CancellationToken cancellationToken)
+    {
+        var response = await _adminHttpClient.RebuildAllProjections(cancellationToken);
+
+        return response.IsSuccessStatusCode
+            ? Ok()
+            : UnprocessableEntity();
+    }
+
+    [HttpPost("admin/detail/rebuild")]
+    public async Task<IActionResult> RebuildAdminProjectionDetail(CancellationToken cancellationToken)
+    {
+        var response = await _adminHttpClient.RebuildDetailProjection(cancellationToken);
+
+        return response.IsSuccessStatusCode
+            ? Ok()
+            : UnprocessableEntity();
+    }
+
+    [HttpPost("admin/historiek/rebuild")]
+    public async Task<IActionResult> RebuildAdminProjectionHistoriek(CancellationToken cancellationToken)
+    {
+        var response = await _adminHttpClient.RebuildHistoriekProjection(cancellationToken);
+
+        return response.IsSuccessStatusCode
+            ? Ok()
+            : UnprocessableEntity();
+    }
+
+    [HttpPost("admin/search/rebuild")]
+    public async Task<IActionResult> RebuildAdminProjectionZoeken(CancellationToken cancellationToken)
+    {
+        var response = await _adminHttpClient.RebuildZoekenProjection(cancellationToken);
+
+        return response.IsSuccessStatusCode
+            ? Ok()
+            : UnprocessableEntity();
+    }
+
+    [HttpGet("admin/status")]
+    public async Task<IActionResult> GetAdminProjectionStatus(CancellationToken cancellationToken)
+    {
+        var response = await _adminHttpClient.GetStatus(cancellationToken);
+
+        if (!response.IsSuccessStatusCode) return BadRequest();
+
+        var projectionProgress = await response.Content.ReadFromJsonAsync<ProjectionStatus[]>(_jsonSerializerOptions, cancellationToken);
+
+        return new OkObjectResult(projectionProgress);
+    }
+
+    [HttpPost("public/detail/rebuild")]
+    public async Task<IActionResult> RebuildPublicProjectionDetail(CancellationToken cancellationToken)
+    {
+        var response = await _publicHttpClient.RebuildDetailProjection(cancellationToken);
+
+        return response.IsSuccessStatusCode
+            ? Ok()
+            : UnprocessableEntity();
+    }
+
+    [HttpPost("public/search/rebuild")]
+    public async Task<IActionResult> RebuildPublicProjectionZoeken(CancellationToken cancellationToken)
+    {
+        var response = await _publicHttpClient.RebuildZoekenProjection(cancellationToken);
+
+        return response.IsSuccessStatusCode
+            ? Ok()
+            : UnprocessableEntity();
+    }
+
+    [HttpGet("public/status")]
+    public async Task<IActionResult> GetPublicProjectionStatus(CancellationToken cancellationToken)
+    {
+        var response = await _publicHttpClient.GetStatus(cancellationToken);
+
+        if (!response.IsSuccessStatusCode) return BadRequest();
+
+        var projectionProgress = await response.Content.ReadFromJsonAsync<ProjectionStatus[]>(_jsonSerializerOptions, cancellationToken);
+
+        return new OkObjectResult(projectionProgress);
+    }
+}

src/AssociationRegistry.Admin.Api/Program.cs

@@ -18,15 +18,14 @@ namespace AssociationRegistry.Admin.Api;
 using FluentValidation;
 using Framework;
 using IdentityModel.AspNetCore.OAuth2Introspection;
-using Infrastructure;
 using Infrastructure.Configuration;
 using Infrastructure.ConfigurationBindings;
 using Infrastructure.ExceptionHandlers;
 using Infrastructure.Extensions;
+using Infrastructure.HttpClients;
 using Infrastructure.Json;
 using Infrastructure.Middleware;
 using JasperFx.CodeGeneration;
-using JasperFx.Core;
 using Kbo;
 using Lamar.Microsoft.DependencyInjection;
 using Magda;
@@ -308,6 +307,14 @@ private static void ConfigureServices(WebApplicationBuilder builder)
                .AddHttpContextAccessor()
                .AddControllers(options => options.Filters.Add<JsonRequestFilter>());
 
+        builder.Services
+               .AddHttpClient<AdminProjectionHostHttpClient>()
+               .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(appSettings.BeheerProjectionHostBaseUrl));
+
+        builder.Services
+               .AddHttpClient<PublicProjectionHostHttpClient>()
+               .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(appSettings.PublicProjectionHostBaseUrl));
+
         builder.Services.TryAddEnumerable(ServiceDescriptor.Transient<IApiControllerSpecification, ApiControllerSpec>());
 
         builder.Services

src/AssociationRegistry.Admin.Api/appsettings.development.json

@@ -32,7 +32,14 @@
     "Authority": "http://127.0.0.1:5051",
     "IntrospectionEndpoint": "http://127.0.0.1:5051/connect/introspect"
   },
+
   "BaseUrl": "http://127.0.0.1:11004/",
+
+  "BeheerApiBaseUrl": "http://127.0.0.1:11004/",
+  "BeheerProjectionHostBaseUrl":  "http://127.0.0.1:11006/",
+  "PublicApiBaseUrl": "http://127.0.0.1:11003/",
+  "PublicProjectionHostBaseUrl":  "http://127.0.0.1:11005/",
+
   "MagdaOptions": {
     "Afzender": "1234",
     "Hoedanigheid": "1234",

src/AssociationRegistry.Admin.ProjectionHost/AssociationRegistry.Admin.ProjectionHost.csproj

@@ -41,6 +41,7 @@
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\AssociationRegistry.Admin.Schema\AssociationRegistry.Admin.Schema.csproj" />
+    <ProjectReference Include="..\AssociationRegistry.Hosts\AssociationRegistry.Hosts.csproj" />
     <ProjectReference Include="..\AssociationRegistry.OpenTelemetry\AssociationRegistry.OpenTelemetry.csproj" />
     <ProjectReference Include="..\AssociationRegistry\AssociationRegistry.csproj" />
   </ItemGroup>

src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs

@@ -12,6 +12,12 @@ public static void CreateVerenigingIndex(this IndicesNamespace indicesNamespace,
             selector: descriptor =>
                 descriptor.Map<VerenigingZoekDocument>(VerenigingZoekDocumentMapping.Get));
 
+    public static Task<CreateIndexResponse> CreateVerenigingIndexAsync(this IndicesNamespace indicesNamespace, IndexName index)
+        => indicesNamespace.CreateAsync(
+            index,
+            selector: descriptor =>
+                descriptor.Map<VerenigingZoekDocument>(VerenigingZoekDocumentMapping.Get));
+
     public static void CreateDuplicateDetectionIndex(this IndicesNamespace indicesNamespace, IndexName index)
     {
         var createIndexResponse = indicesNamespace.Create(
@@ -33,6 +39,24 @@ public static void CreateDuplicateDetectionIndex(this IndicesNamespace indicesNa
             throw createIndexResponse.OriginalException;
     }
 
+    public static async Task<CreateIndexResponse> CreateDuplicateDetectionIndexAsync(
+        this IndicesNamespace indicesNamespace,
+        IndexName index)
+        => await indicesNamespace.CreateAsync(
+            index,
+            selector: c => c
+                          .Settings(s => s
+                                       .Analysis(a => a
+                                                     .CharFilters(cf => cf.PatternReplace(name: "dot_replace",
+                                                                                          selector: prcf
+                                                                                              => prcf.Pattern("\\.").Replacement(""))
+                                                                          .PatternReplace(name: "underscore_replace",
+                                                                                          selector: prcf
+                                                                                              => prcf.Pattern("_").Replacement(" ")))
+                                                     .Analyzers(AddDuplicateDetectionAnalyzer)
+                                                     .TokenFilters(AddDutchStopWordsFilter)))
+                          .Map<DuplicateDetectionDocument>(DuplicateDetectionDocumentMapping.Get));
+
     private static TokenFiltersDescriptor AddDutchStopWordsFilter(TokenFiltersDescriptor tf)
         => tf.Stop(name: "dutch_stop", selector: st => st
                       .StopWords("_dutch_") // Or provide your custom list

src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplication/PrepareElasticSearch.cs

@@ -0,0 +1,33 @@
+namespace AssociationRegistry.Admin.ProjectionHost.Infrastructure.Program.WebApplication;
+
+using ConfigurationBindings;
+using Extensions;
+using Hosts;
+using Nest;
+using Program = ProjectionHost.Program;
+using WebApplication = Microsoft.AspNetCore.Builder.WebApplication;
+
+public static class PrepareElasticSearch
+{
+    public static async Task EnsureElasticSearchIsInitialized(this WebApplication source)
+    {
+        var elasticClient = source.Services.GetRequiredService<IElasticClient>();
+        var elasticSearchOptions = source.Services.GetRequiredService<ElasticSearchOptionsSection>();
+
+        await WaitFor.ElasticSearchToBecomeAvailable(
+            elasticClient, source.Services.GetRequiredService<ILogger<Program>>(), CancellationToken.None);
+
+        await EnsureIndexExists(elasticClient, elasticSearchOptions.Indices!.Verenigingen!);
+    }
+
+    private static async Task EnsureIndexExists(IElasticClient elasticClient, string verenigingenIndexName)
+    {
+        if (!(await elasticClient.Indices.ExistsAsync(verenigingenIndexName)).Exists)
+        {
+            var response = await elasticClient.Indices.CreateVerenigingIndexAsync(verenigingenIndexName);
+
+            if (!response.IsValid)
+                throw response.OriginalException;
+        }
+    }
+}