Forbedre ytelse på identpool (#3647)

* Add repository methods for counting and finding Idents

Introduced new methods in `IdentRepository` to count and find `Ident` entities based on various parameters including `Rekvireringsstatus`, `Identtype`, `Syntetisk`, `Kjoenn`, and date range. Updated `DatabaseService` to utilize these new repository methods, enhancing the service's querying capabilities.

* Refactor Ident fetching logic and add H2 support
#deploy-testnav-ident-pool

Revised methods for fetching and counting Idents in `DatabaseService` to improve query efficiency. Enabled H2 database console for local development and included sample data for testing. Adjusted security configurations to accommodate new endpoints and data sources.

* Update tests to handle potential insufficient identifiers
#deploy-testnav-ident-pool

Modified the tests to ensure that identifier generation does not throw an exception when the desired amount of identifiers cannot be generated. Instead, the tests now assert that the resulting set size is less than the requested amount.

* Configure HikariCP datasource properties
#deploy-testnav-ident-pool

Added HikariCP datasource configuration with minimum idle connections set to 10 and maximum pool size set to 100 to improve database connection management. This change is intended to optimize performance and resource utilization.

* Create index for syntetisk in personidentifikator
#deploy-testnav-ident-pool

This commit adds a new SQL migration script (V1.3.1__IndeksSyntetisk.sql) that creates an index for the 'syntetisk' column in the 'personidentifikator' table. This change is aimed at improving query performance involving the 'syntetisk' column.

* Update PostgreSQL version in config to 16
#deploy-testnav-ident-pool

Changed the PostgreSQL version from 15 to 16 in the configuration file. This update ensures compatibility with the latest features and improvements provided by PostgreSQL 16.

* Increase CPU request limit from 200m to 600m in config.yml
#deploy-testnav-ident-pool

Adjusting the CPU request limit ensures that the application has sufficient resources to handle increased load. This change aims to prevent potential performance issues under higher demand.

* Add new records and tweak logging in PoolService #deploy-testnav-ident-pool

Inserted additional `personidentifikator` entries to the H2 database initialization script and adjusted the sequence restart point. Enhanced logging in `PoolService` to include the count of generated identifiers and increased the number of attempts for synthetic identifier generation.

* Refactor variable names in PoolService

Changed variable names to improve code readability and consistency in the `allocateIdenter` method. Replaced `List<Ident>` with `var` keyword for type inference. These changes should help streamline future modifications and maintenance.

apps/testnav-ident-pool/config.yml

@@ -52,7 +52,7 @@ spec:
     limits:
       memory: 4096Mi
     requests:
-      cpu: 200m
+      cpu: 600m
       memory: 2048Mi
   accessPolicy:
     inbound:
@@ -68,7 +68,7 @@ spec:
           cluster: dev-gcp
   gcp:
     sqlInstances:
-      - type: POSTGRES_15
+      - type: POSTGRES_16
         tier: db-custom-2-7680
         name: testnav-identpool
         databases:

apps/testnav-ident-pool/src/main/java/no/nav/testnav/identpool/config/SecurityConfig.java

@@ -6,6 +6,7 @@
 import org.springframework.security.config.annotation.web.builders.HttpSecurity;
 import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
 import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
+import org.springframework.security.config.annotation.web.configurers.HeadersConfigurer;
 import org.springframework.security.config.http.SessionCreationPolicy;
 import org.springframework.security.web.SecurityFilterChain;
 
@@ -19,15 +20,20 @@ public SecurityFilterChain filterChain(HttpSecurity httpSecurity) throws Excepti
         httpSecurity.sessionManagement(sessionConfig -> sessionConfig.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
                 .csrf(AbstractHttpConfigurer::disable)
                 .authorizeHttpRequests(authorizeConfig -> authorizeConfig.requestMatchers(
-                        "/internal/**",
-                        "/webjars/**",
-                        "/swagger-resources/**",
-                        "/v3/api-docs/**",
-                        "/swagger-ui/**",
-                        "/swagger",
-                        "/error",
-                        "/swagger-ui.html"
-                ).permitAll().requestMatchers("/api/**").fullyAuthenticated())
+                                "/internal/**",
+                                "/webjars/**",
+                                "/swagger-resources/**",
+                                "/v3/api-docs/**",
+                                "/swagger-ui/**",
+                                "/swagger",
+                                "/error",
+                                "/swagger-ui.html",
+                                "/h2/**",
+                                "/member/**")
+                        .permitAll()
+                        .requestMatchers("/api/**")
+                        .fullyAuthenticated())
+                .headers(headers -> headers.frameOptions(HeadersConfigurer.FrameOptionsConfig::disable))
                 .oauth2ResourceServer(oauth2RSConfig -> oauth2RSConfig.jwt(Customizer.withDefaults()));
 
         return httpSecurity.build();

apps/testnav-ident-pool/src/main/java/no/nav/testnav/identpool/repository/IdentRepository.java

@@ -55,4 +55,40 @@ Page<Ident> findAll(@Param("rekvireringsstatus") Rekvireringsstatus rekvirerings
                         @Param("identtype") Identtype identtype, @Param("kjoenn") Kjoenn kjoenn,
                         @Param("foedtFoer") LocalDate foedtFoer, @Param("foedtEtter") LocalDate foedtEtter,
                         @Param("syntetisk") boolean syntetisk, Pageable pageable);
+
+    @Query
+    int countAllByRekvireringsstatusAndIdenttypeAndSyntetiskAndFoedselsdatoBetween(
+            Rekvireringsstatus rekvireringsstatus,
+            Identtype identtype,
+            Boolean syntetisk,
+            LocalDate foedtEtter,
+            LocalDate foedtFoer);
+
+    @Query
+    int countAllByRekvireringsstatusAndIdenttypeAndSyntetiskAndKjoennAndFoedselsdatoBetween(
+            Rekvireringsstatus rekvireringsstatus,
+            Identtype identtype,
+            Boolean syntetisk,
+            Kjoenn kjoenn,
+            LocalDate foedtEtter,
+            LocalDate foedtFoer);
+
+    @Query
+    Page<Ident> findAllByRekvireringsstatusAndIdenttypeAndSyntetiskAndFoedselsdatoBetween(
+            Rekvireringsstatus rekvireringsstatus,
+            Identtype identtype,
+            Boolean syntetisk,
+            LocalDate foedtEtter,
+            LocalDate foedtFoer,
+            Pageable pageable);
+
+    @Query
+    Page<Ident> findAllByRekvireringsstatusAndIdenttypeAndSyntetiskAndKjoennAndFoedselsdatoBetween(
+            Rekvireringsstatus rekvireringsstatus,
+            Identtype identtype,
+            Boolean syntetisk,
+            Kjoenn kjoenn,
+            LocalDate foedtEtter,
+            LocalDate foedtFoer,
+            Pageable pageable);
 }

apps/testnav-ident-pool/src/main/java/no/nav/testnav/identpool/service/DatabaseService.java

@@ -8,59 +8,75 @@
 import no.nav.testnav.identpool.repository.IdentRepository;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Service;
 
 import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
+import java.util.Random;
 import java.util.Set;
 
+import static java.util.Objects.nonNull;
 import static org.apache.commons.lang3.BooleanUtils.isTrue;
 
 @Service
 @RequiredArgsConstructor
 public class DatabaseService {
 
+    private static final Random RANDOM = new SecureRandom();
+
     private final IdentRepository identRepository;
     private final MapperFacade mapperFacade;
 
     public Set<Ident> hentLedigeIdenterFraDatabase(HentIdenterRequest request) {
-        Set<Ident> identEntities = new HashSet<>();
-
-        HentIdenterRequest availableIdentsRequest = mapperFacade.map(request, HentIdenterRequest.class);
-
-        var firstPage = findPage(availableIdentsRequest, Rekvireringsstatus.LEDIG, 0);
-        var pageCache = new HashMap<Integer, Page<Ident>>();
-        pageCache.put(0, firstPage);
-
-        int totalPages = firstPage.getTotalPages();
-        if (totalPages > 0) {
-            List<String> usedIdents = new ArrayList<>();
-            SecureRandom rand = new SecureRandom();
-            for (var i = 0; i < request.getAntall(); i++) {
-                var randomPageNumber = rand.nextInt(totalPages);
-                pageCache.computeIfAbsent(randomPageNumber, k ->
-                        findPage(availableIdentsRequest, Rekvireringsstatus.LEDIG, randomPageNumber));
-
-                List<Ident> content = pageCache.get(randomPageNumber).getContent();
-                for (Ident ident : content) {
-                    if (!usedIdents.contains(ident.getPersonidentifikator())) {
-                        usedIdents.add(ident.getPersonidentifikator());
-                        identEntities.add(ident);
-                        break;
-                    }
-                }
-            }
+
+        var availableIdentsRequest = mapperFacade.map(request, HentIdenterRequest.class);
+
+        var antall = getAntall(availableIdentsRequest);
+
+        if (antall == 0) {
+            return new HashSet<>();
         }
-        return identEntities;
+
+        if (antall > request.getAntall()) {
+            var resultat = getPage(request, PageRequest.of(RANDOM.nextInt(antall/request.getAntall()), request.getAntall()));
+            return new HashSet<>(resultat.getContent());
+        }
+
+        return new HashSet<>(
+                getPage(request, PageRequest.of(0, request.getAntall()))
+                .getContent());
     }
 
-    private Page<Ident> findPage(HentIdenterRequest request, Rekvireringsstatus rekvireringsstatus, int page) {
+    private int getAntall(HentIdenterRequest request) {
+
+        return nonNull(request.getKjoenn()) ?
+
+                identRepository.countAllByRekvireringsstatusAndIdenttypeAndSyntetiskAndKjoennAndFoedselsdatoBetween(
+                        Rekvireringsstatus.LEDIG, request.getIdenttype(),
+                        isTrue(request.getSyntetisk()), request.getKjoenn(),
+                        request.getFoedtEtter(), request.getFoedtFoer()) :
+
+                identRepository.countAllByRekvireringsstatusAndIdenttypeAndSyntetiskAndFoedselsdatoBetween(
+                        Rekvireringsstatus.LEDIG, request.getIdenttype(),
+                        isTrue(request.getSyntetisk()),
+                                request.getFoedtEtter(), request.getFoedtFoer());
+    }
+
+    private Page<Ident> getPage(HentIdenterRequest request, Pageable page) {
+
+        return nonNull(request.getKjoenn()) ?
+
+                identRepository.findAllByRekvireringsstatusAndIdenttypeAndSyntetiskAndKjoennAndFoedselsdatoBetween(
+                        Rekvireringsstatus.LEDIG, request.getIdenttype(),
+                        isTrue(request.getSyntetisk()), request.getKjoenn(),
+                        request.getFoedtEtter(), request.getFoedtFoer(), page
+                ) :
 
-        return identRepository.findAll(
-                rekvireringsstatus, request.getIdenttype(), request.getKjoenn(), request.getFoedtFoer(),
-                request.getFoedtEtter(), isTrue(request.getSyntetisk()), PageRequest.of(page, request.getAntall()));
+                identRepository.findAllByRekvireringsstatusAndIdenttypeAndSyntetiskAndFoedselsdatoBetween(
+                        Rekvireringsstatus.LEDIG, request.getIdenttype(),
+                        isTrue(request.getSyntetisk()),
+                        request.getFoedtEtter(), request.getFoedtFoer(), page
+                );
     }
 }

apps/testnav-ident-pool/src/main/java/no/nav/testnav/identpool/service/IdentGeneratorService.java

@@ -12,6 +12,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -31,7 +32,7 @@
 public class IdentGeneratorService {
 
     private static final int SYNTETISK = 4;
-    private static final SecureRandom random = new SecureRandom();
+    private static final Random random = new SecureRandom();
 
     private static String addSyntetiskIdentifier(String format) {
         return String.format("%s%1d%s", format.substring(0, 2), Integer.parseInt(format.substring(2, 3)) + SYNTETISK, format.substring(3));
@@ -67,45 +68,47 @@ public Set<String> genererIdenter(HentIdenterRequest request, Set<String> idente
             request.setFoedtFoer(request.getFoedtEtter().plusDays(1));
         }
 
-        var identer = identerIIdentPool;
-        var antall = request.getAntall() + identer.size();
+        var antall = request.getAntall() + identerIIdentPool.size();
         var iteratorRange = (request.getKjoenn() == null) ? 1 : 2;
         var numberOfDates = toIntExact(ChronoUnit.DAYS.between(request.getFoedtEtter(), request.getFoedtFoer()));
 
         Function<LocalDate, String> numberFormat =
                 numberFormatter.getOrDefault(request.getIdenttype(), IdentGeneratorUtil::randomFormat);
 
-        while (identer.size() < antall) {
+        while (identerIIdentPool.size() < antall) {
             var birthdate = request.getFoedtEtter().plusDays(random.nextInt(numberOfDates));
             var format = numberFormat.apply(birthdate);
             if (isTrue(request.getSyntetisk())) {
                 format = addSyntetiskIdentifier(format);
             }
 
             var yearRange = getYearRange(birthdate);
-            var originalSize = identer.size();
+            var originalSize = identerIIdentPool.size();
             var genderNumber = getGenderNumber(yearRange, request.getKjoenn());
             var startIndex = getStartIndex(yearRange.get(0), request.getKjoenn());
 
             for (int i = startIndex; identerIIdentPool.size() == originalSize && i < genderNumber; i += iteratorRange) {
                 String fnr = generateFnr(String.format(format, i));
                 if (fnr != null) {
-                    identer.add(fnr);
+                    identerIIdentPool.add(fnr);
                 }
             }
 
-            for (int i = genderNumber; identer.size() == originalSize && i < yearRange.get(1); i += iteratorRange) {
-                String fnr = generateFnr(String.format(format, i));
+            for (int i = genderNumber; identerIIdentPool.size() == originalSize && i < yearRange.get(1); i += iteratorRange) {
+                var fnr = generateFnr(String.format(format, i));
                 if (fnr != null) {
-                    identer.add(fnr);
+                    identerIIdentPool.add(fnr);
                 }
             }
 
             if (identerIIdentPool.size() == originalSize) {
-                throw new IllegalArgumentException("Kan ikke finne ønsket antall fødselsnummer med angitte kriterier");
+                break;
             }
         }
-        return identer;
+        if (identerIIdentPool.isEmpty()) {
+            throw new IllegalArgumentException("Finner ingen fødselsnummer med angitte kriterier");
+        }
+        return identerIIdentPool;
     }
 
     private void validateDates(LocalDate foedtEtter, LocalDate foedtFoer) {

apps/testnav-ident-pool/src/main/java/no/nav/testnav/identpool/service/PoolService.java

@@ -3,18 +3,14 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import no.nav.testnav.identpool.domain.Ident;
-import no.nav.testnav.identpool.domain.Identtype;
 import no.nav.testnav.identpool.domain.Rekvireringsstatus;
 import no.nav.testnav.identpool.dto.TpsStatusDTO;
 import no.nav.testnav.identpool.exception.ForFaaLedigeIdenterException;
 import no.nav.testnav.identpool.providers.v1.support.HentIdenterRequest;
 import no.nav.testnav.identpool.repository.IdentRepository;
 import org.springframework.stereotype.Service;
 
-import java.util.Iterator;
 import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import static java.lang.String.format;
 import static java.time.format.DateTimeFormatter.ISO_DATE;
@@ -64,19 +60,22 @@ private static void logRequest(HentIdenterRequest request) {
 
     public synchronized List<String> allocateIdenter(HentIdenterRequest request) {
 
-        Set<Ident> identEntities = databaseService.hentLedigeIdenterFraDatabase(request);
+        var identEntities = databaseService.hentLedigeIdenterFraDatabase(request);
         int missingIdentCount = request.getAntall() - identEntities.size();
 
         if (missingIdentCount > 0) {
 
-            var tpsStatusDTOS = identerAvailService.generateAndCheckIdenter(request, ATTEMPT_OBTAIN);
+            var statusDTOS = identerAvailService.generateAndCheckIdenter(request,
+                    isTrue(request.getSyntetisk()) ? ATTEMPT_OBTAIN * 12 : ATTEMPT_OBTAIN);
 
-            List<Ident> identerFraTps = tpsStatusDTOS.stream()
+            log.info("Generert {} identer ved mining", statusDTOS.size());
+
+            var identerFraMining = statusDTOS.stream()
                     .map(this::buildIdent)
                     .toList();
-            identRepository.saveAll(identerFraTps);
+            identRepository.saveAll(identerFraMining);
 
-            Iterator<Ident> ledigeIdents = identerFraTps.stream()
+            var ledigeIdents = identerFraMining.stream()
                     .filter(Ident::isLedig)
                     .toList().iterator();
 

apps/testnav-ident-pool/src/main/resources/application-local.yml

@@ -1,27 +1,17 @@
 spring:
-  cloud:
-    vault:
-      connection-timeout: 15000
-      fail-fast: true
-      host: vault.adeo.no
-      port: 443
-      read-timeout: 30000
+  h2:
+    console:
+      enabled: true
+      path: /h2
   datasource:
-    hikari:
-      maximum-pool-size: 3
-      minimum-idle: 1
-    url: jdbc:postgresql://localhost:5432/ident-pool-test
-    username: postgres
-  flyway:
-    baseline-on-migrate: true
-    enabled: true # Disabled by default as you should probably think twice before running Flyway-migrations
-    locations: classpath:/db/migration
-  jpa:
-    properties:
-      hibernate:
-        dialect: org.hibernate.dialect.PostgreSQLDialect
-        default-schema: public
-        showSql: true
+    url: jdbc:h2:mem:testdb
+    username: sa
+    password:
+    driverClassName: org.h2.Driver
+  sql:
+    init:
+      mode: always
+      data-locations: classpath:/db/dev/h2-default-config.sql
 
 consumers:
   tps:

apps/testnav-ident-pool/src/main/resources/application-prod.yaml

@@ -1,8 +1,13 @@
 
 spring:
+  oauth2:
+      tokenx:
+        issuer-uri: ${TOKEN_X_ISSUER}
+        jwk-set-uri: ${TOKEN_X_JWKS_URI}
+        accepted-audience: ${TOKEN_X_CLIENT_ID}
   datasource:
     url: jdbc:postgresql://${NAIS_DATABASE_TESTNAV_IDENTPOOL_TESTNAV_IDENTPOOL_HOST}:${NAIS_DATABASE_TESTNAV_IDENTPOOL_TESTNAV_IDENTPOOL_PORT}/${NAIS_DATABASE_TESTNAV_IDENTPOOL_TESTNAV_IDENTPOOL_DATABASE}?user=${NAIS_DATABASE_TESTNAV_IDENTPOOL_TESTNAV_IDENTPOOL_USERNAME}&password=${NAIS_DATABASE_TESTNAV_IDENTPOOL_TESTNAV_IDENTPOOL_PASSWORD}
     driverClassName: org.postgresql.Driver
     hikari:
       maximum-pool-size: 3
-      minimum-idle: 1
+      minimum-idle: 1