From a960ed67816f2b8b43cf6be018f3597588dc3e0c Mon Sep 17 00:00:00 2001 From: RPKI Team at RIPE NCC Date: Fri, 19 Jul 2024 09:17:00 +0000 Subject: [PATCH] RIPE NCC has merged fd1d936 * Add forRemoval [a009bd1d] * Sonarqube remarks [2418b438] * Update dependency commons-codec:commons-codec to v1.17.1 [ffed33e3] * Update dependency io.sentry:sentry-bom to v7.12.0 [42524de9] * Update dependency gradle to v8.9 [2cad0a10] * Add deprecation javadoc [58dc87b9] * Add a test for no overlap [ccb5c541] * Adjust error handling, fix tests [d224afff] * Deprecate existing announcements end-point [f5606b9b] * Add reasons for empty BGP announcements [f74cb67a] * Take lastUpdated from the downloaded files, store it only in memory [da330df8] * Fix Hibernate error [f7a43604] * Make lastUpdate for RIS updates persistent [06e77007] * Add simple lastUpdated to announcements. [4cd5e9cd] * Update dependency org.wiremock:wiremock-jetty12 to v3.8.0 [bd776aba] * Update dependency org.sonarqube:org.sonarqube.gradle.plugin to v5.1.0.4882 [080df6ab] * Update dependency io.sentry:sentry-bom to v7.11.0 [18d6b90b] * Update dependency net.jqwik:jqwik to v1.9.0 [4e7835c4] * Update dependency org.wiremock:wiremock-jetty12 to v3.7.0 [20ad0c33] * Update dependency org.eclipse.jgit:org.eclipse.jgit to v6.10.0.202406032230-r [504db2af] * Update dependency io.sentry:sentry-bom to v7.10.0 [9a21fd80] * Update dependency gradle to v8.8 [7283bdae] * Update plugin com.google.cloud.tools.jib to v3.4.3 [fed00dbb] * Log exception stacktraces [cf1a9544] * typo [2cfacb90] * Execute after-TA-response-processing actions in the background [9a6cb1ea] --- .gitlab-ci.yml | 2 +- build.gradle | 10 +- buildSrc/build.gradle | 4 +- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 2 +- .../bgpris/BgpRisEntryRepositoryBean.java | 8 +- .../rpki/bgpris/riswhois/RisWhoisFetcher.java | 12 ++- .../property/PropertyEntityRepository.java | 2 - ...oaConfigurationMaintenanceServiceBean.java | 2 +- .../rest/service/AbstractCaRestService.java | 7 +- .../rest/service/AnnouncementService.java | 48 +++++++++- .../ripencc/cache/JpaResourceCacheImpl.java | 1 + .../services/read/BgpRisEntryViewService.java | 7 +- .../background/RisWhoisUpdateServiceBean.java | 8 +- .../impl/jpa/JpaPropertyEntityRepository.java | 7 -- .../ripe/rpki/web/UpstreamCaController.java | 16 +++- .../bgpris/riswhois/RisWhoisFetcherTest.java | 2 +- .../rest/service/AnnouncementServiceTest.java | 93 ++++++++++++++++++- .../RisWhoisUpdateServiceBeanTest.java | 9 +- .../rpki/web/UpstreamCaControllerTest.java | 18 +++- 20 files changed, 207 insertions(+), 53 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index efc456c..46b8450 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ default: - image: gradle:8.7-jdk17 + image: gradle:8.9-jdk17 # Explicit version of the Mergerequests-Pipelines workflow, with the main branch # added. diff --git a/build.gradle b/build.gradle index 0494d1f..fd11679 100644 --- a/build.gradle +++ b/build.gradle @@ -3,7 +3,7 @@ plugins { id 'org.springframework.boot' version "3.2.4" id 'distribution' id 'jacoco' - id "com.google.cloud.tools.jib" version "3.4.2" + id "com.google.cloud.tools.jib" version "3.4.3" id "com.google.osdetector" version "1.7.3" } @@ -44,7 +44,7 @@ dependencies { implementation "org.thymeleaf:thymeleaf:3.1.2.RELEASE" implementation "org.thymeleaf:thymeleaf-spring6:3.1.2.RELEASE" - implementation platform('io.sentry:sentry-bom:7.9.0') + implementation platform('io.sentry:sentry-bom:7.12.0') implementation 'io.sentry:sentry-spring-boot-starter' implementation 'io.sentry:sentry-logback' @@ -58,7 +58,7 @@ dependencies { implementation 'com.google.code.gson:gson:2.11.0' implementation 'com.jamesmurty.utils:java-xmlbuilder:1.3' - implementation 'commons-codec:commons-codec:1.17.0' + implementation 'commons-codec:commons-codec:1.17.1' implementation 'commons-io:commons-io:2.16.1' implementation 'ch.qos.logback.contrib:logback-json-classic:0.1.5' implementation 'ch.qos.logback.contrib:logback-jackson:0.1.5' @@ -72,8 +72,8 @@ dependencies { exclude group: 'org.hamcrest', module: 'hamcrest-core' } - testImplementation "org.wiremock:wiremock-jetty12:3.6.0" - testImplementation 'net.jqwik:jqwik:1.8.5' + testImplementation "org.wiremock:wiremock-jetty12:3.8.0" + testImplementation 'net.jqwik:jqwik:1.9.0' testImplementation "net.ripe.rpki:rpki-commons:$rpki_commons_version:tests" testImplementation 'org.assertj:assertj-core' diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 88fc782..9304182 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -13,6 +13,6 @@ dependencies { implementation('com.gorylenko.gradle-git-properties:com.gorylenko.gradle-git-properties.gradle.plugin:2.4.2') { exclude group: 'org.eclipse.jgit', module: 'org.eclipse.jgit' } - implementation 'org.eclipse.jgit:org.eclipse.jgit:6.9.0.202403050737-r' - implementation 'org.sonarqube:org.sonarqube.gradle.plugin:5.0.0.4638' + implementation 'org.eclipse.jgit:org.eclipse.jgit:6.10.0.202406032230-r' + implementation 'org.sonarqube:org.sonarqube.gradle.plugin:5.1.0.4882' } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b82aa23..09523c0 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 1aa94a4..b740cf1 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. diff --git a/src/main/java/net/ripe/rpki/bgpris/BgpRisEntryRepositoryBean.java b/src/main/java/net/ripe/rpki/bgpris/BgpRisEntryRepositoryBean.java index 2dfd07b..a4bb238 100644 --- a/src/main/java/net/ripe/rpki/bgpris/BgpRisEntryRepositoryBean.java +++ b/src/main/java/net/ripe/rpki/bgpris/BgpRisEntryRepositoryBean.java @@ -1,5 +1,7 @@ package net.ripe.rpki.bgpris; +import lombok.Getter; +import lombok.Setter; import net.ripe.ipresource.IpAddress; import net.ripe.ipresource.IpRange; import net.ripe.ipresource.IpResource; @@ -11,6 +13,7 @@ import net.ripe.rpki.server.api.services.read.BgpRisEntryViewService; import org.springframework.stereotype.Component; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -19,7 +22,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; @Component public class BgpRisEntryRepositoryBean implements BgpRisEntryViewService { @@ -31,6 +33,10 @@ public class BgpRisEntryRepositoryBean implements BgpRisEntryViewService { */ private final AtomicReference>> entries = new AtomicReference<>(emptyEntries()); + @Getter + @Setter + private Instant lastUpdated; + @Override public boolean isEmpty() { return entries.get().isEmpty(); diff --git a/src/main/java/net/ripe/rpki/bgpris/riswhois/RisWhoisFetcher.java b/src/main/java/net/ripe/rpki/bgpris/riswhois/RisWhoisFetcher.java index 864ee6e..68bb849 100644 --- a/src/main/java/net/ripe/rpki/bgpris/riswhois/RisWhoisFetcher.java +++ b/src/main/java/net/ripe/rpki/bgpris/riswhois/RisWhoisFetcher.java @@ -1,6 +1,7 @@ package net.ripe.rpki.bgpris.riswhois; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; import org.joda.time.DateTimeConstants; import org.springframework.stereotype.Component; @@ -16,16 +17,17 @@ public class RisWhoisFetcher { private static final int HTTP_TIMEOUT = 30 * DateTimeConstants.MILLIS_PER_SECOND; - public String fetch(String url) throws IOException { - try (InputStream unzipped = new GZIPInputStream(getContent(url))) { - return IOUtils.toString(unzipped, StandardCharsets.UTF_8); + public Pair fetch(String url) throws IOException { + var content = getContent(url); + try (InputStream unzipped = new GZIPInputStream(content.getLeft())) { + return Pair.of(IOUtils.toString(unzipped, StandardCharsets.UTF_8), content.getRight()); } } - protected InputStream getContent(String url) throws IOException { + protected Pair getContent(String url) throws IOException { URLConnection connection = new URL(url).openConnection(); connection.setConnectTimeout(HTTP_TIMEOUT); connection.setReadTimeout(HTTP_TIMEOUT); - return connection.getInputStream(); + return Pair.of(connection.getInputStream(), connection.getLastModified()); } } diff --git a/src/main/java/net/ripe/rpki/domain/property/PropertyEntityRepository.java b/src/main/java/net/ripe/rpki/domain/property/PropertyEntityRepository.java index e8ead76..58b19cb 100644 --- a/src/main/java/net/ripe/rpki/domain/property/PropertyEntityRepository.java +++ b/src/main/java/net/ripe/rpki/domain/property/PropertyEntityRepository.java @@ -6,7 +6,5 @@ public interface PropertyEntityRepository extends Repository { PropertyEntity findByKey(String key); - PropertyEntity getByKey(String key); - void createOrUpdate(String key, String value); } diff --git a/src/main/java/net/ripe/rpki/domain/roa/RoaConfigurationMaintenanceServiceBean.java b/src/main/java/net/ripe/rpki/domain/roa/RoaConfigurationMaintenanceServiceBean.java index 41a3129..9295255 100644 --- a/src/main/java/net/ripe/rpki/domain/roa/RoaConfigurationMaintenanceServiceBean.java +++ b/src/main/java/net/ripe/rpki/domain/roa/RoaConfigurationMaintenanceServiceBean.java @@ -54,7 +54,7 @@ public void visitIncomingCertificateUpdatedEvent(IncomingCertificateUpdatedEvent private void updateRoaConfigurationsForResources(ManagedCertificateAuthority ca, ImmutableResourceSet nowCurrentResources, CommandContext context) { final Optional maybeConfig = roaConfigurationRepository.findByCertificateAuthority(ca); - if (!maybeConfig.isPresent()) { + if (maybeConfig.isEmpty()) { return; } diff --git a/src/main/java/net/ripe/rpki/rest/service/AbstractCaRestService.java b/src/main/java/net/ripe/rpki/rest/service/AbstractCaRestService.java index 93ab17f..d4afefb 100644 --- a/src/main/java/net/ripe/rpki/rest/service/AbstractCaRestService.java +++ b/src/main/java/net/ripe/rpki/rest/service/AbstractCaRestService.java @@ -1,6 +1,7 @@ package net.ripe.rpki.rest.service; import jakarta.annotation.PostConstruct; +import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import net.ripe.ipresource.ImmutableResourceSet; @@ -135,10 +136,10 @@ enum PrefixValidationResult { OWNERSHIP_ERROR("ownership", "You are not a holder of the prefix %s"), OK("ok", ""); + @Getter private final String type; private final String message; - PrefixValidationResult(String type, String message) { this.type = type; this.message = message; @@ -147,10 +148,6 @@ enum PrefixValidationResult { public String getMessage(String resource) { return String.format(message, resource); } - - public String getType() { - return type; - } } private static class CaNameFormatter implements Formatter { diff --git a/src/main/java/net/ripe/rpki/rest/service/AnnouncementService.java b/src/main/java/net/ripe/rpki/rest/service/AnnouncementService.java index 04f7ffe..50687c1 100644 --- a/src/main/java/net/ripe/rpki/rest/service/AnnouncementService.java +++ b/src/main/java/net/ripe/rpki/rest/service/AnnouncementService.java @@ -34,6 +34,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -68,11 +69,30 @@ public AnnouncementService(BgpRisEntryViewService bgpRisEntryViewService, this.roaAlertConfigurationViewService = roaAlertConfigurationViewService; } + /** + * @deprecated This end-point only exists for the current RPKI dashboard. Remove this call as soon as either it's + * not used from ripe-portal anymore or the whole ripe-portal-based RPKI dashboard is not used anymore. + */ @GetMapping - @Operation(summary = "Get all announcements, as well as not-announced ignored announcements for the CA") - public ResponseEntity> getResourcesForCa(@PathVariable("caName") final CaName caName) { + @Operation(summary = "Get all announcements, as well as not-announced ignored announcements for the CA", deprecated = true) + @Deprecated(since = "2024-07-17", forRemoval = true) + public ResponseEntity getResourcesForCa(@PathVariable("caName") final CaName caName) { log.info("Getting resources for CA: {}", caName); + var response = getAnnouncements(caName); + if (response instanceof AnnouncementResponse.Announcements as) { + return ok(as.announcements); + } + return ok(new AnnouncementResponse.Announcements(Collections.emptyList(), null)); + } + + @GetMapping("extended") + @Operation(summary = "Get all announcements, metadata for them and not-announced ignored announcements for the CA") + public ResponseEntity getResourcesForCaWithMetadata(@PathVariable("caName") final CaName caName) { + log.info("Getting resources for CA: {}", caName); + return ok(getAnnouncements(caName)); + } + private AnnouncementResponse getAnnouncements(CaName caName) { final HostedCertificateAuthorityData ca = getCa(HostedCertificateAuthorityData.class, caName); final ImmutableResourceSet certifiedResources = ca.getResources(); final Map> announcements = bgpRisEntryViewService.findMostSpecificContainedAndNotContained(certifiedResources); @@ -93,7 +113,17 @@ public ResponseEntity> getResourcesForCa(@PathVariable("ca true) ).toList(); - return ok(Stream.concat(announcedAnnouncements.stream(), notSeenAnnouncements.stream()).toList()); + Instant risLastUpdated = bgpRisEntryViewService.getLastUpdated(); + if (certifiedResources.isEmpty()) + return new AnnouncementResponse.Problem(NO_CA_RESOURCES); + else if (risLastUpdated == null) + return new AnnouncementResponse.Problem(NO_RIS_UPDATES); + else if (!announcements.isEmpty() && + announcements.values().stream().allMatch(Collection::isEmpty)) + return new AnnouncementResponse.Problem(NO_OVERLAP_WITH_RIS); + + var announcement = Stream.concat(announcedAnnouncements.stream(), notSeenAnnouncements.stream()).toList(); + return new AnnouncementResponse.Announcements(announcement, risLastUpdated); } private Set bgpRisMapToAnnouncedRoutes(Map> announcements) { @@ -103,7 +133,6 @@ private Set bgpRisMapToAnnouncedRoutes(Map> getAffectedAnnouncementsForCaAndRoa(@PathVariable("caName") final CaName caName, @RequestBody final ApiRoaPrefix roa) { @@ -156,4 +185,15 @@ public ResponseEntity> getAffectedAnnouncementsForCaAndRoa return ok(knownAnnouncements); } + public static final String NO_CA_RESOURCES = "no-ca-resources"; + public static final String NO_RIS_UPDATES = "no-ris-updates"; + public static final String NO_OVERLAP_WITH_RIS = "no-overlap-with-ris"; + + public interface AnnouncementResponse { + record Problem(String emptyAnnouncementsReason) implements AnnouncementResponse {} + + record Announcements(List announcements, + Instant lastUpdated) implements AnnouncementResponse { } + } + } diff --git a/src/main/java/net/ripe/rpki/ripencc/cache/JpaResourceCacheImpl.java b/src/main/java/net/ripe/rpki/ripencc/cache/JpaResourceCacheImpl.java index fd337a6..a205646 100644 --- a/src/main/java/net/ripe/rpki/ripencc/cache/JpaResourceCacheImpl.java +++ b/src/main/java/net/ripe/rpki/ripencc/cache/JpaResourceCacheImpl.java @@ -129,6 +129,7 @@ private void populateWith(Map certifiableResources private void registerUpdateCompleted() { propertyEntityRepository.createOrUpdate(RESOURCE_CACHE_UPDATE_KEY, Instant.now().toString()); } + public void updateEntry(CaName caName, ImmutableResourceSet resources) { entityManager.createNativeQuery( "insert into resource_cache (name, resources) values (:name, :resources)\n" + diff --git a/src/main/java/net/ripe/rpki/server/api/services/read/BgpRisEntryViewService.java b/src/main/java/net/ripe/rpki/server/api/services/read/BgpRisEntryViewService.java index 3f7ff0a..03a427d 100644 --- a/src/main/java/net/ripe/rpki/server/api/services/read/BgpRisEntryViewService.java +++ b/src/main/java/net/ripe/rpki/server/api/services/read/BgpRisEntryViewService.java @@ -3,10 +3,10 @@ import net.ripe.ipresource.ImmutableResourceSet; import net.ripe.rpki.server.api.dto.BgpRisEntry; +import java.time.Instant; import java.util.Collection; import java.util.Map; - public interface BgpRisEntryViewService { boolean isEmpty(); @@ -20,6 +20,7 @@ public interface BgpRisEntryViewService { /** * findMostSpecificContainedAndNotContained + * * @param resources * @return all matching BGP RIS entries that do not have a more specific matching entry, * split into two collections: those fully covered (contained) by the given resource set @@ -28,4 +29,8 @@ public interface BgpRisEntryViewService { Map> findMostSpecificContainedAndNotContained(ImmutableResourceSet resources); void resetEntries(Collection entries); + + Instant getLastUpdated(); + + void setLastUpdated(Instant lastUpdated); } diff --git a/src/main/java/net/ripe/rpki/services/impl/background/RisWhoisUpdateServiceBean.java b/src/main/java/net/ripe/rpki/services/impl/background/RisWhoisUpdateServiceBean.java index 7ecd265..326b701 100644 --- a/src/main/java/net/ripe/rpki/services/impl/background/RisWhoisUpdateServiceBean.java +++ b/src/main/java/net/ripe/rpki/services/impl/background/RisWhoisUpdateServiceBean.java @@ -69,16 +69,17 @@ public boolean isActive() { protected void runService(Map parameters) { List entries = new ArrayList<>(); + AtomicLong lastUpdated = new AtomicLong(0); for (String filename : FILENAMES) { String url = risWhoisBaseUrl + "/" + filename; try { log.info("fetching RIS whois entries from {}", url); - String text = fetcher.fetch(url); - final Collection currentEntries = RisWhoisParser.parse(text); + var result = fetcher.fetch(url); + final Collection currentEntries = RisWhoisParser.parse(result.getLeft()); updateMetrics(url, currentEntries); - entries.addAll(currentEntries); + lastUpdated.updateAndGet(i -> Long.max(result.getRight(), i)); } catch (IOException | NullPointerException e) { log.error(String.format("Exception while handling RIS dump from %s - aborting update", url), e); } @@ -87,6 +88,7 @@ protected void runService(Map parameters) { if (entries.size() >= MINIMUM_EXPECTED_UPDATES) { log.info("fetched {} RIS whois entries.", entries.size()); repository.resetEntries(entries); + repository.setLastUpdated(Instant.ofEpochMilli(lastUpdated.get())); } else { log.error("Found an unusually small number of RIS whois entries, please check files at: {}", risWhoisBaseUrl); } diff --git a/src/main/java/net/ripe/rpki/services/impl/jpa/JpaPropertyEntityRepository.java b/src/main/java/net/ripe/rpki/services/impl/jpa/JpaPropertyEntityRepository.java index 3102ee1..ebae5f8 100644 --- a/src/main/java/net/ripe/rpki/services/impl/jpa/JpaPropertyEntityRepository.java +++ b/src/main/java/net/ripe/rpki/services/impl/jpa/JpaPropertyEntityRepository.java @@ -32,13 +32,6 @@ public PropertyEntity findByKey(String key) { } } - @Override - public PropertyEntity getByKey(String key) { - PropertyEntity entity = findByKey(key); - Preconditions.checkNotNull(entity, "Could not find property by key: " + key); - return entity; - } - @Override public synchronized void createOrUpdate(String key, String value) { PropertyEntity entity = findByKey(key); diff --git a/src/main/java/net/ripe/rpki/web/UpstreamCaController.java b/src/main/java/net/ripe/rpki/web/UpstreamCaController.java index 940a294..5f140e9 100644 --- a/src/main/java/net/ripe/rpki/web/UpstreamCaController.java +++ b/src/main/java/net/ripe/rpki/web/UpstreamCaController.java @@ -5,6 +5,8 @@ import net.ripe.rpki.commons.ta.domain.response.TrustAnchorResponse; import net.ripe.rpki.commons.ta.serializers.TrustAnchorRequestSerializer; import net.ripe.rpki.commons.ta.serializers.TrustAnchorResponseSerializer; +import net.ripe.rpki.core.services.background.BackgroundTaskRunner; +import net.ripe.rpki.core.services.background.SequentialBackgroundQueuedTaskRunner; import net.ripe.rpki.server.api.commands.*; import net.ripe.rpki.server.api.configuration.RepositoryConfiguration; import net.ripe.rpki.server.api.dto.*; @@ -29,6 +31,7 @@ import java.util.*; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Stream; import static java.nio.charset.StandardCharsets.UTF_8; @@ -48,6 +51,7 @@ public class UpstreamCaController extends BaseController { private final CommandService commandService; private final AllCaCertificateUpdateServiceBean allCaCertificateUpdateServiceBean; private final Map backgroundServiceMap; + private final SequentialBackgroundQueuedTaskRunner sequentialBackgroundQueuedTaskRunner; @Inject public UpstreamCaController(RepositoryConfiguration repositoryConfiguration, @@ -56,12 +60,14 @@ public UpstreamCaController(RepositoryConfiguration repositoryConfiguration, CommandService commandService, AllCaCertificateUpdateServiceBean allCaCertificateUpdateServiceBean, Map backgroundServiceMap, - GitProperties gitProperties) { + GitProperties gitProperties, + SequentialBackgroundQueuedTaskRunner sequentialBackgroundQueuedTaskRunner) { super(repositoryConfiguration, activeNodeService, gitProperties); this.certificateAuthorityViewService = certificateAuthorityViewService; this.commandService = commandService; this.allCaCertificateUpdateServiceBean = allCaCertificateUpdateServiceBean; this.backgroundServiceMap = backgroundServiceMap; + this.sequentialBackgroundQueuedTaskRunner = sequentialBackgroundQueuedTaskRunner; } @ModelAttribute(name = "backgroundServices", binding = false) @@ -131,7 +137,9 @@ public Object uploadSignResponse(@RequestParam("response") MultipartFile file, R final X500Principal allResourcesCaName = repositoryConfiguration.getAllResourcesCaPrincipal(); final CertificateAuthorityData allResourcesCa = certificateAuthorityViewService.findCertificateAuthorityByName(allResourcesCaName); commandService.execute(new ProcessTrustAnchorResponseCommand(allResourcesCa.getVersionedId(), response)); - allCaCertificateUpdateServiceBean.execute(Collections.emptyMap()); + sequentialBackgroundQueuedTaskRunner.submit("Updating all certificates after uploading TA response", + () -> allCaCertificateUpdateServiceBean.execute(Collections.emptyMap()), + e -> log.error("Exception in updating certificates", e)); redirectAttributes.addFlashAttribute("success", "Successfully uploaded " + file.getName()); return new RedirectView(UPSTREAM_CA, true); @@ -158,7 +166,9 @@ public Object activateAcaPendingKey() { return withAllResourcesCa(allResourcesCa -> requireACAState(allResourcesCa, KeyPairStatus.PENDING, () -> { commandService.execute(KeyManagementActivatePendingKeysCommand.manualActivationCommand(allResourcesCa.getVersionedId())); - allCaCertificateUpdateServiceBean.execute(Collections.emptyMap()); + sequentialBackgroundQueuedTaskRunner.submit("Updating all certificates after activating All resources CA key pair", + () -> allCaCertificateUpdateServiceBean.execute(Collections.emptyMap()), + e -> log.error("Exception in updating certificates", e)); return new RedirectView(UPSTREAM_CA, true); })); } diff --git a/src/test/java/net/ripe/rpki/bgpris/riswhois/RisWhoisFetcherTest.java b/src/test/java/net/ripe/rpki/bgpris/riswhois/RisWhoisFetcherTest.java index 9ae1aff..9e47d07 100644 --- a/src/test/java/net/ripe/rpki/bgpris/riswhois/RisWhoisFetcherTest.java +++ b/src/test/java/net/ripe/rpki/bgpris/riswhois/RisWhoisFetcherTest.java @@ -33,7 +33,7 @@ void testFetch(WireMockRuntimeInfo wmRuntimeInfo) throws IOException { .willReturn(aResponse().withBody(risDumpContent)) ); - String data = subject.fetch(wmRuntimeInfo.getHttpBaseUrl() + path); + String data = subject.fetch(wmRuntimeInfo.getHttpBaseUrl() + path).getLeft(); assertThat(data).contains("45528\t1.22.52.0/23\t99"); } } diff --git a/src/test/java/net/ripe/rpki/rest/service/AnnouncementServiceTest.java b/src/test/java/net/ripe/rpki/rest/service/AnnouncementServiceTest.java index a613b2c..a226e4a 100644 --- a/src/test/java/net/ripe/rpki/rest/service/AnnouncementServiceTest.java +++ b/src/test/java/net/ripe/rpki/rest/service/AnnouncementServiceTest.java @@ -34,6 +34,7 @@ import org.springframework.test.web.servlet.MockMvc; import javax.security.auth.x500.X500Principal; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -50,8 +51,7 @@ import static net.ripe.rpki.rest.service.AbstractCaRestService.API_URL_PREFIX; import static net.ripe.rpki.rest.service.Rest.TESTING_API_KEY; import static org.hamcrest.Matchers.containsString; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -78,7 +78,6 @@ public class AnnouncementServiceTest { @MockBean private RoaAlertConfigurationViewService roaAlertConfigurationViewService; - @MockBean private HostedCertificateAuthorityData certificateAuthorityData; @Autowired @@ -86,6 +85,7 @@ public class AnnouncementServiceTest { @Before public void init() { + certificateAuthorityData = mock(HostedCertificateAuthorityData.class); when(certificateAuthorityViewService.findCertificateAuthorityByName(any(X500Principal.class))).thenReturn(certificateAuthorityData); when(certificateAuthorityData.getId()).thenReturn(CA_ID); } @@ -93,7 +93,7 @@ public void init() { @Test public void announcements_shouldGetAnnouncement() throws Exception { - final ImmutableResourceSet ipResourceSet = ImmutableResourceSet.empty(); + final ImmutableResourceSet ipResourceSet = ImmutableResourceSet.empty().add(IpRange.parse("192.168.0.0/16")); when(certificateAuthorityData.getResources()).thenReturn(ipResourceSet); final BgpRisEntry e1 = new BgpRisEntry(new Asn(10), IpRange.parse("192.168.0.0/16"), 10); @@ -102,6 +102,7 @@ public void announcements_shouldGetAnnouncement() throws Exception { bgpRisEntries.put(true, Collections.singletonList(e1)); bgpRisEntries.put(false, Collections.singletonList(e2)); when(bgpRisEntryViewService.findMostSpecificContainedAndNotContained(ipResourceSet)).thenReturn(bgpRisEntries); + when(bgpRisEntryViewService.getLastUpdated()).thenReturn(Instant.now()); when(roaAlertConfigurationViewService.findRoaAlertSubscription(CA_ID)).thenReturn(getRoaAlertConfigurationData(e1.getOrigin(), e1.getPrefix())); @@ -115,6 +116,8 @@ public void announcements_shouldGetAnnouncement() throws Exception { .andExpect(status().isOk()) .andExpect(content().contentType(APPLICATION_JSON)) .andExpect(jsonPath("$.length()").value("2")) + .andExpect(jsonPath("$.emptyAnnouncementsReason").doesNotExist()) + .andExpect(jsonPath("$.lastUpdated").doesNotExist()) .andExpect(jsonPath("$.[0].asn").value("AS10")) .andExpect(jsonPath("$.[0].prefix").value("192.168.0.0/16")) .andExpect(jsonPath("$.[0].visibility").value("10")) @@ -125,17 +128,97 @@ public void announcements_shouldGetAnnouncement() throws Exception { } @Test - public void announcements_shouldIncludeSilencesThatAreNotVisibleInBGP() throws Exception { + public void announcements_shouldGetAnnouncementExtended() throws Exception { final ImmutableResourceSet ipResourceSet = ImmutableResourceSet.empty(); when(certificateAuthorityData.getResources()).thenReturn(ipResourceSet); + final BgpRisEntry e1 = new BgpRisEntry(new Asn(10), IpRange.parse("192.168.0.0/16"), 10); + Map> bgpRisEntries = new HashMap<>(); + bgpRisEntries.put(true, Collections.singletonList(e1)); + + when(bgpRisEntryViewService.findMostSpecificContainedAndNotContained(ipResourceSet)).thenReturn(bgpRisEntries); + when(bgpRisEntryViewService.getLastUpdated()).thenReturn(Instant.now()); + + when(roaAlertConfigurationViewService.findRoaAlertSubscription(CA_ID)).thenReturn(getRoaAlertConfigurationData(e1.getOrigin(), e1.getPrefix())); + + when(roaService.getRoaConfiguration(CA_ID)).thenReturn(new RoaConfigurationData(new ArrayList<>())); + + mockMvc.perform( + Rest.get(API_URL_PREFIX + "/123/announcements/extended") + .accept(APPLICATION_JSON) + .contentType(APPLICATION_JSON) + ) + .andExpect(status().isOk()) + .andExpect(content().contentType(APPLICATION_JSON)) + .andExpect(jsonPath("$.emptyAnnouncementsReason").value(AnnouncementService.NO_CA_RESOURCES)); + } + + + @Test + public void announcements_shouldGetAnnouncementNoRisUpdates() throws Exception { + + final ImmutableResourceSet ipResourceSet = ImmutableResourceSet.empty().add(IpRange.parse("192.168.0.0/16")); + when(certificateAuthorityData.getResources()).thenReturn(ipResourceSet); + + final BgpRisEntry e1 = new BgpRisEntry(new Asn(10), IpRange.parse("192.168.0.0/16"), 10); + Map> bgpRisEntries = new HashMap<>(); + bgpRisEntries.put(true, Collections.singletonList(e1)); + + when(bgpRisEntryViewService.findMostSpecificContainedAndNotContained(ipResourceSet)).thenReturn(bgpRisEntries); + + when(roaAlertConfigurationViewService.findRoaAlertSubscription(CA_ID)).thenReturn(getRoaAlertConfigurationData(e1.getOrigin(), e1.getPrefix())); + + when(roaService.getRoaConfiguration(CA_ID)).thenReturn(new RoaConfigurationData(new ArrayList<>())); + + mockMvc.perform( + Rest.get(API_URL_PREFIX + "/123/announcements/extended") + .accept(APPLICATION_JSON) + .contentType(APPLICATION_JSON) + ) + .andExpect(status().isOk()) + .andExpect(content().contentType(APPLICATION_JSON)) + .andExpect(jsonPath("$.emptyAnnouncementsReason").value(AnnouncementService.NO_RIS_UPDATES)); + } + + @Test + public void announcements_shouldGetAnnouncementNoOverlapsBetweenRisAndResources() throws Exception { + + final ImmutableResourceSet ipResourceSet = ImmutableResourceSet.empty().add(IpRange.parse("192.167.0.0/16")); + when(certificateAuthorityData.getResources()).thenReturn(ipResourceSet); + + final BgpRisEntry e1 = new BgpRisEntry(new Asn(10), IpRange.parse("192.168.0.0/16"), 10); + Map> bgpRisEntries = new HashMap<>(); + bgpRisEntries.put(true, Collections.emptyList()); + + when(bgpRisEntryViewService.findMostSpecificContainedAndNotContained(ipResourceSet)).thenReturn(bgpRisEntries); + when(bgpRisEntryViewService.getLastUpdated()).thenReturn(Instant.now()); + + when(roaService.getRoaConfiguration(CA_ID)).thenReturn(new RoaConfigurationData(new ArrayList<>())); + + mockMvc.perform( + Rest.get(API_URL_PREFIX + "/123/announcements/extended") + .accept(APPLICATION_JSON) + .contentType(APPLICATION_JSON) + ) + .andExpect(status().isOk()) + .andExpect(content().contentType(APPLICATION_JSON)) + .andExpect(jsonPath("$.emptyAnnouncementsReason").value(AnnouncementService.NO_OVERLAP_WITH_RIS)); + } + + @Test + public void announcements_shouldIncludeSilencesThatAreNotVisibleInBGP() throws Exception { + + final ImmutableResourceSet ipResourceSet = ImmutableResourceSet.empty().add(IpRange.parse("192.168.0.0/16")); + when(certificateAuthorityData.getResources()).thenReturn(ipResourceSet); + final BgpRisEntry e1 = new BgpRisEntry(new Asn(10), IpRange.parse("192.168.0.0/16"), 10); final BgpRisEntry e2 = new BgpRisEntry(new Asn(20), IpRange.parse("192.168.128.0/24"), 20); Map> bgpRisEntries = new HashMap<>(); bgpRisEntries.put(true, Collections.singletonList(e1)); bgpRisEntries.put(false, Collections.singletonList(e2)); when(bgpRisEntryViewService.findMostSpecificContainedAndNotContained(ipResourceSet)).thenReturn(bgpRisEntries); + when(bgpRisEntryViewService.getLastUpdated()).thenReturn(Instant.now()); when(roaAlertConfigurationViewService.findRoaAlertSubscription(CA_ID)).thenReturn( getRoaAlertConfigurationData(e1.getOrigin(), e1.getPrefix()).withIgnoredAnnouncements( diff --git a/src/test/java/net/ripe/rpki/services/impl/background/RisWhoisUpdateServiceBeanTest.java b/src/test/java/net/ripe/rpki/services/impl/background/RisWhoisUpdateServiceBeanTest.java index 58d5e22..1778ca1 100644 --- a/src/test/java/net/ripe/rpki/services/impl/background/RisWhoisUpdateServiceBeanTest.java +++ b/src/test/java/net/ripe/rpki/services/impl/background/RisWhoisUpdateServiceBeanTest.java @@ -11,6 +11,7 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import org.apache.commons.lang3.tuple.Pair; import java.io.IOException; import java.util.Collection; @@ -59,13 +60,13 @@ public void shouldUpdateRepositoryWhenMoreThan100kEntriesFound() throws IOExcept @Test public void shouldNotFailOnPartiallyBrokenFile() throws IOException { when(fetcher.fetch(IPV4_FILE_URL)).thenReturn(getTestLines(100001)); - when(fetcher.fetch(IPV6_FILE_URL)).thenReturn( + when(fetcher.fetch(IPV6_FILE_URL)).thenReturn(Pair.of( "207841\t::ffff:0.0.0.0/96\t1\n" + "268624\t::ffff:45.164.124.0/120\t1\n" + "268624\t::ffff:45.164.125.0/120\t1\n" + "268624\t::ffff:45.164.126.0/120\t1\n" + "268624\t::ffff:45.164.127.0/120\t1\n" + - "268624\t::ffff:80.94.90.0/120\t1\n" + "268624\t::ffff:80.94.90.0/120\t1\n", 11L) ); subject.runService(Collections.emptyMap()); @@ -93,11 +94,11 @@ public void shouldHandleExceptionsGracefully() throws IOException { subject.runService(Collections.emptyMap()); } - private String getTestLines(int lines) { + private Pair getTestLines(int lines) { StringBuilder responseBuilder = new StringBuilder(); for (int i = 0; i < lines; i++) { responseBuilder.append(i + 1).append("\t10.0.0.0/8\t10\n"); } - return responseBuilder.toString(); + return Pair.of(responseBuilder.toString(), 10L); } } diff --git a/src/test/java/net/ripe/rpki/web/UpstreamCaControllerTest.java b/src/test/java/net/ripe/rpki/web/UpstreamCaControllerTest.java index 18936c9..448ff64 100644 --- a/src/test/java/net/ripe/rpki/web/UpstreamCaControllerTest.java +++ b/src/test/java/net/ripe/rpki/web/UpstreamCaControllerTest.java @@ -1,9 +1,12 @@ package net.ripe.rpki.web; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import lombok.NonNull; import net.ripe.rpki.TestRpkiBootApplication; import net.ripe.rpki.commons.ta.domain.request.TrustAnchorRequest; import net.ripe.rpki.commons.util.VersionedId; +import net.ripe.rpki.core.services.background.BackgroundTaskRunner; +import net.ripe.rpki.core.services.background.SequentialBackgroundQueuedTaskRunner; import net.ripe.rpki.server.api.commands.AllResourcesCaResourcesCommand; import net.ripe.rpki.server.api.commands.KeyManagementActivatePendingKeysCommand; import net.ripe.rpki.server.api.commands.KeyManagementInitiateRollCommand; @@ -35,6 +38,7 @@ import java.util.Collections; import java.util.Map; import java.util.Properties; +import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -69,8 +73,20 @@ public class UpstreamCaControllerTest extends SpringWebControllerTestCase { @NonNull @Override protected UpstreamCaController createSubjectController() { + var sequentialBackgroundQueuedTaskRunner = new SequentialBackgroundQueuedTaskRunner(new BackgroundTaskRunner(activeNodeService, new SimpleMeterRegistry())) { + // Override submit to just execute the action without any background logic + @Override + public void submit(@NonNull String description, @NonNull Runnable action, @NonNull Consumer onException) { + try { + action.run(); + } catch (Exception e) { + onException.accept(e); + } + } + }; return new UpstreamCaController(repositoryConfiguration, activeNodeService, - certificateAuthorityViewService, commandService, allCaCertificateUpdateServiceBean, Collections.emptyMap(), new GitProperties(new Properties())); + certificateAuthorityViewService, commandService, allCaCertificateUpdateServiceBean, Collections.emptyMap(), + new GitProperties(new Properties()), sequentialBackgroundQueuedTaskRunner); } @BeforeEach