From 16e042ef623491e1eb9fa6b22522388b1f6773a7 Mon Sep 17 00:00:00 2001 From: Luca Consalvi <117908483+lucaconsalvi@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:37:20 +0200 Subject: [PATCH 1/3] fix: [RTD-2505] update dependecies, actions and delete some useless imports (#393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [RTD-2505] update dependecies, actions and delete some useless imports * [RTD-2505] fix * [RTD-2505] fix scam action * [RTD-2505] fix --- .github/workflows/pr-title.yml | 2 +- .github/workflows/publish-image.yml | 8 ++++---- .github/workflows/release.yml | 8 ++++---- .github/workflows/scan.yml | 6 +++--- .github/workflows/send-notification.yml | 2 +- .github/workflows/upload-jar.yml | 4 ++-- .../InboundTransactionItemProcessor.java | 2 -- ...TransactionAggregationWriterProcessor.java | 2 -- .../EnforceSenderCodeUniquenessTasklet.java | 2 -- ...eprocessingFilenameAlreadySeenTasklet.java | 2 -- .../batch/step/writer/HpanWriter.java | 2 -- ...terBatchInputFileChecksumDisabledTest.java | 1 - .../TransactionFilterBatchWrongInputTest.java | 1 - .../TransactionItemProcessListenerTest.java | 5 ----- ...TransactionItemProcessorValidatorTest.java | 1 - .../SenderAdeAckFilesRecoveryTaskletTest.java | 2 -- .../TransactionSenderRestTaskletTest.java | 2 -- .../AbiToFiscalCodeRestClientImpl.java | 2 -- .../connector/HpanRestClientImpl.java | 10 ---------- .../connector/HpanRestClientTest.java | 2 -- pom.xml | 20 +++++++++---------- 21 files changed, 25 insertions(+), 61 deletions(-) diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 6fff8b1f..ad959e33 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -17,7 +17,7 @@ jobs: steps: # Please look up the latest version from # https://github.com/amannn/action-semantic-pull-request/releases - - uses: amannn/action-semantic-pull-request@c3cd5d1ea3580753008872425915e343e351ab54 #v5.2.0 + - uses: amannn/action-semantic-pull-request@e9fabac35e210fea40ca5b14c0da95a099eff26f #v5.4.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index f7680ee6..2bb5a725 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1 with: persist-credentials: false fetch-depth: 0 @@ -24,11 +24,11 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 #v3.0.0 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb #v3.3.0 - name: Log in to the Container registry id: docker_login - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d #v3.0.0 + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 #v3.1.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -36,7 +36,7 @@ jobs: - name: Build and push Docker image id: docker_build_push - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 #v5.0.0 + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 #v5.3.0 with: context: . push: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0dbf6be..f30a91d5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,21 +17,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1 with: persist-credentials: false fetch-depth: 0 - name: Setup Node.js - uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d #v3.8.1 + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 #v4.0.2 with: node-version: 20 scope: 'cycjimmy' - name: Release id: sem-release - uses: cycjimmy/semantic-release-action@61680d0e9b02ff86f5648ade99e01be17f0260a4 #v4.0.0 + uses: cycjimmy/semantic-release-action@cb425203a562475bca039ba4dbf90c7f9ac790f4 #v4.1.0 with: extra_plugins: | - semantic-release/release-notes-generator#c466b7165be2a0e5988676174e4bc244c991d669 #v11.0.7 + semantic-release/release-notes-generator#148d97c53d42617d661905eb12166cb1438f0fba #v13.0.0 semantic-release/git#3e934d45f97fd07a63617c0fc098c9ed3e67d97a #v10.0.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml index bef01631..ffa54976 100644 --- a/.github/workflows/scan.yml +++ b/.github/workflows/scan.yml @@ -37,18 +37,18 @@ jobs: CVE_MEDIUM: ${{env.CVE_MEDIUM}} steps: - name: Checkout the code - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1 - name: Build the Docker image run: docker build . --file ${{ env.DOCKERFILE }} --tag localbuild/testimage:latest - name: Run the Trivy scan action itself with GitHub Advanced Security code scanning integration enabled id: scan - uses: aquasecurity/trivy-action@fbd16365eb88e12433951383f5e99bd901fc618f #v0.12.0 + uses: aquasecurity/trivy-action@d710430a6722f083d3b36b8339ff66b32f22ee55 #v0.19.0 with: image-ref: "localbuild/testimage:latest" format: 'sarif' output: 'results.sarif' - name: Upload Anchore Scan Report - uses: github/codeql-action/upload-sarif@6c089f53dd51dc3fc7e599c3cb5356453a52ca9e #v2.20.0 + uses: github/codeql-action/upload-sarif@99c9897648dded3fe63d6f328c46089dd57735ca #codeql bundle v2.17.0 with: sarif_file: 'results.sarif' - name: CVE Description escaped extraction and print diff --git a/.github/workflows/send-notification.yml b/.github/workflows/send-notification.yml index eb52bcf3..c46d5c42 100644 --- a/.github/workflows/send-notification.yml +++ b/.github/workflows/send-notification.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Send notification to Slack id: slack - uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 #v1.24.0 + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 #v1.25.0 with: payload: | { diff --git a/.github/workflows/upload-jar.yml b/.github/workflows/upload-jar.yml index 6987e580..3f446acd 100644 --- a/.github/workflows/upload-jar.yml +++ b/.github/workflows/upload-jar.yml @@ -14,12 +14,12 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 #v3.13.0 + - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 #v4.2.1 with: distribution: 'temurin' # See 'Supported distributions' for available options java-version: '17' diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessor.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessor.java index a81b5aaf..c4828e55 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessor.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessor.java @@ -3,7 +3,6 @@ import it.gov.pagopa.rtd.transaction_filter.batch.model.InboundTransaction; import it.gov.pagopa.rtd.transaction_filter.service.StoreService; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.springframework.batch.item.ItemProcessor; @@ -15,7 +14,6 @@ * to be mapped into a normalized version defined as instances of Transaction */ -@Slf4j @RequiredArgsConstructor public class InboundTransactionItemProcessor implements ItemProcessor { diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/TransactionAggregationWriterProcessor.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/TransactionAggregationWriterProcessor.java index a8bc8595..f5a4ee99 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/TransactionAggregationWriterProcessor.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/TransactionAggregationWriterProcessor.java @@ -4,14 +4,12 @@ import it.gov.pagopa.rtd.transaction_filter.service.store.AggregationKey; import it.gov.pagopa.rtd.transaction_filter.service.StoreService; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.springframework.batch.item.ItemProcessor; /** * ItemProcessor responsible for creating aggregates models suited for CSV * writing from the previously in-memory aggregated data. */ -@Slf4j @RequiredArgsConstructor public class TransactionAggregationWriterProcessor implements ItemProcessor { diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/EnforceSenderCodeUniquenessTasklet.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/EnforceSenderCodeUniquenessTasklet.java index 8358fe79..0dedd1d3 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/EnforceSenderCodeUniquenessTasklet.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/EnforceSenderCodeUniquenessTasklet.java @@ -6,7 +6,6 @@ import java.io.IOException; import java.util.Collection; import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.scope.context.ChunkContext; import org.springframework.batch.core.step.tasklet.Tasklet; @@ -17,7 +16,6 @@ * Tasklet responsible for Sender Code uniqueness enforcing */ @Data -@Slf4j public class EnforceSenderCodeUniquenessTasklet implements Tasklet { private StoreService storeService; diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/PreventReprocessingFilenameAlreadySeenTasklet.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/PreventReprocessingFilenameAlreadySeenTasklet.java index 2f1686d8..0cb56500 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/PreventReprocessingFilenameAlreadySeenTasklet.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/PreventReprocessingFilenameAlreadySeenTasklet.java @@ -4,7 +4,6 @@ import it.gov.pagopa.rtd.transaction_filter.service.TransactionWriterService; import java.io.IOException; import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.scope.context.ChunkContext; import org.springframework.batch.core.step.tasklet.Tasklet; @@ -24,7 +23,6 @@ * but named differently will be processed twice look at the TransactionChecksumTasklet. */ @Data -@Slf4j public class PreventReprocessingFilenameAlreadySeenTasklet implements Tasklet { private StoreService storeService; diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/writer/HpanWriter.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/writer/HpanWriter.java index c8abfec2..93c2df6e 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/writer/HpanWriter.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/writer/HpanWriter.java @@ -2,7 +2,6 @@ import it.gov.pagopa.rtd.transaction_filter.service.StoreService; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; @@ -11,7 +10,6 @@ * Implementation of {@link ItemWriter}, to be used for read/processed Transaction files */ -@Slf4j @RequiredArgsConstructor public class HpanWriter implements ItemWriter { diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchInputFileChecksumDisabledTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchInputFileChecksumDisabledTest.java index 5fa3b651..fa63da26 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchInputFileChecksumDisabledTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchInputFileChecksumDisabledTest.java @@ -44,7 +44,6 @@ import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.cloud.openfeign.FeignAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.test.context.ContextConfiguration; diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchWrongInputTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchWrongInputTest.java index 618c9fa5..b9e5ab90 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchWrongInputTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchWrongInputTest.java @@ -25,7 +25,6 @@ import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.cloud.openfeign.FeignAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.test.context.ContextConfiguration; diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/listener/TransactionItemProcessListenerTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/listener/TransactionItemProcessListenerTest.java index d5fa053b..03a79927 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/listener/TransactionItemProcessListenerTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/listener/TransactionItemProcessListenerTest.java @@ -15,7 +15,6 @@ import jakarta.validation.ConstraintViolationException; import jakarta.validation.Validator; import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -27,7 +26,6 @@ import org.mockito.MockitoAnnotations; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; -@Slf4j public class TransactionItemProcessListenerTest { private final TransactionMaskPolicy maskPolicy = new TransactionMaskPolicyImpl(); @@ -65,7 +63,6 @@ public void initTest() { public void afterProcess_OK() { File folder = tempFolder.newFolder("testProcess"); - PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS"); String executionDate = OffsetDateTime.now().format(fmt); @@ -95,7 +92,6 @@ public void afterProcess_OK() { public void onProcessError_OK() { File folder = tempFolder.newFolder("testProcess"); - PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS"); String executionDate = OffsetDateTime.now().format(fmt); @@ -122,7 +118,6 @@ public void onProcessError_OK() { public void onProcessError_OK_NoFileWritten() { File folder = tempFolder.newFolder("testProcess"); - PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS"); String executionDate = OffsetDateTime.now().format(fmt); diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessorValidatorTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessorValidatorTest.java index c9a95f77..fe0eb478 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessorValidatorTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessorValidatorTest.java @@ -5,7 +5,6 @@ import it.gov.pagopa.rtd.transaction_filter.batch.model.InboundTransaction; import it.gov.pagopa.rtd.transaction_filter.service.StoreService; import org.apache.commons.codec.digest.DigestUtils; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/SenderAdeAckFilesRecoveryTaskletTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/SenderAdeAckFilesRecoveryTaskletTest.java index 9d005adc..99553b7d 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/SenderAdeAckFilesRecoveryTaskletTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/SenderAdeAckFilesRecoveryTaskletTest.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.stream.Collectors; import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -37,7 +36,6 @@ import org.springframework.batch.test.MetaDataInstanceFactory; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; -@Slf4j class SenderAdeAckFilesRecoveryTaskletTest { private ChunkContext chunkContext; diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/TransactionSenderRestTaskletTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/TransactionSenderRestTaskletTest.java index 1bbc6000..98f7cc0b 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/TransactionSenderRestTaskletTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/tasklet/TransactionSenderRestTaskletTest.java @@ -13,7 +13,6 @@ import java.nio.file.Path; import java.util.HashMap; import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -30,7 +29,6 @@ import org.springframework.batch.test.MetaDataInstanceFactory; import org.springframework.core.io.UrlResource; -@Slf4j class TransactionSenderRestTaskletTest { AutoCloseable closeable; diff --git a/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/AbiToFiscalCodeRestClientImpl.java b/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/AbiToFiscalCodeRestClientImpl.java index 431abb97..39b2f0d8 100644 --- a/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/AbiToFiscalCodeRestClientImpl.java +++ b/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/AbiToFiscalCodeRestClientImpl.java @@ -3,7 +3,6 @@ import java.util.Map; import java.util.Objects; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -12,7 +11,6 @@ */ @Service @RequiredArgsConstructor -@Slf4j public class AbiToFiscalCodeRestClientImpl implements AbiToFiscalCodeRestClient { @Value("${rest-client.hpan.api.key}") diff --git a/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/HpanRestClientImpl.java b/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/HpanRestClientImpl.java index 70c7002d..124c37c7 100644 --- a/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/HpanRestClientImpl.java +++ b/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/HpanRestClientImpl.java @@ -18,36 +18,26 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.net.ssl.SSLContext; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; -import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.cookie.StandardCookieSpec; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; -import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder; import org.apache.hc.core5.http.ClassicHttpRequest; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.HttpResponse; import org.apache.hc.core5.http.HttpStatus; -import org.apache.hc.core5.http.io.SocketConfig; import org.apache.hc.core5.http.io.entity.FileEntity; import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; import org.apache.hc.core5.http.message.BasicHeader; -import org.apache.hc.core5.http.ssl.TLS; -import org.apache.hc.core5.pool.PoolConcurrencyPolicy; -import org.apache.hc.core5.pool.PoolReusePolicy; -import org.apache.hc.core5.util.TimeValue; -import org.apache.hc.core5.util.Timeout; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; diff --git a/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/HpanRestClientTest.java b/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/HpanRestClientTest.java index ec7e631e..3ac04a52 100644 --- a/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/HpanRestClientTest.java +++ b/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/HpanRestClientTest.java @@ -29,8 +29,6 @@ import org.springframework.cloud.openfeign.FeignAutoConfiguration; import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.annotation.DirtiesContext.ClassMode; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; diff --git a/pom.xml b/pom.xml index d8e0e014..9d644fbe 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.2.2 + 3.2.4 it.gov.pagopa.rtd.ms @@ -16,18 +16,18 @@ 17 - 42.6.0 - 4.1.0 + 42.7.3 + 4.1.1 2.2 - 2.11.0 + 2.16.1 1.5 1.76 - 2.15.2 - 1.1.1 - 3.1.2 - 3.0.0-M1 - 3.8.0 - 3.0.0-M4 + 2.17.0 + 1.1.2 + 3.3.0 + 3.1.1 + 3.13.0 + 3.2.5 ${java.version} ${java.version} UTF-8 From 9a83914cfc9edf6c89bf4eef450f9fafc243846a Mon Sep 17 00:00:00 2001 From: Luca Consalvi <117908483+lucaconsalvi@users.noreply.github.com> Date: Fri, 3 May 2024 15:25:46 +0200 Subject: [PATCH 2/3] feat: [RTD-2456] change report format from csv to json and enrich it (#387) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [RTD-2456] update actions and dependencies versions * [RTD-2456] update all the actions * [RTD-2456] parameterize application yaml file reporter uri * [RTD-2456] update dependencies, change report extension from csv to json * [RTD-2456] fix * [RTD-2456] fix * [RTD-2456] fix tests * [RTD-2456] add aggregate data summary to file metadata * [RTD-2456] remove some warinings * [RTD-2456] update properties and yml file with new v2 endpoint * [RTD-2456] update tests * [RTD-2456] update test with report v2 * [RTD-2456] update test, add report v2 and v1 tests * [RTD-2456] discarde format changes * [RTD-2456] fix some indentations * [RTD-2456] update test * [RTD-2456] fix sha inside filereportv2.json test file * [RTD-2456] update spring-web cve * [RTD-2456] update spring boot version * [RTD-2456] make filemetadata flexible to future structure changes * [RTD-2456] update datasummary name and delete aggregatedatasummary object * [RTD-2456] update tests * [RTD-2456] delete linefileaggregator and fix tests --- api/batch/pom.xml | 4 +- .../batch/step/TransactionFilterStep.java | 29 +- .../TransactionFilterBatchFileReportTest.java | 243 ++++++++++------- api/pom.xml | 5 +- app/pom.xml | 4 +- app/src/main/resources/config/application.yml | 2 +- core/pom.xml | 4 +- .../service/StoreServiceImpl.java | 2 - integration/jpa/pom.xml | 4 +- integration/pom.xml | 4 +- integration/rest/pom.xml | 4 +- .../connector/model/FileMetadata.java | 19 ++ .../resources/config/rest-client.properties | 2 +- .../connector/FileReportRestClientTest.java | 4 +- .../connector/FileReportV2RestClientTest.java | 256 ++++++++++++++++++ .../resources/stubs/mappings/fileReport.json | 6 +- .../stubs/mappings/fileReportv2.json | 48 ++++ ops_resources/example_config/application.yml | 2 +- .../example_config/application_hbsql.yml | 2 +- pom.xml | 6 +- 20 files changed, 502 insertions(+), 148 deletions(-) create mode 100644 integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/FileReportV2RestClientTest.java create mode 100644 integration/rest/src/test/resources/stubs/mappings/fileReportv2.json diff --git a/api/batch/pom.xml b/api/batch/pom.xml index 4bb8718e..5390132a 100644 --- a/api/batch/pom.xml +++ b/api/batch/pom.xml @@ -4,11 +4,11 @@ rtd-ms-transaction-filter-api it.gov.pagopa.rtd.ms.transaction_filter.api - 2.1.2 + 2.2.0 rtd-ms-transaction-filter-api-batch - 2.1.2 + 2.2.0 diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/TransactionFilterStep.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/TransactionFilterStep.java index b255d1ee..ce19ee6c 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/TransactionFilterStep.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/TransactionFilterStep.java @@ -59,7 +59,6 @@ import org.springframework.batch.item.file.FlatFileItemWriter; import org.springframework.batch.item.file.LineMapper; import org.springframework.batch.item.file.MultiResourceItemWriter; -import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder; import org.springframework.batch.item.file.builder.MultiResourceItemWriterBuilder; import org.springframework.batch.item.file.mapping.FieldSetMapper; import org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor; @@ -67,6 +66,9 @@ import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; import org.springframework.batch.item.file.transform.LineAggregator; import org.springframework.batch.item.file.transform.LineTokenizer; +import org.springframework.batch.item.json.JacksonJsonObjectMarshaller; +import org.springframework.batch.item.json.JsonFileItemWriter; +import org.springframework.batch.item.json.builder.JsonFileItemWriterBuilder; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -261,20 +263,6 @@ public LineAggregator adeTransactionsAggregateLineAggr return delimitedLineAggregator; } - /** - * Composes CSV lines from file report model. - * @return a line aggregator - */ - @Bean - public LineAggregator fileReportLineAggregator() { - BeanWrapperFieldExtractor extractor = new BeanWrapperFieldExtractor<>(); - extractor.setNames(REPORT_CSV_FIELDS); - DelimitedLineAggregator delimitedLineAggregator = new DelimitedLineAggregator<>(); - delimitedLineAggregator.setDelimiter(CSV_DELIMITER); - delimitedLineAggregator.setFieldExtractor(extractor); - return delimitedLineAggregator; - } - /** * Builds a MultiResourceItemWriter for filtered transactions. * Remember MultiResourceItemWriter is not thread safe. @@ -515,7 +503,7 @@ public FileReportItemReader fileReportReader(FileReportRestClient restClient) { @SneakyThrows @Bean @StepScope - public FlatFileItemWriter fileReportWriter() { + public JsonFileItemWriter fileReportWriter() { DateTimeFormatter fmt = DateTimeFormatter.ofPattern(DATE_FORMAT_FOR_FILENAME); String currentDate = OffsetDateTime.now().format(fmt); @@ -525,13 +513,12 @@ public FlatFileItemWriter fileReportWriter() { .concat(fileReportPrefixName) .concat("-") .concat(currentDate) - .concat(".csv")); + .concat(".json")); - return new FlatFileItemWriterBuilder() - .name("file-report-item-writer") + return new JsonFileItemWriterBuilder() + .jsonObjectMarshaller(new JacksonJsonObjectMarshaller<>()) .resource(outputResource) - .headerCallback(writer -> writer.write(String.join(CSV_DELIMITER, REPORT_CSV_FIELDS))) - .lineAggregator(fileReportLineAggregator()) + .name("file-report-item-writer") .build(); } diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchFileReportTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchFileReportTest.java index 53681569..701a5b71 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchFileReportTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchFileReportTest.java @@ -73,60 +73,58 @@ @Transactional(propagation = NOT_SUPPORTED) @Sql({ "classpath:org/springframework/batch/core/schema-drop-hsqldb.sql", - "classpath:org/springframework/batch/core/schema-hsqldb.sql"}) + "classpath:org/springframework/batch/core/schema-hsqldb.sql" }) @ContextConfiguration(classes = { TestConfig.class, JacksonAutoConfiguration.class, TransactionFilterBatch.class, FeignAutoConfiguration.class }) -@TestPropertySource( - properties = { - "spring.main.allow-bean-definition-overriding=true", - "batchConfiguration.TransactionFilterBatch.panList.secretKeyPath=classpath:/test-encrypt/secretKey.asc", - "batchConfiguration.TransactionFilterBatch.panList.passphrase=test", - "batchConfiguration.TransactionFilterBatch.panList.skipLimit=0", - "batchConfiguration.TransactionFilterBatch.panList.hpanDirectoryPath=classpath:/test-encrypt/**/hpan/*pan*.pgp", - "batchConfiguration.TransactionFilterBatch.panList.linesToSkip=0", - "batchConfiguration.TransactionFilterBatch.panList.applyDecrypt=true", - "batchConfiguration.TransactionFilterBatch.panList.applyHashing=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.transactionDirectoryPath=classpath:/test-encrypt/**/transactions/", - "batchConfiguration.TransactionFilterBatch.transactionFilter.outputDirectoryPath=classpath:/test-encrypt/output", - "batchConfiguration.TransactionFilterBatch.transactionFilter.transactionLogsPath=classpath:/test-encrypt/errorLogs", - "batchConfiguration.TransactionFilterBatch.transactionFilter.applyHashing=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.timestampPattern=MM/dd/yyyy HH:mm:ss", - "batchConfiguration.TransactionFilterBatch.transactionFilter.applyEncrypt=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.saveHashing=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.deleteProcessedFiles=false", - "batchConfiguration.TransactionFilterBatch.transactionFilter.deleteOutputFiles=ERROR", - "batchConfiguration.TransactionFilterBatch.successArchivePath=classpath:/test-encrypt/success", - "batchConfiguration.TransactionFilterBatch.errorArchivePath=classpath:/test-encrypt/error", - "batchConfiguration.TransactionFilterBatch.pendingArchivePath=classpath:/test-encrypt/output/pending", - "batchConfiguration.TransactionFilterBatch.saltRecovery.enabled=false", - "batchConfiguration.TransactionFilterBatch.pagopaPublicKeyRecovery.enabled=false", - "batchConfiguration.TransactionFilterBatch.hpanListRecovery.enabled=false", - "batchConfiguration.TransactionFilterBatch.abiToFiscalCodeMapRecovery.enabled=false", - "batchConfiguration.TransactionFilterBatch.transactionSenderRtd.enabled=false", - "batchConfiguration.TransactionFilterBatch.transactionSenderAde.enabled=false", - "batchConfiguration.TransactionFilterBatch.transactionSenderPending.enabled=false", - "batchConfiguration.TransactionFilterBatch.senderAdeAckFilesRecovery.enabled=false", - "batchConfiguration.TransactionFilterBatch.senderAdeAckFilesRecovery.directoryPath=classpath:/test-encrypt/sender-ade-ack", - "batchConfiguration.TransactionFilterBatch.fileReportRecovery.directoryPath=classpath:/test-encrypt/reports", - "batchConfiguration.TransactionFilterBatch.fileReportRecovery.enabled=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableAfterProcessLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableAfterProcessFileLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnReadErrorLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnReadErrorFileLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableAfterProcessFileLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnProcessErrorLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnProcessErrorFileLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnWriteErrorLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnWriteErrorFileLogging=true", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.loggingFrequency=100", - "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.writerPoolSize=5", - "batchConfiguration.TransactionFilterBatch.transactionWriterAde.splitThreshold=1000" - } -) +@TestPropertySource(properties = { + "spring.main.allow-bean-definition-overriding=true", + "batchConfiguration.TransactionFilterBatch.panList.secretKeyPath=classpath:/test-encrypt/secretKey.asc", + "batchConfiguration.TransactionFilterBatch.panList.passphrase=test", + "batchConfiguration.TransactionFilterBatch.panList.skipLimit=0", + "batchConfiguration.TransactionFilterBatch.panList.hpanDirectoryPath=classpath:/test-encrypt/**/hpan/*pan*.pgp", + "batchConfiguration.TransactionFilterBatch.panList.linesToSkip=0", + "batchConfiguration.TransactionFilterBatch.panList.applyDecrypt=true", + "batchConfiguration.TransactionFilterBatch.panList.applyHashing=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.transactionDirectoryPath=classpath:/test-encrypt/**/transactions/", + "batchConfiguration.TransactionFilterBatch.transactionFilter.outputDirectoryPath=classpath:/test-encrypt/output", + "batchConfiguration.TransactionFilterBatch.transactionFilter.transactionLogsPath=classpath:/test-encrypt/errorLogs", + "batchConfiguration.TransactionFilterBatch.transactionFilter.applyHashing=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.timestampPattern=MM/dd/yyyy HH:mm:ss", + "batchConfiguration.TransactionFilterBatch.transactionFilter.applyEncrypt=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.saveHashing=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.deleteProcessedFiles=false", + "batchConfiguration.TransactionFilterBatch.transactionFilter.deleteOutputFiles=ERROR", + "batchConfiguration.TransactionFilterBatch.successArchivePath=classpath:/test-encrypt/success", + "batchConfiguration.TransactionFilterBatch.errorArchivePath=classpath:/test-encrypt/error", + "batchConfiguration.TransactionFilterBatch.pendingArchivePath=classpath:/test-encrypt/output/pending", + "batchConfiguration.TransactionFilterBatch.saltRecovery.enabled=false", + "batchConfiguration.TransactionFilterBatch.pagopaPublicKeyRecovery.enabled=false", + "batchConfiguration.TransactionFilterBatch.hpanListRecovery.enabled=false", + "batchConfiguration.TransactionFilterBatch.abiToFiscalCodeMapRecovery.enabled=false", + "batchConfiguration.TransactionFilterBatch.transactionSenderRtd.enabled=false", + "batchConfiguration.TransactionFilterBatch.transactionSenderAde.enabled=false", + "batchConfiguration.TransactionFilterBatch.transactionSenderPending.enabled=false", + "batchConfiguration.TransactionFilterBatch.senderAdeAckFilesRecovery.enabled=false", + "batchConfiguration.TransactionFilterBatch.senderAdeAckFilesRecovery.directoryPath=classpath:/test-encrypt/sender-ade-ack", + "batchConfiguration.TransactionFilterBatch.fileReportRecovery.directoryPath=classpath:/test-encrypt/reports", + "batchConfiguration.TransactionFilterBatch.fileReportRecovery.enabled=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableAfterProcessLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableAfterProcessFileLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnReadErrorLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnReadErrorFileLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableAfterProcessFileLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnProcessErrorLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnProcessErrorFileLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnWriteErrorLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.enableOnWriteErrorFileLogging=true", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.loggingFrequency=100", + "batchConfiguration.TransactionFilterBatch.transactionFilter.readers.listener.writerPoolSize=5", + "batchConfiguration.TransactionFilterBatch.transactionWriterAde.splitThreshold=1000" +}) @DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD) public class TransactionFilterBatchFileReportTest { @@ -143,10 +141,14 @@ public class TransactionFilterBatchFileReportTest { StoreService storeServiceSpy; @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(new File(getClass().getResource("/test-encrypt").getFile())); + public TemporaryFolder tempFolder = new TemporaryFolder( + new File(getClass().getResource("/test-encrypt").getFile())); PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); + public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS"); + @SneakyThrows @Before public void setUp() { @@ -156,7 +158,7 @@ public void setUp() { deleteFiles("classpath:/test-encrypt/output/*.pgp"); deleteFiles("classpath:/test-encrypt/sender-ade-ack/*.csv"); deleteFiles("classpath:/test-encrypt/output/*.csv"); - deleteFiles("classpath:/test-encrypt/reports/*.csv"); + deleteFiles("classpath:/test-encrypt/reports/*.json"); } @SneakyThrows @@ -222,12 +224,16 @@ public void jobExecutionProducesExpectedFiles() { Set expectedOutputFileAdeContent = getExpectedAdeOutputFileContent(); assertThat(expectedOutputFileTrnContent).containsAll(outputFileTrnContent); - assertThat(outputFileTrnContent.get(0)).isEqualTo("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); + assertThat(outputFileTrnContent.get(0)) + .isEqualTo("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); assertThat(expectedOutputFileAdeContent).containsAll(outputFileAdeContent); - assertThat(outputFileAdeContent.get(0)).isEqualTo("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); + assertThat(outputFileAdeContent.get(0)) + .isEqualTo("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); // Check that encrypted output files have the same content of unencrypted ones - File trxEncFile = outputPgpFiles.stream().filter(p -> p.getName().equals("CSTAR.99999.TRNLOG.20220204.094652.001.01.csv.pgp")).collect(Collectors.toList()).iterator().next(); + File trxEncFile = outputPgpFiles.stream() + .filter(p -> p.getName().equals("CSTAR.99999.TRNLOG.20220204.094652.001.01.csv.pgp")) + .collect(Collectors.toList()).iterator().next(); FileInputStream trxEncFileIS = new FileInputStream(trxEncFile); FileInputStream secretFilePathIS = null; @@ -236,11 +242,13 @@ public void jobExecutionProducesExpectedFiles() { Resource secretKeyResource = resolver.getResource(secretKeyPath); secretFilePathIS = new FileInputStream(secretKeyResource.getFile()); - byte[] trxEncFileDecryptedFileData = EncryptUtil.decryptFile(trxEncFileIS, secretFilePathIS, "test".toCharArray()); + byte[] trxEncFileDecryptedFileData = EncryptUtil.decryptFile(trxEncFileIS, secretFilePathIS, + "test".toCharArray()); File trxEncFileDecryptedFile = tempFolder.newFile("trxEncFileDecrypted.csv"); FileUtils.writeByteArrayToFile(trxEncFileDecryptedFile, trxEncFileDecryptedFileData); - List trxEncFileDecryptedFileContent = Files.readAllLines(trxEncFileDecryptedFile.toPath().toAbsolutePath()); + List trxEncFileDecryptedFileContent = Files + .readAllLines(trxEncFileDecryptedFile.toPath().toAbsolutePath()); assertThat(expectedOutputFileTrnContent).containsAll(trxEncFileDecryptedFileContent); } finally { trxEncFileIS.close(); @@ -249,48 +257,66 @@ public void jobExecutionProducesExpectedFiles() { // Check that logs folder contains expected files Collection outputLogsFiles = FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), new String[]{"csv"}, false); + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), new String[] { "csv" }, false); assertThat(outputLogsFiles).hasSize(2); - FileFilter fileFilter = new WildcardFileFilter("*_Rtd__FilteredRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); - Collection trxFilteredFiles = FileUtils.listFiles(resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, null); + FileFilter fileFilter = new WildcardFileFilter( + "*_Rtd__FilteredRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); + Collection trxFilteredFiles = FileUtils.listFiles( + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, + null); assertThat(trxFilteredFiles).hasSize(1); fileFilter = new WildcardFileFilter("*_Ade__FilteredRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); - Collection adeFilteredFiles = FileUtils.listFiles(resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, null); + Collection adeFilteredFiles = FileUtils.listFiles( + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, + null); assertThat(adeFilteredFiles).hasSize(1); // empty log files get deleted fileFilter = new WildcardFileFilter("*_Rtd__ErrorRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); - Collection trxErrorFiles = FileUtils.listFiles(resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, null); + Collection trxErrorFiles = FileUtils.listFiles( + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, + null); assertThat(trxErrorFiles).isEmpty(); fileFilter = new WildcardFileFilter("*_Ade__ErrorRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); - Collection adeErrorFiles = FileUtils.listFiles(resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, null); + Collection adeErrorFiles = FileUtils.listFiles( + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, + null); assertThat(adeErrorFiles).isEmpty(); // Check that logs files contains expected lines File trxFilteredFile = trxFilteredFiles.iterator().next(); List trxFilteredContent = Files.readAllLines(trxFilteredFile.toPath().toAbsolutePath()); assertThat(trxFilteredContent).hasSize(3) - .contains("99999;00;01;pan4;03/20/2020 13:23:00;4444444444;8888;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;par4", - "99999;00;01;123456******3456;2020-03-20T13:23:00;555555555;9999;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;", - "99999;00;01;123456******3456;03/20/2020 15:23:00;666666666;9999;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;"); + .contains( + "99999;00;01;pan4;03/20/2020 13:23:00;4444444444;8888;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;par4", + "99999;00;01;123456******3456;2020-03-20T13:23:00;555555555;9999;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;", + "99999;00;01;123456******3456;03/20/2020 15:23:00;666666666;9999;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;"); File adeFilteredFile = adeFilteredFiles.iterator().next(); List adeFilteredContent = Files.readAllLines(adeFilteredFile.toPath().toAbsolutePath()); assertThat(adeFilteredContent).hasSize(1) - .contains("99999;00;01;123456******3456;2020-03-20T13:23:00;555555555;9999;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;"); + .contains( + "99999;00;01;123456******3456;2020-03-20T13:23:00;555555555;9999;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;"); Collection fileReportSaved = getFileReportSaved(); assertThat(fileReportSaved).isNotNull().hasSize(1); List fileReportContent = Files.readAllLines(fileReportSaved.stream().findAny() - .orElse(new File("")).toPath()); - assertThat(fileReportContent).isNotNull().containsExactly("name;status;size;transmissionDate", - "file1;RECEIVED;200;" + currentDate, - "file2;RECEIVED;300;" + currentDate.minusDays(4), - "file3;RECEIVED;400;" + currentDate.minusDays(10)); + .orElse(new File("")).toPath()); + assertThat(fileReportContent).isNotNull().containsExactly("[", + " {\"name\":\"file1\",\"size\":200,\"status\":\"RECEIVED\",\"transmissionDate\":\"" + + currentDate.format(DATE_TIME_FORMATTER) + + "\",\"dataSummary\":{}},", + " {\"name\":\"file2\",\"size\":300,\"status\":\"RECEIVED\",\"transmissionDate\":\"" + + currentDate.minusDays(4).format(DATE_TIME_FORMATTER) + + "\",\"dataSummary\":{}},", + " {\"name\":\"file3\",\"size\":400,\"status\":\"RECEIVED\",\"transmissionDate\":\"" + + currentDate.minusDays(10).format(DATE_TIME_FORMATTER) + + "\",\"dataSummary\":{}}", + "]"); } @SneakyThrows @@ -300,7 +326,7 @@ public void givenAReportWhenLaunchFileReportStepThenSaveTheReportOnFile() { BDDMockito.doReturn(getStubFileReport(currentDate)).when(fileReportRestClient).getFileReport(); jobLauncherTestUtils.launchStep("file-report-recovery-step", - new JobParameters()); + new JobParameters()); Mockito.verify(fileReportRestClient, times(1)).getFileReport(); Collection fileReportSaved = getFileReportSaved(); @@ -308,12 +334,23 @@ public void givenAReportWhenLaunchFileReportStepThenSaveTheReportOnFile() { assertThat(fileReportSaved).isNotNull().hasSize(1); List fileReportContent = Files.readAllLines(fileReportSaved.stream().findAny() - .orElse(new File("")).toPath()); - - assertThat(fileReportContent).isNotNull().containsExactly("name;status;size;transmissionDate", - "file1;RECEIVED;200;" + currentDate, - "file2;RECEIVED;300;" + currentDate.minusDays(4), - "file3;RECEIVED;400;" + currentDate.minusDays(10)); + .orElse(new File("")).toPath()); + + assertThat(fileReportContent).isNotNull().containsExactly("[", + " {\"name\":\"file1\",\"size\":200,\"status\":\"RECEIVED\",\"transmissionDate\":\"" + + currentDate.format(DATE_TIME_FORMATTER) + + "\",\"dataSummary\":{}},", + " {\"name\":\"file2\",\"size\":300,\"status\":\"RECEIVED\",\"transmissionDate\":\"" + + currentDate.minusDays(4) + .format(DATE_TIME_FORMATTER) + + "\",\"dataSummary\":{}},", + + " {\"name\":\"file3\",\"size\":400,\"status\":\"RECEIVED\",\"transmissionDate\":\"" + + currentDate.minusDays(10) + .format(DATE_TIME_FORMATTER) + + "\",\"dataSummary\":{}}", + + "]"); } @SneakyThrows @@ -322,7 +359,7 @@ public void givenEmptyReportWhenLaunchFileReportStepThenSaveTheReportWithHeaderO BDDMockito.doReturn(getStubEmptyReport()).when(fileReportRestClient).getFileReport(); jobLauncherTestUtils.launchStep("file-report-recovery-step", - new JobParameters()); + new JobParameters()); Mockito.verify(fileReportRestClient, times(1)).getFileReport(); Collection fileReportSaved = getFileReportSaved(); @@ -330,9 +367,9 @@ public void givenEmptyReportWhenLaunchFileReportStepThenSaveTheReportWithHeaderO assertThat(fileReportSaved).isNotNull().hasSize(1); List fileReportContent = Files.readAllLines(fileReportSaved.stream().findAny() - .orElse(new File("")).toPath()); + .orElse(new File("")).toPath()); - assertThat(fileReportContent).isNotNull().contains("name;status;size;transmissionDate"); + assertThat(fileReportContent).isNotNull().contains("[", "]"); } @SneakyThrows @@ -342,7 +379,7 @@ public void givenMalformedReportWhenLaunchFileReportStepThenSaveTheReportWithHea BDDMockito.doReturn(new FileReport()).when(fileReportRestClient).getFileReport(); jobLauncherTestUtils.launchStep("file-report-recovery-step", - new JobParameters()); + new JobParameters()); Mockito.verify(fileReportRestClient, times(1)).getFileReport(); Collection fileReportSaved = getFileReportSaved(); @@ -350,14 +387,15 @@ public void givenMalformedReportWhenLaunchFileReportStepThenSaveTheReportWithHea assertThat(fileReportSaved).isNotNull().hasSize(1); List fileReportContent = Files.readAllLines(fileReportSaved.stream().findAny() - .orElse(new File("")).toPath()); + .orElse(new File("")).toPath()); - assertThat(fileReportContent).isNotNull().contains("name;status;size;transmissionDate"); + assertThat(fileReportContent).isNotNull().contains("[", "]"); } @SneakyThrows private Collection getFileReportSaved() { - return FileUtils.listFiles(resolver.getResources("classpath:/test-encrypt/reports")[0].getFile(), new String[]{"csv"}, false); + return FileUtils.listFiles(resolver.getResources("classpath:/test-encrypt/reports")[0].getFile(), + new String[] { "json" }, false); } private FileReport getStubFileReport(LocalDateTime dateTime) { @@ -399,7 +437,7 @@ private FileReport getStubEmptyReport() { private String createPublicKey() throws IOException { String publicKeyPath = "file:/" + Objects.requireNonNull( - this.getClass().getResource("/test-encrypt")).getFile() + "/publicKey.asc"; + this.getClass().getResource("/test-encrypt")).getFile() + "/publicKey.asc"; Resource publicKeyResource = resolver.getResource(publicKeyPath); FileInputStream publicKeyFilePathIS = new FileInputStream(publicKeyResource.getFile()); return IOUtils.toString(publicKeyFilePathIS); @@ -413,10 +451,10 @@ private void createPanPGP() { FileOutputStream panPgpFOS = new FileOutputStream(panPgp); EncryptUtil.encryptFile(panPgpFOS, - Objects.requireNonNull(this.getClass().getResource("/test-encrypt/pan")).getFile() + "/pan.csv", - EncryptUtil.readPublicKey( - this.getClass().getResourceAsStream("/test-encrypt/publicKey.asc")), - false, false); + Objects.requireNonNull(this.getClass().getResource("/test-encrypt/pan")).getFile() + "/pan.csv", + EncryptUtil.readPublicKey( + this.getClass().getResourceAsStream("/test-encrypt/publicKey.asc")), + false, false); panPgpFOS.close(); } @@ -424,7 +462,7 @@ private void createPanPGP() { @SneakyThrows private File createAdeOutputFile() { File outputFileAde = new File(resolver.getResource("classpath:/test-encrypt/output") - .getFile().getAbsolutePath() + "/ADE.99999.TRNLOG.20220204.094652.001.01.csv"); + .getFile().getAbsolutePath() + "/ADE.99999.TRNLOG.20220204.094652.001.01.csv"); outputFileAde.createNewFile(); return outputFileAde; @@ -433,7 +471,7 @@ private File createAdeOutputFile() { @SneakyThrows private File createTrnOutputFile() { File outputFileTrn = new File(resolver.getResource("classpath:/test-encrypt/output") - .getFile().getAbsolutePath() + "/CSTAR.99999.TRNLOG.20220204.094652.001.01.csv"); + .getFile().getAbsolutePath() + "/CSTAR.99999.TRNLOG.20220204.094652.001.01.csv"); outputFileTrn.createNewFile(); return outputFileTrn; @@ -442,7 +480,7 @@ private File createTrnOutputFile() { @SneakyThrows private Collection getOutputPgpFiles() { return FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/output")[0].getFile(), new String[]{"pgp"}, false); + resolver.getResources("classpath:/test-encrypt/output")[0].getFile(), new String[] { "pgp" }, false); } private Set getExpectedPgpFilenames() { @@ -455,7 +493,7 @@ private Set getExpectedPgpFilenames() { @SneakyThrows private Collection getOutputCsvFiles() { return FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/output")[0].getFile(), new String[]{"csv"}, false); + resolver.getResources("classpath:/test-encrypt/output")[0].getFile(), new String[] { "csv" }, false); } private Set getExpectedCsvFileNames() { @@ -469,9 +507,12 @@ private Set getExpectedCsvFileNames() { private Set getExpectedTrnOutputFileContent() { Set expectedOutputFileTrnContent = new HashSet<>(); expectedOutputFileTrnContent.add("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); - expectedOutputFileTrnContent.add("99999;00;00;a261f9479522020529213c5336dec371de5b3dacca0a8165c50ac33032c631ac;03/20/2020 10:50:33;1111111111;5555;;1111;978;22222;0000;1;000002;5422;fis123;12345678901;00;"); - expectedOutputFileTrnContent.add("99999;00;01;e2df0a82ac0aa12921c398e1eba9119772db868650ebef22b8919fa0fb7642ed;03/20/2020 11:23:00;333333333;7777;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;"); - expectedOutputFileTrnContent.add("99999;01;00;805f89015f85948f7d7bdd57a0a81e4cd95fc81bdd1195a69c4ab139f0ebed7b;03/20/2020 11:04:53;2222222222;6666;;2222;978;3333;0000;1;000002;5422;fis123;12345678901;00;par2"); + expectedOutputFileTrnContent.add( + "99999;00;00;a261f9479522020529213c5336dec371de5b3dacca0a8165c50ac33032c631ac;03/20/2020 10:50:33;1111111111;5555;;1111;978;22222;0000;1;000002;5422;fis123;12345678901;00;"); + expectedOutputFileTrnContent.add( + "99999;00;01;e2df0a82ac0aa12921c398e1eba9119772db868650ebef22b8919fa0fb7642ed;03/20/2020 11:23:00;333333333;7777;;3333;978;4444;0000;1;000002;5422;fis123;12345678901;00;"); + expectedOutputFileTrnContent.add( + "99999;01;00;805f89015f85948f7d7bdd57a0a81e4cd95fc81bdd1195a69c4ab139f0ebed7b;03/20/2020 11:04:53;2222222222;6666;;2222;978;3333;0000;1;000002;5422;fis123;12345678901;00;par2"); return expectedOutputFileTrnContent; } @@ -480,12 +521,16 @@ private Set getExpectedAdeOutputFileContent() { Set expectedOutputFileAdeContent = new HashSet<>(); expectedOutputFileAdeContent.add("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); - expectedOutputFileAdeContent.add("99999;00;" + transmissionDate + ";03/20/2020;3;9999;978;4444;0000;1;fis123;12345678901;00"); - expectedOutputFileAdeContent.add("99999;01;" + transmissionDate + ";03/20/2020;1;2222;978;3333;0000;1;fis123;12345678901;00"); - expectedOutputFileAdeContent.add("99999;00;" + transmissionDate + ";03/20/2020;1;1111;978;22222;0000;1;fis123;12345678901;00"); + expectedOutputFileAdeContent + .add("99999;00;" + transmissionDate + ";03/20/2020;3;9999;978;4444;0000;1;fis123;12345678901;00"); + expectedOutputFileAdeContent + .add("99999;01;" + transmissionDate + ";03/20/2020;1;2222;978;3333;0000;1;fis123;12345678901;00"); + expectedOutputFileAdeContent + .add("99999;00;" + transmissionDate + ";03/20/2020;1;1111;978;22222;0000;1;fis123;12345678901;00"); return expectedOutputFileAdeContent; } + private String getDateFormattedAsString() { DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd"); return OffsetDateTime.now().format(fmt); diff --git a/api/pom.xml b/api/pom.xml index e9c7da66..787ca033 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,12 +4,12 @@ rtd-ms-transaction-filter it.gov.pagopa.rtd.ms - 2.1.2 + 2.2.0 it.gov.pagopa.rtd.ms.transaction_filter.api rtd-ms-transaction-filter-api - 2.1.2 + 2.2.0 pom @@ -25,4 +25,3 @@ batch - diff --git a/app/pom.xml b/app/pom.xml index c0811f76..2c920c61 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -4,12 +4,12 @@ rtd-ms-transaction-filter it.gov.pagopa.rtd.ms - 2.1.2 + 2.2.0 it.gov.pagopa.rtd.ms.transaction_filter transaction-filter-app - 2.1.2 + 2.2.0 diff --git a/app/src/main/resources/config/application.yml b/app/src/main/resources/config/application.yml index 507713b8..5d23aa52 100644 --- a/app/src/main/resources/config/application.yml +++ b/app/src/main/resources/config/application.yml @@ -177,7 +177,7 @@ rest-client: received: url: /rtd/file-register/ack-received/{id} file-report: - url: /rtd/file-reporter/file-report + url: /rtd/file-reporter/${ACQ_BATCH_FILE_REPORT_URL:v2/file-report} apache: httpclient: diff --git a/core/pom.xml b/core/pom.xml index fbfa5f6e..9f5f50c1 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,12 +4,12 @@ rtd-ms-transaction-filter it.gov.pagopa.rtd.ms - 2.1.2 + 2.2.0 it.gov.pagopa.rtd.ms.transaction_filter rtd-ms-transaction-filter-core - 2.1.2 + 2.2.0 diff --git a/core/src/main/java/it/gov/pagopa/rtd/transaction_filter/service/StoreServiceImpl.java b/core/src/main/java/it/gov/pagopa/rtd/transaction_filter/service/StoreServiceImpl.java index 49a667d2..1f8d7a4e 100644 --- a/core/src/main/java/it/gov/pagopa/rtd/transaction_filter/service/StoreServiceImpl.java +++ b/core/src/main/java/it/gov/pagopa/rtd/transaction_filter/service/StoreServiceImpl.java @@ -13,7 +13,6 @@ import java.util.Set; import java.util.TreeSet; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; /** @@ -21,7 +20,6 @@ */ @Service @RequiredArgsConstructor -@Slf4j public class StoreServiceImpl implements StoreService { private final TreeSet hpanSet; diff --git a/integration/jpa/pom.xml b/integration/jpa/pom.xml index 6ee49e44..52cf1734 100644 --- a/integration/jpa/pom.xml +++ b/integration/jpa/pom.xml @@ -4,12 +4,12 @@ rtd-ms-transaction-filter-integration it.gov.pagopa.rtd.ms.transaction_filter - 2.1.2 + 2.2.0 it.gov.pagopa.rtd.ms.transaction_filter.integration rtd-ms-transaction-filter-integration-jpa - 2.1.2 + 2.2.0 diff --git a/integration/pom.xml b/integration/pom.xml index c95942f6..62f43f4f 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -4,12 +4,12 @@ rtd-ms-transaction-filter it.gov.pagopa.rtd.ms - 2.1.2 + 2.2.0 it.gov.pagopa.rtd.ms.transaction_filter rtd-ms-transaction-filter-integration - 2.1.2 + 2.2.0 pom diff --git a/integration/rest/pom.xml b/integration/rest/pom.xml index e89fd7a6..84f84e8e 100644 --- a/integration/rest/pom.xml +++ b/integration/rest/pom.xml @@ -4,12 +4,12 @@ it.gov.pagopa.rtd.ms.transaction_filter rtd-ms-transaction-filter-integration - 2.1.2 + 2.2.0 it.gov.pagopa.rtd.ms.transaction_filter.integration rtd-ms-transaction-filter-integration-rest - 2.1.2 + 2.2.0 diff --git a/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/model/FileMetadata.java b/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/model/FileMetadata.java index e9b75e7e..8b2de746 100644 --- a/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/model/FileMetadata.java +++ b/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/model/FileMetadata.java @@ -1,8 +1,17 @@ package it.gov.pagopa.rtd.transaction_filter.connector.model; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer; +import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; + import java.time.LocalDateTime; +import java.util.LinkedHashMap; +import java.util.Map; + import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; import lombok.Data; @@ -23,7 +32,17 @@ public class FileMetadata { @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) @JsonDeserialize(using = LocalDateTimeDeserializer.class) + @JsonSerialize(using = LocalDateTimeSerializer.class) + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSS") @NotNull private LocalDateTime transmissionDate; + private Map dataSummary = new LinkedHashMap<>(); + + @JsonAnySetter + void setDataSummary(String key, Object value) { + dataSummary.put(key, value); + } + + } diff --git a/integration/rest/src/main/resources/config/rest-client.properties b/integration/rest/src/main/resources/config/rest-client.properties index d62b25c8..f1aeeaf8 100644 --- a/integration/rest/src/main/resources/config/rest-client.properties +++ b/integration/rest/src/main/resources/config/rest-client.properties @@ -35,7 +35,7 @@ rest-client.hpan.abi-to-fiscalcode-map.url=/rtd/abi-to-fiscalcode/conversion-map rest-client.sender-ade-ack.list.url=/rtd/file-register/sender-ade-ack rest-client.sender-ade-ack.download-file.url=/ade/{id} rest-client.sender-ade-ack.received.url=/rtd/file-register/ack-received/{id} -rest-client.file-report.url=/rtd/file-reporter/file-report +rest-client.file-report.url=/rtd/file-reporter/${ACQ_BATCH_FILE_REPORT_URL:v2/file-report} spring.cloud.openfeign.client.config.hpan-service.connectTimeout=${REST_CLIENT_CONNECT_TIMEOUT:${HPAN_REST_CLIENT_CONNECT_TIMEOUT:5000}} spring.cloud.openfeign.client.config.hpan-service.readTimeout=${REST_CLIENT_READ_TIMEOUT:${HPAN_REST_CLIENT_READ_TIMEOUT:5000}} spring.cloud.openfeign.client.config.hpan-service.loggerLevel=${REST_CLIENT_LOGGER_LEVEL:${HPAN_REST_CLIENT_LOGGER_LEVEL:FULL}} diff --git a/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/FileReportRestClientTest.java b/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/FileReportRestClientTest.java index 354bebfb..66f694d3 100644 --- a/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/FileReportRestClientTest.java +++ b/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/FileReportRestClientTest.java @@ -53,7 +53,9 @@ "rest-client.hpan.key-store.password=secret", "rest-client.hpan.trust-store.file=classpath:certs/client-truststore.jks", "rest-client.hpan.trust-store.password=secret", - "spring.application.name=rtd-ms-transaction-filter-integration-rest"}) + "rest-client.file-report.url=/rtd/file-reporter/file-report", + "spring.application.name=rtd-ms-transaction-filter-integration-rest"} + ) @ContextConfiguration(initializers = FileReportRestClientTest.RandomPortInitializer.class, classes = { HpanRestConnectorConfig.class, FileReportRestClientImpl.class, BasicResponseEntityValidator.class, ValidatorConfig.class, HpanRestConnector.class, diff --git a/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/FileReportV2RestClientTest.java b/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/FileReportV2RestClientTest.java new file mode 100644 index 00000000..b329949a --- /dev/null +++ b/integration/rest/src/test/java/it/gov/pagopa/rtd/transaction_filter/connector/FileReportV2RestClientTest.java @@ -0,0 +1,256 @@ +package it.gov.pagopa.rtd.transaction_filter.connector; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.assertEquals; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.extension.responsetemplating.ResponseTemplateTransformer; +import com.github.tomakehurst.wiremock.junit.WireMockClassRule; +import feign.FeignException; +import it.gov.pagopa.rtd.transaction_filter.connector.config.HpanRestConnectorConfig; +import it.gov.pagopa.rtd.transaction_filter.connector.model.FileMetadata; +import it.gov.pagopa.rtd.transaction_filter.connector.model.FileReport; +import it.gov.pagopa.rtd.transaction_filter.validator.BasicResponseEntityValidator; +import it.gov.pagopa.rtd.transaction_filter.validator.ValidatorConfig; +import java.time.LocalDateTime; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import jakarta.validation.ValidationException; +import lombok.SneakyThrows; +import org.assertj.core.util.Lists; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.http.HttpMessageConvertersAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.openfeign.FeignAutoConfiguration; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.support.TestPropertySourceUtils; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) +@TestPropertySource(locations = "classpath:config/rest-client.properties", properties = { + "rest-client.hpan.list.url=/rtd/payment-instrument-manager/hashed-pans", + "rest-client.hpan.salt.url=/rtd/payment-instrument-manager/salt", + "rest-client.hpan.adesas.url=/rtd/csv-transaction/ade/sas", + "rest-client.hpan.rtdsas.url=/rtd/csv-transaction/rtd/sas", + "rest-client.hpan.mtls.enabled=true", "rest-client.hpan.list.checksumHeaderName=checksum", + "rest-client.hpan.dateValidation.enabled=true", + "rest-client.hpan.list.dateValidationHeaderName=date", + "rest-client.hpan.key-store.file=classpath:certs/client-keystore.jks", + "rest-client.hpan.key-store.password=secret", + "rest-client.hpan.trust-store.file=classpath:certs/client-truststore.jks", + "rest-client.hpan.trust-store.password=secret", + "spring.application.name=rtd-ms-transaction-filter-integration-rest" }) +@ContextConfiguration(initializers = FileReportV2RestClientTest.RandomPortInitializer.class, classes = { + HpanRestConnectorConfig.class, FileReportRestClientImpl.class, + BasicResponseEntityValidator.class, ValidatorConfig.class, HpanRestConnector.class, + FeignAutoConfiguration.class, HttpMessageConvertersAutoConfiguration.class }) +public class FileReportV2RestClientTest { + + @Autowired + private FileReportRestClientImpl restClient; + + ObjectMapper mapper = new ObjectMapper(); + + @ClassRule + public static WireMockClassRule wireMockRule = new WireMockClassRule(wireMockConfig() + .dynamicHttpsPort() + .dynamicPort() + .needClientAuth(true) + .keystorePath("src/test/resources/certs/server-keystore.jks") + .keystorePassword("secret") + .keyManagerPassword("secret") + .trustStorePath("src/test/resources/certs/server-truststore.jks") + .trustStorePassword("secret") + .usingFilesUnderClasspath("stubs") + .extensions(new ResponseTemplateTransformer(true))); + + @After + public void cleanup() { + wireMockRule.resetToDefaultMappings(); + } + + @SneakyThrows + @Test + public void whenGetFileReportV2ThenFileNameAndContentMatch() { + FileReport fileReport = restClient.getFileReport(); + + assertThat(fileReport).isNotNull().extracting(FileReport::getFilesRecentlyUploaded).asList().hasSize(2); + + assertEquals(fileReport.getFilesRecentlyUploaded().get(0).getName(), getDefaultReportV2().get(0).getName()); + assertEquals(fileReport.getFilesRecentlyUploaded().get(0).getSize(), getDefaultReportV2().get(0).getSize()); + assertEquals(fileReport.getFilesRecentlyUploaded().get(0).getStatus(), getDefaultReportV2().get(0).getStatus()); + assertEquals(fileReport.getFilesRecentlyUploaded().get(0).getTransmissionDate(), + getDefaultReportV2().get(0).getTransmissionDate()); + + assertEquals( + fileReport.getFilesRecentlyUploaded().get(0).getDataSummary().get("countNegativeTransactions"), + getDefaultReportV2().get(0).getDataSummary().get("countNegativeTransactions")); + assertEquals( + fileReport.getFilesRecentlyUploaded().get(0).getDataSummary().get("countPositiveTransactions"), + getDefaultReportV2().get(0).getDataSummary().get("countPositiveTransactions")); + assertEquals(fileReport.getFilesRecentlyUploaded().get(0).getDataSummary().get("maxAccountingDate"), + getDefaultReportV2().get(0).getDataSummary().get("maxAccountingDate")); + assertEquals(fileReport.getFilesRecentlyUploaded().get(0).getDataSummary().get("minAccountingDate"), + getDefaultReportV2().get(0).getDataSummary().get("minAccountingDate")); + assertEquals(fileReport.getFilesRecentlyUploaded().get(0).getDataSummary().get("numberOfMerchants"), + getDefaultReportV2().get(0).getDataSummary().get("numberOfMerchants")); + assertEquals(fileReport.getFilesRecentlyUploaded().get(0).getDataSummary().get("sha256OriginFile"), + getDefaultReportV2().get(0).getDataSummary().get("sha256OriginFile")); + assertEquals( + fileReport.getFilesRecentlyUploaded().get(0).getDataSummary().get("sumAmountNegativeTransactions"), + getDefaultReportV2().get(0).getDataSummary().get("sumAmountNegativeTransactions")); + assertEquals( + fileReport.getFilesRecentlyUploaded().get(0).getDataSummary().get("sumAmountPositiveTransactions"), + getDefaultReportV2().get(0).getDataSummary().get("sumAmountPositiveTransactions")); + + assertEquals(fileReport.getFilesRecentlyUploaded().get(1).getName(), getDefaultReportV2().get(1).getName()); + assertEquals(fileReport.getFilesRecentlyUploaded().get(1).getSize(), getDefaultReportV2().get(1).getSize()); + assertEquals(fileReport.getFilesRecentlyUploaded().get(1).getStatus(), getDefaultReportV2().get(1).getStatus()); + assertEquals(fileReport.getFilesRecentlyUploaded().get(1).getTransmissionDate(), + getDefaultReportV2().get(1).getTransmissionDate()); + + assertEquals( + fileReport.getFilesRecentlyUploaded().get(1).getDataSummary().get("countNegativeTransactions"), + getDefaultReportV2().get(1).getDataSummary().get("countNegativeTransactions")); + assertEquals( + fileReport.getFilesRecentlyUploaded().get(1).getDataSummary().get("countPositiveTransactions"), + getDefaultReportV2().get(1).getDataSummary().get("countPositiveTransactions")); + assertEquals(fileReport.getFilesRecentlyUploaded().get(1).getDataSummary().get("maxAccountingDate"), + getDefaultReportV2().get(1).getDataSummary().get("maxAccountingDate")); + assertEquals(fileReport.getFilesRecentlyUploaded().get(1).getDataSummary().get("minAccountingDate"), + getDefaultReportV2().get(1).getDataSummary().get("minAccountingDate")); + assertEquals(fileReport.getFilesRecentlyUploaded().get(1).getDataSummary().get("numberOfMerchants"), + getDefaultReportV2().get(1).getDataSummary().get("numberOfMerchants")); + assertEquals(fileReport.getFilesRecentlyUploaded().get(1).getDataSummary().get("sha256OriginFile"), + getDefaultReportV2().get(1).getDataSummary().get("sha256OriginFile")); + assertEquals( + fileReport.getFilesRecentlyUploaded().get(1).getDataSummary().get("sumAmountNegativeTransactions"), + getDefaultReportV2().get(1).getDataSummary().get("sumAmountNegativeTransactions")); + assertEquals( + fileReport.getFilesRecentlyUploaded().get(1).getDataSummary().get("sumAmountPositiveTransactions"), + getDefaultReportV2().get(1).getDataSummary().get("sumAmountPositiveTransactions")); + + } + + @SneakyThrows + @Test + public void whenGetFileReportV2WithEmptyBodyThenReportIsEmpty() { + wireMockRule.stubFor( + get(urlPathEqualTo("/rtd/file-reporter/v2/file-report")) + .willReturn( + aResponse().withBody(mapper.writeValueAsString(getEmptyFileReportV2())) + .withHeader("Content-type", "application/json"))); + + FileReport fileReport = restClient.getFileReport(); + + assertThat(fileReport).isNotNull().extracting(FileReport::getFilesRecentlyUploaded).asList().isEmpty(); + } + + @SneakyThrows + @Test + public void givenStatus404WhenGetFileReportV2ThenThrowException() { + wireMockRule.stubFor( + get(urlPathEqualTo("/rtd/file-reporter/v2/file-report")) + .willReturn( + aResponse().withStatus(404) + .withBody(mapper.writeValueAsString(getEmptyFileReportV2())) + .withHeader("Content-type", "application/json"))); + + assertThatThrownBy(() -> restClient.getFileReport()).isInstanceOf(FeignException.class); + } + + @SneakyThrows + @Test + public void givenMalformedBodyWhenGetFileReportV2ThenThrowException() { + wireMockRule.stubFor( + get(urlPathEqualTo("/rtd/file-reporter/v2/file-report")) + .willReturn( + aResponse().withBody(mapper.writeValueAsString(getMalformedReportV2())) + .withHeader("Content-type", "application/json"))); + + assertThatThrownBy(() -> restClient.getFileReport()).isInstanceOf( + ValidationException.class); + } + + private List getDefaultReportV2() { + FileMetadata file1 = new FileMetadata(); + Map dataSummary1 = new LinkedHashMap<>(); + file1.setName("ADE.file1.pgp"); + file1.setSize(200L); + file1.setStatus("SUCCESS"); + file1.setTransmissionDate(LocalDateTime.of(2022, 10, 30, 10, 0, 0, 123000000)); + dataSummary1.put("minAccountingDate", LocalDateTime.of(2022, 10, 28, 10, 0, 0, 123000000).toString()); + dataSummary1.put("maxAccountingDate", LocalDateTime.of(2022, 10, 30, 10, 0, 0, 123000000).toString()); + dataSummary1.put("numberOfMerchants", 123); + dataSummary1.put("countNegativeTransactions", 283); + dataSummary1.put("countPositiveTransactions", 980); + dataSummary1.put("sumAmountNegativeTransactions", 3232323); + dataSummary1.put("sumAmountPositiveTransactions", 1231232); + dataSummary1.put("sha256OriginFile", + "#sha256sum:615bbf196371b6f95b738dccf4a4e3873dff569f7a5c1eb3b50ff52b0718f65d"); + file1.setDataSummary(dataSummary1); + + FileMetadata file2 = new FileMetadata(); + Map dataSummary2 = new LinkedHashMap<>(); + file2.setName("ADE.file2.pgp"); + file2.setSize(500L); + file2.setStatus("SUCCESS"); + file2.setTransmissionDate(LocalDateTime.of(2022, 10, 31, 10, 0, 0, 123000000)); + dataSummary2.put("minAccountingDate", LocalDateTime.of(2022, 10, 30, 10, 0, 0, 123000000).toString()); + dataSummary2.put("maxAccountingDate", LocalDateTime.of(2022, 10, 31, 10, 0, 0, 123000000).toString()); + dataSummary2.put("numberOfMerchants", 234); + dataSummary2.put("countNegativeTransactions", 333); + dataSummary2.put("countPositiveTransactions", 1090); + dataSummary2.put("sumAmountNegativeTransactions", 890900); + dataSummary2.put("sumAmountPositiveTransactions", 988898023); + dataSummary2.put("sha256OriginFile", + "#sha256sum:615bbf196371b6f95b738dc9823yt3873dff569f7a5c1eb3b50ff52b0718f65d"); + + file2.setDataSummary(dataSummary2); + + return Lists.list(file1, file2); + } + + private FileReport getEmptyFileReportV2() { + FileReport fileReport = new FileReport(); + fileReport.setFilesRecentlyUploaded(Collections.emptyList()); + return fileReport; + } + + private FileReport getMalformedReportV2() { + FileReport fileReport = new FileReport(); + FileMetadata file = new FileMetadata(); + // missing mandatory fields like filename + file.setSize(200L); + fileReport.setFilesRecentlyUploaded(Collections.singletonList(file)); + return fileReport; + } + + public static class RandomPortInitializer implements + ApplicationContextInitializer { + + @SneakyThrows + @Override + public void initialize(ConfigurableApplicationContext applicationContext) { + TestPropertySourceUtils.addInlinedPropertiesToEnvironment(applicationContext, + String.format("rest-client.hpan.base-url=https://localhost:%d/", + wireMockRule.httpsPort())); + } + } +} \ No newline at end of file diff --git a/integration/rest/src/test/resources/stubs/mappings/fileReport.json b/integration/rest/src/test/resources/stubs/mappings/fileReport.json index 3b373cc9..8b914950 100644 --- a/integration/rest/src/test/resources/stubs/mappings/fileReport.json +++ b/integration/rest/src/test/resources/stubs/mappings/fileReport.json @@ -9,13 +9,13 @@ "filesRecentlyUploaded": [ { "name": "ADE.file1.pgp", - "transmissionDate": "2022-10-30T10:00:00.123Z", + "transmissionDate": "2022-10-30T10:00:00.123", "size": 200, "status": "SUCCESS" }, { "name": "ADE.file2.pgp", - "transmissionDate": "2022-10-31T10:00:00.123Z", + "transmissionDate": "2022-10-31T10:00:00.123", "size": 500, "status": "SUCCESS" } @@ -25,4 +25,4 @@ "Content-Type": "application/json" } } -} +} \ No newline at end of file diff --git a/integration/rest/src/test/resources/stubs/mappings/fileReportv2.json b/integration/rest/src/test/resources/stubs/mappings/fileReportv2.json new file mode 100644 index 00000000..0e5ee55e --- /dev/null +++ b/integration/rest/src/test/resources/stubs/mappings/fileReportv2.json @@ -0,0 +1,48 @@ +{ + "request": { + "method": "GET", + "urlPath": "/rtd/file-reporter/v2/file-report" + }, + "response": { + "status": "200", + "jsonBody": { + "filesRecentlyUploaded": [ + { + "name": "ADE.file1.pgp", + "transmissionDate": "2022-10-30T10:00:00.123", + "size": 200, + "status": "SUCCESS", + "dataSummary": { + "minAccountingDate": "2022-10-28T10:00:00.123", + "maxAccountingDate": "2022-10-30T10:00:00.123", + "numberOfMerchants": 123, + "countNegativeTransactions": 283, + "countPositiveTransactions": 980, + "sumAmountNegativeTransactions": 3232323, + "sumAmountPositiveTransactions": 1231232, + "sha256OriginFile": "#sha256sum:615bbf196371b6f95b738dccf4a4e3873dff569f7a5c1eb3b50ff52b0718f65d" + } + }, + { + "name": "ADE.file2.pgp", + "transmissionDate": "2022-10-31T10:00:00.123", + "size": 500, + "status": "SUCCESS", + "dataSummary": { + "minAccountingDate": "2022-10-30T10:00:00.123", + "maxAccountingDate": "2022-10-31T10:00:00.123", + "numberOfMerchants": 234, + "countNegativeTransactions": 333, + "countPositiveTransactions": 1090, + "sumAmountNegativeTransactions": 890900, + "sumAmountPositiveTransactions": 988898023, + "sha256OriginFile": "#sha256sum:615bbf196371b6f95b738dc9823yt3873dff569f7a5c1eb3b50ff52b0718f65d" + } + } + ] + }, + "headers": { + "Content-Type": "application/json" + } + } +} \ No newline at end of file diff --git a/ops_resources/example_config/application.yml b/ops_resources/example_config/application.yml index 605456f8..0e203c09 100644 --- a/ops_resources/example_config/application.yml +++ b/ops_resources/example_config/application.yml @@ -175,7 +175,7 @@ rest-client: received: url: /rtd/file-register/ack-received/{id} file-report: - url: /rtd/file-reporter/file-report + url: /rtd/file-reporter/${ACQ_BATCH_FILE_REPORT_URL:v2/file-report} apache: httpclient: diff --git a/ops_resources/example_config/application_hbsql.yml b/ops_resources/example_config/application_hbsql.yml index cf261899..99446ddb 100644 --- a/ops_resources/example_config/application_hbsql.yml +++ b/ops_resources/example_config/application_hbsql.yml @@ -153,7 +153,7 @@ rest-client: received: url: /rtd/file-register/ack-received/{id} file-report: - url: /rtd/file-reporter/file-report + url: /rtd/file-reporter/${ACQ_BATCH_FILE_REPORT_URL:v2/file-report} apache: httpclient: diff --git a/pom.xml b/pom.xml index 9d644fbe..93cb7998 100644 --- a/pom.xml +++ b/pom.xml @@ -5,12 +5,12 @@ org.springframework.boot spring-boot-starter-parent - 3.2.4 + 3.2.5 it.gov.pagopa.rtd.ms rtd-ms-transaction-filter - 2.1.2 + 2.2.0 pom @@ -33,7 +33,7 @@ UTF-8 **/enums/**,**/model/**,**/Constants*.java,**/*Config.java,**/*Application.java - 2.1.2 + 2.2.0 From 091a53145455be6ce58ab76c83196c3256f02f49 Mon Sep 17 00:00:00 2001 From: Luca Consalvi <117908483+lucaconsalvi@users.noreply.github.com> Date: Fri, 3 May 2024 16:37:41 +0200 Subject: [PATCH 3/3] fix: [RTD-2513] operation type constraint (#395) [RTD-2513] mod operation type constraint --- .../batch/model/AdeTransactionsAggregate.java | 2 +- .../batch/model/InboundTransaction.java | 2 +- .../batch/step/TransactionFilterStep.java | 2 - .../TransactionFilterBatchFileReportTest.java | 67 +++++++++++-------- ...TransactionItemProcessorValidatorTest.java | 4 +- .../connector/model/FileMetadata.java | 4 +- 6 files changed, 44 insertions(+), 37 deletions(-) diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/model/AdeTransactionsAggregate.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/model/AdeTransactionsAggregate.java index 49ba3184..3ebcb7ab 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/model/AdeTransactionsAggregate.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/model/AdeTransactionsAggregate.java @@ -27,7 +27,7 @@ @NotNull @NotBlank @Size(min = 2, max = 2) - @Pattern(regexp = "\\d{2}") + @Pattern(regexp = "00|01") String operationType; @NotNull diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/model/InboundTransaction.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/model/InboundTransaction.java index 19bb6032..8d587119 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/model/InboundTransaction.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/model/InboundTransaction.java @@ -58,7 +58,7 @@ @NotNull @NotBlank @Size(min = 2, max = 2) - @Pattern(regexp = "\\d{2}") + @Pattern(regexp = "00|01") String operationType; /** diff --git a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/TransactionFilterStep.java b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/TransactionFilterStep.java index ce19ee6c..7a52be80 100644 --- a/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/TransactionFilterStep.java +++ b/api/batch/src/main/java/it/gov/pagopa/rtd/transaction_filter/batch/step/TransactionFilterStep.java @@ -167,8 +167,6 @@ public class TransactionFilterStep { "senderCode", "operationType", "circuitType", "pan", "trxDate", "idTrxAcquirer", "idTrxIssuer", "correlationId", "amount", "amountCurrency", "acquirerId", "merchantId", "terminalId", "bin", "mcc", "fiscalCode", "vat", "posType", "par"}; - private static final String[] REPORT_CSV_FIELDS = new String[]{ - "name", "status", "size", "transmissionDate"}; private static final String CSV_DELIMITER = ";"; private static final String DATE_FORMAT_FOR_FILENAME = "yyyyMMddHHmmssSSS"; diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchFileReportTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchFileReportTest.java index 701a5b71..35549b40 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchFileReportTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/TransactionFilterBatchFileReportTest.java @@ -146,9 +146,6 @@ public class TransactionFilterBatchFileReportTest { PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); - public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter - .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS"); - @SneakyThrows @Before public void setUp() { @@ -212,7 +209,8 @@ public void jobExecutionProducesExpectedFiles() { Set expectedPgpFilenames = getExpectedPgpFilenames(); assertThat(expectedPgpFilenames).containsAll(outputPgpFilenames); - Set outputCsvFilenames = getOutputCsvFiles().stream().map(File::getName).collect(Collectors.toSet()); + Set outputCsvFilenames = getOutputCsvFiles().stream().map(File::getName) + .collect(Collectors.toSet()); Set expectedCsvFilenames = getExpectedCsvFileNames(); assertThat(outputCsvFilenames).containsAll(expectedCsvFilenames); @@ -238,7 +236,8 @@ public void jobExecutionProducesExpectedFiles() { FileInputStream trxEncFileIS = new FileInputStream(trxEncFile); FileInputStream secretFilePathIS = null; try { - String secretKeyPath = "file:/" + this.getClass().getResource("/test-encrypt").getFile() + "/secretKey.asc"; + String secretKeyPath = "file:/" + this.getClass().getResource("/test-encrypt").getFile() + + "/secretKey.asc"; Resource secretKeyResource = resolver.getResource(secretKeyPath); secretFilePathIS = new FileInputStream(secretKeyResource.getFile()); @@ -257,32 +256,38 @@ public void jobExecutionProducesExpectedFiles() { // Check that logs folder contains expected files Collection outputLogsFiles = FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), new String[] { "csv" }, false); + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), + new String[] { "csv" }, false); assertThat(outputLogsFiles).hasSize(2); FileFilter fileFilter = new WildcardFileFilter( "*_Rtd__FilteredRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); Collection trxFilteredFiles = FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), + (IOFileFilter) fileFilter, null); assertThat(trxFilteredFiles).hasSize(1); - fileFilter = new WildcardFileFilter("*_Ade__FilteredRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); + fileFilter = new WildcardFileFilter( + "*_Ade__FilteredRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); Collection adeFilteredFiles = FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), + (IOFileFilter) fileFilter, null); assertThat(adeFilteredFiles).hasSize(1); // empty log files get deleted fileFilter = new WildcardFileFilter("*_Rtd__ErrorRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); Collection trxErrorFiles = FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), + (IOFileFilter) fileFilter, null); assertThat(trxErrorFiles).isEmpty(); fileFilter = new WildcardFileFilter("*_Ade__ErrorRecords_CSTAR.99999.TRNLOG.20220204.094652.001.csv"); Collection adeErrorFiles = FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), (IOFileFilter) fileFilter, + resolver.getResources("classpath:/test-encrypt/errorLogs")[0].getFile(), + (IOFileFilter) fileFilter, null); assertThat(adeErrorFiles).isEmpty(); @@ -308,19 +313,19 @@ public void jobExecutionProducesExpectedFiles() { .orElse(new File("")).toPath()); assertThat(fileReportContent).isNotNull().containsExactly("[", " {\"name\":\"file1\",\"size\":200,\"status\":\"RECEIVED\",\"transmissionDate\":\"" - + currentDate.format(DATE_TIME_FORMATTER) + + currentDate.toString() + "\",\"dataSummary\":{}},", " {\"name\":\"file2\",\"size\":300,\"status\":\"RECEIVED\",\"transmissionDate\":\"" - + currentDate.minusDays(4).format(DATE_TIME_FORMATTER) + + currentDate.minusDays(4).toString() + "\",\"dataSummary\":{}},", " {\"name\":\"file3\",\"size\":400,\"status\":\"RECEIVED\",\"transmissionDate\":\"" - + currentDate.minusDays(10).format(DATE_TIME_FORMATTER) + + currentDate.minusDays(10).toString() + "\",\"dataSummary\":{}}", "]"); } - @SneakyThrows @Test + @SneakyThrows public void givenAReportWhenLaunchFileReportStepThenSaveTheReportOnFile() { LocalDateTime currentDate = LocalDateTime.now(); BDDMockito.doReturn(getStubFileReport(currentDate)).when(fileReportRestClient).getFileReport(); @@ -338,16 +343,14 @@ public void givenAReportWhenLaunchFileReportStepThenSaveTheReportOnFile() { assertThat(fileReportContent).isNotNull().containsExactly("[", " {\"name\":\"file1\",\"size\":200,\"status\":\"RECEIVED\",\"transmissionDate\":\"" - + currentDate.format(DATE_TIME_FORMATTER) + + currentDate.toString() + "\",\"dataSummary\":{}},", " {\"name\":\"file2\",\"size\":300,\"status\":\"RECEIVED\",\"transmissionDate\":\"" - + currentDate.minusDays(4) - .format(DATE_TIME_FORMATTER) + + currentDate.minusDays(4).toString() + "\",\"dataSummary\":{}},", " {\"name\":\"file3\",\"size\":400,\"status\":\"RECEIVED\",\"transmissionDate\":\"" - + currentDate.minusDays(10) - .format(DATE_TIME_FORMATTER) + + currentDate.minusDays(10).toString() + "\",\"dataSummary\":{}}", "]"); @@ -451,7 +454,8 @@ private void createPanPGP() { FileOutputStream panPgpFOS = new FileOutputStream(panPgp); EncryptUtil.encryptFile(panPgpFOS, - Objects.requireNonNull(this.getClass().getResource("/test-encrypt/pan")).getFile() + "/pan.csv", + Objects.requireNonNull(this.getClass().getResource("/test-encrypt/pan")).getFile() + + "/pan.csv", EncryptUtil.readPublicKey( this.getClass().getResourceAsStream("/test-encrypt/publicKey.asc")), false, false); @@ -480,7 +484,8 @@ private File createTrnOutputFile() { @SneakyThrows private Collection getOutputPgpFiles() { return FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/output")[0].getFile(), new String[] { "pgp" }, false); + resolver.getResources("classpath:/test-encrypt/output")[0].getFile(), + new String[] { "pgp" }, false); } private Set getExpectedPgpFilenames() { @@ -493,7 +498,8 @@ private Set getExpectedPgpFilenames() { @SneakyThrows private Collection getOutputCsvFiles() { return FileUtils.listFiles( - resolver.getResources("classpath:/test-encrypt/output")[0].getFile(), new String[] { "csv" }, false); + resolver.getResources("classpath:/test-encrypt/output")[0].getFile(), + new String[] { "csv" }, false); } private Set getExpectedCsvFileNames() { @@ -506,7 +512,8 @@ private Set getExpectedCsvFileNames() { private Set getExpectedTrnOutputFileContent() { Set expectedOutputFileTrnContent = new HashSet<>(); - expectedOutputFileTrnContent.add("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); + expectedOutputFileTrnContent + .add("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); expectedOutputFileTrnContent.add( "99999;00;00;a261f9479522020529213c5336dec371de5b3dacca0a8165c50ac33032c631ac;03/20/2020 10:50:33;1111111111;5555;;1111;978;22222;0000;1;000002;5422;fis123;12345678901;00;"); expectedOutputFileTrnContent.add( @@ -520,13 +527,17 @@ private Set getExpectedAdeOutputFileContent() { String transmissionDate = getDateFormattedAsString(); Set expectedOutputFileAdeContent = new HashSet<>(); - expectedOutputFileAdeContent.add("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); expectedOutputFileAdeContent - .add("99999;00;" + transmissionDate + ";03/20/2020;3;9999;978;4444;0000;1;fis123;12345678901;00"); + .add("#sha256sum:0632500c45de8ef75cd875f0898eaa886659519b615b968a81656e4405320a4d"); + expectedOutputFileAdeContent + .add("99999;00;" + transmissionDate + + ";03/20/2020;3;9999;978;4444;0000;1;fis123;12345678901;00"); expectedOutputFileAdeContent - .add("99999;01;" + transmissionDate + ";03/20/2020;1;2222;978;3333;0000;1;fis123;12345678901;00"); + .add("99999;01;" + transmissionDate + + ";03/20/2020;1;2222;978;3333;0000;1;fis123;12345678901;00"); expectedOutputFileAdeContent - .add("99999;00;" + transmissionDate + ";03/20/2020;1;1111;978;22222;0000;1;fis123;12345678901;00"); + .add("99999;00;" + transmissionDate + + ";03/20/2020;1;1111;978;22222;0000;1;fis123;12345678901;00"); return expectedOutputFileAdeContent; } diff --git a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessorValidatorTest.java b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessorValidatorTest.java index fe0eb478..e3778bfc 100644 --- a/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessorValidatorTest.java +++ b/api/batch/src/test/java/it/gov/pagopa/rtd/transaction_filter/batch/step/processor/InboundTransactionItemProcessorValidatorTest.java @@ -74,7 +74,7 @@ void processTransactionWithValidSenderCode(String senderCode) { } @ParameterizedTest - @ValueSource(strings = {"", " ", " ", "010", "AA"}) + @ValueSource(strings = {"", " ", " ", "010", "AA", "02", "03", "04"}) void processTransactionWithInvalidOperationTypeThrowsException(String operationType) { InboundTransaction transaction = fakeInboundTransaction(); transaction.setOperationType(operationType); @@ -83,7 +83,7 @@ void processTransactionWithInvalidOperationTypeThrowsException(String operationT } @ParameterizedTest - @ValueSource(strings = {"00", "01", "02", "03", "04"}) + @ValueSource(strings = {"00", "01"}) void processTransactionWithValidOperationType(String operationType) { BDDMockito.doReturn(true).when(storeServiceMock).hasHpan(FAKE_ENROLLED_PAN); BDDMockito.doReturn(FAKE_SALT).when(storeServiceMock).getSalt(); diff --git a/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/model/FileMetadata.java b/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/model/FileMetadata.java index 8b2de746..a84d4ff7 100644 --- a/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/model/FileMetadata.java +++ b/integration/rest/src/main/java/it/gov/pagopa/rtd/transaction_filter/connector/model/FileMetadata.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer; @@ -33,7 +32,7 @@ public class FileMetadata { @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) @JsonDeserialize(using = LocalDateTimeDeserializer.class) @JsonSerialize(using = LocalDateTimeSerializer.class) - @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSS") + @JsonFormat(shape = JsonFormat.Shape.STRING) @NotNull private LocalDateTime transmissionDate; @@ -44,5 +43,4 @@ void setDataSummary(String key, Object value) { dataSummary.put(key, value); } - }