From 04b0db5aea39309b7d5c1c31a3e2dcdf4e7c7fdc Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Mon, 7 Oct 2024 09:47:34 -0500 Subject: [PATCH] feat: read WITH_EU_SCREENING_LIST as option to skip EU CSL list --- README.md | 1 + cmd/server/download.go | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 926f4378..d0588f5b 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,7 @@ PONG | `OFAC_DOWNLOAD_TEMPLATE` | HTTP address for downloading raw OFAC files. | `https://www.treasury.gov/ofac/downloads/%s` | | `DPL_DOWNLOAD_TEMPLATE` | HTTP address for downloading the DPL. | `https://www.bis.doc.gov/dpl/%s` | | `EU_CSL_DOWNLOAD_URL` | Use an alternate URL for downloading EU Consolidated Screening List | Subresource of `webgate.ec.europa.eu` | +| `WITH_EU_SCREENING_LIST` | Download and parse the EU Consolidated Screening List | Default: `true` | | `UK_CSL_DOWNLOAD_URL` | Use an alternate URL for downloading UK Consolidated Screening List | Subresource of `www.gov.uk` | | `UK_SANCTIONS_LIST_URL` | Use an alternate URL for downloading UK Sanctions List | Subresource of `www.gov.uk` | | `WITH_UK_SANCTIONS_LIST` | Download and parse the UK Sanctions List on startup. | Default: `false` | diff --git a/cmd/server/download.go b/cmd/server/download.go index e2f569e6..6a94b95c 100644 --- a/cmd/server/download.go +++ b/cmd/server/download.go @@ -6,6 +6,7 @@ package main import ( "bytes" + "cmp" "encoding/json" "errors" "fmt" @@ -14,6 +15,7 @@ import ( "time" "github.com/moov-io/base/log" + "github.com/moov-io/base/strx" "github.com/moov-io/watchman/pkg/csl" "github.com/moov-io/watchman/pkg/dpl" "github.com/moov-io/watchman/pkg/ofac" @@ -284,12 +286,15 @@ func (s *searcher) refreshData(initialDir string) (*DownloadStats, error) { } dps := precomputeDPs(deniedPersons, s.pipe) - euConsolidatedList, err := euCSLRecords(s.logger, initialDir) - if err != nil { - lastDataRefreshFailure.WithLabelValues("EUCSL").Set(float64(time.Now().Unix())) - stats.Errors = append(stats.Errors, fmt.Errorf("EUCSL: %v", err)) + var euConsolidatedList []*csl.EUCSLRecord + withEUScreeningList := cmp.Or(os.Getenv("WITH_EU_SCREENING_LIST"), "true") + if strx.Yes(withEUScreeningList) { + euConsolidatedList, err = euCSLRecords(s.logger, initialDir) + if err != nil { + lastDataRefreshFailure.WithLabelValues("EUCSL").Set(float64(time.Now().Unix())) + stats.Errors = append(stats.Errors, fmt.Errorf("EUCSL: %v", err)) + } } - euCSLs := precomputeCSLEntities[csl.EUCSLRecord](euConsolidatedList, s.pipe) ukConsolidatedList, err := ukCSLRecords(s.logger, initialDir) @@ -301,8 +306,8 @@ func (s *searcher) refreshData(initialDir string) (*DownloadStats, error) { ukCSLs := precomputeCSLEntities[csl.UKCSLRecord](ukConsolidatedList, s.pipe) var ukSLs []*Result[csl.UKSanctionsListRecord] - withSanctionsList := os.Getenv("WITH_UK_SANCTIONS_LIST") - if strings.ToLower(withSanctionsList) == "true" { + withUKSanctionsList := os.Getenv("WITH_UK_SANCTIONS_LIST") + if strings.ToLower(withUKSanctionsList) == "true" { ukSanctionsList, err := ukSanctionsListRecords(s.logger, initialDir) if err != nil { lastDataRefreshFailure.WithLabelValues("UKSanctionsList").Set(float64(time.Now().Unix()))