refactor: split out a few packages for use across v1 and v2 search

moov-io · Jan 3, 2025 · 69cdfdf · 69cdfdf
1 parent 92704ed
commit 69cdfdf
Show file tree

Hide file tree

Showing 30 changed files with 1,355 additions and 542 deletions.
diff --git a/cmd/server/issue115_test.go b/cmd/server/issue115_test.go
@@ -8,14 +8,15 @@ import (
 	"testing"
 
 	"github.com/moov-io/base/log"
+	"github.com/moov-io/watchman/internal/stringscore"
 	"github.com/moov-io/watchman/pkg/ofac"
 )
 
 func TestIssue115__TopSDNs(t *testing.T) {
-	score := jaroWinkler("georgehabbash", "georgebush")
-	eql(t, "george bush jaroWinkler", score, 0.896)
+	score := stringscore.JaroWinkler("georgehabbash", "georgebush")
+	eql(t, "george bush stringscore.JaroWinkler", score, 0.896)
 
-	score = jaroWinkler("g", "geoergebush")
+	score = stringscore.JaroWinkler("g", "geoergebush")
 	eql(t, "g vs geoergebush", score, 0.070)
 
 	pipe := noLogPipeliner

diff --git a/cmd/server/issue326_test.go b/cmd/server/issue326_test.go
@@ -7,26 +7,29 @@ package main
 import (
 	"testing"
 
+	"github.com/moov-io/watchman/internal/prepare"
+	"github.com/moov-io/watchman/internal/stringscore"
+
 	"github.com/stretchr/testify/assert"
 )
 
 func TestIssue326(t *testing.T) {
-	india := precompute("Huawei Technologies India Private Limited")
-	investment := precompute("Huawei Technologies Investment Co. Ltd.")
+	india := prepare.LowerAndRemovePunctuation("Huawei Technologies India Private Limited")
+	investment := prepare.LowerAndRemovePunctuation("Huawei Technologies Investment Co. Ltd.")
 
 	// Cuba
-	score := jaroWinkler(precompute("Huawei Cuba"), precompute("Huawei"))
+	score := stringscore.JaroWinkler(prepare.LowerAndRemovePunctuation("Huawei Cuba"), prepare.LowerAndRemovePunctuation("Huawei"))
 	assert.Equal(t, 0.7444444444444445, score)
 
 	// India
-	score = jaroWinkler(india, precompute("Huawei"))
+	score = stringscore.JaroWinkler(india, prepare.LowerAndRemovePunctuation("Huawei"))
 	assert.Equal(t, 0.4846031746031746, score)
-	score = jaroWinkler(india, precompute("Huawei Technologies"))
+	score = stringscore.JaroWinkler(india, prepare.LowerAndRemovePunctuation("Huawei Technologies"))
 	assert.Equal(t, 0.6084415584415584, score)
 
 	// Investment
-	score = jaroWinkler(investment, precompute("Huawei"))
+	score = stringscore.JaroWinkler(investment, prepare.LowerAndRemovePunctuation("Huawei"))
 	assert.Equal(t, 0.3788888888888889, score)
-	score = jaroWinkler(investment, precompute("Huawei Technologies"))
+	score = stringscore.JaroWinkler(investment, prepare.LowerAndRemovePunctuation("Huawei Technologies"))
 	assert.Equal(t, 0.5419191919191919, score)
 }
diff --git a/cmd/server/main.go b/cmd/server/main.go
@@ -24,6 +24,7 @@ import (
 	"github.com/moov-io/base/http/bind"
 	"github.com/moov-io/base/log"
 	"github.com/moov-io/watchman"
+	"github.com/moov-io/watchman/internal/prepare"
 	searchv2 "github.com/moov-io/watchman/internal/search"
 	"github.com/moov-io/watchman/pkg/ofac"
 	pubsearch "github.com/moov-io/watchman/pkg/search"
@@ -129,11 +130,11 @@ func main() {
 	}()
 	defer adminServer.Shutdown()
 
-	var pipeline *pipeliner
+	var pipeline *prepare.Pipeliner
 	if debug, err := strconv.ParseBool(os.Getenv("DEBUG_NAME_PIPELINE")); debug && err == nil {
-		pipeline = newPipeliner(logger, true)
+		pipeline = prepare.NewPipeliner(logger, true)
 	} else {
-		pipeline = newPipeliner(log.NewNopLogger(), false)
+		pipeline = prepare.NewPipeliner(log.NewNopLogger(), false)
 	}
 
 	searchWorkers := readInt(os.Getenv("SEARCH_MAX_WORKERS"), *flagWorkers)
@@ -299,3 +300,14 @@ func generalizeOFACSDNs(input []*SDN, ofacAddresses []*Address) []pubsearch.Enti
 func addSearchV2Routes(logger log.Logger, r *mux.Router, service searchv2.Service) {
 	searchv2.NewController(logger, service).AppendRoutes(r)
 }
+
+func readInt(override string, value int) int {
+	if override != "" {
+		n, err := strconv.ParseInt(override, 10, 32)
+		if err != nil {
+			panic(fmt.Errorf("unable to parse %q as int", override)) //nolint:forbidigo
+		}
+		return int(n)
+	}
+	return value
+}