Merge master to 1.0.0-SNAPSHOT (#623)

* Fix node resolver for empty/missing files (#587)

* Autogenerated JaCoCo coverage badge

* Bump to v0.26.1 (#588)

* Autogenerated JaCoCo coverage badge

* parallel explain for nrtsearch (#589)

* parallel explain for nrtsearch

* fix nit

* add explain for multi segment parallel

* refactor isExplain in different contexts

* make NRTSearch empty boolean query result constant score 1 (#592)

* make NRTSearch empty boolean query result constant score 1

* fix tests

* soft exception for fvh failures (#593)

* Support unit in SortType (#590)

create sort context and support unit in lat_lon distance sort

* Autogenerated JaCoCo coverage badge

* Upgrade dependencies for snyk check (#596)

* Fix example-plugin test (#598)

* Add page fault metrics (#599)

* Add additional merge metrics (#607)

* Add live setting for verbose index metrics (#608)

* Add live setting for verbose index metrics

* Address review comment

* Autogenerated JaCoCo coverage badge

* Bump to v0.29.0 (#609)

* Add live index settings override from config file (#610)

* Autogenerated JaCoCo coverage badge

* Add ability to update local index live settings (#611)

* Add deadline cancellation for indexing (#612)

* Add geo polygon query type (#613)

* Add geo polygon query type

* Detect closed polygons, update docs

* Autogenerated JaCoCo coverage badge

* Bump to v0.30.0 (#615)

* Testing readthedocs web hooks (#616)

* Add a bare minimum readthedocs config file (#617)

* log more info when fvh failed (#618)

 log more info when fvh failed

* Avoid calling query.toString() (#619)

* add sts for web identity auth (#620)

add sts for web identity auth

* Add search diagnostics to deadline exceeded exceptions (#621)

* Add search diagnostics to deadline exceeded exceptions
---------

Co-authored-by: swekannan <shwethu.kannan93@gmail.com>

* Fixes and spotless apply

* Updated grpc-gateway

---------

Co-authored-by: Andrew Prudhomme <asp@yelp.com>
Co-authored-by: github_actions <runner@fv-az248-700.3twvhzoricxu3figbgb44chhog.cx.internal.cloudapp.net>
Co-authored-by: github_actions <runner@fv-az887-622.kyhrjabtieueri5x0cgfkhfc1a.bx.internal.cloudapp.net>
Co-authored-by: Tao Yu <taoyu@yelp.com>
Co-authored-by: waziqi89 <89210409+waziqi89@users.noreply.github.com>
Co-authored-by: github_actions <runner@fv-az248-372.3twvhzoricxu3figbgb44chhog.cx.internal.cloudapp.net>
Co-authored-by: github_actions <runner@fv-az574-753.esnmgxn14wlejbqjnvhsbsbtxa.dx.internal.cloudapp.net>
Co-authored-by: github_actions <runner@fv-az1272-720.grsihaubamwerhiryzjrxtypna.phxx.internal.cloudapp.net>
Co-authored-by: github_actions <runner@fv-az736-601.alpbqrzxv30uzkvtn2qktnuusd.cx.internal.cloudapp.net>
Co-authored-by: Mohammad Mohtasham <mohm@yelp.com>
Co-authored-by: swekannan <62616258+swekannan@users.noreply.github.com>
Co-authored-by: swekannan <shwethu.kannan93@gmail.com>

.readthedocs.yaml

@@ -0,0 +1,34 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+# python:
+#   install:
+#     - requirements: docs/requirements.txt

build.gradle

@@ -75,6 +75,7 @@ dependencies {
     implementation "org.apache.commons:commons-compress:${commonsCompressVersion}"
     implementation "com.amazonaws:aws-java-sdk-core:${awsJavaSdkVersion}"
     implementation "com.amazonaws:aws-java-sdk-s3:${awsJavaSdkVersion}"
+    runtimeOnly "com.amazonaws:aws-java-sdk-sts:${awsJavaSdkVersion}"
     implementation "javax.xml.bind:jaxb-api:2.3.1"
     implementation "com.guicedee.services:guice:${guicedeeVersion}"
     implementation "org.lz4:lz4-java:${project.ext.lz4Version}"

clientlib/src/main/proto/yelp/nrtsearch/luceneserver.proto

@@ -470,6 +470,8 @@ message LiveSettingsV2Request {
     string indexName = 1;
     // Live settings to merge into existing live settings, or unset to get current live settings
     IndexLiveSettings liveSettings = 2;
+    // When set to true, live settings changes are only applied to the local node. These changes are ephemeral, so will not persist through a restart. Also, the live settings returned in the response will contain the local overrides only when this flag is true.
+    bool local = 3;
 }
 
 message LiveSettingsV2Response {
@@ -1062,6 +1064,8 @@ message IndexLiveSettings {
     google.protobuf.Int32Value defaultTerminateAfter = 13;
     // Merge precopy would be stopped after this time, or 0 for no checks, default: 0
     google.protobuf.UInt64Value maxMergePreCopyDurationSec = 14;
+    // Collect and publish additional index metrics, which may be more expensive in terms of volume, memory and/or compute, default: false
+    google.protobuf.BoolValue verboseMetrics = 15;
 }
 
 message IndexStateInfo {

clientlib/src/main/proto/yelp/nrtsearch/search.proto

@@ -263,6 +263,22 @@ message GeoPointQuery {
     google.type.LatLng point = 2; // point used to query whether the polygon contains it.
 }
 
+// Polygon defined by a list of geo points
+message Polygon {
+    // Points defining the polygon, conforming to the https://geojson.org/ standard. The polygon must not be self-crossing, otherwise may result in unexpected behavior. Polygons cannot cross the 180th meridian. Instead, use two polygons: one on each side.
+    repeated google.type.LatLng points = 1;
+    // Specify holes in the polygon. Hole polygons cannot themselves contain holes.
+    repeated Polygon holes = 2;
+}
+
+// A query that matches documents with geo points within polygons
+message GeoPolygonQuery {
+    // Field in the document to query
+    string field = 1;
+    // Geo polygons to search for containing points
+    repeated Polygon polygons = 2;
+}
+
 // A query that matches documents which contain a value for a field.
 message ExistsQuery {
     string field = 1; // Field in the document to query
@@ -363,6 +379,7 @@ enum QueryType {
     MATCH_PHRASE_PREFIX = 18;
     PREFIX = 19;
     CONSTANT_SCORE_QUERY = 20;
+    GEO_POLYGON = 21;
 }
 
 // Defines a full query consisting of a QueryNode which may be one of several types.
@@ -392,6 +409,7 @@ message Query {
         MatchPhrasePrefixQuery matchPhrasePrefixQuery = 21;
         PrefixQuery prefixQuery = 22;
         ConstantScoreQuery constantScoreQuery = 23;
+        GeoPolygonQuery geoPolygonQuery = 24;
     }
 }
 

docs/queries/geo_polygon.rst

@@ -0,0 +1,19 @@
+Geo-Polygon Query
+==========================
+
+A query that matches documents with geo point within a defined set of polygons.
+
+Polygon definitions must conform to the `GeoJson <https://geojson.org/>`_ standard.
+Polygons must not be self-crossing, otherwise may result in unexpected behavior.
+Polygons cannot cross the 180th meridian. Instead, use two polygons: one on each side.
+
+Proto definition:
+
+.. code-block::
+
+   message GeoPolygonQuery {
+       // Field in the document to query
+       string field = 1;
+       // Geo polygons to search for containing points
+       repeated Polygon polygons = 2;
+   }