Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use FieldDefCreatorContext for field creation #778

Merged
merged 2 commits into from
Nov 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@
public class AtomFieldDef extends TextBaseFieldDef implements Sortable {
private static final Analyzer keywordAnalyzer = new KeywordAnalyzer();

public AtomFieldDef(String name, Field requestField) {
super(name, requestField);
public AtomFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@

/** Field class for 'BOOLEAN' field type. */
public class BooleanFieldDef extends IndexableFieldDef<Boolean> implements TermQueryable {
protected BooleanFieldDef(String name, Field requestField) {
super(name, requestField, Boolean.class);
protected BooleanFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, Boolean.class);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,11 @@ public class ContextSuggestFieldDef extends IndexableFieldDef<Void> {
/**
* @param name name of field
* @param requestField field definition from grpc request
* @param context creation context
*/
protected ContextSuggestFieldDef(String name, Field requestField) {
super(name, requestField, Void.class);
protected ContextSuggestFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, Void.class);
this.indexAnalyzer = this.parseIndexAnalyzer(requestField);
this.searchAnalyzer = this.parseSearchAnalyzer(requestField);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,9 @@ private static DateTimeFormatter createDateTimeFormatter(String dateTimeFormat)
}
}

public DateTimeFieldDef(String name, Field requestField) {
super(name, requestField, Instant.class);
public DateTimeFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, Instant.class);
dateTimeFormat = requestField.getDateTimeFormat();
dateTimeFormatter = createDateTimeFormatter(dateTimeFormat);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@
/** Field class for 'DOUBLE' field type. */
public class DoubleFieldDef extends NumberFieldDef<Double> {

public DoubleFieldDef(String name, Field requestField) {
super(name, requestField, DOUBLE_PARSER, Double.class);
public DoubleFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, DOUBLE_PARSER, context, Double.class);
}

@Override
Expand Down
26 changes: 22 additions & 4 deletions src/main/java/com/yelp/nrtsearch/server/field/FieldDefCreator.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.yelp.nrtsearch.server.grpc.FieldType;
import com.yelp.nrtsearch.server.plugins.FieldTypePlugin;
import com.yelp.nrtsearch.server.plugins.Plugin;
import com.yelp.nrtsearch.server.state.GlobalState;
import com.yelp.nrtsearch.server.utils.StructValueTransformer;
import java.util.HashMap;
import java.util.Map;
Expand All @@ -34,6 +35,13 @@ public class FieldDefCreator {

private final Map<String, FieldDefProvider<? extends FieldDef>> fieldDefMap = new HashMap<>();

/**
* Addition context for field definition creation.
*
* @param config service configuration
*/
public record FieldDefCreatorContext(NrtsearchConfig config) {}

public FieldDefCreator(NrtsearchConfig configuration) {
register("ATOM", AtomFieldDef::new);
register("TEXT", TextFieldDef::new);
Expand All @@ -52,12 +60,12 @@ public FieldDefCreator(NrtsearchConfig configuration) {
// completely registered.
register(
"VIRTUAL",
(name, field) -> {
(name, field, context) -> {
throw new UnsupportedOperationException("Virtual fields should be created directly");
});
register(
"RUNTIME",
(name, field) -> {
(name, field, context) -> {
throw new UnsupportedOperationException("Runtime fields should be created directly");
});
register("VECTOR", VectorFieldDef::createField);
Expand All @@ -72,7 +80,7 @@ public FieldDefCreator(NrtsearchConfig configuration) {
* @param field grpc request field definition
* @return field definition
*/
public FieldDef createFieldDef(String name, Field field) {
public FieldDef createFieldDef(String name, Field field, FieldDefCreatorContext context) {
String type;
if (field.getType().equals(FieldType.CUSTOM)) {
type =
Expand All @@ -86,7 +94,17 @@ public FieldDef createFieldDef(String name, Field field) {
if (provider == null) {
throw new IllegalArgumentException("Invalid field type: " + type);
}
return provider.get(name, field);
return provider.get(name, field, context);
}

/**
* Create a new {@link FieldDefCreatorContext} instance.
*
* @param globalState global state
* @return new context instance
*/
public static FieldDefCreatorContext createContext(GlobalState globalState) {
return new FieldDefCreatorContext(globalState.getConfiguration());
}

private void register(Map<String, FieldDefProvider<? extends FieldDef>> fieldDefs) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ public interface FieldDefProvider<T extends FieldDef> {
*
* @param name field name
* @param requestField field request definition
* @param context creation context
* @return concrete {@link FieldDef} instance for field
*/
T get(String name, Field requestField);
T get(String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context);
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@
/** Field class for 'FLOAT' field type. */
public class FloatFieldDef extends NumberFieldDef<Float> {

public FloatFieldDef(String name, Field requestField) {
super(name, requestField, FLOAT_PARSER, Float.class);
public FloatFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, FLOAT_PARSER, context, Float.class);
}

@Override
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/com/yelp/nrtsearch/server/field/IdFieldDef.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@
/** Field class for defining '_ID' fields which are used to update documents */
public class IdFieldDef extends IndexableFieldDef<String> implements TermQueryable {

protected IdFieldDef(String name, Field requestField) {
super(name, requestField, String.class);
protected IdFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, String.class);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,14 @@ public enum FacetValueType {
*
* @param name name of field
* @param requestField field definition from grpc request
* @param context creation context
* @param docValuesObjectClass class of doc values object
*/
protected IndexableFieldDef(
String name, Field requestField, Class<? super T> docValuesObjectClass) {
String name,
Field requestField,
FieldDefCreator.FieldDefCreatorContext context,
Class<? super T> docValuesObjectClass) {
super(name);

validateRequest(requestField);
Expand Down Expand Up @@ -116,7 +120,7 @@ protected IndexableFieldDef(
for (Field field : requestField.getChildFieldsList()) {
checkChildName(field.getName());
String childName = getName() + IndexState.CHILD_FIELD_SEPARATOR + field.getName();
FieldDef fieldDef = FieldDefCreator.getInstance().createFieldDef(childName, field);
FieldDef fieldDef = FieldDefCreator.getInstance().createFieldDef(childName, field, context);
if (!(fieldDef instanceof IndexableFieldDef)) {
throw new IllegalArgumentException("Child field is not indexable: " + childName);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@
/** Field class for 'INT' field type. */
public class IntFieldDef extends NumberFieldDef<Integer> {

public IntFieldDef(String name, Field requestField) {
super(name, requestField, INT_PARSER, Integer.class);
public IntFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, INT_PARSER, context, Integer.class);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,9 @@

/** Field class for 'LAT_LON' field type. */
public class LatLonFieldDef extends IndexableFieldDef<GeoPoint> implements Sortable, GeoQueryable {
public LatLonFieldDef(String name, Field requestField) {
super(name, requestField, GeoPoint.class);
public LatLonFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, GeoPoint.class);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@
/** Field class for 'LONG' field type. */
public class LongFieldDef extends NumberFieldDef<Long> {

public LongFieldDef(String name, Field requestField) {
super(name, requestField, LONG_PARSER, Long.class);
public LongFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, LONG_PARSER, context, Long.class);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,9 @@ protected NumberFieldDef(
String name,
Field requestField,
Function<String, Number> fieldParser,
FieldDefCreator.FieldDefCreatorContext context,
Class<T> docValuesClass) {
super(name, requestField, docValuesClass);
super(name, requestField, context, docValuesClass);
this.fieldParser = fieldParser;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,9 @@ public class ObjectFieldDef extends IndexableFieldDef<Struct> {
private final Gson gson;
private final boolean isNestedDoc;

protected ObjectFieldDef(String name, Field requestField) {
super(name, requestField, Struct.class);
protected ObjectFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, Struct.class);
this.isNestedDoc = requestField.getNestedDoc();
gson = new GsonBuilder().serializeNulls().create();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@

public class PolygonfieldDef extends IndexableFieldDef<Struct> implements PolygonQueryable {

protected PolygonfieldDef(String name, Field requestField) {
super(name, requestField, Struct.class);
protected PolygonfieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, Struct.class);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,18 @@ public abstract class TextBaseFieldDef extends IndexableFieldDef<String>
private final int ignoreAbove;

/**
* Field constructor. Uses {@link IndexableFieldDef#IndexableFieldDef(String, Field, Class)} to do
* common initialization, then sets up analyzers. Analyzers are parsed through calls to the
* protected methods {@link #parseIndexAnalyzer(Field)} and {@link #parseSearchAnalyzer(Field)}.
* Field constructor. Uses {@link IndexableFieldDef#IndexableFieldDef(String, Field,
* FieldDefCreator.FieldDefCreatorContext, Class)} to do common initialization, then sets up
* analyzers. Analyzers are parsed through calls to the protected methods {@link
* #parseIndexAnalyzer(Field)} and {@link #parseSearchAnalyzer(Field)}.
*
* @param name field name
* @param requestField field definition from grpc request
* @param context creation context
*/
protected TextBaseFieldDef(String name, Field requestField) {
super(name, requestField, String.class);
protected TextBaseFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, String.class);
indexAnalyzer = parseIndexAnalyzer(requestField);
searchAnalyzer = parseSearchAnalyzer(requestField);
eagerFieldGlobalOrdinals = requestField.getEagerFieldGlobalOrdinals();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@

/** Field class for 'TEXT' field type. */
public class TextFieldDef extends TextBaseFieldDef {
public TextFieldDef(String name, Field requestField) {
super(name, requestField);
public TextFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context);
}

@Override
Expand Down
27 changes: 18 additions & 9 deletions src/main/java/com/yelp/nrtsearch/server/field/VectorFieldDef.java
Original file line number Diff line number Diff line change
Expand Up @@ -208,12 +208,14 @@ private static KnnVectorsFormat getHnswScalarQuantizedVectorsFormat(
*
* @param name name of field
* @param field field definition from grpc request
* @param context creation context
* @return new VectorFieldDef
*/
static VectorFieldDef<?> createField(String name, Field field) {
static VectorFieldDef<?> createField(
String name, Field field, FieldDefCreator.FieldDefCreatorContext context) {
return switch (field.getVectorElementType()) {
case VECTOR_ELEMENT_FLOAT -> new FloatVectorFieldDef(name, field);
case VECTOR_ELEMENT_BYTE -> new ByteVectorFieldDef(name, field);
case VECTOR_ELEMENT_FLOAT -> new FloatVectorFieldDef(name, field, context);
case VECTOR_ELEMENT_BYTE -> new ByteVectorFieldDef(name, field, context);
default ->
throw new IllegalArgumentException("Invalid field type: " + field.getVectorElementType());
};
Expand All @@ -222,10 +224,15 @@ static VectorFieldDef<?> createField(String name, Field field) {
/**
* @param name name of field
* @param requestField field definition from grpc request
* @param context creation context
* @param docValuesClass class of doc values object
*/
protected VectorFieldDef(String name, Field requestField, Class<T> docValuesClass) {
super(name, requestField, docValuesClass);
protected VectorFieldDef(
String name,
Field requestField,
FieldDefCreator.FieldDefCreatorContext context,
Class<T> docValuesClass) {
super(name, requestField, context, docValuesClass);
this.vectorDimensions = requestField.getVectorDimensions();
if (isSearchable()) {
VectorSearchType vectorSearchType = getSearchType(requestField.getVectorIndexingOptions());
Expand Down Expand Up @@ -322,8 +329,9 @@ public Query getKnnQuery(KnnQuery knnQuery, Query filterQuery) {

/** Field class for 'FLOAT' vector field type. */
public static class FloatVectorFieldDef extends VectorFieldDef<FloatVectorType> {
public FloatVectorFieldDef(String name, Field requestField) {
super(name, requestField, FloatVectorType.class);
public FloatVectorFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, FloatVectorType.class);
}

@Override
Expand Down Expand Up @@ -462,8 +470,9 @@ public void validateVectorForSearch(float[] vector) {

/** Field class for 'BYTE' vector field type. */
public static class ByteVectorFieldDef extends VectorFieldDef<ByteVectorType> {
public ByteVectorFieldDef(String name, Field requestField) {
super(name, requestField, ByteVectorType.class);
public ByteVectorFieldDef(
String name, Field requestField, FieldDefCreator.FieldDefCreatorContext context) {
super(name, requestField, context, ByteVectorType.class);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package com.yelp.nrtsearch.server.index;

import com.google.common.annotations.VisibleForTesting;
import com.yelp.nrtsearch.server.field.FieldDefCreator;
import com.yelp.nrtsearch.server.grpc.Field;
import com.yelp.nrtsearch.server.grpc.IndexLiveSettings;
import com.yelp.nrtsearch.server.grpc.IndexSettings;
Expand Down Expand Up @@ -87,7 +88,10 @@ public synchronized void load() throws IOException {
stateInfo = fixIndexName(stateInfo, indexName);
UpdatedFieldInfo updatedFieldInfo =
FieldUpdateUtils.updateFields(
new FieldAndFacetState(), Collections.emptyMap(), stateInfo.getFieldsMap().values());
new FieldAndFacetState(),
Collections.emptyMap(),
stateInfo.getFieldsMap().values(),
FieldDefCreator.createContext(globalState));
currentState =
createIndexState(stateInfo, updatedFieldInfo.fieldAndFacetState, liveSettingsOverrides);
}
Expand Down Expand Up @@ -203,7 +207,8 @@ public synchronized String updateFields(List<Field> fields) throws IOException {
FieldUpdateUtils.updateFields(
currentState.getFieldAndFacetState(),
currentState.getIndexStateInfo().getFieldsMap(),
fields);
fields,
FieldDefCreator.createContext(globalState));
IndexStateInfo updatedStateInfo =
replaceFields(currentState.getIndexStateInfo(), updatedFieldInfo.fields);
ImmutableIndexState updatedIndexState =
Expand Down
Loading
Loading