Skip to content

Commit

Permalink
Make sure search functionality is restored immediately upon rollover …
Browse files Browse the repository at this point in the history
…recovery
  • Loading branch information
yrodiere committed Oct 20, 2023
1 parent 509b6f9 commit 7238a5f
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 31 deletions.
52 changes: 29 additions & 23 deletions src/main/java/io/quarkus/search/app/indexing/Rollover.java
Original file line number Diff line number Diff line change
Expand Up @@ -218,35 +218,17 @@ public void addActualIndex(String name, boolean isWrite) {
}
}

public boolean hasMultipleIndexes() {
return allAliasedIndexes.size() > 1;
}

Set<String> allAliasedIndexes() {
public Set<String> allAliasedIndexes() {
return Collections.unmodifiableSet(allAliasedIndexes);
}

Set<String> writeAliasedIndexes() {
return Collections.unmodifiableSet(writeAliasedIndexes);
}

Set<String> readAliasedIndexes() {
public Set<String> readAliasedIndexes() {
return Collections.unmodifiableSet(readAliasedIndexes);
}

public Set<String> extraIndexes() {
Set<String> extra = new HashSet<>(allAliasedIndexes);
// We keep only the oldest write index,
// which hopefully should allow a rollover to start.
if (!writeAliasedIndexes.isEmpty()) {
extra.remove(writeAliasedIndexes.iterator().next());
}
// Failing that, we keep only one index, and hope for the best.
else if (!allAliasedIndexes.isEmpty()) {
extra.remove(allAliasedIndexes.iterator().next());
}
return extra;
}
}

private static void commitAll(RestClient client, Gson gson, List<IndexRolloverResult> rollovers) {
Expand Down Expand Up @@ -286,16 +268,40 @@ private static void rollbackAll(RestClient client, Gson gson, List<IndexRollover
private static boolean recoverInconsistentAliases(RestClient client, Gson gson,
Collection<GetAliasedResult> aliased) {
List<GetAliasedResult> inconsistentList = aliased.stream()
.filter(GetAliasedResult::hasMultipleIndexes)
.filter(a -> a.allAliasedIndexes.size() > 1)
.toList();
if (inconsistentList.isEmpty()) {
return false;
}
List<String> inconsistentNames = inconsistentList.stream().map(r -> r.index.hibernateSearchName()).toList();
Log.infof("Recovering index aliases for %s", inconsistentNames);
try {
changeAliasesAtomically(client, gson, inconsistentList, inconsistent -> inconsistent.extraIndexes().stream()
.map(indexName -> aliasAction("remove_index", Map.of("index", indexName))));
changeAliasesAtomically(client, gson, inconsistentList, inconsistent -> {
Set<String> extraIndexes = new HashSet<>(inconsistent.allAliasedIndexes);
String indexToKeep;
// We keep only the oldest read index,
// which hopefully should restore a complete index with the ability to search.
if (!inconsistent.readAliasedIndexes.isEmpty()) {
indexToKeep = inconsistent.readAliasedIndexes.iterator().next();
}
// Failing that, we keep only one write, and hope for the best.
else {
indexToKeep = inconsistent.allAliasedIndexes.iterator().next();
}
extraIndexes.remove(indexToKeep);
JsonObject restoreIndexToKeepAsWriteIndex = aliasAction("add", Map.of(
"index", indexToKeep,
"alias", inconsistent.index.writeName(),
"is_write_index", "true"));
JsonObject restoreIndexToKeepAsReadIndex = aliasAction("add", Map.of(
"index", indexToKeep,
"alias", inconsistent.index.readName(),
"is_write_index", "false"));
return Stream.concat(
Stream.of(restoreIndexToKeepAsWriteIndex, restoreIndexToKeepAsReadIndex),
extraIndexes.stream()
.map(indexName -> aliasAction("remove_index", Map.of("index", indexName))));
});
} catch (RuntimeException | IOException e) {
throw new IllegalStateException("Failed to recover index aliases for " + inconsistentNames + ": " + e.getMessage(),
e);
Expand Down
20 changes: 12 additions & 8 deletions src/test/java/io/quarkus/search/app/indexing/RolloverTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -154,23 +154,27 @@ void recover() {
// Simulate a JVM crash: the rollover does not get closed
// We restart... and try to recover
assertThat(Rollover.recoverInconsistentAliases(searchMapping)).isTrue();
// Immediately after recovery, search may not work, but write will
assertWriteTargetsIndex(2);
// After recovery and throughout indexing, search must continue to work,
// and target the index most likely to contain (complete) data.
assertSearchWorksAndTargetsIndex(1);
assertWriteTargetsIndex(1);

// Then we will reindex, triggering another rollover...
searchMapping.scope(Guide.class).schemaManager().createIfMissing();
assertWriteTargetsIndex(2);
assertSearchWorksAndTargetsIndex(1);
assertWriteTargetsIndex(1);

try (Rollover rollover = Rollover.start(searchMapping)) {
assertWriteTargetsIndex(3);
assertSearchWorksAndTargetsIndex(1);
assertWriteTargetsIndex(2);

rollover.commit();
assertSearchWorksAndTargetsIndex(3);
assertWriteTargetsIndex(3);
assertSearchWorksAndTargetsIndex(2);
assertWriteTargetsIndex(2);
}
// And all is good in the end!
assertSearchWorksAndTargetsIndex(3);
assertWriteTargetsIndex(3);
assertSearchWorksAndTargetsIndex(2);
assertWriteTargetsIndex(2);
}

}

0 comments on commit 7238a5f

Please sign in to comment.