feat(weaviategh-327): set custom backup location (sync)

bevzzz · Dec 7, 2024 · 88524c5 · 88524c5
1 parent 99593e9
commit 88524c5
Show file tree

Hide file tree

Showing 8 changed files with 193 additions and 7 deletions.
diff --git a/src/main/java/io/weaviate/client/v1/backup/api/BackupCanceler.java b/src/main/java/io/weaviate/client/v1/backup/api/BackupCanceler.java
@@ -1,5 +1,9 @@
 package io.weaviate.client.v1.backup.api;
 
+import java.net.URISyntaxException;
+
+import org.apache.hc.core5.net.URIBuilder;
+
 import io.weaviate.client.Config;
 import io.weaviate.client.base.BaseClient;
 import io.weaviate.client.base.ClientResult;
@@ -16,6 +20,8 @@
 public class BackupCanceler extends BaseClient<Void> implements ClientResult<Void> {
   private String backend;
   private String backupId;
+  private String bucket;
+  private String path;
 
   public BackupCanceler(HttpClient client, Config config) {
     super(client, config);
@@ -26,6 +32,16 @@ public BackupCanceler withBackend(String backend) {
     return this;
   }
 
+  public BackupCanceler withBucket(String bucket) {
+    this.bucket = bucket;
+    return this;
+  }
+
+  public BackupCanceler withPath(String path) {
+    this.path = path;
+    return this;
+  }
+
   public BackupCanceler withBackupId(String backupId) {
     this.backupId = backupId;
     return this;
@@ -38,7 +54,15 @@ public Result<Void> run() {
   }
 
   private String path() {
-    return String.format("/backups/%s/%s", backend, backupId);
+    String base = String.format("/backups/%s/%s", backend, backupId);
+    try {
+      return new URIBuilder(base)
+      .addParameter("bucket", bucket)
+      .addParameter("path", path)
+      .toString();
+    } catch (URISyntaxException e) {
+      return base;
+    }
   }
 }
 
diff --git a/src/main/java/io/weaviate/client/v1/backup/api/BackupCreateStatusGetter.java b/src/main/java/io/weaviate/client/v1/backup/api/BackupCreateStatusGetter.java
@@ -1,6 +1,11 @@
 package io.weaviate.client.v1.backup.api;
 
 import io.weaviate.client.v1.backup.model.BackupCreateStatusResponse;
+
+import java.net.URISyntaxException;
+
+import org.apache.hc.core5.net.URIBuilder;
+
 import io.weaviate.client.Config;
 import io.weaviate.client.base.BaseClient;
 import io.weaviate.client.base.ClientResult;
@@ -12,6 +17,8 @@ public class BackupCreateStatusGetter extends BaseClient<BackupCreateStatusRespo
 
   private String backend;
   private String backupId;
+  private String bucket;
+  private String path;
 
   public BackupCreateStatusGetter(HttpClient httpClient, Config config) {
     super(httpClient, config);
@@ -27,6 +34,16 @@ public BackupCreateStatusGetter withBackupId(String backupId) {
     return this;
   }
 
+  public BackupCreateStatusGetter withBucket(String bucket) {
+    this.bucket = bucket;
+    return this;
+  }
+
+  public BackupCreateStatusGetter withPath(String path) {
+    this.path = path;
+    return this;
+  }
+
   @Override
   public Result<BackupCreateStatusResponse> run() {
     return new Result<>(statusCreate());
@@ -37,6 +54,14 @@ Response<BackupCreateStatusResponse> statusCreate() {
   }
 
   private String path() {
-    return String.format("/backups/%s/%s", backend, backupId);
+    String base = String.format("/backups/%s/%s", backend, backupId);
+    try {
+      return new URIBuilder(base)
+      .addParameter("bucket", bucket)
+      .addParameter("path", path)
+      .toString();
+    } catch (URISyntaxException e) {
+      return base;
+    }
   }
 }
diff --git a/src/main/java/io/weaviate/client/v1/backup/api/BackupCreator.java b/src/main/java/io/weaviate/client/v1/backup/api/BackupCreator.java
@@ -150,6 +150,10 @@ public static class BackupCreateConfig {
     Integer chunkSize;
     @SerializedName("CompressionLevel")
     String compressionLevel;
+    @SerializedName("Bucket")
+    String bucket;
+    @SerializedName("Path")
+    String path;
   }
 
   public interface BackupCompression {

diff --git a/src/main/java/io/weaviate/client/v1/backup/api/BackupRestoreStatusGetter.java b/src/main/java/io/weaviate/client/v1/backup/api/BackupRestoreStatusGetter.java
@@ -1,6 +1,11 @@
 package io.weaviate.client.v1.backup.api;
 
 import io.weaviate.client.v1.backup.model.BackupRestoreStatusResponse;
+
+import java.net.URISyntaxException;
+
+import org.apache.hc.core5.net.URIBuilder;
+
 import io.weaviate.client.Config;
 import io.weaviate.client.base.BaseClient;
 import io.weaviate.client.base.ClientResult;
@@ -12,6 +17,8 @@ public class BackupRestoreStatusGetter extends BaseClient<BackupRestoreStatusRes
 
   private String backend;
   private String backupId;
+  private String bucket;
+  private String path;
 
   public BackupRestoreStatusGetter(HttpClient httpClient, Config config) {
     super(httpClient, config);
@@ -27,6 +34,16 @@ public BackupRestoreStatusGetter withBackupId(String backupId) {
     return this;
   }
 
+  public BackupRestoreStatusGetter withBucket(String bucket) {
+    this.bucket = bucket;
+    return this;
+  }
+
+  public BackupRestoreStatusGetter withPath(String path) {
+    this.path = path;
+    return this;
+  }
+
   @Override
   public Result<BackupRestoreStatusResponse> run() {
     return new Result<>(statusRestore());
@@ -37,6 +54,14 @@ Response<BackupRestoreStatusResponse> statusRestore() {
   }
 
   private String path() {
-    return String.format("/backups/%s/%s/restore", backend, backupId);
+    String base = String.format("/backups/%s/%s/restore", backend, backupId);
+    try {
+      return new URIBuilder(base)
+      .addParameter("bucket", bucket)
+      .addParameter("path", path)
+      .toString();
+    } catch (URISyntaxException e) {
+      return base;
+    }
   }
 }
diff --git a/src/main/java/io/weaviate/client/v1/backup/api/BackupRestorer.java b/src/main/java/io/weaviate/client/v1/backup/api/BackupRestorer.java
@@ -150,5 +150,9 @@ private static class BackupRestore {
   public static class BackupRestoreConfig {
     @SerializedName("CPUPercentage")
     Integer cpuPercentage;
+    @SerializedName("Bucket")
+    String bucket;
+    @SerializedName("Path")
+    String path;
   }
 }
diff --git a/src/test/java/io/weaviate/integration/client/WeaviateVersion.java b/src/test/java/io/weaviate/integration/client/WeaviateVersion.java
@@ -3,12 +3,12 @@
 public class WeaviateVersion {
 
   // docker image version
-  public static final String WEAVIATE_IMAGE = "1.27.0";
+  public static final String WEAVIATE_IMAGE = "stable-v1.28-ac93b01";
 
   // to be set according to weaviate docker image
-  public static final String EXPECTED_WEAVIATE_VERSION = "1.27.0";
+  public static final String EXPECTED_WEAVIATE_VERSION = "1.28.0";
   // to be set according to weaviate docker image
-  public static final String EXPECTED_WEAVIATE_GIT_HASH = "6c571ff";
+  public static final String EXPECTED_WEAVIATE_GIT_HASH = "ac93b01";
 
   private WeaviateVersion() {
   }

diff --git a/src/test/java/io/weaviate/integration/client/backup/ClientBackupTest.java b/src/test/java/io/weaviate/integration/client/backup/ClientBackupTest.java
@@ -3,7 +3,6 @@
 import io.weaviate.client.Config;
 import io.weaviate.client.WeaviateClient;
 import io.weaviate.client.base.Result;
-import io.weaviate.client.v1.async.WeaviateAsyncClient;
 import io.weaviate.client.v1.backup.api.BackupCreator;
 import io.weaviate.client.v1.backup.api.BackupRestorer;
 import io.weaviate.client.v1.backup.model.BackupCreateResponse;
@@ -118,6 +117,43 @@ public void shouldCreateAndRestoreBackupWithoutWaiting() throws InterruptedExcep
       supplierRestoreResult, supplierRestoreStatusResult, supplierDeleteClass, createSupplierGQLOfClass(), backupId);
   }
 
+  @Test
+  public void shouldCreateAndRestoreBackupWithDynamicLocation() throws InterruptedException {
+    String bucket = "test-bucket"; // irrelevant for "filesystem" backend, here only to illustrate
+    String path = "/custom/backup/location";
+
+    Supplier<Result<BackupCreateResponse>> supplierCreateResult = () -> client.backup().creator()
+      .withIncludeClassNames(BackupTestSuite.CLASS_NAME_PIZZA)
+      .withBackend(BackupTestSuite.BACKEND)
+      .withBackupId(backupId)
+      .withConfig(BackupCreator.BackupCreateConfig.builder().bucket(bucket).path(path).build())
+      .run();
+    Supplier<Result<BackupCreateStatusResponse>> supplierCreateStatusResult = () -> client.backup().createStatusGetter()
+      .withBackend(BackupTestSuite.BACKEND)
+      .withBackupId(backupId)
+      .withBucket(bucket)
+      .withPath(path)
+      .run();
+    Supplier<Result<Boolean>> supplierDeleteClass = () -> client.schema().classDeleter()
+      .withClassName(BackupTestSuite.CLASS_NAME_PIZZA)
+      .run();
+    Supplier<Result<BackupRestoreResponse>> supplierRestoreResult = () -> client.backup().restorer()
+      .withIncludeClassNames(BackupTestSuite.CLASS_NAME_PIZZA)
+      .withBackend(BackupTestSuite.BACKEND)
+      .withBackupId(backupId)
+      .withConfig(BackupRestorer.BackupRestoreConfig.builder().bucket(bucket).path(path).build())
+      .run();
+    Supplier<Result<BackupRestoreStatusResponse>> supplierRestoreStatusResult = () -> client.backup().restoreStatusGetter()
+      .withBackend(BackupTestSuite.BACKEND)
+      .withBackupId(backupId)
+      .withBucket(bucket)
+      .withPath(path)
+      .run();
+
+    BackupTestSuite.testCreateWithDynamicLocation(supplierCreateResult, supplierCreateStatusResult,
+      supplierRestoreResult, supplierRestoreStatusResult, supplierDeleteClass, createSupplierGQLOfClass(), backupId, bucket, path);
+  }
+
   @Test
   public void shouldCreateAndRestore1Of2Classes() {
     Supplier<Result<BackupCreateResponse>> supplierCreateResult = () -> client.backup().creator()

diff --git a/src/test/java/io/weaviate/integration/tests/backup/BackupTestSuite.java b/src/test/java/io/weaviate/integration/tests/backup/BackupTestSuite.java
@@ -6,6 +6,7 @@
 import static org.assertj.core.api.InstanceOfAssertFactories.CHAR_SEQUENCE;
 import static org.junit.Assume.assumeTrue;
 
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
@@ -102,6 +103,73 @@ public static void testCreateAndRestoreBackupWithWaiting(Supplier<Result<BackupC
       .returns(null, BackupRestoreStatusResponse::getError);
   }
 
+  public static void testCreateWithDynamicLocation(Supplier<Result<BackupCreateResponse>> supplierCreate,
+                                            Supplier<Result<BackupCreateStatusResponse>> supplierCreateStatus,
+                                            Supplier<Result<BackupRestoreResponse>> supplierRestore,
+                                            Supplier<Result<BackupRestoreStatusResponse>> supplierRestoreStatus,
+                                            Supplier<Result<Boolean>> supplierDeleteClass,
+                                            Function<String, Result<GraphQLResponse>> supplierGQLOfClass,
+                                            String backupId, String bucket, String path) throws InterruptedException {
+    assertThatAllPizzasExist(supplierGQLOfClass);
+    String wantFullPath = Paths.get(path, backupId).toString();
+
+    Result<BackupCreateResponse> createResult = supplierCreate.get();
+    assertThat(createResult.getError()).as("create backup").isNull();
+    assertThat(createResult.getResult()).isNotNull()
+      .returns(backupId, BackupCreateResponse::getId)
+      .returns(wantFullPath, BackupCreateResponse::getPath).as("path in BackupCreateResponse");
+
+    // Wait until created
+    Result<BackupCreateStatusResponse> createStatusResult;
+    while (true) {
+      createStatusResult = supplierCreateStatus.get();
+
+      assertThat(createStatusResult.getError()).as("check backup creation status").isNull();
+      assertThat(createStatusResult.getResult()).isNotNull()
+        .returns(backupId, BackupCreateStatusResponse::getId)
+        .returns(wantFullPath, BackupCreateStatusResponse::getPath)
+        .extracting(BackupCreateStatusResponse::getStatus)
+        .isIn(CreateStatus.STARTED, CreateStatus.TRANSFERRING, CreateStatus.TRANSFERRED, CreateStatus.SUCCESS);
+
+      if (CreateStatus.SUCCESS.equals(createStatusResult.getResult().getStatus())) {
+        break;
+      }
+      Thread.sleep(100);
+    }
+
+    // Delete all data to then restore it from backup.
+    Result<Boolean> delete = supplierDeleteClass.get();
+    assertThat(delete.getError()).as("drop Pizza collection").isNull();
+    assertThat(delete.getResult()).isTrue();
+
+
+    Result<BackupRestoreResponse> restoreResult = supplierRestore.get();
+    assertThat(restoreResult.getError()).as("restore from backup").isNull();
+    assertThat(restoreResult.getResult()).isNotNull()
+      .returns(backupId, BackupRestoreResponse::getId)
+      .returns(wantFullPath, BackupRestoreResponse::getPath);
+
+    // Wait until restored
+    Result<BackupRestoreStatusResponse> restoreStatusResult;
+    while (true) {
+      restoreStatusResult = supplierRestoreStatus.get();
+
+      assertThat(restoreStatusResult.getError()).as("get restore status").isNull();
+      assertThat(restoreStatusResult.getResult()).isNotNull()
+        .returns(backupId, BackupRestoreStatusResponse::getId)
+        .returns(wantFullPath, BackupRestoreStatusResponse::getPath)
+        .extracting(BackupRestoreStatusResponse::getStatus)
+        .isIn(RestoreStatus.STARTED, RestoreStatus.TRANSFERRING, RestoreStatus.TRANSFERRED, RestoreStatus.SUCCESS);
+
+      if (RestoreStatus.SUCCESS.equals(restoreStatusResult.getResult().getStatus())) {
+        break;
+      }
+      Thread.sleep(100);
+    }
+
+    assertThatAllPizzasExist(supplierGQLOfClass);
+  }
+
   public static void testCreateAndRestoreBackupWithoutWaiting(Supplier<Result<BackupCreateResponse>> supplierCreate,
                                                               Supplier<Result<BackupCreateStatusResponse>> supplierCreateStatus,
                                                               Supplier<Result<BackupRestoreResponse>> supplierRestore,