From d9577fa3b4c3ecfd1ed9689d9fc6cc4cf24ded74 Mon Sep 17 00:00:00 2001 From: Chris Riccomini Date: Tue, 1 Nov 2016 20:58:20 -0700 Subject: [PATCH 01/56] Fix Docker integration test to create retriever.tar --- kcbq-connector/test/integrationtest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kcbq-connector/test/integrationtest.sh b/kcbq-connector/test/integrationtest.sh index 03aaf7c0a..a65ae7038 100755 --- a/kcbq-connector/test/integrationtest.sh +++ b/kcbq-connector/test/integrationtest.sh @@ -233,6 +233,7 @@ CONNECT_DOCKER_IMAGE='kcbq/connect' CONNECT_DOCKER_NAME='kcbq_test_connect' cp "$BASE_DIR"/../../bin/tar/kcbq-connector-*-confluent-dist.tar "$DOCKER_DIR/connect/kcbq.tar" +cp "$BASE_DIR"/../../bin/tar/kcbq-connector-*-confluent-dist.tar "$DOCKER_DIR/connect/retriever.tar" cp "$KCBQ_TEST_KEYFILE" "$DOCKER_DIR/connect/key.json" if ! dockerimageexists "$CONNECT_DOCKER_IMAGE"; then From 30f29c57bfa21fdeae6e26abe6660531681f9de6 Mon Sep 17 00:00:00 2001 From: Hongyi Wang Date: Fri, 27 Apr 2018 13:42:09 -0700 Subject: [PATCH 02/56] release version 1.1.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 5d7b7a41b..c0e9c4b03 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=com.wepay.kcbq sourceCompatibility=1.8 -version=1.1.0-SNAPSHOT +version=1.1.0 From bc20a83492e666a2234d6bbc65e87994025f3c4c Mon Sep 17 00:00:00 2001 From: Joy Gao Date: Mon, 30 Jul 2018 14:05:27 -0700 Subject: [PATCH 03/56] Merge pull request #119 from jgao54/close-executor Add executor shutdown when BigQuerySinkTask is stopped --- .../connect/bigquery/BigQuerySinkTask.java | 41 +++++++------ .../bigquery/BigQuerySinkTaskTest.java | 59 +++++++++++++------ 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java index edec55aaa..f72098ebb 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java @@ -21,46 +21,36 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.TableId; - +import com.google.common.annotations.VisibleForTesting; import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; - import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; - import com.wepay.kafka.connect.bigquery.convert.RecordConverter; import com.wepay.kafka.connect.bigquery.convert.SchemaConverter; - import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; - import com.wepay.kafka.connect.bigquery.utils.PartitionedTableId; import com.wepay.kafka.connect.bigquery.utils.TopicToTableResolver; import com.wepay.kafka.connect.bigquery.utils.Version; - import com.wepay.kafka.connect.bigquery.write.batch.KCBQThreadPoolExecutor; import com.wepay.kafka.connect.bigquery.write.batch.TableWriter; import com.wepay.kafka.connect.bigquery.write.row.AdaptiveBigQueryWriter; import com.wepay.kafka.connect.bigquery.write.row.BigQueryWriter; import com.wepay.kafka.connect.bigquery.write.row.SimpleBigQueryWriter; - +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.consumer.OffsetAndMetadata; - import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; - import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.sink.SinkRecord; import org.apache.kafka.connect.sink.SinkTask; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import java.util.Set; -import java.util.concurrent.LinkedBlockingQueue; - /** * A {@link SinkTask} used to translate Kafka Connect {@link SinkRecord SinkRecords} into BigQuery * {@link RowToInsert RowToInserts} and subsequently write them to BigQuery. @@ -79,6 +69,7 @@ public class BigQuerySinkTask extends SinkTask { private TopicPartitionManager topicPartitionManager; private KCBQThreadPoolExecutor executor; + private static final int EXECUTOR_SHUTDOWN_TIMEOUT_SEC = 30; public BigQuerySinkTask() { testBigQuery = null; @@ -146,6 +137,7 @@ private RowToInsert getRecordRow(SinkRecord record) { @Override public void put(Collection records) { + logger.info("Putting {} records in the sink.", records.size()); // create tableWriters Map tableWriterBuilders = new HashMap<>(); @@ -241,7 +233,20 @@ public void start(Map properties) { @Override public void stop() { - logger.trace("task.stop()"); + try { + executor.shutdown(); + executor.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT_SEC, TimeUnit.SECONDS); + } catch (InterruptedException e) { + logger.warn("{} active threads are still executing tasks {}s after shutdown is signaled.", + executor.getActiveCount(), EXECUTOR_SHUTDOWN_TIMEOUT_SEC); + } finally { + logger.trace("task.stop()"); + } + } + + @VisibleForTesting + int getTaskThreadsActiveCount() { + return executor.getActiveCount(); } @Override diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java index 579616266..d58a8b076 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java @@ -18,16 +18,6 @@ */ -import static junit.framework.TestCase.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyObject; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; @@ -42,7 +32,10 @@ import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; -import com.wepay.kafka.connect.bigquery.retrieve.MemorySchemaRetriever; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.RejectedExecutionException; + import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; @@ -50,15 +43,20 @@ import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.sink.SinkRecord; import org.apache.kafka.connect.sink.SinkTaskContext; - import org.junit.BeforeClass; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Captor; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class BigQuerySinkTaskTest { private static SinkTaskPropertiesFactory propertiesFactory; @@ -384,11 +382,34 @@ public void testVersion() { assertNotNull(new BigQuerySinkTask().version()); } - // Doesn't do anything at the moment, but having this here will encourage tests to be written if - // the stop() method ever does anything significant - @Test + // Existing tasks should succeed upon stop is called. New tasks should be rejected once task is stopped. + @Test(expected = RejectedExecutionException.class) public void testStop() { - new BigQuerySinkTask().stop(); + final String dataset = "scratch"; + final String topic = "test_topic"; + + Map properties = propertiesFactory.getProperties(); + properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); + properties.put(BigQuerySinkConfig.DATASETS_CONFIG, String.format(".*=%s", dataset)); + + BigQuery bigQuery = mock(BigQuery.class); + SinkTaskContext sinkTaskContext = mock(SinkTaskContext.class); + InsertAllResponse insertAllResponse = mock(InsertAllResponse.class); + + when(bigQuery.insertAll(anyObject())).thenReturn(insertAllResponse); + when(insertAllResponse.hasErrors()).thenReturn(false); + + BigQuerySinkTask testTask = new BigQuerySinkTask(bigQuery, null); + testTask.initialize(sinkTaskContext); + testTask.start(properties); + testTask.put(Collections.singletonList(spoofSinkRecord(topic))); + + assertEquals(1, testTask.getTaskThreadsActiveCount()); + testTask.stop(); + assertEquals(0, testTask.getTaskThreadsActiveCount()); + verify(bigQuery, times(1)).insertAll(any(InsertAllRequest.class)); + + testTask.put(Collections.singletonList(spoofSinkRecord(topic))); } /** From ebb575027e7e9ca80cd7be0ba012f3fa0e396e8c Mon Sep 17 00:00:00 2001 From: nicolasguyomar Date: Thu, 20 Jun 2019 19:26:03 +0200 Subject: [PATCH 04/56] Prevent context update after/while rebalancing (#161) This call might have been needed before, but it is no longer required --- .../kafka/connect/bigquery/BigQuerySinkTask.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java index f72098ebb..b06dff5d6 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java @@ -89,21 +89,10 @@ public void flush(Map offsets) { } catch (InterruptedException err) { throw new ConnectException("Interrupted while waiting for write tasks to complete.", err); } - updateOffsets(offsets); topicPartitionManager.resumeAll(); } - /** - * This really doesn't do much and I'm not totally clear on whether or not I need it. - * But, in the interest of maintaining old functionality, here we are. - */ - private void updateOffsets(Map offsets) { - for (Map.Entry offsetEntry : offsets.entrySet()) { - context.offset(offsetEntry.getKey(), offsetEntry.getValue().offset()); - } - } - private PartitionedTableId getRecordTable(SinkRecord record) { TableId baseTableId = topicsToBaseTableIds.get(record.topic()); From 96846d6fcc848fc23ca4cef574cfa601f3454fe2 Mon Sep 17 00:00:00 2001 From: Chris Riccomini Date: Wed, 26 Jun 2019 16:57:15 -0700 Subject: [PATCH 05/56] release version 1.1.2 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index c0e9c4b03..301b5832f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=com.wepay.kcbq sourceCompatibility=1.8 -version=1.1.0 +version=1.1.2 From 4514a2feb307c77fd23f23d2715d086a28094a7e Mon Sep 17 00:00:00 2001 From: Chris Riccomini Date: Mon, 1 Jul 2019 11:23:23 -0700 Subject: [PATCH 06/56] release version 1.2.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 5d7b7a41b..711f84905 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=com.wepay.kcbq sourceCompatibility=1.8 -version=1.1.0-SNAPSHOT +version=1.2.0 From 31ffaa83cd32375dd236f0b122ca891a1d696659 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 10 Sep 2020 16:15:53 -0400 Subject: [PATCH 07/56] Bump to next snapshot version for 1.1.x --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 301b5832f..1e4ec7128 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=com.wepay.kcbq sourceCompatibility=1.8 -version=1.1.2 +version=1.1.3-SNAPSHOT From 9cedaeb6c4e74d08be9dd04c6b53883a32b0dfd2 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 10 Sep 2020 16:16:28 -0400 Subject: [PATCH 08/56] Bump to next snapshot version for 1.2.x --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 711f84905..43ce10728 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=com.wepay.kcbq sourceCompatibility=1.8 -version=1.2.0 +version=1.2.1-SNAPSHOT From 4931d3f62ce898612a4206043caca81613488172 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 10 Sep 2020 16:21:37 -0400 Subject: [PATCH 09/56] Bump to next snapshot version for 1.3.x --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 0a3633cb8..a3370a016 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ group=com.wepay.kcbq -version=1.3.0-SNAPSHOT +version=1.3.1-SNAPSHOT From 3f3a3292f6c3de4834bdee436901cd0d5e7af0a0 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 10 Sep 2020 16:22:22 -0400 Subject: [PATCH 10/56] Bump to next snapshot version for 1.4.x --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 7540460ac..cdf539f0e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ group=com.wepay.kcbq -version=1.4.1 +version=1.4.2-SNAPSHOT From dbbe599e1c48a91b61b94a406ea6ad2ac5883345 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 10 Sep 2020 16:22:57 -0400 Subject: [PATCH 11/56] Bump to next snapshot version for 1.5.x --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 23cee74fe..a18c42fd8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ group=com.wepay.kcbq -version=1.5.2 +version=1.5.3-SNAPSHOT From 603e3a3f60ac0084c6fcb5e64e4d7d9a355b524a Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 10 Sep 2020 16:23:45 -0400 Subject: [PATCH 12/56] Bump to next snapshot version for 1.6.x --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 7e259101b..5e43a8f51 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ group=com.wepay.kcbq -version=1.6.5 +version=1.6.6-SNAPSHOT From d9f602dbdf45d2e058e3852c52e1fd2d3cb33eb0 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 23 Sep 2020 08:37:00 -0400 Subject: [PATCH 13/56] GH-32: Switch from Gradle to Maven for build tool (#34) --- .gitignore | 4 +- .travis.yml | 31 -- README.md | 10 +- build.gradle | 315 ---------------- codecov.yml | 43 --- gradle.properties | 3 - gradle/wrapper/gradle-wrapper.jar | Bin 53556 -> 0 bytes gradle/wrapper/gradle-wrapper.properties | 6 - gradlew | 164 --------- kcbq-api/pom.xml | 54 +++ kcbq-confluent/pom.xml | 96 +++++ .../src/test/resources/log4j.properties | 14 + kcbq-connector/logos/BigQuery.png | Bin 0 -> 6210 bytes kcbq-connector/logos/confluent.png | Bin 0 -> 3156 bytes kcbq-connector/pom.xml | 182 +++++++++ .../it/BigQueryConnectorIntegrationTest.java | 0 .../bigquery/it/utils/TableClearer.java | 0 .../src/test/resources/log4j.properties | 14 + .../test/docker/connect/connect-docker.sh | 3 +- kcbq-connector/test/integrationtest.sh | 29 +- pom.xml | 346 ++++++++++++++++++ settings.gradle | 1 - 22 files changed, 727 insertions(+), 588 deletions(-) delete mode 100644 .travis.yml delete mode 100644 build.gradle delete mode 100644 codecov.yml delete mode 100644 gradle.properties delete mode 100644 gradle/wrapper/gradle-wrapper.jar delete mode 100644 gradle/wrapper/gradle-wrapper.properties delete mode 100755 gradlew create mode 100644 kcbq-api/pom.xml create mode 100644 kcbq-confluent/pom.xml create mode 100644 kcbq-confluent/src/test/resources/log4j.properties create mode 100644 kcbq-connector/logos/BigQuery.png create mode 100644 kcbq-connector/logos/confluent.png create mode 100644 kcbq-connector/pom.xml rename kcbq-connector/src/{integration-test => test}/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java (100%) rename kcbq-connector/src/{integration-test => test}/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java (100%) create mode 100644 kcbq-connector/src/test/resources/log4j.properties create mode 100644 pom.xml delete mode 100644 settings.gradle diff --git a/.gitignore b/.gitignore index 0fd805b49..6b35bdd71 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ bin/ build/ +target/ *.class *.jar *.tar +*.zip .gradle @@ -21,5 +23,5 @@ build/ key.json test.conf -kcbq-connector/src/integration-test/resources/test.properties +kcbq-connector/src/test/resources/test.properties kcbq-connector/test/docker/connect/properties/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2ce0f9e86..000000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: java -sudo: true -dist: trusty -group: edge - -jdk: - - oraclejdk8 - - openjdk8 - -matrix: - fast_finish: true - -script: - - gradle test - -after_success: - - if [ -e ./gradlew ]; then ./gradlew jacocoTestReport; else gradle jacocoTestReport; fi - - bash <(curl -s https://codecov.io/bash) - -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ - -cache: - directories: - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - -notifications: - email: - - open-source@wepay.com diff --git a/README.md b/README.md index 91aad2b20..2fad00b76 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ save the properties file. Once you get more familiar with the connector, you might want to revisit the `connector.properties` file and experiment with tweaking its settings. -### Building and Extracting a Tarball +### Building and Extracting a Confluent Hub archive If you haven't already, move into the repository's top-level directory: @@ -46,16 +46,16 @@ If you haven't already, move into the repository's top-level directory: $ cd /path/to/kafka-connect-bigquery/ ``` -Begin by creating a tarball of the connector with the Confluent Schema Retriever included: +Begin by creating Confluent Hub archive of the connector with the Confluent Schema Retriever included: ```bash -$ ./gradlew clean confluentTarBall +$ mvn clean package -DskipTests ``` And then extract its contents: ```bash -$ mkdir bin/jar/ && tar -C bin/jar/ -xf bin/tar/kcbq-connector-*-confluent-dist.tar +$ mkdir -p bin/jar/ && cp kcbq-connector/target/components/packages/wepay-kafka-connect-bigquery-*/wepay-kafka-connect-bigquery-*/lib/*.jar bin/jar/ ``` ### Setting-Up Background Processes @@ -193,7 +193,7 @@ cannot occupy more than one line** (this inconvenience is due to limitations in Console Producer, and may be addressed in future commits). To specify data verification, add a new JUnit test to the file -`src/integration-test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java`. +`src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java`. Rows that are retrieved from BigQuery in the test are only returned as _Lists_ of _Objects_. The names of their columns are not tracked. Construct a _List_ of the _Objects_ that you expect to be stored in the test's BigQuery table, retrieve the actual _List_ of _Objects_ stored via a call to diff --git a/build.gradle b/build.gradle deleted file mode 100644 index 901987f9d..000000000 --- a/build.gradle +++ /dev/null @@ -1,315 +0,0 @@ -// BEGIN ALL PROJECTS // -allprojects { - apply plugin: 'java' -} -// END ALL PROJECTS - -// BEGIN SUBPROJECTS // -subprojects { subproject -> - - apply plugin: 'maven' - apply plugin: 'signing' - apply plugin: 'checkstyle' - apply plugin: 'findbugs' - apply plugin: 'idea' - apply plugin: 'eclipse' - - jar.baseName = subproject.name - - [compileJava, compileTestJava].each { - it.options.compilerArgs << '-Xlint:unchecked' - } - - checkstyle { - configFile = file("${rootDir}/config/checkstyle/google_checks.xml") - toolVersion = '6.18' - } - - task javadocJar(type: Jar) { - classifier = 'javadoc' - from javadoc - } - - task sourcesJar(type: Jar) { - classifier = 'sources' - from sourceSets.main.allSource - } - - task tarBall(type: Tar) { - classifier = 'dist' - baseName = subproject.name - from subproject.configurations.runtime - from jar - } - - signing { - sign configurations.archives - required { - gradle.taskGraph.hasTask('uploadArchives') - } - } - - uploadArchives { - repositories { - mavenDeployer { - beforeDeployment { - MavenDeployment deployment -> signing.signPom(deployment) - } - - repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { - authentication(userName: findProperty('ossrhUsername') ?: '', password: findProperty('ossrhPassword') ?: '') - } - - snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { - authentication(userName: findProperty('ossrhUsername') ?: '', password: findProperty('ossrhPassword') ?: '') - } - - pom.project { - - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - - scm { - connection 'scm:git:git://github.com/wepay/kafka-connect-bigquery.git' - developerConnection 'scm:git:ssh://github.com:wepay/kafka-connect-bigquery.git' - url 'https://github.com/wepay/kafka-connect-bigquery' - } - - developers { - developer { - id 'C0urante' - name 'Chris Egerton' - email 'fearthecellos@gmail.comw' - } - - developer { - id 'moirat' - name 'Moira Tagle' - email 'moirat@wepay.com' - } - } - } - } - } - } -} -// END SUBPROJECTS - -// BEGIN ROOT PROJECT -task confluentTarBall(type: Tar) { - destinationDir = file("${rootDir}/bin/tar/") - baseName = 'kcbq-connector' - classifier = 'confluent-dist' - with project(':kcbq-connector').tarBall - with project(':kcbq-confluent').tarBall - exclude 'jackson-core-2.1.3.jar' // Runtime conflicts occur if this is left in; thankfully, 2.5.4 is compatible -} - -clean { - delete "${rootDir}/bin/" -} -// END ROOT PROJECT - -// BEGIN INDIVIDUAL PROJECTS -project(':kcbq-connector') { - apply plugin: 'jacoco' - - jar { - manifest { - attributes 'Implementation-Title': 'Kafka Connect BigQuery Connector', - 'Implementation-Version': version - - } - } - - repositories { - mavenCentral() - } - - sourceSets { - integrationTest { - java { - compileClasspath += main.output - runtimeClasspath += main.output - srcDir file('src/integration-test/java') - } - resources.srcDir file('src/integration-test/resources') - } - } - - task integrationTestPrep(type: JavaExec) { - main = 'com.wepay.kafka.connect.bigquery.it.utils.TableClearer' - classpath = sourceSets.integrationTest.runtimeClasspath - args findProperty('kcbq_test_keyfile') ?: '' - args findProperty('kcbq_test_project') ?: '' - args findProperty('kcbq_test_dataset') ?: '' - if (findProperty('kcbq_test_tables') != null) - args findProperty('kcbq_test_tables').split(' ') - } - - task integrationTest(type: Test) { - testClassesDir = sourceSets.integrationTest.output.classesDir - classpath = sourceSets.integrationTest.runtimeClasspath - } - - compileIntegrationTestJava.options.compilerArgs << '-Xlint:unchecked' - - configurations { - integrationTestCompile.extendsFrom testCompile - integrationTestRuntime.extendsFrom testRuntime - } - - javadoc { - options.links 'http://docs.oracle.com/javase/8/docs/api/' - options.links 'http://docs.confluent.io/3.2.0/connect/javadocs/' - options.links 'http://googlecloudplatform.github.io/google-cloud-java/0.2.7/apidocs/' - options.links 'https://kafka.apache.org/0100/javadoc/' - options.links 'https://avro.apache.org/docs/1.8.1/api/java/' - } - - jacocoTestReport { - reports { - html.destination "${buildDir}/reports/jacoco/" - xml.enabled true - } - } - - dependencies { - compile project(':kcbq-api') - - compile group: 'com.google.cloud', name: 'google-cloud', version: '0.25.0-alpha' - compile group: 'com.google.auth', name: 'google-auth-library-oauth2-http', version: '0.9.0' - - compile group: 'io.debezium', name: 'debezium-core', version:'0.4.0' - - compile group: 'org.apache.kafka', name: 'connect-api', version: '1.0.0' - compile group: 'org.apache.kafka', name: 'kafka-clients', version: '1.0.0' - - compile group: 'org.slf4j', name: 'slf4j-api', version: '1.6.1' - compile group: 'org.slf4j', name: 'slf4j-simple', version: '1.6.1' - - testCompile group: 'junit', name: 'junit', version: '4.12' - testCompile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' - } - - artifacts { - archives javadocJar, sourcesJar, tarBall, rootProject.confluentTarBall - } - - uploadArchives { - repositories { - mavenDeployer { - pom.project { - name 'Kafka Connect BigQuery Connector' - packaging 'jar' - description 'A Kafka Connector used to load data into BigQuery' - url 'https://github.com/wepay/kafka-connect-bigquery' - } - } - } - } -} - -project('kcbq-api') { - jar { - manifest { - attributes 'Implementation-Title': 'Kafka Connect BigQuery API', - 'Implementation-Version': version - } - } - - repositories { - mavenCentral() - } - - javadoc { - options.links 'http://docs.oracle.com/javase/8/docs/api/' - options.links 'http://docs.confluent.io/3.2.0/connect/javadocs/' - } - - dependencies { - compile group: 'com.google.cloud', name: 'google-cloud', version: '0.25.0-alpha' - - compile group: 'org.apache.kafka', name: 'connect-api', version: '1.0.0' - } - - artifacts { - archives javadocJar, sourcesJar - } - - uploadArchives { - repositories { - mavenDeployer { - pom.project { - name 'Kafka Connect BigQuery Connector API' - packaging 'jar' - description 'A small API for the Kafka Connector used to load data into BigQuery' - url 'https://github.com/wepay/kafka-connect-bigquery' - } - } - } - } -} - -project('kcbq-confluent') { - jar { - manifest { - attributes 'Implementation-Title': 'Kafka Connect BigQuery Schema Registry Schema Retriever', - 'Implementation-Version': version - } - } - - repositories { - mavenCentral() - maven { - url 'http://packages.confluent.io/maven' - } - jcenter() - } - - javadoc { - options.links 'http://docs.oracle.com/javase/8/docs/api/' - options.links 'http://docs.confluent.io/3.2.0/connect/javadocs/' - } - - dependencies { - compile project(':kcbq-api') - - compile group: 'com.google.cloud', name: 'google-cloud', version: '0.25.0-alpha' - - compile group: 'io.confluent', name: 'kafka-connect-avro-converter', version: '3.2.0' - compile group: 'io.confluent', name: 'kafka-schema-registry-client', version: '3.2.0' - - compile group: 'org.apache.avro', name: 'avro', version: '1.8.1' - - compile group: 'org.apache.kafka', name: 'connect-api', version: '1.0.0' - compile group: 'org.apache.kafka', name: 'kafka-clients', version: '1.0.0' - - compile group: 'org.slf4j', name: 'slf4j-api', version: '1.6.1' - - testCompile group: 'junit', name: 'junit', version: '4.12' - testCompile group: 'org.mockito', name: 'mockito-core', version: '1.10.19' - } - - artifacts { - archives javadocJar, sourcesJar - } - - uploadArchives { - repositories { - mavenDeployer { - pom.project { - name 'Kafka Connect BigQuery Connector Schema Registry Schema Retriever' - packaging 'jar' - description 'A Schema Registry-based schema retriever for the Kafka Connector used to load data into BigQuery' - url 'https://github.com/wepay/kafka-connect-bigquery' - } - } - } - } -} -// END INDIVIDUAL PROJECTS diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index c644d5794..000000000 --- a/codecov.yml +++ /dev/null @@ -1,43 +0,0 @@ -codecov: - branch: master - bot: skyzyx - -coverage: - precision: 2 - round: down - range: "70...100" - - status: - project: - default: - target: auto - threshold: 1.25 - branches: - - master - - feature/* - - patch: - default: - target: auto - branches: - - master - - feature/* - - changes: - default: - branches: - - master - - feature/* - - ignore: - - config/.* - - gradle/.* - - test/.* - - .*/vendor/.* - -comment: - layout: "header, diff, changes, sunburst, uncovered, tree" - behavior: default - branches: - - master - - feature/* diff --git a/gradle.properties b/gradle.properties deleted file mode 100644 index 1e4ec7128..000000000 --- a/gradle.properties +++ /dev/null @@ -1,3 +0,0 @@ -group=com.wepay.kcbq -sourceCompatibility=1.8 -version=1.1.3-SNAPSHOT diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index ca78035ef0501d802d4fc55381ef2d5c3ce0ec6e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53556 zcmafaW3XsJ(%7|a+qP}nwr$(CZQFj=wr$(@UA(+xH(#=wO)^z|&iv@9neOWDX^nz3 zFbEU?00abpJ7cBo`loO)|22l7HMDRNfRDr(;s(%6He@B!R zl#>(_RaT*s6?>AMo|2KKrCWfNrlp#lo@-WOSZ3Zod7P#lmzMGa(ZwA{NHx8{)|HLtOGBmL<{ePk& z|0}Aylc9rysnh?l#3IPVtoSeL%3mP<&r3w?-R*4b4NXWG>5Od*ot=GSWT6Hb5JLAX zShc9#=!2lw!t#FMI}pFJc zw6Uj8`Bst|cD2?nsG(d*ZG#%NF?Y80v0PGQSJPsUg@n3BQIkW_dR~d>N{{*bSH}Pd zIWdTJ#iH#>%S&)$tqoH6b*V7fLp<>(xL_ji`jq2`%oD)~iD7`@hsO@Vy3*qM{u`G^ zc0*TD{z`zuUlxn}e`r+pbapYdRdBNZ%Pbd5Q|G@k4^Kf?7YkE67fWM97kj6FFrif0 z)*eX^!4Hihd~D&c(x5hVbJa`bB+7ol01GlU5|UB2N>+y7))3gd&fUa5@v;6n+Lq-3 z{Jl7)Ss;}F5czIs_L}Eunuojl?dWXn4q(#5iYPV+5*ifPnsS@1F)kK`O<80078hB& z!Uu$#cM=e$$6FUI2Uys(|$Fxqmy zG@_F97OGMH;TUgxma36@BQi`!B{e(ZeayiDo z;os4R9{50YQVC-ThdC9S{Ee)4ikHa8|X*ach%>dfECip|EPi!8S zDh{J&bjYD?EYtrlYx3Xq_Uu~2x$3X9ZT$tJ|15Qq|5LU8AycBUzy2x~OxU04i>D z9w@yRqlcbqC}2T_XT5eNHYx5)7rtz8{DE*J?o>>OiS)0JC!ZaB0JL-Ob1w)8zanZ< zR(Xiz3$ioy*%XQmL-bJnNfvE$rI2P~LX90G#gt4nb9mku*6S{mqFw`_kt{LAkj!x21fSFo(-^4px?_hH9-@XW8zqNrs(RYSX5R zn7kQuX>YGYLyM(G>^wtn&><_Q!~W27r537fQwZIqYL965<@&T|=xUF6c$g=5 z9B|kBeu>}r8R@-o3b!=}4_HG6sot1tgjjbmglPS~q)5GX6CU&gxsD0v9llaw7Bh7W zG`o>aya0{@c}L+Gw`1PRqcl6e6}@o3Bcd#mP)9H<2a|Wi{ZWqCzX%93IfRpvQ5Gba z7lEPC4fM4WC?*W3IpV-cRPh5Sc}Q>vS@2qu<+V(nS%!Sm&*^W!gSj)# z5h9&o{KIKp2kov&g`CP%-CqAqA#o0Mw?;q#0Dk{<4VeG4n2LHB+qgPgx|xbu+L#I& z8=E>i%Np7lnw$R9>ZhtnJ0P3l{ISg3VawG!KBZ_pvN2DYtK&W!-f06 z`*U{p=QkVw&*us(0Q^xhL0e%n5Ms&j;)%FBf*#J>kq82xOVpI4<0WK)`n9DXCuv$A zfn4!kd?3Iqh$3+WD+l&4vj>}m@*Jom+}vj&2m=KQGoVRm7M2KY7**ns0|M5px)Deh zez6~hUk1`@NgO%XoGXd)&6$_Hs|(2|X^7HUDkEtbwHV#1wRTpbb)rHlLu^njhFg9S zx+)}U8(USDXm>S%pp;a_Y<5>3i_Hp_vWwtzt5uj8ewqTFEE)E15)Wjvv?x}}8HMiX z;^3-OH85AzcV_0O-Exhrj`RpUZ;j$qjmZ|L#+*_US5`JV%8wqakxhD&XCpyuWo{N- z+bNS}p+afKlpHI>3VBBeq|G8boGeUaC)(Ru3u`YLW30>~)5=GL=sUjLgu65%VcPGs}PA z2_OLv=2)9Xm11f*FTt*o*yc8FG>4G~q{mOUX#}$!=u>KSGyX(=*}&rI;2K(U?Koxp z7F-pc*}}pO@m;7sff=FGTE4TA9ZNTRx%XWeaa|lx9o$qjHByj0HxuO5TvpM}CwTW> z#R=1vZp)76kO?#z;(>6Mu&gCwrlvRCVG_g8sMl;^DrH)&-*)v5ZHl3IWWpPi!|ZNQ z4&vdL!lWNaYH)lo!KJkFQfoCqF_@w-in(c2pNkpCKo6my8_yVs_Uj=zGVLKUT#^z^ z-)|f>)fuk#(@A>3(o0VqQ1$4+z_E9HCQ7R^ z30tu-(OIxDiiOEkGpXw&zReM}VP+C}bFAvU5%L?0cQ@?`fBSwH7!4o)d`OImPc+X< zrwk1#`^<8L8#>HOQb0pxt)HxXg%o|3x3nsPjSioaPqZ^lnSNOaJHg}1zqdDur0PoP zRVh{xV61JsNFuq`Xd6MtK*HtXN?NH20{)o}s_-I*YU7#=qn8b)kV`MS%A%ewrx<5I zY9{WpWlK^G^SP=5nvS-WEy+2%2}G?;#q01CSQ@%UJgw>}sHVEQip4`tToFyKHmwTV z-vWa!(`#8lj^drh)TLYVZLU!F!ak3OPw(qUajt(mO&u~ANUN%r3KUzV%k%|1=7Iat z5Pt`rL>P6u2G|qX<$)j~A0r2ZdE%y2n!@s>8}^KzEQEj6Kc?A%>r0ye>xB@wj|1Ob47`2EH4(rA(O{ zU}u2kj}N3&2?^3EQ{aT{?2g=~RLM;{)T7k%gI$^7qr`&%?-K{7Z|xhUKgd+!`-Yie zuE4Z_s?8kT>|npn6{66?E4$Pc2K(`?YTz3q(aigbu-ShRhKK|(f0cCh1&Q1?!Rr=v&a!K}wA-|$Gr{J~k~ z7@gS_x|i#V?>C5h_S4>+&Y9UC;Z@h2@kZgiJ|M%c)C38h@es^Y`p#a9|M_8mi3pR( z6*QJ0&b&7q+!3NCbBMs(x}XlEUyQp~0K9id;Wx1KycVf%ae(I8KJgjc!$0vE-NSwS zEu2^31P|2W6P)+j90blNtRJ5=DmAN?R}TD4!&z=N=@IeHhDTl-!_-e0hc?;+-;cCJ zm~zCBdd&GjPVt9?QcvkJQtf#Mv5mGLq7;pHYUils+`Yo8=kJB06UOcuYC;cMU2)oG zMH>rDE_p-R8=u3n)w%~+lE$>My@gq^RU(c_#Yk|`!Sjm$ug=Rfte#lnU+3im?EmV# zsQ)8&61KN9vov>gGIX)DxBI8_l58uFEQm1nXX|V=m@g=xsEFu>FsERj84_NVQ56PN z!biByA&vMXZd;f2LD`as@gWp{0NymGSG%BQYnYw6nfWRI`$p&Ub8b!_;Pjp%TsmXI zfGrv)2Ikh0e{6<_{jJk;U`7Zl+LFg){?(TM{#uQ_K{wp6!O_Bx33d!Brgr9~942)4 zchrS8Old{AF_&$zBx^bCTQ74ka9H84%F{rOzJ`rkJjSB_^^pZqe9`VQ^HyUpX_!ZA z+f0In>sw`>{d(L>oA+{4&zo5_^6t%TX0Gj0^M@u0@~^-f=4Gt9HMY&X&b`K%xjauF z8_!X>V|CrL;+a6gp zKd)6{;@wH+A{&U6?dAu>etSxBD)@5z;S~6%oQqH(uVW(Ajr>Dy{pPKUlD+ zFbjJ6c69Zum)+VkzfW(gW7%C{gU6X+a{LH?s2^BS64n$B%cf()0AWRUIbQPhQ|q|& z55=zLH=!8-f5HKjA|4`9M&54<=^^w{`bc~@pMec>@~;_k-6-b93So0uesmwYOL zmrx9lp%heN8h0j@P=!rO5=@h9UIZ^85wMay-2UO?xo>XOHLK<6Q|uyT6%*f4V!dYTC-$swh8fk{pCMlf5hw+9jV|?GlEBEAx zj#np5nqD`peZ6m5`&-xKetv((^8@xo*!!N3lmt=YUou<_xyn#yJp3Y#wf`tEP?IB4 z>Mq>31$Blx^|cr*L09CYlW3$Ek;PY`k@ToRobo6~q}E71Oxr##L$~JJ9_?1@As_if z`YlL&yDtoy733P&wytI4>Gd;vxHw2O@+@KgbPa)>3z8mMkyAS%Fna#8Sg!uWhMEubF;n{i3Ae4j{$p>dYj-^9?1ysjK~i0Q(4XUQE? zq8WLEcE@FsQ%hrS`3O$YbyPGkF6o;%&dxfHG?_n@Z&K4vR@ieBC{}cst~pIc4R0u& zj`QUL>5UQF@PgvVoBbRAtoQ_wyeeA9wsSN9mXX-dN^aFG=EB_B_b{U`BenI&D=;Fj zT!n`sy{aPu9YibsEpvrQ^0t(q&Inj%Pca%Yu&!K1ORT4wD6j-dc+{?5(JAouXgIy8 z%-H6Fbhd6%S=KCeIm`}PC!@`F>UKx&(#(Exk?s77w@&*`_tZ&sgzQ!_QK=DBnare8 z;)ocuEeZw)R1@{BuzGzIj$Z6EqM#s17Zv{q88!cq88!bXFpB=ZG^k$1C)OSWOnz4h zh&DA{Lx8q4*47TCo_gzx?MlHD(Bx{$87ha%T$XB*_{8uv@LhK>VV`UY=tPjwOandObAG0 z65^99S$7U)%^i%0Rnv*|IFjxg{!=`YHMJK^XV#j)p>*^S8FcuGV-BAwAU)a(e+)Wj z<=0$&0zB{usg@89sQBDI-|(HM1iz{8?zwn?5-k8jfM6Uf#vp^D4ozQhw#0tB@N(_V z5G#8|@Ta&(7#{whu<-X6VG66*t5~?Wlg0j8JGkpMEo%Sg1fExMxWXFTg2;1a+bNC~ zMiFaxTcU3ZKjv)V5kM}`LLzVunn%c$N*BoJj-NZ6`Q{g=3;*E#!f_{#*C?+ad~5zZ z=keRIuK5M;04KWI+Ycv(7YzExxp+b(xFaY3Z^kf3mPKNCd{OQbO%F%7nd8P(nBNon z_?lN|<`FF*oN)KZYNm_512Er;<8GEqpFWsK<1M&j{|B zo5C*08{%HJJyGfROq44Q!PMdxq^&J+j?ahYI=`%GLh<*U*BGQ36lvssxuhS-weUq^_|F7sRH2KqhQ2}MFKYfgn|}o{=of1QHP+(v0l0HYK}G+OiNO_D__5DAvd@{ul69am-m8ERsfZLSCNp9cTU% zmH*GrZ`geV`DBTGGoW+_>cFiEGR0sT5#0!Gq3u)$0>Q+2gNXQYFn7##$e~T?O6@UKnaPmHYrr;IL66 zpHCH6FCU(hv{CKW&}j6$b_zL?RWjo+BMls3=9G<#5Tzqzb=To%u9RQYw&j~}FJ@T0 zwqYi7d0bfhOvCF+KQ?e8GFX^6Wr;#sLd>z=9rOo+Sn!Gx#S!8{JZOiICy=>JL!*Db z?0=i<6a%%-Qb$_VMK#jDzwycH@RdM&ODTf(BM+(VE<)*OfvATsOZ?;*Z|+KHl#LYV zwB(~69*ivMM^es;_qv2a`F=yr7hG(h9F_QsJdxq1W);`Gg)XvElwdAOhjO9z zZr>li{sH_~k(_n9ib4ek0I-7t03iF%BB@~LVj<}4Y-(%tUl(nv+J`Z=I^xgjDynBP zN0jq=Yp@Y{EX@X*q%wsh^8JcPZT)X5xy=r1Yhrts;iZ@>npp;KAbS=u^ z7C^t_c%Z%wUF|lirC0D?_B+enX?Etl?DjuDbKmTMIivlD98rUKIU`CqV0Ocly#&IF zVJ8$a8*L_yNF&jX!-@&G+9c#)>ZeLLirXnS+DtWKjc8+nJ|uDRlm6xpN-+4*hewV+ zK>0BT%8ou*`H3UuqFuNnXC^;BIAixsF!~XP(TYBlVf14Qq4mS}s)|2ZF#71(dk7cV zj6Tw*_G9cDz}0~ zXB=I`eTPx>~gi%8(4o7@g1GNnp$hJ_%Mg1`VLZDvLJeHGr+zT1&yk_ z)dbBKq?T{~APy~$Nlig_@z&C!xIWPDo3m~uxHe!qrNb26;xt|ht-7c7np#s+cje~J zZ~taj5)DfMbEaGGQw!+3dN0G2S=fRaa3rl z7Osx|l1jjjIOhCoaPxPQt1`ZxtLxIkA`VmUHN|vTlJRWNz<2C9m^>k4usuSUG})b%|D<wP^rU?JNVjdb*1yWsZBE8HZC}Q5va#I zsBwfZp;FX)RpB3EoWZyd4Bs{TNmbQ{0Kzz-0SgBPl2=f6IWi{9_QZu%rTT_|l31Q_ zycR4qyR5Il(L|CofDAL(ez5(KmRFo@U&>^{qK1eq^QMA`FZE_d6`2iXL�H$uJM z5b&uBBCA_wdL?^xw19P_F!l$XIUCIG0(Uznb36A^l7CS!0R}%?tUXwj0HwXsK4>8v zWE@fGYQ(q1F-!wr2v#*y7wWza-i5khqjQYc`6WHxhz85!iY%{Wb*z~zziBKpL+~P= z5yWtFJwj0m!TPZcI??gVUnnQOG_s*FMi>bxB)n3@mOYG~$F8 zl_Xm}#nH#t1z6WP61iq!0zB{Jh{o+KuI9xVM*x|TC7COi#tnUn_I;MA4`P!sk}}W2 z$gGS}m_|3n{2>Nib`R}0pU=AR9)Uh6;G*?1T2ZSB5`4PjrO>Bt2=i6u=qr=bN)Jho zMV?Wtn1yFbC*Io^`FFE6o|ePN6GG{zD$mtIc0OSsefFkNdF;nI-VNeuPS?6%IPVoN zZsFOKggP&tnTdglp;!r1nb~ME!H<>dW?N62A>Q1QI7WDZr;ehh?{L3L=pIMlpL9<- zCZ-fg1i?An;l=twL*C@`7quCoH<3MF6KapUt`yRJpF@_5T*SKkjpGkuc&h|H=`ud? z`ZbMU&m4ld%TU}+A+8V~1;8C{f84t#jj{05Rv(nfKmS(5<=Ac8!Twv+zNQ2KAo$N0 ztE8Q?i=mCpKTj(+=3sG#PuZ69xtt)EQ_E$H(y>G9(Tc1>K{$_6M z*(L~w^!?vvr`|bde{$}8^!2_!m&7A22>lTX_-4~b$zzFP^|OM2SO6_YC(5x3nDFZF zLEs;<=Rhe2kWFopSdxKt#+6GlvG$4b&}%<@1KN1(I;X?0JG+# zOZ+SI(Rz6pJnLxoojp_o=1!h~JgSvFTm#aA(MK;!EfdNVDQXa* z&OSYBpIIn<0tfRSotyL5B*mozW{+MLZ6NMLdlU~=0cuYk{B}v^W)@XIJ)rGX--$xE zOcvV!YR_%}tq!75cM%KJ4z>o<-#?T-I%Kk_LSFz{9lHk$0c_9Q_`|<#-aCblZ)o=E z*hH(RzI&AO5E03$9B2e^8%VO=Ic`s>OC%|BVCLoQQbv;^DMQ^Uw~-6%GO^F}H0Q~q z^f33U->p7+w08Mu`8u@@tTTdOW34aQ*zLPo3M*ZgM$1;R*;#AtJ6(i#%35VYXVR~_ zpR*$Hu4*h>k<4nGL6_ctd(c>3Fj`0BNeVt%XZj?1n3pFSWG&#xyR5p9Jv$6nTu7ep z?1&YWZQu<{`E%?dM-RU+EZMY2%EDea9xT>s>$*;qAlk-5oOIejvmMX=Dq4!!RUk=a zamTctj!;C0!kjqf;w{^1TIo=<;5h(Fc&cSFE^CdtNLq|vxH@9x>|8h1&ggl0X!ym_ zxDkU%TWQgqxL#tcz=HsPkx1(`m~!V*zIMr!EW@nJ8EsF5D1i?_3bVt6HC-~|(pC+o zolB0hY3Npl)MYwqOg)KHp8bH;7}-IT!ab|vHd#`jh;fZ<<}KC7PEI6)jPuAiRJGC5 z2&o+9RNmrt5uHY7Ei0NyCNA<4mLnKiFYNv_Zb#Nii3WTZ0arZ8AT4M0>{%QkfFKHD z$$+eh87@<>*<{1qeS%#EY7=9pnWpm2e2)YsTnSN=OZ;bh@jzvAJ7{9b^qHwKQXd&- z%P@H^nn=iub17MjB9)=GFUvK6%wfa84NFp5%?$!9s);AdXonKo1(r8TF-+CxrZNsr z&~Nv31)}ejFF>%}r3{F{mBb*6PpWF=m1;g?!&1Yw@g9xX(CztT)5@3!PJ$MraL?jJ zjIfepZ3R}0DTSdM7v5{g4CqqENzH&qX~|~OOAZ?k(03=3VqR=omosOJO0#<^kry}S zMOVziT*;@o#igZ%dH=|V33S4P3X#diBc9o-J2t^IYq9m{K7GEtHmM_yBtV6$dz7+GSDI~g-K~b{o`Ud#% za0>r2$Osa6KCfwq^?pc*f*-YeG33x$$Cz>r@k4A{>e&zlHn~AYPNFAkSGe@|SF%2qflcY{3Q}TP1xU;;lixI`{PI_{1MwPU# zb8@!|+^PX>d@Px~2o3tYZS<^mg8`s&^A%j$#_ecM)T0-=M6*JcsBjG$6!qH-)6k^r z=hP|(rciXq{A45YWNjc*3tE28s-&}Y*eX(?Dl3}SRu~$6>Iiz?;9=wGO3&_yuud9e zI;ydoyIqTk1TB7ZTT{o1+!@^A%5#rZX4&G?bC6Vjp}Q)V%s16{j$h#-0dMi5>oaC* zU7@wAR|uZ!g;*b6%$SP9WYJtzOSYZDh1c(z!EV*QKzo%BvfbkQv*RPPRQm&M)gPX{ zsGE;rsTtrJ$#Y-96Z*&W0@1o8i1XD}SJet-l%J+a?+-Q*x7&~$2T(*W!GkT;zTp0% zNA(Z6)VBxSak^X6;6eB5FV>%~$+vsI)VmXV3FrLDw`e5ziZ6n180=s3hq09zred)+ zgJxaVKHB88?P~L<=_F^?2OWvaMvl_Lf>sx1GE2t38EFH4*y%WGwX9|A`ZH11xDv-% z3(>w@i{-S_vscw(nT*5!zMm)OY9HA?0x+)$lY58XGTd?$B3bT8G>2Nx$&v++LtnP3 zw}ctz1peYD;s&U(-^Myl#2TRgMq>XF?%dT=NcS~K*x?!t!7>qNE z#XC*r*1Tmas=7$c($69)&0Q|gv4u14v;$|>JCPh{TE18`JLEk$4XUNT)N=8{H?x*& zvob>*k&1|Mkkd%B@&YU_Lcn6yuNS9U<3xC>F0xW3NJsSKU{z_OEIUWa!kVhos3p^e znKBiVqZGn&Zfiz_FCObw-B89YT-{>XtOQQPL1W`9eIoGH-yu`;QO593{jOJqGn?rW z=RZk&t9S(Xl|LZ(OCOgW*&y;4vV)EVx-q4}3kS|HZRW|V9K(LmDf^v;cNIA<6Xu;r zr&oQ^+#ynltMZM`QGV&B_LCdX;Ne^G^-p>$C`a&0*)GRI%e-E{tr+g{@f;iM4wUfPv7pnd_ccS(@ z4{d>u?2E(%@tJmuYw(j8bKAF*cbJo=l*&?B*~c9JD0L7D9LGrhr;Cdt zncS<5VKKJXK?NvGezTQjVUEao!!?}QQz%e#pJ`pN*=dEnReH3bA86g#Q&aLzn9ReZ zzJ$1Y2xzkQdOGVMvC7*9JIRk=IPkJQ2Q3hL%S@dl8N9sAYwsaPHJ_V#Ur9yFWa?cX zjz$+PT{j#E`o?A)2J@8F_`LjHqe`B}I=iKBH6G%zkONe{6sF|Z1v_YQ5&iJov>WGX zipwqW?lIMTBKC>nGA2tsNMx`5CdJY5t}Sz&K$ILDLDC^Pxs_SN&B&jwR}-G3CYZ?b zgKQIgD&Y5pU|OO#CgM zDGuh11j==SAiOZK7m6XE5XW7K(-=sL% zH&+Fz#zLnR(xemV8{F6vc-V`jR7;uVCP}E6Ih=qbmD+TbZ0%-$&Jvj$24?|h9`H!y zP_Tq~oX$EP6%+(9dat$vf8(7vrhU`tFbifgmbiJH(c??;^VknrH z0hsB`p0zIK60yzL%uq8HIxikY-MQKue-X0Bb=6c(wEk*{u0TF8t-_|Q3?O!7wDN;z z>J}_l#!p35Wa#!8&${i&4N1dhNxC7AoA!|VwT*p2*5ZBdic8_~ zkfY8g0D2OPVnL0=o~egN@WK#FU(X>U<#}TGn5vFj1{rPxmoMy%^)Wv?A{ASoTusuuqHD7a5BYf}yH8T5&ox(ckKBEO7Rd?Y?Lp&5oNE!c_F zq_zlC1$F{`-KoyC!}LT)RKJ8?u*ioiyHCbjkW@hWoNawAxb?(^dk1pHOkmE}1>J0> zG}DEB*XNnF=GEwAtr6@@RUF?=NFRWh9Yu~`=$C7-iLKM&68Z7$lSa2Q*@8# zr=^)HLw~**-4mMU9p_K_q(NUfgw!mT!&mU6UzRR3?O6+Kf?Bml+DG)4;NHTg#V->s zyl2!8bbaR#xq4a%wC5$AyIvN$3K^|=d2<_Bszp}&D?5ICjvp_Di}EDG=9VygTzAmMB#^O zss~=SJf03Zqu>_Z_sevE`Gw-k0H0vQK&)s_8m#@KSCn1IhS-8236Qy3u!>h&Myz`1Kd8B~HlYtAU=gA11kqTr1`MN9eyqp7elU7>IHRBL9eHY4UWJ;U)t{yN*Rm)~+ss$M3* zIi`3)<{@3Z1heF9@JR!C+xWC##A~Hh6;Jo%oqCK$fPG6;Q%&iwSVez+S&H&4Q3Lap zUzp_C?Bd3k@N0J(XK%I*Y8R~CI>_d(Na+h|_@M&n3!V+t$ONDV-MniLcA-)o=n`-A z<8ttu7TbY&f9C8tiFVKgy;}5p4$ktRr@!JYKa+g+S!26-yZ6r1b6BM82c`o(|AP?0 zWsdI&53A&;EqYJ|$mNdP4zuWK+h<-`H>2EvRYzSDeze~owhCzF^0Iu^xV^Sv!nqE-4@O&@C z!xw^61W&#Ioa2BSBx>;v{M8g!r2;OpS_^Wo%k?M z1ce90s~<)S-q0se_|)Ik!#!_j=fCxaOQcL`BqD`8@WsGWMqEx#v)r zTb_n1GZNvTYT}r9Ag$(i!8X6 zNU$YbD2sh6*}S%!#>qseXVzSBf>J|g&tP1*6;F(7o@z5yBV>-A-B7jDD$%}mKu=Sk zf%YTL_D!P3ujNo-A&!SXL@>`t8oeE<)7Iexa;)be(pOWnJo`y_%5?g?Bb{Z}ptE2I}2DbF^CCr)96 zZd?xW*TqH)B}#ln^QHMl0vFi9DB#20TVb)V^Qgcn0)Pn5QtC|S*aXu1d0YZVxclWn zla0V*_UL8ZB}?}GpxUEvE}5UU{g&yp2-u3POD?+vzbH_ZIN zRg;d~&1^c-`zGviyarVb*dbjO!waqeW4;Cq;S+k3wYM35$?xwUuWHYeBT!~ui^?u2 zDTZnl*=D}kWhrQysw44&$Nj-HI2T1J7ejOO7yPtWc&(=}{Xst2-Xpm5Hw^?R(nORl zSOwG`MxuD_>usNDbhm*wP?Gs$a<)_xk^J>MS8yA#9>Iynllll{WARg{G;EHXW5~Rm zL-|Z^83y%jy-5Zok}|{6-5&6+f3dejs1#g2J()gyET`p4#!=Gv&R=kKKGLVG{l$(k zuBnqP2gKL?<)D89(n(*PI=2Aj@{|2D7901rk8$xu|E<3{jctG{$?BJZ`OP_jqll%=o>SRg|iFp>7h4N6Qe#g*&gbN`CDKxlneuB#GKMN82a|&*-r|8(MUx|XCNs?v_@JrwJ}g0 z1b>lmV2^)q7zrPHc~=+}f7ci!e^K~w(iTHcLQ(?qQO+vdSOVfHybl9#9F<`NjAfiL zpzfSzYhGQp%_aHC$W(cOU0HnZBS5*)rKKjoVXk#yv8|-c70uVW{NZaZa+h72-E7fR zVcaym*Yi3l2bwmQgK^|i|uC9JmO6AKTOo5vSaE7!I z7ZHBuWomktl`=e+6bx-^L31&#i>t|oUVeMQkI}O>)vi3Otn+MRh-9msb!l8`zjS>e zMnz@@b3)gQ)5J>%)w9Zk?$$!iRb}du99&z~D;Ki_0S#o?vL)fjY*wm?^GxM${*Gun zIEbK*(gVC5#6>583s9<3>=)c3k{hbUdh)$UU|bAPFuY&}(krSDl(Zn43%S=hmgshs z=rhpKIIsC!BgObZ!2HuPa&6Q#rAL%7pzPV<=a#n$B&0YL-_V(;Nhr&F=vu37+#xim z{vkE!+&$}q(@;FxP`p?e9ZC z4vpX_#JUbq>_JIgbvIfvrRMIGnav%=hkdOyHPk2j&C_|64`1BE^$=?XOI`Or;6f`i z%+&w0(j-K^MUP-Qc|Xl$J1UgL%$O@>;R1MDR;90qh}(>`OjQIL#PO^Ud7^a} zKEP||e^%jto&@%3V@I!Aq8DlAuW`A;?t{==&x;q%Ah_q{ix0630P2@y;*klP4#WSD zaYvrc6eb!k*X9f+Blw4B+{c_A%nYIP2d0RBGh&eqBaZ_z#;*Yt=}#OjhOqCy=#yQI zhLnTKKJa9b`vB$(Ao&k6%Y3HIpu=gwm5)Ip7dYg$+zm3+8Nuv4&&&(s1N6d8d!kDL zlIe#s9t-S|d?E&24++OCMt$N4hjc`}+dEZx>O6oyo_|611-z}D z72Qwu`{x!>AM|UH_ypY=KYux@1-d~&Lm`*!P$2dQUO7(kmUGD(27|Z}pD-<%rw|?YSLpf58810bgRZon-0n3jtyb004^rTxa-a zKd7jOsj=&SJqSxx_cXv!#rz}NG-1cK6k?auMoCFSYP&ciI<=EVEUAn&zGAbORkS*B z%c8k{9kQ{32LVMvK~;o9gd!qZ+b(zk77BjX0nkOz|t%ZyQwv6Ar9!-%hi0EWRDop&s8J{t(y0 z909e1K0*rT`AAn#<;Vb(bB}h&+k}H;$ou5^)5N2{!G|CKe)3JY>CrILmm~o5W0!tN z9QZxM2S4Fvh-nIpfqDROrU(*+G56EtRg<3&eRzWdV<7qQ+Xp}&Vm}(thcbX3{5}<+k7`Q(^&cHM; zpl;S8UR>zsRN-u#ZSFLxXXd&w^ZzvKkH|Sx|QW;}y zwwjPUwZ>^iUL(>(T;Vp?Oug3rW|qX_4^=p`p$h~p-0jjdiZAZ8#u6qq`J`B(vzM0q zNULLZBad0hD+w7&%@y->WE`Y&H2F)MZLeV;-OxonwCUHW9SFHb;wf~iO&b;(Y@u? z4%$Tw*5v5}98V zAZ>y~BgD&16*=U&=dz6A*+(*dzh4#d=V|EhLBCRaXjJAGzl4-l>$eh+yQQ<~dAmqa zl9#Dzi85)r)=V+bZkEbESsx^rK}j9w%QKNhO3EVOuo4|as4O`0gg{%5M33={#iFwY zV;t7oFqNM>lkPhc4SLqt@NKudj9#nk@;Mm_B2%2BatkFH9*8KcQl|t{KtSjgY z*dyH1Y4R-;uFe>yuk6y09p9}tk*IiQ^&8^Sb@1RwZbDM_s%t=P>0%2-4+(#p&v01E za#7~6OOU}-)7YC^v^1Zg8OOp&zdawbSLKP_iyYi*wnEqBrE)tmr5bIJ9x3%`j7r}x zrGnd+LZ!r@`U&7y(%e?A*VWQee<0^6K6LGn9LX2e#T!d7ldXD>cKA|dyXwhakc>^Y zU|}vjw2zC)R^_3#xlE0`peQcn#`>Y_{xiPi0P;tf?S~YbRn&_m@tTckq9Zo#x#_-- zXdr7e1=gl};Kd#_?fo}C;+H;8`Jv}5%78(8)LH9o3C7p&40<_JO;wcAkjx!LfDGk8DQwau;V^g~l&8@j40GToR?g^-kw zg`U~VD4<;(?gO>o8QOw*o2eOY%b-hogBy+^-P~}9oIk8=OqN)mPV%ErQIVr$u9Zim zPWVp?=}kFPByX$Q9>3O3){Eu(Mmz!xX_{dUCp)ZOqg4dAitL=*7skIWF`qgcKR`=| z73~K%jpmF&%RNio5*}ZrrMQ@dS9P9qEzVREVS!Mjv5?wQ z$NUT#V;GsVUyHZuVn+B#;-QoqrCZjcW86wvJ2!mql*$(h9N|>;flzX+%cPISgz!D)|S2qu8H6sywRqb zH0|YusE-pxerVLq91EJ(4y$S#*5sVlS{7Q1Vm^3dsVzb!C&%owKGo#j+`M5C)`bgSG;KJ7N}V}!HM{-L%%=~hF|}OP z4B=oEPu$ARBWjggMLMW@qnJ2F=a@E5j$x(taAwVba*-i(rC~K~U~CT&AZ^_$pKLC_ zcrJm`yAp)aa#0pU5qG|83u#T|UXiQLGw56RvP9?Plv-;wZG0inQw`1tRbIDlZMG=$ zS|gNO>O<1ZoG2U9Lc!4dAc0qg5MG))j%e(Yjl)iQ)Ae*@?MLAFvMW%2jj zZ2vR`>O-0iRM!3s%B4PpaPN0j&1YI~KjGefFmdX8yi?5`G;JSPJLX19CW%R>L$-2l zg0ubJ)Vj=k4Sqv6*<&4k)JnT|?F343%AoH?&=Y+|^>*VWRx+B?3toG)Nif@!Q1Iad zAo=-XKjdoIpdAq?5jDKyD4h?#;w42Jw}jb;b*m9wl&veNO;Nd&u%acq5R)&6OCxD! zcTzK&>e)#3gsx=jR&3DNKxMOeUipkG=-Fjo@&fs9jJ;EIW!=8+orlHDoo3JJSd@`y+1I$tN#2dj6pE~%ELv|P#LU> zoiF2g3Sa$N)aTgCV{So-dAT@qt|W;9pT34JdcC5%fP$a_bA0s+=%|1Bqa8i?P%GQFXn@ny5sv z$hoFJZ8|eCPH#@tHZK+Tk_}5%!xkj!5;*zf_RumpDb~VeFVHCD+&r(RPP=$s%-meK zfpkJYx{;+d6gVYZPvz&>>KD{MD&A_eUz; z-J>?U)P~OOTL_uhm5ERMn+V;@p2SyC3*99lwtX+3|X>OZn3?WV`e1N zXMW#8K>SF|`4Jx?KQ_Q1E%qsv(Z^0Ie7$A+R*LA{#tw0PH|hO)PDff)ym7Y`Z*&E^ zDZ+Yc_Mo2gbbJf_&bLba=M&AU<83pI@xe zAfIp-=gbZ;@$sWxHKEQuk7E3cXJ^T7d}w9M9Z>>&r;O?BDyV5{s3_nYDCrkn+umNA zOZiEk0Wn2Ny@?YgUS$IccYX#1?rn3#Sd`=nY;)0h7|LD6 z4JU?z?sUhmpzmdYC~N~f`AmT&Mf)%bA!>^fQlb9wjItGcQk(q_d~vMLb==xB60|tB zEF;4Y&$XPOOxnP^N)nQpni)u`BLp{Cu{|h{TG373ctzG70Szai zdfAf((wJP2MV02XykIG=+?}sw7xYe%t{B6UaVTXMqI!xa^+=NHM?&0k*l~#_s6E4Q ze)jCi&R!#Bp-eV%!Th|L=U_jRTp9|PyePmbxDD~5)DLo3j)xuNDrB1@@7j4;1@$KI z^*3w#-=Vm@(fLKcGAtIFAS|eawsoXFid<^@6CwsQmC@&vsL}E_w*8+L5W71w3t^A!F zl?Lt|G9LC=8i4Gwb@DA@+6j_Ik?3s1w|^#r>AzP&-KkbuNJijd=jchdM4=1O>X)08 zKux(&W|)oV8+Rz6@XMlw3dvGNmfk3{DF$t5h*cZ3eq{q4TKgu1J`^u!)RrnAr7jXi zE+v{qGR{^f0gk4a7baDwfg;VSNLGH@$aO{Y&X>RdrQ|@vZEB2Igd-?QyEG`O^kZ8w zy)4Ycu&uY5osWQ{YPMF;Es_aEC@wWyCVHVEufUY#pd8om7#d$T)hG`-V-tnXBFJ*( zn^lHck;P1$k=Wq;AZ(qI6ugCD5*jA_21gs!uFjz*zZM<6srgenF)rCbeo%1*xT?fZ z2vyO1MWI!`SmoTHmLg4U81JUm*YJ%Y@;xzaF~{IC_pSR0M6DLd?BB4>FuvCtXo10OHYn7xB7?}dW9r^o3f0noO8z zF>xgry-GF@6OL`HwL930GNbNg_h<-BW7jz&8XTs|i)sx%VBH-Q#88$Icy+pX!RTK9 zcxw^A8AC{E;u3X*UM@Xm%5Zh}4W*!o2PTvgPls}qtCt*d^J&#!4AO+hLPy4-JZ;0} z)T!r7-3@^#<{=_gkS+&>QH>fC5Rq5jOx0K0-*8oJmN=xdepoqZA&PgVvptyZc<;W0 zX95C&fYzzwnx0%i22m7!auQA+@Zw=&)|kCx@Jg1AVo43 zIOTE=Td=~Y&Lg0d{(~LNCgF0hE^b-V8o3hgviLq-lg|e#AySvbG7Ir|PvIiGjR{X+ zv?YZl{&p>S#N{aQt$fC97*TabZKq+3|BUl zBFl@DF+;NCYxCAoK=CVxf{-T@@t@oJ~7q;_6QAcfWv6uFimU(pZO(^ zF-0ufSPgBLiQYW+*)U8s`<-|_N|@r9^hVDn@C2FKoQ+7sxSc7#yoFr0U# z{|=&N0M`8FhB)*yhb_{b-T^_m=Syi-sgDEWO zE3~Y^lESRO&!w-e?yzhJP2^EcEXmhm{^vN{o^&=(9mlO_jB{NS8<_S?B+k`|W5b8tCkk`ik! zP~h89#WaF*P$$MsOLBLn(4~TKt}W=VgxtUi9R(u{^I_s56?k)T2=0@3{ANXIJhj$1 zsop=_rnp7pnDsO_%p48jW7TsnZtN62+zodXtB-J_dq?mQYM3?SYMfCnZ&t9ZQ2iD< z%s+p%U9>l>s+z3c{<^B~NU2WnysqvAu(B6BSm2}-)mhB=P@bmuALR|h=r}|(Yk_Ld zuX-YtlQG&CU87jzYOT)lgk64hU*=LzTZYkbSx#1!+t#_VtPf!J*XxIbz7!^VP2&!f z$*=J6Lo)4DABzQsAIElQO5W@6#@P3G({;4-Pa$L6xcRq3uFsoqFWi7jS^IF~k-0Lu zxVf?^CFn-|oMv@(tH~H%C1qN^JXBO)Si|rLX%Faj^15i~>OA2)9`zw>p6#0-vw38w z%^KUDx&}Vh7|lSweto0PKO&?3qAF9EBr}9l>_qB=Tbxp(zu3ZPNJ$)AB=eC5uVL^5cMRB{MgKHK|1?ka5N82HCX*|`5o0^Kr*!6s(rJl$ zUi9}JvbAXx_uNlBK;!3`uKyRw>7UW_|3ai?sav_>E};Wga5TetCGoy|Q49fRB%)cB zf`|DgC-jxaUyzAdZf{stdw8BGh9z53oRlIDDYvtqbQZKI)r}C@TpCxalCuyY##ms z9Br^GU+*Occnm#%zBrDsIt_h!DmCg5lM{?WO}oZmK1#GmU=Uf>J0>3pfW??`@d;jn zQ+MxF&^~MjP;FocZ4pzt5>BK;j9D=SU_v)HS4;U`<7O~6pjxceCb_})9L$|h4?(&( zeC{8N-OG%~Kd~r-7HX~cdB>EC*?_3#-Eqh7hzH)|UkJf;3=op9PI;r0b!x>)zA z;p5gSir0i{+gC)(u2$}|Z&nu|G0ds^P~tNfwe%-N1+A&pUu2%1K6B~K-NJQ_d;V$_ zcb1uGMXEV<$G1CiS02>P_rkrV4Dx~n9G^cImHGw$V9}~FbZ(d9eJ2labLk9G=H42C zLU~ggxxVqjC)`8g{u8=@;$65e|Lg=#c%F(PU~+M6z^K1o%pfO$OTPFkdI5+%DQ2%W zLcxjI_rv)O{Wz@+Y+6_?kEr=uFZXuQZppLE$nmq#$oAl&KW)1a6+wb*6q|}hgE0z> zqwhGL1zL5tJzl_+XYpE6b!@0lDs7aK-ddFRex=`|#E@Oi?NT-ES?$rLr>qLlj234~2cbg)dCFsEaUxhCoE zww0TaG%V5#wg_G`j+??MojaIy<4@DgatbDG@`VVOOyd4xC4jX{iP@I_$JlVdg=)*2 z(wel+EVi;yhs+uJ)R}`lfn&}0E!WdnC@b9hYfv8jKcP`aN9|S#2ut9dNuaAKa=6ZAS4Z`GuXW zT8W2UBIBT)zI;ivj1_UmSc%Dey)IGhVLhSUhYTD3Sk_cC$;-$9Ev5Te;LeN%zbX0{nOfuo7z*QMb^k3f#%fd`zl&1JA5gzOCnxado&-u%_+4DYBck!@s#A< zk+9k$Z`H@otY;3_U7CjqPDmA~Z6qs)ly>|;OVFp%{n65d)dIb~SkElpuf-SpHMw6e zfRe=kPA9%ALxxC(v9t~*XxUb!Lq#RoT>@WK&Pvx^JwpqFPCo-A0CN7ZYHQ37Hcvz> zEbopS-zUWaMV8I(1m7npodZ2Z^lX5#$)>j_3`s}@$kC<(LFp>tphVF-2BKU@1qTUrnmoVYOjUiM)UZ^ozdL6Q8~hHW%PC5LhQ zBs_;iO|!EG^~HCyoJRKM&WNq_0+}5r?P?I8Zapm0&tmRc8s87)<#tP-$ZJZ(a@d1V zrTi`?sO#+ER&s94`aX7NxxV=uEvpK(0D_lnSq}^(YQNYr>R8_F_`!a@RU|5gP0jRU zlO>{4Qc=(jk!(>lSwNA8v0Hi5I3235_G;YA2U$n9lFR+kRXFd6HXAm@kA^(kvGZ@4 z$ZPDaAfmj`$ohP}c&48ls=w+4-QE0RE{3%vMb^UvI6CT+zQU?DjNh@cSKjCB-U=vx zH|Mqg4CH<{#JV(T!4M|g+Tr^ok zq9qm#qcJfxqQ!U#jEYP)A}z3OBrq_kM8B8yo)I~w%=|<8WUZ*(zvHPdBjN5%vDyX0 z-v)NE6UL{$M)!O^9^(HI0JZrqBhC!68-dhYu_v9*z0&A$uGwbqSy6J*~BQg z7L03dlL1HDWS`Pr^}s=9I3E^bL^ZP)jG8|PDdLFKa3+wNpkLg?TV{Afm399sb^47Y zI?}$f;mZOnf#RpzrpB71eCy#YID~miHph#Te>sBYtvRHA(;8Vr{hS^?_3R0#EYnRFnTZ;&44bWTgAcK-dcy~?t$qUrAwTw<7ryWu7g=J$OS(UT zN+cMOR%{Ss>N3KF2ZMk6HQI{yqNOU+paXkg_vATjx0A;%)t0=hBbhGG;bZXtU-|dm zEop(9oct!8V7R0PpJiHfMaI=9X%ZKKL<*)ttaxPjQ5HXJ1o5)KT)QDie_5&oL2HfE zcJ1_MV^vB0aBqIq@ri@}rZ!&u?4XAl=cL9_P`ADWbPVBA%qf^APzGsGm&d5MjZUY@ zX1EsL)!D&nc(T>&Tck+M{=Syeid4Jlw`cJxG$2QmnT!!h52Mv8)WcdOW^B@8150}r z%6)i0m)C>n4n;%AyjiCj`lf%!$JL<~ruSEf}2q{)TvJDv4E8I!H5|tKJ8d zN;J!19IOdr1O^#R`6BCqyzAlhDiLB6PTOJHHQUOiq}(f>Y*t6ZxwzY}FjEt@M#WaE z#n~pj9y}fWH=Jy^_t6GOB~hp+lW*3(wsQXGJiPs}lW+Zr#Qk>TYie2|9F~W{ib_ZH zT1|J=LCuc52_76NZfTyvKXP3JoCe)jR@})ZWJsw34iSF<&Z|t`Q#Gpy$T`Qn)!d>^ z4=Kqiqg!)iu;|QqpuuMX(#RB@(l-hbnL(mj}F2LsgwwtRm$e z;>p;v3>W6B5e^6~`+PV6rhEexRyU)}uq-#Aj-Q-@FgU}0363wojO?NfvC8((hnsq< zx7;u`!puGdHiIQ+L;!#+bAd4m2AjcxGY0P9*ilZL_j{BI8~b2ky3mqzf1l`FC+$8u zLduO30@ck)Ij49|NI>Kd^Jg;OqTLmD)nOBao<2L1H@N}yH@yKu5k|sZ!nEI!JKY!0ajCD+xk}j#bA0onRWj}^<*xn%QMxQG_tvgu+zmapC zKg6h4eVcxj;O%PZNxjz8a+uVpYmTq7NX|(GICWQj-E|AtC(i2yS<|sk8>(yv2o(zU zj*pb5wEJ`jcKg)mHDHVeWeqqLw07+TJk1Ox)A!m*?d9g-@P^#;0PVdw7#QsW7iyy} zt3}0@Ej5xGSXJ#8?waSy(&*hQwxb8{WK0($)xL_g8qK6xsn^ainS4zuEmZbOdqw5h z^|PAVR3;AP;dc*=J6QUSvmK=m+~rYlRaJ4A^KxbtZT6K#lm?6qJ$xh)q!{NROG+pG z?$$=`v=#`^iTiaa?Zo-Fv&gR%I@4!oT{&~hFa=UFA6!fYYJ6g_`hSj(v*D4I6X@;A z)CjUxE?Xrk(^xGf_%1Fn2wlV)nh7@H&E}?C4>Bej2MtO5A-ioUoJ`P4BWCv@d$osVx0k5HbVIb`K9FSZDdmXbO+FU(VmfcVWw?4a^wERqZ z0%yOzT&+d;SdVZzwXMwf`aGc)US&7jxIATx3cGD4=>XEr+~F-M(abJK7bklpZV6oF(x}wL*Q}q_dWDYFXW0)b1?@Z43nRbxCV<&Fg$- z5FIy<)2tZE6Om?vBrl$HSa-Wp^G!321jwK`v-Mob-y^7Wr;;k>gIKXnsB#?`-M`3& z!I{g=T1}w#e~r`sVg)HGwt_g0;@8SXf;o$Ei&<;SI9p%!lFwWk5I~RBMY(V zJ^K}>W3fAQeiny1_x`~z`%$e0qm~Y}6`l;0l4#ux8|VY!oHZ;PsP*omSt;HqZRWlR zB6k-I@<;dK)sTdc2zSs=hM$?m-^~Es)sWOR?&~$VR7V^0=p1sJJ#O6gK+sk+xJO>X z*QYoH#I|RmwP$GM7fJ(8NmE`?TV7$-95N6Fg?(O=8YS1@`V~sA!1@*#00^CUOvMeB zseSBQWczm@0~;qT8Z4+l{ASD_tp%RZi>wTSCY*M*IB}=uewB=4DI^v-<=(w zlT8mztmRo1Du}aho(8}ElpxB677Mry!i(F7DdNaBM|`X!w%I$ri9Q}LyS~Ajp1tjo z5d@{<-SQ-GfkSFb8oAgf76~s7|Cxk{w{wQ4+$YcHvamH|Z2)@I6+u;P2Ot%wirk_6 z0BvLwDHTiI;>XCYOwl96=;V|UqLYe|Of!o32>N0{&3^)D!Zb*I$(R zfAZ_;-2Mqxr27X}-u@GdLvR0o!0XD>Q}R?(lByDtvJ;aNv}2Pq`$~^fGs^a~luC@u zs*H>c%&d*f%xdV2kOq9Uy`STz8JE7=t04 z|CF{%DAr@Y5X%>2lqK!%QIWi(XNl1l)$|!TXi7M zo){E*mvAjx*_@2YqN)4TM3_l9j?ANMA$G{LD--m-NEYvxLk$dEQixD|c;r$l0cO%; z9CuTj9JPCdIdx4+F9Nw98zH#$m$r`0Ns%XF@;3?>C;t|8{OdpXeC_{J7~xa!{iFK8 zzbXqDSzG)^ser$3j~#tT=KZ8?DSy(onEw0if`)%Z#EqPV?QCp5A%Zd%wkDs%OxI70 z{(ptVlT>s+nfYjZU~myM&7n3`+p|cA1RV%v+kV3dxNR2FF`mUe|3-M_WJvKfgba_MxO;Fc&AQY{-4lU+`y=o`gKO z@ICM$@I?XcL%(!1O+t_EO5nAC*YmZo@Kxguz<<)stuPilVX0HqWt;qoV0*>*TMdkDTiha*-sp3LP?b zAOR`-NZW9li*1_jgwtdTTE4~v%WB6Xc8duYAwVL63~#=^IW(YJa^8x5iH~+P>WPkN zC&0i;uXnO<8;S|7>m)G=yOJvSoa<*ZrG+u0o==^}kM?ek*}4(?ic{`vvXFr43w;ar z{BbB}Lh7ph+Hgy(b|INkII#sn*o+=mRl)}KUp7CMB>Q`90Fy2&Ng^=6B~v*i_6QKM z!#Prs0gIjFfJ-uw;E73*r686I2YI;+A%r}Xw*ziLVOOV>8UNRL!@fzzP94t17ms+N z1{Psaw?E`6)Obyc4_2D5G~d1poou5JOHbvoNp|39im|J;g8UYgLvu5ag3`yKX(S){ zq9Gc70hE?Vr!APSQq0c(Ev81=@d6hYgBhBQCPiu{7i9R6~sH#@ZA%TU6(SX zrr+}Kl&!y-BJ&TEnBvbSc=CDuEu{Nb%l)?|s9@mu37!8hUp6>W@UPMpq95i>T5zt1 z?V(n}GYV+nqJ3WnT}$aKKqY_K)ARa=pepOM+wK+8oTKrHPve9nb;I_HcJoOKKO`j2xWK&4P9U~HBfTN9ymDTn-VlD#rFs8tq*4-s z!7u&nc2A!UH1B`!cK`idWi6bXENso>?f+Vt3p$#89@ua;`BxGnNmqVBA8q7ghP}P& z+&Gu0n;A2)i^wR{-=92yfk}?FPd`8%sWOcXs63Cc&Cq!}jQdWcCy`Hj+mEyp!kk?~ z=Y%UgoJ@YnB|r0$wbJ+x5MFK&Iy%#V>Y!q10xQ{41vP4FvY9B=ln4{<5F6ysx(kA| z2-67T!)ii~{l?rSLP`gB;Ny2_pdL%x{t4oM&RTuNQ27*1vEC+A)Ly!3g@Ym$uF%sv zdGz;Ws_}4Q_$Q13p=QGGwh6@brmB=Vf)=ga>Kn_KCEgo_3A^=815>iLxJpQfq*ri( z^Y|XdoYBPP{CCZ|2<2KA*`ng|)MTprb}cUR)+>JEiuH#nZ|Dr^Iw}#k)v~q|ZFB&} zmI~$`QU>h!WOG4lm+#L0k1Ov%WXp68Sk!aO+e>n7Zb%C_L?&V62_5-DO=eCRiaKT> z1NYs4Envw3o!H4#WM>iOVxRZlNI;_zi-XivwN0x$0sSQ|yZsml1zA!d@)#x~fxjIj%rIH1V`Q_i0LLMg z-S_<{yoFY@Tnt{m?~2hge_G^|t}fsVFDgP7yoCutdwQ`3(*|- zIq~rQZ+gH#o4)d=J!Nb5*+1+JKAFw`Rk$TfW#$vvjP}R0-Ne8q@2)_C81Y=Jr*~mw+j+EYB}u`1(rqd(w0R#&WWp|B z$PHMNN(19wbh-BdOX1-@n7Ijh#3*mVD{#;wTkl(yI#!M9eD#)sWjy&fw@(x5ULssc z#6>Gu$jRrwUxwn_gEl`vumO)I11N&ZVfDWl%BQ}s9}$wZv-HMhp3E1>l$S+1 zt-a=Sm`z;W)Gg#SL65?K?3ue{;hpnGxL2HMawPU}KlSkI=)EM`3!0h-`M1VpTO1Un zt#8Fb@jR`<1Qd=HqdW9-6C@#C2Nq@cB-v4+J%uun){c2M_^%}I^o*-#FTYr9^h-43 zDdj?@;uAB}7}?kqcV+8&;}d=*vj8ETVTa4~qwkn_5pNq(;cN(uj9JhKg}xLV@DW8U z5&`wU$j81w{9gy|ubJ(H6yZ+%Q{g;6I!tRD@#FBvz86bS^rg|D%46+KxhDCYi-eQXPn}=G!bT&Gpjc0)|)ThluVM+ z=yU;^n+MsOzky%x{@lJo?!Zr>!mctKY={Cy1ADoS14{S;Ui19q3Cl1QQ9R#O98g?i z0N}yWT&CcvIdHBSL!`x!&S(}zM-%>H!sV@F$A-jNH$gjtDbx=_q9Z8x0ij+g%+Y07 zxTC?a4XI%dXI%P7R4Mt=JHxb+=H_KRI>?PF?!SxS$))(yUY6~day9cMe-)vF7j;jn z^j5dsZoE#cmVHT73^Ec5&b^OON4fBw>X{H3H)?Jbf%ABWGd=u1368Iu^~*VXp=04n zMo{nKJv^GMg5Bj1QSDb5Q^ovidJ!k3kuD2-1+y9O1lyyl<8t~Itu3dP57=mD0M$?r zF_|?mSr(39<*?wo!vAj$`Cnf}0Mq3Bn;HB zaz{Hv_w6xG&?E-~1cUrkD@l(vc0&3RG22L-UkLb)D-+qcZr~;Z$-%Obwg!GNB&B@` z)SG2j^Qwbh_xve^D%82CSDXK9IbZ(c(c_iZ=XE=$iqFi{wIKso8z%7kIO9I+db8W< z_w?1!N4DRW?>t*cbr5dVxn#rzUyV>@u!%JyCGYM$^sM#p^mK~lC9#l5cAf*HFtelqM%$T+vi?Dh0-czyF$9rpC*i}W(F9`IrQ>+&vj!$LyHN{Jw{M1AUTy zCadsJ>96^;%M~g=`PfJPR=7u@K?y-?DZzO*H5O;C@d^ z^UJ#7VOEwcv(#7LDOcwX@(jO_?`<`LJ7=F%0$vealnikU{acm62CT56Ne4Fd6#MX2 zpRbTu#Is79%e0>CE;`bM&&f$XAx#cdY=<~u%lrclr`ALMOoo=W~gYcNZIV{~UEg$aF0*BD6^F2>CeNnTX}J9!KzadQ4kmp+W!BaJXAWmzmGO z;VImJY7~a)7kRBrO~zWZ4t)B;Jh+9b;g(<_o7%1VX$i6#*{`V}eE?ij+b(}oiLiM`GF^xIaP zh$cxnT+WBNek$mL4O0u>nzmnw0Mw~{Trdr=(?)WAPVQp;_po}s5wN}^eJAS~Qmv3n zmSXJ%awpB*#xD%JPpE%#cVaFA1$Kp^uix(!ZEYwRjai(QJT!ww zGyG{hjDm>Z>s9HFcECK{>|}*xjy7b+ifoK~1-#|C8j+Wt@+YBh)}llrKbRjfnnhv6 zdDEHg)eKZ@uedah3aW?HM3l+fg4Mf*#WlWQNK8^6ip9gv!9b*nA&ND&G*YXpSogV5Yzx zd}qFZR%m{Y)<1VPi>4-00Yj5>`)y0)JSo0OZVd>!t1RCe5?&9l)aPwKC-6#KD(u)v^$P!LaC`wg9Zg-Sdx>5z~nU0o?HDF zb$7RZ`MtuBQ#SVyCR*tyU<6W%o3|*}{8=h{a+J!f)14|pAal2e%%;%YA5T&a!{lOA za?wQd#H*@3cSY^y4<7rg7RRp_Yr_0F7aYPz|CwO9LOWj*Zcugf=w4djSFa4yTNE{I z(cYy1(;BN++>8=Mr?Ypz7eh;i+`!y;r&Zn%ZmE%1i2>GpS{t0GIC4T$p@3q+PP#wc zE*LhNu*^rzB)-#wUJ*?K=ZX-nN#G( zvQxf+5P`?FGw~;aN69qAz+_A#zBR(0qCM4`cOA^xMcR${(JNv2d=W#Ey}|BOE43@^ zHN$tzHPiOg+2~j8`wpql8y(4dWc+Zaj`SI^8%3_8G=iBx)sxbQi`)B+rYEVff8zop z3WJNP$Kq^*mAq@i{LS&j2eQtX@C@DuePG@#BMJ=oQi-2hh+VqMHnq8e7kDjPbmGIN z1DM>ZGh0;~v&FNDK3YQzRBEOLQl+Jzp9N`@ugd9G@vP^SRj@56z--J`3KJY99JRKy zcq9~z5-q*qL%haz1QXrR4wK%Q>^1td^)jMd&jv8e>*7K_;gsT8P^4R0s_9mFMjI?e z{EQ+}Ze!oy>WkC656{B!h5h7=x|Gij(?P(fAU-?SY0{v1ERkP>8lP0-xJcip^A;q1 z;5VIO7r)lPnQNMxIMs3DcyIw^VOy0<#!L`|W zQ%2pQrrgDMIh+z=vK|7^T2$*b>i``QW;o|~jADj}&?0yE2HbU)Ic*d3?62EeUF&ik z;e{283NT{q;HY(Vp8|+jOW)hPwQ*Hkw&Ghh$@C4dY-8-wos0eH1p@^wW>oVp<`C2; z#iNFr=3tMjl@l0@es*NFs$(Q^@(ekjU)*qQBnf+im!rY8bc@lR;=N#9&%u~M6vtXLu@~Fw7~zShp5_G z{r{-wF4YO8&viT>-`F<;=I_wRx51&5W603Ec_g7EMMbJ;TEX@DE8mp&PmBTSGKoKK ze&|S`$53PX`hV;Uuk=UZacJAScuW;bUlFZ&9W;8e19j&sh)*|LUed_I|VT!LOhX3N<96LN9k=NMEKN%O^5{6`td^m+$qtxeOq z$`^t9t6rAz5@7Nd$IbWizO9F8(eEjlbcyz;soC2mCtE&xdX7<2k}Z5n99e6*wMNRH z`{8FBTk)}8%vlyK^5I5=^II0Vwi}U5di$h~<6HI4Ookj-y*Fn9thFAlTXyx0d{i=e zsZ<8V*kW2=7ABT6!?kCx)AHZTjJUq;MNxasQA~D*+kR7dASx3QObIuD7pu$NBgZIc z9b$Z%S?FV2LfZgYTp&ue5jTF_WycIRU^W5Hk=zGJ4}bQaV&GG>S5z`DPCEt=!Uj z#*(`$O2o?LO6V2vwl7at z@QRC!_!E(eb?t8&=QxNCW0SJDE^1Dw=y*q5K%%iKKe$%Y9*?T3b|%3<52b@!NOT&J z%ASlb0J6cQv;;*cpgdKkiawC^{TNFOEXzpZH+O{U@O5MmQx08(+}!|Lm=T7h#+%Xf z9;>QH7%!@!wW$MN<=fv@pd_ASTJfL$R~iDy-|I^J&GG){s`FodubQ^gf*SIlM68KA zQB?TBT>>J1qpzD7poxVF&@JC3{0k+8b4BY^#Z}^TG>_(gcfG@PK2#kRAvG%Z7fw3A z4hoySQoIVU`--a>uhmNzCxlIBFJ%Mm+m`@as5+nZSZ&)$&9$8*=1bxdA3e^ z;Z1`dirpv4?7{9~HV5f$-KB>&U^W5NMuKAe(bH#T0kN#aU8IHi?zF?XBlhBy+fjYU zeWCZKTwK!~xj%nl>I4-2v4$O+P;~v^>eG(D?pt9zy zRCBU=@K~i~#-dc{xoLO(_pDV34(N7s?WFn2D_SYeP3ZOdh_?JH40yT}j)%?CrpChb zU`0oWPW@S*$G)Ibi z0o-p_#Y^7jWw=dEjzjvU+Cp|SD$WJDFp$pkZdnZlr?oX~c`~TW76Y|c5OvKZP@DwX z@9OH%5)9Z{z2CaI4YUONO*vX_2B{W*luoTGv<_IM*BiJ0qz#Z4U-%eEkshR~Fg$L$ zZ_o9TA3ck`Dc>Qoo^Qn1&DYX1MuXs~lNQtb8Q2B;7%DDiP7QmtmmT>VmOx*o@Ava} zAvYs=WAD-(QtwH`Wu2IFlV+Z!{0-PggPs8So3a2fp;!2vh)c`|rXN;9+xmnIP1>;Y zSo*uiR&Mw%KMYm+)StEbI7nQ#BdAqFyd8I=lihTbCM)+`e@tp{dl9B(cX&qg!Tx|i zHEegYsGD`^LeeoEt4+?qx$_e0m?=eB&^-$&f(;8`M*0Je~WfkLFTSB_qLr#Un;^imfV0Hb73uErgp`POj|0alOCq z2;6?9j1Mr;FKD$Y=$1vE+J3sv$+SNN+ZwNSl7*#zb=CA8CPVdzy(6~t73U$*VKB)S z8s`<>*i>#55d3z}vdkygSRB_t6Dry2Xb*vpN??c^+&Xw47B>M`c#MUZSFvOcxp)j|3z&$SR; z+F4&$!&qzrgX|iVBh5d$!(2KP9!K_ZJwgl+<24>IL-rA_$2y>yBM=Nt%6)pSA>}N6 zdUDMtMXA)g7bGuQF0TDFt{hI0j&j{0cpgC#zhe+YGGG@wHfo-Vj(k^J2(_NmY|f4y z?+@bh4vx|`r!dCwZ{nqY%i!F7A4?nkS|~JayO4&{OZwY=*oOe3gkg=-M=RkJteO>H zx9zre%h8!))600?Dc=KK5{9C)wfW8x)zB1TgL1jLRIa)gm4Pr}sSZ?C>Sa}FYe*Z{ zEN|>}-#clZO}+gO!+*NHnbtZpC7*6@@qbU={%utM*FNU|!%|FA()}xW%h#aU;3_NI zn7-#0NhL;Qi}vFiiTQW50N6O*XLd=z<*2EeDFxX_K~JH4F#j{yYeBdh`xg{A3s-{a ztd8UC2|l+!Z}0E$JIFu0jcZQ_hKfVtLu>#SWh(QTOvdG2HjphSPvFAcR7tJa4?IHK z_i`d>L#CUDiWycG*ZYN5-D5!pyN_d|8bF6EXdv_EY|Unqk`M<;_O}4aktvN3!BP(f zR6&mT&mw(KZD(uz1?}TJaohvmm6VG|V(?RKhW z>)r?39>@;pkaPt_u;Zn z=`T`(jm${Y`Pw0ZjG0Uy{rX-ce+I548vA_wL_#|j1Al&oZf#_zEo=>yr=mCD8p@x- zq;)c(^%Xja99ruciXiQm;EhtNOHQsTc|)*78aFwyHkkeuM?s71ODWI!%= z2v|m57c?QM(^v2Q8GhBo&XLYV7X#h6)j`eqjB(6R+=6x^k3=wcr|#4-kj+M?7<+U5 zw8e7p7VZ2Iy^ntDt7_g!F6YY@R8m~sXJ{j!(IBsTbj3DT;DqZUEjEOP}W!cw(XdQd{t4{@N0BwKhO zeeYB zVc&2TNFZWt5nZ~pRv(mNw3&)Drj=d8&|xNdkWhjw46#p5 z&?EOXo>8;KZHAKTvolyyERY%)Iq)!jvF1)L!DGm9k^}-I_dXjpje2|}0(^63ov+oY zR&?O}?)PwY71kIDZek>DCOW*=tV#3yX#GP0HBnl1VR<;JzpxB0KQMvNnOW^N)yRsP+0ZKbhI5@cghs85i$Ah~><{GmaoK>F$l<7@@m zkNf-6)!~Os~H2L#;zXe3dEjx@Z#c8XS=1y?F zKFIG3e)}7mPCFz@&LA+z7;#~M`-;CYqK`|S+3bCN262^o!+br+PIQlx3pFEMSs6pr*6=;25LB?-~(_9{L z;s!oQ1Z|C!UI^bwd9sS>Oi4MZvcJ0TAxFFGp2w(1t!OVzh;*ZFN#Q3V9*cpG1QVze zd_!ElcJk+yXeETb@~Vg$vS*N~^w-${i}`B$ibQI6wnDm7F*P?T=998nMq{|rK@F@Zm<3U5fGY`% zXmfVDmWWt{&b<}QH4l+yWm!L#gP*m-_Gr7(NsD9Js2@Y;?lTHE2c|9DFQu#eg|WON zj*MHb48iyGp_&zy*mN5nEq*XsWa2q5ty7=Pi>+&i5e5{Dhl+k;c<4(c-C&PEu#CAu zc8YVr>+DM_C**$?v4OEB7Ktd_2{{P0dNP_TyCE)-isKd|;O3*`C*#>fd_`_I>Teq+ z+2)^CZHq`qhRZ8W97J|DcipI)7)TM`>y52gDKDQecIrjAPxt~ zo^U*Bf?+AH-dGojd#b%dDvFGaVKNKZOEeI}O7KYekg5q097f_!`HbPoT$L!y-GNCd zfuOyJ|V<~p1&NNY+KF+1* zZOG=s*BI+0srNv0PV`44+OjL4SK=?Xw-2P-K%cvVEXvOkF4w{tXAD#_;kASq>DdDs zp{v*fic>86eSyX6%0QB%yzR-Vdk6%P zX#Go#)u;|e$@|xuz^JSIpu&Cp^gzpk%q<`%7Hj$JArr@J{h-k@-wqs#|!ZC8>KY#S1c$RQFW1-Cu({B=)HVxRsi2fV}0A7ruZiglW8%MvYmV={vSa>gxq*v zb!8uQfM6lpZxYLeQD>82Tnlo=Gnfa$JcoRgP$qlv<=F$pCQ1>*oX{rC$$l!w>V-qT zT$qeZBlGYE0z=h;?o3 zrBp6&42|3-X9WWM!c9sqJ4A-BRQKj_ONI85_C_Q3NN1&PmPq4}XTTzm&LaFHaHs;` z1i#;I<-ME<;-nx7eCfU5r{gIx9exFgj$2kb7h?C>;82T7^15Lf7izUOA67+i~zUjk) zP@wYF$hNr9`Dg{tazc^aAcq(`4G8rwb1S@0kE6CkazSzQ1)O zFT8x>g2ZU1TqglAUV;EjFe1OV=}%4geW5O>ZL1H^Bh$CAHMTQ$(Eqb9Ql9)@4zWyb zG;2E1bvLR#A@Ow0d3QPl;SxFmBqjor*U!LG4d%@q5&-(0o@+e`$v1D^u0%0UX|ScB z!H@+LU3W(tcSpG$uXf8VSD!I|dinghETh;ysW*3P9IS#}gGr{vTA{alfSx1=6}wK* zJ8E*6vpTLg7;Me$e#c4iH!gkImhvR4_TZg7i0Kpe6d3S4R2l31>Ni!JHxp-ynWOr2 zpW>J-nq!&PgF7w(k%>3O%FUry6XHHK9lGe69tCI7mU@@cbjtWKO)2t1d`!?XhSiV# zfZ@m0)T`C#N;T@Q4{c~R5yF-UhtiJA6ME+y;1sz|2ooqNRqEszXX}hL97RBNn@f*{|d*bZD zi={%gD9boJ3+=+CHW|j~4=l*wMv3eolu6AJ`Z~z!VCf7kUsf63=wz^USJV~}2P|Kj zFqnx%?#vyB;m*c3@pN5zAJ7tv zIPu7!u_;{rbp-Oyt3fwJ0s`s<#OWgY7rphnu}~G-NnyHHi~5{BHugD5G?4F0BKQH_ z7$5%0fA0pGBMr*Qi(}Ga__UJs4nG-v){Ta7nUjsiwDV-l%DFC7rQU> zn4KP9uBb1%TDmT}n5yr$UnM0COTm#{ZEhZMyOy`kEF7Ml);g|yxoJceVh)wvnSi_V zy!|4~gFmoaj`fu`;Xwxfa4Som^Z4yVVX*2ZPMV#uCMV|6%zT$t(hT#JacW8*=kC5j zM}W-jOM%U3PSmsaFGqKMUcT63+G0}MBuaz(gn=J9ZTvEFa;|)m1n+c{Y5N-FRthCV zoKv$a)?I^!*l@rwBuwh^jM->l(%r4Dm&p!_K6DEyT++Ts=gK;%X8SW_e+bmA0+cV+ zI+r|8wUBJBg#%tjm+h8(=9xwsnr&_Gxt-eJIg3`Nb-2usQpRCEb=N+GkDN3T2cbHtjVCS}!+3ye@#T-t26W&Ci0RsX6Cdu--aVtL)mO z)qg_eOlg_!8_9sF-&4mShPd60FPI zJ~~2%$)uN9F1(&Wx{OJ8Cd6tOs?X9pV3dXlJ9yfi$+d## zhb7OWZCPh1hg+BiM)E7M2Jm`Lb1h|PWM?goiy0<1ZZf8# zCa&0MK(xoe+?Y634zmSqXWP$wV8Gr;(I~~R@LQWTG5levz*@>-N`$TIf!M<`W=jUl zP>xN4N*L1owyb7uHg}|%q^LB&SiUOVjN_%_A-W$pl88eC0^hh4ydBMBsD_ofC~(cM zt42n&FhoUK4bmgH*b}Si2_cK^$3v|JvMe1$9f zu{x7OR(ixG`Pj-h>MH#XR0e9rey4he+PVT7*4cZ1&+q@c&(W~TB*&_8A zeqBU^!PCXx<8O($cPt=a8D=M(BG&~O5sBHI{Tc(q4t?2tjK66zlWxo$Y?wrQAk&Q{JeJP7`w$7e8W&?R|_(}%PXF1AOvt$rz}j3OFQwmJarzxTrTbVm@#oP}AEc=bMYx%IEnO>%?rc1D`G zb+45})SH3B4YK;;ZgZ1!fPhTAU`izo8fX|ELSyz` z%y1SDxxIF8BGOWk=L>a7gec9Lxa=kJ{_G}nu7^EL`F#c`;JQ5q5D;S%noB-J1ZK4g zA!u~LN$tj;>PfIo4u-ARk?2^})k27kO{Gg<$wiaRlU0_&dP5ySH;;Rms0x*oYgOwb+g}-6DftAw}7|73aWwqB*#0Fk%#g=akp-mZ*fc1z)Y>^KLBh`Q##f>rQ z-}MC*tYTl5?6lfgzD@HszA9)Jg#{0hJr`kcbh6^y8_;REP5o;10p*4{A#Z)neJ4ls zc7GrDHQm>i{fM5@2!43TE9(}k%#x3s?-f;fUB+lVeVcX+v(N^)%Q2CUVxWvR*P1Hq ztde+%o;P*yp?+CoF3Y{J%gcFW_AlOJp1JLfOgiqO@C#^@fOAJr&&x%Hn*qL5ptsfs zuQ4#AJEnTW?u62?WYLRNvTS{s>Dx4ptHdjk5XXtSdW&mtt<=~mx;e0@Cl@TJ+RVQ~ z?qHXcrGmykp-G^^&~NhCBF&sSK61RVw4^dSqe7G&Dxt(4zd=m0H(6KlK^yvU_;~Rw z%|K5e5ks|gb{MDEmT#sy5DlhYrFmPkBb>Gr0l(a8CAo}1f|Poak$l!oZQePUiQ1uZ zDY-Sj=>k|2$2lWkE!Kw@Pkeb<5=Rk#-k?YB66SsRBC32p67zXLiIsYbravW26gniE zP^UQf4)x#`Yka6j8EfJ2s6z;ML5Iw9XvK*}t90VTh3x3E(M$el^+Y(>&s&7nY`S~H zvO-2^RU{uJSa$s@7GCWkuYvDp>k1YI`uc?7)Z@PuF(Aq`A3HBmv1LwlJ3fpf54(k9 z#ms-#vRG=NpC0`@_A+0kkN6p6`^}VTNcI{37tZ_ep3pK}o-68s4rqQC2$*Mw`*f7Z zsf?}!b1zG?$}noMj`gH*a=XHoyYD-EWb;f7UU6j;Ym^lqFd76Zshwq(OcL)-*D<*r>u&zKlR5PU!Ub$Q6^?!y|+2b^6VOSt-_^ z%Zj-Kwug+V*7zm|^-FH%If>ATTAX%Y2v4`;K3YdBfAuY*jdSIZdth&*-na%thggU> zP55NW&^X>@q{{1@91&BWP^0ykyA)$7v^*l-h%!9acAw`0CMETx06Yk#7#z8THCA+7 zhUPF&qhd0}h4K`maf~H-aJiLv1LF*6Q$UPNE#MTmqBsZAE**)!*B}OgptX6AFlbH` zelmf<&@?UQz0J^Ih~f)wfk>SPh`Xxe^0mjV3yem;!b5_K zkI%6kdAHdv<@x33tG5nv1oE{wa}q>mujS?BRlQt|r39Vv!+WOtjvcSZ+4BY6Ub}eY zTaMje$@;HO3L4^Vkbg<B<2*zN2goBm-=O4XuI)X% zz8YgjIC}QMPWaXS^%mVpR&{YJt3D!y0YvG}?3bJEHi1&w582Qa?-gh{CC8h%AzxQq zy0%a@4Tu&V(W81d;YXNj=U5SLFRQZy zcfd)~HK@`fUIVR$Ge@wFD|9>2YRaIGqp3+MM+JK>8dKZLGigfG+99ioRVoRoVslF# zUm$_*H`j!FfE8U+2;sj5Ps^r{%!G){lSvojYDmo1kg!e{)m#$eawb0BFrOMpvm-st zE4~3bUKcf{$4dbq;}I=4i_+P_;=@A72OQtmpG1$@Z+u^ck449?ZOtgqVY1@ zZ{+Z~!Beiu8ARl`GonjbyIZ{;AYB-|Ic*t;Fw5UH66Tu$L71&IVN2jhJbyt8ssWy+ zx&@ttD$isCH5DnDR49BffwHnzO;I)ANC) zqJa+%=sRO~U-7z6>44p9f(o-b!H}`kqdQ`HeCWOL)NHn# z3#r4>m3ZUNbbZ8LV;grw{=x!j{nk}jl*AJdC!ymr(jA)7k^G;sgLduwG1(3$&BUS6@z zUh0GLzCvxTO~N_kT6+R&_HD=U$IC-^yI{#ZLn4B$OrtpNPzNnYu)JlGebSoAke5EP z(|yL~wczW7k}q&ua+zxN(p0h{XNtEaZj!t^hnDDG$;Sd4O*Msc*C1l6A&8wABG$!s-l)&{$j{CzLL{$%t%8a?!@hpW!{iWjf>Yoo7&hK0?1+v^3&y z&upm#Spa!u@s;{3_SKFk@3T90D$j8HT$j_XI$-pnJ>Cvt@Fo9`Y5SSwd!D{C0eA2~ zRigX#kWuD=`g*hEgNM(_;~R>Wg-?Rv$IJMlT^+(j35&_)LT~O1YYQuAqk+Xx4 z`4!k>wiaW~7pr$8UyIR9jtj1LK_-i_j(D&E-S>K^Es^9I(%H{|quk_fUgw4=P&L2P zI^jclwgL@I zdvSq#qc{xFX@(SE7zCq_{GR1L4(La2c|HzoaDIqXWy|ca1$miYg`gH>Nix5p-6-1- zk*@|y-JSw;V*CLbw`dN$>57KR1!tJ&%&@jw(lkFDBB^A3w<1jD8|{#Q!?3 z%>XaRcyw7XRr+3S1RH@dXwNIbnm{#eR2H&ej`zEwwdyEV}2i}E` z*{yiz!bZG-S70@4O}2YL3m<(S$ZFVpEpW#!a4k=GpPX)f1J5&&12C*o0ye^#{)MTE zgx>%VPv9>%2;0BxR;BO$&u6;tu^#(y4-A_k=p(cbA9P$+b`XP{8^nMRvR!ZsgQF?# zbQz1I@EP%qrW;|fM0PNK2fY5v`r@3bXdeb?myaCRORF5aE4GUn?QLIyUiF56p-y5| zCGL}pD>D=mhC9QOp((^E(lBlvcvKH?7jHPRb~*K+!&VbEY%drr+Ygg#)R>vtuNwLj z+76wiuCaD)*;U<3y(4TrPzRwC>$-EOHV7?f*@@9_*qCip-|mcd(USsKmkA~G+|_>@ z+Gh#ecb(g`<6Ng=?_8`OYl0Vs6N*VjNVaiEd8iZHUOtcg44r?mpPo_Exo6d8a$Bow z3BqraMah5_^R))Eo{eTK%=0#M!S@ZF^i%PRa>k6ASgfv5uH6zZvO{UFS0g`vyj^KJ z{aQ$NtqkVqIvtNghbP{n2u5FmyPg<3uw8)~mj-%E#UzEJ59wRCZW-G2wIjNeVPTtz zE_9eUu*FStC}J&xdLh$f+&i`TF5xk_NRNS8tw;@|`chYF(@0;&-=5lb`oDBMKv8nZk_Bn;-R z_kk)ffhEmn;VKZG<=I7$_-~yzU}T+&u$ab}xCx7_7MR!sK7M4L{Za ziY3XMotWpD>CIu({=}D4bll)52GHkI0hvWyX=|=123Z2G~+6Oe6;8X%oW2>KhkL(BxYwr)y4F zz3F-$z5Umd9m@;Fqw`gITq}^c}ShpKft<&t#Fi5X{#66orY0f}mq9sVL zH*2O`a$4`;_ZWZ5F5vL_U}=7%jdqhF3BvK%i+}YMESElo+jdiDImb%~kYhE|^wpYV z9!vJlBCa~cb2Zu%R=rTRC3wF#?BV3klJX(m%<(U-XUsZ>-i4t_e)Y>2DBm=7>IVv# zMW1ly$tX$|KAQAlRy0P#ghKzo0CVP|3BsS%RKxd4?JVZt9!lEM<=#WHrDl7q&y{Le zGAKeDgVP2hdM7%921ZA#(8vj(3`GrtyquSDx+o)f!?p&}&WFmd8jT$T;x z0ZcEz>y^tj8;@}~m6yq7NSMPSCk1yOPT(Z)0~gnlKE|PKW8U?}pmQ_r64>~$V>$IXD3UmIY)&R|H#^@?lB$Ry3=4u+4VVCNa7WV4s5o?}>7y9N1iI6^pNX6i!4 zXI^voflM;=zo!^_oBH_{4hFdaj6$|fdoVU!XKT`2$eiarh6+PFakM0!_8N4)hrl9_ zh(v&IoM8YSxMWCy4`S1Yso$-X~g7AWAwNqd|hG5-WL{GUJcQm=1cq9A{$Lf#)gT~ z#S;v}RO;QiO)(hDC)^ssSZv1r(Ra|l?m#$^Z7942h>BuC0|9aUKCJ&8E9T#9f&u~q zI$|lJJix(7F(&Q!WU-Kyio>7+!&9&^sgB7QC(xj!p)f3($Joh2ahs8(8BOYx zBFZVJg|@m=8I@TmAZet2pK@x6WM{*>>9n7BZ6xRl?$h&B62@ zAckY(`YMX?u|O&r*<8jtvAk;Cfjw{Nyay{zjNU?Cqg-c)n_YyXV>FUb-#&y zK3}ldPx+zj3buc~F?v-Q+JR^TO>XcY!Pz#CE9ZE7!&9?UOPS8O$O`AGT4aRgy(3F{ zr;#VRyZ2%YK-&gGM0Vlb*^7Mr;kRntx|pYeh|vjhd~&@sZ{#Yev%8hAgp3%k&V+4M0v^eO$__iD zj{53M-z;|ZJTMnlj1_Mv$ZrrLoRk1zj%+AfG^lsdXVw-`ylX9k#hqqZi+?>p`Y6Tg<9Ydgr!N1wjyeIZzZj%xfsGG%lhUg7GP(PJ=HbS5Z$_mP|f zjKg_m5N1o<7Or8!>b4L}gUbg(kK zlLv;*vYe;dW%@M|3t9(sBJS-UsyEXtJ5rVr-y>JS-puI0-puMSqhe#sJwC8CW7Y9zxoj)blmO&LRZU-w})h;h5yZSZ%D#DWIVP{N~Zg# z=#_?B9}Y9y_~Lx#AP|wEyE_BB1w%d^BUFj{g^E@P1)(A2S%!`ITcIWxy?6_AO#zya zc4KpVV{>77{ygv!N3~hvOw)ANTM|v&Cao7(++vM5ustP*^7Fe)#ND^=Xlzm@+?cPB zHeo?BE{DxyRSS<*1**1HJ81=$_xmP4Uoh}k-%b6ba`f$#QfyiaY71a)CIHOMG`|mA zzd2?8eA*&hUj6?1CwG`x14fr-G(;|98 zeI#qU$qbf=5^@J@>3=+Wk%uDgmXyYEpLXiD%E8qB==S*REh06g-m6z~QiMJN@OShX z+1mjjDdIG_QC{i2v@~Sa>K>=>8>ri_x2keC+CspgkX(n&td;rmtA?%;S3dg{D*GMM zQtuT)b?ImgtwR|!c_jE$56}pfyF^rkZ8PSPNOU4;sq!2tujc-ge2U+~_SGYRS`w)Dhz*RzvdialDZ+5wRt(0}qn2 zHi3;aB><1wVEp=)HvtpRfDCf&cFD$@E>oXkXuo|IhE2jpxvd&DiCVLZB(&t>I z2Gc0APSg4QuLer3n>+nUzY@Ifcfe$f)Vhm5G;7%*dPRM|RM66P%$`42)3}@Drw(__ zxR??AVA?dWswDl{&of9HBZ=zxOu6N)ZGjxceWwjpabp3D+zYI#^>mW(ZhHrf-5>(z zlKK0ud!1Z7EBQ(e>e&Vss-K-0x%X5HGl~6cBC1u!7=oBMEp!!nvLi@oidDudLs$a* zUu}mQwo%s6tlw@cv4}CjTtiFNa=|c>Z@zqqkCnJ`ECIJr+ao_3MfgZ(Sh#`r9D}S& znTu;xYq?y9?bKdy3unJFiVQHS+U=)CB$8k?mpb*u zJfbEN@xULK<)?ig|Ct6pe1xFKfI*-VX8V1>k#Oc$5*DIvXULpq=TNsus7(3oe79rk zq5Nfvm7(M_>%r@cWv|lLsd|CaxnXMLgg2S8g;@CF-35QuoU2b;wRd)}53xJAM{(_NQ;||h zB=7)5}m37tuE{8(oj2!aw#7Zh`^kwqF7SBo?U?E?c zhJ=?;(W_A)!T__zak@fEch%1Kr(;gZU6Osh-_F3j8!N|}!oUKVx6oL9h?~pWR+iQq zh$6hGjH(m-+GwxCmHYzCy4~buN!shUZO(OB#@ah{(#CNYNR8Dp6~Ce5(Ufw(6Hn;Q z5r++5wA(Q1>Uo6}KBKqx$+QB&9w;=j@Tt9>V zTEBwhXgdc0k4QJb7s0;@V<(_*U}>W-Vr*k;CvUIwz5f6D`t4CNmq%6xoRY7yvaU7~ zgMC*wC+5qi1;Jm;hX9Qjg%oTa$2wOptui^SH#=`u^bl0ng%Tr4_pj_)Wy{f}$*#=r77`8Z=m`G^)G;3-= zk`1G0!HG1sB@lD4n2bssGhh{?*7ChzJntBSq$5(p5bD@JmOztt;HBkT!7MoNOk$~4!>lz} z8xvtfy`RCruS!rkSIcni@3=A&C)XGmU}m=-=|({tbWzDC2jSqHbVxxrqNa8Q`DnKc zSqBn26Jhr3G(**$f%YXph0JLOIf=ht!)wz?ybiOQbuvnf41Y1;bn>1Q6rG+-#eE2Y zm$Rcv(RhlvOUwQBOmfD9z@&a|650UOI+4YwFj?;*@+8a$-!H=nct-jun_Qq&5=1&l z>qWcKtdZ_O+Y~4l9E^{0rfr8 z!Z@;uO7|8#c$kxZSO3ao!PKri8SIUr0BY*%>iig*b4{leF0DePS~$mf>W#1GVES{L zvuj`BZ`!-1Q@g2&E;6Aexxzqwvs)(n;WOS}U0l0F8n79k6lewac>2?!$sT=pWEydI z%2=4x3D*?FR~PWo>;u=s&S&Y=jdSb5l&dAh?hC^e@A2?H z#k@oQ_`&_=`E%%rpbPSevfC+HfUwhxUSq5vL@np0$PYSuH5Xi?C|?IUnLw`TFKqC$ zvge|4qO}NDofooQ@ly8;f)8NBsuaU2SxDwM8O?lGLOB8-^b=G<+X5h^kjxp9v!mgk z9T5b8;JU|ciR)m!Mj%mba&CB8DmG;+O6!oR)Na*4Y!Em3$EuBX0ppW!SLyIp}tB3Lc5y#8vg&`qc7j%Pg1N~)&IFFn3 zSGJfh_`i-Ju|Ql&-#n|o0LEyJ-^XZqXIndc^M7MgNQ)Vg=;A{O_&8T=URyU~GA+Es zB7iK^?T;RXhW?uF)xJkE-efchGTEfSiiENcG=4`Q61g!#A%C}OD%1JL$C1>=7SEQp zXC2SX5(wbKiOf*4RQ*PP%}_Ii2|Nd1l6{2KTeyqjs~hSQ%Um$TTaj8u3~}YOiFb#}Vb@Tvt`+q2fwGX=^3*mQDXf1&E{)4eX7Aiqk-L z$Ypz+fe@%dCXg_2u4pDs_p3f-6z|Pv66R$_9#y5i_{<#q$0kmtwc{1ArIWT@Mu4z0 zhEqw|76|NL`dA7VH8Wp`c%w|kwA)sIb6l>;4FLy_W^YtsB~c;2v%RO|1ME0JN>J_S zR>J9{Qrr3tQZuwcO@o|}Smn1})OfMBXC=|u(SnZ9WOEf70iG|i)u4)aOpnwaL4Ivg zT2vz+a6of51B^wCzc=Ym)9!c2>fe@^@8nl4CtjgE$WWp{+jcA|Fe9_!(6b)6F=0rP zBqv6hLmI%lHuH5g#i`pa(%$jjZiJHY+<@NzzPQZi^?X5$C(`k+Q%~J?Qx{h~JsyCq zfciwR7FikRMzc*eF&${8Xqh3Bl+!P=XZ;jftp(`0K8%r;IB@UdX@%XF-BH}}xJoR) zCHR7z_0n86)xd7Y-*2h%RaUV}bkJPVBSBs*z4Van!)G)%LdDCjM1g7W^hwAqgnwoqFN{ahS1VOpL#z5IdLpx4sY^qT^T8S4q}i zcEch!1ldo-p-?1KI_Wnvs$Ctf-3%S8n>pGa-0tBB0)!Dqf|w_eP{)0O#H#q|0<0uE zD!djon5YCg61}*9dxf2>W&MKgf$<>3=%-RFrvwNF$I>RkHAoEmi=9bhMv9|z+bRi7 zizyZ5(e!dMF|4cblv$=*`sk+*%^u4ANwsJzLjf_Tonr2aI>$Oe&(*Q1L(UYm24cH2 zCaP^b#90;E=%BclGz03oP30NL6m#Ah)G38T!AykZQ;IOsp+iBbhO^&cu)_szTo}O9 zMv6;2lfXzf#WU!4Nm(Wrl|hOz)-1HRqf$zDy3D7j#jXxUx0GxXVNSlP)o9U}*gbN_ zWW8OB566+!z{GRsSgs;3kPwhW*Pm`{HAhDO6!i?|(D3tmT34uQ&$m{r^J(fd17VBmlO53H<*I809%Yxf}ul$Pr-T0}%fw z>^)$3_+X4=ji5Q#d^XuyB+uBNNTWA~pEw%78 z@58WKBHu!2-vSJJzvdkeAZq%Dyet1D%>l4=7#JJc1L9``V#)tG?|Lr7t1*Bo;Rd`* z^nYg@@T~E^L--@~)Akets709lw~XgG(>EyrG7bc&oo_?N-&c+I0_q>pr7R8qYb}i0 z9EP9*98D|$W&U<9>hG(@+Z><)@`qaZMfUE`#b;lsTgC>wVn={cfZ%UHz_Z4?7m(jS zU;<7B+G(4a{TXe!Ln^o%P?_%lmHBHs;RE``AJ7CWE$zPPZdgfc8(RR3u0PZ^o^}DT znR=2*K>s2J6!n{C!rxbo_X~jN-yfjAcL8B1eO>$igin8p>W7tETm?WC0H9L+4GDPG zc#8`D5%sT^;yd=YO#iteo@(y?4PE2SFY`y-@74O>hM%Vzhd=NL0R#FUO8-mK|2M_M zr?v4^Kko+%welZX{&~cCDx32I&iBoKX3y^f@E>Q;pY!)^ck8L@%@07-xBp!O=PAm! zRNr37Z`U{7n7^)X^BAV~FQxnz!{%w?rz$dkC$I4q`#tgBegZ$O*PmElpTa*?2KfO$ zsry^reuDk}b;?Z^FOFcP5z1MzXYCt3jZ`_`VV+PvwwpB-V*;5LH#M!)8MN=sPygr1=U}b_P?s@ zY5d9`B!Q0qg5;m0Sw1b%({O)3$a-Ap#72PxsJ&ATyQ!hWvYH`V0EcJL*ph@pSL< z2NhY>KT-XUx%BCl-4ED+>VJa$K4ARA2Hw*GJT>h9U>dCdjp^z4!%ubhKMM5J*!+Vg zt?@USpJ2Zi==jD1h7jz91(n*Rm \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn ( ) { - echo "$*" -} - -die ( ) { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules -function splitJvmOpts() { - JVM_OPTS=("$@") -} -eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS -JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" - -exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml new file mode 100644 index 000000000..7e4d3c45e --- /dev/null +++ b/kcbq-api/pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + + + com.wepay.kcbq + kcbq-parent + 1.1.3-SNAPSHOT + .. + + + kcbq-api + kafka-connect-bigquery-api + + + + org.apache.kafka + connect-api + + + + com.google.cloud + google-cloud + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml new file mode 100644 index 000000000..5545e657d --- /dev/null +++ b/kcbq-confluent/pom.xml @@ -0,0 +1,96 @@ + + + + 4.0.0 + + + com.wepay.kcbq + kcbq-parent + 1.1.3-SNAPSHOT + .. + + + kcbq-confluent + kafka-connect-bigquery-confluent + + + + com.wepay.kcbq + kcbq-api + + + + org.apache.kafka + connect-api + + + + com.google.cloud + google-cloud + + + org.slf4j + slf4j-api + + + org.apache.avro + avro + + + io.confluent + kafka-connect-avro-converter + + + io.confluent + kafka-schema-registry-client + + + + junit + junit + + + org.mockito + mockito-core + + + org.slf4j + slf4j-log4j12 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.jacoco + jacoco-maven-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + diff --git a/kcbq-confluent/src/test/resources/log4j.properties b/kcbq-confluent/src/test/resources/log4j.properties new file mode 100644 index 000000000..2a5a0233d --- /dev/null +++ b/kcbq-confluent/src/test/resources/log4j.properties @@ -0,0 +1,14 @@ +log4j.rootLogger=INFO, stdout + +# Send the logs to the console. +# +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + +connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n +log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern} +log4j.appender.connectAppender.layout.ConversionPattern=${connect.log.pattern} + +# These are used in the log4j properties file that ships by default with Connect +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.reflections=ERROR diff --git a/kcbq-connector/logos/BigQuery.png b/kcbq-connector/logos/BigQuery.png new file mode 100644 index 0000000000000000000000000000000000000000..a7e0a715624aa53317e0d5f3f9bd0aab9d5f695c GIT binary patch literal 6210 zcmai3c|4Te+rMYbFvv1Bw#Sq$SsF_yvNS_Tda{-!+k~W$J!@kO6*EX#vkqC3!bJ9E z2&KeCMP{;RknSX( z?sm?TUJeCbO`=6leHL|RO}X{8>~0vOp2A#MD=v5ST#N6P5?2tac2L8)eM)=u(r`s; zMD&TGu6MI?`5fm!5S{a2k(0uy8^??A!V$}Kt)qAy_3osle1%$eA7 zc%Inv;;eccV&t_28~|qEjO;igh;N3AOUWIr)Of&KWH{cX+SCmmlf+io| z){5A|X~%>z?n+ic@mF?~*EXW3hky~1wg#dI$ZXkF04{s)!RVqyAN?SX0zf!QoEV59 zJUS;t4D?`9fsN*k2h}YpMp{QSkO=vk8ySbm-Z>*N&XgiHFs{e7U};z7 z8RJ8D4G6b^2>b2Mk^;K3Hqa8j#|aF5y&a#ZvE^^TsN^3zs#lC};}216(W!08oj5IIPlNT}5-6#Qt2QNwYKJ1>|cnC=W1 zh+~=h!Gl=@XxVLlp55Vg!ucHr#@1vMt$mFLN2ni;O0KWVWl=94SITN<5Lgih3|rw>F61&-|GlRSyBt%eM_I zg1P9!4`%+=Lt!tQ*mKc+TI0+uwle+G4g=Y_Z|E?WlPzrFv9`w^4JJajM zC3t-)1+a!caMSNc1Lo^e?n%a#uKFs6fg$#xNq?nKX+Zbe{p2uRFXM@5e*kpWV{=%F)8P6=?Drx$wgn?jYTDT(Wu${l6}*r zCYC{)HxZ!25J=Z*2O9qE-N{Oi1@pKpWjY@hy>17Ae3NF!_;?-{m|eN=NI^05196+$zh4FUo z^mwe}3;adw)b)V7-UCa&;R-0~Ub4-hwbxQMTBXDSXd8BjSK6-ApDQ$vzwYYU?3fni{ zdH06nk-Uks0ZAP3c1J6o1B*ct)5G;=67_Z%_9B{>}U+3PB4Q=GSNDR!+bq~5%9Sg$R=JDzti0X5fEYscuj zi*t%pUi-R}5}{b5LI8$$bFFe%*3pT7UPo5(X!|&zi47i`E}M?etEc1Hk(L{jhG;j5 z$Pf3u1)2`hOfsgS^x^&^H_H#8BP-BeZ9;!ngPqyBF*^%pYNu-YR6QPu)oDn1p7Wwq zZ@l<;=}?D5zwJir1_VD>=TZ5p>{ut4Z({hc-?L;{Fev-A&CfWJ^vqduEsi0lIq#J) zqVDkYFx5?xM!PwHTo zq=t4N`^Yo8o4{pi?MPlvv&rKVA#*6mjezX9+J<>RlwEdt`}&rgUn5q@ai!2u{C(BQ zf7-1-3f1m>$lv+NKe{K;YKxD4bBJbA$%LxH=2yOlE#?2p$4!6rKYH%loRL@gg{p}6 zrp#EPG(Nu75Hm2D1^BhJ>4M?ey!1NYy&IvDXG}rL#N1qMsy5G({ zis`riv)QClT*V%a3+NI6c7{s9DWXxOxMn3b*|4zY}lFRj_|2eqA_|naZAL*H#+beOQ8sTYmnw%%DJ;(w5cIhSt$c zGfiZ3r3`vTu}h|_w{5_PFk&+`iwz~Mj&xKgt-E>rK29daC{yjYK$N4y7An_w22;d3 zM>hjL$~I{~%<3KY>sKn9peLPE{KyKGFv1o7dMv(tMFa?J*>fJXZWwjB-CG&h@8YQ? zsUuhA_1@I|VKKs_9vEzJ0E>(2TFUR8Yqd+-TcerjZo96A<1-O%9c@D_i+i4KVn(=} zXRg}w47HXhUSe)@#^h|+2y zrnEA_J=F)JWNa+^>(qN!IO~!F`9Bf^qWo>8-J>TVcb)dr zfwk<5iY|;7A;ZrWrf|CZXWcaDI2Qt^t!K2Xjr>;q@;S{4_T#Cd#&Z>W2D=UjOti8KKQD4*!zb^yR%b z%#OIex0AD}6D+{P$1wwool(@IC0Ho*IDDUP~8`jbk$KGKv1n(zi+!~2H?DBw{0`u(>O9I;$R6A zzIMG2Qng-Za;-tC=8r(cUW6Db6s~P`#Gsgd?+Gfpqvm!|-Cdw&(Wc&OFQx9QponxM zS4X2ZO?C(sO{$3nG#1a)Ul7dLedcE_4_#-Vp)Q-j!P_c>On9_7sp4r_MXEGD1rRI` z1V~N0oD(fkrH66X8X^aa5xtg|J4#B=fR%1La-HMCZZDmI8X5c9lT|1!xDvuO1DK2( zNxn5BLUHm?h%kf)C_upMVDdmL_-z1eZEU=FnDvflr#Vf`yY1zt)a zl?!%rqe<(ruOV{qaa4Bx$qH_5;>cxIbw%>=gYCaQr$Ug?)&(G$;u#Y*OwD~c15j8frrB0PRqD!rHpWOjZyyN5PhTjX5jvmL7ur}r7BIvwr9^}MO(hdyvOH1h$`|ORVO2Br`4>8R{z6%y zPdIHIcgap33K#1p4Yu*TIA6@0{A0(w6EEJtlyAMRwe(uAj-qVwc+{oqYO=AkpvuYv zIGkR?jpi{~EFX(sw~*GGSgQUG7KH-$CCYH%$59D@uxV-9bG;%Ok)t}jPCf)!!y4&! z7Lf;Z)4QgyCWMQD21nx{; z`>THVnBa4MUgl&7)^wt)FFUV@Bc|s@O~@!JFU>cq@rwL<|LShzY<{wla7bzQ!-n$C z^}`@tS}G<`GnMZv<47O{QQFqgIZ}q|>ta`$Pc#grR2{r z{A))*g3bpq5?H%Zp$5& zZr0M~D$UqG8QBXHSz6EpxYJ|axYYZ8LF*hH(p0rSzfJwYX`IRI-KTB~m2n8v6pIor zodBP5!u2mq8RCMhEG`G65Y)QH?kG%Ao_x|1j6MX?O=_lu#>=mhwiT~#7e%!58Nzw# z^d3E-I7FTVqSvhE@O;CyKfmi7S;ILZIv_a_)$nP_&3^@|1$xrh`bB;Ad5|Bl{+xHo zeIlyt6m1dzosCO?c-r0kIYtI*0IfGKh26eysq)XuXKa02ap&A`%Q}|Bx@?0Okt1ILb-KTfbbY4!>2Xl5h@1L?9Y(BODn~>RQs!s;CgdG-q znP}XoBkm*-0<2=ZPJoRj^!xoOs}#~^&VPwi|InKun+*JVdRjVf8)G$ih4uGNM3ui8 z^+Z&u#5p(}6@3bLV?QvdyXnX699Gg>Yp`U1$We?W6H!?6PHC$XEj2)fft*~?lw445)?Fq_3-Mueb zoGF8?ZC_McvuH=535e-r@{YF%kRmhwm6X1rdzmcJvMK~pz{rTiZy8p@yVc_9wdzk0 z!$WMQ8dtKW2!peml-|S2?Raw0z7zAj#5xg-AzF${Uv_GOkBg7OdBjEz9r97OEs2+? z0BylzJSm7vv2rw!*v^I)S+SklmS;oT>$c>fR?>KdwLp;c(d`(g_VvYq1iPvB`KMdb zzIacbH~mQ};zm}mSR=e`0mqqC!T%N@h7A&+CJLe)Hc$`+xVcv{gh^}v+CM(=8$VArZ1?6hL@ z?4>1+H6^$*LK7Km2J0w39vPMUESEG35$Tt>vdnvq`G{8p$)$-bMp(c)dpr(fANBh~ z#+f7(M8$A?bJ0hiO!SiWtDml2q)m}e@XOSUrLBGMzMrY0 z+-YBdf|MH3xw-^6y0gvPcfkvuOcm7?nxg{LES@MqmY`o(HK=S_y(+8h;)qUw9~OcD z9e-d!sOV4b9?qXaw!U*?QCI*Sq_)=d#z(6`@Tp*=10&2`m=`HqAG*uPx>#}hpCgX$ z;H#(6lIxQOv(pWLs&Ek^8EV5S9yW(_{dX1FeKpp3u_!TgslWqMpV6Nyjr-}^#>|VX zWu_1}K@--;JE<=tZ-WwjBfjW|MlJ-VVYBo)v~&Ug?M<`@7?+dy{onZWx(KE`xKRtD z1&qXCM+mdOEju+QKtww17~-PHMsg`F{~bS3nP^7ebFXen3tv;b0}=xiBtP&YWp!8k zG%<&FU(n*%@EGFw`ajJ9P`X{R^!WS9`~JNzd|HD-fCRV#l;DOop2<&*{K7M)VTQ1R zZFfjrc5REui8nnlZXn7VE+Av@US5d(>jy9coq?^KqYaxMhwRH`pr>51-_W=|_`dHv z?~`fyxBr7cy#`uyv1{1;9M?V|Rs8#{y!r1DkhkJIf@C6p^YfdtArGuAPjfmV-1J}J z4egTm^rAdJxZITanf1-F5zvOb>hW+H_$~H(<(e4iz|`77RYH0AY-tk*$m#bjgbPW2 zs`yqtILE6FLd%b^y^!q9cF4XSPw$>45O558xdfLvmodo@=L!Av_UY+tVVoO(h?5M^f@>{ZG{nxqlk2aJI=uf52iw*LS> z2;b8TPT1L=0YpK?SU=UB+^{bHyGN{XY6Wdf&&+sQ)K4k_zKUCpv-mUI-_H*KBX-U~ z(4@k!*=a9`NvHH#fY5jF2|~*?W7@zV987!3 zT;j<{AvU}R!hkcl?G~gG7b&f+V}w>Mbzxol%R+>S(BAAR7YX*JGdmoStcel@s-la# zCWye5xdn5sDh;La5kNb7QcN>pu{yf`jefmJpbua~l(z6*!p`5fy`umu8{vu4IHdf` z4J?0e3!l~5cZVSJhjBGyPj8@peDresR`U`_XfVkne|IBP#g8S)2;S+=1}bkCkW4v5QzrMuQ~72ux&z3;zH5PT{C literal 0 HcmV?d00001 diff --git a/kcbq-connector/logos/confluent.png b/kcbq-connector/logos/confluent.png new file mode 100644 index 0000000000000000000000000000000000000000..14cd8c506e46fc70ccd274c4c9f4f655ffb14800 GIT binary patch literal 3156 zcmV-a46E~rP) zd30699mlUwYHdZ^g2uun>JqI2H9W8*6n3Z6Hpv8-!2c@~#0Z6;^@O;5l%OlI9Omnl%sR z6s>q5r+6T6Y>8JTRI}{EzF0$W+7=N-LD%IqyXxH z?Co7)!?Hge;Jn{+( z3E2rrc7yQe=*@XCtL^BE;g}>}*_Qw&Qve`)Dwn-8{a#QM9aANZd%Sl-_6W}3DYEvq&?+&uhcAFqnOpz!eV*Eg27Tv}A>fKdLnlc(P-`IHqDmYRe=3p)D9 zo&v^#zk?lMCnyBJ2M>V;BGy9oWRMBo2HSXE#b7=d46e5<>{g&Rm-0{idMK~`5PdIr z7eBA$*WErqB`$F*BKZyQBv{S!xyvV=jqG1#?#xb*2k9z)4?_Z^`4;#Mzji8@&EOp{ z3EUGPpKF;r%ajFP1?vEn>s}V_=7p%dH-g#VLFOU5M)-rgtQH}U6_9-wWM2x&8youU zW%uIWL14h5q9X(HHyyeGvO8RMXT>{3&6cj+JpjTFq!c&A>bsJ=e@hl7HVP zD#IWE5XLb+L5>3AAeA3y--cQQ!%a+3RlG0pjaSgjH z%J`oVIn4~W$V+B7Wbb1k`++KHGukkg-aG^xA`&dg9{CxvJCNN`WOpFD1KF(xx5@>5 z4U|b2bQ~-IG=O^^H<62{vQW9`i5l4(gCZq%F?gA|Ezk_`H|6*reJbJ~Jnkb<0CKN#nPAR`#{Mu)N zS-dQED>0AxoI8uhl{5FvKyUJC{9F^ji~O^aNS^5}7-a7u%kCk(Ec=3dy^8$S#Xc1HYRzg`1|x_PU`WR`Hlg zrV6{5byH&uf_^Er^e_2T@M0c!6x?VbV=S^$*?!3~-@cM?O*Ju@QkiYxX>|F0m$@@d zB0j|sE7lGyW~P!JVi~PJaN|Q~;Ky|qvgb#AZjwC=Wdule_cNb?88SLKrn5t%U_&2$ zwNw~)d*vg^E_f4c!c8}&0f+SYIA-6H2;}b008NaLeX=Pp$nNkF{}dj727FPDxsY|U z%(Cz+H8VNa@>+w)E@G&w_65s){bX+?mDLK%JgGax2Z_e+nUMWI{&^-jXGgVfRgo1s zA1=R)){g-vLUcTTD8dhaaM>$AvzJ|<#B`Dh?@?Vo9h(q zOtMcgF13W^&%d$XulNM>7uAhF^TgFh~7+*{rmjbL=Pq*qw$cvg+cad%;IS_%O3gt z5uVpcX5W$tx=NMt4UgkNWhY8`nqh`C%X!RYhTKbQ84a=O8f<}8BR|wHSvK$4~P}83hyZs&ySlM2T=Xt#v zV|p==8sTxwD7%N~#5OMb6NYovr0j!O<(tlL8c{3z`Rv;S*Sk)H@2QuRW{gMF~%2E+&x<(Hkl4QgRVyvyP%#V3q zJ7Y*oGKl{EAdll#-57;pp5`e-Oijw(o5xfzcMK7e?9!$+oG0`Nm;DD)8j_b5e`r4a zJp(&9T{P#8MfMRq%?E}UE7_|;qGttM8mD+k_6ynVWaO@pZH4H#L&~drz_b&1y7SCk zd8`iE`4p-4wO!M)cN0^f=SHSB0}kPnuVk*XWm(sEyG{hX@ zG5vHgQNQRGhs1pQob?|$1*Yx2lI#VAWl_Sj%2@-(E8nzoN2E*fcPPhhlaA4A^>&}U zz5`D4_;XDE<>tC5JHfkBNf0P0WRWl5Umt$>b&E>!qTR0zahF4Oo0eUy6Z+Ch&5;6SM=j@bf>w+^qnUA$txa?^KPli-zxgLrk`Gt@J%#TK9-rckq(773cs` zL5fE9ROxsG$es$>Q)JnN@X|@gvS|54L4NG{ zw*T&d(ezlJ%*|7M(Zfs|ok}7zgbjh#)#eqrqx*;)QzKQ$-%sAsa(Fv+{WQj{!#&@rLIqVem?{a{%H)xCksMO z^we`y`lkGzJim1A?mMc&)fjxE`;>H-GNo<0&JMiXXg4M8TJrE!05iPLZ9hFVZD396rZB>UqFcROh>ZSwsiNcYa(5ym9@ z(CL2;6aNZIP<_;K*-3Dj>XtnreSM-1%dU!Eld>n|uMgBw*}X&u%^~|Ukah-`uJ8)~ zcQ{0EAj#emJPT&hamYRcvQLB1cl!ySkiV``2W9V{w^hgdo-?=j{?7<{^5)nF;Em?EeRy<<9*ujhS5l0000 + + + 4.0.0 + + + com.wepay.kcbq + kcbq-parent + 1.1.3-SNAPSHOT + .. + + + kcbq-connector + kafka-connect-bigquery + + + + org.apache.kafka + connect-api + + + + com.fasterxml.jackson.core + jackson-core + + + com.google.guava + guava + + + com.google.code.findbugs + jsr305 + + + com.google.http-client + google-http-client + + + com.google.http-client + google-http-client-jackson2 + + + com.google.cloud + google-cloud + + + com.google.auth + google-auth-library-oauth2-http + + + org.xerial.snappy + snappy-java + + + org.slf4j + slf4j-api + + + io.debezium + debezium-core + + + + com.wepay.kcbq + kcbq-api + + + com.wepay.kcbq + kcbq-confluent + + + + junit + junit + + + org.mockito + mockito-core + + + org.slf4j + slf4j-log4j12 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.jacoco + jacoco-maven-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + io.confluent + kafka-connect-maven-plugin + + + + kafka-connect + + + BigQuery Sink Connector + kafka-connect-bigquery + + A sink connector for writing to Google BigQuery, with support for automatic table creation and schema evolution. + + logos/BigQuery.png + https://docs.confluent.io/current/connect/kafka-connect-bigquery/ + https://github.com/confluentinc/kafka-connect-bigquery + + Confluent, Inc. + supports WePay's BigQuery connector version 1.1.2 and later, as part of a Confluent Platform subscription. + ]]> + https://docs.confluent.io/current/connect/kafka-connect-bigquery/ + logos/confluent.png + + wepay + organization + WePay + https://go.wepay.com/ + + true + + + sink + + + + cloud + analytics + data + gcp + google + bigquery + warehouse + platform + nosql + + + + Apache Kafka 0.11 or higher / Confluent Platform 3.3 or higher + Java 1.8 or higher + Active Google Cloud Platform (GCP) account with authorization to create resources + Kafka Connect 0.11 or higher / Confluent Platform 3.3 or higher + + + + + + + + diff --git a/kcbq-connector/src/integration-test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java similarity index 100% rename from kcbq-connector/src/integration-test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java rename to kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java diff --git a/kcbq-connector/src/integration-test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java similarity index 100% rename from kcbq-connector/src/integration-test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java rename to kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java diff --git a/kcbq-connector/src/test/resources/log4j.properties b/kcbq-connector/src/test/resources/log4j.properties new file mode 100644 index 000000000..2a5a0233d --- /dev/null +++ b/kcbq-connector/src/test/resources/log4j.properties @@ -0,0 +1,14 @@ +log4j.rootLogger=INFO, stdout + +# Send the logs to the console. +# +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + +connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n +log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern} +log4j.appender.connectAppender.layout.ConversionPattern=${connect.log.pattern} + +# These are used in the log4j properties file that ships by default with Connect +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.reflections=ERROR diff --git a/kcbq-connector/test/docker/connect/connect-docker.sh b/kcbq-connector/test/docker/connect/connect-docker.sh index f5d103863..d0f78fafd 100755 --- a/kcbq-connector/test/docker/connect/connect-docker.sh +++ b/kcbq-connector/test/docker/connect/connect-docker.sh @@ -14,8 +14,7 @@ # specific language governing permissions and limitations # under the License. -tar -C /usr/share/java/kafka-connect-bigquery/ -xf /usr/share/java/kafka-connect-bigquery/kcbq.tar -tar -C /usr/share/java/kafka-connect-bigquery/ -xf /usr/share/java/kafka-connect-bigquery/retriever.tar +unzip -j -d /usr/share/java/kafka-connect-bigquery /usr/share/java/kafka-connect-bigquery/kcbq.zip 'wepay-kafka-connect-bigquery-*/lib/*.jar' connect-standalone \ /etc/kafka-connect-bigquery/standalone.properties \ diff --git a/kcbq-connector/test/integrationtest.sh b/kcbq-connector/test/integrationtest.sh index a65ae7038..df6f17de2 100755 --- a/kcbq-connector/test/integrationtest.sh +++ b/kcbq-connector/test/integrationtest.sh @@ -84,7 +84,6 @@ log() { } BASE_DIR=$(dirname "$0") -GRADLEW="$BASE_DIR/../../gradlew" #################################################################################################### # Configuration processing @@ -194,19 +193,17 @@ docker start -a "$POPULATE_DOCKER_NAME" # Deleting existing BigQuery tables warn 'Deleting existing BigQuery test tables' -"$GRADLEW" -p "$BASE_DIR/.." \ - -Pkcbq_test_keyfile="$KCBQ_TEST_KEYFILE" \ - -Pkcbq_test_project="$KCBQ_TEST_PROJECT" \ - -Pkcbq_test_dataset="$KCBQ_TEST_DATASET" \ - -Pkcbq_test_tables="$(basename "$BASE_DIR"/resources/test_schemas/* | sed -E -e 's/[^a-zA-Z0-9_]/_/g' -e 's/^(.*)$/kcbq_test_\1/' | xargs echo -n)" \ - integrationTestPrep +TEST_TABLES="$(basename "$BASE_DIR"/resources/test_schemas/* | sed -E -e 's/[^a-zA-Z0-9_]/_/g' -e 's/^(.*)$/kcbq_test_\1/' | xargs echo -n)" +mvn -f "$BASE_DIR/.." clean test-compile exec:java \ + -Dexec.mainClass=com.wepay.kafka.connect.bigquery.it.utils.TableClearer \ + -Dexec.classpathScope=test \ + -Dexec.args="${KCBQ_TEST_KEYFILE} ${KCBQ_TEST_PROJECT} ${KCBQ_TEST_DATASET} ${TEST_TABLES}" #################################################################################################### # Executing connector in standalone mode (this is the execution portion of the actual test) statusupdate 'Executing Kafka Connect in Docker' -# Run clean task to ensure there's only one connector tarball in the build directory -"$GRADLEW" -q -p "$BASE_DIR/../.." clean confluentTarBall +mvn -f "$BASE_DIR/.." install -Dskip.unit.tests=true [[ ! -e "$DOCKER_DIR/connect/properties" ]] && mkdir "$DOCKER_DIR/connect/properties" RESOURCES_DIR="$BASE_DIR/resources" @@ -232,8 +229,7 @@ echo >> "$CONNECTOR_PROPS" CONNECT_DOCKER_IMAGE='kcbq/connect' CONNECT_DOCKER_NAME='kcbq_test_connect' -cp "$BASE_DIR"/../../bin/tar/kcbq-connector-*-confluent-dist.tar "$DOCKER_DIR/connect/kcbq.tar" -cp "$BASE_DIR"/../../bin/tar/kcbq-connector-*-confluent-dist.tar "$DOCKER_DIR/connect/retriever.tar" +cp "$BASE_DIR"/../target/components/packages/wepay-kafka-connect-bigquery-*.zip "$DOCKER_DIR/connect/kcbq.zip" cp "$KCBQ_TEST_KEYFILE" "$DOCKER_DIR/connect/key.json" if ! dockerimageexists "$CONNECT_DOCKER_IMAGE"; then @@ -242,8 +238,7 @@ fi docker create --name "$CONNECT_DOCKER_NAME" \ --link "$KAFKA_DOCKER_NAME:kafka" --link "$SCHEMA_REGISTRY_DOCKER_NAME:schema-registry" \ -t "$CONNECT_DOCKER_IMAGE" /bin/bash -docker cp "$DOCKER_DIR/connect/kcbq.tar" "$CONNECT_DOCKER_NAME:/usr/share/java/kafka-connect-bigquery/kcbq.tar" -docker cp "$DOCKER_DIR/connect/retriever.tar" "$CONNECT_DOCKER_NAME:/usr/share/java/kafka-connect-bigquery/retriever.tar" +docker cp "$DOCKER_DIR/connect/kcbq.zip" "$CONNECT_DOCKER_NAME:/usr/share/java/kafka-connect-bigquery/kcbq.zip" docker cp "$DOCKER_DIR/connect/properties/" "$CONNECT_DOCKER_NAME:/etc/kafka-connect-bigquery/" docker cp "$DOCKER_DIR/connect/key.json" "$CONNECT_DOCKER_NAME:/tmp/key.json" docker start -a "$CONNECT_DOCKER_NAME" @@ -252,12 +247,12 @@ docker start -a "$CONNECT_DOCKER_NAME" # Checking on BigQuery data via Java test (this is the verification portion of the actual test) statusupdate 'Verifying that test data made it successfully to BigQuery' -INTEGRATION_TEST_RESOURCE_DIR="$BASE_DIR/../src/integration-test/resources" -[[ ! -d "$INTEGRATION_TEST_RESOURCE_DIR" ]] && mkdir -p "$INTEGRATION_TEST_RESOURCE_DIR" -INTEGRATION_TEST_PROPERTIES_FILE="$INTEGRATION_TEST_RESOURCE_DIR/test.properties" +TEST_RESOURCE_DIR="$BASE_DIR/../src/test/resources" +[[ ! -d "$TEST_RESOURCE_DIR" ]] && mkdir -p "$TEST_RESOURCE_DIR" +INTEGRATION_TEST_PROPERTIES_FILE="$TEST_RESOURCE_DIR/test.properties" echo "keyfile=$KCBQ_TEST_KEYFILE" > "$INTEGRATION_TEST_PROPERTIES_FILE" echo "project=$KCBQ_TEST_PROJECT" >> "$INTEGRATION_TEST_PROPERTIES_FILE" echo "dataset=$KCBQ_TEST_DATASET" >> "$INTEGRATION_TEST_PROPERTIES_FILE" -"$GRADLEW" -p "$BASE_DIR/.." cleanIntegrationTest integrationTest +mvn -f "$BASE_DIR/.." -Dskip.unit.tests=true integration-test \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000..b05ee38e7 --- /dev/null +++ b/pom.xml @@ -0,0 +1,346 @@ + + + + 4.0.0 + + com.wepay.kcbq + kcbq-parent + 1.1.3-SNAPSHOT + pom + + + kcbq-api + kcbq-confluent + kcbq-connector + + + + 8 + + 1.8.1 + 3.2.0 + 0.4.0 + 3.0.0 + 0.9.0 + 0.25.0-alpha + 1.22.0 + 20.0 + 2.6.3 + 1.0.0 + 1.7.25 + 1.1.4 + + 4.12 + 1.10.19 + + 2.15 + 6.18 + 3.8.1 + 0.8.5 + 0.11.1 + 2.5.3 + 3.0.0-M4 + + ${maven.test.skip} + + + kafka-connect-bigquery-parent + + https://github.com/confluentinc/kafka-connect-bigquery + + 2016 + + + + Apache License 2.0 + https://www.apache.org/licenses/LICENSE-2.0 + repo + + + + + scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git + scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git + https://github.com/confluentinc/kafka-connect-bigquery + HEAD + + + + + C0urante + Chris Egerton + fearthecellos@gmail.com + America/New_York + + + moirat + Moira Tagle + moirat@wepay.com + America/Los_Angeles + + + + + + confluent + http://packages.confluent.io/maven/ + + + jcenter + https://jcenter.bintray.com + + + + + + confluent + http://packages.confluent.io/maven/ + + + jcenter + https://jcenter.bintray.com + + + + + + + + com.wepay.kcbq + kcbq-api + ${project.version} + + + com.wepay.kcbq + kcbq-confluent + ${project.version} + + + + + org.apache.kafka + connect-api + ${kafka.version} + provided + + + org.apache.kafka + kafka-clients + ${kafka.version} + provided + + + + com.google.cloud + google-cloud + ${google.cloud.version} + + + com.google.auth + google-auth-library-oauth2-http + ${google.auth.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + io.debezium + debezium-core + ${debezium.version} + + + org.apache.avro + avro + ${avro.version} + + + io.confluent + kafka-connect-avro-converter + ${confluent.version} + + + io.confluent + kafka-schema-registry-client + ${confluent.version} + + + + + org.xerial.snappy + snappy-java + ${snappy.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.google.guava + guava + ${guava.version} + + + com.google.code.findbugs + jsr305 + ${findbugs.jsr305.version} + + + com.google.http-client + google-http-client + ${google.http.version} + + + com.google.http-client + google-http-client-jackson2 + ${google.http.version} + + + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + test + + + + + + + + org.apache.maven.plugins + maven-release-plugin + ${release.plugin.version} + + true + false + v@{project.version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${compiler.plugin.version} + + ${java.version} + ${java.version} + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.plugin.version} + + + **/*IntegrationTest.java + + ${skip.unit.tests} + + + + org.apache.maven.plugins + maven-failsafe-plugin + ${surefire.plugin.version} + + + **/*IntegrationTest.java + + + + + integration-test + + integration-test + + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.plugin.version} + + + pre-unit-test + + prepare-agent + + + + report + verify + + report + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${checkstyle.plugin.version} + + + validate + validate + + ${project.parent.basedir}/config/checkstyle/google_checks.xml + + + check + + + + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + + + io.confluent + kafka-connect-maven-plugin + ${kafka.connect.plugin.version} + + + + + diff --git a/settings.gradle b/settings.gradle deleted file mode 100644 index a24368b32..000000000 --- a/settings.gradle +++ /dev/null @@ -1 +0,0 @@ -include 'kcbq-connector', 'kcbq-api', 'kcbq-confluent' From b6e82986866e942dc87fb532e0d476f0d5aceed6 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 23 Sep 2020 14:59:10 -0400 Subject: [PATCH 14/56] MINOR: Add Jenkinsfile (#35) * MINOR: Add Jenkinsfile * Fix build error while running 'mvn site' * Prevent integration test classes from running during `mvn integration-test` * Disable Checkstyle on Jenkins * Fix integration test --- Jenkinsfile | 7 +++++ config/checkstyle/suppressions.xml | 8 +++++ kcbq-connector/pom.xml | 4 --- kcbq-connector/test/integrationtest.sh | 4 +-- pom.xml | 41 ++++++++++++++++++++++---- 5 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 Jenkinsfile create mode 100644 config/checkstyle/suppressions.xml diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..551c15919 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,7 @@ +#!/usr/bin/env groovy +common { + slackChannel = '#connect-warn' + nodeLabel = 'docker-oraclejdk8' + publish = false + downStreamValidate = false +} diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml new file mode 100644 index 000000000..28a683b60 --- /dev/null +++ b/config/checkstyle/suppressions.xml @@ -0,0 +1,8 @@ + + + + + + \ No newline at end of file diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index d1a4a2a4b..6f364dab3 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -107,10 +107,6 @@ org.apache.maven.plugins maven-surefire-plugin - - org.apache.maven.plugins - maven-failsafe-plugin - org.jacoco jacoco-maven-plugin diff --git a/kcbq-connector/test/integrationtest.sh b/kcbq-connector/test/integrationtest.sh index df6f17de2..4ea70bef7 100755 --- a/kcbq-connector/test/integrationtest.sh +++ b/kcbq-connector/test/integrationtest.sh @@ -17,7 +17,7 @@ #################################################################################################### # Basic script setup -set -e +set -ex if [[ -t 1 ]]; then KCBQ_TEST_COLORS='true' @@ -255,4 +255,4 @@ echo "keyfile=$KCBQ_TEST_KEYFILE" > "$INTEGRATION_TEST_PROPERTIES_FILE" echo "project=$KCBQ_TEST_PROJECT" >> "$INTEGRATION_TEST_PROPERTIES_FILE" echo "dataset=$KCBQ_TEST_DATASET" >> "$INTEGRATION_TEST_PROPERTIES_FILE" -mvn -f "$BASE_DIR/.." -Dskip.unit.tests=true integration-test \ No newline at end of file +mvn -f "$BASE_DIR/.." clean test-compile -Dskip.unit.tests=true failsafe:integration-test@verify-docker-test \ No newline at end of file diff --git a/pom.xml b/pom.xml index b05ee38e7..adea1235d 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,7 @@ 0.8.5 0.11.1 2.5.3 + 3.7.1 3.0.0-M4 ${maven.test.skip} @@ -277,17 +278,17 @@ org.apache.maven.plugins maven-failsafe-plugin ${surefire.plugin.version} - - - **/*IntegrationTest.java - - - integration-test + verify-docker-test integration-test + + + **/*IntegrationTest.java + + @@ -321,6 +322,7 @@ validate ${project.parent.basedir}/config/checkstyle/google_checks.xml + ${project.parent.basedir}/config/checkstyle/suppressions.xml check @@ -335,6 +337,15 @@ + + + org.apache.maven.plugins + maven-site-plugin + ${site.plugin.version} + io.confluent kafka-connect-maven-plugin @@ -343,4 +354,22 @@ + + + jenkins + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + + + From f0ec33958239d2e9783d54ba89a4a2ae796f27f7 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 23 Sep 2020 15:51:43 -0400 Subject: [PATCH 15/56] MINOR: Fix BigQueryWriterTest unit test (#36) --- .../connect/bigquery/write/row/BigQueryWriterTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java index 7c0e871e5..ca5743f44 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java @@ -116,8 +116,7 @@ public void testAutoCreateTables() { when(insertAllResponse.hasErrors()).thenReturn(false); when(insertAllResponse.getInsertErrors()).thenReturn(emptyMap); - BigQueryException missTableException = mock(BigQueryException.class); - when(missTableException.getCode()).thenReturn(404); + BigQueryException missTableException = new BigQueryException(404, "Table is missing"); when(bigQuery.insertAll(anyObject())).thenThrow(missTableException).thenReturn(insertAllResponse); @@ -151,8 +150,7 @@ public void testNonAutoCreateTables() { when(insertAllResponse.hasErrors()).thenReturn(false); when(insertAllResponse.getInsertErrors()).thenReturn(emptyMap); - BigQueryException missTableException = mock(BigQueryException.class); - when(missTableException.getCode()).thenReturn(404); + BigQueryException missTableException = new BigQueryException(404, "Table is missing"); when(bigQuery.insertAll(anyObject())).thenThrow(missTableException).thenReturn(insertAllResponse); From 4eee99ed48f2207cf66e88cf219a3c7d80a9bc27 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Fri, 2 Oct 2020 14:44:02 -0400 Subject: [PATCH 16/56] GH-39: Fix NPE when null inner values are encountered with field sanitization enabled (#40) --- .../bigquery/utils/FieldNameSanitizer.java | 21 +++++++++-------- .../utils/FieldNameSanitizerTest.java | 23 +++++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/FieldNameSanitizer.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/FieldNameSanitizer.java index 09aeb70c2..880d5fd22 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/FieldNameSanitizer.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/FieldNameSanitizer.java @@ -1,5 +1,6 @@ package com.wepay.kafka.connect.bigquery.utils; +import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -20,15 +21,17 @@ public static String sanitizeName(String name) { // letters, numbers, and underscores. // Note: a.b and a/b will have the same value after sanitization which will cause Duplicate key // Exception. + @SuppressWarnings("unchecked") public static Map replaceInvalidKeys(Map map) { - return map.entrySet().stream().collect(Collectors.toMap( - (entry) -> sanitizeName(entry.getKey()), - (entry) -> { - if (entry.getValue() instanceof Map) { - return replaceInvalidKeys((Map) entry.getValue()); - } - return entry.getValue(); - } - )); + Map result = new HashMap<>(); + map.forEach((key, value) -> { + String sanitizedKey = sanitizeName(key); + if (value instanceof Map) { + result.put(sanitizedKey, replaceInvalidKeys((Map) value)); + } else { + result.put(sanitizedKey, value); + } + }); + return result; } } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/FieldNameSanitizerTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/FieldNameSanitizerTest.java index 3358c1386..7b02c009b 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/FieldNameSanitizerTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/FieldNameSanitizerTest.java @@ -1,5 +1,6 @@ package com.wepay.kafka.connect.bigquery.utils; +import java.util.Collections; import java.util.Map; import org.junit.Before; import org.junit.Test; @@ -75,4 +76,26 @@ public void testInvalidSymbol() { // Validate map size. assertEquals(5, sanitizedMap.size()); } + + /** + * Verifies that null values are acceptable while sanitizing keys. + */ + @Test + public void testNullValue() { + assertEquals( + Collections.singletonMap("abc", null), + FieldNameSanitizer.replaceInvalidKeys(Collections.singletonMap("abc", null))); + } + + @Test + public void testDeeplyNestedNullValues() { + testMap = new HashMap<>(); + testMap.put("top", null); + testMap.put("middle", Collections.singletonMap("key", null)); + testMap.put("bottom", Collections.singletonMap("key", Collections.singletonMap("key", null))); + assertEquals( + testMap, + FieldNameSanitizer.replaceInvalidKeys(testMap) + ); + } } From 4ab4917209e812bb0edd8130c83795f3f54148bb Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Tue, 6 Oct 2020 13:48:02 -0400 Subject: [PATCH 17/56] MINOR: Modify license headers, add automated license header generation and enforcement (#38) * GH-32: Switch from Gradle to Maven for build tool * Add license plugin and reformat copyright notice on (almost) all files --- Jenkinsfile | 18 +++++ config/checkstyle/suppressions.xml | 20 +++++ config/copyright/custom-header-styles.xml | 44 +++++++++++ kcbq-api/pom.xml | 39 ++++++---- .../connect/bigquery/api/SchemaRetriever.java | 19 +++++ kcbq-confluent/pom.xml | 39 ++++++---- .../SchemaRegistrySchemaRetriever.java | 19 +++++ .../SchemaRegistrySchemaRetrieverConfig.java | 19 +++++ .../SchemaRegistrySchemaRetrieverTest.java | 19 +++++ .../src/test/resources/log4j.properties | 19 +++++ kcbq-connector/pom.xml | 39 ++++++---- .../quickstart/avro-console-producer.sh | 6 +- kcbq-connector/quickstart/connector.sh | 6 +- kcbq-connector/quickstart/kafka.sh | 6 +- .../properties/connector.properties | 6 +- .../properties/standalone.properties | 6 +- kcbq-connector/quickstart/schema-registry.sh | 6 +- kcbq-connector/quickstart/zookeeper.sh | 6 +- .../connect/bigquery/BigQueryHelper.java | 8 +- .../bigquery/BigQuerySinkConnector.java | 7 +- .../connect/bigquery/BigQuerySinkTask.java | 7 +- .../kafka/connect/bigquery/SchemaManager.java | 19 +++++ .../bigquery/config/BigQuerySinkConfig.java | 7 +- .../config/BigQuerySinkConnectorConfig.java | 7 +- .../config/BigQuerySinkTaskConfig.java | 7 +- .../convert/BigQueryRecordConverter.java | 7 +- .../convert/BigQuerySchemaConverter.java | 7 +- .../bigquery/convert/RecordConverter.java | 7 +- .../bigquery/convert/SchemaConverter.java | 7 +- .../kafkadata/KafkaDataBQRecordConverter.java | 7 +- .../kafkadata/KafkaDataBQSchemaConverter.java | 7 +- .../DebeziumLogicalConverters.java | 7 +- .../logicaltype/KafkaLogicalConverters.java | 7 +- .../logicaltype/LogicalConverterRegistry.java | 7 +- .../logicaltype/LogicalTypeConverter.java | 7 +- .../exception/BigQueryConnectException.java | 7 +- .../exception/ConversionConnectException.java | 7 +- .../exception/SinkConfigConnectException.java | 7 +- .../retrieve/MemorySchemaRetriever.java | 19 +++++ .../bigquery/utils/PartitionedTableId.java | 7 +- .../bigquery/utils/TopicToTableResolver.java | 7 +- .../kafka/connect/bigquery/utils/Version.java | 7 +- .../write/batch/CountDownRunnable.java | 7 +- .../write/batch/KCBQThreadPoolExecutor.java | 7 +- .../bigquery/write/batch/TableWriter.java | 7 +- .../write/row/AdaptiveBigQueryWriter.java | 7 +- .../bigquery/write/row/BigQueryWriter.java | 7 +- .../write/row/SimpleBigQueryWriter.java | 7 +- .../bigquery/BigQuerySinkConnectorTest.java | 7 +- .../bigquery/BigQuerySinkTaskTest.java | 7 +- .../connect/bigquery/SchemaManagerTest.java | 7 +- .../SinkConnectorPropertiesFactory.java | 7 +- .../bigquery/SinkPropertiesFactory.java | 7 +- .../bigquery/SinkTaskPropertiesFactory.java | 7 +- .../config/BigQuerySinkConfigTest.java | 7 +- .../BigQuerySinkConnectorConfigTest.java | 7 +- .../config/BigQuerySinkTaskConfigTest.java | 7 +- .../convert/BigQueryRecordConverterTest.java | 7 +- .../convert/BigQuerySchemaConverterTest.java | 7 +- .../KafkaDataBQRecordConvertTest.java | 7 +- .../KafkaDataBQSchemaConverterTest.java | 7 +- .../DebeziumLogicalConvertersTest.java | 7 +- .../KafkaLogicalConvertersTest.java | 7 +- .../it/BigQueryConnectorIntegrationTest.java | 7 +- .../bigquery/it/utils/TableClearer.java | 7 +- .../retrieve/MemorySchemaRetrieverTest.java | 21 ++++- .../utils/PartitionedTableIdTest.java | 7 +- .../utils/TopicToTableResolverTest.java | 8 +- .../write/row/BigQueryWriterTest.java | 7 +- .../src/test/resources/log4j.properties | 19 +++++ .../src/test/resources/test.properties | 22 ++++++ kcbq-connector/test/docker/connect/Dockerfile | 13 +--- .../test/docker/connect/connect-docker.sh | 6 +- .../test/docker/populate/Dockerfile | 13 +--- .../test/docker/populate/populate-docker.sh | 6 +- kcbq-connector/test/integrationtest.sh | 6 +- .../resources/connector-template.properties | 6 +- .../resources/standalone-template.properties | 6 +- pom.xml | 76 +++++++++++++++---- 79 files changed, 670 insertions(+), 236 deletions(-) create mode 100644 config/copyright/custom-header-styles.xml create mode 100644 kcbq-connector/src/test/resources/test.properties diff --git a/Jenkinsfile b/Jenkinsfile index 551c15919..a080d23d6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,22 @@ #!/usr/bin/env groovy +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ common { slackChannel = '#connect-warn' nodeLabel = 'docker-oraclejdk8' diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index 28a683b60..4bf4c40b5 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -1,4 +1,24 @@ + + + + + /* + * + */EOL + (\s|\t)*/\*.*$ + .*\*/(\s|\t)*$ + false + true + false + + + /* + * + */ + #!.* + (\s|\t)*/\*.* + .*\*/(\s|\t)*$ + false + true + false + + \ No newline at end of file diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index 7e4d3c45e..ae804c5d5 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -1,19 +1,24 @@ - + 4.0.0 @@ -27,6 +32,10 @@ kcbq-api kafka-connect-bigquery-api + + ${project.parent.basedir} + + org.apache.kafka diff --git a/kcbq-api/src/main/java/com/wepay/kafka/connect/bigquery/api/SchemaRetriever.java b/kcbq-api/src/main/java/com/wepay/kafka/connect/bigquery/api/SchemaRetriever.java index a46c67097..5b7e66306 100644 --- a/kcbq-api/src/main/java/com/wepay/kafka/connect/bigquery/api/SchemaRetriever.java +++ b/kcbq-api/src/main/java/com/wepay/kafka/connect/bigquery/api/SchemaRetriever.java @@ -1,3 +1,22 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package com.wepay.kafka.connect.bigquery.api; import com.google.cloud.bigquery.TableId; diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 5545e657d..51f555557 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -1,19 +1,24 @@ - + 4.0.0 @@ -27,6 +32,10 @@ kcbq-confluent kafka-connect-bigquery-confluent + + ${project.parent.basedir} + + com.wepay.kcbq diff --git a/kcbq-confluent/src/main/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetriever.java b/kcbq-confluent/src/main/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetriever.java index f85a404f0..420a92e22 100644 --- a/kcbq-confluent/src/main/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetriever.java +++ b/kcbq-confluent/src/main/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetriever.java @@ -1,3 +1,22 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package com.wepay.kafka.connect.bigquery.schemaregistry.schemaretriever; import com.google.cloud.bigquery.TableId; diff --git a/kcbq-confluent/src/main/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetrieverConfig.java b/kcbq-confluent/src/main/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetrieverConfig.java index f43fb411c..069914aaf 100644 --- a/kcbq-confluent/src/main/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetrieverConfig.java +++ b/kcbq-confluent/src/main/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetrieverConfig.java @@ -1,3 +1,22 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package com.wepay.kafka.connect.bigquery.schemaregistry.schemaretriever; import org.apache.kafka.common.config.AbstractConfig; diff --git a/kcbq-confluent/src/test/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetrieverTest.java b/kcbq-confluent/src/test/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetrieverTest.java index 18065ce1d..49a9e9602 100644 --- a/kcbq-confluent/src/test/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetrieverTest.java +++ b/kcbq-confluent/src/test/java/com/wepay/kafka/connect/bigquery/schemaregistry/schemaretriever/SchemaRegistrySchemaRetrieverTest.java @@ -1,3 +1,22 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package com.wepay.kafka.connect.bigquery.schemaregistry.schemaretriever; import static org.junit.Assert.assertEquals; diff --git a/kcbq-confluent/src/test/resources/log4j.properties b/kcbq-confluent/src/test/resources/log4j.properties index 2a5a0233d..94fb72b55 100644 --- a/kcbq-confluent/src/test/resources/log4j.properties +++ b/kcbq-confluent/src/test/resources/log4j.properties @@ -1,3 +1,22 @@ +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + log4j.rootLogger=INFO, stdout # Send the logs to the console. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 6f364dab3..8ecd8281c 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -1,19 +1,24 @@ - + 4.0.0 @@ -27,6 +32,10 @@ kcbq-connector kafka-connect-bigquery + + ${project.parent.basedir} + + org.apache.kafka diff --git a/kcbq-connector/quickstart/avro-console-producer.sh b/kcbq-connector/quickstart/avro-console-producer.sh index a7fe02118..9065f0cb3 100755 --- a/kcbq-connector/quickstart/avro-console-producer.sh +++ b/kcbq-connector/quickstart/avro-console-producer.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# BASE_DIR=`dirname "$0"` diff --git a/kcbq-connector/quickstart/connector.sh b/kcbq-connector/quickstart/connector.sh index 5c9dcecd9..123e9bbe9 100755 --- a/kcbq-connector/quickstart/connector.sh +++ b/kcbq-connector/quickstart/connector.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# BASE_DIR="$(cd "$(dirname "$0")" && pwd)" diff --git a/kcbq-connector/quickstart/kafka.sh b/kcbq-connector/quickstart/kafka.sh index 953c0d3f8..2ce3391ab 100755 --- a/kcbq-connector/quickstart/kafka.sh +++ b/kcbq-connector/quickstart/kafka.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# BASE_DIR=`dirname "$0"` diff --git a/kcbq-connector/quickstart/properties/connector.properties b/kcbq-connector/quickstart/properties/connector.properties index 05ee82fb1..50c9211ae 100644 --- a/kcbq-connector/quickstart/properties/connector.properties +++ b/kcbq-connector/quickstart/properties/connector.properties @@ -1,4 +1,7 @@ -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# name=bigquery-connector connector.class=com.wepay.kafka.connect.bigquery.BigQuerySinkConnector diff --git a/kcbq-connector/quickstart/properties/standalone.properties b/kcbq-connector/quickstart/properties/standalone.properties index 2aee81055..1450e07cc 100644 --- a/kcbq-connector/quickstart/properties/standalone.properties +++ b/kcbq-connector/quickstart/properties/standalone.properties @@ -1,4 +1,7 @@ -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# bootstrap.servers=localhost:9092 key.converter=io.confluent.connect.avro.AvroConverter diff --git a/kcbq-connector/quickstart/schema-registry.sh b/kcbq-connector/quickstart/schema-registry.sh index 5b5dfd3a6..61735fabc 100755 --- a/kcbq-connector/quickstart/schema-registry.sh +++ b/kcbq-connector/quickstart/schema-registry.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# BASE_DIR=`dirname "$0"` diff --git a/kcbq-connector/quickstart/zookeeper.sh b/kcbq-connector/quickstart/zookeeper.sh index ad5a88205..3e5fcbdcc 100755 --- a/kcbq-connector/quickstart/zookeeper.sh +++ b/kcbq-connector/quickstart/zookeeper.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# BASE_DIR=`dirname "$0"` diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQueryHelper.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQueryHelper.java index 1013b7a79..97dd30c7f 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQueryHelper.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQueryHelper.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; + import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java index db8a55e4e..7d2f99bf0 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.TableId; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java index b06dff5d6..f43bd722a 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/SchemaManager.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/SchemaManager.java index 74085228f..e3fe5bfc5 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/SchemaManager.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/SchemaManager.java @@ -1,3 +1,22 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package com.wepay.kafka.connect.bigquery; import com.google.cloud.bigquery.BigQuery; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java index accb3af7a..3b0bd5d85 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.config; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.config; import com.google.cloud.bigquery.Schema; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConnectorConfig.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConnectorConfig.java index 3d8c8b95a..eded455d5 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConnectorConfig.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConnectorConfig.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.config; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.config; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfig.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfig.java index 65b9b056d..482f6b652 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfig.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfig.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.config; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.config; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverter.java index 65de6b172..6fd57a717 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert; import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverter.java index 74c43f7d2..99e02d99d 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert; import com.wepay.kafka.connect.bigquery.convert.logicaltype.DebeziumLogicalConverters; import com.wepay.kafka.connect.bigquery.convert.logicaltype.KafkaLogicalConverters; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/RecordConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/RecordConverter.java index a3a1bfd1e..73f6c4e7e 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/RecordConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/RecordConverter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert; import org.apache.kafka.connect.sink.SinkRecord; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/SchemaConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/SchemaConverter.java index 985c736f8..8ca2e68b7 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/SchemaConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/SchemaConverter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert; import org.apache.kafka.connect.data.Schema; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQRecordConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQRecordConverter.java index 9cf93cf0c..9178105dc 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQRecordConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQRecordConverter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.kafkadata; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.kafkadata; import com.wepay.kafka.connect.bigquery.convert.BigQueryRecordConverter; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQSchemaConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQSchemaConverter.java index 918fd3c74..0d68cd9e9 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQSchemaConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQSchemaConverter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.kafkadata; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.kafkadata; import com.google.cloud.bigquery.Field; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java index 526f4ab4d..24326789c 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.logicaltype; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.logicaltype; import com.google.cloud.bigquery.Field; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConverters.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConverters.java index 84997d07e..3f8781917 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConverters.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConverters.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.logicaltype; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.logicaltype; import com.google.cloud.bigquery.Field; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalConverterRegistry.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalConverterRegistry.java index 36757de47..b21bcf613 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalConverterRegistry.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalConverterRegistry.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.logicaltype; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.logicaltype; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalTypeConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalTypeConverter.java index 2fe93cc66..891d582a7 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalTypeConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalTypeConverter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.logicaltype; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.logicaltype; import com.google.cloud.bigquery.Field; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java index 40fefd7da..a49c77726 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.exception; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.exception; import com.google.cloud.bigquery.BigQueryError; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/ConversionConnectException.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/ConversionConnectException.java index 795ea6749..29e10bd43 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/ConversionConnectException.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/ConversionConnectException.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.exception; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.exception; import org.apache.kafka.connect.errors.ConnectException; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/SinkConfigConnectException.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/SinkConfigConnectException.java index 98a11c069..805cd5643 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/SinkConfigConnectException.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/SinkConfigConnectException.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.exception; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.exception; import org.apache.kafka.connect.errors.ConnectException; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/retrieve/MemorySchemaRetriever.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/retrieve/MemorySchemaRetriever.java index bb6adcb63..a7227ee95 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/retrieve/MemorySchemaRetriever.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/retrieve/MemorySchemaRetriever.java @@ -1,3 +1,22 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package com.wepay.kafka.connect.bigquery.retrieve; import com.google.cloud.bigquery.BigQuery; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/PartitionedTableId.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/PartitionedTableId.java index 4f3e3ee48..28ae9b602 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/PartitionedTableId.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/PartitionedTableId.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.utils; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.utils; import com.google.cloud.bigquery.TableId; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/TopicToTableResolver.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/TopicToTableResolver.java index f1db8286b..994109845 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/TopicToTableResolver.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/TopicToTableResolver.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.utils; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.utils; import com.google.cloud.bigquery.TableId; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/Version.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/Version.java index bbcdfae38..8a6c6f4be 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/Version.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/utils/Version.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.utils; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.utils; /** * Utility class for unifying the version of a project. All other references to version number diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/CountDownRunnable.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/CountDownRunnable.java index cbdca4ea6..70edc8a1a 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/CountDownRunnable.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/CountDownRunnable.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.write.batch; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.write.batch; import org.apache.kafka.connect.errors.ConnectException; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java index 909a53709..2b8058856 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.write.batch; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.write.batch; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java index aef368450..5fb0a7139 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.write.batch; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.write.batch; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/AdaptiveBigQueryWriter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/AdaptiveBigQueryWriter.java index 1031ae166..07d682fee 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/AdaptiveBigQueryWriter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/AdaptiveBigQueryWriter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.write.row; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.write.row; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriter.java index f21a8dd7f..c335254b7 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.write.row; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.write.row; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/SimpleBigQueryWriter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/SimpleBigQueryWriter.java index b1189473a..8d22af876 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/SimpleBigQueryWriter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/row/SimpleBigQueryWriter.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.write.row; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.write.row; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java index 2ebf97e6d..65933de07 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java index d58a8b076..f36500f7d 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SchemaManagerTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SchemaManagerTest.java index 27e5dccd6..bc89accfb 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SchemaManagerTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SchemaManagerTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkConnectorPropertiesFactory.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkConnectorPropertiesFactory.java index 8da3ca467..c29c25394 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkConnectorPropertiesFactory.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkConnectorPropertiesFactory.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConnectorConfig; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java index a87e1cb64..42c80f2bd 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkTaskPropertiesFactory.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkTaskPropertiesFactory.java index cda1d6c94..1d09fac3d 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkTaskPropertiesFactory.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkTaskPropertiesFactory.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java index 7c7327ae6..7547099b9 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.config; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.config; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConnectorConfigTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConnectorConfigTest.java index a27c149e4..e136b99b1 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConnectorConfigTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConnectorConfigTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.config; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.config; import com.wepay.kafka.connect.bigquery.SinkConnectorPropertiesFactory; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfigTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfigTest.java index d989e7ea8..2ca564e00 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfigTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfigTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.config; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.config; import static org.junit.Assert.fail; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverterTest.java index 7529eec3b..a86bffa85 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverterTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert; import static org.junit.Assert.assertEquals; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverterTest.java index 6fb044ba4..5dae59e3c 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverterTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert; import static org.junit.Assert.assertEquals; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQRecordConvertTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQRecordConvertTest.java index 66b5c7c40..39c369821 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQRecordConvertTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQRecordConvertTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.kafkadata; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.kafkadata; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQSchemaConverterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQSchemaConverterTest.java index 3fc1970a9..d5764ce99 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQSchemaConverterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/kafkadata/KafkaDataBQSchemaConverterTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.kafkadata; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.kafkadata; import static org.junit.Assert.assertEquals; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java index ec8f8b6d0..519026715 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.logicaltype; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.logicaltype; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConvertersTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConvertersTest.java index 1808ccfe0..778c68613 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConvertersTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConvertersTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.convert.logicaltype; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.convert.logicaltype; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java index 546c1549f..61d763ee8 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.it; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.it; import static com.google.cloud.bigquery.LegacySQLTypeName.*; import static org.junit.Assert.assertEquals; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java index af981d4ea..64849d0ad 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.it.utils; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.it.utils; import com.google.cloud.bigquery.BigQuery; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/retrieve/MemorySchemaRetrieverTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/retrieve/MemorySchemaRetrieverTest.java index 8cd83e990..83192ddf5 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/retrieve/MemorySchemaRetrieverTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/retrieve/MemorySchemaRetrieverTest.java @@ -1,5 +1,23 @@ -package com.wepay.kafka.connect.bigquery.retrieve; +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.wepay.kafka.connect.bigquery.retrieve; import com.google.cloud.bigquery.TableId; import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; @@ -11,7 +29,6 @@ import java.util.HashMap; - public class MemorySchemaRetrieverTest { public TableId getTableId(String datasetName, String tableName) { return TableId.of(datasetName, tableName); diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/PartitionedTableIdTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/PartitionedTableIdTest.java index 1737bfbdf..d24072714 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/PartitionedTableIdTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/PartitionedTableIdTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.utils; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.utils; import com.google.cloud.bigquery.TableId; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/TopicToTableResolverTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/TopicToTableResolverTest.java index bd69219aa..42106a35c 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/TopicToTableResolverTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/utils/TopicToTableResolverTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.utils; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.utils; + import static org.junit.Assert.assertEquals; import com.google.cloud.bigquery.TableId; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java index 439c70026..4073c186a 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java @@ -1,7 +1,7 @@ -package com.wepay.kafka.connect.bigquery.write.row; - /* - * Copyright 2016 WePay, Inc. + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ * under the License. */ +package com.wepay.kafka.connect.bigquery.write.row; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; diff --git a/kcbq-connector/src/test/resources/log4j.properties b/kcbq-connector/src/test/resources/log4j.properties index 2a5a0233d..94fb72b55 100644 --- a/kcbq-connector/src/test/resources/log4j.properties +++ b/kcbq-connector/src/test/resources/log4j.properties @@ -1,3 +1,22 @@ +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + log4j.rootLogger=INFO, stdout # Send the logs to the console. diff --git a/kcbq-connector/src/test/resources/test.properties b/kcbq-connector/src/test/resources/test.properties new file mode 100644 index 000000000..27b48b485 --- /dev/null +++ b/kcbq-connector/src/test/resources/test.properties @@ -0,0 +1,22 @@ +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +keyfile=/Users/chrise/.gcs-creds.json +project=connect-205118 +dataset=confluentBigquery diff --git a/kcbq-connector/test/docker/connect/Dockerfile b/kcbq-connector/test/docker/connect/Dockerfile index 3ad1d8896..ff296f8da 100644 --- a/kcbq-connector/test/docker/connect/Dockerfile +++ b/kcbq-connector/test/docker/connect/Dockerfile @@ -1,4 +1,7 @@ -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,14 +16,6 @@ # specific language governing permissions and limitations # under the License. # -# Builds a docker image for the Kafka-BigQuery Connector. -# Expects links to "kafka" and "schema-registry" containers. -# -# Usage: -# docker build -t kcbq/connect connect -# docker run --name kcbq_test_connect \ -# --link kcbq_test_kafka:kafka --link kcbq_test_schema-registry:schema-registry \ -# kcbq/connect FROM confluent/platform:3.0.0 diff --git a/kcbq-connector/test/docker/connect/connect-docker.sh b/kcbq-connector/test/docker/connect/connect-docker.sh index d0f78fafd..4443f7d38 100755 --- a/kcbq-connector/test/docker/connect/connect-docker.sh +++ b/kcbq-connector/test/docker/connect/connect-docker.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# unzip -j -d /usr/share/java/kafka-connect-bigquery /usr/share/java/kafka-connect-bigquery/kcbq.zip 'wepay-kafka-connect-bigquery-*/lib/*.jar' diff --git a/kcbq-connector/test/docker/populate/Dockerfile b/kcbq-connector/test/docker/populate/Dockerfile index 756d418ea..3c9fe5b71 100644 --- a/kcbq-connector/test/docker/populate/Dockerfile +++ b/kcbq-connector/test/docker/populate/Dockerfile @@ -1,4 +1,7 @@ -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,14 +16,6 @@ # specific language governing permissions and limitations # under the License. # -# Populates Kafka and Schema Registry with test data -# Expects links to "kafka" and "schema-registry" containers. -# -# Usage: -# docker build -t kcbq/populate populate -# docker run --name kcbq_test_populate \ -# --link kcbq_test_kafka:kafka --link kcbq_test_schema-registry:schema-registry \ -# kcbq/populate FROM confluent/platform:3.0.0 diff --git a/kcbq-connector/test/docker/populate/populate-docker.sh b/kcbq-connector/test/docker/populate/populate-docker.sh index ee7da76f2..37094b2ee 100755 --- a/kcbq-connector/test/docker/populate/populate-docker.sh +++ b/kcbq-connector/test/docker/populate/populate-docker.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# for schema_dir in /tmp/schemas/*; do kafka-avro-console-producer \ diff --git a/kcbq-connector/test/integrationtest.sh b/kcbq-connector/test/integrationtest.sh index 4ea70bef7..76d9b3ad5 100755 --- a/kcbq-connector/test/integrationtest.sh +++ b/kcbq-connector/test/integrationtest.sh @@ -1,5 +1,8 @@ #! /usr/bin/env bash -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# #################################################################################################### # Basic script setup diff --git a/kcbq-connector/test/resources/connector-template.properties b/kcbq-connector/test/resources/connector-template.properties index c37eccb67..5c753c8cb 100644 --- a/kcbq-connector/test/resources/connector-template.properties +++ b/kcbq-connector/test/resources/connector-template.properties @@ -1,4 +1,7 @@ -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# name=bigquery-connector connector.class=com.wepay.kafka.connect.bigquery.BigQuerySinkConnector diff --git a/kcbq-connector/test/resources/standalone-template.properties b/kcbq-connector/test/resources/standalone-template.properties index bf23ba9e6..6ec7dacc7 100644 --- a/kcbq-connector/test/resources/standalone-template.properties +++ b/kcbq-connector/test/resources/standalone-template.properties @@ -1,4 +1,7 @@ -# Copyright 2016 WePay, Inc. +# +# Copyright 2020 Confluent, Inc. +# +# This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# bootstrap.servers=kafka:9092 key.converter=io.confluent.connect.avro.AvroConverter diff --git a/pom.xml b/pom.xml index adea1235d..1ddc1dd7a 100644 --- a/pom.xml +++ b/pom.xml @@ -1,19 +1,24 @@ - + 4.0.0 @@ -56,6 +61,7 @@ 3.7.1 3.0.0-M4 + ${project.basedir} ${maven.test.skip} @@ -251,6 +257,46 @@ v@{project.version} + + com.mycila + license-maven-plugin + 3.0 + + +Copyright 2020 Confluent, Inc. + +This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + + + ${main.dir}/config/copyright/custom-header-styles.xml + + + CUSTOM_JAVA_STYLE + JENKINSFILE_STYLE + + + LICENSE.md + *.log + config/checkstyle/google_checks.xml + + + .ci/* + + + From 63ecc65784e360746b8af5380ca0f3ebb6f1aea7 Mon Sep 17 00:00:00 2001 From: Nigel Liang Date: Fri, 16 Oct 2020 10:08:16 -0700 Subject: [PATCH 18/56] CC-12157: Throw nicer exception on detecting cycle in schema (#47) * CC-12157: Throw nicer exception on detecting cycle in schema * review comments --- .../convert/BigQuerySchemaConverter.java | 33 ++++++++++++++ .../convert/BigQuerySchemaConverterTest.java | 44 +++++++++++++++++++ pom.xml | 2 +- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverter.java index 2924c7fe0..4354d2692 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverter.java @@ -29,7 +29,9 @@ import com.wepay.kafka.connect.bigquery.exception.ConversionConnectException; import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Schema.Type; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -103,6 +105,8 @@ public com.google.cloud.bigquery.Schema convertSchema(Schema kafkaConnectSchema) ConversionConnectException("Top-level Kafka Connect schema must be of type 'struct'"); } + throwOnCycle(kafkaConnectSchema, new ArrayList<>()); + List fields = kafkaConnectSchema.fields().stream() .flatMap(kafkaConnectField -> convertField(kafkaConnectField.schema(), kafkaConnectField.name()) @@ -115,6 +119,35 @@ public com.google.cloud.bigquery.Schema convertSchema(Schema kafkaConnectSchema) return com.google.cloud.bigquery.Schema.of(fields); } + private void throwOnCycle(Schema kafkaConnectSchema, List seenSoFar) { + if (PRIMITIVE_TYPE_MAP.containsKey(kafkaConnectSchema.type())) { + return; + } + + if (seenSoFar.contains(kafkaConnectSchema)) { + throw new ConversionConnectException("Kafka Connect schema contains cycle"); + } + + seenSoFar.add(kafkaConnectSchema); + switch(kafkaConnectSchema.type()) { + case ARRAY: + throwOnCycle(kafkaConnectSchema.valueSchema(), seenSoFar); + break; + case MAP: + throwOnCycle(kafkaConnectSchema.keySchema(), seenSoFar); + throwOnCycle(kafkaConnectSchema.valueSchema(), seenSoFar); + break; + case STRUCT: + kafkaConnectSchema.fields().forEach(f -> throwOnCycle(f.schema(), seenSoFar)); + break; + default: + throw new ConversionConnectException( + "Unrecognized schema type: " + kafkaConnectSchema.type() + ); + } + seenSoFar.remove(seenSoFar.size() - 1); + } + private Optional convertField(Schema kafkaConnectSchema, String fieldName) { Optional result; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverterTest.java index c3f888956..9185fb724 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/BigQuerySchemaConverterTest.java @@ -20,10 +20,12 @@ package com.wepay.kafka.connect.bigquery.convert; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.common.collect.ImmutableList; import com.wepay.kafka.connect.bigquery.exception.ConversionConnectException; import org.apache.kafka.connect.data.Date; @@ -34,6 +36,8 @@ import org.junit.Test; +import io.confluent.connect.avro.AvroData; + public class BigQuerySchemaConverterTest { @Test(expected = ConversionConnectException.class) @@ -628,6 +632,46 @@ public void testAllFieldsNullable() { com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(true).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); + } + + @Test + public void testSimpleRecursiveSchemaThrows() { + final String fieldName = "RecursiveField"; + + // Construct Avro schema with recursion since we cannot directly construct Connect schema with cycle + org.apache.avro.Schema recursiveAvroSchema = org.apache.avro.SchemaBuilder + .record("RecursiveItem") + .namespace("com.example") + .fields() + .name(fieldName) + .type().unionOf().nullType().and().type("RecursiveItem").endUnion() + .nullDefault() + .endRecord(); + + Schema connectSchema = new AvroData(100).toConnectSchema(recursiveAvroSchema); + ConversionConnectException e = assertThrows(ConversionConnectException.class, () -> + new BigQuerySchemaConverter(true).convertSchema(connectSchema)); + assertEquals("Kafka Connect schema contains cycle", e.getMessage()); + } + @Test + public void testComplexRecursiveSchemaThrows() { + final String fieldName = "RecursiveField"; + + // Construct Avro schema with recursion since we cannot directly construct Connect schema with cycle + org.apache.avro.Schema recursiveAvroSchema = org.apache.avro.SchemaBuilder + .record("RecursiveItem") + .namespace("com.example") + .fields() + .name(fieldName) + .type() + .array().items() + .map().values().type("RecursiveItem").noDefault() + .endRecord(); + + Schema connectSchema = new AvroData(100).toConnectSchema(recursiveAvroSchema); + ConversionConnectException e = assertThrows(ConversionConnectException.class, () -> + new BigQuerySchemaConverter(true).convertSchema(connectSchema)); + assertEquals("Kafka Connect schema contains cycle", e.getMessage()); } } diff --git a/pom.xml b/pom.xml index 3b0903092..ea4f4ba23 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2.5.0 1.7.26 - 4.12 + 4.13 3.2.4 2.15 From f24c75d5c42af94ef05b0fbb38c44277e8263bd1 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Sat, 17 Oct 2020 09:22:48 -0400 Subject: [PATCH 19/56] MINOR: Catch 'request payload exceeds size' errors from BigQuery and reduce batch size in response (#49) --- .../kafka/connect/bigquery/write/batch/TableWriter.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java index 2e4839c1f..6584af53a 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java @@ -27,7 +27,6 @@ import com.wepay.kafka.connect.bigquery.write.row.BigQueryWriter; import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.sink.SinkRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +43,7 @@ public class TableWriter implements Runnable { private static final int BAD_REQUEST_CODE = 400; private static final String INVALID_REASON = "invalid"; + private static final String PAYLOAD_TOO_LARGE_REASON = "Request payload size exceeds the limit:"; private final BigQueryWriter writer; private final PartitionedTableId table; @@ -138,6 +138,10 @@ private static boolean isBatchSizeError(BigQueryException exception) { * todo distinguish this from other invalids (like invalid table schema). */ return true; + } else if (exception.getCode() == BAD_REQUEST_CODE + && exception.getMessage() != null + && exception.getMessage().contains(PAYLOAD_TOO_LARGE_REASON)) { + return true; } return false; } From b56fb36b34cf425c78c7c4ee59f96ba9553c9b5e Mon Sep 17 00:00:00 2001 From: atrbgithub <14765982+atrbgithub@users.noreply.github.com> Date: Mon, 23 Nov 2020 18:25:55 +0000 Subject: [PATCH 20/56] 1.6.x backports (#58) * Backport of FMC-463 * Backport of CC-9208 --- .../connect/bigquery/BigQuerySinkTask.java | 5 ++- .../exception/BigQueryConnectException.java | 6 +++ .../write/batch/KCBQThreadPoolExecutor.java | 40 +++++++++++++++++- .../bigquery/write/batch/TableWriter.java | 10 +++-- .../bigquery/BigQuerySinkTaskTest.java | 42 ++++++++++++++++--- .../write/row/BigQueryWriterTest.java | 9 ++-- 6 files changed, 97 insertions(+), 15 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java index feeee65fe..118007756 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java @@ -186,7 +186,10 @@ private String getRowId(SinkRecord record) { @Override public void put(Collection records) { - logger.info("Putting {} records in the sink.", records.size()); + // Periodically poll for errors here instead of doing a stop-the-world check in flush() + executor.maybeThrowEncounteredErrors(); + + logger.debug("Putting {} records in the sink.", records.size()); // create tableWriters Map tableWriterBuilders = new HashMap<>(); diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java index a49c77726..3da4a39ac 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java @@ -62,4 +62,10 @@ private static String formatInsertAllErrors(Map> error } return messageBuilder.toString(); } + + @Override + public String toString() { + return getCause() != null ? + super.toString() + "\nCaused by: " + getCause().getLocalizedMessage() : super.toString(); + } } diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java index 2e2c28fa4..003dca988 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java @@ -45,7 +45,7 @@ public class KCBQThreadPoolExecutor extends ThreadPoolExecutor { private static final Logger logger = LoggerFactory.getLogger(KCBQThreadPoolExecutor.class); - private ConcurrentHashMap.KeySetView encounteredErrors = + private final ConcurrentHashMap.KeySetView encounteredErrors = ConcurrentHashMap.newKeySet(); /** @@ -92,14 +92,28 @@ public void awaitCurrentTasks() throws InterruptedException, BigQueryConnectExce execute(new CountDownRunnable(countDownLatch)); } countDownLatch.await(); + maybeThrowEncounteredErrors(); if (encounteredErrors.size() > 0) { String errorString = createErrorString(encounteredErrors); - encounteredErrors.clear(); throw new BigQueryConnectException("Some write threads encountered unrecoverable errors: " + errorString + "; See logs for more detail"); } } + /** + * Immediately throw an exception if any unrecoverable errors were encountered by any of the write + * tasks. + * + * @throws BigQueryConnectException if any of the tasks failed. + */ + public void maybeThrowEncounteredErrors() { + if (encounteredErrors.size() > 0) { + String errorString = createErrorString(encounteredErrors); + throw new BigQueryConnectException("Some write threads encountered unrecoverable errors: " + + errorString + "; See logs for more detail"); + } + } + private static String createErrorString(Collection errors) { List exceptionTypeStrings = new ArrayList<>(errors.size()); exceptionTypeStrings.addAll(errors.stream() @@ -107,4 +121,26 @@ private static String createErrorString(Collection errors) { .collect(Collectors.toList())); return String.join(", ", exceptionTypeStrings); } + + private static String createDetailedErrorString(Collection errors) { + List exceptionTypeStrings = new ArrayList<>(errors.size()); + exceptionTypeStrings.addAll(errors.stream() + .map(throwable -> + throwable.getClass().getName() + "\nMessage: " + throwable.getLocalizedMessage()) + .collect(Collectors.toList())); + return String.join(", ", exceptionTypeStrings); + } + + /** + * Checks for BigQuery errors. No-op if there isn't any error. + * + * @throws BigQueryConnectException if there have been any unrecoverable errors when writing to BigQuery. + */ + public void maybeFail() throws BigQueryConnectException { + if (encounteredErrors.size() > 0) { + throw new BigQueryConnectException("Encountered unrecoverable errors: " + + createDetailedErrorString(encounteredErrors) + "; See logs for more detail"); + } + } + } diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java index 6584af53a..3ff20037b 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java @@ -23,6 +23,7 @@ import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.wepay.kafka.connect.bigquery.convert.RecordConverter; +import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; import com.wepay.kafka.connect.bigquery.utils.PartitionedTableId; import com.wepay.kafka.connect.bigquery.write.row.BigQueryWriter; @@ -85,7 +86,10 @@ public void run() { logger.warn("Could not write batch of size {} to BigQuery.", currentBatch.size(), err); if (isBatchSizeError(err)) { failureCount++; - currentBatchSize = getNewBatchSize(currentBatchSize); + currentBatchSize = getNewBatchSize(currentBatchSize, err); + } else { + // Throw exception on write errors such as 403. + throw new BigQueryConnectException("Failed to write to table", err); } } } @@ -105,10 +109,10 @@ public void run() { } - private static int getNewBatchSize(int currentBatchSize) { + private static int getNewBatchSize(int currentBatchSize, Throwable err) { if (currentBatchSize == 1) { // todo correct exception type? - throw new ConnectException("Attempted to reduce batch size below 1."); + throw new BigQueryConnectException("Attempted to reduce batch size below 1.", err); } // round batch size up so we don't end up with a dangling 1 row at the end. return (int) Math.ceil(currentBatchSize / 2.0); diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java index cbd212e09..fd16921f4 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java @@ -299,11 +299,43 @@ public void testPutWhenPartitioningOnMessageTimeWhenNoTimestampType() { TimestampType.NO_TIMESTAMP_TYPE, null))); } - // It's important that the buffer be completely wiped after a call to flush, since any execption - // thrown during flush causes Kafka Connect to not commit the offsets for any records sent to the - // task since the last flush - @Test - public void testBufferClearOnFlushError() { + @Test(expected = BigQueryConnectException.class, timeout = 60000L) + public void testSimplePutException() throws InterruptedException { + final String topic = "test-topic"; + Map properties = propertiesFactory.getProperties(); + properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); + properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=scratch"); + + BigQuery bigQuery = mock(BigQuery.class); + Storage storage = mock(Storage.class); + + SinkTaskContext sinkTaskContext = mock(SinkTaskContext.class); + InsertAllResponse insertAllResponse = mock(InsertAllResponse.class); + when(bigQuery.insertAll(any())).thenReturn(insertAllResponse); + when(insertAllResponse.hasErrors()).thenReturn(true); + when(insertAllResponse.getInsertErrors()).thenReturn(Collections.singletonMap( + 0L, Collections.singletonList(new BigQueryError("no such field", "us-central1", "")))); + + SchemaRetriever schemaRetriever = mock(SchemaRetriever.class); + SchemaManager schemaManager = mock(SchemaManager.class); + + BigQuerySinkTask testTask = new BigQuerySinkTask(bigQuery, schemaRetriever, storage, schemaManager); + testTask.initialize(sinkTaskContext); + testTask.start(properties); + + testTask.put(Collections.singletonList(spoofSinkRecord(topic))); + while (true) { + Thread.sleep(100); + testTask.put(Collections.emptyList()); + } + } + + + // Since any exception thrown during flush causes Kafka Connect to not commit the offsets for any + // records sent to the task since the last flush. The task should fail, and next flush should + // also throw an error. + @Test(expected = BigQueryConnectException.class) + public void testFlushException() { final String dataset = "scratch"; final String topic = "test_topic"; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java index aed44f3f3..4ecbb6961 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java @@ -20,6 +20,7 @@ package com.wepay.kafka.connect.bigquery.write.row; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -31,6 +32,7 @@ import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.InsertAllRequest; import com.google.cloud.bigquery.InsertAllResponse; +import com.google.cloud.bigquery.Table; import com.google.cloud.storage.Storage; import com.wepay.kafka.connect.bigquery.BigQuerySinkTask; @@ -137,13 +139,15 @@ public void testAutoCreateTables() { verify(bigQuery, times(2)).insertAll(anyObject()); } - @Test + @Test(expected = BigQueryConnectException.class) public void testNonAutoCreateTables() { final String topic = "test_topic"; final String dataset = "scratch"; final Map properties = makeProperties("3", "2000", topic, dataset); BigQuery bigQuery = mock(BigQuery.class); + Table mockTable = mock(Table.class); + when(bigQuery.getTable(any())).thenReturn(mockTable); Map> emptyMap = mock(Map.class); when(emptyMap.isEmpty()).thenReturn(true); @@ -166,9 +170,6 @@ public void testNonAutoCreateTables() { testTask.put( Collections.singletonList(spoofSinkRecord(topic, 0, 0, "some_field", "some_value"))); testTask.flush(Collections.emptyMap()); - - verify(schemaManager, times(0)).createTable(anyObject(), anyObject()); - verify(bigQuery, times(2)).insertAll(anyObject()); } @Test From 506233d7ec41bcc06ed43e919b64f5ba99a6c563 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Mon, 23 Nov 2020 18:43:11 +0000 Subject: [PATCH 21/56] [maven-release-plugin] prepare release v1.6.6 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index 7dce66e53..bdc430be8 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.6-SNAPSHOT + 1.6.6 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index d7ca9031b..075f17936 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.6-SNAPSHOT + 1.6.6 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 92b7e29cd..cf7b6e381 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.6-SNAPSHOT + 1.6.6 .. diff --git a/pom.xml b/pom.xml index ea4f4ba23..ce2bf6cbb 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.6-SNAPSHOT + 1.6.6 pom @@ -81,7 +81,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.6.6 From 65eae703f13f0c72d2db514ff82a3bcf16365817 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Mon, 23 Nov 2020 18:43:18 +0000 Subject: [PATCH 22/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index bdc430be8..12817236b 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.6 + 1.6.7-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 075f17936..c0f9f24e9 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.6 + 1.6.7-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index cf7b6e381..925405b0b 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.6 + 1.6.7-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index ce2bf6cbb..e01f9954d 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.6 + 1.6.7-SNAPSHOT pom @@ -81,7 +81,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.6.6 + HEAD From dc87644294c15596f028aafcdd309d3485d06164 Mon Sep 17 00:00:00 2001 From: tikimims <39631000+tikimims@users.noreply.github.com> Date: Fri, 8 Jan 2021 18:10:09 -0500 Subject: [PATCH 23/56] Update docs URL path (#66) --- kcbq-connector/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index b3e5c147a..c55786975 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -135,14 +135,14 @@ A sink connector for writing to Google BigQuery, with support for automatic table creation and schema evolution. logos/BigQuery.png - https://docs.confluent.io/current/connect/kafka-connect-bigquery/ + https://docs.confluent.io/kafka-connect-bigquery/current/index.html https://github.com/confluentinc/kafka-connect-bigquery Confluent, Inc. supports WePay's BigQuery connector version 1.1.2 and later, as part of a Confluent Platform subscription. ]]> - https://docs.confluent.io/current/connect/kafka-connect-bigquery/ + https://docs.confluent.io/kafka-connect-bigquery/current/index.html logos/confluent.png wepay From 8f163b06f54caf5299af78417041947cfc6c5eba Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 5 May 2021 15:54:29 -0400 Subject: [PATCH 24/56] MINOR: Remove outdated properties (#107) These properties haven't been used by the connector for a long time; we should remove them from the code base in order to avoid confusing users. --- kcbq-connector/quickstart/properties/connector.properties | 4 ---- 1 file changed, 4 deletions(-) diff --git a/kcbq-connector/quickstart/properties/connector.properties b/kcbq-connector/quickstart/properties/connector.properties index 50c9211ae..9f2d245b5 100644 --- a/kcbq-connector/quickstart/properties/connector.properties +++ b/kcbq-connector/quickstart/properties/connector.properties @@ -30,10 +30,6 @@ autoUpdateSchemas=true schemaRetriever=com.wepay.kafka.connect.bigquery.schemaregistry.schemaretriever.SchemaRegistrySchemaRetriever schemaRegistryLocation=http://localhost:8081 -bufferSize=100000 -maxWriteSize=10000 -tableWriteWait=1000 - ########################################### Fill me in! ########################################### # The name of the BigQuery project to write to project= From a672fcd88b511de599d9af6c785a9402279b62fc Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Tue, 11 May 2021 10:36:49 -0400 Subject: [PATCH 25/56] GH-65: Add support for Kafka logical Time type (#112) --- .../logicaltype/KafkaLogicalConverters.java | 22 ++++++++++++++ .../logicaltype/LogicalTypeConverter.java | 8 ++--- .../KafkaLogicalConvertersTest.java | 29 +++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConverters.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConverters.java index a6d918c12..6d3685ac5 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConverters.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConverters.java @@ -24,6 +24,7 @@ import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Decimal; import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Time; import org.apache.kafka.connect.data.Timestamp; import java.math.BigDecimal; @@ -37,6 +38,7 @@ public class KafkaLogicalConverters { LogicalConverterRegistry.register(Date.LOGICAL_NAME, new DateConverter()); LogicalConverterRegistry.register(Decimal.LOGICAL_NAME, new DecimalConverter()); LogicalConverterRegistry.register(Timestamp.LOGICAL_NAME, new TimestampConverter()); + LogicalConverterRegistry.register(Time.LOGICAL_NAME, new TimeConverter()); } /** @@ -96,4 +98,24 @@ public String convert(Object kafkaConnectObject) { return getBqTimestampFormat().format((java.util.Date) kafkaConnectObject); } } + + + /** + * Class for converting Kafka time logical types to BigQuery time types. + */ + public static class TimeConverter extends LogicalTypeConverter { + /** + * Create a new TimestampConverter. + */ + public TimeConverter() { + super(Time.LOGICAL_NAME, + Schema.Type.INT32, + LegacySQLTypeName.TIME); + } + + @Override + public String convert(Object kafkaConnectObject) { + return getBqTimeFormat().format((java.util.Date) kafkaConnectObject); + } + } } diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalTypeConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalTypeConverter.java index 1815daede..9adaa330b 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalTypeConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/LogicalTypeConverter.java @@ -88,10 +88,10 @@ protected static SimpleDateFormat getBqTimestampFormat() { return bqTimestampFormat; } - protected static SimpleDateFormat getBQDatetimeFormat() { - SimpleDateFormat bqDateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"); - bqDateTimeFormat.setTimeZone(utcTimeZone); - return bqDateTimeFormat; + protected SimpleDateFormat getBqTimeFormat() { + SimpleDateFormat bqTimestampFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + bqTimestampFormat.setTimeZone(utcTimeZone); + return bqTimestampFormat; } protected static SimpleDateFormat getBQDateFormat() { diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConvertersTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConvertersTest.java index 43dc1d1ea..5eb72902f 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConvertersTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/KafkaLogicalConvertersTest.java @@ -27,6 +27,7 @@ import com.wepay.kafka.connect.bigquery.convert.logicaltype.KafkaLogicalConverters.DateConverter; import com.wepay.kafka.connect.bigquery.convert.logicaltype.KafkaLogicalConverters.DecimalConverter; import com.wepay.kafka.connect.bigquery.convert.logicaltype.KafkaLogicalConverters.TimestampConverter; +import com.wepay.kafka.connect.bigquery.convert.logicaltype.KafkaLogicalConverters.TimeConverter; import org.apache.kafka.connect.data.Schema; @@ -101,4 +102,32 @@ public void testTimestampConversion() { assertEquals("2017-03-01 22:20:38.808", formattedTimestamp); } + + + @Test + public void testTimeConversion() { + TimeConverter converter = new KafkaLogicalConverters.TimeConverter(); + + assertEquals(LegacySQLTypeName.TIME, converter.getBQSchemaType()); + + try { + converter.checkEncodingType(Schema.Type.INT32); + } catch (Exception ex) { + fail("Expected encoding type check to succeed."); + } + + try { + converter.checkEncodingType(Schema.Type.INT64); + fail("Expected encoding type check to fail"); + } catch (Exception ex) { + // continue + } + + // Can't use the same timestamp here as the one in other tests as the Time type + // should only fall on January 1st, 1970 + Date date = new Date(166838808); + String formattedTimestamp = converter.convert(date); + + assertEquals("22:20:38.808", formattedTimestamp); + } } From ec09e3e42d3be7cb72e1b525eacc237045d969f0 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 2 Sep 2021 16:20:14 +0000 Subject: [PATCH 26/56] [maven-release-plugin] prepare release v1.6.7 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index 12817236b..f30c77e6d 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.7-SNAPSHOT + 1.6.7 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index c0f9f24e9..a7722583b 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.7-SNAPSHOT + 1.6.7 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index b4412b492..135f88413 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.7-SNAPSHOT + 1.6.7 .. diff --git a/pom.xml b/pom.xml index e01f9954d..3e88a4461 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.7-SNAPSHOT + 1.6.7 pom @@ -81,7 +81,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.6.7 From 0986d116cf3de0a94fa40a8b3dac4964e9e6c8a0 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 2 Sep 2021 16:20:17 +0000 Subject: [PATCH 27/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index f30c77e6d..64748796c 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.7 + 1.6.8-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index a7722583b..6dd426d4a 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.7 + 1.6.8-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 135f88413..869c5e63a 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.7 + 1.6.8-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 3e88a4461..efe178081 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.7 + 1.6.8-SNAPSHOT pom @@ -81,7 +81,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.6.7 + HEAD From 85b8de31c231b804ad29826ce39cbc18a8143b6b Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 8 Sep 2021 14:02:01 -0400 Subject: [PATCH 28/56] Register TimestampConverter in DebeziumLogicalConverters. (#137) Co-authored-by: Bingqin Zhou --- .../bigquery/convert/logicaltype/DebeziumLogicalConverters.java | 1 + 1 file changed, 1 insertion(+) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java index 1ffef1601..abf35c938 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java @@ -47,6 +47,7 @@ public class DebeziumLogicalConverters { LogicalConverterRegistry.register(MicroTimestamp.SCHEMA_NAME, new MicroTimestampConverter()); LogicalConverterRegistry.register(Time.SCHEMA_NAME, new TimeConverter()); LogicalConverterRegistry.register(ZonedTimestamp.SCHEMA_NAME, new ZonedTimestampConverter()); + LogicalConverterRegistry.register(Timestamp.SCHEMA_NAME, new TimestampConverter()); } private static final int MICROS_IN_SEC = 1000000; From 7232276761912aef0a7c5b093d7625875138b981 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 9 Sep 2021 13:52:38 +0000 Subject: [PATCH 29/56] [maven-release-plugin] prepare release v1.6.8 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index 64748796c..bb806333d 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.8-SNAPSHOT + 1.6.8 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 6dd426d4a..88b6260d3 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.8-SNAPSHOT + 1.6.8 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 869c5e63a..c9e864180 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.8-SNAPSHOT + 1.6.8 .. diff --git a/pom.xml b/pom.xml index efe178081..4c5462000 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.8-SNAPSHOT + 1.6.8 pom @@ -81,7 +81,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.6.8 From 98c7c625f2f65314beea368b46715405b0dc0bfd Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 9 Sep 2021 13:52:41 +0000 Subject: [PATCH 30/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index bb806333d..b771a52fb 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.8 + 1.6.9-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 88b6260d3..9d58bef55 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.8 + 1.6.9-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index c9e864180..4932c5908 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.8 + 1.6.9-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 4c5462000..675043611 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.8 + 1.6.9-SNAPSHOT pom @@ -81,7 +81,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.6.8 + HEAD From 986995e2b8eb35b73a712d959a70b0f0e7f0e8fa Mon Sep 17 00:00:00 2001 From: Xiao Fu Date: Fri, 24 Sep 2021 07:53:36 -0700 Subject: [PATCH 31/56] adding table and column information to help debugging (#152) * adding table and column information to BigQueryConnectionException to help debugging * change log format --- .../connect/bigquery/exception/BigQueryConnectException.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java index 3da4a39ac..847b14542 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/BigQueryConnectException.java @@ -53,8 +53,9 @@ private static String formatInsertAllErrors(Map> error for (Map.Entry> errorsEntry : errorsMap.entrySet()) { for (BigQueryError error : errorsEntry.getValue()) { messageBuilder.append(String.format( - "%n\t[row index %d]: %s: %s", + "%n\t[row index %d] (location %s, reason: %s): %s", errorsEntry.getKey(), + error.getLocation(), error.getReason(), error.getMessage() )); From a3fa7c449c681c02c81d6963b30cc8a1bfcdf478 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Mon, 1 Nov 2021 10:25:57 -0400 Subject: [PATCH 32/56] GH-139: Move all user-visible configuration properties to single configuration class (#140) --- .../bigquery/BigQuerySinkConnector.java | 20 +- .../connect/bigquery/BigQuerySinkTask.java | 40 ++- .../bigquery/config/BigQuerySinkConfig.java | 232 ++++++++++++++++- .../config/BigQuerySinkTaskConfig.java | 239 +----------------- .../bigquery/BigQuerySinkConnectorTest.java | 6 +- .../bigquery/BigQuerySinkTaskTest.java | 27 +- .../SinkConnectorPropertiesFactory.java | 46 ---- .../bigquery/SinkPropertiesFactory.java | 9 +- .../bigquery/SinkTaskPropertiesFactory.java | 49 ---- .../config/BigQuerySinkConfigTest.java | 109 ++++++++ .../config/BigQuerySinkTaskConfigTest.java | 175 ------------- .../write/row/BigQueryWriterTest.java | 15 +- .../bigquery/write/row/GCSToBQWriterTest.java | 12 +- 13 files changed, 395 insertions(+), 584 deletions(-) delete mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkConnectorPropertiesFactory.java delete mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkTaskPropertiesFactory.java delete mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfigTest.java diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java index f4ed4eafc..d2fefb8fb 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java @@ -22,12 +22,9 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.TableId; -import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; - import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; -import com.wepay.kafka.connect.bigquery.convert.SchemaConverter; - +import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; @@ -47,7 +44,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; /** * A {@link SinkConnector} used to delegate BigQuery data writes to @@ -57,8 +53,6 @@ public class BigQuerySinkConnector extends SinkConnector { private final BigQuery testBigQuery; private final SchemaManager testSchemaManager; - public static final String GCS_BQ_TASK_CONFIG_KEY = "GCSBQTask"; - public BigQuerySinkConnector() { testBigQuery = null; testSchemaManager = null; @@ -84,16 +78,16 @@ public BigQuerySinkConnector() { @Override public ConfigDef config() { logger.trace("connector.config()"); - return config.getConfig(); + return BigQuerySinkConfig.getConfig(); } private BigQuery getBigQuery() { if (testBigQuery != null) { return testBigQuery; } - String projectName = config.getString(config.PROJECT_CONFIG); + String projectName = config.getString(BigQuerySinkConfig.PROJECT_CONFIG); String key = config.getKeyFile(); - String keySource = config.getString(config.KEY_SOURCE_CONFIG); + String keySource = config.getString(BigQuerySinkConfig.KEY_SOURCE_CONFIG); return new BigQueryHelper().setKeySource(keySource).connect(projectName, key); } @@ -104,7 +98,7 @@ private void ensureExistingTables() { if (bigQuery.getTable(tableId) == null) { logger.warn( "You may want to enable auto table creation by setting {}=true in the properties file", - config.TABLE_CREATE_CONFIG); + BigQuerySinkConfig.TABLE_CREATE_CONFIG); throw new BigQueryConnectException("Table '" + tableId + "' does not exist"); } } @@ -123,7 +117,7 @@ public void start(Map properties) { ); } - if (!config.getBoolean(config.TABLE_CREATE_CONFIG)) { + if (!config.getBoolean(BigQuerySinkConfig.TABLE_CREATE_CONFIG)) { ensureExistingTables(); } } @@ -148,7 +142,7 @@ public List> taskConfigs(int maxTasks) { HashMap taskConfig = new HashMap<>(configProperties); if (i == 0 && !config.getList(BigQuerySinkConfig.ENABLE_BATCH_CONFIG).isEmpty()) { // if batch loading is enabled, configure first task to do the GCS -> BQ loading - taskConfig.put(GCS_BQ_TASK_CONFIG_KEY, "true"); + taskConfig.put(BigQuerySinkTaskConfig.GCS_BQ_TASK_CONFIG, "true"); } taskConfigs.add(taskConfig); } diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java index 118007756..4f59c07d8 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java @@ -171,7 +171,7 @@ private RowToInsert getRecordRow(SinkRecord record) { if (kafkaDataFieldName.isPresent()) { convertedRecord.put(kafkaDataFieldName.get(), KafkaDataBuilder.buildKafkaDataRecord(record)); } - if (config.getBoolean(config.SANITIZE_FIELD_NAME_CONFIG)) { + if (config.getBoolean(BigQuerySinkConfig.SANITIZE_FIELD_NAME_CONFIG)) { convertedRecord = FieldNameSanitizer.replaceInvalidKeys(convertedRecord); } return RowToInsert.of(getRowId(record), convertedRecord); @@ -205,17 +205,17 @@ public void put(Collection records) { if (!tableWriterBuilders.containsKey(table)) { TableWriterBuilder tableWriterBuilder; - if (config.getList(config.ENABLE_BATCH_CONFIG).contains(record.topic())) { + if (config.getList(BigQuerySinkConfig.ENABLE_BATCH_CONFIG).contains(record.topic())) { String topic = record.topic(); String gcsBlobName = topic + "_" + uuid + "_" + Instant.now().toEpochMilli(); - String gcsFolderName = config.getString(config.GCS_FOLDER_NAME_CONFIG); + String gcsFolderName = config.getString(BigQuerySinkConfig.GCS_FOLDER_NAME_CONFIG); if (gcsFolderName != null && !"".equals(gcsFolderName)) { gcsBlobName = gcsFolderName + "/" + gcsBlobName; } tableWriterBuilder = new GCSBatchTableWriter.Builder( gcsToBQWriter, table.getBaseTableId(), - config.getString(config.GCS_BUCKET_NAME_CONFIG), + config.getString(BigQuerySinkConfig.GCS_BUCKET_NAME_CONFIG), gcsBlobName, topic, recordConverter); @@ -255,9 +255,9 @@ private BigQuery getBigQuery() { if (testBigQuery != null) { return testBigQuery; } - String projectName = config.getString(config.PROJECT_CONFIG); + String projectName = config.getString(BigQuerySinkConfig.PROJECT_CONFIG); String keyFile = config.getKeyFile(); - String keySource = config.getString(config.KEY_SOURCE_CONFIG); + String keySource = config.getString(BigQuerySinkConfig.KEY_SOURCE_CONFIG); return new BigQueryHelper().setKeySource(keySource).connect(projectName, keyFile); } @@ -277,10 +277,10 @@ private SchemaManager getSchemaManager(BigQuery bigQuery) { } private BigQueryWriter getBigQueryWriter() { - boolean autoUpdateSchemas = config.getBoolean(config.SCHEMA_UPDATE_CONFIG); - boolean autoCreateTables = config.getBoolean(config.TABLE_CREATE_CONFIG); - int retry = config.getInt(config.BIGQUERY_RETRY_CONFIG); - long retryWait = config.getLong(config.BIGQUERY_RETRY_WAIT_CONFIG); + boolean autoUpdateSchemas = config.getBoolean(BigQuerySinkConfig.SCHEMA_UPDATE_CONFIG); + boolean autoCreateTables = config.getBoolean(BigQuerySinkConfig.TABLE_CREATE_CONFIG); + int retry = config.getInt(BigQuerySinkConfig.BIGQUERY_RETRY_CONFIG); + long retryWait = config.getLong(BigQuerySinkConfig.BIGQUERY_RETRY_WAIT_CONFIG); BigQuery bigQuery = getBigQuery(); if (autoUpdateSchemas || autoCreateTables) { return new AdaptiveBigQueryWriter(bigQuery, @@ -298,18 +298,18 @@ private Storage getGcs() { if (testGcs != null) { return testGcs; } - String projectName = config.getString(config.PROJECT_CONFIG); + String projectName = config.getString(BigQuerySinkConfig.PROJECT_CONFIG); String key = config.getKeyFile(); - String keySource = config.getString(config.KEY_SOURCE_CONFIG); + String keySource = config.getString(BigQuerySinkConfig.KEY_SOURCE_CONFIG); return new GCSBuilder(projectName).setKey(key).setKeySource(keySource).build(); } private GCSToBQWriter getGcsWriter() { BigQuery bigQuery = getBigQuery(); - int retry = config.getInt(config.BIGQUERY_RETRY_CONFIG); - long retryWait = config.getLong(config.BIGQUERY_RETRY_WAIT_CONFIG); - boolean autoCreateTables = config.getBoolean(config.TABLE_CREATE_CONFIG); + int retry = config.getInt(BigQuerySinkConfig.BIGQUERY_RETRY_CONFIG); + long retryWait = config.getLong(BigQuerySinkConfig.BIGQUERY_RETRY_WAIT_CONFIG); + boolean autoCreateTables = config.getBoolean(BigQuerySinkConfig.TABLE_CREATE_CONFIG); // schemaManager shall only be needed for creating table hence do not fetch instance if not // needed. SchemaManager schemaManager = autoCreateTables ? getSchemaManager(bigQuery) : null; @@ -324,8 +324,6 @@ private GCSToBQWriter getGcsWriter() { @Override public void start(Map properties) { logger.trace("task.start()"); - final boolean hasGCSBQTask = - properties.remove(BigQuerySinkConnector.GCS_BQ_TASK_CONFIG_KEY) != null; try { config = new BigQuerySinkTaskConfig(properties); } catch (ConfigException err) { @@ -342,10 +340,10 @@ public void start(Map properties) { executor = new KCBQThreadPoolExecutor(config, new LinkedBlockingQueue<>()); topicPartitionManager = new TopicPartitionManager(); useMessageTimeDatePartitioning = - config.getBoolean(config.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG); + config.getBoolean(BigQuerySinkConfig.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG); usePartitionDecorator = - config.getBoolean(config.BIGQUERY_PARTITION_DECORATOR_CONFIG); - if (hasGCSBQTask) { + config.getBoolean(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG); + if (config.getBoolean(BigQuerySinkTaskConfig.GCS_BQ_TASK_CONFIG)) { startGCSToBQLoadTask(); } } @@ -353,7 +351,7 @@ public void start(Map properties) { private void startGCSToBQLoadTask() { logger.info("Attempting to start GCS Load Executor."); gcsLoadExecutor = Executors.newScheduledThreadPool(1); - String bucketName = config.getString(config.GCS_BUCKET_NAME_CONFIG); + String bucketName = config.getString(BigQuerySinkConfig.GCS_BUCKET_NAME_CONFIG); Storage gcs = getGcs(); // get the bucket, or create it if it does not exist. Bucket bucket = gcs.get(bucketName); diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java index 704288f59..c7cc02593 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java @@ -54,7 +54,6 @@ * Base class for connector and task configs; contains properties shared between the two of them. */ public class BigQuerySinkConfig extends AbstractConfig { - private static final ConfigDef config; private static final Validator validator = new Validator(); private static final Logger logger = LoggerFactory.getLogger(BigQuerySinkConfig.class); @@ -234,8 +233,105 @@ public class BigQuerySinkConfig extends AbstractConfig { private static final String TABLE_CREATE_DOC = "Automatically create BigQuery tables if they don't already exist"; - static { - config = new ConfigDef() + public static final String SCHEMA_UPDATE_CONFIG = "autoUpdateSchemas"; + private static final ConfigDef.Type SCHEMA_UPDATE_TYPE = ConfigDef.Type.BOOLEAN; + public static final Boolean SCHEMA_UPDATE_DEFAULT = false; + private static final ConfigDef.Importance SCHEMA_UPDATE_IMPORTANCE = ConfigDef.Importance.HIGH; + private static final String SCHEMA_UPDATE_DOC = + "Whether or not to automatically update BigQuery schemas"; + + public static final String THREAD_POOL_SIZE_CONFIG = "threadPoolSize"; + private static final ConfigDef.Type THREAD_POOL_SIZE_TYPE = ConfigDef.Type.INT; + public static final Integer THREAD_POOL_SIZE_DEFAULT = 10; + private static final ConfigDef.Validator THREAD_POOL_SIZE_VALIDATOR = ConfigDef.Range.atLeast(1); + private static final ConfigDef.Importance THREAD_POOL_SIZE_IMPORTANCE = + ConfigDef.Importance.MEDIUM; + private static final String THREAD_POOL_SIZE_DOC = + "The size of the BigQuery write thread pool. This establishes the maximum number of " + + "concurrent writes to BigQuery."; + + public static final String QUEUE_SIZE_CONFIG = "queueSize"; + private static final ConfigDef.Type QUEUE_SIZE_TYPE = ConfigDef.Type.LONG; + // should this even have a default? + public static final Long QUEUE_SIZE_DEFAULT = -1L; + private static final ConfigDef.Validator QUEUE_SIZE_VALIDATOR = ConfigDef.Range.atLeast(-1); + private static final ConfigDef.Importance QUEUE_SIZE_IMPORTANCE = ConfigDef.Importance.HIGH; + private static final String QUEUE_SIZE_DOC = + "The maximum size (or -1 for no maximum size) of the worker queue for bigQuery write " + + "requests before all topics are paused. This is a soft limit; the size of the queue can " + + "go over this before topics are paused. All topics will be resumed once a flush is " + + "requested or the size of the queue drops under half of the maximum size."; + + public static final String BIGQUERY_RETRY_CONFIG = "bigQueryRetry"; + private static final ConfigDef.Type BIGQUERY_RETRY_TYPE = ConfigDef.Type.INT; + public static final Integer BIGQUERY_RETRY_DEFAULT = 0; + private static final ConfigDef.Validator BIGQUERY_RETRY_VALIDATOR = ConfigDef.Range.atLeast(0); + private static final ConfigDef.Importance BIGQUERY_RETRY_IMPORTANCE = + ConfigDef.Importance.MEDIUM; + private static final String BIGQUERY_RETRY_DOC = + "The number of retry attempts that will be made per BigQuery request that fails with a " + + "backend error or a quota exceeded error"; + + public static final String BIGQUERY_RETRY_WAIT_CONFIG = "bigQueryRetryWait"; + private static final ConfigDef.Type BIGQUERY_RETRY_WAIT_CONFIG_TYPE = ConfigDef.Type.LONG; + public static final Long BIGQUERY_RETRY_WAIT_DEFAULT = 1000L; + private static final ConfigDef.Validator BIGQUERY_RETRY_WAIT_VALIDATOR = + ConfigDef.Range.atLeast(0); + private static final ConfigDef.Importance BIGQUERY_RETRY_WAIT_IMPORTANCE = + ConfigDef.Importance.MEDIUM; + private static final String BIGQUERY_RETRY_WAIT_DOC = + "The minimum amount of time, in milliseconds, to wait between BigQuery backend or quota " + + "exceeded error retry attempts."; + + public static final String BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG = + "bigQueryMessageTimePartitioning"; + private static final ConfigDef.Type BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG_TYPE = + ConfigDef.Type.BOOLEAN; + public static final Boolean BIGQUERY_MESSAGE_TIME_PARTITIONING_DEFAULT = false; + private static final ConfigDef.Importance BIGQUERY_MESSAGE_TIME_PARTITIONING_IMPORTANCE = + ConfigDef.Importance.HIGH; + private static final String BIGQUERY_MESSAGE_TIME_PARTITIONING_DOC = + "Whether or not to use the message time when inserting records. " + + "Default uses the connector processing time."; + + public static final String BIGQUERY_PARTITION_DECORATOR_CONFIG = + "bigQueryPartitionDecorator"; + private static final ConfigDef.Type BIGQUERY_PARTITION_DECORATOR_CONFIG_TYPE = + ConfigDef.Type.BOOLEAN; + //This has been set to true to preserve the existing behavior. However, we can set it to false if field based partitioning is used in BigQuery + public static final Boolean BIGQUERY_PARTITION_DECORATOR_DEFAULT = true; + private static final ConfigDef.Importance BIGQUERY_PARTITION_DECORATOR_IMPORTANCE = + ConfigDef.Importance.HIGH; + private static final String BIGQUERY_PARTITION_DECORATOR_DOC = + "Whether or not to append partition decorator to BigQuery table name when inserting records. " + + "Default is true. Setting this to true appends partition decorator to table name (e.g. table$yyyyMMdd depending on the configuration set for bigQueryPartitionDecorator). " + + "Setting this to false bypasses the logic to append the partition decorator and uses raw table name for inserts."; + + public static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG = "timestampPartitionFieldName"; + private static final ConfigDef.Type BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_TYPE = ConfigDef.Type.STRING; + private static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DEFAULT = null; + private static final ConfigDef.Importance BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_IMPORTANCE = + ConfigDef.Importance.LOW; + private static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DOC = + "The name of the field in the value that contains the timestamp to partition by in BigQuery" + + " and enable timestamp partitioning for each table. Leave this configuration blank," + + " to enable ingestion time partitioning for each table."; + + public static final String BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG = "clusteringPartitionFieldNames"; + private static final ConfigDef.Type BIGQUERY_CLUSTERING_FIELD_NAMES_TYPE = ConfigDef.Type.LIST; + private static final List BIGQUERY_CLUSTERING_FIELD_NAMES_DEFAULT = null; + private static final ConfigDef.Importance BIGQUERY_CLUSTERING_FIELD_NAMES_IMPORTANCE = + ConfigDef.Importance.LOW; + private static final String BIGQUERY_CLUSTERING_FIELD_NAMES_DOC = + "List of fields on which data should be clustered by in BigQuery, separated by commas"; + + /** + * Return the ConfigDef object used to define this config's fields. + * + * @return The ConfigDef object used to define this config's fields. + */ + public static ConfigDef getConfig() { + return new ConfigDef() .define( TOPICS_CONFIG, TOPICS_TYPE, @@ -366,6 +462,64 @@ public class BigQuerySinkConfig extends AbstractConfig { TABLE_CREATE_DEFAULT, TABLE_CREATE_IMPORTANCE, TABLE_CREATE_DOC + ).define( + SCHEMA_UPDATE_CONFIG, + SCHEMA_UPDATE_TYPE, + SCHEMA_UPDATE_DEFAULT, + SCHEMA_UPDATE_IMPORTANCE, + SCHEMA_UPDATE_DOC + ).define( + THREAD_POOL_SIZE_CONFIG, + THREAD_POOL_SIZE_TYPE, + THREAD_POOL_SIZE_DEFAULT, + THREAD_POOL_SIZE_VALIDATOR, + THREAD_POOL_SIZE_IMPORTANCE, + THREAD_POOL_SIZE_DOC + ).define( + QUEUE_SIZE_CONFIG, + QUEUE_SIZE_TYPE, + QUEUE_SIZE_DEFAULT, + QUEUE_SIZE_VALIDATOR, + QUEUE_SIZE_IMPORTANCE, + QUEUE_SIZE_DOC + ).define( + BIGQUERY_RETRY_CONFIG, + BIGQUERY_RETRY_TYPE, + BIGQUERY_RETRY_DEFAULT, + BIGQUERY_RETRY_VALIDATOR, + BIGQUERY_RETRY_IMPORTANCE, + BIGQUERY_RETRY_DOC + ).define( + BIGQUERY_RETRY_WAIT_CONFIG, + BIGQUERY_RETRY_WAIT_CONFIG_TYPE, + BIGQUERY_RETRY_WAIT_DEFAULT, + BIGQUERY_RETRY_WAIT_VALIDATOR, + BIGQUERY_RETRY_WAIT_IMPORTANCE, + BIGQUERY_RETRY_WAIT_DOC + ).define( + BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, + BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG_TYPE, + BIGQUERY_MESSAGE_TIME_PARTITIONING_DEFAULT, + BIGQUERY_MESSAGE_TIME_PARTITIONING_IMPORTANCE, + BIGQUERY_MESSAGE_TIME_PARTITIONING_DOC + ).define( + BIGQUERY_PARTITION_DECORATOR_CONFIG, + BIGQUERY_PARTITION_DECORATOR_CONFIG_TYPE, + BIGQUERY_PARTITION_DECORATOR_DEFAULT, + BIGQUERY_PARTITION_DECORATOR_IMPORTANCE, + BIGQUERY_PARTITION_DECORATOR_DOC + ).define( + BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, + BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_TYPE, + BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DEFAULT, + BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_IMPORTANCE, + BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DOC + ).define( + BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, + BIGQUERY_CLUSTERING_FIELD_NAMES_TYPE, + BIGQUERY_CLUSTERING_FIELD_NAMES_DEFAULT, + BIGQUERY_CLUSTERING_FIELD_NAMES_IMPORTANCE, + BIGQUERY_CLUSTERING_FIELD_NAMES_DOC ); } /** @@ -697,24 +851,80 @@ private void checkAutoCreateTables() { } } + private void checkAutoUpdateSchemas() { + Class schemaRetriever = getClass(BigQuerySinkConfig.SCHEMA_RETRIEVER_CONFIG); + + boolean autoUpdateSchemas = getBoolean(SCHEMA_UPDATE_CONFIG); + if (autoUpdateSchemas && schemaRetriever == null) { + throw new ConfigException( + "Cannot specify automatic table creation without a schema retriever" + ); + } + + if (schemaRetriever == null) { + logger.warn( + "No schema retriever class provided; auto schema updates are impossible" + ); + } + } + /** - * Return the ConfigDef object used to define this config's fields. - * - * @return The ConfigDef object used to define this config's fields. + * Returns the field name to use for timestamp partitioning. + * @return String that represents the field name. */ - public static ConfigDef getConfig() { - return config; + public Optional getTimestampPartitionFieldName() { + return Optional.ofNullable(getString(BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG)); + } + + /** + * Returns the field names to use for clustering. + * @return List of Strings that represent the field names. + */ + public Optional> getClusteringPartitionFieldName() { + return Optional.ofNullable(getList(BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG)); + } + + /** + * Check the validity of table partitioning configs. + */ + private void checkPartitionConfigs() { + if (getTimestampPartitionFieldName().isPresent() && getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)) { + throw new ConfigException( + "Only one partitioning configuration mode may be specified for the connector. " + + "Use either bigQueryPartitionDecorator OR timestampPartitionFieldName." + ); + } + } + + /** + * Check the validity of table clustering configs. + */ + private void checkClusteringConfigs() { + if (getClusteringPartitionFieldName().isPresent()) { + if (!getTimestampPartitionFieldName().isPresent() && !getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)) { + throw new ConfigException( + "Clustering field name may be specified only on a partitioned table." + ); + } + if (getClusteringPartitionFieldName().get().size() > 4) { + throw new ConfigException( + "You can only specify up to four clustering field names." + ); + } + } } protected BigQuerySinkConfig(ConfigDef config, Map properties) { super(config, properties); verifyBucketSpecified(); + checkAutoCreateTables(); + checkAutoUpdateSchemas(); + checkPartitionConfigs(); + checkClusteringConfigs(); } public BigQuerySinkConfig(Map properties) { - super(config, properties); - verifyBucketSpecified(); - checkAutoCreateTables(); + this(getConfig(), properties); } } diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfig.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfig.java index 0eb3d9dc5..9a72561be 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfig.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfig.java @@ -19,13 +19,7 @@ package com.wepay.kafka.connect.bigquery.config; -import java.util.List; -import java.util.Optional; import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; @@ -33,238 +27,21 @@ * Class for task-specific configuration properties. */ public class BigQuerySinkTaskConfig extends BigQuerySinkConfig { - private static final ConfigDef config; - private static final Logger logger = LoggerFactory.getLogger(BigQuerySinkTaskConfig.class); - - public static final String SCHEMA_UPDATE_CONFIG = "autoUpdateSchemas"; - private static final ConfigDef.Type SCHEMA_UPDATE_TYPE = ConfigDef.Type.BOOLEAN; - public static final Boolean SCHEMA_UPDATE_DEFAULT = false; - private static final ConfigDef.Importance SCHEMA_UPDATE_IMPORTANCE = ConfigDef.Importance.HIGH; - private static final String SCHEMA_UPDATE_DOC = - "Whether or not to automatically update BigQuery schemas"; - - public static final String THREAD_POOL_SIZE_CONFIG = "threadPoolSize"; - private static final ConfigDef.Type THREAD_POOL_SIZE_TYPE = ConfigDef.Type.INT; - public static final Integer THREAD_POOL_SIZE_DEFAULT = 10; - private static final ConfigDef.Validator THREAD_POOL_SIZE_VALIDATOR = ConfigDef.Range.atLeast(1); - private static final ConfigDef.Importance THREAD_POOL_SIZE_IMPORTANCE = - ConfigDef.Importance.MEDIUM; - private static final String THREAD_POOL_SIZE_DOC = - "The size of the BigQuery write thread pool. This establishes the maximum number of " - + "concurrent writes to BigQuery."; - - public static final String QUEUE_SIZE_CONFIG = "queueSize"; - private static final ConfigDef.Type QUEUE_SIZE_TYPE = ConfigDef.Type.LONG; - // should this even have a default? - public static final Long QUEUE_SIZE_DEFAULT = -1L; - private static final ConfigDef.Validator QUEUE_SIZE_VALIDATOR = ConfigDef.Range.atLeast(-1); - private static final ConfigDef.Importance QUEUE_SIZE_IMPORTANCE = ConfigDef.Importance.HIGH; - private static final String QUEUE_SIZE_DOC = - "The maximum size (or -1 for no maximum size) of the worker queue for bigQuery write " - + "requests before all topics are paused. This is a soft limit; the size of the queue can " - + "go over this before topics are paused. All topics will be resumed once a flush is " - + "requested or the size of the queue drops under half of the maximum size."; - - public static final String BIGQUERY_RETRY_CONFIG = "bigQueryRetry"; - private static final ConfigDef.Type BIGQUERY_RETRY_TYPE = ConfigDef.Type.INT; - public static final Integer BIGQUERY_RETRY_DEFAULT = 0; - private static final ConfigDef.Validator BIGQUERY_RETRY_VALIDATOR = ConfigDef.Range.atLeast(0); - private static final ConfigDef.Importance BIGQUERY_RETRY_IMPORTANCE = - ConfigDef.Importance.MEDIUM; - private static final String BIGQUERY_RETRY_DOC = - "The number of retry attempts that will be made per BigQuery request that fails with a " - + "backend error or a quota exceeded error"; - - public static final String BIGQUERY_RETRY_WAIT_CONFIG = "bigQueryRetryWait"; - private static final ConfigDef.Type BIGQUERY_RETRY_WAIT_CONFIG_TYPE = ConfigDef.Type.LONG; - public static final Long BIGQUERY_RETRY_WAIT_DEFAULT = 1000L; - private static final ConfigDef.Validator BIGQUERY_RETRY_WAIT_VALIDATOR = - ConfigDef.Range.atLeast(0); - private static final ConfigDef.Importance BIGQUERY_RETRY_WAIT_IMPORTANCE = - ConfigDef.Importance.MEDIUM; - private static final String BIGQUERY_RETRY_WAIT_DOC = - "The minimum amount of time, in milliseconds, to wait between BigQuery backend or quota " - + "exceeded error retry attempts."; - - public static final String BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG = - "bigQueryMessageTimePartitioning"; - private static final ConfigDef.Type BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG_TYPE = - ConfigDef.Type.BOOLEAN; - public static final Boolean BIGQUERY_MESSAGE_TIME_PARTITIONING_DEFAULT = false; - private static final ConfigDef.Importance BIGQUERY_MESSAGE_TIME_PARTITIONING_IMPORTANCE = - ConfigDef.Importance.HIGH; - private static final String BIGQUERY_MESSAGE_TIME_PARTITIONING_DOC = - "Whether or not to use the message time when inserting records. " - + "Default uses the connector processing time."; - - public static final String BIGQUERY_PARTITION_DECORATOR_CONFIG = - "bigQueryPartitionDecorator"; - private static final ConfigDef.Type BIGQUERY_PARTITION_DECORATOR_CONFIG_TYPE = - ConfigDef.Type.BOOLEAN; - //This has been set to true to preserve the existing behavior. However, we can set it to false if field based partitioning is used in BigQuery - public static final Boolean BIGQUERY_PARTITION_DECORATOR_DEFAULT = true; - private static final ConfigDef.Importance BIGQUERY_PARTITION_DECORATOR_IMPORTANCE = - ConfigDef.Importance.HIGH; - private static final String BIGQUERY_PARTITION_DECORATOR_DOC = - "Whether or not to append partition decorator to BigQuery table name when inserting records. " - + "Default is true. Setting this to true appends partition decorator to table name (e.g. table$yyyyMMdd depending on the configuration set for bigQueryPartitionDecorator). " - + "Setting this to false bypasses the logic to append the partition decorator and uses raw table name for inserts."; - - public static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG = "timestampPartitionFieldName"; - private static final ConfigDef.Type BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_TYPE = ConfigDef.Type.STRING; - private static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DEFAULT = null; - private static final ConfigDef.Importance BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_IMPORTANCE = - ConfigDef.Importance.LOW; - private static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DOC = - "The name of the field in the value that contains the timestamp to partition by in BigQuery" - + " and enable timestamp partitioning for each table. Leave this configuration blank," - + " to enable ingestion time partitioning for each table."; - - public static final String BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG = "clusteringPartitionFieldNames"; - private static final ConfigDef.Type BIGQUERY_CLUSTERING_FIELD_NAMES_TYPE = ConfigDef.Type.LIST; - private static final List BIGQUERY_CLUSTERING_FIELD_NAMES_DEFAULT = null; - private static final ConfigDef.Importance BIGQUERY_CLUSTERING_FIELD_NAMES_IMPORTANCE = - ConfigDef.Importance.LOW; - private static final String BIGQUERY_CLUSTERING_FIELD_NAMES_DOC = - "List of fields on which data should be clustered by in BigQuery, separated by commas"; - - static { - config = BigQuerySinkConfig.getConfig() - .define( - SCHEMA_UPDATE_CONFIG, - SCHEMA_UPDATE_TYPE, - SCHEMA_UPDATE_DEFAULT, - SCHEMA_UPDATE_IMPORTANCE, - SCHEMA_UPDATE_DOC - ).define( - THREAD_POOL_SIZE_CONFIG, - THREAD_POOL_SIZE_TYPE, - THREAD_POOL_SIZE_DEFAULT, - THREAD_POOL_SIZE_VALIDATOR, - THREAD_POOL_SIZE_IMPORTANCE, - THREAD_POOL_SIZE_DOC - ).define( - QUEUE_SIZE_CONFIG, - QUEUE_SIZE_TYPE, - QUEUE_SIZE_DEFAULT, - QUEUE_SIZE_VALIDATOR, - QUEUE_SIZE_IMPORTANCE, - QUEUE_SIZE_DOC - ).define( - BIGQUERY_RETRY_CONFIG, - BIGQUERY_RETRY_TYPE, - BIGQUERY_RETRY_DEFAULT, - BIGQUERY_RETRY_VALIDATOR, - BIGQUERY_RETRY_IMPORTANCE, - BIGQUERY_RETRY_DOC - ).define( - BIGQUERY_RETRY_WAIT_CONFIG, - BIGQUERY_RETRY_WAIT_CONFIG_TYPE, - BIGQUERY_RETRY_WAIT_DEFAULT, - BIGQUERY_RETRY_WAIT_VALIDATOR, - BIGQUERY_RETRY_WAIT_IMPORTANCE, - BIGQUERY_RETRY_WAIT_DOC - ).define( - BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, - BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG_TYPE, - BIGQUERY_MESSAGE_TIME_PARTITIONING_DEFAULT, - BIGQUERY_MESSAGE_TIME_PARTITIONING_IMPORTANCE, - BIGQUERY_MESSAGE_TIME_PARTITIONING_DOC - ).define( - BIGQUERY_PARTITION_DECORATOR_CONFIG, - BIGQUERY_PARTITION_DECORATOR_CONFIG_TYPE, - BIGQUERY_PARTITION_DECORATOR_DEFAULT, - BIGQUERY_PARTITION_DECORATOR_IMPORTANCE, - BIGQUERY_PARTITION_DECORATOR_DOC - ).define( - BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, - BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_TYPE, - BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DEFAULT, - BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_IMPORTANCE, - BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DOC - ).define( - BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, - BIGQUERY_CLUSTERING_FIELD_NAMES_TYPE, - BIGQUERY_CLUSTERING_FIELD_NAMES_DEFAULT, - BIGQUERY_CLUSTERING_FIELD_NAMES_IMPORTANCE, - BIGQUERY_CLUSTERING_FIELD_NAMES_DOC - ); - } - - private void checkAutoUpdateSchemas() { - Class schemaRetriever = getClass(BigQuerySinkConfig.SCHEMA_RETRIEVER_CONFIG); - - boolean autoUpdateSchemas = getBoolean(SCHEMA_UPDATE_CONFIG); - if (autoUpdateSchemas && schemaRetriever == null) { - throw new ConfigException( - "Cannot specify automatic table creation without a schema retriever" - ); - } - - if (schemaRetriever == null) { - logger.warn( - "No schema retriever class provided; auto schema updates are impossible" - ); - } - } - /** - * Returns the field name to use for timestamp partitioning. - * @return String that represents the field name. - */ - public Optional getTimestampPartitionFieldName() { - return Optional.ofNullable(getString(BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG)); - } - - /** - * Returns the field names to use for clustering. - * @return List of Strings that represent the field names. - */ - public Optional> getClusteringPartitionFieldName() { - return Optional.ofNullable(getList(BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG)); - } - - /** - * Check the validity of table partitioning configs. - */ - private void checkPartitionConfigs() { - if (getTimestampPartitionFieldName().isPresent() && getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)) { - throw new ConfigException( - "Only one partitioning configuration mode may be specified for the connector. " - + "Use either bigQueryPartitionDecorator OR timestampPartitionFieldName." - ); - } - } - - /** - * Check the validity of table clustering configs. - */ - private void checkClusteringConfigs() { - if (getClusteringPartitionFieldName().isPresent()) { - if (!getTimestampPartitionFieldName().isPresent() && !getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)) { - throw new ConfigException( - "Clustering field name may be specified only on a partitioned table." - ); - } - if (getClusteringPartitionFieldName().get().size() > 4) { - throw new ConfigException( - "You can only specify up to four clustering field names." - ); - } - } - } + public static final String GCS_BQ_TASK_CONFIG = "GCSBQTask"; + private static final ConfigDef.Type GCS_BQ_TASK_TYPE = ConfigDef.Type.BOOLEAN; + private static final boolean GCS_BQ_TASK_DEFAULT = false; + private static final ConfigDef.Importance GCS_BQ_TASK_IMPORTANCE = ConfigDef.Importance.LOW; - public static ConfigDef getConfig() { - return config; + private static ConfigDef config() { + return BigQuerySinkConfig.getConfig() + .defineInternal(GCS_BQ_TASK_CONFIG, GCS_BQ_TASK_TYPE, GCS_BQ_TASK_DEFAULT, GCS_BQ_TASK_IMPORTANCE); } /** * @param properties A Map detailing configuration properties and their respective values. */ public BigQuerySinkTaskConfig(Map properties) { - super(config, properties); - checkAutoUpdateSchemas(); - checkPartitionConfigs(); - checkClusteringConfigs(); + super(config(), properties); } } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java index 433dc130b..b6558a7fa 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java @@ -53,7 +53,7 @@ import java.util.Map; public class BigQuerySinkConnectorTest { - private static SinkConnectorPropertiesFactory propertiesFactory; + private static SinkPropertiesFactory propertiesFactory; // Would just use Mockito, but can't provide the name of an anonymous class to the config file public static class MockSchemaRetriever implements SchemaRetriever { @@ -75,7 +75,7 @@ public void setLastSeenSchema(TableId table, String topic, Schema schema) { @BeforeClass public static void initializePropertiesFactory() { - propertiesFactory = new SinkConnectorPropertiesFactory(); + propertiesFactory = new SinkPropertiesFactory(); } @Test @@ -127,7 +127,7 @@ public void testTaskConfigs() { @Test public void testConfig() { - assertEquals(BigQuerySinkConfig.getConfig(), new BigQuerySinkConnector().config()); + assertNotNull(new BigQuerySinkConnector().config()); } // Make sure that a config exception is properly translated into a SinkConfigConnectException diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java index fd16921f4..9e86c3fe1 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java @@ -39,7 +39,6 @@ import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; -import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; import org.apache.kafka.common.config.ConfigException; @@ -63,11 +62,11 @@ import java.util.concurrent.RejectedExecutionException; public class BigQuerySinkTaskTest { - private static SinkTaskPropertiesFactory propertiesFactory; + private static SinkPropertiesFactory propertiesFactory; @BeforeClass public static void initializePropertiesFactory() { - propertiesFactory = new SinkTaskPropertiesFactory(); + propertiesFactory = new SinkPropertiesFactory(); } @Test @@ -177,7 +176,7 @@ public void testPutWhenPartitioningOnMessageTime() { Map properties = propertiesFactory.getProperties(); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=scratch"); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, "true"); + properties.put(BigQuerySinkConfig.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, "true"); BigQuery bigQuery = mock(BigQuery.class); Storage storage = mock(Storage.class); @@ -210,8 +209,8 @@ public void testPutWhenPartitioningIsSetToTrue() { Map properties = propertiesFactory.getProperties(); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=scratch"); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "true"); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, "true"); + properties.put(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "true"); + properties.put(BigQuerySinkConfig.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, "true"); BigQuery bigQuery = mock(BigQuery.class); Storage storage = mock(Storage.class); @@ -244,7 +243,7 @@ public void testPutWhenPartitioningIsSetToFalse() { Map properties = propertiesFactory.getProperties(); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=scratch"); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); + properties.put(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); BigQuery bigQuery = mock(BigQuery.class); Storage storage = mock(Storage.class); @@ -278,7 +277,7 @@ public void testPutWhenPartitioningOnMessageTimeWhenNoTimestampType() { Map properties = propertiesFactory.getProperties(); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=scratch"); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, "true"); + properties.put(BigQuerySinkConfig.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, "true"); BigQuery bigQuery = mock(BigQuery.class); Storage storage = mock(Storage.class); @@ -389,8 +388,8 @@ public void testBigQuery5XXRetry() { final String dataset = "scratch"; Map properties = propertiesFactory.getProperties(); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_CONFIG, "3"); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_WAIT_CONFIG, "2000"); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_CONFIG, "3"); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_WAIT_CONFIG, "2000"); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, String.format(".*=%s", dataset)); @@ -425,8 +424,8 @@ public void testBigQuery403Retry() { final String dataset = "scratch"; Map properties = propertiesFactory.getProperties(); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_CONFIG, "2"); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_WAIT_CONFIG, "2000"); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_CONFIG, "2"); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_WAIT_CONFIG, "2000"); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, String.format(".*=%s", dataset)); @@ -462,8 +461,8 @@ public void testBigQueryRetryExceeded() { final String dataset = "scratch"; Map properties = propertiesFactory.getProperties(); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_CONFIG, "1"); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_WAIT_CONFIG, "2000"); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_CONFIG, "1"); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_WAIT_CONFIG, "2000"); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, String.format(".*=%s", dataset)); diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkConnectorPropertiesFactory.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkConnectorPropertiesFactory.java deleted file mode 100644 index e6ea6a5ee..000000000 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkConnectorPropertiesFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2020 Confluent, Inc. - * - * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wepay.kafka.connect.bigquery; - -import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; - -import java.util.Map; - -public class SinkConnectorPropertiesFactory extends SinkPropertiesFactory { - @Override - public Map getProperties() { - Map properties = super.getProperties(); - - properties.put(BigQuerySinkConfig.TABLE_CREATE_CONFIG, "false"); - return properties; - } - - /** - * Make sure that each of the default configuration properties work nicely with the given - * configuration object. - * - * @param config The config object to test - */ - public void testProperties(BigQuerySinkConfig config) { - super.testProperties(config); - - config.getBoolean(config.TABLE_CREATE_CONFIG); - } -} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java index 0d6517c90..3bb981b4e 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java @@ -35,9 +35,9 @@ public Map getProperties() { Map properties = new HashMap<>(); properties.put(BigQuerySinkConfig.TABLE_CREATE_CONFIG, "false"); + properties.put(BigQuerySinkConfig.SCHEMA_UPDATE_CONFIG, "false"); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, "kcbq-test"); properties.put(BigQuerySinkConfig.PROJECT_CONFIG, "test-project"); - properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=test"); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, "kcbq-test=kcbq-test-table"); properties.put(BigQuerySinkConfig.KEYFILE_CONFIG, "key.json"); @@ -59,14 +59,9 @@ public void testProperties(BigQuerySinkConfig config) { config.getMap(config.DATASETS_CONFIG); config.getMap(config.TOPICS_TO_TABLES_CONFIG); - config.getList(config.TOPICS_CONFIG); - config.getList(config.TOPICS_TO_TABLES_CONFIG); - config.getList(config.DATASETS_CONFIG); - - config.getKeyFile(); config.getString(config.PROJECT_CONFIG); - + config.getKeyFile(); config.getBoolean(config.SANITIZE_TOPICS_CONFIG); config.getInt(config.AVRO_DATA_CACHE_SIZE_CONFIG); } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkTaskPropertiesFactory.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkTaskPropertiesFactory.java deleted file mode 100644 index 222d1d881..000000000 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkTaskPropertiesFactory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2020 Confluent, Inc. - * - * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wepay.kafka.connect.bigquery; - -import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; -import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; - -import java.util.Map; - -public class SinkTaskPropertiesFactory extends SinkPropertiesFactory { - @Override - public Map getProperties() { - Map properties = super.getProperties(); - - properties.put(BigQuerySinkTaskConfig.SCHEMA_UPDATE_CONFIG, "false"); - properties.put(BigQuerySinkConfig.TABLE_CREATE_CONFIG, "false"); - - return properties; - } - - /** - * Make sure that each of the default configuration properties work nicely with the given - * configuration object. - * - * @param config The config object to test - */ - public void testProperties(BigQuerySinkTaskConfig config) { - super.testProperties(config); - - config.getBoolean(config.SCHEMA_UPDATE_CONFIG); - } -} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java index e6ec7bea2..8e1f7328c 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java @@ -20,6 +20,7 @@ package com.wepay.kafka.connect.bigquery.config; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -32,8 +33,12 @@ import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; public class BigQuerySinkConfigTest { private SinkPropertiesFactory propertiesFactory; @@ -208,4 +213,108 @@ public void testInvalidAvroCacheSize() { new BigQuerySinkConfig(badConfigProperties); } + + /** + * Test the default for the field name is not present. + */ + @Test + public void testEmptyTimestampPartitionFieldName() { + Map configProperties = propertiesFactory.getProperties(); + BigQuerySinkConfig testConfig = new BigQuerySinkConfig(configProperties); + assertFalse(testConfig.getTimestampPartitionFieldName().isPresent()); + } + + /** + * Test if the field name being non-empty and the decorator default (true) errors correctly. + */ + @Test (expected = ConfigException.class) + public void testTimestampPartitionFieldNameError() { + Map configProperties = propertiesFactory.getProperties(); + configProperties.put(BigQuerySinkConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, "name"); + new BigQuerySinkConfig(configProperties); + } + + /** + * Test the field name being non-empty and the decorator set to false works correctly. + */ + @Test + public void testTimestampPartitionFieldName() { + Map configProperties = propertiesFactory.getProperties(); + configProperties.put(BigQuerySinkConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, "name"); + configProperties.put(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); + BigQuerySinkConfig testConfig = new BigQuerySinkConfig(configProperties); + assertTrue(testConfig.getTimestampPartitionFieldName().isPresent()); + assertFalse(testConfig.getBoolean(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG)); + } + + /** + * Test the default for the field names is not present. + */ + @Test + public void testEmptyClusteringFieldNames() { + Map configProperties = propertiesFactory.getProperties(); + BigQuerySinkConfig testConfig = new BigQuerySinkConfig(configProperties); + assertFalse(testConfig.getClusteringPartitionFieldName().isPresent()); + } + + /** + * Test if the field names being non-empty and the partitioning is not present errors correctly. + */ + @Test (expected = ConfigException.class) + public void testClusteringFieldNamesWithoutTimestampPartitionError() { + Map configProperties = propertiesFactory.getProperties(); + configProperties.put(BigQuerySinkConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, null); + configProperties.put(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); + configProperties.put( + BigQuerySinkConfig.BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, + "column1,column2" + ); + new BigQuerySinkConfig(configProperties); + } + + /** + * Test if the field names are more than four fields errors correctly. + */ + @Test (expected = ConfigException.class) + public void testClusteringPartitionFieldNamesWithMoreThanFourFieldsError() { + Map configProperties = propertiesFactory.getProperties(); + configProperties.put(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "true"); + configProperties.put( + BigQuerySinkConfig.BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, + "column1,column2,column3,column4,column5" + ); + new BigQuerySinkConfig(configProperties); + } + + /** + * Test the field names being non-empty and the partitioning field exists works correctly. + */ + @Test + public void testClusteringFieldNames() { + Map configProperties = propertiesFactory.getProperties(); + configProperties.put(BigQuerySinkConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, "name"); + configProperties.put(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); + configProperties.put( + BigQuerySinkConfig.BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, + "column1,column2" + ); + + ArrayList expectedClusteringPartitionFieldName = new ArrayList<>( + Arrays.asList("column1", "column2") + ); + + BigQuerySinkConfig testConfig = new BigQuerySinkConfig(configProperties); + Optional> testClusteringPartitionFieldName = testConfig.getClusteringPartitionFieldName(); + assertTrue(testClusteringPartitionFieldName.isPresent()); + assertEquals(expectedClusteringPartitionFieldName, testClusteringPartitionFieldName.get()); + } + + @Test(expected = ConfigException.class) + public void testAutoSchemaUpdateWithoutRetriever() { + Map badConfigProperties = propertiesFactory.getProperties(); + badConfigProperties.remove(BigQuerySinkConfig.SCHEMA_RETRIEVER_CONFIG); + badConfigProperties.put(BigQuerySinkConfig.SCHEMA_UPDATE_CONFIG, "true"); + + new BigQuerySinkConfig(badConfigProperties); + } } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfigTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfigTest.java deleted file mode 100644 index 3f55adb48..000000000 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkTaskConfigTest.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright 2020 Confluent, Inc. - * - * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wepay.kafka.connect.bigquery.config; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import com.wepay.kafka.connect.bigquery.SinkTaskPropertiesFactory; - -import org.apache.kafka.common.config.ConfigException; - -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -public class BigQuerySinkTaskConfigTest { - private SinkTaskPropertiesFactory propertiesFactory; - - @Before - public void initializePropertiesFactory() { - propertiesFactory = new SinkTaskPropertiesFactory(); - } - - @Test - public void metaTestBasicConfigProperties() { - Map basicConfigProperties = propertiesFactory.getProperties(); - BigQuerySinkTaskConfig config = new BigQuerySinkTaskConfig(basicConfigProperties); - propertiesFactory.testProperties(config); - } - - @Test() - public void testMaxWriteSize() { - // todo: something like this, maybe. - /* - Map badProperties = propertiesFactory.getProperties(); - badProperties.put(BigQuerySinkTaskConfig.MAX_WRITE_CONFIG, "-1"); - - try { - new BigQuerySinkTaskConfig(badProperties); - } catch (ConfigException err) { - fail("Exception encountered before addition of bad configuration field: " + err); - } - - badProperties.put(BigQuerySinkTaskConfig.MAX_WRITE_CONFIG, "0"); - new BigQuerySinkTaskConfig(badProperties); - */ - } - - /** - * Test the default for the field name is not present. - */ - @Test - public void testEmptyTimestampPartitionFieldName() { - Map configProperties = propertiesFactory.getProperties(); - BigQuerySinkTaskConfig testConfig = new BigQuerySinkTaskConfig(configProperties); - assertFalse(testConfig.getTimestampPartitionFieldName().isPresent()); - } - - /** - * Test if the field name being non-empty and the decorator default (true) errors correctly. - */ - @Test (expected = ConfigException.class) - public void testTimestampPartitionFieldNameError() { - Map configProperties = propertiesFactory.getProperties(); - configProperties.put(BigQuerySinkTaskConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, "name"); - new BigQuerySinkTaskConfig(configProperties); - } - - /** - * Test the field name being non-empty and the decorator set to false works correctly. - */ - @Test - public void testTimestampPartitionFieldName() { - Map configProperties = propertiesFactory.getProperties(); - configProperties.put(BigQuerySinkTaskConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, "name"); - configProperties.put(BigQuerySinkTaskConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); - BigQuerySinkTaskConfig testConfig = new BigQuerySinkTaskConfig(configProperties); - assertTrue(testConfig.getTimestampPartitionFieldName().isPresent()); - assertFalse(testConfig.getBoolean(BigQuerySinkTaskConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG)); - } - - /** - * Test the default for the field names is not present. - */ - @Test - public void testEmptyClusteringFieldNames() { - Map configProperties = propertiesFactory.getProperties(); - BigQuerySinkTaskConfig testConfig = new BigQuerySinkTaskConfig(configProperties); - assertFalse(testConfig.getClusteringPartitionFieldName().isPresent()); - } - - /** - * Test if the field names being non-empty and the partitioning is not present errors correctly. - */ - @Test (expected = ConfigException.class) - public void testClusteringFieldNamesWithoutTimestampPartitionError() { - Map configProperties = propertiesFactory.getProperties(); - configProperties.put(BigQuerySinkTaskConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, null); - configProperties.put(BigQuerySinkTaskConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); - configProperties.put( - BigQuerySinkTaskConfig.BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, - "column1,column2" - ); - new BigQuerySinkTaskConfig(configProperties); - } - - /** - * Test if the field names are more than four fields errors correctly. - */ - @Test (expected = ConfigException.class) - public void testClusteringPartitionFieldNamesWithMoreThanFourFieldsError() { - Map configProperties = propertiesFactory.getProperties(); - configProperties.put(BigQuerySinkTaskConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "true"); - configProperties.put( - BigQuerySinkTaskConfig.BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, - "column1,column2,column3,column4,column5" - ); - new BigQuerySinkTaskConfig(configProperties); - } - - /** - * Test the field names being non-empty and the partitioning field exists works correctly. - */ - @Test - public void testClusteringFieldNames() { - Map configProperties = propertiesFactory.getProperties(); - configProperties.put(BigQuerySinkTaskConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, "name"); - configProperties.put(BigQuerySinkTaskConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); - configProperties.put( - BigQuerySinkTaskConfig.BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, - "column1,column2" - ); - - ArrayList expectedClusteringPartitionFieldName = new ArrayList<>( - Arrays.asList("column1", "column2") - ); - - BigQuerySinkTaskConfig testConfig = new BigQuerySinkTaskConfig(configProperties); - Optional> testClusteringPartitionFieldName = testConfig.getClusteringPartitionFieldName(); - assertTrue(testClusteringPartitionFieldName.isPresent()); - assertEquals(expectedClusteringPartitionFieldName, testClusteringPartitionFieldName.get()); - } - - @Test(expected = ConfigException.class) - public void testAutoSchemaUpdateWithoutRetriever() { - Map badConfigProperties = propertiesFactory.getProperties(); - badConfigProperties.remove(BigQuerySinkTaskConfig.SCHEMA_RETRIEVER_CONFIG); - badConfigProperties.put(BigQuerySinkTaskConfig.SCHEMA_UPDATE_CONFIG, "true"); - - new BigQuerySinkTaskConfig(badConfigProperties); - } -} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java index 4ecbb6961..d55f377cb 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/BigQueryWriterTest.java @@ -37,12 +37,12 @@ import com.wepay.kafka.connect.bigquery.BigQuerySinkTask; import com.wepay.kafka.connect.bigquery.SchemaManager; -import com.wepay.kafka.connect.bigquery.SinkTaskPropertiesFactory; +import com.wepay.kafka.connect.bigquery.SinkPropertiesFactory; import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; -import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; +import com.wepay.kafka.connect.bigquery.retrieve.MemorySchemaRetriever; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; @@ -63,11 +63,11 @@ @SuppressWarnings("unchecked") public class BigQueryWriterTest { - private static SinkTaskPropertiesFactory propertiesFactory; + private static SinkPropertiesFactory propertiesFactory; @BeforeClass public static void initializePropertiesFactory() { - propertiesFactory = new SinkTaskPropertiesFactory(); + propertiesFactory = new SinkPropertiesFactory(); } @Test @@ -109,7 +109,8 @@ public void testAutoCreateTables() { final String topic = "test_topic"; final String dataset = "scratch"; final Map properties = makeProperties("3", "2000", topic, dataset); - properties.put(BigQuerySinkTaskConfig.TABLE_CREATE_CONFIG, "true"); + properties.put(BigQuerySinkConfig.TABLE_CREATE_CONFIG, "true"); + properties.put(BigQuerySinkConfig.SCHEMA_RETRIEVER_CONFIG, MemorySchemaRetriever.class.getName()); BigQuery bigQuery = mock(BigQuery.class); Map> emptyMap = mock(Map.class); @@ -285,8 +286,8 @@ private Map makeProperties(String bigqueryRetry, String topic, String dataset) { Map properties = propertiesFactory.getProperties(); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_CONFIG, bigqueryRetry); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_WAIT_CONFIG, bigqueryRetryWait); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_CONFIG, bigqueryRetry); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_WAIT_CONFIG, bigqueryRetryWait); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, String.format(".*=%s", dataset)); return properties; diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/GCSToBQWriterTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/GCSToBQWriterTest.java index 546095d35..ca1b4d55b 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/GCSToBQWriterTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/write/row/GCSToBQWriterTest.java @@ -21,16 +21,14 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.Table; -import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.wepay.kafka.connect.bigquery.BigQuerySinkTask; import com.wepay.kafka.connect.bigquery.SchemaManager; -import com.wepay.kafka.connect.bigquery.SinkTaskPropertiesFactory; +import com.wepay.kafka.connect.bigquery.SinkPropertiesFactory; import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; -import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; @@ -52,11 +50,11 @@ public class GCSToBQWriterTest { - private static SinkTaskPropertiesFactory propertiesFactory; + private static SinkPropertiesFactory propertiesFactory; @BeforeClass public static void initializePropertiesFactory() { - propertiesFactory = new SinkTaskPropertiesFactory(); + propertiesFactory = new SinkPropertiesFactory(); } @Test @@ -162,8 +160,8 @@ private Map makeProperties(String bigqueryRetry, String topic, String dataset) { Map properties = propertiesFactory.getProperties(); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_CONFIG, bigqueryRetry); - properties.put(BigQuerySinkTaskConfig.BIGQUERY_RETRY_WAIT_CONFIG, bigqueryRetryWait); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_CONFIG, bigqueryRetry); + properties.put(BigQuerySinkConfig.BIGQUERY_RETRY_WAIT_CONFIG, bigqueryRetryWait); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, String.format(".*=%s", dataset)); // gcs config From 1a9bfbda66de24ced45b2c20507889d49d15d96e Mon Sep 17 00:00:00 2001 From: Kanthi Date: Tue, 2 Nov 2021 16:34:25 -0400 Subject: [PATCH 33/56] [GH-161] Fixed cast error of Long in debezium time converter (#162) * [GH-161] Fixed cast error of Long in debezium time converter * [GH-161] Fixed cast error of Long in debezium time converter * [GH-161] Fixed test case for TimeConverter --- .../convert/logicaltype/DebeziumLogicalConverters.java | 2 +- .../convert/logicaltype/DebeziumLogicalConvertersTest.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java index abf35c938..19939bf36 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java @@ -152,7 +152,7 @@ public TimeConverter() { @Override public String convert(Object kafkaConnectObject) { - java.util.Date date = new java.util.Date((Long) kafkaConnectObject); + java.util.Date date = new java.util.Date((Integer) kafkaConnectObject); return getBQTimeFormat().format(date); } } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java index 61ce9d337..059fccbe4 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java @@ -40,6 +40,7 @@ public class DebeziumLogicalConvertersTest { //corresponds to March 1 2017, 22:20:38.808(123) UTC // (March 1 2017, 14:20:38.808(123)-8:00) private static final Integer DAYS_TIMESTAMP = 17226; + private static final Integer MILLI_TIMESTAMP_INT = 1488406838; private static final Long MILLI_TIMESTAMP = 1488406838808L; private static final Long MICRO_TIMESTAMP = 1488406838808123L; @@ -103,8 +104,8 @@ public void testTimeConversion() { fail("Expected encoding type check to succeed."); } - String formattedTime = converter.convert(MILLI_TIMESTAMP); - assertEquals("22:20:38.808", formattedTime); + String formattedTime = converter.convert(MILLI_TIMESTAMP_INT); + assertEquals("05:26:46.838", formattedTime); } @Test From 15fe0f5803d8f5c3da7039d1a08ae33bf24265ad Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Fri, 12 Nov 2021 19:44:51 +0000 Subject: [PATCH 34/56] [maven-release-plugin] prepare release v1.6.9 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index b771a52fb..768bf602d 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.9-SNAPSHOT + 1.6.9 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 9d58bef55..f17b6b069 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.9-SNAPSHOT + 1.6.9 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 4932c5908..4d2203728 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.9-SNAPSHOT + 1.6.9 .. diff --git a/pom.xml b/pom.xml index 675043611..4dcd7d609 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.9-SNAPSHOT + 1.6.9 pom @@ -81,7 +81,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.6.9 From 4c68def60d20bf4f3ee13836d39df69de2fe4be2 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Fri, 12 Nov 2021 19:44:54 +0000 Subject: [PATCH 35/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index 768bf602d..ce992d2ba 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.9 + 1.6.10-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index f17b6b069..3ab210b3b 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.9 + 1.6.10-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 4d2203728..687a17341 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.9 + 1.6.10-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 4dcd7d609..ff27031ae 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.9 + 1.6.10-SNAPSHOT pom @@ -81,7 +81,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.6.9 + HEAD From d8fc535fc19236d3c3db9c35fb30bf05f37e3669 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 17 Nov 2021 10:15:25 -0500 Subject: [PATCH 36/56] GH-149: Improve error messages on write thread failure (#150) * GH-149: Improve error messages on write thread failure * GH-149: Add more detail to error message for batch reduction error --- .../bigquery/BigQuerySinkConnector.java | 11 +-- .../connect/bigquery/BigQuerySinkTask.java | 11 +-- .../write/batch/KCBQThreadPoolExecutor.java | 68 ++++--------------- .../bigquery/write/batch/TableWriter.java | 20 +++++- 4 files changed, 35 insertions(+), 75 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java index d2fefb8fb..8d6cf7ac1 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java @@ -107,15 +107,8 @@ private void ensureExistingTables() { @Override public void start(Map properties) { logger.trace("connector.start()"); - try { - configProperties = properties; - config = new BigQuerySinkConfig(properties); - } catch (ConfigException err) { - throw new SinkConfigConnectException( - "Couldn't start BigQuerySinkConnector due to configuration error", - err - ); - } + configProperties = properties; + config = new BigQuerySinkConfig(properties); if (!config.getBoolean(BigQuerySinkConfig.TABLE_CREATE_CONFIG)) { ensureExistingTables(); diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java index 4f59c07d8..a12a93bee 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java @@ -187,7 +187,7 @@ private String getRowId(SinkRecord record) { @Override public void put(Collection records) { // Periodically poll for errors here instead of doing a stop-the-world check in flush() - executor.maybeThrowEncounteredErrors(); + executor.maybeThrowEncounteredError(); logger.debug("Putting {} records in the sink.", records.size()); @@ -324,14 +324,7 @@ private GCSToBQWriter getGcsWriter() { @Override public void start(Map properties) { logger.trace("task.start()"); - try { - config = new BigQuerySinkTaskConfig(properties); - } catch (ConfigException err) { - throw new SinkConfigConnectException( - "Couldn't start BigQuerySinkTask due to configuration error", - err - ); - } + config = new BigQuerySinkTaskConfig(properties); bigQueryWriter = getBigQueryWriter(); gcsToBQWriter = getGcsWriter(); diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java index 003dca988..491d9b0a1 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/KCBQThreadPoolExecutor.java @@ -21,18 +21,16 @@ import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; +import org.apache.kafka.connect.errors.ConnectException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.util.Optional; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicReference; /** * ThreadPoolExecutor for writing Rows to BigQuery. @@ -44,9 +42,7 @@ public class KCBQThreadPoolExecutor extends ThreadPoolExecutor { private static final Logger logger = LoggerFactory.getLogger(KCBQThreadPoolExecutor.class); - - private final ConcurrentHashMap.KeySetView encounteredErrors = - ConcurrentHashMap.newKeySet(); + private final AtomicReference encounteredError = new AtomicReference<>(); /** * @param config the {@link BigQuerySinkTaskConfig} @@ -66,11 +62,10 @@ protected void afterExecute(Runnable runnable, Throwable throwable) { super.afterExecute(runnable, throwable); if (throwable != null) { - logger.error("Task failed with {} error: {}", - throwable.getClass().getName(), - throwable.getMessage()); - logger.debug("Error Task Stacktrace:", throwable); - encounteredErrors.add(throwable); + // Log at debug level since this will be shown to the user at error level by the Connect framework if it causes + // the task to fail, and will otherwise just pollute logs and potentially mislead users + logger.debug("A write thread has failed with an unrecoverable error", throwable); + encounteredError.compareAndSet(null, throwable); } } @@ -92,12 +87,7 @@ public void awaitCurrentTasks() throws InterruptedException, BigQueryConnectExce execute(new CountDownRunnable(countDownLatch)); } countDownLatch.await(); - maybeThrowEncounteredErrors(); - if (encounteredErrors.size() > 0) { - String errorString = createErrorString(encounteredErrors); - throw new BigQueryConnectException("Some write threads encountered unrecoverable errors: " - + errorString + "; See logs for more detail"); - } + maybeThrowEncounteredError(); } /** @@ -106,41 +96,9 @@ public void awaitCurrentTasks() throws InterruptedException, BigQueryConnectExce * * @throws BigQueryConnectException if any of the tasks failed. */ - public void maybeThrowEncounteredErrors() { - if (encounteredErrors.size() > 0) { - String errorString = createErrorString(encounteredErrors); - throw new BigQueryConnectException("Some write threads encountered unrecoverable errors: " - + errorString + "; See logs for more detail"); - } - } - - private static String createErrorString(Collection errors) { - List exceptionTypeStrings = new ArrayList<>(errors.size()); - exceptionTypeStrings.addAll(errors.stream() - .map(throwable -> throwable.getClass().getName()) - .collect(Collectors.toList())); - return String.join(", ", exceptionTypeStrings); - } - - private static String createDetailedErrorString(Collection errors) { - List exceptionTypeStrings = new ArrayList<>(errors.size()); - exceptionTypeStrings.addAll(errors.stream() - .map(throwable -> - throwable.getClass().getName() + "\nMessage: " + throwable.getLocalizedMessage()) - .collect(Collectors.toList())); - return String.join(", ", exceptionTypeStrings); - } - - /** - * Checks for BigQuery errors. No-op if there isn't any error. - * - * @throws BigQueryConnectException if there have been any unrecoverable errors when writing to BigQuery. - */ - public void maybeFail() throws BigQueryConnectException { - if (encounteredErrors.size() > 0) { - throw new BigQueryConnectException("Encountered unrecoverable errors: " - + createDetailedErrorString(encounteredErrors) + "; See logs for more detail"); - } + public void maybeThrowEncounteredError() { + Optional.ofNullable(encounteredError.get()).ifPresent(t -> { + throw new BigQueryConnectException("A write thread has failed with an unrecoverable error", t); + }); } - } diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java index 3ff20037b..53f49e895 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/write/batch/TableWriter.java @@ -111,8 +111,24 @@ public void run() { private static int getNewBatchSize(int currentBatchSize, Throwable err) { if (currentBatchSize == 1) { - // todo correct exception type? - throw new BigQueryConnectException("Attempted to reduce batch size below 1.", err); + logger.error("Attempted to reduce batch size below 1"); + throw new BigQueryConnectException( + "Failed to write to BigQuery even after reducing batch size to 1 row at a time. " + + "This can indicate an error in the connector's logic for classifying BigQuery errors, as non-retriable" + + "errors may be being treated as retriable." + + "If that appears to be the case, please report the issue to the project's maintainers and include the " + + "complete stack trace for this error as it appears in the logs. " + + "Alternatively, there may be a record that the connector has read from Kafka that is too large to " + + "write to BigQuery using the streaming insert API, which cannot be addressed with a change to the " + + "connector and will need to be handled externally by optionally writing the record to BigQuery using " + + "another means and then reconfiguring the connector to skip the record. " + + "Finally, streaming insert quotas for BigQuery may be causing insertion failures for the connector; " + + "in that case, please ensure that quotas for maximum rows per second, maximum bytes per second, etc. " + + "are being respected before restarting the connector. " + + "The cause of this exception is the error encountered from BigQuery after the last attempt to write a " + + "batch was made.", + err + ); } // round batch size up so we don't end up with a dangling 1 row at the end. return (int) Math.ceil(currentBatchSize / 2.0); From 5280d0e1417e964e6a961b267896bc053bd34a7d Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 17 Nov 2021 11:13:57 -0500 Subject: [PATCH 37/56] GH-139: Improve preflight config validation (#153) * GH-139: Improve preflight config validation Depends on https://github.com/confluentinc/kafka-connect-bigquery/pull/140 Addresses the remainder of https://github.com/confluentinc/kafka-connect-bigquery/issues/139. All multi-property configuration validation is moved from the constructor of the BigQuerySinkConfig class to a new `BigQuerySinkConfig::validate` method, which is invoked from `BigQuerySinkConnector::validate` and leverages the Connect API for defining custom error messages on a per-property basis instead of throwing exceptions on invalid configurations, which only displays one message at a time and doesn't work well with programmatic UIs. Unit tests are added for all new validation logic. The logic for GCP client construction is also updated to conform to a common interface, which should improve readability and maintainability, and makes it easier to validate credentials for either BigQuery or GCS. Finally, the `SinkConfigConnectException` class is removed as it's not really necessary and doesn't bring anything worthwhile to the code base. * Add config revalidation to connector start method * Simplify error message logic for CredentialsValidator * Add PartitioningModeValidator test case for when decorator syntax is enabled but no timestamp partition field is provided --- .../connect/bigquery/BigQueryHelper.java | 108 ------ .../bigquery/BigQuerySinkConnector.java | 67 +--- .../connect/bigquery/BigQuerySinkTask.java | 19 +- .../kafka/connect/bigquery/GCSBuilder.java | 109 ------ .../connect/bigquery/GcpClientBuilder.java | 145 ++++++++ .../bigquery/config/BigQuerySinkConfig.java | 330 ++++++++---------- .../bigquery/config/CredentialsValidator.java | 117 +++++++ .../bigquery/config/GcsBucketValidator.java | 63 ++++ .../config/MultiPropertyValidator.java | 70 ++++ .../config/PartitioningModeValidator.java | 60 ++++ .../config/SchemaRetrieverValidator.java | 105 ++++++ .../config/TableExistenceValidator.java | 108 ++++++ .../exception/SinkConfigConnectException.java | 40 --- .../bigquery/BigQuerySinkConnectorTest.java | 68 +--- .../bigquery/BigQuerySinkTaskTest.java | 21 -- .../bigquery/SinkPropertiesFactory.java | 18 - .../config/BigQuerySinkConfigTest.java | 44 +-- .../config/CredentialsValidatorTest.java | 69 ++++ .../config/GcsBucketValidatorTest.java | 93 +++++ .../config/MultiPropertyValidatorTest.java | 138 ++++++++ .../config/PartitioningModeValidatorTest.java | 80 +++++ .../config/SchemaRetrieverValidatorTest.java | 107 ++++++ .../config/TableExistenceValidatorTest.java | 160 +++++++++ .../it/BigQueryConnectorIntegrationTest.java | 50 ++- .../bigquery/it/utils/BucketClearer.java | 17 +- .../bigquery/it/utils/TableClearer.java | 16 +- 26 files changed, 1540 insertions(+), 682 deletions(-) delete mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQueryHelper.java delete mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/GCSBuilder.java create mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/GcpClientBuilder.java create mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/CredentialsValidator.java create mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/GcsBucketValidator.java create mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/MultiPropertyValidator.java create mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/PartitioningModeValidator.java create mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/SchemaRetrieverValidator.java create mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/TableExistenceValidator.java delete mode 100644 kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/SinkConfigConnectException.java create mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/CredentialsValidatorTest.java create mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/GcsBucketValidatorTest.java create mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/MultiPropertyValidatorTest.java create mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/PartitioningModeValidatorTest.java create mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/SchemaRetrieverValidatorTest.java create mode 100644 kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/TableExistenceValidatorTest.java diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQueryHelper.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQueryHelper.java deleted file mode 100644 index f90ea5f3f..000000000 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQueryHelper.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2020 Confluent, Inc. - * - * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wepay.kafka.connect.bigquery; - -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryOptions; - -import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; - -/** - * Convenience class for creating a default {@link com.google.cloud.bigquery.BigQuery} instance, - * with or without login credentials. - */ -public class BigQueryHelper { - private static final Logger logger = LoggerFactory.getLogger(BigQueryHelper.class); - private static String keySource; - - /** - * Returns a default {@link BigQuery} instance for the specified project with credentials provided - * in the specified file, which can then be used for creating, updating, and inserting into tables - * from specific datasets. - * - * @param projectName The name of the BigQuery project to work with - * @param key The google credentials JSON key that can be used to provide - * credentials to BigQuery, or null if no authentication should be performed. - * @return The resulting BigQuery object. - */ - public BigQuery connect(String projectName, String key) { - if (key == null) { - return connect(projectName); - } - logger.debug("Attempting to open file {} for service account json key", key); - InputStream credentialsStream; - try { - if (keySource != null && keySource.equals("JSON")) { - credentialsStream = new ByteArrayInputStream(key.getBytes(StandardCharsets.UTF_8)); - } else { - credentialsStream = new FileInputStream(key); - } - return new - BigQueryOptions.DefaultBigQueryFactory().create( - BigQueryOptions.newBuilder() - .setProjectId(projectName) - .setCredentials(GoogleCredentials.fromStream(credentialsStream)) - .build() - ); - } catch (IOException err) { - throw new BigQueryConnectException("Failed to access json key file", err); - } - } - /** - * Returns a default {@link BigQuery} instance for the specified project with credentials provided - * in the specified file, which can then be used for creating, updating, and inserting into tables - * from specific datasets. - * - * @param keySource The type of key config we can expect. This is either a String - * representation of the Google credentials file, or the path to the Google credentials file. - * @return The resulting BigQuery object. - */ - public BigQueryHelper setKeySource(String keySource) { - this.keySource = keySource; - return this; - } - - /** - * Returns a default {@link BigQuery} instance for the specified project with no authentication - * credentials, which can then be used for creating, updating, and inserting into tables from - * specific datasets. - * - * @param projectName The name of the BigQuery project to work with - * @return The resulting BigQuery object. - */ - public BigQuery connect(String projectName) { - logger.debug("Attempting to access BigQuery without authentication"); - return new BigQueryOptions.DefaultBigQueryFactory().create( - BigQueryOptions.newBuilder() - .setProjectId(projectName) - .build() - ); - } -} diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java index 8d6cf7ac1..05c912e07 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnector.java @@ -19,21 +19,16 @@ package com.wepay.kafka.connect.bigquery; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.TableId; - import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkTaskConfig; -import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; -import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; -import com.wepay.kafka.connect.bigquery.utils.TopicToTableResolver; import com.wepay.kafka.connect.bigquery.utils.Version; +import org.apache.kafka.common.config.Config; import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.config.ConfigValue; import org.apache.kafka.connect.connector.Task; import org.apache.kafka.connect.sink.SinkConnector; @@ -50,28 +45,9 @@ * {@link org.apache.kafka.connect.sink.SinkTask SinkTasks}. */ public class BigQuerySinkConnector extends SinkConnector { - private final BigQuery testBigQuery; - private final SchemaManager testSchemaManager; - - public BigQuerySinkConnector() { - testBigQuery = null; - testSchemaManager = null; - } - - // For testing purposes only; will never be called by the Kafka Connect framework - BigQuerySinkConnector(BigQuery bigQuery) { - this.testBigQuery = bigQuery; - this.testSchemaManager = null; - } - - // For testing purposes only; will never be called by the Kafka Connect framework - BigQuerySinkConnector(BigQuery bigQuery, SchemaManager schemaManager) { - this.testBigQuery = bigQuery; - this.testSchemaManager = schemaManager; - } - private BigQuerySinkConfig config; - private Map configProperties; + BigQuerySinkConfig config; + Map configProperties; private static final Logger logger = LoggerFactory.getLogger(BigQuerySinkConnector.class); @@ -81,27 +57,16 @@ public ConfigDef config() { return BigQuerySinkConfig.getConfig(); } - private BigQuery getBigQuery() { - if (testBigQuery != null) { - return testBigQuery; - } - String projectName = config.getString(BigQuerySinkConfig.PROJECT_CONFIG); - String key = config.getKeyFile(); - String keySource = config.getString(BigQuerySinkConfig.KEY_SOURCE_CONFIG); - return new BigQueryHelper().setKeySource(keySource).connect(projectName, key); - } - - private void ensureExistingTables() { - BigQuery bigQuery = getBigQuery(); - Map topicsToTableIds = TopicToTableResolver.getTopicsToTables(config); - for (TableId tableId : topicsToTableIds.values()) { - if (bigQuery.getTable(tableId) == null) { - logger.warn( - "You may want to enable auto table creation by setting {}=true in the properties file", - BigQuerySinkConfig.TABLE_CREATE_CONFIG); - throw new BigQueryConnectException("Table '" + tableId + "' does not exist"); - } + @Override + public Config validate(Map properties) { + List singlePropertyValidations = config().validate(properties); + // If any of our properties had malformed syntax or failed a validation to ensure, e.g., that it fell within an + // acceptable numeric range, we only report those errors since they prevent us from being able to construct a + // valid BigQuerySinkConfig instance + if (singlePropertyValidations.stream().anyMatch(v -> !v.errorMessages().isEmpty())) { + return new Config(singlePropertyValidations); } + return new BigQuerySinkConfig(properties).validate(); } @Override @@ -109,10 +74,8 @@ public void start(Map properties) { logger.trace("connector.start()"); configProperties = properties; config = new BigQuerySinkConfig(properties); - - if (!config.getBoolean(BigQuerySinkConfig.TABLE_CREATE_CONFIG)) { - ensureExistingTables(); - } + // Revalidate here in case the connector has been upgraded and its old config is no longer valid + config.ensureValid(); } @Override diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java index a12a93bee..452024165 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTask.java @@ -33,7 +33,6 @@ import com.wepay.kafka.connect.bigquery.convert.KafkaDataBuilder; import com.wepay.kafka.connect.bigquery.convert.RecordConverter; import com.wepay.kafka.connect.bigquery.convert.SchemaConverter; -import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; import com.wepay.kafka.connect.bigquery.utils.FieldNameSanitizer; import com.wepay.kafka.connect.bigquery.utils.PartitionedTableId; import com.wepay.kafka.connect.bigquery.utils.TopicToTableResolver; @@ -48,7 +47,6 @@ import com.wepay.kafka.connect.bigquery.write.row.SimpleBigQueryWriter; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.sink.SinkRecord; @@ -255,10 +253,9 @@ private BigQuery getBigQuery() { if (testBigQuery != null) { return testBigQuery; } - String projectName = config.getString(BigQuerySinkConfig.PROJECT_CONFIG); - String keyFile = config.getKeyFile(); - String keySource = config.getString(BigQuerySinkConfig.KEY_SOURCE_CONFIG); - return new BigQueryHelper().setKeySource(keySource).connect(projectName, keyFile); + return new GcpClientBuilder.BigQueryBuilder() + .withConfig(config) + .build(); } private SchemaManager getSchemaManager(BigQuery bigQuery) { @@ -271,7 +268,7 @@ private SchemaManager getSchemaManager(BigQuery bigQuery) { Optional kafkaKeyFieldName = config.getKafkaKeyFieldName(); Optional kafkaDataFieldName = config.getKafkaDataFieldName(); Optional timestampPartitionFieldName = config.getTimestampPartitionFieldName(); - Optional> clusteringFieldName = config.getClusteringPartitionFieldName(); + Optional> clusteringFieldName = config.getClusteringPartitionFieldNames(); return new SchemaManager(schemaRetriever, schemaConverter, bigQuery, kafkaKeyFieldName, kafkaDataFieldName, timestampPartitionFieldName, clusteringFieldName); } @@ -298,11 +295,9 @@ private Storage getGcs() { if (testGcs != null) { return testGcs; } - String projectName = config.getString(BigQuerySinkConfig.PROJECT_CONFIG); - String key = config.getKeyFile(); - String keySource = config.getString(BigQuerySinkConfig.KEY_SOURCE_CONFIG); - return new GCSBuilder(projectName).setKey(key).setKeySource(keySource).build(); - + return new GcpClientBuilder.GcsBuilder() + .withConfig(config) + .build(); } private GCSToBQWriter getGcsWriter() { diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/GCSBuilder.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/GCSBuilder.java deleted file mode 100644 index 4a0952b2e..000000000 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/GCSBuilder.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2020 Confluent, Inc. - * - * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wepay.kafka.connect.bigquery; - -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; - -import com.wepay.kafka.connect.bigquery.exception.GCSConnectException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; - -/** - * Convenience class for creating a {@link com.google.cloud.storage.Storage} instance - */ -public class GCSBuilder { - private static final Logger logger = LoggerFactory.getLogger(GCSBuilder.class); - - private final String projectName; - private String key; - private String keySource; - - public GCSBuilder(String projectName) { - this.projectName = projectName; - this.key = null; - } - - public GCSBuilder setKeySource(String keySourceType) { - this.keySource = keySourceType; - return this; - } - - public GCSBuilder setKey(String keyFile) { - this.key = keyFile; - return this; - } - public Storage build() { - return connect(projectName, key); - } - - /** - * Returns a default {@link Storage} instance for the specified project with credentials provided - * in the specified file. - * - * @param projectName The name of the GCS project to work with - * @param key The name of a file containing a JSON key that can be used to provide - * credentials to GCS, or null if no authentication should be performed. - * @return The resulting Storage object. - */ - private Storage connect(String projectName, String key) { - if (key == null) { - return connect(projectName); - } - try { - InputStream credentialsStream; - if (keySource != null && keySource.equals("JSON")) { - credentialsStream = new ByteArrayInputStream(key.getBytes(StandardCharsets.UTF_8)); - } else { - credentialsStream = new FileInputStream(key); - } - return StorageOptions.newBuilder() - .setProjectId(projectName) - .setCredentials(GoogleCredentials.fromStream(credentialsStream)) - .build() - .getService(); - } catch (IOException err) { - throw new GCSConnectException("Failed to access json key file", err); - } - } - - /** - * Returns a default {@link Storage} instance for the specified project with no authentication - * credentials. - * - * @param projectName The name of the GCS project to work with - * @return The resulting Storage object. - */ - private Storage connect(String projectName) { - logger.debug("Attempting to access BigQuery without authentication"); - return StorageOptions.newBuilder() - .setProjectId(projectName) - .build() - .getService(); - } -} - diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/GcpClientBuilder.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/GcpClientBuilder.java new file mode 100644 index 000000000..5c79fec87 --- /dev/null +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/GcpClientBuilder.java @@ -0,0 +1,145 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery; + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; +import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.PROJECT_CONFIG; + +public abstract class GcpClientBuilder { + + public enum KeySource { + FILE, JSON + } + + private static final Logger logger = LoggerFactory.getLogger(GcpClientBuilder.class); + + private String project = null; + private KeySource keySource = null; + private String key = null; + + public GcpClientBuilder withConfig(BigQuerySinkConfig config) { + return withProject(config.getString(PROJECT_CONFIG)) + .withKeySource(config.getKeySource()) + .withKey(config.getKey()); + } + + public GcpClientBuilder withProject(String project) { + Objects.requireNonNull(project, "Project cannot be null"); + this.project = project; + return this; + } + + public GcpClientBuilder withKeySource(KeySource keySource) { + Objects.requireNonNull(keySource, "Key cannot be null"); + this.keySource = keySource; + return this; + } + + public GcpClientBuilder withKey(String key) { + this.key = key; + return this; + } + + public Client build() { + return doBuild(project, credentials()); + } + + private GoogleCredentials credentials() { + if (key == null) { + return null; + } + + Objects.requireNonNull(keySource, "Key source must be defined to build a GCP client"); + Objects.requireNonNull(project, "Project must be defined to build a GCP client"); + + InputStream credentialsStream; + switch (keySource) { + case JSON: + credentialsStream = new ByteArrayInputStream(key.getBytes(StandardCharsets.UTF_8)); + break; + case FILE: + try { + logger.debug("Attempting to open file {} for service account json key", key); + credentialsStream = new FileInputStream(key); + } catch (IOException e) { + throw new BigQueryConnectException("Failed to access JSON key file", e); + } + break; + default: + throw new IllegalArgumentException("Unexpected value for KeySource enum: " + keySource); + } + + try { + return GoogleCredentials.fromStream(credentialsStream); + } catch (IOException e) { + throw new BigQueryConnectException("Failed to create credentials from input stream", e); + } + } + + protected abstract Client doBuild(String project, GoogleCredentials credentials); + + public static class BigQueryBuilder extends GcpClientBuilder { + @Override + protected BigQuery doBuild(String project, GoogleCredentials credentials) { + BigQueryOptions.Builder builder = BigQueryOptions.newBuilder() + .setProjectId(project); + + if (credentials != null) { + builder.setCredentials(credentials); + } else { + logger.debug("Attempting to access BigQuery without authentication"); + } + + return builder.build().getService(); + } + } + + public static class GcsBuilder extends GcpClientBuilder { + @Override + protected Storage doBuild(String project, GoogleCredentials credentials) { + StorageOptions.Builder builder = StorageOptions.newBuilder() + .setProjectId(project); + + if (credentials != null) { + builder.setCredentials(credentials); + } else { + logger.debug("Attempting to access GCS without authentication"); + } + + return builder.build().getService(); + } + } +} diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java index c7cc02593..32b0b1821 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java @@ -21,6 +21,7 @@ import com.google.cloud.bigquery.Schema; +import com.wepay.kafka.connect.bigquery.GcpClientBuilder; import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; import com.wepay.kafka.connect.bigquery.convert.BigQueryRecordConverter; @@ -29,13 +30,14 @@ import com.wepay.kafka.connect.bigquery.convert.SchemaConverter; import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.Config; +import org.apache.kafka.common.config.ConfigValue; import org.apache.kafka.common.config.types.Password; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.sink.SinkConnector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -46,17 +48,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Base class for connector and task configs; contains properties shared between the two of them. */ public class BigQuerySinkConfig extends AbstractConfig { - private static final Validator validator = new Validator(); - private static final Logger logger = LoggerFactory.getLogger(BigQuerySinkConfig.class); - // Values taken from https://github.com/apache/kafka/blob/1.1.1/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java#L33 public static final String TOPICS_CONFIG = SinkConnector.TOPICS_CONFIG; private static final ConfigDef.Type TOPICS_TYPE = ConfigDef.Type.LIST; @@ -118,6 +120,7 @@ public class BigQuerySinkConfig extends AbstractConfig { private static final ConfigDef.Importance TOPICS_TO_TABLES_IMPORTANCE = ConfigDef.Importance.MEDIUM; public static final Object TOPICS_TO_TABLES_DEFAULT = null; + private static final ConfigDef.Validator TOPICS_TO_TABLES_VALIDATOR = new MapValidator(); private static final String TOPICS_TO_TABLES_DOC = "A list of mappings from topic regexes to table names. Note the regex must include " + "capture groups that are referenced in the format string using placeholders (i.e. $1) " @@ -132,7 +135,7 @@ public class BigQuerySinkConfig extends AbstractConfig { public static final String DATASETS_CONFIG = "datasets"; private static final ConfigDef.Type DATASETS_TYPE = ConfigDef.Type.LIST; private static final Object DATASETS_DEFAULT = ConfigDef.NO_DEFAULT_VALUE; - private static final ConfigDef.Validator DATASETS_VALIDATOR = validator; + private static final ConfigDef.Validator DATASETS_VALIDATOR = new MapValidator(); private static final ConfigDef.Importance DATASETS_IMPORTANCE = ConfigDef.Importance.HIGH; private static final String DATASETS_DOC = "Names for the datasets kafka topics will write to " @@ -155,9 +158,13 @@ public class BigQuerySinkConfig extends AbstractConfig { public static final String KEY_SOURCE_CONFIG = "keySource"; private static final ConfigDef.Type KEY_SOURCE_TYPE = ConfigDef.Type.STRING; - public static final String KEY_SOURCE_DEFAULT = "FILE"; - private static final ConfigDef.Validator KEY_SOURCE_VALIDATOR = - ConfigDef.ValidString.in("FILE", "JSON"); + public static final String KEY_SOURCE_DEFAULT = GcpClientBuilder.KeySource.FILE.name(); + private static final ConfigDef.Validator KEY_SOURCE_VALIDATOR = ConfigDef.ValidString.in( + Stream.of(GcpClientBuilder.KeySource.values()) + .map(GcpClientBuilder.KeySource::name) + .collect(Collectors.toList()) + .toArray(new String[0]) + ); private static final ConfigDef.Importance KEY_SOURCE_IMPORTANCE = ConfigDef.Importance.MEDIUM; private static final String KEY_SOURCE_DOC = "Determines whether the keyfile config is the path to the credentials json, or the json itself"; @@ -187,6 +194,7 @@ public class BigQuerySinkConfig extends AbstractConfig { public static final String KAFKA_KEY_FIELD_NAME_CONFIG = "kafkaKeyFieldName"; private static final ConfigDef.Type KAFKA_KEY_FIELD_NAME_TYPE = ConfigDef.Type.STRING; public static final String KAFKA_KEY_FIELD_NAME_DEFAULT = null; + private static final ConfigDef.Validator KAFKA_KEY_FIELD_NAME_VALIDATOR = new ConfigDef.NonEmptyString(); private static final ConfigDef.Importance KAFKA_KEY_FIELD_NAME_IMPORTANCE = ConfigDef.Importance.LOW; private static final String KAFKA_KEY_FIELD_NAME_DOC = "The name of the field of Kafka key. " + "Default to be null, which means Kafka Key Field will not be included."; @@ -194,6 +202,7 @@ public class BigQuerySinkConfig extends AbstractConfig { public static final String KAFKA_DATA_FIELD_NAME_CONFIG = "kafkaDataFieldName"; private static final ConfigDef.Type KAFKA_DATA_FIELD_NAME_TYPE = ConfigDef.Type.STRING; public static final String KAFKA_DATA_FIELD_NAME_DEFAULT = null; + private static final ConfigDef.Validator KAFKA_DATA_FIELD_NAME_VALIDATOR = new ConfigDef.NonEmptyString(); private static final ConfigDef.Importance KAFKA_DATA_FIELD_NAME_IMPORTANCE = ConfigDef.Importance.LOW; private static final String KAFKA_DATA_FIELD_NAME_DOC = "The name of the field of Kafka Data. " + "Default to be null, which means Kafka Data Field will not be included. "; @@ -310,6 +319,7 @@ public class BigQuerySinkConfig extends AbstractConfig { public static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG = "timestampPartitionFieldName"; private static final ConfigDef.Type BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_TYPE = ConfigDef.Type.STRING; private static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DEFAULT = null; + private static final ConfigDef.Validator BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_VALIDATOR = new ConfigDef.NonEmptyString(); private static final ConfigDef.Importance BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_IMPORTANCE = ConfigDef.Importance.LOW; private static final String BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DOC = @@ -320,6 +330,17 @@ public class BigQuerySinkConfig extends AbstractConfig { public static final String BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG = "clusteringPartitionFieldNames"; private static final ConfigDef.Type BIGQUERY_CLUSTERING_FIELD_NAMES_TYPE = ConfigDef.Type.LIST; private static final List BIGQUERY_CLUSTERING_FIELD_NAMES_DEFAULT = null; + private static final ConfigDef.Validator BIGQUERY_CLUSTERING_FIELD_NAMES_VALIDATOR = (name, value) -> { + if (value == null) { + return; + } + + @SuppressWarnings("unchecked") + List parsedValue = (List) value; + if (parsedValue.size() > 4) { + throw new ConfigException(name, value, "You may only specify up to four clustering field names."); + } + }; private static final ConfigDef.Importance BIGQUERY_CLUSTERING_FIELD_NAMES_IMPORTANCE = ConfigDef.Importance.LOW; private static final String BIGQUERY_CLUSTERING_FIELD_NAMES_DOC = @@ -380,6 +401,7 @@ public static ConfigDef getConfig() { TOPICS_TO_TABLES_CONFIG, TOPICS_TO_TABLES_TYPE, TOPICS_TO_TABLES_DEFAULT, + TOPICS_TO_TABLES_VALIDATOR, TOPICS_TO_TABLES_IMPORTANCE, TOPICS_TO_TABLES_DOC ).define( @@ -429,12 +451,14 @@ public static ConfigDef getConfig() { KAFKA_KEY_FIELD_NAME_CONFIG, KAFKA_KEY_FIELD_NAME_TYPE, KAFKA_KEY_FIELD_NAME_DEFAULT, + KAFKA_KEY_FIELD_NAME_VALIDATOR, KAFKA_KEY_FIELD_NAME_IMPORTANCE, KAFKA_KEY_FIELD_NAME_DOC ).define( KAFKA_DATA_FIELD_NAME_CONFIG, KAFKA_DATA_FIELD_NAME_TYPE, KAFKA_DATA_FIELD_NAME_DEFAULT, + KAFKA_DATA_FIELD_NAME_VALIDATOR, KAFKA_DATA_FIELD_NAME_IMPORTANCE, KAFKA_DATA_FIELD_NAME_DOC ).define( @@ -512,133 +536,145 @@ public static ConfigDef getConfig() { BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_TYPE, BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DEFAULT, + BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_VALIDATOR, BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_IMPORTANCE, BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_DOC ).define( BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, BIGQUERY_CLUSTERING_FIELD_NAMES_TYPE, BIGQUERY_CLUSTERING_FIELD_NAMES_DEFAULT, + BIGQUERY_CLUSTERING_FIELD_NAMES_VALIDATOR, BIGQUERY_CLUSTERING_FIELD_NAMES_IMPORTANCE, BIGQUERY_CLUSTERING_FIELD_NAMES_DOC ); } - /** - * Throw an exception if the passed-in properties do not constitute a valid sink. - * @param props sink configuration properties - */ - public static void validate(Map props) { - final boolean hasTopicsConfig = hasTopicsConfig(props); - final boolean hasTopicsRegexConfig = hasTopicsRegexConfig(props); - - if (hasTopicsConfig && hasTopicsRegexConfig) { - throw new ConfigException(TOPICS_CONFIG + " and " + TOPICS_REGEX_CONFIG + - " are mutually exclusive options, but both are set."); - } - if (!hasTopicsConfig && !hasTopicsRegexConfig) { - throw new ConfigException("Must configure one of " + - TOPICS_CONFIG + " or " + TOPICS_REGEX_CONFIG); - } - } + private static final List> MULTI_PROPERTY_VALIDATIONS = new ArrayList<>(); + + static { + // Note that order matters here: validations are performed in the order they're added to this list, and if a + // property or any of the properties that it depends on has an error, validation for it gets skipped. + // This comes in handy for things like checking for the existence of tables, which requires valid BigQuery + // credentials. We validate those credentials before checking for tables so that we can safely assume while + // checking for those tables that the credentials are already valid. + MULTI_PROPERTY_VALIDATIONS.add(new CredentialsValidator.BigQueryCredentialsValidator()); + MULTI_PROPERTY_VALIDATIONS.add(new CredentialsValidator.GcsCredentialsValidator()); + MULTI_PROPERTY_VALIDATIONS.add(new TableExistenceValidator()); + MULTI_PROPERTY_VALIDATIONS.add(new SchemaRetrieverValidator.TableCreationValidator()); + MULTI_PROPERTY_VALIDATIONS.add(new SchemaRetrieverValidator.SchemaUpdateValidator()); + MULTI_PROPERTY_VALIDATIONS.add(new GcsBucketValidator()); + MULTI_PROPERTY_VALIDATIONS.add(new PartitioningModeValidator()); + } - public static boolean hasTopicsConfig(Map props) { - String topicsStr = props.get(TOPICS_CONFIG); - return topicsStr != null && !topicsStr.trim().isEmpty(); - } + /** + * Used in conjunction with {@link com.wepay.kafka.connect.bigquery.BigQuerySinkConnector#validate(Map)} to perform + * preflight configuration checks. Simple validations that only require a single property value at a time (such as + * ensuring that boolean properties only contain true/false values, or that values for required properties are + * provided) are handled automatically by the {@link #getConfig() ConfigDef} for this class and optionally-defined + * custom {@link ConfigDef.Validator validators}. Other, more sophisticated validations that require multiple + * property values at a time (such as checking if all of the tables the connector will write to already exist if + * automatic table creation is disabled) are performed manually in a subsequent step. + * + * @return a {@link Config} object containing all errors that the connector was able to detect during preflight + * validation of this configuration; never null + */ + public Config validate() { + List initialValidation = getConfig().validate(originalsStrings()); + Map valuesByName = initialValidation + .stream() + .collect(Collectors.toMap(ConfigValue::name, Function.identity())); + MULTI_PROPERTY_VALIDATIONS.forEach(validator -> { + ConfigValue value = valuesByName.get(validator.propertyName()); + validator.validate(value, this, valuesByName).ifPresent(value::addErrorMessage); + }); + return new Config(initialValidation); + } - public static boolean hasTopicsRegexConfig(Map props) { - String topicsRegexStr = props.get(TOPICS_REGEX_CONFIG); - return topicsRegexStr != null && !topicsRegexStr.trim().isEmpty(); + /** + * Ensure that this config is valid (including multi-property validations performed in {@link #validate()}, and if any errors + * are detected, throw an exception. + * @throws ConnectException if this config is invalid + */ + public void ensureValid() { + Config config = validate(); + List errors = config.configValues().stream() + .filter(v -> !v.errorMessages().isEmpty()) + .map(v -> "For property '" + v.name() + "': " + String.join(",", v.errorMessages())) + .collect(Collectors.toList()); + if (!errors.isEmpty()) { + throw new ConnectException( + "The connector config is invalid and contains the following errors:\n" + + String.join("\n", errors) + ); } + } - @SuppressWarnings("unchecked") - public static class Validator implements ConfigDef.Validator { + public static class MapValidator implements ConfigDef.Validator { @Override public void ensureValid(String name, Object value) { - switch (name) { - case DATASETS_CONFIG: - ensureValidMap(name, (List) value); - break; - case TOPICS_TO_TABLES_CONFIG: - ensureValidMap(name, (List) value); - break; - default: - break; - } - } - - protected static void ensureValidMap(String name, List values) { - if (values == null) { + if (value == null) { return; } - values.forEach((entry) -> parseMapping(entry, name)); + + @SuppressWarnings("unchecked") + List parsedValue = (List) value; + + parsedValue.forEach(BigQuerySinkConfig::parseMapping); } - /** - * Ensures the mapping given is valid, then returns an entry containing its key and value. - * Checks to make sure that the given String adheres to the specified format, and throws - * an exception if it does not. Trims leading and trailing whitespace, and then checks to make - * sure that both Strings are still non-empty. - * - * @param mapping The mapping to parse (should be of the form <key>=<value>) - * @param name The name of the field. Used in error messages. - * @return A Map.Entry containing the parsed key/value pair. - */ - protected static Map.Entry parseMapping(String mapping, String name) { - String[] keyValue = mapping.split("="); - if (keyValue.length != 2) { - throw new ConfigException( - "Invalid mapping for " + name - + " property: '" + mapping - + "' (must follow format '=')" - ); - } + @Override + public String toString() { + return "A list of key-value pairs in the format =, =, ..."; + } + } - String key = keyValue[0].trim(); - if (key.isEmpty()) { - throw new ConfigException( - "Empty key found in mapping '" + mapping - + "' for " + name + " property" - ); - } + /** + * Ensures the mapping given is valid, then returns an entry containing its key and value. + * Checks to make sure that the given String adheres to the specified format, and throws + * an exception if it does not. Trims leading and trailing whitespace, and then checks to make + * sure that both Strings are still non-empty. + * + * @param mapping The mapping to parse (should be of the form <key>=<value>) + * @return A Map.Entry containing the parsed key/value pair. + */ + static Map.Entry parseMapping(String mapping) { + String[] keyValue = mapping.split("="); + if (keyValue.length != 2) { + throw new ConfigException("Invalid mapping '" + mapping + "' (must follow format '=')"); + } - String value = keyValue[1].trim(); - if (value.isEmpty()) { - throw new ConfigException( - "Empty value found in mapping '" + mapping - + "' for " + name + " property" - ); - } + String key = keyValue[0].trim(); + if (key.isEmpty()) { + throw new ConfigException("Invalid mapping '" + mapping + "' (key cannot be empty)"); + } - return new AbstractMap.SimpleEntry<>(key, value); + String value = keyValue[1].trim(); + if (value.isEmpty()) { + throw new ConfigException("Invalid mapping '" + mapping + "' (value cannot be empty)"); } + + return new AbstractMap.SimpleEntry<>(key, value); } /** - * Returns the keyfile + * @return the key, which is (depending on the key source property) either a path to a file or a raw JSON string */ - public String getKeyFile() { + public String getKey() { return Optional.ofNullable(getPassword(KEYFILE_CONFIG)).map(Password::value).orElse(null); } /** - * Parses a config map, which must be provided as a list of Strings of the form - * '<key>=<value>' into a Map. Locates that list, splits its key and value pairs, and - * returns they Map they represent. - * - * @param name The name of the property the mapping is given for. Used in exception messages. - * @return A Map containing the given key and value pairs. + * @return the {@link com.wepay.kafka.connect.bigquery.GcpClientBuilder.KeySource key source type} that dictates how + * the {@link #getKey()} should be be interpreted */ - public Map getMap(String name) { - List assocList = getList(name); - Map configMap = new HashMap<>(); - if (assocList != null) { - for (String mapping : assocList) { - Map.Entry entry = validator.parseMapping(mapping, name); - configMap.put(entry.getKey(), entry.getValue()); - } + public GcpClientBuilder.KeySource getKeySource() { + String rawKeySource = getString(KEY_SOURCE_CONFIG); + try { + return GcpClientBuilder.KeySource.valueOf(rawKeySource); + } catch (IllegalArgumentException e) { + // Should never happen with preflight validation of the key source property + throw new ConnectException("Invalid key source type: " + rawKeySource); } - return configMap; } /** @@ -653,7 +689,7 @@ public List> getSinglePatterns(String property) { List> patternList = new ArrayList<>(); if (propList != null) { for (String propValue : propList) { - Map.Entry mapping = validator.parseMapping(propValue, property); + Map.Entry mapping = parseMapping(propValue); Pattern propPattern = Pattern.compile(mapping.getKey()); Map.Entry patternEntry = new AbstractMap.SimpleEntry<>(propPattern, mapping.getValue()); @@ -775,7 +811,7 @@ public SchemaRetriever getSchemaRetriever() { Class schemaRetrieverClass = userSpecifiedClass.asSubclass(SchemaRetriever.class); - Constructor schemaRetrieverConstructor = null; + Constructor schemaRetrieverConstructor; try { schemaRetrieverConstructor = schemaRetrieverClass.getConstructor(); } catch (NoSuchMethodException nsme) { @@ -785,7 +821,7 @@ public SchemaRetriever getSchemaRetriever() { ); } - SchemaRetriever schemaRetriever = null; + SchemaRetriever schemaRetriever; try { schemaRetriever = schemaRetrieverConstructor.newInstance(); } catch (InstantiationException @@ -804,7 +840,6 @@ public SchemaRetriever getSchemaRetriever() { } /** - * * If the connector is configured to load Kafka data into BigQuery, this config defines * the name of the kafka data field. A structure is created under the field name to contain * kafka data schema including topic, offset, partition and insertTime. @@ -816,7 +851,6 @@ public Optional getKafkaKeyFieldName() { } /** - * * If the connector is configured to load Kafka keys into BigQuery, this config defines * the name of the kafka key field. A structure is created under the field name to contain * a topic's Kafka key schema. @@ -827,47 +861,6 @@ public Optional getKafkaDataFieldName() { return Optional.ofNullable(getString(KAFKA_DATA_FIELD_NAME_CONFIG)); } - /** - * Verifies that a bucket is specified if GCS batch loading is enabled. - * @throws ConfigException Exception thrown if no bucket is specified and batch loading is on. - */ - private void verifyBucketSpecified() throws ConfigException { - // Throw an exception if GCS Batch loading will be used but no bucket is specified - if (getString(GCS_BUCKET_NAME_CONFIG).equals("") - && !getList(ENABLE_BATCH_CONFIG).isEmpty()) { - throw new ConfigException("Batch loading enabled for some topics, but no bucket specified"); - } - } - - private void checkAutoCreateTables() { - - Class schemaRetriever = getClass(BigQuerySinkConfig.SCHEMA_RETRIEVER_CONFIG); - boolean autoCreateTables = getBoolean(TABLE_CREATE_CONFIG); - - if (autoCreateTables && schemaRetriever == null) { - throw new ConfigException( - "Cannot specify automatic table creation without a schema retriever" - ); - } - } - - private void checkAutoUpdateSchemas() { - Class schemaRetriever = getClass(BigQuerySinkConfig.SCHEMA_RETRIEVER_CONFIG); - - boolean autoUpdateSchemas = getBoolean(SCHEMA_UPDATE_CONFIG); - if (autoUpdateSchemas && schemaRetriever == null) { - throw new ConfigException( - "Cannot specify automatic table creation without a schema retriever" - ); - } - - if (schemaRetriever == null) { - logger.warn( - "No schema retriever class provided; auto schema updates are impossible" - ); - } - } - /** * Returns the field name to use for timestamp partitioning. * @return String that represents the field name. @@ -880,51 +873,18 @@ public Optional getTimestampPartitionFieldName() { * Returns the field names to use for clustering. * @return List of Strings that represent the field names. */ - public Optional> getClusteringPartitionFieldName() { - return Optional.ofNullable(getList(BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG)); - } - - /** - * Check the validity of table partitioning configs. - */ - private void checkPartitionConfigs() { - if (getTimestampPartitionFieldName().isPresent() && getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)) { - throw new ConfigException( - "Only one partitioning configuration mode may be specified for the connector. " - + "Use either bigQueryPartitionDecorator OR timestampPartitionFieldName." - ); - } - } - - /** - * Check the validity of table clustering configs. - */ - private void checkClusteringConfigs() { - if (getClusteringPartitionFieldName().isPresent()) { - if (!getTimestampPartitionFieldName().isPresent() && !getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)) { - throw new ConfigException( - "Clustering field name may be specified only on a partitioned table." - ); - } - if (getClusteringPartitionFieldName().get().size() > 4) { - throw new ConfigException( - "You can only specify up to four clustering field names." - ); - } - } + public Optional> getClusteringPartitionFieldNames() { + return Optional + .ofNullable(getList(BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG)) + // With Java 11 there's Predicate::not, but for now we have to just manually invert the isEmpty check + .filter(l -> !l.isEmpty()); } protected BigQuerySinkConfig(ConfigDef config, Map properties) { super(config, properties); - verifyBucketSpecified(); - checkAutoCreateTables(); - checkAutoUpdateSchemas(); - checkPartitionConfigs(); - checkClusteringConfigs(); } public BigQuerySinkConfig(Map properties) { this(getConfig(), properties); } - } diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/CredentialsValidator.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/CredentialsValidator.java new file mode 100644 index 000000000..76007d11a --- /dev/null +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/CredentialsValidator.java @@ -0,0 +1,117 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.storage.Storage; +import com.wepay.kafka.connect.bigquery.GcpClientBuilder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.ENABLE_BATCH_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.GCS_BUCKET_NAME_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.KEYFILE_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.KEY_SOURCE_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.PROJECT_CONFIG; + +public abstract class CredentialsValidator> extends MultiPropertyValidator { + + public CredentialsValidator() { + super(KEYFILE_CONFIG); + } + + private static final Collection DEPENDENTS = Collections.unmodifiableCollection(Arrays.asList( + PROJECT_CONFIG, KEY_SOURCE_CONFIG + )); + + @Override + protected Collection dependents() { + return DEPENDENTS; + } + + @Override + protected Optional doValidate(BigQuerySinkConfig config) { + String keyFile = config.getKey(); + if (keyFile == null || keyFile.isEmpty()) { + // No credentials to validate + return Optional.empty(); + } + + try { + clientBuilder() + .withConfig(config) + .build(); + return Optional.empty(); + } catch (RuntimeException e) { + String errorMessage = "An unexpected error occurred while validating credentials for " + gcpService(); + if (e.getMessage() != null) { + errorMessage += ": " + e.getMessage(); + } + return Optional.of(errorMessage); + } + } + + protected abstract String gcpService(); + protected abstract ClientBuilder clientBuilder(); + + public static class BigQueryCredentialsValidator extends CredentialsValidator> { + @Override + public String gcpService() { + return "BigQuery"; + } + + @Override + protected GcpClientBuilder clientBuilder() { + return new GcpClientBuilder.BigQueryBuilder(); + } + } + + public static class GcsCredentialsValidator extends CredentialsValidator> { + + private static final Collection DEPENDENTS; + + static { + List dependents = new ArrayList<>(CredentialsValidator.DEPENDENTS); + dependents.add(ENABLE_BATCH_CONFIG); + dependents.add(GCS_BUCKET_NAME_CONFIG); + DEPENDENTS = Collections.unmodifiableCollection(dependents); + } + + @Override + public Collection dependents() { + return DEPENDENTS; + } + + @Override + public String gcpService() { + return "GCS"; + } + + @Override + protected GcpClientBuilder clientBuilder() { + return new GcpClientBuilder.GcsBuilder(); + } + } +} diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/GcsBucketValidator.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/GcsBucketValidator.java new file mode 100644 index 000000000..19141c0c7 --- /dev/null +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/GcsBucketValidator.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.ENABLE_BATCH_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.GCS_BUCKET_NAME_CONFIG; + +public class GcsBucketValidator extends MultiPropertyValidator { + + public GcsBucketValidator() { + super(GCS_BUCKET_NAME_CONFIG); + } + + private static final Collection DEPENDENTS = Collections.unmodifiableCollection(Arrays.asList( + ENABLE_BATCH_CONFIG + )); + + @Override + protected Collection dependents() { + return DEPENDENTS; + } + + @Override + protected Optional doValidate(BigQuerySinkConfig config) { + List batchLoadedTopics = config.getList(ENABLE_BATCH_CONFIG); + if (batchLoadedTopics == null || batchLoadedTopics.isEmpty()) { + // Batch loading is disabled; no need to validate the GCS bucket + return Optional.empty(); + } + + String bucket = config.getString(GCS_BUCKET_NAME_CONFIG); + if (bucket == null || bucket.trim().isEmpty()) { + return Optional.of("When GCS batch loading is enabled, a bucket must be provided"); + } + + // No need to validate that the bucket exists; we create it automatically if it doesn't + + return Optional.empty(); + } +} diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/MultiPropertyValidator.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/MultiPropertyValidator.java new file mode 100644 index 000000000..95b9c2da6 --- /dev/null +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/MultiPropertyValidator.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import org.apache.kafka.common.config.ConfigValue; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public abstract class MultiPropertyValidator { + + private final String propertyName; + + protected MultiPropertyValidator(String propertyName) { + this.propertyName = propertyName; + } + + public String propertyName() { + return propertyName; + } + + public Optional validate(ConfigValue value, Config config, Map valuesByName) { + // Only perform follow-up validation if the property doesn't already have an error associated with it + if (!value.errorMessages().isEmpty()) { + return Optional.empty(); + } + + boolean dependentsAreValid = dependents().stream() + .map(valuesByName::get) + .filter(Objects::nonNull) + .map(ConfigValue::errorMessages) + .allMatch(List::isEmpty); + // Also ensure that all of the other properties that the validation for this one depends on don't already have errors + if (!dependentsAreValid) { + return Optional.empty(); + } + + try { + return doValidate(config); + } catch (RuntimeException e) { + return Optional.of( + "An unexpected error occurred during validation" + + (e.getMessage() != null ? ": " + e.getMessage() : "") + ); + } + } + + protected abstract Collection dependents(); + protected abstract Optional doValidate(Config config); +} diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/PartitioningModeValidator.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/PartitioningModeValidator.java new file mode 100644 index 000000000..65389e5fd --- /dev/null +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/PartitioningModeValidator.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG; + +public class PartitioningModeValidator extends MultiPropertyValidator { + public PartitioningModeValidator() { + super(BIGQUERY_PARTITION_DECORATOR_CONFIG); + } + + private static final Collection DEPENDENTS = Collections.unmodifiableCollection(Arrays.asList( + BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG + )); + + @Override + protected Collection dependents() { + return DEPENDENTS; + } + + @Override + protected Optional doValidate(BigQuerySinkConfig config) { + if (!config.getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)) { + return Optional.empty(); + } + + if (config.getTimestampPartitionFieldName().isPresent()) { + return Optional.of(String.format("Only one partitioning mode may be specified for the connector. " + + "Use either %s OR %s.", + BIGQUERY_PARTITION_DECORATOR_CONFIG, + BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG + )); + } else { + return Optional.empty(); + } + } +} diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/SchemaRetrieverValidator.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/SchemaRetrieverValidator.java new file mode 100644 index 000000000..9cb6a3894 --- /dev/null +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/SchemaRetrieverValidator.java @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.SCHEMA_RETRIEVER_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.SCHEMA_UPDATE_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.TABLE_CREATE_CONFIG; + +public abstract class SchemaRetrieverValidator extends MultiPropertyValidator { + + protected SchemaRetrieverValidator(String propertyName) { + super(propertyName); + } + + private static final Collection DEPENDENTS = Collections.unmodifiableCollection(Arrays.asList( + SCHEMA_RETRIEVER_CONFIG + )); + + @Override + protected Collection dependents() { + return DEPENDENTS; + } + + @Override + protected Optional doValidate(BigQuerySinkConfig config) { + if (!schemaRetrieverRequired(config)) { + return Optional.empty(); + } + + SchemaRetriever schemaRetriever = config.getSchemaRetriever(); + if (schemaRetriever != null) { + return Optional.empty(); + } else { + return Optional.of(missingSchemaRetrieverMessage()); + } + } + + /** + * @param config the user-provided configuration + * @return whether a schema retriever class is required for the property this validator is responsible for + */ + protected abstract boolean schemaRetrieverRequired(BigQuerySinkConfig config); + + /** + * @return an error message explaining why a schema retriever class is required for the property this validator is + * responsible for + */ + protected abstract String missingSchemaRetrieverMessage(); + + public static class TableCreationValidator extends SchemaRetrieverValidator { + public TableCreationValidator() { + super(TABLE_CREATE_CONFIG); + } + + @Override + protected boolean schemaRetrieverRequired(BigQuerySinkConfig config) { + return config.getBoolean(TABLE_CREATE_CONFIG); + } + + @Override + protected String missingSchemaRetrieverMessage() { + return "A schema retriever class is required when automatic table creation is enabled"; + } + } + + public static class SchemaUpdateValidator extends SchemaRetrieverValidator { + public SchemaUpdateValidator() { + super(SCHEMA_UPDATE_CONFIG); + } + + @Override + protected boolean schemaRetrieverRequired(BigQuerySinkConfig config) { + return config.getBoolean(SCHEMA_UPDATE_CONFIG); + } + + @Override + protected String missingSchemaRetrieverMessage() { + return "A schema retriever class is required when automatic schema updates are enabled"; + } + } +} diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/TableExistenceValidator.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/TableExistenceValidator.java new file mode 100644 index 000000000..149b20380 --- /dev/null +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/TableExistenceValidator.java @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.TableId; +import com.google.common.annotations.VisibleForTesting; +import com.wepay.kafka.connect.bigquery.GcpClientBuilder; +import com.wepay.kafka.connect.bigquery.utils.TopicToTableResolver; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.DATASETS_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.KEYFILE_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.KEY_SOURCE_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.PROJECT_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.SANITIZE_TOPICS_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.TABLE_CREATE_CONFIG; + +public class TableExistenceValidator extends MultiPropertyValidator { + + public TableExistenceValidator() { + super(TABLE_CREATE_CONFIG); + } + + private static final Collection DEPENDENTS = Collections.unmodifiableCollection(Arrays.asList( + SANITIZE_TOPICS_CONFIG, + KEY_SOURCE_CONFIG, + KEYFILE_CONFIG, + PROJECT_CONFIG, + DATASETS_CONFIG + )); + + @Override + protected Collection dependents() { + return DEPENDENTS; + } + + @Override + protected Optional doValidate(BigQuerySinkConfig config) { + BigQuery bigQuery; + try { + bigQuery = new GcpClientBuilder.BigQueryBuilder() + .withConfig(config) + .build(); + } catch (RuntimeException e) { + return Optional.of(String.format( + "Failed to construct BigQuery client%s", + e.getMessage() != null ? ": " + e.getMessage() : "" + )); + } + + return doValidate(bigQuery, config); + } + + @VisibleForTesting + Optional doValidate(BigQuery bigQuery, BigQuerySinkConfig config) { + boolean autoCreateTables = config.getBoolean(TABLE_CREATE_CONFIG); + // No need to check if tables already exist if we're allowed to create them ourselves + if (autoCreateTables) { + return Optional.empty(); + } + + List missingTables = missingTables(bigQuery, config); + + if (missingTables.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(String.format( + "Automatic table creation is disabled and the following tables do not appear to exist: %s. " + + "Please either manually create these tables before restarting the connector or enable automatic table " + + "creation by the connector.", + missingTables.stream().map(t -> t.getDataset() + ":" + t.getTable()).collect(Collectors.joining(", ")) + )); + } + + @VisibleForTesting + List missingTables(BigQuery bigQuery, BigQuerySinkConfig config) { + return TopicToTableResolver.getTopicsToTables(config).values().stream() + .filter(t -> bigQuery.getTable(t) == null) + .collect(Collectors.toList()); + } +} diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/SinkConfigConnectException.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/SinkConfigConnectException.java deleted file mode 100644 index 805cd5643..000000000 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/exception/SinkConfigConnectException.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2020 Confluent, Inc. - * - * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.wepay.kafka.connect.bigquery.exception; - -import org.apache.kafka.connect.errors.ConnectException; - -/** - * Class for exceptions that occur while attempting to process configuration files, including both - * formatting and logical errors. - */ -public class SinkConfigConnectException extends ConnectException { - public SinkConfigConnectException(String msg) { - super(msg); - } - - public SinkConfigConnectException(String msg, Throwable thr) { - super(msg, thr); - } - - public SinkConfigConnectException(Throwable thr) { - super(thr); - } -} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java index b6558a7fa..06d182694 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkConnectorTest.java @@ -19,32 +19,13 @@ package com.wepay.kafka.connect.bigquery; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; - -import static org.mockito.Matchers.any; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; - import com.wepay.kafka.connect.bigquery.api.KafkaSchemaRecordType; import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; - import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; - -import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; -import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; -import org.apache.kafka.common.config.ConfigException; - import org.apache.kafka.connect.data.Schema; - import org.junit.BeforeClass; import org.junit.Test; @@ -52,27 +33,16 @@ import java.util.List; import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class BigQuerySinkConnectorTest { private static SinkPropertiesFactory propertiesFactory; - // Would just use Mockito, but can't provide the name of an anonymous class to the config file - public static class MockSchemaRetriever implements SchemaRetriever { - @Override - public void configure(Map properties) { - // Shouldn't be called - } - - @Override - public Schema retrieveSchema(TableId table, String topic, KafkaSchemaRecordType schemaType) { - // Shouldn't be called - return null; - } - - @Override - public void setLastSeenSchema(TableId table, String topic, Schema schema) { - } - } - @BeforeClass public static void initializePropertiesFactory() { propertiesFactory = new SinkPropertiesFactory(); @@ -87,15 +57,10 @@ public void testTaskClass() { public void testTaskConfigs() { Map properties = propertiesFactory.getProperties(); - Table fakeTable = mock(Table.class); - - BigQuery bigQuery = mock(BigQuery.class); - when(bigQuery.getTable(any(TableId.class))).thenReturn(fakeTable); + BigQuerySinkConnector testConnector = new BigQuerySinkConnector(); - SchemaManager schemaManager = mock(SchemaManager.class); - BigQuerySinkConnector testConnector = new BigQuerySinkConnector(bigQuery, schemaManager); - - testConnector.start(properties); + testConnector.configProperties = properties; + testConnector.config = new BigQuerySinkConfig(properties); for (int i : new int[] { 1, 2, 10, 100 }) { Map expectedProperties = new HashMap<>(properties); @@ -130,19 +95,6 @@ public void testConfig() { assertNotNull(new BigQuerySinkConnector().config()); } - // Make sure that a config exception is properly translated into a SinkConfigConnectException - @Test(expected = SinkConfigConnectException.class) - public void testConfigException() { - try { - Map badProperties = propertiesFactory.getProperties(); - badProperties.remove(BigQuerySinkConfig.TOPICS_CONFIG); - BigQuerySinkConfig.validate(badProperties); - new BigQuerySinkConnector().start(badProperties); - } catch (ConfigException e) { - throw new SinkConfigConnectException(e); - } - } - @Test public void testVersion() { assertNotNull(new BigQuerySinkConnector().version()); diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java index 9e86c3fe1..e5224e32d 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/BigQuerySinkTaskTest.java @@ -40,8 +40,6 @@ import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; import com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig; import com.wepay.kafka.connect.bigquery.exception.BigQueryConnectException; -import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; -import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.connect.data.Schema; @@ -520,25 +518,6 @@ public void testInterruptedException() { testTask.flush(Collections.emptyMap()); } - // Make sure that a ConfigException is properly translated into a SinkConfigConnectException - @Test(expected = SinkConfigConnectException.class) - public void testConfigException() { - try { - Map badProperties = propertiesFactory.getProperties(); - badProperties.remove(BigQuerySinkConfig.TOPICS_CONFIG); - BigQuerySinkConfig.validate(badProperties); - - SchemaRetriever schemaRetriever = mock(SchemaRetriever.class); - SchemaManager schemaManager = mock(SchemaManager.class); - - BigQuerySinkTask testTask = - new BigQuerySinkTask(mock(BigQuery.class), schemaRetriever, mock(Storage.class), schemaManager); - testTask.start(badProperties); - } catch (ConfigException e) { - throw new SinkConfigConnectException(e); - } - } - @Test public void testVersion() { assertNotNull(new BigQuerySinkTask().version()); diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java index 3bb981b4e..93db65926 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/SinkPropertiesFactory.java @@ -47,22 +47,4 @@ public Map getProperties() { return properties; } - - /** - * Make sure that each of the default configuration properties work nicely with the given - * configuration object. - * - * @param config The config object to test - */ - public void testProperties(BigQuerySinkConfig config) { - config.getTopicsToDatasets(); - - config.getMap(config.DATASETS_CONFIG); - config.getMap(config.TOPICS_TO_TABLES_CONFIG); - config.getList(config.TOPICS_CONFIG); - config.getString(config.PROJECT_CONFIG); - config.getKeyFile(); - config.getBoolean(config.SANITIZE_TOPICS_CONFIG); - config.getInt(config.AVRO_DATA_CACHE_SIZE_CONFIG); - } } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java index 8e1f7328c..4890f3716 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfigTest.java @@ -53,7 +53,11 @@ public void initializePropertiesFactory() { public void metaTestBasicConfigProperties() { Map basicConfigProperties = propertiesFactory.getProperties(); BigQuerySinkConfig config = new BigQuerySinkConfig(basicConfigProperties); - propertiesFactory.testProperties(config); + config.getList(BigQuerySinkConfig.TOPICS_CONFIG); + config.getString(BigQuerySinkConfig.PROJECT_CONFIG); + config.getKey(); + config.getBoolean(BigQuerySinkConfig.SANITIZE_TOPICS_CONFIG); + config.getInt(BigQuerySinkConfig.AVRO_DATA_CACHE_SIZE_CONFIG); } @Test @@ -224,16 +228,6 @@ public void testEmptyTimestampPartitionFieldName() { assertFalse(testConfig.getTimestampPartitionFieldName().isPresent()); } - /** - * Test if the field name being non-empty and the decorator default (true) errors correctly. - */ - @Test (expected = ConfigException.class) - public void testTimestampPartitionFieldNameError() { - Map configProperties = propertiesFactory.getProperties(); - configProperties.put(BigQuerySinkConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, "name"); - new BigQuerySinkConfig(configProperties); - } - /** * Test the field name being non-empty and the decorator set to false works correctly. */ @@ -254,22 +248,7 @@ public void testTimestampPartitionFieldName() { public void testEmptyClusteringFieldNames() { Map configProperties = propertiesFactory.getProperties(); BigQuerySinkConfig testConfig = new BigQuerySinkConfig(configProperties); - assertFalse(testConfig.getClusteringPartitionFieldName().isPresent()); - } - - /** - * Test if the field names being non-empty and the partitioning is not present errors correctly. - */ - @Test (expected = ConfigException.class) - public void testClusteringFieldNamesWithoutTimestampPartitionError() { - Map configProperties = propertiesFactory.getProperties(); - configProperties.put(BigQuerySinkConfig.BIGQUERY_TIMESTAMP_PARTITION_FIELD_NAME_CONFIG, null); - configProperties.put(BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG, "false"); - configProperties.put( - BigQuerySinkConfig.BIGQUERY_CLUSTERING_FIELD_NAMES_CONFIG, - "column1,column2" - ); - new BigQuerySinkConfig(configProperties); + assertFalse(testConfig.getClusteringPartitionFieldNames().isPresent()); } /** @@ -304,17 +283,8 @@ public void testClusteringFieldNames() { ); BigQuerySinkConfig testConfig = new BigQuerySinkConfig(configProperties); - Optional> testClusteringPartitionFieldName = testConfig.getClusteringPartitionFieldName(); + Optional> testClusteringPartitionFieldName = testConfig.getClusteringPartitionFieldNames(); assertTrue(testClusteringPartitionFieldName.isPresent()); assertEquals(expectedClusteringPartitionFieldName, testClusteringPartitionFieldName.get()); } - - @Test(expected = ConfigException.class) - public void testAutoSchemaUpdateWithoutRetriever() { - Map badConfigProperties = propertiesFactory.getProperties(); - badConfigProperties.remove(BigQuerySinkConfig.SCHEMA_RETRIEVER_CONFIG); - badConfigProperties.put(BigQuerySinkConfig.SCHEMA_UPDATE_CONFIG, "true"); - - new BigQuerySinkConfig(badConfigProperties); - } } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/CredentialsValidatorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/CredentialsValidatorTest.java new file mode 100644 index 000000000..7a55d5ad8 --- /dev/null +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/CredentialsValidatorTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import com.wepay.kafka.connect.bigquery.GcpClientBuilder; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CredentialsValidatorTest { + + @Test + public void testNoCredentialsSkipsValidation() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getKey()).thenReturn(null); + + assertEquals( + Optional.empty(), + new CredentialsValidator.BigQueryCredentialsValidator().doValidate(config) + ); + assertEquals( + Optional.empty(), + new CredentialsValidator.GcsCredentialsValidator().doValidate(config) + ); + } + + @Test + public void testFailureToConstructClient() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getKey()).thenReturn("key"); + + @SuppressWarnings("unchecked") + GcpClientBuilder mockClientBuilder = mock(GcpClientBuilder.class); + when(mockClientBuilder.withConfig(eq(config))).thenReturn(mockClientBuilder); + when(mockClientBuilder.build()).thenThrow(new RuntimeException("Provided credentials are invalid")); + + assertNotEquals( + Optional.empty(), + new CredentialsValidator.BigQueryCredentialsValidator().doValidate(config) + ); + assertNotEquals( + Optional.empty(), + new CredentialsValidator.GcsCredentialsValidator().doValidate(config) + ); + } +} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/GcsBucketValidatorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/GcsBucketValidatorTest.java new file mode 100644 index 000000000..d46832678 --- /dev/null +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/GcsBucketValidatorTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import org.junit.Test; + +import java.util.Collections; +import java.util.Optional; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.ENABLE_BATCH_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.GCS_BUCKET_NAME_CONFIG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class GcsBucketValidatorTest { + + @Test + public void testNullBatchLoadingSkipsValidation() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getList(ENABLE_BATCH_CONFIG)).thenReturn(null); + + assertEquals( + Optional.empty(), + new GcsBucketValidator().doValidate(config) + ); + } + + @Test + public void testEmptyBatchLoadingSkipsValidation() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getList(ENABLE_BATCH_CONFIG)).thenReturn(Collections.emptyList()); + + assertEquals( + Optional.empty(), + new GcsBucketValidator().doValidate(config) + ); + } + + @Test + public void testNullBucketWithBatchLoading() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getList(ENABLE_BATCH_CONFIG)).thenReturn(Collections.singletonList("t1")); + when(config.getString(GCS_BUCKET_NAME_CONFIG)).thenReturn(null); + + assertNotEquals( + Optional.empty(), + new GcsBucketValidator().doValidate(config) + ); + } + + @Test + public void testBlankBucketWithBatchLoading() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getList(ENABLE_BATCH_CONFIG)).thenReturn(Collections.singletonList("t1")); + when(config.getString(GCS_BUCKET_NAME_CONFIG)).thenReturn(" \t "); + + assertNotEquals( + Optional.empty(), + new GcsBucketValidator().doValidate(config) + ); + } + + @Test + public void testValidBucketWithBatchLoading() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getList(ENABLE_BATCH_CONFIG)).thenReturn(Collections.singletonList("t1")); + when(config.getString(GCS_BUCKET_NAME_CONFIG)).thenReturn("gee_cs"); + + assertEquals( + Optional.empty(), + new GcsBucketValidator().doValidate(config) + ); + } +} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/MultiPropertyValidatorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/MultiPropertyValidatorTest.java new file mode 100644 index 000000000..205bb56a3 --- /dev/null +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/MultiPropertyValidatorTest.java @@ -0,0 +1,138 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import com.google.common.collect.ImmutableMap; +import org.apache.kafka.common.config.ConfigValue; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.fail; + +public class MultiPropertyValidatorTest { + + private static class TestValidator extends MultiPropertyValidator { + + private final List dependents; + private final Function> validationFunction; + + public TestValidator(String propertyName, List dependents, Function> validationFunction) { + super(propertyName); + this.dependents = dependents; + this.validationFunction = validationFunction; + } + + @Override + protected Collection dependents() { + return dependents; + } + + @Override + protected Optional doValidate(Config config) { + return validationFunction.apply(config); + } + } + + @Test + public void testExistingErrorSkipsValidation() { + MultiPropertyValidator validator = new TestValidator<>( + "p", + Arrays.asList("d1", "d2", "d3"), + o -> { + fail("Validation should have been performed on property that already has an error"); + return null; + } + ); + + ConfigValue configValue = new ConfigValue("p", "v", Collections.emptyList(), Collections.singletonList("an error")); + + assertEquals( + Optional.empty(), + validator.validate(configValue, null, Collections.emptyMap()) + ); + } + + @Test + public void testDependentErrorSkipsValidation() { + MultiPropertyValidator validator = new TestValidator<>( + "p", + Arrays.asList("d1", "d2", "d3"), + o -> { + fail("Validation should have been performed on property whose dependent already has an error"); + return null; + } + ); + + ConfigValue configValue = new ConfigValue("p", "v", Collections.emptyList(), Collections.emptyList()); + Map valuesByName = ImmutableMap.of( + "d1", new ConfigValue("d1", "v1", Collections.emptyList(), Collections.emptyList()), + "d2", new ConfigValue("d2", "v1", Collections.emptyList(), Collections.singletonList("an error")) + ); + + assertEquals( + Optional.empty(), + validator.validate(configValue, null, valuesByName) + ); + } + + @Test + public void testValidationFails() { + Optional expectedError = Optional.of("an error"); + MultiPropertyValidator validator = new TestValidator<>( + "p", + Collections.emptyList(), + o -> expectedError + ); + + ConfigValue configValue = new ConfigValue("p", "v", Collections.emptyList(), Collections.emptyList()); + + assertEquals( + expectedError, + validator.validate(configValue, null, Collections.emptyMap()) + ); + } + + @Test + public void testUnexpectedErrorDuringValidation() { + MultiPropertyValidator validator = new TestValidator<>( + "p", + Collections.emptyList(), + o -> { + throw new RuntimeException("Some unexpected error"); + } + ); + + ConfigValue configValue = new ConfigValue("p", "v", Collections.emptyList(), Collections.emptyList()); + + assertNotEquals( + Optional.empty(), + validator.validate(configValue, null, Collections.emptyMap()) + ); + } +} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/PartitioningModeValidatorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/PartitioningModeValidatorTest.java new file mode 100644 index 000000000..a4b79a14c --- /dev/null +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/PartitioningModeValidatorTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import org.junit.Test; + +import java.util.Optional; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.BIGQUERY_PARTITION_DECORATOR_CONFIG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PartitioningModeValidatorTest { + + @Test + public void testDisabledDecoratorSyntaxSkipsValidation() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)).thenReturn(false); + + assertEquals( + Optional.empty(), + new PartitioningModeValidator().doValidate(config) + ); + } + + @Test + public void testDecoratorSyntaxWithoutTimestampPartitionFieldName() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)).thenReturn(true); + when(config.getTimestampPartitionFieldName()).thenReturn(Optional.empty()); + + assertEquals( + Optional.empty(), + new PartitioningModeValidator().doValidate(config) + ); + } + + @Test + public void testDecoratorSyntaxWithTimestampPartitionFieldName() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)).thenReturn(true); + when(config.getTimestampPartitionFieldName()).thenReturn(Optional.of("f1")); + + assertNotEquals( + Optional.empty(), + new PartitioningModeValidator().doValidate(config) + ); + } + + @Test + public void testTimestampPartitionFieldNameWithoutDecoratorSyntax() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(BIGQUERY_PARTITION_DECORATOR_CONFIG)).thenReturn(false); + when(config.getTimestampPartitionFieldName()).thenReturn(Optional.of("f1")); + + assertEquals( + Optional.empty(), + new PartitioningModeValidator().doValidate(config) + ); + } +} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/SchemaRetrieverValidatorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/SchemaRetrieverValidatorTest.java new file mode 100644 index 000000000..19a1ae28b --- /dev/null +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/SchemaRetrieverValidatorTest.java @@ -0,0 +1,107 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import com.wepay.kafka.connect.bigquery.api.SchemaRetriever; +import org.junit.Test; + +import java.util.Optional; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.SCHEMA_UPDATE_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.TABLE_CREATE_CONFIG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SchemaRetrieverValidatorTest { + + @Test + public void testDisabledTableCreationSkipsValidation() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(TABLE_CREATE_CONFIG)).thenReturn(false); + + assertEquals( + Optional.empty(), + new SchemaRetrieverValidator.TableCreationValidator().doValidate(config) + ); + } + + @Test + public void testDisabledSchemaUpdatesSkipsValidation() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(TABLE_CREATE_CONFIG)).thenReturn(true); + + assertEquals( + Optional.empty(), + new SchemaRetrieverValidator.SchemaUpdateValidator().doValidate(config) + ); + } + + @Test + public void testTableCreationEnabledWithNoRetriever() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(TABLE_CREATE_CONFIG)).thenReturn(true); + when(config.getSchemaRetriever()).thenReturn(null); + + assertNotEquals( + Optional.empty(), + new SchemaRetrieverValidator.TableCreationValidator().doValidate(config) + ); + } + + @Test + public void testSchemaUpdatesEnabledWithNoRetriever() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(SCHEMA_UPDATE_CONFIG)).thenReturn(true); + when(config.getSchemaRetriever()).thenReturn(null); + + assertNotEquals( + Optional.empty(), + new SchemaRetrieverValidator.SchemaUpdateValidator().doValidate(config) + ); + } + + @Test + public void testTableCreationEnabledWithValidRetriever() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(TABLE_CREATE_CONFIG)).thenReturn(true); + SchemaRetriever mockRetriever = mock(SchemaRetriever.class); + when(config.getSchemaRetriever()).thenReturn(mockRetriever); + + assertEquals( + Optional.empty(), + new SchemaRetrieverValidator.TableCreationValidator().doValidate(config) + ); + } + + @Test + public void testSchemaUpdatesEnabledWithValidRetriever() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(SCHEMA_UPDATE_CONFIG)).thenReturn(true); + SchemaRetriever mockRetriever = mock(SchemaRetriever.class); + when(config.getSchemaRetriever()).thenReturn(mockRetriever); + + assertEquals( + Optional.empty(), + new SchemaRetrieverValidator.SchemaUpdateValidator().doValidate(config) + ); + } +} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/TableExistenceValidatorTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/TableExistenceValidatorTest.java new file mode 100644 index 000000000..fb9ffc32f --- /dev/null +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/config/TableExistenceValidatorTest.java @@ -0,0 +1,160 @@ +/* + * Copyright 2020 Confluent, Inc. + * + * This software contains code derived from the WePay BigQuery Kafka Connector, Copyright WePay, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.wepay.kafka.connect.bigquery.config; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; +import com.google.common.collect.ImmutableMap; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.TABLE_CREATE_CONFIG; +import static com.wepay.kafka.connect.bigquery.config.BigQuerySinkConfig.TOPICS_CONFIG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TableExistenceValidatorTest { + + @Test + public void testMissingTableWithAutoCreationDisabled() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getTopicsToDatasets()) + .thenReturn(ImmutableMap.of( + "t1", "d1", + "t2", "d2" + )); + when(config.getBoolean(eq(TABLE_CREATE_CONFIG))).thenReturn(false); + when(config.getList(TOPICS_CONFIG)).thenReturn(Arrays.asList("t1", "t2")); + + BigQuery bigQuery = bigQuery(TableId.of("d1", "t1")); + + assertNotEquals( + Optional.empty(), + new TableExistenceValidator().doValidate(bigQuery, config) + ); + } + + @Test + public void testEmptyTopicsListWithAutoCreationDisabled() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getTopicsToDatasets()) + .thenReturn(ImmutableMap.of( + "t1", "d1", + "t2", "d2" + )); + when(config.getBoolean(eq(TABLE_CREATE_CONFIG))).thenReturn(false); + when(config.getList(TOPICS_CONFIG)).thenReturn(Collections.emptyList()); + + BigQuery bigQuery = bigQuery(); + + assertEquals( + Optional.empty(), + new TableExistenceValidator().doValidate(bigQuery, config) + ); + } + + @Test + public void testMissingTableWithAutoCreationEnabled() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getBoolean(eq(TABLE_CREATE_CONFIG))).thenReturn(true); + + assertEquals( + Optional.empty(), + new TableExistenceValidator().doValidate(null, config) + ); + } + + @Test + public void testExactListOfMissingTables() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getTopicsToDatasets()) + .thenReturn(ImmutableMap.of( + "t1", "d1", + "t2", "d2", + "t3", "d1", + "t4", "d2", + "t5", "d1" + )); + when(config.getList(TOPICS_CONFIG)).thenReturn(Arrays.asList("t1", "t2", "t3", "t4", "t5")); + + BigQuery bigQuery = bigQuery( + TableId.of("d1", "t1"), + TableId.of("d3", "t2"), + TableId.of("d2", "t5") + ); + Set expectedMissingTables = new HashSet<>(Arrays.asList( + TableId.of("d2", "t2"), + TableId.of("d1", "t3"), + TableId.of("d2", "t4"), + TableId.of("d1", "t5") + )); + + assertEquals( + expectedMissingTables, + new HashSet<>(new TableExistenceValidator().missingTables(bigQuery, config)) + ); + } + + @Test + public void testExactEmptyListOfMissingTables() { + BigQuerySinkConfig config = mock(BigQuerySinkConfig.class); + when(config.getTopicsToDatasets()) + .thenReturn(ImmutableMap.of( + "t1", "d1", + "t2", "d2", + "t3", "d1", + "t4", "d2", + "t5", "d1" + )); + when(config.getList(TOPICS_CONFIG)).thenReturn(Arrays.asList("t1", "t2", "t3", "t4", "t5")); + + BigQuery bigQuery = bigQuery( + TableId.of("d1", "t1"), + TableId.of("d2", "t2"), + TableId.of("d1", "t3"), + TableId.of("d2", "t4"), + TableId.of("d1", "t5") + ); + + assertEquals( + Collections.emptyList(), + new TableExistenceValidator().missingTables(bigQuery, config) + ); + } + + private static BigQuery bigQuery(TableId... existingTables) { + BigQuery result = mock(BigQuery.class); + Stream.of(existingTables).forEach(table -> { + Table mockTable = mock(Table.class); + when(result.getTable(eq(table))).thenReturn(mockTable); + }); + return result; + } +} diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java index 1ce64e5d3..a3287d112 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/BigQueryConnectorIntegrationTest.java @@ -19,16 +19,6 @@ package com.wepay.kafka.connect.bigquery.it; -import static com.google.cloud.bigquery.LegacySQLTypeName.BOOLEAN; -import static com.google.cloud.bigquery.LegacySQLTypeName.BYTES; -import static com.google.cloud.bigquery.LegacySQLTypeName.DATE; -import static com.google.cloud.bigquery.LegacySQLTypeName.FLOAT; -import static com.google.cloud.bigquery.LegacySQLTypeName.INTEGER; -import static com.google.cloud.bigquery.LegacySQLTypeName.STRING; -import static com.google.cloud.bigquery.LegacySQLTypeName.TIMESTAMP; - -import static org.junit.Assert.assertEquals; - import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldValue; @@ -36,16 +26,13 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableResult; - -import com.wepay.kafka.connect.bigquery.BigQueryHelper; -import com.wepay.kafka.connect.bigquery.exception.SinkConfigConnectException; - +import com.wepay.kafka.connect.bigquery.GcpClientBuilder; +import org.apache.kafka.common.config.ConfigException; import org.junit.BeforeClass; import org.junit.Test; import java.io.FileNotFoundException; import java.io.InputStream; - import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -54,17 +41,24 @@ import java.util.List; import java.util.Properties; +import static com.google.cloud.bigquery.LegacySQLTypeName.BOOLEAN; +import static com.google.cloud.bigquery.LegacySQLTypeName.BYTES; +import static com.google.cloud.bigquery.LegacySQLTypeName.DATE; +import static com.google.cloud.bigquery.LegacySQLTypeName.FLOAT; +import static com.google.cloud.bigquery.LegacySQLTypeName.INTEGER; +import static com.google.cloud.bigquery.LegacySQLTypeName.STRING; +import static com.google.cloud.bigquery.LegacySQLTypeName.TIMESTAMP; +import static org.junit.Assert.assertEquals; + public class BigQueryConnectorIntegrationTest { public static final String TEST_PROPERTIES_FILENAME = "/test.properties"; public static final String KEYFILE_PROPERTY = "keyfile"; public static final String PROJECT_PROPERTY = "project"; public static final String DATASET_PROPERTY = "dataset"; - public static final String KEY_SOURCE_PROPERTY = "keySource"; - private static String keyfile; + private static String key; private static String project; private static String dataset; - private static String keySource; private static BigQuery bigQuery; @@ -87,9 +81,9 @@ private static void initializeTestProperties() throws Exception { Properties properties = new Properties(); properties.load(propertiesFile); - keyfile = properties.getProperty(KEYFILE_PROPERTY); - if (keyfile == null) { - throw new SinkConfigConnectException( + key = properties.getProperty(KEYFILE_PROPERTY); + if (key == null) { + throw new ConfigException( "'" + KEYFILE_PROPERTY + "' property must be specified in test properties file" ); @@ -97,7 +91,7 @@ private static void initializeTestProperties() throws Exception { project = properties.getProperty(PROJECT_PROPERTY); if (project == null) { - throw new SinkConfigConnectException( + throw new ConfigException( "'" + PROJECT_PROPERTY + "' property must be specified in test properties file" ); @@ -105,18 +99,20 @@ private static void initializeTestProperties() throws Exception { dataset = properties.getProperty(DATASET_PROPERTY); if (dataset == null) { - throw new SinkConfigConnectException( + throw new ConfigException( "'" + DATASET_PROPERTY + "' property must be specified in test properties file" ); } - - keySource = properties.getProperty(KEY_SOURCE_PROPERTY); } } - private static void initializeBigQuery() throws Exception { - bigQuery = new BigQueryHelper().setKeySource(keySource).connect(project, keyfile); + private static void initializeBigQuery() { + bigQuery = new GcpClientBuilder.BigQueryBuilder() + .withKeySource(GcpClientBuilder.KeySource.FILE) + .withKey(key) + .withProject(project) + .build(); } private static List boxByteArray(byte[] bytes) { diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/BucketClearer.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/BucketClearer.java index ce647d5b2..845be68cc 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/BucketClearer.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/BucketClearer.java @@ -20,27 +20,26 @@ package com.wepay.kafka.connect.bigquery.it.utils; import com.google.cloud.storage.Storage; -import com.wepay.kafka.connect.bigquery.GCSBuilder; +import com.wepay.kafka.connect.bigquery.GcpClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class BucketClearer { private static final Logger logger = LoggerFactory.getLogger(BucketClearer.class); - private static String keySource; /** * Clears tables in the given project and dataset, using a provided JSON service account key. */ public static void main(String[] args) { - if (args.length < 3 || args.length > 4) { + if (args.length != 3) { usage(); - } else if (args.length == 3) { - keySource = "FILE"; - } else { - keySource = args[3]; } - Storage gcs = new GCSBuilder(args[1]).setKey(args[0]).setKeySource(keySource).build(); + Storage gcs = new GcpClientBuilder.GcsBuilder() + .withKeySource(GcpClientBuilder.KeySource.FILE) + .withKey(args[0]) + .withProject(args[1]) + .build(); // if bucket exists, delete it. String bucketName = args[2]; @@ -53,7 +52,7 @@ public static void main(String[] args) { private static void usage() { System.err.println( - "usage: BucketClearer []" + "usage: BucketClearer " ); System.exit(1); } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java index 20e7d8fa2..c66606eae 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java @@ -21,16 +21,15 @@ import com.google.cloud.bigquery.BigQuery; -import com.wepay.kafka.connect.bigquery.BigQueryHelper; +import com.wepay.kafka.connect.bigquery.GcpClientBuilder; import com.wepay.kafka.connect.bigquery.utils.FieldNameSanitizer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TableClearer { - private static final Logger logger = LoggerFactory.getLogger(TableClearer.class); - private static String keySource; + private static final Logger logger = LoggerFactory.getLogger(TableClearer.class); /** * Clears tables in the given project and dataset, using a provided JSON service account key. @@ -39,9 +38,14 @@ public static void main(String[] args) { if (args.length < 4) { usage(); } - int tablesStart = 3; - BigQuery bigQuery = new BigQueryHelper().connect(args[1], args[0]); - for (int i = tablesStart; i < args.length; i++) { + + BigQuery bigQuery = new GcpClientBuilder.BigQueryBuilder() + .withKeySource(GcpClientBuilder.KeySource.FILE) + .withKey(args[0]) + .withProject(args[1]) + .build(); + + for (int i = 3; i < args.length; i++) { // May be consider using sanitizeTopics property value in future to decide table name // sanitization but as currently we always run test cases with sanitizeTopics value as true // hence sanitize table name prior delete. This is required else it makes test cases flaky. From 56aa0cf7ce9c3909bcc3bf9284408f873d5c7f1f Mon Sep 17 00:00:00 2001 From: Manasjyoti Sharma Date: Thu, 4 Aug 2022 10:22:34 +0530 Subject: [PATCH 38/56] CCDB-4929: Fixing "Code scanning"/CodeQL warnings. (#229) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1ddc1dd7a..b24ffcb6d 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,7 @@ confluent - http://packages.confluent.io/maven/ + https://packages.confluent.io/maven/ jcenter @@ -115,7 +115,7 @@ confluent - http://packages.confluent.io/maven/ + https://packages.confluent.io/maven/ jcenter From c129f21c0e884f1a68ea448b6baff63ae0179a29 Mon Sep 17 00:00:00 2001 From: Shaik Zakir Hussain Date: Mon, 5 Dec 2022 14:33:54 +0530 Subject: [PATCH 39/56] Updating the debeziumLogicalConverters to honor the leading 0 in microsecond remainder from a given epoch timestamp long value. --- .../DebeziumLogicalConverters.java | 4 ++-- .../DebeziumLogicalConvertersTest.java | 23 +++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java index 24326789c..67ea1a6df 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConverters.java @@ -101,7 +101,7 @@ public String convert(Object kafkaConnectObject) { Long microRemainder = microTimestamp % MICROS_IN_SEC; - return formattedSecondsTimestamp + "." + microRemainder; + return formattedSecondsTimestamp + "." + String.format("%06d", microRemainder); } } @@ -132,7 +132,7 @@ public String convert(Object kafkaConnectObject) { Long microRemainder = microTimestamp % MICROS_IN_SEC; - return formattedSecondsTimestamp + "." + microRemainder; + return formattedSecondsTimestamp + "." + String.format("%06d", microRemainder); } } diff --git a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java index 519026715..d8e65d9f1 100644 --- a/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java +++ b/kcbq-connector/src/test/java/com/wepay/kafka/connect/bigquery/convert/logicaltype/DebeziumLogicalConvertersTest.java @@ -61,6 +61,13 @@ public void testDateConversion() { @Test public void testMicroTimeConversion() { + testMicroTimeConversionHelper(MICRO_TIMESTAMP, "22:20:38.808123"); + // Test case where microseconds have a leading 0. + long microTimestamp = 1592511382050720L; + testMicroTimeConversionHelper(microTimestamp, "20:16:22.050720"); + } + + private void testMicroTimeConversionHelper(long microTimestamp, String s) { MicroTimeConverter converter = new MicroTimeConverter(); assertEquals(Field.Type.time(), converter.getBQSchemaType()); @@ -71,12 +78,20 @@ public void testMicroTimeConversion() { fail("Expected encoding type check to succeed."); } - String formattedMicroTime = converter.convert(MICRO_TIMESTAMP); - assertEquals("22:20:38.808123", formattedMicroTime); + String formattedMicroTime = converter.convert(microTimestamp); + assertEquals(s, formattedMicroTime); } + @Test public void testMicroTimestampConversion() { + testMicroTimestampConversionHelper(MICRO_TIMESTAMP, "2017-03-01 22:20:38.808123"); + // Test timestamp where microseconds have a leading 0 + Long timestamp = 1592511382050720L; + testMicroTimestampConversionHelper(timestamp, "2020-06-18 20:16:22.050720"); + } + + private void testMicroTimestampConversionHelper(Long timestamp, String s) { MicroTimestampConverter converter = new MicroTimestampConverter(); assertEquals(Field.Type.timestamp(), converter.getBQSchemaType()); @@ -87,8 +102,8 @@ public void testMicroTimestampConversion() { fail("Expected encoding type check to succeed."); } - String formattedMicroTimestamp = converter.convert(MICRO_TIMESTAMP); - assertEquals("2017-03-01 22:20:38.808123", formattedMicroTimestamp); + String formattedMicroTimestamp = converter.convert(timestamp); + assertEquals(s, formattedMicroTimestamp); } @Test From 4557b2123b5d9b241caf2de7c5b45c1db030a27e Mon Sep 17 00:00:00 2001 From: sp-gupta Date: Wed, 11 Jan 2023 11:35:52 +0530 Subject: [PATCH 40/56] [CCDB-5307] Upgrade protobuf-java to 3.19.6 --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index b24ffcb6d..81538cb56 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,7 @@ 0.9.0 0.25.0-alpha 1.22.0 + 3.19.6 20.0 2.6.3 1.0.0 @@ -223,6 +224,11 @@ google-http-client-jackson2 ${google.http.version} + + com.google.protobuf + protobuf-java + ${google.protobuf.version} + junit From a61e5286004d90a993ba036b39aa7c515600da72 Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Tue, 17 Jan 2023 11:44:48 +0000 Subject: [PATCH 41/56] [maven-release-plugin] prepare release v1.1.3 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index ae804c5d5..be171ef81 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.1.3-SNAPSHOT + 1.1.3 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 51f555557..28c53c09d 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.1.3-SNAPSHOT + 1.1.3 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 8ecd8281c..3fa37cddd 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.1.3-SNAPSHOT + 1.1.3 .. diff --git a/pom.xml b/pom.xml index 81538cb56..719d95ada 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.1.3-SNAPSHOT + 1.1.3 pom @@ -84,7 +84,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.1.3 From 8e3c66df6243617d8e8878936baddded1bf16e36 Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Tue, 17 Jan 2023 11:44:52 +0000 Subject: [PATCH 42/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index be171ef81..ec2e3cc92 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.1.3 + 1.1.4-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 28c53c09d..0a165b930 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.1.3 + 1.1.4-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 3fa37cddd..0d1ed7946 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.1.3 + 1.1.4-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 719d95ada..049e0fdf8 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.1.3 + 1.1.4-SNAPSHOT pom @@ -84,7 +84,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.1.3 + HEAD From 36bee9ed9489c3f10e244e176ba411bf72bd7ede Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Tue, 17 Jan 2023 12:07:18 +0000 Subject: [PATCH 43/56] [maven-release-plugin] prepare release v1.3.1 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index bc73307c9..aaea47049 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.3.1-SNAPSHOT + 1.3.1 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 346551e26..566124017 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.3.1-SNAPSHOT + 1.3.1 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index a9fe712dc..8feab3319 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.3.1-SNAPSHOT + 1.3.1 .. diff --git a/pom.xml b/pom.xml index 2d438083d..4b53d3cba 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.3.1-SNAPSHOT + 1.3.1 pom @@ -83,7 +83,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.3.1 From c606c54f7e8ef4bf0823b0983bcf55182e9ac34d Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Tue, 17 Jan 2023 12:07:21 +0000 Subject: [PATCH 44/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index aaea47049..bde3f606d 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.3.1 + 1.3.2-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 566124017..b8e5045fd 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.3.1 + 1.3.2-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 8feab3319..6477341f9 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.3.1 + 1.3.2-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 4b53d3cba..8ca0dc08d 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.3.1 + 1.3.2-SNAPSHOT pom @@ -83,7 +83,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.3.1 + HEAD From 8210d2eef2fd27aa98739a49f0efeca71a4e6ee0 Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Tue, 17 Jan 2023 12:17:14 +0000 Subject: [PATCH 45/56] [maven-release-plugin] prepare release v1.2.1 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index f6bdf9ea9..d07f4438a 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.2.1-SNAPSHOT + 1.2.1 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index c0cfed9ff..9b11ea5fd 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.2.1-SNAPSHOT + 1.2.1 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index c55786975..aac195b63 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.2.1-SNAPSHOT + 1.2.1 .. diff --git a/pom.xml b/pom.xml index fab9572c7..a5e3e2e85 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.2.1-SNAPSHOT + 1.2.1 pom @@ -83,7 +83,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.2.1 From 2e275444b51bf1dd5ca342c5a50d0942e7615441 Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Tue, 17 Jan 2023 12:17:17 +0000 Subject: [PATCH 46/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index d07f4438a..c7a8522f6 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.2.1 + 1.2.2-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 9b11ea5fd..0ba81a3b9 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.2.1 + 1.2.2-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index aac195b63..631947ccd 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.2.1 + 1.2.2-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index a5e3e2e85..da52cce98 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.2.1 + 1.2.2-SNAPSHOT pom @@ -83,7 +83,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.2.1 + HEAD From a1c7499fdfed988a039cf095b2ad1353fd5d16a0 Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Tue, 17 Jan 2023 12:25:58 +0000 Subject: [PATCH 47/56] [maven-release-plugin] prepare release v1.6.10 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index ce992d2ba..b475335d6 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.10-SNAPSHOT + 1.6.10 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 3ab210b3b..36e4b1897 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.10-SNAPSHOT + 1.6.10 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 687a17341..c6480b025 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.10-SNAPSHOT + 1.6.10 .. diff --git a/pom.xml b/pom.xml index c7e905ac8..f83b5e1cc 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.10-SNAPSHOT + 1.6.10 pom @@ -82,7 +82,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.6.10 From bb245872a3db757759e27935e2a169884ace2925 Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Tue, 17 Jan 2023 12:26:01 +0000 Subject: [PATCH 48/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index b475335d6..f60fd834a 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.10 + 1.6.11-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 36e4b1897..dea5a02f7 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.10 + 1.6.11-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index c6480b025..0428b78b6 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.6.10 + 1.6.11-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index f83b5e1cc..d49d417ed 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.6.10 + 1.6.11-SNAPSHOT pom @@ -82,7 +82,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.6.10 + HEAD From d5fbf14e409b08094dbd95c6a50e41a94954684f Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Thu, 19 Jan 2023 16:43:10 +0000 Subject: [PATCH 49/56] [maven-release-plugin] prepare release v1.4.2 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index bde47b946..59f11c96d 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.4.2-SNAPSHOT + 1.4.2 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 82dd0029c..4a4ba185a 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.4.2-SNAPSHOT + 1.4.2 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index bd8351b1d..150512b84 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.4.2-SNAPSHOT + 1.4.2 .. diff --git a/pom.xml b/pom.xml index ef5fce9a6..c92ba34d0 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.4.2-SNAPSHOT + 1.4.2 pom @@ -83,7 +83,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.4.2 From e6367264e811d6965ba90a3e2a7a26695b66b81f Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Thu, 19 Jan 2023 16:43:13 +0000 Subject: [PATCH 50/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index 59f11c96d..476b0b7dc 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.4.2 + 1.4.3-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 4a4ba185a..4648e28b9 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.4.2 + 1.4.3-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 150512b84..68344d0ab 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.4.2 + 1.4.3-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index c92ba34d0..2fb10d0a9 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.4.2 + 1.4.3-SNAPSHOT pom @@ -83,7 +83,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.4.2 + HEAD From 31998bc4ac87edc4ab5910535506a6967e97731a Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Thu, 19 Jan 2023 16:52:47 +0000 Subject: [PATCH 51/56] [maven-release-plugin] prepare release v1.5.3 --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index 60ebfa9ca..4e27ce11a 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.5.3-SNAPSHOT + 1.5.3 .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 4ebf30d17..0ce78c350 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.5.3-SNAPSHOT + 1.5.3 .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index c6c1d9af4..119edfe0d 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.5.3-SNAPSHOT + 1.5.3 .. diff --git a/pom.xml b/pom.xml index f0ce428af..5a1d111ef 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.5.3-SNAPSHOT + 1.5.3 pom @@ -83,7 +83,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - HEAD + v1.5.3 From 7055e4270624d5b72c23194de7edbada8809cf92 Mon Sep 17 00:00:00 2001 From: Sparsh Gupta Date: Thu, 19 Jan 2023 16:52:50 +0000 Subject: [PATCH 52/56] [maven-release-plugin] prepare for next development iteration --- kcbq-api/pom.xml | 2 +- kcbq-confluent/pom.xml | 2 +- kcbq-connector/pom.xml | 2 +- pom.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kcbq-api/pom.xml b/kcbq-api/pom.xml index 4e27ce11a..d6185e913 100644 --- a/kcbq-api/pom.xml +++ b/kcbq-api/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.5.3 + 1.5.4-SNAPSHOT .. diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 0ce78c350..6e311744b 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.5.3 + 1.5.4-SNAPSHOT .. diff --git a/kcbq-connector/pom.xml b/kcbq-connector/pom.xml index 119edfe0d..6b7372a05 100644 --- a/kcbq-connector/pom.xml +++ b/kcbq-connector/pom.xml @@ -25,7 +25,7 @@ com.wepay.kcbq kcbq-parent - 1.5.3 + 1.5.4-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 5a1d111ef..7b12c82fc 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ com.wepay.kcbq kcbq-parent - 1.5.3 + 1.5.4-SNAPSHOT pom @@ -83,7 +83,7 @@ scm:git:git://github.com/confluentinc/kafka-connect-bigquery.git scm:git:git@github.com:confluentinc/kafka-connect-bigquery.git https://github.com/confluentinc/kafka-connect-bigquery - v1.5.3 + HEAD From c525134e16c0a6c9fc41e13cd70d90fbbcc79fc4 Mon Sep 17 00:00:00 2001 From: sp-gupta Date: Fri, 10 Mar 2023 09:44:05 +0530 Subject: [PATCH 53/56] [CCDB-4843] Register TimestampConverter in DebeziumLogicalConverters behind a config for BigQuerySink Connector --- .../bigquery/config/BigQuerySinkConfig.java | 13 ++- .../convert/BigQueryRecordConverter.java | 8 +- .../convert/BigQueryRecordConverterTest.java | 107 +++++++++++++----- 3 files changed, 95 insertions(+), 33 deletions(-) diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java index 32b0b1821..6f2d05446 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/config/BigQuerySinkConfig.java @@ -346,6 +346,12 @@ public class BigQuerySinkConfig extends AbstractConfig { private static final String BIGQUERY_CLUSTERING_FIELD_NAMES_DOC = "List of fields on which data should be clustered by in BigQuery, separated by commas"; + public static final String CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_CONFIG = "convertDebeziumTimestampToInteger"; + private static final ConfigDef.Type CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_TYPE = ConfigDef.Type.BOOLEAN; + private static final Boolean CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_DEFAULT = false; + private static final ConfigDef.Importance CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_IMPORTANCE = + ConfigDef.Importance.MEDIUM; + /** * Return the ConfigDef object used to define this config's fields. * @@ -546,6 +552,11 @@ public static ConfigDef getConfig() { BIGQUERY_CLUSTERING_FIELD_NAMES_VALIDATOR, BIGQUERY_CLUSTERING_FIELD_NAMES_IMPORTANCE, BIGQUERY_CLUSTERING_FIELD_NAMES_DOC + ).defineInternal( + CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_CONFIG, + CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_TYPE, + CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_DEFAULT, + CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_IMPORTANCE ); } @@ -780,7 +791,7 @@ public SchemaConverter getSchemaConverter() { * @return a {@link RecordConverter} for BigQuery. */ public RecordConverter> getRecordConverter() { - return new BigQueryRecordConverter(getBoolean(CONVERT_DOUBLE_SPECIAL_VALUES_CONFIG)); + return new BigQueryRecordConverter(getBoolean(CONVERT_DOUBLE_SPECIAL_VALUES_CONFIG), getBoolean(CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER_CONFIG)); } /** diff --git a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverter.java b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverter.java index 4ba97174c..d5985000b 100644 --- a/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverter.java +++ b/kcbq-connector/src/main/java/com/wepay/kafka/connect/bigquery/convert/BigQueryRecordConverter.java @@ -54,6 +54,7 @@ public class BigQueryRecordConverter implements RecordConverter bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -93,7 +94,7 @@ public void testInteger() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); final Short fieldShortValue = (short) 4242; @@ -110,7 +111,7 @@ public void testInteger() { kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); final Integer fieldIntegerValue = 424242; @@ -127,7 +128,7 @@ public void testInteger() { kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); final Long fieldLongValue = 424242424242L; @@ -144,7 +145,7 @@ public void testInteger() { kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -165,7 +166,7 @@ public void testInteger() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); final Double fieldDoubleValue = 4242424242.4242; @@ -183,7 +184,7 @@ public void testInteger() { kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -210,7 +211,7 @@ public void testInteger() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } } @@ -233,7 +234,7 @@ public void testString() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -265,7 +266,7 @@ public void testStruct() { SinkRecord kafkaConnectInnerSinkRecord = spoofSinkRecord(kafkaConnectInnerSchema, kafkaConnectInnerStruct, false); Map bigQueryTestInnerRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE) + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER) .convertRecord(kafkaConnectInnerSinkRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedInnerRecord, bigQueryTestInnerRecord); @@ -287,7 +288,7 @@ public void testStruct() { SinkRecord kafkaConnectMiddleSinkRecord = spoofSinkRecord(kafkaConnectMiddleSchema, kafkaConnectMiddleStruct, true); Map bigQueryTestMiddleRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE) + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER) .convertRecord(kafkaConnectMiddleSinkRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedMiddleRecord, bigQueryTestMiddleRecord); @@ -309,7 +310,7 @@ public void testStruct() { SinkRecord kafkaConnectOuterSinkRecord = spoofSinkRecord(kafkaConnectOuterSchema, kafkaConnectOuterStruct, false); Map bigQueryTestOuterRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE) + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER) .convertRecord(kafkaConnectOuterSinkRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedOuterRecord, bigQueryTestOuterRecord); } @@ -325,7 +326,7 @@ public void testEmptyStruct() { SinkRecord kafkaConnectSinkRecord = spoofSinkRecord(kafkaConnectInnerSchema, kafkaConnectInnerStruct, false); Map bigQueryTestInnerRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE) + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER) .convertRecord(kafkaConnectSinkRecord, KafkaSchemaRecordType.VALUE); assertEquals(new HashMap(), bigQueryTestInnerRecord); } @@ -358,7 +359,7 @@ public void testEmptyInnerStruct() { SinkRecord kafkaConnectOuterSinkRecord = spoofSinkRecord(kafkaConnectOuterSchema, kafkaConnectOuterStruct, false); Map bigQueryTestOuterRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE) + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER) .convertRecord(kafkaConnectOuterSinkRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedOuterRecord, bigQueryTestOuterRecord); @@ -398,7 +399,7 @@ public void testMap() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -420,7 +421,7 @@ public void testIntegerArray() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -447,7 +448,7 @@ public void testStructArray() { SinkRecord kafkaConnectInnerSinkRecord = spoofSinkRecord(kafkaConnectInnerSchema, kafkaConnectInnerStruct, true); Map bigQueryTestInnerRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE) + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER) .convertRecord(kafkaConnectInnerSinkRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedInnerRecord, bigQueryTestInnerRecord); @@ -468,7 +469,7 @@ public void testStructArray() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -491,7 +492,7 @@ public void testStringArray() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -515,12 +516,13 @@ public void testBytes() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @Test public void testDebeziumLogicalType() { + // Test-1 final String fieldName = "DebeziumDate"; final int fieldDate = 17226; @@ -537,7 +539,50 @@ public void testDebeziumLogicalType() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); + + // Test-2 + String timeStampFieldName = "DebeziumTimestamp"; + long fieldValue = 1611854944000l; + + bigQueryExpectedRecord = new HashMap<>(); + bigQueryExpectedRecord.put(timeStampFieldName, "2021-01-28 17:29:04.000"); + + kafkaConnectSchema = SchemaBuilder + .struct() + .field(timeStampFieldName, io.debezium.time.Timestamp.schema()) + .build(); + + kafkaConnectStruct = new Struct(kafkaConnectSchema); + kafkaConnectStruct.put(timeStampFieldName, fieldValue); + kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); + + bigQueryTestRecord = + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); + + // Test-3 + timeStampFieldName = "DebeziumTimestamp"; + fieldValue = 1611854944000l; + + // By default, it is set to false + SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER = true; + + bigQueryExpectedRecord = new HashMap<>(); + bigQueryExpectedRecord.put(timeStampFieldName, 1611854944000l); + + kafkaConnectSchema = SchemaBuilder + .struct() + .field(timeStampFieldName, io.debezium.time.Timestamp.schema()) + .build(); + + kafkaConnectStruct = new Struct(kafkaConnectSchema); + kafkaConnectStruct.put(timeStampFieldName, fieldValue); + kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); + + bigQueryTestRecord = + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -559,7 +604,7 @@ public void testKafkaLogicalType() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -586,7 +631,7 @@ public void testNullable() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, true); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -609,7 +654,7 @@ public void testNullableStruct() { SinkRecord kafkaConnectRecord = spoofSinkRecord(kafkaConnectSchema, kafkaConnectStruct, false); Map bigQueryTestRecord = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(bigQueryExpectedRecord, bigQueryTestRecord); } @@ -631,7 +676,7 @@ public void testValidMapSchemaless() { SinkRecord kafkaConnectRecord = spoofSinkRecord(null, kafkaConnectMap, true); Map convertedMap = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(kafkaConnectMap, convertedMap); } @@ -653,7 +698,7 @@ public void testInvalidMapSchemaless() { SinkRecord kafkaConnectRecord = spoofSinkRecord(null, kafkaConnectMap, false); Map convertedMap = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); } @Test @@ -665,7 +710,7 @@ public void testInvalidMapSchemalessNullValue() { }}; SinkRecord kafkaConnectRecord = spoofSinkRecord(null, kafkaConnectMap, true); - Map stringObjectMap = new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + Map stringObjectMap = new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); Assert.assertEquals(kafkaConnectMap, stringObjectMap ); } @@ -682,7 +727,7 @@ public void testInvalidMapSchemalessNestedMapNullValue() { }}; SinkRecord kafkaConnectRecord = spoofSinkRecord(null, kafkaConnectMap, true); - Map stringObjectMap = new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE) + Map stringObjectMap = new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER) .convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); Assert.assertEquals(kafkaConnectMap, stringObjectMap); } @@ -705,7 +750,7 @@ public void testMapSchemalessConvertDouble() { SinkRecord kafkaConnectRecord = spoofSinkRecord(null, kafkaConnectMap, true); Map convertedMap = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.KEY); assertEquals(convertedMap.get("f1"), Double.MAX_VALUE); assertEquals(((Map)(convertedMap.get("f3"))).get("f4"), Double.MAX_VALUE); assertEquals(((ArrayList)((Map)(convertedMap.get("f3"))).get("f6")).get(1), Double.MAX_VALUE); @@ -731,7 +776,7 @@ public void testMapSchemalessConvertBytes() { SinkRecord kafkaConnectRecord = spoofSinkRecord(null, kafkaConnectMap, false); Map convertedMap = - new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); + new BigQueryRecordConverter(SHOULD_CONVERT_DOUBLE, SHOULD_CONVERT_DEBEZIUM_TIMESTAMP_TO_INTEGER).convertRecord(kafkaConnectRecord, KafkaSchemaRecordType.VALUE); assertEquals(convertedMap.get("f1"), Base64.getEncoder().encodeToString(helloWorld)); assertEquals(((Map)(convertedMap.get("f3"))).get("f4"), Base64.getEncoder().encodeToString(helloWorld)); } From c2aa14bd6c33afa4aa4ad65f253e7f59bb1f023e Mon Sep 17 00:00:00 2001 From: amitr17 Date: Mon, 28 Aug 2023 22:04:26 +0530 Subject: [PATCH 54/56] CC-22130 : Adding jacoco code coverage limits --- kcbq-confluent/pom.xml | 94 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index dea5a02f7..4c18b3dd7 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -84,6 +84,100 @@ + + org.jacoco + jacoco-maven-plugin + + + prepare-agent + + prepare-agent + + + + prepare-agent-it + + prepare-agent-integration + + pre-integration-test + + + merge-coverage-reports + verify + + merge + + + + + ${project.basedir} + + /target/jacoco.exec + /target/jacoco-it.exec + + + + ${project.basedir}/target/jacoco-aggregate.exec + + + + check + + check + + + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + 0.60 + + + BRANCH + COVEREDRATIO + 0.60 + + + COMPLEXITY + COVEREDRATIO + 0.60 + + + LINE + COVEREDRATIO + 0.60 + + + METHOD + COVEREDRATIO + 0.60 + + + CLASS + COVEREDRATIO + 0.80 + + + + + ${project.basedir}/target/jacoco-aggregate.exec + + + + report + test + + report + + + ${project.basedir}/target/jacoco-aggregate.exec + + + + org.apache.maven.plugins maven-compiler-plugin From a7be65540e30f861d4a429c0bca3a1d8d66def69 Mon Sep 17 00:00:00 2001 From: amitr17 Date: Mon, 28 Aug 2023 22:05:50 +0530 Subject: [PATCH 55/56] CC-22130 : Removed jacoco code coverage limits --- kcbq-confluent/pom.xml | 94 ------------------------------------------ 1 file changed, 94 deletions(-) diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index 4c18b3dd7..dea5a02f7 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -84,100 +84,6 @@ - - org.jacoco - jacoco-maven-plugin - - - prepare-agent - - prepare-agent - - - - prepare-agent-it - - prepare-agent-integration - - pre-integration-test - - - merge-coverage-reports - verify - - merge - - - - - ${project.basedir} - - /target/jacoco.exec - /target/jacoco-it.exec - - - - ${project.basedir}/target/jacoco-aggregate.exec - - - - check - - check - - - - - BUNDLE - - - INSTRUCTION - COVEREDRATIO - 0.60 - - - BRANCH - COVEREDRATIO - 0.60 - - - COMPLEXITY - COVEREDRATIO - 0.60 - - - LINE - COVEREDRATIO - 0.60 - - - METHOD - COVEREDRATIO - 0.60 - - - CLASS - COVEREDRATIO - 0.80 - - - - - ${project.basedir}/target/jacoco-aggregate.exec - - - - report - test - - report - - - ${project.basedir}/target/jacoco-aggregate.exec - - - - org.apache.maven.plugins maven-compiler-plugin From 188428b91a300a9f8ddfa4e4c85c0533c1b91b59 Mon Sep 17 00:00:00 2001 From: amitr17 Date: Mon, 28 Aug 2023 22:07:03 +0530 Subject: [PATCH 56/56] CC-22130 : Added jacoco code coverage limits --- kcbq-confluent/pom.xml | 94 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/kcbq-confluent/pom.xml b/kcbq-confluent/pom.xml index dea5a02f7..4c18b3dd7 100644 --- a/kcbq-confluent/pom.xml +++ b/kcbq-confluent/pom.xml @@ -84,6 +84,100 @@ + + org.jacoco + jacoco-maven-plugin + + + prepare-agent + + prepare-agent + + + + prepare-agent-it + + prepare-agent-integration + + pre-integration-test + + + merge-coverage-reports + verify + + merge + + + + + ${project.basedir} + + /target/jacoco.exec + /target/jacoco-it.exec + + + + ${project.basedir}/target/jacoco-aggregate.exec + + + + check + + check + + + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + 0.60 + + + BRANCH + COVEREDRATIO + 0.60 + + + COMPLEXITY + COVEREDRATIO + 0.60 + + + LINE + COVEREDRATIO + 0.60 + + + METHOD + COVEREDRATIO + 0.60 + + + CLASS + COVEREDRATIO + 0.80 + + + + + ${project.basedir}/target/jacoco-aggregate.exec + + + + report + test + + report + + + ${project.basedir}/target/jacoco-aggregate.exec + + + + org.apache.maven.plugins maven-compiler-plugin