Merge branch 'master' into harshil/redirect-advanced-json-changes

config/server.yaml

@@ -202,6 +202,9 @@ sentry:
   tags:
     component: coordinator
 
+quotas:
+  pinotMaxQPSQuotaOverride: 100
+
 scheduler:
   # Run the Quartz Scheduler.
   # Only 1 instance of scheduler should run. This responsibility is currently on the user!!

jacoco-code-coverage/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>jacoco-code-coverage</artifactId>

kubernetes/helm/startree-thirdeye/Chart.yaml

@@ -16,7 +16,7 @@ apiVersion: v2
 appVersion: 0.5.0-SNAPSHOT # Should be maintained automatically by a pipeline
 name: startree-thirdeye
 description: One Stop Shop For Anomaly Detection.
-version: 2.50.0
+version: 2.51.0
 keywords:
   - thirdeye
   - metrics

kubernetes/helm/startree-thirdeye/templates/coordinator/configmap.yaml

@@ -133,7 +133,11 @@ data:
       # Some characters are forbidden - see https://docs.sentry.io/platforms/java/configuration/environments
       environment: {{regexReplaceAll `(https?:|\n| |/)` .Values.ui.publicUrl ""}}
       tags:
-        component: "{{ .Values.coordinator.name }}"  
+        component: "{{ .Values.coordinator.name }}"
+{{ if .Values.quotas }}
+    quotas:
+{{- toYaml .Values.quotas  | nindent 6 }}
+{{- end }}
 
     notifications:
       useSendgridEmail: true

pinot-test-container/pom.xml

@@ -18,7 +18,7 @@
     <parent>
         <artifactId>thirdeye</artifactId>
         <groupId>ai.startree.thirdeye</groupId>
-        <version>1.401.0-SNAPSHOT</version>
+        <version>1.404.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

pom.xml

@@ -18,7 +18,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>ai.startree.thirdeye</groupId>
   <artifactId>thirdeye</artifactId>
-  <version>1.401.0-SNAPSHOT</version>
+  <version>1.404.0-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>thirdeye</name>
 

thirdeye-benchmarks/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>thirdeye-benchmarks</artifactId>

thirdeye-core/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <artifactId>thirdeye-core</artifactId>
 

thirdeye-core/src/main/java/ai/startree/thirdeye/ThirdEyeCoreModule.java

@@ -18,15 +18,14 @@
 import ai.startree.thirdeye.datasource.loader.DefaultAggregationLoader;
 import ai.startree.thirdeye.datasource.loader.DefaultMinMaxTimeLoader;
 import ai.startree.thirdeye.rootcause.configuration.RcaConfiguration;
-import ai.startree.thirdeye.spi.api.NamespaceConfigurationApi;
+import ai.startree.thirdeye.spi.config.QuotasConfiguration;
 import ai.startree.thirdeye.spi.config.TimeConfiguration;
 import ai.startree.thirdeye.spi.datalayer.dto.NamespaceConfigurationDTO;
 import ai.startree.thirdeye.spi.datasource.loader.AggregationLoader;
 import ai.startree.thirdeye.spi.datasource.loader.MinMaxTimeLoader;
 import com.google.inject.AbstractModule;
 import com.google.inject.Scopes;
 import com.google.inject.util.Providers;
-import java.security.Provider;
 import org.apache.tomcat.jdbc.pool.DataSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,18 +39,21 @@ public class ThirdEyeCoreModule extends AbstractModule {
   private final UiConfiguration uiConfiguration;
   private final TimeConfiguration timeConfiguration;
   private final NamespaceConfigurationDTO defaultNamespaceConfiguration;
+  private final QuotasConfiguration quotasConfiguration;
 
   public ThirdEyeCoreModule(final DataSource dataSource,
       final RcaConfiguration rcaConfiguration,
       final UiConfiguration uiConfiguration,
       final TimeConfiguration timeConfiguration,
-      final NamespaceConfigurationDTO defaultNamespaceConfiguration) {
+      final NamespaceConfigurationDTO defaultNamespaceConfiguration,
+      final QuotasConfiguration quotasConfiguration) {
     this.dataSource = dataSource;
 
     this.rcaConfiguration = rcaConfiguration;
     this.uiConfiguration = uiConfiguration;
     this.timeConfiguration = timeConfiguration;
     this.defaultNamespaceConfiguration = defaultNamespaceConfiguration;
+    this.quotasConfiguration = quotasConfiguration;
   }
 
   @Override
@@ -70,5 +72,6 @@ protected void configure() {
       bind(TimeConfiguration.class).toProvider(Providers.of(null));
     }
     bind(NamespaceConfigurationDTO.class).toInstance(defaultNamespaceConfiguration);
+    bind(QuotasConfiguration.class).toInstance(quotasConfiguration);
   }
 }

thirdeye-core/src/main/java/ai/startree/thirdeye/datasource/DataSourceOnboarder.java

@@ -77,6 +77,8 @@ public List<DatasetConfigDTO> onboardAll(final DataSourceDTO dataSourceDto) {
         .toList();
 
     final List<DatasetConfigDTO> addedDatasets = datasetsToOnboard.stream()
+        .peek(datasetConfigDTO -> dataSource.prepareDatasetForOnboarding(
+            datasetConfigDTO.getDataset()))
         .map(datasetConfigDTO -> persist(datasetConfigDTO, dataSourceDto.getAuth()))
         .collect(Collectors.toList());
 
@@ -91,6 +93,7 @@ public DatasetConfigDTO onboardDataset(final DataSourceDTO dataSourceDto,
     checkThirdEye(dataSource != null, ThirdEyeStatus.ERR_DATASOURCE_NOT_LOADED, dataSourceDto.getName());
     final DatasetConfigDTO newDataset = dataSource.getDataset(datasetName);
     checkThirdEye(newDataset != null, ThirdEyeStatus.ERR_DATASET_NOT_FOUND, datasetName);
+    dataSource.prepareDatasetForOnboarding(datasetName);
     final DatasetConfigDTO datasetConfigDTO = persist(newDataset, dataSourceDto.getAuth());
     checkThirdEye(datasetConfigDTO != null, ThirdEyeStatus.ERR_DATASET_NOT_FOUND, datasetName);
 

thirdeye-core/src/main/java/ai/startree/thirdeye/datasource/DataSourcesLoader.java

@@ -17,6 +17,7 @@
 import static com.google.common.base.Preconditions.checkState;
 import static java.util.Objects.requireNonNull;
 
+import ai.startree.thirdeye.spi.config.QuotasConfiguration;
 import ai.startree.thirdeye.spi.datalayer.dto.DataSourceDTO;
 import ai.startree.thirdeye.spi.datasource.ThirdEyeDataSource;
 import ai.startree.thirdeye.spi.datasource.ThirdEyeDataSourceContext;
@@ -40,8 +41,11 @@ public class DataSourcesLoader {
 
   private final Map<String, ThirdEyeDataSourceFactory> dataSourceFactoryMap = new HashMap<>();
 
+  private final QuotasConfiguration quotasConfiguration;
+
   @Inject
-  public DataSourcesLoader() {
+  public DataSourcesLoader(final QuotasConfiguration quotasConfiguration) {
+    this.quotasConfiguration = quotasConfiguration;
   }
 
   public void addThirdEyeDataSourceFactory(ThirdEyeDataSourceFactory f) {
@@ -79,7 +83,8 @@ private ThirdEyeDataSourceContext buildContext(final DataSourceDTO dataSource) {
     try {
       final DataSourceDTO dataSourceWithEnvVarResolved = StringTemplateUtils.applyContext(
           dataSource, values);
-      return new ThirdEyeDataSourceContext().setDataSourceDTO(dataSourceWithEnvVarResolved);
+      return new ThirdEyeDataSourceContext().setDataSourceDTO(dataSourceWithEnvVarResolved)
+          .setQuotasConfiguration(quotasConfiguration);
     } catch (IOException e) {
       throw new RuntimeException(
           "Error while replacing env variables in datasource spec. spec: " + dataSource);

thirdeye-core/src/main/java/ai/startree/thirdeye/datasource/cache/MeteredDataSource.java

@@ -59,6 +59,11 @@ public List<DatasetConfigDTO> getDatasets() {
     return delegate.getDatasets();
   }
 
+  @Override
+  public void prepareDatasetForOnboarding(final String datasetName) {
+    delegate.prepareDatasetForOnboarding(datasetName);
+  }
+
   @Override
   public DatasetConfigDTO getDataset(final String datasetName) {
     return delegate.getDataset(datasetName);

thirdeye-dataframe/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

thirdeye-detectionpipeline/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <artifactId>thirdeye-detectionpipeline</artifactId>
 

thirdeye-distribution/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye</artifactId>
     <groupId>ai.startree.thirdeye</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-integration-tests/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye</artifactId>
     <groupId>ai.startree.thirdeye</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-notification/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <groupId>ai.startree.thirdeye</groupId>
     <artifactId>thirdeye</artifactId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <artifactId>thirdeye-notification</artifactId>
 

thirdeye-persistence/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye</artifactId>
     <groupId>ai.startree.thirdeye</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-plugins/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye</artifactId>
     <groupId>ai.startree.thirdeye</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-plugins/thirdeye-bootstrap-open-core/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye-plugins</artifactId>
     <groupId>ai.startree.thirdeye.plugins</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-plugins/thirdeye-contributors-simple/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye-plugins</artifactId>
     <groupId>ai.startree.thirdeye.plugins</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-plugins/thirdeye-detectors/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye-plugins</artifactId>
     <groupId>ai.startree.thirdeye.plugins</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-plugins/thirdeye-enumerators/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye-plugins</artifactId>
     <groupId>ai.startree.thirdeye.plugins</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-plugins/thirdeye-notification-email/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye-plugins</artifactId>
     <groupId>ai.startree.thirdeye.plugins</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-plugins/thirdeye-oauth/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye-plugins</artifactId>
     <groupId>ai.startree.thirdeye.plugins</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

thirdeye-plugins/thirdeye-pinot/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>thirdeye-plugins</artifactId>
     <groupId>ai.startree.thirdeye.plugins</groupId>
-    <version>1.401.0-SNAPSHOT</version>
+    <version>1.404.0-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>

thirdeye-plugins/thirdeye-pinot/src/main/java/ai/startree/thirdeye/plugins/datasource/pinot/PinotDatasetReader.java

@@ -108,6 +108,16 @@ public DatasetConfigDTO getTable(final String tableName, final String dataSource
         dataSourceName);
   }
 
+  public void prepareDatasetForOnboarding(final String datasetName)
+      throws IOException {
+    final JsonNode tableConfigJson = pinotControllerRestClient
+        .getTableConfigFromPinotEndpoint(datasetName);
+    checkArgument(tableConfigJson != null && !tableConfigJson.isNull(),
+        "Onboarding Preparation Error: table config is null for pinot table: " + datasetName);
+
+    pinotControllerRestClient.updateTableMaxQPSQuota(datasetName, tableConfigJson);
+  }
+
   public void close() {
     pinotControllerRestClient.close();
   }

thirdeye-plugins/thirdeye-pinot/src/main/java/ai/startree/thirdeye/plugins/datasource/pinot/PinotQueryExecutor.java

@@ -173,6 +173,7 @@ private static ThirdEyeDataFrameResultSet fromPinotResultSet(final ResultSet res
         try {
           valueString = resultSet.getString(rowIdx, metricColumnIdx);
         } catch (final Exception e) {
+          LOG.error("Failed to get value as String from result set. Replacing with a null value. This may generate errors downstream. Please reach out to support.", e);
           // Do nothing and subsequently insert a null value to the current series.
         }
         columnsOfTheRow[metricColumnIdx + groupByColumnCount] = valueString;

thirdeye-plugins/thirdeye-pinot/src/main/java/ai/startree/thirdeye/plugins/datasource/pinot/PinotThirdEyeDataSource.java

@@ -259,6 +259,15 @@ public List<DatasetConfigDTO> getDatasets() {
     }
   }
 
+  @Override
+  public void prepareDatasetForOnboarding(final String datasetName) {
+    try {
+      datasetReader.prepareDatasetForOnboarding(datasetName);
+    } catch (final IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   @Override
   public DatasetConfigDTO getDataset(final String datasetName) {
     try {