Add a flag to enable security feature while running the regression.

dtests/src/resources/scripts/alterTable/alterTable.sql

@@ -4,9 +4,9 @@ describe orders;
 
 SELECT count(*) FROM orders;
 
-insert into app.orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000','1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuailles en stock 2','rue du Commerce','Lyon',NULL,'69004','France','empFirstName2');
-insert into app.orders values(10260,'CENTC',13,'1996-07-19 00:00:00.000','1996-08-16 00:00:00.000','1996-07-29 00:00:00.000',13,55.09,'Ottilies Kaoseladen','Mehrheimerstr. 369','Kaqln',NULL,'50739','Germany','empFirstName4');
-insert into app.orders values(10265,'DUMON',18,'1996-07-25 00:00:00.000','1996-08-22 00:00:00.000','1996-08-12 00:00:00.000',18,55.28,'Blondel pare et fils 24', 'place Klacber','Strasbourg',NULL,'67000','France','empFirstName5');
+insert into orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000','1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuailles en stock 2','rue du Commerce','Lyon',NULL,'69004','France','empFirstName2');
+insert into orders values(10260,'CENTC',13,'1996-07-19 00:00:00.000','1996-08-16 00:00:00.000','1996-07-29 00:00:00.000',13,55.09,'Ottilies Kaoseladen','Mehrheimerstr. 369','Kaqln',NULL,'50739','Germany','empFirstName4');
+insert into orders values(10265,'DUMON',18,'1996-07-25 00:00:00.000','1996-08-22 00:00:00.000','1996-08-12 00:00:00.000',18,55.28,'Blondel pare et fils 24', 'place Klacber','Strasbourg',NULL,'67000','France','empFirstName5');
 
 SELECT * FROM orders where FirstName='empFirstName2';
 SELECT * FROM orders where FirstName='empFirstName4';
@@ -22,9 +22,9 @@ ALTER TABLE order_details ADD CustomerID String;
 
 SELECT count(*) FROM order_details;
 
-insert into app.order_details values(10250,72,34.8,5,0,'custID4');
-insert into app.order_details values(10249,42,9.8,10,0,'custID14');
-insert into app.order_details values(10253,41,7.7,10,0,'custID7');
+insert into order_details values(10250,72,34.8,5,0,'custID4');
+insert into order_details values(10249,42,9.8,10,0,'custID14');
+insert into order_details values(10253,41,7.7,10,0,'custID7');
 
 SELECT * FROM order_details where CustomerID='custID4';
 SELECT * FROM order_details where CustomerID='custID14';

dtests/src/resources/scripts/diskFullTests/createAndLoadColumnTable.sql

@@ -8,7 +8,7 @@ CREATE TABLE AGREEMENT( AGREE_ID BIGINT NOT NULL,
   VLD_FRM_DT TIMESTAMP NOT NULL,
   VLD_TO_DT TIMESTAMP,
   SRC_SYS_REF_ID VARCHAR(200) NOT NULL,
-  SRC_SYS_REC_ID VARCHAR(200)) USING column OPTIONS(partition_by 'AGREE_ID', buckets '32',key_columns 'CLIENT_ID,AGREE_ID ' );
+  SRC_SYS_REC_ID VARCHAR(200)) USING column OPTIONS(partition_by 'AGREE_ID', buckets '32',key_columns 'CLIENT_ID,AGREE_ID ',redundancy '1' );
   INSERT into AGREEMENT select id,abs(rand()*1000),abs(rand()*1000),'agree_cd','description','2018-01-01','2019-01-01',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' FROM range(10000000);
 
 CREATE TABLE BANK(
@@ -21,5 +21,5 @@ CREATE TABLE BANK(
   VLD_FRM_DT TIMESTAMP NOT NULL,
   VLD_TO_DT TIMESTAMP,
   SRC_SYS_REF_ID VARCHAR(10) NOT NULL,
-  SRC_SYS_REC_ID VARCHAR(150)) USING column OPTIONS(partition_by 'BNK_ORG_ID', buckets '32',key_columns 'CLIENT_ID,BNK_ORG_ID,BNK_ID ' ) ;
+  SRC_SYS_REC_ID VARCHAR(150)) USING column OPTIONS(partition_by 'BNK_ORG_ID', buckets '32',key_columns 'CLIENT_ID,BNK_ORG_ID,BNK_ID ',redundancy '1' ) ;
   INSERT into BANK select id,id,abs(rand()*1000),abs(rand()*1000),'BNK_FULL_NM','RTNG_NUM',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' from range(4000000);

dtests/src/resources/scripts/diskFullTests/createAndLoadPartitionedRowTable.sql

@@ -1,4 +1,4 @@
-CREATE TABLE AGREEMENT( AGREE_ID BIGINT NOT NULL,
+CREATE TABLE AGREEMENTrow( AGREE_ID BIGINT NOT NULL,
   VER BIGINT NOT NULL,
   CLIENT_ID BIGINT NOT NULL,
   AGREE_CD VARCHAR(200),
@@ -8,4 +8,4 @@ CREATE TABLE AGREEMENT( AGREE_ID BIGINT NOT NULL,
   VLD_FRM_DT TIMESTAMP NOT NULL,
   VLD_TO_DT TIMESTAMP,
   SRC_SYS_REF_ID VARCHAR(200) NOT NULL,
-  SRC_SYS_REC_ID VARCHAR(200)) USING row OPTIONS(partition_by 'AGREE_ID', buckets '32');
+  SRC_SYS_REC_ID VARCHAR(200)) USING row OPTIONS(partition_by 'AGREE_ID', buckets '32',redundancy '1');

dtests/src/resources/scripts/diskFullTests/createAndLoadReplicatedRowTable.sql

@@ -1,4 +1,4 @@
-CREATE TABLE AGREEMENT( AGREE_ID BIGINT NOT NULL,
+CREATE TABLE AGREEMENTrr( AGREE_ID BIGINT NOT NULL,
   VER BIGINT NOT NULL,
   CLIENT_ID BIGINT NOT NULL,
   AGREE_CD VARCHAR(200),

dtests/src/resources/scripts/diskFullTests/insertQuery.sql

@@ -1,2 +1,2 @@
-INSERT into AGREEMENT select id,abs(rand()*1000),abs(rand()*1000),'agree_cd','description','2018-01-01','2019-01-01',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' FROM range(2000000);
-INSERT into BANK select id,id,abs(rand()*1000),abs(rand()*1000),'BNK_FULL_NM','RTNG_NUM',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' from range(2000000);
+INSERT into AGREEMENT select id,abs(rand()*1000),abs(rand()*1000),'agree_cd','description','2018-01-01','2019-01-01',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' FROM range(200000000);
+INSERT into BANK select id,id,abs(rand()*1000),abs(rand()*1000),'BNK_FULL_NM','RTNG_NUM',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' from range(5000000);

dtests/src/resources/scripts/security/dmlExtTblQuery.sql

@@ -0,0 +1,10 @@
+SELECT count(*) FROM gemfire2.staging_employees;
+SELECT * FROM gemfire2.staging_employees limit 5;
+SELECT count(*) FROM gemfire2.staging_categories;
+SELECT * FROM gemfire2.staging_categories;
+SELECT City, COUNT(EmployeeID) AS NumEmployees FROM gemfire2.staging_employees WHERE Title = 'Sales Representative' GROUP BY City HAVING COUNT(EmployeeID) > 1 ORDER BY NumEmployees;
+SELECT * FROM gemfire2.staging_customers;
+SELECT * from gemfire2.staging_suppliers;
+SELECT * FROM gemfire2.staging_orders limit 5;
+SELECT count(*) FROM gemfire2.staging_orders;
+SELECT ProductID, AVG(UnitPrice) AS AveragePrice FROM gemfire2.staging_products GROUP BY ProductID HAVING AVG(UnitPrice) > 70 ORDER BY AveragePrice;

dtests/src/resources/scripts/security/dmlRowColQuery.sql

@@ -1,22 +1,22 @@
-SELECT count(*) FROM user2.employees;
-SELECT * FROM user2.employees limit 5;
-SELECT count(*) FROM user2.categories;
-SELECT * FROM user2.categories;
-SELECT City, COUNT(EmployeeID) AS NumEmployees FROM user2.employees WHERE Title = 'Sales Representative' GROUP BY City HAVING COUNT(EmployeeID) > 1 ORDER BY NumEmployees;
-INSERT INTO user2.employees values(2,'empFirstName2','empLastName2','VicePresidentSales','Mrs.','1952-02-19 00:00:00.000','1992-08-14 00:00:00.000','908W.CapitalWay','Tacoma','WA','98401','USA','(206)555-9482',3457,'000','StudiedMBA.',NULL,'http://accweb/emmployees/fuller.bmp');
-INSERT INTO user2.categories values(10,'Pulses','dalandsprouts','00');
-UPDATE user2.employees SET LastName = 'last name' WHERE EmployeeID = 7;
-DELETE FROM user2.employees WHERE LastName = 'last name' AND EmployeeID = 7;
-UPDATE user2.categories SET CategoryName = 'Sweet Candies' WHERE CategoryId = 10;
-DELETE FROM user2.categories WHERE categoryId = 10;
-SELECT * FROM user2.customers;
-SELECT * from user2.suppliers;
-SELECT * FROM user2.orders limit 5;
-SELECT count(*) FROM user2.orders;
-SELECT ProductID, AVG(UnitPrice) AS AveragePrice FROM user2.products GROUP BY ProductID HAVING AVG(UnitPrice) > 70 ORDER BY AveragePrice;
-INSERT into user2.customers values('custID4','abc','custName7','MarketingManager','aasdfasdfasd7','Strasbourg',NULL,'67000','France','88.60.15.31','88.60.15.32');
-INSERT into user2.order_details values(10253,41,7.7,10,0);
-INSERT into user2.orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000','1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuaillesenstock2','rueduCommerce','Lyon',NULL,'69004','France');
-UPDATE user2.customers SET companyName = 'New Company' WHERE CustomerId = 'ANTON';
-UPDATE user2.orders SET ShipCountry = 'INDIA' WHERE OrderID = 10250;
-DELETE FROM user2.orders WHERE ShipCountry = 'INDIA' AND OrderID = 10250;
+SELECT count(*) FROM gemfire2.employees;
+SELECT * FROM gemfire2.employees limit 5;
+SELECT count(*) FROM gemfire2.categories;
+SELECT * FROM gemfire2.categories;
+SELECT City, COUNT(EmployeeID) AS NumEmployees FROM gemfire2.employees WHERE Title = 'Sales Representative' GROUP BY City HAVING COUNT(EmployeeID) > 1 ORDER BY NumEmployees;
+INSERT INTO gemfire2.employees values(2,'empFirstName2','empLastName2','VicePresidentSales','Mrs.','1952-02-19 00:00:00.000','1992-08-14 00:00:00.000','908W.CapitalWay','Tacoma','WA','98401','USA','(206)555-9482',3457,'000','StudiedMBA.',NULL,'http://accweb/emmployees/fuller.bmp');
+INSERT INTO gemfire2.categories values(10,'Pulses','dalandsprouts','00');
+UPDATE gemfire2.employees SET LastName = 'last name' WHERE EmployeeID = 7;
+DELETE FROM gemfire2.employees WHERE LastName = 'last name' AND EmployeeID = 7;
+UPDATE gemfire2.categories SET CategoryName = 'Sweet Candies' WHERE CategoryId = 10;
+DELETE FROM gemfire2.categories WHERE categoryId = 10;
+SELECT * FROM gemfire2.customers;
+SELECT * from gemfire2.suppliers;
+SELECT * FROM gemfire2.orders limit 5;
+SELECT count(*) FROM gemfire2.orders;
+SELECT ProductID, AVG(UnitPrice) AS AveragePrice FROM gemfire2.products GROUP BY ProductID HAVING AVG(UnitPrice) > 70 ORDER BY AveragePrice;
+INSERT into gemfire2.customers values('custID4','abc','custName7','MarketingManager','aasdfasdfasd7','Strasbourg',NULL,'67000','France','88.60.15.31','88.60.15.32');
+INSERT into gemfire2.order_details values(10253,41,7.7,10,0);
+INSERT into gemfire2.orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000','1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuaillesenstock2','rueduCommerce','Lyon',NULL,'69004','France');
+UPDATE gemfire2.customers SET companyName = 'New Company' WHERE CustomerId = 'ANTON';
+UPDATE gemfire2.orders SET ShipCountry = 'INDIA' WHERE OrderID = 10250;
+DELETE FROM gemfire2.orders WHERE ShipCountry = 'INDIA' AND OrderID = 10250;

dtests/src/resources/scripts/security/user3Credentials.txt

@@ -1 +1 @@
--u user3:user3123
+-u gemfire3:gemfire3

dtests/src/test/java/io/snappydata/hydra/cdcConnector/CDCIngestionApp.java

@@ -27,14 +27,16 @@ public class CDCIngestionApp implements Runnable {
   private int startRange;
   private int endRange;
   private String endPoint;
+  private boolean isSecurityEnabled;
 
-  public CDCIngestionApp(int sRange, int eRange, int i, String path, String sqlServerInst, String hostName){
+  public CDCIngestionApp(int sRange, int eRange, int i, String path, String sqlServerInst, String hostName, boolean isSecurity){
     threadName = "Thread-" + i;
     startRange = sRange;
     endRange = eRange;
     filePath = path + "/insert" + i + ".sql";
     sqlServer = sqlServerInst;
     endPoint = hostName;
+    isSecurityEnabled = isSecurity;
   }
 
   public void run() {
@@ -57,9 +59,14 @@ public Connection getSnappyConnection() {
     Connection conn = null;
     String url = "jdbc:snappydata://" + endPoint;
     String driver = "io.snappydata.jdbc.ClientDriver";
-    try {
+    Properties props = new Properties();
+    if(isSecurityEnabled){
+      props.setProperty("user","gemfire");
+      props.setProperty("password","gemfire");
+    }
+   try {
       Class.forName(driver);
-      conn = DriverManager.getConnection(url);
+      conn = DriverManager.getConnection(url,props);
     } catch (Exception ex) {
       System.out.println("Caught exception in getSnappyConnection() method" + ex.getMessage());
     }
@@ -74,7 +81,7 @@ public Connection getSqlServerConnection() {
       Class.forName(driver);
       String url;
       if (sqlServer.equals("sqlServer1")) {
-        url = "jdbc:sqlserver://sqlent.westus.cloudapp.azure.com:1433";
+        url = "jdbc:sqlserver://sqlserver-ent16.copfedn1qbcz.us-west-2.rds.amazonaws.com:1433";
       } else
         url = "jdbc:sqlserver://sqlserver2-et16.copfedn1qbcz.us-west-2.rds.amazonaws.com:1435";
       String username = "sqldb";
@@ -93,69 +100,69 @@ public Connection getSqlServerConnection() {
     return conn;
   }
 
-public void insertData(ArrayList<String> queryArray, Connection conn) {
+  public void insertData(ArrayList<String> queryArray, Connection conn) {
     PreparedStatement ps = null;
-  try {
-    final int batchSize = 1000;
-    int count = 0;
-    Random rnd = new Random();
-    for (int i = 0; i < queryArray.size(); i++) {
-      String qStr = queryArray.get(i);
-      System.out.println("Query = " + qStr);
-      System.out.println("The startRange = " + startRange + " the endRange = " + endRange);
-      if (qStr.contains("PUT INTO")) {
-        for (int j = startRange; j <= endRange; j++) {
-          String newStr;
-          if (qStr.contains("?"))
-            newStr = qStr.replace("?", Integer.toString(j));
-          else
-            newStr = qStr;
-          System.out.println("The new query String is " + newStr);
-          conn.createStatement().execute(newStr);
-          updateData(queryArray,conn);
-        }
-      } else if (qStr.contains("UPDATE")) {
-        ps = conn.prepareStatement(qStr);
-        int updateKey = rnd.nextInt(startRange);
-        ps.setInt(1, updateKey);
-        ps.execute();
-        System.out.println("Key to be updated is " + updateKey);
-      } else if (qStr.contains("DELETE")) {
-        ps = conn.prepareStatement(qStr);
-        int delKey = rnd.nextInt(startRange);
-        ps.setInt(1, delKey);
-        ps.execute();
-        System.out.println("Key to be deleted is " + delKey);
-      } else if(qStr.contains("INSERT INTO")){
-        ps = conn.prepareStatement(qStr);
-        for (int j = startRange; j <= endRange; j++) {
-          int KEY_ID = j;
-          ps.setInt(1, KEY_ID);
-          ps.addBatch();
-          if (++count % batchSize == 0) {
-            ps.executeBatch();
+    try {
+      final int batchSize = 1000;
+      int count = 0;
+      Random rnd = new Random();
+      for (int i = 0; i < queryArray.size(); i++) {
+        String qStr = queryArray.get(i);
+        System.out.println("Query = " + qStr);
+        System.out.println("The startRange = " + startRange + " the endRange = " + endRange);
+        if (qStr.contains("PUT INTO")) {
+          for (int j = startRange; j <= endRange; j++) {
+            String newStr;
+            if (qStr.contains("?"))
+              newStr = qStr.replace("?", Integer.toString(j));
+            else
+              newStr = qStr;
+            System.out.println("The new query String is " + newStr);
+            conn.createStatement().execute(newStr);
+            updateData(queryArray, conn);
           }
-        }
-        System.out.println("Thread " + threadName + " finished  ingesting " + (endRange - startRange) + " rows in a table");
+        } else if (qStr.contains("UPDATE")) {
+          ps = conn.prepareStatement(qStr);
+          int updateKey = rnd.nextInt(startRange);
+          ps.setInt(1, updateKey);
+          ps.execute();
+          System.out.println("Key to be updated is " + updateKey);
+        } else if (qStr.contains("DELETE")) {
+          ps = conn.prepareStatement(qStr);
+          int delKey = rnd.nextInt(startRange);
+          ps.setInt(1, delKey);
+          ps.execute();
+          System.out.println("Key to be deleted is " + delKey);
+        } else if (qStr.contains("INSERT INTO")) {
+          ps = conn.prepareStatement(qStr);
+          for (int j = startRange; j <= endRange; j++) {
+            int KEY_ID = j;
+            ps.setInt(1, KEY_ID);
+            ps.addBatch();
+            if (++count % batchSize == 0) {
+              ps.executeBatch();
+            }
+          }
+          System.out.println("Thread " + threadName + " finished  ingesting " + (endRange - startRange) + " rows in a table");
+        } else
+          conn.createStatement().execute(qStr);
+      }
+      System.out.println("FINISHED: Thread " + threadName + " finished ingestion in all the tables");
+    } catch (Exception e) {
+      System.out.println("Caught exception " + e.getMessage());
+    } finally {
+      if (ps != null) try {
+        ps.close();
+      } catch (SQLException ex) {
+      }
+      if (conn != null) try {
+        conn.close();
+      } catch (SQLException ex) {
       }
-     else
-       conn.createStatement().execute(qStr);
-    }
-    System.out.println("FINISHED: Thread " + threadName + " finished ingestion in all the tables");
-  } catch (Exception e) {
-    System.out.println("Caught exception " + e.getMessage());
-  } finally {
-    if (ps != null) try {
-      ps.close();
-    } catch (SQLException ex) {
-    }
-    if (conn != null) try {
-      conn.close();
-    } catch (SQLException ex) {
-    }
 
+    }
   }
-}
+
   public void updateData(ArrayList<String> queryArray, Connection conn) {
     try {
       for (int i = 0; i < queryArray.size(); i++) {
@@ -210,12 +217,12 @@ public void start() {
     }
   }
 
-  public static void runIngestionApp(int sRange, int eRange, int thnCnt, String path, String sqlServerInst, String hostName) {
+  public static void runIngestionApp(int sRange, int eRange, int thnCnt, String path, String sqlServerInst, String hostName, boolean isSecurity) {
     ExecutorService executor = Executors.newFixedThreadPool(thnCnt);
     for (int i = 1; i <= thnCnt; i++) {
       String threadName = "Thread-" + i;
       System.out.println("Creating " + threadName);
-      executor.execute(new CDCIngestionApp(sRange, eRange, i,path, sqlServerInst, hostName));
+      executor.execute(new CDCIngestionApp(sRange, eRange, i,path, sqlServerInst, hostName, isSecurity));
     }
     executor.shutdown();
     try {
@@ -235,7 +242,7 @@ public static void main(String args[]) {
       String sqlServerInstance = args[4];
       String hostname = args[5];
       System.out.println("The startRange is " + sRange + " and the endRange is " + eRange);
-      runIngestionApp(sRange, eRange, threadCnt, insertQPAth, sqlServerInstance, hostname);
+      runIngestionApp(sRange, eRange, threadCnt, insertQPAth, sqlServerInstance, hostname, false);
     } catch (Exception e) {
       System.out.println("Caught exception in main " + e.getMessage());
     } finally {