2022 kie updates (#2032)

* Update the rule engine to never save the KB blob to the database.

The KB from the database is not required to be saved to function.

This will save a large data-call to the database as the rule blob is
saved as a large text file.

#2020

* Slight cleanup for #2020

* Update rule runner for modular process and read/write separation.

** Update knowledge bases to be broken into sets of 10,000
    * Add a way to re-balance rules. When a rule or watchlist item is
added the rules will re-balance. This will page through all rule and
watchlist items and assign them to knowledge bases.
** Update rule runner to accept multiple KBs of watchlist or UDRs
(instead of just one of each)
** Break rule runner scheduler into several stepped parts to separate
out fact gathering, kb processes, hit detail generation, and hit detail
persistence.
    * Separate out updated rule runner parts to only read from the
database with no modifications.
** Update KB generation to not hook in individual rules to the KB, but
instead use the UDR

* Update rule runner to go horizontal.

These code changes are to assist in horizontal drools rules. Instead of
using vertical scaling threads the rule engine for drools switch over to
a queue based system.

Hits are also always persisted in an async manner.

This allows the rule engine to process messages on a queue instead of a
scheduled.

Some  infrastructure changes were also done in order to support the new
rule runing application

* Add the gtas-rule-runner to the GTAS project.

Break out rule runner from the rule engine. This changes from a
scheduled task to a queue based system. There are no write executions on
the gtas-rule-runner which allow for scaling horizontally and
vertically.

* Docker updates:

Update ETL job dockerfile reference (was older reference which was
preventing builds

Add local properties for rule runner

Remove HTTP-PROXY as it has been superceded by the gtas-ui.

* Update local deployment

*to reference gtas-rule-runner instead of rule-runner.

* Update ConditionalOnProperty for Notificaiton Services

To use conditional on property to use email.hit.notification instead of
the manual enable.email.notification", name = "service was incorrectly
erroring when running the rules due to not instantiating the
HitEmailNotificationService.

* Upload loader jar to rule-runner

* Upload gtas-information-share jar

* Update gtas-commons

* Update gtas-parsers

* Move GTAS-Rule-Runner to gtas-parent

Gtas-rule-runner was very tightly coupled with gtas-parent and so it
made the most sense to add it as a module instead of a separate project.

* update docker compose and local docker compose

* Add gtas-information-share

* Updates to Rule Runner

Updated rule runner to return a list of hit details instead of raw
results that would need to be converted to hit details.

Updated the fuzzy matcher to return a list of hit details instead of
persisting the list. This tightens the logic on what happens on the
final step of the queue.

Updated hit detail's hitmakerid to not be json ignored. This allows the
hit maker id to be used by the rule engine.

Updated notification service instantiation to be agnostic to whether the
email service was toggled on or off.

* Add scripts for db change.

* Fix test

* Updates for deployment

Control memory
control dev deployment
control local deployment

* Update main.yml

* Minor adjustments and addendums to docker related stuff

* memory

Co-authored-by: simbam1 <simbam1@umbc.edu>

.github/workflows/main.yml

@@ -19,6 +19,15 @@ jobs:
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
+      - uses: docker/build-push-action@v2
+        name: Build and push GTAS rule runner image
+        with:
+          repository: wcogtas/gtas-rule-runner
+          context: .
+          platforms: linux/amd64, ppc64le
+          push: true
+          file: ./gtas-parent/gtas-rule-runner/gtas-rule-runner.Dockerfile
+          tags: wcogtas/gtas-rule-runner:dev
       - uses: docker/build-push-action@v2
         name: Build and push GTAS web application image
         with:

docker-compose.yml

@@ -1,30 +1,32 @@
 version: '3.7'
 services:
 
-  http-proxy: 
-    image: wcogtas/http-proxy:dev
-    container_name: http-proxy
-    build:  
-      context: ./gtas-parent
-      dockerfile: docker-resources/proxy.Dockerfile
-    restart: unless-stopped
-    ports:
-      - 443:443
-      - 80:80
-    networks:
-      - gtas-webapp-network
-    secrets:
-      - elastic-bootstrap-password
+#  http-proxy: 
+#    image: wcogtas/http-proxy:dev
+#    container_name: http-proxy
+#    build:  
+#      context: ./gtas-parent
+#      dockerfile: docker-resources/proxy.Dockerfile
+#    restart: unless-stopped
+#    ports:
+#      - 443:443
+#      - 80:80
+#      - 8443:8443
+#      - 8080:8080
+#    networks:
+#      - gtas-webapp-network
+#    secrets:
+#      - elastic-bootstrap-password
 
   web-app:
     image: wcogtas/webapp:dev
     container_name: web-app
     build: 
       context: ./gtas-parent
       dockerfile: web-app.Dockerfile
-    ports:
-      - 8080:8080
-      - 8443:8443
+#    ports:
+#      - 8080:8080
+#      - 8443:8443
     environment:
       - KIBANA_HOST=kibana
       - NEO4J_HOST=neo4j
@@ -115,7 +117,34 @@ services:
     restart: unless-stopped
     volumes:
       - db-data:/var/lib/mysql
-
+
+  rule-runner:
+    container_name: rule-runner
+    image: wcogtas/gtas-rule-runner:dev
+    build:
+      context: .
+      dockerfile: ./gtas-parent/gtas-rule-runner/gtas-rule-runner.Dockerfile
+    environment:
+      KB_LIST: UDR Knowledge Base_1,WL_Knowledge_Base_3,WL_Knowledge_Base_2,WL_Knowledge_Base_1
+      INBOUND_QUEUE: RULE_RUNNER_1
+      OUTBOUND_QUEUE: GTAS_RULE_ENGINE
+      ACTIVE_MQ_HOST: activemq
+    restart: unless-stopped
+    secrets:
+      - mysql-etl-user
+      - mysql-etl-password
+      - etl-neo4j-user
+      - etl-neo4j-password
+      - mysql-report-etl-user
+      - mysql-report-etl-password
+      - pentaho-to-elasticsearch-user
+      - pentaho-to-elasticsearch-password
+      - elastic-ca
+    networks: 
+      - gtas-webapp-network
+    depends_on: 
+      - activemq
+
   etl-job:
     container_name: etl-job
     image: wcogtas/neo4j-etl-job:dev
@@ -145,7 +174,7 @@ services:
     image: wcogtas/neo4j:dev
     build:
       context: .
-      dockerfile: ./gtas-neo4j-etl/neo4j.Dockerfile
+      dockerfile: ./gtas-etl-main/neo4j.Dockerfile
     restart: unless-stopped
     environment:
       NEO4J_bolt_ssl__policy: "default"

docker-properties.yml

@@ -1,13 +1,13 @@
 version: '3.7'
 services:
-  http-proxy:
-    environment:
-      BOOTSTRAP_PATH: '/run/secrets/elastic-bootstrap-password'
-    secrets:
-      - source: webapp-cert
-        target: /temp-cert/wcogtas.org.crt
-      - source: webapp-key
-        target: /temp-cert/wcogtas.org.key
+#  http-proxy:
+#    environment:
+#      BOOTSTRAP_PATH: '/run/secrets/elastic-bootstrap-password'
+#    secrets:
+#      - source: webapp-cert
+#        target: /temp-cert/wcogtas.org.crt
+#      - source: webapp-key
+#        target: /temp-cert/wcogtas.org.key
   web-app:
     environment:
       DB_HOST: mariadb

gtas-parent/gtas-commons/src/main/java/gov/gtas/config/TestCommonServicesConfig.java

@@ -23,12 +23,12 @@
  * <context:annotation-config/>
  * <bean class="gov.gtas.config.TestCommonServicesConfig"/>
  */
-@Configuration
-@ComponentScan("gov.gtas")
-@PropertySource({ "classpath:default.application.properties"})
-@EnableJpaRepositories("gov.gtas")
-@EnableTransactionManagement
-@Import({ AsyncConfig.class, CommonServicesConfig.class })
+//@Configuration
+//@ComponentScan("gov.gtas")
+//@PropertySource({ "classpath:default.application.properties"})
+//@EnableJpaRepositories("gov.gtas")
+//@EnableTransactionManagement
+//@Import({ AsyncConfig.class, CommonServicesConfig.class })
 public class TestCommonServicesConfig {
 
  /*

gtas-parent/gtas-commons/src/main/java/gov/gtas/constant/WatchlistConstants.java

@@ -6,7 +6,7 @@
 package gov.gtas.constant;
 
 public class WatchlistConstants {
-	public static final String WL_KNOWLEDGE_BASE_NAME = "WL Knowledge Base";
+	public static final String WL_KNOWLEDGE_BASE_NAME = "WL_Knowledge_Base";
 	public static final String DELETE_OP_NAME = "delete";
 	//////////////////////////////////////////////////////////////////////////////////
 	// VALIDATION FIELDS and ERROR codes

gtas-parent/gtas-commons/src/main/java/gov/gtas/config/ElasticConfig.java → gtas-parent/gtas-commons/src/main/java/gov/gtas/elasticsearchconfig/ElasticConfig.java

@@ -1,4 +1,4 @@
-package gov.gtas.config;
+package gov.gtas.elasticsearchconfig;
 
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
@@ -8,9 +8,9 @@
 import javax.annotation.Resource;
 
 @Configuration
-@ComponentScan("gov.gtas")
-@PropertySource("classpath:default.application.properties")
-@PropertySource(value = "file:${catalina.home}/conf/application.properties", ignoreResourceNotFound = true)
+//@ComponentScan("gov.gtas")
+//@PropertySource("classpath:default.application.properties")
+//@PropertySource(value = "file:${catalina.home}/conf/application.properties", ignoreResourceNotFound = true)
 public class ElasticConfig {
 
     @Resource

gtas-parent/gtas-commons/src/main/java/gov/gtas/model/ExternalHit.java

@@ -2,15 +2,18 @@
 
 import gov.gtas.enumtype.HitTypeEnum;
 
+import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Table;
 
 @Entity
 @Table(name = "external_hit")
 public class ExternalHit extends HitMaker {
 
+	@Column(name = "description")
     private String description;
 
+	@Column(name = "lookout")
     private Boolean lookout;
 
     public ExternalHit() {

gtas-parent/gtas-commons/src/main/java/gov/gtas/model/FlightLeg.java

@@ -22,6 +22,7 @@ public FlightLeg() {
 	private Message message;
 
 	@ManyToOne
+	@JoinColumn(name = "bookingDetail_id")
 	private BookingDetail bookingDetail;
 
 	@Column(name = "leg_number", nullable = false)

gtas-parent/gtas-commons/src/main/java/gov/gtas/model/GraphRuleParameter.java

@@ -2,6 +2,7 @@
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.JoinColumn;
 import javax.persistence.ManyToOne;
 import javax.persistence.Table;
 
@@ -11,6 +12,7 @@
 public class GraphRuleParameter extends BaseEntity {
 
 	@ManyToOne
+	@JoinColumn(name = "graphRule_id")
 	private GraphRule graphRule;
 
 	@Column

gtas-parent/gtas-commons/src/main/java/gov/gtas/model/HitDetail.java

@@ -12,6 +12,8 @@
 
 import javax.persistence.*;
 
+import org.apache.commons.lang3.StringUtils;
+
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import gov.gtas.enumtype.HitTypeEnum;
 
@@ -49,7 +51,6 @@ public HitDetail(HitTypeEnum hitEnum) {
 	@JsonIgnore
 	private HitMaker hitMaker;
 
-	@JsonIgnore
 	@Column(name = "hm_id", columnDefinition = "bigint unsigned", nullable = false)
 	private Long hitMakerId;
 
@@ -226,13 +227,18 @@ public static HitDetail from(PendingHitDetails pendingHitDetails) {
 
 	public static HitDetail from(RuleHitDetail ruleHitDetail) {
 		HitDetail hitDetail = new HitDetail(ruleHitDetail.getHitType());
+		if (!StringUtils.isBlank(ruleHitDetail.getTitle())) {
+			hitDetail.setTitle(ruleHitDetail.getTitle());
+		} else {
+			hitDetail.setTitle(ruleHitDetail.getHitRule());
+		}
 		hitDetail.setHitType(ruleHitDetail.getHitType().toString());
 		hitDetail.setPassengerId(ruleHitDetail.getPassengerId());
 		hitDetail.setHitMakerId(ruleHitDetail.getHitMakerId());
 		hitDetail.setRuleId(ruleHitDetail.getRuleId());
+		hitDetail.setFlightId(ruleHitDetail.getFlightId());
 		hitDetail.setDescription(ruleHitDetail.getDescription());
 		hitDetail.setCreatedDate(new Date());
-		hitDetail.setTitle(ruleHitDetail.getTitle());
 		if (HitTypeEnum.GRAPH_HIT == ruleHitDetail.getHitType()) {
 			hitDetail.setRuleConditions(ruleHitDetail.getGraphHitDisplay());
 		} else {

gtas-parent/gtas-commons/src/main/java/gov/gtas/model/LookoutRequest.java

@@ -2,16 +2,23 @@
 
 import gov.gtas.services.dto.LookoutSendRequest;
 
+import javax.persistence.Column;
 import javax.persistence.Entity;
 
 @Entity(name="lookout_request")
 public class LookoutRequest extends BaseEntityAudit {
+	@Column(name="passengerId")
     private Long passengerId;
+	@Column(name="flightId")
     private Long flightId;
     private String note;
+	@Column(name="noteRtf")
     private String noteRtf;
+	@Column(name="noteCat")
     private String noteCat;
+	@Column(name="sendRaw")
     private Boolean sendRaw;
+	@Column(name="countryGroupName")
     private String countryGroupName;
 
     public static LookoutRequest from(LookoutSendRequest lsr) {

gtas-parent/gtas-commons/src/main/java/gov/gtas/model/Message.java

@@ -59,6 +59,9 @@ public Message() {
 	@ManyToMany
 	@JoinTable(name = "message_document", joinColumns = @JoinColumn(name = "document_id"), inverseJoinColumns = @JoinColumn(name = "message_id"))
 	private Set<Document> documents = new HashSet<>();
+
+	@OneToMany(orphanRemoval = true, mappedBy = "mostRecentMessage")
+	private Set<PassengerTripDetails> passengerTripDetails = new HashSet<>();
 
 	@Column(name = "passenger_count")
 	protected Integer passengerCount;
@@ -69,6 +72,14 @@ public Message() {
 	@Transient
 	private List<PassengerNote> passengerNotes = new ArrayList<>();
 
+	public Set<PassengerTripDetails> getPassengerTripDetails() {
+		return passengerTripDetails;
+	}
+
+	public void setPassengerTripDetails(Set<PassengerTripDetails> passengerTripDetails) {
+		this.passengerTripDetails = passengerTripDetails;
+	}
+
 	public List<PendingHitDetails> getPendingHitDetails() {
 		return pendingHitDetails;
 	}

gtas-parent/gtas-commons/src/main/java/gov/gtas/model/PassengerTripDetails.java

@@ -78,13 +78,36 @@ public void setPassenger(Passenger passenger) {
 
 	@Column(name = "hours_before_takeoff")
 	private Integer hoursBeforeTakeOff;
+
+	@Column(name="most_recent_message", columnDefinition = "bigint unsigned")
+	private Long mostRecentMessageId;
+
+	@ManyToOne(fetch = FetchType.LAZY)
+	@JoinColumn(name = "most_recent_message", updatable = false, insertable = false)
+	private Message mostRecentMessage;
 
 	@Transient
 	private String totalBagWeight;
 
 	@Transient
 	private String bagNum;
 
+	public Long getMostRecentMessageId() {
+		return mostRecentMessageId;
+	}
+
+	public void setMostRecentMessageId(Long messageId) {
+		this.mostRecentMessageId = messageId;
+	}
+
+	public Message getMostRecentMessage() {
+		return mostRecentMessage;
+	}
+
+	public void setMostRecentMessage(Message mostRecentMessage) {
+		this.mostRecentMessage = mostRecentMessage;
+	}
+
 	public String getPnrReservationReferenceNumber() {
 		return pnrReservationReferenceNumber;
 	}

gtas-parent/gtas-commons/src/main/java/gov/gtas/model/PendingHitDetails.java

@@ -13,7 +13,9 @@
 
 import javax.persistence.*;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.Objects;
+import java.util.Set;
 import java.util.UUID;
 
 @Entity
@@ -221,4 +223,24 @@ public Boolean getLookout() {
     public void setLookout(Boolean lookout) {
         this.lookout = lookout;
     }
+
+	public static Set<PendingHitDetails> convertHits(Set<HitDetail> hits) {
+		Set<PendingHitDetails> phdSet = new HashSet<>();
+		for (HitDetail hd : hits) {
+			PendingHitDetails phd = new PendingHitDetails(hd.getHitEnum());
+			phd.setTitle(hd.getTitle());
+			phd.setFlightId(hd.getFlightId());
+			phd.setHitType(hd.getHitType());
+			phd.setHitEnum(hd.getHitEnum());
+			phd.setPassengerId(hd.getPassengerId());
+			phd.setHitMakerId(hd.getHitMakerId());
+			phd.setDescription(hd.getDescription());
+			phd.setCreatedDate(hd.getCreatedDate());
+			phd.setTitle(hd.getTitle());
+			phd.setRuleConditions(hd.getRuleConditions());
+			phd.setPercentage(hd.getPercentage());
+			phdSet.add(phd);
+		}
+		return phdSet;
+	}
 }