diff --git a/pom.xml b/pom.xml
index 30bb273..d6c32c5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -8,7 +8,7 @@
NFRR Audit Collector
jar
https://github.com/hygieia/${repository.name}
- 3.1.7-SNAPSHOT
+ 3.1.8-SNAPSHOT
org.springframework.boot
diff --git a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java
index 3531fa9..4c86b8f 100644
--- a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java
+++ b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java
@@ -1,19 +1,19 @@
package com.capitalone.dashboard.collector;
+
import com.capitalone.dashboard.client.RestClient;
-import com.capitalone.dashboard.model.Dashboard;
import com.capitalone.dashboard.model.Audit;
import com.capitalone.dashboard.model.AuditResult;
import com.capitalone.dashboard.model.AuditType;
-import com.capitalone.dashboard.model.DashboardType;
import com.capitalone.dashboard.model.Cmdb;
-
-import com.capitalone.dashboard.repository.DashboardRepository;
-import com.capitalone.dashboard.repository.AuditResultRepository;
+import com.capitalone.dashboard.model.Dashboard;
+import com.capitalone.dashboard.model.DashboardType;
import com.capitalone.dashboard.repository.AuditCollectorRepository;
-import com.capitalone.dashboard.repository.CmdbRepository;
+import com.capitalone.dashboard.repository.AuditResultRepository;
import com.capitalone.dashboard.repository.BaseCollectorRepository;
-import com.capitalone.dashboard.repository.ComponentRepository;
+import com.capitalone.dashboard.repository.CmdbRepository;
import com.capitalone.dashboard.repository.CollectorItemRepository;
+import com.capitalone.dashboard.repository.ComponentRepository;
+import com.capitalone.dashboard.repository.DashboardRepository;
import org.apache.commons.collections4.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -21,14 +21,12 @@
import org.springframework.scheduling.TaskScheduler;
import org.springframework.stereotype.Component;
-
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.util.Date;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
/**
* AuditCollectorTask
@@ -69,11 +67,18 @@ public AuditCollectorTask(TaskScheduler taskScheduler, DashboardRepository dashb
@Override
public void collect(AuditCollector collector) {
+ long startTime = System.currentTimeMillis();
+ LOGGER.info(String.format("NFRRCollectorTask:collect start collector_run_id=%d", startTime));
LOGGER.info("NFRR Audit Collector pulls all the team dashboards");
- Iterable dashboards = dashboardRepository.findAllByType(DashboardType.Team);
- collectAuditResults(dashboards);
+ Iterable dashboards = dashboardRepository.findAllByType(DashboardType.Team);
+ collectAuditResults(dashboards, startTime);
+ long endTime = System.currentTimeMillis();
+ long elapsedSeconds = (endTime-startTime)/1000;
+ collector.setLastExecutedSeconds(elapsedSeconds);
+ collector.setLastExecutionRecordCount(CollectionUtils.size(dashboards));
LOGGER.info("NFRR Audit Collector executed successfully");
+ LOGGER.info(String.format("NFRRCollectorTask:collect stop, collector_process_time=%d collector_item_count=%d collector_run_id=%d", elapsedSeconds, CollectionUtils.size(dashboards), startTime));
}
/**
@@ -81,15 +86,15 @@ public void collect(AuditCollector collector) {
*
* @param dashboards
*/
- protected void collectAuditResults(Iterable dashboards) {
+ protected void collectAuditResults(Iterable dashboards, long collector_run_id) {
int numberOfAuditDays = settings.getDays();
long auditBeginDateTimeStamp = Instant.now().minus(Duration.ofDays(numberOfAuditDays)).toEpochMilli();
long auditEndDateTimeStamp = Instant.now().toEpochMilli();
int totTeamDbdCount = CollectionUtils.size(dashboards);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd:HH:mm:ss");
- LOGGER.info(String.format("Audit time range start=%d (%s) end=%d (%s) dashboards=%d",
+ LOGGER.info(String.format("Audit time range time_range_start=%d (%s) time_range_end=%d (%s) dashboard_count=%d collector_run_id=%d",
auditBeginDateTimeStamp, sdf.format(new Date(auditBeginDateTimeStamp)),
- auditEndDateTimeStamp, sdf.format(new Date(auditEndDateTimeStamp)), totTeamDbdCount));
+ auditEndDateTimeStamp, sdf.format(new Date(auditEndDateTimeStamp)), totTeamDbdCount, collector_run_id));
AuditCollector collector = getCollectorRepository().findByName(COLLECTOR_NAME);
AuditCollectorUtil auditCollectorUtil = new AuditCollectorUtil(collector, componentRepository,
@@ -104,11 +109,11 @@ protected void collectAuditResults(Iterable dashboards) {
List latestAuditResults = AuditCollectorUtil.getAuditResults(dashboard, auditMap, cmdb, auditEndDateTimeStamp);
refreshAuditResults(dashboard, latestAuditResults);
} catch (Exception e) {
- LOGGER.error("Exception in collecting audit result for dashboard="+dashboard.getTitle(), e);
+ LOGGER.error("Exception in collecting audit result for error_dashboard=" + dashboard.getTitle() +" collector_run_id=" + collector_run_id , e);
}
long endTime = System.currentTimeMillis();
- LOGGER.info(String.format("Adding audit results - dashboard=%s [%d/%d] timeTaken=%d",
- dashboard.getTitle(), ++index, totTeamDbdCount, endTime-startTime));
+ LOGGER.info(String.format("Adding audit results - dashboard=%s [%d/%d] duration=%d collector_run_id=%d",
+ dashboard.getTitle(), ++index, totTeamDbdCount, endTime-startTime, collector_run_id));
}
}
diff --git a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java
index 2857412..912555b 100644
--- a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java
+++ b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java
@@ -2,30 +2,26 @@
import com.capitalone.dashboard.client.RestClient;
import com.capitalone.dashboard.model.Audit;
+import com.capitalone.dashboard.model.AuditResult;
import com.capitalone.dashboard.model.AuditStatus;
-import com.capitalone.dashboard.model.DataStatus;
import com.capitalone.dashboard.model.AuditType;
-import com.capitalone.dashboard.model.AuditResult;
-import com.capitalone.dashboard.model.Dashboard;
import com.capitalone.dashboard.model.Cmdb;
-import com.capitalone.dashboard.model.CollectorItem;
import com.capitalone.dashboard.model.Collector;
+import com.capitalone.dashboard.model.CollectorItem;
import com.capitalone.dashboard.model.CollectorType;
import com.capitalone.dashboard.model.Component;
-
+import com.capitalone.dashboard.model.Dashboard;
+import com.capitalone.dashboard.model.DataStatus;
import com.capitalone.dashboard.repository.CollectorItemRepository;
import com.capitalone.dashboard.repository.ComponentRepository;
-
-
+import com.capitalone.dashboard.status.ArtifactAuditStatus;
+import com.capitalone.dashboard.status.CodeQualityAuditStatus;
import com.capitalone.dashboard.status.CodeReviewAuditStatus;
import com.capitalone.dashboard.status.DashboardAuditStatus;
-import com.capitalone.dashboard.status.CodeQualityAuditStatus;
-import com.capitalone.dashboard.status.PerformanceTestAuditStatus;
+import com.capitalone.dashboard.status.DeployAuditStatus;
import com.capitalone.dashboard.status.LibraryPolicyAuditStatus;
+import com.capitalone.dashboard.status.PerformanceTestAuditStatus;
import com.capitalone.dashboard.status.TestResultAuditStatus;
-import com.capitalone.dashboard.status.ArtifactAuditStatus;
-import com.capitalone.dashboard.status.DeployAuditStatus;
-
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.http.client.utils.URIBuilder;
import org.bson.types.ObjectId;
@@ -37,23 +33,22 @@
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
-import org.springframework.data.domain.Sort;
-import org.springframework.data.web.PageableDefault;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.jmx.MBeanServerNotFoundException;
import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
-import java.util.List;
+import java.net.URISyntaxException;
import java.util.ArrayList;
-import java.util.Set;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import java.util.Arrays;
import java.util.Optional;
+import java.util.Set;
import java.util.StringJoiner;
-import java.util.Collections;
-
import java.util.function.Supplier;
import java.util.stream.Stream;
@@ -192,7 +187,7 @@ private Audit doBasicAuditCheck(JSONArray jsonArray, JSONArray global, AuditType
* Check for collector error
*/
private boolean isCollectorError(JSONArray jsonArray) {
- Stream jsonObjectStream = jsonArray.stream().map((Object object) -> (JSONObject) object);
+ Stream jsonObjectStream = jsonArray.stream().map((Object object) -> object);
Stream auditStatusArray = jsonObjectStream.map(jsonObject -> (JSONArray) jsonObject.get(STR_AUDITSTATUSES));
return auditStatusArray.anyMatch(aSArray -> aSArray.toJSONString().contains(DashboardAuditStatus.COLLECTOR_ITEM_ERROR.name()));
}
@@ -203,31 +198,31 @@ private boolean isCollectorError(JSONArray jsonArray) {
@SuppressWarnings("PMD.NPathComplexity")
private boolean isConfigured(AuditType auditType, JSONArray jsonArray) {
if (auditType.equals(AuditType.CODE_REVIEW)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_REPO_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_REPO_CONFIGURED.name()));
}
if (auditType.equals(AuditType.CODE_QUALITY)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_CODEQUALITY_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_CODEQUALITY_CONFIGURED.name()));
}
if (auditType.equals(AuditType.STATIC_SECURITY_ANALYSIS)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_STATIC_SECURITY_ANALYSIS_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_STATIC_SECURITY_ANALYSIS_CONFIGURED.name()));
}
if (auditType.equals(AuditType.LIBRARY_POLICY)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_LIBRARY_POLICY_ANALYSIS_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_LIBRARY_POLICY_ANALYSIS_CONFIGURED.name()));
}
if (auditType.equals(AuditType.TEST_RESULT)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_TEST_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_TEST_CONFIGURED.name()));
}
if (auditType.equals(AuditType.PERF_TEST)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_PERFORMANCE_TEST_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_PERFORMANCE_TEST_CONFIGURED.name()));
}
if (auditType.equals(AuditType.BUILD_REVIEW)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_BUILD_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_BUILD_CONFIGURED.name()));
}
if (auditType.equals(AuditType.ARTIFACT)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_ARTIFACT_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_ARTIFACT_CONFIGURED.name()));
}
if (auditType.equals(AuditType.DEPLOY)) {
- return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_DEPLOYMENT_SCRIPTS_CONFIGURED.name()) ? true : false);
+ return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_DEPLOYMENT_SCRIPTS_CONFIGURED.name()));
}
return false;
}
@@ -646,6 +641,7 @@ private void updateComponent(Dashboard dashboard) {
*/
protected JSONObject parseObject(String url, AuditSettings settings){
JSONObject responseObj = null;
+ if(StringUtils.isEmpty(url)) return responseObj;
try {
ResponseEntity response = restClient.makeRestCallGet(url, getHeaders(settings));
JSONParser jsonParser = new JSONParser();
@@ -665,8 +661,12 @@ protected String getAuditAPIUrl(Dashboard dashboard, AuditSettings settings, lon
LOGGER.error("No Server Found to run NoFearRelease audit collector");
throw new MBeanServerNotFoundException("No Server Found to run NoFearRelease audit collector");
}
- URIBuilder auditURI = new URIBuilder();
- auditURI.setPath(settings.getServers().get(0) + HYGIEIA_AUDIT_URL);
+ URIBuilder auditURI = null;
+ try {
+ auditURI = new URIBuilder(settings.getServers().get(0) + HYGIEIA_AUDIT_URL);
+ } catch (URISyntaxException e) {
+ return null;
+ }
auditURI.addParameter(AUDIT_PARAMS.title.name(), dashboard.getTitle());
auditURI.addParameter(AUDIT_PARAMS.businessService.name(), dashboard.getConfigurationItemBusServName());
auditURI.addParameter(AUDIT_PARAMS.businessApplication.name(), dashboard.getConfigurationItemBusAppName());
@@ -789,7 +789,6 @@ private Double getAvgTracePercent(Stream stream) {
protected CollectorItem createCollectorItem(AuditType auditType){
String description = getDescription(auditType);
Pageable pageable = new PageRequest(0, 25);
-// Iterable collectorItems = collectorItemRepository.findByDescription(description);
Page collectorItems = collectorItemRepository.findByCollectorIdAndSearchField(Collections.singletonList(collector.getId()),
OPT_OPT_URL, description, pageable);
Optional optCollectorItem = Optional.ofNullable(collectorItems.iterator().hasNext() ? collectorItems.iterator().next() : null);
@@ -805,7 +804,8 @@ protected CollectorItem createCollectorItem(AuditType auditType){
collectorItem.setDescription(description);
collectorItem.getOptions().put(OPT_DBRD_ID, getDashboard().getId());
collectorItem.getOptions().put(OPT_URL, description);
- LOGGER.info(String.format("new collector item created, description=%s", description));
+ LOGGER.info(String.format("new collector item created, description=%s updated=%d dashboard_id=%s audit_collector_item_id=%s",
+ description, collectorItem.getLastUpdated(), collectorItem.getOptions().get(OPT_DBRD_ID), collectorItem.getId()));
return collectorItem;
}));
return collectorItemRepository.save(optCollectorItem.get());