From 3ee30b6ee797fc5376a7b6eba29f12b289ab8e5d Mon Sep 17 00:00:00 2001 From: rvema Date: Tue, 20 Apr 2021 12:31:48 -0400 Subject: [PATCH 1/3] Fix the rest calls from collector to audit api instance --- pom.xml | 2 +- .../collector/AuditCollectorUtil.java | 63 ++++++++++--------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index 30bb273..d6c32c5 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ NFRR Audit Collector jar https://github.com/hygieia/${repository.name} - 3.1.7-SNAPSHOT + 3.1.8-SNAPSHOT org.springframework.boot diff --git a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java index 2857412..0940821 100644 --- a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java +++ b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java @@ -2,30 +2,26 @@ import com.capitalone.dashboard.client.RestClient; import com.capitalone.dashboard.model.Audit; +import com.capitalone.dashboard.model.AuditResult; import com.capitalone.dashboard.model.AuditStatus; -import com.capitalone.dashboard.model.DataStatus; import com.capitalone.dashboard.model.AuditType; -import com.capitalone.dashboard.model.AuditResult; -import com.capitalone.dashboard.model.Dashboard; import com.capitalone.dashboard.model.Cmdb; -import com.capitalone.dashboard.model.CollectorItem; import com.capitalone.dashboard.model.Collector; +import com.capitalone.dashboard.model.CollectorItem; import com.capitalone.dashboard.model.CollectorType; import com.capitalone.dashboard.model.Component; - +import com.capitalone.dashboard.model.Dashboard; +import com.capitalone.dashboard.model.DataStatus; import com.capitalone.dashboard.repository.CollectorItemRepository; import com.capitalone.dashboard.repository.ComponentRepository; - - +import com.capitalone.dashboard.status.ArtifactAuditStatus; +import com.capitalone.dashboard.status.CodeQualityAuditStatus; import com.capitalone.dashboard.status.CodeReviewAuditStatus; import com.capitalone.dashboard.status.DashboardAuditStatus; -import com.capitalone.dashboard.status.CodeQualityAuditStatus; -import com.capitalone.dashboard.status.PerformanceTestAuditStatus; +import com.capitalone.dashboard.status.DeployAuditStatus; import com.capitalone.dashboard.status.LibraryPolicyAuditStatus; +import com.capitalone.dashboard.status.PerformanceTestAuditStatus; import com.capitalone.dashboard.status.TestResultAuditStatus; -import com.capitalone.dashboard.status.ArtifactAuditStatus; -import com.capitalone.dashboard.status.DeployAuditStatus; - import org.apache.commons.lang3.math.NumberUtils; import org.apache.http.client.utils.URIBuilder; import org.bson.types.ObjectId; @@ -37,23 +33,22 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Sort; -import org.springframework.data.web.PageableDefault; import org.springframework.http.HttpHeaders; import org.springframework.http.ResponseEntity; import org.springframework.jmx.MBeanServerNotFoundException; import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; -import java.util.List; +import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Set; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Arrays; import java.util.Optional; +import java.util.Set; import java.util.StringJoiner; -import java.util.Collections; - import java.util.function.Supplier; import java.util.stream.Stream; @@ -192,7 +187,7 @@ private Audit doBasicAuditCheck(JSONArray jsonArray, JSONArray global, AuditType * Check for collector error */ private boolean isCollectorError(JSONArray jsonArray) { - Stream jsonObjectStream = jsonArray.stream().map((Object object) -> (JSONObject) object); + Stream jsonObjectStream = jsonArray.stream().map((Object object) -> object); Stream auditStatusArray = jsonObjectStream.map(jsonObject -> (JSONArray) jsonObject.get(STR_AUDITSTATUSES)); return auditStatusArray.anyMatch(aSArray -> aSArray.toJSONString().contains(DashboardAuditStatus.COLLECTOR_ITEM_ERROR.name())); } @@ -203,31 +198,31 @@ private boolean isCollectorError(JSONArray jsonArray) { @SuppressWarnings("PMD.NPathComplexity") private boolean isConfigured(AuditType auditType, JSONArray jsonArray) { if (auditType.equals(AuditType.CODE_REVIEW)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_REPO_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_REPO_CONFIGURED.name())); } if (auditType.equals(AuditType.CODE_QUALITY)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_CODEQUALITY_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_CODEQUALITY_CONFIGURED.name())); } if (auditType.equals(AuditType.STATIC_SECURITY_ANALYSIS)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_STATIC_SECURITY_ANALYSIS_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_STATIC_SECURITY_ANALYSIS_CONFIGURED.name())); } if (auditType.equals(AuditType.LIBRARY_POLICY)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_LIBRARY_POLICY_ANALYSIS_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_LIBRARY_POLICY_ANALYSIS_CONFIGURED.name())); } if (auditType.equals(AuditType.TEST_RESULT)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_TEST_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_TEST_CONFIGURED.name())); } if (auditType.equals(AuditType.PERF_TEST)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_PERFORMANCE_TEST_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_PERFORMANCE_TEST_CONFIGURED.name())); } if (auditType.equals(AuditType.BUILD_REVIEW)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_BUILD_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_BUILD_CONFIGURED.name())); } if (auditType.equals(AuditType.ARTIFACT)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_ARTIFACT_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_ARTIFACT_CONFIGURED.name())); } if (auditType.equals(AuditType.DEPLOY)) { - return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_DEPLOYMENT_SCRIPTS_CONFIGURED.name()) ? true : false); + return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_DEPLOYMENT_SCRIPTS_CONFIGURED.name())); } return false; } @@ -646,6 +641,7 @@ private void updateComponent(Dashboard dashboard) { */ protected JSONObject parseObject(String url, AuditSettings settings){ JSONObject responseObj = null; + if(StringUtils.isEmpty(url)) return responseObj; try { ResponseEntity response = restClient.makeRestCallGet(url, getHeaders(settings)); JSONParser jsonParser = new JSONParser(); @@ -665,8 +661,13 @@ protected String getAuditAPIUrl(Dashboard dashboard, AuditSettings settings, lon LOGGER.error("No Server Found to run NoFearRelease audit collector"); throw new MBeanServerNotFoundException("No Server Found to run NoFearRelease audit collector"); } - URIBuilder auditURI = new URIBuilder(); - auditURI.setPath(settings.getServers().get(0) + HYGIEIA_AUDIT_URL); + URIBuilder auditURI = null; + try { + auditURI = new URIBuilder(settings.getServers().get(0) + HYGIEIA_AUDIT_URL); + } catch (URISyntaxException e) { + return null; + } + //auditURI.setPath(settings.getServers().get(0) + HYGIEIA_AUDIT_URL); auditURI.addParameter(AUDIT_PARAMS.title.name(), dashboard.getTitle()); auditURI.addParameter(AUDIT_PARAMS.businessService.name(), dashboard.getConfigurationItemBusServName()); auditURI.addParameter(AUDIT_PARAMS.businessApplication.name(), dashboard.getConfigurationItemBusAppName()); From 0ec690ae65d0210ebf762858b98b3245c0085f44 Mon Sep 17 00:00:00 2001 From: rvema Date: Tue, 20 Apr 2021 15:30:55 -0400 Subject: [PATCH 2/3] added diagnostic log statements --- .../collector/AuditCollectorTask.java | 39 +++++++++++-------- .../collector/AuditCollectorUtil.java | 3 +- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java index 3531fa9..4c86b8f 100644 --- a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java +++ b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java @@ -1,19 +1,19 @@ package com.capitalone.dashboard.collector; + import com.capitalone.dashboard.client.RestClient; -import com.capitalone.dashboard.model.Dashboard; import com.capitalone.dashboard.model.Audit; import com.capitalone.dashboard.model.AuditResult; import com.capitalone.dashboard.model.AuditType; -import com.capitalone.dashboard.model.DashboardType; import com.capitalone.dashboard.model.Cmdb; - -import com.capitalone.dashboard.repository.DashboardRepository; -import com.capitalone.dashboard.repository.AuditResultRepository; +import com.capitalone.dashboard.model.Dashboard; +import com.capitalone.dashboard.model.DashboardType; import com.capitalone.dashboard.repository.AuditCollectorRepository; -import com.capitalone.dashboard.repository.CmdbRepository; +import com.capitalone.dashboard.repository.AuditResultRepository; import com.capitalone.dashboard.repository.BaseCollectorRepository; -import com.capitalone.dashboard.repository.ComponentRepository; +import com.capitalone.dashboard.repository.CmdbRepository; import com.capitalone.dashboard.repository.CollectorItemRepository; +import com.capitalone.dashboard.repository.ComponentRepository; +import com.capitalone.dashboard.repository.DashboardRepository; import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,14 +21,12 @@ import org.springframework.scheduling.TaskScheduler; import org.springframework.stereotype.Component; - import java.text.SimpleDateFormat; import java.time.Duration; import java.time.Instant; import java.util.Date; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; /** *

AuditCollectorTask

@@ -69,11 +67,18 @@ public AuditCollectorTask(TaskScheduler taskScheduler, DashboardRepository dashb @Override public void collect(AuditCollector collector) { + long startTime = System.currentTimeMillis(); + LOGGER.info(String.format("NFRRCollectorTask:collect start collector_run_id=%d", startTime)); LOGGER.info("NFRR Audit Collector pulls all the team dashboards"); - Iterable dashboards = dashboardRepository.findAllByType(DashboardType.Team); - collectAuditResults(dashboards); + Iterable dashboards = dashboardRepository.findAllByType(DashboardType.Team); + collectAuditResults(dashboards, startTime); + long endTime = System.currentTimeMillis(); + long elapsedSeconds = (endTime-startTime)/1000; + collector.setLastExecutedSeconds(elapsedSeconds); + collector.setLastExecutionRecordCount(CollectionUtils.size(dashboards)); LOGGER.info("NFRR Audit Collector executed successfully"); + LOGGER.info(String.format("NFRRCollectorTask:collect stop, collector_process_time=%d collector_item_count=%d collector_run_id=%d", elapsedSeconds, CollectionUtils.size(dashboards), startTime)); } /** @@ -81,15 +86,15 @@ public void collect(AuditCollector collector) { * * @param dashboards */ - protected void collectAuditResults(Iterable dashboards) { + protected void collectAuditResults(Iterable dashboards, long collector_run_id) { int numberOfAuditDays = settings.getDays(); long auditBeginDateTimeStamp = Instant.now().minus(Duration.ofDays(numberOfAuditDays)).toEpochMilli(); long auditEndDateTimeStamp = Instant.now().toEpochMilli(); int totTeamDbdCount = CollectionUtils.size(dashboards); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd:HH:mm:ss"); - LOGGER.info(String.format("Audit time range start=%d (%s) end=%d (%s) dashboards=%d", + LOGGER.info(String.format("Audit time range time_range_start=%d (%s) time_range_end=%d (%s) dashboard_count=%d collector_run_id=%d", auditBeginDateTimeStamp, sdf.format(new Date(auditBeginDateTimeStamp)), - auditEndDateTimeStamp, sdf.format(new Date(auditEndDateTimeStamp)), totTeamDbdCount)); + auditEndDateTimeStamp, sdf.format(new Date(auditEndDateTimeStamp)), totTeamDbdCount, collector_run_id)); AuditCollector collector = getCollectorRepository().findByName(COLLECTOR_NAME); AuditCollectorUtil auditCollectorUtil = new AuditCollectorUtil(collector, componentRepository, @@ -104,11 +109,11 @@ protected void collectAuditResults(Iterable dashboards) { List latestAuditResults = AuditCollectorUtil.getAuditResults(dashboard, auditMap, cmdb, auditEndDateTimeStamp); refreshAuditResults(dashboard, latestAuditResults); } catch (Exception e) { - LOGGER.error("Exception in collecting audit result for dashboard="+dashboard.getTitle(), e); + LOGGER.error("Exception in collecting audit result for error_dashboard=" + dashboard.getTitle() +" collector_run_id=" + collector_run_id , e); } long endTime = System.currentTimeMillis(); - LOGGER.info(String.format("Adding audit results - dashboard=%s [%d/%d] timeTaken=%d", - dashboard.getTitle(), ++index, totTeamDbdCount, endTime-startTime)); + LOGGER.info(String.format("Adding audit results - dashboard=%s [%d/%d] duration=%d collector_run_id=%d", + dashboard.getTitle(), ++index, totTeamDbdCount, endTime-startTime, collector_run_id)); } } diff --git a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java index 0940821..51adb81 100644 --- a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java +++ b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java @@ -806,7 +806,8 @@ protected CollectorItem createCollectorItem(AuditType auditType){ collectorItem.setDescription(description); collectorItem.getOptions().put(OPT_DBRD_ID, getDashboard().getId()); collectorItem.getOptions().put(OPT_URL, description); - LOGGER.info(String.format("new collector item created, description=%s", description)); + LOGGER.info(String.format("new collector item created, description=%s updated=%d dashboard_id=%s audit_collector_item_id=%s", + description, collectorItem.getLastUpdated(), collectorItem.getOptions().get(OPT_DBRD_ID), collectorItem.getId())); return collectorItem; })); return collectorItemRepository.save(optCollectorItem.get()); From 4298784f0438e0904b4c9e069a2346d2505a9b21 Mon Sep 17 00:00:00 2001 From: rvema Date: Tue, 20 Apr 2021 15:34:19 -0400 Subject: [PATCH 3/3] linting --- .../com/capitalone/dashboard/collector/AuditCollectorUtil.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java index 51adb81..912555b 100644 --- a/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java +++ b/src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java @@ -667,7 +667,6 @@ protected String getAuditAPIUrl(Dashboard dashboard, AuditSettings settings, lon } catch (URISyntaxException e) { return null; } - //auditURI.setPath(settings.getServers().get(0) + HYGIEIA_AUDIT_URL); auditURI.addParameter(AUDIT_PARAMS.title.name(), dashboard.getTitle()); auditURI.addParameter(AUDIT_PARAMS.businessService.name(), dashboard.getConfigurationItemBusServName()); auditURI.addParameter(AUDIT_PARAMS.businessApplication.name(), dashboard.getConfigurationItemBusAppName()); @@ -790,7 +789,6 @@ private Double getAvgTracePercent(Stream stream) { protected CollectorItem createCollectorItem(AuditType auditType){ String description = getDescription(auditType); Pageable pageable = new PageRequest(0, 25); -// Iterable collectorItems = collectorItemRepository.findByDescription(description); Page collectorItems = collectorItemRepository.findByCollectorIdAndSearchField(Collections.singletonList(collector.getId()), OPT_OPT_URL, description, pageable); Optional optCollectorItem = Optional.ofNullable(collectorItems.iterator().hasNext() ? collectorItems.iterator().next() : null);