Skip to content
This repository has been archived by the owner on Oct 2, 2023. It is now read-only.

Commit

Permalink
Merge pull request #23 from hygieia/fix-maven-release
Browse files Browse the repository at this point in the history 
Fix the rest calls
  • Loading branch information
@vidhya9lakshmi
vidhya9lakshmi authored Apr 20, 2021
2 parents 2db9c03 + 4298784 commit 2e04209
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 51 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<description>NFRR Audit Collector</description>
<packaging>jar</packaging>
<url>https://github.com/hygieia/${repository.name}</url>
<version>3.1.7-SNAPSHOT</version>
<version>3.1.8-SNAPSHOT</version>

<parent>
<groupId>org.springframework.boot</groupId>
Expand Down
39 changes: 22 additions & 17 deletions src/main/java/com/capitalone/dashboard/collector/AuditCollectorTask.java
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,32 @@
package com.capitalone.dashboard.collector;

import com.capitalone.dashboard.client.RestClient;
import com.capitalone.dashboard.model.Dashboard;
import com.capitalone.dashboard.model.Audit;
import com.capitalone.dashboard.model.AuditResult;
import com.capitalone.dashboard.model.AuditType;
import com.capitalone.dashboard.model.DashboardType;
import com.capitalone.dashboard.model.Cmdb;

import com.capitalone.dashboard.repository.DashboardRepository;
import com.capitalone.dashboard.repository.AuditResultRepository;
import com.capitalone.dashboard.model.Dashboard;
import com.capitalone.dashboard.model.DashboardType;
import com.capitalone.dashboard.repository.AuditCollectorRepository;
import com.capitalone.dashboard.repository.CmdbRepository;
import com.capitalone.dashboard.repository.AuditResultRepository;
import com.capitalone.dashboard.repository.BaseCollectorRepository;
import com.capitalone.dashboard.repository.ComponentRepository;
import com.capitalone.dashboard.repository.CmdbRepository;
import com.capitalone.dashboard.repository.CollectorItemRepository;
import com.capitalone.dashboard.repository.ComponentRepository;
import com.capitalone.dashboard.repository.DashboardRepository;
import org.apache.commons.collections4.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.stereotype.Component;


import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

/**
* <h1>AuditCollectorTask</h1>
Expand Down Expand Up @@ -69,27 +67,34 @@ public AuditCollectorTask(TaskScheduler taskScheduler, DashboardRepository dashb

@Override
public void collect(AuditCollector collector) {
long startTime = System.currentTimeMillis();
LOGGER.info(String.format("NFRRCollectorTask:collect start collector_run_id=%d", startTime));
LOGGER.info("NFRR Audit Collector pulls all the team dashboards");
Iterable<Dashboard> dashboards = dashboardRepository.findAllByType(DashboardType.Team);

collectAuditResults(dashboards);
Iterable<Dashboard> dashboards = dashboardRepository.findAllByType(DashboardType.Team);
collectAuditResults(dashboards, startTime);
long endTime = System.currentTimeMillis();
long elapsedSeconds = (endTime-startTime)/1000;
collector.setLastExecutedSeconds(elapsedSeconds);
collector.setLastExecutionRecordCount(CollectionUtils.size(dashboards));
LOGGER.info("NFRR Audit Collector executed successfully");
LOGGER.info(String.format("NFRRCollectorTask:collect stop, collector_process_time=%d collector_item_count=%d collector_run_id=%d", elapsedSeconds, CollectionUtils.size(dashboards), startTime));
}

/**
* Collect audit results of dashboards
*
* @param dashboards
*/
protected void collectAuditResults(Iterable<Dashboard> dashboards) {
protected void collectAuditResults(Iterable<Dashboard> dashboards, long collector_run_id) {
int numberOfAuditDays = settings.getDays();
long auditBeginDateTimeStamp = Instant.now().minus(Duration.ofDays(numberOfAuditDays)).toEpochMilli();
long auditEndDateTimeStamp = Instant.now().toEpochMilli();
int totTeamDbdCount = CollectionUtils.size(dashboards);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd:HH:mm:ss");
LOGGER.info(String.format("Audit time range start=%d (%s) end=%d (%s) dashboards=%d",
LOGGER.info(String.format("Audit time range time_range_start=%d (%s) time_range_end=%d (%s) dashboard_count=%d collector_run_id=%d",
auditBeginDateTimeStamp, sdf.format(new Date(auditBeginDateTimeStamp)),
auditEndDateTimeStamp, sdf.format(new Date(auditEndDateTimeStamp)), totTeamDbdCount));
auditEndDateTimeStamp, sdf.format(new Date(auditEndDateTimeStamp)), totTeamDbdCount, collector_run_id));

AuditCollector collector = getCollectorRepository().findByName(COLLECTOR_NAME);
AuditCollectorUtil auditCollectorUtil = new AuditCollectorUtil(collector, componentRepository,
Expand All @@ -104,11 +109,11 @@ protected void collectAuditResults(Iterable<Dashboard> dashboards) {
List<AuditResult> latestAuditResults = AuditCollectorUtil.getAuditResults(dashboard, auditMap, cmdb, auditEndDateTimeStamp);
refreshAuditResults(dashboard, latestAuditResults);
} catch (Exception e) {
LOGGER.error("Exception in collecting audit result for dashboard="+dashboard.getTitle(), e);
LOGGER.error("Exception in collecting audit result for error_dashboard=" + dashboard.getTitle() +" collector_run_id=" + collector_run_id , e);
}
long endTime = System.currentTimeMillis();
LOGGER.info(String.format("Adding audit results - dashboard=%s [%d/%d] timeTaken=%d",
dashboard.getTitle(), ++index, totTeamDbdCount, endTime-startTime));
LOGGER.info(String.format("Adding audit results - dashboard=%s [%d/%d] duration=%d collector_run_id=%d",
dashboard.getTitle(), ++index, totTeamDbdCount, endTime-startTime, collector_run_id));
}
}

Expand Down
66 changes: 33 additions & 33 deletions src/main/java/com/capitalone/dashboard/collector/AuditCollectorUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,30 +2,26 @@

import com.capitalone.dashboard.client.RestClient;
import com.capitalone.dashboard.model.Audit;
import com.capitalone.dashboard.model.AuditResult;
import com.capitalone.dashboard.model.AuditStatus;
import com.capitalone.dashboard.model.DataStatus;
import com.capitalone.dashboard.model.AuditType;
import com.capitalone.dashboard.model.AuditResult;
import com.capitalone.dashboard.model.Dashboard;
import com.capitalone.dashboard.model.Cmdb;
import com.capitalone.dashboard.model.CollectorItem;
import com.capitalone.dashboard.model.Collector;
import com.capitalone.dashboard.model.CollectorItem;
import com.capitalone.dashboard.model.CollectorType;
import com.capitalone.dashboard.model.Component;

import com.capitalone.dashboard.model.Dashboard;
import com.capitalone.dashboard.model.DataStatus;
import com.capitalone.dashboard.repository.CollectorItemRepository;
import com.capitalone.dashboard.repository.ComponentRepository;


import com.capitalone.dashboard.status.ArtifactAuditStatus;
import com.capitalone.dashboard.status.CodeQualityAuditStatus;
import com.capitalone.dashboard.status.CodeReviewAuditStatus;
import com.capitalone.dashboard.status.DashboardAuditStatus;
import com.capitalone.dashboard.status.CodeQualityAuditStatus;
import com.capitalone.dashboard.status.PerformanceTestAuditStatus;
import com.capitalone.dashboard.status.DeployAuditStatus;
import com.capitalone.dashboard.status.LibraryPolicyAuditStatus;
import com.capitalone.dashboard.status.PerformanceTestAuditStatus;
import com.capitalone.dashboard.status.TestResultAuditStatus;
import com.capitalone.dashboard.status.ArtifactAuditStatus;
import com.capitalone.dashboard.status.DeployAuditStatus;

import org.apache.commons.lang3.math.NumberUtils;
import org.apache.http.client.utils.URIBuilder;
import org.bson.types.ObjectId;
Expand All @@ -37,23 +33,22 @@
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.web.PageableDefault;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.jmx.MBeanServerNotFoundException;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;

import java.util.List;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Set;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Arrays;
import java.util.Optional;
import java.util.Set;
import java.util.StringJoiner;
import java.util.Collections;

import java.util.function.Supplier;
import java.util.stream.Stream;

Expand Down Expand Up @@ -192,7 +187,7 @@ private Audit doBasicAuditCheck(JSONArray jsonArray, JSONArray global, AuditType
* Check for collector error
*/
private boolean isCollectorError(JSONArray jsonArray) {
Stream<JSONObject> jsonObjectStream = jsonArray.stream().map((Object object) -> (JSONObject) object);
Stream<JSONObject> jsonObjectStream = jsonArray.stream().map((Object object) -> object);
Stream<JSONArray> auditStatusArray = jsonObjectStream.map(jsonObject -> (JSONArray) jsonObject.get(STR_AUDITSTATUSES));
return auditStatusArray.anyMatch(aSArray -> aSArray.toJSONString().contains(DashboardAuditStatus.COLLECTOR_ITEM_ERROR.name()));
}
Expand All @@ -203,31 +198,31 @@ private boolean isCollectorError(JSONArray jsonArray) {
@SuppressWarnings("PMD.NPathComplexity")
private boolean isConfigured(AuditType auditType, JSONArray jsonArray) {
if (auditType.equals(AuditType.CODE_REVIEW)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_REPO_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_REPO_CONFIGURED.name()));
}
if (auditType.equals(AuditType.CODE_QUALITY)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_CODEQUALITY_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_CODEQUALITY_CONFIGURED.name()));
}
if (auditType.equals(AuditType.STATIC_SECURITY_ANALYSIS)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_STATIC_SECURITY_ANALYSIS_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_STATIC_SECURITY_ANALYSIS_CONFIGURED.name()));
}
if (auditType.equals(AuditType.LIBRARY_POLICY)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_LIBRARY_POLICY_ANALYSIS_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_LIBRARY_POLICY_ANALYSIS_CONFIGURED.name()));
}
if (auditType.equals(AuditType.TEST_RESULT)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_TEST_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_TEST_CONFIGURED.name()));
}
if (auditType.equals(AuditType.PERF_TEST)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_PERFORMANCE_TEST_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_PERFORMANCE_TEST_CONFIGURED.name()));
}
if (auditType.equals(AuditType.BUILD_REVIEW)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_BUILD_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_BUILD_CONFIGURED.name()));
}
if (auditType.equals(AuditType.ARTIFACT)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_ARTIFACT_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_ARTIFACT_CONFIGURED.name()));
}
if (auditType.equals(AuditType.DEPLOY)) {
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_DEPLOYMENT_SCRIPTS_CONFIGURED.name()) ? true : false);
return (jsonArray.toJSONString().contains(DashboardAuditStatus.DASHBOARD_DEPLOYMENT_SCRIPTS_CONFIGURED.name()));
}
return false;
}
Expand Down Expand Up @@ -646,6 +641,7 @@ private void updateComponent(Dashboard dashboard) {
*/
protected JSONObject parseObject(String url, AuditSettings settings){
JSONObject responseObj = null;
if(StringUtils.isEmpty(url)) return responseObj;
try {
ResponseEntity<String> response = restClient.makeRestCallGet(url, getHeaders(settings));
JSONParser jsonParser = new JSONParser();
Expand All @@ -665,8 +661,12 @@ protected String getAuditAPIUrl(Dashboard dashboard, AuditSettings settings, lon
LOGGER.error("No Server Found to run NoFearRelease audit collector");
throw new MBeanServerNotFoundException("No Server Found to run NoFearRelease audit collector");
}
URIBuilder auditURI = new URIBuilder();
auditURI.setPath(settings.getServers().get(0) + HYGIEIA_AUDIT_URL);
URIBuilder auditURI = null;
try {
auditURI = new URIBuilder(settings.getServers().get(0) + HYGIEIA_AUDIT_URL);
} catch (URISyntaxException e) {
return null;
}
auditURI.addParameter(AUDIT_PARAMS.title.name(), dashboard.getTitle());
auditURI.addParameter(AUDIT_PARAMS.businessService.name(), dashboard.getConfigurationItemBusServName());
auditURI.addParameter(AUDIT_PARAMS.businessApplication.name(), dashboard.getConfigurationItemBusAppName());
Expand Down Expand Up @@ -789,7 +789,6 @@ private Double getAvgTracePercent(Stream stream) {
protected CollectorItem createCollectorItem(AuditType auditType){
String description = getDescription(auditType);
Pageable pageable = new PageRequest(0, 25);
// Iterable<CollectorItem> collectorItems = collectorItemRepository.findByDescription(description);
Page<CollectorItem> collectorItems = collectorItemRepository.findByCollectorIdAndSearchField(Collections.singletonList(collector.getId()),
OPT_OPT_URL, description, pageable);
Optional<CollectorItem> optCollectorItem = Optional.ofNullable(collectorItems.iterator().hasNext() ? collectorItems.iterator().next() : null);
Expand All @@ -805,7 +804,8 @@ protected CollectorItem createCollectorItem(AuditType auditType){
collectorItem.setDescription(description);
collectorItem.getOptions().put(OPT_DBRD_ID, getDashboard().getId());
collectorItem.getOptions().put(OPT_URL, description);
LOGGER.info(String.format("new collector item created, description=%s", description));
LOGGER.info(String.format("new collector item created, description=%s updated=%d dashboard_id=%s audit_collector_item_id=%s",
description, collectorItem.getLastUpdated(), collectorItem.getOptions().get(OPT_DBRD_ID), collectorItem.getId()));
return collectorItem;
}));
return collectorItemRepository.save(optCollectorItem.get());
Expand Down

0 comments on commit 2e04209

Please sign in to comment.