From 4f937ee58c9a44c35f716726fead3ea45f68cc10 Mon Sep 17 00:00:00 2001 From: bobong Date: Thu, 20 Jun 2024 17:41:52 +0800 Subject: [PATCH] [dag inspector] fixed the unittest of dag API in Starcoin Scan service --- .../org/starcoin/bean/DagInspectorBlock.java | 18 +++- .../org/starcoin/bean/DagInspectorEdge.java | 12 +++ .../bean/DagInspectorHeightGroup.java | 8 ++ .../handler/DagInspectorIndexerHandler.java | 14 +-- .../indexer/handler/SwapIndexerTest.java | 1 - .../scan/service/DagInspectorService.java | 95 +++++++++++-------- .../vo/DIBlocksAndEdgesAndHeightGroupsVo.java | 9 ++ .../scan/service/DagInspectServiceTest.java | 43 +++++++-- 8 files changed, 143 insertions(+), 57 deletions(-) diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java index 82fd261..7af5287 100644 --- a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java @@ -1,6 +1,5 @@ package org.starcoin.bean; -import java.util.List; import java.util.List; public class DagInspectorBlock { @@ -103,4 +102,21 @@ public List getMergeSetBlueIds() { public void setMergeSetBlueIds(List mergeSetBlueIds) { this.mergeSetBlueIds = mergeSetBlueIds; } + + @Override + public String toString() { + return "DagInspectorBlock{" + + "blockHash='" + blockHash + '\'' + + ", timestamp=" + timestamp + + ", parentIds=" + parentIds + + ", height=" + height + + ", daaScore=" + daaScore + + ", heightGroupIndex=" + heightGroupIndex + + ", selectedParentHash='" + selectedParentHash + '\'' + + ", color='" + color + '\'' + + ", isInVirtualSelectedParentChain=" + isInVirtualSelectedParentChain + + ", mergeSetRedIds=" + mergeSetRedIds + + ", mergeSetBlueIds=" + mergeSetBlueIds + + '}'; + } } diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java index 7d31995..08f3c0e 100644 --- a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java @@ -55,4 +55,16 @@ public Integer getToHeightGroupIndex() { public void setToHeightGroupIndex(Integer toHeightGroupIndex) { this.toHeightGroupIndex = toHeightGroupIndex; } + + @Override + public String toString() { + return "DagInspectorEdge{" + + "fromBlockHash='" + fromBlockHash + '\'' + + ", toBlockHash='" + toBlockHash + '\'' + + ", fromHeight=" + fromHeight + + ", toHeight=" + toHeight + + ", fromHeightGroupIndex=" + fromHeightGroupIndex + + ", toHeightGroupIndex=" + toHeightGroupIndex + + '}'; + } } diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java index 4191534..fa24b24 100644 --- a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java @@ -19,4 +19,12 @@ public Integer getSize() { public void setSize(Integer size) { this.size = size; } + + @Override + public String toString() { + return "DagInspectorHeightGroup{" + + "height=" + height + + ", size=" + size + + '}'; + } } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index fa43a4b..f0e5fd0 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -139,12 +139,12 @@ List buildEdgeDataFromDagBlockDataMaybeUpate( } List newDagBlocks = new ArrayList<>(); - for (DagInspectorBlock block : dagBlockMap.values()) { - if (block.getParentIds().isEmpty()) { + for (DagInspectorBlock dagBlock : dagBlockMap.values()) { + if (dagBlock.getParentIds().isEmpty()) { continue; } - for (String parentHash : block.getParentIds()) { + for (String parentHash : dagBlock.getParentIds()) { DagInspectorBlock parentDagBlock = dagBlockMap.get(parentHash); if (parentDagBlock == null) { logger.info("Parent block not found: {} ", parentHash); @@ -154,12 +154,12 @@ List buildEdgeDataFromDagBlockDataMaybeUpate( newDagBlocks.add(parentDagBlock); } DagInspectorEdge edge = new DagInspectorEdge(); - edge.setFromBlockHash(block.getBlockHash()); + edge.setFromBlockHash(dagBlock.getBlockHash()); edge.setToBlockHash(parentDagBlock.getBlockHash()); - edge.setFromHeight(block.getHeight()); + edge.setFromHeight(dagBlock.getHeight()); edge.setToHeight(parentDagBlock.getHeight()); - edge.setFromHeightGroupIndex(block.getHeightGroupIndex()); - edge.setToHeightGroupIndex(parentDagBlock.getHeightGroupIndex()); + edge.setFromHeightGroupIndex(dagBlock.getHeightGroupIndex() == null ? 0 : dagBlock.getHeightGroupIndex()); + edge.setToHeightGroupIndex(parentDagBlock.getHeightGroupIndex() == null ? 0 :parentDagBlock.getHeightGroupIndex()); edgeList.add(edge); } } diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java index 023759b..b1a343d 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.math.BigDecimal; -import java.util.Calendar; import java.util.List; import static org.starcoin.utils.DateTimeUtils.getTimeStamp; diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 67b7bbc..315eb6a 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -1,5 +1,6 @@ package org.starcoin.scan.service; +import com.alibaba.fastjson.JSONObject; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -8,8 +9,12 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.Max; +import org.elasticsearch.search.aggregations.metrics.TopHits; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +29,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; @Service @@ -38,7 +44,7 @@ public class DagInspectorService extends BaseService { private RestHighLevelClient client; - public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String network, Long startHeight, Long endHeight) { + public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String network, Long startHeight, Long endHeight) throws IOException { DIBlocksAndEdgesAndHeightGroupsVo groups = new DIBlocksAndEdgesAndHeightGroupsVo(); List blockList = getBlockList(network, startHeight, endHeight); @@ -49,16 +55,20 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String groups.setBlocks(blockList); groups.setEdges(getEdgeList(network, startHeight, endHeight)); - List heightList = + Set heightList = blockList.stream() .map(DagInspectorBlock::getHeight) - .collect(Collectors.toList()); + .collect(Collectors.toSet()); groups.setHeightGroups(getHeightGroup(network, heightList)); return groups; } - public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash(String network, String targetHash, Integer heightDifference) { + public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( + String network, + String targetHash, + Integer heightDifference + ) throws IOException { DagInspectorBlock block = getBlockWithHashFromStorage(network, targetHash); if (block == null) { throw new RuntimeException("Cannot find block by block hash"); @@ -71,7 +81,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash(String network, String tar return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } - public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Integer targetDAAScore, Integer heightDifference) { + public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Integer targetDAAScore, Integer heightDifference) throws IOException { DagInspectorBlock block = getHeightWithDAAScoreFromStorage(network, targetDAAScore); if (block == null) { throw new RuntimeException("Cannot find block by block hash"); @@ -84,8 +94,8 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Intege return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } - public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDifference) { - long endHeight = getMaxHeightFromStorage(); + public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Integer heightDifference) throws IOException { + long endHeight = getMaxHeightFromStorage(network); long startHeight = endHeight - heightDifference; if (startHeight < 0L) { startHeight = 0L; @@ -101,19 +111,14 @@ public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDiff * @param endHeight * @return */ - private List getBlockList(String network, Long startHeight, Long endHeight) { + private List getBlockList(String network, Long startHeight, Long endHeight) throws IOException { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight))); - sourceBuilder .sort("height", SortOrder.ASC); + sourceBuilder.sort("height", SortOrder.ASC); + sourceBuilder.size(endHeight.intValue() - startHeight.intValue()); searchRequest.source(sourceBuilder); - SearchResponse searchResponse; - try { - searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - logger.error("getEdgeList failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); - return null; - } + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); return result.getContents(); } @@ -152,13 +157,11 @@ List getEdgeList(String network, Long startHeight, Long endHei * @param heights * @return */ - List getHeightGroup(String network, List heights) { + List getHeightGroup(String network, Set heights) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (Long height : heights) { - boolQueryBuilder.should(QueryBuilders.termQuery("height", height)); - } - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(QueryBuilders.termsQuery("height", heights)); + sourceBuilder.size(heights.size()); searchRequest.source(sourceBuilder); SearchResponse searchResponse; try { @@ -175,27 +178,35 @@ public DIAppConfigVo getAppConfig() { return new DIAppConfigVo(); } - private Long getMaxHeightFromStorage() { - final String MAX_HEIGHT_FIELD = "max_height"; - final String HEIGHT_FIELD = "height"; - - try { - SearchRequest searchRequest = new SearchRequest(Constant.DAG_INSPECTOR_NODE); - - // Build the SearchSourceBuilder with max aggregation - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.aggregation(AggregationBuilders.max(MAX_HEIGHT_FIELD).field(HEIGHT_FIELD)); - searchRequest.source(sourceBuilder); - - // Execute the search request - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - - // Handle the response - Max maxHeight = searchResponse.getAggregations().get("max_height"); - return (long) maxHeight.getValue(); + private Long getMaxHeightFromStorage(String network) throws IOException { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); - } catch (IOException e) { - e.printStackTrace(); + // Build the SearchSourceBuilder with max aggregation + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + TermsAggregationBuilder maxHeightAgg = AggregationBuilders + .terms("max_height_agg") + .field("height") + .size(1); + TopHitsAggregationBuilder topHitsAgg = AggregationBuilders + .topHits("top_hits") + .size(1); + + maxHeightAgg.subAggregation(topHitsAgg); + searchSourceBuilder.aggregation(maxHeightAgg); + searchRequest.source(searchSourceBuilder); + + // Execute the search request + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + Terms maxHeightTerms = searchResponse.getAggregations().get("max_height_agg"); + if (maxHeightTerms.getBuckets().size() > 0) { + Terms.Bucket bucket = maxHeightTerms.getBuckets().get(0); + TopHits topHits = bucket.getAggregations().get("top_hits"); + if (topHits.getHits().getHits().length > 0) { + String topHitsJson = topHits.getHits().getHits()[0].getSourceAsString(); + logger.info("Object with the max height: " + topHitsJson); + DagInspectorBlock block = JSONObject.parseObject(topHitsJson, DagInspectorBlock.class); + return block.getHeight(); + } } return 0L; } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java index e93975c..1ae3ddd 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java @@ -37,4 +37,13 @@ public List getHeightGroups() { public void setHeightGroups(List heightGroups) { this.heightGroups = heightGroups; } + + @Override + public String toString() { + return "DIBlocksAndEdgesAndHeightGroupsVo{" + + "blocks=" + blocks + + ", edges=" + edges + + ", heightGroups=" + heightGroups + + '}'; + } } \ No newline at end of file diff --git a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java index 4ecc748..4ce0a5d 100644 --- a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java +++ b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java @@ -2,17 +2,16 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; import org.starcoin.scan.ScanApplication; import org.starcoin.scan.service.vo.DIBlocksAndEdgesAndHeightGroupsVo; -@RunWith(SpringRunner.class) +import java.io.IOException; + @SpringBootTest(classes = ScanApplication.class) -@TestPropertySource(locations = "classpath:application.properties") +@TestPropertySource(locations = "classpath:application-unittest.properties") public class DagInspectServiceTest { final static String TEST_NETWORK = "halley"; @@ -21,10 +20,42 @@ public class DagInspectServiceTest { DagInspectorService dagInspectorService; @Test - public void testGetBlocksAndEdgesAndHeightGroups() { - DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlocksAndEdgesAndHeightGroups(TEST_NETWORK, 0L, 100L); + public void testGetBlocksAndEdgesAndHeightGroups() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlocksAndEdgesAndHeightGroups(TEST_NETWORK, 10000L, 10050L); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + } + + @Test + public void testGetBlockHash() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockHash( + TEST_NETWORK, + "0x93a4fc71929be2e435efe682d02260f8dd46824fe90e926e3b3ea5839f31e67c", + 10 + ); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); + } + + @Test + public void testGetBlockDAAScore() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockDAAScore(TEST_NETWORK, 10409, 1); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); + } + + + @Test + public void testGetHead() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getHead(TEST_NETWORK, 100); Assertions.assertNotEquals(0, vo.getBlocks().size()); Assertions.assertNotEquals(0, vo.getEdges().size()); Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); } }