From b1e1841de25106f6eae542a34581363cbc6c41c7 Mon Sep 17 00:00:00 2001 From: Phodal Huang Date: Mon, 8 Jan 2024 19:54:09 +0800 Subject: [PATCH] refactor: extract for show logs --- README.md | 2 +- docs/home.md | 2 +- .../cc/unitmesh/pick/threshold/Filter.kt | 11 ++++++- .../cc/unitmesh/pick/threshold/Pipeline.kt | 9 ++++-- .../pick/threshold/ThresholdChecker.kt | 29 ++++++++++++------- 5 files changed, 37 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7972852..a67f89c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

UnitGen

- Chapi Logo + UnitGen Logo

diff --git a/docs/home.md b/docs/home.md index 9fbcf51..c1d71e9 100644 --- a/docs/home.md +++ b/docs/home.md @@ -8,7 +8,7 @@ permalink: /

UnitGen

- Chapi Logo + UnitGen Logo

diff --git a/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/Filter.kt b/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/Filter.kt index e27c1eb..b016aa7 100644 --- a/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/Filter.kt +++ b/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/Filter.kt @@ -1,5 +1,14 @@ package cc.unitmesh.pick.threshold +import kotlinx.serialization.Serializable + +@Serializable +data class FilterResult( + val result: Boolean, + val reason: String = "", + val isCritical: Boolean = false, +) + interface Filter { - fun filter(data: T): Boolean + fun filter(data: T): FilterResult } \ No newline at end of file diff --git a/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/Pipeline.kt b/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/Pipeline.kt index 075c5a1..725249a 100644 --- a/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/Pipeline.kt +++ b/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/Pipeline.kt @@ -14,10 +14,13 @@ class Pipeline { fun process(data: T): Boolean { return filters.all { val result = it.filter(data) - if (!result) { - logger.info("not met filter: ${it.javaClass.simpleName}") + if (!result.result) { + if (result.isCritical) { + logger.info("not met filter: ${it.javaClass.simpleName}, reason: ${result.reason}") + } } - result + + result.result } } } \ No newline at end of file diff --git a/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/ThresholdChecker.kt b/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/ThresholdChecker.kt index 97bf070..f838e50 100644 --- a/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/ThresholdChecker.kt +++ b/unit-picker/src/main/kotlin/cc/unitmesh/pick/threshold/ThresholdChecker.kt @@ -67,26 +67,31 @@ class ExtensionFilter(private val qualityThreshold: InsQualityThreshold) : Filte language.getExtension(it.extension) }.toSet() - override fun filter(data: FileSummary): Boolean { - return supportedExtensions.contains(data.extension) + override fun filter(data: FileSummary): FilterResult { + val ext = data.extension + return FilterResult(supportedExtensions.contains(ext), "extension not supported: $ext") } } class ComplexityFilter(private val qualityThreshold: InsQualityThreshold) : Filter { - override fun filter(data: FileSummary): Boolean { - return data.complexity <= qualityThreshold.complexity + override fun filter(data: FileSummary): FilterResult { + return FilterResult( + data.complexity <= qualityThreshold.complexity, + "complexity too high: ${data.complexity}", + true + ) } } class BinaryGeneratedMinifiedFilter(qualityThreshold: InsQualityThreshold) : Filter { - override fun filter(data: FileSummary): Boolean { - return !(data.binary || data.generated || data.minified) + override fun filter(data: FileSummary): FilterResult { + return FilterResult(!(data.binary || data.generated || data.minified), "binary or generated or minified", true) } } class SizeFilter(private val qualityThreshold: InsQualityThreshold) : Filter { - override fun filter(data: FileSummary): Boolean { - return data.bytes <= qualityThreshold.fileSize + override fun filter(data: FileSummary): FilterResult { + return FilterResult(data.bytes <= qualityThreshold.fileSize, "file size too large: ${data.bytes}", true) } } @@ -94,10 +99,14 @@ class TokenLengthFilter(private val qualityThreshold: InsQualityThreshold) : Fil private var registry: EncodingRegistry = Encodings.newDefaultEncodingRegistry() private var enc: Encoding = registry.getEncoding(EncodingType.CL100K_BASE) - override fun filter(data: FileSummary): Boolean { + override fun filter(data: FileSummary): FilterResult { val encoded = enc.encode(data.content.toString()) val length = encoded.size val codeWithBuffer = 1.25 - return length <= qualityThreshold.maxTokenLength * codeWithBuffer + return FilterResult( + length <= qualityThreshold.maxTokenLength * codeWithBuffer, + "token length too long: $length", + true + ) } }