From d0e5baccc6bf25f85a95751ae1f3e82684860d13 Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:59:19 +0900 Subject: [PATCH 1/6] fix: relevances are not sent to OpenAI if empty --- .../formatting-evaluator-config.ts | 1 + src/parser/content-evaluator-module.ts | 8 +- src/parser/github-comment-module.ts | 4 + .../results/valid-configuration.json | 90 ++++++++----------- 4 files changed, 45 insertions(+), 58 deletions(-) diff --git a/src/configuration/formatting-evaluator-config.ts b/src/configuration/formatting-evaluator-config.ts index 0a9161bd..0d8d816e 100644 --- a/src/configuration/formatting-evaluator-config.ts +++ b/src/configuration/formatting-evaluator-config.ts @@ -32,6 +32,7 @@ const htmlType = Type.Record(Type.String(), Type.Number(), { ul: 1, td: 1, hr: 0, + pre: 1, }, }); diff --git a/src/parser/content-evaluator-module.ts b/src/parser/content-evaluator-module.ts index ec306e5c..c34ca8eb 100644 --- a/src/parser/content-evaluator-module.ts +++ b/src/parser/content-evaluator-module.ts @@ -18,7 +18,7 @@ import { GithubCommentScore, Module, Result } from "./processor"; * Evaluates and rates comments. */ export class ContentEvaluatorModule implements Module { - readonly _openAi = new OpenAI({ apiKey: OPENAI_API_KEY }); + readonly _openAi = new OpenAI({ apiKey: OPENAI_API_KEY, baseURL: "https://api.groq.com/openai/v1" }); readonly _configuration: ContentEvaluatorConfiguration | null = configuration.incentives.contentEvaluator; private readonly _fixedRelevances: { [k: string]: number } = {}; @@ -86,7 +86,9 @@ export class ContentEvaluatorModule implements Module { } } - const relevancesByAI = await this._evaluateComments(specificationBody, commentsToEvaluate); + const relevancesByAI = commentsToEvaluate.length + ? await this._evaluateComments(specificationBody, commentsToEvaluate) + : {}; if (Object.keys(relevancesByAI).length !== commentsToEvaluate.length) { console.error("Relevance / Comment length mismatch! \nWill use 1 as relevance for missing comments."); @@ -137,7 +139,7 @@ export class ContentEvaluatorModule implements Module { const maxTokens = this._calculateMaxTokens(dummyResponse); const response: OpenAI.Chat.ChatCompletion = await this._openAi.chat.completions.create({ - model: "gpt-4o-2024-08-06", + model: "llama3-groq-70b-8192-tool-use-preview", response_format: { type: "json_object" }, messages: [ { diff --git a/src/parser/github-comment-module.ts b/src/parser/github-comment-module.ts index da9ef3a5..0aec633c 100644 --- a/src/parser/github-comment-module.ts +++ b/src/parser/github-comment-module.ts @@ -77,6 +77,10 @@ export class GithubCommentModule implements Module { async postComment(body: string, updateLastComment = true) { const { eventPayload } = program; + if (!this._configuration?.post) { + logger.debug("Won't post a comment since posting is disabled.", { body }); + return; + } if (updateLastComment && this._lastCommentId !== null) { await getOctokitInstance().issues.updateComment({ body, diff --git a/tests/__mocks__/results/valid-configuration.json b/tests/__mocks__/results/valid-configuration.json index f0d3108c..aa751eac 100644 --- a/tests/__mocks__/results/valid-configuration.json +++ b/tests/__mocks__/results/valid-configuration.json @@ -11,33 +11,23 @@ "contentEvaluator": { "multipliers": [ { - "role": [ - "ISSUE_SPECIFICATION" - ], + "role": ["ISSUE_SPECIFICATION"], "relevance": 1 }, { - "role": [ - "PULL_AUTHOR" - ], + "role": ["PULL_AUTHOR"], "relevance": 1 }, { - "role": [ - "PULL_ASSIGNEE" - ], + "role": ["PULL_ASSIGNEE"], "relevance": 1 }, { - "role": [ - "PULL_COLLABORATOR" - ], + "role": ["PULL_COLLABORATOR"], "relevance": 1 }, { - "role": [ - "PULL_CONTRIBUTOR" - ], + "role": ["PULL_CONTRIBUTOR"], "relevance": 1 } ] @@ -50,9 +40,7 @@ "multipliers": [ { "multiplier": 1, - "role": [ - "ISSUE_SPECIFICATION" - ], + "role": ["ISSUE_SPECIFICATION"], "rewards": { "regex": { "\\b\\w+\\b": 0.1 @@ -75,15 +63,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 1, - "role": [ - "ISSUE_AUTHOR" - ], + "role": ["ISSUE_AUTHOR"], "rewards": { "regex": { "\\b\\w+\\b": 0.2 @@ -106,15 +93,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 0, - "role": [ - "ISSUE_ASSIGNEE" - ], + "role": ["ISSUE_ASSIGNEE"], "rewards": { "regex": { "\\b\\w+\\b": 0 @@ -137,15 +123,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 1, - "role": [ - "ISSUE_COLLABORATOR" - ], + "role": ["ISSUE_COLLABORATOR"], "rewards": { "regex": { "\\b\\w+\\b": 0.1 @@ -168,15 +153,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 0.25, - "role": [ - "ISSUE_CONTRIBUTOR" - ], + "role": ["ISSUE_CONTRIBUTOR"], "rewards": { "regex": { "\\b\\w+\\b": 0.1 @@ -199,15 +183,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 0, - "role": [ - "PULL_SPECIFICATION" - ], + "role": ["PULL_SPECIFICATION"], "rewards": { "regex": { "\\b\\w+\\b": 0 @@ -230,15 +213,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 2, - "role": [ - "PULL_AUTHOR" - ], + "role": ["PULL_AUTHOR"], "rewards": { "regex": { "\\b\\w+\\b": 0.2 @@ -261,15 +243,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 1, - "role": [ - "PULL_ASSIGNEE" - ], + "role": ["PULL_ASSIGNEE"], "rewards": { "regex": { "\\b\\w+\\b": 0.1 @@ -292,15 +273,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 1, - "role": [ - "PULL_COLLABORATOR" - ], + "role": ["PULL_COLLABORATOR"], "rewards": { "regex": { "\\b\\w+\\b": 0.1 @@ -323,15 +303,14 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } }, { "multiplier": 0.25, - "role": [ - "PULL_CONTRIBUTOR" - ], + "role": ["PULL_CONTRIBUTOR"], "rewards": { "regex": { "\\b\\w+\\b": 0.1 @@ -354,7 +333,8 @@ "ul": 1, "p": 1, "strong": 0, - "td": 1 + "td": 1, + "pre": 1 } } } From bab8e3a61b03eb54bbb7d5288cbaeae67a41171c Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:15:46 +0900 Subject: [PATCH 2/6] feat: added configuration for OpenAI settings --- src/configuration/content-evaluator-config.ts | 15 +++++++++++++++ src/parser/content-evaluator-module.ts | 7 +++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/configuration/content-evaluator-config.ts b/src/configuration/content-evaluator-config.ts index 6e60a35a..91a155c8 100644 --- a/src/configuration/content-evaluator-config.ts +++ b/src/configuration/content-evaluator-config.ts @@ -1,7 +1,22 @@ import { Static, Type } from "@sinclair/typebox"; import { commentType } from "./formatting-evaluator-config"; +const openAiType = Type.Object( + { + /** + * AI model to use for comment evaluation. + */ + model: Type.String({ default: "gpt-4o" }), + /** + * Specific endpoint to send the comments to. + */ + endpoint: Type.String({ default: "" }), + }, + { default: {} } +); + export const contentEvaluatorConfigurationType = Type.Object({ + openAi: openAiType, /** * Multipliers applied to different types of comments */ diff --git a/src/parser/content-evaluator-module.ts b/src/parser/content-evaluator-module.ts index c34ca8eb..56a0cbb2 100644 --- a/src/parser/content-evaluator-module.ts +++ b/src/parser/content-evaluator-module.ts @@ -18,8 +18,11 @@ import { GithubCommentScore, Module, Result } from "./processor"; * Evaluates and rates comments. */ export class ContentEvaluatorModule implements Module { - readonly _openAi = new OpenAI({ apiKey: OPENAI_API_KEY, baseURL: "https://api.groq.com/openai/v1" }); readonly _configuration: ContentEvaluatorConfiguration | null = configuration.incentives.contentEvaluator; + readonly _openAi = new OpenAI({ + apiKey: OPENAI_API_KEY, + ...(this._configuration?.openAi.endpoint && { baseURL: this._configuration.openAi.endpoint }), + }); private readonly _fixedRelevances: { [k: string]: number } = {}; _getEnumValue(key: CommentType) { @@ -139,7 +142,7 @@ export class ContentEvaluatorModule implements Module { const maxTokens = this._calculateMaxTokens(dummyResponse); const response: OpenAI.Chat.ChatCompletion = await this._openAi.chat.completions.create({ - model: "llama3-groq-70b-8192-tool-use-preview", + model: this._configuration?.openAi.model || "gpt-4o-2024-08-06", response_format: { type: "json_object" }, messages: [ { From b7d1fdb1d2728ee2e7fb5262a8829de980838abd Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Tue, 27 Aug 2024 18:57:24 +0900 Subject: [PATCH 3/6] chore: fixed unit test --- .../results/valid-configuration.json | 176 +++++++++--------- 1 file changed, 90 insertions(+), 86 deletions(-) diff --git a/tests/__mocks__/results/valid-configuration.json b/tests/__mocks__/results/valid-configuration.json index aa751eac..d02b48bd 100644 --- a/tests/__mocks__/results/valid-configuration.json +++ b/tests/__mocks__/results/valid-configuration.json @@ -7,44 +7,40 @@ "evmNetworkId": 100, "evmPrivateEncrypted": "kmpTKq5Wh9r9x5j3U9GqZr3NYnjK2g0HtbzeUBOuLC2y3x8ja_SKBNlB2AZ6LigXHP_HeMitftVUtzmoj8CFfVP9SqjWoL6IPku1hVTWkdTn97g1IxzmjydFxjdcf0wuDW1hvVtoq3Uw5yALABqxcQ", "incentives": { - "requirePriceLabel": true, "contentEvaluator": { "multipliers": [ { - "role": ["ISSUE_SPECIFICATION"], - "relevance": 1 + "relevance": 1, + "role": ["ISSUE_SPECIFICATION"] }, { - "role": ["PULL_AUTHOR"], - "relevance": 1 + "relevance": 1, + "role": ["PULL_AUTHOR"] }, { - "role": ["PULL_ASSIGNEE"], - "relevance": 1 + "relevance": 1, + "role": ["PULL_ASSIGNEE"] }, { - "role": ["PULL_COLLABORATOR"], - "relevance": 1 + "relevance": 1, + "role": ["PULL_COLLABORATOR"] }, { - "role": ["PULL_CONTRIBUTOR"], - "relevance": 1 + "relevance": 1, + "role": ["PULL_CONTRIBUTOR"] } - ] - }, - "userExtractor": { - "redeemTask": true + ], + "openAi": { + "endpoint": "", + "model": "gpt-4o" + } }, "dataPurge": {}, "formattingEvaluator": { "multipliers": [ { "multiplier": 1, - "role": ["ISSUE_SPECIFICATION"], "rewards": { - "regex": { - "\\b\\w+\\b": 0.1 - }, "html": { "a": 1, "blockquote": 0, @@ -60,21 +56,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0.1 } - } + }, + "role": ["ISSUE_SPECIFICATION"] }, { "multiplier": 1, - "role": ["ISSUE_AUTHOR"], "rewards": { - "regex": { - "\\b\\w+\\b": 0.2 - }, "html": { "a": 1, "blockquote": 0, @@ -90,21 +86,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0.2 } - } + }, + "role": ["ISSUE_AUTHOR"] }, { "multiplier": 0, - "role": ["ISSUE_ASSIGNEE"], "rewards": { - "regex": { - "\\b\\w+\\b": 0 - }, "html": { "a": 1, "blockquote": 0, @@ -120,21 +116,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0 } - } + }, + "role": ["ISSUE_ASSIGNEE"] }, { "multiplier": 1, - "role": ["ISSUE_COLLABORATOR"], "rewards": { - "regex": { - "\\b\\w+\\b": 0.1 - }, "html": { "a": 1, "blockquote": 0, @@ -150,21 +146,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0.1 } - } + }, + "role": ["ISSUE_COLLABORATOR"] }, { "multiplier": 0.25, - "role": ["ISSUE_CONTRIBUTOR"], "rewards": { - "regex": { - "\\b\\w+\\b": 0.1 - }, "html": { "a": 1, "blockquote": 0, @@ -180,21 +176,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0.1 } - } + }, + "role": ["ISSUE_CONTRIBUTOR"] }, { "multiplier": 0, - "role": ["PULL_SPECIFICATION"], "rewards": { - "regex": { - "\\b\\w+\\b": 0 - }, "html": { "a": 1, "blockquote": 0, @@ -210,21 +206,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0 } - } + }, + "role": ["PULL_SPECIFICATION"] }, { "multiplier": 2, - "role": ["PULL_AUTHOR"], "rewards": { - "regex": { - "\\b\\w+\\b": 0.2 - }, "html": { "a": 1, "blockquote": 0, @@ -240,21 +236,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0.2 } - } + }, + "role": ["PULL_AUTHOR"] }, { "multiplier": 1, - "role": ["PULL_ASSIGNEE"], "rewards": { - "regex": { - "\\b\\w+\\b": 0.1 - }, "html": { "a": 1, "blockquote": 0, @@ -270,21 +266,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0.1 } - } + }, + "role": ["PULL_ASSIGNEE"] }, { "multiplier": 1, - "role": ["PULL_COLLABORATOR"], "rewards": { - "regex": { - "\\b\\w+\\b": 0.1 - }, "html": { "a": 1, "blockquote": 0, @@ -300,21 +296,21 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0.1 } - } + }, + "role": ["PULL_COLLABORATOR"] }, { "multiplier": 0.25, - "role": ["PULL_CONTRIBUTOR"], "rewards": { - "regex": { - "\\b\\w+\\b": 0.1 - }, "html": { "a": 1, "blockquote": 0, @@ -330,13 +326,17 @@ "hr": 0, "img": 0, "li": 1, - "ul": 1, "p": 1, + "pre": 1, "strong": 0, "td": 1, - "pre": 1 + "ul": 1 + }, + "regex": { + "\\b\\w+\\b": 0.1 } - } + }, + "role": ["PULL_CONTRIBUTOR"] } ] }, @@ -344,6 +344,10 @@ "debug": true, "post": false }, - "permitGeneration": {} + "permitGeneration": {}, + "requirePriceLabel": true, + "userExtractor": { + "redeemTask": true + } } } From b85d7accd45f420da230981e0ab534991e81ca23 Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:34:29 +0900 Subject: [PATCH 4/6] chore: updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 103231f4..bfaabd59 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,9 @@ with: incentives: requirePriceLabel: true contentEvaluator: + openAi: + model: "gpt-4o" + endpoint: "https://api.openai.com/v1" multipliers: - role: [ISSUE_SPECIFICATION] relevance: 1 From 56175197f8a13e14698cac9314f2903dd8b27c28 Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Tue, 27 Aug 2024 21:07:43 +0900 Subject: [PATCH 5/6] chore: set pre to 0 --- .../formatting-evaluator-config.ts | 2 +- .../results/valid-configuration.json | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/configuration/formatting-evaluator-config.ts b/src/configuration/formatting-evaluator-config.ts index 0d8d816e..30368c12 100644 --- a/src/configuration/formatting-evaluator-config.ts +++ b/src/configuration/formatting-evaluator-config.ts @@ -32,7 +32,7 @@ const htmlType = Type.Record(Type.String(), Type.Number(), { ul: 1, td: 1, hr: 0, - pre: 1, + pre: 0, }, }); diff --git a/tests/__mocks__/results/valid-configuration.json b/tests/__mocks__/results/valid-configuration.json index d02b48bd..9ad69d3f 100644 --- a/tests/__mocks__/results/valid-configuration.json +++ b/tests/__mocks__/results/valid-configuration.json @@ -57,7 +57,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -87,7 +87,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -117,7 +117,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -147,7 +147,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -177,7 +177,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -207,7 +207,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -237,7 +237,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -267,7 +267,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -297,7 +297,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 @@ -327,7 +327,7 @@ "img": 0, "li": 1, "p": 1, - "pre": 1, + "pre": 0, "strong": 0, "td": 1, "ul": 1 From 91f08b2518f9282670a1e467f614edeeb8c6e636 Mon Sep 17 00:00:00 2001 From: gentlementlegen Date: Tue, 27 Aug 2024 23:17:55 +0900 Subject: [PATCH 6/6] chore: fixed octokit type --- src/octokit.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/octokit.ts b/src/octokit.ts index 7bad219e..2d6a35eb 100644 --- a/src/octokit.ts +++ b/src/octokit.ts @@ -2,11 +2,11 @@ import { Octokit } from "@octokit/rest"; import { retry } from "@octokit/plugin-retry"; import program from "./parser/command-line"; import configuration from "./configuration/config-reader"; -import { paginateGraphQL, paginateGraphQLInterface } from "@octokit/plugin-paginate-graphql"; +import { paginateGraphQL } from "@octokit/plugin-paginate-graphql"; const customOctokit = Octokit.plugin(retry, paginateGraphQL); -let octokitInstance: (Octokit & paginateGraphQLInterface) | null = null; +let octokitInstance: InstanceType | null = null; function getOctokitInstance() { if (!octokitInstance) {