Skip to content

Commit

Permalink
Look in specific extension path for commits
Browse files Browse the repository at this point in the history
  • Loading branch information
@holly-cummins
holly-cummins committed Oct 24, 2023
1 parent 898be25 commit 6709e1b
Show file tree
Hide file tree
Showing 4 changed files with 105 additions and 60 deletions.
34 changes: 22 additions & 12 deletions plugins/github-enricher/gatsby-node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ let getLabels
exports.onPreBootstrap = async ({}) => {
repoCache = new PersistableCache({ key: "github-api-for-repos", stdTTL: 3 * DAY_IN_SECONDS })

// The location of extension metadata files is unlikely to change often, and if it does, the link checker will flag the issue
// The location of extension files is unlikely to change often, and if it does, the link checker will flag the issue
extensionYamlCache = new PersistableCache({
key: "github-api-for-extension-metadata",
key: "github-api-for-extension-paths",
stdTTL: 10 * DAY_IN_SECONDS
})

Expand Down Expand Up @@ -222,9 +222,6 @@ const fetchGitHubInfo = async (scmUrl, groupId, artifactId, labels) => {

const scmInfo = { url: scmUrl, project }

scmInfo.sponsors = await findSponsor(coords.owner, project)
scmInfo.contributors = await getContributors(coords.owner, project)

// Always set the issuesUrl and labels since the cached one might be invalid
scmInfo.issuesUrl = issuesUrl
scmInfo.labels = labels
Expand Down Expand Up @@ -349,10 +346,14 @@ const fetchGitHubInfo = async (scmUrl, groupId, artifactId, labels) => {
scmInfo.ownerImageUrl = data?.repositoryOwner?.avatarUrl


let extensionYamlUrl

let extensionPathInRepo
if (hasMetadataFileLocationCache) {
extensionYamlUrl = extensionYamlCache.get(artifactKey)
const paths = extensionYamlCache.get(artifactKey)
const { extensionYamlUrl, extensionRootUrl } = paths
extensionPathInRepo = paths.extensionPathInRepo
scmInfo.extensionYamlUrl = extensionYamlUrl
scmInfo.extensionPathInRepo = extensionPathInRepo
scmInfo.extensionRootUrl = extensionRootUrl
} else {
const allMetaInfs = [
...(metaInfs ? metaInfs.entries : []),
Expand All @@ -370,15 +371,24 @@ const fetchGitHubInfo = async (scmUrl, groupId, artifactId, labels) => {
)
// We should only have one extension yaml - if we have more, don't guess, and if we have less, don't set anything
if (extensionYamls.length === 1) {
const extensionYamlPath = extensionYamls[0].path

// If we didn't get a branch ref from the cache or from github we're a bit stuck and will have to try again next time
if (defaultBranchRef) {
extensionYamlUrl = `${scmUrl}/blob/${defaultBranchRef.name}/${extensionYamlPath}`
extensionYamlCache.set(artifactKey, extensionYamlUrl)
const extensionYamlPath = extensionYamls[0].path
extensionPathInRepo = extensionYamlPath.replace("runtime/src/main/resources/META-INF/quarkus-extension.yaml", "")
const extensionRootUrl = `${scmUrl}/blob/${defaultBranchRef.name}/${extensionPathInRepo}`
const extensionYamlUrl = `${scmUrl}/blob/${defaultBranchRef.name}/${extensionYamlPath}`
extensionYamlCache.set(artifactKey, { extensionYamlUrl, extensionPathInRepo, extensionRootUrl })

scmInfo.extensionYamlUrl = extensionYamlUrl
scmInfo.extensionPathInRepo = extensionPathInRepo
scmInfo.extensionRootUrl = extensionRootUrl
}
}
}
scmInfo.extensionYamlUrl = extensionYamlUrl

scmInfo.sponsors = await findSponsor(coords.owner, project, extensionPathInRepo)
scmInfo.contributors = await getContributors(coords.owner, project, extensionPathInRepo)

scmInfo.owner = coords.owner

Expand Down
70 changes: 48 additions & 22 deletions plugins/github-enricher/gatsby-node.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,24 @@ describe("the github data handler", () => {
)
})

it("fills in a path for the extension", () => {
expect(createNode).toHaveBeenCalledWith(
expect.objectContaining({
extensionPathInRepo:
"some-folder-name/",
})
)
})

it("fills in a url for the extension", () => {
expect(createNode).toHaveBeenCalledWith(
expect.objectContaining({
extensionRootUrl:
"http://fake.github.com/someuser/somerepo/blob/unusual/some-folder-name/",
})
)
})

it("invokes the github graphql api", async () => {
expect(queryGraphQl).toHaveBeenCalled()
expect(queryGraphQl).toHaveBeenCalledWith(
Expand All @@ -236,7 +254,7 @@ describe("the github data handler", () => {
})

it("caches the issue count", async () => {
expect(queryGraphQl).toHaveBeenLastCalledWith(expect.stringMatching(/issues\(states:OPEN/))
expect(queryGraphQl).toHaveBeenCalledWith(expect.stringMatching(/issues\(states:OPEN/))

// Reset call counts and histories, since the code may not even do a query
jest.clearAllMocks()
Expand All @@ -253,7 +271,7 @@ describe("the github data handler", () => {
)

// But it should not ask for the issues
expect(queryGraphQl).not.toHaveBeenLastCalledWith(expect.stringMatching(/issues\(states:OPEN/))
expect(queryGraphQl).not.toHaveBeenCalledWith(expect.stringMatching(/issues\(states:OPEN/))

// It should set an issue count, even though it didn't ask for one
expect(createNode).toHaveBeenLastCalledWith(
Expand All @@ -262,7 +280,7 @@ describe("the github data handler", () => {
})

it("caches the top-level quarkus-extension.yaml", async () => {
expect(queryGraphQl).toHaveBeenLastCalledWith(expect.stringMatching(
expect(queryGraphQl).toHaveBeenCalledWith(expect.stringMatching(
/HEAD:runtime\/src\/main\/resources\/META-INF/
)
)
Expand All @@ -282,7 +300,7 @@ describe("the github data handler", () => {
)

// But it should not ask for the top-level meta-inf listing
expect(queryGraphQl).not.toHaveBeenLastCalledWith(
expect(queryGraphQl).not.toHaveBeenCalledWith(
expect.stringMatching(/HEAD:runtime\/src\/main\/resources\/META-INF/),
)

Expand All @@ -292,6 +310,8 @@ describe("the github data handler", () => {
extensionYamlUrl:
url +
"/blob/unusual/some-folder-name/runtime/src/main/resources/META-INF/quarkus-extension.yaml",
extensionPathInRepo:
"some-folder-name/",
})
)

Expand All @@ -305,7 +325,7 @@ describe("the github data handler", () => {

it("caches the quarkus-extension.yaml in subfolders", async () => {
// Sense check of what happened in beforeEach
expect(queryGraphQl).toHaveBeenLastCalledWith(
expect(queryGraphQl).toHaveBeenCalledWith(
expect.stringMatching(
/HEAD:something\/runtime\/src\/main\/resources\/META-INF/
),
Expand All @@ -326,13 +346,13 @@ describe("the github data handler", () => {

// And it should not ask for the subfolder meta-inf listing
// It possibly won't ask for anything at all
expect(queryGraphQl).not.toHaveBeenLastCalledWith(
expect(queryGraphQl).not.toHaveBeenCalledWith(
expect.stringMatching(
/HEAD:something\/runtime\/src\/main\/resources\/META-INF/
),
)

// It should set an extension descriptor path
// It should set an extension descriptor path and extension path
expect(createNode).toHaveBeenLastCalledWith(
expect.objectContaining({
extensionYamlUrl:
Expand All @@ -357,7 +377,7 @@ describe("the github data handler", () => {
{}
)

expect(queryGraphQl).toHaveBeenLastCalledWith(
expect(queryGraphQl).toHaveBeenCalledWith(
expect.stringMatching(/issues\(states:OPEN/),
)

Expand Down Expand Up @@ -543,7 +563,7 @@ describe("the github data handler", () => {
})

it("does not cache the issue count", async () => {
expect(queryGraphQl).toHaveBeenLastCalledWith(
expect(queryGraphQl).toHaveBeenCalledWith(
// This is a bit fragile with the assumptions about whitespace and a bit fiddly with the slashes, but it checks we did a query and got the project name right
expect.stringMatching(/issues\(states:OPEN/),
)
Expand All @@ -562,17 +582,17 @@ describe("the github data handler", () => {
{}
)

expect(queryGraphQl).toHaveBeenLastCalledWith(
expect(queryGraphQl).toHaveBeenCalledWith(
expect.stringMatching(/issues\(states:OPEN/),
)

expect(createNode).toHaveBeenLastCalledWith(
expect(createNode).toHaveBeenCalledWith(
expect.objectContaining({ issues: newIssueCount })
)
})

it("does not cache the labels and issue url", async () => {
expect(queryGraphQl).toHaveBeenLastCalledWith(
expect(queryGraphQl).toHaveBeenCalledWith(
// This is a bit fragile with the assumptions about whitespace and a bit fiddly with the slashes, but it checks we did a query and got the project name right
expect.stringMatching(/issues\(states:OPEN/),
)
Expand All @@ -592,17 +612,19 @@ describe("the github data handler", () => {
{}
)

expect(createNode).toHaveBeenLastCalledWith(
expect(createNode).toHaveBeenCalledWith(
expect.objectContaining({ issuesUrl: secondIssuesUrl })
)

// It should return the labels for this extension
expect(createNode).toHaveBeenLastCalledWith(
expect(createNode).toHaveBeenCalledWith(
expect.objectContaining({ labels: secondLabels })
)
})

it("caches the top-level quarkus-extension.yaml", async () => {
// Reset call counts
jest.clearAllMocks()
// Now re-trigger the invocation
await onCreateNode(
{
Expand All @@ -615,14 +637,14 @@ describe("the github data handler", () => {
)

// But it should not ask for the top-level meta-inf listing
expect(queryGraphQl).not.toHaveBeenLastCalledWith(
expect(queryGraphQl).not.toHaveBeenCalledWith(
expect.stringMatching(
/HEAD:runtime\/src\/main\/resources\/META-INF/
),
)

// It should set an extension descriptor path, even though it didn't ask for one
expect(createNode).toHaveBeenLastCalledWith(
expect(createNode).toHaveBeenCalledWith(
expect.objectContaining({
extensionYamlUrl:
url +
Expand All @@ -632,6 +654,9 @@ describe("the github data handler", () => {
})

it("caches the quarkus-extension.yaml in subfolders", async () => {
// Reset call counts
jest.clearAllMocks()

// Now re-trigger the invocation
await onCreateNode(
{
Expand All @@ -644,14 +669,14 @@ describe("the github data handler", () => {
)

// And it should not ask for the subfolder meta-inf listing
expect(queryGraphQl).not.toHaveBeenLastCalledWith(
expect(queryGraphQl).not.toHaveBeenCalledWith(
expect.stringMatching(
/HEAD:second\/runtime\/src\/main\/resources\/META-INF/
),
)

// It should set an extension descriptor path
expect(createNode).toHaveBeenLastCalledWith(
expect(createNode).toHaveBeenCalledWith(
expect.objectContaining({
extensionYamlUrl:
url +
Expand All @@ -661,13 +686,14 @@ describe("the github data handler", () => {
})

it("caches the image location", async () => {
expect(queryGraphQl).toHaveBeenLastCalledWith(
expect(queryGraphQl).toHaveBeenCalledWith(
expect.stringMatching(/issues\(states:OPEN/),
)

response.data.repository.issues.totalCount = 7

const callCount = queryGraphQl.mock.calls.length
// Reset call counts
jest.clearAllMocks()

// Now re-trigger the invocation
await onCreateNode(
Expand All @@ -680,10 +706,10 @@ describe("the github data handler", () => {
{}
)

expect(queryGraphQl).toHaveBeenCalledTimes(callCount + 1)
expect(queryGraphQl).toHaveBeenCalledTimes(1)

// We shouldn't be asking for image urls or file paths, since those are totally cacheable
expect(queryGraphQl).not.toHaveBeenLastCalledWith(
expect(queryGraphQl).not.toHaveBeenCalledWith(
expect.stringMatching(/openGraphImageUrl/),
)

Expand Down
51 changes: 26 additions & 25 deletions plugins/github-enricher/sponsorFinder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,46 +83,47 @@ const getOrSetFromCache = async (cache, key, functionThatReturnsAPromise) => {
}


const getUserContributions = async (org, project) => {
const getUserContributions = async (org, project, inPath) => {
if (org && project) {
const key = org + ":" + project
const key = org + ":" + project + inPath
return await getOrSetFromCache(
repoContributorCache,
key,
getUserContributionsNoCache.bind(this, org, project)
getUserContributionsNoCache.bind(this, org, project, inPath)
)
}
}

const getUserContributionsNoCache = async (org, project) => {
const getUserContributionsNoCache = async (org, project, inPath) => {
const path = inPath || ""
// We're only doing one, easy, date manipulation, so don't bother with a library
const timePeriodInDays = 180
const someMonthsAgo = new Date(Date.now() - timePeriodInDays * DAY_IN_MILLISECONDS).toISOString()
const query = `query {
repository(owner: "${org}", name: "${project}") {
defaultBranchRef{
target{
... on Commit{
history(since: "${someMonthsAgo}"){
edges{
node{
... on Commit{
author {
user {
login
name
company
url
... on Commit{
history(path: "${path}", since: "${someMonthsAgo}"){
edges{
node{
... on Commit{
author {
user {
login
name
company
url
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}`

const body = await queryGraphQl(query)
Expand Down Expand Up @@ -152,9 +153,9 @@ const getUserContributionsNoCache = async (org, project) => {

}

const findSponsor = async (org, project) => {
const findSponsor = async (org, project, path) => {
// Cache the github response and aggregation, but not the calculation of sponsors, since we may change the algorithm for it
const collatedHistory = await getUserContributions(org, project)
const collatedHistory = await getUserContributions(org, project, path)
if (collatedHistory) {
// We don't want to persist the sponsor calculations across builds; we could cache it locally but it's probably not worth it
return findSponsorFromContributorList(collatedHistory)
Expand All @@ -165,8 +166,8 @@ const notBot = (user) => {
return user.login && !user.login.includes("[bot]") && user.login !== "actions-user" && user.login !== "quarkiversebot"
}

const getContributors = async (org, project) => {
const collatedHistory = await getUserContributions(org, project)
const getContributors = async (org, project, path) => {
const collatedHistory = await getUserContributions(org, project, path)
return collatedHistory?.map(user => {
const { name, login, contributions, url } = user
return { name: name || login, login, contributions, url }
Expand Down
Loading

0 comments on commit 6709e1b

Please sign in to comment.