Improve usability of caches

.github/workflows/build_and_publish.yml

@@ -16,7 +16,7 @@ defaults:
 
 
 concurrency:
-  group: ${{ github.workflow }} # do not allow any concurrency or the different builds will fight for the github rate limit and all take far longer
+  group: uses-github-api # do not allow any concurrency or the different builds will fight for the github rate limit and all take far longer
 
 jobs:
   unit-test:

.github/workflows/check-external-links.yml

@@ -1,10 +1,7 @@
 name: Check external links
 
 on:
-  push:
-    branches:
-      [ main ]
-  schedule: ## Do a run once times daily, to catch new regressions
+  schedule: ## Do a run once daily, to catch new regressions
     - cron: '45 10 * * *'
   workflow_dispatch:
 
@@ -13,7 +10,7 @@ defaults:
     shell: bash
 
 concurrency:
-  group: ${{ github.workflow }} # do not allow any concurrency or the different builds will risk creating multiple issues
+  group: uses-github-api # do not allow any concurrency or the different builds will risk creating multiple issues
 
 jobs:
   build:

README.md

@@ -30,6 +30,15 @@ nvm use 18
 Information is more complete if a GitHub access token is provided. It should only be granted read access. 
 Set it as an environment variable called `GITHUB_TOKEN`. (In the CI, this will be provided by the platform.)
 
+## Caching 
+
+The site pulls down a lot of content through the GitHub API. 
+A full build of the site will trigger the rate limiter several times. Each time the rate limiter is hit, the build needs to wait an hour for it to roll over.
+Because of this, a fresh build could take two or three hours – be prepared! To build more quickly (but with incomplete information), use the `npm run develop:quickly` command.
+
+The build caches GitHub content in a cache in the `.cache-github-api/` directory, so once a build has been done, subsequent builds should be quicker. 
+Most cache contents have a lifespan of a few days (with some jitter so everything doesn't expire at once).
+
 In one terminal, run tests
 ```
 npm install
@@ -41,6 +50,8 @@ In another terminal, run the site
 npm run develop
 ```
 
+(or `npm run develop:quickly` if you're in a hurry and don't need all the source control data)
+
 You can then see changes live on http://localhost:8000. 
 
 # Local production-like development 

package.json

@@ -91,6 +91,7 @@
   "scripts": {
     "build": "cp ./node_modules/gatsby-page-utils/dist/apply-trailing-slash-option.js ./node_modules/gatsby-page-utils && gatsby build",
     "develop": "gatsby develop",
+    "develop:quickly": "DONT_WAIT=true gatsby develop",
     "format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md}\"",
     "start": "cp ./node_modules/gatsby-page-utils/dist/apply-trailing-slash-option.js ./node_modules/gatsby-page-utils && gatsby develop",
     "serve": "gatsby serve",

plugins/github-enricher/gatsby-node.js

@@ -206,7 +206,7 @@ const createContributingCompanies = async ({ actions, createNodeId, createConten
     }, { sponsor: company, source: extensionCatalogContributingCompany }))
 
   } else {
-    console.warn("Could not fetch sponsor opt in information from", url, ". Does the file exist?")
+    console.warn("Could not fetch sponsor opt in information from", org, repo, path, ". Does the file exist?")
   }
 }
 
@@ -589,6 +589,7 @@ exports.createSchemaCustomization = ({ actions }) => {
     url: String
     ownerImageUrl: String
     extensionYamlUrl: String
+    extensionRootUrl: String
     issues: String
     lastUpdated: String
     contributors: [ContributorInfo]

plugins/github-enricher/github-helper.js

@@ -14,6 +14,7 @@ const RATE_LIMIT_PREQUERY = `rateLimit {
   }`
 
 let resetTime
+const allowSlowBuild = !process.env.DONT_WAIT
 
 // We can add more errors we know are non-recoverable here, which should help build times
 const isRecoverableError = (ghBody, params) => {
@@ -46,13 +47,15 @@ async function tolerantFetch(url, params, isSuccessful, getContents) {
         if (!isSuccessful(ghBody) && isRecoverableError(ghBody, params)) {
           const responseString = JSON.stringify(ghBody)
 
-          if (responseString?.includes("RATE_LIMITED") && resetTime) {
+          if (allowSlowBuild && responseString?.includes("RATE_LIMITED") && resetTime) {
             console.warn("Hit the rate limit. Waiting until", resetTime)
             await waitUntil(resetTime)
           }
-          retry(
-            `Unsuccessful GitHub fetch for ${url} - response is ${responseString}`
-          )
+          if (allowSlowBuild) {
+            retry(
+              `Unsuccessful GitHub fetch for ${url} - response is ${responseString}`
+            )
+          }
         }
         return ghBody
       },