From 243d648c0b8af4f7e3fa9c6fa10c58eede455ebd Mon Sep 17 00:00:00 2001 From: Holly Cummins Date: Tue, 21 Nov 2023 17:05:14 +0000 Subject: [PATCH] Add (and document) a quickly option which does not wait for the rate limiter. --- README.md | 11 +++++++++++ package.json | 1 + plugins/github-enricher/github-helper.js | 11 +++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 05f04d166e12..f128445de0a7 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,15 @@ nvm use 18 Information is more complete if a GitHub access token is provided. It should only be granted read access. Set it as an environment variable called `GITHUB_TOKEN`. (In the CI, this will be provided by the platform.) +## Caching + +The site pulls down a lot of content through the GitHub API. +A full build of the site will trigger the rate limiter several times. Each time the rate limiter is hit, the build needs to wait an hour for it to roll over. +Because of this, a fresh build could take two or three hours – be prepared! To build more quickly (but with incomplete information), use the `npm run develop:quickly` command. + +The build caches GitHub content in a cache in the `.cache-github-api/` directory, so once a build has been done, subsequent builds should be quicker. +Most cache contents have a lifespan of a few days (with some jitter so everything doesn't expire at once). + In one terminal, run tests ``` npm install @@ -41,6 +50,8 @@ In another terminal, run the site npm run develop ``` +(or `npm run develop:quickly` if you're in a hurry and don't need all the source control data) + You can then see changes live on http://localhost:8000. # Local production-like development diff --git a/package.json b/package.json index 3a8886860c23..358b68af9f7f 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "scripts": { "build": "cp ./node_modules/gatsby-page-utils/dist/apply-trailing-slash-option.js ./node_modules/gatsby-page-utils && gatsby build", "develop": "gatsby develop", + "develop:quickly": "DONT_WAIT=true gatsby develop", "format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md}\"", "start": "cp ./node_modules/gatsby-page-utils/dist/apply-trailing-slash-option.js ./node_modules/gatsby-page-utils && gatsby develop", "serve": "gatsby serve", diff --git a/plugins/github-enricher/github-helper.js b/plugins/github-enricher/github-helper.js index 234f953093b9..fdcfc2ba131c 100644 --- a/plugins/github-enricher/github-helper.js +++ b/plugins/github-enricher/github-helper.js @@ -14,6 +14,7 @@ const RATE_LIMIT_PREQUERY = `rateLimit { }` let resetTime +const allowSlowBuild = !process.env.DONT_WAIT // We can add more errors we know are non-recoverable here, which should help build times const isRecoverableError = (ghBody, params) => { @@ -46,13 +47,15 @@ async function tolerantFetch(url, params, isSuccessful, getContents) { if (!isSuccessful(ghBody) && isRecoverableError(ghBody, params)) { const responseString = JSON.stringify(ghBody) - if (responseString?.includes("RATE_LIMITED") && resetTime) { + if (allowSlowBuild && responseString?.includes("RATE_LIMITED") && resetTime) { console.warn("Hit the rate limit. Waiting until", resetTime) await waitUntil(resetTime) } - retry( - `Unsuccessful GitHub fetch for ${url} - response is ${responseString}` - ) + if (allowSlowBuild) { + retry( + `Unsuccessful GitHub fetch for ${url} - response is ${responseString}` + ) + } } return ghBody },