Merge pull request #35659 from github/repo-sync

Repo sync

.github/workflows/azure-preview-env-deploy-public.yml

@@ -123,6 +123,7 @@ jobs:
         uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755
         with:
           context: .
+          file: Dockerfile.azure
           push: true
           target: preview
           tags: ${{ env.DOCKER_IMAGE }}

.github/workflows/azure-preview-env-deploy.yml

@@ -174,6 +174,7 @@ jobs:
         uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755
         with:
           context: .
+          file: Dockerfile.azure
           push: true
           target: ${{ steps.with-translations.outputs.result == 'true'  && 'production' || 'preview' }}
           tags: ${{ env.DOCKER_IMAGE }}

.github/workflows/azure-prod-build-deploy.yml

@@ -104,6 +104,7 @@ jobs:
         uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755
         with:
           context: .
+          file: Dockerfile.azure
           push: true
           target: production
           tags: ${{ env.DOCKER_IMAGE }}, ${{ env.DOCKER_IMAGE_CACHE_REF }}

.github/workflows/azure-staging-build-deploy.yml

@@ -94,6 +94,7 @@ jobs:
         uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755
         with:
           context: .
+          file: Dockerfile.azure
           push: true
           target: production
           tags: ${{ env.DOCKER_IMAGE }}

.github/workflows/main-preview-docker-cache.yml

@@ -71,6 +71,7 @@ jobs:
         uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755
         with:
           context: .
+          file: Dockerfile.azure
           push: true
           target: preview
           tags: ${{ env.DOCKER_IMAGE_CACHE_REF }}

.github/workflows/moda-ci.yaml

@@ -12,47 +12,83 @@ on:
     types: [checks_requested]
 
 jobs:
+  ##########################
+  # Generate Vault keys
+  ##########################
+  set-vault-keys:
+    runs-on: ubuntu-latest
+    outputs:
+      modified_vault_keys: ${{ steps.modify_vault_keys.outputs.modified }}
+    steps:
+      - name: Set vault-keys output
+        id: modify_vault_keys
+        run: |
+          if [ -z "${{ vars.VAULT_KEYS }}" ]; then
+            # We want to add the DOCS_BOT_PAT_READPUBLICKEY to the list of keys
+            # so that builds fetch the secret from the docs-internal vault
+            # where --environment is "ci"
+            echo "modified=DOCS_BOT_PAT_READPUBLICKEY" >> $GITHUB_OUTPUT
+          else
+            echo "modified=${{ vars.VAULT_KEYS }},DOCS_BOT_PAT_READPUBLICKEY" >> $GITHUB_OUTPUT
+          fi
+
+  #############
+  # Moda jobs
+  #############
   moda-config-bundle:
     if: ${{ github.repository == 'github/docs-internal' }}
     name: ${{ matrix.ci_job.job }}
+    needs: set-vault-keys
     strategy:
       fail-fast: false
       matrix:
         ci_job: [{ 'job': 'docs-internal-moda-config-bundle' }]
     uses: github/internal-actions/.github/workflows/moda.yml@main
     with:
       ci-formatted-job-name: ${{ matrix.ci_job.job }}
-      vault-keys: ${{ vars.VAULT_KEYS }}
+      vault-keys: ${{ needs.set-vault-keys.outputs.modified_vault_keys }}
     secrets:
       dx-bot-token: ${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}
       datadog-api-key: ${{ secrets.DATADOG_API_KEY }}
 
+  #############
+  # Docker Image jobs
+  #############
   docker-image:
     if: ${{ github.repository == 'github/docs-internal' }}
     name: ${{ matrix.ci_job.job }}
+    needs: set-vault-keys
     strategy:
       fail-fast: false
       matrix:
         ci_job: [{ 'job': 'docs-internal-docker-image' }]
     uses: github/internal-actions/.github/workflows/kube.yml@main
     with:
       ci-formatted-job-name: ${{ matrix.ci_job.job }}
-      vault-keys: ${{ vars.VAULT_KEYS }}
+      vault-keys: ${{ needs.set-vault-keys.outputs.modified_vault_keys }}
+      # Passes 'DOCS_BOT_PAT_READPUBLICKEY' secret from Vault to docker as --secret id=DOCS_BOT_PAT_READPUBLICKEY,src=<PAT value>
+      docker-build-env-secrets: 'DOCS_BOT_PAT_READPUBLICKEY'
     secrets:
       dx-bot-token: ${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}
       datadog-api-key: ${{ secrets.DATADOG_API_KEY }}
 
+  #############
+  # Docker Security jobs
+  #############
   docker-security:
     if: ${{ github.repository == 'github/docs-internal' }}
     name: ${{ matrix.ci_job.job }}
+    needs: set-vault-keys
     strategy:
       fail-fast: false
       matrix:
         ci_job: [{ 'job': 'docs-internal-docker-security' }]
     uses: github/internal-actions/.github/workflows/docker_security.yml@main
     with:
       ci-formatted-job-name: ${{ matrix.ci_job.job }}
-      vault-keys: ${{ vars.VAULT_KEYS }}
+      vault-keys: ${{ needs.set-vault-keys.outputs.modified_vault_keys }}
+      # Passes 'DOCS_BOT_PAT_READPUBLICKEY' secret from Vault to docker as --secret id=DOCS_BOT_PAT_READPUBLICKEY,src=<PAT value>
+      docker-build-env-secrets: 'DOCS_BOT_PAT_READPUBLICKEY'
     secrets:
       dx-bot-token: ${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}
       datadog-api-key: ${{ secrets.DATADOG_API_KEY }}

Dockerfile

@@ -1,4 +1,6 @@
-# This Dockerfile is used for docker-based deployments to Azure for both preview environments and production
+# This Dockerfile is used solely for production deployments to Moda
+# For staging deployments, see src/deployments/staging/Dockerfile
+# For building this file locally, see src/deployments/production/README.md
 
 # --------------------------------------------------------------------------------
 # BASE IMAGE
@@ -9,104 +11,150 @@ FROM node:22-alpine@sha256:c13b26e7e602ef2f1074aef304ce6e9b7dd284c419b35d89fcf3c
 
 # This directory is owned by the node user
 ARG APP_HOME=/home/node/app
-
-# Make sure we don't run anything as the root user
-USER node
-
+RUN mkdir -p $APP_HOME && chown -R node:node $APP_HOME
 WORKDIR $APP_HOME
 
+# Switch to root to ensure we have permissions to copy, chmod, and install
+USER root
 
-# ---------------
-# ALL DEPS
-# ---------------
-FROM base AS all_deps
+# Install git for cloning docs-early-access & translations repos
+# Install curl for determining the early access branch
+RUN apk add --no-cache git curl
 
-COPY --chown=node:node package.json package-lock.json ./
+# Copy in build scripts
+COPY src/deployments/production/build-scripts/*.sh ./build-scripts/
 
-RUN npm ci --no-optional --registry https://registry.npmjs.org/
+# Make scripts executable
+RUN chmod +x build-scripts/*.sh
 
-# For Next.js v12+
-# This the appropriate necessary extra for node:VERSION-alpine
-# Other options are https://www.npmjs.com/search?q=%40next%2Fswc
-RUN npm i @next/swc-linux-x64-musl --no-save || npm i @next/swc-linux-arm64-musl --no-save
+# We need to copy over content that will be merged with early-access
+COPY content ./content
+COPY assets ./assets
+COPY data ./data
 
+# Use the mounted --secret to:
+# - 1. Fetch the docs-internal repo
+# - 2. Fetch the docs-early-access repo & override docs-internal with early access content
+# - 3. Fetch each translations repo to the repo/translations directory
+# We use --mount-type=secret to avoid the secret being copied into the image layers for security
+# The secret passed via --secret can only be used in this RUN command
+RUN --mount=type=secret,id=DOCS_BOT_PAT_READPUBLICKEY \
+    # We don't cache because Docker can't know if we need to fetch new content from remote repos
+    echo "Don't cache this step by printing date: $(date)" && \
+    . ./build-scripts/fetch-repos.sh
+
+# Give node user access to the copied content since we cloned as root
+RUN chown -R node:node $APP_HOME/content 
+RUN chown -R node:node $APP_HOME/assets
+RUN chown -R node:node $APP_HOME/data
+# Give node user access to translations repos
+RUN chown -R node:node $APP_HOME/translations
+
+# Change back to node to make sure we don't run anything as the root user
+USER node
 
 # ---------------
-# PROD DEPS
+# ALL DEPS Image
 # ---------------
-FROM all_deps AS prod_deps
+FROM base AS all_deps
 
-RUN npm prune --production
+ARG APP_HOME=/home/node/app
+USER node
+WORKDIR $APP_HOME
+
+# Copy what is needed to run npm ci
+COPY --chown=node:node package.json package-lock.json ./
 
+RUN npm ci --no-optional --registry https://registry.npmjs.org/
 
 # ---------------
-# BUILDER
+# BUILDER Image
 # ---------------
 FROM all_deps AS builder
 
-COPY src ./src
-# The star is because it's an optional directory
-COPY .remotejson-cache* ./.remotejson-cache
-# The star is because it's an optional file
-COPY .pageinfo-cache.json.br* ./.pageinfo-cache.json.br
-# Certain content is necessary for being able to build
-COPY content/index.md ./content/index.md
-COPY content/rest ./content/rest
-COPY data ./data
+ARG APP_HOME=/home/node/app
+USER node
+WORKDIR $APP_HOME
 
-COPY next.config.js ./next.config.js
-COPY tsconfig.json ./tsconfig.json
+# Copy what is needed to:
+# 1. Build the app
+# 2. run warmup-remotejson script
+# 3. run precompute-pageinfo script
+# Dependencies
+COPY --chown=node:node --from=all_deps $APP_HOME/node_modules $APP_HOME/node_modules
+# Content with merged early-access content
+COPY --chown=node:node --from=base $APP_HOME/data ./data
+COPY --chown=node:node --from=base $APP_HOME/assets ./assets
+COPY --chown=node:node --from=base $APP_HOME/content ./content
+# Source code
+COPY --chown=node:node --from=all_deps $APP_HOME/package.json ./
+COPY src ./src
+COPY next.config.js ./
+COPY tsconfig.json ./
 
+# 1. Build
 RUN npm run build
 
+# 2. Warm up the remotejson cache
+RUN npm run warmup-remotejson
+
+# 3. Precompute the pageinfo cache
+RUN npm run precompute-pageinfo -- --max-versions 2
+
+# Prune deps for prod image
+RUN npm prune --production
+
 # --------------------------------------------------------------------------------
-# PREVIEW IMAGE - no translations
+# PRODUCTION IMAGE 
 # --------------------------------------------------------------------------------
+FROM base AS production
 
-FROM base AS preview
+ARG APP_HOME=/home/node/app
+USER node
+WORKDIR $APP_HOME
 
-# Copy just prod dependencies
-COPY --chown=node:node --from=prod_deps $APP_HOME/node_modules $APP_HOME/node_modules
+# Copy the content with merged early-access content
+COPY --chown=node:node --from=base $APP_HOME/data ./data
+COPY --chown=node:node --from=base $APP_HOME/assets ./assets
+COPY --chown=node:node --from=base $APP_HOME/content ./content
 
-# Copy our front-end code
-COPY --chown=node:node --from=builder $APP_HOME/.next $APP_HOME/.next
+# Include cloned translations
+COPY --chown=node:node --from=base $APP_HOME/translations ./translations
 
-# We should always be running in production mode
-ENV NODE_ENV=production
+# Copy prod dependencies
+COPY --chown=node:node --from=builder $APP_HOME/package.json ./
+COPY --chown=node:node --from=builder $APP_HOME/node_modules $APP_HOME/node_modules
 
-# Preferred port for server.js
-ENV PORT=4000
+# Copy built artifacts needed at runtime for the server
+COPY --chown=node:node --from=builder $APP_HOME/.next $APP_HOME/.next
 
-ENV ENABLED_LANGUAGES="en"
+# Copy cache files generated during build scripts
+COPY --chown=node:node --from=builder $APP_HOME/.remotejson-cache ./.remotejson-cache
+COPY --chown=node:node --from=builder $APP_HOME/.pageinfo-cache.json.br* ./.pageinfo-cache.json.br
 
+# Copy only what's needed to run the server
+COPY --chown=node:node --from=builder $APP_HOME/src ./src
+COPY --chown=node:node --from=builder $APP_HOME/.remotejson-cache ./.remotejson-cache
+COPY --chown=node:node --from=builder $APP_HOME/.pageinfo-cache.json.br* ./.pageinfo-cache.json.br
+COPY --chown=node:node --from=builder $APP_HOME/next.config.js ./
+COPY --chown=node:node --from=builder $APP_HOME/tsconfig.json ./
+
+# - - -
+# Environment variables
+# - - -
 # This makes it possible to set `--build-arg BUILD_SHA=abc123`
 # and it then becomes available as an environment variable in the docker run.
 ARG BUILD_SHA
 ENV BUILD_SHA=$BUILD_SHA
-
-# Copy only what's needed to run the server
-COPY --chown=node:node package.json ./
-COPY --chown=node:node assets ./assets
-COPY --chown=node:node content ./content
-COPY --chown=node:node src ./src
-COPY --chown=node:node .remotejson-cache* ./.remotejson-cache
-COPY --chown=node:node .pageinfo-cache.json.br* ./.pageinfo-cache.json.br
-COPY --chown=node:node data ./data
-COPY --chown=node:node next.config.js ./
-COPY --chown=node:node tsconfig.json ./
+# We should always be running in production mode
+ENV NODE_ENV=production
+# Preferred port for server.js
+ENV PORT=4000
+# Include all languages
+ENV ENABLED_LANGUAGES="en,zh,es,pt,ru,ja,fr,de,ko"
 
 EXPOSE $PORT
 
+# Entrypoint to start the server
+# Note: Currently we have to use tsx because we have a mix of `.ts` and `.js` files with multiple import patterns
 CMD ["node_modules/.bin/tsx", "src/frame/server.ts"]
-
-# --------------------------------------------------------------------------------
-# PRODUCTION IMAGE - includes all translations
-# --------------------------------------------------------------------------------
-FROM preview AS production
-
-# Override what was set for previews
-# Make this match the default of `Object.keys(languages)` in src/languages/lib/languages.js
-ENV ENABLED_LANGUAGES "en,zh,es,pt,ru,ja,fr,de,ko"
-
-# Copy in all translations
-COPY --chown=node:node translations ./translations