From 13599930cb314ca9335fbb9dcb31acc4342f748d Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 26 Jun 2024 00:51:40 +0200 Subject: [PATCH 1/6] ci: init directory This creates a new directory for all CI support files, will be populated in future commits. --- .github/CODEOWNERS | 3 ++- ci/README.md | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 ci/README.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 35f1bacbf5cff..89d8d57cd73df 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,8 +11,9 @@ # This also holds true for GitHub teams. Since almost none of our teams have write # permissions, you need to list all members of the team with commit access individually. -# GitHub actions +# CI /.github/workflows @NixOS/Security @Mic92 @zowoq +/ci @infinisil # EditorConfig /.editorconfig @Mic92 @zowoq diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000000000..79dc4b3bce714 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,4 @@ +# CI support files + +This directory contains files to support CI, such as [GitHub Actions](https://github.com/NixOS/nixpkgs/tree/master/.github/workflows) and [Ofborg](https://github.com/nixos/ofborg). +This is in contrast with [`maintainers/scripts`](`../maintainers/scripts`) which is for human use instead. From b33ac05d043c03cf98397e5f026ee5a93b781199 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 26 Jun 2024 00:52:13 +0200 Subject: [PATCH 2/6] ci: reusable Nix format Nixpkgs pin This is needed such that in the next commit, we can re-use the same version from a shell.nix, allowing people to have a guaranteed matching nixfmt version. --- .github/workflows/check-nix-format.yml | 11 ++++++++--- ci/README.md | 8 ++++++++ ci/pinned-nixpkgs.json | 4 ++++ ci/update-pinned-nixpkgs.sh | 16 ++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 ci/pinned-nixpkgs.json create mode 100755 ci/update-pinned-nixpkgs.sh diff --git a/.github/workflows/check-nix-format.yml b/.github/workflows/check-nix-format.yml index 131803213cb57..5a3b7fe40cb92 100644 --- a/.github/workflows/check-nix-format.yml +++ b/.github/workflows/check-nix-format.yml @@ -19,13 +19,18 @@ jobs: with: # pull_request_target checks out the base branch by default ref: refs/pull/${{ github.event.pull_request.number }}/merge + - name: Get Nixpkgs revision for nixfmt + run: | + # pin to a commit from nixpkgs-unstable to avoid e.g. building nixfmt + # from staging + # This should not be a URL, because it would allow PRs to run arbitrary code in CI! + rev=$(jq -r .rev ci/pinned-nixpkgs.json) + echo "url=https://github.com/NixOS/nixpkgs/archive/$rev.tar.gz" >> "$GITHUB_ENV" - uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26 with: # explicitly enable sandbox extra_nix_config: sandbox = true - # fix a commit from nixpkgs-unstable to avoid e.g. building nixfmt - # from staging - nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/4b455dc2048f73a79eb3713f342369ff58f93e0b.tar.gz + nix_path: nixpkgs=${{ env.url }} - name: Install nixfmt run: "nix-env -f '' -iAP nixfmt-rfc-style" - name: Check that Nix files are formatted according to the RFC style diff --git a/ci/README.md b/ci/README.md index 79dc4b3bce714..64cd8e5bcea98 100644 --- a/ci/README.md +++ b/ci/README.md @@ -2,3 +2,11 @@ This directory contains files to support CI, such as [GitHub Actions](https://github.com/NixOS/nixpkgs/tree/master/.github/workflows) and [Ofborg](https://github.com/nixos/ofborg). This is in contrast with [`maintainers/scripts`](`../maintainers/scripts`) which is for human use instead. + +## Pinned Nixpkgs + +CI may need certain packages from Nixpkgs. +In order to ensure that the needed packages are generally available without building, +[`pinned-nixpkgs.json`](./pinned-nixpkgs.json) contains a pinned Nixpkgs version tested by Hydra. + +Run [`update-pinned-nixpkgs.sh`](./update-pinned-nixpkgs.sh) to update it. diff --git a/ci/pinned-nixpkgs.json b/ci/pinned-nixpkgs.json new file mode 100644 index 0000000000000..29af1b0225816 --- /dev/null +++ b/ci/pinned-nixpkgs.json @@ -0,0 +1,4 @@ +{ + "rev": "cfb89a95f19bea461fc37228dc4d07b22fe617c2", + "sha256": "1yhsacvry6j8r02lk70p9dphjpi8lpzgq2qay8hiy4nqlys0mrch" +} diff --git a/ci/update-pinned-nixpkgs.sh b/ci/update-pinned-nixpkgs.sh new file mode 100755 index 0000000000000..7765581300572 --- /dev/null +++ b/ci/update-pinned-nixpkgs.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p jq + +set -euo pipefail + +# https://stackoverflow.com/a/246128 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +repo=https://github.com/nixos/nixpkgs +branch=nixpkgs-unstable +file=$SCRIPT_DIR/pinned-nixpkgs.json + +rev=$(git ls-remote "$repo" refs/heads/"$branch" | cut -f1) +sha256=$(nix-prefetch-url --unpack "$repo/archive/$rev.tar.gz" --name source) + +jq -n --arg rev "$rev" --arg sha256 "$sha256" '$ARGS.named' | tee /dev/stderr > $file From a70ab58960c254c58c5289b18b769c9d2311dce7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 26 Jun 2024 01:07:09 +0200 Subject: [PATCH 3/6] root: shell.nix for nixfmt This uses the reusable pinned Nixpkgs from the parent commit to create a shell.nix file for an environment with a pinned nixfmt version. --- .github/CODEOWNERS | 3 ++- .github/workflows/check-shell.yml | 29 +++++++++++++++++++++++++++++ shell.nix | 30 ++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/check-shell.yml create mode 100644 shell.nix diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 89d8d57cd73df..ebdefd539999d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -15,8 +15,9 @@ /.github/workflows @NixOS/Security @Mic92 @zowoq /ci @infinisil -# EditorConfig +# Develompent support /.editorconfig @Mic92 @zowoq +/shell.nix @infinisil @NixOS/Security # Libraries /lib @infinisil diff --git a/.github/workflows/check-shell.yml b/.github/workflows/check-shell.yml new file mode 100644 index 0000000000000..01ea97dfff20a --- /dev/null +++ b/.github/workflows/check-shell.yml @@ -0,0 +1,29 @@ +name: "Check shell" + +on: + pull_request_target: + +permissions: {} + +jobs: + x86_64-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 + with: + # pull_request_target checks out the base branch by default + ref: refs/pull/${{ github.event.pull_request.number }}/merge + - uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26 + - name: Build shell + run: nix-build shell.nix + + aarch64-darwin: + runs-on: macos-latest + steps: + - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 + with: + # pull_request_target checks out the base branch by default + ref: refs/pull/${{ github.event.pull_request.number }}/merge + - uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26 + - name: Build shell + run: nix-build shell.nix diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000000..d9e94eb2816d5 --- /dev/null +++ b/shell.nix @@ -0,0 +1,30 @@ +# A shell to get tooling for Nixpkgs development +# +# Note: We intentionally don't use Flakes here, +# because every time you change any file and do another `nix develop`, +# it would create another copy of the entire ~500MB tree in the store. +# See https://github.com/NixOS/nix/pull/6530 for the future +{ + system ? builtins.currentSystem, +}: +let + pinnedNixpkgs = builtins.fromJSON (builtins.readFile ci/pinned-nixpkgs.json); + + nixpkgs = fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/${pinnedNixpkgs.rev}.tar.gz"; + sha256 = pinnedNixpkgs.sha256; + }; + + pkgs = import nixpkgs { + inherit system; + config = {}; + overlays = []; + }; +in +pkgs.mkShellNoCC { + packages = [ + # The default formatter for Nix code + # https://github.com/NixOS/nixfmt + pkgs.nixfmt-rfc-style + ]; +} From 53b517c6851ef52a86e44ea16220d93f88bf9849 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 26 Jun 2024 01:08:17 +0200 Subject: [PATCH 4/6] workflows/check-nix-format: show command to fix When some files are not formatted properly, this shows how people can fix the problem. This notably uses the shell.nix introduced in the parent commit to ensure that the nixfmt version matches what CI expects. --- .github/workflows/check-nix-format.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-nix-format.yml b/.github/workflows/check-nix-format.yml index 5a3b7fe40cb92..5352b6fa53f95 100644 --- a/.github/workflows/check-nix-format.yml +++ b/.github/workflows/check-nix-format.yml @@ -56,6 +56,7 @@ jobs: NIX_FMT_PATHS_BUILD_SUPPORT_PHP: pkgs/build-support/php # Iterate over all environment variables beginning with NIX_FMT_PATHS_. run: | + unformattedPaths=() for env_var in "${!NIX_FMT_PATHS_@}"; do readarray -t paths <<< "${!env_var}" if [[ "${paths[*]}" == "" ]]; then @@ -64,7 +65,12 @@ jobs: fi echo "Checking paths: ${paths[@]}" if ! nixfmt --check "${paths[@]}"; then - echo "Error: nixfmt failed." - exit 1 + unformattedPaths+=("${paths[@]}") fi done + if (( "${#unformattedPaths[@]}" > 0 )); then + echo "Some required Nix files are not properly formatted" + echo "Please run the following in \`nix-shell\`:" + echo "nixfmt ${unformattedPaths[*]@Q}" + exit 1 + fi From d0bebb7d6b86a705f943dc926527400d7ebc144b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 26 Jun 2024 01:37:20 +0200 Subject: [PATCH 5/6] workflows/check-nix-format: strip newline from file listings Otherwise it prints the command to run with an empty entry: nixfmt '' '' Thanks to https://yaml-multiline.info/ for the great help with this --- .github/workflows/check-nix-format.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-nix-format.yml b/.github/workflows/check-nix-format.yml index 5352b6fa53f95..5459240766a1a 100644 --- a/.github/workflows/check-nix-format.yml +++ b/.github/workflows/check-nix-format.yml @@ -40,14 +40,14 @@ jobs: NIX_FMT_PATHS_BSD: pkgs/os-specific/bsd NIX_FMT_PATHS_MPVSCRIPTS: pkgs/applications/video/mpv/scripts # Format paths related to the Nixpkgs CUDA ecosystem. - NIX_FMT_PATHS_CUDA: | + NIX_FMT_PATHS_CUDA: |- pkgs/development/cuda-modules pkgs/test/cuda pkgs/top-level/cuda-packages.nix - NIX_FMT_PATHS_MAINTAINERS: | + NIX_FMT_PATHS_MAINTAINERS: |- maintainers/maintainer-list.nix maintainers/team-list.nix - NIX_FMT_PATHS_K3S: | + NIX_FMT_PATHS_K3S: |- nixos/modules/services/cluster/k3s nixos/tests/k3s pkgs/applications/networking/cluster/k3s From 58f98e4b1bc487a519b0f57e9723eb2ce223c229 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 26 Jun 2024 01:52:31 +0200 Subject: [PATCH 6/6] CODEOWNERS: set myself as code owner of workflows/check-nix-format --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ebdefd539999d..5cdc19fc098f2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,7 @@ # CI /.github/workflows @NixOS/Security @Mic92 @zowoq +/.github/workflows/check-nix-format.yml @infinisil /ci @infinisil # Develompent support