llvm on disk randomly fails with 'failed: absolute path inclusion(s) found in'

[spags-lacework]

Hi, I'm trying to have my llvm on disk to speed up CI times however I recently started seeing random failures.

http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

# Partially sourced from https://github.com/marvin-hansen/bazel_rust_cross_llvm/blob/small-clang/MODULE.bazel as reference.

http_archive(
    name = "org_chromium_sysroot_linux_x64",
    build_file_content = """
filegroup(
  name = "sysroot",
  srcs = glob(["*/**"]),
  visibility = ["//visibility:public"],
)
""",
    sha256 = "5df5be9357b425cdd70d92d4697d07e7d55d7a923f037c22dc80a78e85842d2c",
    urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/4f611ec025be98214164d4bf9fbe8843f58533f7/debian_bullseye_amd64_sysroot.tar.xz"],
)

llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm")
llvm.toolchain(
    name = "llvm_toolchain_with_urls",
    llvm_versions = {
        "darwin-aarch64": "16.0.0",
        "darwin-x86_64": "16.0.0",
    },
)
use_repo(llvm, "llvm_toolchain_with_urls")

register_toolchains("@llvm_toolchain_with_urls//:all")

llvm.toolchain(
    name = "llvm_toolchain",
    llvm_version = "16.0.0",
)
llvm.toolchain_root(
    name = "llvm_toolchain",
    label = "@@llvm_toolchain_with_urls//:all",
    targets = [
        "darwin-aarch64",
        "darwin-x86_64",
    ],
)
llvm.toolchain_root(
    name = "llvm_toolchain",
    # This is managed by devspaces for local development and in our runner images for CI.
    path = "/opt/llvm",
    targets = ["linux-x86_64"],
)
use_repo(llvm, "llvm_toolchain")

register_toolchains("@llvm_toolchain//:all")

A few weeks ago I started seeing this intermittent issue.

(02:50:59) [4,446 / 4,502] Compiling src/google/protobuf/compiler/cpp/message.cc [for tool]; 39s disk-cache, remote-cache, processwrapper-sandbox
(02:51:09) [4,446 / 4,502] Compiling src/google/protobuf/compiler/cpp/message.cc [for tool]; 50s disk-cache, remote-cache, processwrapper-sandbox
(02:51:12) ERROR: /home/runner/_work/cache/409a1b0f6b631ca8c4910634f2effa88/external/protobuf~/src/google/protobuf/compiler/cpp/BUILD.bazel:64:11: Compiling src/google/protobuf/compiler/cpp/message.cc [for tool] failed: absolute path inclusion(s) found in rule '@@protobuf~//src/google/protobuf/compiler/cpp:cpp':
the source file 'src/google/protobuf/compiler/cpp/message.cc' includes the following non-builtin files with absolute paths (if these are builtin files, make sure these paths are in your toolchain):
'/opt/llvm/lib/clang/16/include/stddef.h'
'/opt/llvm/lib/clang/16/include/__stddef_max_align_t.h'
'/opt/llvm/lib/clang/16/include/stdint.h'
'/opt/llvm/lib/clang/16/include/stdarg.h'
'/opt/llvm/lib/clang/16/include/limits.h'
'/opt/llvm/lib/clang/16/include/xmmintrin.h'
'/opt/llvm/lib/clang/16/include/mmintrin.h'
'/opt/llvm/lib/clang/16/include/mm_malloc.h'
'/opt/llvm/lib/clang/16/include/emmintrin.h'

Versions I have set

bazel_dep(name = "abseil-cpp", version = "20240722.0", repo_name = "com_google_absl")
bazel_dep(name = "aspect_bazel_lib", version = "2.9.4")
bazel_dep(name = "aspect_rules_esbuild", version = "0.21.0")
bazel_dep(name = "aspect_rules_js", version = "2.1.2")
bazel_dep(name = "aspect_rules_ts", version = "3.3.2")
bazel_dep(name = "aspect_rules_webpack", version = "0.16.0")
bazel_dep(name = "bazel_features", version = "1.22.0")
bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "contrib_rules_jvm", version = "0.27.0")
bazel_dep(name = "gazelle", version = "0.40.0", repo_name = "bazel_gazelle")
bazel_dep(name = "protobuf", version = "29.1", repo_name = "com_google_protobuf")
bazel_dep(name = "platforms", version = "0.0.10")
bazel_dep(name = "rules_cc", version = "0.0.17")
bazel_dep(name = "rules_go", version = "0.50.1", repo_name = "io_bazel_rules_go")
bazel_dep(name = "rules_java", version = "8.6.2")
bazel_dep(name = "rules_jvm_external", version = "6.5")
bazel_dep(name = "rules_nodejs", version = "6.3.2")
bazel_dep(name = "rules_oci", version = "2.0.1")
bazel_dep(name = "rules_pkg", version = "1.0.1")
bazel_dep(name = "rules_python", version = "1.0.0")

# There's an issue with rdkafka and rules_rust 54+: https://github.com/bazelbuild/rules_rust/issues/3099
bazel_dep(name = "rules_rust", version = "0.53.0")
bazel_dep(name = "stardoc", version = "0.7.2", repo_name = "io_bazel_stardoc")
bazel_dep(name = "toolchains_llvm", version = "1.2.0")
bazel_dep(name = "toolchains_protoc", version = "0.3.6")

# protobuf wants rules_ruby
bazel_dep(name = "rules_ruby", version = "0.14.1")

# We don't directly use apple_support rules_apple but it is needed by grpc_grpc.
bazel_dep(name = "apple_support", version = "1.17.1", repo_name = "build_bazel_apple_support")
bazel_dep(name = "rules_license", version = "1.0.0")
bazel_dep(name = "rules_apple", version = "3.16.0", repo_name = "build_bazel_rules_apple")

I wish I could provide more details but the issue is fairly random and not consistently reproducible.

I tried bazel fetch --configure --force as seen here " bazelbuild/bazel#21718 (comment)" but that was not successful.