Skip to content

Commit

Permalink
Add target_toolchain_roots and make sysroot more hermetic
Browse files Browse the repository at this point in the history
It is useful when cross compiling to have 1 LLVM build used as the
compiler, and multiple others used for their libc++/target libraries.

This concept forces us to be much more robust about what comes from the
target toolchain, and what comes from the compiler.  Leave
toolchain_roots for the compiler roots, but add a new flag,
target_toolchain_roots with the same semantics for target toolchains.
This necesitates being much more clear internally about what comes from
what.

This isn't fully enough to make everything hermetic, clang still has
some autodetection logic left over when using sysroots.  We want to very
explicity specify the resource directory it should use, as well as where
the standard C/C++ libraries come from so it finds only what we want it
to find.

When pulling from a sysroot, we need to know the version of stdc++ to
find includes.  Use the stdlib variable for that by supporting
"stdc++-12" for example, and parsing the number out.
  • Loading branch information
AustinSchuh committed Jan 14, 2025
1 parent 08ccb5e commit 170ca55
Show file tree
Hide file tree
Showing 5 changed files with 245 additions and 58 deletions.
5 changes: 4 additions & 1 deletion toolchain/BUILD.llvm_repo
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ filegroup(
filegroup(
name = "include",
srcs = glob([
"include/**/c++/**",
"include/**",
"lib/clang/*/include/**",
]),
)
Expand All @@ -70,8 +70,11 @@ filegroup(
name = "lib",
srcs = glob(
[
"lib/lib*.a",
"lib/**/lib*.a",
"lib/**/lib*.a.syms",
"lib/clang/*/lib/**/*.a",
"lib/clang/*/lib/**/*.a.syms",
"lib/clang/*/lib/**/*.dylib",
# clang_rt.*.o supply crtbegin and crtend sections.
"lib/**/clang_rt.*.o",
Expand Down
191 changes: 156 additions & 35 deletions toolchain/cc_toolchain_config.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,12 @@ def cc_toolchain_config(
target_os,
target_system_name,
toolchain_path_prefix,
target_toolchain_path_prefix,
tools_path_prefix,
wrapper_bin_prefix,
compiler_configuration,
cxx_builtin_include_directories,
major_llvm_version):
llvm_version):
exec_os_arch_key = _os_arch_pair(exec_os, exec_arch)
target_os_arch_key = _os_arch_pair(target_os, target_arch)
_check_os_arch_keys([exec_os_arch_key, target_os_arch_key])
Expand All @@ -56,6 +57,7 @@ def cc_toolchain_config(
compiler,
abi_version,
abi_libc_version,
multiarch,
) = {
"darwin-x86_64": (
"clang-x86_64-darwin",
Expand All @@ -64,6 +66,7 @@ def cc_toolchain_config(
"clang",
"darwin_x86_64",
"darwin_x86_64",
None,
),
"darwin-aarch64": (
"clang-aarch64-darwin",
Expand All @@ -72,6 +75,7 @@ def cc_toolchain_config(
"clang",
"darwin_aarch64",
"darwin_aarch64",
None,
),
"linux-aarch64": (
"clang-aarch64-linux",
Expand All @@ -80,6 +84,7 @@ def cc_toolchain_config(
"clang",
"clang",
"glibc_unknown",
"aarch64-linux-gnu",
),
"linux-x86_64": (
"clang-x86_64-linux",
Expand All @@ -88,6 +93,7 @@ def cc_toolchain_config(
"clang",
"clang",
"glibc_unknown",
"x86_64-linux-gnu",
),
"wasm32": (
"clang-wasm32",
Expand All @@ -96,6 +102,7 @@ def cc_toolchain_config(
"clang",
"unknown",
"unknown",
None,
),
"wasm64": (
"clang-wasm64",
Expand All @@ -104,6 +111,7 @@ def cc_toolchain_config(
"clang",
"unknown",
"unknown",
None,
),
}[target_os_arch_key]

Expand All @@ -120,12 +128,30 @@ def cc_toolchain_config(
"-D__TIME__=\"redacted\"",
"-fdebug-prefix-map={}=__bazel_toolchain_llvm_repo__/".format(toolchain_path_prefix),
]
if target_toolchain_path_prefix != toolchain_path_prefix:
unfiltered_compile_flags.extend([
"-fdebug-prefix-map={}=__bazel_toolchain_llvm_repo__/".format(target_toolchain_path_prefix),
])

is_xcompile = not (exec_os == target_os and exec_arch == target_arch)

# Default compiler flags:
compile_flags = [
major_llvm_version = int(llvm_version.split(".")[0])

external_include_paths = None

resource_dir_version = llvm_version if major_llvm_version < 16 else major_llvm_version

resource_dir = [
"-resource-dir",
"{}lib/clang/{}".format(target_toolchain_path_prefix, resource_dir_version),
]

target_flags = [
"--target=" + target_system_name,
]

# Default compiler flags:
compile_flags = target_flags + [
# Security
"-U_FORTIFY_SOURCE", # https://github.com/google/sanitizers/issues/247
"-fstack-protector",
Expand All @@ -135,22 +161,36 @@ def cc_toolchain_config(
"-Wall",
"-Wthread-safety",
"-Wself-assign",
]
"-B{}bin/".format(toolchain_path_prefix),
] + resource_dir

dbg_compile_flags = ["-g", "-fstandalone-debug"]

opt_compile_flags = [
"-g0",
# Disable debug info
"-ggdb0",
"-O2",
"-D_FORTIFY_SOURCE=1",
"-DNDEBUG",
"-ffunction-sections",
"-fdata-sections",
]

link_flags = [
"--target=" + target_system_name,
link_flags = target_flags + [
"-no-canonical-prefixes",
] + resource_dir

# Use the standard libc++ location when not using msan
non_msan_compile_flags = [
"-isystem",
target_toolchain_path_prefix + "include/c++/v1/",
"-isystem",
target_toolchain_path_prefix + "include/" + target_system_name + "/c++/v1/",
]

non_msan_link_flags = [
"-L{}lib".format(target_toolchain_path_prefix),
"-L{}lib/{}".format(target_toolchain_path_prefix, target_system_name),
]

stdlib = compiler_configuration["stdlib"]
Expand Down Expand Up @@ -210,13 +250,30 @@ def cc_toolchain_config(
cxx_standard = compiler_configuration["cxx_standard"]
conly_flags = compiler_configuration["conly_flags"]
sysroot_path = compiler_configuration["sysroot_path"]

# Flags related to C++ standard.
cxx_flags = [
"-std=" + cxx_standard,
]
compile_flags_post_language = []

# We only support getting libc++ from the toolchain for now. Is it worth
# supporting libc++ from the sysroot? Or maybe just part of a LLVM distribution
# that's built for the target?
if not stdlib and is_xcompile:
print("WARNING: Using libc++ for host architecture while cross compiling, this is " +
"probably not what you want. Explicitly set standard_libraries to libc++ to silence.")

if stdlib == "builtin-libc++" and is_xcompile:
stdlib = "stdc++"
if stdlib == "builtin-libc++":
cxx_flags = [
"-std=" + cxx_standard,
cxx_flags.extend([
"-stdlib=libc++",
]
])
compile_flags_post_language.extend([
"-isystem",
target_toolchain_path_prefix + "lib/clang/{}/include".format(resource_dir_version),
])
if major_llvm_version >= 14:
# With C++20, Clang defaults to using C++ rather than Clang modules,
# which breaks Bazel's `use_module_maps` feature, which is used by
Expand Down Expand Up @@ -260,37 +317,80 @@ def cc_toolchain_config(
]

elif stdlib == "libc++":
cxx_flags = [
"-std=" + cxx_standard,
cxx_flags.extend([
"-stdlib=libc++",
]
])

link_flags.extend([
"-l:c++.a",
"-l:c++abi.a",
])
elif stdlib == "dynamic-stdc++":
cxx_flags = [
cxx_flags.extend([
"-std=" + cxx_standard,
"-stdlib=libstdc++",
]
])

link_flags.extend([
"-lstdc++",
])
elif stdlib == "stdc++":
cxx_flags = [
"-std=" + cxx_standard,
"-stdlib=libstdc++",
]
elif stdlib.startswith("stdc++"):
if not use_lld:
fail("libstdc++ only supported with lld")

# We use libgcc when using libstdc++ from a sysroot. Most libstdc++
# builds link to libgcc, which means we need to use libgcc's exception
# handling implementation, not the separate one in compiler-rt.
# Unfortunately, clang sometimes emits code incompatible with libgcc,
# see <https://bugs.llvm.org/show_bug.cgi?id=27455> and
# <https://lists.llvm.org/pipermail/cfe-dev/2016-April/048466.html> for
# example. This seems to be a commonly-used configuration with clang
# though, so it's probably good enough for most people.
link_flags.extend([
"-L{}lib".format(target_toolchain_path_prefix),
"-L{}lib/{}".format(target_toolchain_path_prefix, target_system_name),
])

# We expect to pick up these libraries from the sysroot.
link_flags.extend([
"-l:libstdc++.a",
])
if stdlib == "stdc++":
cxx_flags.extend([
"-stdlib=libstdc++",
])
elif stdlib.startswith("stdc++-"):
if sysroot_path == None:
fail("Need a sysroot to link against stdc++")

# -stdlib does nothing when using -nostdinc besides produce a warning
# that it's unused, so don't use it here.
libstdcxx_version = stdlib[len("stdc++-"):]

cxx_flags.extend([
"-nostdinc++",
"-isystem",
sysroot_path + "/usr/include/c++/" + libstdcxx_version,
"-isystem",
sysroot_path + "/usr/include/" + multiarch + "/c++/" + libstdcxx_version,
"-isystem",
sysroot_path + "/usr/include/c++/" + libstdcxx_version + "/backward",
])

# Clang really wants C system header includes after C++ ones.
compile_flags_post_language.extend([
"-nostdinc",
"-isystem",
target_toolchain_path_prefix + "lib/clang/{}/include".format(resource_dir_version),
])

link_flags.extend([
"-L" + sysroot_path + "/usr/lib/gcc/" + multiarch + "/" + libstdcxx_version,
])
else:
fail("Invalid stdlib: " + stdlib)
elif stdlib == "libc":
cxx_flags = [
"-std=" + cxx_standard,
]
pass
elif stdlib == "none":
cxx_flags = [
"-nostdlib",
Expand Down Expand Up @@ -343,27 +443,46 @@ def cc_toolchain_config(

# Replace flags with any user-provided overrides.
if compiler_configuration["compile_flags"] != None:
compile_flags = _fmt_flags(compiler_configuration["compile_flags"], toolchain_path_prefix)
compile_flags.extend(_fmt_flags(compiler_configuration["compile_flags"], toolchain_path_prefix))
if compiler_configuration["cxx_flags"] != None:
cxx_flags = _fmt_flags(compiler_configuration["cxx_flags"], toolchain_path_prefix)
cxx_flags.extend(_fmt_flags(compiler_configuration["cxx_flags"], toolchain_path_prefix))
if compiler_configuration["link_flags"] != None:
link_flags = _fmt_flags(compiler_configuration["link_flags"], toolchain_path_prefix)
link_flags.extend(_fmt_flags(compiler_configuration["link_flags"], toolchain_path_prefix))
if compiler_configuration["archive_flags"] != None:
archive_flags = _fmt_flags(compiler_configuration["archive_flags"], toolchain_path_prefix)
archive_flags.extend(_fmt_flags(compiler_configuration["archive_flags"], toolchain_path_prefix))
if compiler_configuration["link_libs"] != None:
link_libs = _fmt_flags(compiler_configuration["link_libs"], toolchain_path_prefix)
link_libs.extend(_fmt_flags(compiler_configuration["link_libs"], toolchain_path_prefix))
if compiler_configuration["opt_compile_flags"] != None:
opt_compile_flags = _fmt_flags(compiler_configuration["opt_compile_flags"], toolchain_path_prefix)
opt_compile_flags.extend(_fmt_flags(compiler_configuration["opt_compile_flags"], toolchain_path_prefix))
if compiler_configuration["opt_link_flags"] != None:
opt_link_flags = _fmt_flags(compiler_configuration["opt_link_flags"], toolchain_path_prefix)
opt_link_flags.extend(_fmt_flags(compiler_configuration["opt_link_flags"], toolchain_path_prefix))
if compiler_configuration["dbg_compile_flags"] != None:
dbg_compile_flags = _fmt_flags(compiler_configuration["dbg_compile_flags"], toolchain_path_prefix)
dbg_compile_flags.extend(_fmt_flags(compiler_configuration["dbg_compile_flags"], toolchain_path_prefix))
if compiler_configuration["coverage_compile_flags"] != None:
coverage_compile_flags = _fmt_flags(compiler_configuration["coverage_compile_flags"], toolchain_path_prefix)
coverage_compile_flags.extend(_fmt_flags(compiler_configuration["coverage_compile_flags"], toolchain_path_prefix))
if compiler_configuration["coverage_link_flags"] != None:
coverage_link_flags = _fmt_flags(compiler_configuration["coverage_link_flags"], toolchain_path_prefix)
coverage_link_flags.extend(_fmt_flags(compiler_configuration["coverage_link_flags"], toolchain_path_prefix))
if compiler_configuration["unfiltered_compile_flags"] != None:
unfiltered_compile_flags = _fmt_flags(compiler_configuration["unfiltered_compile_flags"], toolchain_path_prefix)
unfiltered_compile_flags.extend(_fmt_flags(compiler_configuration["unfiltered_compile_flags"], toolchain_path_prefix))

# TODO(AustinSchuh): Add msan support and make this conditional.
cxx_flags = non_msan_compile_flags + cxx_flags

# Add the sysroot flags here, as we want to check these last
if sysroot_path != None:
external_include_paths = [
sysroot_path + "usr/local/include",
]
if multiarch != None:
external_include_paths.extend([
sysroot_path + "usr/" + multiarch + "/include",
sysroot_path + "usr/include/" + multiarch,
])

external_include_paths.extend([
sysroot_path + "usr/include",
sysroot_path + "include",
])

# Source: https://cs.opensource.google/bazel/bazel/+/master:tools/cpp/unix_cc_toolchain_config.bzl
unix_cc_toolchain_config(
Expand All @@ -383,8 +502,10 @@ def cc_toolchain_config(
opt_compile_flags = opt_compile_flags,
conly_flags = conly_flags,
cxx_flags = cxx_flags,
compile_flags_post_language = compile_flags_post_language,
link_flags = link_flags + select({str(Label("@toolchains_llvm//toolchain/config:use_libunwind")): libunwind_link_flags, "//conditions:default": []}) +
select({str(Label("@toolchains_llvm//toolchain/config:use_compiler_rt")): compiler_rt_link_flags, "//conditions:default": []}),
select({str(Label("@toolchains_llvm//toolchain/config:use_compiler_rt")): compiler_rt_link_flags, "//conditions:default": []}) +
non_msan_link_flags,
archive_flags = archive_flags,
link_libs = link_libs,
opt_link_flags = opt_link_flags,
Expand Down
Loading

0 comments on commit 170ca55

Please sign in to comment.