Implement clang resource directory logic

.circleci/config.yml

@@ -69,6 +69,7 @@ commonSteps: &commonSteps
             -DCMAKE_BUILD_TYPE=Release \
             ${HOST_LDC_VERSION:+-DD_COMPILER=$PWD/../host-ldc/bin/ldmd2} \
             -DLDC_LINK_MANUALLY=OFF \
+            -DCOMPILER_RT_LIBDIR_OS=linux \
             ${EXTRA_CMAKE_FLAGS:-}
           ninja -j$PARALLELISM obj/ldc2.o all ldc2-unittest all-test-runners
           bin/ldc2 -version

CMakeLists.txt

@@ -790,14 +790,8 @@ endif()
 # LLVM_LIBRARY_DIRS/clang/<version>/lib/<OS>/ , for example
 # LLVM_LIBRARY_DIRS/clang/4.0.0/lib/darwin/ , but we allow the user to specify
 # another directory.
-set(COMPILER_RT_BASE_DIR "${LLVM_LIBRARY_DIRS}" CACHE PATH "Base path of compiler-rt libraries. If they in are /usr/lib/clang/17/lib/linux/libclang_rt* you should set this value to /usr/lib")
-# If it's different than the default it will need to be added to the config files
-if(COMPILER_RT_BASE_DIR STREQUAL LLVM_LIBRARY_DIRS)
-    set(WANT_COMPILER_RT_LIBDIR_CONFIG FALSE)
-else()
-    set(WANT_COMPILER_RT_LIBDIR_CONFIG TRUE)
-endif()
-set(COMPILER_RT_LIBDIR "${COMPILER_RT_BASE_DIR}/clang")
+set(COMPILER_RT_BASE_DIR "${LLVM_LIBRARY_DIRS}/clang" CACHE PATH "Base path of compiler-rt libraries. If they in are /usr/lib/clang/17/lib/linux/libclang_rt* you should set this value to /usr/lib/clang")
+set(COMPILER_RT_LIBDIR "${COMPILER_RT_BASE_DIR}")
 if(LDC_LLVM_VER LESS 1600)
     set(COMPILER_RT_LIBDIR "${COMPILER_RT_LIBDIR}/${LLVM_VERSION_BASE_STRING}")
 else()
@@ -832,9 +826,6 @@ message(STATUS "-- Including LLVM compiler-rt libraries (LDC_INSTALL_LLVM_RUNTIM
 if (LDC_INSTALL_LLVM_RUNTIME_LIBS)
     # Locate LLVM sanitizer runtime libraries, and copy them to our lib folder
 
-    # No need to add another libdir, the default ldc one will have the libraries
-    set(WANT_COMPILER_RT_LIBDIR_CONFIG FALSE)
-
     if(APPLE)
         copy_compilerrt_lib("libclang_rt.asan_osx_dynamic.dylib" "libldc_rt.asan.dylib" TRUE)
         copy_compilerrt_lib("libclang_rt.lsan_osx_dynamic.dylib" "libldc_rt.lsan.dylib" TRUE)
@@ -879,10 +870,13 @@ if (LDC_INSTALL_LLVM_RUNTIME_LIBS)
     endif()
 endif()
 
-if(WANT_COMPILER_RT_LIBDIR_CONFIG)
-    message(STATUS "Adding ${COMPILER_RT_LIBDIR} to libdir in configuration files")
-    set(OPTIONAL_COMPILER_RT_DIR "\n        \"${COMPILER_RT_LIBDIR}\",")
+# If everything else if insufficient, the user can specify
+# -DCOMPILER_RT_LIBDIR_CONFIG to embed an arbitrary value.
+if(NOT DEFINED COMPILER_RT_LIBDIR_CONFIG)
+    set(COMPILER_RT_LIBDIR_CONFIG "${COMPILER_RT_LIBDIR}")
 endif()
+message(STATUS "Adding ${COMPILER_RT_LIBDIR_CONFIG} to lib-dirs in configuration file")
+set(OPTIONAL_COMPILER_RT_DIR "\n        \"${COMPILER_RT_LIBDIR_CONFIG}\", // compiler-rt directory")
 
 #
 # Auxiliary build and test utils.

docs/compiler_rt.md

@@ -0,0 +1,155 @@
+# Configuring LDC to use compiler-rt libraries
+
+LDC provides some functionalities like PGO, sanitizers and fuzzing, which require the presence of some libraries that are part of the compiler-rt LLVM sub-project.
+This document aims to describe how to tell LDC the locations of these libraries, based on your installation method.
+It is meant mostly for people using LDC through a Linux package manager or manually building it, which includes package maintainers.
+
+## Using the prebuilt packages or [dlang.org/install.sh](https://dlang.org/install.sh)
+
+The tarballs at https://github.com/ldc-developers/ldc/releases come with the compiler-rt libraries and you don't need to do any configuration.
+The dlang.org install.sh script also uses these tarballs so the same things applies to it.
+
+## Using a Linux distribution's package manager
+
+Given a survey of the available options at the time of writing,
+most distributions don't add a dependency on compiler-rt nor do they configure LDC to find them when installed manually,
+therefore, it is needed to do both of these steps manually.
+
+### Installing compiler-rt
+
+The name of the actual package varies quite a lot across distributions but you can usually find it be searching for `compiler-rt` or `libclang`.
+Below you will find a list of popular Linux distributions and the name of the `compiler-rt` package:
+
+- Debian/Ubuntu: `libclang-rt-dev`
+- Fedora: `compiler-rt`
+- Arch: `compiler-rt`
+- Gentoo: `compiler-rt-sanitizers`
+
+### Adding the compiler-rt path to ldc2.conf
+
+After you've installed the packages chances are that the libraries still won't be found because the path differs from what LDC expects.
+To solve this you need to first determine the path of the libraries.
+Again, this path depends on distribution but it is easy to find if you use your package manager to list the contents of the `compiler-rt` package installed in the previous step.
+
+Usually the path has the form: `<some_directory>/lib/clang/<major_version>/lib/linux/`.
+Where  `<major_version>` is a number like 18, 17, etc.
+`linux` may be, instead, a target triple like `x86_64-redhat-linux-gnu`.
+If the directory you found contains a bunch of `libclang_rt.*` files then you've found the right path, if not, try again.
+
+For simplicity, below you can find the paths of some Linux distributions.
+Remember to adapt them to the appropriate version of the package you have installed.
+
+- Debian/Ubuntu: `/usr/lib/llvm-<major_version>/lib/clang/<major_version>/lib/linux/`
+- Fedora: `/usr/lib/clang/<major_version>/lib/x86_64-redhat-linux-gnu/`
+- Arch: `/usr/lib/clang/<major_version>/lib/linux/`
+- Gentoo: `/usr/lib/clang/<major_version>/lib/linux/`
+
+You now need to edit the ldc2 configuration file.
+Since you're installing ldc2 through your package manager the config file is probably in `/etc/ldc2.conf`.
+If it's not there it should be in `../etc/ldc2.conf` relative to the directory of the ldc2 executable.
+
+Use your favorite text editor to edit the file and jump to the `lib-dirs = ` snippet.
+
+On recent versions you should see:
+```
+    lib-dirs = [
+        "/usr/lib",
+        "/usr/lib/clang/17/lib/linux", // compiler-rt directory
+    ];
+```
+You should edit the line with the `compiler-rt directory` comment and put the path you determined above in there.
+
+On older versions you wouldn't have the second entry so you would only see:
+```
+    lib-dirs = [
+        "/usr/lib",
+    ];
+```
+
+In that case just add another line with the path surrounded  by `"`.
+
+In the end, you should have:
+```
+    lib-dirs = [
+        "/usr/lib",
+		"YOUR_PATH_HERE", // compiler-rt directory
+    ];
+```
+With or without the `compiler-rt` comment.
+
+## Building from source
+
+When building from source you can pass a number of options to cmake related to compiler-rt libraries.
+
+### `COMPILER_RT_BASE_DIR`
+
+This option tells cmake a path that will be transformed, assuming standard layout, in the final compiler-rt directory.
+If you are using a distribution tarball for llvm then the default value for this option, `<LLVM_LIB_DIR>/clang`, should be the correct one.
+
+If on Linux and using the llvm provided through your package manager set this option to something like `/usr/lib/clang`.
+How to find the correct path is described above, in the Linux section.
+
+### `COMPILER_RT_LIBDIR_OS`
+
+This option only applies to non-Mac Unixes and refers to the final directory name in the full path to the compiler-rt libraries.
+
+If the path is `[...]/clang/<version>/lib/linux` then this value should be `linux`.
+If it is `[...]/clang/<version>/lib/x86_64-redhat-linux-gnu` then this value should be `x86_64-redhat-linux-gnu`.
+
+### `LDC_INSTALL_LLVM_RUNTIME_LIBS_ARCH`
+
+The option only applied to non-Mac Unixes and refers to the architecture component in the filename of the libraries.
+It only makes sense specifying this if `LDC_INSTALL_LLVM_RUNTIME_LIBS` is set to `ON`, see below.
+
+If the filename of the libraries is `libclang_rt.asan-x86_64.a` then this value should be `x86_64`.
+If the filename of the libraries is `libclang_rt.asan.a` then this value should be `""` (empty).
+
+### `LDC_INSTALL_LLVM_RUNTIME_LIBS`
+
+This option tells cmake to copy the library files from the compiler-rt directory specified above to the library directory of ldc2.
+If you enable this you don't need to configure anything else, the libraries will be found when you run ldc2.
+
+### `COMPILER_RT_LIBDIR_CONFIG`
+
+The final path that is stored in the configuration file in regards to compiler-rt is `${COMPILER_RT_BASE_DIR}/${LLVM_MAJOR_VERSION}/lib/${COMPILER_RT_LIBDIR_OS}`.
+If this setting doesn't match your layout, as a last resort, you can specify your custom path with `-DCOMPILER_RT_LIBDIR_CONFIG` and that unaltered path will be stored in the config.
+
+## Checking that everything works
+
+Try to compile the empty program below with `ldc2 -fprofile-generate -vv sample.d`
+```d
+void main () {}
+```
+
+And check the end of the output:
+```
+[...]
+*** Linking executable ***
+Searching profile runtime: /usr/lib/libldc_rt.profile.a
+Searching profile runtime: /usr/lib/llvm-17/lib/clang/17/lib/linux/libldc_rt.profile.a
+Searching profile runtime: /usr/lib/libldc_rt.profile.a
+Searching profile runtime: /usr/lib/llvm-17/lib/libldc_rt.profile.a
+Searching profile runtime: /usr/lib/libclang_rt.profile-x86_64.a
+Searching profile runtime: /usr/lib/llvm-17/lib/clang/17/lib/linux/libclang_rt.profile-x86_64.a
+Found, linking with /usr/lib/llvm-17/lib/clang/17/lib/linux/libclang_rt.profile-x86_64.a
+Linking with:
+[...]
+```
+The import line is `Found, linking with ...`.
+If you see this it means that you configured LDC correctly and the library was found.
+
+If you're missing that line, like in:
+```
+*** Linking executable ***
+Searching profile runtime: /usr/lib/libldc_rt.profile.a
+Searching profile runtime: /usr/lib/libldc_rt.profile.a
+Searching profile runtime: /usr/lib/llvm-17/lib/libldc_rt.profile.a
+Searching profile runtime: /usr/lib/libclang_rt.profile-x86_64.a
+Searching profile runtime: /usr/lib/libclang_rt.profile-x86_64.a
+Searching profile runtime: /usr/lib/llvm-17/lib/libclang_rt.profile-x86_64.a
+Searching profile runtime: /usr/lib/clang/17.0.6/lib/linux/libclang_rt.profile-x86_64.a
+Searching profile runtime: /usr/lib/clang/17.0.6/lib/linux/libclang_rt.profile-x86_64.a
+Searching profile runtime: /usr/lib/llvm-17/lib/clang/17.0.6/lib/linux/libclang_rt.profile-x86_64.a
+Linking with:
+```
+You should recheck the paths and make the necessary adjustments in the config file until `ldc2` can find it.

driver/linker-gcc.cpp

@@ -235,45 +235,54 @@ bool ArgsBuilder::isLldDefaultLinker() {
 
 //////////////////////////////////////////////////////////////////////////////
 
-// Returns the arch name as used in the compiler_rt libs.
-// FIXME: implement correctly for non-x86 platforms (e.g. ARM)
-// See clang/lib/Driver/Toolchain.cpp.
-llvm::StringRef getCompilerRTArchName(const llvm::Triple &triple) {
-  return triple.getArchName();
+// Keep as much as possible in sync with
+// clang/lib/Driver/Toolchain.cpp : getArchNameForCompilerRTLib.
+std::string getCompilerRTArchName(const llvm::Triple &triple) {
+  auto result = [&]() -> std::string {
+    bool IsWindows = triple.isOSWindows();
+    auto floatABI = gTargetMachine->Options.FloatABIType;
+
+    if (triple.getArch() == llvm::Triple::arm
+	|| triple.getArch() == llvm::Triple::armeb)
+      return (floatABI == llvm::FloatABI::Hard && !IsWindows)
+	? "armhf"
+	: "arm";
+
+    // For historic reasons, Android library is using i686 instead of i386.
+    if (triple.getArch() == llvm::Triple::x86 && triple.isAndroid())
+      return "i686";
+
+    if (triple.getArch() == llvm::Triple::x86_64 && triple.isX32())
+      return "x32";
+
+    return llvm::Triple::getArchTypeName(triple.getArch()).str();
+  }();
+  if (triple.isAndroid())
+    result += "-android";
+
+  return result;
 }
 
 // Appends arch suffix and extension.
 // E.g., for name="libclang_rt.fuzzer" and sharedLibrary=false, returns
-// "libclang_rt.fuzzer_osx.a" on Darwin.
+// "libclang_rt.fuzzer_osx.a" on Darwin. Without appendArch the result
+// would be "libclang_rt.fuzzer.a"
 std::string getCompilerRTLibFilename(const llvm::Twine &name,
-                                     const llvm::Triple &triple,
-                                     bool sharedLibrary) {
-  return (triple.isOSDarwin()
-              ? name + (sharedLibrary ? "_osx_dynamic.dylib" : "_osx.a")
-              : name + "-" + getCompilerRTArchName(triple) +
-                    (sharedLibrary ? ".so" : ".a"))
-      .str();
-}
+				     const llvm::Triple &triple,
+				     bool sharedLibrary,
+				     bool appendArch) {
+  std::string result = name.str();
+  if (triple.isOSDarwin()) {
+    if (appendArch)
+      result += "_osx";
+    result += sharedLibrary ? "_dynamic.dylib" : ".a";
+  } else {
+    if (appendArch)
+      result += "-" + getCompilerRTArchName(triple);
+    result += sharedLibrary ? ".so" : ".a";
+  }
 
-// Clang's RT libs are in a subdir of the lib dir.
-// E.g., for name="libclang_rt.asan" and sharedLibrary=true, returns
-// "clang/6.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" on
-// Darwin.
-// This function is "best effort", the path may not be what Clang does...
-// See clang/lib/Driver/Toolchain.cpp.
-std::string getRelativeClangCompilerRTLibPath(const llvm::Twine &name,
-                                              const llvm::Triple &triple,
-                                              bool sharedLibrary) {
-  llvm::StringRef OSName =
-      triple.isOSDarwin()
-          ? "darwin"
-          : triple.isOSFreeBSD() ? "freebsd" : triple.getOSName();
-
-  std::string relPath = (llvm::Twine("clang/") + ldc::llvm_version_base +
-                         "/lib/" + OSName + "/" + name)
-                            .str();
-
-  return getCompilerRTLibFilename(relPath, triple, sharedLibrary);
+  return result;
 }
 
 void appendFullLibPathCandidates(std::vector<std::string> &paths,
@@ -287,19 +296,14 @@ void appendFullLibPathCandidates(std::vector<std::string> &paths,
 
   // for backwards compatibility
   paths.push_back(exe_path::prependLibDir(filename));
-
-#ifdef LDC_LLVM_LIBDIR
-  candidate = LDC_LLVM_LIBDIR;
-  llvm::sys::path::append(candidate, filename);
-  paths.emplace_back(candidate.data(), candidate.size());
-#endif
 }
 
 // Returns candidates of full paths to a compiler-rt lib.
 // E.g., for baseName="asan" and sharedLibrary=false, returns something like
 // [ "<libDir>/libldc_rt.asan.a",
 //   "<libDir>/libclang_rt.asan_osx.a",
-//   "<libDir>/clang/6.0.0/lib/darwin/libclang_rt.asan_osx.a" ].
+//   "<libDir>/libclang_rt.asan.a",
+// ]
 std::vector<std::string>
 getFullCompilerRTLibPathCandidates(llvm::StringRef baseName,
                                    const llvm::Triple &triple,
@@ -310,12 +314,12 @@ getFullCompilerRTLibPathCandidates(llvm::StringRef baseName,
        (!sharedLibrary ? ".a" : triple.isOSDarwin() ? ".dylib" : ".so"))
           .str();
   appendFullLibPathCandidates(r, ldcRT);
-  const auto clangRT = getCompilerRTLibFilename("libclang_rt." + baseName,
-                                                triple, sharedLibrary);
-  appendFullLibPathCandidates(r, clangRT);
-  const auto fullClangRT = getRelativeClangCompilerRTLibPath(
-      "libclang_rt." + baseName, triple, sharedLibrary);
-  appendFullLibPathCandidates(r, fullClangRT);
+  const auto clangRTWithArch = getCompilerRTLibFilename("libclang_rt." + baseName,
+							triple, sharedLibrary, true);
+  appendFullLibPathCandidates(r, clangRTWithArch);
+  const auto clangRTWithoutArch = getCompilerRTLibFilename("libclang_rt." + baseName,
+							   triple, sharedLibrary, false);
+  appendFullLibPathCandidates(r, clangRTWithoutArch);
   return r;
 }