Initial DTLTO commit

This follows on from our original RFC: here

This PR includes a design document which details the DTLTO design. There are some areas where we would like feedback from the upstream community to inform the design, those areas are noted in the design document.

This first commit just adds the initial scaffolding to show how DTLTO will work end-to-end. Support for additional features such as archive handling/caching etc.. will be added in later commits. Once this commit is in place we intend to add those features in a series of subsequent commits.

Questions to discuss with upstream contributors:

- What can we call the feature? DTLTO doesn't really distinguish this from Bazel-style DTLTO. Bazel-style DTLTO doesn't mean anything to anyone else!

- Perhaps there is a subset of arguments that should be automatically passed to the codegen tool from the driver level. For example perhaps the compiler should pass the relevant -mllm options to LLD if it sees: -fdiagnostics-format=msvc?

- Where to store temporary files? In particular we might want to discuss our downstream performance optimization (not included in this patch) for creating the native object files in the cache so that when we rename them to populate the cache we are guaranteed not to cross a device (which caused a measurable performance problem downstream).

- Our proposals for how to support bitcode archive members.

- Perhaps the (current) default tool should be clang -cc1 instead of just clang.

Author: bd1976bris

clang/docs/ThinLTO.rst

@@ -240,6 +240,33 @@ The ``BOOTSTRAP_LLVM_ENABLE_LTO=Thin`` will enable ThinLTO for stage 2 and
 stage 3 in case the compiler used for stage 1 does not support the ThinLTO
 option.
 
+Distributed ThinLTO (DTLTO)
+---------------------------
+
+DTLTO allows for the distribution of backend ThinLTO compilations 
+internally as part of the link step and therefore should be usable
+via any build that can use ThinLTO.
+
+DTLTO requires the LLD linker (-fuse-ld=lld).
+
+-fthinlto-distributor=<path>
+   - Specifies the <path> to the distributor process executable.
+
+-Xdist <arg>
+   - Pass <arg> to the distributor process (see -fthinlto-distributor=).
+   - Can be specified multiple times to pass multiple options.
+
+Examples:
+  clang -flto=thin -fthinlto-distributor=sndbs-dtlto.exe -Xdist --verbose -fuse-ld=lld
+  clang -flto=thin -fthinlto-distributor=%python -Xdist %llvm_src_root/utils/dtlto/local.py -fuse-ld=lld
+
+Example distributor scripts can be found under `llvm/utils/dtlto`.
+
+Clang supplies the the path to itself to LLD, so that it can be invoked
+to perform the backend compilations.
+
+See `DTLTO <https://lld.llvm.org/dtlto.html>`_ for more information.
+
 More Information
 ================
 

clang/include/clang/Driver/Options.td

@@ -1041,6 +1041,9 @@ def offload_link : Flag<["--"], "offload-link">, Group<Link_Group>,
 def Xlinker : Separate<["-"], "Xlinker">, Flags<[LinkerInput, RenderAsInput]>,
   HelpText<"Pass <arg> to the linker">, MetaVarName<"<arg>">,
   Group<Link_Group>;
+def Xdist : Separate<["-"], "Xdist">, Flags<[LinkerInput, RenderAsInput]>,
+  HelpText<"Pass <arg> to the ThinLTO distributor">, MetaVarName<"<arg>">,
+  Group<Link_Group>;
 def Xoffload_linker : JoinedAndSeparate<["-"], "Xoffload-linker">,
   HelpText<"Pass <arg> to the offload linkers or the ones idenfied by -<triple>">,
   MetaVarName<"<triple> <arg>">, Group<Link_Group>;
@@ -3997,7 +4000,9 @@ def ffinite_loops: Flag<["-"],  "ffinite-loops">, Group<f_Group>,
 def fno_finite_loops: Flag<["-"], "fno-finite-loops">, Group<f_Group>,
   HelpText<"Do not assume that any loop is finite.">,
   Visibility<[ClangOption, CC1Option]>;
-
+def fthinlto_distributor_EQ : Joined<["-", "--"], "fthinlto-distributor=">,
+  HelpText<"Specifies the <path> to the distributor process executable.">, MetaVarName<"<path>">,
+  Visibility<[ClangOption, CLOption]>;
 def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>,
   HelpText<"Process trigraph sequences">, Visibility<[ClangOption, CC1Option]>;
 def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>,

clang/lib/Driver/ToolChains/CommonArgs.cpp

@@ -859,6 +859,15 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
       CmdArgs.push_back("--fat-lto-objects");
   }
 
+  if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) {
+    CmdArgs.push_back(Args.MakeArgString(Twine("--thinlto-distributor=") +
+                                         A->getValue()));
+    const char *ClangPath =
+        Args.MakeArgString(ToolChain.getDriver().getClangProgramPath());
+    CmdArgs.push_back(
+        Args.MakeArgString(Twine("--thinlto-remote-opt-tool=") + ClangPath));
+  }
+
   const char *PluginOptPrefix = IsOSAIX ? "-bplugin_opt:" : "-plugin-opt=";
   const char *ExtraDash = IsOSAIX ? "-" : "";
   const char *ParallelismOpt = IsOSAIX ? "-threads=" : "jobs=";

cross-project-tests/CMakeLists.txt

@@ -22,6 +22,7 @@ set(CROSS_PROJECT_TEST_DEPS
   llvm-dwarfdump
   llvm-config
   llvm-objdump
+  llvm-profdata
   split-file
   not
   )
@@ -94,6 +95,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}

cross-project-tests/dtlto/README.txt

@@ -0,0 +1,2 @@
+                                                                   -*- rst -*-
+This is a collection of tests to check distributed thinLTO (DTLTO) functionality

cross-project-tests/dtlto/cspgouse.test.c

@@ -0,0 +1,43 @@
+## Test that --lto-cs-profile-file= can be specified.
+## This is interesting as it requires an additional input file.
+
+# RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir
+
+## compile bitcode.
+# RUN: echo "-triple x86_64-unknown-unknown -funified-lto -emit-llvm-bc -flto=thin" > c.rsp
+# RUN: %clang_cc1 -x c foo.c -o foo.o @c.rsp
+# RUN: %clang_cc1 -x c bar.c -o bar.o @c.rsp
+# RUN: %clang_cc1 -x c _start.c -o _start.o @c.rsp
+
+## Create an empty profile.
+# RUN: touch empty.proftext
+# RUN: llvm-profdata merge empty.proftext -o empty.profdata
+
+# RUN: echo "foo.o bar.o _start.o \
+# RUN:       --build-id=none \
+# RUN:       --lto=thin --lto-O3 \
+# RUN:       --thinlto-distributor=%python  \
+# RUN:       -mllvm -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+# RUN:       --thinlto-remote-opt-tool=%clang" > l.rsp
+
+## link passes with an empty profile.
+# RUN: ld.lld @l.rsp --lto-cs-profile-file=empty.profdata
+
+## Generate a broken profile to show that the profile
+## is consumed by the backend compilations.
+# RUN: echo "WOBBLER" > bad.profdata
+# RUN: not ld.lld @l.rsp --lto-cs-profile-file=bad.profdata 2>&1 | FileCheck %s
+
+# CHECK: Error in reading profile 
+# CHECK: error: backend compilation error
+
+#--- foo.c
+void foo() {}
+
+#--- bar.c
+extern void foo();
+void bar() {foo();}
+
+#--- _start.c
+extern void bar();
+void _start() {bar();}

cross-project-tests/dtlto/dtlto.c

@@ -0,0 +1,48 @@
+// RUN: rm -rf %t && mkdir -p %t && cd %t
+
+// RUN: %clang -target x86_64-linux-gnu %s -flto=thin \
+// RUN:   -fthinlto-distributor=%python  \
+// RUN:   -Xdist %llvm_src_root/utils/dtlto/local.py \
+// RUN:   --save-temps \
+// RUN:   -fuse-ld=lld \
+// RUN:   -Wl,--save-temps \
+// RUN:   -nostdlib \
+// RUN:   -nostartfiles
+
+/// Check that the required output files have been created.
+// RUN: ls | tee %t.log | count 20
+// RUN: ls | FileCheck %s --check-prefix=COMPILE
+// RUN: ls | FileCheck %s --check-prefix=DTLTO
+// RUN: ls | FileCheck %s --check-prefix=LTO
+
+/// Files produced by the compiler.
+// COMPILE: dtlto.bc
+// COMPILE: dtlto.i
+// COMPILE: dtlto.o
+
+/// DTLTO native object output file for t.o.
+// DTLTO: t{{.*}}.{{.*}}.native.o{{$}}
+/// DTLTO imports file for t.o.
+// DTLTO: {{.*}}.{{.*}}.native.o.imports{{$}}
+/// DTLTO summary slice for t.o.
+// DTLTO: t{{.*}}.{{.*}}.native.o.thinlto.bc{{$}}
+
+// LTO: a.out{{$}}
+/// save-temps incremental files for a.out
+// LTO: a.out.0.0.preopt.bc{{$}}
+// LTO: a.out.0.2.internalize.bc{{$}}
+// LTO: a.out.dist-file.json{{$}}
+// LTO: a.out.index.bc{{$}}
+// LTO: a.out.index.dot{{$}}
+// : a.out.lto.dtlto.o{{$}}
+// LTO: a.out.resolution.txt{{$}}
+/// save-temps incremental files for t.o.
+// LTO: dtlto-{{.*}}.{{.*}}.native.o.0.preopt.bc{{$}}
+// LTO: dtlto-{{.*}}.{{.*}}.native.o.1.promote.bc{{$}}
+// LTO: dtlto-{{.*}}.{{.*}}.native.o.2.internalize.bc{{$}}
+// LTO: dtlto-{{.*}}.{{.*}}.native.o.3.import.bc{{$}}
+// LTO: dtlto-{{.*}}.{{.*}}.native.o.4.opt.bc{{$}}
+// LTO: dtlto-{{.*}}.{{.*}}.native.o.5.precodegen.bc{{$}}
+// LTO: dtlto-{{.*}}.{{.*}}.native.o.resolution.txt{{$}}
+
+int _start() {return 0;}

cross-project-tests/dtlto/lit.local.cfg

@@ -0,0 +1,2 @@
+if "clang" not in config.available_features:
+    config.unsupported = True

lld/ELF/Config.h

@@ -206,6 +206,8 @@ struct Config {
   llvm::SmallVector<llvm::StringRef, 0> searchPaths;
   llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
   llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
+  llvm::StringRef DTLTODistributor;
+  llvm::StringRef DTLTORemoteOptTool;
   llvm::SmallVector<llvm::StringRef, 0> undefined;
   llvm::SmallVector<SymbolVersion, 0> dynamicList;
   llvm::SmallVector<uint8_t, 0> buildIdVector;

lld/ELF/Driver.cpp

@@ -1443,6 +1443,12 @@ static void readConfigs(opt::InputArgList &args) {
   }
   config->thinLTOModulesToCompile =
       args::getStrings(args, OPT_thinlto_single_module_eq);
+
+  config->DTLTODistributor =
+      args.getLastArgValue(OPT_thinlto_distributor_eq);
+  config->DTLTORemoteOptTool =
+      args.getLastArgValue(OPT_thinlto_remote_opt_tool_eq);
+
   config->timeTraceEnabled = args.hasArg(OPT_time_trace_eq);
   config->timeTraceGranularity =
       args::getInteger(args, OPT_time_trace_granularity, 500);
@@ -1624,6 +1630,22 @@ static void readConfigs(opt::InputArgList &args) {
 
   config->passPlugins = args::getStrings(args, OPT_load_pass_plugins);
 
+  if (!config->DTLTODistributor.empty()) {
+    for (auto o : {"-thinlto-cc1-arg=-ferror-limit", "-thinlto-cc1-arg=19",
+                   "-thinlto-cc1-arg=-fmessage-length=120"}) {
+      parseClangOption(o, "-mllvm");
+      config->mllvmOpts.emplace_back(o);
+    }
+
+    if (errorHandler().vsDiagnostics) {
+      for (auto o :
+           {"-thinlto-cc1-arg=-fdiagnostics-format", "-thinlto-cc1-arg=msvc"}) {
+        parseClangOption(o, "-mllvm");
+        config->mllvmOpts.emplace_back(o);
+      }
+    }
+  }
+
   // Parse -mllvm options.
   for (const auto *arg : args.filtered(OPT_mllvm)) {
     parseClangOption(arg->getValue(), arg->getSpelling());