--- a/clang/include/clang/Driver/Driver.h +++ b/clang/include/clang/Driver/Driver.h @@ -247,6 +247,9 @@ /// stored in it, and will clean them up when torn down. mutable llvm::StringMap> ToolChains; + /// Number of parallel jobs. + unsigned NumParallelJobs; + private: /// TranslateInputArgs - Create a new derived argument list from the input /// arguments, after applying the standard argument translations. @@ -549,6 +552,12 @@ /// Get the specific kind of LTO being performed. LTOKind getLTOMode() const { return LTOMode; } + /// Get the number of parallel jobs. + unsigned getNumberOfParallelJobs() const { return NumParallelJobs; } + + /// Set the number of parallel jobs. + void setNumberOfParallelJobs(unsigned N) { NumParallelJobs = N; } + private: /// Tries to load options from configuration file. --- a/clang/include/clang/Driver/Job.h +++ b/clang/include/clang/Driver/Job.h @@ -73,6 +73,9 @@ /// See Command::setEnvironment std::vector Environment; + /// Dependent actions + llvm::SmallVector DependentActions; + /// When a response file is needed, we try to put most arguments in an /// exclusive file, while others remains as regular command line arguments. /// This functions fills a vector with the regular command line arguments, @@ -227,6 +227,10 @@ return ProcStat; } + const llvm::SmallVector &getDependentActions() const { + return DependentActions; + } + protected: /// Optionally print the filenames to be compiled void PrintFileNames() const; --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -726,6 +726,8 @@ def P : Flag<["-"], "P">, Flags<[CC1Option,FlangOption,FC1Option]>, Group, HelpText<"Disable linemarker output in -E mode">, MarshallingInfoNegativeFlag>; +def parallel_jobs_EQ : Joined<["-"], "parallel-jobs=">, Flags<[NoXarchOption]>, + HelpText<"Number of parallel jobs">; def Qy : Flag<["-"], "Qy">, Flags<[CC1Option]>, HelpText<"Emit metadata containing compiler name and version">; def Qn : Flag<["-"], "Qn">, Flags<[CC1Option]>, --- a/clang/lib/Driver/Compilation.cpp +++ b/clang/lib/Driver/Compilation.cpp @@ -15,6 +15,7 @@ #include "clang/Driver/Options.h" #include "clang/Driver/ToolChain.h" #include "clang/Driver/Util.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" @@ -25,8 +26,11 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/raw_ostream.h" #include +#include +#include #include #include +#include #include using namespace clang; @@ -220,22 +224,134 @@ return !ActionFailed(&C.getSource(), FailingCommands); } +namespace { +class JobScheduler { +public: + enum JobState { JS_WAIT, JS_RUN, JS_DONE, JS_FAIL }; + JobScheduler(const JobList &Jobs, size_t NJobs = 1) + : Jobs(Jobs), NumJobs(NJobs) { +#if !LLVM_ENABLE_THREADS + NumJobs = 1; +#endif + for (auto &Job : Jobs) { + JState[&Job] = JS_WAIT; + for (const auto *AI : Job.getDependentActions()) { + for (const auto *CI : ActToCmds[AI]) { + DependentCmds[&Job].push_back(CI); + } + } + for (const auto *CI : ActToCmds[&Job.getSource()]) { + DependentCmds[&Job].push_back(CI); + } + ActToCmds[&Job.getSource()].push_back(&Job); + } + } + /// \return true if all jobs are done. Otherwise, \p Next contains the + /// the next job ready to be executed if it is not null pointer. Otherwise + /// all jobs are running or waiting. + bool IsDone(const Command *&Next) { + std::lock_guard lock(Mutex); + Next = nullptr; + unsigned Done = 0; + unsigned Running = 0; + for (auto &Cmd : Jobs) { + switch (JState[&Cmd]) { + case JS_RUN: + ++Running; + break; + case JS_DONE: + case JS_FAIL: + ++Done; + break; + case JS_WAIT: { + bool InputsReady = true; + for (const auto *CI : DependentCmds[&Cmd]) { + if (JState[CI] == JS_FAIL) { + JState[&Cmd] = JS_FAIL; + ++Done; + InputsReady = false; + break; + } + if (JState[CI] != JS_DONE) { + InputsReady = false; + break; + } + } + if (!Next && InputsReady) { + Next = &Cmd; + } + break; + } + } + } + if (Running >= NumJobs) + Next = nullptr; + return Done == Jobs.size(); + } + + void setJobState(const Command *Cmd, JobState JS) { + std::lock_guard lock(Mutex); + JState[Cmd] = JS; + } + + void launch(std::function Work) { +#if LLVM_ENABLE_THREADS + if (NumJobs == 1) { + Work(); + return; + } + std::thread Th(Work); + Th.detach(); +#else + Work(); +#endif + } + +private: + std::mutex Mutex; + const JobList &Jobs; + llvm::DenseMap JState; + llvm::DenseMap> + ActToCmds; + llvm::DenseMap> + DependentCmds; + size_t NumJobs; // Number of parallel jobs to run +}; +} // namespace void Compilation::ExecuteJobs(const JobList &Jobs, FailingCommandList &FailingCommands) const { // According to UNIX standard, driver need to continue compiling all the // inputs on the command line even one of them failed. // In all but CLMode, execute all the jobs unless the necessary inputs for the // job is missing due to previous failures. - for (const auto &Job : Jobs) { - if (!InputsOk(Job, FailingCommands)) + JobScheduler JS(Jobs, getDriver().getNumberOfParallelJobs()); + + const Command *Next = nullptr; + while (!JS.IsDone(Next)) { + if (!Next) { + std::this_thread::yield(); continue; - const Command *FailingCommand = nullptr; - if (int Res = ExecuteCommand(Job, FailingCommand)) { - FailingCommands.push_back(std::make_pair(Res, FailingCommand)); + } + + if (!InputsOk(*Next, FailingCommands)) { + JS.setJobState(Next, JobScheduler::JS_FAIL); // Bail as soon as one command fails in cl driver mode. if (TheDriver.IsCLMode()) return; + continue; } + + JS.setJobState(Next, JobScheduler::JS_RUN); + auto Work = [&, Next]() { + const Command *FailingCommand = nullptr; + if (int Res = ExecuteCommand(*Next, FailingCommand)) { + JS.setJobState(Next, JobScheduler::JS_FAIL); + FailingCommands.push_back(std::make_pair(Res, FailingCommand)); + } else { + JS.setJobState(Next, JobScheduler::JS_DONE); + } + }; + JS.launch(Work); } } --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -51,6 +51,7 @@ #include "ToolChains/XCore.h" #include "ToolChains/ZOS.h" #include "clang/Basic/TargetID.h" +#include "clang/Driver/OptionUtils.h" #include "clang/Basic/Version.h" #include "clang/Config/config.h" #include "clang/Driver/Action.h" @@ -63,6 +64,7 @@ #include "clang/Driver/Tool.h" #include "clang/Driver/ToolChain.h" #include "clang/Driver/Types.h" +#include "clang/Driver/Util.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" @@ -194,7 +196,7 @@ CCPrintHeaders(false), CCLogDiagnostics(false), CCGenDiagnostics(false), CCPrintProcessStats(false), TargetTriple(TargetTriple), Saver(Alloc), CheckInputsExist(true), GenReproducer(false), - SuppressMissingInputWarning(false) { + SuppressMissingInputWarning(false), NumParallelJobs(1) { // Provide a sane fallback if no VFS is specified. if (!this->VFS) this->VFS = llvm::vfs::getRealFileSystem(); @@ -1103,6 +1105,9 @@ BitcodeEmbed = static_cast(Model); } + setNumberOfParallelJobs( + getLastArgIntValue(Args, options::OPT_parallel_jobs_EQ, 1, Diags)); + std::unique_ptr UArgs = std::make_unique(std::move(Args)); --- a/clang/lib/Driver/Job.cpp +++ b/clang/lib/Driver/Job.cpp @@ -44,9 +44,11 @@ for (const auto &II : Inputs) if (II.isFilename()) InputInfoList.push_back(II); - for (const auto &II : Outputs) + for (const auto &II : Outputs) { if (II.isFilename()) OutputFilenames.push_back(II.getFilename()); + DependentActions.push_back(II.getAction()); + } } /// Check if the compiler flag in question should be skipped when --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -8026,7 +8026,7 @@ C.addCommand(std::make_unique( JA, *this, ResponseFileSupport::None(), TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())), - CmdArgs, None, Output)); + CmdArgs, Inputs, Output)); } void OffloadBundler::ConstructJobMultipleOutputs( @@ -8110,7 +8110,7 @@ C.addCommand(std::make_unique( JA, *this, ResponseFileSupport::None(), TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())), - CmdArgs, None, Outputs)); + CmdArgs, Inputs, Outputs)); } void OffloadWrapper::ConstructJob(Compilation &C, const JobAction &JA,