Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
diff -u -r -P ./mainline/src/disk_interface.cc ./mainline/src/disk_interface.cc
--- ./mainline/src/disk_interface.cc 2011-10-24 15:12:45.771598100 +0200
+++ ./patched/src/disk_interface.cc 2011-10-24 15:13:22.746552100 +0200
@@ -17,6 +17,7 @@
#include <errno.h>
#include <stdio.h>
#include <string.h>
+#include <sstream>
#include <sys/stat.h>
#include "util.h"
@@ -85,6 +85,81 @@
return true;
}
+// Method to open, read and divide and aggregated depfile, saving ints individual
+// components into internal cache of RealDiskInterface
+// see RealDiskInterface::ReadAggregatedFile()
+bool RealDiskInterface::loadIntoCache(const string & aggregated_path, string* err)
+{
+ string content = ReadFile(aggregated_path, err);
+
+ if (!err->empty())
+ return false;
+ if (content.empty())
+ return true;
+
+ map<string, string> & fileMap = cache[aggregated_path];
+ istringstream stream(content);
+ size_t colon;
+ string line, filename;
+ while (getline(stream, line))
+ {
+ if ((colon = line.find(":")) != string::npos)
+ {
+ size_t beginning = line.find_first_not_of(" \t");
+ filename = line.substr(beginning, colon - beginning);
+ }
+
+ if (!filename.empty())
+ {
+ fileMap[filename] += line + "\n";
+ }
+ }
+
+ return true;
+}
+
+// Method to support aggregated depfiles, which contain multiple depfiles concatenated one after each other.
+// When called the first time, the file is read as a whole and individual sections are saved in internal cache.
+// Subsequent calls asking for different parts of the same aggregated file return the cached data, while
+// removing it from the cache - no more than one call asking for a particular *.d file is supported.
+string RealDiskInterface::ReadAggregatedFile(const string& group_path, const string& element_path, string* err)
+{
+ DepFileMap::iterator aggregated = cache.find(group_path);
+
+ // make sure the aggregated file is in cache
+ if (cache.end() == aggregated)
+ {
+ // not yet cached -> populate the cache
+ if (!loadIntoCache(group_path, err))
+ {
+ return "";
+ }
+
+ aggregated = cache.find(group_path);
+ }
+
+ DepFileMap::mapped_type::iterator element = aggregated->second.find(element_path);
+ if (aggregated->second.end() == element)
+ {
+ // no record for the underlying .d file -> not an error
+ return "";
+ }
+
+ // load the contens
+ string contens = element->second;
+ if (contens.empty())
+ {
+ *err = "Empty depfile record for " + element_path;
+ return "";
+ }
+
+ // drop the element contens
+ aggregated->second.erase(element);
+
+ // return the contens
+ return contens;
+}
+
std::string RealDiskInterface::ReadFile(const std::string& path,
std::string* err) {
std::string contents;
diff -u -r -P ./mainline/src/disk_interface.h ./mainline/src/disk_interface.h
--- ./mainline/src/disk_interface.h 2011-10-24 15:12:45.771598100 +0200
+++ ./patched/src/disk_interface.h 2011-10-24 15:13:22.746552100 +0200
@@ -15,6 +15,7 @@
#ifndef NINJA_DISK_INTERFACE_H_
#define NINJA_DISK_INTERFACE_H_
+#include <map>
#include <string>
/// Interface for accessing the disk.
@@ -34,6 +34,8 @@
/// Read a file to a string. Fill in |err| on error.
virtual std::string ReadFile(const std::string& path, std::string* err) = 0;
+ virtual std::string ReadAggregatedFile(const std::string& group_path, const std::string& element_path, std::string* err) = 0;
+
/// Remove the file named @a path. It behaves like 'rm -f path' so no errors
/// are reported if it does not exists.
/// @returns 0 if the file has been removed,
@@ -46,13 +48,19 @@
bool MakeDirs(const std::string& path);
};
+typedef std::map<std::string, std::map<std::string, std::string> > DepFileMap;
+
/// Implementation of DiskInterface that actually hits the disk.
struct RealDiskInterface : public DiskInterface {
virtual ~RealDiskInterface() {}
virtual int Stat(const std::string& path);
virtual bool MakeDir(const std::string& path);
virtual std::string ReadFile(const std::string& path, std::string* err);
+ virtual std::string ReadAggregatedFile(const std::string& group_path, const std::string& element_path, std::string* err);
virtual int RemoveFile(const std::string& path);
+
+ bool loadIntoCache( const std::string & project, std::string* err);
+ DepFileMap cache;
};
#endif // NINJA_DISK_INTERFACE_H_
diff -u -r -P ./mainline/src/disk_interface_test.cc ./patched/src/disk_interface_test.cc
--- ./mainline/src/disk_interface_test.cc 2011-10-24 15:12:45.787198500 +0200
+++ ./patched/src/disk_interface_test.cc 2011-10-24 15:13:22.762152500 +0200
@@ -171,6 +171,10 @@
assert(false);
return "";
}
+ virtual string ReadAggregatedFile(const string &, const string & , string *) {
+ assert(false);
+ return "";
+ }
virtual int RemoveFile(const string& path) {
assert(false);
return 0;
diff -u -r -P ./mainline/src/graph.cc ./patched/src/graph.cc
--- ./mainline/src/graph.cc 2011-10-24 15:12:45.677995700 +0200
+++ ./patched/src/graph.cc 2011-10-24 15:13:22.621748900 +0200
@@ -229,7 +229,62 @@
bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface,
string* err) {
string path = EvaluateDepFile();
- string content = disk_interface->ReadFile(path, err);
+
+ string content;
+ size_t colon;
+ string pathActuallyLoaded = path;
+ if ((colon = path.find(":")) != string::npos)
+ {
+ size_t colon2 = path.find(":", colon + 1);
+ if (colon2 == string::npos)
+ {
+ *err = path + ": invalid depfile format";
+ return false;
+ }
+
+ // D:o:d case
+ const string group = path.substr(0, colon);
+ const string filename = path.substr(colon + 1, colon2 - colon - 1);
+ const string fallback = path.substr(colon2 + 1);
+
+ // fallback logic
+ time_t most_recent_output = 1;
+ for (vector<Node *>::iterator node = outputs_.begin(); node != outputs_.end(); node++)
+ {
+ // stat the output file, but don't mark it as processed
+ (*node)->StatIfNecessary(disk_interface, false);
+ most_recent_output = max((*node)->mtime(), most_recent_output);
+ }
+
+ // get the aggregated depfile timestamp
+ Node * aggregated = state->GetNode(group);
+ if (aggregated == NULL)
+ {
+ *err = "Unable to acquire a STAT object for " + group;
+ return false;
+ }
+ aggregated->StatIfNecessary(disk_interface, false);
+
+ if (aggregated->exists() && (aggregated->mtime() >= most_recent_output))
+ {
+ // use the aggregated depfile (D)
+ content = disk_interface->ReadAggregatedFile(group, filename, err);
+ pathActuallyLoaded = group;
+ }
+ else
+ {
+ // fallback to using the individual .d
+ content = disk_interface->ReadFile(fallback, err);
+ pathActuallyLoaded = fallback;
+ }
+ }
+ else
+ {
+ // plain vanilla .d file
+ content = disk_interface->ReadFile(path, err);
+ }
+
+
if (!err->empty())
return false;
if (content.empty())
@@ -238,7 +293,7 @@
DepfileParser depfile;
string depfile_err;
if (!depfile.Parse(&content, &depfile_err)) {
- *err = path + ": " + depfile_err;
+ *err = pathActuallyLoaded + ": " + depfile_err;
return false;
}
diff -u -r -P ./mainline/src/graph.h ./patched/src/graph.h
--- ./mainline/src/graph.h 2011-10-24 15:12:45.693596100 +0200
+++ ./patched/src/graph.h 2011-10-24 15:13:22.652949700 +0200
@@ -29,7 +29,7 @@
/// it's dirty, mtime, etc.
struct Node {
Node(const string& path) : path_(path), mtime_(-1), dirty_(false),
- in_edge_(NULL) {}
+ in_edge_(NULL), to_be_processed_(true) {}
/// Return true if the file exists (mtime_ got a value).
bool Stat(DiskInterface* disk_interface);
@@ -34,14 +34,28 @@
/// Return true if the file exists (mtime_ got a value).
bool Stat(DiskInterface* disk_interface);
- /// Return true if we needed to stat.
- bool StatIfNecessary(DiskInterface* disk_interface) {
- if (status_known())
- return false;
- Stat(disk_interface);
- return true;
+ /// Flag 'for_processing' indicates, that the caller only asks for the timestamp and does not
+ /// intend to process the node in terms of dependency analysis. The flag is remembered
+ /// and its value is used in the return value of the next call. Thus, the method returns
+ /// true
+ bool StatIfNecessary(DiskInterface* disk_interface, bool for_processing = true) {
+ if (!status_known()) {
+ Stat(disk_interface);
+ }
+
+ if (for_processing) {
+ // caller will process this file right now
+ bool ret = to_be_processed_;
+ to_be_processed_ = false;
+ return ret;
+ }
+ else {
+ // caller is not interested to process this file right now
+ return to_be_processed_;
+ }
}
+
/// Mark as not-yet-stat()ed and not dirty.
void ResetState() {
mtime_ = -1;
@@ -60,6 +60,7 @@
void ResetState() {
mtime_ = -1;
dirty_ = false;
+ to_be_processed_ = true;
}
/// Mark the Node as already-stat()ed and missing.
@@ -107,6 +107,9 @@
/// All Edges that use this Node as an input.
vector<Edge*> out_edges_;
+
+ /// Indicates if the aggregated dep file has to be processed
+ bool to_be_processed_;
};
/// An invokable build command and associated metadata (description, etc.).
diff -u -r -P ./mainline/src/test.cc ./patched/src/test.cc
--- ./mainline/src/test.cc 2011-10-24 15:12:45.927602100 +0200
+++ ./patched/src/test.cc 2011-10-24 15:13:22.902556100 +0200
@@ -59,6 +59,14 @@
return "";
}
+string VirtualFileSystem::ReadAggregatedFile(const string& group_path, const string& element_path, string* err) {
+ files_read_.push_back(group_path);
+ FileMap::iterator i = files_.find(group_path);
+ if (i != files_.end())
+ return i->second.contents;
+ return "";
+}
+
int VirtualFileSystem::RemoveFile(const string& path) {
if (find(directories_made_.begin(), directories_made_.end(), path)
!= directories_made_.end())
diff -u -r -P ./mainline/src/test.h ./patched/src/test.h
--- ./mainline/src/test.h 2011-10-24 15:12:45.927602100 +0200
+++ ./patched/src/test.h 2011-10-24 15:13:22.902556100 +0200
@@ -45,6 +45,7 @@
virtual int Stat(const string& path);
virtual bool MakeDir(const string& path);
virtual string ReadFile(const string& path, string* err);
+ virtual string ReadAggregatedFile(const string& group_path, const string& element_path, string* err);
virtual int RemoveFile(const string& path);
/// An entry for a single in-memory file.
diff -u -r -P ./patched/src/build.cc ./patched/src/build.cc
--- ./mainline/src/build.cc 2011-12-01 11:25:25.000000000 +0000
+++ ./patched/src/build.cc 2011-12-01 11:24:17.000000000 +0000
@@ -32,6 +32,7 @@
#include "state.h"
#include "subprocess.h"
#include "util.h"
+#include <time.h>
/// Tracks the status of a build: completion fraction, printing updates.
struct BuildStatus {
@@ -614,26 +615,7 @@
}
if (node_cleaned) {
- // If any output was cleaned, find the most recent mtime of any
- // (existing) non-order-only input or the depfile.
- for (vector<Node*>::iterator i = edge->inputs_.begin();
- i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
- time_t input_mtime = disk_interface_->Stat((*i)->path());
- if (input_mtime == 0) {
- restat_mtime = 0;
- break;
- }
- if (input_mtime > restat_mtime)
- restat_mtime = input_mtime;
- }
-
- if (restat_mtime != 0 && !edge->rule().depfile().empty()) {
- time_t depfile_mtime = disk_interface_->Stat(edge->EvaluateDepFile());
- if (depfile_mtime == 0)
- restat_mtime = 0;
- else if (depfile_mtime > restat_mtime)
- restat_mtime = depfile_mtime;
- }
+ restat_mtime = time(NULL);
// The total number of edges in the plan may have changed as a result
// of a restat.
diff -u -r -P ./mainline/src/build.cc ./patched/src/build.cc
--- ./mainline/src/build.cc 2011-08-22 16:04:17.684730100 +0200
+++ ./patched/src/build.cc 2011-08-22 16:08:43.803459100 +0200
@@ -22,6 +22,7 @@
#include <sys/ioctl.h>
#include <sys/time.h>
#include <sys/termios.h>
+#include <errno.h>
#endif
#include "build_log.h"
@@ -321,7 +322,11 @@
if ((*ni)->file_->mtime_ > most_recent_input)
most_recent_input = (*ni)->file_->mtime_;
string command = (*ei)->EvaluateCommand();
-
+ if ((*ei)->HasRspFile())
+ {
+ command += " " + (*ei)->GetRspFileContent();
+ }
+
// Now, recompute the dirty state of each output.
bool all_outputs_clean = true;
for (vector<Node*>::iterator ni = (*ei)->outputs_.begin();
@@ -364,6 +369,36 @@
printf("ready: %d\n", (int)ready_.size());
}
+void CreateRspFile(const char * rspFileName, const char * rspFileContent) {
+ FILE * rspFile = fopen(rspFileName, "w");
+ if (rspFile == NULL) {
+#ifdef _WIN32
+ fprintf(stderr, "Unable to create rsp file %s\n", rspFileName);
+ Fatal("CreateRspFile : %s", GetLastErrorString().c_str());
+#else
+ Fatal("Unable to create rsp file %s: %s\n", rspFileName, strerror( errno ));
+#endif
+ }
+
+ if (fputs(rspFileContent, rspFile) == EOF) {
+#ifdef _WIN32
+ fprintf(stderr, "Unable to write to the rsp file %s\n", rspFileName);
+ Fatal("CreateRspFile : %s", GetLastErrorString().c_str());
+#else
+ Fatal("Unable to write the rsp file %s: %s\n", rspFileName, strerror( errno ));
+#endif
+ }
+
+ if (fclose(rspFile) == EOF) {
+#ifdef _WIN32
+ fprintf(stderr, "Unable to close the rsp file %s\n", rspFileName);
+ Fatal("CreateRspFile : %s", GetLastErrorString().c_str());
+#else
+ Fatal("Unable to close the rsp file %s: %s\n", rspFileName, strerror( errno ));
+#endif
+ }
+}
+
struct RealCommandRunner : public CommandRunner {
RealCommandRunner(const BuildConfig& config) : config_(config) {}
virtual ~RealCommandRunner() {}
@@ -380,8 +415,15 @@
return ((int)subprocs_.running_.size()) < config_.parallelism;
}
-bool RealCommandRunner::StartCommand(Edge* edge) {
+bool RealCommandRunner::StartCommand(Edge* edge) {
+ // create a RSP file, if specified
+ if (edge->HasRspFile()) {
+ CreateRspFile(edge->GetRspFile().c_str(), edge->GetRspFileContent().c_str());
+ }
+
+ // run the command
string command = edge->EvaluateCommand();
+
Subprocess* subproc = new Subprocess;
subproc_to_edge_.insert(make_pair(subproc, edge));
if (!subproc->Start(&subprocs_, command))
diff -u -r -P ./mainline/src/build_log.cc ./patched/src/build_log.cc
--- ./mainline/src/build_log.cc 2011-08-22 15:20:37.451195100 +0200
+++ ./patched/src/build_log.cc 2011-08-22 16:08:43.803459100 +0200
@@ -100,7 +100,9 @@
void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
time_t restat_mtime) {
- const string command = edge->EvaluateCommand();
+ string command = edge->EvaluateCommand();
+ if (edge->HasRspFile())
+ command += " " + edge->GetRspFileContent();
for (vector<Node*>::iterator out = edge->outputs_.begin();
out != edge->outputs_.end(); ++out) {
const string& path = (*out)->file_->path_;
diff -u -r -P ./mainline/src/clean.cc ./patched/src/clean.cc
--- ./mainline/src/clean.cc 2011-08-22 16:06:20.507324600 +0200
+++ ./patched/src/clean.cc 2011-08-22 16:08:43.819059500 +0200
@@ -115,6 +115,8 @@
}
if (!(*e)->rule().depfile_.empty())
Remove((*e)->EvaluateDepFile());
+ if ((*e)->HasRspFile())
+ Remove((*e)->GetRspFile());
}
PrintFooter();
return status_;
@@ -121,6 +121,11 @@
void Cleaner::DoCleanTarget(Node* target) {
if (target->in_edge()) {
Remove(target->path());
+
+ if (target->in_edge()->HasRspFile()) {
+ Remove(target->in_edge()->GetRspFile());
+ }
+
for (vector<Node*>::iterator n = target->in_edge()->inputs_.begin();
n != target->in_edge()->inputs_.end();
++n) {
@@ -182,6 +187,10 @@
out_node != (*e)->outputs_.end(); ++out_node) {
Remove((*out_node)->path());
}
+
+ if ((*e)->HasRspFile()) {
+ Remove((*e)->GetRspFile());
+ }
}
}
}
diff -u -r -P ./mainline/src/clean_test.cc ./patched/src/clean_test.cc
--- ./mainline/src/clean_test.cc 2011-08-22 16:06:19.586901000 +0200
+++ ./patched/src/clean_test.cc 2011-08-22 16:08:43.819059500 +0200
@@ -249,6 +249,63 @@
EXPECT_EQ(2u, fs_.files_removed_.size());
}
+TEST_F(CleanTest, CleanRspFile) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n"
+" command = cc $in > $out\n"
+" rspfile = $rspfile\n"
+"build out1: cc in1\n"
+" rspfile = cc1.rsp\n"));
+ fs_.Create("out1", 1, "");
+ fs_.Create("cc1.rsp", 1, "");
+
+ Cleaner cleaner(&state_, config_, &fs_);
+ EXPECT_EQ(0, cleaner.CleanAll());
+ EXPECT_EQ(2, cleaner.cleaned_files_count());
+ EXPECT_EQ(2u, fs_.files_removed_.size());
+}
+
+TEST_F(CleanTest, CleanRsp) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cat_rsp \n"
+" command = cat $rspfile > $out\n"
+" rspfile = $rspfile\n"
+" rspfile_content = $in\n"
+"build in1: cat src1\n"
+"build out1: cat in1\n"
+"build in2: cat_rsp src2\n"
+" rspfile=in2.rsp\n"
+"build out2: cat_rsp in2\n"
+" rspfile=out2.rsp\n"));
+ fs_.Create("in1", 1, "");
+ fs_.Create("out1", 1, "");
+ fs_.Create("in2.rsp", 1, "");
+ fs_.Create("out2.rsp", 1, "");
+ fs_.Create("in2", 1, "");
+ fs_.Create("out2", 1, "");
+
+ Cleaner cleaner(&state_, config_, &fs_);
+ ASSERT_EQ(0, cleaner.cleaned_files_count());
+ ASSERT_EQ(0, cleaner.CleanTarget("out1"));
+ EXPECT_EQ(2, cleaner.cleaned_files_count());
+ ASSERT_EQ(0, cleaner.CleanTarget("in2"));
+ EXPECT_EQ(2, cleaner.cleaned_files_count());
+ ASSERT_EQ(0, cleaner.CleanRule("cat_rsp"));
+ EXPECT_EQ(2, cleaner.cleaned_files_count());
+
+ EXPECT_EQ(6u, fs_.files_removed_.size());
+
+ // Check they are removed.
+ EXPECT_EQ(0, fs_.Stat("in1"));
+ EXPECT_EQ(0, fs_.Stat("out1"));
+ EXPECT_EQ(0, fs_.Stat("in2"));
+ EXPECT_EQ(0, fs_.Stat("out2"));
+ EXPECT_EQ(0, fs_.Stat("in2.rsp"));
+ EXPECT_EQ(0, fs_.Stat("out2.rsp"));
+
+ fs_.files_removed_.clear();
+}
+
TEST_F(CleanTest, CleanFailure) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build dir: cat src1\n"));
diff -u -r -P ./mainline/src/graph.cc ./patched/src/graph.cc
--- ./mainline/src/graph.cc 2011-08-22 15:20:37.341992300 +0200
+++ ./patched/src/graph.cc 2011-08-22 16:30:44.429976500 +0200
@@ -75,6 +75,8 @@
if (!dirty) {
BuildLog* build_log = state ? state->build_log_ : 0;
string command = EvaluateCommand();
+ if (HasRspFile())
+ command += " " + GetRspFileContent();
for (vector<Node*>::iterator i = outputs_.begin();
i != outputs_.end(); ++i) {
@@ -129,6 +131,20 @@
return rule_->description_.Evaluate(&env);
}
+bool Edge::HasRspFile() {
+ return !rule_->rspfile_.empty();
+}
+
+string Edge::GetRspFile() {
+ EdgeEnv env(this);
+ return rule_->rspfile_.Evaluate(&env);
+}
+
+string Edge::GetRspFileContent() {
+ EdgeEnv env(this);
+ return rule_->rspfile_content_.Evaluate(&env);
+}
+
bool Edge::LoadDepFile(State* state, DiskInterface* disk_interface,
string* err) {
EdgeEnv env(this);
diff -u -r -P ./mainline/src/graph.h ./patched/src/graph.h
--- ./mainline/src/graph.h 2011-08-22 16:07:00.707566800 +0200
+++ ./patched/src/graph.h 2011-08-22 16:08:43.834659900 +0200
@@ -109,6 +109,8 @@
const EvalString& description() const { return description_; }
const EvalString& depfile() const { return depfile_; }
+ EvalString rspfile_;
+ EvalString rspfile_content_;
private:
// Allow the parsers to reach into this object and fill out its fields.
friend class ManifestParser;
@@ -85,6 +87,9 @@
bool dirty, const string& command, Node* output);
string EvaluateCommand(); // XXX move to env, take env ptr
string GetDescription();
+ bool HasRspFile();
+ string GetRspFile();
+ string GetRspFileContent();
bool LoadDepFile(State* state, DiskInterface* disk_interface, string* err);
void Dump();
diff -u -r -P ./mainline/src/parsers.cc ./patched/src/parsers.cc
--- ./mainline/src/parsers.cc 2011-08-22 15:20:37.341992300 +0200
+++ ./patched/src/parsers.cc 2011-08-22 16:08:43.834659900 +0200
@@ -395,6 +395,10 @@
if (!tokenizer_.ReadToNewline(&dummy, err))
return false;
continue;
+ } else if (key == "rspfile") {
+ eval_target = &rule->rspfile_;
+ } else if (key == "rspfile_content") {
+ eval_target = &rule->rspfile_content_;
} else {
// Die on other keyvals for now; revisit if we want to add a
// scope here.
diff -u -r -P ./mainline/src/parsers_test.cc ./patched/src/parsers_test.cc
--- ./mainline/src/parsers_test.cc 2011-08-22 15:20:37.373193100 +0200
+++ ./patched/src/parsers_test.cc 2011-08-22 16:08:43.850260300 +0200
@@ -64,6 +64,24 @@
EXPECT_EQ("cat $in > $out", rule->command().unparsed());
}
+TEST_F(ParserTest, ResponseFiles) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"rule cat_rsp\n"
+" command = cat $rspfile > $out\n"
+" rspfile = $rspfile\n"
+" rspfile_content = $in\n"
+"\n"
+"build out: cat_rsp in\n"
+" rspfile=out.rsp\n"));
+
+ ASSERT_EQ(2u, state.rules_.size());
+ const Rule* rule = state.rules_.begin()->second;
+ EXPECT_EQ("cat_rsp", rule->name());
+ EXPECT_EQ("cat $rspfile > $out", rule->command().unparsed());
+ EXPECT_EQ("$rspfile", rule->rspfile_.unparsed());
+ EXPECT_EQ("$in", rule->rspfile_content_.unparsed());
+}
+
TEST_F(ParserTest, Variables) {
ASSERT_NO_FATAL_FAILURE(AssertParse(
"l = one-letter-test\n"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.