Skip to content

Instantly share code, notes, and snippets.

Created April 16, 2013 18:00
Show Gist options
  • Save anonymous/5398076 to your computer and use it in GitHub Desktop.
Save anonymous/5398076 to your computer and use it in GitHub Desktop.
diff --git a/configure.py b/configure.py
index 1203386..a33ed90 100755
--- a/configure.py
+++ b/configure.py
@@ -132,6 +132,7 @@ if platform == 'windows':
if not options.debug:
cflags += ['/Ox', '/DNDEBUG', '/GL']
ldflags += ['/LTCG', '/OPT:REF', '/OPT:ICF']
+ libs = []
else:
cflags = ['-g', '-Wall', '-Wextra',
'-Wno-deprecated',
@@ -151,7 +152,7 @@ else:
if platform == 'mingw':
cflags += ['-D_WIN32_WINNT=0x0501']
ldflags = ['-L$builddir']
-libs = []
+ libs = ['-lpthread']
if platform == 'mingw':
cflags.remove('-fvisibility=hidden');
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index a581114..3a1a96b 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -16,6 +16,7 @@
#include <assert.h>
#include <errno.h>
+#include <pthread.h>
#include <stdio.h>
#include <string.h>
@@ -25,10 +26,91 @@
#include "util.h"
#include "version.h"
-ManifestParser::ManifestParser(State* state, FileReader* file_reader)
+struct Lock {
+ Lock();
+};
+
+Lock::Lock() {
+}
+
+struct ScopedLock {
+ explicit ScopedLock(pthread_mutex_t* l) : lock_(l) {
+ pthread_mutex_lock(l);
+ }
+ ~ScopedLock() {
+ pthread_mutex_unlock(lock_);
+ }
+
+ pthread_mutex_t* lock_;
+};
+
+struct ManifestParser::ThreadPool {
+ ThreadPool();
+ pthread_mutex_t lock_;
+ vector<pthread_t> threads_;
+
+ struct WorkItem {
+ ManifestParser* parser;
+ string path;
+ };
+
+ void Start(WorkItem* work);
+ void Wait();
+
+ static void* Thread(void* arg);
+};
+
+ManifestParser::ThreadPool::ThreadPool() {
+ pthread_mutex_init(&lock_, NULL);
+}
+
+void ManifestParser::ThreadPool::Start(WorkItem* work) {
+ pthread_t thread;
+ ScopedLock l(&lock_);
+ pthread_create(&thread, NULL, Thread, work);
+ threads_.push_back(thread);
+}
+
+void ManifestParser::ThreadPool::Wait() {
+ for (;;) {
+ vector<pthread_t> threads;
+ {
+ ScopedLock l(&lock_);
+ swap(threads, threads_);
+ }
+
+ if (threads.empty())
+ break;
+ for (vector<pthread_t>::iterator i = threads.begin();
+ i != threads.end(); ++i) {
+ pthread_join(*i, NULL);
+ }
+ }
+}
+
+void* ManifestParser::ThreadPool::Thread(void* arg) {
+ WorkItem* work = reinterpret_cast<WorkItem*>(arg);
+
+ string err;
+ if (!work->parser->Load(work->path, &err))
+ assert(false);
+
+ return NULL;
+}
+
+ManifestParser::ManifestParser(State* state, FileReader* file_reader,
+ ThreadPool* thread_pool)
: state_(state), file_reader_(file_reader) {
env_ = &state->bindings_;
+ if (thread_pool) {
+ thread_pool_ = thread_pool;
+ wait_ = false;
+ } else {
+ thread_pool_ = new ThreadPool;
+ wait_ = true;
+ }
}
+
bool ManifestParser::Load(const string& filename, string* err) {
string contents;
string read_err;
@@ -37,7 +119,11 @@ bool ManifestParser::Load(const string& filename, string* err) {
return false;
}
contents.resize(contents.size() + 10);
- return Parse(filename, contents, err);
+ bool ret = Parse(filename, contents, err);
+ if (wait_) {
+ thread_pool_->Wait();
+ }
+ return ret;
}
bool ManifestParser::Parse(const string& filename, const string& input,
@@ -110,6 +196,7 @@ bool ManifestParser::ParsePool(string* err) {
if (!ExpectToken(Lexer::NEWLINE, err))
return false;
+ ScopedLock l(&thread_pool_->lock_);
if (state_->LookupPool(name) != NULL)
return lexer_.Error("duplicate pool '" + name + "'", err);
@@ -147,6 +234,7 @@ bool ManifestParser::ParseRule(string* err) {
if (!ExpectToken(Lexer::NEWLINE, err))
return false;
+ ScopedLock l(&thread_pool_->lock_);
if (state_->LookupRule(name) != NULL) {
*err = "duplicate rule '" + name + "'";
return false;
@@ -199,6 +287,7 @@ bool ManifestParser::ParseDefault(string* err) {
if (eval.empty())
return lexer_.Error("expected target name", err);
+ ScopedLock l(&thread_pool_->lock_);
do {
string path = eval.Evaluate(env_);
string path_err;
@@ -244,10 +333,6 @@ bool ManifestParser::ParseEdge(string* err) {
if (!lexer_.ReadIdent(&rule_name))
return lexer_.Error("expected build command name", err);
- const Rule* rule = state_->LookupRule(rule_name);
- if (!rule)
- return lexer_.Error("unknown build rule '" + rule_name + "'", err);
-
for (;;) {
// XXX should we require one path here?
EvalString in;
@@ -301,6 +386,12 @@ bool ManifestParser::ParseEdge(string* err) {
env->AddBinding(key, val.Evaluate(env_));
}
+ ScopedLock l(&thread_pool_->lock_);
+
+ const Rule* rule = state_->LookupRule(rule_name);
+ if (!rule)
+ return lexer_.Error("unknown build rule '" + rule_name + "'", err);
+
Edge* edge = state_->AddEdge(rule);
edge->env_ = env;
@@ -341,26 +432,21 @@ bool ManifestParser::ParseEdge(string* err) {
}
bool ManifestParser::ParseFileInclude(bool new_scope, string* err) {
- // XXX this should use ReadPath!
EvalString eval;
if (!lexer_.ReadPath(&eval, err))
return false;
string path = eval.Evaluate(env_);
- string contents;
- string read_err;
- if (!file_reader_->ReadFile(path, &contents, &read_err))
- return lexer_.Error("loading '" + path + "': " + read_err, err);
-
- ManifestParser subparser(state_, file_reader_);
+ ThreadPool::WorkItem* work = new ThreadPool::WorkItem;
+ work->parser = new ManifestParser(state_, file_reader_, thread_pool_);
if (new_scope) {
- subparser.env_ = new BindingEnv(env_);
+ work->parser->env_ = new BindingEnv(env_);
} else {
- subparser.env_ = env_;
+ work->parser->env_ = env_;
}
+ work->path = path;
- if (!subparser.Parse(path, contents, err))
- return false;
+ thread_pool_->Start(work);
if (!ExpectToken(Lexer::NEWLINE, err))
return false;
diff --git a/src/manifest_parser.h b/src/manifest_parser.h
index a08e5af..6b41c55 100644
--- a/src/manifest_parser.h
+++ b/src/manifest_parser.h
@@ -35,7 +35,9 @@ struct ManifestParser {
virtual bool ReadFile(const string& path, string* content, string* err) = 0;
};
- ManifestParser(State* state, FileReader* file_reader);
+ struct ThreadPool;
+ ManifestParser(State* state, FileReader* file_reader,
+ ThreadPool* thread_pool=NULL);
/// Load and parse a file.
bool Load(const string& filename, string* err);
@@ -67,6 +69,8 @@ private:
BindingEnv* env_;
FileReader* file_reader_;
Lexer lexer_;
+ ThreadPool* thread_pool_;
+ bool wait_;
};
#endif // NINJA_MANIFEST_PARSER_H_
diff --git a/src/ninja.cc b/src/ninja.cc
index 8ba1aa6..8ee290c 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -884,6 +884,7 @@ int NinjaMain(int argc, char** argv) {
bool rebuilt_manifest = false;
+ int i = 0;
reload:
RealFileReader file_reader;
ManifestParser parser(globals.state, &file_reader);
@@ -893,6 +894,11 @@ reload:
return 1;
}
+ if (++i < 10) {
+ globals.ResetState();
+ goto reload;
+ }
+
if (tool && tool->when == Tool::RUN_AFTER_LOAD)
return tool->func(&globals, argc, argv);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment