Skip to content

Instantly share code, notes, and snippets.

@TylerBrock

TylerBrock/find_stuff.py

Last active January 3, 2016 12:19
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save TylerBrock/8461921 to your computer and use it in GitHub Desktop.
Save TylerBrock/8461921 to your computer and use it in GitHub Desktop.
Download ZIP
Script to find headers required for building libmongoclient.
Raw
find_stuff.py
#!/usr/bin/env python
import subprocess
import argparse
import shutil
import os
from sets import Set
client_source_basic = [
#'mongo/base/error_codes.cpp',
'mongo/base/global_initializer.cpp',
'mongo/base/global_initializer_registerer.cpp',
'mongo/base/init.cpp',
'mongo/base/initializer.cpp',
'mongo/base/initializer_context.cpp',
'mongo/base/initializer_dependency_graph.cpp',
'mongo/base/make_string_vector.cpp',
'mongo/base/parse_number.cpp',
'mongo/base/status.cpp',
'mongo/base/string_data.cpp',
'mongo/bson/bson_validate.cpp',
'mongo/bson/oid.cpp',
'mongo/bson/util/bson_extract.cpp',
#'mongo/buildinfo.cpp',
'mongo/client/auth_helpers.cpp',
'mongo/client/clientAndShell.cpp',
'mongo/client/clientOnly.cpp',
'mongo/client/connpool.cpp',
'mongo/client/dbclient.cpp',
'mongo/client/dbclient_rs.cpp',
'mongo/client/dbclientcursor.cpp',
'mongo/client/gridfs.cpp',
'mongo/client/init.cpp',
'mongo/client/sasl_client_authenticate.cpp',
'mongo/client/syncclusterconnection.cpp',
'mongo/db/jsobj.cpp',
'mongo/db/json.cpp',
'mongo/db/lasterror.cpp',
'mongo/db/dbmessage.cpp',
'mongo/db/server_options.cpp',
'mongo/logger/console.cpp',
'mongo/logger/log_manager.cpp',
'mongo/logger/log_severity.cpp',
'mongo/logger/logger.cpp',
'mongo/logger/logstream_builder.cpp',
'mongo/logger/message_event_utf8_encoder.cpp',
'mongo/logger/message_log_domain.cpp',
'mongo/logger/ramlog.cpp',
'mongo/logger/rotatable_file_manager.cpp',
'mongo/logger/rotatable_file_writer.cpp',
'mongo/pch.cpp',
'mongo/platform/backtrace.cpp',
'mongo/platform/posix_fadvise.cpp',
'mongo/platform/process_id.cpp',
'mongo/platform/random.cpp',
'mongo/util/assert_util.cpp',
'mongo/util/background.cpp',
'mongo/util/base64.cpp',
'mongo/util/concurrency/rwlockimpl.cpp',
'mongo/util/concurrency/spin_lock.cpp',
'mongo/util/concurrency/synchronization.cpp',
'mongo/util/concurrency/task.cpp',
'mongo/util/concurrency/thread_name.cpp',
'mongo/util/concurrency/thread_pool.cpp',
'mongo/util/concurrency/mutexdebugger.cpp',
'mongo/util/debug_util.cpp',
'mongo/util/stacktrace.cpp',
'mongo/util/file.cpp',
'mongo/util/file_allocator.cpp',
'mongo/util/fail_point.cpp',
'mongo/util/fail_point_registry.cpp',
'mongo/util/fail_point_service.cpp',
'mongo/util/hex.cpp',
'mongo/util/histogram.cpp',
'mongo/util/intrusive_counter.cpp',
'mongo/util/log.cpp',
'mongo/util/md5.cpp',
'mongo/util/md5main.cpp',
'mongo/util/net/httpclient.cpp',
'mongo/util/net/listen.cpp',
'mongo/util/net/message.cpp',
'mongo/util/net/message_port.cpp',
'mongo/util/net/sock.cpp',
"mongo/util/net/socket_poll.cpp",
'mongo/util/net/ssl_manager.cpp',
'mongo/util/password.cpp',
'mongo/util/paths.cpp',
'mongo/util/processinfo.cpp',
'mongo/util/signal_handlers.cpp',
'mongo/util/stringutils.cpp',
'mongo/util/text.cpp',
'mongo/util/time_support.cpp',
'mongo/util/timer.cpp',
'mongo/util/trace.cpp',
'mongo/util/util.cpp',
'mongo/util/version.cpp',
'third_party/murmurhash3/MurmurHash3.cpp',
]
client_source_sasl = [
'mongo/client/sasl_client_authenticate_impl.cpp',
'mongo/client/sasl_client_session.cpp'
]
client_source_process_info = [
'mongo/util/processinfo_darwin.cpp',
'mongo/util/processinfo_freebsd.cpp',
'mongo/util/processinfo_linux2.cpp',
'mongo/util/processinfo_none.cpp',
'mongo/util/processinfo_sunos5.cpp',
'mongo/util/processinfo_win32.cpp'
]
test_source = [
'mongo/client/include_bsonh_test.cpp',
'mongo/client/include_dbclienth_test.cpp',
]
example_source = [
'mongo/client/examples/authTest.cpp',
'mongo/client/examples/clientTest.cpp',
'mongo/client/examples/first.cpp',
'mongo/client/examples/httpClientTest.cpp',
#'mongo/client/examples/insert_demo.cpp',
'mongo/client/examples/rs.cpp',
'mongo/client/examples/second.cpp',
'mongo/client/examples/tutorial.cpp',
'mongo/client/examples/whereExample.cpp',
'mongo/bson/bsondemo/bsondemo.cpp',
]
# src level relative
client_sources = (
client_source_basic +
client_source_sasl +
client_source_process_info +
test_source +
example_source
)
# top level relative
build_files = [
'SConstruct',
'Sconscript.buildinfo',
'buildscripts/__init__.py',
'buildscripts/moduleconfig.py',
'buildscripts/utils.py',
'buildscripts/errorcodes.py',
'site_scons/libdeps.py',
'site_scons/site_tools/gch.py',
'site_scons/site_tools/jsheader.py',
'site_scons/site_tools/mergelib.py',
'site_scons/site_tools/unittest.py',
'jstests/libs/key1',
'jstests/libs/key2',
'src/SConscript.client',
'src/mongo/base/generate_error_codes.py',
'src/mongo/base/error_codes.err',
'src/mongo/db/auth/generate_action_types.py',
'src/mongo/db/auth/action_types.txt',
'src/third_party/SConscript',
'src/third_party/shim_boost.cpp',
'src/SConscript',
]
defines = [
'_WIN32',
'_DEBUG',
'_SCONS',
'_MSC_EXTENSIONS',
'_M_IX86',
'MONGO_GCOV',
'MONGO_SSL',
'MONGO_MALLOC',
'MONGO_EXPOSE_MACROS',
'MONGO_MACROS_PUSHED',
'__APPLE__',
'__openbsd__',
'__sunos__',
'__linux__',
'__clang__',
'__openbsd__',
'ARCH_IS_BIG_ENDIAN',
'OPENSSL_FIPS',
'SO_NOSIGPIPE',
'SO_KEEPALIVE',
'MSG_NOSIGNAL',
'EAI_NODATA',
'SOL_TCP',
'USE_ASIO',
'EINTR',
'POSIX_FADV_DONTNEED',
'POSIX_FADV_SEQUENTIAL',
'BOOST_RNG_HACK'
'LOKI_SCOPEGUARD_H_',
'MAP_NORESERVE',
'O_NOATIME',
'md5_INCLUDED',
'TH_FLAGS',
'UTIL_HISTOGRAM_HEADER',
'UTIL_VERSION_HEADER',
'LATER',
]
header_excludes = [
'malloc.h',
'procfs.h',
'kvm.h',
'psapi',
'vm/vm_param.h',
'sys/systeminfo.h',
'gnu/libc-version.h',
'psapi.h',
'mongo/db/auth/action_type.h',
'mongo/base/error_codes.h',
]
def process_source(source, src_path):
definitions = ['-D_XOPEN_SOURCE', '-D_DEBUG', '-D_SCONS', '-DMONGO_SSL']
# ['-D' + define for define in defines]
cmd = ['clang++', '-MM', '-MG', '-I.']
cmd.extend(definitions)
cmd.append(source)
#print 'CMD: ' + str(cmd)
process = subprocess.Popen(
cmd,
cwd=os.path.join(src_path, 'src'),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
return process.communicate()
def process_output(output):
lines = [line for line in output.split() if line != '\\']
file_name = lines[1]
headers = lines[2:]
return (file_name, headers)
def get_headers(src_path):
required_headers = Set()
for source in client_sources:
#print 'Required HEADERS: ' + str(required_headers)
output, error = process_source(source, src_path)
if error:
print("ERROR:" + error)
else:
_, headers = process_output(output)
required_headers.update(headers)
return required_headers
def get_args():
parser = argparse.ArgumentParser(
description='Find required headers for mongoclient'
)
parser.add_argument(
'--srcpath',
dest='src_path',
required=True,
help='path to mongo repository'
)
parser.add_argument(
'--destpath',
dest='dest_path',
help='path to new repository'
)
return parser.parse_args()
def copy_file(src_file, dest_file):
dir_name = os.path.dirname(dest_file)
if not os.path.exists(dir_name):
print 'Making Directory: ' + dir_name
os.makedirs(dir_name)
try:
shutil.copyfile(
os.path.abspath(src_file),
os.path.abspath(dest_file)
)
except IOError as error:
print 'Could not copy: ' + src_file
print error
def copy_files(src_path, dest_path, file_list, relative=''):
for single_file in file_list:
src_file = os.path.join(src_path, relative, single_file)
dest_file = os.path.join(dest_path, relative, single_file)
copy_file(src_file, dest_file)
if __name__ == '__main__':
args = get_args()
required_headers = get_headers(args.src_path) - Set(header_excludes)
for header in sorted(list(required_headers)):
print header
if args.dest_path:
copy_files(args.src_path, args.dest_path, build_files)
copy_files(args.src_path, args.dest_path, client_sources, 'src')
copy_files(args.src_path, args.dest_path, required_headers, 'src')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment