Skip to content

Instantly share code, notes, and snippets.

@blorente
Last active June 26, 2019 14:14
Show Gist options
  • Save blorente/43ac94ec47b63c202ea31a7f783e444e to your computer and use it in GitHub Desktop.
Save blorente/43ac94ec47b63c202ea31a7f783e444e to your computer and use it in GitHub Desktop.
function _launch_dockerized_pants() {
(
set -xe
pants_location=$1
docker_image=$2
cd "${pants_location}"
# Build the docker image locally
build_hash=$( docker build "build-support/docker/${docker_image}" | tail -n 1 | awk '{print $3}' )
# Launch the image with ~/.cache/pants and your local pants copy as mounted volumes
docker run -v $(realpath "${pants_location}"):/pants -v ~/.cache/pants:/home/.cache/pants -it "${build_hash}" /bin/bash
)
}
# E.g. dockerized_pants centos7
alias dockerized_pants="_launch_dockerized_pants $(pwd) "
source ~/aliases.sh
$ cd copy/of/pants
$ dockerized_pants centos7
$docker> export LANG=en_US.UTF-8
$docker> export LC_ALL=en_US.UTF-8
$docker> ./pants test tests/python/pants_test/util:dirutil -- -s
[root@dfae087454e6 pants]# ./pants test tests/python/pants_test/util:dirutil -- -s
13:40:11 00:00 [main]
(To run a reporting server: ./pants server)
13:40:18 00:07 [setup]
13:40:22 00:11 [parse]
Executing tasks in goals: bootstrap -> imports -> unpack-jars -> unpack-wheels -> deferred-sources -> native-compile -> link -> jvm-platform-validate -> gen -> resolve -> resources -> pyprep -> compile -> test
13:40:23 00:12 [bootstrap]
13:40:23 00:12 [substitute-aliased-targets]
13:40:23 00:12 [jar-dependency-management]
13:40:23 00:12 [bootstrap-jvm-tools]
13:40:23 00:12 [provide-tools-jar]
13:40:24 00:13 [background]
13:40:24 00:13 [workdir_build_cleanup]
13:40:24 00:13 [imports]
13:40:24 00:13 [ivy-imports]
13:40:24 00:13 [unpack-jars]
13:40:24 00:13 [unpack-jars]
13:40:24 00:13 [workdir_build_cleanup]
13:40:24 00:13 [unpack-wheels]
13:40:24 00:13 [unpack-wheels]
13:40:24 00:13 [workdir_build_cleanup]
13:40:24 00:13 [deferred-sources]
13:40:24 00:13 [deferred-sources]
13:40:24 00:13 [native-compile]
13:40:24 00:13 [conan-prep]
13:40:24 00:13 [conan-fetch]
13:40:24 00:13 [c-for-ctypes]
13:40:24 00:13 [workdir_build_cleanup]
13:40:24 00:13 [cpp-for-ctypes]
13:40:24 00:13 [workdir_build_cleanup]
13:40:24 00:13 [link]
13:40:24 00:13 [shared-libraries]
13:40:24 00:13 [workdir_build_cleanup]
13:40:24 00:13 [jvm-platform-validate]
13:40:24 00:13 [jvm-platform-validate]
13:40:24 00:13 [workdir_build_cleanup]
13:40:24 00:13 [gen]
13:40:24 00:13 [antlr-java]
13:40:24 00:13 [antlr-py]
13:40:24 00:13 [jaxb]
13:40:24 00:13 [protoc]
13:40:24 00:13 [ragel]
13:40:24 00:13 [thrift-java]
13:40:24 00:13 [thrift-py]
13:40:24 00:13 [py-thrift-namespace-clash-check]
13:40:24 00:13 [grpcio-prep]
13:40:24 00:13 [grpcio-run]
13:40:24 00:13 [wire]
13:40:24 00:13 [avro-java]
13:40:25 00:14 [go-thrift]
13:40:25 00:14 [go-protobuf]
13:40:25 00:14 [jax-ws]
13:40:25 00:14 [scrooge]
13:40:25 00:14 [thrifty]
13:40:25 00:14 [resolve]
13:40:25 00:14 [ivy]
13:40:25 00:14 [coursier]
13:40:25 00:14 [go]
13:40:25 00:14 [scala-js-compile]
13:40:25 00:14 [scala-js-link]
13:40:25 00:14 [workdir_build_cleanup]
13:40:25 00:14 [node]
13:40:25 00:14 [resources]
13:40:25 00:14 [prepare]
13:40:25 00:14 [services]
13:40:25 00:14 [pyprep]
13:40:25 00:14 [interpreter]
Invalidated 11 targets.
13:40:26 00:15 [workdir_build_cleanup]
13:40:26 00:15 [build-local-dists]
13:40:26 00:15 [requirements]
Invalidated 6 targets.
13:40:36 00:25 [workdir_build_cleanup]
13:40:36 00:25 [sources]
Invalidated 14 targets.
13:40:37 00:26 [workdir_build_cleanup]
13:40:37 00:26 [compile]
13:40:37 00:26 [node]
13:40:37 00:26 [workdir_build_cleanup]
13:40:37 00:26 [compile-jvm-prep-command]
13:40:37 00:26 [jvm_prep_command]
13:40:37 00:26 [compile-prep-command]
13:40:37 00:26 [compile]
13:40:37 00:26 [rsc]
13:40:37 00:26 [zinc]
13:40:37 00:26 [javac]
13:40:37 00:26 [cpp]
13:40:37 00:26 [workdir_build_cleanup]
13:40:37 00:26 [errorprone]
13:40:37 00:26 [workdir_build_cleanup]
13:40:37 00:26 [findbugs]
13:40:37 00:26 [workdir_build_cleanup]
13:40:37 00:26 [go]
13:40:37 00:26 [workdir_build_cleanup]
13:40:37 00:26 [test]
13:40:37 00:26 [test-jvm-prep-command]
13:40:37 00:26 [jvm_prep_command]
13:40:37 00:26 [test-prep-command]
13:40:37 00:26 [legacy]
13:40:37 00:26 [pytest-prep]
Invalidated 20 targets.
13:40:41 00:30 [workdir_build_cleanup]
13:40:42 00:31 [pytest]
Invalidated 1 target.
scrubbed PYTHONPATH=/pants/src/python:/opt/rh/devtoolset-7/root/usr/lib64/python2.7/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python2.7/site-packages from pytest environment
13:40:42 00:31 [run]
============== test session starts ===============
platform linux -- Python 3.6.8, pytest-3.6.4, py-1.8.0, pluggy-0.7.1
rootdir: /pants, inifile: /dev/null
plugins: timeout-1.2.1, cov-2.4.0
collected 50 items
tests/python/pants_test/util/test_dirutil.py ..................................F.. def move(src, dst, copy_function=copy2):
"""Recursively move a file or directory to another location. This is
similar to the Unix "mv" command. Return the file or directory's
destination.
If the destination is a directory or a symlink to a directory, the source
is moved inside the directory. The destination path must not already
exist.
If the destination already exists but is not a directory, it may be
overwritten depending on os.rename() semantics.
If the destination is on our current filesystem, then rename() is used.
Otherwise, src is copied to the destination and then removed. Symlinks are
recreated under the new name if os.rename() fails because of cross
filesystem renames.
The optional `copy_function` argument is a callable that will be used
to copy the source or it will be delegated to `copytree`.
By default, copy2() is used, but any function that supports the same
signature (like copy()) can be used.
A lot more could be done here... A look at a mv.c shows a lot of
the issues this implementation glosses over.
"""
real_dst = dst
if os.path.isdir(dst):
if _samefile(src, dst):
# We might be on a case insensitive filesystem,
# perform the rename anyway.
os.rename(src, dst)
return
real_dst = os.path.join(dst, _basename(src))
if os.path.exists(real_dst):
raise Error("Destination path '%s' already exists" % real_dst)
try:
> os.rename(src, real_dst)
E NotADirectoryError: [Errno 20] Not a directory: '/tmp/tmpxy3fluh9/link/' -> '/tmp/tmpozim3fdg'
build-support/pants_dev_deps.py36.venv/lib/python3.6/shutil.py:550: NotADirectoryError
During handling of the above exception, another exception occurred:
self = <pants_test.util.test_dirutil.DirutilTest testMethod=test_safe_concurrent_rename_with_link_recursive>
def test_safe_concurrent_rename_with_link_recursive(self):
with temporary_dir() as td:
with temporary_dir() as td_renamed:
td_link = os.path.join(td, 'link')
os.symlink(td, td_link)
self.assertTrue(os.path.exists(td))
self.assertTrue(os.path.exists(td_renamed))
> safe_concurrent_rename(td_link, td_renamed, recursive=True)
.pants.d/pyprep/sources/9b5651ff1b5ba24487c7a9e14b637df0f444ec27/pants_test/util/test_dirutil.py:473:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.pants.d/pyprep/sources/9b5651ff1b5ba24487c7a9e14b637df0f444ec27/pants/util/dirutil.py:354: in safe_concurrent_rename
shutil.move(src, dst)
build-support/pants_dev_deps.py36.venv/lib/python3.6/shutil.py:562: in move
rmtree(src)
build-support/pants_dev_deps.py36.venv/lib/python3.6/shutil.py:490: in rmtree
onerror(os.rmdir, path, sys.exc_info())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
path = '/tmp/tmpxy3fluh9/link/'
ignore_errors = False
onerror = <function rmtree.<locals>.onerror at 0x7fa469a572f0>
def rmtree(path, ignore_errors=False, onerror=None):
"""Recursively delete a directory tree.
If ignore_errors is set, errors are ignored; otherwise, if onerror
is set, it is called to handle the error with arguments (func,
path, exc_info) where func is platform and implementation dependent;
path is the argument to that function that caused it to fail; and
exc_info is a tuple returned by sys.exc_info(). If ignore_errors
is false and onerror is None, an exception is raised.
"""
if ignore_errors:
def onerror(*args):
pass
elif onerror is None:
def onerror(*args):
raise
if _use_fd_functions:
# While the unsafe rmtree works fine on bytes, the fd based does not.
if isinstance(path, bytes):
path = os.fsdecode(path)
# Note: To guard against symlink races, we use the standard
# lstat()/open()/fstat() trick.
try:
orig_st = os.lstat(path)
except Exception:
onerror(os.lstat, path, sys.exc_info())
return
try:
fd = os.open(path, os.O_RDONLY)
except Exception:
onerror(os.lstat, path, sys.exc_info())
return
try:
if os.path.samestat(orig_st, os.fstat(fd)):
_rmtree_safe_fd(fd, path, onerror)
try:
> os.rmdir(path)
E FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmpxy3fluh9/link/'
build-support/pants_dev_deps.py36.venv/lib/python3.6/shutil.py:488: FileNotFoundError
__ DirutilTest.test_safe_rmtree_recursive_link ___
self = <pants_test.util.test_dirutil.DirutilTest testMethod=test_safe_rmtree_recursive_link>
def test_safe_rmtree_recursive_link(self):
with temporary_dir() as td:
real = os.path.join(td, 'real')
link = os.path.join(td, 'link')
os.mkdir(real)
os.symlink(real, link)
self.assertTrue(os.path.exists(real))
self.assertTrue(os.path.exists(link))
safe_rmtree(link, recursive=True)
> self.assertFalse(os.path.exists(link))
E AssertionError: True is not false
.pants.d/pyprep/sources/9b5651ff1b5ba24487c7a9e14b637df0f444ec27/pants_test/util/test_dirutil.py:543: AssertionError
- generated xml file: /pants/.pants.d/test/pytest/tests.python.pants_test.util.dirutil/junitxml/TEST-tests.python.pants_test.util.dirutil.xml -
====== 2 failed, 48 passed in 4.84 seconds =======
tests/python/pants_test/util:dirutil ..... FAILURE
FAILURE: FAILURE
Waiting for background workers to finish.
13:40:53 00:42 [complete]
FAILURE
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment