Skip to content

Instantly share code, notes, and snippets.

@msarahan

msarahan/post_diff.diff

Created October 3, 2016 23:43
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save msarahan/3198be752f2d6bb3e698b02061a1315a to your computer and use it in GitHub Desktop.
Save msarahan/3198be752f2d6bb3e698b02061a1315a to your computer and use it in GitHub Desktop.
Download ZIP
Diff of post module from 1.21.14..2.0.2
Raw
post_diff.diff
diff --git a/conda_build/post.py b/conda_build/post.py
index 9ef0068..b94ee0b 100644
--- a/conda_build/post.py
+++ b/conda_build/post.py
@@ -1,6 +1,7 @@
from __future__ import absolute_import, division, print_function
from collections import defaultdict
+from functools import partial
from glob import glob
import io
import locale
@@ -9,29 +10,28 @@ import re
import os
from os.path import (basename, dirname, join, splitext, isdir, isfile, exists,
islink, realpath, relpath, normpath)
-import shutil
import stat
-from subprocess import call
+from subprocess import call, check_output
import sys
try:
from os import readlink
except ImportError:
readlink = False
+from conda_build.os_utils import external
from .conda_interface import lchmod
from .conda_interface import walk_prefix
from .conda_interface import md5_file
+from .conda_interface import PY3
-from conda_build.config import config
-from conda_build import external
from conda_build import environ
from conda_build import utils
from conda_build import source
if sys.platform.startswith('linux'):
- from conda_build import elf
+ from conda_build.os_utils import elf
elif sys.platform == 'darwin':
- from conda_build import macho
+ from conda_build.os_utils import macho
SHEBANG_PAT = re.compile(br'^#!.+$', re.M)
@@ -42,12 +42,14 @@ def is_obj(path):
(sys.platform == 'darwin' and macho.is_macho(path)))
-def fix_shebang(f, osx_is_app=False):
- path = join(config.build_prefix, f)
+def fix_shebang(f, prefix, build_python, osx_is_app=False):
+ path = join(prefix, f)
if is_obj(path):
return
elif os.path.islink(path):
return
+ elif not os.path.isfile(path):
+ return
if os.stat(path).st_size == 0:
return
@@ -70,9 +72,9 @@ def fix_shebang(f, osx_is_app=False):
encoding = sys.stdout.encoding or 'utf8'
- py_exec = ('/bin/bash ' + config.build_prefix + '/bin/python.app'
+ py_exec = ('/bin/bash ' + prefix + '/bin/python.app'
if sys.platform == 'darwin' and osx_is_app else
- config.build_prefix + '/bin/' + basename(config.build_python))
+ prefix + '/bin/' + basename(build_python))
new_data = SHEBANG_PAT.sub(b'#!' + py_exec.encode(encoding), data, count=1)
if new_data == data:
return
@@ -82,25 +84,25 @@ def fix_shebang(f, osx_is_app=False):
os.chmod(path, int('755', 8))
-def write_pth(egg_path):
+def write_pth(egg_path, config):
fn = basename(egg_path)
- with open(join(environ.get_sp_dir(),
+ with open(join(environ.get_sp_dir(config),
'%s.pth' % (fn.split('-')[0])), 'w') as fo:
fo.write('./%s\n' % fn)
-def remove_easy_install_pth(files, preserve_egg_dir=False):
+def remove_easy_install_pth(files, prefix, config, preserve_egg_dir=False):
"""
remove the need for easy-install.pth and finally remove easy-install.pth
itself
"""
- absfiles = [join(config.build_prefix, f) for f in files]
- sp_dir = environ.get_sp_dir()
+ absfiles = [join(prefix, f) for f in files]
+ sp_dir = environ.get_sp_dir(config)
for egg_path in glob(join(sp_dir, '*-py*.egg')):
if isdir(egg_path):
if preserve_egg_dir or not any(join(egg_path, i) in absfiles for i
in walk_prefix(egg_path, False, windows_forward_slashes=False)):
- write_pth(egg_path)
+ write_pth(egg_path, config=config)
continue
print('found egg dir:', egg_path)
@@ -118,7 +120,7 @@ def remove_easy_install_pth(files, preserve_egg_dir=False):
# so the package directory already exists
# from another installed dependency
if os.path.exists(join(sp_dir, fn)):
- utils.copy_into(join(egg_path, fn), join(sp_dir, fn))
+ utils.copy_into(join(egg_path, fn), join(sp_dir, fn), config.timeout)
utils.rm_rf(join(egg_path, fn))
else:
os.rename(join(egg_path, fn), join(sp_dir, fn))
@@ -127,49 +129,38 @@ def remove_easy_install_pth(files, preserve_egg_dir=False):
if egg_path not in absfiles:
continue
print('found egg:', egg_path)
- write_pth(egg_path)
+ write_pth(egg_path, config=config)
utils.rm_rf(join(sp_dir, 'easy-install.pth'))
-def rm_py_along_so():
+def rm_py_along_so(prefix):
"remove .py (.pyc) files alongside .so or .pyd files"
- for root, dirs, files in os.walk(config.build_prefix):
+ for root, _, files in os.walk(prefix):
for fn in files:
if fn.endswith(('.so', '.pyd')):
- name, unused_ext = splitext(fn)
- for ext in '.py', '.pyc':
+ name, _ = splitext(fn)
+ for ext in '.py', '.pyc', '.pyo':
if name + ext in files:
os.unlink(join(root, name + ext))
-def coerce_pycache_to_old_style(files, cwd):
- """
- For the sake of simplicity, remove the filename additions, like .cpython-35.pyc
+def rm_pyo(files, prefix):
+ """pyo considered harmful: https://www.python.org/dev/peps/pep-0488/
- Since Conda allows only one Python install in a given prefix, there is no reason
- for these additional suffixes. Newer Python will find these pyc files without issue.
- """
- for f in files:
- if not os.path.exists(f):
- f = os.path.join(cwd, f)
- if not os.path.isfile(f) or not f.endswith('.py'):
- continue
- if '/' in f or '\\' in f:
- folder = os.path.join(cwd, os.path.dirname(f), '__pycache__')
- else:
- folder = os.path.join(cwd, '__pycache__')
- fname = os.path.join(folder, os.path.splitext(os.path.basename(f))[0] +
- '.cpython-{0}{1}.pyc'.format(sys.version_info.major,
- sys.version_info.minor))
- if os.path.isfile(fname):
- os.rename(fname, f + 'c')
- for root, folders, files in os.walk(cwd):
- if root.endswith("__pycache__") and not files:
- os.rmdir(root)
-
-
-def compile_missing_pyc(files, cwd=config.build_prefix, python_exe=config.build_python):
+ The build may have proceeded with:
+ [install]
+ optimize = 1
+ .. in setup.cfg in which case we can end up with some stdlib __pycache__
+ files ending in .opt-N.pyc on Python 3, as well as .pyo files for the
+ package's own python. """
+ re_pyo = re.compile(r'.*(?:\.pyo$|\.opt-[0-9]\.pyc)')
+ for fn in files:
+ if re_pyo.match(fn):
+ os.unlink(os.path.join(prefix, fn))
+
+
+def compile_missing_pyc(files, cwd, python_exe):
compile_files = []
for fn in files:
# omit files in Library/bin, Scripts, and the root prefix - they are not generally imported
@@ -180,27 +171,29 @@ def compile_missing_pyc(files, cwd=config.build_prefix, python_exe=config.build_
else:
if fn.startswith('bin'):
continue
- if fn.endswith(".py"):
+ cache_prefix = ("__pycache__" + os.sep) if PY3 else ""
+ if (fn.endswith(".py") and
+ os.path.dirname(fn) + cache_prefix + os.path.basename(fn) + 'c' not in files):
compile_files.append(fn)
- if compile_files:
+ if compile_files and os.path.isfile(python_exe):
print('compiling .pyc files...')
for f in compile_files:
call([python_exe, '-Wi', '-m', 'py_compile', f], cwd=cwd)
- coerce_pycache_to_old_style(compile_files, cwd=cwd)
-def post_process(files, preserve_egg_dir=False):
- compile_missing_pyc(files)
- remove_easy_install_pth(files, preserve_egg_dir=preserve_egg_dir)
- rm_py_along_so()
+def post_process(files, prefix, config, preserve_egg_dir=False):
+ rm_pyo(files, prefix)
+ compile_missing_pyc(files, cwd=prefix, python_exe=config.build_python)
+ remove_easy_install_pth(files, prefix, config, preserve_egg_dir=preserve_egg_dir)
+ rm_py_along_so(prefix)
-def find_lib(link, path=None):
+def find_lib(link, prefix, path=None):
from conda_build.build import prefix_files
- files = prefix_files()
- if link.startswith(config.build_prefix):
- link = normpath(link[len(config.build_prefix) + 1:])
+ files = prefix_files(prefix)
+ if link.startswith(prefix):
+ link = normpath(link[len(prefix) + 1:])
if link not in files:
sys.exit("Error: Could not find %s" % link)
return link
@@ -225,7 +218,7 @@ def find_lib(link, path=None):
# multiple places.
md5s = set()
for f in file_names[link]:
- md5s.add(md5_file(join(config.build_prefix, f)))
+ md5s.add(md5_file(join(prefix, f)))
if len(md5s) > 1:
sys.exit("Error: Found multiple instances of %s: %s" % (link, file_names[link]))
else:
@@ -236,15 +229,15 @@ def find_lib(link, path=None):
print("Don't know how to find %s, skipping" % link)
-def osx_ch_link(path, link_dict):
+def osx_ch_link(path, link_dict, prefix):
link = link_dict['name']
print("Fixing linking of %s in %s" % (link, path))
- link_loc = find_lib(link, path)
+ link_loc = find_lib(link, prefix, path)
if not link_loc:
return
lib_to_link = relpath(dirname(link_loc), 'lib')
- # path_to_lib = utils.relative(path[len(config.build_prefix) + 1:])
+ # path_to_lib = utils.relative(path[len(prefix) + 1:])
# e.g., if
# path = '/build_prefix/lib/some/stuff/libstuff.dylib'
@@ -275,9 +268,9 @@ def osx_ch_link(path, link_dict):
return ret
-def mk_relative_osx(path, build_prefix=None):
+def mk_relative_osx(path, prefix, build_prefix=None):
'''
- if build_prefix is None, then this is a standard conda build. The path
+ if build_prefix is None, the_n this is a standard conda build. The path
and all dependencies are in the build_prefix.
if package is built in develop mode, build_prefix is specified. Object
@@ -285,19 +278,19 @@ def mk_relative_osx(path, build_prefix=None):
build_prefix/lib/. Also, in develop mode, 'path' is not in 'build_prefix'
'''
if build_prefix is None:
- assert path.startswith(config.build_prefix + '/')
+ assert path.startswith(prefix + '/')
else:
- config.short_build_prefix = build_prefix
+ prefix = build_prefix
assert sys.platform == 'darwin' and is_obj(path)
- s = macho.install_name_change(path, osx_ch_link)
+ s = macho.install_name_change(path, partial(osx_ch_link, prefix=prefix))
names = macho.otool(path)
if names:
# Add an rpath to every executable to increase the chances of it
# being found.
rpath = join('@loader_path',
- relpath(join(config.build_prefix, 'lib'),
+ relpath(join(prefix, 'lib'),
dirname(path)), '').replace('/./', '/')
macho.add_rpath(path, rpath, verbose=True)
@@ -309,50 +302,87 @@ def mk_relative_osx(path, build_prefix=None):
if s:
# Skip for stub files, which have to use binary_has_prefix_files to be
# made relocatable.
- assert_relative_osx(path)
-
-
-def mk_relative_linux(f, rpaths=('lib',)):
- path = join(config.build_prefix, f)
- rpath = ':'.join('$ORIGIN/' + utils.relative(f, d) if not
- d.startswith('/') else d for d in rpaths)
- patchelf = external.find_executable('patchelf')
- print('patchelf: file: %s\n setting rpath to: %s' % (path, rpath))
- call([patchelf, '--force-rpath', '--set-rpath', rpath, path])
-
-
-def assert_relative_osx(path):
+ assert_relative_osx(path, prefix)
+
+
+def mk_relative_linux(f, prefix, rpaths=('lib',)):
+ 'Respects the original values and converts abs to $ORIGIN-relative'
+
+ elf = join(prefix, f)
+ origin = dirname(elf)
+
+ patchelf = external.find_executable('patchelf', prefix)
+ existing = check_output([patchelf, '--print-rpath', elf]).decode('utf-8').splitlines()[0]
+ existing = existing.split(os.pathsep)
+ new = []
+ for old in existing:
+ if old.startswith('$ORIGIN/'):
+ new.append(old)
+ elif old.startswith('/'):
+ # Test if this absolute path is outside of prefix. That is fatal.
+ relpath = os.path.relpath(old, prefix)
+ assert not relpath.startswith('..' + os.sep), \
+ 'rpath {0} is outside prefix {1}'.format(old, prefix)
+ relpath = '$ORIGIN/' + os.path.relpath(old, origin)
+ if relpath not in new:
+ new.append(relpath)
+ # Ensure that the asked-for paths are also in new.
+ for rpath in rpaths:
+ if not rpath.startswith('/'):
+ # IMHO utils.relative shouldn't exist, but I am too paranoid to remove
+ # it, so instead, make sure that what I think it should be replaced by
+ # gives the same result and assert if not. Yeah, I am a chicken.
+ rel_ours = utils.relative(f, rpath)
+ rel_stdlib = os.path.relpath(rpath, os.path.dirname(f))
+ assert rel_ours == rel_stdlib, \
+ 'utils.relative {0} and relpath {1} disagree for {2}, {3}'.format(
+ rel_ours, rel_stdlib, f, rpath)
+ rpath = '$ORIGIN/' + rel_stdlib
+ if rpath not in new:
+ new.append(rpath)
+ rpath = ':'.join(new)
+ print('patchelf: file: %s\n setting rpath to: %s' % (elf, rpath))
+ call([patchelf, '--force-rpath', '--set-rpath', rpath, elf])
+
+
+def assert_relative_osx(path, prefix):
for name in macho.get_dylibs(path):
- assert not name.startswith(config.build_prefix), path
+ assert not name.startswith(prefix), path
-def mk_relative(m, f):
+def mk_relative(m, f, prefix):
assert sys.platform != 'win32'
- path = join(config.build_prefix, f)
+ path = join(prefix, f)
if not is_obj(path):
return
+ # skip over this file
+ if (m.ignore_prefix_files() and (type(m.ignore_prefix_files()) is bool or
+ f in m.ignore_prefix_files())):
+ print("Skipping relocation path patch for " + f)
+ return
+
if sys.platform.startswith('linux'):
- mk_relative_linux(f, rpaths=m.get_value('build/rpaths', ['lib']))
+ mk_relative_linux(f, prefix=prefix, rpaths=m.get_value('build/rpaths', ['lib']))
elif sys.platform == 'darwin':
- mk_relative_osx(path)
+ mk_relative_osx(path, prefix=prefix)
-def fix_permissions(files):
+def fix_permissions(files, prefix):
print("Fixing permissions")
- for root, dirs, unused_files in os.walk(config.build_prefix):
+ for root, dirs, _ in os.walk(prefix):
for dn in dirs:
lchmod(join(root, dn), int('755', 8))
for f in files:
- path = join(config.build_prefix, f)
+ path = join(prefix, f)
st = os.lstat(path)
lchmod(path, stat.S_IMODE(st.st_mode) | stat.S_IWUSR) # chmod u+w
-def post_build(m, files):
+def post_build(m, files, prefix, build_python, croot):
print('number of files:', len(files))
- fix_permissions(files)
+ fix_permissions(files, prefix)
if sys.platform == 'win32':
return
@@ -364,19 +394,19 @@ def post_build(m, files):
for f in files:
if f.startswith('bin/'):
- fix_shebang(f, osx_is_app=osx_is_app)
+ fix_shebang(f, prefix=prefix, build_python=build_python, osx_is_app=osx_is_app)
if binary_relocation:
- mk_relative(m, f)
- make_hardlink_copy(f)
+ mk_relative(m, f, prefix)
+ make_hardlink_copy(f, prefix)
- check_symlinks(files)
+ check_symlinks(files, prefix, croot)
-def check_symlinks(files):
+def check_symlinks(files, prefix, croot):
if readlink is False:
return # Not on Unix system
msgs = []
- real_build_prefix = realpath(config.build_prefix)
+ real_build_prefix = realpath(prefix)
for f in files:
path = join(real_build_prefix, f)
if islink(path):
@@ -395,7 +425,7 @@ def check_symlinks(files):
os.symlink(relpath(real_link_path, dirname(path)), path)
else:
# Symlinks to absolute paths on the system (like /usr) are fine.
- if real_link_path.startswith(config.croot):
+ if real_link_path.startswith(croot):
msgs.append("%s is a symlink to a path that may not "
"exist after the build is completed (%s)" % (f, link_path))
@@ -405,47 +435,51 @@ def check_symlinks(files):
sys.exit(1)
-def make_hardlink_copy(path):
+def make_hardlink_copy(path, prefix):
"""Hardlinks create invalid packages. Copy files to break the link.
Symlinks are OK, and unaffected here."""
if not os.path.isabs(path) and not os.path.exists(path):
- path = os.path.normpath(os.path.join(config.build_prefix, path))
+ path = os.path.normpath(os.path.join(prefix, path))
nlinks = os.lstat(path).st_nlink
+ dest = 'tmpfile'
+ if os.path.isabs(path):
+ dest = os.path.join(os.getcwd(), dest)
if nlinks > 1:
# copy file to new name
- shutil.copy2(path, "tmpfile")
+ utils.copy_into(path, dest)
# remove old file
- os.remove(path)
+ utils.rm_rf(path)
# rename copy to original filename
- os.rename("tmpfile", path)
+ utils.copy_into(dest, path)
+ utils.rm_rf(dest)
-def get_build_metadata(m):
- src_dir = source.get_dir()
+def get_build_metadata(m, config):
+ src_dir = config.work_dir
if "build" not in m.meta:
m.meta["build"] = {}
if exists(join(src_dir, '__conda_version__.txt')):
- print("Deprecation warning: support for __conda_version__ will be removed in Conda build 2.0." # noqa
+ print("Deprecation warning: support for __conda_version__ will be removed in Conda build 3.0." # noqa
"Try Jinja templates instead: "
- "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa
+ "http://conda.pydata.org/docs/building/meta-yaml.html#templating-with-jinja")
with open(join(src_dir, '__conda_version__.txt')) as f:
version = f.read().strip()
print("Setting version from __conda_version__.txt: %s" % version)
m.meta['package']['version'] = version
if exists(join(src_dir, '__conda_buildnum__.txt')):
- print("Deprecation warning: support for __conda_buildnum__ will be removed in Conda build 2.0." # noqa
+ print("Deprecation warning: support for __conda_buildnum__ will be removed in Conda build 3.0." # noqa
"Try Jinja templates instead: "
- "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa
+ "http://conda.pydata.org/docs/building/meta-yaml.html#templating-with-jinja")
with open(join(src_dir, '__conda_buildnum__.txt')) as f:
build_number = f.read().strip()
print("Setting build number from __conda_buildnum__.txt: %s" %
build_number)
m.meta['build']['number'] = build_number
if exists(join(src_dir, '__conda_buildstr__.txt')):
- print("Deprecation warning: support for __conda_buildstr__ will be removed in Conda build 2.0." # noqa
+ print("Deprecation warning: support for __conda_buildstr__ will be removed in Conda build 3.0." # noqa
"Try Jinja templates instead: "
- "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa
+ "http://conda.pydata.org/docs/building/meta-yaml.html#templating-with-jinja")
with open(join(src_dir, '__conda_buildstr__.txt')) as f:
buildstr = f.read().strip()
print("Setting version from __conda_buildstr__.txt: %s" % buildstr)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment