Skip to content

Instantly share code, notes, and snippets.

@mcg1969
Last active May 31, 2019 19:08
Show Gist options
  • Save mcg1969/152c86aca2f0c561bb2a8447aecc5bd5 to your computer and use it in GitHub Desktop.
Save mcg1969/152c86aca2f0c561bb2a8447aecc5bd5 to your computer and use it in GitHub Desktop.
rearch.py: building arch-dependent versions of noarch python packages
# usage: python rearch.py <spec-that-resolves-to-noarch-package>
# 1. creates a directory derived from the spec (replacing ':/=' with dashes)
# 2. creates a conda recipe to build a python-version-specific package
# 3. runs the conda recipe for PYTHON_VERSIONS
# 4. runs conda-convert to obtain the other platforms
# leaves everything in the given directory.
import os
import shutil
import sys
import subprocess
import uuid
import ruamel_yaml as yaml
import tempfile
import json
import re
SUBDIRS = ('win-64', 'osx-64', 'linux-64')
PYTHON_VERSIONS = ('2.7', '3.5', '3.6', '3.7')
package_spec = build_dir = sys.argv[1]
package_name = package_spec.split('=', 1)[0]
if '::' in package_name:
channel_name, package_name = package_name.split('::', 1)
build_dir = package_spec.replace(':', '-')
build_dir = build_dir.replace('=', '-')
os.mkdir(build_dir)
os.chdir(build_dir)
build_specs = []
with tempfile.TemporaryDirectory() as env_dir:
conda_command = ['conda', 'create', '--yes', '--prefix={}'.format(env_dir), package_spec ]
print(' '.join(conda_command))
p = subprocess.Popen(conda_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
retcode = p.returncode
if retcode != 0:
msg = ['conda returned an unexpected return code {}'.format(retcode),
'---- begin stdout', stdout.decode().strip(), '---- end stdout',
'---- begin stderr', stderr.decode().strip(), '---- end stderr']
raise RuntimeError('\n'.join(msg))
meta_dir = os.path.join(env_dir, 'conda-meta')
for meta_file in os.listdir(meta_dir):
if meta_file.endswith('.json'):
with open(os.path.join(meta_dir, meta_file)) as fp:
meta_data = json.load(fp)
if meta_data['name'] == package_name:
package_data = meta_data
elif meta_data['name'] == 'python':
subdir = meta_data['subdir']
if 'noarch' not in package_data:
raise RuntimeError('Package is arch-specific, no need for conversion')
unpacked = package_data['extracted_package_dir']
license_txt = os.path.join(unpacked, 'info', 'LICENSE.txt')
recipe_txt = os.path.join(unpacked, 'info', 'recipe', 'meta.yaml')
with open(recipe_txt) as fp:
recipe = yaml.load(fp.read(), Loader=yaml.Loader)
def replace_python_spec(spec):
package_name = spec.split(' ')
if package_name == 'python':
return 'python {}'.format(python_spec.replace('=', ' '))
return spec
if 'extra' in recipe:
del recipe['extra']
build_specs = ['python {{ python }}']
build_specs.extend(spec for spec in package_data['depends']
if spec != 'python' and not spec.startswith('python' ))
recipe['requirements']['host'] = ['python {{ python }}']
if 'build' in recipe['requirements']:
del recipe['requirements']['build']
if 'license_file' in recipe['about']:
recipe['about']['license_file'] = 'LICENSE.txt'
recipe['source'] = {'path': './'}
tarball = package_data['package_tarball_full_path']
tarfile = os.path.basename(tarball)
shutil.copyfile(tarball, tarfile)
recipe['build'] = {
'number': int(recipe['build']['number']),
'script': 'conda install {}/{}'.format('{{ RECIPE_DIR }}', tarfile),
'merge_build_host': True
}
if 'test' in recipe:
if 'imports' in recipe['test']:
recipe['test'] = {'imports': recipe['test']['imports']}
else:
del recipe['test']
if os.path.exists(license_txt):
shutil.copyfile(license_txt, 'LICENSE.txt')
FIELD_ORDER = ['package', 'source', 'build', 'requirements',
'test', 'outputs', 'about', 'app', 'extra']
FIELD_ORDER.extend(k for k in recipe if k not in FIELD_ORDER)
recipe = {k: recipe[k] for k in FIELD_ORDER if k in recipe}
with open('meta.yaml', 'w') as fp:
fp.write(yaml.dump(recipe, Dumper=yaml.RoundTripDumper,
default_flow_style=False, width=10000))
with open('conda_build_config.yaml', 'w') as fp:
fp.write('\n'.join(['python:'] + [' - ' + p for p in PYTHON_VERSIONS]))
build_command = ['conda', 'build', '.']
print(' '.join(build_command))
p = subprocess.Popen(build_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
retcode = p.returncode
if retcode != 0:
msg = ['conda build returned an unexpected return code {}'.format(retcode),
'---- begin stdout', stdout.decode().strip(), '---- end stdout',
'---- begin stderr', stderr.decode().strip(), '---- end stderr']
raise RuntimeError('\n'.join(msg))
package_paths = []
os.mkdir(subdir)
convert_command = ['conda', 'convert']
for new_subdir in SUBDIRS:
if new_subdir != subdir:
convert_command.extend(['-p', new_subdir])
for line in stdout.decode().splitlines():
if line.startswith('anaconda upload '):
src_path = line.rsplit(' ', 1)[-1]
dst_path = os.path.join(subdir, os.path.basename(src_path))
shutil.copyfile(src_path, dst_path)
convert_command.append(dst_path)
print('{}'.format(' '.join(convert_command)))
p = subprocess.Popen(convert_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.communicate()
retcode = p.returncode
if retcode != 0:
msg = ['conda convert returned an unexpected return code {}'.format(retcode),
'---- begin stdout', stdout.decode().strip(), '---- end stdout',
'---- begin stderr', stderr.decode().strip(), '---- end stderr']
raise RuntimeError('\n'.join(msg))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment