rearch.py: building arch-dependent versions of noarch python packages

rearch.py

# usage: python rearch.py <spec-that-resolves-to-noarch-package>
# 1. creates a directory derived from the spec (replacing ':/=' with dashes)
# 2. creates a conda recipe to build a python-version-specific package
# 3. runs the conda recipe for PYTHON_VERSIONS
# 4. runs conda-convert to obtain the other platforms
# leaves everything in the given directory.

import os
import shutil
import sys
import subprocess
import uuid
import ruamel_yaml as yaml
import tempfile
import json
import re

SUBDIRS = ('win-64', 'osx-64', 'linux-64')
PYTHON_VERSIONS = ('2.7', '3.5', '3.6', '3.7')

package_spec = build_dir = sys.argv[1]
package_name = package_spec.split('=', 1)[0]
if '::' in package_name:
	channel_name, package_name = package_name.split('::', 1)
	build_dir = package_spec.replace(':', '-')
build_dir = build_dir.replace('=', '-')
os.mkdir(build_dir)
os.chdir(build_dir)

build_specs = []
with tempfile.TemporaryDirectory() as env_dir:
	conda_command = ['conda', 'create', '--yes', '--prefix={}'.format(env_dir), package_spec ]
	print(' '.join(conda_command))
	p = subprocess.Popen(conda_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	stdout, stderr = p.communicate()
	retcode = p.returncode
	if retcode != 0:
		msg = ['conda returned an unexpected return code {}'.format(retcode),
		       '---- begin stdout', stdout.decode().strip(), '---- end stdout',
		       '---- begin stderr', stderr.decode().strip(), '---- end stderr']
		raise RuntimeError('\n'.join(msg))
	meta_dir = os.path.join(env_dir, 'conda-meta')
	for meta_file in os.listdir(meta_dir):
		if meta_file.endswith('.json'):
			with open(os.path.join(meta_dir, meta_file)) as fp:
				meta_data = json.load(fp)
			if meta_data['name'] == package_name:
				package_data = meta_data
			elif meta_data['name'] == 'python':
				subdir = meta_data['subdir']

if 'noarch' not in package_data:
	raise RuntimeError('Package is arch-specific, no need for conversion')

unpacked = package_data['extracted_package_dir']
license_txt = os.path.join(unpacked, 'info', 'LICENSE.txt')
recipe_txt = os.path.join(unpacked, 'info', 'recipe', 'meta.yaml')

with open(recipe_txt) as fp:
	recipe = yaml.load(fp.read(), Loader=yaml.Loader)

def replace_python_spec(spec):
	package_name = spec.split(' ')
	if package_name == 'python':
		return 'python {}'.format(python_spec.replace('=', ' '))
	return spec

if 'extra' in recipe:
	del recipe['extra']
build_specs = ['python {{ python }}']
build_specs.extend(spec for spec in package_data['depends']
	               if spec != 'python' and not spec.startswith('python' ))
recipe['requirements']['host'] = ['python {{ python }}']
if 'build' in recipe['requirements']:
	del recipe['requirements']['build']
if 'license_file' in recipe['about']:
	recipe['about']['license_file'] = 'LICENSE.txt'
recipe['source'] = {'path': './'}

tarball = package_data['package_tarball_full_path']
tarfile = os.path.basename(tarball)
shutil.copyfile(tarball, tarfile)
recipe['build'] = {
	'number': int(recipe['build']['number']),
	'script': 'conda install {}/{}'.format('{{ RECIPE_DIR }}', tarfile),
	'merge_build_host': True
}

if 'test' in recipe:
	if 'imports' in recipe['test']:
		recipe['test'] = {'imports': recipe['test']['imports']}
	else:
		del recipe['test']

if os.path.exists(license_txt):
	shutil.copyfile(license_txt, 'LICENSE.txt')

FIELD_ORDER = ['package', 'source', 'build', 'requirements',
               'test', 'outputs', 'about', 'app', 'extra']
FIELD_ORDER.extend(k for k in recipe if k not in FIELD_ORDER)
recipe = {k: recipe[k] for k in FIELD_ORDER if k in recipe}

with open('meta.yaml', 'w') as fp:
	fp.write(yaml.dump(recipe, Dumper=yaml.RoundTripDumper,
                           default_flow_style=False, width=10000))
with open('conda_build_config.yaml', 'w') as fp:
	fp.write('\n'.join(['python:'] + ['  - ' + p for p in PYTHON_VERSIONS]))

build_command = ['conda', 'build', '.']
print(' '.join(build_command))
p = subprocess.Popen(build_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
retcode = p.returncode
if retcode != 0:
	msg = ['conda build returned an unexpected return code {}'.format(retcode),
	       '---- begin stdout', stdout.decode().strip(), '---- end stdout',
	       '---- begin stderr', stderr.decode().strip(), '---- end stderr']
	raise RuntimeError('\n'.join(msg))

package_paths = []
os.mkdir(subdir)
convert_command = ['conda', 'convert']
for new_subdir in SUBDIRS:
	if new_subdir != subdir:
		convert_command.extend(['-p', new_subdir])
for line in stdout.decode().splitlines():
	if line.startswith('anaconda upload '):
		src_path = line.rsplit(' ', 1)[-1]
		dst_path = os.path.join(subdir, os.path.basename(src_path))
		shutil.copyfile(src_path, dst_path)
		convert_command.append(dst_path)
print('{}'.format(' '.join(convert_command)))
p = subprocess.Popen(convert_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.communicate()
retcode = p.returncode
if retcode != 0:
	msg = ['conda convert returned an unexpected return code {}'.format(retcode),
	       '---- begin stdout', stdout.decode().strip(), '---- end stdout',
	       '---- begin stderr', stderr.decode().strip(), '---- end stderr']
	raise RuntimeError('\n'.join(msg))