Conda recipe porting script.

patch_meta_forge.py

#!/usr/bin/env python

import ruamel.yaml
import yaml
import hashlib
import urllib.request
# import urllib.error
# import urllib.parse
import requests
import json
try:
    from packaging.version import parse
except ImportError:
    from pip._vendor.packaging.version import parse
# import codecs


URL_PATTERN = 'https://pypi.python.org/pypi/{package}/json'


def get_version(package, url_pattern=URL_PATTERN):
    """Return version of package on pypi.python.org using json."""
    # reader = codecs.getreader("utf-8")
    req = requests.get(url_pattern.format(package=package))
    version = parse('0')
    if req.status_code == requests.codes.ok:
        # j = json.loads(req.text.encode(req.encoding))
        j = json.loads(req.text)
        if 'releases' in j:
            releases = j['releases']
            for release in releases:
                ver = parse(release)
                if not ver.is_prerelease:
                    version = max(version, ver)
    return version

EX_DOC = '''
package:
  name: example
  version: 1.0.0.0.1.000

source:
  fn: bob.extension-.zip

build:
  number: 0

requirements:
  build:
    - python
  run:
    - python

test:
  imports:
    - example
  commands:
    - python -c "import example;"

about:
  home: https://github.com/conda-forge/staged-recipes
  license: BSD
  summary: An example package

extra:
  recipe-maintainers:
    - HomerSimpson
    - LukeSkywalker
'''


def get_remote_md5_sum(url, max_file_size=100 * 1024 * 1024):
    remote = urllib.request.urlopen(url)
    hash = hashlib.md5()

    total_read = 0
    while True:
        data = remote.read(4096)
        total_read += 4096

        if not data or total_read > max_file_size:
            break

        hash.update(data)

    return hash.hexdigest()


def space_out(doc, key1, key2):
    i1 = doc.find('{}:\n'.format(key1))
    i2 = doc.find('  {}:\n'.format(key2), i1)
    if i1 < 0 or i2 < 0:
        print('{} {} not found!'.format(key1, key2))
        return doc
    return doc[:i2].rstrip() + '\n\n\n' + doc[i2:]


def main(path):
    with open(path) as f:
        doc = f.read()
    # remove comments
    lines = doc.split('\n')
    lines = [l for l in lines if l.strip() and l.lstrip()[0] != '#']
    doc = '\n'.join(lines)

    data = ruamel.yaml.load(doc, Loader=ruamel.yaml.RoundTripLoader)
    # data = yaml.load(doc)
    # import ipdb; ipdb.set_trace()
    metaversion = data['package']['version']
    print('version from meta.yaml: ', metaversion)
    package = data['package']['name']
    version = get_version(package)
    print('version from pypi:      ', version)
    url = 'https://pypi.python.org/packages/source/{p}/{package}/{package}-{version}.zip'
    md5 = get_remote_md5_sum(url.format(p=package[0], package=package, version=version))

    doc = doc.replace(str(metaversion), '1.0.0.0.1.000')
    lines = doc.split('\n')
    lines = [l for l in lines if ('nomkl' not in l) and ('gcc' not in l)]
    doc = '\n'.join(lines)

    data = ruamel.yaml.load(doc, Loader=ruamel.yaml.RoundTripLoader)
    # data = yaml.load(doc)
    data['extra'] = {'recipe-maintainers': ['183amir']}
    data['source']['md5'] = md5
    data['source']['url'] = url.format(p=package[0], package=package, version='1.0.0.0.1.000')
    data['build']['number'] = 0
    data['build'].pop('preserve_egg_dir', None)
    if 'setuptools' in data['requirements']['run']:
        data['requirements']['run'].remove('setuptools')
    data['build']['skip'] = True
    data['build'].yaml_add_eol_comment('[not linux]', 'skip', column=18)
    if package in ['bob.core', 'bob.io.base']:
        with open(path[:-9]+'build.sh', 'w') as f:
            f.write('''#!/usr/bin/env bash

if [[ `uname` == 'Darwin' ]]; then
    CFLAGS="${CFLAGS} -DBOOST_NO_CXX11_RVALUE_REFERENCES"
	CXXFLAGS="${CXXFLAGS} -DBOOST_NO_CXX11_RVALUE_REFERENCES"
    LDFLAGS="${LDFLAGS} -DBOOST_NO_CXX11_RVALUE_REFERENCES"
	$PYTHON -B setup.py install --single-version-externally-managed --record record.txt
else
	$PYTHON -B setup.py install --single-version-externally-managed --record record.txt
fi
''')
    else:
        data['build']['script'] = 'python -B setup.py install --single-version-externally-managed --record record.txt'
    if 'BSD' in data['about']['license']:
        data['about']['license'] = 'Modified BSD License (3-clause)'
    if 'numpy' in data['requirements']['build']:
        for brun in ['build', 'run']:
            i = data['requirements'][brun].index('numpy')
            data['requirements'][brun][i] = 'numpy x.x'
    if 'pkg-config' not in data['requirements']['build']:
        for pkg in ['cmake', 'boost', 'opencv', 'ffmpeg', 'blitz', 'sox', 'libsvm']:
            if pkg in data['requirements']['build']:
                data['requirements']['build'].append('pkg-config')
                break
    list_size = len(data['requirements']['build'])
    data['requirements']['build'].append('gcc')
    data['requirements']['build'].yaml_add_eol_comment('[linux]', list_size, column=18)
    list_size = len(data['requirements']['run'])
    data['requirements']['run'].append('libgcc')
    data['requirements']['run'].yaml_add_eol_comment('[linux]', list_size, column=16)

    # use example as template
    ex = ruamel.yaml.load(EX_DOC, Loader=ruamel.yaml.RoundTripLoader)
    ex.update(data)
    # for key in ex:
    #     ex[key].update(data[key])
    # with open(path, 'w') as f:
    #     f.write('{% set version = "' + str(version) + '" %}\n')
    doc = '{% set version = "' + str(version) + '" %}\n'
    doc += ruamel.yaml.dump(ex, Dumper=ruamel.yaml.RoundTripDumper, width=1000, indent=4, block_seq_indent=2)
    doc = doc.replace('    ', '  ')
  #   doc = doc.replace('''  script: python -B setup.py install --single-version-externally-managed --record record.txt''',
  #       '''  script: python -B setup.py install --single-version-externally-managed --record record.txt
  # script: LINKFLAGS="$LINKFLAGS -stdlib=libc++ -mmacosx-version-min=10.7 " CXXFLAGS="$CXXFLAGS -stdlib=libc++ -mmacosx-version-min=10.7 " python -B setup.py install --single-version-externally-managed --record record.txt  # [osx]''')
    # yaml.dump(data, f, default_flow_style=False)
    # with open(path) as f:
    #     doc = f.read()
    # doc = space_out(doc, 'requirements', 'build')
    doc = space_out(doc, 'requirements', 'run')
    doc = space_out(doc, 'requirements', 'run')
    doc = space_out(doc, 'test', 'commands')
    doc = space_out(doc, 'test', 'commands')
    # doc = space_out(doc, 'test', 'imports')
    doc = space_out(doc, 'test', 'requires')
    doc = space_out(doc, 'test', 'requires')
    with open(path, 'w') as f:
        f.write(doc.replace('1.0.0.0.1.000', '{{ version }}').replace('\n\n', '\n'))


#     with open(path[:-9]+'build.sh') as f:
#         doc = f.read()
#     with open(path[:-9]+'build.sh', 'w') as f:
#         f.write(doc.replace('''
# # Add more build steps here, if they are necessary.

# # See
# # http://docs.continuum.io/conda/build.html
# # for a list of environment variables that are set during the build process.
# ''', ''))

#     with open(path[:-9]+'bld.bat') as f:
#         doc = f.read()
#     with open(path[:-9]+'bld.bat', 'w') as f:
#         f.write(doc.replace('''
# :: Add more build steps here, if they are necessary.

# :: See
# :: http://docs.continuum.io/conda/build.html
# :: for a list of environment variables that are set during the build process.
# ''', ''))
if __name__ == '__main__':
    import sys
    main(sys.argv[1])

port.sh

#!/bin/bash

set -ex

	# bob.extension \
	# bob.blitz \
	# bob.core \
	# bob.math \
	# bob.io.base \
	# bob.io.matlab \

	# bob.io.audio \
	# bob.sp \
	# bob.measure \
	# bob.db.base \
	# bob.io.image \
	# bob.io.video \
	# bob.learn.activation \
	# bob.learn.boosting \
for package in \
	bob.ap \
	# bob.learn.libsvm \
	# bob.ip.base \
	# bob.ip.color \
	# bob.ip.draw \
	# bob.ip.gabor \
	# bob.learn.linear \
	# bob.learn.mlp \
	# bob.learn.em \
	# bob.db.verification.utils \
	# bob.db.iris \
	# bob.db.wine \
	# bob.db.mnist \
	# bob.db.atnt \
	# bob.ip.facedetect \
	# bob.ip.optflow.hornschunck \
	# bob.ip.optflow.liu \
	# bob.ip.flandmark \
	# bob.db.verification.filelist \
	# bob.db.arface \
	# bob.db.banca \
	# bob.db.biosecure \
	# bob.db.caspeal \
	# bob.db.frgc \
	# bob.db.gbu \
	# bob.db.lfw \
	# bob.db.mobio \
	# bob.db.multipie \
	# bob.db.scface \
	# bob.db.xm2vts \
	# bob.db.youtube \
	# bob.db.nist_sre12 \
	# bob.db.voxforge \
	# bob.db.utfvp \
	# bob.db.livdet2013 \
	# bob.db.atvskeystroke \
	# bob.db.kboc16 \
	# bob.db.biosecurid.face \
	# bob.db.casme2 \
	# antispoofing.utils \
	# bob.db.replay \
	# bob.db.casia_fasd \
	# bob.bio.base \
	# bob.bio.gmm \
	# bob.bio.spear \
	# bob.bio.face \
	# bob.bio.video \
	# bob.db.msu_mfsd_mod \
	# bob.db.avspoof \
	# bob \
	# bob-devel
do
	# checkout to master
	cd ~/idiap/git/staged-recipes
	# git checkout master

	# create branch and checkout
	# git checkout -b $package || (git branch -D $package && git checkout -b $package)

	# copy files over
	mkdir -p $HOME/idiap/git/staged-recipes/recipes/$package
	cp "$HOME/idiap/git/bob.conda/$package"/meta.yaml $HOME/idiap/git/staged-recipes/recipes/$package/meta.yaml
	# rm "$HOME/idiap/git/staged-recipes/recipes/$package/bld.bat"
	# rm "$HOME/idiap/git/staged-recipes/recipes/$package/build.sh"

	# patch the meta.yaml file
	python $HOME/idiap/git/bob.conda/patch_meta_forge.py "$HOME/idiap/git/staged-recipes/recipes/$package/meta.yaml"
	# sed  '/^$/d' -i $HOME/idiap/git/staged-recipes/recipes/$package/meta.yaml
	sed '/^[a-z]\+:/{x;p;x;}' -i "$HOME/idiap/git/staged-recipes/recipes/$package/meta.yaml"

	# commit changes
	# git add $HOME/idiap/git/staged-recipes/recipes/$package
	# git commit -am "Add $package recipe [skip appveyor]"

	# # build the recipe
	# conda build --python=2.7 recipes/$package

	# # push changes
	# git push --set-upstream origin $package

	# # open pull request
	# hub pull-request --browse -b conda-forge:master -h bioidiap:$package -m "Add $package recipe"

# # checkout to master
# 	cd ~/idiap/git/staged-recipes
# 	git checkout master
# # checkout
# 	git checkout $package
# # copy files over
# 	cp "$HOME/idiap/git/bob.conda/$package"/meta.yaml $HOME/idiap/git/staged-recipes/recipes/$package/meta.yaml
# # patch the meta.yaml file
# 	python $HOME/idiap/git/bob.conda/patch_meta_forge.py "$HOME/idiap/git/staged-recipes/recipes/$package/meta.yaml"
# 	sed '/^[a-z]\+:/{x;p;x;}' -i "$HOME/idiap/git/staged-recipes/recipes/$package/meta.yaml"
# # commit changes
# 	git commit -am "osx min deployment version [skip appveyor]"
# # build the recipe
# 	# conda build --python=2.7 recipes/$package
# # push changes
# 	git push origin $package


	# git reset --soft HEAD~
	# git stash
	# git stash drop

done