Skip to content

Instantly share code, notes, and snippets.

@beckermr

beckermr/conda_recipe_parser.ipynb

Created February 10, 2020 15:29
Show Gist options
  • Save beckermr/870414e5f168d3aec7f0d9e1c91adccf to your computer and use it in GitHub Desktop.
Save beckermr/870414e5f168d3aec7f0d9e1c91adccf to your computer and use it in GitHub Desktop.
Download ZIP
conda recipe parser
Display the source blob
Display the rendered blob
Raw
conda_recipe_parser.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"meta_yaml = \"\"\"\\\n",
"{% set name = \"antlr4-python3-runtime\" %} # [py3k]\n",
"{% set name = \"antlr4-python2-runtime\" %} # [py2k]\n",
"{% set version = \"4.7.2\" %}\n",
"\n",
"package:\n",
" name: antlr-python-runtime\n",
" version: {{ version }}\n",
"\n",
"source:\n",
" - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz\n",
" sha256: 580825bdd89ed6200170710cb26cc1e64f96f145870d8c2cfdf162cb0b8b9212 # [py2k]\n",
" sha256: 168cdcec8fb9152e84a87ca6fd261b3d54c8f6358f42ab3b813b14a7193bb50b # [py3k]\n",
" - url: https://raw.githubusercontent.com/antlr/antlr4/{{ version }}/LICENSE.txt\n",
" sha256: b1b379fcaf3219593a4c433feb1b35c780bed23fafaae440b1ae2771a9521e3a\n",
"\n",
"build:\n",
" number: 1001\n",
" script: \"{{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv\"\n",
"\n",
"requirements:\n",
" build:\n",
" - python\n",
" - pip\n",
" run:\n",
" - python\n",
"\n",
"test:\n",
" imports:\n",
" - antlr4\n",
" - antlr4.atn\n",
" - antlr4.dfa\n",
" - antlr4.error\n",
" - antlr4.tree\n",
" - antlr4.xpath\n",
"\n",
"about:\n",
" home: https://www.antlr.org\n",
" license: BSD-3-Clause\n",
" license_family: BSD\n",
" license_file: LICENSE.txt\n",
" summary: This is the Python runtime for ANTLR.\n",
" dev_url: https://github.com/antlr/antlr4\n",
" doc_url: https://github.com/antlr/antlr4/blob/master/doc/python-target.md\n",
"\n",
"extra:\n",
" recipe-maintainers:\n",
" - bollwyvl\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import jinja2\n",
"import re\n",
"\n",
"SELECTOR_RE = re.compile(r'^.*#\\s*\\[(.*)\\]')\n",
"\n",
"CONDA_SELECTOR = '__###conda-selector###__'\n",
"\n",
"def config_has_key_with_selectors(cfg, key):\n",
" for _key in cfg:\n",
" if _key == key or _key.startswith(key + CONDA_SELECTOR):\n",
" return True\n",
" return False\n",
"\n",
"def parse_jinja2_variables(meta_yaml):\n",
" env = jinja2.Environment()\n",
" parsed_content = env.parse(meta_yaml)\n",
" all_nodes = list(parsed_content.iter_child_nodes())\n",
"\n",
" jinja2_vals = {}\n",
" for i, n in enumerate(all_nodes):\n",
" if isinstance(n, jinja2.nodes.Assign):\n",
" if config_has_key_with_selectors(jinja2_vals, n.target.name):\n",
" # selectors!\n",
" \n",
" # this block runs if we see the key for the \n",
" # first time\n",
" if n.target.name in jinja2_vals:\n",
" # we need to adjust the previous key\n",
" # first get the data right after the key we have\n",
" jinja2_data = all_nodes[jinja2_vals[n.target.name][1]+1].nodes[0].data\n",
" \n",
" # now pull out the selector and reset the key\n",
" selector_re = SELECTOR_RE.match(jinja2_data)\n",
" if selector_re is not None:\n",
" selector = selector_re.group(1)\n",
" new_key = n.target.name + CONDA_SELECTOR + selector\n",
" jinja2_vals[new_key] = jinja2_vals[n.target.name]\n",
" del jinja2_vals[n.target.name]\n",
" else:\n",
" assert False, jinja2_data\n",
"\n",
" # now insert this key - selector is the next thing\n",
" jinja2_data = all_nodes[i+1].nodes[0].data\n",
" selector_re = SELECTOR_RE.match(jinja2_data)\n",
" if selector_re is not None:\n",
" selector = selector_re.group(1)\n",
" new_key = n.target.name + CONDA_SELECTOR + selector\n",
" jinja2_vals[new_key] = (n.node.value, i)\n",
" else:\n",
" assert False, jinja2_data\n",
" else:\n",
" jinja2_vals[n.target.name] = (n.node.value, i)\n",
"\n",
" # we don't need the indexes into the jinja2 node list anymore\n",
" for key, val in jinja2_vals.items():\n",
" jinja2_vals[key] = jinja2_vals[key][0] \n",
" \n",
" return jinja2_vals"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"import io\n",
"import collections.abc\n",
"import json\n",
"\n",
"from ruamel.yaml import YAML\n",
"\n",
"SPC_KEY_VAL_SELECTOR_RE = re.compile(r'^(\\s*)(.*):(.*)#\\s*\\[(.*)\\]')\n",
"\n",
"MUNGED_LINE_RE = re.compile(r'^(\\s*)(\\S*)' + CONDA_SELECTOR + r'(.*):(.*)')\n",
"\n",
"YAML_JINJA2 = YAML(typ='jinja2')\n",
"YAML_JINJA2.indent(mapping=2, sequence=4, offset=2)\n",
"YAML_JINJA2.width = 120\n",
"\n",
"def _munge_line(line):\n",
" m = SPC_KEY_VAL_SELECTOR_RE.match(line)\n",
" if m:\n",
" spc, key, val, selector = m.group(1, 2, 3, 4)\n",
" new_key = key + CONDA_SELECTOR + selector\n",
" return spc + new_key + ':' + val + '\\n'\n",
" else:\n",
" return line\n",
"\n",
"\n",
"def _unmunge_line(line):\n",
" m = MUNGED_LINE_RE.match(line)\n",
" if m:\n",
" spc, key, selector, val = m.group(1, 2, 3, 4)\n",
" return spc + key + ': ' + val.strip() + ' # [' + selector + ']\\n'\n",
" else:\n",
" return line\n",
"\n",
"\n",
"def _demunge_jinja2_vars(meta):\n",
" if isinstance(meta, collections.abc.MutableMapping):\n",
" for key, val in meta.items():\n",
" meta[key] = _demunge_jinja2_vars(val)\n",
" return meta\n",
" elif isinstance(meta, collections.abc.MutableSequence):\n",
" for i in range(len(meta)):\n",
" meta[i] = _demunge_jinja2_vars(meta[i])\n",
" return meta\n",
" elif isinstance(meta, str):\n",
" return meta.replace('<{', '{{')\n",
" else:\n",
" return meta\n",
"\n",
"def _remunge_jinja2_vars(meta):\n",
" if isinstance(meta, collections.abc.MutableMapping):\n",
" for key, val in meta.items():\n",
" meta[key] = _remunge_jinja2_vars(val)\n",
" return meta\n",
" elif isinstance(meta, collections.abc.MutableSequence):\n",
" for i in range(len(meta)):\n",
" meta[i] = _remunge_jinja2_vars(meta[i])\n",
" return meta\n",
" elif isinstance(meta, str):\n",
" return meta.replace('{{', '<{')\n",
" else:\n",
" return meta\n",
"\n",
"\n",
"def _quote_string(val):\n",
" return json.dumps(val)\n",
"\n",
"def _replace_jinja2_vars(lines, jinja2_vars):\n",
" jinja2_re = re.compile(r'^(\\s*){%\\s*set\\s*(.*)=\\s*(.*)%}(.*)')\n",
" jinja2_re_selector = re.compile(r'^(\\s*){%\\s*set\\s*(.*)=\\s*(.*)%}\\s*#\\s*\\[(.*)\\]')\n",
"\n",
" new_lines = []\n",
" for line in lines:\n",
" _re_sel = jinja2_re_selector.match(line)\n",
" _re = jinja2_re.match(line)\n",
" \n",
" if _re_sel:\n",
" spc, var, val, sel = _re_sel.group(1, 2, 3, 4)\n",
" key = var.strip() + CONDA_SELECTOR + sel\n",
" if key in jinja2_vars:\n",
" _new_line = (\n",
" spc\n",
" + '{% set '\n",
" + var.strip()\n",
" + ' = '\n",
" + _quote_string(jinja2_vars[key])\n",
" + ' %} # ['\n",
" + sel \n",
" + ']\\n'\n",
" )\n",
" else:\n",
" assert False, \"variable '%s' not in %s\" % (key, jinja2_vars)\n",
" elif _re:\n",
" # no selector\n",
" spc, var, val, end = _re.group(1, 2, 3, 4)\n",
" if var.strip() in jinja2_vars:\n",
" _new_line = (\n",
" spc \n",
" + '{% set ' \n",
" + var.strip() \n",
" + ' = ' \n",
" + _quote_string(jinja2_vars[var.strip()]) \n",
" + ' %}' \n",
" + end\n",
" )\n",
" else:\n",
" assert False, \"variable '%s' not in %s\" % (var.strip(), jinja2_vars)\n",
" else:\n",
" _new_line = line\n",
" \n",
" if _new_line[-1] != '\\n':\n",
" _new_line = _new_line + '\\n'\n",
" \n",
" new_lines.append(_new_line)\n",
" \n",
" return new_lines\n",
"\n",
"\n",
"class CondaMetaYAML(object):\n",
" def __init__(self, meta_yaml):\n",
" # get any variables set in the file by jinja2\n",
" self.jinja2_vars = parse_jinja2_variables(meta_yaml)\n",
"\n",
" # munge any duplicate keys\n",
" in_data = io.StringIO(meta_yaml)\n",
" in_data.seek(0)\n",
" lines = []\n",
" for line in in_data.readlines():\n",
" lines.append(_munge_line(line))\n",
"\n",
" # parse with yaml\n",
" self.meta = YAML_JINJA2.load(''.join(lines))\n",
" \n",
" # undo munging of jinja2 variables '<{ var }}' -> '{{ var }}'\n",
" self.meta = _demunge_jinja2_vars(self.meta)\n",
" \n",
" def dump(self, fp):\n",
" # redo jinja2 changes\n",
" self.meta = _remunge_jinja2_vars(self.meta)\n",
" \n",
" try:\n",
" # first dump to yaml\n",
" s = io.StringIO()\n",
" YAML_JINJA2.dump(self.meta, s)\n",
" s.seek(0)\n",
"\n",
" # now unmunge\n",
" lines = []\n",
" for line in s.readlines():\n",
" lines.append(_unmunge_line(line))\n",
"\n",
" # put in new jinja2 vars\n",
" lines = _replace_jinja2_vars(lines, self.jinja2_vars)\n",
" \n",
" # now write to final loc\n",
" for line in lines:\n",
" fp.write(line)\n",
" finally:\n",
" # always put things back!\n",
" self.meta = _demunge_jinja2_vars(self.meta)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"cmy = CondaMetaYAML(meta_yaml)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'about': {'dev_url': 'https://github.com/antlr/antlr4',\n",
" 'doc_url': 'https://github.com/antlr/antlr4/blob/master/doc/python-target.md',\n",
" 'home': 'https://www.antlr.org',\n",
" 'license': 'BSD-3-Clause',\n",
" 'license_family': 'BSD',\n",
" 'license_file': 'LICENSE.txt',\n",
" 'summary': 'This is the Python runtime for ANTLR.'},\n",
" 'build': {'number': 1001,\n",
" 'script': '{{ PYTHON }} -m pip install . --no-deps '\n",
" '--ignore-installed --no-cache-dir -vvv'},\n",
" 'extra': ordereddict([('recipe-maintainers', ['bollwyvl'])]),\n",
" 'package': {'name': 'antlr-python-runtime',\n",
" 'version': '{{ version }}'},\n",
" 'requirements': {'build': ['python', 'pip'],\n",
" 'run': ['python']},\n",
" 'source': [ordereddict([('url', 'https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz'), ('sha256__###conda-selector###__py2k', '580825bdd89ed6200170710cb26cc1e64f96f145870d8c2cfdf162cb0b8b9212'), ('sha256__###conda-selector###__py3k', '168cdcec8fb9152e84a87ca6fd261b3d54c8f6358f42ab3b813b14a7193bb50b')]), ordereddict([('url', 'https://raw.githubusercontent.com/antlr/antlr4/{{ version }}/LICENSE.txt'), ('sha256', 'b1b379fcaf3219593a4c433feb1b35c780bed23fafaae440b1ae2771a9521e3a')])],\n",
" 'test': {'imports': ['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.error', 'antlr4.tree', 'antlr4.xpath']}}\n",
"{'name__###conda-selector###__py2k': 'antlr4-python2-runtime',\n",
" 'name__###conda-selector###__py3k': 'antlr4-python3-runtime',\n",
" 'version': '4.7.2'}\n"
]
}
],
"source": [
"import pprint\n",
"pprint.pprint(cmy.meta)\n",
"pprint.pprint(cmy.jinja2_vars)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"cmy.jinja2_vars['version'] = '4.7.3'\n",
"cmy.jinja2_vars['name__###conda-selector###__py2k'] = 'blah'"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{% set name = \"antlr4-python3-runtime\" %} # [py3k]\n",
"{% set name = \"blah\" %} # [py2k]\n",
"{% set version = \"4.7.3\" %}\n",
"\n",
"package:\n",
" name: antlr-python-runtime\n",
" version: {{ version }}\n",
"\n",
"source:\n",
" - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz\n",
" sha256: 580825bdd89ed6200170710cb26cc1e64f96f145870d8c2cfdf162cb0b8b9212 # [py2k]\n",
" sha256: 168cdcec8fb9152e84a87ca6fd261b3d54c8f6358f42ab3b813b14a7193bb50b # [py3k]\n",
" - url: https://raw.githubusercontent.com/antlr/antlr4/{{ version }}/LICENSE.txt\n",
" sha256: b1b379fcaf3219593a4c433feb1b35c780bed23fafaae440b1ae2771a9521e3a\n",
"\n",
"build:\n",
" number: 1001\n",
" script: {{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv\n",
"\n",
"requirements:\n",
" build:\n",
" - python\n",
" - pip\n",
" run:\n",
" - python\n",
"\n",
"test:\n",
" imports:\n",
" - antlr4\n",
" - antlr4.atn\n",
" - antlr4.dfa\n",
" - antlr4.error\n",
" - antlr4.tree\n",
" - antlr4.xpath\n",
"\n",
"about:\n",
" home: https://www.antlr.org\n",
" license: BSD-3-Clause\n",
" license_family: BSD\n",
" license_file: LICENSE.txt\n",
" summary: This is the Python runtime for ANTLR.\n",
" dev_url: https://github.com/antlr/antlr4\n",
" doc_url: https://github.com/antlr/antlr4/blob/master/doc/python-target.md\n",
"\n",
"extra:\n",
" recipe-maintainers:\n",
" - bollwyvl\n",
"\n"
]
}
],
"source": [
"s = io.StringIO()\n",
"cmy.dump(s)\n",
"s.seek(0)\n",
"print(s.read())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment