Skip to content

Instantly share code, notes, and snippets.

@blink1073
Last active May 28, 2022 21:47
Show Gist options
  • Save blink1073/e77218a824af4ace9711f6de5b226092 to your computer and use it in GitHub Desktop.
Save blink1073/e77218a824af4ace9711f6de5b226092 to your computer and use it in GitHub Desktop.
"""Compare the dist file created by a migrated package to one created by the original."""
import argparse
import glob
import os
import shutil
import subprocess
import sys
import tarfile
import zipfile
parser = argparse.ArgumentParser()
parser.add_argument(dest='source_dir', help="Source Directory")
parser.add_argument(dest='target_dir', help="Target Directory")
parser.add_argument(dest='dist_name', help="Dist name")
args = parser.parse_args()
subprocess.run([sys.executable, '-m', 'pip', 'install', 'build'])
def build_file(dirname):
orig_dir = os.getcwd()
os.chdir(dirname)
if os.path.exists('dist'):
shutil.rmtree('dist')
subprocess.run([sys.executable, '-m', 'build', f'--{args.dist_name}'])
os.chdir(orig_dir)
def get_tar_names(dirname):
dist_file = glob.glob(f'{dirname}/dist/*.tar.gz')[0]
tarf = tarfile.open(dist_file, 'r:gz')
return set(tarf.getnames())
def get_zip_names(dirname):
wheel_file = glob.glob(f'{dirname}/dist/*.whl')[0]
with zipfile.ZipFile(wheel_file, 'r') as f:
return set(f.namelist())
def filter_file(path):
if 'egg-info' in path:
return True
_, ext = os.path.splitext(path)
if not ext:
return True
if os.path.basename(path) in [path, 'setup.py', 'setup.cfg', 'MANIFEST.in']:
return True
return False
build_file(args.source_dir)
build_file(args.target_dir)
if args.dist_name == 'sdist':
source_names = get_tar_names(args.source_dir)
target_names = get_tar_names(args.target_dir)
else:
source_names = get_zip_names(args.source_dir)
target_names = get_zip_names(args.target_dir)
removed = source_names - target_names
if removed:
print('Removed_files:')
[print(f'{f}\n') for f in removed if not filter_file(f)]
added = target_names - source_names
if added:
print('\nAdded files:')
[print(f'{f}\n') for f in added if not filter_file(f)]
"""Migrate a Jupyter project from setuptools/jupyter-packaging to hatch and
hatch_jupyter_builder."""
import argparse
import os
import re
import subprocess
import sys
from pathlib import Path
parser = argparse.ArgumentParser()
parser.add_argument(dest='target_dir', help="Target Directory")
# Parse and print the results
args = parser.parse_args()
os.chdir(args.target_dir)
# Install prereqs.
subprocess.run([sys.executable, '-m', 'pip', 'install', '-e', '.'])
subprocess.run([sys.executable, '-m', 'pip', 'install', 'jupyter_packaging'])
subprocess.run([sys.executable, '-m', 'pip', 'install', 'tomli_w'])
subprocess.run([sys.executable, '-m', 'pip', 'install', 'tomli'])
subprocess.run([sys.executable, '-m', 'pip', 'install', 'hatch'])
# Automatic migration from hatch.
subprocess.run([sys.executable, '-m', 'hatch', 'new', '--init'])
# Handle setup.cfg
# Move flake8 config to separate file, preserving comments.
# Add .flake8 file to git.
# Remove file when done.
setup_cfg = Path('setup.cfg')
flake8 = ['[flake8]']
if setup_cfg.exists():
lines = setup_cfg.read_text('utf-8').splitlines()
match = False
for line in lines:
if line.strip() == '[flake8]':
match = True
continue
if not match:
continue
if match and line.startswith('['):
break
flake8.append(line)
Path('.flake8').write_text('\n'.join(flake8) + '\n', 'utf-8')
subprocess.run(['git', 'add', '.flake'])
subprocess.run(['git', 'rm', 'setup.cfg'])
# Handle pyproject.toml config.
# Migrate and remove unused config.
pyproject = Path('pyproject.toml')
text = pyproject.read_text('utf-8')
import tomli # Lazy import in case it was installed above.
data = tomli.loads(text)
tool_table = data.setdefault('tool', {})
# Remove old check-manifest config.
if 'check-manifest' in tool_table:
del tool_table['check-manifest']
# Build up the hatch config.
hatch_table = tool_table.setdefault('hatch', {})
build_table = hatch_table.setdefault('build', {})
targets_table = build_table.setdefault('targets', {})
# Remove any auto-generated sdist config.
if 'sdist' in targets_table:
del targets_table['sdist']
# Remove leading slashes in shared-data paths.
wheel_table = targets_table.setdefault('wheel', {})
shared_data_table = targets_table.setdefault('shared-data', {})
for key in list(shared_data_table):
if key.startswith('/'):
shared_data_table[key[1:]] = shared_data_table[key]
del shared_data_table[key]
hooks_table = build_table.setdefault('hooks', {})
builder_table = hooks_table['jupyter-builder'] = {}
builder_table['dependencies'] = ["hatch-jupyter-builder>=0.3.3"]
# Migrate the jupyter-packaging static data.
if 'jupyter-packaging' in tool_table:
packaging_table = tool_table.get('jupyter-packaging', {})
del tool_table['jupyter-packaging']
options_table = packaging_table.setdefault('options', {})
build_args_table = packaging_table.setdefault('build-args', {})
builder_table['build-function'] = "hatch_jupyter_builder.npm_builder"
if 'ensured-targets' in options_table:
builder_table['ensured-targets'] = options_table['ensured-targets']
if build_args_table:
builder_table['build-kwargs'] = build_args_table.copy()
# Add artifacts config for package data that would be ignored.
project_name = data.get('project', {}).get('name', '')
gitignore = Path('.gitignore')
artifacts = []
if gitignore.exists() and project_name and Path(project_name).exists():
text = gitignore.read_text('utf-8')
for line in text.splitlines():
if line.startswith(project_name):
artifacts.append(f'/{line}')
if artifacts:
build_table['artifacts'] = artifacts
# Handle setup.py - jupyter_packaging and pre-commit config.
# Remove the file when finished.
setup_py = Path('setup.py')
if setup_py.exists():
text = setup_py.read_text('utf-8')
if 'pre-commit' in text:
builder_table['install-pre-commit'] = True
build_kwargs = builder_table.setdefault('build-kwargs', {})
editable_build_command = None
if 'build_cmd' in text:
match = re.search("build_cmd=\"(.*?)\"", text, re.MULTILINE)
assert match is not None
editable_build_command = match.groups()[0]
if 'source_dir' in text or 'build_dir' in text:
builder_table['editable-build-kwargs'] = editable_build_kwargs = {}
editable_build_kwargs['build_cmd'] = editable_build_command
for name in ["source_dir", "build_dir"]:
match = re.search(f"{name}=\"(.*?)\"", text, re.MULTILINE)
if match is not None:
editable_build_kwargs[name] = match.groups()[0]
else:
editable_build_kwargs[name] = "!!! needs manual input !!!"
elif editable_build_command:
build_kwargs['editable_build_cmd'] = editable_build_command
subprocess.run(['git', 'rm', 'setup.py'])
# Remove manifest file if it exists.
if os.path.exists('MANIFEST.in'):
subprocess.run(['git', 'rm', 'MANIFEST.in'])
# Write out the new config.
import tomli_w
pyproject.write_text(tomli_w.dumps(data), 'utf-8')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment