Skip to content

Instantly share code, notes, and snippets.

@mkrizek
Created February 10, 2020 14:14
Show Gist options
  • Save mkrizek/19c4431dddfce2c8fe212eff024b3d29 to your computer and use it in GitHub Desktop.
Save mkrizek/19c4431dddfce2c8fe212eff024b3d29 to your computer and use it in GitHub Desktop.
diff --git a/migrate.py b/migrate.py
index 014b90c..fed8256 100755
--- a/migrate.py
+++ b/migrate.py
@@ -119,7 +119,7 @@ REWRITE_CLASS_PROPERTY_PLUGINS= {
VARDIR = os.environ.get('GRAVITY_VAR_DIR', '.cache')
LOGFILE = os.path.join(VARDIR, 'errors.log')
-REMOVE = set()
+REMOVE = defaultdict(lambda: defaultdict(set))
core = {}
manual_check = defaultdict(list)
@@ -193,8 +193,6 @@ def ensure_cmd_succeeded(ssh_agent, cmd, cwd):
print(cmd_out)
-
-
def add_core(ptype, name):
global core
@@ -246,12 +244,12 @@ def checkout_repo(
### FILE utils
-def remove(path):
+def remove(path, namespace, collection):
global REMOVE
- REMOVE.add(path)
+ REMOVE[namespace][collection].add(path)
-def actually_remove(checkout_path, namespace, collection):
+def actually_remove(checkout_path):
global REMOVE
# load sanity/ignore.txt, the files being removed below need to be removed from ignore.txt too
@@ -263,41 +261,43 @@ def actually_remove(checkout_path, namespace, collection):
# actually remove files we marked for removal
init_files = set()
- for path in REMOVE:
- actual_devel_path = os.path.relpath(path, checkout_path)
- if actual_devel_path.endswith('__init__.py'):
- init_files.add(path)
- continue
+ for namespace in REMOVE:
+ for collection in REMOVE[namespace]:
+ for path in REMOVE[namespace][collection]:
+ actual_devel_path = os.path.relpath(path, checkout_path)
+ if actual_devel_path.endswith('__init__.py'):
+ init_files.add(path)
+ continue
- subprocess.check_call(('git', 'rm', '-f', actual_devel_path), cwd=checkout_path)
- new_sanity_ignore.pop(actual_devel_path, None)
+ subprocess.check_call(('git', 'rm', '-f', actual_devel_path), cwd=checkout_path)
+ new_sanity_ignore.pop(actual_devel_path, None)
- # process the __init__.py files from dirs now that all files are removed,
- # that way we can check if there are no files left in the dirs they are in
- # so we can remove __init__.py as well
- for init in sorted(init_files, key=lambda x: len(x.split('/')), reverse=True):
- if os.listdir(os.path.dirname(init)) != ['__init__.py']:
- continue
+ # process the __init__.py files from dirs now that all files are removed,
+ # that way we can check if there are no files left in the dirs they are in
+ # so we can remove __init__.py as well
+ for init in sorted(init_files, key=lambda x: len(x.split('/')), reverse=True):
+ if os.listdir(os.path.dirname(init)) != ['__init__.py']:
+ continue
- actual_devel_path = os.path.relpath(init, checkout_path)
- subprocess.check_call(
- ('git', 'rm', actual_devel_path),
- cwd=checkout_path,
- )
- new_sanity_ignore.pop(actual_devel_path, None)
+ actual_devel_path = os.path.relpath(init, checkout_path)
+ subprocess.check_call(
+ ('git', 'rm', actual_devel_path),
+ cwd=checkout_path,
+ )
+ new_sanity_ignore.pop(actual_devel_path, None)
- # save modified sanity/ignore.txt
- res = ''
- for filename, values in new_sanity_ignore.items():
- for value in values:
- # value contains '\n' which is preserved from the original file
- res += '%s %s' % (filename, value)
+ # save modified sanity/ignore.txt
+ res = ''
+ for filename, values in new_sanity_ignore.items():
+ for value in values:
+ # value contains '\n' which is preserved from the original file
+ res += '%s %s' % (filename, value)
- write_text_into_file(os.path.join(checkout_path, 'test/sanity/ignore.txt'), res)
- subprocess.check_call(('git', 'add', 'test/sanity/ignore.txt'), cwd=checkout_path)
+ write_text_into_file(os.path.join(checkout_path, 'test/sanity/ignore.txt'), res)
+ subprocess.check_call(('git', 'add', 'test/sanity/ignore.txt'), cwd=checkout_path)
- # commit the changes
- subprocess.check_call(('git', 'commit', '-m', 'Migrated to %s.%s' % (namespace, collection), '--allow-empty'), cwd=checkout_path)
+ # commit the changes
+ subprocess.check_call(('git', 'commit', '-m', 'Migrated to %s.%s' % (namespace, collection), '--allow-empty'), cwd=checkout_path)
def read_yaml_file(path):
@@ -1175,7 +1175,7 @@ def create_unit_tests_copy_map(checkout_path, plugin_type, plugin):
return copy_map
-def copy_unit_tests(copy_map, collection_dir, checkout_path):
+def copy_unit_tests(copy_map, collection_dir, checkout_path, namespace, collection):
"""Copy unit tests into a collection using a copy map."""
if not copy_map:
logger.info(
@@ -1200,7 +1200,7 @@ def copy_unit_tests(copy_map, collection_dir, checkout_path):
if should_be_preserved:
continue
- remove(src)
+ remove(src, namespace, collection)
inject_requirements_into_unit_tests(checkout_path, collection_dir)
@@ -1304,12 +1304,12 @@ def assemble_collections(checkout_path, spec, args, target_github_org):
# plugin, plugin_type, namespace, collection)
# continue
- remove(src)
+ remove(src, namespace, collection)
if plugin_type in ('modules',) and '/' in plugin:
init_py_path = os.path.join(os.path.dirname(src), '__init__.py')
if os.path.exists(init_py_path):
- remove(init_py_path)
+ remove(init_py_path, namespace, collection)
do_preserve_subdirs = (
(args.preserve_module_subdirs and plugin_type == 'modules')
@@ -1353,7 +1353,7 @@ def assemble_collections(checkout_path, spec, args, target_github_org):
unit_tests_copy_map.update(plugin_unit_tests_copy_map)
if not args.skip_tests:
- copy_unit_tests(unit_tests_copy_map, collection_dir, checkout_path)
+ copy_unit_tests(unit_tests_copy_map, collection_dir, checkout_path, namespace, collection)
migrated_to_collection.update(unit_tests_copy_map)
inject_init_into_tree(
@@ -1413,11 +1413,8 @@ def assemble_collections(checkout_path, spec, args, target_github_org):
checkout_path, namespace, collection, migrated_to_collection,
)
- if args.move_plugins:
- actually_remove(checkout_path, namespace, collection)
-
- global REMOVE
- REMOVE = set()
+ if args.move_plugins:
+ actually_remove(checkout_path)
def init_galaxy_metadata(collection, namespace, target_github_org):
@@ -1747,7 +1744,7 @@ def rewrite_integration_tests(test_dirs, checkout_dir, collection_dir, namespace
shutil.copy2(src, dest)
if to_remove:
- remove(src)
+ remove(src, namespace, collection)
relative_src_path = os.path.relpath(src, checkout_dir)
relative_dest_path = os.path.relpath(dest, collection_dir)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment