Unreal Paragon Asset Consolidation

process.py

import unreal

#####
# For Texture2D, Material and TextureCube
#   BEFORE - 72.8 GB
#   AFTER - 49.0 GB, deleted 6869 duplicate files 
#   22GB savings of data in /Game/Content
#
# Regarding MaterialFunctions, there are over 3200 of them, unfortunately the editor crashes when
# processing these, and the crash results in your character no longer having a material
# you could try adding MaterialFucntion to classesOfInterest array, but do make a backup of your project first
#####

# Change this as you like, this is the new location for all consolidated global files
newGlobalPath = "/Game/ParagonGlobal"

# If your system does not have a lot of memory, run this script once per array item (make the array have only 1 class)
classesOfInterest = ['Texture2D', 'Material', 'TextureCube']

searchPaths = ['/Characters/Global/','/FX/']
dupesProcessed = []
dupeCount = 0
registry = unreal.AssetRegistryHelpers.get_asset_registry()
lib = unreal.EditorAssetLibrary()
assets = registry.get_assets(unreal.ARFilter(class_names=classesOfInterest, recursive_paths=True))

def is_in_path(obj):
	return (searchPaths[0] in str(obj.object_path) or searchPaths[1] in str(obj.object_path))

def has_same_parent(obj1, obj2):
	return str(obj1.object_path).split('/')[-2] == str(obj2.object_path).split('/')[-2]

# TO DO: ideally Advanced Search would be nice, it would simplify this code imensly if I could search by "asset_name"
#   https://answers.unrealengine.com/questions/997513/how-to-script-content-browser-advanced-search.html
for outter in assets:
	if is_in_path(outter) and str(outter.object_path) not in dupesProcessed:
		dupesOfOutter = []

		for inner in assets:
			if (is_in_path(inner) and                                   # only the assets that would have dupes
				inner.asset_name == outter.asset_name and           # the asset name must match
				inner.asset_class == outter.asset_class and         # the asset class must match
				inner.object_path != outter.object_path and         # must not be the same as outter asset
				has_same_parent(outter, inner) and                  # must have same parent directory name (edge cases for /rubber/same-asset.name and /steel/same-asset.name)
				str(inner.object_path) not in dupesProcessed):      # since we are changing the contents of the obj we are iterating

				dupesOfOutter.append(str(inner.object_path))

		if (len(dupesOfOutter) > 0):
			dupeCount += 1

			source = str(outter.object_path)
			destination = newGlobalPath
			if "/FX/" in source:
				destination += '/FX' + source.split('/FX')[1]
			else:
				destination += source.split('/Characters/Global')[1]

			# Move to a new top level global
			if not lib.do_assets_exist([destination]):
				result = lib.rename_asset(source, destination)
				print("### Move Success" if result else "### Move Fail")

			print('#### DUPES FOUND FOR : ' + str(outter.asset_class) + ' - ' + str(outter.object_path))
			dupeObjects = []
			for dupe in dupesOfOutter:
				if lib.do_assets_exist([dupe]):
				  dupeObjects.append(lib.load_asset(dupe))
				  dupesProcessed.append(dupe)
				  print('####    ' + dupe)
				
			# Do the consolidation
			if (len(dupeObjects) > 0):
				newAsset = lib.load_asset(destination)
				result = lib.consolidate_assets(newAsset, dupeObjects) 
				lib.save_loaded_asset(newAsset) # if we don't do this, and there is a crash, were fucked 
				print("### Consolidate Success" if result else "### Consolidate Fail")

print("### " + str(dupeCount) + " newly sourced global files, consolidated " + str(len(dupesProcessed)) + " duplicated files.")