sort-arm.py

#!/usr/bin/python

import re, mmap
from pathlib import Path

from git import Repo
import json

repo = Repo('.')
index = repo.index

DTS_DIR = 'arch/arm/boot/dts'
DTS_ARM64_DIR = 'arch/arm64/boot/dts/'

ignore_dep =  ['arm64/qcom/apq8016-sbc.dts', 'arm64/qcom/msm8916-samsung-serranove.dts',]
blacklist = [ 'cros-ec-keyboard.dtsi', 'armv7-m.dtsi', 'tps65910.dtsi']

re_arch = re.compile(r'^dtb-\$\(CONFIG_(.*)\) \+=.*')
re_dtb_sameline = re.compile(r'^dtb-\$\(CONFIG_.*\) \+= (.*).dtb\n')
re_dtb = re.compile(r'\t(.*)\.dtb[ \\n]*')
re_dep = re.compile(rb'[/s/t]*[#/]include[/]* "(.*dtsi*)"')
re_dep_h = re.compile(rb'[/s/t]*[#/]include[/]* "([a-z0-9-]*\.h)"')
re_64_dep = re.compile(rb'[/s/t]*[#/]include[/]* ["<]*(arm/.*dtsi*)[">]*')
re_64_dep_h = re.compile(rb'[/s/t]*[#/]include[/]* ["<]*(arm/[a-z0-9-]*\.h)[">]*')

oems = {
        'marvell': ["mach_armada_370","mach_armada_375","mach_armada_38x","mach_armada_39x","mach_armada_xp", "arch_orion5x", "arch_pxa", "arch_mmp", "arch_berlin",],
        'nvidia': ["arch_tegra_2x_soc","arch_tegra_3x_soc","arch_tegra_114_soc","arch_tegra_124_soc",],
        'freescale': ["soc_imx1", "soc_imx25", "soc_imx27", "soc_imx31", "soc_imx35", "soc_imx50", "soc_imx51", "soc_imx53", "soc_imx6q", "soc_imx6sl", "soc_imx6sll", "soc_imx6sx", "soc_imx6ul", "soc_imx7d", "soc_imx7ulp", "arch_mxs",],
        'allwinner': ["mach_sun4i", "mach_sun5i", "mach_sun6i", "mach_sun7i", "mach_sun8i", "mach_sun9i", "mach_suniv",],
        'broadcom': [ "arch_bcm2835", "arch_bcm_5301x", "arch_bcm_53573", "arch_bcm_63xx", "arch_bcm_cygnus", "arch_bcm_hr2", "arch_bcm_mobile", "arch_bcm_nsp",  "arch_brcmstb"],
        'ti': ["arch_omap2", "arch_omap3", "arch_omap4", "soc_omap5", "soc_ti81xx", "soc_dra7xx", "arch_nspire", "arch_keystone", "arch_davinci", "soc_am43xx", "soc_am33xx",],
        'amlogic': ["mach_meson6", "mach_meson8",],
        'exynos': ["arch_exynos3", "arch_exynos4", "arch_exynos5",],
        'qcom': ["arch_qcom",],
        'st': ["arch_spear13xx", "arch_spear3xx", "arch_spear6xx",],
        'stm': ["arch_sti", "arch_stm32",],
        'intel': ["arch_intel_socfpga"],
        'samsung': ["arch_s3c24xx", "arch_s3c64xx", "arch_s5pv210",],
        'hisilicon': ["arch_hip01", "arch_hip04", "arch_hi3xxx", "arch_hisi", "arch_hix5hd2", "arch_sd5203"],
        'realtek': ["arch_realtek"],
        'mediatek': ["arch_mediatek",],
        'microchip': ["soc_at91rm9200", "soc_at91sam9", "soc_sam9x60", "soc_sam_v7", "soc_sama7g5", "soc_lan966",],
        'alphascale': ["mach_asm9260"],
        'aspeed': ["arch_aspeed"],
        'mstart': ["arch_mstarv7"],
        'socionext': ["arch_milbeaut", "arch_uniphier",],
        'dove': ["mach_dove"],
        'xilinx': ["arch_zynq",],
        'via': ["arch_vt8500"],
        'st_ericsson': ["arch_u8500", "arch_nomadik"],
        'rockchip': ["arch_rockchip"],
        'renasas': ["arch_renesas"],
        'vybrid': ["soc_vf610",],
        "axix": ["mach_artpec6"],
        "kirkwood": ["mach_kirkwood"],
        "edac": ["arch_highbank"],
        "layerscape": ["soc_ls1021a",],
        'nxp': ["arch_lpc18xx", "arch_lpc32xx",],
        "oxnas": ["arch_oxnas"],
        "xscale": ["arch_ixp4xx"],
        "realview": ["arch_realview"],
        "moxart": ["arch_moxart"],
        "rda": ["arch_rda"],
        "actions": ["arch_actions"],
        "nuvoton": ["arch_npcm7xx", "arch_wpcm450"],
        "cortina": ["arch_gemini"],
        "airoha": ["arch_airoha"],
        "conexant": ["arch_digicolor"],
        "cirrus": ["arch_clps711x",],
        "axxia": ["arch_axxia"],
        "annapurna": ["arch_alpine"],
        # ARM OEM SUBDIR?
        "arm": ["arch_mps2", "arch_integrator", "arch_versatile", "arch_vexpress"],
        # Very strange
        'virtual': ["arch_virt"],
}

orpahn_dts = {
        "marvell": ["armada-380.dtsi", "armada-390-db.dts", "armada-390.dtsi", "armada-395-gp.dts", "pxa27x.dtsi", "pxa25x.dtsi"],
        "ti": ["am3703.dtsi", "am3715.dtsi", "omap3430es1-clocks.dtsi", "logicpd-torpedo-35xx-devkit.dts", "logicpd-som-lv-35xx-devkit.dts", "logicpd-torpedo-37xx-devkit-28.dts"],
        "aspeed": ["aspeed-bmc-inspur-on5263m5.dts"],
        "microchip": ["at91rm9200_pqfp.dtsi", "usb_a9g20-dab-mmx.dtsi", "sama5d29.dtsi"],
        "freescale": ["imx6dl-kontron-samx6i.dtsi", "imx6q-kontron-samx6i.dtsi", "imx6qdl-kontron-samx6i.dtsi", "imx6qdl-pico-dwarf.dtsi", "imx6qdl-pico-nymph.dtsi", "imx6ul-kontron-n6311-s.dts", "imx6ul-kontron-n6311-som.dtsi", "imx6ull-kontron-n6411-s.dts", "imx6ull-kontron-n6411-som.dtsi", "imxrt1050-pinfunc.h"],
        "kirkwood": ["kirkwood-km_fixedeth.dts"],
        "mediatek": ["mt6582-prestigio-pmt5008-3g.dts", "mt6582.dtsi"],
        "rockchip": ["rockchip-radxa-dalang-carrier.dtsi"],
        "socionext": ["uniphier-pro5.dtsi"],
        "stm": ["stm32mp13xc.dtsi", "stm32mp15xxab-pinctrl.dtsi", "stm32mp15xxad-pinctrl.dtsi"],
        "allwinner": ["sunxi-h3-h5-emlid-neutis.dtsi", "sunxi-libretech-all-h3-it.dtsi"],
        "st": ["spear320s.dtsi"],
        "samsung": ["s3c6400.dtsi"],
        "renasas": ["r8a77xx-aa104xd12-panel.dtsi", "r8a77xx-aa121td01-panel.dtsi", "gr-peach-audiocamerashield.dtsi"],
}

def validate_makefile():
        makefile_list = []
        makefile_r = Path(DTS_DIR).joinpath('Makefile').open('r')
        makefile_list = makefile_r.readlines()
        makefile_r.close()

        for index, line in enumerate(makefile_list):
                if index==0:
                        continue
                if "dtb-" in line in line:
                        to_check = makefile_list[index-1]
                        if to_check.endswith(" \\\n"):
                                makefile_list[index-1] = to_check.replace(' \\\n','\n')
                        if "dtb-" in to_check and not ".dtb" in to_check:
                                makefile_list[index-1] = ''
                        to_check = makefile_list[index-2]
                        if '#' in to_check and not '# SPDX' in to_check:
                                makefile_list[index-2] = ''

                if "subdir-y" in line:
                        to_check = makefile_list[index-1]
                        if "dtb-" in to_check:
                                makefile_list[index-1] = ''

        makefile_w = Path(DTS_DIR).joinpath('Makefile_tmp').open('w')
        makefile_w.write("".join(makefile_list))
        makefile_w.close()
        Path(DTS_DIR+'/Makefile_tmp').rename(DTS_DIR+'/Makefile')

def remove_from_makefile(name):
        makefile_r = Path(DTS_DIR).joinpath('Makefile').open('r')
        makefile_w = Path(DTS_DIR).joinpath('Makefile_tmp').open('w')
        for line in makefile_r.readlines():
                if name in line:
                        continue
                else:
                        makefile_w.write(line)
        makefile_r.close()
        makefile_w.close()
        Path(DTS_DIR+'/Makefile_tmp').rename(DTS_DIR+'/Makefile')

def fix_dep(name, broke_dep_data, oem):
        value = broke_dep_data['value']
        file = broke_dep_data['file']
        dep_oem = broke_dep_data['oem']
        if broke_dep_data['type'] == 'dts':
                if name in blacklist:
                        print("Ignoring blacklisted dep {} in {} arm64 dir with {}".format(file, dep_oem, 'arm/{}/{}'.format(oem,name)))
                        return
                print("Fixing {} in {} arm64 dir with {}".format(file, dep_oem, 'arm/{}/{}'.format(oem,name)))
                dts_r = Path(DTS_ARM64_DIR).joinpath(dep_oem, file).open('r')
                dts_w = Path(DTS_ARM64_DIR).joinpath(dep_oem, file+'_tmp').open('w')
                for line in dts_r.readlines():
                        if value in line:
                                dts_w.write(line.replace(value,'arm/{}/{}'.format(oem, name)))
                        else:
                                dts_w.write(line)
                dts_r.close()
                dts_w.close()
                Path(DTS_ARM64_DIR).joinpath(dep_oem, file+'_tmp').rename(Path(DTS_ARM64_DIR).joinpath(dep_oem, file))
        elif broke_dep_data['type'] == 'symbolic':
                print("Fixing symbolic {} in {} arm64 dir with {}".format(file, dep_oem, '../../../../arm/boot/dts/{}/{}'.format(oem, name)))
                Path(DTS_ARM64_DIR).joinpath(dep_oem, file).unlink()
                Path(DTS_ARM64_DIR).joinpath(dep_oem, file).symlink_to('../../../../arm/boot/dts/{}/{}'.format(oem, name))

def enumerate_archs_from_makefile():
        curr_arch=''
        res = {}

        Makefile = Path(DTS_DIR).joinpath('Makefile').open('r')
        lines = Makefile.readlines()

        # enumerate dts
        for line in lines:
                if line.startswith('#'):
                        continue
                if line.startswith('dtb'):
                        curr_arch=re_arch.findall(line)[0].lower()
                        res[curr_arch] = []
                        dtb = re_dtb_sameline.findall(line)
                        if len(dtb) > 0:
                                res[curr_arch].append(dtb[0])
                if line.startswith('\t'):
                        dtb = re_dtb.findall(line)
                        res[curr_arch].append(dtb[0])

        Makefile.close()

        return res

def check_dependency(dts_name):
        res = []
        dts = Path(DTS_DIR).joinpath(dts_name).open('r')
        lines = mmap.mmap(dts.fileno(), 0, prot=mmap.PROT_READ)
        dependencies = re_dep.findall(lines)
        if len(dependencies) > 0:
                for _, dependency in enumerate(dependencies):
                        dependency = dependency.decode("utf-8")
                        if dependency in ignore_dep:
                                continue
                        res.append(dependency)
                        res += check_dependency(dependency)
        dependencies = re_dep_h.findall(lines)
        if len(dependencies) > 0:
                for _, dependency in enumerate(dependencies):
                        dependency = dependency.decode("utf-8")
                        if dependency in ignore_dep:
                                continue
                        res.append(dependency)
                        res += check_dependency(dependency)
        return res

def get_oem_from_soc(soc):
        for oem in oems:
                if soc in oems[oem]:
                        return oem
                else:
                        continue
        return None

def generate_action_list_file_details():
        file_details = {}
        actions_list = {}
        dependency = {}
        output = enumerate_archs_from_makefile()

        for socs in output:
                dependency[socs] = []

        # check dependency
        for socs in output:
                for dtb in output[socs]:
                        dependency_list = check_dependency(dtb+'.dts')
                        dependency[socs].extend(x for x in dependency_list if x not in dependency[socs])

        for oem in oems:
                actions_list[oem] = []

        # Actually composing the final move list
        for soc in output:
                for dtb in output[soc]:
                        oem = get_oem_from_soc(soc)
                        if oem == None:
                                print("OEM NOT FOUND for {}!".format(soc))
                                exit()
                        actions_list[oem].append(dtb+'.dts')
                        file_details[dtb+'.dts'] = {'soc': soc, 'oem': oem}
                for dep in dependency[soc]:
                        oem = get_oem_from_soc(soc)
                        if oem == None:
                                print("OEM NOT FOUND for {}!".format(soc))
                                exit()
                        if not dep in actions_list[oem]:
                                actions_list[oem].append(dep)

        for oem in orpahn_dts:
                for dtb in orpahn_dts[oem]:
                        actions_list[oem].append(dtb)

        return actions_list, file_details

def check_dep_in_array(array, dep_name):
        res = []
        for oem in array:
                for dtb in array[oem]:
                        for data in array[oem][dtb]:
                                if type(data) == str: # this is a subtarget
                                        for real_data in array[oem][dtb][data]:
                                                if dep_name in real_data['value']:
                                                        res.append(real_data)
                                else:
                                        if dep_name in data['value']:
                                                res.append(data)
        return res

def gen_dep_in_dir(path):
        res = None
        if path.is_symlink():
                res = []
                res.append({'file': path.name, 'value':path.readlink().name,'type': 'symbolic','oem':path.parents[0].name})
        elif path.is_file():
                res = []
                file = path.open('r')
                lines = mmap.mmap(file.fileno(), 0, prot=mmap.PROT_READ)
                dependencies = re_64_dep.findall(lines)
                for _, dependency in enumerate(dependencies):
                        res.append({'file': path.name, 'value':dependency.decode("utf-8"),'type':'dts','oem':path.parents[0].name})
                dependencies = re_64_dep_h.findall(lines)
                for _, dependency in enumerate(dependencies):
                        res.append({'file': path.name, 'value':dependency.decode("utf-8"),'type':'header','oem':path.parents[0].name})
                file.close()
        elif path.is_dir():
                res = {}
                for dir in path.iterdir():
                        res[dir.name] = gen_dep_in_dir(dir)
        return res

#Generate dep list for arm64
arm64_dep_list = gen_dep_in_dir(Path(DTS_ARM64_DIR))

commit_count = 0

actions_list, file_details = generate_action_list_file_details()
moved_file = []

# print(json.dumps(actions_list, indent=4, sort_keys=True))
# exit()

for oem in actions_list:
        # create oems directory
        if not Path(DTS_DIR).joinpath(oem).is_dir():
                print('Creating {}'.format(DTS_DIR+oem))
                Path(DTS_DIR).joinpath(oem).mkdir()
        makefile = open(DTS_DIR+'/'+oem+'/Makefile', 'w')
        makefile.write('# SPDX-License-Identifier: GPL-2.0\n')
        makefile.close()

        # Generate table makefile
        makefile_oem_action = {}
        for name in file_details:
                data = file_details[name]
                if data['oem'] == oem:
                        if not data['soc'] in makefile_oem_action:
                                makefile_oem_action[data['soc']] = []
                        makefile_oem_action[data['soc']].append(name)

        # Apply makefile
        makefile = Path(DTS_DIR).joinpath('Makefile').open('a')
        makefile.write('subdir-y += {}\n'.format(oem))
        makefile.close()
        makefile_oem = Path(DTS_DIR).joinpath(oem,'Makefile').open('a')
        for soc in makefile_oem_action:
                makefile_oem.write('dtb-$(CONFIG_{}) += \\\n'.format(soc.upper()))
                count = len(makefile_oem_action[soc])
                for index, file in enumerate(makefile_oem_action[soc]):
                        name = file.replace('.dts','')
                        remove_from_makefile(name)
                        if index == count-1:
                                makefile_oem.write('\t{}\n'.format(name+'.dtb'))
                        else:
                                makefile_oem.write('\t{} \\\n'.format(name+'.dtb'))

        makefile_oem.close()
        validate_makefile()

        broken_arm64_oem = None
        commit_arm64 = False

        # actualy sort
        for file in actions_list[oem]:
                if file in moved_file:
                        print("WARNING ALREADY MOVED FILE {}!".format(file))
                        exit()
                if file in blacklist:
                        print('Linking from {} to {}'.format(DTS_DIR+'/'+oem+'/'+file, DTS_DIR+'/'+file))
                        Path(DTS_DIR).joinpath(oem, file).symlink_to('../'+file)
                else:
                        moved_file.append(file)
                        print('Moving from {} to {}'.format(DTS_DIR+'/'+file, DTS_DIR+'/'+oem+'/'+file))
                        Path(DTS_DIR).joinpath(file).rename(Path(DTS_DIR).joinpath(oem, file))

                        #check dependency with arm64
                        broke_dep_data = check_dep_in_array(arm64_dep_list, file)
                        if len(broke_dep_data) > 0:
                                for broken_dep in broke_dep_data:
                                        commit_arm64 = True
                                        broken_arm64_oem = broken_dep['oem']
                                        fix_dep(file, broken_dep, oem)

        # print("Adding Makefiles")
        repo.git.add(DTS_DIR)
        # print("Committing Makefiles")
        # repo.git.commit('-s','-m', "ARM: dts: {}: move dts to dedicated directory".format(oem), '-m', "Move {} dts, dtsi and header to dedicated directory.".format(oem))
        # commit_count+=1

        if commit_arm64:
        #         print("Adding Dependency Fixes")
                repo.git.add(DTS_ARM64_DIR)
        #         print("Committing Dependency Fixes")
        #         repo.git.commit('-s','-m', "arm64: dts: {}: fix broken include from arm move".format(broken_arm64_oem), '-m', "Fix broken include from moving arm dts to {} directory.".format(oem))
        #         commit_count+=1

print("Final commit count {}".format(commit_count))