SelfAU3 de-obfuscation script, it removes junk code, un-accessed local and global variables and de-obfuscate all strings.

SelfAU3-deobfusct.py

import re
from itertools import filterfalse
from collections import Counter
from typing import List, Tuple, Set

reg_DENIS = re.compile(r"(dennis\(\")([0-9\]\"]{1,})(,)([0-9]{1,})([\/])([0-9]{1,})([\)])")
reg_INT_VAR = re.compile(r"(\$)(?P<name>[a-zA-Z0-9]{1,})( )=( )([0-9]{1,})")


def dennis(string, diff):
    try:
        diff = int(diff)
    except ZeroDivisionError:
        diff = 0
    string = string.split("]")
    out = ""
    for i in string:
        out += chr(int(i) - diff)
    return out


def replace_dennis(file_content: List[str]) -> List[str]:
    """string deobfuscation"""

    output_file = []
    for line in file_content:
        temp = line
        output = reg_DENIS.findall(line)
        if output:
            for match in output:
                try:
                    diff = int(match[3].replace("'",'')) / int(match[5].replace("'", ''))
                except ZeroDivisionError:
                    diff = 0
                clear = dennis(match[1].replace('"',''), diff)
                temp = temp.replace(''.join(match), clear) # replace by the deobfuscated code

        output_file.append(temp)
    return output_file


def remove_uncalled_function(file_content: List[str]) -> List[str]:
    function_list: dict = {}

    for line_nb, line in enumerate(file_content):
        if line.startswith("Func "):
            temp = line.split("(")[0].replace("Func ", '')
            for line_nb_2, line in enumerate(file_content[line_nb:]):
                if line.startswith("EndFunc"):
                    function_list[temp] = {"start": line_nb, "end": line_nb_2 + line_nb}
                    break

    to_remove = {}
    for func in function_list.keys():
        called = False
        for line in file_content:
            if f"{func}(" in line:
                if not f"Func {func}(" in line:
                    called = True
        if not called:
            to_remove[func] = function_list[func]
          
    buff = []

    ranges = {tuple(v.values()) for v in to_remove.values()}

    for nb, line in enumerate(file_content):
        if len(list(filterfalse(lambda x: x[0] <= nb and x[1] >= nb, ranges))) == len(ranges):
            buff.append(line)
                
    return buff

def remove_lines_range(file_content: List[str], ranges: Set[Tuple]) -> List[str]:
    """remove line from the script based on a list of range of line to exclude:
    ex: {(1, 2), (5, 9)} means remove line 1, 2, 5, 6, 7, 8, 9"""

    buff: List[str] = []

    for nb, line in enumerate(file_content):
        if len(list(filterfalse(lambda x: x[0] <= nb and x[1] >= nb, ranges))) == len(ranges):
            buff.append(line)

    return buff

def replace_lines_range(file_content: List[str], ranges: Set[Tuple]) -> List[str]:
    """remove line from the script based on a list of range of line to exclude:
    ex: {(1, 2, "toto"), (5, 9, "MsgBox(titi)")} means:
    replace lines: 1, 2 by "toto" and
    replace lines: 5, 6, 7, 8, 9 by MsgBox(titi)"""

    buff: List[str] = []

    for nb, line in enumerate(file_content):
        if len(list(filterfalse(lambda x: x[0] <= nb and x[1] >= nb, ranges))) == len(ranges):
            buff.append(line)
        else:
            for _range in ranges:
                if nb >= _range[0] and nb <= _range[1]:
                    if _range[2]:
                        buff.append(_range[2])
                        ranges.remove(_range)
                        ranges.add((_range[0], _range[1], ''))
                        break
    return buff


def remove_deadcode(file_content: List[str]) -> List[str]:
    """remove dead code example:
    $a = 12
    $c = 11
    For $b = 15 to 303
      If $a = 1 Then
        do something useless
      EndIf
      If $a = 42 Then
         another usless if case
      EndIf
      If $a = 12 Then
        MsgBox("test")
      EndIf
    Next
    The above code could be replace by:
    MsgBox("test")
    """

    ranges = set()
    for nb, line in enumerate(file_content):
        if line.startswith("For "):
            start = nb - 2
            end = 0
            replace_value = ""
            constante_pattern = file_content[nb - 2].strip()
            for nb2, line2 in enumerate(file_content[nb:]):
                if line2.startswith("Next"):
                    end = nb + nb2
                    ranges.add((start, end, replace_value))
                    break
                if constante_pattern in line2 and "Then" in line2:
                    replace_value = file_content[nb + nb2 + 1]

    for nb, line in enumerate(file_content):
        if line.lower().startswith("while "):
            start = nb
            end = 0
            replace_value = ""
            constante_pattern = file_content[nb - 2].strip()
            if not constante_pattern.startswith("$"):
                continue
            for nb2, line2 in enumerate(file_content[nb:]):
                if line2.lower().startswith("wend"):
                    end = nb + nb2
                    ranges.add((start, end, replace_value))
                    break
                if constante_pattern in line2 and "Then" in line2:
                    replace_value = file_content[nb + nb2 + 1]


    return replace_lines_range(file_content, ranges)
                    

def remove_unused_local_var(file_content: List[str]) -> List[str]:
    """remove unused local variable, ex:
    Func a(toto, titi)
      $BLABLA = 123
      $DDD = 333
      $CCC = 111
      $x = DllCall()
      $DDD = $x + 1
    EndFunc
    $BLABLA and $CCC will be deleted from the function scope
    """

    _file_content = "\n".join(file_content)

    to_remove = list(filter(lambda x: x[1] == 1, Counter(re.findall(r"(\$[a-zA-Z0-9]{1,})", _file_content)).items()))

    buff: List[str] = []
    
    for line in file_content:
        to_removed = False
        for remove in to_remove:
            if remove[0] in line and "global" not in line.lower():
                to_removed = True
        if not to_removed:
            buff.append(line)

    return buff
                
    
def tabify_script(file_content: List[str]) -> List[str]:

    buff: List[str] = []
    tab_counter = 0

    for line in file_content:
        buff.append("\t" * tab_counter + line)
        _line = line.lower()
        if _line.startswith("while ") or _line.startswith("for ") or _line.startswith("if "):
            tab_counter += 1
        if _line.startswith("endif") or _line.startswith("next") or _line.startswith("wend"):
            buff.pop(-1)
            tab_counter -= 1
            buff.append("\t" * tab_counter + line)

        if _line.startswith("endfunc"):
            tab_counter = 0
            buff.pop(-1)
            buff.append("\t" * tab_counter + line)
        if _line.startswith("func "):
            buff.pop(-1)
            tab_counter = 1
            buff.append(line)

    return buff

        

def write_outputfile(name, output_file: List[str]):

    with open(f"deob-{name}", "w") as f:
        f.write("\n".join(output_file))
        
if __name__ == "__main__":
    import sys
    filename = sys.argv[1]

    with open(filename, "r") as f:
        d = f.read().split("\n")

    
    print(f"Original script: {len(d)}")
    out = replace_dennis(d)
    print(f"deobfuscate strings {len(out)}")
    out = remove_uncalled_function(out)
    print(f"remove uncalled function {len(out)}")
    out = remove_deadcode(out)
    print(f"remove dead code in used function {len(out)}")
    out = remove_unused_local_var(out)
    print(f"remove unaccessed local & global variable {len(out)}")

    out = tabify_script(out)
    
    write_outputfile(filename, out)