ios_symbolication.py

import sys
import re
import json
import os

from downloads import download_logs
from downloads import download_symbols

from macholib.MachO import MachO
from macholib.SymbolTable import SymbolTable

from shutil import which
from shutil import rmtree

from argparse import ArgumentParser

import subprocess

class DemangleMode:
    """
    Represents a mode that should be used for demangling swift symbols.

    WHEN_AVAILABLE: Will attempt to demangling swift symbols provided that the `swift` command is found in your $PATH
                    variable. If the symbols cannot be demangled and verbose mode is enabled, a warning will be
                    printed to the console indicating that demangling is not available.

    ALWAYS:         Will attempt to demangling swift symbols provided that the `swift` command is found in your $PATH
                    variable. If the symbols cannot be demangled an error message will be printed to console and
                    execution will terminate.

    NEVER:          Demangling will not occur. 
    """
    WHEN_AVAILABLE = None
    ALWAYS         = None
    NEVER          = None

    def __init__(self, code):
        self.code = code

    def __eq__(self, other):
        if isinstance(other, DemangleMode):
            return self.code == other.code
        return False

# Set the initial values for the damangling options
DemangleMode.WHEN_AVAILABLE = DemangleMode(0)
DemangleMode.ALWAYS         = DemangleMode(1)
DemangleMode.NEVER          = DemangleMode(2)

class iOSSymbol:
    """
    Represents a single Symbol in an iOSImage instance.
    """
    def __init__(self, macho_sym, offset, demangle_mode = DemangleMode.WHEN_AVAILABLE, compact_demangle=True):
        """
        Initializes a new instance of iOSSymbol

        :param macho_sym:        The Macho-O Symbol information taken from the Macho-O Symbol Table
        :param offset:           The offset from the symbol at which the instruction resides.
        :param demangle_mode:    The mode to use for demangling symbol names.
        :param compact_demangle: Whether or not to use compact formatting when demangling symbols.
        """
        self.macho_sym = macho_sym
        self.offset = offset
        self.demangle_mode = demangle_mode
        self.compact_demangle = compact_demangle

    def describe_string(self):
        """
        Describes this symbol as a stack-trace friendly string.

        :return: The string describing this symbol.
        """
        offset_str = ""
        symbol_str = self.macho_sym[1].decode("utf-8") 
        if self.demangle_mode == DemangleMode.WHEN_AVAILABLE or self.demangle_mode == DemangleMode.ALWAYS:
            if which("swift") is not None:
                if self.compact_demangle:
                    symbol_str = subprocess.check_output(
                        ['swift', 'demangle', '--compact', "{0}".format(symbol_str)]
                    ).rstrip().decode("utf-8")
                else:
                    symbol_str = subprocess.check_output(
                        ['swift', 'demangle', "{0}".format(symbol_str)]
                    ).rstrip().decode("utf-8")
            elif self.demangle_mode == DemangleMode.ALWAYS:
                print("failed to demangle a symbol when demangle mode set to ALWAYS: swift command not available")
                quit(5)
        if self.offset != 0:
            offset_str = "+ {0}".format(self.offset)
        return "{0} {1}".format(symbol_str, offset_str)

class iOSImage:
    """
    Represents a binary image within an iOS Crash Dump
    """

    def __init__(self, symbols_dir, data, compute_slide_value=False):
        """
        Initialize a new instance of iOSImage

        :param symbols_dir:         The directory in which Mach-O symbol binaries are stored.
        :param data:                A map containing load_address, architecture, uuid, name and path.
        :param compute_slide_value: When set, will attempt to determine a proper slide value for the image based on it's
                                    Macho-O symbol binary. This will only be performed if the `otool` command is
                                    available in the users $PATH.
        """
        self.load_address = int(data["load_address"], 16)
        self.architecture = data["architecture"]
        self.uuid = data["uuid"]
        self.name = data["name"]
        self.path = data["path"]
        self.macho = MachO("{0}/{1}".format(symbols_dir, self.name).replace("/", os.sep))
        self.symbol_table = SymbolTable(self.macho)
        self.slide = 0x0

        # Note: I'm uncertain if the slide value is actually required to symbolicate. Right now, it seems like it works
        #       without it, so i'm making it optional. If it were to be required, it would be added to the end of the
        #       address calculation that's done in the `symbolicate_stack(self, address)` function.
        #
        if compute_slide_value:
            if which("otool") is not None:
                cmd = "otool -arch {0} -l {1} | grep __TEXT -m 2 -A 1 | grep vmaddr | awk '{{print $2}}'".format(
                    self.architecture, "{0}/{1}".format(symbols_dir, self.name).replace("/", os.sep)
                )
                ps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                self.slide = int(ps.communicate()[0].decode("utf-8").rstrip('\n'), 16)

    def symbolicate_stack(self, address, demangle_mode = DemangleMode.WHEN_AVAILABLE, compact_demangle=True):
        """
        Will symbolicate a stack address using the Mach-O symbol binary corresponding to this image and the load
        address provided for the binary.

        :param address:          The stack address to symbolicate.
        :param demangle_mode:    The mode to use for demangling symbol names.
        :param compact_demangle: Whether or not to use compact formatting when demangling symbols.

        :return:                 An instance of iOSSymbol or None if symbolication failed.
        """
        addr = address - self.load_address + self.slide
        return self.symbolicate(addr, demangle_mode, compact_demangle)

    def symbolicate(self, address, demangle_mode = DemangleMode.WHEN_AVAILABLE, compact_demangle=True):
        """
        Will symbolicate a address using the Mach-O symbol binary corresponding to this image.

        :param address:          The address to symbolicate.
        :param demangle_mode:    The mode to use for demangling symbol names.
        :param compact_demangle: Whether or not to use compact formatting when demangling symbols.

        :return:                 An instance of iOSSymbol or None if symbolication failed.
        """
        sym_lower = None

        for symbol in self.symbol_table.nlists:
            if symbol[0].n_value <= address:
                sym_lower = symbol
            elif symbol[0].n_value > address:
                break

        if sym_lower is None:
            return None

        if (sym_lower[0].n_value == self.symbol_table.nlists[0][0].n_value and
            address >= self.symbol_table.nlists[1][0].n_value):
            return None

        return iOSSymbol(sym_lower, address - sym_lower[0].n_value, demangle_mode, compact_demangle)

class iosCrashReport:
    """
    Represents a crash report associated with an iOS Clients Log File.
    """

    def __init__(self, log_id):
        """
        Initialize a new instance of an iOSCrashReport

        :param log_id: The identifier of the log containing a crash report you would like to symbolicate. This will be
                       downloaded and information will be extracted from it for use.
        """
        self.log_id = log_id
        self.crash_report = None
        self.load_address = -1
        self.binary_file = None
        self.elf_file = None
        self.images = []
        self.symbolicated = []

    def load_crash_report(self):
        """
        Will attempt to load the iOSCrashReport object from the log file into memory. If none is found, an error message
        will be printed to the console and execution will be terminated.

        It is important that the log file be downloaded before you call this function.
        """
        with open("./{0}.log".format(self.log_id).replace("/", os.sep)) as f:
            for line in f:
                if re.search(r'\sOBJECT:\s', line):
                    json_data = (re.split(r'\sOBJECT:\s', line)[1]).rstrip('\n')
                    obj = json.loads(json_data)
                    if obj["_objectType"] == "iOSCrashReport":
                        self.crash_report = obj
                        return
        print("no crash report found in log file")
        quit(1)

    def download_symbol_files(self, verbose=True):
        """
        Will attempt to download the symbol files corresponding to the application version found in the crash report.
        
        It is important that you call load_crash_report before calling this function.

        :param verbose: Whether or not to print out informative messages regarding the download process.
        """
        if self.crash_report == None:
            print("cannot download symbol files, no crash report")
            quit(1)
        if verbose:
            print("")
        download_symbols('client-ios', self.crash_report['app_version'], verbose)

    def load_images(self, compute_slide_values=False, verbose=True):
        """
        Will load information regarding the binary images found in the crash dump into memory. This includes some
        information that comes from the actual Mach-O binary files themselves.

        It's important that you call download_symbol_files before calling this function.

        :param compute_slide_values: When set, will attempt to determine a proper slide value for each image based on
                                     it's Macho-O symbol binary. This will only be performed if the `otool` command is
                                     available in the users $PATH. If `otool` is not available in the users $PATH and
                                     verbose is set to True, an error message will be printed to the logs and slide
                                     values will default to 0x0.
        :param verbose:              Whether or not to print out warning messages and other informative messages during
                                     the process of loading binary images.
        """
        image_not_symbolicated = 0

        if compute_slide_values:
            if which("otool") is None:
                if verbose:
                    print("")
                    print(("{0}warning: can not determine slide values for binary images due to {1}otool{0} not being" 
                           " found in $PATH{2}").format(bcolors.WARNING(), bcolors.OKBLUE(), bcolors.ENDC()))
                    print("")

        self.images = []
        for image in self.crash_report["binary_images"]:
            if os.path.isfile("./client-ios_{0}_symbols/{1}".format(
                self.crash_report['app_version'], image["name"]
            ).replace("/", os.sep)):
                img = iOSImage("./client-ios_{0}_symbols".format(
                    self.crash_report['app_version']
                ).replace("/", os.sep), image, compute_slide_values)
                if verbose:
                    print("loaded symbols for image {0} at ./client-ios_{1}_symbols/{0} with slide value {2}".format(
                        image["name"], self.crash_report['app_version'], hex(img.slide)
                    ).replace("/", os.sep))
                self.images.append(img)
            else:
                image_not_symbolicated += 1
        if image_not_symbolicated > 0:
            if verbose:
                print("")
                print(("{0}warning: could not locate symbols for {1}{2}{3} images that were listed in binary images"
                       " section of crash dump{4}").format(
                           bcolors.WARNING(), bcolors.HEADER(), image_not_symbolicated,
                           bcolors.WARNING(), bcolors.ENDC()
                       ))
                print(("{0}         this may indicate system libraries and is normally"
                       " not a cause for concern{1}").format(bcolors.BOLD(), bcolors.ENDC()))

    def symbolicate(self, demangle_mode = DemangleMode.WHEN_AVAILABLE, compact_demangle=True, 
                    compute_slide_values=False, verbose=True):
        """
        Will attempt to symbolicate every stack frame found in the crash log and store it in self.symbolicated

        :param demangle_mode:        The mode to use for demangling symbol names.
        :param compact_demangle:     Whether or not to use compact formatting when demangling symbols.
        :param compute_slide_values: When set, will attempt to determine a proper slide value for each image based on
                                     it's Macho-O symbol binary. This will only be performed if the `otool` command is
                                     available in the users $PATH. If `otool` is not available in the users $PATH and
                                     verbose is set to True, an error message will be printed to the logs and slide
                                     values will default to 0x0.
        :param verbose:              Whether or not to print out warning messages and other informative messages during
                                     the process of symbolication.

        :return:                     True when symbolication has completed, or False otherwise.
        """
        download_logs([self.log_id], force_reassembly=True, verbose=verbose)

        self.load_crash_report()
        if self.crash_report == None:
            return False

        self.download_symbol_files(verbose)

        self.load_images(compute_slide_values, verbose=verbose)

        if demangle_mode == DemangleMode.WHEN_AVAILABLE:
            if which("swift") is None:
                print("")
                print("{0}warning: demangling unavailable{1}".format(bcolors.WARNING(), bcolors.ENDC()))
                print("")

        symbolicated = []

        for stack_frame in self.crash_report["stack_trace"]:
            was_symbolicated = False
            for image in self.images:
                if image.name == stack_frame["image"]:
                    symbol = image.symbolicate_stack(
                        int(stack_frame["stack_address"], 16), demangle_mode, compact_demangle
                    )
                    if symbol is not None:
                        stack_frame["location"] = symbol.describe_string()
                        was_symbolicated = True
                    break
            symbolicated.append((stack_frame, was_symbolicated))

        self.symbolicated = symbolicated
        return True

    def write(self, output=sys.stdout, fmt='text', demangle_mode = DemangleMode.WHEN_AVAILABLE, compact_demangle=True,
              compute_slide_values=False, colored=False, verbose=False):
        """
        Will write the crash dump to the provided output using the provided fromat and configuration.

        :param output:               The output to write to. Defaults to sys.stdout
        :param fmt:                  The format to use. Supported formats include text and json.
        :param demangle_mode:        The mode to use for demangling symbol names.
        :param compact_demangle:     Whether or not to use compact formatting when demangling symbols.
        :param compute_slide_values: When set, will attempt to determine a proper slide value for each image based on
                                     it's Macho-O symbol binary. This will only be performed if the `otool` command is
                                     available in the users $PATH. If `otool` is not available in the users $PATH and
                                     verbose is set to True, an error message will be printed to the logs and slide
                                     values will default to 0x0.
        :param colored:              Whether or not to color output. This will only be applied if using the text format.
        :param verbose:              Whether or not to print out warning messages and other informative messages during
                                     the process of symbolication. This will only be applied when using the text format.
        """
        global enable_colors
        old_colored = enable_colors
        enable_colors = colored

        if not self.symbolicate(demangle_mode, compact_demangle, compute_slide_values, verbose=verbose):
            print("failed to symbolicate crash report")
            quit(3)

        if fmt == 'text':
            longest_image_name = 0
            for frame in self.crash_report["stack_trace"]:
                if len(frame["image"]) > longest_image_name:
                    longest_image_name = len(frame["image"])

            if verbose:
                output.write("\n")
            output.write("Error Message:\n")
            output.write("\n")
            output.write("{0}{1}{2}\n".format(bcolors.BOLD(), self.crash_report["message"], bcolors.ENDC()))
            output.write("\n")
            output.write("Stack Trace:\n")
            output.write("\n")
            output.write("Frame {0} {1} {2}\n".format(
                "Image".ljust(longest_image_name, ' '), "Stack Address".ljust(18, ' '), "Location"
            ))
            for idx,stack_frame in enumerate(self.symbolicated):
                location = stack_frame[0]["location"]
                if stack_frame[1]:
                    location = "{0}{1}{2}".format(bcolors.BOLD(), stack_frame[0]["location"], bcolors.ENDC())
                output.write("{0}    {1}{2}{3} {4}{5}{6} {7}\n".format(
                    str(idx).zfill(2), bcolors.HEADER(), stack_frame[0]["image"].ljust(longest_image_name, ' '),
                    bcolors.ENDC(), bcolors.OKBLUE(), stack_frame[0]["stack_address"], bcolors.ENDC(), location
                ))

            longest_image_name = 0
            for image in self.images:
                if len(image.name) > longest_image_name:
                    longest_image_name = len(image.name)

            output.write("\n")
            output.write("Binary Images:\n")
            output.write("\n")
            output.write("{0} {1} {2}  {3}\n".format(
                "Image ".ljust(longest_image_name, ' '), " Architecture", " Load Address", "Slide (vmaddr)"
            ))
            for image in self.images:
                output.write("{0} {1}  {2}     {3}\n".format(
                    image.name.ljust(longest_image_name + 1, ' '), image.architecture.ljust(12, ' '),
                    hex(image.load_address), hex(image.slide)
                ))
            if verbose:
                output.write("\n")
        elif fmt == 'json':
            crash_report = {
                "_objectType": "symbolicatediOSCrashReport",
                "message": self.crash_report["message"]
            }

            stack_frames = []
            binary_images = self.crash_report["binary_images"]
            image_information = []

            for idx,stack_frame in enumerate(self.symbolicated):    
                stack_frames.append(stack_frame[0])

            for image in self.images:
                oimg = image.__dict__.copy()
                del(oimg["macho"])
                del(oimg["symbol_table"])
                oimg["load_address"] = hex(oimg["load_address"])
                image_information.append(oimg)

            crash_report["stack_frames"] = stack_frames
            crash_report["binary_images"] = binary_images
            crash_report["image_information"] = image_information

            output.write(json.dumps(crash_report))
            pass
        enable_colors = old_colored

        if verbose:
            output.write("Cleaning up... ")
        rmtree("./client-ios_{0}_symbols".format(self.crash_report['app_version']).replace("/", os.sep))
        os.remove("./{0}.log".format(self.log_id).replace("/", os.sep))
        if verbose:
            output.write("{0}Done.{1}\n".format(bcolors.OKBLUE(), bcolors.ENDC()))

    def print_stack_trace(self, demangle_mode = DemangleMode.WHEN_AVAILABLE, compact_demangle=True,
                          compute_slide_values=False, colored=True, verbose=True):
        """
        Will print the stack trace and any related information to standard output.

        This is equivalent to running `write(sys.stdout, fmt='text')`.

        :param demangle_mode:        The mode to use for demangling symbol names.
        :param compact_demangle:     Whether or not to use compact formatting when demangling symbols.
        :param compute_slide_values: When set, will attempt to determine a proper slide value for each image based on
                                     it's Macho-O symbol binary. This will only be performed if the `otool` command is
                                     available in the users $PATH. If `otool` is not available in the users $PATH and
                                     verbose is set to True, an error message will be printed to the logs and slide
                                     values will default to 0x0.
        :param colored:              Whether or not to color output. This will only be applied if using the text format.
        :param verbose:              Whether or not to print out warning messages and other informative messages during
                                     the process of symbolication. This will only be applied when using the text format.
        """
        self.write(sys.stdout, fmt='text', demangle_mode = demangle_mode, compact_demangle=compact_demangle,
                  compute_slide_values=compute_slide_values, colored=colored, verbose=verbose)

enable_colors = True
class bcolors:
    """
    Utility class for ANSI escape sequences corresponding to color values.
    """
    @staticmethod
    def HEADER():
        return '\033[95m' if enable_colors else ''

    @staticmethod
    def OKBLUE():
        return '\033[94m' if enable_colors else ''

    @staticmethod
    def OKBLUE():
        return '\033[94m' if enable_colors else ''

    @staticmethod
    def WARNING():
        return '\033[93m' if enable_colors else ''

    @staticmethod
    def FAIL():
        return '\033[91m' if enable_colors else ''

    @staticmethod
    def ENDC():
        return '\033[0m' if enable_colors else ''

    @staticmethod
    def BOLD():
        return '\033[1m' if enable_colors else ''

    @staticmethod
    def UNDERLINE():
        return '\033[4m' if enable_colors else ''

def cli_entry_point():
    parser = ArgumentParser(description="Utility script for symbolicating iOS Crash Reports.")
    parser.add_argument("--log-id", dest="log_id", type=str,
                     help="A log ID corresponding to a log file that contains an iOS Crash Report.")
    parser.add_argument("--format", dest="fmt", type=str,
                     help="The format in which to display output. Options are 'json' and 'text'. Defaults to 'text'.")
    parser.add_argument("--demangle-mode", dest="demangle_mode", type=str,
                     help=("The mode to use for demangling swift symbols. Options are 'WHEN_AVAILABLE', 'ALWAYS' and "
                           "'NEVER'. Default value is 'WHEN_AVAILABLE'."))
    parser.add_argument("--expanded-demangle", dest="expanded_demangle", action="store_true", default=False,
                     help=("When swift symbol demangling is enabled, providing this option will tell the script to "
                           "display both the mangled and demangled versions of the symbol."))
    parser.add_argument("--compute-slide-values", dest="compute_slide_values", action="store_true", default=False,
                     help=("When set, will attempt to determine a proper slide value for each binary image based on "
                           "it's Macho-O symbol binary. This will only be performed if the `otool` command is "
                           "available in the users $PATH. If `otool` is not available in the users $PATH and the "
                           "--verbose flag is set, an error message will be printed to the logs and slide values will "
                           "default to 0x0."))
    parser.add_argument("--disable-color", dest="disable_color", action="store_true", default=False,
                     help=("When set, ANSI escape sequences that correspond to color codes will be omitted from the "
                           "output. This only takes effect when using the 'text' format."))
    parser.add_argument("--verbose", dest="verbose", action="store_true", default=False,
                     help=("When set, will print out warning messages and other informative messages during the "
                           "process. This only takes effect when using the 'text' format."))

    args = parser.parse_args()

    if args.log_id is None:
        print("error: please provide a valid log identifier")
        quit(1)

    fmt = 'text'
    demanglemode = DemangleMode.WHEN_AVAILABLE
    compact_demangle = True
    compute_slide_values = False
    colored = True
    verbose = False

    if args.fmt is not None:
        if args.fmt.lower() in ['text', 'json']:
            fmt = args.fmt.lower()
        else:
            print("Warning: Invalid format '{0}' provided. Defaulting to 'text'.".format(args.fmt))

    if args.demangle_mode is not None:
        dmm = args.demangle_mode.lower()
        if dmm in ['when_available', 'always', 'never']:
            if dmm == 'always':
                demanglemode = DemangleMode.ALWAYS
            elif dmm == 'never':
                demanglemode = DemangleMode.NEVER
            elif dmm == 'when_available':
                demanglemode = DemangleMode.WHEN_AVAILABLE
        else:
            print(("Warning: Invalid demangle mode provide. Options are 'WHEN_AVAILABLE', 'ALWAYS' and 'NEVER'. "
                   "Defaulting to 'WHEN_AVAILABLE'."))

    if args.expanded_demangle is not None:
        compact_demangle = not args.expanded_demangle

    if args.compute_slide_values is not None:
        compute_slide_values = args.compute_slide_values

    if args.disable_color is not None:
        colored = not args.disable_color

    if args.verbose is not None:
        verbose = args.verbose

    crash = iosCrashReport(args.log_id)
    crash.write(sys.stdout, fmt=fmt, demangle_mode=demanglemode, compact_demangle=compact_demangle,
                  compute_slide_values=compute_slide_values, colored=colored, verbose=verbose)
    if fmt == 'json':
        sys.stdout.write("\n")