essential_extract.py

#!/usr/bin/env python3
import gzip
import os
import shutil
import subprocess
import zipfile
from defusedxml import ElementTree
import yaml


"""
*Run this on Linux, first of all
*Python 3.5+ is needed
*It will ask you for your sudo password, if you have a password set for it
*Have simg2img, abootimg, cpio, and Java (8 or 9) in your path (apt install simg2img abootimg cpio on *buntu)
*Have pyyaml and defusedxml installed via pip (pip3/pip install pyyaml defusedxml)
*Have the apktool jar as ~/apktool_2.3.3.jar (change in script as needed)
*Run this script from a terminal in the same folder as downloaded factory image (PH1-Image-blabla.zip)
*Comment out the functions in the bottom block if necessary
"""


def simg2img(indir, exts):
    for ext in exts:
        print("DECOMPRESSING: {0}".format(os.path.basename(ext)))
        subprocess.run(["simg2img", ext, ext.replace(".img", ".raw.img")])


def indiv_abootimg(imgfile):
    imgdir = os.path.dirname(imgfile)
    imgname = os.path.basename(imgfile).split(".")[0]
    outdir = os.path.join(imgdir, "output", imgname)
    here = os.getcwd()
    os.chdir(outdir)
    with open(os.devnull, "wb") as dnull:
        subprocess.run(["abootimg", "-x", imgfile], stdout=dnull, stderr=subprocess.STDOUT)
    for image in ["initrd.img"]:
        imagepath = os.path.join(outdir, image)
        if not os.path.exists(imagepath):
            pass
        else:
            imagedir = os.path.join(outdir, image.split(".")[0])
            if not os.path.exists(imagedir):
                os.makedirs(imagedir)
            os.chdir(imagedir)
            try:
                gzdata = gunzip(imagepath)
            except OSError:
                continue
            else:
                with open(os.devnull, "wb") as dnull:
                    subprocess.run(["cpio", "-id"], input=gzdata, stdout=dnull, stderr=subprocess.STDOUT)
    os.chdir(here)


def gunzip(gzfile):
    with gzip.open(gzfile) as gunfile:
        data = gunfile.read()
    return data


def boot_extract(indir):
    prep_output_dirs(indir, ["boot"])
    for img in ["boot.img"]:
        indiv_abootimg(os.path.join(indir, img))


def prep_dirs(indir):
    loopdir = os.path.join(indir, "loop")
    outdir = os.path.join(indir, "output")
    for dir in (loopdir, outdir):
        if not os.path.exists(dir):
            os.makedirs(dir)


def prep_output_dirs(indir, dirlist):
    for dir in dirlist:
        if not os.path.exists(dir):
            os.makedirs(os.path.join(indir, "output", dir))


def remove_loop(indir):
    if os.path.exists(os.path.join(indir, "loop")):
        os.rmdir(os.path.join(indir, "loop"))


def indiv_imgextract(imgfile, fstype="ext4"):
    imgdir = os.path.dirname(imgfile)
    imgname = os.path.basename(imgfile).split(".")[0]
    subprocess.run(["sudo", "mount", "-t", fstype, "-o", "loop", os.path.join(imgdir, imgfile), os.path.join(imgdir, "loop")])
    with open(os.devnull, "wb") as dnull:
        subprocess.run(["sudo", "cp", "-r", os.path.join(imgdir, "loop"), os.path.join(imgdir, "output", imgname)], stdout=dnull, stderr=subprocess.STDOUT)
    subprocess.run(["sudo", "umount", os.path.join(imgdir, "loop")])


def imgextract(indir, dirnames, fstype="ext4"):
    for dir in dirnames:
        print("EXTRACTING IMAGE: {0}".format(os.path.basename(dir)))
        indiv_imgextract(os.path.join(indir, dir), fstype)


def get_yml_info(yamlfile):
    with open(yamlfile, "r") as afile:
        skiptag = next(afile)
        ydata = yaml.load(afile.read())
    sdkinfo = ydata["sdkInfo"]
    if sdkinfo is None:
        sdkinfo = {"minSdkVersion": "unknown"}
    elif "minSdkVersion" not in sdkinfo.keys():
        sdkinfo = {"minSdkVersion": "unknown"}
    return sdkinfo, ydata["versionInfo"]


def get_xml_info(xmlfile):
    tree = ElementTree.parse(xmlfile)
    root = tree.getroot()
    return root.attrib["package"]


def is_odexed(apkdir):
    status = "deodexed" if "classes.dex" in os.listdir(apkdir) else "odexed"
    return status


def get_apk_filename(apkdir):
    sdkinfo, versioninfo = get_yml_info(os.path.join(apkdir, "apktool.yml"))
    packname = get_xml_info(os.path.join(apkdir, "AndroidManifest.xml"))
    status = is_odexed(apkdir)
    filename = "{0}_{1}-{2}_minAPI{3}_{4}.apk".format(packname, versioninfo["versionName"], versioninfo["versionCode"], sdkinfo["minSdkVersion"], status)
    return filename


def apktool(apkdir, basedir, apktoolpath=None, framedir=None):
    if apktoolpath is None:
        apktoolversion = "2.3.3"
        apktoolpath = os.path.join(os.path.expanduser("~"), "apktool_{0}.jar".format(apktoolversion))
    apkname = "{0}.apk".format(apkdir.split(os.sep)[-1])
    if framedir is None:
        framedir = os.path.join(basedir, "output", "system", "system", "framework")
    with open(os.devnull, "wb") as dnull:
        subprocess.run(["java", "-jar", apktoolpath, "d", os.path.join(apkdir, apkname), "-s", "-f", "-p", framedir, "-o", os.path.join(apkdir, apkname).replace(".apk", "")], stdout=dnull, stderr=subprocess.STDOUT)
    #subprocess.run(["java", "-jar", apktoolpath, "d", os.path.join(apkdir, apkname), "-s", "-f", "-p", framedir, "-o", os.path.join(apkdir, apkname).replace(".apk", "")])


def indiv_apkextract(basedir, outdir, apkdir, apktoolpath=None, framedir=None):
    apktool(apkdir, basedir, apktoolpath, framedir)
    apkname = apkdir.split(os.sep)[-1]
    try:
        fname = get_apk_filename(os.path.join(apkdir, apkname))
    except FileNotFoundError:
        pass
    else:
        aspl = apkdir.split(os.sep)
        bottom = aspl[-2]
        top = aspl[-3]
        newdir = os.path.join(basedir, "apks", top, bottom)
        if not os.path.exists(newdir):
            os.makedirs(newdir)
        shutil.copy(os.path.join(apkdir, "{0}.apk".format(apkname)), os.path.join(newdir, fname))
        shutil.rmtree(os.path.join(apkdir, apkname), ignore_errors=True)


def apk_extract(indir, apkdirs):
    for apkdir in apkdirs:
        aspl = apkdir.split(os.sep)
        bottom = aspl[-1]
        top = aspl[-2]
        outdir = os.path.join(indir, "apks", top, bottom)
        apklist = [os.path.join(apkdir, x) for x in os.listdir(apkdir) if are_there_apks(os.path.join(apkdir, x))]
        for apk in apklist:
            bname = os.path.basename(apk)
            print("COPYING APK: {0}.apk".format(os.path.join(top, bottom, bname)))
            indiv_apkextract(indir, outdir, apk)


def prep_apks(indir, apkdirs):
    for apkdir in apkdirs:
        aspl = apkdir.split(os.sep)
        bottom = aspl[-1]
        top = aspl[-2]
        if not os.path.exists(os.path.join(indir, "apks", top, bottom)):
            os.makedirs(os.path.join(indir, "apks", top, bottom))


def are_there_apks(indir):
    for root, dirs, files in os.walk(indir):
        for file in files:
            if file.lower().endswith(".apk"):
                return True
    return False


def unpack_images(indir):
    simg2img(indir, ["system.img"])


def dump_images(indir):
    prep_dirs(indir)
    baselist = ["system.raw.img", "dsp.img"]
    raws = [os.path.join(indir, x) for x in baselist]
    dirlist = [os.path.basename(x) for x in raws]
    prep_output_dirs(indir, dirlist)
    imgextract(indir, raws)
    boot_extract(indir)


def dump_radios(indir):
    baselist = ["bluetooth.img", "modem.img"]
    rads = [os.path.join(indir, x) for x in baselist]
    if rads:
        dirlist = [os.path.basename(x) for x in rads]
        prep_output_dirs(indir, dirlist)
        imgextract(indir, rads, fstype="vfat")


def collect_appdirs(indir):
    basedirs = [os.path.join(indir, "output", "system", "system"), os.path.join(indir, "output", "system", "system", "vendor")]
    apklistdirs = [x for x in basedirs if are_there_apks(x)]
    apkdirs = []
    for dir in apklistdirs:
        for tail in ("app", "priv-app"):
            if os.path.exists(os.path.join(dir, tail)) and are_there_apks(os.path.join(dir, tail)):
                apkdirs.append(os.path.join(dir, tail))
    return apkdirs


def dump_apks(indir):
    apkdirs = collect_appdirs(indir)
    prep_apks(indir, apkdirs)
    apk_extract(indir, apkdirs)


def extract_zip(indir):
    fzip = [x for x in os.listdir(indir) if x.endswith(".zip")][0]
    print("EXTRACTING: {0}".format(fzip))
    with zipfile.ZipFile(os.path.join(indir, fzip), allowZip64=True) as zipx:
        zipx.extractall(indir)


if __name__ == "__main__":
    indir = os.path.abspath(os.getcwd())
    print("EXTRACTING ZIP...")
    extract_zip(indir)
    print("DECOMPRESSING IMAGES...")
    unpack_images(indir)
    print("EXTRACTING IMAGES...")
    dump_images(indir)
    dump_radios(indir)
    print("EXTRACTING APKS...")
    dump_apks(indir)
    remove_loop(indir)
    print("EXTRACTION COMPLETE!")