Remote control apps running on a smartphone

AppsOverADB.kt

/*
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You can find a copy of the GNU General Public License at <https://www.gnu.org/licenses/>. 
*/

/*
Usage:
Start the attached service
Forward TCP port: adb forward 9900 9900
Connect your browser to http://localhost:9900
Enjoy
*/
package me.phh.treble.app

import android.annotation.SuppressLint
import android.app.ActivityOptions
import android.app.Service
import android.content.ComponentName
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.content.pm.ResolveInfo
import android.graphics.*
import android.hardware.display.DisplayManager
import android.hardware.display.VirtualDisplay
import android.media.*
import android.os.*
import android.util.Log
import android.view.*
import fi.iki.elonen.NanoHTTPD
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.lang.StringBuilder
import java.util.*
import kotlin.concurrent.thread

class AppsOverAdb : Service() {
    private val handler = Handler(HandlerThread("AppsOverADB").also { it.start() }.looper)

    val componentToDisplay = mutableMapOf<String, VirtualDisplay>()
    val componentToReader = mutableMapOf<String, ImageReader>()
    var encoder: MediaCodec? = null
    @SuppressLint("WrongConstant")
    private fun getDisplay(component: String): VirtualDisplay {
        if(componentToDisplay.containsKey(component)) return componentToDisplay[component]!!
        val dm = getSystemService(DisplayManager::class.java)

        val saveAsBitmap = "true".toBoolean()
        val compId = component.hashCode()

        val width = 720
        val height = 1280
        val surface = if(saveAsBitmap) {
            val imageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 5)
            componentToReader[component] = imageReader

            imageReader.setOnImageAvailableListener({ p0 ->
                Log.d("PHH-AOA", "Got new image ${System.currentTimeMillis()}")
                try {
                    val image = p0.acquireLatestImage() ?: return@setOnImageAvailableListener
                    val plane = image.planes[0]
                    val rowPadding = plane.rowStride - plane.pixelStride * image.width
                    val bmp = Bitmap.createBitmap(image.width + rowPadding / plane.pixelStride, image.height, Bitmap.Config.ARGB_8888)
                    bmp.copyPixelsFromBuffer(plane.buffer)
                    image.close()
                    val fos = FileOutputStream("/sdcard/test-tmp-$compId.png")
                    bmp.compress(Bitmap.CompressFormat.PNG, 100, fos)
                    fos.close()
                    File("/sdcard/test-$compId.png").delete()
                    File("/sdcard/test-tmp-$compId.png").renameTo(File("/sdcard/test-$compId.png"))
                    Log.d("PHH-AOA", "New image ${System.currentTimeMillis()}")
                } catch(t: Throwable) {
                    Log.d("PHH-AOA", "Failed dumping image", t)
                }
            }, handler)
            imageReader.surface
        } else {
          // Please note that so far this code path DOES NOT WORK, because muxer can't do real-time muxing!
            val mimeType = "video/x-vnd.on2.vp9"
            //encoder = MediaCodec.createEncoderByType(mimeType)
            encoder = MediaCodec.createByCodecName("OMX.google.vp9.encoder")
            val muxer = MediaMuxer("/data/data/me.phh.treble.app/test.webm", MediaMuxer.OutputFormat.MUXER_OUTPUT_WEBM)

            encoder!!.setCallback(object: MediaCodec.Callback() {
                var trackId: Int = -1
                var timeOffset: Long = -1L
                override fun onOutputBufferAvailable(p0: MediaCodec, p1: Int, p2: MediaCodec.BufferInfo) {
                    try {
                        Log.d("PHH-AOA", "output buffer available (size = ${p2.size} $p1, $p2 to track $trackId")
                        if (trackId == -1) {
                            Log.d("PHH-AOA", "Ignoring buffers")
                            encoder!!.getOutputBuffer(p1)
                            encoder!!.releaseOutputBuffer(p1, false)
                            return
                        }
                        if ((p2.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
                            Log.d("PHH-AOA", "Got codec config stuff. Ignoring, it is in format already")
                            encoder!!.getOutputBuffer(p1)
                            encoder!!.releaseOutputBuffer(p1, false)
                            return
                        }

                        if (p2.size == 0) {
                            Log.d("PHH-AOA", "Receive empty packet?")
                            encoder!!.getOutputBuffer(p1)
                            encoder!!.releaseOutputBuffer(p1, false)
                            return
                        }
                        val encodedData = encoder!!.getOutputBuffer(p1)!!

                        if(p2.presentationTimeUs != 0L) {
                            if (timeOffset == -1L) {
                                timeOffset = p2.presentationTimeUs
                                p2.presentationTimeUs = 0
                            } else {
                                p2.presentationTimeUs -= timeOffset
                            }
                        }

                        encodedData.position(p2.offset)
                        encodedData.limit(p2.size + p2.offset)
                        muxer.writeSampleData(trackId, encodedData, p2)

                        encoder!!.releaseOutputBuffer(p1, false)
                        Log.d("PHH-AOA", "Released output buffer")
                    } catch(t: Throwable) {
                        Log.d("PHH-AOA", "Failed reading buffer...", t)
                    }
                }

                override fun onInputBufferAvailable(p0: MediaCodec, p1: Int) {
                    Log.d("PHH-AOA", "input buffer available $p1")
                }

                override fun onOutputFormatChanged(p0: MediaCodec, p1: MediaFormat) {
                    if(trackId == -1) {
                        trackId = muxer.addTrack(p1)
                        muxer.start()
                    }
                    Log.d("PHH-AOA", "output format changed $p1")
                }

                override fun onError(p0: MediaCodec, p1: MediaCodec.CodecException) {
                    Log.d("PHH-AOA", "media error $p1")
                }
            }, handler)

            val format = MediaFormat.createVideoFormat(mimeType, width, height)
            format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
            format.setInteger(MediaFormat.KEY_BIT_RATE, 3000000)
            format.setInteger(MediaFormat.KEY_FRAME_RATE, 60)
            format.setInteger("max-fps-to-encoder", 30)
            format.setInteger("low-latency", 1)
            format.setInteger(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 100*1000)
            format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5)
            format.setLong(MediaFormat.KEY_DURATION, 1000*1000*1000L)
            encoder!!.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)

            val surface = encoder!!.createInputSurface()

            encoder!!.start()

            surface
        }

        Log.d("PHH-AOA", "Creating virtual display")
        val virtualDisplay = dm.createVirtualDisplay(
                "Phh-AppsOverAdb-$component", width, height, 320,
                surface, DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC or (1 shl 6) or (1 shl 10)) //Supports touch and is trusted
        componentToDisplay[component] = virtualDisplay

        handler.postDelayed({
            try {
                val s = component.split("/")
                val comp = ComponentName(s[0], s[1])
                val i = Intent().setComponent(comp)
                val options = ActivityOptions.makeBasic()
                options.launchDisplayId = virtualDisplay.display.displayId
                val m = Context::class.java.getMethod("startActivityAsUser", Intent::class.java, Bundle::class.java, UserHandle::class.java)
                m.invoke(this, i, options.toBundle(), UserHandle.getUserHandleForUid(10105))
            } catch(t: Throwable) {
                Log.d("PHH-AOA", "Failed sending intent", t)
            }
            Log.d("PHH-AOA", "Started $component")
        }, 500L)

        return virtualDisplay
    }

    override fun onCreate() {
        super.onCreate()

        thread {

            try {
                val httpServer = object : NanoHTTPD(9900) {
                    init {
                        Log.d("PHH-AOA", "Starting http server")
                        start(NanoHTTPD.SOCKET_READ_TIMEOUT, false)
                        Log.d("PHH-AOA", "Started http server")
                    }

                    override fun serve(session: IHTTPSession): Response {
                        Log.d("PHH-AOA", "Receive request for ${session.uri}")
                        try {
                            val component = session.parameters["act"]?.firstOrNull()
                            val displayId = (if(component != null) componentToDisplay[component] else null)?.display?.displayId
                            if (session.uri.endsWith("screen.png")) {
                                val compId = component!!.hashCode()

                                val input = (0..10).map {
                                    try {
                                        FileInputStream("/sdcard/test-$compId.png")
                                    } catch (t: Throwable) {
                                        Thread.sleep(10); null
                                    }
                                }.filterNotNull().firstOrNull()
                                return newChunkedResponse(Response.Status.OK, "image/png", input).apply { addHeader("Cache-Control", "no-store") }
                            } else if (session.uri.endsWith("/click")) {
                                val reqX = session.parameters["x"]!!.first().toFloat()
                                val reqY = session.parameters["y"]!!.first().toFloat()

                                val im = Class.forName("android.hardware.input.InputManager").getDeclaredMethod("getInstance").invoke(null)
                                val injectEventMethod = im.javaClass.getDeclaredMethod("injectInputEvent", InputEvent::class.java, Int::class.java)
                                val downTime = SystemClock.uptimeMillis()
                                for (action in listOf(MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP)) {
                                    val eventTime = if (action == MotionEvent.ACTION_DOWN) downTime else downTime + 10
                                    val ev = MotionEvent.obtain(
                                            downTime,
                                            eventTime,
                                            action,
                                            1, // number of pointers
                                            Array(1) { MotionEvent.PointerProperties().apply { toolType = MotionEvent.TOOL_TYPE_FINGER; id = 12 } },
                                            Array(1) { MotionEvent.PointerCoords().apply { x = reqX; y = reqY; pressure = 1.0f; size = 1.0f; } },
                                            0, // metaState
                                            0, // buttonState
                                            1.0f, 1.0f, //x/y precision
                                            0, //deviceId
                                            0, //edgeFlags
                                            InputDevice.SOURCE_MOUSE,
                                            0 //flags
                                    )
                                    val setDisplayIdMethod = ev.javaClass.getDeclaredMethod("setDisplayId", Int::class.java)
                                    setDisplayIdMethod(ev, displayId!!)

                                    injectEventMethod.invoke(im, ev, 1 /* INJECT_INPUT_EVENT_MODE_WAIT_FOR_RESULT */)
                                }
                                return newFixedLengthResponse(Response.Status.OK, "text/text", "Yay")
                            } else if (session.uri.endsWith("/wheel")) {
                                val reqX = session.parameters["x"]!!.first().toFloat()
                                val reqY = session.parameters["y"]!!.first().toFloat()
                                val reqDY = session.parameters["dy"]!!.first().toFloat()

                                val im = Class.forName("android.hardware.input.InputManager").getDeclaredMethod("getInstance").invoke(null)
                                val injectEventMethod = im.javaClass.getDeclaredMethod("injectInputEvent", InputEvent::class.java, Int::class.java)
                                val downTime = SystemClock.uptimeMillis()
                                val ev = MotionEvent.obtain(
                                        downTime,
                                        downTime,
                                        MotionEvent.ACTION_SCROLL,
                                        1, // number of pointers
                                        Array(1) { MotionEvent.PointerProperties().apply { toolType = MotionEvent.TOOL_TYPE_MOUSE; id = 12 } },
                                        Array(1) { MotionEvent.PointerCoords().apply { x = reqX; y = reqY; pressure = 1.0f; size = 1.0f; setAxisValue(MotionEvent.AXIS_VSCROLL, -reqDY)} },
                                        0, // metaState
                                        0, // buttonState
                                        1.0f, 1.0f, //x/y precision
                                        0, //deviceId
                                        0, //edgeFlags
                                        InputDevice.SOURCE_MOUSE,
                                        0 //flags
                                )
                                val setDisplayIdMethod = ev.javaClass.getDeclaredMethod("setDisplayId", Int::class.java)
                                setDisplayIdMethod(ev, displayId!!)

                                injectEventMethod.invoke(im, ev, 1 /* INJECT_INPUT_EVENT_MODE_WAIT_FOR_RESULT */)
                                return newFixedLengthResponse(Response.Status.OK, "text/text", "Yay")
                            } else if(session.uri.endsWith("/keypress")) {
                                val preprocessedKey = session.parameters["y"]!!.first().toString()
                                val key = if(preprocessedKey.toLowerCase() == "enter") "\n" else preprocessedKey

                                Log.d("PHH-AOA", "Pressing '$key'")

                                val im = Class.forName("android.hardware.input.InputManager").getDeclaredMethod("getInstance").invoke(null)
                                val injectEventMethod = im.javaClass.getDeclaredMethod("injectInputEvent", InputEvent::class.java, Int::class.java)

                                val kcm = KeyCharacterMap.load(0)
                                val events = kcm.getEvents(key.toCharArray())
                                for(ev in events) {
                                    val setDisplayIdMethod = ev.javaClass.getDeclaredMethod("setDisplayId", Int::class.java)
                                    setDisplayIdMethod(ev, displayId!!)

                                    injectEventMethod.invoke(im, ev, 1 /* INJECT_INPUT_EVENT_MODE_WAIT_FOR_RESULT */)
                                }

                                return newFixedLengthResponse(Response.Status.OK, "text/text", "Yay")
                            } else if(session.uri.endsWith("/back")) {

                                Log.d("PHH-AOA", "Pressing BACK")

                                val im = Class.forName("android.hardware.input.InputManager").getDeclaredMethod("getInstance").invoke(null)
                                val injectEventMethod = im.javaClass.getDeclaredMethod("injectInputEvent", InputEvent::class.java, Int::class.java)

                                val downTime = SystemClock.uptimeMillis()
                                for (action in listOf(KeyEvent.ACTION_DOWN, KeyEvent.ACTION_UP)) {
                                    val eventTime = if (action == KeyEvent.ACTION_DOWN) downTime else downTime + 10
                                    val ev = KeyEvent(downTime, eventTime, action, KeyEvent.KEYCODE_BACK, 0)
                                    val setDisplayIdMethod = ev.javaClass.getDeclaredMethod("setDisplayId", Int::class.java)
                                    setDisplayIdMethod(ev, displayId!!)

                                    injectEventMethod.invoke(im, ev, 1 /* INJECT_INPUT_EVENT_MODE_WAIT_FOR_RESULT */)
                                }

                                return newFixedLengthResponse(Response.Status.OK, "text/text", "Yay")
                            } else if(session.uri.endsWith("/close")) {
                                val display = componentToDisplay[component]!!
                                display.release()
                                val imReader = componentToReader[component]!!
                                imReader.close()
                                val compId = component!!.hashCode()
                                File("/sdcard/test-$compId.png").delete()
                                return newFixedLengthResponse(Response.Status.OK, "text/text", "Yay")
                            } else if(session.uri.endsWith("display")) {
                                val component = session.parameters["act"]!!.first()
                                getDisplay(component)
                                return newChunkedResponse(Response.Status.OK, "text/html", resources.openRawResource(R.raw.index))
                            } else {
                                val i = Intent(Intent.ACTION_MAIN).addCategory(Intent.CATEGORY_LAUNCHER)
                                val activities = packageManager
                                        .queryIntentActivities(i, PackageManager.MATCH_ALL)

                                activities.sortWith(ResolveInfo.DisplayNameComparator(packageManager))
                                val sb = StringBuilder()
                                sb.append("<html><body>")
                                for (resolveInfo in activities) {
                                    val pkgName = resolveInfo.activityInfo.applicationInfo.packageName
                                    val className = resolveInfo.activityInfo.name
                                    sb.append("<a href=\"/display?act=$pkgName/$className\">")

                                    val name = resolveInfo.loadLabel(packageManager)
                                    val icon = resolveInfo.loadIcon(packageManager)
                                    sb.append(name)
                                    val bitmap = Bitmap.createBitmap(96, 96, Bitmap.Config.ARGB_8888)
                                    val canvas = Canvas(bitmap)
                                    icon.setBounds(0, 0, 95, 95)
                                    icon.draw(canvas)

                                    val os = ByteArrayOutputStream()
                                    bitmap.compress(Bitmap.CompressFormat.PNG, 100, os)
                                    val byteArray = os.toByteArray()
                                    val b64 = Base64.getEncoder().encodeToString(byteArray)
                                    sb.append("<img src=\"data:image/png;base64, $b64\" />")
                                    sb.append("</a>")

                                    sb.append("<br>")
                                }
                                sb.append("</body></html>")
                                return newFixedLengthResponse(Response.Status.OK, "text/html", sb.toString())
                            }
                        } catch(t: Throwable) {
                            Log.d("PHH", "Hello", t)
                        }
                        return super.serve(session)
                    }
                }
            } catch(t: Throwable) {
                Log.d("PHH", "http server gave bip", t)
            }
        }
    }

    override fun onBind(p0: Intent?): IBinder? {
        return null
    }
}