Rendering with a TextureView

EglBaseProviderReflectionUtils.java

import org.webrtc.EglBase;

import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

/*
 * Uses reflection to interact with non public class EglBaseProvider.
 */
public class EglBaseProviderReflectionUtils {

    public static Object getEglBaseProvider(Object owner) {
        Object eglBaseProvider = null;
        try {
            Class<?> eglBaseProviderClass = Class.forName("com.twilio.video.EglBaseProvider");
            Method instanceMethod = eglBaseProviderClass.getDeclaredMethod("instance",
                    Object.class);
            instanceMethod.setAccessible(true);
            eglBaseProvider = instanceMethod.invoke(null, owner);
        } catch (Throwable e) {
            e.printStackTrace();
        }

        return eglBaseProvider;
    }

    public static EglBase.Context getRootEglBaseContext(Object eglBaseProvider) {
        EglBase.Context rootEglBaseContext = null;

        try {
            Field rootEglBaseField = eglBaseProvider.getClass().getDeclaredField("rootEglBase");
            rootEglBaseField.setAccessible(true);
            Object rootEglBase = rootEglBaseField.get(eglBaseProvider);
            Method getEglBaseContextMethod = rootEglBase.getClass()
                    .getDeclaredMethod("getEglBaseContext");
            getEglBaseContextMethod.setAccessible(true);
            rootEglBaseContext = (EglBase.Context) getEglBaseContextMethod.invoke(rootEglBase);
        } catch (NoSuchFieldException e) {
            e.printStackTrace();
        } catch (IllegalAccessException e) {
            e.printStackTrace();
        } catch (NoSuchMethodException e) {
            e.printStackTrace();
        } catch (InvocationTargetException e) {
            e.printStackTrace();
        }

        return rootEglBaseContext;
    }

    public static void relaseEglBaseProvider(Object eglBaseProvider, Object owner) {
        try {
            Method eglBaseProviderReleaseMethod = eglBaseProvider.getClass()
                    .getDeclaredMethod("release", Object.class);
            eglBaseProviderReleaseMethod.setAccessible(true);
            eglBaseProviderReleaseMethod.invoke(eglBaseProvider, owner);
        } catch (NoSuchMethodException e) {
            e.printStackTrace();
        } catch (InvocationTargetException e) {
            e.printStackTrace();
        } catch (IllegalAccessException e) {
            e.printStackTrace();
        }
    }
}

VideoTextureView.java

import android.content.Context;
import android.content.res.Resources;
import android.graphics.Point;
import android.graphics.SurfaceTexture;
import android.os.Handler;
import android.os.Looper;
import android.util.AttributeSet;
import android.view.TextureView;

import com.twilio.video.I420Frame;
import com.twilio.video.VideoRenderer;

import org.webrtc.EglBase;
import org.webrtc.EglRenderer;
import org.webrtc.GlRectDrawer;
import org.webrtc.Logging;
import org.webrtc.RendererCommon;
import org.webrtc.ThreadUtils;

import java.lang.reflect.Field;
import java.util.concurrent.CountDownLatch;

public class VideoTextureView extends TextureView
    implements VideoRenderer, TextureView.SurfaceTextureListener {

    private static final String TAG = "VideoTextureView";

    // Cached resource name.
    private final String resourceName;
    private final RendererCommon.VideoLayoutMeasure videoLayoutMeasure =
        new RendererCommon.VideoLayoutMeasure();
    private final EglRenderer eglRenderer;

    // Callback for reporting renderer events. Read-only after initilization so no lock required.
    private RendererCommon.RendererEvents rendererEvents = new RendererCommon.RendererEvents() {
        @Override
        public void onFirstFrameRendered() {
            if (listener != null) {
                listener.onFirstFrame();
            }
        }

        @Override
        public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation) {
            if (listener != null) {
                listener.onFrameDimensionsChanged(videoWidth, videoHeight, rotation);
            }
        }
    };
    private VideoRenderer.Listener listener;

    private final Object layoutLock = new Object();
    private Handler uiThreadHandler = new Handler(Looper.getMainLooper());
    private boolean isFirstFrameRendered;
    private int rotatedFrameWidth;
    private int rotatedFrameHeight;
    private int frameRotation;

    // Accessed only on the main thread.
    private int surfaceWidth;
    private int surfaceHeight;

    private Object eglBaseProvider;
    private Field webRtcI420FrameField;

    public VideoTextureView(Context context) throws NoSuchFieldException {
        this(context, null);
    }

    public VideoTextureView(Context context, AttributeSet attrs) throws NoSuchFieldException {
        super(context, attrs);
        this.resourceName = getResourceName();
        eglRenderer = new EglRenderer(resourceName);
        setSurfaceTextureListener(this);
        webRtcI420FrameField = I420Frame.class.getDeclaredField("webRtcI420Frame");
        webRtcI420FrameField.setAccessible(true);
    }

    @Override
    protected void onAttachedToWindow() {
        super.onAttachedToWindow();
        // Do not setup the renderer when using developer tools to avoid EGL14 runtime exceptions
        if(!isInEditMode()) {
            eglBaseProvider = EglBaseProviderReflectionUtils.getEglBaseProvider(this);
            init(EglBaseProviderReflectionUtils.getRootEglBaseContext(eglBaseProvider), rendererEvents);
        }
    }

    @Override
    protected void onDetachedFromWindow() {
        eglRenderer.release();
        EglBaseProviderReflectionUtils.relaseEglBaseProvider(eglBaseProvider, this);
        super.onDetachedFromWindow();
    }

    /**
     * Set if the video stream should be mirrored or not.
     */
    public void setMirror(final boolean mirror) {
        eglRenderer.setMirror(mirror);
    }

    /**
     * Set how the video will fill the allowed layout area.
     */
    public void setScalingType(RendererCommon.ScalingType scalingType) {
        ThreadUtils.checkIsOnMainThread();
        videoLayoutMeasure.setScalingType(scalingType);
    }

    public void setScalingType(RendererCommon.ScalingType scalingTypeMatchOrientation,
            RendererCommon.ScalingType scalingTypeMismatchOrientation) {
        ThreadUtils.checkIsOnMainThread();
        videoLayoutMeasure.setScalingType(scalingTypeMatchOrientation,
                scalingTypeMismatchOrientation);
    }

    /**
     * Sets listener of rendering events.
     */
    public void setListener(VideoRenderer.Listener listener) {
        this.listener = listener;
    }

    @Override
    public void renderFrame(I420Frame frame) {
        updateFrameDimensionsAndReportEvents(frame);
        eglRenderer.renderFrame(getWebRtcI420Frame(frame));
    }

    @Override
    protected void onMeasure(int widthSpec, int heightSpec) {
        ThreadUtils.checkIsOnMainThread();
        final Point size;
        synchronized (layoutLock) {
            size = videoLayoutMeasure.measure(widthSpec,
                    heightSpec,
                    rotatedFrameWidth,
                    rotatedFrameHeight);
        }
        setMeasuredDimension(size.x, size.y);
        logV("onMeasure(). New size: " + size.x + "x" + size.y);
    }

    @Override
    protected void onLayout(boolean changed, int left, int top, int right, int bottom) {
        ThreadUtils.checkIsOnMainThread();
        eglRenderer.setLayoutAspectRatio((right - left) / (float) (bottom - top));
        updateSurfaceSize();
    }

    private void init(EglBase.Context sharedContext,
            RendererCommon.RendererEvents rendererEvents) {
        init(sharedContext, rendererEvents, EglBase.CONFIG_PLAIN, new GlRectDrawer());
    }

    private void init(final EglBase.Context sharedContext,
            RendererCommon.RendererEvents rendererEvents,
            final int[] configAttributes,
            RendererCommon.GlDrawer drawer) {
        ThreadUtils.checkIsOnMainThread();
        this.rendererEvents = rendererEvents;
        synchronized (layoutLock) {
            rotatedFrameWidth = 0;
            rotatedFrameHeight = 0;
            frameRotation = 0;
        }
        eglRenderer.init(sharedContext, configAttributes, drawer);
    }

    /*
     * Use reflection on I420 frame to get access to WebRTC frame since EglRenderer only renders
     * WebRTC frames.
     */
    private org.webrtc.VideoRenderer.I420Frame getWebRtcI420Frame(I420Frame i420Frame) {
        org.webrtc.VideoRenderer.I420Frame webRtcI420Frame = null;

        try {
            webRtcI420Frame = (org.webrtc.VideoRenderer.I420Frame)
                webRtcI420FrameField.get(i420Frame);
        } catch (IllegalAccessException e) {
            e.printStackTrace();
        }

        return webRtcI420Frame;
    }

    private void updateSurfaceSize() {
        ThreadUtils.checkIsOnMainThread();
        synchronized (layoutLock) {
            if (rotatedFrameWidth != 0 &&
                    rotatedFrameHeight != 0 && getWidth() != 0
                    && getHeight() != 0) {
                final float layoutAspectRatio = getWidth() / (float) getHeight();
                final float frameAspectRatio = rotatedFrameWidth / (float) rotatedFrameHeight;
                final int drawnFrameWidth;
                final int drawnFrameHeight;
                if (frameAspectRatio > layoutAspectRatio) {
                    drawnFrameWidth = (int) (rotatedFrameHeight * layoutAspectRatio);
                    drawnFrameHeight = rotatedFrameHeight;
                } else {
                    drawnFrameWidth = rotatedFrameWidth;
                    drawnFrameHeight = (int) (rotatedFrameWidth / layoutAspectRatio);
                }
                // Aspect ratio of the drawn frame and the view is the same.
                final int width = Math.min(getWidth(), drawnFrameWidth);
                final int height = Math.min(getHeight(), drawnFrameHeight);
                logV("updateSurfaceSize. Layout size: " + getWidth() + "x" + getHeight() +
                        ", frame size: " + rotatedFrameWidth + "x" + rotatedFrameHeight +
                        ", requested surface size: " + width + "x" + height +
                        ", old surface size: " + surfaceWidth + "x" + surfaceHeight);
                if (width != surfaceWidth || height != surfaceHeight) {
                    surfaceWidth = width;
                    surfaceHeight = height;
                }
            } else {
                surfaceWidth = surfaceHeight = 0;
            }
        }
    }

    @Override
    public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int width, int height) {
        ThreadUtils.checkIsOnMainThread();
        eglRenderer.createEglSurface(surfaceTexture);
        surfaceWidth = width;
        surfaceHeight = height;
        updateSurfaceSize();
    }

    @Override
    public boolean onSurfaceTextureDestroyed(SurfaceTexture surfaceTexture) {
        ThreadUtils.checkIsOnMainThread();
        final CountDownLatch completionLatch = new CountDownLatch(1);
        eglRenderer.releaseEglSurface(new Runnable() {
            @Override
            public void run() {
                completionLatch.countDown();
            }
        });
        ThreadUtils.awaitUninterruptibly(completionLatch);
        return true;
    }

    @Override
    public void onSurfaceTextureSizeChanged(SurfaceTexture surfaceTexture, int width, int height) {
        ThreadUtils.checkIsOnMainThread();
        logV("surfaceChanged: size: " + width + "x" + height);
    }

    @Override
    public void onSurfaceTextureUpdated(SurfaceTexture surfaceTexture) {
        ThreadUtils.checkIsOnMainThread();
        logV("onSurfaceTextureUpdated");
    }

    private String getResourceName() {
        try {
            return getResources().getResourceEntryName(getId()) + ": ";
        } catch (Resources.NotFoundException e) {
            return "";
        }
    }

    // Update frame dimensions and report any changes to |rendererEvents|.
    private void updateFrameDimensionsAndReportEvents(I420Frame frame) {
        synchronized (layoutLock) {
            if (!isFirstFrameRendered) {
                isFirstFrameRendered = true;
                logV("Reporting first rendered frame.");
                if (rendererEvents != null) {
                    rendererEvents.onFirstFrameRendered();
                }
            }
            if (rotatedFrameWidth != frame.rotatedWidth() ||
                    rotatedFrameHeight != frame.rotatedHeight() ||
                    frameRotation != frame.rotationDegree) {
                logV("Reporting frame resolution changed to " + frame.width + "x" + frame.height
                        + " with rotation " + frame.rotationDegree);
                if (rendererEvents != null) {
                    rendererEvents.onFrameResolutionChanged(frame.width,
                            frame.height,
                            frame.rotationDegree);
                }
                rotatedFrameWidth = frame.rotatedWidth();
                rotatedFrameHeight = frame.rotatedHeight();
                frameRotation = frame.rotationDegree;
                uiThreadHandler.post(new Runnable() {
                    @Override
                    public void run() {
                        updateSurfaceSize();
                        requestLayout();
                    }
                });
                    }
        }
    }

    private void logV(String string) {
        Logging.v(TAG, resourceName + string);
    }

    private void logD(String string) {
        Logging.d(TAG, resourceName + string);
    }
}

it Needs setListener(VideoRenderer.Listener listener) method

@shivthepro added a setListener method

Hey @aaalaniz ... Thanks for adding the listener method. I hope you should check this out there is some minor issue with the code.. The copy/paste dint go well I guess ... Check line no 236 .. I have done it for you .. https://gist.github.com/shivthepro/61c5b52af8a213333adb7446704a6a39#file-videotextureview-java

Hey @shivthepro

Sorry about that incorrect Gist. Thanks for pointing that out! It should be fixed now.

hi @aaalaniz
Can i get frame from remot video participant from heare and convert this to bitmap??. Thank you

Hey @minhkang0107

Yes, you can but there is a known issue with capturing a remote video track to a bitmap. You will need this workaround.

https://github.com/aaalaniz/video-quickstart-android/commits/task/GSDK-2042-blurkit-workaround

hi @aaalaniz thank you so much!!!.

Hi @aaalaniz,

I am implementing a feature to take a snapshot from remote video.
Currently managed to get it work for certain devices when it's running with captureBitmapFromYuvFrame.

For captureBitmapFromTexture, I am getting error with
java.lang.RuntimeException: glCreateShader() failed. GLES20 error: 0.

And after a lot of reading, I found your comments for issues on https://github.com/twilio/video-quickstart-android to be very useful.
Mostly the issue is regarding with threading and active EGL context.

I have already added the workaround you mentioned in the commit aaalaniz/video-quickstart-android@9ee09e7.

However not able to get getEglHandler method to return anything significant. (Getting null due to NoSuchFieldException)

---

I am using https://github.com/blackuy/react-native-twilio-video-webrtc and extending the feature to support snapshot.

Rendering both local and remote video in the app.

---

Approaches I have tried:

1. Change to extend from VideoTextureView for the custom video view https://github.com/blackuy/react-native-twilio-video-webrtc/blob/master/android/src/main/java/com/twiliorn/library/PatchedVideoView.java
2. Add SnapshotVideoRenderer from the workaround gist.
3. Add the remoteSnapshotVideoRenderer to track.
4. And other combinations of these (without renderer etc etc)

---

Appreciate if you can shed some light on this, regarding how to fix/workaround NoSuchFieldException. Or what is that issue is about.
I am not an Android developer but I am trying to get this work.

Thanks in advance.

---

Update: Managed to get it working with getBitmap, anyway, still a big thank you for the example code and gist and all the work. @aaalaniz