视频画中画实际就是两张纹理叠加，其中之一为前景，另外一个为背景。前景通常是一个小窗口叠加在背景上。
在 Android 应用中，通常视频源来自 Camera 或者其他经过硬解码（MediaCodec）而来的视频流。我们知道不论是 Camera 还是硬解码后的视频流都是可以直接送显到 Surface 上的（存在相应的 API 直接调用），所以可以通过 OpenGL ES 生成对应的纹理，然后包装成 Surface 交给对应的视频源去输出。需要注意的是，片段着色器中需要声明为 GL_OES_EGL_image_external，也就是外部扩展纹理。
有了以上方案基础就不难实现相应的代码了。首先来开发前景和背景纹理使用 Drawer。前景可经过一定的矩阵变换（缩小、平移），可以改变其位置和大小。
/*** author : liuhongwei* e-mail : * date : 2021/7/22 16:28* desc : OES绘制器* version: 1.0*/
public class OesDrawer {private static final String TAG = "OesDrawer";private final float[] mVertexCoors = new float[]{-1.0F, -1.0F,1.0F, -1.0F,-1.0F, 1.0F,1.0F, 1.0F};private final float[] mTextureCoors = new float[]{0.0F, 1.0F,1.0F, 1.0F,0.0F, 0.0F,1.0F, 0.0F};private int mTextureId = -1;private SurfaceTexture mSurfaceTexture;private int mProgram = -1;private int mVertexMatrixHandler = -1;private int mVertexPosHandler = -1;private int mTexturePosHandler = -1;private int mTextureHandler = -1;private int mAlphaHandler = -1;private FloatBuffer mVertexBuffer;private FloatBuffer mTextureBuffer;private final float[] mMatrix = new float[16];private float mAlpha = 1.0F;public OesDrawer(){initPos();Matrix.setIdentityM(mMatrix, 0);}public void scale(float sx, float sy) {Matrix.scaleM(mMatrix, 0, sx, sy, 1f);}public void translate(float x, float y, float z) {Matrix.translateM(mMatrix, 0, x, y, z);}private void initPos() {ByteBuffer bb = ByteBuffer.allocateDirect(mVertexCoors.length * 4);bb.order(ByteOrder.nativeOrder());//将坐标数据转换为FloatBuffer，用以传入给OpenGL ES程序mVertexBuffer = bb.asFloatBuffer();mVertexBuffer.put(mVertexCoors);mVertexBuffer.position(0);ByteBuffer cc = ByteBuffer.allocateDirect(mTextureCoors.length * 4);cc.order(ByteOrder.nativeOrder());mTextureBuffer = cc.asFloatBuffer();mTextureBuffer.put(mTextureCoors);mTextureBuffer.position(0);}public void setAlpha(float alpha) {this.mAlpha = alpha;}public void setTextureID(int id) {mTextureId = id;mSurfaceTexture = new SurfaceTexture(id);}public SurfaceTexture getSurfaceTexture() {return mSurfaceTexture;}private void createGLPrg() {if (mProgram == -1) {int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, getVertexShader());int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, getFragmentShader());//创建OpenGL ES程序，注意：需要在OpenGL渲染线程中创建，否则无法渲染mProgram = GLES20.glCreateProgram();//将顶点着色器加入到程序GLES20.glAttachShader(mProgram, vertexShader);//将片元着色器加入到程序中GLES20.glAttachShader(mProgram, fragmentShader);//连接到着色器程序GLES20.glLinkProgram(mProgram);mVertexMatrixHandler = GLES20.glGetUniformLocation(mProgram, "uMatrix");mVertexPosHandler = GLES20.glGetAttribLocation(mProgram, "aPosition");mTextureHandler = GLES20.glGetUniformLocation(mProgram, "uTexture");mTexturePosHandler = GLES20.glGetAttribLocation(mProgram, "aCoordinate");mAlphaHandler = GLES20.glGetAttribLocation(mProgram, "alpha");}//使用OpenGL程序GLES20.glUseProgram(mProgram);}private void activateTexture() {//激活指定纹理单元GLES20.glActiveTexture(GLES20.GL_TEXTURE0);//绑定纹理ID到纹理单元GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureId);//将激活的纹理单元传递到着色器里面GLES20.glUniform1i(mTextureHandler, 0);//配置边缘过渡参数GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR * 1.0f);GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR * 1.0f);GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);}public void updateTexture() {if (null != mSurfaceTexture) {mSurfaceTexture.updateTexImage();}}public void doDraw() {if (mTextureId != -1) {createGLPrg();activateTexture();//启用顶点的句柄GLES20.glEnableVertexAttribArray(mVertexPosHandler);GLES20.glEnableVertexAttribArray(mTexturePosHandler);GLES20.glUniformMatrix4fv(mVertexMatrixHandler, 1, false, mMatrix, 0);//设置着色器参数， 第二个参数表示一个顶点包含的数据数量，这里为xy，所以为2GLES20.glVertexAttribPointer(mVertexPosHandler, 2, GLES20.GL_FLOAT, false, 0, mVertexBuffer);GLES20.glVertexAttribPointer(mTexturePosHandler, 2, GLES20.GL_FLOAT, false, 0, mTextureBuffer);GLES20.glVertexAttrib1f(mAlphaHandler, mAlpha);//开始绘制GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);}}public void release() {GLES20.glDisableVertexAttribArray(mVertexPosHandler);GLES20.glDisableVertexAttribArray(mTexturePosHandler);GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);GLES20.glDeleteTextures(1, new int[]{mTextureId}, 0);GLES20.glDeleteProgram(mProgram);}private String getVertexShader() {return "attribute vec4 aPosition;" +"precision mediump float;" +"uniform mat4 uMatrix;" +"attribute vec2 aCoordinate;" +"varying vec2 vCoordinate;" +"attribute float alpha;" +"varying float inAlpha;" +"void main() {" +" gl_Position = uMatrix*aPosition;" +" vCoordinate = aCoordinate;" +" inAlpha = alpha;" +"}";}private String getFragmentShader() {//一定要加换行"\n"，否则会和下一行的precision混在一起，导致编译出错return "#extension GL_OES_EGL_image_external : require\n" +"precision mediump float;" +"varying vec2 vCoordinate;" +"varying float inAlpha;" +"uniform samplerExternalOES uTexture;" +"void main() {" +" vec4 color = texture2D(uTexture, vCoordinate);" +" gl_FragColor = vec4(color.r, color.g, color.b, inAlpha);" +"}";}private int loadShader(int type, String shaderCode) {//根据type创建顶点着色器或者片元着色器int shader = GLES20.glCreateShader(type);//将资源加入到着色器中，并编译GLES20.glShaderSource(shader, shaderCode);GLES20.glCompileShader(shader);return shader;}
}
前景和背景使用的 Drawer 开发好之后，需要更进一步，也就是将两张纹理直接绘制到 FBO，如此可以更加灵活的定制，比如还要贴一个水印 logo 都可以在 FBO Drawer 中处理。
/*** author : liuhongwei* e-mail : * date : 2021/7/26 14:59* desc : FBO绘制器* version: 1.0*/
public class FBODrawer {private static final String TAG = "FBODrawer";//单画面或PIP模式private Mode mMode;private int[] mFrameBuffers;private int[] mFBOTextures;private int mFrameWidth;private int mFrameHeight;private OesDrawer mBgOesDrawer;private OesDrawer mFgOesDrawer;public FBODrawer(){mBgOesDrawer = new OesDrawer();mFgOesDrawer = new OesDrawer();}public SurfaceTexture getBgSurfaceTexture() {return mBgOesDrawer.getSurfaceTexture();}public SurfaceTexture getFgSurfaceTexture() {return mFgOesDrawer.getSurfaceTexture();}private void loadFBO(int frameWidth, int frameHeight) {mFrameWidth = frameWidth;mFrameHeight = frameHeight;if (mFrameBuffers != null) {destroyFrameBuffers();}//创建FrameBuffermFrameBuffers = new int[1];GLES20.glGenFramebuffers(mFrameBuffers.length, mFrameBuffers, 0);//创建FBO中的纹理mFBOTextures = new int[1];GlesUtils.genTextures(mFBOTextures);GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mFBOTextures[0]);Log.d(TAG, "mFBOTextures[0]=" + mFBOTextures[0]);//指定FBO纹理的输出图像的格式 RGBAGLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, frameWidth, frameHeight,0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null);GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFrameBuffers[0]);//将fbo绑定到2d的纹理上GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,GLES20.GL_TEXTURE_2D, mFBOTextures[0], 0);GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);}private void destroyFrameBuffers() {//删除fbo的纹理if (mFBOTextures != null) {GLES20.glDeleteTextures(1, mFBOTextures, 0);mFBOTextures = null;}//删除fboif (mFrameBuffers != null) {GLES20.glDeleteFramebuffers(1, mFrameBuffers, 0);mFrameBuffers = null;}}/*** 前置纹理应用矩阵转换，右上角* @param w 纹理宽* @param h 纹理高*/private void applyMatrix2Fg(int w, int h){// 先缩放，后平移float scale = 0.4f;mFgOesDrawer.scale(scale, scale);float scaleW = w * scale;float scaleH = h * scale;float startX = w / 2.0f + scaleW / 2.0f;float startY = h / 2.0f + scaleH / 2.0f;float deltaX = w - startX;float deltaY = h - startY;mFgOesDrawer.translate(deltaX / scaleW * 2, deltaY / scaleH * 2, 0);mFgOesDrawer.setAlpha(1.0f);}public void prepareDraw(int frameWidth, int frameHeight) {applyMatrix2Fg(frameWidth, frameHeight);loadFBO(frameWidth, frameHeight);int[] textureIds = GlesUtils.createTextureIds(2);mBgOesDrawer.setTextureID(textureIds[0]);mFgOesDrawer.setTextureID(textureIds[1]);Log.d(TAG, "textureIds[0]=" + textureIds[0]);Log.d(TAG, "textureIds[1]=" + textureIds[1]);//开启混合，即半透明GLES20.glEnable(GLES20.GL_BLEND);GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);}public void destroyDraw() {mBgOesDrawer.release();mFgOesDrawer.release();if (mFrameBuffers != null) {destroyFrameBuffers();}}public int doDraw() {//锁定绘制的区域 绘制是从左下角开始的GLES20.glViewport(0, 0, mFrameWidth, mFrameHeight);//绑定FBO，在FBO上操作GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFrameBuffers[0]);GLES20.glClearColor(0f, 0f, 0f, 0f);mBgOesDrawer.doDraw();if (mMode == Mode.PIP) {mFgOesDrawer.doDraw();}GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);return mFBOTextures[0];}public void setMode(Mode mode) {this.mMode = mode;}public Mode getMode() {return mMode;}public enum Mode {SINGLE, PIP}}
GlesUtils 工具类实现。
/*** author : liuhongwei* e-mail : * date : 2021/7/22 13:46* desc : openGL ES 工具类* version: 1.0*/
public class GlesUtils {private final static String TAG = "GlesUtils";public static int[] createTextureIds(int count) {int[] texture = new int[count];GLES20.glGenTextures(count, texture, 0); //生成纹理return texture;}public static void genTextures(int[] textures) {GLES20.glGenTextures(textures.length, textures, 0);for (int texture : textures) {GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texture);GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);}}
}
最后，直接将 FBO 绘制到需要显示的 EGL 上，整个画中画预览就实现了。