MaskTest2.java

package mdesl.line2dx.test;

import com.badlogic.gdx.ApplicationListener;
import com.badlogic.gdx.Gdx;
import com.badlogic.gdx.graphics.GL10;
import com.badlogic.gdx.graphics.OrthographicCamera;
import com.badlogic.gdx.graphics.Texture;
import com.badlogic.gdx.graphics.Texture.TextureFilter;
import com.badlogic.gdx.graphics.g2d.SpriteBatch;

 
public class MaskTest2 implements ApplicationListener {
 
  OrthographicCamera cam;
	SpriteBatch batch;
	Texture bg, sprite, alphaMask;
 
	@Override
	public void create () {
		cam = new OrthographicCamera();
		batch = new SpriteBatch();
		
		sprite = new Texture("data/grass.png");
		alphaMask = new Texture("data/mask.png");
		alphaMask.setFilter(TextureFilter.Linear, TextureFilter.Linear);
	}
 
	@Override
	public void resize (int width, int height) {
		cam.setToOrtho(false, width, height);
		batch.setProjectionMatrix(cam.combined);
	}
	
	private void drawBackground(SpriteBatch batch) {
		//regular blending mode
		batch.setBlendFunction(GL10.GL_SRC_ALPHA, GL10.GL_ONE_MINUS_SRC_ALPHA);
		
		
		//... draw background entities/tiles here ...
		
		
		//flush the batch to the GPU
		batch.flush();
	}
	
	private void drawAlphaMask(SpriteBatch batch, float x, float y, float width, float height) {
		//disable RGB color, only enable ALPHA to the frame buffer
		Gdx.gl.glColorMask(false, false, false, true);
		
		//change the blending function for our alpha map
		batch.setBlendFunction(GL10.GL_ONE, GL10.GL_ZERO);
		
		//draw alpha mask sprite(s)
		batch.draw(alphaMask, x, y, width, height);
		
		//flush the batch to the GPU
		batch.flush();
	}
	
	private void drawForeground(SpriteBatch batch, int clipX, int clipY, int clipWidth, int clipHeight) {
		//now that the buffer has our alpha, we simply draw the sprite with the mask applied
		Gdx.gl.glColorMask(true, true, true, true);
		batch.setBlendFunction(GL10.GL_DST_ALPHA, GL10.GL_ONE_MINUS_DST_ALPHA);
		
		//The scissor test is optional, but it depends 
		Gdx.gl.glEnable(GL10.GL_SCISSOR_TEST);
		Gdx.gl.glScissor(clipX, clipY, clipWidth, clipHeight);
		
		//draw our sprite to be masked
		batch.draw(sprite, 0, 0, 250, 250);
		
		//remember to flush before changing GL states again
		batch.flush();
		
		//disable scissor before continuing
		Gdx.gl.glDisable(GL10.GL_SCISSOR_TEST);
	}
 
	@Override
	public void render () {
		Gdx.gl.glClearColor(0.5f, 0.5f, 0.5f, 1.0f);
		Gdx.gl.glClear(GL10.GL_COLOR_BUFFER_BIT);
		
		
		int w = Gdx.graphics.getWidth();
		int h = Gdx.graphics.getHeight();
		
		//start the batch
		batch.begin();
		
		//draw background
		drawBackground(batch);
		
 
		//the sprite we want the circle mask applied to
		int x = 25;
		int y = 50;
		int spriteWidth = 200;
		int spriteHeight = 200;
		
		//draw the alpha mask
		drawAlphaMask(batch, x, y, spriteWidth, spriteHeight);
		
		//draw our foreground elements
		drawForeground(batch, x, y, spriteWidth, spriteHeight);
		
		batch.end();
	}
 
	@Override
	public void pause () {
		
	}
 
	@Override
	public void resume () {
		
	}
 
	@Override
	public void dispose () {
		batch.dispose();
		alphaMask.dispose();
		sprite.dispose();
	}
	
	
}

I wanted to add three notes on this gist based on my personal experiences:
First, the code requires an image buffer that has a dedicated alpha channel. When I adapted this code for our android project, I had to look for hours until I finally found out that the default AndroidApplicationConfiguration with the libgdx version we were using has zero bit set for alpha channel.

Secondly, this code uses flush on the batch after each step. For the batch to work more efficiently, in my opinion, I think it is best to wait with that and only flush before the gl state is altered, so that batched operations will not draw with the new settings. But that is just a matter of personal taste, I guess. If you completely forget flushing, though, things will look really messy.

Third thing is that some devices (Xperia Z in my case) will still look into the alpha value when rendering the image buffer onto the screen and perform blending with (I think) the clear color. This caused me having a black background with every actor that used this technique. A fix for that is, for example, when the program is done with rendering to set the color mask to only alpha and clear the screen with clear color (0, 0, 0, 1). Don't forget to reset this afterwards, though.

Still a big thank you for that gist and your great tutorial on masking! Really helped me getting started with it.

You hava to write some code in render method.

batch.setBlendFunction(GL20.GL_SRC_ALPHA, GL20.GL_ONE_MINUS_SRC_ALPHA); batch.flush();