nonInstancedBilboardSprites.vs

//Bilboard shader
//draw sprites in 3D from any angle

const vec2 pos[6] = vec2[](
  vec2(0, 0),
  vec2(1, 0),
  vec2(0, 1),
                           
  vec2(0, 1),
  vec2(1, 0),
  vec2(1, 1)
);

const vec2 tc[6] = vec2[](
  vec2(0, 0),
  vec2(1, 0),
  vec2(0, 1),
                           
  vec2(0, 1),
  vec2(1, 0),
  vec2(1, 1)
);

out vec2 varTexcoord;
out flat float varDistanceToCamera;
out flat float varAlpha;
out flat float textureCutoffT;

const int MAX_TRANSFORMS = 512;

struct PerInstance
{
  //.xyz are xyz pos in 3D
  //abs(.w) is texture atlas x-shift
  //sign(.w) can be -1 for h-flip
  vec4 position;
  
  float alpha, scale;
  float textureCutoffT;
};

#ifdef VGL_VULKAN
layout (std140, set = 1, binding = 16) uniform InstancedTransforms
{
  PerInstance instanceData[MAX_TRANSFORMS];
};
#else
#define gl_VertexIndex gl_VertexID
layout (std140) uniform InstancedTransforms
{
  PerInstance instanceData[MAX_TRANSFORMS];
};
#endif

uniform mat4 projectionMatrix;
uniform mat4 modelViewMatrix;
uniform vec3 cameraPosition;

//.xy is sprite size, .z is number of sprites in entire sheet
uniform vec3 spriteSize;

mat4 rotationX(float angle)
{
  return mat4(1.0, 0.0, 0.0 ,0.0,
              0.0, cos(angle), -sin(angle), 0.0,
              0.0, sin(angle), cos(angle), 0.0,
              0.0, 0.0, 0.0, 1.0);
}

void main()
{
  int vertexId = gl_VertexIndex % 6;
  int instanceId = gl_VertexIndex / 6;
  vec4 instancePos = instanceData[instanceId].position;
  
  vec2 size = vec2(1.0/spriteSize.z, 1.0);
  float flipped = sign(instancePos.w);
  vec2 texCoord = tc[vertexId] * size + flipped*vec2(abs(instancePos.w), 0.0);
  
  if(flipped < 0)
  {
    texCoord.x = size.x-texCoord.x;
  }
  
  varTexcoord = texCoord;
  varAlpha = instanceData[instanceId].alpha;

  mat4 translate = mat4(1.0);
  translate[3] = vec4(instancePos.xyz, 1.0);
  
  mat4 localModelViewMatrix = modelViewMatrix * translate;
  
#ifndef NO_BILBOARDING
  mat4 mvBilboard = mat4(mat3(SPRITE_SCALE*instanceData[instanceId].scale));
  mvBilboard[3] = localModelViewMatrix[3];
#elif !defined NO_3D
  mat4 rot = rotationX(radians(90.0));
  mat4 mvBilboard = localModelViewMatrix * mat4(mat3(SPRITE_SCALE*instanceData[instanceId].scale)) * rot;
#else
  mat4 mvBilboard = localModelViewMatrix * mat4(mat3(SPRITE_SCALE*instanceData[instanceId].scale));
#endif

  vec4 mvPos = (mvBilboard * (vec4(pos[vertexId], 0.0, 1.0) * vec4(spriteSize.xy, 1.0, 1.0)));
                
  //somewhat hacking this (tired of dealing with it)
  varDistanceToCamera = length(cameraPosition-mvPos.xyz);// / (spriteSize.y/16.0);
  
  textureCutoffT = instanceData[instanceId].textureCutoffT;
  
  gl_Position = projectionMatrix * mvPos;
}