Skip to content

Instantly share code, notes, and snippets.

@aras-p
Created August 2, 2012 12:30
Show Gist options
  • Save aras-p/3236705 to your computer and use it in GitHub Desktop.
Save aras-p/3236705 to your computer and use it in GitHub Desktop.
d3dcompiler_43.dll hull shader optimization bug

Hull shader constant function, in triangle domain:

HSConstData HS_FlatConstant(InputPatch<InterpData, 3> I)
{
    float3 f;
    // factors computed per vertex
    f.x = I[0].fact;
    f.y = I[1].fact;
    f.z = I[2].fact;

    float4 tess;
    // each edge - average of vertex factors
    tess.x = 0.5 * (f.y + f.z);
    tess.y = 0.5 * (f.z + f.x);
    tess.z = 0.5 * (f.x + f.y);
    tess.w = 2.5; // fixed factor for inside

    HSConstData O;
    O.fTessFactor[0] = tess.x;
    O.fTessFactor[1] = tess.y;
    O.fTessFactor[2] = tess.z;
    O.fInsideTessFactor = tess.w;
    
    return O;
}

This works as expected, and compiles down to this assembly:

hs_5_0
hs_decls 
dcl_input_control_point_count 3
dcl_output_control_point_count 3
dcl_tessellator_domain domain_tri
dcl_tessellator_partitioning partitioning_fractional_odd
dcl_tessellator_output_primitive output_triangle_cw
dcl_globalFlags refactoringAllowed
; forks into 3 instances to compute each of the edge factors?
hs_fork_phase 
dcl_hs_fork_phase_instance_count 3
dcl_input vForkInstanceID
dcl_input vicp[3][4].x
dcl_output_siv o0.x, finalTriUeq0EdgeTessFactor
dcl_output_siv o1.x, finalTriVeq0EdgeTessFactor
dcl_output_siv o2.x, finalTriWeq0EdgeTessFactor
dcl_temps 1
dcl_indexrange o0.x 3
iadd r0.xy, vForkInstanceID.xxxx, l(1, 2, 0, 0)
udiv null, r0.xy, r0.xyxx, l(3, 3, 0, 0)
add r0.x, vicp[r0.y + 0][4].x, vicp[r0.x + 0][4].x
mov r0.y, vForkInstanceID.x
mul o[r0.y + 0].x, r0.x, l(0.500000)
ret 
; computes inside factor
hs_fork_phase 
dcl_output_siv o3.x, finalTriInsideTessFactor
mov o3.x, l(2.500000)
ret

Now, let's introduce some trivial math. Just multiply factors by 1.0 (or any other constant really) in hull shader:

HSConstData HS_FlatConstant(InputPatch<InterpData, 3> I)
{
    float3 f;
    // factors computed per vertex
    f.x = I[0].fact * 1.0; // ****
    f.y = I[1].fact * 1.0; // **** these "* 1.0" is the only changed thing !
    f.z = I[2].fact * 1.0; // ****

    float4 tess;
    // each edge - average of vertex factors
    tess.x = 0.5 * (f.y + f.z);
    tess.y = 0.5 * (f.z + f.x);
    tess.z = 0.5 * (f.x + f.y);
    tess.w = 2.5; // fixed factor for inside

    HSConstData O;
    O.fTessFactor[0] = tess.x;
    O.fTessFactor[1] = tess.y;
    O.fTessFactor[2] = tess.z;
    O.fInsideTessFactor = tess.w;
    
    return O;
}

Compiles down to something completely different:

hs_5_0
hs_decls 
dcl_input_control_point_count 3
dcl_output_control_point_count 3
dcl_tessellator_domain domain_tri
dcl_tessellator_partitioning partitioning_fractional_odd
dcl_tessellator_output_primitive output_triangle_cw
dcl_globalFlags refactoringAllowed
; forks into 2 instances (?), each outputs one of the edge factors, and
; factor for the 3rd edge using some convoluted math?
hs_fork_phase 
dcl_hs_fork_phase_instance_count 2
dcl_input vForkInstanceID
dcl_input vicp[3][4].x
dcl_output_siv o0.x, finalTriUeq0EdgeTessFactor
dcl_output_siv o1.x, finalTriVeq0EdgeTessFactor
dcl_output_siv o2.x, finalTriWeq0EdgeTessFactor
dcl_temps 1
dcl_indexrange o0.x 2
iadd r0.x, vForkInstanceID.x, l(2)
udiv null, r0.x, r0.x, l(3)
mov r0.y, vForkInstanceID.x
add r0.x, vicp[r0.x + 0][4].x, vicp[r0.y + 1][4].x
mul o[r0.y + 0].x, r0.x, l(0.500000)
add r0.x, vicp[r0.y + 1][4].x, vicp[0][4].x
mul o2.x, r0.x, l(0.500000)
ret
; computes inside factor
hs_fork_phase 
dcl_output_siv o3.x, finalTriInsideTessFactor
mov o3.x, l(2.500000)
ret 

Result: Wrong rendering; one of triangle edges gets a wrong tessellation factor! Checked both in hardware device (GTX 680), and with reference rasterizer. Displacement cracks ahoy!

Turning off compiler optimizations (D3D10_SHADER_SKIP_OPTIMIZATION) fixes the problem but, well, no optimization then.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment