Fix for NanoGui's ColorWheel::adjustPosition() method.

colorwheel.cpp

ColorWheel::Region ColorWheel::adjustPosition(const ivec2 &p, Region consideredRegions)
{
	float x = p.x - mPos.x, y = p.y - mPos.y, w = mSize.x, h = mSize.y;

	float cx = w*0.5f;
	float cy = h*0.5f;
	float r1 = (w < h ? w : h) * 0.5f - 5.0f;
	float r0 = r1 * .75f;

	x -= cx;
	y -= cy;

	float mr = glm::sqrt(x*x + y*y);

	if ((consideredRegions & OuterCircle) &&
		((mr >= r0 && mr <= r1) || (consideredRegions == OuterCircle))) {
		if (!(consideredRegions & OuterCircle))
			return None;
		mHue = std::atan(y / x);
		if (x < 0)
			mHue += NVG_PI;
		mHue /= 2 * NVG_PI;

		if (mCallback)
			mCallback(color());

		return OuterCircle;
	}

	float r = r0 - 6;

	float hueAngle = float( NVG_PI * 2 ) * mHue;
	const float angle = float( NVG_PI * 120 / 180 );
	glm::vec2 a( r * glm::cos( hueAngle + angle ), r * glm::sin( hueAngle + angle ) );
	glm::vec2 b( r * glm::cos( hueAngle - angle ), r * glm::sin( hueAngle - angle ) );
	glm::vec2 c( r * glm::cos( hueAngle ), r * glm::sin( hueAngle ) );

	vec3 bary = calcBarycentric( vec2( x, y ), a, b, c );

	float l0 = bary[0], l1 = bary[1], l2 = bary[2];
	bool triangleTest = l0 >= 0 && l0 <= 1.f && l1 >= 0.f && l1 <= 1.f && l2 >= 0.f && l2 <= 1.f;

	if ((consideredRegions & InnerTriangle) &&
		(triangleTest || consideredRegions == InnerTriangle)) {
		if (!(consideredRegions & InnerTriangle))
			return None;
		l0 = std::min(std::max(0.f, l0), 1.f);
		l1 = std::min(std::max(0.f, l1), 1.f);
		l2 = std::min(std::max(0.f, l2), 1.f);
		float sum = l0 + l1 + l2;
		l0 /= sum;
		l1 /= sum;
		mWhite = l0;
		mBlack = l1;
		if (mCallback)
			mCallback(color());
		return InnerTriangle;
	}
	return None;
}

vec3 ColorWheel::calcBarycentric( const vec2 &pt, const vec2 &a, const vec2 &b, const vec2 &c ) const
{
	vec3 result;

	vec2 v0 = b - a, v1 = c - a, v2 = pt - a;

	float d00 = dot( v0, v0 );
	float d01 = dot( v0, v1 );
	float d11 = dot( v1, v1 );
	float d20 = dot( v2, v0 );
	float d21 = dot( v2, v1 );
	float denom = d00 * d11 - d01 * d01;
	
	result.y = ( d11 * d20 - d01 * d21 ) / denom;
	result.z = ( d00 * d21 - d01 * d20 ) / denom;
	result.x = 1.0f - result.y - result.z;

	return result;
}

Note that I created a separate calcBarycentric method to keep things readable. Don't forget to add its declaration to the header file.