Updated to do maths relative to centre point. Added extra check to first condition to ensure points in the 3->9 o'clock arc aren't sorted on x coordinate in the same manner as those above.

gistfile1.txt

function less(center,a,b)
  local adx=a.x-center.x
  local ady=a.y-center.y
  local bdx=b.x-center.x
  local bdy=b.y-center.y

  if adx >= 0 and bdx<0 then
    return true
  end

  if adx < 0 and bdx>=0 then
    return false
  end

  if adx*adx==0 and bdx*bdx==0 then
    if ady >= 0 or bdy >= 0 then
      return a.y > b.y
    end
    return b.y > a.y
  end

  -- compute the cross product of vectors (center -> a) x (center -> b)
  local det = adx*bdy - bdx*ady
  if det < 0 then
    return true
  end
  if det > 0 then
    return false
  end

  -- // points a and b are on the same line from the center
  -- // check which point is closer to the center
  local d1 = adx*adx + ady*ady
  local d2 = bdx*bdx + bdy*bdy

  return d1 > d2
end