Skip to content

Instantly share code, notes, and snippets.

@Roffild

Roffild/test_autopytorch_metrics.py

Last active February 24, 2021 10:23
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Roffild/982ba8ffb1f7ee8a5f4f0183cbbf1cc0 to your computer and use it in GitHub Desktop.
Save Roffild/982ba8ffb1f7ee8a5f4f0183cbbf1cc0 to your computer and use it in GitHub Desktop.
Download ZIP
Metrics from XGBoost for PyTorch
Raw
test_autopytorch_metrics.py
# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# https://github.com/Roffild/RoffildLibrary
# ==============================================================================
import unittest
import torch
import math
# from roffild.autopytorch import Metrics
class Metrics:
@staticmethod
def rmse(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
result = (output - input).pow_(2)
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum).sqrt_()
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1]).sqrt_()
return ("rmse", result)
@staticmethod
def rmsle(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
result = (output.log1p() - input.log1p()).pow_(2)
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum).sqrt_()
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1]).sqrt_()
return ("rmsle", result)
@staticmethod
def mae(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
result = (output - input).abs_()
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum)
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1])
return ("mae", result)
@staticmethod
def mape(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
result = (output - input).div_(output).abs_()
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum)
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1])
return ("mape", result)
@staticmethod
def mphe(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
result = (output - input).pow_(2).add_(1.0).sqrt_().sub_(1.0)
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum)
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1])
return ("mphe", result)
@staticmethod
def error(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
result = torch.where(input > 0.5, 1.0 - output, output)
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum)
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1])
return ("error", result)
@staticmethod
def logloss(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
result = input.clone()
ilog = input.log()
i1m = 1.0 - input
i1mlog = i1m.log()
oneg = output.neg()
o1m = 1.0 - output
eps = torch.tensor(1e-16, dtype=torch.float32, device=result.device)
epslog = eps.log()
eps1mlog = (1.0 - eps).log_() ### BUG == nan == 0 !!!
con1 = input < eps
con2 = i1m < eps
con2 = con2.bitwise_xor(con1).bitwise_and_(con2)
con3 = con2.bitwise_or(con1).bitwise_not_()
if bool(con1.any()):
result[con1] = oneg[con1] * epslog - o1m[con1] * eps1mlog
if bool(con2.any()):
result[con2] = oneg[con2] * eps1mlog - o1m[con2] * epslog
if bool(con3.any()):
result[con3] = oneg[con3] * ilog[con3] - o1m[con3] * i1mlog[con3]
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum)
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1])
return ("logloss", result)
@staticmethod
def poisson_nloglik(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
eps = torch.tensor(1e-16, dtype=torch.float32, device=input.device)
result = torch.where(input < eps, eps, input)
result = (output + 1.0).lgamma_().add_(result).sub_(result.log_().mul_(output))
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum)
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1])
return ("poisson-nloglik", result)
@staticmethod
def gamma_deviance(input: torch.Tensor, output: torch.Tensor, weights: torch.Tensor = None) -> (str, torch.Tensor):
"""Multiclass classification error."""
with torch.no_grad():
eps = torch.tensor(1e-16, dtype=torch.float32, device=input.device)
result = input.div(output.add(eps)).log_().add_(output.div(input.add(eps))).sub_(1.0).mul_(2.0)
if weights is not None:
result.mul_(weights)
weights_sum = weights.sum(dim=1, keepdim=True)
weights_sum[weights_sum == 0.0] = 1.0
result = result.sum(dim=1, keepdim=True).div_(weights_sum)
else:
result = result.sum(dim=1, keepdim=True).div_(output.shape[1])
return ("gamma-deviance", result)
class AutoPytorchMetricsTest(unittest.TestCase):
def test_rmse(self):
metric = Metrics.rmse
self.EXPECT_NEAR(self.GetMetricEval(metric, [0, 1], [0, 1]), 0, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[ 0.1, 0.9, 0.1, 0.9 ],
[ 0, 0, 1, 1 ]),
0.6403, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ -1, 1, 9, -9]),
2.8284, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ 1, 2, 9, 8]),
0.6708, 0.001)
def test_rmsle(self):
metric = Metrics.rmsle
self.EXPECT_NEAR(self.GetMetricEval(metric, [0, 1], [0, 1]), 0, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.2, 0.4, 0.8, 1.6],
[1.0, 1.0, 1.0, 1.0, 1.0]),
0.40632, 1e-4)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.2, 0.4, 0.8, 1.6],
[1.0, 1.0, 1.0, 1.0, 1.0],
[ 0, -1, 1, -9, 9]),
0.6212, 1e-4)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.2, 0.4, 0.8, 1.6],
[1.0, 1.0, 1.0, 1.0, 1.0],
[ 0, 1, 2, 9, 8]),
0.2415, 1e-4)
def test_mae(self):
metric = Metrics.mae
self.EXPECT_NEAR(self.GetMetricEval(metric, [0, 1], [0, 1]), 0, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1]),
0.5, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ -1, 1, 9, -9]),
8.0, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ 1, 2, 9, 8]),
0.54, 0.001)
def test_mape(self):
metric = Metrics.mape
self.EXPECT_NEAR(self.GetMetricEval(metric, [150, 300], [100, 200]), 0.5, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[50, 400, 500, 4000],
[100, 200, 500, 1000]),
1.125, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[50, 400, 500, 4000],
[100, 200, 500, 1000],
[ -1, 1, 9, -9]),
-26.5, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[50, 400, 500, 4000],
[100, 200, 500, 1000],
[ 1, 2, 9, 8]),
1.3250, 0.001)
def test_mphe(self):
metric = Metrics.mphe
self.EXPECT_NEAR(self.GetMetricEval(metric, [0, 1], [0, 1]), 0, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1]),
0.17517, 1e-4)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ -1, 1, 9, -9]),
3.4037, 1e-4)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ 1, 2, 9, 8]),
0.1922, 1e-4)
def test_logloss(self):
metric = Metrics.logloss
self.EXPECT_NEAR(self.GetMetricEval(metric, [0, 1], [0, 1]), 0, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.5, 1e-17, 1.0+1e-17, 0.9],
[ 0, 0, 1, 1]),
0.1996, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1]),
1.2039, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ -1, 1, 9, -9]),
21.9722, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ 1, 2, 9, 8]),
1.3138, 0.001)
def test_error(self):
metric = Metrics.error
# For error@0.5
self.EXPECT_NEAR(self.GetMetricEval(metric, [0, 1], [0, 1]), 0, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1]),
0.5, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ -1, 1, 9, -9]),
10.0, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ 1, 2, 9, 8]),
0.55, 0.001)
def test_poisson_nloglik(self):
metric = Metrics.poisson_nloglik
self.EXPECT_NEAR(self.GetMetricEval(metric, [0, 1], [0, 1]), 0.5, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.5, 1e-17, 1.0+1e-17, 0.9],
[ 0, 0, 1, 1]),
0.6263, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1]),
1.1019, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ -1, 1, 9, -9]),
13.3750, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ 1, 2, 9, 8]),
1.5783, 0.001)
def test_gamma_deviance(self):
metric = Metrics.gamma_deviance
self.EXPECT_NEAR(self.GetMetricEval(metric, [0, 1], [0, 1]), 0.5, 1e-10)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1]),
1.1020, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ -1, 1, 9, -9]),
4.4079, 0.001)
self.EXPECT_NEAR(self.GetMetricEval(metric,
[0.1, 0.9, 0.1, 0.9],
[ 0, 0, 1, 1],
[ 1, 2, 9, 8]),
0.2204, 0.001)
def Stest_merror(self):
metric = Metrics.merror
def Stest_mlogloss(self):
metric = Metrics.mlogloss
def EXPECT_NEAR(self, val1, val2, abs_error):
if abs(val1 - val2) >= abs_error: print(f"{val1} != {val2} >= {abs_error} = ", abs(val1-val2))
return 0
self.assertTrue(abs(val1 - val2) <= abs_error, f"{val1} != {val2}")
def GetMetricEval(self, metric, preds, labels, weights=None): # , groups = None):
input = torch.tensor(list(preds) * 5, dtype=torch.float32).reshape((5, -1))
output = torch.tensor(list(labels) * 5, dtype=torch.float32).reshape((5, -1))
input_old = input.clone()
output_old = output.clone()
if weights is not None:
weights = torch.tensor(list(weights) * 5, dtype=torch.float32).reshape((5, -1))
weights_old = weights.clone()
name, result = metric(input, output, weights)
self.assertEqual(name.replace("-", "_"), metric.__name__)
self.assertTrue(bool(input.eq(input_old).all()))
self.assertTrue(bool(output.eq(output_old).all()))
if weights is not None:
self.assertTrue(bool(weights.eq(weights_old).all()))
print(name, " = ", result.mean())
return float(result.mean())
if __name__ == "__main__":
unittest.main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment