Last active
July 6, 2023 16:54
-
-
Save astanin/5270668 to your computer and use it in GitHub Desktop.
Benchmark various sigmoid functions
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <stdio.h> | |
#include <time.h> | |
#include <stdlib.h> | |
#include <math.h> | |
#define M_PI_2 1.57079632679489661923 /* pi/2 */ | |
#define M_PI_2_INV (1.0/M_PI_2) | |
#define M_2_SQRTPI 1.12837916709551257390 /* 2/sqrt(pi) */ | |
#define ERF_COEF (1.0/M_2_SQRTPI) | |
const int SIZE=100; | |
const int CYCLES=10000000; | |
double benchmark(const char* name, double (*fun)(double)) { | |
clock_t start, stop; | |
double xs[SIZE]; | |
double t_ns; | |
for (int i=0; i<SIZE; i++) { | |
xs[i] = rand(); | |
} | |
start = clock(); | |
for (int repeat=0; repeat<CYCLES; repeat++) { | |
for (int i=0; i<SIZE; i++) { | |
(*fun)(xs[i]); | |
} | |
} | |
stop = clock(); | |
t_ns = (stop-start)*1.0e9/CLOCKS_PER_SEC/CYCLES/SIZE; | |
printf("%-17s %6.1f ns\n", name, t_ns); | |
return t_ns; | |
} | |
double with_atan(double x) { | |
/* normalized atan */ | |
return M_PI_2_INV*atan(M_PI_2*x); | |
} | |
double with_exp(double x) { | |
return 1.0/(1.0 + exp(-x)); | |
} | |
double with_sqrt(double x) { | |
return 1.0/sqrt(1.0 + x*x); | |
} | |
double with_erf(double x) { | |
return erf(ERF_COEF*x); | |
} | |
double with_fabs(double x) { | |
return x/(1.0 + fabs(x)); | |
} | |
int main(int argc, char **argv) { | |
benchmark("atan(pi*x/2)*2/pi", with_atan); | |
benchmark("atan(x)", atan); | |
benchmark("1/(1+exp(-x))", with_exp); | |
benchmark("1/sqrt(1+x^2)", with_sqrt); | |
benchmark("erf(sqrt(pi)*x/2)", with_erf); | |
benchmark("tanh(x)", tanh); | |
benchmark("x/(1+|x|)", with_fabs); | |
} | |
Lenovo E480
intel core i5 8th gen - model name: Intel(R) Core(TM) i5-8250U CPU @ 1.60GHz
$ gcc --version
gcc (Ubuntu 8.3.0-6ubuntu1) 8.3.0
Copyright (C) 2018 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
$ gcc -Wall -O2 -o sigmoid-bench sigmoid.c -std=c99 -lm && ./sigmoid-bench
atan(pi*x/2)*2/pi 11.5 ns
atan(x) 10.4 ns
1/(1+exp(-x)) 10.3 ns
1/sqrt(1+x^2) 6.3 ns
erf(sqrt(pi)*x/2) 4.6 ns
tanh(x) 3.7 ns
x/(1+|x|) 2.1 ns
These functions don't have the same domains(some go [0, 1] others [-1, 1]) so I'm not sure what the purpose of this benchmark is.
@miltondts In several applications it's convenient to have a sigmoid function but there is no particular requirement which one should be used. Yes, the function may have to be scaled or shifted according to the application requirements. And indeed, the images of these functions are different, but in some applications the image of the function may be irrelevant or can be accounted for. Exclude or modify those that don't fit your case and choose what's best for you.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Intel(R) Xeon(R) CPU E5-2609 0 @ 2.40GHz, SuSE Linux
gcc Version 6.3.0:
icc version 16.0.3 (Intel's commercial compiler):
Note that the Intel compiler simply elides the function invocations because it detects that the return values are not saved (!). I modified the
benchmark()
function by saving the return values in an arrayys
:TL;DR:
tanh(x)
still wins. And different compilers optimize differently :-)