Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Benchmark various sigmoid functions
#include <stdio.h>
#include <time.h>
#include <stdlib.h>
#include <math.h>
#define M_PI_2 1.57079632679489661923 /* pi/2 */
#define M_PI_2_INV (1.0/M_PI_2)
#define M_2_SQRTPI 1.12837916709551257390 /* 2/sqrt(pi) */
#define ERF_COEF (1.0/M_2_SQRTPI)
const int SIZE=100;
const int CYCLES=10000000;
double benchmark(const char* name, double (*fun)(double)) {
clock_t start, stop;
double xs[SIZE];
double t_ns;
for (int i=0; i<SIZE; i++) {
xs[i] = rand();
}
start = clock();
for (int repeat=0; repeat<CYCLES; repeat++) {
for (int i=0; i<SIZE; i++) {
(*fun)(xs[i]);
}
}
stop = clock();
t_ns = (stop-start)*1.0e9/CLOCKS_PER_SEC/CYCLES/SIZE;
printf("%-17s %6.1f ns\n", name, t_ns);
return t_ns;
}
double with_atan(double x) {
/* normalized atan */
return M_PI_2_INV*atan(M_PI_2*x);
}
double with_exp(double x) {
return 1.0/(1.0 + exp(-x));
}
double with_sqrt(double x) {
return 1.0/sqrt(1.0 + x*x);
}
double with_erf(double x) {
return erf(ERF_COEF*x);
}
double with_fabs(double x) {
return x/(1.0 + fabs(x));
}
int main(int argc, char **argv) {
benchmark("atan(pi*x/2)*2/pi", with_atan);
benchmark("atan(x)", atan);
benchmark("1/(1+exp(-x))", with_exp);
benchmark("1/sqrt(1+x^2)", with_sqrt);
benchmark("erf(sqrt(pi)*x/2)", with_erf);
benchmark("tanh(x)", tanh);
benchmark("x/(1+|x|)", with_fabs);
}
Owner

astanin commented Mar 29, 2013

On my Core i5-3317U with GCC 4.7.2:

% gcc -Wall -O2 -lm -o sigmoid-bench{,.c} -std=c99 && ./sigmoid-bench
atan(pi*x/2)*2/pi   24.1 ns
atan(x)             23.0 ns
1/(1+exp(-x))       20.4 ns
1/sqrt(1+x^2)       13.4 ns
erf(sqrt(pi)*x/2)    6.7 ns
tanh(x)              5.5 ns
x/(1+|x|)            5.5 ns

On AMD 1090T @ gcc 4.8.2:

% gcc -Wall -O2 -o sigmoid-bench{,.c} -std=c99 -lm && ./sigmoid-bench
atan(pi*x/2)*2/pi   30.5 ns
atan(x)             26.0 ns
1/(1+exp(-x))       81.6 ns
1/sqrt(1+x^2)       12.7 ns
erf(sqrt(pi)*x/2)    6.6 ns
tanh(x)              5.8 ns
x/(1+|x|)            5.1 ns

On Intel i7-4702MQ Notebook with gcc 4.8.3 20140627:

atan(pi*x/2)*2/pi   16.5 ns
atan(x)             14.9 ns
1/(1+exp(-x))       12.1 ns
1/sqrt(1+x^2)       10.7 ns
erf(sqrt(pi)*x/2)    3.9 ns
tanh(x)              3.2 ns
x/(1+|x|)            4.4 ns

But note: if I evaluate a real neural net with them, tanh(x) and 1/(1+exp(-x)) are about the same speed, and x/(1+|x|) is more than twice as fast as those two. Trying to investigate why...

Edit:

If I change the line:

(*fun)(xs[i]);

to this:

xs[i] = (*fun)(xs[i]);

I get this:

atan(pi*x/2)*2/pi    9.2 ns
atan(x)              7.9 ns
1/(1+exp(-x))       19.3 ns
1/sqrt(1+x^2)       10.8 ns
erf(sqrt(pi)*x/2)    6.5 ns
tanh(x)             15.4 ns
x/(1+|x|)            4.4 ns

Which seems closer to what I get with a real neural net...

And if I use:

xs[i] = (*fun)(xs[i]) + 0.5;

I get:

atan(pi*x/2)*2/pi   20.5 ns
atan(x)             19.4 ns
1/(1+exp(-x))       19.3 ns
1/sqrt(1+x^2)       10.8 ns
erf(sqrt(pi)*x/2)   49.9 ns
tanh(x)             28.1 ns
x/(1+|x|)            2.5 ns

As you see, some of these functions depend very much on the value they have to evaluate...

Wiffzack commented Sep 30, 2017

Test on BashOnWindows!
Intel® Core™ i3-5010U
gcc version 6.3.0 20170519
almost every compiler option was used :-)

atan(pi*x/2)*2/pi   13.0 ns
atan(x)             11.5 ns
1/(1+exp(-x))       14.0 ns
1/sqrt(1+x^2)       10.6 ns
erf(sqrt(pi)*x/2)    7.0 ns
tanh(x)              5.3 ns
x/(1+|x|)            2.8 ns
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment