Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Benchmark various sigmoid functions
#include <stdio.h>
#include <time.h>
#include <stdlib.h>
#include <math.h>
#define M_PI_2 1.57079632679489661923 /* pi/2 */
#define M_PI_2_INV (1.0/M_PI_2)
#define M_2_SQRTPI 1.12837916709551257390 /* 2/sqrt(pi) */
#define ERF_COEF (1.0/M_2_SQRTPI)
const int SIZE=100;
const int CYCLES=10000000;
double benchmark(const char* name, double (*fun)(double)) {
clock_t start, stop;
double xs[SIZE];
double t_ns;
for (int i=0; i<SIZE; i++) {
xs[i] = rand();
}
start = clock();
for (int repeat=0; repeat<CYCLES; repeat++) {
for (int i=0; i<SIZE; i++) {
(*fun)(xs[i]);
}
}
stop = clock();
t_ns = (stop-start)*1.0e9/CLOCKS_PER_SEC/CYCLES/SIZE;
printf("%-17s %6.1f ns\n", name, t_ns);
return t_ns;
}
double with_atan(double x) {
/* normalized atan */
return M_PI_2_INV*atan(M_PI_2*x);
}
double with_exp(double x) {
return 1.0/(1.0 + exp(-x));
}
double with_sqrt(double x) {
return 1.0/sqrt(1.0 + x*x);
}
double with_erf(double x) {
return erf(ERF_COEF*x);
}
double with_fabs(double x) {
return x/(1.0 + fabs(x));
}
int main(int argc, char **argv) {
benchmark("atan(pi*x/2)*2/pi", with_atan);
benchmark("atan(x)", atan);
benchmark("1/(1+exp(-x))", with_exp);
benchmark("1/sqrt(1+x^2)", with_sqrt);
benchmark("erf(sqrt(pi)*x/2)", with_erf);
benchmark("tanh(x)", tanh);
benchmark("x/(1+|x|)", with_fabs);
}
Owner

astanin commented Mar 29, 2013

On my Core i5-3317U with GCC 4.7.2:

% gcc -Wall -O2 -lm -o sigmoid-bench{,.c} -std=c99 && ./sigmoid-bench
atan(pi*x/2)*2/pi   24.1 ns
atan(x)             23.0 ns
1/(1+exp(-x))       20.4 ns
1/sqrt(1+x^2)       13.4 ns
erf(sqrt(pi)*x/2)    6.7 ns
tanh(x)              5.5 ns
x/(1+|x|)            5.5 ns

On AMD 1090T @ gcc 4.8.2:

% gcc -Wall -O2 -o sigmoid-bench{,.c} -std=c99 -lm && ./sigmoid-bench
atan(pi*x/2)*2/pi   30.5 ns
atan(x)             26.0 ns
1/(1+exp(-x))       81.6 ns
1/sqrt(1+x^2)       12.7 ns
erf(sqrt(pi)*x/2)    6.6 ns
tanh(x)              5.8 ns
x/(1+|x|)            5.1 ns

On Intel i7-4702MQ Notebook with gcc 4.8.3 20140627:

atan(pi*x/2)*2/pi   16.5 ns
atan(x)             14.9 ns
1/(1+exp(-x))       12.1 ns
1/sqrt(1+x^2)       10.7 ns
erf(sqrt(pi)*x/2)    3.9 ns
tanh(x)              3.2 ns
x/(1+|x|)            4.4 ns

But note: if I evaluate a real neural net with them, tanh(x) and 1/(1+exp(-x)) are about the same speed, and x/(1+|x|) is more than twice as fast as those two. Trying to investigate why...

Edit:

If I change the line:

(*fun)(xs[i]);

to this:

xs[i] = (*fun)(xs[i]);

I get this:

atan(pi*x/2)*2/pi    9.2 ns
atan(x)              7.9 ns
1/(1+exp(-x))       19.3 ns
1/sqrt(1+x^2)       10.8 ns
erf(sqrt(pi)*x/2)    6.5 ns
tanh(x)             15.4 ns
x/(1+|x|)            4.4 ns

Which seems closer to what I get with a real neural net...

And if I use:

xs[i] = (*fun)(xs[i]) + 0.5;

I get:

atan(pi*x/2)*2/pi   20.5 ns
atan(x)             19.4 ns
1/(1+exp(-x))       19.3 ns
1/sqrt(1+x^2)       10.8 ns
erf(sqrt(pi)*x/2)   49.9 ns
tanh(x)             28.1 ns
x/(1+|x|)            2.5 ns

As you see, some of these functions depend very much on the value they have to evaluate...

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment