JavaScript Performance Benchmarks: Looping with `for` and `yield`

benchmarks-raw.coffee

'use strict'


############################################################################################################
log   = console.log
warn  = console.warn
fn    = (  new Intl.NumberFormat 'en-US' ).format


#-----------------------------------------------------------------------------------------------------------
@_get_addup_probe = ( n ) ->
  base        = 1e6
  probe       = [ base ... base + n ]
  matcher     = 0
  matcher    += number for number in probe
  return { probe, matcher, base, }

#-----------------------------------------------------------------------------------------------------------
@addup_for_in_loop = ( n ) -> new Promise ( resolve ) =>
  { probe
    matcher } = @_get_addup_probe n
  #.........................................................................................................
  resolve => new Promise addup_for_in_loop = ( resolve ) =>
    count       = 0
    sum         = 0
    for number in probe
      count++
      sum += number
    unless sum is matcher then warn "incorrect result, expected #{matcher}, got #{sum}"
    resolve count
    return null
  #.........................................................................................................
  return null

#-----------------------------------------------------------------------------------------------------------
@addup_for_from_loop = ( n ) -> new Promise ( resolve ) =>
  { probe
    matcher } = @_get_addup_probe n
  #.........................................................................................................
  resolve => new Promise addup_for_from_loop = ( resolve ) =>
    count       = 0
    sum         = 0
    for number from probe
      count++
      sum += number
    unless sum is matcher then warn "incorrect result, expected #{matcher}, got #{sum}"
    resolve count
    return null
  #.........................................................................................................
  return null

#-----------------------------------------------------------------------------------------------------------
@addup_forEach_loop = ( n ) -> new Promise ( resolve ) =>
  { probe
    matcher } = @_get_addup_probe n
  #.........................................................................................................
  resolve => new Promise addup_forEach_loop = ( resolve ) =>
    count       = 0
    sum         = 0
    probe.forEach ( number ) ->
      count++
      sum += number
    unless sum is matcher then warn "incorrect result, expected #{matcher}, got #{sum}"
    resolve count
    return null
  #.........................................................................................................
  return null

#-----------------------------------------------------------------------------------------------------------
@addup_yield_for_in_array = ( n ) -> new Promise ( resolve ) =>
  { probe
    matcher } = @_get_addup_probe n
  numbers     = probe
  probe       = -> yield n for n in numbers
  #.........................................................................................................
  resolve => new Promise addup_yield_for_in_array = ( resolve ) =>
    count       = 0
    sum         = 0
    for number from probe()
      count++
      sum += number
    unless sum is matcher then warn "incorrect result, expected #{matcher}, got #{sum}"
    resolve count
    return null
  #.........................................................................................................
  return null

#-----------------------------------------------------------------------------------------------------------
@addup_yield_for_from_values = ( n ) -> new Promise ( resolve ) =>
  { probe
    matcher } = @_get_addup_probe n
  numbers     = probe
  probe       = -> yield n for n from numbers.values()
  #.........................................................................................................
  resolve => new Promise addup_yield_for_from_values = ( resolve ) =>
    count       = 0
    sum         = 0
    for number from probe()
      count++
      sum += number
    unless sum is matcher then warn "incorrect result, expected #{matcher}, got #{sum}"
    resolve count
    return null
  #.........................................................................................................
  return null

#-----------------------------------------------------------------------------------------------------------
@addup_yield_from_array = ( n ) -> new Promise ( resolve ) =>
  { probe
    matcher } = @_get_addup_probe n
  numbers     = probe
  probe       = -> yield from numbers
  #.........................................................................................................
  resolve => new Promise addup_yield_from_array = ( resolve ) =>
    count       = 0
    sum         = 0
    for number from probe()
      count++
      sum += number
    unless sum is matcher then warn "incorrect result, expected #{matcher}, got #{sum}"
    resolve count
    return null
  #.........................................................................................................
  return null

#-----------------------------------------------------------------------------------------------------------
@addup_yield_from_values = ( n ) -> new Promise ( resolve ) =>
  { probe
    matcher } = @_get_addup_probe n
  numbers     = probe
  probe       = -> yield from numbers.values()
  #.........................................................................................................
  resolve => new Promise addup_yield_from_values = ( resolve ) =>
    count       = 0
    sum         = 0
    for number from probe()
      count++
      sum += number
    unless sum is matcher then warn "incorrect result, expected #{matcher}, got #{sum}"
    resolve count
    return null
  #.........................................................................................................
  return null

#-----------------------------------------------------------------------------------------------------------
@addup_values_iterator = ( n ) -> new Promise ( resolve ) =>
  { probe
    matcher } = @_get_addup_probe n
  numbers     = probe
  probe       = numbers.values()
  #.........................................................................................................
  resolve => new Promise addup_values_iterator = ( resolve ) =>
    { probe
      matcher } = @_get_addup_probe n
    count       = 0
    sum         = 0
    for number from probe
      count++
      sum += number
    unless sum is matcher then warn "incorrect result, expected #{matcher}, got #{sum}"
    resolve count
    return null
  #.........................................................................................................
  return null


#-----------------------------------------------------------------------------------------------------------
@benchmark = ->
  # n           = 5e6
  n           = 5e6
  test_names = [
    'addup_for_in_loop'
    'addup_for_from_loop'
    'addup_values_iterator'
    'addup_forEach_loop'
    'addup_yield_from_values'
    'addup_yield_for_in_array'
    'addup_yield_from_array'
    'addup_yield_for_from_values'
    ]
  delta_ts = []
  #.........................................................................................................
  for test_name in test_names
    log "test: #{test_name}"
    test  = await @[ test_name ] n
    t0    = Date.now()
    await test()
    t1    = Date.now()
    delta_ts.push { test_name, dt: t1 - t0, }
  #.........................................................................................................
  delta_ts.sort ( a, b ) ->
    return -1 if a.dt < b.dt
    return +1 if a.dt > b.dt
    return  0
  #.........................................................................................................
  baseline_dt = delta_ts[ 0 ].dt
  for { test_name, dt, } in delta_ts
    test_name_txt   = "#{test_name} ".padEnd 30, '.'
    dt_txt          = " #{fn dt} ms".padStart 10, ' '
    frequency       = n / ( dt / 1000 )
    frequency_txt   = " #{fn frequency.toFixed 0} Hz".padStart 25, ' '
    percentage      = baseline_dt / dt * 100
    percentage_txt  = " #{percentage.toFixed 1} %".padStart 8, ' '
    log test_name_txt + dt_txt + frequency_txt + percentage_txt
  #.........................................................................................................
  return null


############################################################################################################
do =>
  await @benchmark()
  log 'ok'

benchmarks-raw.js

(function() {
  'use strict';
  var fn, log, warn;

  //###########################################################################################################
  log = console.log;

  warn = console.warn;

  fn = (new Intl.NumberFormat('en-US')).format;

  //-----------------------------------------------------------------------------------------------------------
  this._get_addup_probe = function(n) {
    var base, i, len, matcher, number, probe, ref;
    base = 1e6;
    probe = (function() {
      var results = [];
      for (var i = base, ref = base + n; base <= ref ? i < ref : i > ref; base <= ref ? i++ : i--){ results.push(i); }
      return results;
    }).apply(this);
    matcher = 0;
    for (i = 0, len = probe.length; i < len; i++) {
      number = probe[i];
      matcher += number;
    }
    return {probe, matcher, base};
  };

  //-----------------------------------------------------------------------------------------------------------
  this.addup_for_in_loop = function(n) {
    return new Promise((resolve) => {
      var matcher, probe;
      ({probe, matcher} = this._get_addup_probe(n));
      //.........................................................................................................
      resolve(() => {
        var addup_for_in_loop;
        return new Promise(addup_for_in_loop = (resolve) => {
          var count, i, len, number, sum;
          count = 0;
          sum = 0;
          for (i = 0, len = probe.length; i < len; i++) {
            number = probe[i];
            count++;
            sum += number;
          }
          if (sum !== matcher) {
            warn(`incorrect result, expected ${matcher}, got ${sum}`);
          }
          resolve(count);
          return null;
        });
      });
      //.........................................................................................................
      return null;
    });
  };

  //-----------------------------------------------------------------------------------------------------------
  this.addup_for_from_loop = function(n) {
    return new Promise((resolve) => {
      var matcher, probe;
      ({probe, matcher} = this._get_addup_probe(n));
      //.........................................................................................................
      resolve(() => {
        var addup_for_from_loop;
        return new Promise(addup_for_from_loop = (resolve) => {
          var count, number, sum;
          count = 0;
          sum = 0;
          for (number of probe) {
            count++;
            sum += number;
          }
          if (sum !== matcher) {
            warn(`incorrect result, expected ${matcher}, got ${sum}`);
          }
          resolve(count);
          return null;
        });
      });
      //.........................................................................................................
      return null;
    });
  };

  //-----------------------------------------------------------------------------------------------------------
  this.addup_forEach_loop = function(n) {
    return new Promise((resolve) => {
      var matcher, probe;
      ({probe, matcher} = this._get_addup_probe(n));
      //.........................................................................................................
      resolve(() => {
        var addup_forEach_loop;
        return new Promise(addup_forEach_loop = (resolve) => {
          var count, sum;
          count = 0;
          sum = 0;
          probe.forEach(function(number) {
            count++;
            return sum += number;
          });
          if (sum !== matcher) {
            warn(`incorrect result, expected ${matcher}, got ${sum}`);
          }
          resolve(count);
          return null;
        });
      });
      //.........................................................................................................
      return null;
    });
  };

  //-----------------------------------------------------------------------------------------------------------
  this.addup_yield_for_in_array = function(n) {
    return new Promise((resolve) => {
      var matcher, numbers, probe;
      ({probe, matcher} = this._get_addup_probe(n));
      numbers = probe;
      probe = function*() {
        var i, len, results;
        results = [];
        for (i = 0, len = numbers.length; i < len; i++) {
          n = numbers[i];
          results.push((yield n));
        }
        return results;
      };
      //.........................................................................................................
      resolve(() => {
        var addup_yield_for_in_array;
        return new Promise(addup_yield_for_in_array = (resolve) => {
          var count, number, ref, sum;
          count = 0;
          sum = 0;
          ref = probe();
          for (number of ref) {
            count++;
            sum += number;
          }
          if (sum !== matcher) {
            warn(`incorrect result, expected ${matcher}, got ${sum}`);
          }
          resolve(count);
          return null;
        });
      });
      //.........................................................................................................
      return null;
    });
  };

  //-----------------------------------------------------------------------------------------------------------
  this.addup_yield_for_from_values = function(n) {
    return new Promise((resolve) => {
      var matcher, numbers, probe;
      ({probe, matcher} = this._get_addup_probe(n));
      numbers = probe;
      probe = function*() {
        var ref, results;
        ref = numbers.values();
        results = [];
        for (n of ref) {
          results.push((yield n));
        }
        return results;
      };
      //.........................................................................................................
      resolve(() => {
        var addup_yield_for_from_values;
        return new Promise(addup_yield_for_from_values = (resolve) => {
          var count, number, ref, sum;
          count = 0;
          sum = 0;
          ref = probe();
          for (number of ref) {
            count++;
            sum += number;
          }
          if (sum !== matcher) {
            warn(`incorrect result, expected ${matcher}, got ${sum}`);
          }
          resolve(count);
          return null;
        });
      });
      //.........................................................................................................
      return null;
    });
  };

  //-----------------------------------------------------------------------------------------------------------
  this.addup_yield_from_array = function(n) {
    return new Promise((resolve) => {
      var matcher, numbers, probe;
      ({probe, matcher} = this._get_addup_probe(n));
      numbers = probe;
      probe = function*() {
        return (yield* numbers);
      };
      //.........................................................................................................
      resolve(() => {
        var addup_yield_from_array;
        return new Promise(addup_yield_from_array = (resolve) => {
          var count, number, ref, sum;
          count = 0;
          sum = 0;
          ref = probe();
          for (number of ref) {
            count++;
            sum += number;
          }
          if (sum !== matcher) {
            warn(`incorrect result, expected ${matcher}, got ${sum}`);
          }
          resolve(count);
          return null;
        });
      });
      //.........................................................................................................
      return null;
    });
  };

  //-----------------------------------------------------------------------------------------------------------
  this.addup_yield_from_values = function(n) {
    return new Promise((resolve) => {
      var matcher, numbers, probe;
      ({probe, matcher} = this._get_addup_probe(n));
      numbers = probe;
      probe = function*() {
        return (yield* numbers.values());
      };
      //.........................................................................................................
      resolve(() => {
        var addup_yield_from_values;
        return new Promise(addup_yield_from_values = (resolve) => {
          var count, number, ref, sum;
          count = 0;
          sum = 0;
          ref = probe();
          for (number of ref) {
            count++;
            sum += number;
          }
          if (sum !== matcher) {
            warn(`incorrect result, expected ${matcher}, got ${sum}`);
          }
          resolve(count);
          return null;
        });
      });
      //.........................................................................................................
      return null;
    });
  };

  //-----------------------------------------------------------------------------------------------------------
  this.addup_values_iterator = function(n) {
    return new Promise((resolve) => {
      var matcher, numbers, probe;
      ({probe, matcher} = this._get_addup_probe(n));
      numbers = probe;
      probe = numbers.values();
      //.........................................................................................................
      resolve(() => {
        var addup_values_iterator;
        return new Promise(addup_values_iterator = (resolve) => {
          var count, number, sum;
          ({probe, matcher} = this._get_addup_probe(n));
          count = 0;
          sum = 0;
          for (number of probe) {
            count++;
            sum += number;
          }
          if (sum !== matcher) {
            warn(`incorrect result, expected ${matcher}, got ${sum}`);
          }
          resolve(count);
          return null;
        });
      });
      //.........................................................................................................
      return null;
    });
  };

  //-----------------------------------------------------------------------------------------------------------
  this.benchmark = async function() {
    var baseline_dt, delta_ts, dt, dt_txt, frequency, frequency_txt, i, j, len, len1, n, percentage, percentage_txt, t0, t1, test, test_name, test_name_txt, test_names;
    // n           = 5e6
    n = 5e6;
    test_names = ['addup_for_in_loop', 'addup_for_from_loop', 'addup_values_iterator', 'addup_forEach_loop', 'addup_yield_from_values', 'addup_yield_for_in_array', 'addup_yield_from_array', 'addup_yield_for_from_values'];
    delta_ts = [];
//.........................................................................................................
    for (i = 0, len = test_names.length; i < len; i++) {
      test_name = test_names[i];
      log(`test: ${test_name}`);
      test = (await this[test_name](n));
      t0 = Date.now();
      await test();
      t1 = Date.now();
      delta_ts.push({
        test_name,
        dt: t1 - t0
      });
    }
    //.........................................................................................................
    delta_ts.sort(function(a, b) {
      if (a.dt < b.dt) {
        return -1;
      }
      if (a.dt > b.dt) {
        return +1;
      }
      return 0;
    });
    //.........................................................................................................
    baseline_dt = delta_ts[0].dt;
    for (j = 0, len1 = delta_ts.length; j < len1; j++) {
      ({test_name, dt} = delta_ts[j]);
      test_name_txt = `${test_name} `.padEnd(30, '.');
      dt_txt = ` ${fn(dt)} ms`.padStart(10, ' ');
      frequency = n / (dt / 1000);
      frequency_txt = ` ${fn(frequency.toFixed(0))} Hz`.padStart(25, ' ');
      percentage = baseline_dt / dt * 100;
      percentage_txt = ` ${percentage.toFixed(1)} %`.padStart(8, ' ');
      log(test_name_txt + dt_txt + frequency_txt + percentage_txt);
    }
    //.........................................................................................................
    return null;
  };

  (async() => {    //###########################################################################################################
    await this.benchmark();
    return log('ok');
  })();

}).call(this);

output.md

Output from script above (runs in NodeJS, Chrome, Firefox)

➜  benchmarks (master) node lib/frameworkless-pipelines/benchmarks-raw.js                                                                                         ✭ ✱
test: addup_for_in_loop
test: addup_for_from_loop
test: addup_values_iterator
test: addup_forEach_loop
test: addup_yield_from_values
test: addup_yield_for_in_array
test: addup_yield_from_array
test: addup_yield_for_from_values
addup_for_in_loop ............     20 ms           250,000,000 Hz 100.0 %
addup_for_from_loop ..........    210 ms            23,809,524 Hz   9.5 %
addup_forEach_loop ...........    278 ms            17,985,612 Hz   7.2 %
addup_yield_from_values ......    440 ms            11,363,636 Hz   4.5 %
addup_values_iterator ........    503 ms             9,940,358 Hz   4.0 %
addup_yield_from_array .......    523 ms             9,560,229 Hz   3.8 %
addup_yield_for_in_array .....    788 ms             6,345,178 Hz   2.5 %
addup_yield_for_from_values ..  1,043 ms             4,793,864 Hz   1.9 %
ok

Equivalent results obtained with my benchmarking lib

➜  benchmarks (master) node lib/frameworkless-pipelines/benchmarks.js                                                                                             ✭ ✱
addup_yield_from_array                     0.451 s       5,000,000 items      11,098,016⏶Hz              90⏷nspc             440⏷CPU/u               9⏷CPU/s
addup_yield_for_in_array                   0.810 s       5,000,000 items       6,170,025⏶Hz             162⏷nspc             679⏷CPU/u             165⏷CPU/s
addup_values_iterator                      0.537 s       5,000,000 items       9,304,792⏶Hz             107⏷nspc             364⏷CPU/u             191⏷CPU/s
addup_for_in_loop                          0.020 s       5,000,000 items     247,248,935⏶Hz               4⏷nspc              37⏷CPU/u               0⏷CPU/s
addup_forEach_loop                         0.246 s       5,000,000 items      20,304,442⏶Hz              49⏷nspc             233⏷CPU/u              29⏷CPU/s
addup_yield_for_from_values                1.070 s       5,000,000 items       4,674,917⏶Hz             214⏷nspc             947⏷CPU/u             163⏷CPU/s
addup_for_from_loop                        0.232 s       5,000,000 items      21,540,226⏶Hz              46⏷nspc             211⏷CPU/u              29⏷CPU/s
addup_yield_from_values                    0.504 s       5,000,000 items       9,923,114⏶Hz             101⏷nspc             485⏷CPU/u              22⏷CPU/s

00:06 BENCHMARKS  ▶  addup_for_in_loop                            247,248,935 Hz   100.0 % │████████████▌│
00:06 BENCHMARKS  ▶  addup_for_from_loop                           21,540,226 Hz     8.7 % │█▏           │
00:06 BENCHMARKS  ▶  addup_forEach_loop                            20,304,442 Hz     8.2 % │█            │
00:06 BENCHMARKS  ▶  addup_yield_from_array                        11,098,016 Hz     4.5 % │▌            │
00:06 BENCHMARKS  ▶  addup_yield_from_values                        9,923,114 Hz     4.0 % │▌            │
00:06 BENCHMARKS  ▶  addup_values_iterator                          9,304,792 Hz     3.8 % │▌            │
00:06 BENCHMARKS  ▶  addup_yield_for_in_array                       6,170,025 Hz     2.5 % │▎            │
00:06 BENCHMARKS  ▶  addup_yield_for_from_values                    4,674,917 Hz     1.9 % │▎            │

If you spot any flaws in the code above, please comment. The whole point of this exercise is to find out why yield is such a performance drag in JS; naturally, this can only be done with fair tests, not with flawed ones.

Caveats and Discussion

• Sorry for the formatting of the JS part, this is the CoffeeScript compiler's output.
• The tests were originally written in CS not JS and this explains my choice for naming the loops; when I say 'for/in loops' that's a JS for loop with numerical index; when I say for/from loops that's a JS for/of loop. Sorry for the confusion.
• The overall run time of the test suite could have been reduced somewhat by eschewing to produce one new probes list for each test run; however, the time needed for these setups is not included in the timings.
• Per se none of the loop implementations is claimed to be best practice; you would probably not want to iterate over array.values() when you can iterate over array itself. Rather, the intent was to provide enough similar scenarios so as to enable a differential analysis of which factors in a given looping construct could have caused an improvement or a deterioration.
• The results have also been obtained with up to 10 repetitions of the entire test suite; averages same as shown.
• Test runs with repetitions have also involved shuffling of the order in which tests were run so as to disperse any hidden smudging effects.
• A simple guard is included such that inaccurate test setups will produce warnings.
• The time needed to set up the probe lists is not included in the timings; this is why each test case is a set of two nested async functions.
• The tests above use lists with 5 million integers, counting upwards from 1e6. Tests have been run with significantly shorter probe lists such as 1.000 and 100.000 numbers; then, results are more volatile, and, more interestingly, the differences between wieners and lusers become less pronounced.
• Again, iterating over lists of millions of consecutive integers is not a typical workload, but the reason I'm doing this is because I first spotted issues with yield's effect on data throughput in a rather more complicated setting (testing a conceptually MUCH simpler, yield-based re-implementation of SteamPipes, which internally uses classical for loops). The ~10 : 1 ... ~20 : 1 speed advantage of for loops was enough to allow my old, convoluted formulation to run circles around a very clean and simple, three-layers deep cascade of nested yield from generator statements.

Verdicts

• I believe the tests show that using yield has a downright incredible/horrible effect on JS performance.
• Please please tell me I'm wrong on this (and why, if so); yield is such a great tool it should be fast, too.
• It's not only yield that has problems, the performance of Array::forEach() isn't great, either, and even replacing (JS) for (i = 0, len = probe.length; i < len; i++) { ... } with (JS) for (number of probe) { ... } basically destroys loop throughput, and this is another thing I find hard to believe.