Skip to content

Instantly share code, notes, and snippets.

@ippeiukai
Last active March 22, 2021 12:57
Show Gist options
  • Star 6 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ippeiukai/1d9f717711414118b360eff0718c07c0 to your computer and use it in GitHub Desktop.
Save ippeiukai/1d9f717711414118b360eff0718c07c0 to your computer and use it in GitHub Desktop.
Sequelize port of find_each in ActiveRecord. (https://github.com/sequelize/sequelize/issues/686 )
"use strict";
const Sequelize = require('sequelize');
const Promise = Sequelize.Promise;
const DEFAULT_BATCH_SIZE = 3000;
/**
* Port of ActiveRecord::Base.find_each of Rails.
*
* @param model - Sequelize model
* @param [findOptions] - The options to be passed to findAll
* @param [options]
* @param [options.batchSize] - Number of records to be loaded at each loading. Default is 3000.
* @param callbackFn - Will be called with each instance. If you return a Promise, it will be waited before calling with next instance.
* @return {Promsie}
*/
const sequelizeFindEach = module.exports.findEach = function (model, findOptions, options, callbackFn) {
if (typeof findOptions === 'function' && options == null && callbackFn == null) {
callbackFn = findOptions;
findOptions = null;
} else if (typeof options === 'function' && callbackFn == null) {
callbackFn = options;
options = null;
}
if (findOptions == null) {
findOptions = {};
}
if (options == null) {
options = {};
}
return sequelizeFindInBatches(model, findOptions, options, Promise.coroutine(function* (batch) {
for (let record of batch) {
yield Promise.resolve(callbackFn(record));
}
}));
};
/**
* Port of ActiveRecord::Base.find_in_batches of Rails.
*
* @param model - Sequelize model
* @param [findOptions] - The options to be passed to findAll
* @param [options]
* @param [options.batchSize] - Number of records to be loaded in each batch. Default is 3000.
* @param callbackFn - Will be called with each batch of instances. If you return a Promise, it will be waited before loading the next batch.
* @return {Promsie}
*/
const sequelizeFindInBatches = module.exports.findInBatches = Promise.coroutine(function* (model, findOptions, options, callbackFn) {
if (typeof findOptions === 'function' && options == null && callbackFn == null) {
callbackFn = findOptions;
findOptions = null;
} else if (typeof options === 'function' && callbackFn == null) {
callbackFn = options;
options = null;
}
if (findOptions == null) {
findOptions = {};
}
if (options == null) {
options = {};
}
const batchSize = ('batchSize' in options ? options.batchSize : DEFAULT_BATCH_SIZE);
const idCol = model.primaryKeyAttribute;
let records = yield model.findAll(Object.assign({}, findOptions, {
limit: batchSize,
order: idCol,
}));
yield Promise.resolve(callbackFn(records));
while (!(records.length < batchSize)) {
let lastRecord = records[records.length -1];
records = yield model.findAll(mergeWhereConditions(findOptions, {
[idCol]: {
$gt: lastRecord[idCol]
}
}, {
limit: batchSize,
order: idCol,
}));
if (records.length) {
yield Promise.resolve(callbackFn(records));
}
}
});
function mergeWhereConditions(findOptions, additionalWhere, overrides) {
let mergedWhere;
if ('where' in findOptions) {
mergedWhere = {
$and: [
findOptions.where,
additionalWhere
]
};
} else {
mergedWhere = additionalWhere;
}
return Object.assign({}, findOptions, {where: mergedWhere}, overrides);
}
@ravirahman
Copy link

This is cool -- thanks for sharing! Can I use this code under the MIT license?

@ippeiukai
Copy link
Author

Can I use this code under the MIT license?

It’s probably too late, but for the sake of any late comers, I hereby grant anyone to use this code under the MIT or any other licenses by all means, as long as:

  • you don’t publish the above code claiming it is yours (just put a comment with URL if you are in doubt).
  • you don’t put any responsibility on me for whatever using this code caused you.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment