Skip to content

Instantly share code, notes, and snippets.

@arvobowen
Forked from patt0/cbl.js
Created March 16, 2016 11:56
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save arvobowen/5a073c07a7573fb29b24 to your computer and use it in GitHub Desktop.
Save arvobowen/5a073c07a7573fb29b24 to your computer and use it in GitHub Desktop.
ContinuousBatchLibrary is a Google Apps Script library that manages large batches and works around the 5 minute limitation of GAS execution. It does this by setting time based triggers in the future as well as memorising the last processed key in the batch in order to restart from the correct position. At the end of the batch a cleanup function …
/**
* --- Continous Execution Library ---
*
* Copyright (c) 2013 Patrick Martinent
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*************************************************************************
* Call this function at the start of your batch script
* it will create the necessary UserProperties with the fname
* so that it can keep managing the triggers until the batch
* execution is complete. It will store the start time for the
* email it sends out to you when the batch has completed
*
* @param {fname} str The batch function to invoke repeatedly.
*/
function startOrResumeContinousExecutionInstance(fname){
var userProperties = PropertiesService.getUserProperties();
var start = userProperties.getProperty('GASCBL_' + fname + '_START_BATCH');
if (start === "" || start === null)
{
start = new Date();
userProperties.setProperty('GASCBL_' + fname + '_START_BATCH', start);
userProperties.setProperty('GASCBL_' + fname + '_KEY', "");
}
userProperties.setProperty('GASCBL_' + fname + '_START_ITERATION', new Date());
deleteCurrentTrigger_(fname);
enableNextTrigger_(fname);
}
/*************************************************************************
* In order to be able to understand where your batch last executed you
* set the key ( or counter ) everytime a new item in your batch is complete
* when you restart the batch through the trigger, use getBatchKey to start
* at the right place
*
* @param {fname} str The batch function we are continuously triggering.
* @param {key} str The batch key that was just completed.
*/
function setBatchKey(fname, key){
var userProperties = PropertiesService.getUserProperties();
userProperties.setProperty('GASCBL_' + fname + '_KEY', key);
}
/*************************************************************************
* This function returns the current batch key, so you can start processing at
* the right position when your batch resumes from the execution of the trigger
*
* @param {fname} str The batch function we are continuously triggering.
* @returns {string} The batch key which was last completed.
*/
function getBatchKey(fname){
var userProperties = PropertiesService.getUserProperties();
return userProperties.getProperty('GASCBL_' + fname + '_KEY');
}
/*************************************************************************
* When the batch is complete run this function, and pass it an email and
* custom title so you have an indication that the process is complete as
* well as the time it took
*
* @param {fname} str The batch function we are continuously triggering.
* @param {emailRecipient} str The email address to which the email will be sent.
* @param {customTitle} str The custom title for the email.
*/
function endContinuousExecutionInstance(fname, emailRecipient, customTitle){
var userProperties = PropertiesService.getUserProperties();
var end = new Date();
var start = userProperties.getProperty('GASCBL_' + fname + '_START_BATCH');
var key = userProperties.getProperty('GASCBL_' + fname + '_KEY');
var emailTitle = customTitle + " : Continuous Execution Script for " + fname;
var body = "Started : " + start + "<br>" + "Ended :" + end + "<br>" + "LAST KEY : " + key;
MailApp.sendEmail(emailRecipient, emailTitle, "", {htmlBody:body});
deleteCurrentTrigger_(fname);
userProperties.deleteProperty('GASCBL_' + fname + '_START_ITERATION');
userProperties.deleteProperty('GASCBL_' + fname + '_START_BATCH');
userProperties.deleteProperty('GASCBL_' + fname + '_KEY');
userProperties.deleteProperty('GASCBL_' + fname);
}
/*************************************************************************
* Call this function when finishing a batch item to find out if we have
* time for one more. if not exit elegantly and let the batch restart with
* the trigger
*
* @param {fname} str The batch function we are continuously triggering.
* @returns (boolean) wether we are close to reaching the exec time limit
*/
function isTimeRunningOut(fname){
var userProperties = PropertiesService.getUserProperties();
var start = new Date(userProperties.getProperty('GASCBL_' + fname + '_START_ITERATION'));
var now = new Date();
var timeElapsed = Math.floor((now.getTime() - start.getTime())/1000);
return (timeElapsed > 270);
}
/*
* Set the next trigger, 7 minutes in the future
*/
function enableNextTrigger_(fname) {
var userProperties = PropertiesService.getUserProperties();
var nextTrigger = ScriptApp.newTrigger(fname).timeBased().after(7 * 60 * 1000).create();
var triggerId = nextTrigger.getUniqueId();
userProperties.setProperty('GASCBL_' + fname, triggerId);
}
/*
* Deletes the current trigger, so we don't end up with undeleted
* time based triggers all over the place
*/
function deleteCurrentTrigger_(fname) {
var userProperties = PropertiesService.getUserProperties();
var triggerId = userProperties.getProperty('GASCBL_' + fname);
var triggers = ScriptApp.getProjectTriggers();
for (var i in triggers) {
if (triggers[i].getUniqueId() === triggerId)
ScriptApp.deleteTrigger(triggers[i]);
}
userProperties.setProperty('GASCBL_' + fname, "");
}
function testCBL() {
// simulate a trigger on batch process, i.e if you run the batch
// everyday at a particular time
var triggerId = ScriptApp.newTrigger("batchProcess").timeBased().after(60 * 1000).create();
// clean test by deleting it the initial triggers
Utilities.sleep(90000);
ScriptApp.deleteTrigger(triggerId);
}
function batchProcess() {
// initiate CBL for the function
CBL.startOrResumeContinousExecutionInstance("batchProcess");
// this is approach is valid is we are looking to process a for loop
// this is because the key start value is ""
if (CBL.getBatchKey("batchProcess") === "")
CBL.setBatchKey("batchProcess", 0);
// find out where we left off (again with a for loop)
var counter = Number(CBL.getBatchKey("batchProcess"));
for(var i = 0 + counter; i &lt; 80; i++) {
// perform batch
Utilities.sleep(5000);
Logger.log("batchProcess_" + i)
CBL.setBatchKey("batchProcess", i);
// find out wether we have been running this iteration for more that 5 minutes
// in which case exit the batch elegantly
if (CBL.isTimeRunningOut("batchProcess"))
return;
}
// if we get to this point, it means we have completed the batch and we can cleanup
// this will also send an email with the last batch key, start and end times
CBL.endContinuousExecutionInstance("batchProcess", "you@example.com", "My Custom Email Title");
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment