deduplicate.js

const new_items = [];
const key_field =  $("Globals").first()["json"]["dedup"]["key"];
console.log("key_field from json.key: " + key_field);
const now = $now

// Get static data stored with the workflow
data = $getWorkflowStaticData("node");

// delete the old cache, that didn't have time 
data.ids = null;
// new cache of id => time added
data.seen = data.seen || {};

for (let i = items.length - 1; i >= 0; i--) {
    var item = items[i]
    if(item.json[key_field]){
        console.log("item: " + item.json[key_field])
    	// Check if data is already present
	    if (!data.seen[item.json[key_field]]) {

		   // if new data then add it to an array
		   new_items.push(item);
           data.seen[item.json[key_field]] = now
	    }
    }  // errors have no guid, just drop 'em
}
// destroy everything older that 24 hours from data.seen
var yesterday = new Date(new Date().getTime() - (24 * 60 * 60 * 1000));
const keys = Object.keys(data.seen)
for (var key in keys){
  var orig_date = data.seen[key];
  if (orig_date < yesterday){
    console.log("removing old item: " + key + " from " + orig_date)
    data.seen.delete(key);
  }
}

// return new items
return new_items;