undupe() removes duplicate entries or objects by shape from an array...

undupe.js

// 7 April 2019
// removes duplicate entries or objects by shape from an array...

// 9 April 2019 - support recursive array dedupe.

// Correction: returns an array of unique entries in an array or an object,
// where the keys are whole numbers, and the key values of the entire path
// through an object structure are unique.

// Safe for arrays, objects, null, undefined, empty strings, and NaN.

// Operates on the indexical keys of an object, e.g., { 0: 'zero' }, iterating indexical
// key count into a `length` property.
// For more detail, see https://gist.github.com/dfkaye/499f2e21a5b0f1725b4d08743ecba396

// Spurred by tweet https://twitter.com/ireaderinokun/status/1105378900962623488
// regarding post, "Removing duplicate objects from an Array (is hard)",
// at https://bitsofco.de/removing-duplicate-objects-from-an-array-is-hard/

function undupe(arr) {
  var O = /Object|Array/
  var A = /Array/;
  var S = {}.toString;
  var M = { /* map of visited values by name */ };
  
  /*
   * Recursive sort for creating key-sorted copies of an object.
   */
  function visit(item) {
    var o = {};
    
    // Work on a local sorted copy of an array.
    
    A.test(S.call(item)) && (item = item.slice().sort());
    
    Object.keys(item).sort().map(function(key) {
      o[key] = O.test(S.call(item[key])) ? visit(item[key]) : item[key]
    });

    return o;
  }
  
  // Gratuitous coercion of object into an array-like structure, to show that
  // it can be done. Attempt to coerce an object with indexical keys and add
  // a length property in preparation for the next step.
  // For more info, see https://gist.github.com/dfkaye/499f2e21a5b0f1725b4d08743ecba396
  
  var temp;
  (arr.length >= 0) || (
    temp = Object.assign({}, arr),
    temp.length = Object.keys(arr).reduce(function(length, k) {
      return k == +k && k >= 0 ? length + 1 : length;
    }, 0),
    arr = temp
  );

  // Copy or coerce the argument to an array and capture entries on new array
  // using with array.reduce() and return the new array.
  
  return [].slice.call(arr).reduce(function(N, item) {
    // recursive undupe for arrays
    A.test(S.call(item)) && (item = undupe(item));
    
    var name = O.test(S.call(item))

      // If object, get new object by sorted keys, if any,      
      // and stringify the new object by its entire key sorted object chain.
    
      ? JSON.stringify(visit(item))

      // If not object, use item's string value.
    
      : String(item);
    
    // console.warn(name);
    if (name in M) {
      
      // If already mapped we have a dupe
      
      console.error('dupe', name)        
    } else {
      
      // Otherwise we're good to go.
      // Add to the map, push to the new array.

      M[name] = item
      N.push(item);
    }
    
    // Return the new array we're creating with reduce for the next iteration.
    
    return N;
  }, [ /* new array */ ])
}

// test it out 

var test = [
  // primitives - should see each only once
  '', '',
  NaN, NaN,
  null, 'null',
  undefined, 'undefined',
  1, 1, 
  'a', 'a',
  Date.now(), Date.now(), // on a slower runtime might see more than one
  
  // objects
  { name: 'a', value: 'b' },
  { name: 'a', value: 'b' }, // dupe
  { name: 'a', value: 'c' },
  { value: 'c', name: 'a' }, // dupe
  { name: 'b', value: 'c' },
  
  // nested objects
  { object: { name: 'a', value: 'b' } },
  { object: { name: 'a', value: 'b' } }, // dupe  
  { object: { name: 'a', value: 'c' } },
  { object: { value: 'c', name: 'a' } }, // dupe
  { object: { name: 'b', value: 'c' } },
  
  // arrays
  [ 1, 2, 3 ],
  [ 1, 2, 3 ], // dupe
  [ 2, 1, 3 ], // dupe
  [ 3, 2, 1 ], // dupe
  [ 1, 2, 3, 3 ], // RECURSIVE dupe, because [1,2,3,3] reduces to [1,2,3] which is already mapped   
  [ 1, 2, 3, undefined ], // dupe because undefined is unprocessed
  [ 1, 2, 3, undefined, , 3 ], // dupe because undefined and empty slot are unprocessed
  // and 1,2,3,3 is recursively unduped
  
  [ 1, 2, 3, null ] // not a dupe because null is valid
];

/* test it out */

console.dir(
  undupe(test)
);

/*
0: ""
1: NaN
2: null
3: undefined
4: 1
5: "a"
6: 1554702108396
7: 1554702108397 // only see this on a "slower" runtime
8: Object { name: "a", value: "b" }
9: Object { name: "a", value: "c" }
10: Object { name: "b", value: "c" }
11: { object: Object { name: "a", value: "b" } }
12: { object: Object { name: "a", value: "c" } }
13: { object: Object { name: "b", value: "c" } }
14: Array(3) [ 1, 2, 3 ]
15: Array(4) [ 1, 2, 3, null ]
*/

// Dubious removal of matches from object pretending to be an array.
// Notice that -1 isn't picked up as an index at all
var o = { 0: ['a'], 1: ['b'], 2: ['c'], 3: ['a'], '-1': '-1' }
console.log(
  undupe(o),
  JSON.stringify(undupe(o), null, 2)
);

/*
[
  [ "a" ],
  [ "b" ],
  [ "c" ]
]
*/