Delete duplicates from mongodb

removeDuplicateFromMongoDB.js

`dropDups: true` option is not available in 3.0. 

I have solution with aggregation framework for collecting duplicates and then removing in one go.

It might be somewhat slower than system level "index" changes. But it is good by considering way you want to remove duplicate documents. 

a. Remove all documents in one go

    var duplicates = [];
    
    db.collectionName.aggregate([
      { $match: { 
        name: { "$ne": '' }  // discard selection criteria
      }},
      { $group: { 
        _id: { name: "$name"}, // can be grouped on multiple properties 
        dups: { "$addToSet": "$_id" }, 
        count: { "$sum": 1 } 
      }}, 
      { $match: { 
        count: { "$gt": 1 }    // Duplicates considered as count greater than one
      }}
    ])               // You can display result until this and check duplicates 
    // If your result getting response in "result" then use else don't use ".result" in query    
    .result          
    .forEach(function(doc) {
        doc.dups.shift();      // First element skipped for deleting
        doc.dups.forEach( function(dupId){ 
            duplicates.push(dupId);   // Getting all duplicate ids
            }
        )    
    })

    // If you want to Check all "_id" which you are deleting else print statement not needed
    printjson(duplicates);     

    // Remove all duplicates in one go    
    db.collectionName.remove({_id:{$in:duplicates}})  
    

b. You can delete documents one by one.


    db.collectionName.aggregate([
      // discard selection criteria, You can remove "$match" section if you want
      { $match: { 
        source_references.key: { "$ne": '' }  
      }},
      { $group: { 
        _id: { source_references.key: "$source_references.key"}, // can be grouped on multiple properties 
        dups: { "$addToSet": "$_id" }, 
        count: { "$sum": 1 } 
      }}, 
      { $match: { 
        count: { "$gt": 1 }    // Duplicates considered as count greater than one
      }}
    ])               // You can display result until this and check duplicates 
    // If your result getting response in "result" then use else don't use ".result" in query    
    .result          
    .forEach(function(doc) {
        doc.dups.shift();      // First element skipped for deleting
        db.collectionName.remove({_id : {$in: doc.dups }});  // Delete remaining duplicates
    })

From stackoverflow URL: http://stackoverflow.com/a/33364353