Apple Image Capture Deduper

dedupe.js

// Deduper for Apple Image Capture as it doesn't compare files and duplicates on import
// Is currently prone to crashing on certain (large?) files with malloc error
// drop in dir that images are imported into

const fs = require( "fs" );
const crypto = require( "crypto" );

let names = {};

function init() {

	fs.readdirSync( "./" ).forEach( ( file ) => {

		console.log( "file", file );

		let stripped_file = stripName( file );

		if ( ! names[ stripped_file ] ) {
			names[ stripped_file ] = [];
		}

		names[ stripped_file ].push( file );

	});

	Object.keys( names ).filter( ( key ) => {
		const name_list = names[ key ];
		return ( name_list.length > 1 );
	}).forEach( ( key ) => {

		const name_list = names[ key ];

		console.log( "name_list", name_list )

		let md5s = {};
		let skip = false;

		name_list.forEach( ( file ) => {

			if ( skip === true ) {
				console.log( "skipping b" )
				return;
			}
      
			const stats = fs.statSync( "./" + file );
			console.log( "[" + file + "]" )

			let md5;
			try {
				md5 = getMD5( fs.readFileSync( "./" + file, { encoding: "utf8", flag: "r" } ) );
			} catch ( error ) {
				const stats = fs.statSync( "./" + file );
				console.log( "stats", stats )
				skip = true;
				console.log( "skipping a" )
				return;
			}

			if ( ! md5s[ md5 ] ) {
				md5s[ md5 ] = [];
			}

			md5s[ md5 ].push( file );

		});

		if ( skip === true ) {
			console.log( "skipping c" )
			return;
		}

		Object.keys( md5s ).forEach( ( md5 ) => {

			if ( md5s[ md5 ].length > 1 ) {

				let unlinks = [];
				let keeps = [];
				md5s[ md5 ].forEach( ( file ) => {
					if ( / \d+/.test( file ) ) {
						unlinks.push( file );
					}
					else {
						keeps.push( file );
					}
				});

				console.log( "keeps", keeps )
				console.log( "unlinks", unlinks )

				if ( ! keeps.length ) {
					console.log( "empty keeps" )
					if ( unlinks.length ) {
						keeps = unlinks.splice( 0, 1 );
					}

					console.log( "keeps 2", keeps )
					console.log( "unlinks 2", unlinks )
				}

				if ( unlinks.length ) {
					unlinks.forEach( ( file ) => {
						console.log( "unlinking", file )
						fs.unlinkSync( "./" + file );
					});
				}

			}

		});

	});

}

function stripName( name ) {
	return name.replace( /\s\d+/, "" );
}

function getMD5( content ) {
	return crypto.createHash( "md5" ).update( content ).digest( "hex" );
}

init();