Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Print groups of files that are duplicates.
#!/usr/bin/perl -w
use Getopt::Long;
use Digest::MD5;
#use diagnostics;
BEGIN {
my $help;
my $xml;
our $minimum = 0;
our $header = '';
our $footer = '';
our $group_open_a = 'duplicate files (md5: ';
our $group_open_b = ')';
our $group_close = '';
our $file_open = '';
our $file_close = '';
GetOptions(
'help' => \$help,
'h' => \$help,
'minimum=i' => \$minimum,
'm=i' => \$minimum,
'xml' => \$xml,
'x' => \$xml
);
if (defined $help) {
print <<'EOT';
Usage: group-dupes.pl [options] file1 file2 ...
Print groups of files that are duplicates.
If no files are passed on the command line, files are read from stdin (tip: feed with find).
Anything that is not a regular file is ignored.
Options:
-m, --minimum minimum number of duplicates to print, default is 1
-x, --xml print as xml
-h, --help print this message
EOT
exit;
}
if (defined $xml) {
$header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<duplicates>\n";
$footer = "</duplicates>\n";
$group_open_a = '<group md5="'; $group_open_b = '">';
$group_close = "</group>\n";
$file_open = '<file>';
$file_close = '</file>';
}
my %md5_hash=();
}
if ($#ARGV > -1) {
while ($#ARGV > -1) { # process every file on the command line
group_dupe($ARGV[0]);
shift;
}
} else {
while (<>) { # read from standard in if there's nothing on the command line
chomp;
group_dupe($_);
}
}
END {
print $header;
foreach $md5 (keys %md5_hash) {
my @files = @{$md5_hash{$md5}};
if ($#files > $minimum) {
print "$group_open_a$md5$group_open_b\n";
for $file (@files) {
print "\t$file_open$file$file_close\n";
}
print $group_close;
}
}
print $footer;
}
sub group_dupe {
my ($path) = @_;
(! -f $path) && return; # skip anything that's not a regular file
if (-r $path) {
open(FILE, '<', $path) or die "Can't open $path: $!";
binmode(FILE);
if (my $digest = Digest::MD5->new->addfile(*FILE)->hexdigest, $path) {
push(@{$md5_hash{$digest}}, $path);
} else {
print STDERR "Can't make md5 digest of file $path: $!\n";
}
} else {
print STDERR "Can't read file: $path ... skipping\n";
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment