hashcat-rules-uniq.pl

# use Digest::xxHash qw/xxhash64/; # many collisions
use Digest::SHA qw(sha512_hex);
use File::Temp qw/tempfile/;
use Data::Dumper;
use strict;
use warnings;
use Time::HiRes qw(time);

my $wordlist = 'C:\pentest\lab\temp\hashcat-wlist.txt';
# my $wordlist = 'C:\pentest\lab\temp\test.txt';
my $rule_file = 'C:\pentest\tools\hashcat\rules\d3ad0ne.rule';
my $uniq_rules_out = 'uniq-rules-new.txt';
my %rules_hash;
my (undef, $temp_filename) = tempfile();
my $rules_count = 0;
my $tick = time;

open my $rules, $rule_file or die $!;
while(my $rule = <$rules>)
{
	chomp $rule;
	next if substr($rule, 0, 1) eq '#'; # skip comments in rule file
	print "Checking $rule\n";
	$rules_count++;
	save_rule($temp_filename, $rule);
	my $content = `C:/pentest/tools/hashcat/hashcat-cli64.exe -a 0 -r $temp_filename --stdout $wordlist`;
	$rules_hash{$rule} = sha512_hex($content);
	# $rules_hash{$rule} = xxhash64($content, SEED);
}
close $rules;

print "Checked $rules_count rules in ". stime(time - $tick) ."s\n";

my %uniq_rules = reverse %rules_hash;
my @uniq_rules = values %uniq_rules;
my $uniq_rules_count = scalar(@uniq_rules);

open my $out, '>', $uniq_rules_out or die $!;
foreach my $rule (@uniq_rules){
	print $out $rule,"\n";
}
close $out;

my $uniq_percentage = int($uniq_rules_count * 1000 * 100 / $rules_count ) / 1000;
print "Found $uniq_rules_count uniq rules from $rules_count total ($uniq_percentage%).\n";

my %seen;
foreach my $r_key (keys %rules_hash)
{
	if(!exists($seen{$rules_hash{$r_key}}))
	{
		$seen{$rules_hash{$r_key}} = $r_key;
	}else{
		print "Found duplicate $r_key with ". $seen{$rules_hash{$r_key}} ."\n";
	}
}

sub save_rule {
	my ($file, $rule) = @_;
	open my $out, '>', $file or die $!;
	print $out $rule;
	close $out;
}

sub stime {
	my $time = shift;
	return int($time*1000)/1000;
}

sub count_lines {
	my $file = shift;
	open my $in, '<', $file or die $!;
	while(<$in>){}
	my $count = $.;
	close $in;
	return $count;
}