Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
A PowerShell script which goes through the a Pwned Password list (available from https://haveibeenpwned.com/Passwords) and produces multiple smaller 'partition' files which contain the SHA-1 passwords specific to that partition
# A PowerShell script which goes through the a Pwned Password list (available from https://haveibeenpwned.com/Passwords)
# and produces multiple smaller 'partition' files which contain the SHA-1 passwords specific to that partition
#
# First released by Stuart Clarkson via https://gist.github.com/Tras2/66f80f2af1f1e125b1e4cc9ad3c12e3c
# Updated for increased speed by Christian Arnold via https://gist.github.com/meilon/d034ccf366d28343bf47ef59891acbaa
Param (
# The filename of the Pwned password source file
[Parameter(Mandatory=$true)]
[ValidateScript({Test-Path -Path $_})]
$InputFilename,
# Output directory is where the output partition .txt files will end up here
[Parameter(Mandatory=$true)]
[ValidateScript({Test-Path -Path $_})]
$OutputDirectory,
# Partition width is the number of characters from the SHA-1 hash that are used to determine a partition
[int]$PartitionWidth = 3
)
# Calculate the number of partitions that will be processed (used for creating the partition files & progress bar purposes)
$NumberOfPartitions = [int32]("0x"+("F".PadLeft($PartitionWidth, "F")))
# initialise a stopwatch (used for progress bar time remaining calculations)
$StopWatch = New-Object -TypeName System.Diagnostics.Stopwatch
# Array of StreamWriters for the paritioned files
$writers = @{}
# Pre-create the partition files
(0..$NumberOfPartitions) | Foreach-Object {
# Create the partition name (for a $PartitionWidth of 3, this generates '000' to 'fff')
$Partition = ([Convert]::ToString($_, 16))
$Partition = $Partition.PadLeft($PartitionWidth,"0")
# Test if the partition file exists
if (-not (Test-Path -path "$OutputDirectory\$Partition.txt"))
{
# create an empty partition file
New-Item -Path "$OutputDirectory\$Partition.txt" -ItemType File
}
# Initialize FileWriter for the partition file
$writers[$Partition] = [System.io.File]::Open("$OutputDirectory\$Partition.txt", "Append", "Write", "None")
}
# Preset the $Partition variable to be the first partition we'll be working with
$Partition = "0".PadLeft($PartitionWidth, "0")
# As the input file is likely to be large, I'm not using Get-Content (causes massive memory usage)
# Using the .NET System.IO.File class to create a file handle for reading line-by-line later
$reader = [System.IO.File]::OpenText($InputFilename)
# Pre-set variables used for progress bar purposes
$TimeElapsed = 0 # Total time the script has been running
$PartitionsDone = 0 # A count of how many partitions have been completed
$lineNumber = 0 # What line number we're at
$AverageTimePerPartition = -1 # On average, how many seconds does a partition take
$SecondsRemaining = -1 # Estimate of the number of seconds remaining
$enc = [System.Text.Encoding]::UTF8 # Set encoder
# Write the initial progress bar
Write-Progress -Activity "Parsing $InputFilename" -Status "Processing partion $Partition (line $lineNumber)" -PercentComplete (($PartitionsDone/$NumberOfPartitions)*100) -SecondsRemaining $SecondsRemaining
# Start the stopwatch
$StopWatch.start()
# Loop round each line in the inputfile until we get to the end
while($null -ne ($line = $reader.ReadLine())) {
# Increment the line number counter
$lineNumber++
# If the SHA-1 hash isn't for the current partition then we'll move to the next one
if ($line.Substring(0,$PartitionWidth) -ne $Partition)
{
# Stop the stopwatch
$StopWatch.Stop()
# We've completed a partiton, so increment the counter
$PartitionsDone++
# Add the stopwatch elapsed seconds to the total time
$TimeElapsed+=$StopWatch.Elapsed.TotalSeconds
# Calculate the Average time per partition
$AverageTimePerPartition = $TimeElapsed / $PartitionsDone
# Update the estimate for the time remaining
$SecondsRemaining = ($NumberOfPartitions - $PartitionsDone) * $AverageTimePerPartition
# Flush the file, so the OS can write some stuff to disk
$writers[$Partition].Flush()
# Update the $Parition variable to be the new partition
$Partition = $line.Substring(0,$PartitionWidth)
# Update the progress bar
Write-Progress -Activity "Parsing $InputFilename" -Status "Processing partion $Partition (line $lineNumber)" -PercentComplete (($PartitionsDone/$NumberOfPartitions)*100) -SecondsRemaining $SecondsRemaining
# Reset and restart the stopwatch
$StopWatch.Reset()
$StopWatch.start()
}
# This will update the progress bar every 10000 lines so at least it gives an indication that the script is still working
if ($lineNumber % 10000 -eq 0)
{
Write-Progress -Activity "Parsing $InputFilename" -Status "Processing partion $Partition (line $lineNumber)" -PercentComplete (($PartitionsDone/$NumberOfPartitions)*100) -SecondsRemaining $SecondsRemaining
}
# Convert string to byte with new line (FileStream doesn't have WriteLine
$byte = $enc.GetBytes($line + "`r`n")
# Append the line to the partition file
$writers[$partition].Write($byte,0,$byte.length)
}
# Cleaning up FileStreams
(0..$NumberOfPartitions) | Foreach-Object {
# Create the partition name (for a $PartitionWidth of 3, this generates '000' to 'fff')
$Partition = ([Convert]::ToString($_, 16))
$Partition = $Partition.PadLeft($PartitionWidth,"0")
# Final flush and closing the handle
$writers[$Partition].Flush()
$writers[$Partition].Close()
}
# Create a timespan based on the counter $TimeElapsed which is the number of seconds the script has ran for
$Runtime = New-TimeSpan -Seconds $TimeElapsed
# Writes "Process file.txt containing 12345 lines in 1.00:50:00" for example
Write-Host "Processed ${InputFilename}"
Write-Host "${linenumber}" -ForegroundColor Green -NoNewline
Write-Host "lines were processed in " -NoNewline
Write-host $Runtime.ToString() -ForegroundColor Green
# Writes "4096 partition files were created or updated in c:\temp" for example
Write-Host (${NumberOfPartitions} + 1) -ForegroundColor Green -NoNewline
Write-Host " partition files were created or updated in " -NoNewLine
Write-Host "'${OutputDirectory}'"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.