A PowerShell script which goes through the a Pwned Password list (available from https://haveibeenpwned.com/Passwords) and produces multiple smaller 'partition' files which contain the SHA-1 passwords specific to that partition

ParsePwnedPasswordFile.ps1

# A PowerShell script which goes through the a Pwned Password list (available from https://haveibeenpwned.com/Passwords)
# and produces multiple smaller 'partition' files which contain the SHA-1 passwords specific to that partition
#
# First released by Stuart Clarkson via https://gist.github.com/Tras2/66f80f2af1f1e125b1e4cc9ad3c12e3c
# Updated for increased speed by Christian Arnold via https://gist.github.com/meilon/d034ccf366d28343bf47ef59891acbaa
Param (
    # The filename of the Pwned password source file
    [Parameter(Mandatory=$true)]
    [ValidateScript({Test-Path -Path $_})]
    $InputFilename,

    # Output directory is where the output partition .txt files will end up here
    [Parameter(Mandatory=$true)]
    [ValidateScript({Test-Path -Path $_})]
    $OutputDirectory,

    # Partition width is the number of characters from the SHA-1 hash that are used to determine a partition
    [int]$PartitionWidth = 3
)

# Calculate the number of partitions that will be processed (used for creating the partition files & progress bar purposes)
$NumberOfPartitions = [int32]("0x"+("F".PadLeft($PartitionWidth, "F")))

# initialise a stopwatch (used for progress bar time remaining calculations)
$StopWatch = New-Object -TypeName System.Diagnostics.Stopwatch

# Array of StreamWriters for the paritioned files
$writers = @{}

# Pre-create the partition files
(0..$NumberOfPartitions) | Foreach-Object {

    # Create the partition name (for a $PartitionWidth of 3, this generates '000' to 'fff')
    $Partition = ([Convert]::ToString($_, 16))
    $Partition = $Partition.PadLeft($PartitionWidth,"0")
    
    # Test if the partition file exists
    if (-not (Test-Path -path "$OutputDirectory\$Partition.txt"))
    {
        # create an empty partition file
        New-Item -Path "$OutputDirectory\$Partition.txt" -ItemType File
    }
    
    # Initialize FileWriter for the partition file
    $writers[$Partition] = [System.io.File]::Open("$OutputDirectory\$Partition.txt", "Append", "Write", "None")
}

# Preset the $Partition variable to be the first partition we'll be working with
$Partition = "0".PadLeft($PartitionWidth, "0")

# As the input file is likely to be large, I'm not using Get-Content (causes massive memory usage)
# Using the .NET System.IO.File class to create a file handle for reading line-by-line later
$reader = [System.IO.File]::OpenText($InputFilename)

# Pre-set variables used for progress bar purposes
$TimeElapsed = 0                # Total time the script has been running
$PartitionsDone = 0             # A count of how many partitions have been completed
$lineNumber =  0                # What line number we're at
$AverageTimePerPartition = -1   # On average, how many seconds does a partition take
$SecondsRemaining = -1          # Estimate of the number of seconds remaining
$enc = [System.Text.Encoding]::UTF8 # Set encoder

# Write the initial progress bar
Write-Progress -Activity "Parsing $InputFilename" -Status "Processing partion $Partition (line $lineNumber)" -PercentComplete (($PartitionsDone/$NumberOfPartitions)*100) -SecondsRemaining $SecondsRemaining

# Start the stopwatch
$StopWatch.start()

# Loop round each line in the inputfile until we get to the end
while($null -ne ($line = $reader.ReadLine())) {

    # Increment the line number counter
    $lineNumber++
    
    # If the SHA-1 hash isn't for the current partition then we'll move to the next one
    if ($line.Substring(0,$PartitionWidth) -ne $Partition)
    {
        # Stop the stopwatch
        $StopWatch.Stop()

        # We've completed a partiton, so increment the counter
        $PartitionsDone++
        
        # Add the stopwatch elapsed seconds to the total time
        $TimeElapsed+=$StopWatch.Elapsed.TotalSeconds

        # Calculate the Average time per partition
        $AverageTimePerPartition = $TimeElapsed / $PartitionsDone

        # Update the estimate for the time remaining
        $SecondsRemaining = ($NumberOfPartitions - $PartitionsDone) * $AverageTimePerPartition
    
        # Flush the file, so the OS can write some stuff to disk
        $writers[$Partition].Flush()
                
        # Update the $Parition variable to be the new partition
        $Partition = $line.Substring(0,$PartitionWidth)

        # Update the progress bar
        Write-Progress -Activity "Parsing $InputFilename" -Status "Processing partion $Partition (line $lineNumber)" -PercentComplete (($PartitionsDone/$NumberOfPartitions)*100) -SecondsRemaining $SecondsRemaining

        # Reset and restart the stopwatch
        $StopWatch.Reset()
        $StopWatch.start()
    }

    # This will update the progress bar every 10000 lines so at least it gives an indication that the script is still working
    if ($lineNumber % 10000 -eq 0)
    {
        Write-Progress -Activity "Parsing $InputFilename" -Status "Processing partion $Partition (line $lineNumber)" -PercentComplete (($PartitionsDone/$NumberOfPartitions)*100) -SecondsRemaining $SecondsRemaining
    }
    
    # Convert string to byte with new line (FileStream doesn't have WriteLine
    $byte = $enc.GetBytes($line + "`r`n") 
    
    # Append the line to the partition file
    $writers[$partition].Write($byte,0,$byte.length)
}

# Cleaning up FileStreams
(0..$NumberOfPartitions) | Foreach-Object {

    # Create the partition name (for a $PartitionWidth of 3, this generates '000' to 'fff')
    $Partition = ([Convert]::ToString($_, 16))
    $Partition = $Partition.PadLeft($PartitionWidth,"0")
    
    # Final flush and closing the handle
    $writers[$Partition].Flush()
    $writers[$Partition].Close()
    
}

# Create a timespan based on the counter $TimeElapsed which is the number of seconds the script has ran for
$Runtime = New-TimeSpan -Seconds $TimeElapsed

# Writes "Process file.txt containing 12345 lines in 1.00:50:00" for example
Write-Host "Processed ${InputFilename}"
Write-Host "${linenumber}" -ForegroundColor Green -NoNewline
Write-Host "lines were processed in " -NoNewline
Write-host $Runtime.ToString() -ForegroundColor Green 

# Writes "4096 partition files were created or updated in c:\temp" for example
Write-Host (${NumberOfPartitions} + 1) -ForegroundColor Green -NoNewline
Write-Host " partition files were created or updated in " -NoNewLine
Write-Host "'${OutputDirectory}'"