Split big CSV quickly (multiple gigs < minute) based on column, powershell with embedded C#

readme.md

bigcsvsplitter.ps1

# C# syntax works under both pwsh core and legacy Windows Powershell which is capped at C# 5.0 and .Net Framework vs dotnet core
Add-Type @"

using System;
using System.IO;

public static class BigCsvSplitter
{

    public static void Run()
    {

        var lastmac = "";
        var fileCount = 0;
        var fileLineCount = 0;
        var totalLineCount = 0;

        using (var fs = new FileStream("$($pwd -replace "\\", "/")/prime.csv", FileMode.Open, FileAccess.Read, FileShare.None, 4096, FileOptions.SequentialScan))
        using (var sr = new StreamReader(fs))
        {
            FileStream outfs = null;
            // BufferedStream outbfs = null;
            StreamWriter outsw = null;

            sr.ReadLine(); //eat the first header line

            while (sr.Peek() >= 0)
            {
                var line = sr.ReadLine();
                var mac = (line == null ? null : line.Substring(0, line.IndexOf(",")));
                if (mac != lastmac)
                {
                    if (fileCount > 0) Console.WriteLine("   closing, total lines: " + fileLineCount);

                    lastmac = mac;
                    if (outsw != null)
                    {
                        outsw.Flush();
                        outsw.Close();
                        outsw.Dispose();
                        // outbfs?.Flush();
                        // outbfs?.Close();
                        // outbfs?.Dispose();
                        // outfs?.Flush();
                        // outfs?.Close();
                        outfs.Dispose();
                    }
                    outfs = new FileStream("$($pwd -replace "\\", "/")/" + mac + ".csv", FileMode.CreateNew, FileAccess.Write);
                    // outbfs = new BufferedStream(fs, 1024);
                    outsw = new StreamWriter(outfs);

                    fileCount++;
                    fileLineCount = 0;
                    Console.WriteLine("new file #" + fileCount + ": " + mac);
                }
                fileLineCount++;
                totalLineCount++;
                outsw.WriteLine(line);
                if (fileLineCount % 1000 == 0) Console.WriteLine("  file #: " + fileCount + ", processed lines: " + fileLineCount);
            }
            if (outsw != null)
            {
                outsw.Flush();
                outsw.Close();
                outsw.Dispose();
                // outbfs?.Flush();
                // outbfs?.Close();
                // outbfs?.Dispose();
                // outfs?.Flush();
                // outfs?.Close();
                outfs.Dispose();
            }
        }

        Console.WriteLine("DONE! " + fileCount + " total files, " + totalLineCount + " total lines.\r\n");
    }
}
"@

$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
[BigCsvSplitter]::Run()
$stopwatch.Elapsed

prime.csv