Skip to content

Instantly share code, notes, and snippets.

@peaeater
Created March 12, 2024 00:27
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save peaeater/1b8c6e402d6b8cb71bd1b2527099e067 to your computer and use it in GitHub Desktop.
Save peaeater/1b8c6e402d6b8cb71bd1b2527099e067 to your computer and use it in GitHub Desktop.
Split csv files with or without header rows, multiline columns with newlines ok.
<#
Split csv files with or without header rows, multiline columns with newlines ok.
#>
param (
[string]$in,
[string]$outdir = [System.IO.Path]::GetDirectoryName($in),
[int]$count = 1000,
[string]$delimiter = ',',
[string[]]$header = @()
)
function createOutPath($in, $outdir, $fileIndex) {
$name = [System.IO.Path]::GetFileNameWithoutExtension($in)
$ext = [System.IO.Path]::GetExtension($in)
# append file index to original file name in outdir
return join-path $outdir "$name-$fileIndex$ext"
}
# create out dir if it doesn't exist
if (!(test-path $outdir)) {
mkdir $outdir | Out-Null
}
$fileIndex = 1
$rowIndex = 0
$file = @()
if ($null -ne $header -and $header.count -ne 0) {
# import with supplied header (file does not have its own header)
Import-Csv -Path $in -Delimiter $delimiter -Header $header | ForEach-Object {
$file += $_
$rowIndex++
$out = createOutPath $in $outdir $fileIndex
if ($rowIndex -eq $count) {
$file | Export-Csv -Path $out -Encoding utf8NoBOM -NoTypeInformation -UseQuotes AsNeeded
$fileIndex++
$rowIndex = 0
$file = @()
write-host $out
}
}
}
else {
# import and assume first row is header
Import-Csv -Path $in -Delimiter $delimiter | ForEach-Object {
$file += $_
$rowIndex++
$out = createOutPath $in $outdir $fileIndex
if ($rowIndex -eq $count) {
$file | Export-Csv -Path $out -Encoding utf8NoBOM -NoTypeInformation -UseQuotes AsNeeded
$fileIndex++
$rowIndex = 0
$file = @()
write-host $out
}
}
}
if ($file.Count -gt 0) {
$out = createOutPath $in $outdir $fileIndex
$file | Export-Csv -Path $out -Encoding utf8NoBOM -NoTypeInformation -UseQuotes AsNeeded
write-host $out
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment