Last active
November 9, 2016 18:16
-
-
Save MAOliver/e26a92027e5a6192791dda06baca00b0 to your computer and use it in GitHub Desktop.
Something I created to allow me to rapidly iterate on my data model in Excel and re-seed the database as I adjust it. Uses powershell and some assumptions on the format of the data to accomplish it.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Function Export-WorksheetToCSV | |
{ | |
<# | |
.Synopsis | |
This cmdlet converts .xlsx files to .csv. | |
.Description | |
This cmdlet uses COM to open XLSX and then saves every tab as a separate CSV file. This only works if the | |
individual tab CAN be converted to CSV. | |
Ignores tabs named "_Ignore". | |
.Parameter Root | |
The root directory where the files should be opened / saved to. | |
.Parameter FileBaseName | |
The base filename without the extension. | |
.Example | |
Export-WorksheetToCSV -Root "C:\ExcelFileDirectory\" -BaseFileName "TheFileToConvert" | |
Opens the file at C:\ExcelFileDirectory\TheFileToConvert.xlsx and saves every tab as C:\ExcelFileDirectory\TheFileToConvert_(TabName).csv | |
#> | |
[CmdletBinding()] | |
Param( | |
[parameter(Mandatory=$true)] | |
[string]$Source, | |
[parameter(Mandatory=$true)] | |
[string]$FileBaseName | |
) | |
$excelFile = Join-Path $Source "$FileBaseName.xlsx" | |
$E = New-Object -ComObject Excel.Application | |
$E.Visible = $false | |
$E.DisplayAlerts = $false | |
$wb = $E.Workbooks.Open($excelFile) | |
foreach ($ws in $wb.Worksheets) | |
{ | |
if($ws.Name -notmatch "_Ignore") { | |
$n = $FileBaseName + "_" + $ws.Name | |
$savePath = Join-Path $root "$n.csv" | |
$ws.SaveAs($savePath, 6) | |
} | |
} | |
$E.Quit() | |
} | |
Function Export-CSVToSQL | |
{ | |
<# | |
.Synopsis | |
This cmdlet converts CSV files to Database Post-Deploy SQL Scripts. | |
.Description | |
This cmdlet converts CSV files named like (schema).Data_*.csv to $schema.Seed.sql post-deploy scripts. | |
CSV data must contain a header formatted like, e.g.: | |
"Id","Name","Abbrev","_SeedScript" | |
The Id field must be the unique id for the table. Columns named starting with "_" will be ignored when building | |
the merge into script. The column named "_SeedScript" will be used to populate the seed data (required for now...). | |
.Parameter Source | |
Required. The source directory in which the CSV files reside. | |
.Parameter Destination | |
Required. The destination for the (schema).Seed.sql files. | |
.Parameter Schemas | |
Required. A string array of schemas. Used to filter files and aggregate post-deploy scripts. | |
.Example | |
Export-CSVToSQL -Root "C:\CsvFileDirectory\" -Destination "C:\TFS\Solution1\DatabaseProj\PostDeploy\" -Schemas "dbo","products" | |
Opens the CSVs at C:\CsvFileDirectory\dbo.Data_*.csv and converts each (tab) into a table seed script. All "dbo" scripts will | |
be aggregated at C:\TFS\Solution1\DatabaseProj\PostDeploy\dbo.Seed.sql. The same thing will then occur for | |
C:\CsvFileDirectory\products.Data_*.csv which will be saved to C:\TFS\Solution1\DatabaseProj\PostDeploy\products.Seed.sql. | |
#> | |
[CmdletBinding()] | |
Param( | |
[parameter(Mandatory=$true)] | |
[string]$Source, | |
[parameter(Mandatory=$true)] | |
[string]$Destination, | |
[parameter(Mandatory=$true)] | |
[string[]]$Schemas | |
) | |
foreach($schema in $Schemas) | |
{ | |
$files = Get-ChildItem $Source -Filter "$schema.Data_*.csv" | |
$seedFileName = "$schema.Seed.sql" | |
$seedFilePath = Join-Path "$Source" "$seedFileName" | |
New-Item -path $Source -Name $seedFileName -Value 'USE [$(DatabaseName)]' -ItemType file -force | |
Add-Content -Path $seedFilePath -Value "`r`nGO" | |
Add-Content -Path $seedFilePath -Value "`r`nexec sp_msforeachtable 'ALTER TABLE ? NOCHECK CONSTRAINT all'`r`nGO" | |
#Add-Content $source+$seedFileName "" | |
foreach ($file in $files) | |
{ | |
$csv = Import-Csv $file.FullName | |
$linesInFile = 0 | |
$baseName = $file.BaseName.Split("_")[1] | |
$content = @" | |
SET IDENTITY_INSERT [$schema].[$baseName] ON | |
GO | |
MERGE INTO [$schema].[$baseName] AS Target | |
USING (VALUES | |
"@ | |
foreach($row in $csv) | |
{ | |
$content+="`r`n "+$row._SeedScript | |
$linesInFile++; | |
} | |
$content = $content.TrimEnd(",")+"`r`n)`r`n" | |
$temp = ($csv[0].psobject.Properties | ? { $_.Name -notlike "_*" -and $_.Name -ne "Id" -and $_.MemberType -eq "NoteProperty" } | Select -ExpandProperty Name) | |
$updateSetParams = New-Object object[] $temp.Length | |
for($i=0; $i -lt $temp.Length; $i++) | |
{ | |
$updateSetParams[$i] = ($temp[$i]) +" = Source."+($temp[$i]) | |
} | |
$commaSeparatedHeaders = "Id, "+($temp -join ", ") | |
$content += "AS SOURCE ($commaSeparatedHeaders)`r`n" | |
$content += "ON Target.Id = Source.Id`r`n" | |
$content += "--update matched rows`r`n" | |
$content += "WHEN MATCHED THEN`r`n" | |
$content += "UPDATE SET " + ($updateSetParams -join ", ")+"`r`n" | |
$content += "--insert new rows`r`n" | |
$content += "WHEN NOT MATCHED BY TARGET THEN`r`n" | |
$content += "INSERT ($commaSeparatedHeaders) VALUES ($commaSeparatedHeaders)`r`n" | |
$content += "--delete rows that are in the target but not the source`r`n" | |
$content += "WHEN NOT MATCHED BY SOURCE THEN`r`n" | |
$content += "DELETE;`r`n`r`n" | |
$content += "DBCC CHECKIDENT ('$schema.$baseName', RESEED, $linesInFile);`r`n`r`n" | |
$content += "SET IDENTITY_INSERT [$schema].[$baseName] OFF`r`nGO" | |
Add-Content -Path $seedFilePath -Value $content | |
} | |
Add-Content -Path $seedFilePath -Value "exec sp_msforeachtable 'ALTER TABLE ? WITH CHECK CHECK CONSTRAINT all'`r`nGO" | |
Copy-Item -Path $seedFilePath -Destination $Destination | |
} | |
} | |
Function Export-ExcelToSql | |
{ | |
<# | |
.Synopsis | |
This cmdlet converts Excel files to Database Post-Deploy SQL Scripts. | |
.Description | |
This cmdlet converts Excel files to CSV via Export-WorksheetToCSV and then to post-deploy sql scripts via Export-CSVToSQL. | |
.Parameter Source | |
Required. The source directory in which the excel files reside. | |
.Parameter Destination | |
Required. The destination for the (schema).Seed.sql files. | |
.Parameter Schemas | |
Required. A string array of schemas. Used to filter files and aggregate post-deploy scripts. | |
.Example | |
Export-CSVToSQL -Source "C:\CsvFileDirectory\" -Destination "C:\TFS\Solution1\DatabaseProj\PostDeploy\" -Schemas "dbo","products" | |
Opens the CSVs at C:\CsvFileDirectory\dbo.Data_*.csv and converts each (tab) into a table seed script. All "dbo" scripts will | |
be aggregated at C:\TFS\Solution1\DatabaseProj\PostDeploy\dbo.Seed.sql. The same thing will then occur for | |
C:\CsvFileDirectory\products.Data_*.csv which will be saved to C:\TFS\Solution1\DatabaseProj\PostDeploy\products.Seed.sql. | |
#> | |
[CmdletBinding()] | |
Param( | |
[parameter(Mandatory=$true)] | |
[string]$Source, | |
[parameter(Mandatory=$true)] | |
[string]$Destination, | |
[parameter(Mandatory=$true)] | |
[string[]]$Schemas) | |
$allSchemaCsvs = $schemas | % { Join-Path $Source "\$_.*.csv" } | |
$allSQL = $schemas | % { Join-Path $Source "\$_.Seed.sql" } | |
foreach ($schema in $Schemas) | |
{ | |
#Remove-Item $allSchemaCsvs | |
$files = Get-ChildItem $Source -Filter "$schema.Data.xlsx"#"*.Data.xlsx" | |
foreach ($e in $files) | |
{ | |
Export-WorksheetToCSV -root $Source -excelFileName $e.BaseName | |
} | |
Export-CSVToSQL -source $Source -destination $Destination -schemas $schema | |
} | |
$allSchemaCsvs | Remove-Item | |
$allSQL | Remove-Item | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment