Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Migrating GitLab issues + history to Azure DevOps

Migrate-GitlabToDevops

Dependencies

You will need an access token from azure devops. Get yours here. This script will need contributor access to the project in question.

How to run

# populate user map in Map-IssueToWorkitem
# Load all functions in functions.ps1 into scope
# Setup global variables as per below
$DevOpsAccessToken = 'get yours here https://dev.azure.com/datainsight/_usersSettings/tokens'
$GitLabAccessToken = 'get yours here https://gitlab.com/profile/personal_access_tokens'
$DevOpsOrganization = 'this is the path segmment right after https://dev.azure.com/ you use to access your devops instance ie: myorg'
$DevOpsProjectId = 'get yours here https://dev.azure.com/$DevOpsOrganization/_apis/projects'
$GitlabContainerPath = 'either "groups/groupID" get yours here https://gitlab.com/api/v4/groups or "projects/projectId" get yours here https://gitlab.com/api/v4/projects;'
# Now run all the steps one by one in process.ps1
function Get-GitlabIssues {
[CmdletBinding()]
param (
[parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[System.String]
$AccessToken,
[ValidateNotNullOrEmpty()]
[System.String]
$ContainerPath)
Begin {
$issues = $null;
$result = $null;
$page = 1;
Write-Progress -Activity "Get-GitlabIssues" -Status "0 Complete:" -PercentComplete 0;
}
Process {
$total = (Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues_statistics" -Headers @{ 'Authorization' = "Bearer $AccessToken" }).statistics.counts.all;
while($result -ne 0) {
$result = Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" }
$issues += $result;
if($issues.Count -eq 0) {
$count = 1;
} else {
$count = $issues.Count;
}
$percent = $count/$total*100;
$page ++;
Write-Progress -Activity "Get-GitlabIssues" -Status "$percent% Complete:" -PercentComplete $percent;
}
}
End {
return $issues;
}
}
function Get-GitlabComments {
[CmdletBinding()]
param (
[parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[System.String]
$AccessToken,
[parameter(Mandatory=$true, ValueFromPipeline)]
[ValidateNotNullOrEmpty()]
[System.String]
$NotesLink)
Begin {
}
Process {
try{
Invoke-RestMethod -Method Get -Uri "$NotesLink`?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" };
} catch {
$issueIid = $NotesLink.Split("/") | Select-Object -Last 2 | Select-Object -First 1
Write-Error -Message "Failed to get comments for issue: $issueIid" -ErrorId 500 -TargetObject $issueIid -Category InvalidResult;
}
}
End {
}
}
function Get-GitLabAttachment {
[CmdletBinding(
SupportsShouldProcess,
ConfirmImpact="Medium"
)]
param (
[parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[System.String]
$AccessToken,
[parameter(Mandatory)]
[System.String]
$Root,
[parameter(Mandatory)]
[System.String]
$Issue,
[parameter(Mandatory)]
[System.String]
$File)
Begin {
}
Process {
$path = Join-Path -Path $Root -ChildPath $Issue | Join-Path -ChildPath "Attachments";
New-Item -Path $path -Type Directory -Force | Out-Null;
$path = $path | Join-Path -ChildPath ("~"+([string]::Join("~", ($File.Split("/") | Select-Object -Last 3))));
if(-Not (Test-Path -Path $path)) {
Try {
Invoke-WebRequest -OutFile $path -Uri $File -Headers @{ 'Authorization' = "Bearer $AccessToken" };
} Catch {
Write-Error -Message "Failed to get attachment: $File for issue: $Issue" -ErrorId 500 -TargetObject $Issue -Category InvalidResult;
}
}
}
End {
}
}
function Add-DevopsWorkItem {
[CmdletBinding(
SupportsShouldProcess=$true,
ConfirmImpact="Medium"
)]
param (
[parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[System.String]
$AccessToken,
[ValidateNotNullOrEmpty()]
[System.String]
$Organization,
[ValidateNotNullOrEmpty()]
[System.String]
$Project,
[ValidateNotNullOrEmpty()]
[System.String]
[ValidateSet("Epic", "Feature", "User Story", "Bug", "Issue", "Task")]
$Type,
[System.Int32]
$Id,
[parameter(Mandatory=$true,ValueFromPipeline)]
[ValidateNotNullOrEmpty()]
[System.String]
$Data,
[Switch]
$Test)
Begin {
$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workitems/`$$($Type)?api-version=5.1&bypassRules=true&suppressNotifications=true";
}
Process {
if ($PSCmdlet.ShouldProcess($Data)) {
Try {
if($Test) {
Invoke-RestMethod -Method Patch -Uri $uri+"&validateOnly=true" -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
} else {
Invoke-RestMethod -Method Patch -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
}
} Catch {
Write-Error -Message "Failed to add issue for issue: $($Id)" -ErrorId 500 -TargetObject $Data -Category InvalidResult;
}
}
}
End {
}
}
function Add-DevopsComment {
[CmdletBinding(
SupportsShouldProcess=$true,
ConfirmImpact="Medium"
)]
param (
[parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[System.String]
$AccessToken,
[ValidateNotNullOrEmpty()]
[System.String]
$Organization,
[ValidateNotNullOrEmpty()]
[System.String]
$Project,
[ValidateNotNullOrEmpty()]
[System.Int32]
$Id,
[parameter(Mandatory=$true,ValueFromPipeline)]
[ValidateNotNullOrEmpty()]
[System.String]
$Data)
Begin {
$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workItems/$Id/comments?api-version=5.1-preview.3";
}
Process {
if ($PSCmdlet.ShouldProcess($Data)) {
Try{
Invoke-RestMethod -Method Post -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json' -Body $Data
} Catch {
Write-Error -Message "Failed to add comments for issue: $($Id)" -ErrorId 500 -TargetObject $Id -Category InvalidResult;
}
}
}
End {
}
}
function Add-DevopsAttachment {
[CmdletBinding(
SupportsShouldProcess=$true,
ConfirmImpact="Medium"
)]
param (
[parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[System.String]
$AccessToken,
[ValidateNotNullOrEmpty()]
[System.String]
$Organization,
[ValidateNotNullOrEmpty()]
[System.String]
$Project,
[parameter(Mandatory=$true,ValueFromPipeline)]
[System.IO.FileInfo]
$File)
Begin {
$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)));
$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/attachments?api-version=5.0";
}
Process {
if ($PSCmdlet.ShouldProcess($FileName)) {
Try{
$image = [System.IO.File]::ReadAllBytes($File.FullName);
Invoke-RestMethod -Uri $uri+"&fileName=$([System.Web.HttpUtility]::UrlEncode($File.Name))" -Method Post -Headers @{ Authorization=("Basic {0}" -f $basicAuth) } -ContentType application/json -Body $image;
} Catch {
Write-Error -Message "Failed to add attachments for issue: $($File.FullName)" -ErrorId 500 -TargetObject $File -Category InvalidResult;
}
}
}
End {
}
}
function Convert-IssueToWorkItem {
[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
[CmdletBinding()]
[OutputType([string])]
param (
[parameter(Mandatory=$true)]
$RootIterationPath,
[parameter(Mandatory=$true,ValueFromPipeline)]
$Issue
)
Begin {
Import-Module MarkdownToHtml -Force;
$userMap = @{
#https://gitlab.com/api/v4/groups/mygroupid/members GitLabUserUserId="AzureDevOpsDisplayName",
123 ="John Doe";
};
}
Process {
if([String]::IsNullOrWhiteSpace($Issue.description)) {
$content = "`" `"";
} else {
$content = ((Convert-MarkdownToHTMLFragment -Markdown ($Issue.description | Format-BadCharacters)).HtmlFragment).Replace("", "₹").Replace("'", "'") | ConvertTo-Json;
}
$title = [System.Web.HttpUtility]::HtmlEncode(($Issue.title | Format-BadCharacters));
if($title.Length -gt 255){$title = $title.Substring(0,247);}
$author = $($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.author.id}).Value;
$assignee = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
$closedby = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.closed_by.id}).Value;
$tags = [String]::Join(";", $Issue.labels+$Issue.state);
if($Issue.milestone) {
$iteration = $RootIterationPath+"\\"+$Issue.milestone.title.Replace("?", "").Replace("&", "and");
} else {
$iteration = $RootIterationPath;
}
if($null -ne $Issue.assignee){
$user = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
$assignee = ",
{
`"op`": `"add`",
`"path`": `"/fields/System.AssignedTo`",
`"from`": null,
`"value`": `"$($user)`"
}"
} else {
$assignee = "";
}
$type = @("User Story", "Bug")[$Issue.labels.Contains("Bug")];
if($type -eq "Bug") {
$body = ",
{
`"op`": `"add`",
`"path`": `"/fields/Microsoft.VSTS.TCM.ReproSteps`",
`"from`": null,
`"value`": $content
}"
} else {
$body = ",
{
`"op`": `"add`",
`"path`": `"/fields/System.Description`",
`"from`": null,
`"value`": $content
}"
}
if($Issue.state -eq "closed"){
$closed = ",
{
`"op`": `"add`",
`"path`": `"/fields/Microsoft.VSTS.Common.ClosedBy`",
`"from`": null,
`"value`": `"$($closedby)`"
},
{
`"op`": `"add`",
`"path`": `"/fields/Microsoft.VSTS.Common.ClosedDate`",
`"from`": null,
`"value`": `"$($Issue.closed_at)`"
}";
} else {
$closed = "";
}
return "[
{
`"op`": `"add`",
`"path`": `"/fields/System.IterationPath`",
`"from`": null,
`"value`": `"$($iteration)`"
},
{
`"op`": `"add`",
`"path`": `"/fields/System.Title`",
`"from`": null,
`"value`": `"$($Issue.iid): $($title)`"
}$($body),
{
`"op`": `"add`",
`"path`": `"/fields/System.State`",
`"from`": null,
`"value`": `"$(@{"opened"="New"; "closed" = "New"}[$Issue.state])`"
},
{
`"op`": `"add`",
`"path`": `"/fields/System.Tags`",
`"from`": null,
`"value`": `"$($tags)`"
},
{
`"op`": `"add`",
`"path`": `"/fields/System.CreatedBy`",
`"from`": null,
`"value`": `"$($author)`"
},
{
`"op`": `"add`",
`"path`": `"/fields/System.CreatedDate`",
`"from`": null,
`"value`": `"$($Issue.created_at)`"
}$($assignee)$($closed)
]";
}
End {
}
}
function Convert-NoteToComment {
[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
[CmdletBinding()]
[OutputType([string])]
param(
[parameter(Mandatory,ValueFromPipeline)]
$Note
)
Begin {
Import-Module MarkdownToHtml -Force;
}
Process {
$header = "<p><b>$($Note.created_at.ToLocalTime().ToString('yyyy-MM-dd HH:mm:ss')) by $($Note.author.name)</b></p>";
if([String]::IsNullOrWhiteSpace($Note.body)) {
$content = "`" `"";
} else {
$content = ($header + (Convert-MarkdownToHTMLFragment -Markdown ($Note.body | Format-BadCharacters)).HtmlFragment).Replace("", "&#x20b9;") | ConvertTo-Json;
}
return "{ `"text`": $content }";
}
End {
}
}
function Format-BadCharacters {
[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
[CmdletBinding()]
[OutputType([string])]
param (
[parameter(
Mandatory,
ValueFromPipeline,
ValueFromPipelineByPropertyName,
Position = 0)]
[string] $Input
)
Process {
return $Input.Replace("`“", "`"").Replace("`”", "`"").Replace(" ", " ").Replace("ä", "a").Replace("ë", "e");
}
}
#Requires -PSEdition Core
#Requires -Modules @{ ModuleName="GitLab"; ModuleVersion="0.0"; }, @{ ModuleName="AzureDevops"; ModuleVersion="0.0"; }, @{ ModuleName="GitLabDevOpsMigration"; ModuleVersion="0.0"; }, @{ ModuleName="MarkdownToHtml"; ModuleVersion="2.0.0"}
throw "This is purely here so you dont accidentaly trigger this script, it is not designed to run end to end in one step, Please read the instructions"
### Setup working directory
$root = Join-Path -Path $(Get-Location) -ChildPath "MigrationWorkSpace"
New-Item -Path $root -Type Directory -Force;
### Step 1 - Download all GitLab Issues
$issues = Get-GitlabIssues -AccessToken $GitLabAccessToken -ContainerPath $GitlabContainerPath;
### Step 2 - Had to run this a few times so I always kept track of migrated ids in this file
$MigratedIssueSucessPath = $root | Join-Path -ChildPath "GtDIssueSucess.txt";
if(Test-Path -Path $MigratedIssueSucessPath) {
$migratedIssueIds = Get-Content $MigratedIssueSucessPath | ForEach-Object{[int64]::Parse($_)}
} else { $migratedIssueIds = @()}
$issuesToMigrate = $issues | Where-Object{ -not $migratedIssueIds.Contains($_.id)}
### Step 3 - Save all issues into a folder
$issuesToMigrate | ForEach-Object {
$folder = Join-Path -Path $root -ChildPath $_.id;
New-Item -Path $folder -Type Directory -Force | Out-Null;
$_ | ConvertTo-Json | Out-File -FilePath (Join-Path -Path $folder -ChildPath issue.json) -Encoding utf8 -Force};
### Step 4 - Get Unique Milestones and their status
### These need to be created as sprints in devops manualy, I only had to create about a dozen so was easy enough
$issuesToMigrate | Select-Object -ExpandProperty milestone | Select-Object title, state -Unique | Sort-Object state, name
### Step 5 - Download Comments
$total = $issuesToMigrate.Count;
$counter = 1;
$issuesToMigrate | Sort-Object id | ForEach-Object -Begin {
Write-Progress -Activity "Get-GitlabComments" -Status "0% Complete:" -PercentComplete 0;
} -Process {
$percent = $counter/$total*100;
$counter++;
Write-Progress -Activity "Get-GitlabComments" -Status "$percent% Complete: $($_.id)" -PercentComplete $percent;
$folder = Join-Path -Path $root -ChildPath $_.id;
Get-GitlabComments -AccessToken $GitLabAccessToken -NotesLink $_._links.notes |
ConvertTo-Json |
Out-File -FilePath (Join-Path -Path $folder -ChildPath notes.json) -Encoding utf8 -Force;
} -End {
Write-Progress -Activity "Get-GitlabComments" -Status "100% Complete:" -PercentComplete 100 -Completed
}
### Step 6 - Get all attachment links
$files = @();
#### Get attachment links from issues
Get-ChildItem -Path $root |
ForEach-Object { $id = $_.Name;
Get-ChildItem -Path $_.FullName -Filter issue.json |
Get-Content -Raw |
ConvertFrom-Json |
Where-Object { -not [String]::IsNullOrWhiteSpace($_.description) } |
ForEach-Object {
$urlParts = $_.web_url.Split("/");
$rootPath = [string]::Join("/", ($urlParts | Select-Object -First ($urlParts.Count-2)));
$_.description.Split("`n") |
ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} |
Where-Object { $_.Success } |
Select-Object -ExpandProperty Groups |
Where-Object { -Not $_.Value.StartsWith("]")} |
Select-Object -ExpandProperty Value |
ForEach-Object {
$files+=New-Object PSObject -Property @{
issue=$id;
file=$rootPath+$_;}
}
}
}
#### Get attachment links from comments
Get-ChildItem -Path $root |
ForEach-Object {
$id = $_.Name;
$urlParts = (Get-ChildItem -Path $_.FullName -Filter issue.json |
Get-Content -Raw |
ConvertFrom-Json).web_url.Split("/");
$rootPath = [string]::Join("/", ($urlParts | Select-Object -First ($urlParts.Count-2)));
Get-ChildItem -Path $_.FullName -Filter notes.json |
Get-Content -Raw |
ConvertFrom-Json |
Where-Object { -not [String]::IsNullOrWhiteSpace($_.body) } |
ForEach-Object { $_.body.Split("`n") } |
ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} |
Where-Object { $_.Success } |
Select-Object -ExpandProperty Groups |
Where-Object { -Not $_.Value.StartsWith("]")} |
Select-Object -ExpandProperty Value |
ForEach-Object {
$files+=New-Object PSObject -Property @{
issue=$id;
file=$rootPath+$_;} }
}
### Step 7 - Downlaod the attachments from gitlab NOTE: This can take while
$total = $files.Count;
$counter = 1;
$files | ForEach-Object -Begin {
Write-Progress -Activity "Get-GitlabAttachments" -Status "0% Complete:" -PercentComplete 0;
} -Process {
$percent = $counter/$total*100;
$counter++;
Write-Progress -Activity "Get-GitlabAttachments" -Status "$percent% Complete: $($_.issue)" -PercentComplete $percent;
Get-GitLabAttachment -AccessToken $GitLabAccessToken -Root $root -Issue $_.Issue -File $_.File;
} -End {
Write-Progress -Activity "Get-GitlabAttachments" -Status "100% Complete:" -PercentComplete 100 -Completed
}
### Step 8 - Upload and map attachments to offlined notes and comments
$total = (Get-ChildItem -Path $root).Count;
$counter = 1;
Get-ChildItem -Path $root |
ForEach-Object -Begin {
Write-Progress -Activity "Map-Attachments" -Status "0% Complete:" -PercentComplete 0;
} -Process {
$percent = $counter/$total*100;
$counter++;
Write-Progress -Activity "Map-Attachments" -Status "$percent% Complete: $($_.Name)" -PercentComplete $percent;
Push-Location -Path $_.FullName;
if(Test-Path -Path Attachments) {
$replace = Get-ChildItem -Path Attachments | ForEach-Object {
$name = $_ | Select-Object -Property @{Name="Name";Expression={$_.Name.Replace("~","/")}} | Select-Object -ExpandProperty Name;
$upld = Add-DevopsAttachment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -File $_;
Write-Verbose "$($name) $($upld.url)" -Verbose;
@{Old=$name; New=$upld.url};
}
$replace | ForEach-Object {
$old = $_.Old;
$new = $_.New;
Get-ChildItem -File | ForEach-Object {
$fileName = $_.FullName;
$content = Get-Content $_ | ForEach-Object {
$_.Replace($old, $new);
};
$content | Out-File -FilePath $fileName -Encoding utf8 -Force;
}
}
}
Pop-Location;
} -End {
Write-Progress -Activity "Map-Attachments" -Status "100% Complete:" -PercentComplete 100 -Completed
}
###
# At this stage I had to do some manual fixes sometimes, important at this stage to re-load all the issues and commments from the filesystem as they have the updated data after uploading all the attachments
###
$issuesToMigrate = Get-ChildItem -Path $root -Filter issue.json -Recurse | Get-Content -Raw | ConvertFrom-Json
### Step 9 - Issue + Comment Upload
$failed = @();
$total = $issuesToMigrate.Count;
$test=$true; # This will allow you to validate before import, set to false for actual import
$counter = 1;
#### The Sort helped me judge how far into the process I was
$issuesToMigrate | Sort-Object id |
ForEach-Object -Begin {
Write-Progress -Activity "Create Work Items and comments" -Status "0% Complete:" -PercentComplete 0;
} -Process {
$percent = $counter/$total*100;
$counter++;
$id = $_.id;
### Note the new bug template needs to map the content to another field other than description, think it was called reposteps or similar
$type = @("User Story", "Bug")[$_.labels.Contains("Bug")];
Write-Progress -Activity "Create Work Items and comments" -Status "$percent% Complete: $($id)" -PercentComplete $percent;
$_ | Convert-IssueToWorkItem -RootIterationPath "Squad.Umenit" | ForEach-Object{
$workItem = $null;
$workItem = Add-DevopsWorkItem -Test:$test -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $id -Type $type -Data $_;
if($null -eq $workItem){
$failed += @{$id=$_}
} else {
if(-not $test) {
$comments = Get-Content -Path ($root | Join-Path -ChildPath $id | Join-Path -ChildPath "notes.json") -Raw | ConvertFrom-Json | Sort-Object -Property created_at;
$comments.GetEnumerator() | Convert-NoteToComment | Add-DevopsComment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $workItem.id | Out-Null;
}
}
}
} -End {
Write-Progress -Activity "Create Work Items and comments" -Status "100% Complete:" -PercentComplete 100 -Completed
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment