markoutso/Readme.md

## Readme.md

      
    Raw
  

              Readme.md
            
          
    Migrate-GitlabToDevops

Dependencies

You will need an access token from azure devops. Get yours here. This script will need contributor access to the project in question.
How to run

# populate user map in Map-IssueToWorkitem
# Load all functions in functions.ps1 into scope
# Setup global variables as per below
$DevOpsAccessToken = 'get yours here https://dev.azure.com/datainsight/_usersSettings/tokens'
$GitLabAccessToken = 'get yours here https://gitlab.com/profile/personal_access_tokens'
$DevOpsOrganization = 'this is the path segmment right after https://dev.azure.com/ you use to access your devops instance ie: myorg'
$DevOpsProjectId = 'get yours here https://dev.azure.com/$DevOpsOrganization/_apis/projects'
$GitlabContainerPath = 'either "groups/groupID" get yours here https://gitlab.com/api/v4/groups or "projects/projectId" get yours here https://gitlab.com/api/v4/projects;'
# Now run all the steps one by one in process.ps1

  
## functions.ps1
function Get-GitlabIssues {
	[CmdletBinding()]
	param (
		[parameter(Mandatory=$true)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$AccessToken,
		[ValidateNotNullOrEmpty()]
		[System.String]
		$ContainerPath)
	Begin {
		$issues = $null;
		$result = $null;
		$page = 1;
		Write-Progress -Activity "Get-GitlabIssues" -Status "0 Complete:" -PercentComplete 0;
	}
	Process {
		$total = (Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues_statistics" -Headers @{ 'Authorization' = "Bearer $AccessToken" }).statistics.counts.all;
		while($result -ne 0) {
			$result = Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" }
			$issues += $result;
			if($issues.Count  -eq 0) {
				$count = 1;
			} else {
				$count = $issues.Count;
			}
			$percent = $count/$total*100;
			$page ++;
			Write-Progress -Activity "Get-GitlabIssues" -Status "$percent% Complete:" -PercentComplete $percent;
		}
	}
	End {
		return $issues;
	}
}


function Get-GitlabComments {
	[CmdletBinding()]
	param (
		[parameter(Mandatory=$true)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$AccessToken,
		[parameter(Mandatory=$true, ValueFromPipeline)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$NotesLink)
	Begin {
	}
	Process {
		try{
			Invoke-RestMethod -Method Get -Uri "$NotesLink`?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" };
		} catch {
			$issueIid = $NotesLink.Split("/") | Select-Object -Last 2 | Select-Object -First 1
			Write-Error -Message "Failed to get comments for issue: $issueIid" -ErrorId 500 -TargetObject $issueIid -Category InvalidResult;
		}
	}
	End {
	}
}


function Get-GitLabAttachment {
	[CmdletBinding(
		SupportsShouldProcess,
		ConfirmImpact="Medium"
	)]
	param (
		[parameter(Mandatory)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$AccessToken,
		[parameter(Mandatory)]
		[System.String]
		$Root,
		[parameter(Mandatory)]
		[System.String]
		$Issue,
		[parameter(Mandatory)]
		[System.String]
		$File)
	Begin {
	}
	Process {
		$path = Join-Path -Path $Root -ChildPath $Issue | Join-Path -ChildPath "Attachments";
		New-Item -Path $path -Type Directory -Force | Out-Null;
		$path = $path | Join-Path -ChildPath ("~"+([string]::Join("~", ($File.Split("/") | Select-Object -Last 3))));
		if(-Not (Test-Path -Path $path)) {
			Try {
				Invoke-WebRequest -OutFile $path -Uri $File -Headers @{ 'Authorization' = "Bearer $AccessToken" };
			} Catch {
				Write-Error -Message "Failed to get attachment: $File for issue: $Issue" -ErrorId 500 -TargetObject $Issue -Category InvalidResult;
			}
		}
	}
	End {
	}
}


function Add-DevopsWorkItem {
	[CmdletBinding(
		SupportsShouldProcess=$true,
		ConfirmImpact="Medium"
	)]
	param (
		[parameter(Mandatory=$true)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$AccessToken,
		[ValidateNotNullOrEmpty()]
		[System.String]
		$Organization,
		[ValidateNotNullOrEmpty()]
		[System.String]
		$Project,
		[ValidateNotNullOrEmpty()]
		[System.String]
		[ValidateSet("Epic", "Feature", "User Story", "Bug", "Issue", "Task")]
		$Type,
		[System.Int32]
		$Id,
		[parameter(Mandatory=$true,ValueFromPipeline)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$Data,
		[Switch]
		$Test)
	Begin {
		$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
		$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workitems/`$$($Type)?api-version=5.1&bypassRules=true&suppressNotifications=true";
	}
	Process {
		if ($PSCmdlet.ShouldProcess($Data)) {
			Try {
				if($Test) {
					Invoke-RestMethod -Method Patch -Uri $uri+"&validateOnly=true" -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
				} else {
						Invoke-RestMethod -Method Patch -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
				}
			} Catch {
				Write-Error -Message "Failed to add issue for issue: $($Id)" -ErrorId 500 -TargetObject $Data -Category InvalidResult;
			}
		}
	}
	End {
	}
}


function Add-DevopsComment {
	[CmdletBinding(
		SupportsShouldProcess=$true,
		ConfirmImpact="Medium"
	)]
	param (
		[parameter(Mandatory=$true)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$AccessToken,
		[ValidateNotNullOrEmpty()]
		[System.String]
		$Organization,
		[ValidateNotNullOrEmpty()]
		[System.String]
		$Project,
		[ValidateNotNullOrEmpty()]
		[System.Int32]
		$Id,
		[parameter(Mandatory=$true,ValueFromPipeline)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$Data)
	Begin {
		$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
		$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workItems/$Id/comments?api-version=5.1-preview.3";
	}
	Process {
		if ($PSCmdlet.ShouldProcess($Data)) {
			Try{
				Invoke-RestMethod -Method Post -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json' -Body $Data
			} Catch {
				Write-Error -Message "Failed to add comments for issue: $($Id)" -ErrorId 500 -TargetObject $Id -Category InvalidResult;
			}
		}
	}
	End {
	}
}


function Add-DevopsAttachment {
	[CmdletBinding(
		SupportsShouldProcess=$true,
		ConfirmImpact="Medium"
	)]
	param (
		[parameter(Mandatory=$true)]
		[ValidateNotNullOrEmpty()]
		[System.String]
		$AccessToken,
		[ValidateNotNullOrEmpty()]
		[System.String]
		$Organization,
		[ValidateNotNullOrEmpty()]
		[System.String]
		$Project,
		[parameter(Mandatory=$true,ValueFromPipeline)]
		[System.IO.FileInfo]
		$File)
	Begin {
		$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)));
		$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/attachments?api-version=5.0";
	}
	Process {
		if ($PSCmdlet.ShouldProcess($FileName)) {
			Try{
				$image = [System.IO.File]::ReadAllBytes($File.FullName);
				Invoke-RestMethod -Uri $uri+"&fileName=$([System.Web.HttpUtility]::UrlEncode($File.Name))" -Method Post -Headers @{ Authorization=("Basic {0}" -f $basicAuth) } -ContentType application/json -Body $image;
			} Catch {
				Write-Error -Message "Failed to add attachments for issue: $($File.FullName)" -ErrorId 500 -TargetObject $File -Category InvalidResult;
			}
		}
	}
	End {
	}
}


function Convert-IssueToWorkItem {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param (
		[parameter(Mandatory=$true)]
		$RootIterationPath,
		[parameter(Mandatory=$true,ValueFromPipeline)]
		$Issue
	)
	Begin {
		Import-Module MarkdownToHtml -Force;
		$userMap = @{
			#https://gitlab.com/api/v4/groups/mygroupid/members GitLabUserUserId="AzureDevOpsDisplayName",
			123 ="John Doe";
		};
	}
	Process {
		if([String]::IsNullOrWhiteSpace($Issue.description)) {
			$content = "`" `"";
		} else {
			$content = ((Convert-MarkdownToHTMLFragment -Markdown ($Issue.description | Format-BadCharacters)).HtmlFragment).Replace("₹", "&#x20b9;").Replace("'", "&#39;") | ConvertTo-Json;
		}
		$title = [System.Web.HttpUtility]::HtmlEncode(($Issue.title | Format-BadCharacters));
		if($title.Length -gt 255){$title = $title.Substring(0,247);}
		$author = $($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.author.id}).Value;
		$assignee = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
		$closedby = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.closed_by.id}).Value;
		$tags = [String]::Join(";", $Issue.labels+$Issue.state);
		if($Issue.milestone) {
			$iteration = $RootIterationPath+"\\"+$Issue.milestone.title.Replace("?", "").Replace("&", "and");
		} else {
			$iteration = $RootIterationPath;
		}
		if($null -ne $Issue.assignee){
			$user = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
			$assignee = ",
	{
		`"op`": `"add`",
		`"path`": `"/fields/System.AssignedTo`",
		`"from`": null,
		`"value`": `"$($user)`"
	}"
		} else {
			$assignee = "";
		}
		$type = @("User Story", "Bug")[$Issue.labels.Contains("Bug")];
		if($type -eq "Bug") {
			$body = ",
			{
				`"op`": `"add`",
				`"path`": `"/fields/Microsoft.VSTS.TCM.ReproSteps`",
				`"from`": null,
				`"value`": $content
			}"
		} else {
			$body = ",
			{
				`"op`": `"add`",
				`"path`": `"/fields/System.Description`",
				`"from`": null,
				`"value`": $content
			}"
		}
		if($Issue.state -eq "closed"){
			$closed = ",
	{
		`"op`": `"add`",
		`"path`": `"/fields/Microsoft.VSTS.Common.ClosedBy`",
		`"from`": null,
		`"value`": `"$($closedby)`"
	},
	{
		`"op`": `"add`",
		`"path`": `"/fields/Microsoft.VSTS.Common.ClosedDate`",
		`"from`": null,
		`"value`": `"$($Issue.closed_at)`"
	}";
		} else {
			$closed = "";
		}
return "[
	{
		`"op`": `"add`",
		`"path`": `"/fields/System.IterationPath`",
		`"from`": null,
		`"value`": `"$($iteration)`"
	},
	{
		`"op`": `"add`",
		`"path`": `"/fields/System.Title`",
		`"from`": null,
		`"value`": `"$($Issue.iid): $($title)`"
	}$($body),
	{
		`"op`": `"add`",
		`"path`": `"/fields/System.State`",
		`"from`": null,
		`"value`": `"$(@{"opened"="New"; "closed" = "New"}[$Issue.state])`"
	},
	{
		`"op`": `"add`",
		`"path`": `"/fields/System.Tags`",
		`"from`": null,
		`"value`": `"$($tags)`"
	},
	{
		`"op`": `"add`",
		`"path`": `"/fields/System.CreatedBy`",
		`"from`": null,
		`"value`": `"$($author)`"
	},
	{
		`"op`": `"add`",
		`"path`": `"/fields/System.CreatedDate`",
		`"from`": null,
		`"value`": `"$($Issue.created_at)`"
	}$($assignee)$($closed)
]";
	}
	End {
	}
}

function Convert-NoteToComment {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param(
		[parameter(Mandatory,ValueFromPipeline)]
		$Note
	)
	Begin {
		Import-Module MarkdownToHtml -Force;
	}
	Process {
		$header = "<p><b>$($Note.created_at.ToLocalTime().ToString('yyyy-MM-dd HH:mm:ss')) by $($Note.author.name)</b></p>";
		if([String]::IsNullOrWhiteSpace($Note.body)) {
			$content = "`" `"";
		} else {
			$content = ($header + (Convert-MarkdownToHTMLFragment -Markdown ($Note.body | Format-BadCharacters)).HtmlFragment).Replace("₹", "&#x20b9;") | ConvertTo-Json;
		}
		return "{ `"text`": $content }";
	}
	End {
	}
}

function Format-BadCharacters {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param (
		[parameter(
			Mandatory,
			ValueFromPipeline,
			ValueFromPipelineByPropertyName,
			Position = 0)]
		[string] $Input
	)
	Process {
		return $Input.Replace("`“", "`"").Replace("`”", "`"").Replace(" ", " ").Replace("ä", "a").Replace("ë", "e");
	}
}

## process.ps1
#Requires -PSEdition Core
#Requires -Modules @{ ModuleName="GitLab"; ModuleVersion="0.0"; }, @{ ModuleName="AzureDevops"; ModuleVersion="0.0"; }, @{ ModuleName="GitLabDevOpsMigration"; ModuleVersion="0.0"; }, @{ ModuleName="MarkdownToHtml"; ModuleVersion="2.0.0"}

throw "This is purely here so you dont accidentaly trigger this script, it is not designed to run end to end in one step, Please read the instructions"

### Setup working directory
$root = Join-Path -Path $(Get-Location) -ChildPath "MigrationWorkSpace"
New-Item -Path $root -Type Directory -Force;

### Step 1 - Download all GitLab Issues
$issues = Get-GitlabIssues -AccessToken $GitLabAccessToken -ContainerPath $GitlabContainerPath;

### Step 2 - Had to run this a few times so I always kept track of migrated ids in this file
$MigratedIssueSucessPath = $root | Join-Path -ChildPath "GtDIssueSucess.txt";
if(Test-Path -Path $MigratedIssueSucessPath) {
	$migratedIssueIds = Get-Content $MigratedIssueSucessPath | ForEach-Object{[int64]::Parse($_)}
} else { $migratedIssueIds = @()}
$issuesToMigrate = $issues | Where-Object{ -not $migratedIssueIds.Contains($_.id)}


### Step 3 - Save all issues into a folder
$issuesToMigrate | ForEach-Object {
	$folder = Join-Path -Path $root -ChildPath $_.id;
	New-Item -Path $folder -Type Directory -Force | Out-Null;
	$_ | ConvertTo-Json | Out-File -FilePath (Join-Path -Path $folder -ChildPath issue.json) -Encoding utf8 -Force};

### Step 4 - Get Unique Milestones and their status
### These need to be created as sprints in devops manualy, I only had to create about a dozen so was easy enough
$issuesToMigrate | Select-Object -ExpandProperty milestone | Select-Object title, state -Unique | Sort-Object state, name

### Step 5 - Download Comments
$total = $issuesToMigrate.Count;
$counter = 1;
$issuesToMigrate | Sort-Object id | ForEach-Object -Begin {
		Write-Progress -Activity "Get-GitlabComments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
		$percent = $counter/$total*100;
		$counter++;
		Write-Progress -Activity "Get-GitlabComments" -Status "$percent% Complete: $($_.id)" -PercentComplete $percent;
			$folder = Join-Path -Path $root -ChildPath $_.id;
			Get-GitlabComments -AccessToken $GitLabAccessToken -NotesLink $_._links.notes |
			ConvertTo-Json |
			Out-File -FilePath (Join-Path -Path $folder -ChildPath notes.json) -Encoding utf8 -Force;
	} -End {
		Write-Progress -Activity "Get-GitlabComments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}

### Step 6 - Get all attachment links
$files = @();
#### Get attachment links from issues
Get-ChildItem -Path $root |
	ForEach-Object { $id = $_.Name;
		Get-ChildItem -Path $_.FullName -Filter issue.json |
		Get-Content -Raw |
		ConvertFrom-Json |
		Where-Object   { -not [String]::IsNullOrWhiteSpace($_.description) } |
		ForEach-Object {
			$urlParts = $_.web_url.Split("/");
			$rootPath = [string]::Join("/", ($urlParts | Select-Object -First ($urlParts.Count-2)));
			$_.description.Split("`n")  |
			ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} |
			Where-Object   { $_.Success } |
			Select-Object -ExpandProperty Groups |
			Where-Object   { -Not $_.Value.StartsWith("]")} |
			Select-Object -ExpandProperty Value |
			ForEach-Object {
				$files+=New-Object PSObject -Property @{
					issue=$id;
					file=$rootPath+$_;}
			}
		}
	}

#### Get attachment links from comments
Get-ChildItem -Path $root |
	ForEach-Object {
		$id = $_.Name;
		$urlParts = (Get-ChildItem -Path $_.FullName -Filter issue.json |
		Get-Content -Raw |
		ConvertFrom-Json).web_url.Split("/");
		$rootPath = [string]::Join("/", ($urlParts | Select-Object -First ($urlParts.Count-2)));
		Get-ChildItem -Path $_.FullName -Filter notes.json |
		Get-Content -Raw |
		ConvertFrom-Json |
		Where-Object   { -not [String]::IsNullOrWhiteSpace($_.body) } |
		ForEach-Object { $_.body.Split("`n") } |
		ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} |
		Where-Object   { $_.Success } |
		Select-Object -ExpandProperty Groups |
		Where-Object   { -Not $_.Value.StartsWith("]")} |
		Select-Object -ExpandProperty Value |
		ForEach-Object {
			$files+=New-Object PSObject -Property @{
				issue=$id;
				file=$rootPath+$_;} }
	}


### Step 7 - Downlaod the attachments from gitlab NOTE: This can take while
$total = $files.Count;
$counter = 1;
$files | ForEach-Object -Begin {
		Write-Progress -Activity "Get-GitlabAttachments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
		$percent = $counter/$total*100;
		$counter++;
		Write-Progress -Activity "Get-GitlabAttachments" -Status "$percent% Complete: $($_.issue)" -PercentComplete $percent;
		Get-GitLabAttachment -AccessToken $GitLabAccessToken -Root $root -Issue $_.Issue -File $_.File;
	} -End {
		Write-Progress -Activity "Get-GitlabAttachments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}

### Step 8 - Upload and map attachments to offlined notes and comments
$total = (Get-ChildItem -Path $root).Count;
$counter = 1;
Get-ChildItem -Path $root |
	ForEach-Object -Begin {
		Write-Progress -Activity "Map-Attachments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
		$percent = $counter/$total*100;
		$counter++;
		Write-Progress -Activity "Map-Attachments" -Status "$percent% Complete: $($_.Name)" -PercentComplete $percent;
			Push-Location -Path $_.FullName;
				if(Test-Path -Path Attachments) {
					$replace = Get-ChildItem -Path Attachments | ForEach-Object {
						$name = $_ | Select-Object -Property @{Name="Name";Expression={$_.Name.Replace("~","/")}} | Select-Object -ExpandProperty Name;
						$upld = Add-DevopsAttachment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -File $_;
						Write-Verbose "$($name) $($upld.url)" -Verbose;
						@{Old=$name; New=$upld.url};
					}

					$replace | ForEach-Object {
						$old = $_.Old;
						$new = $_.New;
						Get-ChildItem -File | ForEach-Object {
							$fileName = $_.FullName;
							$content = Get-Content $_ | ForEach-Object {
								$_.Replace($old, $new);
							};
							$content | Out-File -FilePath $fileName -Encoding utf8 -Force;
						}
					}
				}
			Pop-Location;
	} -End {
		Write-Progress -Activity "Map-Attachments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}
###
# At this stage I had to do some manual fixes sometimes, important at this stage to re-load all the issues and commments from the filesystem as they have the updated data after uploading all the attachments
###
$issuesToMigrate = Get-ChildItem -Path $root -Filter issue.json -Recurse | Get-Content -Raw | ConvertFrom-Json

### Step 9 - Issue + Comment Upload
$failed = @();
$total = $issuesToMigrate.Count;
$test=$true; # This will allow you to validate before import, set to false for actual import
$counter = 1;
#### The Sort helped me judge how far into the process I was
$issuesToMigrate | Sort-Object id |
	ForEach-Object -Begin {
        Write-Progress -Activity "Create Work Items and comments" -Status "0% Complete:" -PercentComplete 0;
    } -Process {
        $percent = $counter/$total*100;
        $counter++;
		$id = $_.id;
		### Note the new bug template needs to map the content to another field other than description, think it was called reposteps or similar
		$type = @("User Story", "Bug")[$_.labels.Contains("Bug")];
        Write-Progress -Activity "Create Work Items and comments" -Status "$percent% Complete: $($id)" -PercentComplete $percent;
		$_ | Convert-IssueToWorkItem -RootIterationPath "Squad.Umenit" | ForEach-Object{
			$workItem = $null;
			$workItem = Add-DevopsWorkItem -Test:$test -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $id -Type $type -Data $_;
			if($null -eq $workItem){
				$failed += @{$id=$_}
			} else {
				if(-not $test) {
					$comments = Get-Content -Path ($root | Join-Path -ChildPath $id | Join-Path -ChildPath "notes.json") -Raw | ConvertFrom-Json | Sort-Object -Property created_at;
					$comments.GetEnumerator() | Convert-NoteToComment | Add-DevopsComment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $workItem.id | Out-Null;
				}
			}
		}
    } -End {
        Write-Progress -Activity "Create Work Items and comments" -Status "100% Complete:" -PercentComplete 100 -Completed
    }
	function Get-GitlabIssues {
	[CmdletBinding()]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$ContainerPath)
	Begin {
	$issues = $null;
	$result = $null;
	$page = 1;
	Write-Progress -Activity "Get-GitlabIssues" -Status "0 Complete:" -PercentComplete 0;
	}
	Process {
	$total = (Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues_statistics" -Headers @{ 'Authorization' = "Bearer $AccessToken" }).statistics.counts.all;
	while($result -ne 0) {
	$result = Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" }
	$issues += $result;
	if($issues.Count -eq 0) {
	$count = 1;
	} else {
	$count = $issues.Count;
	}
	$percent = $count/$total*100;
	$page ++;
	Write-Progress -Activity "Get-GitlabIssues" -Status "$percent% Complete:" -PercentComplete $percent;
	}
	}
	End {
	return $issues;
	}
	}


	function Get-GitlabComments {
	[CmdletBinding()]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[parameter(Mandatory=$true, ValueFromPipeline)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$NotesLink)
	Begin {
	}
	Process {
	try{
	Invoke-RestMethod -Method Get -Uri "$NotesLink`?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" };
	} catch {
	$issueIid = $NotesLink.Split("/") \| Select-Object -Last 2 \| Select-Object -First 1
	Write-Error -Message "Failed to get comments for issue: $issueIid" -ErrorId 500 -TargetObject $issueIid -Category InvalidResult;
	}
	}
	End {
	}
	}


	function Get-GitLabAttachment {
	[CmdletBinding(
	SupportsShouldProcess,
	ConfirmImpact="Medium"
	)]
	param (
	[parameter(Mandatory)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[parameter(Mandatory)]
	[System.String]
	$Root,
	[parameter(Mandatory)]
	[System.String]
	$Issue,
	[parameter(Mandatory)]
	[System.String]
	$File)
	Begin {
	}
	Process {
	$path = Join-Path -Path $Root -ChildPath $Issue \| Join-Path -ChildPath "Attachments";
	New-Item -Path $path -Type Directory -Force \| Out-Null;
	$path = $path \| Join-Path -ChildPath ("~"+([string]::Join("~", ($File.Split("/") \| Select-Object -Last 3))));
	if(-Not (Test-Path -Path $path)) {
	Try {
	Invoke-WebRequest -OutFile $path -Uri $File -Headers @{ 'Authorization' = "Bearer $AccessToken" };
	} Catch {
	Write-Error -Message "Failed to get attachment: $File for issue: $Issue" -ErrorId 500 -TargetObject $Issue -Category InvalidResult;
	}
	}
	}
	End {
	}
	}


	function Add-DevopsWorkItem {
	[CmdletBinding(
	SupportsShouldProcess=$true,
	ConfirmImpact="Medium"
	)]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Organization,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Project,
	[ValidateNotNullOrEmpty()]
	[System.String]
	[ValidateSet("Epic", "Feature", "User Story", "Bug", "Issue", "Task")]
	$Type,
	[System.Int32]
	$Id,
	[parameter(Mandatory=$true,ValueFromPipeline)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Data,
	[Switch]
	$Test)
	Begin {
	$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
	$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workitems/`$$($Type)?api-version=5.1&bypassRules=true&suppressNotifications=true";
	}
	Process {
	if ($PSCmdlet.ShouldProcess($Data)) {
	Try {
	if($Test) {
	Invoke-RestMethod -Method Patch -Uri $uri+"&validateOnly=true" -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
	} else {
	Invoke-RestMethod -Method Patch -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
	}
	} Catch {
	Write-Error -Message "Failed to add issue for issue: $($Id)" -ErrorId 500 -TargetObject $Data -Category InvalidResult;
	}
	}
	}
	End {
	}
	}


	function Add-DevopsComment {
	[CmdletBinding(
	SupportsShouldProcess=$true,
	ConfirmImpact="Medium"
	)]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Organization,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Project,
	[ValidateNotNullOrEmpty()]
	[System.Int32]
	$Id,
	[parameter(Mandatory=$true,ValueFromPipeline)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Data)
	Begin {
	$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
	$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workItems/$Id/comments?api-version=5.1-preview.3";
	}
	Process {
	if ($PSCmdlet.ShouldProcess($Data)) {
	Try{
	Invoke-RestMethod -Method Post -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json' -Body $Data
	} Catch {
	Write-Error -Message "Failed to add comments for issue: $($Id)" -ErrorId 500 -TargetObject $Id -Category InvalidResult;
	}
	}
	}
	End {
	}
	}


	function Add-DevopsAttachment {
	[CmdletBinding(
	SupportsShouldProcess=$true,
	ConfirmImpact="Medium"
	)]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Organization,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Project,
	[parameter(Mandatory=$true,ValueFromPipeline)]
	[System.IO.FileInfo]
	$File)
	Begin {
	$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)));
	$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/attachments?api-version=5.0";
	}
	Process {
	if ($PSCmdlet.ShouldProcess($FileName)) {
	Try{
	$image = [System.IO.File]::ReadAllBytes($File.FullName);
	Invoke-RestMethod -Uri $uri+"&fileName=$([System.Web.HttpUtility]::UrlEncode($File.Name))" -Method Post -Headers @{ Authorization=("Basic {0}" -f $basicAuth) } -ContentType application/json -Body $image;
	} Catch {
	Write-Error -Message "Failed to add attachments for issue: $($File.FullName)" -ErrorId 500 -TargetObject $File -Category InvalidResult;
	}
	}
	}
	End {
	}
	}


	function Convert-IssueToWorkItem {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param (
	[parameter(Mandatory=$true)]
	$RootIterationPath,
	[parameter(Mandatory=$true,ValueFromPipeline)]
	$Issue
	)
	Begin {
	Import-Module MarkdownToHtml -Force;
	$userMap = @{
	#https://gitlab.com/api/v4/groups/mygroupid/members GitLabUserUserId="AzureDevOpsDisplayName",
	123 ="John Doe";
	};
	}
	Process {
	if([String]::IsNullOrWhiteSpace($Issue.description)) {
	$content = "`" `"";
	} else {
	$content = ((Convert-MarkdownToHTMLFragment -Markdown ($Issue.description \| Format-BadCharacters)).HtmlFragment).Replace("₹", "₹").Replace("'", "'") \| ConvertTo-Json;
	}
	$title = [System.Web.HttpUtility]::HtmlEncode(($Issue.title \| Format-BadCharacters));
	if($title.Length -gt 255){$title = $title.Substring(0,247);}
	$author = $($userMap.GetEnumerator() \| Where-Object{$_.Name -eq $Issue.author.id}).Value;
	$assignee = ($userMap.GetEnumerator() \| Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
	$closedby = ($userMap.GetEnumerator() \| Where-Object{$_.Name -eq $Issue.closed_by.id}).Value;
	$tags = [String]::Join(";", $Issue.labels+$Issue.state);
	if($Issue.milestone) {
	$iteration = $RootIterationPath+"\\"+$Issue.milestone.title.Replace("?", "").Replace("&", "and");
	} else {
	$iteration = $RootIterationPath;
	}
	if($null -ne $Issue.assignee){
	$user = ($userMap.GetEnumerator() \| Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
	$assignee = ",
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.AssignedTo`",
	`"from`": null,
	`"value`": `"$($user)`"
	}"
	} else {
	$assignee = "";
	}
	$type = @("User Story", "Bug")[$Issue.labels.Contains("Bug")];
	if($type -eq "Bug") {
	$body = ",
	{
	`"op`": `"add`",
	`"path`": `"/fields/Microsoft.VSTS.TCM.ReproSteps`",
	`"from`": null,
	`"value`": $content
	}"
	} else {
	$body = ",
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.Description`",
	`"from`": null,
	`"value`": $content
	}"
	}
	if($Issue.state -eq "closed"){
	$closed = ",
	{
	`"op`": `"add`",
	`"path`": `"/fields/Microsoft.VSTS.Common.ClosedBy`",
	`"from`": null,
	`"value`": `"$($closedby)`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/Microsoft.VSTS.Common.ClosedDate`",
	`"from`": null,
	`"value`": `"$($Issue.closed_at)`"
	}";
	} else {
	$closed = "";
	}
	return "[
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.IterationPath`",
	`"from`": null,
	`"value`": `"$($iteration)`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.Title`",
	`"from`": null,
	`"value`": `"$($Issue.iid): $($title)`"
	}$($body),
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.State`",
	`"from`": null,
	`"value`": `"$(@{"opened"="New"; "closed" = "New"}[$Issue.state])`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.Tags`",
	`"from`": null,
	`"value`": `"$($tags)`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.CreatedBy`",
	`"from`": null,
	`"value`": `"$($author)`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.CreatedDate`",
	`"from`": null,
	`"value`": `"$($Issue.created_at)`"
	}$($assignee)$($closed)
	]";
	}
	End {
	}
	}

	function Convert-NoteToComment {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param(
	[parameter(Mandatory,ValueFromPipeline)]
	$Note
	)
	Begin {
	Import-Module MarkdownToHtml -Force;
	}
	Process {
	$header = "<p><b>$($Note.created_at.ToLocalTime().ToString('yyyy-MM-dd HH:mm:ss')) by $($Note.author.name)</b></p>";
	if([String]::IsNullOrWhiteSpace($Note.body)) {
	$content = "`" `"";
	} else {
	$content = ($header + (Convert-MarkdownToHTMLFragment -Markdown ($Note.body \| Format-BadCharacters)).HtmlFragment).Replace("₹", "₹") \| ConvertTo-Json;
	}
	return "{ `"text`": $content }";
	}
	End {
	}
	}

	function Format-BadCharacters {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param (
	[parameter(
	Mandatory,
	ValueFromPipeline,
	ValueFromPipelineByPropertyName,
	Position = 0)]
	[string] $Input
	)
	Process {
	return $Input.Replace("`“", "`"").Replace("`”", "`"").Replace(" ", " ").Replace("ä", "a").Replace("ë", "e");
	}
	}
	#Requires -PSEdition Core
	#Requires -Modules @{ ModuleName="GitLab"; ModuleVersion="0.0"; }, @{ ModuleName="AzureDevops"; ModuleVersion="0.0"; }, @{ ModuleName="GitLabDevOpsMigration"; ModuleVersion="0.0"; }, @{ ModuleName="MarkdownToHtml"; ModuleVersion="2.0.0"}

	throw "This is purely here so you dont accidentaly trigger this script, it is not designed to run end to end in one step, Please read the instructions"

	### Setup working directory
	$root = Join-Path -Path $(Get-Location) -ChildPath "MigrationWorkSpace"
	New-Item -Path $root -Type Directory -Force;

	### Step 1 - Download all GitLab Issues
	$issues = Get-GitlabIssues -AccessToken $GitLabAccessToken -ContainerPath $GitlabContainerPath;

	### Step 2 - Had to run this a few times so I always kept track of migrated ids in this file
	$MigratedIssueSucessPath = $root \| Join-Path -ChildPath "GtDIssueSucess.txt";
	if(Test-Path -Path $MigratedIssueSucessPath) {
	$migratedIssueIds = Get-Content $MigratedIssueSucessPath \| ForEach-Object{[int64]::Parse($_)}
	} else { $migratedIssueIds = @()}
	$issuesToMigrate = $issues \| Where-Object{ -not $migratedIssueIds.Contains($_.id)}


	### Step 3 - Save all issues into a folder
	$issuesToMigrate \| ForEach-Object {
	$folder = Join-Path -Path $root -ChildPath $_.id;
	New-Item -Path $folder -Type Directory -Force \| Out-Null;
	$_ \| ConvertTo-Json \| Out-File -FilePath (Join-Path -Path $folder -ChildPath issue.json) -Encoding utf8 -Force};

	### Step 4 - Get Unique Milestones and their status
	### These need to be created as sprints in devops manualy, I only had to create about a dozen so was easy enough
	$issuesToMigrate \| Select-Object -ExpandProperty milestone \| Select-Object title, state -Unique \| Sort-Object state, name

	### Step 5 - Download Comments
	$total = $issuesToMigrate.Count;
	$counter = 1;
	$issuesToMigrate \| Sort-Object id \| ForEach-Object -Begin {
	Write-Progress -Activity "Get-GitlabComments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
	$percent = $counter/$total*100;
	$counter++;
	Write-Progress -Activity "Get-GitlabComments" -Status "$percent% Complete: $($_.id)" -PercentComplete $percent;
	$folder = Join-Path -Path $root -ChildPath $_.id;
	Get-GitlabComments -AccessToken $GitLabAccessToken -NotesLink $_._links.notes \|
	ConvertTo-Json \|
	Out-File -FilePath (Join-Path -Path $folder -ChildPath notes.json) -Encoding utf8 -Force;
	} -End {
	Write-Progress -Activity "Get-GitlabComments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}

	### Step 6 - Get all attachment links
	$files = @();
	#### Get attachment links from issues
	Get-ChildItem -Path $root \|
	ForEach-Object { $id = $_.Name;
	Get-ChildItem -Path $_.FullName -Filter issue.json \|
	Get-Content -Raw \|
	ConvertFrom-Json \|
	Where-Object { -not [String]::IsNullOrWhiteSpace($_.description) } \|
	ForEach-Object {
	$urlParts = $_.web_url.Split("/");
	$rootPath = [string]::Join("/", ($urlParts \| Select-Object -First ($urlParts.Count-2)));
	$_.description.Split("`n") \|
	ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} \|
	Where-Object { $_.Success } \|
	Select-Object -ExpandProperty Groups \|
	Where-Object { -Not $_.Value.StartsWith("]")} \|
	Select-Object -ExpandProperty Value \|
	ForEach-Object {
	$files+=New-Object PSObject -Property @{
	issue=$id;
	file=$rootPath+$_;}
	}
	}
	}

	#### Get attachment links from comments
	Get-ChildItem -Path $root \|
	ForEach-Object {
	$id = $_.Name;
	$urlParts = (Get-ChildItem -Path $_.FullName -Filter issue.json \|
	Get-Content -Raw \|
	ConvertFrom-Json).web_url.Split("/");
	$rootPath = [string]::Join("/", ($urlParts \| Select-Object -First ($urlParts.Count-2)));
	Get-ChildItem -Path $_.FullName -Filter notes.json \|
	Get-Content -Raw \|
	ConvertFrom-Json \|
	Where-Object { -not [String]::IsNullOrWhiteSpace($_.body) } \|
	ForEach-Object { $_.body.Split("`n") } \|
	ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} \|
	Where-Object { $_.Success } \|
	Select-Object -ExpandProperty Groups \|
	Where-Object { -Not $_.Value.StartsWith("]")} \|
	Select-Object -ExpandProperty Value \|
	ForEach-Object {
	$files+=New-Object PSObject -Property @{
	issue=$id;
	file=$rootPath+$_;} }
	}



	### Step 7 - Downlaod the attachments from gitlab NOTE: This can take while
	$total = $files.Count;
	$counter = 1;
	$files \| ForEach-Object -Begin {
	Write-Progress -Activity "Get-GitlabAttachments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
	$percent = $counter/$total*100;
	$counter++;
	Write-Progress -Activity "Get-GitlabAttachments" -Status "$percent% Complete: $($_.issue)" -PercentComplete $percent;
	Get-GitLabAttachment -AccessToken $GitLabAccessToken -Root $root -Issue $_.Issue -File $_.File;
	} -End {
	Write-Progress -Activity "Get-GitlabAttachments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}

	### Step 8 - Upload and map attachments to offlined notes and comments
	$total = (Get-ChildItem -Path $root).Count;
	$counter = 1;
	Get-ChildItem -Path $root \|
	ForEach-Object -Begin {
	Write-Progress -Activity "Map-Attachments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
	$percent = $counter/$total*100;
	$counter++;
	Write-Progress -Activity "Map-Attachments" -Status "$percent% Complete: $($_.Name)" -PercentComplete $percent;
	Push-Location -Path $_.FullName;
	if(Test-Path -Path Attachments) {
	$replace = Get-ChildItem -Path Attachments \| ForEach-Object {
	$name = $_ \| Select-Object -Property @{Name="Name";Expression={$_.Name.Replace("~","/")}} \| Select-Object -ExpandProperty Name;
	$upld = Add-DevopsAttachment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -File $_;
	Write-Verbose "$($name) $($upld.url)" -Verbose;
	@{Old=$name; New=$upld.url};
	}

	$replace \| ForEach-Object {
	$old = $_.Old;
	$new = $_.New;
	Get-ChildItem -File \| ForEach-Object {
	$fileName = $_.FullName;
	$content = Get-Content $_ \| ForEach-Object {
	$_.Replace($old, $new);
	};
	$content \| Out-File -FilePath $fileName -Encoding utf8 -Force;
	}
	}
	}
	Pop-Location;
	} -End {
	Write-Progress -Activity "Map-Attachments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}
	###
	# At this stage I had to do some manual fixes sometimes, important at this stage to re-load all the issues and commments from the filesystem as they have the updated data after uploading all the attachments
	###
	$issuesToMigrate = Get-ChildItem -Path $root -Filter issue.json -Recurse \| Get-Content -Raw \| ConvertFrom-Json

	### Step 9 - Issue + Comment Upload
	$failed = @();
	$total = $issuesToMigrate.Count;
	$test=$true; # This will allow you to validate before import, set to false for actual import
	$counter = 1;
	#### The Sort helped me judge how far into the process I was
	$issuesToMigrate \| Sort-Object id \|
	ForEach-Object -Begin {
	Write-Progress -Activity "Create Work Items and comments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
	$percent = $counter/$total*100;
	$counter++;
	$id = $_.id;
	### Note the new bug template needs to map the content to another field other than description, think it was called reposteps or similar
	$type = @("User Story", "Bug")[$_.labels.Contains("Bug")];
	Write-Progress -Activity "Create Work Items and comments" -Status "$percent% Complete: $($id)" -PercentComplete $percent;
	$_ \| Convert-IssueToWorkItem -RootIterationPath "Squad.Umenit" \| ForEach-Object{
	$workItem = $null;
	$workItem = Add-DevopsWorkItem -Test:$test -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $id -Type $type -Data $_;
	if($null -eq $workItem){
	$failed += @{$id=$_}
	} else {
	if(-not $test) {
	$comments = Get-Content -Path ($root \| Join-Path -ChildPath $id \| Join-Path -ChildPath "notes.json") -Raw \| ConvertFrom-Json \| Sort-Object -Property created_at;
	$comments.GetEnumerator() \| Convert-NoteToComment \| Add-DevopsComment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $workItem.id \| Out-Null;
	}
	}
	}
	} -End {
	Write-Progress -Activity "Create Work Items and comments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}