Skip to content

Instantly share code, notes, and snippets.

@tonyarnold
Created March 24, 2014 03:02
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tonyarnold/9733445 to your computer and use it in GitHub Desktop.
Save tonyarnold/9733445 to your computer and use it in GitHub Desktop.
Really rough script to copy GitHub issues to GitLab, including scaffolding users with the correct name/username, but dummy emails. Prepare to be spammed by notifications. Current issues include comments not being associated with their proper owner, pull requests aren't properly transferred and wiki page imports have not been tested.
# Community contributed script to import from GitHub to GitLab
# It imports repositories, issues and the wiki's.
# This script is not maintained, please send merge requests to improve it, do not file bugs.
# The issue import might concatenate all comments of an issue into one, if so feel free to fix this.
require 'bundler/setup'
require 'octokit'
require 'optparse'
require 'git'
require 'gitlab'
require 'pp'
OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
def random_string
o = [('a'..'z'), ('A'..'Z')].map { |i| i.to_a }.flatten
(0...50).map { o[rand(o.length)] }.join
end
def find_or_create_user(username, github_client, gitlab_client)
found_user = nil
# Look for an existing user
gitlab_client.users.each do |u|
if u.username == username
found_user = u
end
end
# If an existing user can't be found, create a new one
if found_user.nil?
gh_user = github_client.user(username)
found_user = gitlab_client.create_user("gitlab-#{gh_user[:login]}@yourdomain.com", random_string, {
:name => gh_user[:name],
:username => gh_user[:login]
})
end
found_user
end
def find_or_create_milestone(project_id, milestone_title, description, due_on, github_client, gitlab_client)
found_milestone = nil
# Look for an existing milestone
gitlab_client.milestones(project_id).each do |m|
if m.title == milestone_title && m.project == project_id
found_milestone = m
end
end
# If an existing milestone can't be found, create a new one
if found_milestone.nil?
found_milestone = gitlab_client.create_milestone(project_id, milestone_title, {
:description => description,
:due_date => due_on
})
end
found_milestone
end
#deal with options from cli, like username and pw
options = {:usr => nil,
:pw => nil,
:repository => nil,
:api => 'https://github.com/api/v3',
:web => 'https://github.com/',
:enterprise => false,
:group => nil,
:gitlab_api => 'https://gitlab.yourdomain.com/api/v3',
:gitlab_token => 'YOUR_TOKEN'
}
optparse = OptionParser.new do |opts|
opts.on('-u', '--user USER', "user to connect to github with") do |u|
options[:usr] = u
end
opts.on('-p', '--pw PASSWORD', 'password for user to connect to github with') do |p|
options[:pw] = p
end
opts.on('-r', '--repository "USERNAME/REPOSITORY"', 'The repository to import — must be of the format \'username/repository\'') do |r|
options[:repository] = r
end
opts.on('-g', '--group GROUP', 'The group to import repositories to') do |g|
options[:group] = g
end
opts.on('-e', '--enterprise true/false', 'Is this an enterprise GitHub server?') do |e|
options[:enterprise] = e
end
opts.on('--api', 'API endpoint for github') do |a|
options[:api] = a
end
opts.on('--web', 'Web endpoint for GitHub') do |w|
options[:web] = w
end
opts.on('-h', '--help', 'Display this screen') do
puts opts
exit
end
end
optparse.parse!
if options[:usr].nil? or options[:pw].nil? or options[:repository].nil? or options[:group].nil?
puts "Missing parameter ..."
puts options
exit
end
#setup octokit to deal with github enterprise
if options[:enterprise]
Octokit.configure do |c|
c.api_endpoint = options[:api]
c.web_endpoint = options[:web]
end
end
#set the gitlab options
Gitlab.configure do |c|
c.endpoint = options[:gitlab_api]
c.private_token = options[:gitlab_token]
end
# Setup the clients
gh_client = Octokit::Client.new(:login => options[:usr], :password => options[:pw])
gl_client = Gitlab.client()
#get all of the repos that are in the specified space (user or org)
gh_r = gh_client.repository("#{options[:repository]}")
# Clone the repo from the github server
git_repo = nil
if File.directory?("/tmp/clones/#{gh_r.name}")
puts "Pulling changes from repository #{gh_r.name}"
git_repo = Git.open("/tmp/clones/#{gh_r.name}")
git_repo.pull
else
puts "Cloning from repository #{gh_r.ssh_url}"
git_repo = Git.clone(gh_r.ssh_url, gh_r.name, :path => '/tmp/clones')
end
#
## Push the cloned repo to gitlab
#
project_list = []
push_group = nil
#I should be able to search for a group by name
gl_client.groups.each do |g|
if g.name == options[:group]
push_group = g
end
end
#if the group wasn't found, create it
if push_group.nil?
push_group = gl_client.create_group(options[:group], options[:group])
end
#edge case, gitlab didn't like names that didn't start with an alpha. Can't remember how I ran into this.
project_path = gh_r.name.gsub!('.', '-')
# if gh_r.name !~ /^[a-zA-Z]/
# project_path = "#{gh_r.name}"
# end
project = nil
#I should be able to search for a group by name
gl_client.projects.each do |p|
if p.path == project_path
project = p
end
end
# If the project wasn't found, create it
if project.nil?
project = gl_client.create_project(gh_r.name, {
:default_branch => gh_r.default_branch,
:description => gh_r.description,
:public => not(gh_r.private)
})
end
remote = nil
remote_name = 'gitlab'
git_repo.remotes.each do |r|
if r.name == remote_name
remote = r
end
end
if not(remote.nil?)
git_repo.remote(remote_name).remove
end
puts "Adding remote '#{remote_name}' to repository #{gh_r.name}"
git_repo.add_remote(remote_name, project.ssh_url_to_repo)
git_repo.branches.each do |b|
next if b.name =~ /^HEAD/
next if git_repo.is_local_branch? b
puts "Pushing branch '#{b.name}' to remote '#{remote_name}' of repository #{gh_r.name}"
git_repo.checkout(b.name)
git_repo.push(remote_name, b.name, true)
end
milestone_hash = {}
#
## Look for milestones in GitHub for this project and push them to GitLab
#
puts "Copying #{gh_r.name} milestones: "
milestones = gh_client.milestones(gh_r.full_name)
milestones.each do |m|
gl_milestone = find_or_create_milestone(project.id, m.title, m.description, m.due_on, gh_client, gl_client)
milestone_hash[gl_milestone.title] = gl_milestone.id
puts " ✓ Copied milestone #{gl_milestone.title} (#{gl_milestone.id})"
end
#
## Look for issues in GitHub for this project and push them to GitLab
#
if gh_r.has_issues
puts "Copying issues for #{gh_r.name}: "
issues = gh_client.issues(gh_r.full_name, {:state => :open}).concat(gh_client.issues(gh_r.full_name, {:state => :closed}))
issues.each do |i|
unless i.user.nil?
issue_author = find_or_create_user(i.user[:login], gh_client, gl_client)
end
unless i.assignee.nil?
issue_assignee = find_or_create_user(i.assignee[:login], gh_client, gl_client)
end
issue_options = {
:description => i.body
}
unless issue_author.nil?
issue_options[:author_id] = issue_author.id
end
unless issue_assignee.nil?
issue_options[:assignee_id] = issue_assignee.id
end
unless i.milestone.nil? or i.milestone.title.nil? or milestone_hash[i.milestone.title].nil?
issue_options[:milestone_id] = milestone_hash[i.milestone.title]
end
labels_for_issue = gh_client.labels_for_issue(gh_r.full_name, i.number).map{|label|
label.name
}.join(",")
unless labels_for_issue.empty?
issue_options[:labels] = labels_for_issue
end
gl_issue = gl_client.create_issue(project.id, i.title, issue_options)
if i.state.eql? "closed"
gl_client.close_issue(project.id, gl_issue.id)
end
puts " ✓ Copied issue #{gl_issue.id} - #{gl_issue.title} by #{issue_author.username} with state #{i.state}"
comments = gh_client.issue_comments(gh_r.full_name, i.number)
if comments.any?
comments.each do |c|
comment_body = <<-COMMENT_BODY
#{c.body}
Comment originally by [#{c.user.login}](#{c.user.url}) on #{c.created_at}
COMMENT_BODY
gl_issue_note = gl_client.create_issue_note(project.id, gl_issue.id, comment_body)
puts " ✓ Copied comment for issue #{gl_issue.id} - #{gl_issue.title} with ID #{gl_issue_note.id}"
end
end
end
end
#
## Look for wiki pages for this repo in GitHub and migrate them to GitLab
#
if gh_r.has_wiki
puts "Copying #{gh_r.name} wiki: "
#this is dumb. The only way to know if a repo has a wiki is to attempt to clone it and then ignore failure if it doesn't have one
begin
gh_wiki_url = gh_r.git_url.gsub(/\.git/, ".wiki.git")
wiki_name = gh_r.name + '.wiki'
wiki_repo = Git.clone(gh_wiki_url, wiki_name, :path => '/tmp/clones')
#this is a pain, have to visit the wiki page on the web ui before being able to work with it as a git repo
`wget -q --save-cookies /tmp/junk/gl_login.txt -P /tmp/junk --post-data "username=#{options[:usr]}&password=#{options[:pw]}" gitlab.example.com/users/auth/ldap/callback`
`wget -q --load-cookies /tmp/junk/gl_login.txt -P /tmp/junk -p #{project.web_url}/wikis/home`
`rm -fr /tmp/junk/*`
gl_wiki_url = project.ssh_url_to_repo.gsub(/\.git/, ".wiki.git")
wiki_repo.add_remote('gitlab', gl_wiki_url)
wiki_repo.push('gitlab')
rescue
end
end
# change the owner of this new project to the group we found it in
gl_client.transfer_project_to_group(push_group.id, project.id)
puts "Adding remote '#{remote_name}' to repository #{gh_r.name}"
git_repo.remote(remote_name).remove
git_repo.add_remote(remote_name, project.ssh_url_to_repo)
module API
# Issues API
class Issues < Grape::API
before { authenticate! }
resource :issues do
# Get currently authenticated user's issues
#
# Example Request:
# GET /issues
get do
present paginate(current_user.issues), with: Entities::Issue
end
end
resource :projects do
# Get a list of project issues
#
# Parameters:
# id (required) - The ID of a project
# Example Request:
# GET /projects/:id/issues
get ":id/issues" do
present paginate(user_project.issues), with: Entities::Issue
end
# Get a single project issue
#
# Parameters:
# id (required) - The ID of a project
# issue_id (required) - The ID of a project issue
# Example Request:
# GET /projects/:id/issues/:issue_id
get ":id/issues/:issue_id" do
@issue = user_project.issues.find(params[:issue_id])
present @issue, with: Entities::Issue
end
# Create a new project issue
#
# Parameters:
# id (required) - The ID of a project
# title (required) - The title of an issue
# description (optional) - The description of an issue
# author_id (optional) - The ID of a user to author the issue
# assignee_id (optional) - The ID of a user to assign issue
# milestone_id (optional) - The ID of a milestone to assign issue
# labels (optional) - The labels of an issue
# state_event (optional) - The state event of an issue (open|close|reopen)
# Example Request:
# POST /projects/:id/issues
post ":id/issues" do
set_current_user_for_thread do
required_attributes! [:title]
attrs = attributes_for_keys [:title, :description, :author_id, :assignee_id, :milestone_id, :state_event]
attrs[:label_list] = params[:labels] if params[:labels].present?
@issue = user_project.issues.new attrs
unless attrs[:author_id].nil?
@issue.author = User.find(attrs[:author_id])
else
@issue.author = current_user
end
if @issue.save
present @issue, with: Entities::Issue
else
not_found!
end
end
end
# Update an existing issue
#
# Parameters:
# id (required) - The ID of a project
# issue_id (required) - The ID of a project issue
# title (optional) - The title of an issue
# description (optional) - The description of an issue
# assignee_id (optional) - The ID of a user to assign issue
# milestone_id (optional) - The ID of a milestone to assign issue
# labels (optional) - The labels of an issue
# state_event (optional) - The state event of an issue (close|reopen)
# Example Request:
# PUT /projects/:id/issues/:issue_id
put ":id/issues/:issue_id" do
set_current_user_for_thread do
@issue = user_project.issues.find(params[:issue_id])
authorize! :modify_issue, @issue
attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id, :state_event]
attrs[:label_list] = params[:labels] if params[:labels].present?
if @issue.update_attributes attrs
present @issue, with: Entities::Issue
else
not_found!
end
end
end
# Delete a project issue (deprecated)
#
# Parameters:
# id (required) - The ID of a project
# issue_id (required) - The ID of a project issue
# Example Request:
# DELETE /projects/:id/issues/:issue_id
delete ":id/issues/:issue_id" do
not_allowed!
end
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment