sangyye/README.md Secret

## README.md

      
    Raw
  

              README.md
            
          
    Manual for rietveld migration

I would recommend to use vagrant for running an testing, you have to install django to get this working. You need to have ruby version 1.9.3 or higher to run the scripts.

Create a folder rietveld and cd in it
Download all migration scripts https://gist.github.com/sangyye/2b32cfb91f70af275749
hg clone https://code.google.com/p/django-gae2django/ and cd in it.
Change in the folder examples/rietveld and open the Makefile and change the hg repo link to hg clone ssh://hg@bitbucket.org/adblockplus/rietveld -r importer rietveld.hg and comment out the patch calls. (you need permissions for the repo, ask in the irc #adblockplus on irc.mozilla.org)
go back to the main folder and run vagrant up to start up our virtual Machine and then vagrant ssh to enter it. and change to /vagrant/django-gae2django/examples/rietveld
run sudo apt-get update && sudo apt-get install python-django make mercurial
run make all & follow the instructions
run ./manage.py runserver 10.0.2.15:8000 congrants your own rietveld with my changes is running open your browser with the localhost:3030

Export Data of the existing rietveld

Create a folder called export and cd in it. Just start the ruby ../fetch_issue.rb script, it will end with ../fetch_issues.rb:35:in `<main>': undefined method `click' for nil:NilClass (NoMethodError) this will be fixed in a future version.
Import the Data in the created rietveld instance

First of all you need to copy the upload.py (should be at django-gae2django/examples/rietveld/upload.py it's a symlink, don't copy that) in the same place upload_issues.rb, and you should change the server name if it's something other than localhost:3030, all you need to do than is call the skript out of the folder were your backup is saved.

  
## fetch_issues.rb
# encoding: utf-8
require 'rubygems'
require 'mechanize'
require 'json'

agent = Mechanize.new
agent.pluggable_parser.default = Mechanize::Download
page = agent.get('http://codereview.adblockplus.org/all?limit=100')

count = 0 #just count the number of issues handelt so far

begin
	page.links.each do |link|
 		next if not link.text =~ /^\d+$/
 		count = count + 1
 		puts "This is issue:" + count.to_s
 		issue = JSON.parse(agent.get('http://codereview.adblockplus.org/api/' + link.text + '/?messages=true').body)
 		Dir.mkdir(link.text) unless File.directory?(link.text)
 		File.open(File.join(link.text, link.text + ".json"),"w") do |f|
  			f.write(issue.to_json)
		end
		pwd = Dir.getwd
		Dir.chdir(link.text)
 		issue["patchsets"].each do |patches|
 			filename = "issue" + link.text + "_" + patches.to_s + ".diff"
 			begin
 				agent.get("http://codereview.adblockplus.org/download/" + filename).save(filename)
 				agent.back
 			rescue StandardError => bang
				puts bang
 			end
 		end
 		Dir.chdir(pwd)
 		agent.back
 	end
end while page = agent.page.links_with(:text => 'Older ›')[1].click

## fetch_users.rb
require 'rubygems'
require 'json'

array = []

Dir['*'].each do |n|
	next if not File.directory?(n)
	n = n.to_i
	array.push(n)
end

users = {}

array.sort.each do |n|
	Dir.chdir(n.to_s) do
		jsonfile = n.to_s + ".json"
		hash = JSON.parse( IO.read(jsonfile) )
		users[hash['owner']] = hash['owner_email']
	end
end

print "Users\n"
users.each do |n|
	print "\t#{n[0]}\t => #{n[1]}\n"
end

## upload_issues.rb
# encoding: utf-8
require 'rubygems'
require 'json'
require 'uri'
require "yaml/store"

#global var
array = []
store = YAML::Store.new("/home/christian/src/rietveld/data.yml")

def escape_characters_in_string(string)
	#string = string.sub(/\s-\s/, ' ')
	pattern = /(\`|\'|\"|\.|\*|\(|\))/
    string.gsub(pattern){|match|"\\"  + match} # <-- Trying to take the currently found match and add a \ before it I have no idea how to do that).
end


Dir['*'].each do |n|
	next if not File.directory?(n)
	n = n.to_i
	array.push(n)
end

array.sort.each do |n|
	Dir.chdir(n.to_s) do
		print "Issue number: " + n.to_s + "\n"
		jsonfile = n.to_s + ".json"
		issues = Dir['issue*'].sort
		hash = JSON.parse( IO.read(jsonfile) )
		subject	= escape_characters_in_string(hash['subject'])
		if subject.length >= 100
			subject = subject.slice(0,90) + "..."
		end
		print "subject: #{subject}\n"
		#end
		print "\n"
		#store.transaction do
  		#	store[hash['issue']] = issue_infos[-1].split('/')[-1]
		#end
		issues.each do |i|
			print "Start uploading #{i}..."
			%x(python upload.py -m "#{subject}" --difffile #{i} -i #{n.to_s} --vcs Importer -s localhost:3030)
			print "Done.\n"
			sleep(1)
		end

		print "\nUpload json description file\n"
		%x(curl -H 'Content-Type: application/json' --data-urlencode "fields=$(cat #{jsonfile})" http://localhost:3030/#{n.to_s}/fields)

		print "\n"
		print "Sleep for 3 seconds..."
		sleep(3)
		print "Done.\n"
    end
end


## Vagrantfile
# -*- mode: ruby -*-
# vi: set ft=ruby :

# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"

Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  # All Vagrant configuration is done here. The most common configuration
  # options are documented and commented below. For a complete reference,
  # please see the online documentation at vagrantup.com.

  # Every Vagrant virtual environment requires a box to build off of.
  config.vm.box = "precise32"

  # The url from where the 'config.vm.box' box will be fetched if it
  # doesn't already exist on the user's system.
  config.vm.box_url = "http://files.vagrantup.com/precise32.box"

  # Create a forwarded port mapping which allows access to a specific port
  # within the machine from a port on the host machine. In the example below,
  # accessing "localhost:8080" will access port 80 on the guest machine.
  config.vm.network :forwarded_port, guest: 8000, host: 3030

  # Create a private network, which allows host-only access to the machine
  # using a specific IP.
  # config.vm.network :private_network, ip: "192.168.33.10"

  # Create a public network, which generally matched to bridged network.
  # Bridged networks make the machine appear as another physical device on
  # your network.
  # config.vm.network :public_network

  # If true, then any SSH connections made will enable agent forwarding.
  # Default value: false
  # config.ssh.forward_agent = true

  # Share an additional folder to the guest VM. The first argument is
  # the path on the host to the actual folder. The second argument is
  # the path on the guest to mount the folder. And the optional third
  # argument is a set of non-required options.
  # config.vm.synced_folder "../data", "/vagrant_data"

  # Provider-specific configuration so you can fine-tune various
  # backing providers for Vagrant. These expose provider-specific options.
  # Example for VirtualBox:
  #
  # config.vm.provider :virtualbox do |vb|
  #   # Don't boot with headless mode
  #   vb.gui = true
  #
  #   # Use VBoxManage to customize the VM. For example to change memory:
  #   vb.customize ["modifyvm", :id, "--memory", "1024"]
  # end
  #
  # View the documentation for the provider you're using for more
  # information on available options.

  # Enable provisioning with Puppet stand alone.  Puppet manifests
  # are contained in a directory path relative to this Vagrantfile.
  # You will need to create the manifests directory and a manifest in
  # the file precise32.pp in the manifests_path directory.
  #
  # An example Puppet manifest to provision the message of the day:
  #
  # # group { "puppet":
  # #   ensure => "present",
  # # }
  # #
  # # File { owner => 0, group => 0, mode => 0644 }
  # #
  # # file { '/etc/motd':
  # #   content => "Welcome to your Vagrant-built virtual machine!
  # #               Managed by Puppet.\n"
  # # }
  #
  # config.vm.provision :puppet do |puppet|
  #   puppet.manifests_path = "manifests"
  #   puppet.manifest_file  = "site.pp"
  # end

  # Enable provisioning with chef solo, specifying a cookbooks path, roles
  # path, and data_bags path (all relative to this Vagrantfile), and adding
  # some recipes and/or roles.
  #
  # config.vm.provision :chef_solo do |chef|
  #   chef.cookbooks_path = "../my-recipes/cookbooks"
  #   chef.roles_path = "../my-recipes/roles"
  #   chef.data_bags_path = "../my-recipes/data_bags"
  #   chef.add_recipe "mysql"
  #   chef.add_role "web"
  #
  #   # You may also specify custom JSON attributes:
  #   chef.json = { :mysql_password => "foo" }
  # end

  # Enable provisioning with chef server, specifying the chef server URL,
  # and the path to the validation key (relative to this Vagrantfile).
  #
  # The Opscode Platform uses HTTPS. Substitute your organization for
  # ORGNAME in the URL and validation key.
  #
  # If you have your own Chef Server, use the appropriate URL, which may be
  # HTTP instead of HTTPS depending on your configuration. Also change the
  # validation key to validation.pem.
  #
  # config.vm.provision :chef_client do |chef|
  #   chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME"
  #   chef.validation_key_path = "ORGNAME-validator.pem"
  # end
  #
  # If you're using the Opscode platform, your validator client is
  # ORGNAME-validator, replacing ORGNAME with your organization name.
  #
  # If you have your own Chef Server, the default validation client name is
  # chef-validator, unless you changed the configuration.
  #
  #   chef.validation_client_name = "ORGNAME-validator"
end
	# encoding: utf-8
	require 'rubygems'
	require 'mechanize'
	require 'json'

	agent = Mechanize.new
	agent.pluggable_parser.default = Mechanize::Download
	page = agent.get('http://codereview.adblockplus.org/all?limit=100')

	count = 0 #just count the number of issues handelt so far

	begin
	page.links.each do \|link\|
	next if not link.text =~ /^\d+$/
	count = count + 1
	puts "This is issue:" + count.to_s
	issue = JSON.parse(agent.get('http://codereview.adblockplus.org/api/' + link.text + '/?messages=true').body)
	Dir.mkdir(link.text) unless File.directory?(link.text)
	File.open(File.join(link.text, link.text + ".json"),"w") do \|f\|
	f.write(issue.to_json)
	end
	pwd = Dir.getwd
	Dir.chdir(link.text)
	issue["patchsets"].each do \|patches\|
	filename = "issue" + link.text + "_" + patches.to_s + ".diff"
	begin
	agent.get("http://codereview.adblockplus.org/download/" + filename).save(filename)
	agent.back
	rescue StandardError => bang
	puts bang
	end
	end
	Dir.chdir(pwd)
	agent.back
	end
	end while page = agent.page.links_with(:text => 'Older ›')[1].click
	require 'rubygems'
	require 'json'

	array = []

	Dir['*'].each do \|n\|
	next if not File.directory?(n)
	n = n.to_i
	array.push(n)
	end

	users = {}

	array.sort.each do \|n\|
	Dir.chdir(n.to_s) do
	jsonfile = n.to_s + ".json"
	hash = JSON.parse( IO.read(jsonfile) )
	users[hash['owner']] = hash['owner_email']
	end
	end

	print "Users\n"
	users.each do \|n\|
	print "\t#{n[0]}\t => #{n[1]}\n"
	end
	# -- mode: ruby --
	# vi: set ft=ruby :

	# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
	VAGRANTFILE_API_VERSION = "2"

	Vagrant.configure(VAGRANTFILE_API_VERSION) do \|config\|
	# All Vagrant configuration is done here. The most common configuration
	# options are documented and commented below. For a complete reference,
	# please see the online documentation at vagrantup.com.

	# Every Vagrant virtual environment requires a box to build off of.
	config.vm.box = "precise32"

	# The url from where the 'config.vm.box' box will be fetched if it
	# doesn't already exist on the user's system.
	config.vm.box_url = "http://files.vagrantup.com/precise32.box"

	# Create a forwarded port mapping which allows access to a specific port
	# within the machine from a port on the host machine. In the example below,
	# accessing "localhost:8080" will access port 80 on the guest machine.
	config.vm.network :forwarded_port, guest: 8000, host: 3030

	# Create a private network, which allows host-only access to the machine
	# using a specific IP.
	# config.vm.network :private_network, ip: "192.168.33.10"

	# Create a public network, which generally matched to bridged network.
	# Bridged networks make the machine appear as another physical device on
	# your network.
	# config.vm.network :public_network

	# If true, then any SSH connections made will enable agent forwarding.
	# Default value: false
	# config.ssh.forward_agent = true

	# Share an additional folder to the guest VM. The first argument is
	# the path on the host to the actual folder. The second argument is
	# the path on the guest to mount the folder. And the optional third
	# argument is a set of non-required options.
	# config.vm.synced_folder "../data", "/vagrant_data"

	# Provider-specific configuration so you can fine-tune various
	# backing providers for Vagrant. These expose provider-specific options.
	# Example for VirtualBox:
	#
	# config.vm.provider :virtualbox do \|vb\|
	# # Don't boot with headless mode
	# vb.gui = true
	#
	# # Use VBoxManage to customize the VM. For example to change memory:
	# vb.customize ["modifyvm", :id, "--memory", "1024"]
	# end
	#
	# View the documentation for the provider you're using for more
	# information on available options.

	# Enable provisioning with Puppet stand alone. Puppet manifests
	# are contained in a directory path relative to this Vagrantfile.
	# You will need to create the manifests directory and a manifest in
	# the file precise32.pp in the manifests_path directory.
	#
	# An example Puppet manifest to provision the message of the day:
	#
	# # group { "puppet":
	# # ensure => "present",
	# # }
	# #
	# # File { owner => 0, group => 0, mode => 0644 }
	# #
	# # file { '/etc/motd':
	# # content => "Welcome to your Vagrant-built virtual machine!
	# # Managed by Puppet.\n"
	# # }
	#
	# config.vm.provision :puppet do \|puppet\|
	# puppet.manifests_path = "manifests"
	# puppet.manifest_file = "site.pp"
	# end

	# Enable provisioning with chef solo, specifying a cookbooks path, roles
	# path, and data_bags path (all relative to this Vagrantfile), and adding
	# some recipes and/or roles.
	#
	# config.vm.provision :chef_solo do \|chef\|
	# chef.cookbooks_path = "../my-recipes/cookbooks"
	# chef.roles_path = "../my-recipes/roles"
	# chef.data_bags_path = "../my-recipes/data_bags"
	# chef.add_recipe "mysql"
	# chef.add_role "web"
	#
	# # You may also specify custom JSON attributes:
	# chef.json = { :mysql_password => "foo" }
	# end

	# Enable provisioning with chef server, specifying the chef server URL,
	# and the path to the validation key (relative to this Vagrantfile).
	#
	# The Opscode Platform uses HTTPS. Substitute your organization for
	# ORGNAME in the URL and validation key.
	#
	# If you have your own Chef Server, use the appropriate URL, which may be
	# HTTP instead of HTTPS depending on your configuration. Also change the
	# validation key to validation.pem.
	#
	# config.vm.provision :chef_client do \|chef\|
	# chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME"
	# chef.validation_key_path = "ORGNAME-validator.pem"
	# end
	#
	# If you're using the Opscode platform, your validator client is
	# ORGNAME-validator, replacing ORGNAME with your organization name.
	#
	# If you have your own Chef Server, the default validation client name is
	# chef-validator, unless you changed the configuration.
	#
	# chef.validation_client_name = "ORGNAME-validator"
	end