Skip to content

Embed URL

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
DEB/RPM repository maintenance script
# GistID: 1924422
# Author:: Couchbase <info@couchbase.com>
# Copyright:: 2011, 2012 Couchbase, Inc.
# License:: Apache License, Version 2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This is a collection of the task helping to maintain RPM and DEB
# repositories.
#
# USAGE
# =====
#
# The most interesting and useful tasks are:
#
# Synchronize DEB/RPM repositories with Amazon S3. (First time it should
# create master copy of the repositories at HOME. The target s3 bucket
# should be specified in S3_PKG_MIRROR environment variable (for example
# s3://packages.example.com)
#
# * rake -g master:sync
#
# Tasks to upload artifacts from current directory. You should specify the
# target master host in MASTER_HOST environment variable (for example
# master@masternode.com)
#
# * rake -g builder:deb:upload:lucid
#
# $ sudo apt-get install rake reprepro createrepo s3cmd
# $ mkdir $HOME/.rake
# $ cp repositories.rake $HOME/.rake/
#
# Also make sure that all instances have GPG keys to sign the packages and
# the repository itself.
#
# Following crontab stuff helps to import incoming packages automatically
# every 5 minutes
#
# S3_PKG_MIRROR=s3://package.example.net
# GPG_KEY=872CF7D3
# HOME=/home/master
# */5 * * * * /usr/bin/rake -g master:sync > /dev/null
#
# To create special branches (like preview) use PREFIX environment variable
#
require 'pathname'
namespace :builder do
desc "Check the builder readiness"
task :check do
abort if tool_is_missing("ssh -V", "openssh-client") ||
tool_is_missing("gpg --version", "gnupg") ||
var_is_missing("MASTER_HOST")
end
namespace :deb do
["lucid", "oneiric"].each do |dist|
desc "Upload DEB packages for Ubuntu #{dist}"
task "upload:#{dist}" => :check do
Dir["*.{changes,deb,dsc,tar.gz}"].each do |file|
sh("scp #{file} #{ENV['MASTER_HOST']}:#{ENV['PREFIX']}ubuntu/incoming/#{dist}")
end
end
end
end
namespace :rpm do
["5.5", "6.2"].each do |dist|
desc "Upload RPM packages for CentOS #{dist}"
task "upload:centos#{dist}" => :check do
Dir["*.rpm"].each do |file|
sh("scp #{file} #{ENV['MASTER_HOST']}:#{ENV['PREFIX']}rpm/#{dist}/incoming")
end
end
end
end
end
namespace :master do
desc "Check the master readiness"
task :check do
abort if tool_is_missing("reprepro --version", "reprepro") ||
tool_is_missing("s3cmd --version", "s3cmd") ||
tool_is_missing("createrepo --version", "createrepo") ||
tool_is_missing("gpg --version", "gnupg") ||
tool_is_missing("expect -v", "expect") ||
var_is_missing("S3_PKG_MIRROR") ||
(var_is_missing("GPG_KEY") && var_is_missing("RPM_GPG_KEY")) ||
var_is_missing("HOME") ||
file_is_missing(File.join(ENV['HOME'], '.s3cfg'), "run 's3cmd --configure' to create") ||
file_is_missing(File.join(File.dirname(__FILE__), 'sign_rpm.expect'), "copy sign_rpm.expect script to #{File.dirname(__FILE__)}")
PREFIX = Pathname.new(ENV['HOME'])
if ENV['PREFIX']
PREFIX = PREFIX.join(ENV['PREFIX'])
end
end
namespace :deb do
desc "Populate DEB repository structure"
task :seed => :check do
repo = PREFIX.join("ubuntu")
unless repo.join(".checkpoint").exist?
mkdir_p(repo.join("conf"))
ubuntu_dists = {"lucid" => "10.04", "oneiric" => "11.10"}
File.open(repo.join("conf", "distributions"), "w+") do |f|
ubuntu_dists.each do |name, ver|
mkdir_p(repo.join("pool"))
mkdir_p(repo.join("dists", name))
mkdir_p(repo.join("incoming", name))
f.puts(<<-EOC.gsub(/^\s+/, ''))
Origin: couchbase
SignWith: #{ENV['GPG_KEY']}
Suite: #{name}
Codename: #{name}
Version: #{ver}
Components: #{name}/main
Architectures: amd64 i386 source
Description: Couchbase package repository
EOC
f.puts
end
end
touch(repo.join(".checkpoint"))
end
end
desc "Import DEB packages from incoming queues"
task :import => :seed do
repo = PREFIX.join("ubuntu")
# import from incoming (only .change files)
incoming = repo.join('incoming')
Dir[incoming.join('*')].each do |path|
if File.directory?(path)
codename = File.basename(path)
Dir[File.join(path, "*.changes")].each do |change|
sh("reprepro -T deb -V --ignore=wrongdistribution -b #{repo} include #{codename} #{change}")
end
FileUtils.rm_rf(Dir[File.join(path, "*")])
end
end
# import plain deb files (this is how the server ship packages)
incoming = repo.join('incoming-deb')
Dir[incoming.join('*')].each do |path|
if File.directory?(path)
codename = File.basename(path)
Dir[File.join(path, "*.deb")].each do |package|
sh("reprepro -T deb -V --ignore=wrongdistribution -b #{repo} includedeb #{codename} #{package}")
end
FileUtils.rm_rf(Dir[File.join(path, "*")])
end
end
# fix S3 space handling
names = `find #{repo.join('pool')} -name '*+*'`.split
names.each do |name|
FileUtils.cp(name, name.sub('+', ' '))
FileUtils.cp(name, name.sub('+', '%2B'))
task :import => :seed do
repo = PREFIX.join("rpm")
["5.5", "6.2"].each do |ver|
incoming = repo.join(ver, 'incoming')
map = {'*.src.rpm' => 'SRPMS',
'*.x86_64.rpm' => 'x86_64',
'*.x86_64*rel.rpm' => 'x86_64', # HACK for server
'*.i686.rpm' => 'i686',
'*.x86_[0-9].*rel.rpm' => 'i386', # HACK for server
'*.i386.rpm' => 'i386'}
map.each do |glob, target|
Dir[incoming.join(glob)].each do |pkg|
FileUtils.mv(pkg, repo.join(ver, target))
end
sh("createrepo --update #{repo.join(ver, target)}")
end
end
end
desc "Sign RPM repository"
task :sign => :seed do
repo = PREFIX.join("rpm")
abort if var_is_missing('RPM_GPG_KEY')
sh("expect #{File.join(File.dirname(__FILE__), "sign_rpm.expect")} #{ENV['RPM_GPG_KEY']} #{repo}")
["5.5", "6.2"].each do |ver|
['SRPMS', 'x86_64', 'i386', 'i686'].each do |target|
sh("gpg --batch --yes -u #{ENV['RPM_GPG_KEY']} --detach-sign --armor #{repo.join(ver, target, 'repodata/repomd.xml')}")
end
end
end
desc "Syncronize Amazon S3 mirrors for RPM repositories"
task :sync => [:import, :sign] do
repo = PREFIX.join("rpm")
sh("s3cmd sync -P #{repo}/ #{ENV['S3_PKG_MIRROR']}/#{ENV['PREFIX']}rpm/")
end
end
desc "Synchronyze all repositories"
task :sync => ["deb:sync", "rpm:sync"]
end
def tool_is_missing(command, package)
print "checking for #{package} tool ... "
`#{command} >/dev/null 2>&1`
unless $?.success?
puts("missing. apt-get install #{package}")
return true
end
puts `which #{command.split(' ').first}`.chomp
return false
end
def var_is_missing(name)
print "checking for #{name} variable ... "
if ENV[name].nil? || ENV[name].empty?
puts "missing"
return true
end
puts ENV[name]
return false
end
def file_is_missing(name, message)
print "checking for #{name} file ... "
if name.nil? || !File.exist?(name)
puts("missing. #{message}")
return true
end
puts 'ok'
return false
end
#!/usr/bin/expect -f
set gpgName [lrange $argv 0 0]
set repo [lrange $argv 1 1]
foreach rpmfile [exec find $repo -name *.rpm] {
spawn rpm --resign -D "_signature gpg" -D "_gpg_name $gpgName" $rpmfile
expect -exact "Enter pass phrase: "
send -- "\r"
expect eof
wait
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Something went wrong with that request. Please try again.