public
Last active

DEB/RPM repository maintenance script

  • Download Gist
repositories.rake
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
# GistID: 1924422
# Author:: Couchbase <info@couchbase.com>
# Copyright:: 2011, 2012 Couchbase, Inc.
# License:: Apache License, Version 2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
 
# This is a collection of the task helping to maintain RPM and DEB
# repositories.
#
# USAGE
# =====
#
# The most interesting and useful tasks are:
#
# Synchronize DEB/RPM repositories with Amazon S3. (First time it should
# create master copy of the repositories at HOME. The target s3 bucket
# should be specified in S3_PKG_MIRROR environment variable (for example
# s3://packages.example.com)
#
# * rake -g master:sync
#
# Tasks to upload artifacts from current directory. You should specify the
# target master host in MASTER_HOST environment variable (for example
# master@masternode.com)
#
# * rake -g builder:deb:upload:lucid
#
# $ sudo apt-get install rake reprepro createrepo s3cmd
# $ mkdir $HOME/.rake
# $ cp repositories.rake $HOME/.rake/
#
# Also make sure that all instances have GPG keys to sign the packages and
# the repository itself.
#
# Following crontab stuff helps to import incoming packages automatically
# every 5 minutes
#
# S3_PKG_MIRROR=s3://package.example.net
# GPG_KEY=872CF7D3
# HOME=/home/master
# */5 * * * * /usr/bin/rake -g master:sync > /dev/null
#
# To create special branches (like preview) use PREFIX environment variable
#
require 'pathname'
 
namespace :builder do
desc "Check the builder readiness"
task :check do
abort if tool_is_missing("ssh -V", "openssh-client") ||
tool_is_missing("gpg --version", "gnupg") ||
var_is_missing("MASTER_HOST")
end
 
namespace :deb do
["lucid", "oneiric"].each do |dist|
desc "Upload DEB packages for Ubuntu #{dist}"
task "upload:#{dist}" => :check do
Dir["*.{changes,deb,dsc,tar.gz}"].each do |file|
sh("scp #{file} #{ENV['MASTER_HOST']}:#{ENV['PREFIX']}ubuntu/incoming/#{dist}")
end
end
end
end
 
namespace :rpm do
["5.5", "6.2"].each do |dist|
desc "Upload RPM packages for CentOS #{dist}"
task "upload:centos#{dist}" => :check do
Dir["*.rpm"].each do |file|
sh("scp #{file} #{ENV['MASTER_HOST']}:#{ENV['PREFIX']}rpm/#{dist}/incoming")
end
end
end
end
end
 
namespace :master do
desc "Check the master readiness"
task :check do
abort if tool_is_missing("reprepro --version", "reprepro") ||
tool_is_missing("s3cmd --version", "s3cmd") ||
tool_is_missing("createrepo --version", "createrepo") ||
tool_is_missing("gpg --version", "gnupg") ||
tool_is_missing("expect -v", "expect") ||
var_is_missing("S3_PKG_MIRROR") ||
(var_is_missing("GPG_KEY") && var_is_missing("RPM_GPG_KEY")) ||
var_is_missing("HOME") ||
file_is_missing(File.join(ENV['HOME'], '.s3cfg'), "run 's3cmd --configure' to create") ||
file_is_missing(File.join(File.dirname(__FILE__), 'sign_rpm.expect'), "copy sign_rpm.expect script to #{File.dirname(__FILE__)}")
PREFIX = Pathname.new(ENV['HOME'])
if ENV['PREFIX']
PREFIX = PREFIX.join(ENV['PREFIX'])
end
end
 
namespace :deb do
desc "Populate DEB repository structure"
task :seed => :check do
repo = PREFIX.join("ubuntu")
unless repo.join(".checkpoint").exist?
mkdir_p(repo.join("conf"))
ubuntu_dists = {"lucid" => "10.04", "oneiric" => "11.10"}
File.open(repo.join("conf", "distributions"), "w+") do |f|
ubuntu_dists.each do |name, ver|
mkdir_p(repo.join("pool"))
mkdir_p(repo.join("dists", name))
mkdir_p(repo.join("incoming", name))
f.puts(<<-EOC.gsub(/^\s+/, ''))
Origin: couchbase
SignWith: #{ENV['GPG_KEY']}
Suite: #{name}
Codename: #{name}
Version: #{ver}
Components: #{name}/main
Architectures: amd64 i386 source
Description: Couchbase package repository
EOC
f.puts
end
end
touch(repo.join(".checkpoint"))
end
end
 
desc "Import DEB packages from incoming queues"
task :import => :seed do
repo = PREFIX.join("ubuntu")
# import from incoming (only .change files)
incoming = repo.join('incoming')
Dir[incoming.join('*')].each do |path|
if File.directory?(path)
codename = File.basename(path)
Dir[File.join(path, "*.changes")].each do |change|
sh("reprepro -T deb -V --ignore=wrongdistribution -b #{repo} include #{codename} #{change}")
end
FileUtils.rm_rf(Dir[File.join(path, "*")])
end
end
# import plain deb files (this is how the server ship packages)
incoming = repo.join('incoming-deb')
Dir[incoming.join('*')].each do |path|
if File.directory?(path)
codename = File.basename(path)
Dir[File.join(path, "*.deb")].each do |package|
sh("reprepro -T deb -V --ignore=wrongdistribution -b #{repo} includedeb #{codename} #{package}")
end
FileUtils.rm_rf(Dir[File.join(path, "*")])
end
end
# fix S3 space handling
names = `find #{repo.join('pool')} -name '*+*'`.split
names.each do |name|
FileUtils.cp(name, name.sub('+', ' '))
FileUtils.cp(name, name.sub('+', '%2B'))
task :import => :seed do
repo = PREFIX.join("rpm")
["5.5", "6.2"].each do |ver|
incoming = repo.join(ver, 'incoming')
map = {'*.src.rpm' => 'SRPMS',
'*.x86_64.rpm' => 'x86_64',
'*.x86_64*rel.rpm' => 'x86_64', # HACK for server
'*.i686.rpm' => 'i686',
'*.x86_[0-9].*rel.rpm' => 'i386', # HACK for server
'*.i386.rpm' => 'i386'}
 
map.each do |glob, target|
Dir[incoming.join(glob)].each do |pkg|
FileUtils.mv(pkg, repo.join(ver, target))
end
sh("createrepo --update #{repo.join(ver, target)}")
end
end
end
 
desc "Sign RPM repository"
task :sign => :seed do
repo = PREFIX.join("rpm")
abort if var_is_missing('RPM_GPG_KEY')
sh("expect #{File.join(File.dirname(__FILE__), "sign_rpm.expect")} #{ENV['RPM_GPG_KEY']} #{repo}")
["5.5", "6.2"].each do |ver|
['SRPMS', 'x86_64', 'i386', 'i686'].each do |target|
sh("gpg --batch --yes -u #{ENV['RPM_GPG_KEY']} --detach-sign --armor #{repo.join(ver, target, 'repodata/repomd.xml')}")
end
end
end
 
desc "Syncronize Amazon S3 mirrors for RPM repositories"
task :sync => [:import, :sign] do
repo = PREFIX.join("rpm")
sh("s3cmd sync -P #{repo}/ #{ENV['S3_PKG_MIRROR']}/#{ENV['PREFIX']}rpm/")
end
end
 
desc "Synchronyze all repositories"
task :sync => ["deb:sync", "rpm:sync"]
end
 
def tool_is_missing(command, package)
print "checking for #{package} tool ... "
`#{command} >/dev/null 2>&1`
unless $?.success?
puts("missing. apt-get install #{package}")
return true
end
puts `which #{command.split(' ').first}`.chomp
return false
end
 
def var_is_missing(name)
print "checking for #{name} variable ... "
if ENV[name].nil? || ENV[name].empty?
puts "missing"
return true
end
puts ENV[name]
return false
end
 
def file_is_missing(name, message)
print "checking for #{name} file ... "
if name.nil? || !File.exist?(name)
puts("missing. #{message}")
return true
end
puts 'ok'
return false
end
sign_rpm.expect
1 2 3 4 5 6 7 8 9 10 11 12
#!/usr/bin/expect -f
 
set gpgName [lrange $argv 0 0]
set repo [lrange $argv 1 1]
 
foreach rpmfile [exec find $repo -name *.rpm] {
spawn rpm --resign -D "_signature gpg" -D "_gpg_name $gpgName" $rpmfile
expect -exact "Enter pass phrase: "
send -- "\r"
expect eof
wait
}

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.