public
Last active

Debian packaging continuous integration

  • Download Gist
00-packages_builder.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
# Chef resources describing how to set up package repository server,
# simplified fromactual cookbook (not open sourced yet). Sets up apt
# repository in /srv/apt directory, with system user "apt-repo". Packages
# are GPG-signed to prevent apt-get from complaining on every install.
#
# Directory /srv/apt should be reachable to clients via http or other
# means. This is left as an exercise for the reader.
#
# For extra explanations, see:
# http://joseph.ruscio.org/blog/2010/08/19/setting-up-an-apt-repository/
# http://www.roethof.net/techblog/setting-up-an-apt-repository-with-reprepro/
# http://blog.mycrot.ch/2011/04/26/creating-your-own-signed-apt-repository-and-debian-packages/
 
package "reprepro" # manages apt repository
package "gnupg" # signs packages / indexes
package "build-essential" # glibc, gcc, and company
package "dpkg-dev" # basic debian packaging tools - just in case
package "fakeroot" # let us pretend we can chown stuff to root
gem_package "fpm" # the actual tool that packages stuff
 
# System user & group
group "apt-repo"
user "apt-repo" do
system true
group "apt-repo"
home "/srv/apt"
end
 
# Home & config directory
%w(/srv/apt /srv/apt/conf).each do |dir_path|
directory dir_path do
owner "apt-repo"
group "apt-repo"
mode "0755"
end
end
 
# Distributions file - see reprepro docs for details
template "/srv/apt/conf/distributions" do
owner "apt-repo"
group "apt-repo"
mode "0644"
variables :distributions => node.apt_repository_distributions
end
 
# Specification for non-interactive GPG key generation
file "#{Chef::Config[:file_cache_path]}/apt-key-spec.txt" do
content <<EOF
Key-Type: 1
Key-Length: 2048
Name-Real: #{node[:apt_repository][:key_real_name]}
Name-Email: #{node[:apt_repository][:key_email]}
Expire-Date: 0
EOF
end
 
# Generate GPG key (non-interactively with --batch)
execute "generate-apt-gpg-key" do
command "gpg --gen-key --batch < #{Chef::Config[:file_cache_path]}/apt-key-spec.txt"
user 'apt-repo'
group 'apt-repo'
environment "HOME" => '/srv/apt'
not_if { File.exist?('/srv/apt/.gnupg') }
end
 
# Remember the public key as a Chef attribute
ruby_block "save-apt-gpg-key" do
block do
node[:apt_repository][:public_gpg_key] = `sudo -u apt-repo -H gpg --export --armor`
end
end
 
# Helper script for simplicity
cookbook_file "/srv/apt/add_package.sh" do
owner "apt-repo"
group "apt-repo"
mode "0755"
end
00b-add_package.sh
Shell
1 2 3 4 5 6 7 8 9 10 11 12
#!/bin/sh
 
# Usage: /srv/apt/add_package.sh package.deb [package2.deb [package3.deb [...]]]
 
DISTRO=${DISTRO:-squeeze}
 
set -e -x
 
while [ -f "$1" ] ; do
sudo -u apt-repo reprepro -Vb /srv/apt includedeb $DISTRO "$1"
shift
done
01_client.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
# Chef resources to set up repository client.
 
package "lsb-release"
 
# Save public GPG key from Chef attribute to file
cached_keyfile = "#{Chef::Config[:file_cache_path]}/#{server.name}.apt-key"
file cached_keyfile do
content server[:apt_repository][:public_gpg_key]
end
 
# Add key as trusted
execute "install-apt-key #{server.name}" do
command "apt-key add #{cached_keyfile}"
end
 
# Figure out full repository URL
url = "http://"
url << "#{server[:apt_repository][:http_username]}:#{server[:apt_repository][:http_password]}@" if server[:apt_repository][:http_username]
url << "#{server[:apt_repository][:http_domain]}/"
 
# Add repository list
file "/etc/apt/sources.list.d/chef_apt_repository.list" do
content "deb #{url} #{server[:apt_repository][:codename]} #{server[:apt_repository][:components]}\n"
owner 'root'
group 'sysadmin'
mode '0640'
notifies :run, 'execute[apt-get update]', :immediately
end
02_packages_Gemfile.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12
# -*- ruby -*-
# Ruby gems used to build packages
 
source "http://rubygems.org"
 
gem "rake" # Build tool
gem "evoker", ">= 0.0.9" # Add-on to Rake for downloading upstream stuff
gem "fpm" # Effing Package Management
gem "escape" # Shell escapes for scripting
 
gem "vagrant", '~> 1.0.3', :group => :development
# VirtualBox command-line tool for developing packages on a clean VM
03_selenium-server_Rakefile.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
# Each package lives in its own subdirectory of the packages repo.
# Each package directory has its own Rakefile.
# Here's the structure of the Rakefile:
 
# First, some general constants used by magic later on.
PKG_VERSION = '2.21.0'
PKG_ITERATION = 2
PKG_ARCHITECTURE = 'all'
PKG_DEPS = ['sun-java6-jre', 'runit', 'xvfb', 'gnowsis-firefox (>= 11.0)']
 
# More specific constants, used by code below
CHECKSUM = '34983fbf870cefeab892786e3d4f9169ed7e9e17e6937b5fba5e2133603ca03a'
URL="http://selenium.googlecode.com/files/selenium-server-standalone-#{PKG_VERSION}.jar"
 
# Include the magic
load '../packaging.rake'
require 'evoker'
require 'evoker/local_cache'
include Evoker
 
# Download file to package
dl = cached_wget(URL,
:output_file => 'root/opt/selenium-server/selenium-server-standalone.jar',
:checksum => CHECKSUM)
 
# Call 'fpm' script via utility method defined in packaging.rake to build actual package
 
file PKG_FILE_NAME => dl do
fpm '-s dir --after-install after-install -C root opt/selenium-server'
end
05_dwim_all.sh
Shell
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
# This is the Buildbot step to rebuild all updated packages and add them to
# apt repository. It calls apt_dwim task for every subdirectory with a Rakefile.
#
# DWIM stands for "Do What I Mean": if the package to be built is already in the
# apt repository, this task does nothing; if the package is not there, it does a clean
# rebuild and adds it to the apt repository.
 
set -e
for rf in */Rakefile ; do
d=`dirname $rf`
cd $d
echo "*** $d"
fakeroot bundle exec rake apt_dwim
cd ..
done
06_packaging.rake
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
# -*- ruby -*-
 
# The magic happens here.
# This file is included from all the Rakefiles after defining basic constants
 
require 'rake/clean'
 
APT_ROOT = ENV['APT_ROOT'] || '/srv/apt'
 
# Set a constant if it's not defined yet
def _setdefault(const, value)
Object.const_set(const, value) unless Object.const_defined?(const)
end
 
# Try to figure out default values for constants that haven't been yet defined.
_setdefault(:PKG_NAME, File.basename(Dir.pwd))
_setdefault(:PKG_VERSION, '0.unspecified')
_setdefault(:PKG_ITERATION, 1)
_setdefault(:PKG_ARCHITECTURE, 'amd64')
_setdefault(:PKG_DISTROS, %w(squeeze))
_setdefault(:PKG_DEPS, [])
_setdefault(:PKG_BUILD_DEPS, [])
 
_setdefault(:PKG_FILE_NAME, "gnowsis-#{PKG_NAME}_#{PKG_VERSION}-gnowsis#{PKG_ITERATION}_#{PKG_ARCHITECTURE}.deb")
_setdefault(:PKG_FILES, [PKG_FILE_NAME]) # This will be defined earlier if the Rakefile
# builds multiple .deb files
 
CACHE_PATH = '../cache' # Download cache is shared between packages
 
CLEAN.include(*PKG_FILES)
 
# Call fpm tool, filling in the defaults
def fpm(*args)
require 'escape'
cmd = "bundle exec fpm -t deb -n gnowsis-#{PKG_NAME} "
cmd << "-v #{PKG_VERSION} --iteration gnowsis#{PKG_ITERATION} "
cmd << "-a #{PKG_ARCHITECTURE} -m info@gnowsis.com --license proprietary "
PKG_DEPS.each { |d| cmd << "-d #{Escape.shell_single_word(d)} " }
cmd << args.join(' ')
sh cmd
end
 
### Check build dependencies
 
# Return true if package is installed in the system.
def pkg_installed?(package)
system "dpkg-query -s #{package} > /dev/null 2>&1"
return $?.exitstatus.zero?
end
 
missing = PKG_BUILD_DEPS.select { |pkg| !pkg_installed?(pkg) }
unless missing.empty? || ENV['IGNORE_MISSING'] # Allow user to ignore missing build deps
puts "Missing packages:\n - #{missing.join("\n - ")}"
 
if ENV['INSTALL_DEPS'] # Allow user to let rake automatically install missing build deps
sh "sudo apt-get install --yes #{missing.join(' ')}"
else
puts "Install with: sudo apt-get install #{missing.join(' ')}"
raise "Cannot proceed."
end
end
 
### Apt repository stuff
 
# True if package_filename package is already in the apt repo
def already_in_apt?(package_filename)
!Dir[
File.join(APT_ROOT, 'pool', '*', '*', '*', File.basename(package_filename))
].empty?
end
 
desc "Build package and add it to apt"
task :add_to_apt => PKG_FILES do
PKG_FILES.each do |pkg|
sh "#{APT_ROOT}/add_package.sh #{pkg}"
end
end
 
desc "If package is not in apt repository, do clean rebuild, and add to apt."
if PKG_FILES.all? { |pkg| already_in_apt?(pkg) } and not ENV['FORCE_REBUILD']
task :apt_dwim do
puts "All packages already in repo, doing nothing."
end
else
task :apt_dwim => [ :clean, :add_to_apt ]
end
 
desc "Build package"
task :default => PKG_FILES
07_Vagrantfile.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
# -*- mode: ruby -*-
# vi: set ft=ruby :
 
# Configuration file for Vagrant (http://vagrantup.com/) to set up a virtual machine
# for package development.
 
# Figure out user data to set up git on the VM
USER_EMAIL = `git config user.email`.strip
USER_NAME = `git config user.name`.strip
 
# Find address of our custom apt repository to configure it automatically
# on the VM.
APT_REPOSITORY = `ssh [APT_HOST] cat /etc/apt/sources.list.d/chef_apt_repository.list`.strip
APT_KEY = `ssh [APT_HOST] sudo apt-key export apt-repo@[APT_HOST]`.strip
 
Vagrant::Config.run do |config|
config.ssh.forward_agent = true
 
config.vm.define "gnowsis-packages-vm" do |vm_cfg|
# Clean Debian Lenny base box
# http://dominique.broeglin.fr/2011/03/26/squeeze-64-vagrant-base-box.html
vm_cfg.vm.box = "squeeze64"
vm_cfg.vm.box_url = "http://dl.dropbox.com/u/937870/VMs/squeeze64.box"
 
vm_cfg.vm.provision :shell, :inline => <<EOF
# Provisioning script, ran as root when VM is first started
set -e -x
 
# Configure VM as client for custom apt repository
cat > /etc/apt/sources.list.d/chef_apt_repository.list <<EOR
#{APT_REPOSITORY}
EOR
 
cat <<EOK | apt-key add -
#{APT_KEY}
EOK
 
# General upgrade
apt-get update
apt-get upgrade --yes
 
# Needed packages (libxml2 & libxslt are needed as gems' prereqs later on)
apt-get install --yes git build-essential libxml2-dev libxslt1-dev
 
# Update rubygems themselves
gem install rubygems-update
update_rubygems
 
# Install gem bundler (http://gembundler.com/) to automatically install
# Ruby gems from the Gemfile
gem install bundler
 
# Share cache path with VM's host (directory of the Vagrantfile is seen by the
# VM as /vagrant/); install build dependencies automatically.
cat > /etc/profile.d/gnowsis_packages_settings.sh <<EOS
export CACHE_PATH=/vagrant/cache
export INSTALL_DEPS=1
EOS
 
# Now set up non-root account which we are `vagrant ssh'-ing.
su vagrant <<EOSU
cd ~
set -e -x
 
git config --global user.email #{USER_EMAIL.inspect}
git config --global user.name #{USER_NAME.inspect}
 
# Clone host VM's repository, using .git dir as bare repo
git clone /vagrant/.git packages
 
cd packages
 
# Create "develop" branch. It will fail if develop branch exists on host machine,
# but is not merged to master.
git checkout -b develop
git push origin develop:develop
 
# Download Ruby gems needed to run the Rakefiles
bundle --binstubs --path bundle/
EOSU
 
EOF
end
end
08_java_Rakefile.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
# With that setup, we don't even need to use fpm to build
# packages. For example, it's possible to build traditionally packaged
# https://github.com/rraptorr/sun-java6/ Oracle Java debs, by calling
# out to dpkg-buildpackage instead of fpm.
#
# If we want to use third-party packages that don't have their apt
# repository, we can skip the "build" part altogether and just
# download the .deb files and consider them ready.
 
PKG_VERSION = '6.32-1'
PKG_BUILD_DEPS = %w(lib32asound2 ia32-libs unixodbc libx11-6 libxext6 libxi6 libxt6 libxtst6)
 
PKG_FILES = { 'ia32-sun-java6-bin' => 'amd64',
'sun-java6-bin' => 'amd64',
'sun-java6-fonts' => 'all',
'sun-java6-javadb' => 'all',
'sun-java6-jdk' => 'amd64',
'sun-java6-jre' => 'all',
'sun-java6-plugin' => 'amd64',
'sun-java6-source' => 'all'
}.map { |name, arch| "#{name}_#{PKG_VERSION}_#{arch}.deb" }
 
GIT_TAG = "v#{PKG_VERSION}"
 
# These files are downloaded automatically from S3 by packaging.rake - this part of
# that file has been left out for clarity.
 
S3_FILES = {
"jdk-6u32-linux-i586.bin" => 'ab840c7de8f452b09d53ebe9477ffdb4622fa8cabbf962474d1914fde9a20f19',
"jdk-6u32-linux-x64.bin" => '269d05b8d88e583e4e66141514d8294e636f537f55eb50962233c9e33d8f8f49'
}
 
load '../packaging.rake'
require 'evoker'
include Evoker
 
pkg_repo = git('rraptor-sun-java6',
:url => 'git://github.com/rraptorr/sun-java6.git',
:revision => GIT_TAG)
 
task :build => [:s3_files, pkg_repo] do
S3_FILES.keys.each do |binf|
ln_sf "../#{binf}", "#{pkg_repo}/#{binf}"
end
sh "cd #{pkg_repo} && dpkg-buildpackage -b -uc -us"
end
 
PKG_FILES.each { |deb_file| file deb_file => [ :build ] }
 
CLEAN << pkg_repo
CLEAN << "sun-java6_#{PKG_VERSION}_#{PKG_ARCHITECTURE}.changes"
09_jetty_Rakefile.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
# A more involved fpm-using Rakefile
 
PKG_VERSION = '8.1.3.v20120416'
PKG_ITERATION = 5
PKG_ARCHITECTURE = 'all'
PKG_DEPS = %w(sun-java6-jdk runit)
 
CHECKSUM = 'd92baa45e100b348a8238f6199d194122719e14de23ece86821068d4bededbec'
URL="http://download.eclipse.org/jetty/#{PKG_VERSION}/dist/jetty-distribution-#{PKG_VERSION}.tar.gz"
 
load '../packaging.rake'
require 'evoker'
require 'evoker/local_cache'
include Evoker
 
dl = cached_wget(URL, :checksum => CHECKSUM)
 
task :install => dl do
sh "tar -C root/opt/jetty --strip-components 1 --exclude '*/javadoc' -xzf #{dl}"
chown_R 'root', 'root', 'root' if Process::Sys.getuid.zero? # fakeroot?
end
 
task :clean do
sh "git clean -fdX root/"
end
 
file PKG_FILE_NAME => :install do
cfcf = '--config-files /opt/jetty/etc/keystore'
Dir.chdir 'root' do
Dir['opt/jetty/etc/*.{xml,properties,conf,ini}'].each do |cf|
cfcf << " --config-files /#{cf}"
end
end
fpm "-s dir --before-install preinst.sh --after-install postinst.sh --before-remove prerm.sh -C root #{cfcf} opt/"
end

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.