Skip to content

Instantly share code, notes, and snippets.

@mpasternacki
Created May 29, 2012 09:44
  • Star 5 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save mpasternacki/2823575 to your computer and use it in GitHub Desktop.
Debian packaging continuous integration
# Chef resources describing how to set up package repository server,
# simplified fromactual cookbook (not open sourced yet). Sets up apt
# repository in /srv/apt directory, with system user "apt-repo". Packages
# are GPG-signed to prevent apt-get from complaining on every install.
#
# Directory /srv/apt should be reachable to clients via http or other
# means. This is left as an exercise for the reader.
#
# For extra explanations, see:
# http://joseph.ruscio.org/blog/2010/08/19/setting-up-an-apt-repository/
# http://www.roethof.net/techblog/setting-up-an-apt-repository-with-reprepro/
# http://blog.mycrot.ch/2011/04/26/creating-your-own-signed-apt-repository-and-debian-packages/
package "reprepro" # manages apt repository
package "gnupg" # signs packages / indexes
package "build-essential" # glibc, gcc, and company
package "dpkg-dev" # basic debian packaging tools - just in case
package "fakeroot" # let us pretend we can chown stuff to root
gem_package "fpm" # the actual tool that packages stuff
# System user & group
group "apt-repo"
user "apt-repo" do
system true
group "apt-repo"
home "/srv/apt"
end
# Home & config directory
%w(/srv/apt /srv/apt/conf).each do |dir_path|
directory dir_path do
owner "apt-repo"
group "apt-repo"
mode "0755"
end
end
# Distributions file - see reprepro docs for details
template "/srv/apt/conf/distributions" do
owner "apt-repo"
group "apt-repo"
mode "0644"
variables :distributions => node.apt_repository_distributions
end
# Specification for non-interactive GPG key generation
file "#{Chef::Config[:file_cache_path]}/apt-key-spec.txt" do
content <<EOF
Key-Type: 1
Key-Length: 2048
Name-Real: #{node[:apt_repository][:key_real_name]}
Name-Email: #{node[:apt_repository][:key_email]}
Expire-Date: 0
EOF
end
# Generate GPG key (non-interactively with --batch)
execute "generate-apt-gpg-key" do
command "gpg --gen-key --batch < #{Chef::Config[:file_cache_path]}/apt-key-spec.txt"
user 'apt-repo'
group 'apt-repo'
environment "HOME" => '/srv/apt'
not_if { File.exist?('/srv/apt/.gnupg') }
end
# Remember the public key as a Chef attribute
ruby_block "save-apt-gpg-key" do
block do
node[:apt_repository][:public_gpg_key] = `sudo -u apt-repo -H gpg --export --armor`
end
end
# Helper script for simplicity
cookbook_file "/srv/apt/add_package.sh" do
owner "apt-repo"
group "apt-repo"
mode "0755"
end
#!/bin/sh
# Usage: /srv/apt/add_package.sh package.deb [package2.deb [package3.deb [...]]]
DISTRO=${DISTRO:-squeeze}
set -e -x
while [ -f "$1" ] ; do
sudo -u apt-repo reprepro -Vb /srv/apt includedeb $DISTRO "$1"
shift
done
# Chef resources to set up repository client.
package "lsb-release"
# Save public GPG key from Chef attribute to file
cached_keyfile = "#{Chef::Config[:file_cache_path]}/#{server.name}.apt-key"
file cached_keyfile do
content server[:apt_repository][:public_gpg_key]
end
# Add key as trusted
execute "install-apt-key #{server.name}" do
command "apt-key add #{cached_keyfile}"
end
# Figure out full repository URL
url = "http://"
url << "#{server[:apt_repository][:http_username]}:#{server[:apt_repository][:http_password]}@" if server[:apt_repository][:http_username]
url << "#{server[:apt_repository][:http_domain]}/"
# Add repository list
file "/etc/apt/sources.list.d/chef_apt_repository.list" do
content "deb #{url} #{server[:apt_repository][:codename]} #{server[:apt_repository][:components]}\n"
owner 'root'
group 'sysadmin'
mode '0640'
notifies :run, 'execute[apt-get update]', :immediately
end
# -*- ruby -*-
# Ruby gems used to build packages
source "http://rubygems.org"
gem "rake" # Build tool
gem "evoker", ">= 0.0.9" # Add-on to Rake for downloading upstream stuff
gem "fpm" # Effing Package Management
gem "escape" # Shell escapes for scripting
gem "vagrant", '~> 1.0.3', :group => :development
# VirtualBox command-line tool for developing packages on a clean VM
# Each package lives in its own subdirectory of the packages repo.
# Each package directory has its own Rakefile.
# Here's the structure of the Rakefile:
# First, some general constants used by magic later on.
PKG_VERSION = '2.21.0'
PKG_ITERATION = 2
PKG_ARCHITECTURE = 'all'
PKG_DEPS = ['sun-java6-jre', 'runit', 'xvfb', 'gnowsis-firefox (>= 11.0)']
# More specific constants, used by code below
CHECKSUM = '34983fbf870cefeab892786e3d4f9169ed7e9e17e6937b5fba5e2133603ca03a'
URL="http://selenium.googlecode.com/files/selenium-server-standalone-#{PKG_VERSION}.jar"
# Include the magic
load '../packaging.rake'
require 'evoker'
require 'evoker/local_cache'
include Evoker
# Download file to package
dl = cached_wget(URL,
:output_file => 'root/opt/selenium-server/selenium-server-standalone.jar',
:checksum => CHECKSUM)
# Call 'fpm' script via utility method defined in packaging.rake to build actual package
file PKG_FILE_NAME => dl do
fpm '-s dir --after-install after-install -C root opt/selenium-server'
end
# This is the Buildbot step to rebuild all updated packages and add them to
# apt repository. It calls apt_dwim task for every subdirectory with a Rakefile.
#
# DWIM stands for "Do What I Mean": if the package to be built is already in the
# apt repository, this task does nothing; if the package is not there, it does a clean
# rebuild and adds it to the apt repository.
set -e
for rf in */Rakefile ; do
d=`dirname $rf`
cd $d
echo "*** $d"
fakeroot bundle exec rake apt_dwim
cd ..
done
# -*- ruby -*-
# The magic happens here.
# This file is included from all the Rakefiles after defining basic constants
require 'rake/clean'
APT_ROOT = ENV['APT_ROOT'] || '/srv/apt'
# Set a constant if it's not defined yet
def _setdefault(const, value)
Object.const_set(const, value) unless Object.const_defined?(const)
end
# Try to figure out default values for constants that haven't been yet defined.
_setdefault(:PKG_NAME, File.basename(Dir.pwd))
_setdefault(:PKG_VERSION, '0.unspecified')
_setdefault(:PKG_ITERATION, 1)
_setdefault(:PKG_ARCHITECTURE, 'amd64')
_setdefault(:PKG_DISTROS, %w(squeeze))
_setdefault(:PKG_DEPS, [])
_setdefault(:PKG_BUILD_DEPS, [])
_setdefault(:PKG_FILE_NAME, "gnowsis-#{PKG_NAME}_#{PKG_VERSION}-gnowsis#{PKG_ITERATION}_#{PKG_ARCHITECTURE}.deb")
_setdefault(:PKG_FILES, [PKG_FILE_NAME]) # This will be defined earlier if the Rakefile
# builds multiple .deb files
CACHE_PATH = '../cache' # Download cache is shared between packages
CLEAN.include(*PKG_FILES)
# Call fpm tool, filling in the defaults
def fpm(*args)
require 'escape'
cmd = "bundle exec fpm -t deb -n gnowsis-#{PKG_NAME} "
cmd << "-v #{PKG_VERSION} --iteration gnowsis#{PKG_ITERATION} "
cmd << "-a #{PKG_ARCHITECTURE} -m info@gnowsis.com --license proprietary "
PKG_DEPS.each { |d| cmd << "-d #{Escape.shell_single_word(d)} " }
cmd << args.join(' ')
sh cmd
end
### Check build dependencies
# Return true if package is installed in the system.
def pkg_installed?(package)
system "dpkg-query -s #{package} > /dev/null 2>&1"
return $?.exitstatus.zero?
end
missing = PKG_BUILD_DEPS.select { |pkg| !pkg_installed?(pkg) }
unless missing.empty? || ENV['IGNORE_MISSING'] # Allow user to ignore missing build deps
puts "Missing packages:\n - #{missing.join("\n - ")}"
if ENV['INSTALL_DEPS'] # Allow user to let rake automatically install missing build deps
sh "sudo apt-get install --yes #{missing.join(' ')}"
else
puts "Install with: sudo apt-get install #{missing.join(' ')}"
raise "Cannot proceed."
end
end
### Apt repository stuff
# True if package_filename package is already in the apt repo
def already_in_apt?(package_filename)
!Dir[
File.join(APT_ROOT, 'pool', '*', '*', '*', File.basename(package_filename))
].empty?
end
desc "Build package and add it to apt"
task :add_to_apt => PKG_FILES do
PKG_FILES.each do |pkg|
sh "#{APT_ROOT}/add_package.sh #{pkg}"
end
end
desc "If package is not in apt repository, do clean rebuild, and add to apt."
if PKG_FILES.all? { |pkg| already_in_apt?(pkg) } and not ENV['FORCE_REBUILD']
task :apt_dwim do
puts "All packages already in repo, doing nothing."
end
else
task :apt_dwim => [ :clean, :add_to_apt ]
end
desc "Build package"
task :default => PKG_FILES
# -*- mode: ruby -*-
# vi: set ft=ruby :
# Configuration file for Vagrant (http://vagrantup.com/) to set up a virtual machine
# for package development.
# Figure out user data to set up git on the VM
USER_EMAIL = `git config user.email`.strip
USER_NAME = `git config user.name`.strip
# Find address of our custom apt repository to configure it automatically
# on the VM.
APT_REPOSITORY = `ssh [APT_HOST] cat /etc/apt/sources.list.d/chef_apt_repository.list`.strip
APT_KEY = `ssh [APT_HOST] sudo apt-key export apt-repo@[APT_HOST]`.strip
Vagrant::Config.run do |config|
config.ssh.forward_agent = true
config.vm.define "gnowsis-packages-vm" do |vm_cfg|
# Clean Debian Lenny base box
# http://dominique.broeglin.fr/2011/03/26/squeeze-64-vagrant-base-box.html
vm_cfg.vm.box = "squeeze64"
vm_cfg.vm.box_url = "http://dl.dropbox.com/u/937870/VMs/squeeze64.box"
vm_cfg.vm.provision :shell, :inline => <<EOF
# Provisioning script, ran as root when VM is first started
set -e -x
# Configure VM as client for custom apt repository
cat > /etc/apt/sources.list.d/chef_apt_repository.list <<EOR
#{APT_REPOSITORY}
EOR
cat <<EOK | apt-key add -
#{APT_KEY}
EOK
# General upgrade
apt-get update
apt-get upgrade --yes
# Needed packages (libxml2 & libxslt are needed as gems' prereqs later on)
apt-get install --yes git build-essential libxml2-dev libxslt1-dev
# Update rubygems themselves
gem install rubygems-update
update_rubygems
# Install gem bundler (http://gembundler.com/) to automatically install
# Ruby gems from the Gemfile
gem install bundler
# Share cache path with VM's host (directory of the Vagrantfile is seen by the
# VM as /vagrant/); install build dependencies automatically.
cat > /etc/profile.d/gnowsis_packages_settings.sh <<EOS
export CACHE_PATH=/vagrant/cache
export INSTALL_DEPS=1
EOS
# Now set up non-root account which we are `vagrant ssh'-ing.
su vagrant <<EOSU
cd ~
set -e -x
git config --global user.email #{USER_EMAIL.inspect}
git config --global user.name #{USER_NAME.inspect}
# Clone host VM's repository, using .git dir as bare repo
git clone /vagrant/.git packages
cd packages
# Create "develop" branch. It will fail if develop branch exists on host machine,
# but is not merged to master.
git checkout -b develop
git push origin develop:develop
# Download Ruby gems needed to run the Rakefiles
bundle --binstubs --path bundle/
EOSU
EOF
end
end
# With that setup, we don't even need to use fpm to build
# packages. For example, it's possible to build traditionally packaged
# https://github.com/rraptorr/sun-java6/ Oracle Java debs, by calling
# out to dpkg-buildpackage instead of fpm.
#
# If we want to use third-party packages that don't have their apt
# repository, we can skip the "build" part altogether and just
# download the .deb files and consider them ready.
PKG_VERSION = '6.32-1'
PKG_BUILD_DEPS = %w(lib32asound2 ia32-libs unixodbc libx11-6 libxext6 libxi6 libxt6 libxtst6)
PKG_FILES = { 'ia32-sun-java6-bin' => 'amd64',
'sun-java6-bin' => 'amd64',
'sun-java6-fonts' => 'all',
'sun-java6-javadb' => 'all',
'sun-java6-jdk' => 'amd64',
'sun-java6-jre' => 'all',
'sun-java6-plugin' => 'amd64',
'sun-java6-source' => 'all'
}.map { |name, arch| "#{name}_#{PKG_VERSION}_#{arch}.deb" }
GIT_TAG = "v#{PKG_VERSION}"
# These files are downloaded automatically from S3 by packaging.rake - this part of
# that file has been left out for clarity.
S3_FILES = {
"jdk-6u32-linux-i586.bin" => 'ab840c7de8f452b09d53ebe9477ffdb4622fa8cabbf962474d1914fde9a20f19',
"jdk-6u32-linux-x64.bin" => '269d05b8d88e583e4e66141514d8294e636f537f55eb50962233c9e33d8f8f49'
}
load '../packaging.rake'
require 'evoker'
include Evoker
pkg_repo = git('rraptor-sun-java6',
:url => 'git://github.com/rraptorr/sun-java6.git',
:revision => GIT_TAG)
task :build => [:s3_files, pkg_repo] do
S3_FILES.keys.each do |binf|
ln_sf "../#{binf}", "#{pkg_repo}/#{binf}"
end
sh "cd #{pkg_repo} && dpkg-buildpackage -b -uc -us"
end
PKG_FILES.each { |deb_file| file deb_file => [ :build ] }
CLEAN << pkg_repo
CLEAN << "sun-java6_#{PKG_VERSION}_#{PKG_ARCHITECTURE}.changes"
# A more involved fpm-using Rakefile
PKG_VERSION = '8.1.3.v20120416'
PKG_ITERATION = 5
PKG_ARCHITECTURE = 'all'
PKG_DEPS = %w(sun-java6-jdk runit)
CHECKSUM = 'd92baa45e100b348a8238f6199d194122719e14de23ece86821068d4bededbec'
URL="http://download.eclipse.org/jetty/#{PKG_VERSION}/dist/jetty-distribution-#{PKG_VERSION}.tar.gz"
load '../packaging.rake'
require 'evoker'
require 'evoker/local_cache'
include Evoker
dl = cached_wget(URL, :checksum => CHECKSUM)
task :install => dl do
sh "tar -C root/opt/jetty --strip-components 1 --exclude '*/javadoc' -xzf #{dl}"
chown_R 'root', 'root', 'root' if Process::Sys.getuid.zero? # fakeroot?
end
task :clean do
sh "git clean -fdX root/"
end
file PKG_FILE_NAME => :install do
cfcf = '--config-files /opt/jetty/etc/keystore'
Dir.chdir 'root' do
Dir['opt/jetty/etc/*.{xml,properties,conf,ini}'].each do |cf|
cfcf << " --config-files /#{cf}"
end
end
fpm "-s dir --before-install preinst.sh --after-install postinst.sh --before-remove prerm.sh -C root #{cfcf} opt/"
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment