Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
build package with cowbuilder or git-buildpackage on Jenkins
#!/usr/bin/python
# -*- coding: utf-8 -*-
''' debbuild_wrapper.py
The debian package build wrapper on Jenkins.
This script support three way of building debian package.
1-1. Backport from source package with cowbuilder.
1-2. Pre-build and Backport from source package with cowbuilder.
2. Build original package with git-buildpackage.
Support flavors are "debian" or "ubuntu".
'''
import deb822
import shutil
import argparse
import subprocess
import pexpect
import os.path
import sys
import pwd
import shlex
from glob import glob
from pydebsign import debsign
def parse_options():
""" building options """
parser = argparse.ArgumentParser(description='usage')
parser.add_argument('-c', '--codename', action='store', required=True,
help='specify codename of distribution')
# for piuparts
parser.add_argument('-f', '--flavor', choices=['debian', 'ubuntu'],
default='debian',
help='specify flavor debian or ubuntu (for piuparts)')
parser.add_argument('-m', '--mirror', action='store',
help='specify Debian mirror (for piuparts)')
# Backport version is different with an existing version in the arvhive,
# so generally upgrade test fails.
parser.add_argument('-n', '--no-upgrade-test', action='store_true',
help=('Skip testing upgrade from '
'an existing version in the archive.'))
# without lintian checking
parser.add_argument('--without-lintian', action='store_true',
help=('Skip checking with lintian'))
# for dput
parser.add_argument('-p', '--passphrase', action='store', required=True,
help='GPG private key passphrase of uploder')
parser.add_argument('-r', '--reprepro-passphrase', action='store',
required=True,
help='GPG Private key passphrase of reprepro')
subparser = parser.add_subparsers(title='subcommands',
dest='subparser_name')
backport = subparser.add_parser('backport', help='backport package')
subparser.add_parser('original', help='original package')
backport.add_argument('-u', '--url', action='store', required=True,
help='specify .dsc url')
backport.add_argument('-P', '--prebuild', action='store_true',
help='pre-build debian package')
backport.add_argument('--debfullname', action='store')
backport.add_argument('--debemail', action='store')
return parser.parse_args()
def set_work_dirpath():
""" set work_dirpath variable
expected value is WORKSPACE on Jenkins,
current directory is on local test.
"""
if os.environ.get('WORKSPACE'):
work_dirpath = os.environ.get('WORKSPACE')
else:
work_dirpath = os.path.abspath(os.path.curdir)
return work_dirpath
def make_directories(dir_list):
""" make directories for results """
[os.mkdir(dir_name) for dir_name in dir_list
if os.path.isdir(dir_name) is False]
def parse_changes(changes_filepath):
""" parse .changes file """
with open(changes_filepath, 'rb') as changes_file:
changes = deb822.Changes(changes_file)
return [_file for _file in changes['Files']]
def get_architecture():
""" get DEB_BUILD_ARCH """
command = 'dpkg-architecture -qDEB_BUILD_ARCH'
return subprocess.check_output(
shlex.split(command)).split()[0].decode('utf-8')
def retrieve_source_package(dsc_url):
""" retrieve source package with dget command """
command = 'dget -d %s' % dsc_url
return subprocess.call(shlex.split(command))
def get_changes_name(dsc_name, arch):
""" get .changes file name """
pkg_name = dsc_name.split('.dsc')[0]
changes_name = '%s_%s.changes' % (pkg_name, arch)
return changes_name
def get_basepath(codename):
""" get cowbuilder basepath """
return '/var/cache/pbuilder/base-%s.cow' % codename
def get_basetgz(codename):
""" get pbuilder basetgz path """
return '/var/cache/pbuilder/base-%s.tgz' % codename
def update_cowbuilder(basepath):
""" update cowbuilder image """
command = 'sudo /usr/sbin/cowbuilder --update --basepath %s' % basepath
return subprocess.call(shlex.split(command))
def extract_source_package(dsc_path):
""" extract source package and meta data from .control file """
command = 'dpkg-source -x %s' % dsc_path
subprocess.call(shlex.split(command))
with open(dsc_path, 'rb') as dsc_file:
srcs = deb822.Sources(dsc_file)
build_depends = srcs.get('Build-Depends').split(',')
if srcs.get('Build-Depends-Indep'):
build_depends += (srcs.get('Build-Depends-Indep').split(','))
data = {'source_dirname': '%s-%s' % (srcs.get('Source'),
srcs.get('Version').split('-')[0]),
'build_depends': build_depends,
'dsc_name': os.path.basename(dsc_path),
'upstream_version': srcs.get('Version').split('-')[0],
'source_package_name': srcs.get('Source'),
'debian_version': srcs.get('Version'),
'batch_name': '%s-batch.sh' % srcs.get('Source')}
return data
def append_param(param_dict, args):
""" append variables to param_dict from args """
param_dict['dsc_url'] = args.url
param_dict['debfullname'] = args.debfullname
param_dict['debemail'] = args.debemail
return param_dict
def generate_batch_script(param_dict):
""" generate batch script for cowbuilder """
return ('#!/bin/sh -x\n'
'export DEBFULLNAME="%(debfullname)s"\n'
'export DEBEMAIL=%(debemail)s\n'
'apt-get -y install curl devscripts quilt '
'patch libdistro-info-perl fakeroot\n'
'apt-get -y build-dep %(source_package_name)s\n'
'dget -d %(dsc_url)s\n'
'dpkg-source -x %(dsc_name)s\n'
'(\n'
'cd %(source_dirname)s\n'
'debuild -us -uc\n'
')\n'
'cp -f %(source_package_name)s_%(debian_version)s.debian.tar.gz '
'%(source_package_name)s_%(upstream_version)s.orig.tar.gz '
'%(source_package_name)s_%(debian_version)s.dsc '
'%(temp_dirpath)s/\n'
% param_dict)
def prepare_pbuilderrc(workspace_dirpath, codename):
""" prepare .pbuilderrc file of cowbuilder for pre-building
---
DISTRIBUTION: codename
BINDMOUNTS: for "temp" results of pre-build
BUILDRESULT: for "unsigned_results" of backports build
"""
content = ('DISTRIBUTION="%s"\n'
'BINDMOUNTS="%s/temp"\n'
'DEBBUILDOPTS="-sa"\n'
'BUILDRESULT="%s/unsigned_results"\n'
% (codename, workspace_dirpath, workspace_dirpath))
pbuilderrc_path = os.path.join(workspace_dirpath, '.pbuilderrc')
with open(pbuilderrc_path, 'w') as _file:
_file.write(content)
return pbuilderrc_path
def pre_build_package(pbuilderrc_path, codename, basepath, batch_filepath):
""" pre-building source package for backport """
command = ('sudo /usr/sbin/cowbuilder --execute --configfile %s '
'--distribution %s --basepath %s -- %s' % (pbuilderrc_path,
codename,
basepath,
batch_filepath))
subprocess.call(shlex.split(command))
def build_package(basepath, pbuilderrc_path, dsc_path):
""" build package for backport """
command = ('sudo /usr/sbin/cowbuilder --build --configfile %s '
'--basepath %s %s' % (pbuilderrc_path,
basepath,
dsc_path))
return subprocess.call(shlex.split(command))
def git_build_package(codename, result_dirpath):
''' invoke git-buildpackage
Must prepare ~/.gbp.conf as folloing.
---
git-pbuilder-options = '--debbuildopts "-sa"'
---
'''
command = ('sudo /usr/bin/git-buildpackage --git-ignore-branch '
'--git-pbuilder --git-export-dir=%s --git-ignore-new '
'--git-dist=%s' % (result_dirpath, codename))
return subprocess.call(shlex.split(command))
def change_owner(result_dirpath):
username = pwd.getpwuid(os.geteuid()).pw_name
command = 'sudo chown -R %s: %s' % (username, result_dirpath)
return subprocess.call(shlex.split(command))
def update_pbuilder(basetgz):
""" update pbuilder image """
command = 'sudo /usr/sbin/pbuilder --update --basetgz %s' % basetgz
return subprocess.call(shlex.split(command))
def exec_piuparts(codename, basetgz, changes_path,
flavor='debian', mirror=None, no_upgrade_test=False):
""" execute piuparts for install/uninstall test """
command_param = ['sudo', '/usr/sbin/piuparts', '-d', codename,
'-D', flavor, '--basetgz', basetgz, changes_path]
if mirror:
# for ubuntu
command_param.insert(2, mirror)
command_param.insert(2, '-m')
if no_upgrade_test:
# workaround of backport firstly
command_param.insert(2, '--no-upgrade-test')
return subprocess.call(command_param)
def copy_files(filename_list, src_dirpath, dest_dirpath):
""" copy files to specified destination directory
:param filename_list: expected Files in .changes file and .changes file
:param src_dirpath: expected diretory of generated source package
:param dest_dirpath: expected directory of signing key files
"""
[shutil.copy(os.path.join(src_dirpath, filename), dest_dirpath)
for filename in filename_list]
return 0
def find_dsc_name(dir_path):
""" find .dsc file name """
return os.path.basename(glob('%s/*.dsc' % dir_path)[0])
def upload_files(codename, changes_path, reprepro_passphrase):
""" uploading to local archive incoming directory of reprepro """
os.environ['LANG'] = 'C'
command = "dput %s %s" % (codename, changes_path)
exp = pexpect.spawn(command, maxread=4000)
exp.logfile_read = sys.stdout
exp.expect('Please enter passphrase:')
exp.sendline(reprepro_passphrase)
exp.expect('Please enter passphrase:')
exp.sendline(reprepro_passphrase)
exp.expect(pexpect.EOF)
exp.close()
return exp.status
def main():
"""
[backport with pre-build]
1. wget .dsc file to "workspace". (as ${WORKSPACE} or current dir)
2. parse .dsc file, generate batch_script.sh of cowbuilder for pre-build
3. launch cowbuilder
3-1. installing packages
3-2. dget source package and extract it
3-3. rebuild source package
3-4. copy the rebuilding results to "temp/" (as BINDMOUNT of .pbuilderrc)
4. clean build from rebuilten source package existed at "temp/".
the building results genrate to "unsigned_results/". (as BUILDRESULT)
5. GPG sign .dsc and .changes files in "unsigned_results/".
6. move the source packages from "unsigned_results/" to "signed_results/".
7. push signed source package to "incoming" with dput.
[backport]
1. dget source package (as ${WORKSPACE} of or current dir)
2. clean build from downloaded source package existed at "workspace".
the building results genrate to "unsigned_results/". (as BUILDRESULT)
3. GPG sign .dsc and .changes files in "unsigned_results/".
4. move the source packages from "unsigned_results/" to "signed_results/".
5. push signed source package to "incoming" with dput.
[build original]
1. clean build from Git repository with git-buildpackage,
and export results to unsigned_results/"
2. GPG sign .dsc and .changes files in "unsigned_results/".
3. move the source packages from "unsigned_results/" to "signed_results/".
4. push signed source package to "incoming" with dput.
"""
args = parse_options()
work_dirpath = set_work_dirpath()
temp_dirpath = os.path.join(work_dirpath, 'temp')
unsigned_results_dirpath = os.path.join(work_dirpath, 'unsigned_results')
signed_results_dirpath = os.path.join(work_dirpath, 'signed_results')
make_directories([temp_dirpath,
unsigned_results_dirpath,
signed_results_dirpath])
basepath = get_basepath(args.codename)
basetgz = get_basetgz(args.codename)
# update cowbuilder image tree
print('### update cowbuilder ###')
if update_cowbuilder(basepath) != 0:
sys.exit(1)
# generate .pbuilderrc
pbuilderrc_filepath = prepare_pbuilderrc(work_dirpath, args.codename)
if args.subparser_name == 'backport':
# for backporting
dsc_name = os.path.basename(args.url)
dsc_filepath = os.path.join(work_dirpath, dsc_name)
# retrieve source package
print('### retreive source package ###')
retrieve_source_package(args.url)
if args.prebuild:
print('### extract source package ###')
param_dict = extract_source_package(
os.path.join(work_dirpath, dsc_name))
param_dict = append_param(param_dict, args)
# temp_dirpath is for mount point with BINDMOUNTS
param_dict['temp_dirpath'] = temp_dirpath
batch_content = generate_batch_script(param_dict)
batch_filepath = os.path.join(work_dirpath,
param_dict.get('batch_name'))
with open(batch_filepath, 'w') as batch_file:
batch_file.write(batch_content)
print('### pre build package ###')
pre_build_package(pbuilderrc_filepath,
args.codename,
basepath,
batch_filepath)
dsc_filepath = os.path.join(temp_dirpath, dsc_name)
# build package for backport
print('### build package with cowbuilder ###')
if build_package(basepath, pbuilderrc_filepath, dsc_filepath) != 0:
sys.exit(1)
elif args.subparser_name == 'original':
# build package with git-buildpackage
print('### build package with git-buildpackage ###')
if git_build_package(args.codename, unsigned_results_dirpath) != 0:
sys.exit(1)
# for original package
dsc_name = find_dsc_name(unsigned_results_dirpath)
arch = get_architecture()
changes_name = get_changes_name(dsc_name, arch)
# update pbuilder basetgz image
print('### update pbuilder image ###')
if update_pbuilder(basetgz) != 0:
sys.exit(1)
# install and uninstall test
unsigned_changes_filepath = os.path.join(unsigned_results_dirpath,
changes_name)
print('### install and uninstall test ###')
if exec_piuparts(args.codename,
basetgz,
unsigned_changes_filepath,
flavor=args.flavor,
mirror=args.mirror,
no_upgrade_test=args.no_upgrade_test) != 0:
sys.exit(1)
# change owner
if change_owner(unsigned_results_dirpath) != 0:
sys.exit(1)
# debsign
print('### debsign ###')
changes_path = os.path.join(unsigned_results_dirpath, changes_name)
if args.without_lintian:
if debsign.debsign_process(changes_path,
passphrase=args.passphrase,
lintian=False) is False:
sys.exit(1)
else:
if debsign.debsign_process(changes_path,
passphrase=args.passphrase) is False:
sys.exit(1)
# move singed source packages
# file list format is md5sum, size, section, priority, name
files_list = parse_changes(unsigned_changes_filepath)
filename_list = [_file.get('name') for _file in files_list]
filename_list.append(changes_name)
if copy_files(filename_list,
unsigned_results_dirpath,
signed_results_dirpath) != 0:
sys.exit(1)
# upload incoming directory
print('### upload files ###')
if upload_files(args.codename,
os.path.join(signed_results_dirpath, changes_name),
args.reprepro_passphrase) != 0:
sys.exit(1)
if __name__ == '__main__':
main()
@mkouhei

This comment has been minimized.

Copy link
Owner Author

@mkouhei mkouhei commented May 17, 2014

cd packagetree
../jobs/debbuild/job.py -c wheezy \
         -d ${WORKSPACE} \
         -p ${passphrase} \
         -r ${repo_passphrase} \ 
         -f debian
@mkouhei

This comment has been minimized.

Copy link
Owner Author

@mkouhei mkouhei commented May 21, 2014

cowbuilder のイメージに予め deb-srcを追加しておく必要あり。

@mkouhei

This comment has been minimized.

Copy link
Owner Author

@mkouhei mkouhei commented Jun 3, 2014

Jenkins 1.565 on Wheezy

debbuild/debbuild.py \
         -c wheezy \
         -p ${passphrase} \
         -r ${repo_passphrase} \
         backport \
         -u http://cdn.debian.net/debian/pool/main/j/jq/jq_1.3-1.1.dsc \
         --debfullname "Firstname Lastname" \
         --debemail user@example.org

の場合はうまく行くが、

cd pycontrol
../jobs/debbuild/debbuild.py -c trusty \
         -f ubuntu \
         -m http://repo.example.org/ubuntu/ \
         -p ${passphrase} \
         -r ${repo_passphrase} \
         --without-lintian \
         original

の場合はだめ。

@mkouhei

This comment has been minimized.

Copy link
Owner Author

@mkouhei mkouhei commented Jun 3, 2014

ダメな原因は、未登録状態(=exportされただけで、ひとつも登録されていない場合)のrepreproの場合で、何か一つでもDBに登録されていればOK。

@mkouhei

This comment has been minimized.

Copy link
Owner Author

@mkouhei mkouhei commented Jun 3, 2014

コードで対応する方法が現時点ではわからないので、とりあえずディストロ毎に、repreproで何か最低限一つ登録しておく、というワークアラウンドをとることにした。

@mkouhei

This comment has been minimized.

Copy link
Owner Author

@mkouhei mkouhei commented Jun 3, 2014

by hand tasks

  • 最低限一つはrepreproにパッケージを登録する
  • Debian, Ubuntu用のpbuilder/cowbuilder イメージをそれぞれDebian/Ubuntu上で作成した後、Debian -> Ubuntu, Ubuntu -> Debianにコピーする
  • 各イメージの apt lineにdebian-security, -{security,updates}, および各-customを追加する
@mkouhei

This comment has been minimized.

Copy link
Owner Author

@mkouhei mkouhei commented Jun 5, 2014

登録できたりできなかったりする問題は、登録状態でも発生したので、そもそもsubprocessで行うのをやめて、pexpectで処理することにした。

@mkouhei

This comment has been minimized.

Copy link
Owner Author

@mkouhei mkouhei commented Jun 5, 2014

複数ディストリビューションに対応する場合(wheezy, jessieとか、precise, trustyとか) は、repreproのpoolでバッティングするので、debian versionを変更しないとアカン問題。

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment