build package with cowbuilder or git-buildpackage on Jenkins
#!/usr/bin/python | |
# -*- coding: utf-8 -*- | |
''' debbuild_wrapper.py | |
The debian package build wrapper on Jenkins. | |
This script support three way of building debian package. | |
1-1. Backport from source package with cowbuilder. | |
1-2. Pre-build and Backport from source package with cowbuilder. | |
2. Build original package with git-buildpackage. | |
Support flavors are "debian" or "ubuntu". | |
''' | |
import deb822 | |
import shutil | |
import argparse | |
import subprocess | |
import pexpect | |
import os.path | |
import sys | |
import pwd | |
import shlex | |
from glob import glob | |
from pydebsign import debsign | |
def parse_options(): | |
""" building options """ | |
parser = argparse.ArgumentParser(description='usage') | |
parser.add_argument('-c', '--codename', action='store', required=True, | |
help='specify codename of distribution') | |
# for piuparts | |
parser.add_argument('-f', '--flavor', choices=['debian', 'ubuntu'], | |
default='debian', | |
help='specify flavor debian or ubuntu (for piuparts)') | |
parser.add_argument('-m', '--mirror', action='store', | |
help='specify Debian mirror (for piuparts)') | |
# Backport version is different with an existing version in the arvhive, | |
# so generally upgrade test fails. | |
parser.add_argument('-n', '--no-upgrade-test', action='store_true', | |
help=('Skip testing upgrade from ' | |
'an existing version in the archive.')) | |
# without lintian checking | |
parser.add_argument('--without-lintian', action='store_true', | |
help=('Skip checking with lintian')) | |
# for dput | |
parser.add_argument('-p', '--passphrase', action='store', required=True, | |
help='GPG private key passphrase of uploder') | |
parser.add_argument('-r', '--reprepro-passphrase', action='store', | |
required=True, | |
help='GPG Private key passphrase of reprepro') | |
subparser = parser.add_subparsers(title='subcommands', | |
dest='subparser_name') | |
backport = subparser.add_parser('backport', help='backport package') | |
subparser.add_parser('original', help='original package') | |
backport.add_argument('-u', '--url', action='store', required=True, | |
help='specify .dsc url') | |
backport.add_argument('-P', '--prebuild', action='store_true', | |
help='pre-build debian package') | |
backport.add_argument('--debfullname', action='store') | |
backport.add_argument('--debemail', action='store') | |
return parser.parse_args() | |
def set_work_dirpath(): | |
""" set work_dirpath variable | |
expected value is WORKSPACE on Jenkins, | |
current directory is on local test. | |
""" | |
if os.environ.get('WORKSPACE'): | |
work_dirpath = os.environ.get('WORKSPACE') | |
else: | |
work_dirpath = os.path.abspath(os.path.curdir) | |
return work_dirpath | |
def make_directories(dir_list): | |
""" make directories for results """ | |
[os.mkdir(dir_name) for dir_name in dir_list | |
if os.path.isdir(dir_name) is False] | |
def parse_changes(changes_filepath): | |
""" parse .changes file """ | |
with open(changes_filepath, 'rb') as changes_file: | |
changes = deb822.Changes(changes_file) | |
return [_file for _file in changes['Files']] | |
def get_architecture(): | |
""" get DEB_BUILD_ARCH """ | |
command = 'dpkg-architecture -qDEB_BUILD_ARCH' | |
return subprocess.check_output( | |
shlex.split(command)).split()[0].decode('utf-8') | |
def retrieve_source_package(dsc_url): | |
""" retrieve source package with dget command """ | |
command = 'dget -d %s' % dsc_url | |
return subprocess.call(shlex.split(command)) | |
def get_changes_name(dsc_name, arch): | |
""" get .changes file name """ | |
pkg_name = dsc_name.split('.dsc')[0] | |
changes_name = '%s_%s.changes' % (pkg_name, arch) | |
return changes_name | |
def get_basepath(codename): | |
""" get cowbuilder basepath """ | |
return '/var/cache/pbuilder/base-%s.cow' % codename | |
def get_basetgz(codename): | |
""" get pbuilder basetgz path """ | |
return '/var/cache/pbuilder/base-%s.tgz' % codename | |
def update_cowbuilder(basepath): | |
""" update cowbuilder image """ | |
command = 'sudo /usr/sbin/cowbuilder --update --basepath %s' % basepath | |
return subprocess.call(shlex.split(command)) | |
def extract_source_package(dsc_path): | |
""" extract source package and meta data from .control file """ | |
command = 'dpkg-source -x %s' % dsc_path | |
subprocess.call(shlex.split(command)) | |
with open(dsc_path, 'rb') as dsc_file: | |
srcs = deb822.Sources(dsc_file) | |
build_depends = srcs.get('Build-Depends').split(',') | |
if srcs.get('Build-Depends-Indep'): | |
build_depends += (srcs.get('Build-Depends-Indep').split(',')) | |
data = {'source_dirname': '%s-%s' % (srcs.get('Source'), | |
srcs.get('Version').split('-')[0]), | |
'build_depends': build_depends, | |
'dsc_name': os.path.basename(dsc_path), | |
'upstream_version': srcs.get('Version').split('-')[0], | |
'source_package_name': srcs.get('Source'), | |
'debian_version': srcs.get('Version'), | |
'batch_name': '%s-batch.sh' % srcs.get('Source')} | |
return data | |
def append_param(param_dict, args): | |
""" append variables to param_dict from args """ | |
param_dict['dsc_url'] = args.url | |
param_dict['debfullname'] = args.debfullname | |
param_dict['debemail'] = args.debemail | |
return param_dict | |
def generate_batch_script(param_dict): | |
""" generate batch script for cowbuilder """ | |
return ('#!/bin/sh -x\n' | |
'export DEBFULLNAME="%(debfullname)s"\n' | |
'export DEBEMAIL=%(debemail)s\n' | |
'apt-get -y install curl devscripts quilt ' | |
'patch libdistro-info-perl fakeroot\n' | |
'apt-get -y build-dep %(source_package_name)s\n' | |
'dget -d %(dsc_url)s\n' | |
'dpkg-source -x %(dsc_name)s\n' | |
'(\n' | |
'cd %(source_dirname)s\n' | |
'debuild -us -uc\n' | |
')\n' | |
'cp -f %(source_package_name)s_%(debian_version)s.debian.tar.gz ' | |
'%(source_package_name)s_%(upstream_version)s.orig.tar.gz ' | |
'%(source_package_name)s_%(debian_version)s.dsc ' | |
'%(temp_dirpath)s/\n' | |
% param_dict) | |
def prepare_pbuilderrc(workspace_dirpath, codename): | |
""" prepare .pbuilderrc file of cowbuilder for pre-building | |
--- | |
DISTRIBUTION: codename | |
BINDMOUNTS: for "temp" results of pre-build | |
BUILDRESULT: for "unsigned_results" of backports build | |
""" | |
content = ('DISTRIBUTION="%s"\n' | |
'BINDMOUNTS="%s/temp"\n' | |
'DEBBUILDOPTS="-sa"\n' | |
'BUILDRESULT="%s/unsigned_results"\n' | |
% (codename, workspace_dirpath, workspace_dirpath)) | |
pbuilderrc_path = os.path.join(workspace_dirpath, '.pbuilderrc') | |
with open(pbuilderrc_path, 'w') as _file: | |
_file.write(content) | |
return pbuilderrc_path | |
def pre_build_package(pbuilderrc_path, codename, basepath, batch_filepath): | |
""" pre-building source package for backport """ | |
command = ('sudo /usr/sbin/cowbuilder --execute --configfile %s ' | |
'--distribution %s --basepath %s -- %s' % (pbuilderrc_path, | |
codename, | |
basepath, | |
batch_filepath)) | |
subprocess.call(shlex.split(command)) | |
def build_package(basepath, pbuilderrc_path, dsc_path): | |
""" build package for backport """ | |
command = ('sudo /usr/sbin/cowbuilder --build --configfile %s ' | |
'--basepath %s %s' % (pbuilderrc_path, | |
basepath, | |
dsc_path)) | |
return subprocess.call(shlex.split(command)) | |
def git_build_package(codename, result_dirpath): | |
''' invoke git-buildpackage | |
Must prepare ~/.gbp.conf as folloing. | |
--- | |
git-pbuilder-options = '--debbuildopts "-sa"' | |
--- | |
''' | |
command = ('sudo /usr/bin/git-buildpackage --git-ignore-branch ' | |
'--git-pbuilder --git-export-dir=%s --git-ignore-new ' | |
'--git-dist=%s' % (result_dirpath, codename)) | |
return subprocess.call(shlex.split(command)) | |
def change_owner(result_dirpath): | |
username = pwd.getpwuid(os.geteuid()).pw_name | |
command = 'sudo chown -R %s: %s' % (username, result_dirpath) | |
return subprocess.call(shlex.split(command)) | |
def update_pbuilder(basetgz): | |
""" update pbuilder image """ | |
command = 'sudo /usr/sbin/pbuilder --update --basetgz %s' % basetgz | |
return subprocess.call(shlex.split(command)) | |
def exec_piuparts(codename, basetgz, changes_path, | |
flavor='debian', mirror=None, no_upgrade_test=False): | |
""" execute piuparts for install/uninstall test """ | |
command_param = ['sudo', '/usr/sbin/piuparts', '-d', codename, | |
'-D', flavor, '--basetgz', basetgz, changes_path] | |
if mirror: | |
# for ubuntu | |
command_param.insert(2, mirror) | |
command_param.insert(2, '-m') | |
if no_upgrade_test: | |
# workaround of backport firstly | |
command_param.insert(2, '--no-upgrade-test') | |
return subprocess.call(command_param) | |
def copy_files(filename_list, src_dirpath, dest_dirpath): | |
""" copy files to specified destination directory | |
:param filename_list: expected Files in .changes file and .changes file | |
:param src_dirpath: expected diretory of generated source package | |
:param dest_dirpath: expected directory of signing key files | |
""" | |
[shutil.copy(os.path.join(src_dirpath, filename), dest_dirpath) | |
for filename in filename_list] | |
return 0 | |
def find_dsc_name(dir_path): | |
""" find .dsc file name """ | |
return os.path.basename(glob('%s/*.dsc' % dir_path)[0]) | |
def upload_files(codename, changes_path, reprepro_passphrase): | |
""" uploading to local archive incoming directory of reprepro """ | |
os.environ['LANG'] = 'C' | |
command = "dput %s %s" % (codename, changes_path) | |
exp = pexpect.spawn(command, maxread=4000) | |
exp.logfile_read = sys.stdout | |
exp.expect('Please enter passphrase:') | |
exp.sendline(reprepro_passphrase) | |
exp.expect('Please enter passphrase:') | |
exp.sendline(reprepro_passphrase) | |
exp.expect(pexpect.EOF) | |
exp.close() | |
return exp.status | |
def main(): | |
""" | |
[backport with pre-build] | |
1. wget .dsc file to "workspace". (as ${WORKSPACE} or current dir) | |
2. parse .dsc file, generate batch_script.sh of cowbuilder for pre-build | |
3. launch cowbuilder | |
3-1. installing packages | |
3-2. dget source package and extract it | |
3-3. rebuild source package | |
3-4. copy the rebuilding results to "temp/" (as BINDMOUNT of .pbuilderrc) | |
4. clean build from rebuilten source package existed at "temp/". | |
the building results genrate to "unsigned_results/". (as BUILDRESULT) | |
5. GPG sign .dsc and .changes files in "unsigned_results/". | |
6. move the source packages from "unsigned_results/" to "signed_results/". | |
7. push signed source package to "incoming" with dput. | |
[backport] | |
1. dget source package (as ${WORKSPACE} of or current dir) | |
2. clean build from downloaded source package existed at "workspace". | |
the building results genrate to "unsigned_results/". (as BUILDRESULT) | |
3. GPG sign .dsc and .changes files in "unsigned_results/". | |
4. move the source packages from "unsigned_results/" to "signed_results/". | |
5. push signed source package to "incoming" with dput. | |
[build original] | |
1. clean build from Git repository with git-buildpackage, | |
and export results to unsigned_results/" | |
2. GPG sign .dsc and .changes files in "unsigned_results/". | |
3. move the source packages from "unsigned_results/" to "signed_results/". | |
4. push signed source package to "incoming" with dput. | |
""" | |
args = parse_options() | |
work_dirpath = set_work_dirpath() | |
temp_dirpath = os.path.join(work_dirpath, 'temp') | |
unsigned_results_dirpath = os.path.join(work_dirpath, 'unsigned_results') | |
signed_results_dirpath = os.path.join(work_dirpath, 'signed_results') | |
make_directories([temp_dirpath, | |
unsigned_results_dirpath, | |
signed_results_dirpath]) | |
basepath = get_basepath(args.codename) | |
basetgz = get_basetgz(args.codename) | |
# update cowbuilder image tree | |
print('### update cowbuilder ###') | |
if update_cowbuilder(basepath) != 0: | |
sys.exit(1) | |
# generate .pbuilderrc | |
pbuilderrc_filepath = prepare_pbuilderrc(work_dirpath, args.codename) | |
if args.subparser_name == 'backport': | |
# for backporting | |
dsc_name = os.path.basename(args.url) | |
dsc_filepath = os.path.join(work_dirpath, dsc_name) | |
# retrieve source package | |
print('### retreive source package ###') | |
retrieve_source_package(args.url) | |
if args.prebuild: | |
print('### extract source package ###') | |
param_dict = extract_source_package( | |
os.path.join(work_dirpath, dsc_name)) | |
param_dict = append_param(param_dict, args) | |
# temp_dirpath is for mount point with BINDMOUNTS | |
param_dict['temp_dirpath'] = temp_dirpath | |
batch_content = generate_batch_script(param_dict) | |
batch_filepath = os.path.join(work_dirpath, | |
param_dict.get('batch_name')) | |
with open(batch_filepath, 'w') as batch_file: | |
batch_file.write(batch_content) | |
print('### pre build package ###') | |
pre_build_package(pbuilderrc_filepath, | |
args.codename, | |
basepath, | |
batch_filepath) | |
dsc_filepath = os.path.join(temp_dirpath, dsc_name) | |
# build package for backport | |
print('### build package with cowbuilder ###') | |
if build_package(basepath, pbuilderrc_filepath, dsc_filepath) != 0: | |
sys.exit(1) | |
elif args.subparser_name == 'original': | |
# build package with git-buildpackage | |
print('### build package with git-buildpackage ###') | |
if git_build_package(args.codename, unsigned_results_dirpath) != 0: | |
sys.exit(1) | |
# for original package | |
dsc_name = find_dsc_name(unsigned_results_dirpath) | |
arch = get_architecture() | |
changes_name = get_changes_name(dsc_name, arch) | |
# update pbuilder basetgz image | |
print('### update pbuilder image ###') | |
if update_pbuilder(basetgz) != 0: | |
sys.exit(1) | |
# install and uninstall test | |
unsigned_changes_filepath = os.path.join(unsigned_results_dirpath, | |
changes_name) | |
print('### install and uninstall test ###') | |
if exec_piuparts(args.codename, | |
basetgz, | |
unsigned_changes_filepath, | |
flavor=args.flavor, | |
mirror=args.mirror, | |
no_upgrade_test=args.no_upgrade_test) != 0: | |
sys.exit(1) | |
# change owner | |
if change_owner(unsigned_results_dirpath) != 0: | |
sys.exit(1) | |
# debsign | |
print('### debsign ###') | |
changes_path = os.path.join(unsigned_results_dirpath, changes_name) | |
if args.without_lintian: | |
if debsign.debsign_process(changes_path, | |
passphrase=args.passphrase, | |
lintian=False) is False: | |
sys.exit(1) | |
else: | |
if debsign.debsign_process(changes_path, | |
passphrase=args.passphrase) is False: | |
sys.exit(1) | |
# move singed source packages | |
# file list format is md5sum, size, section, priority, name | |
files_list = parse_changes(unsigned_changes_filepath) | |
filename_list = [_file.get('name') for _file in files_list] | |
filename_list.append(changes_name) | |
if copy_files(filename_list, | |
unsigned_results_dirpath, | |
signed_results_dirpath) != 0: | |
sys.exit(1) | |
# upload incoming directory | |
print('### upload files ###') | |
if upload_files(args.codename, | |
os.path.join(signed_results_dirpath, changes_name), | |
args.reprepro_passphrase) != 0: | |
sys.exit(1) | |
if __name__ == '__main__': | |
main() |
This comment has been minimized.
This comment has been minimized.
cowbuilder のイメージに予め deb-srcを追加しておく必要あり。 |
This comment has been minimized.
This comment has been minimized.
Jenkins 1.565 on Wheezy debbuild/debbuild.py \
-c wheezy \
-p ${passphrase} \
-r ${repo_passphrase} \
backport \
-u http://cdn.debian.net/debian/pool/main/j/jq/jq_1.3-1.1.dsc \
--debfullname "Firstname Lastname" \
--debemail user@example.org の場合はうまく行くが、 cd pycontrol
../jobs/debbuild/debbuild.py -c trusty \
-f ubuntu \
-m http://repo.example.org/ubuntu/ \
-p ${passphrase} \
-r ${repo_passphrase} \
--without-lintian \
original の場合はだめ。 |
This comment has been minimized.
This comment has been minimized.
ダメな原因は、未登録状態(=exportされただけで、ひとつも登録されていない場合)のrepreproの場合で、何か一つでもDBに登録されていればOK。 |
This comment has been minimized.
This comment has been minimized.
コードで対応する方法が現時点ではわからないので、とりあえずディストロ毎に、repreproで何か最低限一つ登録しておく、というワークアラウンドをとることにした。 |
This comment has been minimized.
This comment has been minimized.
by hand tasks
|
This comment has been minimized.
This comment has been minimized.
登録できたりできなかったりする問題は、登録状態でも発生したので、そもそもsubprocessで行うのをやめて、pexpectで処理することにした。 |
This comment has been minimized.
This comment has been minimized.
複数ディストリビューションに対応する場合(wheezy, jessieとか、precise, trustyとか) は、repreproのpoolでバッティングするので、debian versionを変更しないとアカン問題。 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This comment has been minimized.