Yocto AB reproducible build output compare

pull_repro.py

#! /usr/bin/env python3
# Copyright 2020 Joshua Watt
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import argparse
import subprocess
import sys
import os
import shutil

THIS_DIR = os.path.abspath(os.path.dirname(__file__))

def main():
    parser = argparse.ArgumentParser(description="Process reproducible build output from autobuilder")
    parser.add_argument("number", help="Build number")
    parser.add_argument("--force-fetch", action="store_true", help="Force fetching, even if it looks like it's already been done")
    parser.add_argument("--force-diff", action="store_true", help="Rerun diff, even if it looks like it's already been done")

    args = parser.parse_args()

    target = "autobuilder.yocto.io/pub/repro-fail/build-st-%s/" % args.number
    dest_dir = os.path.join(THIS_DIR, target)

    if os.path.exists(dest_dir) and not args.force_fetch:
        print("It looks like %s has been downloaded already. Skipping fetch" % args.number)
    else:
        subprocess.run(['wget', "--no-clobber", "--convert-links", "--random-wait", "--no-parent", "-r", "-p", "--level=10", "-E", "-e", "robots=off", "-U", "mozilla", "https://%s" % target], check=True, cwd=THIS_DIR)

    delete_files = set()
    for root, dirs, files in os.walk(dest_dir):
        for f in files:
            if f.endswith('.html') or f.endswith('.htm'):
                delete_files.add(os.path.join(root, f))

    for f in delete_files:
        os.unlink(f)

    diff_dir = os.path.join(dest_dir, 'diff')

    if os.path.exists(diff_dir) and not args.force_diff:
        print("It looks like %s has already been compared. Skipping diff" % args.number)
    else:
        if os.path.exists(diff_dir):
            shutil.rmtree(diff_dir)
        os.mkdir(diff_dir)
        subprocess.run(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', '--progress', '--html-dir', diff_dir, 'reproducibleA', 'reproducibleB'], cwd=dest_dir)

    subprocess.run(['firefox', 'file://%s' % os.path.join(diff_dir, 'index.html')])

if __name__ == "__main__":
    sys.exit(main())