gitsm.py

# TODO: figure out how to get the submodules handled for shallow,
# ideally without having to unpack the shallow tarball. since we
# already limit history, and we know what revs the tarball provides,
# we may be able to explicitly include the .gitmodules for each rev we
# care about. We also need to make sure gitshallow is handling
# destsuffix correctly, to use shallow submodule tarballs
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import collections
import os
import bb
from   bb.fetch2.git import Git
from   bb.fetch2 import runfetchcmd
from   bb.fetch2 import logger

class GitSM(Git):
    scheme = 'gitsm2'

    def supports(self, ud, d):
        return ud.type == self.scheme

    def need_update(self, ud, d):
        if super().need_update(ud, d):
            return True

        if os.path.exists(ud.clonedir) and self.uses_submodules(ud, d, ud.clonedir):
            fetcher = self.create_submodule_fetcher(ud, d)
            for sm_url, sm_ud in fetcher.ud.items():
                if sm_ud.method.need_update(sm_ud, d):
                    return True

        return False

    def uses_submodules(self, ud, d, wd):
        for name in ud.names:
            try:
                runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=wd)
                return True
            except bb.fetch.FetchError:
                pass
        return False

    def download(self, ud, d):
        if super().need_update(ud, d):
            super().download(ud, d)

        if not ud.shallow or ud.localpath != ud.fullshallow:
            if self.uses_submodules(ud, d, ud.clonedir):
                fetcher = self.create_submodule_fetcher(ud, d)
                fetcher.download()

    def unpack(self, ud, destdir, d):
        super().unpack(ud, destdir, d)

        if self.uses_submodules(ud, d, ud.clonedir):
            destsuffix = ud.parm.get('destsuffix', 'git/')
            dest = os.path.join(destdir, destsuffix)

            gitdir = runfetchcmd('%s rev-parse --git-dir' % ud.basecmd, d, workdir=dest).rstrip()
            gitdir = os.path.join(dest, gitdir)
            modulesdir = os.path.join(gitdir, 'modules')

            fetcher = self.create_submodule_fetcher(ud, d)
            fetcher.unpack(modulesdir)

            # We mark these bare to avoid a checkout, but submodules use them
            # as non-bare repos with worktrees, so disable bare
            for ud in fetcher.ud.values():
                sm_dest = os.path.join(modulesdir, ud.parm['destsuffix'])
                runfetchcmd(ud.basecmd + " config core.bare false", d, workdir=sm_dest)

            runfetchcmd(ud.basecmd + " submodule update --init --recursive --no-fetch", d, workdir=dest)
            runfetchcmd(ud.basecmd + " submodule sync --recursive", d, workdir=dest)

    def create_submodule_fetcher(self, ud, d):
        urls = list(self._get_submodule_urls(ud, d))
        l = d.createCopy()
        # Avoid conflict with explicit ;rev= in the urls
        l.delVar('SRCREV')
        return bb.fetch.Fetch(urls, l, cache=False)

    def _get_submodule_urls(self, ud, d):
        submodules = collections.defaultdict(dict)
        for name in ud.names:
            try:
                config_lines = runfetchcmd("%s show %s:.gitmodules | git config -f - -l" % (ud.basecmd, ud.revisions[name]), d, workdir=ud.clonedir)
            except bb.fetch.FetchError:
                continue

            for line in config_lines.splitlines():
                full_key, value = line.split('=', 1)
                sm_name, key = full_key.replace('submodule.', '').split('.', 1)
                submodules[sm_name][key] = value

            for sm_name, sm_data in submodules.items():
                tree_info = runfetchcmd("%s ls-tree %s %s" % (ud.basecmd, ud.revisions[name], sm_data['path']), d, workdir=ud.clonedir)
                sm_revision = tree_info.split()[2]

                # Construct a bitbake fetcher url from the git url
                scheme, netloc, path, user, pw, param = bb.fetch2.decodeurl(sm_data['url'])
                param['bareclone'] = '1'
                param['protocol'] = scheme
                param['rev'] = sm_revision
                param['destsuffix'] = sm_name + os.sep
                if 'branch' in sm_data:
                    param['branch'] = sm_data['branch']
                else:
                    param['nobranch'] = '1'

                yield bb.fetch2.encodeurl([self.scheme, netloc, path, user, pw, param])