Simple python script to fetch gitlab's artifact from a job / from privately

getlab.py

#!/usr/bin/env python

import logging
import sys
import os
import subprocess
import argparse
import configparser
try:
    import gitlab
except ImportError:
    print('ERROR: cannot import gitlab package (python-gitlab). Please install them with pip install python-gitlab')
    sys.exit(-1)

here = os.path.dirname(__file__)
assert sys.version_info >= (3, 6)

logging.basicConfig(
    format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
    datefmt='%H:%M:%S',
    level=logging.INFO)
l = logging.getLogger()


"""
Parse command line arguments.
"""
p = argparse.ArgumentParser(
    description='GetLab - Fetch artifacts from GitLab server.')
p.add_argument('--config-file', default='getlab.ini',
               help=('Config file of getlab. The config file contains sections. Each section defines'
                     ' an artifact. The section header is the defines the directory where the '
                     'artifact should be extracted as well.'),
               )
p.add_argument('-v', '--verbose', help='verbose', action='store_true')
p.add_argument('--url', help="URL of the GitLab server",
               default=os.environ.get('CI_SERVER_URL', None))
p.add_argument('--job-token',
               help="GitLab's Job token: Use this from pipeline-job. More: https://docs.gitlab.com/ee/ci/triggers/#ci-job-token",
               default=os.environ.get('CI_JOB_TOKEN', None))
p.add_argument('--private-token',
               help="GitLab's private token: Use this from custom build. More: https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html",
               default=os.environ.get('GITLAB_PRIVATE_TOKEN', None))

args = p.parse_args()

if args.verbose:
    l.setLevel(logging.DEBUG)
    print(args)

config = configparser.ConfigParser()
try:
    with open(args.config_file) as f:
        config.read_file(f)
except IOError as e:
    l.error(f'Failed to open config file!')
    print(e)
    sys.exit(-1)
except Exception as e:
    l.error(f'Config file parse error')
    print(e)
    sys.exit(-1)

if args.verbose:
    print(config._sections)


"""
Do the fetch. The config file contains sections. Each section defines an artifact. The section
header is the defines the directory where the artifact should be extracted as well.
"""
for dependency in config.sections():
    try:
        url = args.url if args.url else config[dependency].get('url', 'https://gitlab.com/')
        project_id = config[dependency]['project-id']
        job_id = config[dependency]['job-id']
    except Exception as e:
        l.error(
            f'One of < url, project_id, job_id > cannot fetched for {dependency}')
        print(e)
        sys.exit(-1)

    if args.private_token or args.job_token:
        gl = gitlab.Gitlab(
            url, private_token=args.private_token, job_token=args.job_token)
    else:
        print(
            f'Error: Cannot find any token. Please generate it at: {url}/-/profile/personal_access_tokens and give to me with --private-token')
        sys.exit(-1)

    # Job token cannot fetch project information https://gitlab.com/gitlab-org/gitlab/-/issues/17511
    project = gl.projects.get(project_id, lazy=True)
    buildjob = project.jobs.get(job_id, lazy=True)

    # Create directory for the unzipped artifact.
    if not os.path.exists(dependency):
        os.makedirs(dependency)

    try:
        # Fetch the artifact and extract it.
        zipfn = "___artifacts.zip"
        with open(zipfn, "wb") as f:
            buildjob.artifacts(streamed=True, action=f.write)
        subprocess.run(["unzip", "-bo", zipfn, '-d', dependency])
        os.unlink(zipfn)
    except gitlab.exceptions.GitlabAuthenticationError as e:
        l.error(f'Cannot reach site: {url} with token ****')
        print(e)
        sys.exit(-1)
    except gitlab.exceptions.GitlabGetError as e:
        l.error(f'Project id: {project_id}')
        l.error(f'Job id: {job_id}')
        print(e)
        sys.exit(-1)