gitlab_clone_all.py

#!/usr/bin/env python3
import argparse
import requests
import json
import urllib3
from urllib.parse import urlparse
import os
import re
from getpass import getpass


# SUPPRESS WARNINGS ############################################################
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


# CLONE ########################################################################
def do_clone_ssh(repo_list):
    for repo in repo_list:
        directory = OUTPUT_DIR + repo.split(':')[1]
        os.system('git clone \'{}\' \'{}\''.format(repo, directory))

def do_clone_http(repo_list):
    for repo in repo_list:
        parsed_http_url = urlparse(repo)
        directory = OUTPUT_DIR + re.sub('.git', '', parsed_http_url.path)

        if args.user and args.password:
            # for some unknown reason, urlparse cannot replace username/password directly ...
            parsed_http_url = parsed_http_url._replace(netloc=args.user + ':' + args.password + '@' + parsed_http_url.netloc)

        os.system('git clone \'{}\' \'{}\''.format(parsed_http_url.geturl(), directory))

def do_clone(repo_list, method):
    if method == 'HTTP':
        do_clone_http(repo_list)
    if method == 'SSH':
        do_clone_ssh(repo_list)


# API ##########################################################################
def gitlab_get_cookie(session, url, user, password, ldap=False):

    # get the CSRF token
    content = session.get(url + '/users/sign_in').content
    tokens = re.findall(b'"authenticity_token" value="([^"]+)"', content)

    print(tokens)
    if ldap:
        data = {'authenticity_token': tokens[0].decode('utf-8'), 'username': user, 'password': password}
        url = url + '/users/auth/ldapmain/callback'
    elif len(tokens) > 1:
        data = {'authenticity_token': tokens[1].decode('utf-8'), 'user[login]': user, 'user[password]': password}
        url = url + '/users/sign_in'
    else:
        data = {'authenticity_token': tokens[0].decode('utf-8'), 'user[login]': user, 'user[password]': password}
        url = url + '/users/sign_in'


    r = session.post(url, data=data)
    if r.status_code != 200:
        print('Failed logging in: {}'.format(r.status_code))
        exit(1)


def gitlab_get_repo_list(session, url, method):
    repo_list = []
    page_number = 1

    while True:
        r = session.get(url + '/api/v4/projects?per_page=1000&page={}'.format(page_number))
        projects_page = json.loads(r.text)
        if len(projects_page) == 0:
            break

        print(len(projects_page))
        for project in projects_page:
            if args.group is not None:
                if args.group.casefold() + ' /' not in project['name_with_namespace'].casefold():
                    continue

            if method == 'SSH':
                repo_list.append(project['ssh_url_to_repo'])
            if method == 'HTTP':
                repo_list.append(project['http_url_to_repo'])

        page_number += 1

    return repo_list


# MAIN #########################################################################
parser = argparse.ArgumentParser()
parser.add_argument('url', type=str)
parser.add_argument('-u', '--user', type=str)
parser.add_argument('-p', '--password', type=str)
parser.add_argument('-c', '--cookie', type=str)
parser.add_argument('-o', '--output-dir', type=str, required=True)
parser.add_argument('-m', '--method', type=str, default='HTTP', help='Cloning method: HTTP or SSH (default: HTTP)')
parser.add_argument('-l', '--ldap', action='store_true')
parser.add_argument('-g', '--group', type=str)

args = parser.parse_args()

s = requests.Session()
s.verify = False

if not args.cookie:
    if args.user:
        if not args.password:
            args.password = getpass("password: ")

        gitlab_get_cookie(s, args.url.rstrip('/'), args.user, args.password, args.ldap)
        args.password = args.password.replace('@', '%40') # escape for HTTP later on


if args.cookie:
    cookie = requests.cookies.create_cookie(name='_gitlab_session', value=args.cookie)
    s.cookies.set_cookie(cookie)

OUTPUT_DIR = args.output_dir + '/'

print('[+] Getting list of repositories...')
repo_list = gitlab_get_repo_list(s, args.url.rstrip('/'), args.method)
print(repo_list)
print('[+] Cloning {} repositories...'.format(len(repo_list)))
do_clone(repo_list, args.method)