Skip to content

Instantly share code, notes, and snippets.

@gquere

gquere/gitlab_clone_all.py

Last active Jul 17, 2020
Embed
What would you like to do?
#!/usr/bin/env python3
import argparse
import requests
import json
import urllib3
from urllib.parse import urlparse
import os
import re
from getpass import getpass
# SUPPRESS WARNINGS ############################################################
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# CLONE ########################################################################
def do_clone_ssh(repo_list):
for repo in repo_list:
directory = OUTPUT_DIR + repo.split(':')[1]
os.system('git clone \'{}\' \'{}\''.format(repo, directory))
def do_clone_http(repo_list):
for repo in repo_list:
parsed_http_url = urlparse(repo)
directory = OUTPUT_DIR + re.sub('.git', '', parsed_http_url.path)
if args.user and args.password:
# for some unknown reason, urlparse cannot replace username/password directly ...
parsed_http_url = parsed_http_url._replace(netloc=args.user + ':' + args.password + '@' + parsed_http_url.netloc)
os.system('git clone \'{}\' \'{}\''.format(parsed_http_url.geturl(), directory))
def do_clone(repo_list, method):
if method == 'HTTP':
do_clone_http(repo_list)
if method == 'SSH':
do_clone_ssh(repo_list)
# API ##########################################################################
def gitlab_get_cookie(session, url, user, password, ldap=False):
# get the CSRF token
content = session.get(url + '/users/sign_in').content
tokens = re.findall(b'"authenticity_token" value="([^"]+)"', content)
if ldap:
data = {'authenticity_token': tokens[0].decode('utf-8'), 'username': user, 'password': password}
url = url + '/users/auth/ldapmain/callback'
else:
data = {'authenticity_token': tokens[1].decode('utf-8'), 'user[login]': user, 'user[password]': password}
url = url + '/users/sign_in'
r = session.post(url, data=data)
if r.status_code != 200:
print('Failed logging in: {}'.format(r.status_code))
exit(1)
def gitlab_get_repo_list(session, url, method):
repo_list = []
page_number = 0
while True:
r = session.get(url + '/api/v4/projects?per_page=100&page={}'.format(page_number))
projects_page = json.loads(r.text)
if len(projects_page) == 0:
break
for project in projects_page:
if method == 'SSH':
repo_list.append(project['ssh_url_to_repo'])
if method == 'HTTP':
repo_list.append(project['http_url_to_repo'])
page_number += 1
return repo_list
# MAIN #########################################################################
parser = argparse.ArgumentParser()
parser.add_argument('url', type=str)
parser.add_argument('-u', '--user', type=str)
parser.add_argument('-p', '--password', type=str)
parser.add_argument('-c', '--cookie', type=str)
parser.add_argument('-o', '--output-dir', type=str, required=True)
parser.add_argument('-m', '--method', type=str, default='HTTP', help='Cloning method: HTTP or SSH (default: HTTP)')
parser.add_argument('-l', '--ldap', action='store_true')
args = parser.parse_args()
s = requests.Session()
s.verify = False
if args.user:
if not args.password:
args.password = getpass("password: ")
gitlab_get_cookie(s, args.url.rstrip('/'), args.user, args.password, args.ldap)
args.password = args.password.replace('@', '%40') # escape for HTTP later on
if args.cookie:
cookie = requests.cookies.create_cookie(name='_gitlab_session', value=args.cookie)
s.cookies.set_cookie(cookie)
OUTPUT_DIR = args.output_dir + '/'
print('[+] Getting list of repositories...')
repo_list = gitlab_get_repo_list(s, args.url.rstrip('/'), args.method)
print('[+] Cloning {} repositories...'.format(len(repo_list)))
do_clone(repo_list, args.method)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.