Skip to content

Instantly share code, notes, and snippets.

@bogus34
Last active June 30, 2020 11:24
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save bogus34/1da6409e972eb116b70359ac9c1eec26 to your computer and use it in GitHub Desktop.
Save bogus34/1da6409e972eb116b70359ac9c1eec26 to your computer and use it in GitHub Desktop.
clone all repos from bitbucket
In order to get private repos you should add application key on bitbucket w/ permissions to read account,
repos and workspaces. Assume you have ssh-key for your repositories configured.
Dependencies:
npm install node-fetch qs base-64
Cheers!
{
"username": "...",
"password": "..."
}
const fetch = require('node-fetch');
const qs = require('qs');
const base64 = require('base-64');
const credentials = require('./credentials.json');
const { exec } = require('child_process');
const fs = require('fs');
const API_ROOT = 'https://bitbucket.org/api/2.0';
async function apiCall(method, path, query, body) {
path = API_ROOT + path;
if (query) {
q = qs.stringify(query);
path = path + '?' + q;
}
const headers = {
Authorization: 'Basic ' + base64.encode(`${credentials.username}:${credentials.password}`)
};
let response;
if (method.toLowerCase() == 'get') {
response = await fetch(path, {
method,
headers
});
} else {
response = await fetch(path, {
method,
headers,
body: JSON.stringify(body)
});
}
if (!response.ok) {
throw response;
}
return response.json();
}
function getWorkspaces() {
return apiCall('get', '/workspaces');
}
function getRepositories(workspace) {
return apiCall('get', `/repositories/${workspace}`, { pagelen: 100, role: 'member' });
}
function shell(cmd, options) {
return new Promise((resolve, reject) => {
const cp = exec(cmd, options);
cp.on('error', (e) => reject(e));
cp.on('exit', (code) => resolve(code));
});
}
async function cloneHg(repo) {
console.log('>>> ', repo.full_name);
const cwd = `./repositories/${repo.workspace.slug}`;
fs.mkdirSync(cwd, { recursive: true });
await shell(`hg clone ssh://hg@bitbucket.org/${repo.full_name}`, { cwd });
console.log('>>> cloned to', repo.full_name);
}
async function cloneGit(repo) {
console.log('>>> ', repo.full_name);
const cwd = `./repositories/${repo.workspace.slug}`;
fs.mkdirSync(cwd, { recursive: true });
await shell(`git clone git@bitbucket.org:${repo.full_name}.git`, { cwd });
console.log('>>> cloned to', repo.full_name);
}
async function run() {
const workspaces = await getWorkspaces();
console.log(
'>>>',
workspaces.values.map((w) => w.name)
);
const allRepos = await Promise.all(workspaces.values.map((w) => getRepositories(w.slug)));
const repos = allRepos.reduce((sum, r) => sum.concat(r.values), []);
console.log(
'>>>',
repos.map((r) => r.full_name)
);
console.log('>>>', repos.length);
for (r of repos) {
if (r.scm == 'hg') {
await cloneHg(r);
} else {
await cloneGit(r);
}
}
}
run().catch((e) => console.log(e));
@sebastianberm
Copy link

Please add:
const fs = require('fs');

If you get an error that fs.mkdirSync doesn't exist.

@bogus34
Copy link
Author

bogus34 commented Jun 30, 2020

Thanks, Sebastian! Of course that 'require' should be there.

@erkiesken
Copy link

erkiesken commented Jun 30, 2020

Thanks for the script @bogus34!

Some of my older repos were getting errors on cloning:

applying clone bundle from https://api.media.atlassian.com/file/25bb90ae-.../binary?client=d5951f7d-...
error fetching bundle: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1108)
abort: error applying bundle

I had to add --insecure option to hg clone for things to work.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment