Skip to content

Instantly share code, notes, and snippets.

@archenroot
Last active March 5, 2021 00:42
Show Gist options
  • Save archenroot/1d11f58d182163ce55bdafcdfe11d411 to your computer and use it in GitHub Desktop.
Save archenroot/1d11f58d182163ce55bdafcdfe11d411 to your computer and use it in GitHub Desktop.
Clone Stash/BitBucket - all repositories from all projects which user has access to
package org.prokyon.management.bitbucket_cloner;
import com.google.common.base.CaseFormat;
import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.JsonNode;
import com.mashape.unirest.http.Unirest;
import com.mashape.unirest.http.exceptions.UnirestException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.File;
/**
* Created by zangetsu on 1/31/17.
*/
public class Cloner {
private static final Logger log = LogManager.getLogger(Cloner.class);
String BIT_BUCKET_SERVER = "http://stash.domain.com";
String REST_API_SUFFIX ="/rest/api/1.0";
String API_PROJECTS = "/projects";
String API_REPOSITORIES ="/repos";
String DEFAULT_DIR = "/home/user/projects..";
String USER = "user";
String PASSWORD = "password";
public static JSONArray projects = null;
public static JSONArray repositories = null;
private void obtainProjects() {
log.debug("Going to obtain projects from: " + BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS);
try {
projects = Unirest.get(BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS)
.basicAuth(USER, PASSWORD)
.header("accept", "application/json")
.asJson()
.getBody()
.getObject().getJSONArray("values");
} catch (UnirestException e) {
log.error("Cannot obtain list of projects from " + BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS);
}
}
private void obtainRepositories(String projectKey) {
String projectReposURL = BIT_BUCKET_SERVER +REST_API_SUFFIX + API_PROJECTS + "/" + projectKey + API_REPOSITORIES;
log.info("Obtaining repos from " + projectReposURL);
try {
repositories = Unirest.get(projectReposURL)
.basicAuth(USER, PASSWORD)
.header("accept", "application/json")
.asJson()
.getBody()
.getObject().getJSONArray("values");
} catch (UnirestException e) {
log.error("Cannot obtain project repositories from " + projectReposURL);
}
//log.debug("Repositories obtained: " + repositories);
}
private void cloneRepository(String gitURL,String repoDir){
log.info("Going to clone repo " + gitURL);
log.info("Repository will be stored at " + repoDir);
try {
Git git = Git.cloneRepository()
.setURI(gitURL)
.setDirectory(new File(repoDir))
.setCloneAllBranches(true)
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(USER, PASSWORD))
.call();
} catch (GitAPIException e) {
log.error("Error on cloning repository from " + gitURL + " into local directory " + repoDir + ". Check the path.");
}
}
public void cloner(){
// Obtain projects
obtainProjects();
if (projects.length()<1){
log.info("There are no projects available to process.");
} else{
for (int i = 0; i < projects.length();i++) {
JSONObject project = (JSONObject) projects.get(i);
//log.debug("Project data: " + project);
String projectName = project.getString("name").toLowerCase().replace(" ","_");
log.debug("Project name: " + projectName);
String projectKey = project.get("key").toString();
log.debug("Project key: " + projectKey);
// Obtain repos
obtainRepositories(projectKey);
for (int k = 0; k < repositories.length();k++){
JSONObject repository = (JSONObject) repositories.get(k);
//log.debug("DEBUG Repository " + repository);
String repoName = repository.getString("name");
log.debug("Repository name: " + repoName);
String repoDir = DEFAULT_DIR + "/" + projectName + "/" + repoName;
log.debug("Repository local directory where clone to: " + repoDir);
final JSONArray cloneURLs = (JSONArray) ((JSONObject) repository.get("links")).get("clone");
for (int r = 0; r < cloneURLs.length();r++){
if (((JSONObject)cloneURLs.get(r)).get("name").toString().equals("http")){
log.debug("HTTP repository link for clone found.");
String repoURL = ((JSONObject) cloneURLs.get(r)).getString("href");
cloneRepository(repoURL,repoDir);
} else{
log.debug(((JSONObject)cloneURLs.get(r)).get("name").toString());
}
}
}
}
}
}
public static void main(String[] args) throws UnirestException {
Cloner c = new Cloner();
c.cloner();
}
}
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO">
<Appenders>
<Console name="console-log" target="SYSTEM_OUT">
<PatternLayout pattern="[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Logger name="org.prokyon.management.bitbucket_cloner" level="debug" additivity="false">
<appender-ref ref="console-log" level="debug"/>
</Logger>
<Root level="info" additivity="false">
<AppenderRef ref="console-log"/>
</Root>
</Loggers>
</Configuration>
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.prokyon.management</groupId>
<artifactId>bitbucket-cloner</artifactId>
<version>1.0-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>com.mashape.unirest</groupId>
<artifactId>unirest-java</artifactId>
<version>1.4.9</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.4.1</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>RELEASE</version>
</dependency>
</dependencies>
</project>
@jauyzed
Copy link

jauyzed commented May 11, 2017

This is a great solution. Can you please add a readme on how to use this?

@archenroot
Copy link
Author

archenroot commented May 18, 2017

Hi, not a problem, actually I might little bit redesign it to be more externally configurable... via some properties file or similar, so you can just provide url, credentials, etc. and you go..

@archenroot
Copy link
Author

maybe we can create a generic service for cloning repositories and keep them uptodate accross all git/svn/hg systems (they have some remote API), if someone is interested ping me, I have quite nice template about how to implement such system.

The functionality could be for example:

  • clone
  • continuously compile after clone (so will support build commands/command sequence for every possible build system - autogen, cmake, maven, gradle, nodejs, etc.... whatever)

Ping me if someone would be interested in this... I will join the mission as this is handy.

@mahyadav1
Copy link

mahyadav1 commented Jul 4, 2018

Hii Archenroot,
I am new in git, can you explain little bit about below lines
String BIT_BUCKET_SERVER = "http://stash.domain.com";
String REST_API_SUFFIX ="/rest/api/1.0";
String API_PROJECTS = "/projects";
String API_REPOSITORIES ="/repos";
String DEFAULT_DIR = "/home/user/projects..";

how we should construct url to hit git

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment