Created
August 17, 2015 02:00
-
-
Save bryantrobbins/8aa6a43a625026a8aeb3 to your computer and use it in GitHub Desktop.
Jenkins parallel grid search; I've copied some code from a helper class here to simplify the example (you'll notice that JenkinsClient is really written as "Java" instead of Groovy). I just wanted the example to be self-contained - there are definitely much more "Groovy" ways to make HTTP calls like the ones our Groovy code needs to make.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apply plugin: 'java' | |
apply plugin: 'maven' | |
apply plugin: 'groovy' | |
import org.apache.commons.io.IOUtils | |
import org.apache.commons.io.FileUtils | |
import static com.xlson.groovycsv.CsvParser.parseCsv | |
// Load properties or defaults | |
def min_gamma_val = hasProperty('min_gamma') ? min_gamma : '' | |
def max_gamma_val = hasProperty('max_gamma') ? max_gamma : '' | |
def min_cost_val = hasProperty('min_cost') ? min_cost : '' | |
def max_cost_val = hasProperty('max_cost') ? max_cost : '' | |
def access_key_val = hasProperty('access_key') ? access_key : '' | |
def secret_key_val = hasProperty('secret_key') ? secret_key : '' | |
repositories { | |
mavenCentral() | |
} | |
dependencies { | |
compile 'org.codehaus.groovy:groovy-all:2.3.8' | |
compile 'com.amazonaws:aws-java-sdk-s3:1.10.11' | |
} | |
task tuningGrid(dependsOn: 'classes', type: JavaExec) { | |
main = 'tuningGrid' | |
classpath = sourceSets.main.runtimeClasspath | |
args dataset_val, min_gamma_val, max_gamma_val, min_cost_val, max_cost_val, access_key_val, secret_key_val | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import com.amazonaws.services.s3.AmazonS3Client | |
import com.amazonaws.auth.BasicAWSCredentials | |
import com.amazonaws.services.s3.model.GetObjectRequest | |
import com.google.gson.JsonElement; | |
import com.google.gson.JsonObject; | |
import com.google.gson.JsonParser; | |
import org.apache.http.HttpEntity; | |
import org.apache.http.HttpHost; | |
import org.apache.http.NameValuePair; | |
import org.apache.http.auth.AuthScope; | |
import org.apache.http.auth.UsernamePasswordCredentials; | |
import org.apache.http.client.AuthCache; | |
import org.apache.http.client.CredentialsProvider; | |
import org.apache.http.client.entity.UrlEncodedFormEntity; | |
import org.apache.http.client.methods.CloseableHttpResponse; | |
import org.apache.http.client.methods.HttpPost; | |
import org.apache.http.client.protocol.HttpClientContext; | |
import org.apache.http.impl.auth.BasicScheme; | |
import org.apache.http.impl.client.BasicAuthCache; | |
import org.apache.http.impl.client.BasicCredentialsProvider; | |
import org.apache.http.impl.client.CloseableHttpClient; | |
import org.apache.http.impl.client.HttpClients; | |
import org.apache.http.message.BasicNameValuePair; | |
import org.apache.http.util.EntityUtils; | |
import org.apache.log4j.LogManager; | |
import org.apache.log4j.Logger; | |
import java.io.IOException; | |
import java.util.ArrayList; | |
import java.util.List; | |
import java.util.Map; | |
import java.util.Map.Entry; | |
def master = "jenkins-master-host" | |
def dataset = args[0] | |
def minGammaExp = Integer.parseInt(args[1]) | |
def maxGammaExp = Integer.parseInt(args[2]) | |
def minCostExp = Integer.parseInt(args[3]) | |
def maxCostExp = Integer.parseInt(args[4]) | |
def accessKey = args[5] | |
def secretKey = args[6] | |
// Clients | |
def jenkinsClient = new JenkinsClient(master, "someport", "", "someuser", "secret") | |
def awsClient = new AwsUtils(accessKey, secretKey) | |
println "Removing old reports for this dataset" | |
awsClient.deleteOldReports("reports/${dataset}_report") | |
println "Train models with various parameters" | |
int reportCount = 0 | |
for (int gamma=minGammaExp; gamma<=maxGammaExp; gamma++) { | |
for (int cost=minCostExp; cost<=maxCostExp; cost++) { | |
reportCount++ | |
// build Map of params | |
// I have only used text params, but perhaps others supported via Jenkins Remote API | |
def jobParams = new HashMap<String, String>(); | |
jobParams.put("DATASET", dataset.toString()) | |
jobParams.put("GAMMA_EXPONENT", gamma.toString()) | |
jobParams.put("COST_EXPONENT", cost.toString()) | |
jobParams.put("ACCESS_KEY", accessKey) | |
jobParams.put("SECRET_KEY", secretKey) | |
// Use Jenkins client to launch job | |
jenkinsClient.submitJob("train-model", jobParams) | |
// ZZZ to let the master recover | |
sleep(1000) | |
} | |
} | |
// Wait for report files to show up in S3 | |
println "Waiting for reports" | |
def utils = new AwsUtils(accessKey, secretKey) | |
def reportKeys = utils.waitForReports("reports/${dataset}_report", reportCount) | |
// Find best accuracy from among reports | |
println "Finding the best model" | |
def bestAcc = 0.0 | |
def modelKey = null | |
reportKeys.each { | |
def text = utils.getTextRow(it) | |
def chunks = text.split(",") | |
def summary = [:] | |
def acc = Double.parseDouble(chunks[3]) | |
if (acc > bestAcc) { | |
bestAcc = acc | |
modelKey = chunks[2] | |
} | |
} | |
println "Best accuracy is ${bestAcc} by model ${modelKey}" | |
utils.writeBestFile(dataset, modelKey) | |
class AwsUtils { | |
def client | |
static final long POLL_INTERVAL= 30000 | |
static final long POLL_MAX = 1200000 | |
static final String BUCKET = "com.btr3.research" | |
def AwsUtils(accessKey, secretKey) { | |
client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey)) | |
} | |
def getReportKeys(String prefix){ | |
def allObjects = client.listObjects(BUCKET, prefix).getObjectSummaries() | |
allObjects.collect { | |
it.getKey() | |
} | |
} | |
def waitForReports(String prefix, int desiredCount) { | |
int waited = 0 | |
def actualKeys = getReportKeys(prefix) | |
while ((actualKeys.size() < desiredCount) && (waited < POLL_MAX)) { | |
if (actualKeys.size() == desiredCount) { | |
break; | |
} else if (actualKeys.size() > desiredCount) { | |
throw new RuntimeException("Asked for ${desiredCount} reports, but already found ${actualKeys.size()}") | |
} else { | |
println "Waiting for ${desiredCount} reports, found ${actualKeys.size()}" | |
Thread.sleep(POLL_INTERVAL) | |
waited += POLL_INTERVAL | |
actualKeys = getReportKeys(prefix) | |
} | |
} | |
if (waited >= POLL_MAX) { | |
throw new RuntimeException("Did not find ${desiredCount} reports after waiting max of ${POLL_MAX}ms") | |
} else { | |
actualKeys | |
} | |
} | |
def getTextRow(String key) { | |
client.getObject(BUCKET, key).getObjectContent().getText() | |
} | |
def deleteOldReports(String prefix) { | |
def keys = getReportKeys(prefix) | |
keys.each { | |
client.deleteObject(BUCKET, it) | |
} | |
} | |
def writeBestFile(String dataset, String modelKey) { | |
def bestFile = new File("${dataset}_best") | |
bestFile.withWriter('UTF-8') { writer -> | |
writer.write("${modelKey}") | |
} | |
def gor = new GetObjectRequest(BUCKET, "models/${modelKey}") | |
client.getObject(gor, new File("${modelKey}")) | |
} | |
} | |
public class JenkinsClient { | |
/** | |
* A Logger. | |
*/ | |
private static Logger logger = LogManager.getLogger(JenkinsClient.class); | |
/** | |
* We will wait this long between successive calls checking status. | |
*/ | |
private static final long DEFAULT_POLLING_INTERVAL = 30000; | |
/** | |
* HTTP Page not found code. | |
*/ | |
private static final int HTTP_NOT_FOUND = 404; | |
/** | |
* HTTP Internal Server Error code. | |
*/ | |
private static final int HTTP_INTERNAL_SERVER_ERROR = 500; | |
/** | |
* The host of the Jenkins master. | |
*/ | |
private String host; | |
/** | |
* The Jenkins port. | |
*/ | |
private String port; | |
/** | |
* The path to the jenkins installation. | |
*/ | |
private String path; | |
/** | |
* The Jenkins user to use for checking status. | |
*/ | |
private String user; | |
/** | |
* The password for the Jenkins user above. | |
*/ | |
private String password; | |
/** | |
* A default constructor with Jenkins server details. | |
* | |
* @param hostVal the Jenkins host | |
* @param portVal the port of the Jenkins server | |
* @param pathVal the path to the Jenkins installation | |
* @param userVal a user on the Jenkins machine | |
* @param passwordVal the user's password | |
*/ | |
public JenkinsClient(final String hostVal, final String portVal, | |
final String pathVal, final String userVal, | |
final String passwordVal) { | |
this.host = hostVal; | |
this.port = portVal; | |
this.path = pathVal; | |
this.user = userVal; | |
this.password = passwordVal; | |
} | |
/** | |
* Helper method for carrying out an authenticated HTTP Request given URL | |
* and Body. | |
* | |
* @param url the URL to be requested | |
* @param body the body of the request | |
* @return the response body | |
* @throws IOException if response cannot be read | |
*/ | |
private String makeHttpRequest(final String url, final String body) throws | |
IOException { | |
// This usage of HttpClient based on example: | |
// http://hc.apache.org/httpcomponents-client-ga/httpclient/examples | |
// /org/apache/http/examples/client/ClientAuthentication.java | |
// Basic auth usage based on second example: | |
// http://hc.apache.org/httpcomponents-client-ga/httpclient/examples | |
// /org/apache/http/examples/client | |
// /ClientPreemptiveBasicAuthentication.java | |
// Content to be returned | |
String ret = null; | |
// Register handler for Basic Auth on Jenkins server | |
HttpHost targetHost = new HttpHost(host, Integer.parseInt(port), | |
"http"); | |
CredentialsProvider credsProvider = new BasicCredentialsProvider(); | |
credsProvider.setCredentials(new AuthScope(targetHost.getHostName(), | |
targetHost.getPort()), new UsernamePasswordCredentials(user, | |
password)); | |
CloseableHttpClient httpclient = HttpClients.custom() | |
.setDefaultCredentialsProvider(credsProvider).build(); | |
// Create AuthCache instance | |
AuthCache authCache = new BasicAuthCache(); | |
// Generate BASIC scheme object and add it to the local | |
// auth cache | |
BasicScheme basicAuth = new BasicScheme(); | |
authCache.put(targetHost, basicAuth); | |
// Add AuthCache to the execution context | |
HttpClientContext localContext = HttpClientContext.create(); | |
localContext.setAuthCache(authCache); | |
try { | |
HttpPost httppost = new HttpPost(url); | |
// Add payload, if available | |
if (body != null) { | |
List<NameValuePair> nvps = new ArrayList<NameValuePair>(); | |
nvps.add(new BasicNameValuePair("json", body)); | |
httppost.setEntity(new UrlEncodedFormEntity(nvps)); | |
logger.debug(httppost.getEntity().toString()); | |
} | |
System.out.println("executing request" + httppost.getRequestLine()); | |
CloseableHttpResponse response = httpclient.execute(targetHost, | |
httppost, localContext); | |
try { | |
HttpEntity entity = response.getEntity(); | |
System.out.println("----------------------------------------"); | |
System.out.println(response.getStatusLine()); | |
System.out.println(response.getStatusLine().getStatusCode()); | |
if (entity != null) { | |
System.out.println("Response content length: " | |
+ entity.getContentLength()); | |
} | |
if ((response.getStatusLine().getStatusCode() | |
== HTTP_NOT_FOUND) || (response.getStatusLine() | |
.getStatusCode() == HTTP_INTERNAL_SERVER_ERROR)) { | |
ret = null; | |
} else { | |
ret = EntityUtils.toString(entity); | |
} | |
EntityUtils.consume(entity); | |
} finally { | |
response.close(); | |
} | |
} finally { | |
httpclient.close(); | |
} | |
return ret; | |
} | |
/** | |
* Return the status of a previously submitted job given job name and | |
* build number. | |
* | |
* @param jobName the job name | |
* @param buildNumber the build number | |
* @return job JSON content, which includes status | |
*/ | |
public String getJobStatus(final String jobName, final String buildNumber) { | |
String buildUrl = "http://" + host + ":" + port + "/" + path + "/job/" | |
+ jobName + "/build/" + buildNumber + "/api/json"; | |
try { | |
String buildStatusJson = this.makeHttpRequest(buildUrl, null); | |
JsonElement buildElement = new JsonParser().parse(buildStatusJson); | |
JsonObject buildObject = buildElement.getAsJsonObject(); | |
return buildObject.getAsJsonPrimitive("result").toString(); | |
} catch (IOException e) { | |
// We can safely ignore this exception as we will retry later | |
e.printStackTrace(); | |
return null; | |
} | |
} | |
/** | |
* Method to submit Jenkins job given job name and job parameters. | |
* | |
* @param jobName the job name | |
* @param params the job parameters as a String, String Map | |
* @throws IOException if valid HTTP Request cannot be constructed | |
*/ | |
public void submitJob(final String jobName, final Map<String, | |
String> params) throws | |
IOException { | |
String urlBuild = "http://" + host + ":" + port + "/" + path + "/job/" | |
+ jobName + "/build"; | |
StringBuffer payload = null; | |
if (params != null) { | |
// Build the Jenkins payload | |
payload = new StringBuffer(); | |
payload.append("{\"parameter\": ["); | |
boolean comma = false; | |
for (Entry<String, String> pair : params.entrySet()) { | |
if (comma) { | |
payload.append(","); | |
} else { | |
comma = true; | |
} | |
payload.append("{\"name\": \""); | |
payload.append(pair.getKey()); | |
payload.append("\", \"value\": \""); | |
payload.append(pair.getValue()); | |
payload.append("\"}"); | |
} | |
payload.append("], \"\": \"\"}"); | |
logger.debug(payload); | |
} | |
String pString = null; | |
if (payload != null) { | |
pString = payload.toString(); | |
} | |
this.makeHttpRequest(urlBuild, pString); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
TODO: Use Groovy RestClient or similar to avoid need for hefty "JenkinsClient" class.