Backend Test Results Generator

readme.md

Description:

Currently, it is very difficult to get a consolidated list of backend test failures from the PR tester. Both the comment and the jenkins-report.html don't present the information in a way that is convenient for stakeholders to analyze. This tool, will take a given jenkins-report.html and generate an html file that consolidates the list of backend test failures (any integration, modules-integration, modules-unit, or unit tests) by their batch name and provide links to all errors.

Usage:

1. Download the ci.xml file
2. Place the file in the liferay-portal/master repo
3. Run: ant -f ci.xml prepare-classpath
4. Run: ant -f ci.xml -Djenkins.report.url=VALID_REPORT_HTML_URL (For OSX, ( or ) may need to be escaped with \)
5. backend-test-failures.html will be generated in the root directory

Sample output

http://bl.ocks.org/kenjiheigel/raw/d183247d4adac64c83558645e1927bfb/

Future updates

• Grab the information from the PR link directly using the github API
• Streamline classpath dependencies
• Polish methods and simplify
• Potentially convert to gradle?
• Long term goal: move functionality to jenkins-results-parser

ci.xml

<?xml version="1.0"?>

<project basedir="." default="generate-backend-results" name="ci" xmlns:antelope="antlib:ise.antelope.tasks">
	<import file="build-test.xml" />

	<target name="prepare-classpath">
		<gradle-execute dir="modules/test/jenkins-results-parser" task="jar" />

		<copy todir="lib/development">
			<fileset dir="tools/sdk/dist" includes="com.liferay.jenkins.results.parser*.jar" />
		</copy>
	</target>

	<target name="generate-backend-results">
		<fail message="Please set jenkins.report.url to a valid jenkins-report.html URL.">
			<condition>
				<not>
					<isset property="jenkins.report.url" />
				</not>
			</condition>
		</fail>

		<beanshell>
			<![CDATA[
import com.liferay.jenkins.results.parser.JenkinsResultsParserUtil;

import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.json.JSONArray;
import org.json.JSONObject;

String getURLContent(URL url) throws Exception {
	URLConnection conn = url.openConnection();

	BufferedReader br = new BufferedReader(
		new InputStreamReader(conn.getInputStream()));

	String inputLine;

	StringBuilder sb = new StringBuilder();

	while ((inputLine = br.readLine()) != null) {
		sb.append(inputLine);
	}

	br.close();

	return sb.toString();
}

Map getJenkinsBatchJSONURLs(String htmlContent) throws Exception {
	List regexs = new ArrayList();

	regexs.add("/(integration.*?)<.*?\"(.*?)//console");
	regexs.add("/(modules-integration.*?)<.*?\"(.*?)//console");
	regexs.add("/(unit.*?)<.*?\"(.*?)//console");
	regexs.add("/(modules-unit.*?)<.*?\"(.*?)//console");

	Map batchURLs = new TreeMap();

	for (String regex : regexs) {
		Pattern pattern = Pattern.compile(regex);

		Matcher matcher = pattern.matcher(htmlContent);

		while (matcher.find()) {
			JSONObject batchJSONObject = JenkinsResultsParserUtil.toJSONObject(
				matcher.group(2) +
				"/api/json?tree=actions[parameters[name,value]],result");

			JSONArray actionsJSONArray = batchJSONObject.getJSONArray(
				"actions");

			String batchName = null;

			for (int i = 0; i < actionsJSONArray.length(); i++) {
				if (actionsJSONArray.isNull(i)) {
					break;
				}

				JSONObject actionsJSONObject = actionsJSONArray.getJSONObject(
					i);

				if (actionsJSONObject.isNull("parameters")) {
					break;
				}

				JSONArray parametersJSONArray = actionsJSONObject.getJSONArray(
					"parameters");

				for (int i = 0; i < parametersJSONArray.length(); i++) {
					JSONObject parameterJSONObject =
						parametersJSONArray.getJSONObject(i);

					String name = parameterJSONObject.getString(
						"name");

					if (name.equals("JOB_VARIANT")) {
						batchName = parameterJSONObject.getString("value");
					}
				}
			}

			resultString = batchJSONObject.getString("result");

			if (!resultString.equals("SUCCESS")) {
				batchURLs.put(batchName, matcher.group(2));
			}
		}
	}

	return batchURLs;
}

Map getFailingTestURLs(Map batchURLs) {
	Map testURLMap = new TreeMap();

	for (Map.Entry batchURL : batchURLs.entrySet()) {
		Map testURLs = new TreeMap();

		StringBuilder sb = new StringBuilder();

		sb.append(batchURL.getValue());
		sb.append("/testReport/api/json?pretty&tree=childReports[child[url],");
		sb.append("result[suites[cases[className,name,status]]]]");

		JSONObject batchJSONObject = JenkinsResultsParserUtil.toJSONObject(
			sb.toString());

		JSONArray childReportsJSONArray = batchJSONObject.getJSONArray(
			"childReports");

		for (int i = 0; i < childReportsJSONArray.length(); i++) {
			JSONObject childReportsJSONObject =
				childReportsJSONArray.getJSONObject(i);

			JSONObject childJSONObject = childReportsJSONObject.getJSONObject(
				"child");

			JSONObject resultJSONObject = childReportsJSONObject.getJSONObject(
				"result");

			JSONArray suitesJSONArray = resultJSONObject.getJSONArray("suites");

			for (int i = 0; i < suitesJSONArray.length(); i++) {
				JSONObject suiteJSONObject = suitesJSONArray.getJSONObject(i);

				JSONArray casesJSONArray = suiteJSONObject.getJSONArray(
					"cases");

				for (int i = 0; i < casesJSONArray.length(); i++) {
					JSONObject caseJSONObject = casesJSONArray.getJSONObject(i);

					String status = caseJSONObject.getString("status");

					if (status.equals("FAILED") ||
						status.equals("REGRESSION")) {

						String className = caseJSONObject.getString(
							"className");
						String name = caseJSONObject.getString("name");
						String url = childJSONObject.getString("url");

						testURLs.put(className + "." + name, url);
					}
				}
			}
		}

		testURLMap.put(batchURL.getKey(), testURLs);
	}

	return testURLMap;
}

void writeTestResults(Map testURLMaps) {
	StringBuilder sb = new StringBuilder();

	sb.append("<html>");

	for (Map.Entry testURLMap : testURLMaps.entrySet()) {
		String batchName = testURLMap.getKey();
		Map testURLs = testURLMap.getValue();

		sb.append("<strong>");
		sb.append(batchName);
		sb.append("</strong><ol>");

		for (Map.Entry testURL : testURLs.entrySet()) {
			String testName = testURL.getKey();
			String axisURL = testURL.getValue();

			sb.append("<li><a href=\"");
			sb.append(axisURL);
			sb.append("testReport/junit/");

			String testNameURL = testName;

			if (testNameURL.contains("TestSuite")) {
				testNameURL = testNameURL.replace(".TestSuite.","/TestSuite/");

				testNameURL = testNameURL.replace(".","_");

				testNameURL = testNameURL.replace(
					"junit_framework","junit.framework");
			}

			testNameURL = testNameURL.replaceAll("([a-z])[.]([A-Z])","$1/$2");
			testNameURL = testNameURL.replaceAll("(Test)[.](test)","$1/$2");

			sb.append(testNameURL);
			sb.append("\">");
			sb.append(testName);
			sb.append("</a></li>");

			System.out.println(axisURL + "testReport/junit/" + testNameURL);
		}

		sb.append("</ol>");
	}

	sb.append("</html>");

	PrintWriter writer = new PrintWriter("backend-test-failures.html", "UTF-8");

	writer.println(sb.toString());

	writer.close();

	File file = new File("backend-test-failures.html");
	System.out.println("Backend test failures have been generated at: ");
	System.out.println(file.getAbsolutePath());
}

URL jenkinsURL = new URL(project.getProperty("jenkins.report.url"));

String jenkinsURLContent = getURLContent(jenkinsURL);

Map failingBatchURLs = getJenkinsBatchJSONURLs(jenkinsURLContent);

Map failingTestURLs = getFailingTestURLs(failingBatchURLs);

writeTestResults(failingTestURLs);


			]]>
		</beanshell>
	</target>
</project>

How about adding the target="_blank" attribute to each test result link? That way they'll open in new tabs.