Skip to content

Instantly share code, notes, and snippets.

@rivasdiaz
Last active November 21, 2021 02:49
Show Gist options
  • Save rivasdiaz/f2e9efd1bf2079f0101f865846af2502 to your computer and use it in GitHub Desktop.
Save rivasdiaz/f2e9efd1bf2079f0101f865846af2502 to your computer and use it in GitHub Desktop.
Maven Project Statistics
///usr/bin/env jbang "$0" "$@" ; exit $?
//JDK 17+
//DEPS io.quarkus:quarkus-bom:2.4.1.Final@pom
//DEPS io.quarkus:quarkus-picocli
//DEPS io.quarkus:quarkus-jackson
//DEPS io.quarkus:quarkus-smallrye-context-propagation
//DEPS com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.12.5
//DEPS com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.5
//DEPS org.zeroturnaround:zt-exec:1.12
//DEPS commons-io:commons-io:2.11.0
//Q:CONFIG quarkus.banner.enabled=false
//Q:CONFIG quarkus.log.level=WARN
//Q:CONFIG quarkus.log.console.format=%d{HH:mm:ss} %-5p %s%e%n
//Q:CONFIG quarkus.log.category."cli".level=${logging.level:INFO}
//Q:CONFIG quarkus.log.console.stderr=${logging.stderr:true}
//TODO find out why these properties do not work if set as Q:CONFIG
//JAVA_OPTIONS -Dmavenstats.parallel.enabled=${parallel.enabled:true}
//JAVA_OPTIONS -Dmavenstats.parallel.max-async=${parallel.max-async:0}
//JAVA_OPTIONS -Dmavenstats.parallel.max-queue=${parallel.max-queue:2048}
/*
Example use:
stats from:
- quarkus project
- only tags 2.4.x.Final (--tag-regex=...)
- only analyze modules: (--module=regex=...)
* core/...
* extensions/...
- reuse previously generated tag information (--no-override-existing-release)
j! -Dlogging.level=DEBUG mavenstats.java \
--tag-regex='2\.4\..*\.Final' \
--module-regex='(core|extensions)/.*' \
--no-override-existing-release \
--maven-define=skipTests \
--maven-define=skipITs \
--continue-on-go-offline-failure \
--maven-settings=/usr/local/Cellar/maven/3.8.2/libexec/conf/settings.xml \
--output=quarkus \
~/Projects/Upstream/quarkus
*/
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.cfg.MapperBuilder;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.quarkus.arc.log.LoggerName;
import io.quarkus.runtime.Quarkus;
import io.smallrye.config.ConfigMapping;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.inject.Produces;
import javax.inject.Inject;
import javax.inject.Named;
import org.apache.commons.io.FilenameUtils;
import org.eclipse.microprofile.context.ManagedExecutor;
import org.jboss.logging.BasicLogger;
import org.jboss.logging.Logger;
import org.jboss.logging.Logger.Level;
import org.zeroturnaround.exec.ProcessExecutor;
import org.zeroturnaround.exec.stream.LogOutputStream;
import picocli.CommandLine;
/**
* app configuration
*/
@ConfigMapping(prefix = "mavenstats")
interface MavenStatsConfig {
MavenStatsParallelExecutionConfig parallel();
}
interface MavenStatsParallelExecutionConfig {
boolean enabled();
int maxAsync();
int maxQueue();
}
/**
* CLI: receives, parses and validates user supplied parameters
*/
@CommandLine.Command(mixinStandardHelpOptions = true, name = "mavenstats", version = "0.5.0")
public class mavenstats implements Runnable {
@LoggerName("cli.main")
Logger logger;
@CommandLine.Option(
names = {"-b", "--main-branch"}, paramLabel = "branch-name", defaultValue = "main",
description = "Main branch name (default: ${DEFAULT-VALUE})")
String mainBranch;
@CommandLine.Option(
names = {"--tag-regex"}, paramLabel = "tag-regex", defaultValue = "",
description = "Regular expression to filter tags (default: ${DEFAULT-VALUE})")
String tagRegex;
@CommandLine.Option(
names = {"--module-regex"}, paramLabel = "module-regex", defaultValue = "",
description = "Regular expression to filter list of modules (default: ${DEFAULT-VALUE})")
String moduleRegex;
@CommandLine.Option(
names = {"--maven-settings"}, paramLabel = "settings.xml",
description = "Use a different maven settings file")
Path mavenSettingsFile;
@CommandLine.Option(
names = {"-o", "--output"}, paramLabel = "output-dir",
description = "Folder to write output files (default: standard output)")
Path outputDir;
@CommandLine.Option(
names = {"--exclude-dependency-stats"}, defaultValue = "false",
description = "Exclude stats about dependencies (default: ${DEFAULT-VALUE})")
boolean excludeDependencyStats;
@CommandLine.Option(
names = {"--exclude-file-stats"}, defaultValue = "false",
description = "Exclude stats about files (default: ${DEFAULT-VALUE})")
boolean excludeFileStats;
@CommandLine.Option(
names = {"--exclude-compilation-stats"}, defaultValue = "false",
description = "Exclude stats about compiling the project (default: ${DEFAULT-VALUE})")
boolean excludeCompilationStats;
@CommandLine.Option(
names = {"-E", "--no-each",
"--no-save-each-release"}, negatable = true, defaultValue = "true",
description = "Output a JSON file on each release. Only if output-dir set. (default: ${DEFAULT-VALUE})")
boolean saveEachRelease;
@CommandLine.Option(
names = {"-A", "--no-all", "--no-save-all-releases"}, negatable = true, defaultValue = "true",
description = "Output a JSON file with all releases (default: ${DEFAULT-VALUE})")
boolean saveAllReleases;
@CommandLine.Option(
names = {"--no-override-existing-release"}, negatable = true, defaultValue = "true",
description = "Override a previously generated JSON release file. If negated, load previous instead (default: ${DEFAULT-VALUE})")
boolean overrideExistingRelease;
@CommandLine.Option(
names = {"--no-go-offline-before-release"}, negatable = true, defaultValue = "true",
description = "Download dependencies/plugins before processing each release. Timestamps won't depend on network (default: ${DEFAULT-VALUE})")
boolean goOfflineBeforeRelease;
@CommandLine.Option(
names = {"--continue-on-go-offline-failure"}, negatable = true, defaultValue = "false",
description = "If go offline fails, ignore and continue. Set it to true only if you want the optimization but you don't care about the timestamps (default: ${DEFAULT-VALUE})")
boolean continueOnGoOfflineFailure;
@CommandLine.Option(
names = {"--maven-define", "--maven-property"}, arity = "*",
description = "Define a system property. Passed in all maven invocations")
List<String> mavenProperties;
@CommandLine.Option(
names = {"--maven-profiles", "--maven-active-profiles"}, arity = "*",
description = "List of profiles to activate. Passed in all maven invocations")
List<String> mavenProfiles;
@CommandLine.Option(
names = {"-e", "--env"}, arity = "*",
description = "Environment variables to add. Passed to all processes")
Map<String, String> environment;
@CommandLine.Parameters(
index = "0", paramLabel = "project-dir",
description = "The root folder of the project")
Path projectDir;
@Inject
ProcessManager proc;
@Inject
MavenStatsAnalyzer analyzer;
@Override
public void run() {
// some validations to avoid possible mistakes
logger.debugf("project path: %s", projectDir);
if (!Files.isDirectory(projectDir)) {
logger.fatalf("project-dir is not a valid directory: %s", projectDir);
Quarkus.asyncExit(1);
return;
}
if (!Files.isDirectory(projectDir.resolve(".git"))) {
logger.fatalf("project doesn't contain a git repository");
Quarkus.asyncExit(1);
return;
}
if (!proc.output(projectDir, environment, "git", "status", "--porcelain=v1").trim().isEmpty()) {
logger.fatalf("project contains git uncommitted changes");
Quarkus.asyncExit(1);
return;
}
if (outputDir != null) {
if (!Files.exists(outputDir)) {
try {
Files.createDirectories(outputDir);
} catch (IOException e) {
logger.fatalf("I/O error creating output directory: %s", e.getMessage());
Quarkus.asyncExit(1);
return;
}
}
if (!Files.isDirectory(outputDir)) {
logger.fatalf("output-dir is not a valid directory: %s", projectDir);
Quarkus.asyncExit(1);
return;
}
}
analyzer.runAnalysis(
new Parameters(
mainBranch, tagRegex, moduleRegex, mavenSettingsFile,
mavenProperties, mavenProfiles,
!excludeDependencyStats, !excludeFileStats, !excludeCompilationStats,
saveEachRelease, saveAllReleases, overrideExistingRelease,
goOfflineBeforeRelease, continueOnGoOfflineFailure,
outputDir, projectDir, environment));
}
}
/**
* User supplied parameters
*/
record Parameters(
String mainBranch, String tagRegex,
String moduleRegex, Path mavenSettingsFile,
List<String> mavenProperties, List<String> mavenProfiles,
boolean includeDependencyStats, boolean includeFileStats, boolean includeCompilationStats,
boolean saveEachRelease, boolean saveAllReleases, boolean overrideExistingRelease,
boolean goOfflineBeforeRelease, boolean continueOnGoOfflineFailure,
Path outputDir, Path projectDir, Map<String, String> environment) {
}
/**
* Maven project stats
*/
@ApplicationScoped
class MavenStatsAnalyzer {
public static final String COMMIT_DATE_REGEX = "(?m)^Date:\\s+(.*)$";
public static final Pattern COMMIT_DATE_PATTERN = Pattern.compile(COMMIT_DATE_REGEX);
public static final String COMMIT_AUTHOR_REGEX = "(?m)^Author:\\s+(.*)\\s+<(.*)>$";
public static final Pattern COMMIT_AUTHOR_PATTERN = Pattern.compile(COMMIT_AUTHOR_REGEX);
@LoggerName("cli.mavenstatsanalyzer")
Logger logger;
@Inject
ProcessManager proc;
@Inject
@Named("xmlMapper")
Mapper xml;
@Inject
@Named("jsonMapper")
Mapper json;
@Inject
@Named("cpuBoundExecutor")
ManagedExecutor executor;
@Inject
MavenStatsConfig config;
public void runAnalysis(Parameters params) {
logger.debugf("project analysis starting with configuration:");
json.log(Level.DEBUG, params);
final var projectHistory = analyzeProjectHistory(params);
logger.debugf("project analysis completed");
if (params.saveAllReleases()) {
if (params.outputDir() != null) {
json.write(
projectHistory,
params.outputDir().resolve("releases.json"),
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
} else {
json.write(
projectHistory,
System.out);
}
}
}
private ProjectHistoryInfo analyzeProjectHistory(Parameters params) {
// switch to the main branch before pulling the list of tags
if (!proc.output(params.projectDir(), params.environment(), "git", "branch", "--show-current")
.trim()
.equals(params.mainBranch())) {
logger.debugf("project not in main branch. switching to: %s", params.mainBranch());
proc.run(
params.projectDir(), params.environment(),
"git", "checkout", params.mainBranch(), "--quiet");
} else {
logger.debugf("project already in main branch: %s", params.mainBranch());
}
final var fullTagList = Arrays.asList(
proc.output(
params.projectDir(), params.environment(),
"git", "tag", "--list").split("\\s+"));
final var selectedTagList = fullTagList.stream()
.filter(tag -> params.tagRegex().isEmpty() || tag.matches(params.tagRegex()))
.sorted(TagComparator.INSTANCE)
.toList();
logger.debugf("number of tags: %d (of %d)", selectedTagList.size(), fullTagList.size());
logger.debug(selectedTagList);
// go over each tag detected
final var releases = new ArrayList<ProjectInfo>();
for (String tag : selectedTagList) {
try {
final var outputReleaseFile = params.outputDir() != null
? params.outputDir().resolve("release.%s.json".formatted(tag.replaceAll("[/: \\t]", "_")))
: null;
final ProjectInfo projectInfo;
if (!params.overrideExistingRelease()
&& outputReleaseFile != null && Files.isRegularFile(outputReleaseFile)) {
logger.debugf("loading already existing tag: %s", tag);
projectInfo = json.readAs(ProjectInfo.class, outputReleaseFile, StandardOpenOption.READ);
} else {
projectInfo = analyzeProjectTag(params, tag);
if (params.saveEachRelease() && params.outputDir() != null) {
json.write(
projectInfo,
params.outputDir().resolve("release.%s.json".formatted(tag)),
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
}
}
releases.add(projectInfo);
} catch (RuntimeException e) {
logger.warnf("skipping tag: %s, unexpected exception: %s", tag, e.getMessage());
}
}
return new ProjectHistoryInfo(
selectedTagList.size(),
releases);
}
private ProjectInfo analyzeProjectTag(Parameters params, String tag) {
logger.infof("checking out tag: %s", tag);
proc.run(params.projectDir(), params.environment(), "git", "checkout", tag, "--quiet");
return analyzeProjectWorkingTree(params, tag);
}
private ProjectInfo analyzeProjectWorkingTree(Parameters params, String releaseTag) {
logger.debugf("querying last commit");
final var commitContent = proc.output(
params.projectDir(), params.environment(),
"git", "log", "HEAD~1..HEAD", "--date=iso8601-strict");
final var commitDateMatcher = COMMIT_DATE_PATTERN.matcher(commitContent);
final var commitDate = commitDateMatcher.find()
? OffsetDateTime.parse(commitDateMatcher.group(1))
: null;
logger.debugf("commit date: %s", commitDate);
final var commitAuthorMatcher = COMMIT_AUTHOR_PATTERN.matcher(commitContent);
final String commitAuthorName;
final String commitAuthorEmail;
if (commitAuthorMatcher.find()) {
commitAuthorName = commitAuthorMatcher.group(1);
commitAuthorEmail = commitAuthorMatcher.group(2);
} else {
commitAuthorName = null;
commitAuthorEmail = null;
}
logger.debugf("commit author: %s <%s>", commitAuthorName, commitAuthorEmail);
final boolean offline;
if (params.goOfflineBeforeRelease()) {
boolean goOfflineSuccessful;
try {
logger.debugf("downloading dependencies for maven to run offline");
proc.run(
params.projectDir(), params.environment(),
mvnCmdLine(
params, false,
"--quiet",
config.parallel().enabled() ? "--threads=1C" : "--threads=1",
"dependency:go-offline"));
goOfflineSuccessful = true;
} catch (NonZeroExitCodeRuntimeException e) {
if (params.continueOnGoOfflineFailure()) {
goOfflineSuccessful = false;
} else {
logger.warnf("failed to go offline: %s", e.getMessage());
throw e;
}
}
offline = goOfflineSuccessful && !params.continueOnGoOfflineFailure();
} else {
offline = false;
}
// find modules
final var fullModuleList =
extractFullModuleList(params.projectDir().resolve("pom.xml"));
final var selectedModuleList = fullModuleList.stream()
.filter(module -> params.moduleRegex().isEmpty() || module.matches(params.moduleRegex()))
.toList();
logger.debugf("number of modules: %d (of %d)", selectedModuleList.size(),
fullModuleList.size());
final List<ModuleInfo> moduleInfoList;
if (config.parallel().enabled()) {
try {
moduleInfoList = executor.invokeAll(
selectedModuleList.stream()
.map(
module -> ((Callable<ModuleInfo>) (() -> analyzeProjectModule(params, offline, module))))
.toList())
.stream()
.map(moduleInfoF -> {
try {
return moduleInfoF.get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e.getMessage(), e);
}
}).toList();
} catch (InterruptedException | RuntimeException e) {
logger.warnf("unexpected interruption: %s", e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
} else {
moduleInfoList = selectedModuleList.stream()
.map(module -> analyzeProjectModule(params, offline, module))
.toList();
}
final var uniqueDependencyCount = (params.includeDependencyStats())
? moduleInfoList.stream()
.filter(info -> info.dependencies() != null)
.flatMap(info -> info.dependencies().stream())
.collect(Collectors.toSet())
.size()
: null;
final var fileCount = (params.includeFileStats())
? moduleInfoList.stream()
.filter(info -> info.fileCount() != null)
.mapToLong(ModuleInfo::fileCount)
.sum()
: null;
final var fileCountByExtension = (params.includeFileStats())
? moduleInfoList.stream()
.map(ModuleInfo::fileCountByExtension)
.filter(Objects::nonNull)
.reduce(
new HashMap<>(),
(a, b) -> {
final var result = new HashMap<>(a);
b.forEach((ext, count) -> result.merge(ext, count, Long::sum));
return result;
})
: null;
final var moduleAnalysisDuration = moduleInfoList.stream()
.map(ModuleInfo::analysisDuration)
.reduce(Duration.ZERO, Duration::plus);
final Duration compilationDuration;
if (params.includeCompilationStats()) {
logger.debugf("cleaning pre-compile");
proc.run(
params.projectDir(), params.environment(),
mvnCmdLine(params, offline, "--quiet", "clean"));
final var compilationStartTime = Instant.now();
logger.debugf("compiling project");
proc.run(
params.projectDir(), params.environment(),
mvnCmdLine(params, offline, "--quiet", "install"));
final var compilationEndTime = Instant.now();
compilationDuration =
Duration.between(compilationStartTime, compilationEndTime);
} else {
compilationDuration = Duration.ZERO;
}
final var totalAnalysisDuration = moduleAnalysisDuration.plus(compilationDuration);
return new ProjectInfo(
releaseTag,
commitDate,
commitAuthorName,
commitAuthorEmail,
uniqueDependencyCount,
fileCount,
fileCountByExtension,
selectedModuleList.size(),
moduleInfoList,
moduleAnalysisDuration,
compilationDuration,
totalAnalysisDuration);
}
private ModuleInfo analyzeProjectModule(Parameters params, boolean offline, String module) {
logger.debugf("analyzing module: %s", module);
try {
var analysisStartTime = Instant.now();
final List<DependencyInfo> dependencies;
if (params.includeDependencyStats()) {
final var effectivePomPath = Files.createTempFile(
"mavenstats.%s.".formatted(module.replaceAll("[/: \\t]", "_")),
".effective-pom.xml");
proc.run(
params.projectDir(),
params.environment(),
mvnCmdLine(
params, offline,
"--quiet",
"--projects=%s".formatted(module),
"-Doutput=%s".formatted(effectivePomPath),
"help:effective-pom"));
final var projectNode = xml.read(
effectivePomPath,
StandardOpenOption.READ,
StandardOpenOption.DELETE_ON_CLOSE);
dependencies = extractDependencies(projectNode);
logger.debugf("module: %s, dependencies: %d", module, dependencies.size());
} else {
dependencies = null;
}
final Long fileCount;
final Map<String, Long> fileCountByExtension;
if (params.includeFileStats()) {
fileCountByExtension = Files.find(
params.projectDir().resolve(module),
Integer.MAX_VALUE,
(file, attrs) -> attrs.isRegularFile())
.collect(
Collectors.groupingBy(
path -> FilenameUtils.getExtension(path.toString()),
Collectors.counting()));
fileCount = fileCountByExtension.values().stream().mapToLong(Long::longValue).sum();
logger.debugf("module: %s, files: %d", module, fileCount);
} else {
fileCountByExtension = null;
fileCount = null;
}
final var analysisEndTime = Instant.now();
final var analysisDuration = Duration.between(analysisStartTime, analysisEndTime);
return new ModuleInfo(
module,
dependencies != null ? dependencies.size() : null,
dependencies,
fileCount,
fileCountByExtension,
analysisDuration
);
} catch (IOException e) {
logger.warnf("unexpected I/O exception: %s", e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
private List<DependencyInfo> extractDependencies(JsonNode projectNode) {
final var dependenciesNode = projectNode.path("dependencies").path("dependency");
if (dependenciesNode.isMissingNode() || !dependenciesNode.isArray()) {
return List.of();
}
return StreamSupport.stream(dependenciesNode.spliterator(), false)
.map(dependencyNode -> new DependencyInfo(
dependencyNode.path("groupId").textValue(),
dependencyNode.path("artifactId").textValue(),
dependencyNode.path("version").textValue(),
dependencyNode.path("scope").textValue()))
.toList();
}
private List<String> extractFullModuleList(Path parentPom) {
final var modules = extractModuleList(parentPom);
return modules != null ? modules : List.of();
}
private List<String> extractModuleList(Path pom) {
final var projectNode = xml.read(pom, StandardOpenOption.READ);
// is this a pom?
final var packaging = projectNode.path("packaging").textValue();
if (!"pom".equals(packaging)) {
return null; // leaf node
}
final var modulesNode = projectNode.path("modules").path("module");
if (!modulesNode.isArray()) {
return List.of(); // unexpected, but no leaves
}
final var modules =
StreamSupport.stream(modulesNode.spliterator(), false)
.map(JsonNode::textValue)
.toList();
final var result = new ArrayList<String>();
for (final var module : modules) {
final var submodules =
extractModuleList(pom.getParent().resolve(module).resolve("pom.xml"));
if (submodules != null) {
result.addAll(
submodules.stream()
.map(submodule -> "%s/%s".formatted(module, submodule))
.toList());
} else {
result.add(module);
}
}
return result;
}
private List<String> mvnCmdLine(Parameters params, boolean offline, String... args) {
final var command = new ArrayList<String>();
command.add("mvn");
if (params.mavenSettingsFile() != null) {
command.add("--settings=%s".formatted(params.mavenSettingsFile()));
}
if (params.mavenProfiles() != null && !params.mavenProfiles().isEmpty()) {
command.add("--active-profiles=%s".formatted(String.join(",", params.mavenProfiles())));
}
if (params.mavenProperties() != null && !params.mavenProperties().isEmpty()) {
for (final var property: params.mavenProperties()) {
command.add("--define=%s".formatted(property));
}
}
if (offline) {
command.add("--offline");
}
command.addAll(List.of(args));
return command;
}
}
class TagComparator implements Comparator<String> {
public static TagComparator INSTANCE = new TagComparator();
@Override
public int compare(String tag1, String tag2) {
if (Objects.equals(tag1, tag2)) {
return 0;
}
return Arrays.compare(
(tag1 != null) ? tag1.split("\\.") : null,
(tag2 != null) ? tag2.split("\\.") : null,
TagPartComparator.INSTANCE);
}
}
class TagPartComparator implements Comparator<String> {
public static TagPartComparator INSTANCE = new TagPartComparator();
@Override
public int compare(String part1, String part2) {
//noinspection StringEquality
if (part1 == part2) {
return 0;
}
if (part1 == null || part2 == null) {
return part1 == null ? -1 : 1;
}
try {
return Integer.compare(Integer.parseInt(part1), Integer.parseInt(part2));
} catch (NumberFormatException e) {
return part1.compareTo(part2);
}
}
}
@JsonInclude(Include.NON_NULL)
record ProjectHistoryInfo(
int tagCount,
List<ProjectInfo> releases
) {
}
@JsonInclude(Include.NON_NULL)
record ProjectInfo(
String releaseTag,
OffsetDateTime releaseDate,
String authorName,
String authorEmail,
Integer uniqueDependencyCount,
Long fileCount,
Map<String, Long> fileCountByExtension,
int moduleCount,
List<ModuleInfo> modules,
Duration moduleAnalysisDuration,
Duration compilationDuration,
Duration totalAnalysisDuration
) {
}
@JsonInclude(Include.NON_NULL)
record ModuleInfo(
String module,
Integer dependencyCount,
List<DependencyInfo> dependencies,
Long fileCount,
Map<String, Long> fileCountByExtension,
Duration analysisDuration
) {
}
@JsonInclude(Include.NON_NULL)
record DependencyInfo(
String groupId,
String artifactId,
String version,
String scope
) {
}
/**
* Executor factory used to run system jobs
*/
@SuppressWarnings("unused")
@ApplicationScoped
class ManagedExecutorFactory {
@LoggerName("cli.managedexecutorfactory")
Logger logger;
@Inject
MavenStatsConfig config;
@Produces
@Named("cpuBoundExecutor")
public ManagedExecutor executor() {
if (config != null) {
final int parallelMaxAsyncResolved = config.parallel().maxAsync() <= 0
? (Runtime.getRuntime().availableProcessors() * 2 / 3)
: Math.max(config.parallel().maxAsync(), Runtime.getRuntime().availableProcessors() * 2);
logger.debugf(
"cpu bound executor max async size: %d, max queue size: %d",
parallelMaxAsyncResolved, config.parallel().maxQueue());
return ManagedExecutor.builder()
.maxAsync(parallelMaxAsyncResolved)
.maxQueued(config.parallel().maxQueue())
.build();
}
throw new IllegalStateException("no configuration received yet");
}
}
/**
* XML and JSON I/O with Jackson Mappers
*/
@SuppressWarnings("unused")
@ApplicationScoped
class ObjectMapperFactory {
@LoggerName("cli.objectmapperfactory")
Logger logger;
private ObjectMapper jsonObjectMapper() {
return objectMapper(JsonMapper.builder());
}
@Produces
@ApplicationScoped
@Named("jsonMapper")
public Mapper jsonMapper() {
return new Mapper(jsonObjectMapper(), logger);
}
private ObjectMapper xmlObjectMapper() {
return objectMapper(XmlMapper.builder());
}
@Produces
@ApplicationScoped
@Named("xmlMapper")
public Mapper xmlMapper() {
return new Mapper(xmlObjectMapper(), logger);
}
private <M extends ObjectMapper, B extends MapperBuilder<M, B>> M objectMapper(B builder) {
final var mapper = builder
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.enable(SerializationFeature.INDENT_OUTPUT)
.build();
mapper.registerModule(new JavaTimeModule());
return mapper;
}
}
/**
* Jackson ObjectMapper wrapper with I/O extension methods
*/
@SuppressWarnings("ClassCanBeRecord")
class Mapper {
private final ObjectMapper objectMapper;
private final Logger logger;
@Inject
Mapper(ObjectMapper objectMapper, Logger logger) {
this.objectMapper = objectMapper;
this.logger = logger;
}
public JsonNode read(Path file, OpenOption... options) {
try (final var in = Files.newInputStream(file, options)) {
return objectMapper.readTree(in);
} catch (IOException e) {
logger.warnf(
"unexpected I/O exception when reading: %s. message: %s", file, e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
public <T> T readAs(Class<T> type, Path file, OpenOption... options) {
try (final var in = Files.newInputStream(file, options)) {
return objectMapper.readValue(in, type);
} catch (IOException e) {
logger.warnf(
"unexpected I/O exception when reading: %s. message: %s", file, e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
public void write(Object node, Path file, OpenOption... options) {
try (final var out = Files.newOutputStream(file, options)) {
write(node, out);
} catch (IOException e) {
logger.warnf(
"unexpected I/O exception when writing: %s. message: %s", file, e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
public void write(Object node, OutputStream out) {
try {
objectMapper.writeValue(out, node);
} catch (IOException e) {
logger.warnf("unexpected I/O exception when writing. message: %s", e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
public void log(Level level, Object node) {
try (final var out = new JBossLogOutputStream(logger, level)) {
write(node, out);
} catch (IOException e) {
logger.warnf("unexpected I/O exception when logging. message: %s", e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
}
/**
* System process management
*/
@SuppressWarnings("unused")
@ApplicationScoped
class ProcessManager {
@LoggerName("cli.procmanager")
Logger logger;
public void run(Path workingDir, Map<String, String> env, String... command) {
run(workingDir, env, List.of(command));
}
public void run(Path workingDir, Map<String, String> env, List<String> command) {
try {
run(pe ->
pe.directory(workingDir != null ? workingDir.toFile() : null)
.environment(env != null ? env : Map.of())
.command(command));
} catch (NonZeroExitCodeRuntimeException e) {
throw new NonZeroExitCodeRuntimeException(e.exitCode, command);
}
}
public void run(Consumer<ProcessExecutor> processExecutorConfigurator) {
final var exitCode = exitCode(processExecutorConfigurator);
if (exitCode != 0) {
throw new NonZeroExitCodeRuntimeException(exitCode);
}
}
public int exitCode(Path workingDir, Map<String, String> env, String... command) {
return exitCode(workingDir, env, List.of(command));
}
public int exitCode(Path workingDir, Map<String, String> env, List<String> command) {
return exitCode(pe ->
pe.directory(workingDir != null ? workingDir.toFile() : null)
.environment(env != null ? env : Map.of())
.command(command));
}
public int exitCode(Consumer<ProcessExecutor> processExecutorConfigurator) {
final var executor = new ProcessExecutor();
processExecutorConfigurator.accept(executor);
return invoke(() ->
executor.redirectOutput(loggerOutputStream())
.execute()
.getExitValue());
}
public String output(Path workingDir, Map<String, String> env, String... command) {
return output(workingDir, env, List.of(command));
}
public String output(Path workingDir, Map<String, String> env, List<String> command) {
return output(pe ->
pe.directory(workingDir != null ? workingDir.toFile() : null)
.environment(env != null ? env : Map.of())
.command(command));
}
public String output(Consumer<ProcessExecutor> processExecutorConfigurator) {
final var executor = new ProcessExecutor();
processExecutorConfigurator.accept(executor);
return invoke(() ->
executor.readOutput(true)
.execute()
.outputUTF8());
}
private <V> V invoke(Callable<V> job) {
try {
return job.call();
} catch (Exception e) {
logger.errorf(e, e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
}
private OutputStream loggerOutputStream() {
return new JBossLogOutputStream(logger, Level.DEBUG);
}
}
class NonZeroExitCodeRuntimeException extends RuntimeException {
final int exitCode;
final List<String> command;
public NonZeroExitCodeRuntimeException(int exitCode, List<String> command) {
super(
"command failed. exit code: %d. command line: %s"
.formatted(exitCode, command));
this.exitCode = exitCode;
this.command = command;
}
public NonZeroExitCodeRuntimeException(int exitCode) {
this(exitCode, null);
}
}
/**
* Bridge for APIs that require an OutputStream, to send output to JBoss logger
*/
class JBossLogOutputStream extends LogOutputStream {
final BasicLogger logger;
final Level level;
JBossLogOutputStream(BasicLogger logger, Level level) {
this.logger = logger;
this.level = level;
}
@Override
protected void processLine(String line) {
logger.log(level, line);
}
}
@rivasdiaz
Copy link
Author

rivasdiaz commented Nov 4, 2021

Example execution:

Invocation:

j! -Dlogging.level=DEBUG \
  https://gist.githubusercontent.com/rivasdiaz/f2e9efd1bf2079f0101f865846af2502/raw/dfbdaaa177c2a33efc9e38b2a5af80d70b382cd4/mavenstats.java \
  --tag-regex='0\.[1-9]\.0' \
  --module-regex='core/.*' \
  --no-override-existing-release \
    --maven-define=skipTests \
    --maven-define=skipITs \
    --continue-on-go-offline-failure \
  --output=quarkus \
  ~/Projects/Upstream/quarkus

Log:

23:14:08 DEBUG project path: /Users/rrivas/Projects/Upstream/quarkus
23:14:09 DEBUG cpu bound executor max async size: 8, max queue size: 2048
23:14:09 DEBUG project analysis starting with configuration:
23:14:10 DEBUG {
23:14:10 DEBUG   "mainBranch" : "main",
23:14:10 DEBUG   "tagRegex" : "0\\.[1-9]\\.0",
23:14:10 DEBUG   "moduleRegex" : "core/.*",
23:14:10 DEBUG   "mavenSettingsFile" : null,
23:14:10 DEBUG   "includeDependencyStats" : true,
23:14:10 DEBUG   "includeFileStats" : true,
23:14:10 DEBUG   "saveEachRelease" : true,
23:14:10 DEBUG   "saveAllReleases" : true,
23:14:10 DEBUG   "overrideExistingRelease" : false,
23:14:10 DEBUG   "outputDir" : "file:///private/tmp/stats/quarkus/",
23:14:10 DEBUG   "projectDir" : "file:///Users/rrivas/Projects/Upstream/quarkus/"
23:14:10 DEBUG }
23:14:10 DEBUG project already in main branch: main
23:14:10 DEBUG number of tags: 9 (of 147)
23:14:10 DEBUG checking out tag: 0.1.0
23:14:14 DEBUG querying last commit
23:14:14 DEBUG commit date: 2018-12-13T19:24:09+01:00
23:14:14 DEBUG commit author: Clement Escoffier <clement.escoffier@gmail.com>
23:14:14 DEBUG number of modules: 3 (of 66)
23:14:14 DEBUG analyzing module: core/deployment
23:14:14 DEBUG analyzing module: core/deployment-api
23:14:14 DEBUG analyzing module: core/runtime
23:14:19 DEBUG module: core/deployment-api, dependencies: 3
23:14:19 DEBUG module: core/runtime, dependencies: 11
23:14:19 DEBUG module: core/deployment, dependencies: 13
...
23:15:28 DEBUG module: core/devmode, dependencies: 2
23:15:28 DEBUG module: core/processor, dependencies: 0
23:15:28 DEBUG module: core/devmode, files: 7
23:15:28 DEBUG module: core/processor, files: 3
23:15:28 DEBUG module: core/creator, dependencies: 7
23:15:28 DEBUG module: core/creator, files: 97
23:15:28 DEBUG module: core/deployment, dependencies: 11
23:15:28 DEBUG module: core/deployment, files: 123
23:15:28 DEBUG module: core/builder, dependencies: 5
23:15:28 DEBUG module: core/builder, files: 36
23:15:28 DEBUG module: core/runtime, dependencies: 14
23:15:28 DEBUG module: core/runtime, files: 62
23:15:28 DEBUG project analysis completed

Example of how to use resulting json:

jq '[.releases[] | {tag:.releaseTag, "java-files":.fileCountByExtension.java}]' quarkus/releases.json

Results in:

[
  {
    "tag": "0.1.0",
    "java-files": 127
  },
  {
    "tag": "0.2.0",
    "java-files": 128
  },
  {
    "tag": "0.3.0",
    "java-files": 129
  },
  {
    "tag": "0.4.0",
    "java-files": 220
  },
  {
    "tag": "0.5.0",
    "java-files": 220
  },
  {
    "tag": "0.6.0",
    "java-files": 211
  },
  {
    "tag": "0.7.0",
    "java-files": 241
  },
  {
    "tag": "0.8.0",
    "java-files": 285
  },
  {
    "tag": "0.9.0",
    "java-files": 319
  }
]

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment