Created
March 20, 2022 19:24
-
-
Save michael-simons/9a90a44295b10b6ae92523f16b6c6528 to your computer and use it in GitHub Desktop.
Create a bunch of Trivial Graph Format (tgf) files from Maven and import them into Neo4j.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
///usr/bin/env jbang "$0" "$@" ; exit $? | |
//JAVA 17 | |
//DEPS info.picocli:picocli:4.6.3 | |
//DEPS org.neo4j.driver:neo4j-java-driver:4.4.5 | |
import picocli.CommandLine; | |
import picocli.CommandLine.Command; | |
import picocli.CommandLine.Option; | |
import picocli.CommandLine.Parameters; | |
import java.io.BufferedReader; | |
import java.io.InputStreamReader; | |
import java.net.URI; | |
import java.nio.file.Files; | |
import java.nio.file.Path; | |
import java.util.List; | |
import java.util.Map; | |
import java.util.concurrent.Callable; | |
import java.util.concurrent.atomic.AtomicInteger; | |
import java.util.function.Predicate; | |
import java.util.stream.Collectors; | |
import org.neo4j.driver.AuthTokens; | |
import org.neo4j.driver.GraphDatabase; | |
import org.neo4j.driver.Session; | |
/** | |
* find all artifacts with `javax` in the groupId and the the paths leading to them: | |
* {@code match p=(a:Artifact)<-[]-(b) where a.groupId =~'.*javax.*' return p} | |
*/ | |
@Command(name = "import_tgf_recursive") | |
public class import_tgf_recursive implements Callable<Integer> { | |
@Parameters(index = "0", description = "Root folder") | |
private Path root; | |
@Option( | |
names = { "-a", "--address" }, | |
description = "The address this migration should connect to. The driver supports bolt, bolt+routing or neo4j as schemes.", | |
required = true, | |
defaultValue = "bolt://localhost:7687" | |
) | |
private URI address; | |
@Option( | |
names = { "-u", "--username" }, | |
description = "The login of the user connecting to the database.", | |
required = true, | |
defaultValue = "neo4j" | |
) | |
private String user; | |
@Option( | |
names = { "-p", "--password" }, | |
description = "The password of the user connecting to the database.", | |
arity = "0..1", interactive = true, | |
defaultValue = "secret" | |
) | |
private char[] password; | |
public static void main(String... args) { | |
int exitCode = new CommandLine(new import_tgf_recursive()).execute(args); | |
System.exit(exitCode); | |
} | |
@Override | |
public Integer call() throws Exception { | |
System.out.println("Creating trivial graph files..."); | |
var mvnw = new ProcessBuilder("./mvnw", | |
"dependency:tree", | |
"-DoutputFile=pom.tgf", | |
"-DoutputType=tgf" | |
).directory(root.toFile()) | |
.redirectErrorStream(true) | |
.start(); | |
try (var reader = new BufferedReader(new InputStreamReader(mvnw.getInputStream()))) { | |
String line; | |
while ((line = reader.readLine()) != null) { | |
System.out.println(line); | |
} | |
} | |
System.out.println("Loading files..."); | |
try (var driver = GraphDatabase.driver(address, AuthTokens.basic(user, new String(password))); | |
var session = driver.session()) { | |
session.run("MATCH (n) DETACH DELETE n").consume(); | |
session.run( | |
"CREATE INDEX artifact_index IF NOT EXISTS FOR (a:Artifact) ON (a.groupId, a.artifactId, a.version)") | |
.consume(); | |
var pomTgf = Path.of("pom.tgf"); | |
Predicate<Path> isTgf = p -> Files.isRegularFile(p) && p.getFileName().endsWith(pomTgf); | |
Files.walk(root) | |
.filter(isTgf) | |
.peek(path -> System.out.println("Loading " + path)) | |
.map(this::process) | |
.forEach(dependencies -> mergeDependencies(session, dependencies)); | |
} | |
return 0; | |
} | |
private void mergeDependencies(Session session, List<Dependency> dependencies) { | |
session.writeTransaction(tx -> { | |
dependencies.forEach(d -> { | |
var dependant = d.dependant; | |
var dependency = d.dependency; | |
var cypher = """ | |
MERGE (dependant:Artifact {groupId: $g1, artifactId: $a1, version: $v1}) | |
MERGE (dependency:Artifact {groupId: $g2, artifactId: $a2, version: $v2}) | |
MERGE (dependant) -[:%s]-> (dependency) | |
""".formatted(d.type); | |
tx.run(cypher, | |
Map.of("g1", dependant.groupId, "a1", dependant.artifactId, "v1", dependant.version, | |
"g2", dependency.groupId, "a2", dependency.artifactId, "v2", dependency.version) | |
); | |
}); | |
return null; | |
}); | |
} | |
record Artifact(String groupId, String artifactId, String version, String scope) { | |
} | |
record Dependency(Artifact dependant, Artifact dependency, String type) { | |
} | |
List<Dependency> process(Path tgf) { | |
try { | |
var content = Files.readString(tgf).split("#"); | |
var cnt = new AtomicInteger(0); | |
var artifacts = content[0].lines() | |
.map(l -> { | |
var indexOfFirstBlank = l.indexOf(' '); | |
var id = Long.parseLong(l.substring(0, indexOfFirstBlank)); | |
var details = l.substring(indexOfFirstBlank + 1).split(":"); | |
if (cnt.getAndIncrement() == 0) { | |
return Map.entry(id, new Artifact(details[0], details[1], details[3], null)); | |
} else { | |
return Map.entry(id, new Artifact(details[0], details[1], details[details.length - 2], | |
details[details.length - 1])); | |
} | |
}).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); | |
return content[1].lines() | |
.filter(Predicate.not(String::isBlank)) | |
.map(l -> { | |
var details = l.split(" "); | |
return new Dependency( | |
artifacts.get(Long.parseLong(details[0])), | |
artifacts.get(Long.parseLong(details[1])), | |
details[2] | |
); | |
}) | |
.toList(); | |
} catch (Exception e) { | |
throw new RuntimeException(e); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment