JanusGraph Dynamic Graphs Docker Config: use cassandra and es

docker-compose.yml

version: "3"

services:
  jce-janusgraph:
    image: janusgraph/janusgraph:0.5
    container_name: jce-janusgraph
    environment:
      JANUS_PROPS_TEMPLATE: cql-es
      janusgraph.storage.hostname: 192.168.0.1
      janusgraph.storage.port: 9042
      janusgraph.index.search.hostname: 192.168.0.1
      janusgraph.index.search.port: 9200
      janusgraph.index.search.directory: /var/lib/janusgraph/index  # option
      janusgraph.storage.directory: /var/lib/janusgraph/data  # option
      janusgraph.gremlin.graph: org.janusgraph.core.ConfiguredGraphFactory  # important
      janusgraph.graph.graphname: ConfigurationManagementGraph  # important
      gremlinserver.graphManager: org.janusgraph.graphdb.management.JanusGraphManager # important
      gremlinserver.channelizer: org.janusgraph.channelizers.JanusGraphWsAndHttpChannelizer # important
      gremlinserver%d.graphs.graph: ''  # important, delete default graph
      gremlinserver.graphs.ConfigurationManagementGraph: /etc/opt/janusgraph/janusgraph.properties  # important, add ConfigurationManagementGraph
      gremlinserver.scriptEngines.gremlin-groovy.plugins[org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin].files[0]: scripts/empty-configuration.groovy
    ports:
      - "8182:8182"
    networks:
      - jce-network
    volumes:
      - ./janusgraph/conf:/etc/opt/janusgraph # for check gremlin-server.yaml and janusgraph.properties correct or not
      - ./janusgraph/scripts:/opt/janusgraph/scripts  # use empty-configuration.groovy, comment // globals << [g : graph.traversal()]
    depends_on:
      - jce-cassandra
      - jce-elastic
    # option
    healthcheck:
      test: ["CMD", "bin/gremlin.sh", "-e", "scripts/remote-connect.groovy"]
      interval: 10s
      timeout: 30s
      retries: 3

  jce-cassandra:
    image: cassandra:3.11.10
    container_name: jce-cassandra
    ports:
      - "9042:9042"
      - "9160:9160"
    volumes:
      - ./cassandra/data:/var/lib/cassandra # option, can check graphname in ./cassandra/data/data
    networks:
      - jce-network

  jce-elastic:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
    container_name: jce-elastic
    environment:
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - "http.host=0.0.0.0"
      - "network.host=0.0.0.0"
      - "transport.host=127.0.0.1"
      - "cluster.name=docker-cluster"
      - "xpack.security.enabled=false"
      - "discovery.zen.minimum_master_nodes=1"
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - jce-network

networks:
  jce-network:
volumes:
  janusgraph-default-data:

empty-configuration.groovy

// Copyright 2019 JanusGraph Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// an init script that returns a Map allows explicit setting of global bindings.
def globals = [:]

// defines a sample LifeCycleHook that prints some output to the Gremlin Server console.
// note that the name of the key in the "global" map is unimportant.
globals << [hook : [
        onStartUp: { ctx ->
            ctx.logger.info("Executed once at startup of Gremlin Server.")
        },
        onShutDown: { ctx ->
            ctx.logger.info("Executed once at shutdown of Gremlin Server.")
        }
] as LifeCycleHook]

// define the default TraversalSource to bind queries to - this one will be named "g".
// globals << [g : graph.traversal()]

gremlin-server.yaml(janusgraph docker auto generate)

#
# NOTE: THIS FILE IS GENERATED VIA "update.sh"
# DO NOT EDIT IT DIRECTLY; CHANGES WILL BE OVERWRITTEN.
#
# Copyright 2021 JanusGraph Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

host: 0.0.0.0
port: 8182
scriptEvaluationTimeout: 30000
channelizer: org.janusgraph.channelizers.JanusGraphWsAndHttpChannelizer
graphs:
  ConfigurationManagementGraph: /etc/opt/janusgraph/janusgraph.properties
scriptEngines:
  gremlin-groovy:
    plugins:
      org.janusgraph.graphdb.tinkerpop.plugin.JanusGraphGremlinPlugin: {}
      org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}
      org.apache.tinkerpop.gremlin.tinkergraph.jsr223.TinkerGraphGremlinPlugin: {}
      org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin:
        classImports:
          - java.lang.Math
        methodImports:
          - java.lang.Math#*
      org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin:
        files:
          - scripts/empty-configuration.groovy
serializers:
  - className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV3d0
    config:
      ioRegistries:
        - org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry
  - className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV3d0
    config:
      serializeResultToString: true
  - className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0
    config:
      ioRegistries:
        - org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry
  # Older serialization versions for backwards compatibility:
  - className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0
    config:
      ioRegistries:
        - org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry
  - className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0
    config:
      ioRegistries:
        - org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry
  - className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0
    config:
      serializeResultToString: true
  - className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0
    config:
      ioRegistries:
        - org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry
  - className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0
    config:
      ioRegistries:
        - org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistryV1d0
  - className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0
    config:
      ioRegistries:
        - org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistryV1d0
processors:
  - className: org.apache.tinkerpop.gremlin.server.op.session.SessionOpProcessor
    config:
      sessionTimeout: 28800000
  - className: org.apache.tinkerpop.gremlin.server.op.traversal.TraversalOpProcessor
    config:
      cacheExpirationTime: 600000
      cacheMaxSize: 1000
metrics:
  consoleReporter:
    enabled: true
    interval: 180000
  csvReporter:
    enabled: true
    interval: 180000
    fileName: /tmp/gremlin-server-metrics.csv
  jmxReporter:
    enabled: true
  slf4jReporter:
    enabled: true
    interval: 180000
  gangliaReporter:
    enabled: false
    interval: 180000
    addressingMode: MULTICAST
  graphiteReporter:
    enabled: false
    interval: 180000
maxInitialLineLength: 4096
maxHeaderSize: 8192
maxChunkSize: 8192
maxContentLength: 65536
maxAccumulationBufferComponents: 1024
resultIterationBatchSize: 64
writeBufferLowWaterMark: 32768
writeBufferHighWaterMark: 65536
threadPoolWorker: 1
gremlinPool: 8
graphManager: org.janusgraph.graphdb.management.JanusGraphManager

janusgraph.properties(janusgraph docker auto generate)

#
# NOTE: THIS FILE IS GENERATED VIA "update.sh"
# DO NOT EDIT IT DIRECTLY; CHANGES WILL BE OVERWRITTEN.
#
# Copyright 2021 JanusGraph Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

gremlin.graph=org.janusgraph.core.ConfiguredGraphFactory

# JanusGraph configuration sample: Cassandra & Elasticsearch over sockets
#
# This file connects to Cassandra and Elasticsearch services running
# on localhost over the CQL API and the Elasticsearch native
# "Transport" API on their respective default ports.  The Cassandra
# and Elasticsearch services must already be running before starting
# JanusGraph with this file.

# The primary persistence provider used by JanusGraph.  This is required.  It
# should be set one of JanusGraph's built-in shorthand names for its standard
# storage backends or to the full package and classname of a custom/third-party
# StoreManager implementation.
#
# Default:    (no default value)
# Data Type:  String
# Mutability: LOCAL
storage.backend=cql

# The hostname or comma-separated list of hostnames of storage backend
# servers.  This is only applicable to some storage backends, such as
# cassandra and hbase.
#
# Default:    127.0.0.1
# Data Type:  class java.lang.String[]
# Mutability: LOCAL
storage.hostname=192.168.0.1

# The name of JanusGraph's keyspace.  It will be created if it does not
# exist.
#
# Default:    janusgraph
# Data Type:  String
# Mutability: LOCAL
storage.cql.keyspace=janusgraph

# Whether to enable JanusGraph's database-level cache, which is shared across
# all transactions. Enabling this option speeds up traversals by holding
# hot graph elements in memory, but also increases the likelihood of
# reading stale data.  Disabling it forces each transaction to
# independently fetch graph elements from storage before reading/writing
# them.
#
# Default:    false
# Data Type:  Boolean
# Mutability: MASKABLE
cache.db-cache = true

# How long, in milliseconds, database-level cache will keep entries after
# flushing them.  This option is only useful on distributed storage
# backends that are capable of acknowledging writes without necessarily
# making them immediately visible.
#
# Default:    50
# Data Type:  Integer
# Mutability: GLOBAL_OFFLINE
#
# Settings with mutability GLOBAL_OFFLINE are centrally managed in JanusGraph's
# storage backend.  After starting the database for the first time, this
# file's copy of this setting is ignored.  Use JanusGraph's Management System
# to read or modify this value after bootstrapping.
cache.db-cache-clean-wait = 20

# Default expiration time, in milliseconds, for entries in the
# database-level cache. Entries are evicted when they reach this age even
# if the cache has room to spare. Set to 0 to disable expiration (cache
# entries live forever or until memory pressure triggers eviction when set
# to 0).
#
# Default:    10000
# Data Type:  Long
# Mutability: GLOBAL_OFFLINE
#
# Settings with mutability GLOBAL_OFFLINE are centrally managed in JanusGraph's
# storage backend.  After starting the database for the first time, this
# file's copy of this setting is ignored.  Use JanusGraph's Management System
# to read or modify this value after bootstrapping.
cache.db-cache-time = 180000

# Size of JanusGraph's database level cache.  Values between 0 and 1 are
# interpreted as a percentage of VM heap, while larger values are
# interpreted as an absolute size in bytes.
#
# Default:    0.3
# Data Type:  Double
# Mutability: MASKABLE
cache.db-cache-size = 0.25

# Connect to an already-running ES instance on localhost

# The indexing backend used to extend and optimize JanusGraph's query
# functionality. This setting is optional.  JanusGraph can use multiple
# heterogeneous index backends.  Hence, this option can appear more than
# once, so long as the user-defined name between "index" and "backend" is
# unique among appearances.Similar to the storage backend, this should be
# set to one of JanusGraph's built-in shorthand names for its standard index
# backends (shorthands: lucene, elasticsearch, es, solr) or to the full
# package and classname of a custom/third-party IndexProvider
# implementation.
#
# Default:    elasticsearch
# Data Type:  String
# Mutability: GLOBAL_OFFLINE
#
# Settings with mutability GLOBAL_OFFLINE are centrally managed in JanusGraph's
# storage backend.  After starting the database for the first time, this
# file's copy of this setting is ignored.  Use JanusGraph's Management System
# to read or modify this value after bootstrapping.
index.search.backend=elasticsearch

# The hostname or comma-separated list of hostnames of index backend
# servers.  This is only applicable to some index backends, such as
# elasticsearch and solr.
#
# Default:    127.0.0.1
# Data Type:  class java.lang.String[]
# Mutability: MASKABLE
index.search.hostname=192.168.0.1

# The Elasticsearch node.client option is set to this boolean value, and
# the Elasticsearch node.data option is set to the negation of this value.
# True creates a thin client which holds no data.  False creates a regular
# Elasticsearch cluster node that may store data.
#
# Default:    true
# Data Type:  Boolean
# Mutability: GLOBAL_OFFLINE
#
# Settings with mutability GLOBAL_OFFLINE are centrally managed in JanusGraph's
# storage backend.  After starting the database for the first time, this
# file's copy of this setting is ignored.  Use JanusGraph's Management System
# to read or modify this value after bootstrapping.
index.search.elasticsearch.client-only=true

# Or start ES inside the JanusGraph JVM
#index.search.backend=elasticsearch
#index.search.directory=db/es
#index.search.elasticsearch.client-only=false
#index.search.elasticsearch.local-mode=true
storage.port=9042
storage.directory=/var/lib/janusgraph/data
index.search.port=9200
index.search.directory=/var/lib/janusgraph/index
graph.graphname=ConfigurationManagementGraph

remote-connect.groovy

//
// NOTE: THIS FILE IS GENERATED VIA "update.sh"
// DO NOT EDIT IT DIRECTLY; CHANGES WILL BE OVERWRITTEN.
//
// Copyright 2019 JanusGraph Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

:remote connect tinkerpop.server conf/remote.yaml
:> ConfiguredGraphFactory

step.md

1. use docker-compose.yml, edit hostname and port for janusgraph.storage and janusgraph.index
2. mkdir -p ./janusgraph/conf
3. mkdir -p ./janusgraph/scripts, copy empty-configuration.groovy and remote-connect.groovy here
4. mkdir -p ./cassandra/data
5. startup: docker-compose up -d
6. after start success, check gremlin-server.yaml and janusgraph.properties in ./janusgraph/conf correct.

reference：

• https://github.com/JanusGraph/janusgraph-docker
• https://docs.janusgraph.org/operations/configured-graph-factory/
• https://docs.janusgraph.org/operations/dynamic-graphs/
• JanusGraph/janusgraph#1268

In the Spring application, here is a example

1. application-dev.yml

# ...
janus:
  config:
    remote: ${JANUS_REMOTE_OBJECTS_YML:conf/remote-objects.yaml}
    storage_backend: cql
    storage_hostname: ${JANUS_CASSANDRA_HOST:192.168.0.1}
    storage_port: ${JANUS_CASSANDRA_PORT:9042}
    index_search_backend: elasticsearch
    index_search_hostname: ${JANUS_ES_HOST:192.168.0.1}
    index_search_port: ${JANUS_ES_PORT:9200}
# ...

2. remote-objects.yaml

# Copyright 2019 JanusGraph Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

hosts: [192.168.0.1]
port: 8182
serializer: { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV3d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }}

3. JanusConfig.java

package me.simon.config;

import lombok.Data;
import org.apache.tinkerpop.gremlin.driver.Client;
import org.apache.tinkerpop.gremlin.driver.Cluster;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;

@Data
@org.springframework.context.annotation.Configuration
@ConfigurationProperties(prefix = "janus.config")
public class JanusConfig {
  private String storageBackend;

  private String storageHostname;

  private String storagePort;

  private String indexSearchBackend;

  private String indexSearchHostname;

  private String indexSearchPort;

  private String remote;

  @Bean
  public Client client() throws Exception {
    return Cluster.open(remote).connect();
  }
}

4. JanusRepository.java (Include create and drop graph operation, and I use graphId to distinguish different graph)

package me.simon.modules.graph.repository;

import cn.hutool.core.util.StrUtil;
import lombok.RequiredArgsConstructor;
import me.simon.config.JanusConfig;
import org.apache.tinkerpop.gremlin.driver.Client;
import org.apache.tinkerpop.gremlin.driver.Result;
import org.apache.tinkerpop.gremlin.driver.remote.DriverRemoteConnection;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.springframework.stereotype.Service;

import static org.apache.tinkerpop.gremlin.process.traversal.AnonymousTraversalSource.traversal;

@Service
@RequiredArgsConstructor
public class JanusRepository {

    private final Client client;
    private final JanusConfig janusConfig;

    public GraphTraversalSource g(String graphId) {
        return traversal().withRemote(DriverRemoteConnection.using(client, graphName(graphId) + "_traversal"));
    }

    public void createGraph(String graphId) {
        createTemplateConfig();

        String str = StrUtil.format("ConfiguredGraphFactory.create(\"{}\");", graphName(graphId));
        client.submit(str);
    }

    public void deleteGraph(String graphId) {
        String str = StrUtil.format("ConfiguredGraphFactory.drop(\"{}\");", graphName(graphId));
        client.submit(str);
    }

    private String graphName(String graphId) {
        return "graph_" + graphId;
    }

    private boolean templateConfigExists() {
        String str = "ConfiguredGraphFactory.getTemplateConfiguration()";
        Result result = client.submit(str).one();
        return !result.isNull();
    }

    private void createTemplateConfig() {
        if (templateConfigExists()) return;

        StringBuilder stringBuilder = StrUtil.builder();

        stringBuilder.append("Map map = new HashMap();\n");
        stringBuilder.append(StrUtil.format("map.put(\"storage.backend\", \"{}\");", janusConfig.getStorageBackend()));
        stringBuilder.append(StrUtil.format("map.put(\"storage.hostname\", \"{}\");", janusConfig.getStorageHostname()));
        stringBuilder.append(StrUtil.format("map.put(\"storage.port\", {});", janusConfig.getStoragePort()));
        stringBuilder.append("map.put(\"storage.connection-timeout\", \"30000\");\n");
        stringBuilder.append("map.put(\"storage.cql.protocol-version\", 3);\n");
        stringBuilder.append("map.put(\"storage.cql.local-core-connections-per-host\", 20);\n");
        stringBuilder.append("map.put(\"storage.cql.local-max-connections-per-host\", 40);\n");
        stringBuilder.append("map.put(\"storage.cql.local-max-requests-per-connection\", 2048);\n");
        stringBuilder.append("map.put(\"storage.buffer-size\", 10240);\n");

        stringBuilder.append(StrUtil.format("map.put(\"index.search.backend\", \"{}\");", janusConfig.getIndexSearchBackend()));
        stringBuilder.append(StrUtil.format("map.put(\"index.search.hostname\", \"{}\");", janusConfig.getIndexSearchHostname()));
        stringBuilder.append(StrUtil.format("map.put(\"index.search.port\", {});", janusConfig.getIndexSearchPort()));

        stringBuilder.append("ConfiguredGraphFactory.createTemplateConfiguration(new MapConfiguration(map));");

        client.submit(stringBuilder.toString()).one();
    }
}