Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Integration testing elasticsearch take 1
@echo off
SETLOCAL
if NOT DEFINED JAVA_HOME goto err
set SCRIPT_DIR=%~dp0
for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
REM ***** JAVA options *****
if "%ES_MIN_MEM%" == "" (
set ES_MIN_MEM=256m
)
if "%ES_MAX_MEM%" == "" (
set ES_MAX_MEM=1g
)
if NOT "%ES_HEAP_SIZE%" == "" (
set ES_MIN_MEM=%ES_HEAP_SIZE%
set ES_MAX_MEM=%ES_HEAP_SIZE%
)
set JAVA_OPTS=%JAVA_OPTS% -Xms%ES_MIN_MEM% -Xmx%ES_MAX_MEM%
if NOT "%ES_HEAP_NEWSIZE%" == "" (
set JAVA_OPTS=%JAVA_OPTS% -Xmn%ES_HEAP_NEWSIZE%
)
if NOT "%ES_DIRECT_SIZE%" == "" (
set JAVA_OPTS=%JAVA_OPTS% -XX:MaxDirectMemorySize=%ES_DIRECT_SIZE%
)
set JAVA_OPTS=%JAVA_OPTS% -Xss256k
REM Enable aggressive optimizations in the JVM
REM - Disabled by default as it might cause the JVM to crash
REM set JAVA_OPTS=%JAVA_OPTS% -XX:+AggressiveOpts
set JAVA_OPTS=%JAVA_OPTS% -XX:+UseParNewGC
set JAVA_OPTS=%JAVA_OPTS% -XX:+UseConcMarkSweepGC
set JAVA_OPTS=%JAVA_OPTS% -XX:CMSInitiatingOccupancyFraction=75
set JAVA_OPTS=%JAVA_OPTS% -XX:+UseCMSInitiatingOccupancyOnly
REM When running under Java 7
REM JAVA_OPTS=%JAVA_OPTS% -XX:+UseCondCardMark
REM GC logging options -- uncomment to enable
REM JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCDetails
REM JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCTimeStamps
REM JAVA_OPTS=%JAVA_OPTS% -XX:+PrintClassHistogram
REM JAVA_OPTS=%JAVA_OPTS% -XX:+PrintTenuringDistribution
REM JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCApplicationStoppedTime
REM JAVA_OPTS=%JAVA_OPTS% -Xloggc:/var/log/elasticsearch/gc.log
REM Causes the JVM to dump its heap on OutOfMemory.
set JAVA_OPTS=%JAVA_OPTS% -XX:+HeapDumpOnOutOfMemoryError
REM The path to the heap dump location, note directory must exists and have enough
REM space for a full heap dump.
REM JAVA_OPTS=%JAVA_OPTS% -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof
set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/elasticsearch-1.1.0.jar;%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*
set ES_PARAMS=-Delasticsearch -Des-foreground=yes -Des.path.home="%ES_HOME%"
rem "%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% %* -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch"
rem dell search notes - begin modifcations
rem we changed the one line invokation to make it echo two lines: the command, and the arguments
echo %JAVA_HOME%\bin\java
echo %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% %* -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch"
rem end dell search modifications
goto finally
:err
echo JAVA_HOME environment variable must be set!
pause
:finally
ENDLOCAL
##################### Elasticsearch Configuration Example #####################
# This file contains an overview of various configuration settings,
# targeted at operations staff. Application developers should
# consult the guide at <http://elasticsearch.org/guide>.
#
# The installation procedure is covered at
# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html>.
#
# Elasticsearch comes with reasonable defaults for most settings,
# so you can try it out without bothering with configuration.
#
# Most of the time, these defaults are just fine for running a production
# cluster. If you're fine-tuning your cluster, or wondering about the
# effect of certain configuration option, please _do ask_ on the
# mailing list or IRC channel [http://elasticsearch.org/community].
# Any element in the configuration can be replaced with environment variables
# by placing them in ${...} notation. For example:
#
# node.rack: ${RACK_ENV_VAR}
# For information on supported formats and syntax for the config file, see
# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html>
################################### Cluster ###################################
# Cluster name identifies your cluster for auto-discovery. If you're running
# multiple clusters on the same network, make sure you're using unique names.
#
cluster.name: test.cluster
#################################### Node #####################################
# Node names are generated dynamically on startup, so you're relieved
# from configuring them manually. You can tie this node to a specific name:
#
node.name: "Test Node"
node.local: true
# Every node can be configured to allow or deny being eligible as the master,
# and to allow or deny to store the data.
#
# Allow this node to be eligible as a master node (enabled by default):
#
# node.master: true
#
# Allow this node to store data (enabled by default):
#
# node.data: true
# You can exploit these settings to design advanced cluster topologies.
#
# 1. You want this node to never become a master node, only to hold data.
# This will be the "workhorse" of your cluster.
#
# node.master: false
# node.data: true
#
# 2. You want this node to only serve as a master: to not store any data and
# to have free resources. This will be the "coordinator" of your cluster.
#
# node.master: true
# node.data: false
#
# 3. You want this node to be neither master nor data node, but
# to act as a "search load balancer" (fetching data from nodes,
# aggregating results, etc.)
#
# node.master: false
# node.data: false
# Use the Cluster Health API [http://localhost:9200/_cluster/health], the
# Node Info API [http://localhost:9200/_nodes] or GUI tools
# such as <http://www.elasticsearch.org/overview/marvel/>,
# <http://github.com/karmi/elasticsearch-paramedic>,
# <http://github.com/lukas-vlcek/bigdesk> and
# <http://mobz.github.com/elasticsearch-head> to inspect the cluster state.
# A node can have generic attributes associated with it, which can later be used
# for customized shard allocation filtering, or allocation awareness. An attribute
# is a simple key value pair, similar to node.key: value, here is an example:
#
# node.rack: rack314
# By default, multiple nodes are allowed to start from the same installation location
# to disable it, set the following:
node.max_local_storage_nodes: 1
#################################### Index ####################################
# You can set a number of options (such as shard/replica options, mapping
# or analyzer definitions, translog settings, ...) for indices globally,
# in this file.
#
# Note, that it makes more sense to configure index settings specifically for
# a certain index, either when creating it or by using the index templates API.
#
# See <http://elasticsearch.org/guide/en/elasticsearch/reference/current/index-modules.html> and
# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/indices-create-index.html>
# for more information.
# Set the number of shards (splits) of an index (5 by default):
#
# index.number_of_shards: 5
# Set the number of replicas (additional copies) of an index (1 by default):
#
# index.number_of_replicas: 1
# Note, that for development on a local machine, with small indices, it usually
# makes sense to "disable" the distributed features:
#
index.number_of_shards: 1
index.number_of_replicas: 0
# These settings directly affect the performance of index and search operations
# in your cluster. Assuming you have enough machines to hold shards and
# replicas, the rule of thumb is:
#
# 1. Having more *shards* enhances the _indexing_ performance and allows to
# _distribute_ a big index across machines.
# 2. Having more *replicas* enhances the _search_ performance and improves the
# cluster _availability_.
#
# The "number_of_shards" is a one-time setting for an index.
#
# The "number_of_replicas" can be increased or decreased anytime,
# by using the Index Update Settings API.
#
# Elasticsearch takes care about load balancing, relocating, gathering the
# results from nodes, etc. Experiment with different settings to fine-tune
# your setup.
index.store.type: memory
# Use the Index Status API (<http://localhost:9200/A/_status>) to inspect
# the index status.
#################################### Paths ####################################
# Path to directory containing configuration (this file and logging.yml):
#
# path.conf: /path/to/conf
# Path to directory where to store index data allocated for this node.
#
path.data: C:/temp/elastic.test/data
#
# Can optionally include more than one location, causing data to be striped across
# the locations (a la RAID 0) on a file level, favouring locations with most free
# space on creation. For example:
#
# path.data: /path/to/data1,/path/to/data2
# Path to temporary files:
#
path.work: C:/temp/elastic.test/temp
# Path to log files:
#
path.logs: C:/temp/elastic.test/logs
# Path to where plugins are installed:
#
# path.plugins: /path/to/plugins
#################################### Plugin ###################################
# If a plugin listed here is not installed for current node, the node will not start.
#
# plugin.mandatory: mapper-attachments,lang-groovy
################################### Memory ####################################
# Elasticsearch performs poorly when JVM starts swapping: you should ensure that
# it _never_ swaps.
#
# Set this property to true to lock the memory:
#
# bootstrap.mlockall: true
# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set
# to the same value, and that the machine has enough memory to allocate
# for Elasticsearch, leaving enough memory for the operating system itself.
#
# You should also make sure that the Elasticsearch process is allowed to lock
# the memory, eg. by using `ulimit -l unlimited`.
############################## Network And HTTP ###############################
# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens
# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node
# communication. (the range means that if the port is busy, it will automatically
# try the next port).
# Set the bind address specifically (IPv4 or IPv6):
#
# network.bind_host: 192.168.0.1
# Set the address other nodes will use to communicate with this node. If not
# set, it is automatically derived. It must point to an actual IP address.
#
# network.publish_host: 192.168.0.1
# Set both 'bind_host' and 'publish_host':
#
# network.host: 192.168.0.1
# Set a custom port for the node to node communication (9300 by default):
#
# transport.tcp.port: 9300
# Enable compression for all communication between nodes (disabled by default):
#
# transport.tcp.compress: true
# Set a custom port to listen for HTTP traffic:
#
http.port: 9220
# Set a custom allowed content length:
#
# http.max_content_length: 100mb
# Disable HTTP completely:
#
# http.enabled: false
################################### Gateway ###################################
# The gateway allows for persisting the cluster state between full cluster
# restarts. Every change to the state (such as adding an index) will be stored
# in the gateway, and when the cluster starts up for the first time,
# it will read its state from the gateway.
# There are several types of gateway implementations. For more information, see
# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-gateway.html>.
# The default gateway type is the "local" gateway (recommended):
#
gateway.type: none
# Settings below control how and when to start the initial recovery process on
# a full cluster restart (to reuse as much local data as possible when using shared
# gateway).
# Allow recovery process after N nodes in a cluster are up:
#
# gateway.recover_after_nodes: 1
# Set the timeout to initiate the recovery process, once the N nodes
# from previous setting are up (accepts time value):
#
# gateway.recover_after_time: 5m
# Set how many nodes are expected in this cluster. Once these N nodes
# are up (and recover_after_nodes is met), begin recovery process immediately
# (without waiting for recover_after_time to expire):
#
# gateway.expected_nodes: 2
############################# Recovery Throttling #############################
# These settings allow to control the process of shards allocation between
# nodes during initial recovery, replica allocation, rebalancing,
# or when adding and removing nodes.
# Set the number of concurrent recoveries happening on a node:
#
# 1. During the initial recovery
#
# cluster.routing.allocation.node_initial_primaries_recoveries: 4
#
# 2. During adding/removing nodes, rebalancing, etc
#
# cluster.routing.allocation.node_concurrent_recoveries: 2
# Set to throttle throughput when recovering (eg. 100mb, by default 20mb):
#
# indices.recovery.max_bytes_per_sec: 20mb
# Set to limit the number of open concurrent streams when
# recovering a shard from a peer:
#
# indices.recovery.concurrent_streams: 5
################################## Discovery ##################################
# Discovery infrastructure ensures nodes can be found within a cluster
# and master node is elected. Multicast discovery is the default.
# Set to ensure a node sees N other master eligible nodes to be considered
# operational within the cluster. Its recommended to set it to a higher value
# than 1 when running more than 2 nodes in the cluster.
#
# discovery.zen.minimum_master_nodes: 1
# Set the time to wait for ping responses from other nodes when discovering.
# Set this option to a higher value on a slow or congested network
# to minimize discovery failures:
#
# discovery.zen.ping.timeout: 3s
# For more information, see
# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html>
# Unicast discovery allows to explicitly control which nodes will be used
# to discover the cluster. It can be used when multicast is not present,
# or to restrict the cluster communication-wise.
#
# 1. Disable multicast discovery (enabled by default):
#
discovery.zen.ping.multicast.enabled: false
#
# 2. Configure an initial list of master nodes in the cluster
# to perform discovery when new nodes (master or data) are started:
#
# discovery.zen.ping.unicast.hosts: ["host1", "host2:port"]
# EC2 discovery allows to use AWS EC2 API in order to perform discovery.
#
# You have to install the cloud-aws plugin for enabling the EC2 discovery.
#
# For more information, see
# <http://elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-ec2.html>
#
# See <http://elasticsearch.org/tutorials/elasticsearch-on-ec2/>
# for a step-by-step tutorial.
# GCE discovery allows to use Google Compute Engine API in order to perform discovery.
#
# You have to install the cloud-gce plugin for enabling the GCE discovery.
#
# For more information, see <https://github.com/elasticsearch/elasticsearch-cloud-gce>.
# Azure discovery allows to use Azure API in order to perform discovery.
#
# You have to install the cloud-azure plugin for enabling the Azure discovery.
#
# For more information, see <https://github.com/elasticsearch/elasticsearch-cloud-azure>.
################################## Slow Log ##################################
# Shard level query and fetch threshold logging.
#index.search.slowlog.threshold.query.warn: 10s
#index.search.slowlog.threshold.query.info: 5s
#index.search.slowlog.threshold.query.debug: 2s
#index.search.slowlog.threshold.query.trace: 500ms
#index.search.slowlog.threshold.fetch.warn: 1s
#index.search.slowlog.threshold.fetch.info: 800ms
#index.search.slowlog.threshold.fetch.debug: 500ms
#index.search.slowlog.threshold.fetch.trace: 200ms
#index.indexing.slowlog.threshold.index.warn: 10s
#index.indexing.slowlog.threshold.index.info: 5s
#index.indexing.slowlog.threshold.index.debug: 2s
#index.indexing.slowlog.threshold.index.trace: 500ms
################################## GC Logging ################################
#monitor.jvm.gc.young.warn: 1000ms
#monitor.jvm.gc.young.info: 700ms
#monitor.jvm.gc.young.debug: 400ms
#monitor.jvm.gc.old.warn: 10s
#monitor.jvm.gc.old.info: 5s
#monitor.jvm.gc.old.debug: 2s
using System;
using NUnit.Framework;
using Search.QueryUnderstanding.Api.Application;
[SetUpFixture]
public class ElasticSearchIntegrationTestSetupFixture
{
private readonly ElasticSearchTestClient client = new ElasticSearchTestClient(new ElasticSearchConfiguration());
private readonly ElasticSearchTestContext context = new ElasticSearchTestContext();
[SetUp]
public void Start_elastic_node()
{
Console.WriteLine("{0}: Starting elasticsearch node", GetType().Name);
context.StartServer();
Console.WriteLine("{0}: elasticsearch node started", GetType().Name);
Console.WriteLine("{0}: deleting all indexes", GetType().Name);
client.DeleteAllIndexes();
}
[TearDown]
public void Kill_elastic_node()
{
Console.WriteLine("\r\n{0}: Killing elasticsearch node", GetType().Name);
context.KillServer();
}
}
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
/// <summary>
/// ElasticSearch helper, manages the node
/// </summary>
public class ElasticSearchTestContext
{
private string _configPath;
private string _exePath;
private Process _process;
public ElasticSearchTestContext()
{
// $root\tools\elasticsearch\elasticsearch-1.1.0\bin
// expecting test configs at $root\tools\elasticsearch
// expecting elasticsearch.echo.bat next to elasticsearch.bat <- horrible idea?
SetPaths();
AssertPathsAreValid();
}
private void AssertPathsAreValid()
{
if (!File.Exists(_exePath))
{
throw new InvalidOperationException(string.Format("ExePath does not exist '{0}'", _exePath));
}
if (!Directory.Exists(_configPath))
{
throw new InvalidOperationException(string.Format("ConfigPath does not exist '{0}'", _configPath));
}
}
private void SetPaths()
{
var directoryInfo = GetToolsDirectory();
var toolsElasticSearch = directoryInfo.GetDirectories("elasticsearch").Single();
_configPath = toolsElasticSearch.FullName;
var elasticVersionedPath = toolsElasticSearch.GetDirectories("elasticsearch*").Single().FullName;
_exePath = Path.Combine(elasticVersionedPath, "bin", "elasticsearch.echo.bat");
}
private DirectoryInfo GetToolsDirectory()
{
var currentPath = Path.GetDirectoryName(GetType().Assembly.Location);
var pwd = new DirectoryInfo(currentPath);
var list = new List<string>();
while (!pwd.GetDirectories("tools").Any())
{
list.Add(pwd.FullName);
pwd = pwd.Parent;
if (pwd == null)
{
throw new InvalidOperationException(string.Format("Error looking for tools directory. The following directories were searched:\r\n{0}", string.Join("\r\n", list)));
}
}
var directoryInfo = pwd.GetDirectories("tools").Single();
return directoryInfo;
}
public void StartServer()
{
ValidateProcessCanBeStarted();
var processStartInfo = GetRealProcessStartInfo();
_process = new Process { StartInfo = processStartInfo, };
_process.Start();
// wait for elasticsearch to finish initialization
while (true)
{
var readLine = _process.StandardOutput.ReadLine();
Console.WriteLine(readLine);
Debug.WriteLine(readLine);
if (( readLine ?? "" ).Trim().EndsWith("started")) return;
}
}
private ProcessStartInfo GetRealProcessStartInfo()
{
var info = new ProcessStartInfo(_exePath, string.Format("-Des.path.conf={0}", _configPath))
{
UseShellExecute = false,
RedirectStandardOutput = true,
CreateNoWindow = true
};
var getCmdProc = new Process
{
StartInfo = info
};
getCmdProc.Start();
var cmd = getCmdProc.StandardOutput.ReadLine();
var args = getCmdProc.StandardOutput.ReadLine();
Console.WriteLine("{0}: Java: {1}", GetType().Name, cmd);
Console.WriteLine("{0}: Args: {1}", GetType().Name, args);
var processStartInfo = new ProcessStartInfo(cmd, args)
{
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false,
RedirectStandardOutput = true
};
return processStartInfo;
}
public void KillServer()
{
if (_process == null) return;
if (_process.WaitForExit(5000)) return;
if (!_process.HasExited) _process.Kill();
}
public override string ToString()
{
return string.Format("ConfigPath: {0}, ExePath: {1}", _configPath, _exePath);
}
private void ValidateProcessCanBeStarted()
{
if (_process != null)
{
throw new InvalidOperationException("ElasticSearch is already started");
}
if (Environment.GetEnvironmentVariable("JAVA_HOME") == null)
{
throw new InvalidOperationException("JAVA_HOME environment variable not set");
}
}
}
@mhinze
Copy link
Author

mhinze commented Apr 10, 2014

Output looks like this:

ProcessModel: Separate    DomainUsage: Multiple
Execution Runtime: net-4.5
Included categories: Slow
ElasticSearchIntegrationTestSetupFixture: Starting elasticsearch node
ElasticSearchTestContext: Java: C:\Progra~1\Java\jre7\bin\java
ElasticSearchTestContext: Args:  -Xms256m -Xmx1g -Xss256k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly
-XX:+HeapDumpOnOutOfMemoryError  -Delasticsearch -Des-foreground=yes -Des.path.home="C:\search\Api\Project\tools\elasticsearch\elasticsearch-1.1.0" -Des.path.conf=C:\search\Api\Project\tools\elasticsearch -cp ";C:\search\Api\Project\tools\elasticsearch\elasticsearch-1.1.0/lib/elasticsearch-1.1.0.jar;C:\search\Api\Project\tools\elasticsearch\elasticsearch-1.1.0/lib/*;C:\search\Api\Project\tools\elasticsearch\elasticsearch-1.1.0/lib/sigar/*" "org.elasticsearch.bootstrap.Elasticsearch"
[2014-04-10 09:17:11,154][INFO ][node                     ] [Test Node] version[1.1.0], pid[43552], build[2181e11/2014-03-25T15:59:51Z]
[2014-04-10 09:17:11,154][INFO ][node                     ] [Test Node] initializing ...
[2014-04-10 09:17:11,172][INFO ][plugins                  ] [Test Node] loaded [], sites []
[2014-04-10 09:17:13,670][INFO ][node                     ] [Test Node] initialized
[2014-04-10 09:17:13,670][INFO ][node                     ] [Test Node] starting ...
[2014-04-10 09:17:13,676][INFO ][transport                ] [Test Node] bound_address {local[1]}, publish_address {local[1]}
[2014-04-10 09:17:13,717][INFO ][cluster.service          ] [Test Node] new_master [Test Node][kZ0O1KW7QbKol4835VJIAQ][my-box][local[1]]{local=true, max_local_storage_nodes=1}, reason: local-disco-initial_connect(master)
[2014-04-10 09:17:13,720][INFO ][discovery                ] [Test Node] test.cluster/kZ0O1KW7QbKol4835VJIAQ
[2014-04-10 09:17:13,737][INFO ][gateway                  ] [Test Node] recovered [0] indices into cluster_state
[2014-04-10 09:17:13,978][INFO ][http                     ] [Test Node] bound_address {inet[/0:0:0:0:0:0:0:0:9220]}, publish_address {inet[/10.210.54.78:9220]}
[2014-04-10 09:17:13,980][INFO ][node                     ] [Test Node] started
ElasticSearchIntegrationTestSetupFixture: elasticsearch node started
ElasticSearchIntegrationTestSetupFixture: deleting all indexes
..
ElasticSearchIntegrationTestSetupFixture: Killing elasticsearch node

Tests run: 2, Errors: 0, Failures: 0, Inconclusive: 0, Time: 11.1736673554647 seconds
  Not run: 0, Invalid: 0, Ignored: 0, Skipped: 0

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment