elasticsearch.yml

node.name: "Gardevoir"
index.number_of_shards: 1
index.number_of_replicas: 0
script.default_lang: groovy
script.disable_dynamic: false

ElasticSearchTest.java

package elasticrtb;

import java.io.IOException;
import java.util.Map;

import elasticrtb.es.SearchHelpers;
import test.helpers.TestClient;

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.rest.RestStatus;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

/**
 * Le but de cette classe n'est pas de tester des fonctionnalités de notre plugin mais de tester directement le fonctionnement d'elasticSearch
 * afin de permettre au déveoppeurs de mieux appréhender le fonctionnement de ce dernier.
 * 
 * Les tests de cette classent ne doivent en aucun cas être considérés comme validant des parties du plugin. Des tests précis pour chaque partie du plugin
 * doivent être effectués.
 */
public class ElasticSearchTest {
	static TestClient	client;

	@BeforeClass
	public static void setUpClass() throws IOException {
		client = TestClient.getDefaultInstance();
	}

	/**
	 * @link http://stackoverflow.com/questions/14595988/queries-vs-filters
	 */
	@Test
	public void test_queryWithFiltersAndCutomScore() {
		// Query
		QueryBuilder query = QueryBuilders
			.functionScoreQuery(QueryBuilders
				.boolQuery()
				.must(QueryBuilders.termQuery("width", 1024))
				.must(QueryBuilders.termQuery("height", 768))
			).add(ScoreFunctionBuilders.scriptFunction("_score * (doc['doc_boost'].empty ? 1 : doc['doc_boost'].value)", "groovy"));

		FilterBuilder filter = FilterBuilders.boolFilter()
			.must(SearchHelpers.buildMustTermsOrMissing("categories", "painting"))
			.must(SearchHelpers.buildMustNotTermsOrMissing("not_categories", "drawing", "landscape"));

		SearchResponse searchResponse = client.executeQuery(QueryBuilders.filteredQuery(query, filter));

		Assert.assertEquals(RestStatus.OK, searchResponse.status());
		Assert.assertEquals("entry-1", searchResponse.getHits().getAt(0).getSource().get("test_ref"));

		for (int totalHits = (int) searchResponse.getHits().getTotalHits() - 1; totalHits >= 0; totalHits--) {
			Map<String, Object> source = searchResponse.getHits().getAt(totalHits).getSource();
			Assert.assertEquals(1024, source.get("width"));
			Assert.assertEquals(768, source.get("height"));
		}
	}
}

IndexHelpers.java

package elasticrtb.es;

import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;

public class IndexHelpers {

	public static boolean indexExists(Client client, String index) {
		return client.admin().indices().exists(new IndicesExistsRequest(index)).actionGet().isExists();
	}

	public static void createIndex(Client client, String index, String source) {
		if (!indexExists(client, index)) {
			CreateIndexRequestBuilder q = client.admin().indices().prepareCreate(index);
			if (source != null) {
				q.setSource(source);
			}
			q.execute().actionGet();
		}
	}

	public static IndexRequestBuilder prepareIndex(Client client, String index, String type, String source) {
		return client.prepareIndex(index, type).setSource(source);
	}

	public static void index(Client client, String index, String type, String source) {
		prepareIndex(client, index, type, source).execute().actionGet();
	}

	public static void deleteIndex(Client client, String index) {
		if (indexExists(client, index)) {
			client.admin().indices().prepareDelete(index).execute().actionGet();
		}
	}
}

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>org.elasticsearch.testsample</groupId>
	<artifactId>testsample</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<dependencies>
		<dependency>
			<groupId>org.elasticsearch</groupId>
			<artifactId>elasticsearch</artifactId>
			<version>1.3.0</version>
		</dependency>
		<dependency>
			<groupId>commons-io</groupId>
			<artifactId>commons-io</artifactId>
			<version>2.4</version>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>4.11</version>
			<scope>test</scope>
		</dependency>
	</dependencies>
	<build>
		<pluginManagement>
			<plugins>
				<plugin>
					<groupId>org.apache.maven.plugins</groupId>
					<artifactId>maven-compiler-plugin</artifactId>
					<version>3.1</version>
					<configuration>
						<source>1.7</source>
						<target>1.7</target>
					</configuration>
				</plugin>
			</plugins>
		</pluginManagement>
	</build>
</project>

SearchHelpers.java

package elasticrtb.es;

import java.util.Collection;

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;

public class SearchHelpers {

	public static Object getResultField(SearchResponse rsp, Integer index, String field) {
		return rsp.getHits().getAt(index).getSource().get(field);
	}

	/**
	 * @link http://www.elasticsearch.org/guide/reference/query-dsl/bool-filter/
	 * @link http://www.elasticsearch.org/guide/reference/query-dsl/terms-filter/
	 * @param column
	 * @param terms
	 * @return OrFilterBuilder permettant de satisfaire qu'au moins un des termes fournis soit présent dans la colonne spécifiée ou que la colonne spécifiée
	 *         n'existe pas (soit null) dans l'index
	 */
	public static FilterBuilder buildMustTermsOrMissing(String column, Object... terms) {
		return FilterBuilders.boolFilter()
			.should(
				FilterBuilders.termsFilter(column, terms),
				buildMissing(column)
			);
	}

	public static FilterBuilder buildMustTermsOrMissing(String column, Collection<?> terms) {
		return buildMustTermsOrMissing(column, terms.toArray());
	}

	/**
	 * @link http://www.elasticsearch.org/guide/reference/query-dsl/bool-filter/
	 * @link http://www.elasticsearch.org/guide/reference/query-dsl/terms-filter/
	 * @param column
	 * @param terms
	 * @return OrFilterBuilder permettant de satisfaire qu'aucun des termes fournis ne soit présent dans la colonne spécifiée ou que la colonne spécifiée
	 *         n'existe pas (soit null) dans l'index
	 */
	public static FilterBuilder buildMustNotTermsOrMissing(String column, Object... terms) {
		return FilterBuilders.boolFilter()
			.should(
				FilterBuilders.notFilter(FilterBuilders.termsFilter(column, terms)),
				buildMissing(column)
			);
	}

	public static FilterBuilder buildMustNotTermsOrMissing(String column, Collection<?> terms) {
		return buildMustNotTermsOrMissing(column, terms.toArray());
	}

	public static FilterBuilder buildMissing(String column) {
		return FilterBuilders.missingFilter(column).existence(true).nullValue(true);
	}

	/**
	 * @link http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets
	 * @param documentColumn
	 * @param scriptValue
	 * @return
	 */
	public static FilterBuilder buildSegmentationScriptFilter(String documentColumn, String scriptValue) {
		return FilterBuilders.orFilter()
			.add(FilterBuilders.scriptFilter(scriptValue))
			.add(buildMissing(documentColumn));
	}

	public static String getGeoDistanceScript(String fieldGeopoint, String fieldRadius, Object latitude, Object longitude) {
		return "!doc['" + fieldGeopoint + "'].empty && doc['" + fieldGeopoint + "'].distanceInKm(" + String.valueOf(latitude) + "," + String.valueOf(longitude) + ") <= doc['" + fieldRadius + "'].value";
	}
}

TestClient.data.json

{
	"test_ref": "entry-1",
	"width": 1024,
	"height": 768,
	"deal_identifier": ["smaato:deal-1"],
	"doc_boost": 1.0,
	"categories": ["painting", "landscape"]
}

{
	"test_ref": "entry-2",
	"width": 1024,
	"height": 768,
	"doc_boost": 2.0,
	"not_categories": ["landscape"]
}

{
	"test_ref": "entry-3",
	"width": 1024,
	"height": 768,
	"doc_boost": 1.0,
	"segcategories": ["landscape"]
}

TestClient.index.json

{
	"settings": {
		"analysis": {
			"analyzer": {
				"full_text": {
					"type": "custom",
					"tokenizer": "keyword",
					"filter": ["lowercase"]
				}
			}
		},
		"auto_expand_replicas": "0-all"
	},
	"mappings": {
		"testtype": {
			"properties": {
				"doc_boost": {
					"type": "double"
				},
				"categories": {
					"type": "string",
					"index" : "not_analyzed"
				},
				"not_categories": {
					"type": "string",
					"index" : "not_analyzed"
				}
			}
		}
	}
}

TestClient.java

package test.helpers;

import java.io.IOException;

import elasticrtb.es.IndexHelpers;

import org.apache.commons.io.IOUtils;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.NodeBuilder;

public class TestClient {

	//ELASTIC
	final static String		className		= TestClient.class.getSimpleName();

	final static String		ES_TYPE			= "testtype";
	final static String		ES_SETTINGS;
	final static String[]	ES_DATA;
	final static String		ES_INDEX		= "test_" + className.toLowerCase();

	static boolean			setupDone		= false;
	static boolean			lastLogWasLine	= true;
	static final Client		client;
	static {
		logLine("Loading config files... ");
		Class<TestClient> clazz = TestClient.class;
		try {
			ES_SETTINGS = IOUtils.toString(clazz.getResourceAsStream("/" + clazz.getSimpleName() + ".index.json"));
			ES_DATA = IOUtils.toString(clazz.getResourceAsStream("/" + clazz.getSimpleName() + ".data.json")).split("\n\n");
		} catch (IOException e) {
			throw new RuntimeException(e);
		}

		logLine("Creating local node... ");
		client = NodeBuilder.nodeBuilder().local(true).node().client();
		setup();

		Runtime.getRuntime().addShutdownHook(new Thread() {
			@Override
			public void run() {
				cleanup();
				log("Shuting down local node... ");
				client.close();
				logLine("Done.");
			}
		});
	}

	public static TestClient getDefaultInstance() throws IOException {
		return new TestClient();
	}

	private static void log(String str) {
		if (lastLogWasLine) {
			System.out.print(className + ": " + str);
		}
		else {
			System.out.print(str);
		}
		lastLogWasLine = false;
	}

	private static void logLine(String str) {
		if (lastLogWasLine) {
			System.out.println(className + ": " + str);
		}
		else {
			System.out.println(str);
		}
		lastLogWasLine = true;
	}

	public TestClient() {
		//		client = new TransportClient().addTransportAddress(new InetSocketTransportAddress(ES_HOST, ES_PORT));
	}

	public static void setup() {
		if (!setupDone) {
			setupDone = true;
			logLine("Creating index " + ES_INDEX + "...");
			IndexHelpers.createIndex(client, ES_INDEX, ES_SETTINGS);

			logLine("Seeding test data...");
			int seeds = 0;
			BulkRequestBuilder bulk = client.prepareBulk();
			for (String request : ES_DATA) {
				seeds++;
				bulk.add(IndexHelpers.prepareIndex(client, ES_INDEX, ES_TYPE, request));
			}
			BulkResponse bulkResponse = bulk.execute().actionGet();
			if (bulkResponse.hasFailures()) {
				logLine("Seeding failed!");
				for (BulkItemResponse bulkItemResponse : bulkResponse) {
					logLine(bulkItemResponse.getFailureMessage());
				}
			}
			log("Seeded");
			long done;
			while ((done = countSeeds()) < seeds) {
				log(".");
				try {
					Thread.sleep(100);
				} catch (InterruptedException e) {
					Thread.currentThread().interrupt();
				}
			}
			logLine(" " + done + " documents");
		}
	}

	public Client getClient() {
		return client;
	}

	public String getIndex() {
		return ES_INDEX;
	}

	public String getType() {
		return ES_TYPE;
	}

	public static void cleanup() {
		if (setupDone) {
			logLine("Deleting index " + ES_INDEX + "...");
			IndexHelpers.deleteIndex(client, ES_INDEX);
		}
	}

	public SearchResponse executeFilter(FilterBuilder filterBuilder) {
		return this.executeQueryWithFilters(null, filterBuilder);
	}

	public SearchResponse executeQuery(QueryBuilder queryBuilder) {
		return this.executeQueryWithFilters(queryBuilder, null);
	}

	public SearchResponse executeQueryWithFilters(QueryBuilder queryBuilder, FilterBuilder filterBuilder) {

		SearchRequestBuilder search = client
			.prepareSearch(ES_INDEX)
			.setTypes(ES_TYPE)
			.setSearchType(SearchType.QUERY_AND_FETCH)
			.setQuery(queryBuilder == null ? QueryBuilders.matchAllQuery() : queryBuilder);

		if (filterBuilder != null) {
			search.setPostFilter(filterBuilder);
		}

		//		System.out.println(search);

		return search
			.setFrom(0)
			.setSize(100)
			.setExplain(true)
			.execute()
			.actionGet();
	}

	private static long countSeeds() {
		return client
			.prepareSearch(ES_INDEX)
			.setTypes(ES_TYPE)
			.setSearchType(SearchType.QUERY_AND_FETCH)
			.setQuery(QueryBuilders.matchAllQuery())
			.setFrom(0)
			.setSize(100)
			.execute()
			.actionGet()
			.getHits()
			.getTotalHits();
	}

	public long getTotalHits(QueryBuilder queryBuilder) {
		return getTotalHits(queryBuilder, null);
	}

	public long getTotalHits(QueryBuilder queryBuilder, FilterBuilder filterBuilder) {
		return this
			.executeQueryWithFilters(queryBuilder, filterBuilder)
			.getHits()
			.getTotalHits();
	}
}