Last active
January 13, 2017 21:24
-
-
Save brianmhess/4882b5253a9371f9d481b6076dc6e7fc to your computer and use it in GitHub Desktop.
DSEFS FieldInputTransformer
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
cqlsh -e "CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}" | |
cqlsh -e "CREATE TABLE test.dtest(pkey INT, info TEXT, link TEXT, dsefs_contents TEXT, PRIMARY KEY ((pkey)))" | |
echo This is a test > /tmp/test1.txt | |
echo This is another test > /tmp/test2.txt | |
echo Hello World > /tmp/test3.txt | |
echo Thank You > /tmp/test4.txt | |
dse fs "mkdir /fittest" | |
dse fs "put /tmp/test1.txt /fittest/test1.txt" | |
dse fs "put /tmp/test2.txt /fittest/test2.txt" | |
dse fs "put /tmp/test3.txt /fittest/test3.txt" | |
dse fs "put /tmp/test4.txt /fittest/test4.txt" | |
dsetool create_core test.dtest schema=dsefs_schema.xml solrconfig=dsefs_solrconfig.xml | |
cqlsh -e "INSERT INTO test.dtest(pkey,info,link) VALUES (100,'Test #1','dsefs:///fittest/test1.txt')" | |
cqlsh -e "INSERT INTO test.dtest(pkey,info,link) VALUES (200,'Test #2','dsefs:///fittest/test2.txt')" | |
cqlsh -e "INSERT INTO test.dtest(pkey,info,link) VALUES (300,'Hello Test','dsefs:///fittest/test3.txt')" | |
cqlsh -e "INSERT INTO test.dtest(pkey,info,link) VALUES (400,'Gracias Test','dsefs:///fittest/test4.txt')" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0" encoding="UTF-8" standalone="no"?> | |
<schema name="autoSolrSchema" version="1.5"> | |
<types> | |
<fieldType class="org.apache.solr.schema.TextField" name="TextField"> | |
<analyzer> | |
<tokenizer class="solr.StandardTokenizerFactory"/> | |
<filter class="solr.LowerCaseFilterFactory"/> | |
</analyzer> | |
</fieldType> | |
<fieldType class="org.apache.solr.schema.TrieIntField" name="TrieIntField"/> | |
</types> | |
<fields> | |
<field indexed="true" multiValued="false" name="info" stored="true" type="TextField"/> | |
<field indexed="true" multiValued="false" name="pkey" stored="true" type="TrieIntField"/> | |
<field indexed="true" multiValued="false" name="link" stored="true" type="TextField"/> | |
<field indexed="true" multiValued="false" name="dsefs_contents" stored="false" type="TextField"/> | |
</fields> | |
<uniqueKey>pkey</uniqueKey> | |
</schema> | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0" encoding="UTF-8" standalone="no"?> | |
<!-- | |
======= | |
Copyright DataStax, Inc. | |
Please see the included license file for details. | |
--><!-- | |
For more details about configurations options that may appear in | |
this file, see http://wiki.apache.org/solr/SolrConfigXml. | |
--><config> | |
<!-- In all configuration below, a prefix of "solr." for class names | |
is an alias that causes solr to search appropriate packages, | |
including org.apache.solr.(search|update|request|core|analysis) | |
You may also specify a fully qualified Java classname if you | |
have your own custom plugins. | |
--> | |
<!-- Set this to 'false' if you want solr to continue working after | |
it has encountered an severe configuration error. In a | |
production environment, you may want solr to keep working even | |
if one handler is mis-configured. | |
You may also set this to false using by setting the system | |
property: | |
-Dsolr.abortOnConfigurationError=false | |
--> | |
<abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError> | |
<!-- Controls what version of Lucene various components of Solr | |
adhere to. Generally, you want to use the latest version to | |
get all bug fixes and improvements. It is highly recommended | |
that you fully re-index after changing this setting as it can | |
affect both how text is indexed and queried. | |
--> | |
<luceneMatchVersion>LUCENE_46</luceneMatchVersion> | |
<!-- Enable DSE Search new type mappings --> | |
<dseTypeMappingVersion>2</dseTypeMappingVersion> | |
<!-- lib directives can be used to instruct Solr to load an Jars | |
identified and use them to resolve any "plugins" specified in | |
your solrconfig.xml or schema.xml (ie: Analyzers, Request | |
Handlers, etc...). | |
All directories and paths are resolved relative to the | |
instanceDir. | |
If a "./lib" directory exists in your instanceDir, all files | |
found in it are included as if you had used the following | |
syntax... | |
<lib dir="./lib" /> | |
--> | |
<!-- A dir option by itself adds any files found in the directory to | |
the classpath, this is useful for including all jars in a | |
directory. | |
--> | |
<!-- an exact path can be used to specify a specific file. This | |
will cause a serious error to be logged if it can't be loaded. | |
--> | |
<!-- | |
<lib path="../a-jar-that-does-not-exist.jar" /> | |
--> | |
<!-- Data Directory | |
Used to specify an alternate directory to hold all index data | |
other than the default ./data under the Solr home. If | |
replication is in use, this should match the replication | |
configuration. | |
<dataDir>${solr.data.dir}</dataDir> | |
--> | |
<!-- The DirectoryFactory to use for indexes. | |
solr.StandardDirectoryFactory, the default, is filesystem | |
based and tries to pick the best implementation for the current | |
JVM and platform. One can force a particular implementation | |
via solr.MMapDirectoryFactory, solr.NIOFSDirectoryFactory, or | |
solr.SimpleFSDirectoryFactory. | |
solr.RAMDirectoryFactory is memory based, not | |
persistent, and doesn't work with replication. | |
--> | |
<directoryFactory class="solr.StandardDirectoryFactory" name="DirectoryFactory"/> | |
<indexConfig> | |
<rt>false</rt> | |
<useCompoundFile>false</useCompoundFile> | |
<ramBufferSizeMB>512</ramBufferSizeMB> | |
<mergeFactor>10</mergeFactor> | |
<!-- Unlock On Startup | |
If true, unlock any held write or commit locks on startup. | |
This defeats the locking mechanism that allows multiple | |
processes to safely access a lucene index, and should be used | |
with care. | |
This is not needed if lock type is 'none' or 'single' | |
--> | |
<unlockOnStartup>true</unlockOnStartup> | |
<!-- If true, IndexReaders will be reopened (often more efficient) | |
instead of closed and then opened. | |
--> | |
<reopenReaders>true</reopenReaders> | |
<!-- Commit Deletion Policy | |
Custom deletion policies can specified here. The class must | |
implement org.apache.lucene.index.IndexDeletionPolicy. | |
http://lucene.apache.org/java/2_9_1/api/all/org/apache/lucene/index/IndexDeletionPolicy.html | |
The standard Solr IndexDeletionPolicy implementation supports | |
deleting index commit points on number of commits, age of | |
commit point and optimized status. | |
The latest commit point should always be preserved regardless | |
of the criteria. | |
--> | |
<deletionPolicy class="solr.SolrDeletionPolicy"> | |
<!-- The number of commit points to be kept --> | |
<str name="maxCommitsToKeep">1</str> | |
<!-- The number of optimized commit points to be kept --> | |
<str name="maxOptimizedCommitsToKeep">0</str> | |
<!-- | |
Delete all commit points once they have reached the given age. | |
Supports DateMathParser syntax e.g. | |
--> | |
<!-- | |
<str name="maxCommitAge">30MINUTES</str> | |
<str name="maxCommitAge">1DAY</str> | |
--> | |
</deletionPolicy> | |
<!-- Lucene Infostream | |
To aid in advanced debugging, Lucene provides an "InfoStream" | |
of detailed information when indexing. | |
Setting The value to true will instruct the underlying Lucene | |
IndexWriter to write its debugging info the specified file | |
--> | |
<infoStream file="INFOSTREAM.txt">false</infoStream> | |
</indexConfig> | |
<!-- JMX | |
This example enables JMX if and only if an existing MBeanServer | |
is found, use this if you want to configure JMX through JVM | |
parameters. Remove this to disable exposing Solr configuration | |
and statistics to JMX. | |
For more details see http://wiki.apache.org/solr/SolrJmx | |
--> | |
<jmx/> | |
<!-- If you want to connect to a particular server, specify the | |
agentId | |
--> | |
<!-- <jmx agentId="myAgent" /> --> | |
<!-- If you want to start a new MBeanServer, specify the serviceUrl --> | |
<!-- <jmx serviceUrl="service:jmx:rmi:///jndi/rmi://localhost:9999/solr"/> | |
--> | |
<!-- The default high-performance update handler --> | |
<!-- IN DSE THIS CANNOT BE CHANGED --> | |
<updateHandler class="solr.DirectUpdateHandler2"> | |
<!-- AutoCommit | |
Perform a <commit/> automatically under certain conditions. | |
Instead of enabling autoCommit, consider using "commitWithin" | |
when adding documents. | |
http://wiki.apache.org/solr/UpdateXmlMessages | |
maxDocs - Maximum number of documents to add since the last | |
commit before automatically triggering a new commit. | |
maxTime - Maximum amount of time that is allowed to pass | |
since a document was added before automaticly | |
triggering a new commit. | |
--> | |
<autoSoftCommit> | |
<maxTime>10000</maxTime> | |
</autoSoftCommit> | |
<!-- Update Related Event Listeners | |
Various IndexWriter related events can trigger Listeners to | |
take actions. | |
postCommit - fired after every commit or optimize command | |
postOptimize - fired after every optimize command | |
--> | |
<!-- The RunExecutableListener executes an external command from a | |
hook such as postCommit or postOptimize. | |
exe - the name of the executable to run | |
dir - dir to use as the current working directory. (default=".") | |
wait - the calling thread waits until the executable returns. | |
(default="true") | |
args - the arguments to pass to the program. (default is none) | |
env - environment variables to set. (default is none) | |
--> | |
<!-- This example shows how RunExecutableListener could be used | |
with the script based replication... | |
http://wiki.apache.org/solr/CollectionDistribution | |
--> | |
<!-- | |
<listener event="postCommit" class="solr.RunExecutableListener"> | |
<str name="exe">solr/bin/snapshooter</str> | |
<str name="dir">.</str> | |
<bool name="wait">true</bool> | |
<arr name="args"> <str>arg1</str> <str>arg2</str> </arr> | |
<arr name="env"> <str>MYVAR=val1</str> </arr> | |
</listener> | |
--> | |
</updateHandler> | |
<query> | |
<!-- Max Boolean Clauses | |
Maximum number of clauses in each BooleanQuery, an exception | |
is thrown if exceeded. | |
** WARNING ** | |
This option actually modifies a global Lucene property that | |
will affect all SolrCores. If multiple solrconfig.xml files | |
disagree on this property, the value at any given moment will | |
be based on the last SolrCore to be initialized. | |
--> | |
<maxBooleanClauses>1024</maxBooleanClauses> | |
<!-- Solr Internal Query Caches | |
There are two implementations of cache available for Solr, | |
LRUCache, based on a synchronized LinkedHashMap, and | |
FastLRUCache, based on a ConcurrentHashMap. | |
FastLRUCache has faster gets and slower puts in single | |
threaded operation and thus is generally faster than LRUCache | |
when the hit ratio of the cache is high (> 75%), and may be | |
faster under other scenarios on multi-cpu systems. | |
--> | |
<!-- Filter Cache | |
Cache used by SolrIndexSearcher for filters (DocSets), | |
unordered sets of *all* documents that match a query. | |
There are two types of filter caches. | |
Per segment size-based caches, with the following parameters: | |
class - LFUCache, LRUCache or FastLRUCache. | |
size - the maximum number of entries in the cache. | |
initialSize - the initial capacity (number of entries) of the cache. | |
Per segment memory-based caches with LRU eviction, with the following parameters: | |
class - SolrFilterCache. | |
highWaterMarkMB - the maximum number of used MBs before triggering asynchronous LRU eviction. | |
lowWaterMarkMB - the maximum number of used MBs after eviction is completed. | |
--> | |
<filterCache class="solr.SolrFilterCache" highWaterMarkMB="256" lowWaterMarkMB="128"/> | |
<!-- Query Result Cache | |
Caches results of searches - ordered lists of document ids | |
(DocList) based on a query, a sort, and the range of documents requested. | |
--> | |
<!-- on DSE, queryResultCache is commented out as it is cleared on each soft commit --> | |
<!-- <queryResultCache class="solr.LRUCache" | |
size="512" | |
initialSize="512" | |
autowarmCount="0"/> --> | |
<!-- Field Value Cache | |
Cache used to hold field values that are quickly accessible | |
by document id. The fieldValueCache is created by default | |
even if not configured here. | |
--> | |
<!-- | |
<fieldValueCache class="solr.FastLRUCache" | |
size="512" | |
autowarmCount="128" | |
showItems="32" /> | |
--> | |
<!-- Lazy Field Loading | |
If true, stored fields that are not requested will be loaded | |
lazily. This can result in a significant speed improvement | |
if the usual case is to not load all stored fields, | |
especially if the skipped fields are large compressed text | |
fields. | |
--> | |
<enableLazyFieldLoading>true</enableLazyFieldLoading> | |
<!-- Use Filter For Sorted Query | |
A possible optimization that attempts to use a filter to | |
satisfy a search. If the requested sort does not include | |
score, then the filterCache will be checked for a filter | |
matching the query. If found, the filter will be used as the | |
source of document ids, and then the sort will be applied to | |
that. | |
For most situations, this will not be useful unless you | |
frequently get the same search repeatedly with different sort | |
options, and none of them ever use "score" | |
--> | |
<!-- | |
<useFilterForSortedQuery>true</useFilterForSortedQuery> | |
--> | |
<!-- Result Window Size | |
An optimization for use with the queryResultCache. When a search | |
is requested, a superset of the requested number of document ids | |
are collected. For example, if a search for a particular query | |
requests matching documents 10 through 19, and queryWindowSize is 50, | |
then documents 0 through 49 will be collected and cached. Any further | |
requests in that range can be satisfied via the cache. | |
<queryResultWindowSize>20</queryResultWindowSize> | |
--> | |
<!-- Maximum number of documents to cache for any entry in the | |
queryResultCache. | |
<queryResultMaxDocsCached>200</queryResultMaxDocsCached> | |
--> | |
<!-- Query Related Event Listeners | |
Various IndexSearcher related events can trigger Listeners to | |
take actions. | |
newSearcher - fired whenever a new searcher is being prepared | |
and there is a current searcher handling requests (aka | |
registered). It can be used to prime certain caches to | |
prevent long request times for certain requests. | |
firstSearcher - fired whenever a new searcher is being | |
prepared but there is no current registered searcher to handle | |
requests or to gain autowarming data from. | |
--> | |
<!-- QuerySenderListener takes an array of NamedList and executes a | |
local query request for each NamedList in sequence. | |
--> | |
<!-- | |
<listener event="newSearcher" class="solr.QuerySenderListener"> | |
<arr name="queries"> | |
<lst><str name="q">solr</str><str name="sort">price asc</str></lst> | |
<lst><str name="q">rocks</str><str name="sort">weight asc</str></lst> | |
</arr> | |
</listener> | |
<listener event="firstSearcher" class="solr.QuerySenderListener"> | |
<arr name="queries"> | |
<lst> | |
<str name="q">static firstSearcher warming in solrconfig.xml</str> | |
</lst> | |
</arr> | |
</listener> | |
--> | |
<!-- Use Cold Searcher | |
If a search request comes in and there is no current | |
registered searcher, then immediately register the still | |
warming searcher and use it. If "false" then all requests | |
will block until the first searcher is done warming. | |
--> | |
<useColdSearcher>true</useColdSearcher> | |
<!-- Max Warming Searchers | |
Maximum number of searchers that may be warming in the | |
background concurrently. An error is returned if this limit | |
is exceeded. | |
Recommend values of 1-2 for read-only slaves, higher for | |
masters w/o cache warming. | |
--> | |
<maxWarmingSearchers>16</maxWarmingSearchers> | |
</query> | |
<!-- Request Dispatcher | |
This section contains instructions for how the SolrDispatchFilter | |
should behave when processing requests for this SolrCore. | |
handleSelect affects the behavior of requests such as /select?qt=XXX | |
handleSelect="true" will cause the SolrDispatchFilter to process | |
the request and will result in consistent error handling and | |
formatting for all types of requests. | |
handleSelect="false" will cause the SolrDispatchFilter to | |
ignore "/select" requests and fallback to using the legacy | |
SolrServlet and it's Solr 1.1 style error formatting | |
--> | |
<requestDispatcher handleSelect="true"> | |
<!-- Request Parsing | |
These settings indicate how Solr Requests may be parsed, and | |
what restrictions may be placed on the ContentStreams from | |
those requests | |
enableRemoteStreaming - enables use of the stream.file | |
and stream.url parameters for specifying remote streams. | |
multipartUploadLimitInKB - specifies the max size of | |
Multipart File Uploads that Solr will allow in a Request. | |
*** WARNING *** | |
The settings below authorize Solr to fetch remote files, You | |
should make sure your system has some authentication before | |
using enableRemoteStreaming="true" | |
--> | |
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000"/> | |
<!-- HTTP Caching | |
Set HTTP caching related parameters (for proxy caches and clients). | |
The options below instruct Solr not to output any HTTP Caching | |
related headers | |
--> | |
<httpCaching never304="true"/> | |
<!-- If you include a <cacheControl> directive, it will be used to | |
generate a Cache-Control header (as well as an Expires header | |
if the value contains "max-age=") | |
By default, no Cache-Control header is generated. | |
You can use the <cacheControl> option even if you have set | |
never304="true" | |
--> | |
<!-- | |
<httpCaching never304="true" > | |
<cacheControl>max-age=30, public</cacheControl> | |
</httpCaching> | |
--> | |
<!-- To enable Solr to respond with automatically generated HTTP | |
Caching headers, and to response to Cache Validation requests | |
correctly, set the value of never304="false" | |
This will cause Solr to generate Last-Modified and ETag | |
headers based on the properties of the Index. | |
The following options can also be specified to affect the | |
values of these headers... | |
lastModFrom - the default value is "openTime" which means the | |
Last-Modified value (and validation against If-Modified-Since | |
requests) will all be relative to when the current Searcher | |
was opened. You can change it to lastModFrom="dirLastMod" if | |
you want the value to exactly correspond to when the physical | |
index was last modified. | |
etagSeed="..." is an option you can change to force the ETag | |
header (and validation against If-None-Match requests) to be | |
different even if the index has not changed (ie: when making | |
significant changes to your config file) | |
(lastModifiedFrom and etagSeed are both ignored if you use | |
the never304="true" option) | |
--> | |
<!-- | |
<httpCaching lastModifiedFrom="openTime" | |
etagSeed="Solr"> | |
<cacheControl>max-age=30, public</cacheControl> | |
</httpCaching> | |
--> | |
</requestDispatcher> | |
<!-- Request Handlers | |
http://wiki.apache.org/solr/SolrRequestHandler | |
incoming queries will be dispatched to the correct handler | |
based on the path or the qt (query type) param. | |
Names starting with a '/' are accessed with the a path equal to | |
the registered name. Names without a leading '/' are accessed | |
with: http://host/app/[core/]select?qt=name | |
If a /select request is processed with out a qt param | |
specified, the requestHandler that declares default="true" will | |
be used. | |
If a Request Handler is declared with startup="lazy", then it will | |
not be initialized until the first request that uses it. | |
--> | |
<!-- SearchHandler | |
http://wiki.apache.org/solr/SearchHandler | |
For processing Search Queries, the primary Request Handler | |
provided with Solr is "SearchHandler" It delegates to a sequent | |
of SearchComponents (see below) and supports distributed | |
queries across multiple shards | |
--> | |
<requestHandler class="solr.SearchHandler" default="true" name="search"> | |
<!-- default values for query parameters can be specified, these | |
will be overridden by parameters in the request | |
--> | |
<lst name="defaults"> | |
<int name="rows">10</int> | |
</lst> | |
<!-- In addition to defaults, "appends" params can be specified | |
to identify values which should be appended to the list of | |
multi-val params from the query (or the existing "defaults"). | |
--> | |
<!-- In this example, the param "fq=instock:true" would be appended to | |
any query time fq params the user may specify, as a mechanism for | |
partitioning the index, independent of any user selected filtering | |
that may also be desired (perhaps as a result of faceted searching). | |
NOTE: there is *absolutely* nothing a client can do to prevent these | |
"appends" values from being used, so don't use this mechanism | |
unless you are sure you always want it. | |
--> | |
<!-- | |
<lst name="appends"> | |
<str name="fq">inStock:true</str> | |
</lst> | |
--> | |
<!-- "invariants" are a way of letting the Solr maintainer lock down | |
the options available to Solr clients. Any params values | |
specified here are used regardless of what values may be specified | |
in either the query, the "defaults", or the "appends" params. | |
In this example, the facet.field and facet.query params would | |
be fixed, limiting the facets clients can use. Faceting is | |
not turned on by default - but if the client does specify | |
facet=true in the request, these are the only facets they | |
will be able to see counts for; regardless of what other | |
facet.field or facet.query params they may specify. | |
NOTE: there is *absolutely* nothing a client can do to prevent these | |
"invariants" values from being used, so don't use this mechanism | |
unless you are sure you always want it. | |
--> | |
<!-- | |
<lst name="invariants"> | |
<str name="facet.field">cat</str> | |
<str name="facet.field">manu_exact</str> | |
<str name="facet.query">price:[* TO 500]</str> | |
<str name="facet.query">price:[500 TO *]</str> | |
</lst> | |
--> | |
<!-- If the default list of SearchComponents is not desired, that | |
list can either be overridden completely, or components can be | |
prepended or appended to the default list. (see below) | |
--> | |
<!-- | |
<arr name="components"> | |
<str>nameOfCustomComponent1</str> | |
<str>nameOfCustomComponent2</str> | |
</arr> | |
--> | |
</requestHandler> | |
<!-- SearchHandler for CQL Solr queries: | |
this handler doesn't support any additional components, only default parameters | |
--> | |
<requestHandler class="com.datastax.bdp.search.solr.handler.component.CqlSearchHandler" name="solr_query"> | |
<lst name="defaults"> | |
<int name="rows">10</int> | |
</lst> | |
</requestHandler> | |
<!-- XML Update Request Handler. | |
http://wiki.apache.org/solr/UpdateXmlMessages | |
The canonical Request Handler for Modifying the Index through | |
commands specified using XML. | |
Note: Since solr1.1 requestHandlers requires a valid content | |
type header if posted in the body. For example, curl now | |
requires: -H 'Content-type:text/xml; charset=utf-8' | |
--> | |
<requestHandler class="solr.UpdateRequestHandler" name="/update"> | |
<!-- See below for information on defining | |
updateRequestProcessorChains that can be used by name | |
on each Update Request | |
--> | |
<!-- | |
<lst name="defaults"> | |
<str name="update.chain">dedupe</str> | |
</lst> | |
--> | |
</requestHandler> | |
<!-- CSV Update Request Handler | |
http://wiki.apache.org/solr/UpdateCSV | |
--> | |
<requestHandler class="solr.CSVRequestHandler" name="/update/csv" startup="lazy"/> | |
<!-- JSON Update Request Handler | |
http://wiki.apache.org/solr/UpdateJSON | |
--> | |
<requestHandler class="solr.JsonUpdateRequestHandler" name="/update/json" startup="lazy"/> | |
<!-- Solr Cell Update Request Handler | |
http://wiki.apache.org/solr/ExtractingRequestHandler | |
<requestHandler name="/update/extract" | |
startup="lazy" | |
class="solr.extraction.ExtractingRequestHandler" > | |
<lst name="defaults"> | |
<str name="fmap.content">text</str> | |
<str name="lowernames">true</str> | |
<str name="uprefix">ignored_</str> | |
<str name="captureAttr">true</str> | |
<str name="fmap.a">links</str> | |
<str name="fmap.div">ignored_</str> | |
</lst> | |
</requestHandler> | |
--> | |
<!-- Field Analysis Request Handler | |
RequestHandler that provides much the same functionality as | |
analysis.jsp. Provides the ability to specify multiple field | |
types and field names in the same request and outputs | |
index-time and query-time analysis for each of them. | |
Request parameters are: | |
analysis.fieldname - field name whose analyzers are to be used | |
analysis.fieldtype - field type whose analyzers are to be used | |
analysis.fieldvalue - text for index-time analysis | |
q (or analysis.q) - text for query time analysis | |
analysis.showmatch (true|false) - When set to true and when | |
query analysis is performed, the produced tokens of the | |
field value analysis will be marked as "matched" for every | |
token that is produces by the query analysis | |
--> | |
<requestHandler class="solr.FieldAnalysisRequestHandler" name="/analysis/field" startup="lazy"/> | |
<!-- Document Analysis Handler | |
http://wiki.apache.org/solr/AnalysisRequestHandler | |
An analysis handler that provides a breakdown of the analysis | |
process of provided docuemnts. This handler expects a (single) | |
content stream with the following format: | |
<docs> | |
<doc> | |
<field name="id">1</field> | |
<field name="name">The Name</field> | |
<field name="text">The Text Value</field> | |
</doc> | |
<doc>...</doc> | |
<doc>...</doc> | |
... | |
</docs> | |
Note: Each document must contain a field which serves as the | |
unique key. This key is used in the returned response to associate | |
an analysis breakdown to the analyzed document. | |
Like the FieldAnalysisRequestHandler, this handler also supports | |
query analysis by sending either an "analysis.query" or "q" | |
request parameter that holds the query text to be analyzed. It | |
also supports the "analysis.showmatch" parameter which when set to | |
true, all field tokens that match the query tokens will be marked | |
as a "match". | |
--> | |
<requestHandler class="solr.DocumentAnalysisRequestHandler" name="/analysis/document" startup="lazy"/> | |
<!-- Admin Handlers | |
Admin Handlers - This will register all the standard admin | |
RequestHandlers. | |
--> | |
<requestHandler class="solr.admin.AdminHandlers" name="/admin/"/> | |
<!-- This single handler is equivalent to the following... --> | |
<!-- | |
<requestHandler name="/admin/luke" class="solr.admin.LukeRequestHandler" /> | |
<requestHandler name="/admin/system" class="solr.admin.SystemInfoHandler" /> | |
<requestHandler name="/admin/plugins" class="solr.admin.PluginInfoHandler" /> | |
<requestHandler name="/admin/threads" class="solr.admin.ThreadDumpHandler" /> | |
<requestHandler name="/admin/properties" class="solr.admin.PropertiesRequestHandler" /> | |
<requestHandler name="/admin/file" class="solr.admin.ShowFileRequestHandler" > | |
--> | |
<!-- If you wish to hide files under ${solr.home}/conf, explicitly | |
register the ShowFileRequestHandler using: | |
--> | |
<!-- | |
<requestHandler name="/admin/file" | |
class="solr.admin.ShowFileRequestHandler" > | |
<lst name="invariants"> | |
<str name="hidden">synonyms.txt</str> | |
<str name="hidden">anotherfile.txt</str> | |
</lst> | |
</requestHandler> | |
--> | |
<!-- ping/healthcheck --> | |
<requestHandler class="solr.PingRequestHandler" name="/admin/ping"> | |
<lst name="invariants"> | |
<str name="qt">search</str> | |
<str name="q">solrpingquery</str> | |
</lst> | |
<lst name="defaults"> | |
<str name="echoParams">all</str> | |
</lst> | |
</requestHandler> | |
<!-- Echo the request contents back to the client --> | |
<requestHandler class="solr.DumpRequestHandler" name="/debug/dump"> | |
<lst name="defaults"> | |
<str name="echoParams">explicit</str> | |
<str name="echoHandler">true</str> | |
</lst> | |
</requestHandler> | |
<!-- Search Components | |
Search components are registered to SolrCore and used by | |
instances of SearchHandler (which can access them by name) | |
By default, the following components are available: | |
<searchComponent name="query" class="solr.QueryComponent" /> | |
<searchComponent name="facet" class="solr.FacetComponent" /> | |
<searchComponent name="mlt" class="solr.MoreLikeThisComponent" /> | |
<searchComponent name="highlight" class="solr.HighlightComponent" /> | |
<searchComponent name="stats" class="solr.StatsComponent" /> | |
<searchComponent name="debug" class="solr.DebugComponent" /> | |
Default configuration in a requestHandler would look like: | |
<arr name="components"> | |
<str>query</str> | |
<str>facet</str> | |
<str>mlt</str> | |
<str>highlight</str> | |
<str>stats</str> | |
<str>debug</str> | |
</arr> | |
If you register a searchComponent to one of the standard names, | |
that will be used instead of the default. | |
To insert components before or after the 'standard' components, use: | |
<arr name="first-components"> | |
<str>myFirstComponentName</str> | |
</arr> | |
<arr name="last-components"> | |
<str>myLastComponentName</str> | |
</arr> | |
NOTE: The component registered with the name "debug" will | |
always be executed after the "last-components" | |
--> | |
<!-- Spell Check | |
The spell check component can return a list of alternative spelling | |
suggestions. | |
http://wiki.apache.org/solr/SpellCheckComponent | |
<searchComponent name="spellcheck" class="solr.SpellCheckComponent"> | |
<lst name="spellchecker"> | |
<str name="name">default</str> | |
<str name="field">name</str> | |
<str name="classname">solr.DirectSolrSpellChecker</str> | |
<str name="distanceMeasure">internal</str> | |
<float name="accuracy">0.5</float> | |
<int name="maxEdits">2</int> | |
<int name="minPrefix">1</int> | |
<int name="maxInspections">5</int> | |
<int name="minQueryLength">4</int> | |
<float name="maxQueryFrequency">0.01</float> | |
<float name="thresholdTokenFrequency">.01</float> | |
</lst> | |
</searchComponent> | |
--> | |
<fieldInputTransformer name="dse" class="hessian.DsefsFieldInputTransformer"> | |
</fieldInputTransformer> | |
<fieldOutputTransformer name="dse" class="hessian.DsefsFieldOutputTransformer"> | |
</fieldOutputTransformer> | |
<!-- A request handler for demonstrating the spellcheck component. | |
NOTE: This is purely as an example. The whole purpose of the | |
SpellCheckComponent is to hook it into the request handler that | |
handles your normal user queries so that a separate request is | |
not needed to get suggestions. | |
IN OTHER WORDS, THERE IS REALLY GOOD CHANCE THE SETUP BELOW IS | |
NOT WHAT YOU WANT FOR YOUR PRODUCTION SYSTEM! | |
See http://wiki.apache.org/solr/SpellCheckComponent for details | |
on the request parameters. | |
<requestHandler name="/spell" class="solr.SearchHandler" startup="lazy"> | |
<lst name="defaults"> | |
<str name="spellcheck.onlyMorePopular">false</str> | |
<str name="spellcheck.extendedResults">false</str> | |
<str name="spellcheck.count">1</str> | |
</lst> | |
<arr name="last-components"> | |
<str>spellcheck</str> | |
</arr> | |
</requestHandler> | |
--> | |
<!-- Term Vector Component | |
http://wiki.apache.org/solr/TermVectorComponent | |
<searchComponent name="tvComponent" class="solr.TermVectorComponent"/> | |
--> | |
<!-- A request handler for demonstrating the term vector component | |
This is purely as an example. | |
In reality you will likely want to add the component to your | |
already specified request handlers. | |
<requestHandler name="tvrh" class="solr.SearchHandler" startup="lazy"> | |
<lst name="defaults"> | |
<bool name="tv">true</bool> | |
</lst> | |
<arr name="last-components"> | |
<str>tvComponent</str> | |
</arr> | |
</requestHandler> | |
--> | |
<!-- Terms Component | |
http://wiki.apache.org/solr/TermsComponent | |
A component to return terms and document frequency of those | |
terms | |
<searchComponent name="terms" class="solr.TermsComponent"/> | |
--> | |
<!-- A request handler for demonstrating the terms component | |
<requestHandler name="/terms" class="solr.SearchHandler" startup="lazy"> | |
<lst name="defaults"> | |
<bool name="terms">true</bool> | |
</lst> | |
<arr name="components"> | |
<str>terms</str> | |
</arr> | |
</requestHandler> | |
--> | |
<!-- Query Elevation Component | |
http://wiki.apache.org/solr/QueryElevationComponent | |
a search component that enables you to configure the top | |
results for a given query regardless of the normal lucene | |
scoring. | |
--> | |
<!-- | |
<searchComponent name="elevator" class="solr.QueryElevationComponent" > | |
<str name="queryFieldType">string</str> | |
<str name="config-file">elevate.xml</str> | |
</searchComponent> | |
--> | |
<!-- A request handler for demonstrating the elevator component | |
<requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy"> | |
<lst name="defaults"> | |
<str name="echoParams">explicit</str> | |
</lst> | |
<arr name="last-components"> | |
<str>elevator</str> | |
</arr> | |
</requestHandler> | |
--> | |
<!-- Highlighting Component | |
http://wiki.apache.org/solr/HighlightingParameters | |
<searchComponent class="solr.HighlightComponent" name="highlight"> | |
<highlighting> | |
<fragmenter name="gap" | |
default="true" | |
class="solr.highlight.GapFragmenter"> | |
<lst name="defaults"> | |
<int name="hl.fragsize">100</int> | |
</lst> | |
</fragmenter> | |
<fragmenter name="regex" | |
class="solr.highlight.RegexFragmenter"> | |
<lst name="defaults"> | |
<int name="hl.fragsize">70</int> | |
<float name="hl.regex.slop">0.5</float> | |
<str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str> | |
</lst> | |
</fragmenter> | |
<formatter name="html" | |
default="true" | |
class="solr.highlight.HtmlFormatter"> | |
<lst name="defaults"> | |
<str name="hl.simple.pre"><![CDATA[<em>]]></str> | |
<str name="hl.simple.post"><![CDATA[</em>]]></str> | |
</lst> | |
</formatter> | |
<encoder name="html" | |
class="solr.highlight.HtmlEncoder" /> | |
<fragListBuilder name="simple" | |
default="true" | |
class="solr.highlight.SimpleFragListBuilder"/> | |
<fragListBuilder name="single" | |
class="solr.highlight.SingleFragListBuilder"/> | |
<fragmentsBuilder name="default" | |
default="true" | |
class="solr.highlight.ScoreOrderFragmentsBuilder"> | |
</fragmentsBuilder> | |
<fragmentsBuilder name="colored" | |
class="solr.highlight.ScoreOrderFragmentsBuilder"> | |
<lst name="defaults"> | |
<str name="hl.tag.pre"><![CDATA[ | |
<b style="background:yellow">,<b style="background:lawgreen">, | |
<b style="background:aquamarine">,<b style="background:magenta">, | |
<b style="background:palegreen">,<b style="background:coral">, | |
<b style="background:wheat">,<b style="background:khaki">, | |
<b style="background:lime">,<b style="background:deepskyblue">]]></str> | |
<str name="hl.tag.post"><![CDATA[</b>]]></str> | |
</lst> | |
</fragmentsBuilder> | |
<boundaryScanner name="default" | |
default="true" | |
class="solr.highlight.SimpleBoundaryScanner"> | |
<lst name="defaults"> | |
<str name="hl.bs.maxScan">10</str> | |
<str name="hl.bs.chars">.,!? 	 </str> | |
</lst> | |
</boundaryScanner> | |
<boundaryScanner name="breakIterator" | |
class="solr.highlight.BreakIteratorBoundaryScanner"> | |
<lst name="defaults"> | |
<str name="hl.bs.type">WORD</str> | |
<str name="hl.bs.language">en</str> | |
<str name="hl.bs.country">US</str> | |
</lst> | |
</boundaryScanner> | |
</highlighting> | |
</searchComponent> | |
--> | |
<!-- Response Writers | |
http://wiki.apache.org/solr/QueryResponseWriter | |
Request responses will be written using the writer specified by | |
the 'wt' request parameter matching the name of a registered | |
writer. | |
The "default" writer is the default and will be used if 'wt' is | |
not specified in the request. | |
--> | |
<!-- The following response writers are implicitly configured unless | |
overridden... | |
--> | |
<!-- | |
<queryResponseWriter name="xml" | |
default="true" | |
class="solr.XMLResponseWriter" /> | |
<queryResponseWriter name="json" class="solr.JSONResponseWriter"/> | |
<queryResponseWriter name="python" class="solr.PythonResponseWriter"/> | |
<queryResponseWriter name="ruby" class="solr.RubyResponseWriter"/> | |
<queryResponseWriter name="php" class="solr.PHPResponseWriter"/> | |
<queryResponseWriter name="phps" class="solr.PHPSerializedResponseWriter"/> | |
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter"/> | |
<queryResponseWriter name="csv" class="solr.CSVResponseWriter"/> | |
--> | |
<!-- | |
Custom response writers can be declared as needed... | |
--> | |
<!-- | |
<queryResponseWriter name="custom" class="com.example.MyResponseWriter"/> | |
--> | |
<!-- XSLT response writer transforms the XML output by any xslt file found | |
in Solr's conf/xslt directory. Changes to xslt files are checked for | |
every xsltCacheLifetimeSeconds. | |
<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter"> | |
<int name="xsltCacheLifetimeSeconds">5</int> | |
</queryResponseWriter> | |
--> | |
<!-- Query Parsers | |
http://wiki.apache.org/solr/SolrQuerySyntax | |
Multiple QParserPlugins can be registered by name, and then | |
used in either the "defType" param for the QueryComponent (used | |
by SearchHandler) or in LocalParams | |
--> | |
<!-- example of registering a query parser --> | |
<!-- | |
<queryParser name="myparser" class="com.mycompany.MyQParserPlugin"/> | |
--> | |
<!-- Function Parsers | |
http://wiki.apache.org/solr/FunctionQuery | |
Multiple ValueSourceParsers can be registered by name, and then | |
used as function names when using the "func" QParser. | |
--> | |
<!-- example of registering a custom function parser --> | |
<!-- | |
<valueSourceParser name="myfunc" | |
class="com.mycompany.MyValueSourceParser" /> | |
--> | |
<!-- Legacy config for the admin interface --> | |
<admin> | |
<defaultQuery>*:*</defaultQuery> | |
<!-- configure a healthcheck file for servers behind a | |
loadbalancer | |
--> | |
<!-- | |
<healthcheck type="file">server-enabled</healthcheck> | |
--> | |
</admin> | |
</config> | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package hessian; | |
import java.io.IOException; | |
import java.io.PrintWriter; | |
import java.net.URI; | |
import org.apache.commons.codec.binary.Hex; | |
import org.apache.commons.lang.StringUtils; | |
import org.apache.lucene.document.Document; | |
import org.apache.solr.core.SolrCore; | |
import org.apache.solr.schema.SchemaField; | |
import com.datastax.bdp.search.solr.FieldInputTransformer; | |
import org.apache.solr.schema.IndexSchema; | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.fs.FSDataInputStream; | |
import org.apache.hadoop.fs.FileSystem; | |
import org.apache.hadoop.fs.Path; | |
public class DsefsFieldInputTransformer extends FieldInputTransformer | |
{ | |
@Override | |
public boolean evaluate(String field) | |
{ | |
return field.equals("link"); | |
} | |
@Override | |
public void addFieldToDocument(SolrCore core, | |
IndexSchema schema, | |
String key, | |
Document doc, | |
SchemaField fieldInfo, | |
String fieldValue, | |
//float boost, | |
DocumentHelper helper) | |
throws IOException | |
{ | |
try | |
{ | |
URI uri = URI.create(fieldValue); | |
Configuration conf = new Configuration(); | |
FileSystem file = FileSystem.get(uri, conf); | |
FSDataInputStream in = file.open(new Path(uri)); | |
byte buf[] = new byte[1024000]; | |
int numRead = in.read(buf); | |
String bufString = new String(buf, 0, numRead, "UTF-8"); | |
SchemaField dsefs_contents = core.getLatestSchema().getFieldOrNull("dsefs_contents"); | |
helper.addFieldToDocument(core, core.getLatestSchema(), key, doc, dsefs_contents, bufString); | |
} | |
catch (Exception ex) | |
{ | |
throw new RuntimeException(ex); | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package hessian; | |
import java.io.IOException; | |
import java.io.PrintWriter; | |
import java.net.URI; | |
import org.apache.commons.lang.StringUtils; | |
import org.apache.lucene.index.FieldInfo; | |
import org.apache.lucene.index.StoredFieldVisitor; | |
import com.datastax.bdp.search.solr.FieldOutputTransformer; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.fs.FSDataInputStream; | |
import org.apache.hadoop.fs.FileSystem; | |
import org.apache.hadoop.fs.Path; | |
public class DsefsFieldOutputTransformer extends FieldOutputTransformer | |
{ | |
private static final Logger LOGGER = LoggerFactory.getLogger(DsefsFieldOutputTransformer.class); | |
@Override | |
public void stringField(FieldInfo fieldInfo, String value, | |
StoredFieldVisitor visitor, DocumentHelper helper) | |
throws IOException | |
{ | |
if (fieldInfo.name.equalsIgnoreCase("link")) { | |
URI uri = URI.create(value); | |
Configuration conf = new Configuration(); | |
FileSystem file = FileSystem.get(uri, conf); | |
FSDataInputStream in = file.open(new Path(uri)); | |
byte buf[] = new byte[1024000]; | |
int numRead = in.read(buf); | |
String bufString = new String(buf, 0, numRead, "UTF-8"); | |
FieldInfo dsefs_contents_fi = helper.getFieldInfo("dsefs_contents"); | |
visitor.stringField(dsefs_contents_fi, bufString); | |
} | |
visitor.stringField(fieldInfo, value); | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
cqlsh -e "SELECT * FROM test.dtest" | |
cqlsh -e "SELECT * FROM test.dtest WHERE solr_query = '{\"q\":\"dsefs_contents: test\"}'" | |
cqlsh -e "SELECT * FROM test.dtest WHERE solr_query = '{\"q\":\"dsefs_contents: Hello\"}'" | |
cqlsh -e "SELECT * FROM test.dtest WHERE solr_query = '{\"q\":\"dsefs_contents: Thank\"}'" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment