jnioche

<fieldtype name="payloads" stored="false" indexed="true" class="solr.TextField" >
      <analyzer>
        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
        <!--
        The DelimitedPayloadTokenFilter can put payloads on tokens... for example,
        a token of "foo|1.4"  would be indexed as "foo" with a payload of 1.4f
        Attributes of the DelimitedPayloadTokenFilterFactory : 
         "delimiter" - a one character delimiter. Default is | (pipe)
  "encoder" - how to encode the following value into a playload
     float -> org.apache.lucene.analysis.payloads.FloatEncoder,

jnioche

package com.digitalpebble.solr;

import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.BooleanClause;

jnioche

package com.digitalpebble.solr;

import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.search.DefaultSimilarity;

public class PayloadSimilarity extends DefaultSimilarity
{
    @Override public float scorePayload(int docId, String fieldName, int start, int end, byte[] payload, int offset, int length)
    {
        if (length > 0) {

jnioche

package com.digitalpebble.solr;

import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QParserPlugin;

public class PLDisMaxQParserPlugin extends QParserPlugin {
    public void init(NamedList args) {

jnioche

<queryParser name="payload" class="com.digitalpebble.solr.PLDisMaxQParserPlugin" />

jnioche

<document id='xxxx'>
<label>category_of_document</label>
<field name='text'>every document has some text</field>
<field name='title'>some even have a title</field>
<field name='description'>or some meaningful description</field>
</document>

jnioche

package org.apache.nutch.protocol.httpclient;

import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

jnioche

package com.digitalpebble.crawl;

import java.util.List;

import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

jnioche

selenium.capabilities:
    goog:chromeOptions:
      args: 
         - "--headless"
         - "--disable-gpu"

jnioche

name: "crawler"

includes:
    - resource: true
      file: "/crawler-default.yaml"
      override: false

    - resource: false
      file: "crawler-conf.yaml"
      override: true