search-index script that simulates searches while indexing

data.json

{
  "id": "page/Z2l0aHViLmNvbS9ubHAtY29tcHJvbWlzZS9jb21wcm9taXNl",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified when",
  "url": "github.com/nlp-compromise/compromise",
  "visits": [
    "1504836098717",
    "1505134022188"
  ],
  "latest": "1505134022188",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9Xb3JsZEJyYWluL1dlYk1lbWV4L3B1bGwvbmV3L21hc3Rlcg%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Unwatch Notifications watching notified w",
  "url": "github.com/WorldBrain/WebMemex/pull/new/master",
  "visits": [
    "1498448586231",
    "1499218625365",
    "1499219629872",
    "1505101643843",
    "1505182505378"
  ],
  "latest": "1505182505378",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images",
  "visits": [
    "1505127870156",
    "1505127883028",
    "1505127898544",
    "1505128033557"
  ],
  "latest": "1505128033557",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0L4ltYWdlcw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images",
  "visits": [
    "1505127870156",
    "1505127883028",
    "1505127898544",
    "1505128033557"
  ],
  "latest": "1505128033557",
  "bookmarks": []
}{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWlbYWdlcw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images",
  "visits": [
    "1505127870156",
    "1505127883028",
    "1505127898544",
    "1505128033557"
  ],
  "latest": "1505128033557",
  "bookmarks": []
}{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0vWltYWdlcw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images",
  "visits": [
    "1505127870156",
    "1505127883028",
    "1505127898544",
    "1505128033557"
  ],
  "latest": "1505128033557",
  "bookmarks": []
}{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjx3JlYWN0LWltYWdlcw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images",
  "visits": [
    "1505127870156",
    "1505127883028",
    "1505127898544",
    "1505128033557"
  ],
  "latest": "1505128033557",
  "bookmarks": []
}{
  "id": "page/Z2l0aHViLmNvbS9qb3zzbWFjL3JlYWN0LWltYWdlcw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images",
  "visits": [
    "1505127870156",
    "1505127883028",
    "1505127898544",
    "1505128033557"
  ],
  "latest": "1505128033557",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNsbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXMvMTk%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues/19",
  "visits": [
    "1505127894708",
    "1505127895522"
  ],
  "latest": "1505127895522",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues",
  "visits": [
    "1505127883053",
    "1505127884131",
    "1505127897608"
  ],
  "latest": "1505127892342",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%2FcT1pczppc3N1ZSB5b3V0dWJlICZ1dGY4PeKckw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20youtube%20",
  "visits": [
    "1505127891824",
    "1505127892342"
  ],
  "latest": "1505127892342",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%2FcT1pczppc3N1ZSB2aWRlbyZ1dGY4PeKckw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20video",
  "visits": [
    "1505127887114",
    "1505127887641",
    "1505127893642",
    "1505127897012"
  ],
  "latest": "1505127897012",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9yZWFjdGpzL3JlYWN0LXRyYW5zaXRpb24tZ3JvdXA%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/reactjs/react-transition-group",
  "visits": [
    "1504158672857",
    "1505130270443",
    "1505133013351"
  ],
  "latest": "1505133013351",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9cc3N1ZXM%2FcT1pczppc3N1ZSB2aWRlbyZ1dGY4PeKckw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20video",
  "visits": [
    "1505127887114",
    "1505127887641",
    "1505127893642",
    "1505127897011"
  ],
  "latest": "1505127897011",
  "bookmarks": []
}

index.js

const fs = require('fs')

// Constants
const searchIntervalMS = 50  // # ms between each search
const searchTerm = '*'       // What to search for


const getQuery = term => ({
  "query": [
    {
      "AND": {
        "content": [
          term,
        ],
        "bookmarks": [],
        "visits": [],
        "url": []
      }
    }
  ],
  "pageSize": 10,
})

// Promisifies search-index.search, resolving to # of results, rejecting any error
const initSearch = index => query => {
  const results = []

  return new Promise((resolve, reject) =>
    index.search(query)
      .on('data', datum => results.push(datum))
      .on('error', reject)
      .on('finish', () => resolve(results.length)))
}

// Main
function buildIndex(err, index) {
  if (err) {
    throw err
  }

  const search = initSearch(index)

  // Start searching every N ms, logging output/errors to stdout/stderr
  const intervalID = setInterval(() =>
    search(getQuery(searchTerm))
      .then(console.log)
      .catch(console.error)
  , searchIntervalMS)

  // Start indexing test data
  console.time('indexing time')
  fs.createReadStream('data.json')
    .pipe(index.feed())
    .on('finish', () => {
      // Stop re-running search now that indexing is complete
      clearInterval(intervalID)
      console.timeEnd('indexing time')
    })
}

// Entrypoint
require('search-index')(require('./indexopts'), buildIndex)

indexopts.js

module.exports = {
  batchSize: 500,
  appendOnly: false,
  indexPath: 'test',
  logLevel: 'warn',
  preserveCase: false,
  compositeField: false,
  // separator: /[|' .,\-|(\n)]+/,
  nGramLength: 1,
  fieldOptions: {
    visits: {
      fieldedSearch: true,
    },
    bookmarks: {
      fieldedSearch: true,
    },
    latest: {
      sortable: true,
    },
    content: {
      fieldedSearch: true,
      wildcard: true,
    },
    url: {
      weight: 10,
      fieldedSearch: true,
      separator: '/',
    },
  },
}

I think this is now fixed in master. I'm not totally sure what the expected output of the program is, but the error is at least gone.