data.json

{
  "id": "page/Z2l0aHViLmNvbS9ubHAtY29tcHJvbWlzZS9jb21wcm9taXNl",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified when",
  "url": "github.com/nlp-compromise/compromise",
  "visits": [
    "1504836098717",
    "1505134022188"
  ],
  "latest": "1505134022188",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9Xb3JsZEJyYWluL1dlYk1lbWV4L3B1bGwvbmV3L21hc3Rlcg%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Unwatch Notifications watching notified w",
  "url": "github.com/WorldBrain/WebMemex/pull/new/master",
  "visits": [
    "1498448586231",
    "1499218625365",
    "1499219629872",
    "1505101643843",
    "1505182505378"
  ],
  "latest": "1505182505378",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images",
  "visits": [
    "1505127870156",
    "1505127883028",
    "1505127898544",
    "1505128033557"
  ],
  "latest": "1505128033557",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXMvMTk%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues/19",
  "visits": [
    "1505127894708",
    "1505127895522"
  ],
  "latest": "1505127895522",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues",
  "visits": [
    "1505127883053",
    "1505127884131",
    "1505127897608"
  ],
  "latest": "1505127892342",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%2FcT1pczppc3N1ZSB5b3V0dWJlICZ1dGY4PeKckw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20youtube%20",
  "visits": [
    "1505127891824",
    "1505127892342"
  ],
  "latest": "1505127892342",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%2FcT1pczppc3N1ZSB2aWRlbyZ1dGY4PeKckw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20video",
  "visits": [
    "1505127887114",
    "1505127887641",
    "1505127893642",
    "1505127897012"
  ],
  "latest": "1505127897012",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9yZWFjdGpzL3JlYWN0LXRyYW5zaXRpb24tZ3JvdXA%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/reactjs/react-transition-group",
  "visits": [
    "1504158672857",
    "1505130270443",
    "1505133013351"
  ],
  "latest": "1505133013351",
  "bookmarks": []
}
{
  "id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9cc3N1ZXM%2FcT1pczppc3N1ZSB2aWRlbyZ1dGY4PeKckw%3D%3D",
  "content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
  "url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20video",
  "visits": [
    "1505127887114",
    "1505127887641",
    "1505127893642",
    "1505127897011"
  ],
  "latest": "1505127897011",
  "bookmarks": []
}

index.js

const fs = require('fs')

function buildIndex(err, index) {
  if (err) {
    throw err
  }

  fs.createReadStream('data.json')
  .pipe(index.feed())
  .on('finish', () => console.log('indexing done'))
}

// Entrypoint
require('search-index')(require('./indexopts'), buildIndex)

indexopts.js

module.exports = {
  batchSize: 500,
  appendOnly: true,
  indexPath: 'test',
  logLevel: 'warn',
  preserveCase: false,
  compositeField: false,
  separator: /[|' .,\-|(\n)]+/,
  nGramLength: 1,
  fieldOptions: {
    visits: {
      fieldedSearch: true,
    },
    bookmarks: {
      fieldedSearch: true,
    },
    latest: {
      sortable: true,
    },
    content: {
      fieldedSearch: true,
      wildcard: true,
    },
    url: {
      weight: 10,
      fieldedSearch: true,
      separator: '/',
    },
  },
}

search.js

const getPromisifiedSearch = index => query => {
  let results = []

  // Batches results from stream, resolving when stream finishes
  return new Promise((resolve, reject) =>
    index.search(query)
      .on('data', datum => results.push(datum))
      .on('error', reject)
      .on('finish', () => resolve(results)))
}

const promisifiedCountDocs = index =>
  new Promise((resolve, reject) =>
    index.countDocs((err, count) => err ? reject(err) : resolve(count)))

const sortByLatestField = {
  sort: { field: 'latest', direction: 'desc' },
}

async function testSearch(err, index) {
  if (err) {
    throw err
  }

  const q = ({ withSort }) => ({
    query: {
      AND: {
        content: ['*'],
      }
    },
    pageSize: 100,
    ...(withSort ? sortByLatestField : {}),
  })

  try {
    const promisifiedSearch = getPromisifiedSearch(index)

    const totalCount = await promisifiedCountDocs(index)
    console.log(`# total docs indexed: ${totalCount}\n`)

    console.log('searching WITHOUT sort...\n')
    const unsortedResults = await promisifiedSearch(q({ withSort: false }))
    console.log('`latest` field from each result in result order:')
    console.log(unsortedResults.map(res => res.document.latest))
    console.log(`# unsorted results: ${unsortedResults.length}\n`)

    console.log('searching with sort...\n')
    const sortedResults = await promisifiedSearch(q({ withSort: true }))
    console.log('`latest` field from each result in result order:')
    console.log(sortedResults.map(res => res.document.latest))
    console.log(`# sorted results: ${sortedResults.length}\n`)
  } catch (err) {
    console.error(err)
  }
}

// Entrypoint
require('search-index')(require('./indexopts'), testSearch)

Test SI setup that leads to same TypeError as reported in search-index issue #371, but only after reindexing some data with the same ID and searching with a query that has a sort param.

Note that this only happens after reindexing with appendOnly set in indexopts.js.

Instructions to reproduce:

1. run node index to do initial indexing of test data
2. run node search to do two wildcard searches; one without sort, one with sort (on latest sortable field). latest timestamp field printed out in results order for each search
3. run node index to reindex the same unchanged test data
4. run node search to repeat the searches - second (sorted) search should now throw the error

You can repeat the above steps with appendOnly set to false and it should not happen.

Have also done a more minimal version of step3, only reindexing a single doc with an existing ID and it happens as well.