Skip to content

Instantly share code, notes, and snippets.

@poltak
Last active October 11, 2017 10:20
Show Gist options
  • Save poltak/7eb8128d9bbc6d784a38a922f8d8560c to your computer and use it in GitHub Desktop.
Save poltak/7eb8128d9bbc6d784a38a922f8d8560c to your computer and use it in GitHub Desktop.
{
"id": "page/Z2l0aHViLmNvbS9ubHAtY29tcHJvbWlzZS9jb21wcm9taXNl",
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified when",
"url": "github.com/nlp-compromise/compromise",
"visits": [
"1504836098717",
"1505134022188"
],
"bookmarks": []
}
{
"id": "page/Z2l0aHViLmNvbS9Xb3JsZEJyYWluL1dlYk1lbWV4L3B1bGwvbmV3L21hc3Rlcg%3D%3D",
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Unwatch Notifications watching notified w",
"url": "github.com/WorldBrain/WebMemex/pull/new/master",
"visits": [
"1498448586231",
"1499218625365",
"1499219629872",
"1505101643843",
"1505182505378"
],
"bookmarks": []
}
{
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcw%3D%3D",
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
"url": "github.com/jossmac/react-images",
"visits": [
"1505127870156",
"1505127883028",
"1505127898544",
"1505128033557"
],
"bookmarks": []
}
{
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXMvMTk%3D",
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
"url": "github.com/jossmac/react-images/issues/19",
"visits": [
"1505127894708",
"1505127895522"
],
"bookmarks": []
}
{
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%3D",
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
"url": "github.com/jossmac/react-images/issues",
"visits": [
"1505127883053",
"1505127884131",
"1505127897608"
],
"bookmarks": []
}
{
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%2FcT1pczppc3N1ZSB5b3V0dWJlICZ1dGY4PeKckw%3D%3D",
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
"url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20youtube%20",
"visits": [
"1505127891824",
"1505127892342"
],
"bookmarks": []
}
{
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%2FcT1pczppc3N1ZSB2aWRlbyZ1dGY4PeKckw%3D%3D",
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
"url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20video",
"visits": [
"1505127887114",
"1505127887641",
"1505127893642",
"1505127897012"
],
"bookmarks": []
}
{
"id": "page/Z2l0aHViLmNvbS9yZWFjdGpzL3JlYWN0LXRyYW5zaXRpb24tZ3JvdXA%3D",
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe",
"url": "github.com/reactjs/react-transition-group",
"visits": [
"1504158672857",
"1505130270443",
"1505133013351"
],
"bookmarks": []
}
const fs = require('fs')
function buildIndex(err, index) {
if (err) {
throw err
}
fs.createReadStream('data.json')
.pipe(index.feed())
.on('finish', () => console.log('indexing done'))
}
// Entrypoint
require('search-index')(require('./indexopts'), buildIndex)
module.exports = {
batchSize: 500,
appendOnly: true,
indexPath: 'test',
logLevel: 'warn',
preserveCase: false,
compositeField: false,
separator: /[|' .,\-|(\n)]+/,
nGramLength: 1,
fieldOptions: {
visits: {
fieldedSearch: true,
},
bookmarks: {
fieldedSearch: true,
},
content: {
fieldedSearch: true,
wildcard: true,
},
url: {
weight: 10,
fieldedSearch: true,
separator: '/',
},
},
}
const getPromisifiedSearch = index => query => {
let results = []
// Batches results from stream, resolving when stream finishes
return new Promise((resolve, reject) =>
index.search(query)
.on('data', datum => results.push(datum))
.on('error', reject)
.on('finish', () => resolve(results)))
}
const promisifiedCountDocs = index =>
new Promise((resolve, reject) =>
index.countDocs((err, count) => err ? reject(err) : resolve(count)))
async function testSearch(err, index) {
if (err) {
throw err
}
const q = offset => ({
query: {
AND: {
content: ['*'],
visits: [{
gte: '0000000000000',
lte: '1505195898400', // Try playing with these values
}],
},
},
})
try {
const totalCount = await promisifiedCountDocs(index)
console.log(`# docs indexed: ${totalCount}\n`)
console.log('searching...\n')
const promisifiedSearch = getPromisifiedSearch(index)
let results
results = await promisifiedSearch(q(0))
console.log(`# results (offset: 0): ${results.length}\n`)
console.log(JSON.stringify(results, null, ' '))
// results = await promisifiedSearch(q(5))
// console.log(`# results (offset: 5): ${results.length}\n`)
// results = await promisifiedSearch(q(10))
// console.log(`# results (offset: 10): ${results.length}\n`)
// results = await promisifiedSearch(q(15))
// console.log(`# results (offset: 15): ${results.length}\n`)
// results = await promisifiedSearch(q(20))
// console.log(`# results (offset: 20): ${results.length}\n`)
// results = await promisifiedSearch(q(25))
// console.log(`# results (offset: 25): ${results.length}\n`)
// results = await promisifiedSearch(q(30))
// console.log(`# results (offset: 30): ${results.length}\n`)
} catch (err) {
console.error(err)
}
}
// Entrypoint
require('search-index')(require('./indexopts'), testSearch)
@fergiemcdowall
Copy link

Yes, can confirm that its broken here too- investigating...

@fergiemcdowall
Copy link

I think this is going to require a rewrite of the code that generates scoringCriteria

@fergiemcdowall
Copy link

This should now work in search-index@0.14.0

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment