Last active
October 11, 2017 10:20
-
-
Save poltak/7eb8128d9bbc6d784a38a922f8d8560c to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"id": "page/Z2l0aHViLmNvbS9ubHAtY29tcHJvbWlzZS9jb21wcm9taXNl", | |
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified when", | |
"url": "github.com/nlp-compromise/compromise", | |
"visits": [ | |
"1504836098717", | |
"1505134022188" | |
], | |
"bookmarks": [] | |
} | |
{ | |
"id": "page/Z2l0aHViLmNvbS9Xb3JsZEJyYWluL1dlYk1lbWV4L3B1bGwvbmV3L21hc3Rlcg%3D%3D", | |
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Unwatch Notifications watching notified w", | |
"url": "github.com/WorldBrain/WebMemex/pull/new/master", | |
"visits": [ | |
"1498448586231", | |
"1499218625365", | |
"1499219629872", | |
"1505101643843", | |
"1505182505378" | |
], | |
"bookmarks": [] | |
} | |
{ | |
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcw%3D%3D", | |
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe", | |
"url": "github.com/jossmac/react-images", | |
"visits": [ | |
"1505127870156", | |
"1505127883028", | |
"1505127898544", | |
"1505128033557" | |
], | |
"bookmarks": [] | |
} | |
{ | |
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXMvMTk%3D", | |
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe", | |
"url": "github.com/jossmac/react-images/issues/19", | |
"visits": [ | |
"1505127894708", | |
"1505127895522" | |
], | |
"bookmarks": [] | |
} | |
{ | |
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%3D", | |
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe", | |
"url": "github.com/jossmac/react-images/issues", | |
"visits": [ | |
"1505127883053", | |
"1505127884131", | |
"1505127897608" | |
], | |
"bookmarks": [] | |
} | |
{ | |
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%2FcT1pczppc3N1ZSB5b3V0dWJlICZ1dGY4PeKckw%3D%3D", | |
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe", | |
"url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20youtube%20", | |
"visits": [ | |
"1505127891824", | |
"1505127892342" | |
], | |
"bookmarks": [] | |
} | |
{ | |
"id": "page/Z2l0aHViLmNvbS9qb3NzbWFjL3JlYWN0LWltYWdlcy9pc3N1ZXM%2FcT1pczppc3N1ZSB2aWRlbyZ1dGY4PeKckw%3D%3D", | |
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe", | |
"url": "github.com/jossmac/react-images/issues?utf8=%E2%9C%93&q=is%3Aissue%20video", | |
"visits": [ | |
"1505127887114", | |
"1505127887641", | |
"1505127893642", | |
"1505127897012" | |
], | |
"bookmarks": [] | |
} | |
{ | |
"id": "page/Z2l0aHViLmNvbS9yZWFjdGpzL3JlYWN0LXRyYW5zaXRpb24tZ3JvdXA%3D", | |
"content": "Skip content This repository Pull requests Issues Marketplace Explore Import gist organization issue Signed poltak Your profile stars Gists Help Settings Sign Watch Notifications watching notified whe", | |
"url": "github.com/reactjs/react-transition-group", | |
"visits": [ | |
"1504158672857", | |
"1505130270443", | |
"1505133013351" | |
], | |
"bookmarks": [] | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs') | |
function buildIndex(err, index) { | |
if (err) { | |
throw err | |
} | |
fs.createReadStream('data.json') | |
.pipe(index.feed()) | |
.on('finish', () => console.log('indexing done')) | |
} | |
// Entrypoint | |
require('search-index')(require('./indexopts'), buildIndex) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module.exports = { | |
batchSize: 500, | |
appendOnly: true, | |
indexPath: 'test', | |
logLevel: 'warn', | |
preserveCase: false, | |
compositeField: false, | |
separator: /[|' .,\-|(\n)]+/, | |
nGramLength: 1, | |
fieldOptions: { | |
visits: { | |
fieldedSearch: true, | |
}, | |
bookmarks: { | |
fieldedSearch: true, | |
}, | |
content: { | |
fieldedSearch: true, | |
wildcard: true, | |
}, | |
url: { | |
weight: 10, | |
fieldedSearch: true, | |
separator: '/', | |
}, | |
}, | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const getPromisifiedSearch = index => query => { | |
let results = [] | |
// Batches results from stream, resolving when stream finishes | |
return new Promise((resolve, reject) => | |
index.search(query) | |
.on('data', datum => results.push(datum)) | |
.on('error', reject) | |
.on('finish', () => resolve(results))) | |
} | |
const promisifiedCountDocs = index => | |
new Promise((resolve, reject) => | |
index.countDocs((err, count) => err ? reject(err) : resolve(count))) | |
async function testSearch(err, index) { | |
if (err) { | |
throw err | |
} | |
const q = offset => ({ | |
query: { | |
AND: { | |
content: ['*'], | |
visits: [{ | |
gte: '0000000000000', | |
lte: '1505195898400', // Try playing with these values | |
}], | |
}, | |
}, | |
}) | |
try { | |
const totalCount = await promisifiedCountDocs(index) | |
console.log(`# docs indexed: ${totalCount}\n`) | |
console.log('searching...\n') | |
const promisifiedSearch = getPromisifiedSearch(index) | |
let results | |
results = await promisifiedSearch(q(0)) | |
console.log(`# results (offset: 0): ${results.length}\n`) | |
console.log(JSON.stringify(results, null, ' ')) | |
// results = await promisifiedSearch(q(5)) | |
// console.log(`# results (offset: 5): ${results.length}\n`) | |
// results = await promisifiedSearch(q(10)) | |
// console.log(`# results (offset: 10): ${results.length}\n`) | |
// results = await promisifiedSearch(q(15)) | |
// console.log(`# results (offset: 15): ${results.length}\n`) | |
// results = await promisifiedSearch(q(20)) | |
// console.log(`# results (offset: 20): ${results.length}\n`) | |
// results = await promisifiedSearch(q(25)) | |
// console.log(`# results (offset: 25): ${results.length}\n`) | |
// results = await promisifiedSearch(q(30)) | |
// console.log(`# results (offset: 30): ${results.length}\n`) | |
} catch (err) { | |
console.error(err) | |
} | |
} | |
// Entrypoint | |
require('search-index')(require('./indexopts'), testSearch) |
Yes, can confirm that its broken here too- investigating...
I think this is going to require a rewrite of the code that generates scoringCriteria
This should now work in search-index@0.14.0
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Instructions: assuming
search-index
is installed in anode_modules
dir somewhere up the dir tree, run:node index.js
to index the test datanode search.js
to run the test search and log results to stdout (or errors to stderr)