Instantly share code, notes, and snippets.

@Ir1d Ir1d/init.js Secret
Last active Jan 24, 2019

Embed
What would you like to do?
elasticsearch
const fs = require("fs");
let remark = require("remark");
const strip = require("strip-markdown-math");
const math = require("remark-math");
remark = remark()
.use(math)
.use(strip);
const elasticsearch = require("elasticsearch");
var client = new elasticsearch.Client({
host: "localhost:9200",
log: "trace"
});
modified = [
"docs/graph/topo.md",
"docs/graph/basic.md"
];
let ops = [],
yml = "";
function getContent(e) {
try {
file = String(fs.readFileSync("/OI-wiki/" + e));
} catch (e) {
return ["", "", ""];
}
lines = file.split("\n");
let h1reg = /^# .+$/gm,
h2reg = /^## .+$/gm;
others = lines.filter(e => {
return !e.match(h1reg) && !e.match(h2reg);
});
let title = lines[0];
let filename = e.replace("docs/", "");
if (!title.match(h1reg)) {
if (!yml)
yml = String(fs.readFileSync("/OI-wiki/mkdocs.yml")).split(
"\n"
);
yml.forEach(cur => {
if (cur.indexOf(filename) > -1) {
title = cur.split(":")[0].replace(/^\s+- /m, "");
}
});
} else {
title = title.replace("# ", "");
}
h2 = lines.filter(e => {
return e.match(h2reg);
});
remark.process(others.join("\n"), function(err, file) {
if (err) throw err;
// others = String(file);
others = String(file)
.replace('"', "")
.replace("\\n\\n", "\\n");
});
return [title, others, h2];
}
modified.forEach(e => {
ops.push({ index: { _index: "oiwiki", _type: "article", _id: e } });
let [title, article, h2] = getContent(e);
ops.push({
title: title,
content: article,
url: e.replace("docs", "").replace(".md", "/"),
h2: h2
});
});
let removed = [];
removed.forEach(e => {
ops.push({ delete: { _index: "oiwiki", _type: "article", _id: e } });
});
client.bulk({ body: ops, refresh: "true" }, function(err, res) {
if (err) {
console.log("Failed Bulk opoeration", err);
res.statusCode = 504;
res.end("elasticsearch bulk op failed");
return;
}
console.log("Elasticsearch bulk op success");
});
const elasticsearch = require('elasticsearch');
var client = new elasticsearch.Client({
host: 'localhost:9200',
});
const express = require('express');
const app = express();
app.set('port', process.env.PORT || 8000);
app.get('/status', function(req, res) {
if (req.query.TOKEN == TOKEN) {
client.ping({
requestTimeout: 1000,
}, function (error) {
if (error) {
console.error('elasticsearch cluster is down!');
res.end('elasticsearch cluster is down!');
} else {
console.log('All is well');
res.end('All is well');
}
});
}
});
app.get('/', function(req, res) {
if (!req.headers.referer || req.headers.referer.indexOf('oi-wiki.org') < 0) {
res.send([]);
return;
}
// console.log(req.headers);
if (!req.query.s) {
res.send([]);
return;
}
let keyword = req.query.s;
console.log(keyword);
client.search({
index: "oiwiki",
type: "article",
from: 0,
size: 12,
body: {
query: {
dis_max: {
queries: [
{
match: {
title: {
query: keyword,
minimum_should_match: "75%",
boost: 4
}
}
},
{
match: {
h2: {
query: keyword,
minimum_should_match: "75%",
boost: 3
}
}
},
{
match: {
content: {
query: keyword,
minimum_should_match: "75%",
boost: 2
}
}
},
{
match: {
url: {
query: keyword,
minimum_should_match: "75%",
boost: 3
}
}
}
],
tie_breaker: 0.3
}
},
highlight: {
pre_tags: ["<em>"],
post_tags: ["</em>"],
fields: {
title: { number_of_fragments: 1 },
h2: { number_of_fragments: 1 },
content: { number_of_fragments: 1 },
url: { number_of_fragments: 1 }
},
fragment_size: 20,
}
}
})
.then(results => {
results = results.hits.hits;
results = results.map((e) => {
return {
url: e._source.url,
title: e._source.title,
highlight: e.highlight.content
}
});
res.send(results);
})
.catch(err=> {
console.log(err);
res.send([]);
});
});
app.listen(app.get('port'), function() {
console.log('Search server running on port ' + app.get('port'));
});
const elasticsearch = require("elasticsearch");
var client = new elasticsearch.Client({
host: "localhost:9200",
log: "trace"
});
let keyword = "线段";
const response = client.search({
index: "oiwiki",
type: "article",
from: 0,
size: 10,
body: {
query: {
dis_max: {
queries: [
{
match: {
title: {
query: keyword,
minimum_should_match: "75%",
boost: 4
}
}
},
{
match: {
h2: {
query: keyword,
minimum_should_match: "75%",
boost: 3
}
}
},
{
match: {
content: {
query: keyword,
minimum_should_match: "75%",
boost: 2
}
}
},
{
match: {
url: {
query: keyword,
minimum_should_match: "75%",
boost: 3
}
}
}
],
tie_breaker: 0.3
}
},
highlight: {
pre_tags: ["<b>"],
post_tags: ["</b>"],
fields: {
title: {},
content: {}
}
}
}
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment