elasticsearch

init.js

const fs = require("fs");
let remark = require("remark");
const strip = require("strip-markdown-math");
const math = require("remark-math");
remark = remark()
  .use(math)
  .use(strip);

const elasticsearch = require("elasticsearch");
var client = new elasticsearch.Client({
  host: "localhost:9200",
  log: "trace"
});

modified = [
  "docs/graph/topo.md",
  "docs/graph/basic.md"
];

let ops = [],
  yml = "";
function getContent(e) {
  try {
    file = String(fs.readFileSync("/OI-wiki/" + e));
  } catch (e) {
    return ["", "", ""];
  }
  lines = file.split("\n");
  let h1reg = /^# .+$/gm,
    h2reg = /^## .+$/gm;
  others = lines.filter(e => {
    return !e.match(h1reg) && !e.match(h2reg);
  });
  let title = lines[0];
  let filename = e.replace("docs/", "");
  if (!title.match(h1reg)) {
    if (!yml)
      yml = String(fs.readFileSync("/OI-wiki/mkdocs.yml")).split(
        "\n"
      );
    yml.forEach(cur => {
      if (cur.indexOf(filename) > -1) {
        title = cur.split(":")[0].replace(/^\s+- /m, "");
      }
    });
  } else {
    title = title.replace("# ", "");
  }

  h2 = lines.filter(e => {
    return e.match(h2reg);
  });
  remark.process(others.join("\n"), function(err, file) {
    if (err) throw err;
   // others = String(file);
      others = String(file)
      .replace('"', "")
      .replace("\\n\\n", "\\n");
  });
  return [title, others, h2];
}
modified.forEach(e => {
  ops.push({ index: { _index: "oiwiki", _type: "article", _id: e } });
  let [title, article, h2] = getContent(e);
  ops.push({
    title: title,
    content: article,
    url: e.replace("docs", "").replace(".md", "/"),
    h2: h2
  });
});
let removed = [];
removed.forEach(e => {
  ops.push({ delete: { _index: "oiwiki", _type: "article", _id: e } });
});
client.bulk({ body: ops, refresh: "true" }, function(err, res) {
  if (err) {
    console.log("Failed Bulk opoeration", err);
    res.statusCode = 504;
    res.end("elasticsearch bulk op failed");
    return;
  }
  console.log("Elasticsearch bulk op success");
});

main.js

const elasticsearch = require('elasticsearch');
var client = new elasticsearch.Client({
  host: 'localhost:9200',
});
const express = require('express');
const app     = express();

app.set('port', process.env.PORT || 8000);

app.get('/status', function(req, res) {
  if (req.query.TOKEN == TOKEN) {
    client.ping({
      requestTimeout: 1000,
    }, function (error) {
      if (error) {
        console.error('elasticsearch cluster is down!');
        res.end('elasticsearch cluster is down!');
      } else {
        console.log('All is well');
        res.end('All is well');
      }
    });
  }
});

app.get('/', function(req, res) {
  if (!req.headers.referer || req.headers.referer.indexOf('oi-wiki.org') < 0) {
    res.send([]);
    return;
  }
  // console.log(req.headers);
  if (!req.query.s) {
    res.send([]);
    return;
  }
  let keyword = req.query.s;
  console.log(keyword);
  client.search({
    index: "oiwiki",
    type: "article",
    from: 0,
    size: 12,
    body: {
      query: {
        dis_max: {
          queries: [
            {
              match: {
                title: {
                  query: keyword,
                  minimum_should_match: "75%",
                  boost: 4
                }
              }
            },
            {
              match: {
                h2: {
                  query: keyword,
                  minimum_should_match: "75%",
                  boost: 3
                }
              }
            },
            {
              match: {
                content: {
                  query: keyword,
                  minimum_should_match: "75%",
                  boost: 2
                }
              }
            },
            {
              match: {
                url: {
                  query: keyword,
                  minimum_should_match: "75%",
                  boost: 3
                }
              }
            }
          ],
          tie_breaker: 0.3
        }
      },
      highlight: {
        pre_tags: ["<em>"],
        post_tags: ["</em>"],
        fields: {
          title: { number_of_fragments: 1 },
          h2: { number_of_fragments: 1 },
          content: { number_of_fragments: 1 },
          url: { number_of_fragments: 1 }
        },
        fragment_size: 20,
      }
    }
  })
  .then(results => {
    results = results.hits.hits;
    results = results.map((e) => {
      return {
        url: e._source.url,
        title: e._source.title,
        highlight: e.highlight.content
      }
    });
    res.send(results);
  })
  .catch(err=> {
    console.log(err);
    res.send([]);
  });
});

app.listen(app.get('port'), function() {
  console.log('Search server running on port ' + app.get('port'));
});

test.js

const elasticsearch = require("elasticsearch");
var client = new elasticsearch.Client({
  host: "localhost:9200",
  log: "trace"
});
let keyword = "线段";
const response = client.search({
  index: "oiwiki",
  type: "article",
  from: 0,
  size: 10,
  body: {
    query: {
      dis_max: {
        queries: [
          {
            match: {
              title: {
                query: keyword,
                minimum_should_match: "75%",
                boost: 4
              }
            }
          },
          {
            match: {
              h2: {
                query: keyword,
                minimum_should_match: "75%",
                boost: 3
              }
            }
          },
          {
            match: {
              content: {
                query: keyword,
                minimum_should_match: "75%",
                boost: 2
              }
            }
          },
          {
            match: {
              url: {
                query: keyword,
                minimum_should_match: "75%",
                boost: 3
              }
            }
          }
        ],
        tie_breaker: 0.3
      }
    },
    highlight: {
      pre_tags: ["<b>"],
      post_tags: ["</b>"],
      fields: {
        title: {},
        content: {}
      }
    }
  }
});

统计：

curl -H'Content-Type: application/json' -XGET http://localhost:PORT/oiwiki/_count

清空

curl -H'Content-Type: application/json' -XPOST "http://localhost:PORT/oiwiki/_delete_by_query" -d'
{
     "query":{
          "match_all":{}
      }
}'

这个代码有一些 bug，后续改动待我有空再同步过来

放到这里好了 https://github.com/OI-wiki/elasticsearch-config