colection of coreseek/sphinx configuration & reindex script on biliplus.com

0_coreseek_sphinx_index-reindex-collection

shit title placeholder

cs-index.conf

#MySQL数据源配置，详情请查看：http://www.coreseek.cn/products-install/mysql/
#请先将var/test/documents.sql导入数据库，并配置好以下的MySQL用户密码数据库

#源定义
source avinfo
{
    type                    = mysql

    sql_host                 = <deleted>
    sql_user                 = <deleted>
    sql_pass                 = <deleted>
    sql_db                   = <deleted>
    sql_port                 = <deleted>
    sql_query_pre            = SET NAMES utf8

    sql_query                = SELECT /*+ MAX_EXECUTION_TIME(1800000) */ aid, created, tid, mid, video_review, play, title, description, tag FROM cache_av
    sql_attr_timestamp       = created
    sql_attr_uint            = tid
    sql_attr_uint            = mid
    sql_attr_uint            = video_review
    sql_attr_uint            = play
}
index avinfo
{
    source                   = avinfo
    path                     = /data/index/temp/avinfo
    docinfo                  = extern
    mlock                    = 0
    morphology               = none
    min_word_len             = 1
    html_strip               = 0

    charset_dictpath = /usr/local/coreseek/etc/
    charset_type        = zh_cn.utf-8
}

source avinfo_delta
{
    type                    = mysql

    sql_host                 = <deleted>
    sql_user                 = <deleted>
    sql_pass                 = <deleted>
    sql_db                   = <deleted>
    sql_port                 = <deleted>
    sql_query_pre            = SET NAMES utf8

    sql_query_pre            = REPLACE INTO sphinx_last_ts SELECT 1, ts FROM sphinx_last_ts WHERE id=2
    sql_query_pre            = REPLACE INTO sphinx_last_ts SELECT 2, MAX(lastupdatets) FROM cache_av
    sql_query                = SELECT /*+ MAX_EXECUTION_TIME(1800000) */ aid, created, tid, mid, video_review, play, title, description, tag FROM cache_av WHERE lastupdatets > (SELECT ts FROM sphinx_last_ts WHERE id=1)
    sql_attr_timestamp       = created
    sql_attr_uint            = tid
    sql_attr_uint            = mid
    sql_attr_uint            = video_review
    sql_attr_uint            = play
}
index avinfo_delta
{
    source                   = avinfo_delta
    path                     = /data/index/temp/avinfo_delta
    docinfo                  = extern
    mlock                    = 0
    morphology               = none
    min_word_len             = 1
    html_strip               = 0

    charset_dictpath = /usr/local/coreseek/etc/
    charset_type        = zh_cn.utf-8
}

#全局index定义
indexer
{
    mem_limit            = 512M
    write_buffer         = 8M
    max_iops             = 50
}

cs.conf

#MySQL数据源配置，详情请查看：http://www.coreseek.cn/products-install/mysql/
#请先将var/test/documents.sql导入数据库，并配置好以下的MySQL用户密码数据库

#源定义
source avinfo
{
    type                    = mysql

    sql_host                 = <deleted>
    sql_user                 = <deleted>
    sql_pass                 = <deleted>
    sql_db                   = <deleted>
    sql_port                 = <deleted>
    sql_query_pre            = SET NAMES utf8

    sql_query                = SELECT /*+ MAX_EXECUTION_TIME(1800000) */ aid, created, tid, mid, video_review, play, title, description, tag FROM cache_av
    sql_attr_timestamp       = created
    sql_attr_uint            = tid
    sql_attr_uint            = mid
    sql_attr_uint            = video_review
    sql_attr_uint            = play
}
index avinfo
{
    source            = avinfo
    path            = /data/index/avinfo
    #path            = var/data/avinfo
    docinfo            = extern
    mlock            = 0
    morphology        = none
    min_word_len        = 1
    html_strip                = 0

    #中文分词配置，详情请查看：http://www.coreseek.cn/products-install/coreseek_mmseg/
    #charset_dictpath = /usr/local/mmseg3/etc/ #BSD、Linux环境下设置，/符号结尾
    charset_dictpath = /usr/local/coreseek/etc/                             #Windows环境下设置，/符号结尾，最好给出绝对路径，例如：C:/usr/local/coreseek/etc/...
    charset_type        = zh_cn.utf-8
}

source fav
{
    type                    = mysql

    sql_host                 = <deleted>
    sql_user                 = <deleted>
    sql_pass                 = <deleted>
    sql_db                   = <deleted>
    sql_port                 = <deleted>
    sql_query_pre            = SET NAMES utf8

    sql_query                = SELECT aid, created, typeid, mid, video_review, play, title, description, keywords FROM bilibili_all
    sql_attr_timestamp       = created
    sql_attr_uint            = typeid
    sql_attr_uint            = mid
    sql_attr_uint            = video_review
    sql_attr_uint            = play
}
index fav
{
    source            = fav
    path            = /data/index/fav
    #path            = var/data/fav
    docinfo            = extern
    mlock            = 0
    morphology        = none
    min_word_len        = 1
    html_strip                = 0

    #中文分词配置，详情请查看：http://www.coreseek.cn/products-install/coreseek_mmseg/
    #charset_dictpath = /usr/local/mmseg3/etc/ #BSD、Linux环境下设置，/符号结尾
    charset_dictpath = /usr/local/coreseek/etc/                             #Windows环境下设置，/符号结尾，最好给出绝对路径，例如：C:/usr/local/coreseek/etc/...
    charset_type        = zh_cn.utf-8
}

#全局index定义
indexer
{
    mem_limit            = 128M
}

#searchd服务定义
searchd
{
    listen                  =   9312
    read_timeout        = 5
    max_children        = 30
    max_matches            = 1000
    seamless_rotate        = 0
    preopen_indexes        = 0
    unlink_old            = 1
    pid_file = /usr/local/coreseek/var/log/searchd_mysql.pid  #请修改为实际使用的绝对路径，例如：/usr/local/coreseek/var/...
    log = /usr/local/coreseek/var/log/searchd_mysql.log        #请修改为实际使用的绝对路径，例如：/usr/local/coreseek/var/...
    query_log = /usr/local/coreseek/var/log/query_mysql.log #请修改为实际使用的绝对路径，例如：/usr/local/coreseek/var/...
    binlog_path =                                #关闭binlog日志
}

reindex.sh

#!/bin/sh

\cp -f /data/index/avinfo.sp* /data/index/temp/ &
sudo -u mysql /usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/cs-index.conf avinfo_delta
wait
chown -R mysql:mysql /data/index/temp
#sudo -u mysql /usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/cs-index.conf --merge avinfo avinfo_delta --merge-dst-range deleted 0 0
sudo -u mysql /usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/cs-index.conf --merge avinfo avinfo_delta

rm -rf /data/index/temp/avinfo_delta*
touch /data/home/web/task/.searchd_maintaince
killall searchd
mv -f /data/index/temp/* /data/index/
sudo -u mysql /usr/local/coreseek/bin/searchd --config /usr/local/coreseek/etc/cs.conf >/dev/null 2>&1
rm -f /data/home/web/task/.searchd_maintaince

exit

sudo -u mysql /usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/cs-index.conf avinfo

touch /data/home/web/task/.searchd_maintaince
killall searchd
mv -f /data/index/temp/* /data/index/
sudo -u mysql /usr/local/coreseek/bin/searchd --config /usr/local/coreseek/etc/cs.conf >/dev/null 2>&1 &
rm -f /data/home/web/task/.searchd_maintaince