From 85bfcd74479b2f50120d658e36194163f60a5fb7 Mon Sep 17 00:00:00 2001 From: dami Date: Mon, 21 Jul 2025 16:13:34 +0800 Subject: [PATCH] =?UTF-8?q?manticoresearch=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugins/manticoresearch/class/sphinx_make.py | 486 +++++++ plugins/manticoresearch/class/sphinxapi.py | 1256 +++++++++++++++++ plugins/manticoresearch/conf/sphinx.conf | 28 + plugins/manticoresearch/ico.png | Bin 0 -> 2268 bytes plugins/manticoresearch/index.html | 36 + plugins/manticoresearch/index.py | 483 +++++++ plugins/manticoresearch/info.json | 19 + .../manticoresearch/init.d/sphinx.service.tpl | 12 + plugins/manticoresearch/init.d/sphinx.tpl | 82 ++ plugins/manticoresearch/install.sh | 28 + plugins/manticoresearch/js/sphinx.js | 311 ++++ plugins/manticoresearch/tool_cron.py | 211 +++ plugins/manticoresearch/tpl/discuz.conf | 127 ++ plugins/manticoresearch/tpl/maccms.conf | 95 ++ plugins/manticoresearch/tpl/none.conf | 28 + plugins/manticoresearch/tpl/qbittorrent.conf | 54 + plugins/manticoresearch/tpl/simdht.conf | 59 + plugins/manticoresearch/tpl/simdht_delta.conf | 79 ++ plugins/manticoresearch/tpl/video.conf | 58 + .../manticoresearch/versions/apt/install.sh | 171 +++ .../manticoresearch/versions/yum/install.sh | 171 +++ 21 files changed, 3794 insertions(+) create mode 100644 plugins/manticoresearch/class/sphinx_make.py create mode 100644 plugins/manticoresearch/class/sphinxapi.py create mode 100755 plugins/manticoresearch/conf/sphinx.conf create mode 100644 plugins/manticoresearch/ico.png create mode 100755 plugins/manticoresearch/index.html create mode 100755 plugins/manticoresearch/index.py create mode 100755 plugins/manticoresearch/info.json create mode 100644 plugins/manticoresearch/init.d/sphinx.service.tpl create mode 100644 plugins/manticoresearch/init.d/sphinx.tpl create mode 100755 plugins/manticoresearch/install.sh create mode 100755 plugins/manticoresearch/js/sphinx.js create mode 100644 plugins/manticoresearch/tool_cron.py create mode 100644 plugins/manticoresearch/tpl/discuz.conf create mode 100644 plugins/manticoresearch/tpl/maccms.conf create mode 100755 plugins/manticoresearch/tpl/none.conf create mode 100755 plugins/manticoresearch/tpl/qbittorrent.conf create mode 100755 plugins/manticoresearch/tpl/simdht.conf create mode 100755 plugins/manticoresearch/tpl/simdht_delta.conf create mode 100755 plugins/manticoresearch/tpl/video.conf create mode 100755 plugins/manticoresearch/versions/apt/install.sh create mode 100755 plugins/manticoresearch/versions/yum/install.sh diff --git a/plugins/manticoresearch/class/sphinx_make.py b/plugins/manticoresearch/class/sphinx_make.py new file mode 100644 index 000000000..0ccb1f409 --- /dev/null +++ b/plugins/manticoresearch/class/sphinx_make.py @@ -0,0 +1,486 @@ +# coding:utf-8 + +import sys +import io +import os +import time +import subprocess +import re +import json + + +sys.path.append(os.getcwd() + "/class/core") +import mw + + +def getServerDir(): + return mw.getServerDir() + '/mysql' + +def getPluginDir(): + return mw.getPluginDir() + '/mysql' + +def getConf(): + path = getServerDir() + '/etc/my.cnf' + return path + +def getDbPort(): + file = getConf() + content = mw.readFile(file) + rep = 'port\s*=\s*(.*)' + tmp = re.search(rep, content) + return tmp.groups()[0].strip() + +def getSocketFile(): + file = getConf() + content = mw.readFile(file) + rep = 'socket\s*=\s*(.*)' + tmp = re.search(rep, content) + return tmp.groups()[0].strip() + +def pSqliteDb(dbname='databases'): + file = getServerDir() + '/mysql.db' + name = 'mysql' + + conn = mw.M(dbname).dbPos(getServerDir(), name) + return conn + +def pMysqlDb(): + # pymysql + db = mw.getMyORM() + + db.setPort(getDbPort()) + db.setSocket(getSocketFile()) + # db.setCharset("utf8") + db.setPwd(pSqliteDb('config').where('id=?', (1,)).getField('mysql_root')) + return db + +class sphinxMake(): + + pdb = None + psdb = None + + pkey_name_cache = {} + delta = 'sph_counter' + ver = '' + + + def __init__(self): + self.pdb = pMysqlDb() + + def setDeltaName(self, name): + self.delta = name + return True + + def setVersion(self, ver): + self.ver = ver + + def createSql(self, db): + conf = ''' +CREATE TABLE IF NOT EXISTS `{$DB_NAME}`.`{$TABLE_NAME}` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `table` varchar(200) NOT NULL, + `max_id` bigint(20) unsigned NOT NULL DEFAULT '0', + PRIMARY KEY (`id`), + UNIQUE KEY `table_uniq` (`table`), + KEY `table` (`table`) +) ENGINE=InnoDB AUTO_INCREMENT=1 CHARSET=utf8mb4; +''' + conf = conf.replace("{$TABLE_NAME}", self.delta) + conf = conf.replace("{$DB_NAME}", db) + return conf + + def eqVerField(self, field): + ver = self.ver.replace(".1",'') + if float(ver) >= 3.6: + if field == 'sql_attr_timestamp': + return 'attr_bigint' + + if field == 'sql_attr_bigint': + return 'attr_bigint' + + if field == 'sql_attr_float': + return 'attr_float' + + if field == 'sql_field_string': + return 'field_string' + + if float(ver) >= 3.3: + if field == 'sql_attr_timestamp': + return 'sql_attr_bigint' + + return field + + def pathVerName(self): + ver = self.ver.replace(".1",'') + # if float(ver) >= 3.6: + # return 'datadir' + return 'path' + + def getTablePk(self, db, table): + key = db+'_'+table + if key in self.pkey_name_cache: + return self.pkey_name_cache[key] + + # SHOW INDEX FROM bbs.bbs_ucenter_vars WHERE Key_name = 'PRIMARY' + pkey_sql = "SHOW INDEX FROM {}.{} WHERE Key_name = 'PRIMARY';".format(db,table,); + pkey_data = self.pdb.query(pkey_sql) + + # print(db, table) + # print(pkey_data) + key = '' + if len(pkey_data) == 1: + pkey_name = pkey_data[0]['Column_name'] + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}' and `COLUMN_NAME`='{}';" + sql = sql.format(db,table,pkey_name,) + # print(sql) + fields = self.pdb.query(sql) + + if len(fields) == 1: + # print(fields[0]['DATA_TYPE']) + if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], fields[0]['DATA_TYPE']): + key = pkey_name + return key + + + def getTableFieldStr(self, db, table): + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" + sql = sql.format(db,table,) + fields = self.pdb.query(sql) + + field_str = '' + for x in range(len(fields)): + field_str += '`'+fields[x]['COLUMN_NAME']+'`,' + + field_str = field_str.strip(',') + return field_str + + def makeSphinxHeader(self): + conf = ''' +indexer +{ + mem_limit = 128M +} + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$server_dir}/sphinx/index/searchd.log + query_log = {$server_dir}/sphinx/index/query.log + read_timeout = 5 + max_children = 0 + pid_file = {$server_dir}/sphinx/index/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + #workers = threads # for RT to work + binlog_path = {$server_dir}/sphinx/index/binlog +} + ''' + conf = conf.replace("{$server_dir}", mw.getServerDir()) + return conf + + def makeSphinxDbSourceRangeSql(self, db, table): + pkey_name = self.getTablePk(db,table) + sql = "SELECT min("+pkey_name+"), max("+pkey_name+") FROM "+table + return sql + + def makeSphinxDbSourceQuerySql(self, db, table): + pkey_name = self.getTablePk(db,table) + field_str = self.getTableFieldStr(db,table) + # print(field_str) + if pkey_name == 'id': + sql = "SELECT " + field_str + " FROM " + table + " where id >= $start AND id <= $end" + else: + sql = "SELECT `"+pkey_name+'` as `id`,' + field_str + " FROM " + table + " where "+pkey_name+" >= $start AND "+pkey_name+" <= $end" + return sql + + + def makeSphinxDbSourceDeltaRange(self, db, table): + pkey_name = self.getTablePk(db,table) + conf = "SELECT (SELECT max_id FROM `{$SPH_TABLE}` where `table`='{$TABLE_NAME}') as min, (SELECT max({$PK_NAME}) FROM {$TABLE_NAME}) as max" + conf = conf.replace("{$DB_NAME}", db) + conf = conf.replace("{$TABLE_NAME}", table) + conf = conf.replace("{$SPH_TABLE}", self.delta) + conf = conf.replace("{$PK_NAME}", pkey_name) + return conf + + def makeSphinxDbSourcePost(self, db, table): + pkey_name = self.getTablePk(db,table) + conf = "sql_query_post = UPDATE {$SPH_TABLE} SET max_id=(SELECT MAX({$PK_NAME}) FROM {$TABLE_NAME}) where `table`='{$TABLE_NAME}'" + # conf = "REPLACE INTO {$SPH_TABLE} (`table`,`max_id`) VALUES ('{$TABLE_NAME}',(SELECT MAX({$PK_NAME}) FROM {$TABLE_NAME}))" + conf = conf.replace("{$DB_NAME}", db) + conf = conf.replace("{$TABLE_NAME}", table) + conf = conf.replace("{$SPH_TABLE}", self.delta) + conf = conf.replace("{$PK_NAME}", pkey_name) + return conf + + def makeSphinxDbSourceDelta(self, db, table): + conf = ''' +source {$DB_NAME}_{$TABLE_NAME}_delta:{$DB_NAME}_{$TABLE_NAME} +{ + sql_query_pre = SET NAMES utf8 + sql_query_range = {$DELTA_RANGE} + sql_query = {$DELTA_QUERY} + {$DELTA_UPDATE} + +{$SPH_FIELD} +} + +index {$DB_NAME}_{$TABLE_NAME}_delta:{$DB_NAME}_{$TABLE_NAME} +{ + source = {$DB_NAME}_{$TABLE_NAME}_delta + {$PATH_NAME} = {$server_dir}/sphinx/index/db/{$DB_NAME}.{$TABLE_NAME}/delta + + html_strip = 1 + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F + +{$SPH_FIELD_INDEX} +} +'''; + conf = conf.replace("{$server_dir}", mw.getServerDir()) + conf = conf.replace("{$PATH_NAME}", self.pathVerName()) + + conf = conf.replace("{$DB_NAME}", db) + conf = conf.replace("{$TABLE_NAME}", table) + + delta_range = self.makeSphinxDbSourceDeltaRange(db, table) + conf = conf.replace("{$DELTA_RANGE}", delta_range) + + delta_query = self.makeSphinxDbSourceQuerySql(db, table) + conf = conf.replace("{$DELTA_QUERY}", delta_query) + + delta_update = self.makeSphinxDbSourcePost(db, table) + conf = conf.replace("{$DELTA_UPDATE}", delta_update) + + + sph_field = self.makeSqlToSphinxTable(db, table) + conf = self.makeSphinxDbFieldRepalce(conf, sph_field) + + return conf; + + def makeSphinxDbSource(self, db, table, create_sphinx_table = False): + db_info = pSqliteDb('databases').field('username,password').where('name=?', (db,)).find() + port = getDbPort() + + conf = ''' +source {$DB_NAME}_{$TABLE_NAME} +{ + type = mysql + sql_host = 127.0.0.1 + sql_user = {$DB_USER} + sql_pass = {$DB_PASS} + sql_db = {$DB_NAME} + sql_port = {$DB_PORT} + + sql_query_pre = SET NAMES utf8 + + {$UPDATE} + + sql_query_range = {$DB_RANGE_SQL} + sql_range_step = 1000 + + sql_query = {$DB_QUERY_SQL} + +{$SPH_FIELD} +} + +index {$DB_NAME}_{$TABLE_NAME} +{ + source = {$DB_NAME}_{$TABLE_NAME} + {$PATH_NAME} = {$server_dir}/sphinx/index/db/{$DB_NAME}.{$TABLE_NAME}/index + + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F + +{$SPH_FIELD_INDEX} +} + ''' + conf = conf.replace("{$server_dir}", mw.getServerDir()) + conf = conf.replace("{$PATH_NAME}", self.pathVerName()) + + conf = conf.replace("{$DB_NAME}", db) + conf = conf.replace("{$TABLE_NAME}", table) + conf = conf.replace("{$DB_USER}", db_info['username']) + conf = conf.replace("{$DB_PASS}", db_info['password']) + conf = conf.replace("{$DB_PORT}", port) + + range_sql = self.makeSphinxDbSourceRangeSql(db, table) + conf = conf.replace("{$DB_RANGE_SQL}", range_sql) + + query_sql = self.makeSphinxDbSourceQuerySql(db, table) + conf = conf.replace("{$DB_QUERY_SQL}", query_sql) + + sph_field = self.makeSqlToSphinxTable(db, table) + # conf = conf.replace("{$SPH_FIELD}", sph_field) + + + conf = self.makeSphinxDbFieldRepalce(conf, sph_field) + + if create_sphinx_table: + update = self.makeSphinxDbSourcePost(db, table) + conf = conf.replace("{$UPDATE}", update) + else: + conf = conf.replace("{$UPDATE}", '') + + if create_sphinx_table: + sph_sql = self.createSql(db) + self.pdb.query(sph_sql) + sql_find = "select * from {}.{} where `table`='{}'".format(db,self.delta,table) + find_data = self.pdb.query(sql_find) + if len(find_data) == 0: + insert_sql = "insert into `{}`.`{}`(`table`,`max_id`) values ('{}',{}) ".format(db,self.delta,table,0) + # print(insert_sql) + self.pdb.execute(insert_sql) + conf += self.makeSphinxDbSourceDelta(db,table) + + # print(ver) + # print(conf) + + return conf + + def makeSphinxDbFieldRepalce(self, content, sph_field): + ver = self.ver.replace(".1",'') + ver = float(ver) + if ver >= 3.6: + content = content.replace("{$SPH_FIELD}", '') + content = content.replace("{$SPH_FIELD_INDEX}", '') + else: + content = content.replace("{$SPH_FIELD}", sph_field) + content = content.replace("{$SPH_FIELD_INDEX}", '') + + return content + + + def makeSqlToSphinxDb(self, db, table = [], is_delta = False): + conf = '' + + + for tn in table: + pkey_name = self.getTablePk(db,tn) + if pkey_name == '': + continue + conf += self.makeSphinxDbSource(db, tn,is_delta) + + if len(table) == 0: + tables = self.pdb.query("show tables in "+ db) + for x in range(len(tables)): + key = 'Tables_in_'+db + table_name = tables[x][key] + pkey_name = self.getTablePk(db, table_name, is_delta) + if pkey_name == '': + continue + + if self.makeSqlToSphinxTableIsHaveFulltext(db, table_name): + conf += self.makeSphinxDbSource(db, table_name) + return conf + + def makeSqlToSphinxTableIsHaveFulltext(self, db, table): + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" + sql = sql.format(db,table,) + cols = self.pdb.query(sql) + cols_len = len(cols) + + for x in range(cols_len): + data_type = cols[x]['DATA_TYPE'] + column_name = cols[x]['COLUMN_NAME'] + + if mw.inArray(['varchar'], data_type): + return True + if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type): + return True + return False + + def makeSqlToSphinxTable(self,db,table): + pkey_name = self.getTablePk(db,table) + sql = "select COLUMN_NAME,DATA_TYPE from information_schema.COLUMNS where `TABLE_SCHEMA`='{}' and `TABLE_NAME` = '{}';" + sql = sql.format(db,table,) + cols = self.pdb.query(sql) + cols_len = len(cols) + conf = '' + run_pos = 0 + for x in range(cols_len): + data_type = cols[x]['DATA_TYPE'] + column_name = cols[x]['COLUMN_NAME'] + # print(column_name+":"+data_type) + + # if mw.inArray(['tinyint'], data_type): + # conf += 'sql_attr_bool = '+ column_name + "\n" + + if pkey_name == column_name: + # run_pos += 1 + # conf += '\tsql_attr_bigint = '+column_name+"\n" + continue + + if mw.inArray(['enum'], data_type): + run_pos += 1 + conf += '\t'+self.eqVerField('sql_attr_string')+' = '+ column_name + "\n" + continue + + if mw.inArray(['decimal'], data_type): + run_pos += 1 + conf += '\t'+self.eqVerField('sql_attr_float')+' = '+ column_name + "\n" + continue + + if mw.inArray(['bigint','smallint','tinyint','int','mediumint'], data_type): + run_pos += 1 + conf += '\t'+self.eqVerField('sql_attr_bigint')+' = '+ column_name + "\n" + continue + + + if mw.inArray(['float'], data_type): + run_pos += 1 + conf += '\t'+self.eqVerField('sql_attr_float')+' = '+ column_name + "\n" + continue + + if mw.inArray(['char'], data_type): + conf += '\t'+self.eqVerField('sql_attr_string')+' = '+ column_name + "\n" + continue + + if mw.inArray(['varchar'], data_type): + run_pos += 1 + conf += '\t'+self.eqVerField('sql_field_string')+' = '+ column_name + "\n" + continue + + if mw.inArray(['text','mediumtext','tinytext','longtext'], data_type): + run_pos += 1 + conf += '\t'+self.eqVerField('sql_field_string')+' = '+ column_name + "\n" + continue + + if mw.inArray(['datetime','date'], data_type): + run_pos += 1 + conf += '\t'+self.eqVerField('sql_attr_timestamp')+' = '+ column_name + "\n" + continue + + return conf + + def checkDbName(self, db): + filter_db = ['information_schema','performance_schema','sys','mysql'] + if db in filter_db: + return False + return True + + def makeSqlToSphinx(self, db, tables = [], is_delta = False): + conf = '' + conf += self.makeSphinxHeader() + conf += self.makeSqlToSphinxDb(db, tables, is_delta) + return conf + + def makeSqlToSphinxAll(self): + filter_db = ['information_schema','performance_schema','sys','mysql'] + + dblist = self.pdb.query('show databases') + + conf = '' + conf += self.makeSphinxHeader() + + # conf += makeSqlToSphinxDb(pdb, 'bbs') + for x in range(len(dblist)): + dbname = dblist[x]['Database'] + if mw.inArray(filter_db, dbname): + continue + conf += self.makeSqlToSphinxDb(dbname) + return conf + + diff --git a/plugins/manticoresearch/class/sphinxapi.py b/plugins/manticoresearch/class/sphinxapi.py new file mode 100644 index 000000000..88f5afde4 --- /dev/null +++ b/plugins/manticoresearch/class/sphinxapi.py @@ -0,0 +1,1256 @@ +# +# $Id$ +# +# Python version of Sphinx searchd client (Python API) +# +# Copyright (c) 2006, Mike Osadnik +# Copyright (c) 2006-2016, Andrew Aksyonoff +# Copyright (c) 2008-2016, Sphinx Technologies Inc +# All rights reserved +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library General Public License. You should +# have received a copy of the LGPL license along with this program; if you +# did not, you can find it at http://www.gnu.org/ +# +# WARNING!!! +# +# As of 2015, we strongly recommend to use either SphinxQL or REST APIs +# rather than the native SphinxAPI. +# +# While both the native SphinxAPI protocol and the existing APIs will +# continue to exist, and perhaps should not even break (too much), exposing +# all the new features via multiple different native API implementations +# is too much of a support complication for us. +# +# That said, you're welcome to overtake the maintenance of any given +# official API, and remove this warning ;) +# + +from __future__ import print_function +import sys +import select +import socket +import re +from struct import * + +if sys.version_info > (3,): + long = int + text_type = str +else: + text_type = unicode + +# known searchd commands +SEARCHD_COMMAND_SEARCH = 0 +SEARCHD_COMMAND_EXCERPT = 1 +SEARCHD_COMMAND_UPDATE = 2 +SEARCHD_COMMAND_KEYWORDS = 3 +SEARCHD_COMMAND_PERSIST = 4 +SEARCHD_COMMAND_STATUS = 5 +SEARCHD_COMMAND_FLUSHATTRS = 7 + +# current client-side command implementation versions +VER_COMMAND_SEARCH = 0x120 +VER_COMMAND_EXCERPT = 0x104 +VER_COMMAND_UPDATE = 0x103 +VER_COMMAND_KEYWORDS = 0x100 +VER_COMMAND_STATUS = 0x101 +VER_COMMAND_FLUSHATTRS = 0x100 + +# known searchd status codes +SEARCHD_OK = 0 +SEARCHD_ERROR = 1 +SEARCHD_RETRY = 2 +SEARCHD_WARNING = 3 + +# known ranking modes (extended2 mode only) +SPH_RANK_PROXIMITY_BM15 = 0 # default mode, phrase proximity major factor and BM15 minor one +SPH_RANK_BM15 = 1 # statistical mode, BM15 ranking only (faster but worse quality) +SPH_RANK_NONE = 2 # no ranking, all matches get a weight of 1 +SPH_RANK_WORDCOUNT = 3 # simple word-count weighting, rank is a weighted sum of per-field keyword occurence counts +SPH_RANK_PROXIMITY = 4 +SPH_RANK_MATCHANY = 5 +SPH_RANK_FIELDMASK = 6 +SPH_RANK_SPH04 = 7 +SPH_RANK_EXPR = 8 +SPH_RANK_TOTAL = 9 + +# aliases; to be retired +SPH_RANK_PROXIMITY_BM25 = 0 +SPH_RANK_BM25 = 1 + +# known sort modes +SPH_SORT_RELEVANCE = 0 +SPH_SORT_ATTR_DESC = 1 +SPH_SORT_ATTR_ASC = 2 +SPH_SORT_TIME_SEGMENTS = 3 +SPH_SORT_EXTENDED = 4 + +# known filter types +SPH_FILTER_VALUES = 0 +SPH_FILTER_RANGE = 1 +SPH_FILTER_FLOATRANGE = 2 +SPH_FILTER_STRING = 3 +SPH_FILTER_STRING_LIST = 6 + +# known attribute types +SPH_ATTR_NONE = 0 +SPH_ATTR_INTEGER = 1 +SPH_ATTR_TIMESTAMP = 2 +SPH_ATTR_ORDINAL = 3 +SPH_ATTR_BOOL = 4 +SPH_ATTR_FLOAT = 5 +SPH_ATTR_BIGINT = 6 +SPH_ATTR_STRING = 7 +SPH_ATTR_FACTORS = 1001 +SPH_ATTR_MULTI = long(0X40000001) +SPH_ATTR_MULTI64 = long(0X40000002) + +SPH_ATTR_TYPES = (SPH_ATTR_NONE, + SPH_ATTR_INTEGER, + SPH_ATTR_TIMESTAMP, + SPH_ATTR_ORDINAL, + SPH_ATTR_BOOL, + SPH_ATTR_FLOAT, + SPH_ATTR_BIGINT, + SPH_ATTR_STRING, + SPH_ATTR_MULTI, + SPH_ATTR_MULTI64) + +# known grouping functions +SPH_GROUPBY_DAY = 0 +SPH_GROUPBY_WEEK = 1 +SPH_GROUPBY_MONTH = 2 +SPH_GROUPBY_YEAR = 3 +SPH_GROUPBY_ATTR = 4 +SPH_GROUPBY_ATTRPAIR = 5 + + +class SphinxClient: + def __init__ (self): + """ + Create a new client object, and fill defaults. + """ + self._host = 'localhost' # searchd host (default is "localhost") + self._port = 9312 # searchd port (default is 9312) + self._path = None # searchd unix-domain socket path + self._socket = None + self._offset = 0 # how much records to seek from result-set start (default is 0) + self._limit = 20 # how much records to return from result-set starting at offset (default is 20) + self._weights = [] # per-field weights (default is 1 for all fields) + self._sort = SPH_SORT_RELEVANCE # match sorting mode (default is SPH_SORT_RELEVANCE) + self._sortby = bytearray() # attribute to sort by (defualt is "") + self._min_id = 0 # min ID to match (default is 0) + self._max_id = 0 # max ID to match (default is UINT_MAX) + self._filters = [] # search filters + self._groupby = bytearray() # group-by attribute name + self._groupfunc = SPH_GROUPBY_DAY # group-by function (to pre-process group-by attribute value with) + self._groupsort = str_bytes('@group desc') # group-by sorting clause (to sort groups in result set with) + self._groupdistinct = bytearray() # group-by count-distinct attribute + self._maxmatches = 1000 # max matches to retrieve + self._cutoff = 0 # cutoff to stop searching at + self._retrycount = 0 # distributed retry count + self._retrydelay = 0 # distributed retry delay + self._indexweights = {} # per-index weights + self._ranker = SPH_RANK_PROXIMITY_BM15 # ranking mode + self._rankexpr = bytearray() # ranking expression for SPH_RANK_EXPR + self._maxquerytime = 0 # max query time, milliseconds (default is 0, do not limit) + self._timeout = 1.0 # connection timeout + self._fieldweights = {} # per-field-name weights + self._select = str_bytes('*') # select-list (attributes or expressions, with optional aliases) + self._query_flags = SetBit ( 0, 6, True ) # default idf=tfidf_normalized + self._predictedtime = 0 # per-query max_predicted_time + self._outerorderby = bytearray() # outer match sort by + self._outeroffset = 0 # outer offset + self._outerlimit = 0 # outer limit + self._hasouter = False # sub-select enabled + self._tokenfilterlibrary = bytearray() # token_filter plugin library name + self._tokenfiltername = bytearray() # token_filter plugin name + self._tokenfilteropts = bytearray() # token_filter plugin options + + self._error = '' # last error message + self._warning = '' # last warning message + self._reqs = [] # requests array for multi-query + + def __del__ (self): + if self._socket: + self._socket.close() + + + def GetLastError (self): + """ + Get last error message (string). + """ + return self._error + + + def GetLastWarning (self): + """ + Get last warning message (string). + """ + return self._warning + + + def SetServer (self, host, port = None): + """ + Set searchd server host and port. + """ + assert(isinstance(host, str)) + if host.startswith('/'): + self._path = host + return + elif host.startswith('unix://'): + self._path = host[7:] + return + self._host = host + if isinstance(port, int): + assert(port>0 and port<65536) + self._port = port + self._path = None + + def SetConnectTimeout ( self, timeout ): + """ + Set connection timeout ( float second ) + """ + assert (isinstance(timeout, float)) + # set timeout to 0 make connaection non-blocking that is wrong so timeout got clipped to reasonable minimum + self._timeout = max ( 0.001, timeout ) + + def _Connect (self): + """ + INTERNAL METHOD, DO NOT CALL. Connects to searchd server. + """ + if self._socket: + # we have a socket, but is it still alive? + sr, sw, _ = select.select ( [self._socket], [self._socket], [], 0 ) + + # this is how alive socket should look + if len(sr)==0 and len(sw)==1: + return self._socket + + # oops, looks like it was closed, lets reopen + self._socket.close() + self._socket = None + + try: + if self._path: + af = socket.AF_UNIX + addr = self._path + desc = self._path + else: + af = socket.AF_INET + addr = ( self._host, self._port ) + desc = '%s;%s' % addr + sock = socket.socket ( af, socket.SOCK_STREAM ) + sock.settimeout ( self._timeout ) + sock.connect ( addr ) + except socket.error as msg: + if sock: + sock.close() + self._error = 'connection to %s failed (%s)' % ( desc, msg ) + return + + v = unpack('>L', sock.recv(4))[0] + if v<1: + sock.close() + self._error = 'expected searchd protocol version, got %s' % v + return + + # all ok, send my version + sock.send(pack('>L', 1)) + return sock + + + def _GetResponse (self, sock, client_ver): + """ + INTERNAL METHOD, DO NOT CALL. Gets and checks response packet from searchd server. + """ + (status, ver, length) = unpack('>2HL', sock.recv(8)) + response = bytearray() + left = length + while left>0: + chunk = sock.recv(left) + if chunk: + response += chunk + left -= len(chunk) + else: + break + + if not self._socket: + sock.close() + + # check response + read = len(response) + if not response or read!=length: + if length: + self._error = 'failed to read searchd response (status=%s, ver=%s, len=%s, read=%s)' \ + % (status, ver, length, read) + else: + self._error = 'received zero-sized searchd response' + return None + + # check status + if status==SEARCHD_WARNING: + wend = 4 + unpack ( '>L', response[0:4] )[0] + self._warning = bytes_str(response[4:wend]) + return response[wend:] + + if status==SEARCHD_ERROR: + self._error = 'searchd error: ' + bytes_str(response[4:]) + return None + + if status==SEARCHD_RETRY: + self._error = 'temporary searchd error: ' + bytes_str(response[4:]) + return None + + if status!=SEARCHD_OK: + self._error = 'unknown status code %d' % status + return None + + # check version + if ver>8, ver&0xff, client_ver>>8, client_ver&0xff) + + return response + + + def _Send ( self, sock, req ): + """ + INTERNAL METHOD, DO NOT CALL. send request to searchd server. + """ + total = 0 + while True: + sent = sock.send ( req[total:] ) + if sent<=0: + break + + total = total + sent + + return total + + + def SetLimits (self, offset, limit, maxmatches=0, cutoff=0): + """ + Set offset and count into result set, and optionally set max-matches and cutoff limits. + """ + assert ( type(offset) in [int,long] and 0<=offset<16777216 ) + assert ( type(limit) in [int,long] and 0=0) + self._offset = offset + self._limit = limit + if maxmatches>0: + self._maxmatches = maxmatches + if cutoff>=0: + self._cutoff = cutoff + + + def SetMaxQueryTime (self, maxquerytime): + """ + Set maximum query time, in milliseconds, per-index. 0 means 'do not limit'. + """ + assert(isinstance(maxquerytime,int) and maxquerytime>0) + self._maxquerytime = maxquerytime + + + def SetRankingMode ( self, ranker, rankexpr='' ): + """ + Set ranking mode. + """ + assert(ranker>=0 and ranker=0) + assert(isinstance(delay,int) and delay>=0) + self._retrycount = count + self._retrydelay = delay + + + def SetSelect (self, select): + assert(isinstance(select, (str,text_type))) + self._select = str_bytes(select) + + + def SetQueryFlag ( self, name, value ): + known_names = [ "reverse_scan", "sort_method", "max_predicted_time", "boolean_simplify", "idf", "global_idf" ] + flags = { "reverse_scan":[0, 1], "sort_method":["pq", "kbuffer"],"max_predicted_time":[0], "boolean_simplify":[True, False], "idf":["normalized", "plain", "tfidf_normalized", "tfidf_unnormalized"], "global_idf":[True, False] } + assert ( name in known_names ) + assert ( value in flags[name] or ( name=="max_predicted_time" and isinstance(value, (int, long)) and value>=0)) + + if name=="reverse_scan": + self._query_flags = SetBit ( self._query_flags, 0, value==1 ) + if name=="sort_method": + self._query_flags = SetBit ( self._query_flags, 1, value=="kbuffer" ) + if name=="max_predicted_time": + self._query_flags = SetBit ( self._query_flags, 2, value>0 ) + self._predictedtime = int(value) + if name=="boolean_simplify": + self._query_flags= SetBit ( self._query_flags, 3, value ) + if name=="idf" and ( value=="plain" or value=="normalized" ) : + self._query_flags = SetBit ( self._query_flags, 4, value=="plain" ) + if name=="global_idf": + self._query_flags= SetBit ( self._query_flags, 5, value ) + if name=="idf" and ( value=="tfidf_normalized" or value=="tfidf_unnormalized" ) : + self._query_flags = SetBit ( self._query_flags, 6, value=="tfidf_normalized" ) + + def SetOuterSelect ( self, orderby, offset, limit ): + assert(isinstance(orderby, (str,text_type))) + assert(isinstance(offset, (int, long))) + assert(isinstance(limit, (int, long))) + assert ( offset>=0 ) + assert ( limit>0 ) + + self._outerorderby = str_bytes(orderby) + self._outeroffset = offset + self._outerlimit = limit + self._hasouter = True + + def SetTokenFilter ( self, library, name, opts='' ): + assert(isinstance(library, str)) + assert(isinstance(name, str)) + assert(isinstance(opts, str)) + + self._tokenfilterlibrary = str_bytes(library) + self._tokenfiltername = str_bytes(name) + self._tokenfilteropts = str_bytes(opts) + + + def ResetFilters (self): + """ + Clear all filters (for multi-queries). + """ + self._filters = [] + + + def ResetGroupBy (self): + """ + Clear groupby settings (for multi-queries). + """ + self._groupby = bytearray() + self._groupfunc = SPH_GROUPBY_DAY + self._groupsort = str_bytes('@group desc') + self._groupdistinct = bytearray() + + def ResetQueryFlag (self): + self._query_flags = SetBit ( 0, 6, True ) # default idf=tfidf_normalized + self._predictedtime = 0 + + def ResetOuterSelect (self): + self._outerorderby = bytearray() + self._outeroffset = 0 + self._outerlimit = 0 + self._hasouter = False + + def Query (self, query, index='*', comment=''): + """ + Connect to searchd server and run given search query. + Returns None on failure; result set hash on success (see documentation for details). + """ + assert(len(self._reqs)==0) + self.AddQuery(query,index,comment) + results = self.RunQueries() + self._reqs = [] # we won't re-run erroneous batch + + if not results or len(results)==0: + return None + self._error = results[0]['error'] + self._warning = results[0]['warning'] + if results[0]['status'] == SEARCHD_ERROR: + return None + return results[0] + + + def AddQuery (self, query, index='*', comment=''): + """ + Add query to batch. + """ + # build request + # 6 == match_mode extended2 + req = bytearray() + req.extend(pack('>5L', self._query_flags, self._offset, self._limit, 6, self._ranker)) + if self._ranker==SPH_RANK_EXPR: + req.extend(pack('>L', len(self._rankexpr))) + req.extend(self._rankexpr) + req.extend(pack('>L', self._sort)) + req.extend(pack('>L', len(self._sortby))) + req.extend(self._sortby) + + query = str_bytes(query) + assert(isinstance(query,bytearray)) + + req.extend(pack('>L', len(query))) + req.extend(query) + + req.extend(pack('>L', len(self._weights))) + for w in self._weights: + req.extend(pack('>L', w)) + index = str_bytes(index) + assert(isinstance(index,bytearray)) + req.extend(pack('>L', len(index))) + req.extend(index) + req.extend(pack('>L',1)) # id64 range marker + req.extend(pack('>Q', self._min_id)) + req.extend(pack('>Q', self._max_id)) + + # filters + req.extend ( pack ( '>L', len(self._filters) ) ) + for f in self._filters: + attr = str_bytes(f['attr']) + req.extend ( pack ( '>L', len(f['attr'])) + attr) + filtertype = f['type'] + req.extend ( pack ( '>L', filtertype)) + if filtertype == SPH_FILTER_VALUES: + req.extend ( pack ('>L', len(f['values']))) + for val in f['values']: + req.extend ( pack ('>q', val)) + elif filtertype == SPH_FILTER_RANGE: + req.extend ( pack ('>2q', f['min'], f['max'])) + elif filtertype == SPH_FILTER_FLOATRANGE: + req.extend ( pack ('>2f', f['min'], f['max'])) + elif filtertype == SPH_FILTER_STRING: + val = str_bytes(f['value']) + req.extend ( pack ( '>L', len(val) ) ) + req.extend ( val ) + elif filtertype == SPH_FILTER_STRING_LIST: + req.extend ( pack ('>L', len(f['values']))) + for sval in f['values']: + val = str_bytes( sval ) + req.extend ( pack ( '>L', len(val) ) ) + req.extend(val) + req.extend ( pack ( '>L', f['exclude'] ) ) + + # group-by, max-matches, group-sort + req.extend ( pack ( '>2L', self._groupfunc, len(self._groupby) ) ) + req.extend ( self._groupby ) + req.extend ( pack ( '>2L', self._maxmatches, len(self._groupsort) ) ) + req.extend ( self._groupsort ) + req.extend ( pack ( '>LLL', self._cutoff, self._retrycount, self._retrydelay)) + req.extend ( pack ( '>L', len(self._groupdistinct))) + req.extend ( self._groupdistinct) + + # geoanchor point + req.extend ( pack ('>L', 0) ) + + # per-index weights + req.extend ( pack ('>L',len(self._indexweights))) + for indx,weight in list(self._indexweights.items()): + indx = str_bytes(indx) + req.extend ( pack ('>L',len(indx)) + indx + pack ('>L',weight)) + + # max query time + req.extend ( pack ('>L', self._maxquerytime) ) + + # per-field weights + req.extend ( pack ('>L',len(self._fieldweights) ) ) + for field,weight in list(self._fieldweights.items()): + field = str_bytes(field) + req.extend ( pack ('>L',len(field)) + field + pack ('>L',weight) ) + + # comment + comment = str_bytes(comment) + req.extend ( pack('>L',len(comment)) + comment ) + + # attribute overrides + req.extend ( pack('>L', 0) ) + + # select-list + req.extend ( pack('>L', len(self._select)) ) + req.extend ( self._select ) + if self._predictedtime>0: + req.extend ( pack('>L', self._predictedtime ) ) + + # outer + req.extend ( pack('>L',len(self._outerorderby)) + self._outerorderby ) + req.extend ( pack ( '>2L', self._outeroffset, self._outerlimit ) ) + if self._hasouter: + req.extend ( pack('>L', 1) ) + else: + req.extend ( pack('>L', 0) ) + + # token_filter + req.extend ( pack('>L',len(self._tokenfilterlibrary)) + self._tokenfilterlibrary ) + req.extend ( pack('>L',len(self._tokenfiltername)) + self._tokenfiltername ) + req.extend ( pack('>L',len(self._tokenfilteropts)) + self._tokenfilteropts ) + + # send query, get response + + self._reqs.append(req) + return + + + def RunQueries (self): + """ + Run queries batch. + Returns None on network IO failure; or an array of result set hashes on success. + """ + if len(self._reqs)==0: + self._error = 'no queries defined, issue AddQuery() first' + return None + + sock = self._Connect() + if not sock: + return None + + req = bytearray() + for r in self._reqs: + req.extend(r) + length = len(req)+8 + req_all = bytearray() + req_all.extend(pack('>HHLLL', SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, length, 0, len(self._reqs))) + req_all.extend(req) + self._Send ( sock, req_all ) + + response = self._GetResponse(sock, VER_COMMAND_SEARCH) + if not response: + return None + + nreqs = len(self._reqs) + + # parse response + max_ = len(response) + p = 0 + + results = [] + for i in range(0,nreqs,1): + result = {} + results.append(result) + + result['error'] = '' + result['warning'] = '' + status = unpack('>L', response[p:p+4])[0] + p += 4 + result['status'] = status + if status != SEARCHD_OK: + length = unpack('>L', response[p:p+4])[0] + p += 4 + message = bytes_str(response[p:p+length]) + p += length + + if status == SEARCHD_WARNING: + result['warning'] = message + else: + result['error'] = message + continue + + # read schema + fields = [] + attrs = [] + + nfields = unpack('>L', response[p:p+4])[0] + p += 4 + while nfields>0 and pL', response[p:p+4])[0] + p += 4 + fields.append(bytes_str(response[p:p+length])) + p += length + + result['fields'] = fields + + nattrs = unpack('>L', response[p:p+4])[0] + p += 4 + while nattrs>0 and pL', response[p:p+4])[0] + p += 4 + attr = bytes_str(response[p:p+length]) + p += length + type_ = unpack('>L', response[p:p+4])[0] + p += 4 + attrs.append([attr,type_]) + + result['attrs'] = attrs + + # read match count + count = unpack('>L', response[p:p+4])[0] + p += 4 + id64 = unpack('>L', response[p:p+4])[0] + p += 4 + + # read matches + result['matches'] = [] + while count>0 and pQL', response[p:p+12]) + p += 12 + else: + doc, weight = unpack('>2L', response[p:p+8]) + p += 8 + + match = { 'id':doc, 'weight':weight, 'attrs':{} } + for i in range(len(attrs)): + if attrs[i][1] == SPH_ATTR_FLOAT: + match['attrs'][attrs[i][0]] = unpack('>f', response[p:p+4])[0] + elif attrs[i][1] == SPH_ATTR_BIGINT: + match['attrs'][attrs[i][0]] = unpack('>q', response[p:p+8])[0] + p += 4 + elif attrs[i][1] == SPH_ATTR_STRING: + slen = unpack('>L', response[p:p+4])[0] + p += 4 + match['attrs'][attrs[i][0]] = '' + if slen>0: + match['attrs'][attrs[i][0]] = bytes_str(response[p:p+slen]) + p += slen-4 + elif attrs[i][1] == SPH_ATTR_FACTORS: + slen = unpack('>L', response[p:p+4])[0] + p += 4 + match['attrs'][attrs[i][0]] = '' + if slen>0: + match['attrs'][attrs[i][0]] = response[p:p+slen-4] + p += slen-4 + p -= 4 + elif attrs[i][1] == SPH_ATTR_MULTI: + match['attrs'][attrs[i][0]] = [] + nvals = unpack('>L', response[p:p+4])[0] + p += 4 + for n in range(0,nvals,1): + match['attrs'][attrs[i][0]].append(unpack('>L', response[p:p+4])[0]) + p += 4 + p -= 4 + elif attrs[i][1] == SPH_ATTR_MULTI64: + match['attrs'][attrs[i][0]] = [] + nvals = unpack('>L', response[p:p+4])[0] + nvals = nvals/2 + p += 4 + for n in range(0,nvals,1): + match['attrs'][attrs[i][0]].append(unpack('>q', response[p:p+8])[0]) + p += 8 + p -= 4 + else: + match['attrs'][attrs[i][0]] = unpack('>L', response[p:p+4])[0] + p += 4 + + result['matches'].append ( match ) + + result['total'], result['total_found'], result['time'], words = unpack('>4L', response[p:p+16]) + + result['time'] = '%.3f' % (result['time']/1000.0) + p += 16 + + result['words'] = [] + while words>0: + words -= 1 + length = unpack('>L', response[p:p+4])[0] + p += 4 + word = bytes_str(response[p:p+length]) + p += length + docs, hits = unpack('>2L', response[p:p+8]) + p += 8 + + result['words'].append({'word':word, 'docs':docs, 'hits':hits}) + + self._reqs = [] + return results + + + def BuildExcerpts (self, docs, index, words, opts=None): + """ + Connect to searchd server and generate exceprts from given documents. + """ + if not opts: + opts = {} + + assert(isinstance(docs, list)) + assert(isinstance(index, (str,text_type))) + assert(isinstance(words, (str,text_type))) + assert(isinstance(opts, dict)) + + sock = self._Connect() + + if not sock: + return None + + # fixup options + opts.setdefault('before_match', '') + opts.setdefault('after_match', '') + opts.setdefault('chunk_separator', ' ... ') + opts.setdefault('html_strip_mode', 'index') + opts.setdefault('limit', 256) + opts.setdefault('limit_passages', 0) + opts.setdefault('limit_words', 0) + opts.setdefault('around', 5) + opts.setdefault('start_passage_id', 1) + opts.setdefault('passage_boundary', 'none') + + # build request + # v.1.0 req + + flags = 1 # (remove spaces) + if opts.get('exact_phrase'): flags |= 2 + if opts.get('single_passage'): flags |= 4 + if opts.get('use_boundaries'): flags |= 8 + if opts.get('weight_order'): flags |= 16 + if opts.get('query_mode'): flags |= 32 + if opts.get('force_all_words'): flags |= 64 + if opts.get('load_files'): flags |= 128 + if opts.get('allow_empty'): flags |= 256 + if opts.get('emit_zones'): flags |= 512 + if opts.get('load_files_scattered'): flags |= 1024 + + # mode=0, flags + req = bytearray() + req.extend(pack('>2L', 0, flags)) + + # req index + index = str_bytes(index) + req.extend(pack('>L', len(index))) + req.extend(index) + + # req words + words = str_bytes(words) + req.extend(pack('>L', len(words))) + req.extend(words) + + # options + opts_before_match = str_bytes(opts['before_match']) + req.extend(pack('>L', len(opts_before_match))) + req.extend(opts_before_match) + + opts_after_match = str_bytes(opts['after_match']) + req.extend(pack('>L', len(opts_after_match))) + req.extend(opts_after_match) + + opts_chunk_separator = str_bytes(opts['chunk_separator']) + req.extend(pack('>L', len(opts_chunk_separator))) + req.extend(opts_chunk_separator) + + req.extend(pack('>L', int(opts['limit']))) + req.extend(pack('>L', int(opts['around']))) + + req.extend(pack('>L', int(opts['limit_passages']))) + req.extend(pack('>L', int(opts['limit_words']))) + req.extend(pack('>L', int(opts['start_passage_id']))) + opts_html_strip_mode = str_bytes(opts['html_strip_mode']) + req.extend(pack('>L', len(opts_html_strip_mode))) + req.extend(opts_html_strip_mode) + opts_passage_boundary = str_bytes(opts['passage_boundary']) + req.extend(pack('>L', len(opts_passage_boundary))) + req.extend(opts_passage_boundary) + + # documents + req.extend(pack('>L', len(docs))) + for doc in docs: + doc = str_bytes(doc) + req.extend(pack('>L', len(doc))) + req.extend(doc) + + # send query, get response + length = len(req) + + # add header + req_head = bytearray() + req_head.extend(pack('>2HL', SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, length)) + req_all = req_head + req + self._Send ( sock, req_all ) + + response = self._GetResponse(sock, VER_COMMAND_EXCERPT ) + if not response: + return [] + + # parse response + pos = 0 + res = [] + rlen = len(response) + + for i in range(len(docs)): + length = unpack('>L', response[pos:pos+4])[0] + pos += 4 + + if pos+length > rlen: + self._error = 'incomplete reply' + return [] + + res.append(bytes_str(response[pos:pos+length])) + pos += length + + return res + + + def UpdateAttributes ( self, index, attrs, values, mva=False, ignorenonexistent=False ): + """ + Update given attribute values on given documents in given indexes. + Returns amount of updated documents (0 or more) on success, or -1 on failure. + + 'attrs' must be a list of strings. + 'values' must be a dict with int key (document ID) and list of int values (new attribute values). + optional boolean parameter 'mva' points that there is update of MVA attributes. + In this case the 'values' must be a dict with int key (document ID) and list of lists of int values + (new MVA attribute values). + Optional boolean parameter 'ignorenonexistent' points that the update will silently ignore any warnings about + trying to update a column which is not exists in current index schema. + + Example: + res = cl.UpdateAttributes ( 'test1', [ 'group_id', 'date_added' ], { 2:[123,1000000000], 4:[456,1234567890] } ) + """ + assert ( isinstance ( index, str ) ) + assert ( isinstance ( attrs, list ) ) + assert ( isinstance ( values, dict ) ) + for attr in attrs: + assert ( isinstance ( attr, str ) ) + for docid, entry in list(values.items()): + AssertUInt32(docid) + assert ( isinstance ( entry, list ) ) + assert ( len(attrs)==len(entry) ) + for val in entry: + if mva: + assert ( isinstance ( val, list ) ) + for vals in val: + AssertInt32(vals) + else: + AssertInt32(val) + + # build request + req = bytearray() + index = str_bytes(index) + req.extend( pack('>L',len(index)) + index ) + + req.extend ( pack('>L',len(attrs)) ) + ignore_absent = 0 + if ignorenonexistent: ignore_absent = 1 + req.extend ( pack('>L', ignore_absent ) ) + mva_attr = 0 + if mva: mva_attr = 1 + for attr in attrs: + attr = str_bytes(attr) + req.extend ( pack('>L',len(attr)) + attr ) + req.extend ( pack('>L', mva_attr ) ) + + req.extend ( pack('>L',len(values)) ) + for docid, entry in list(values.items()): + req.extend ( pack('>Q',docid) ) + for val in entry: + val_len = val + if mva: val_len = len ( val ) + req.extend ( pack('>L',val_len ) ) + if mva: + for vals in val: + req.extend ( pack ('>L',vals) ) + + # connect, send query, get response + sock = self._Connect() + if not sock: + return None + + length = len(req) + req_all = bytearray() + req_all.extend( pack ( '>2HL', SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, length ) ) + req_all.extend( req ) + self._Send ( sock, req_all ) + + response = self._GetResponse ( sock, VER_COMMAND_UPDATE ) + if not response: + return -1 + + # parse response + updated = unpack ( '>L', response[0:4] )[0] + return updated + + + def BuildKeywords ( self, query, index, hits ): + """ + Connect to searchd server, and generate keywords list for a given query. + Returns None on failure, or a list of keywords on success. + """ + assert ( isinstance ( query, str ) ) + assert ( isinstance ( index, str ) ) + assert ( isinstance ( hits, int ) ) + + # build request + req = bytearray() + query = str_bytes(query) + req.extend(pack ( '>L', len(query) ) + query) + index = str_bytes(index) + req.extend ( pack ( '>L', len(index) ) + index ) + req.extend ( pack ( '>L', hits ) ) + + # connect, send query, get response + sock = self._Connect() + if not sock: + return None + + length = len(req) + req_all = bytearray() + req_all.extend(pack ( '>2HL', SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, length )) + req_all.extend(req) + self._Send ( sock, req_all ) + + response = self._GetResponse ( sock, VER_COMMAND_KEYWORDS ) + if not response: + return None + + # parse response + res = [] + + nwords = unpack ( '>L', response[0:4] )[0] + p = 4 + max_ = len(response) + + while nwords>0 and pL', response[p:p+4] )[0] + p += 4 + tokenized = response[p:p+length] + p += length + + length = unpack ( '>L', response[p:p+4] )[0] + p += 4 + normalized = response[p:p+length] + p += length + + entry = { 'tokenized':bytes_str(tokenized), 'normalized':bytes_str(normalized) } + if hits: + entry['docs'], entry['hits'] = unpack ( '>2L', response[p:p+8] ) + p += 8 + + res.append ( entry ) + + if nwords>0 or p>max_: + self._error = 'incomplete reply' + return None + + return res + + def Status ( self, session=False ): + """ + Get the status + """ + + # connect, send query, get response + sock = self._Connect() + if not sock: + return None + + sess = 1 + if session: + sess = 0 + + req = pack ( '>2HLL', SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, sess ) + self._Send ( sock, req ) + + response = self._GetResponse ( sock, VER_COMMAND_STATUS ) + if not response: + return None + + # parse response + res = [] + + p = 8 + max_ = len(response) + + while pL', response[p:p+4] )[0] + k = response[p+4:p+length+4] + p += 4+length + length = unpack ( '>L', response[p:p+4] )[0] + v = response[p+4:p+length+4] + p += 4+length + res += [[bytes_str(k), bytes_str(v)]] + + return res + + ### persistent connections + + def Open(self): + if self._socket: + self._error = 'already connected' + return None + + server = self._Connect() + if not server: + return None + + # command, command version = 0, body length = 4, body = 1 + request = pack ( '>hhII', SEARCHD_COMMAND_PERSIST, 0, 4, 1 ) + self._Send ( server, request ) + + self._socket = server + return True + + def Close(self): + if not self._socket: + self._error = 'not connected' + return + self._socket.close() + self._socket = None + + def EscapeString(self, string): + return re.sub(r"([=\(\)|\-!@~\"&/\\\^\$\=\<])", r"\\\1", string) + + + def FlushAttributes(self): + sock = self._Connect() + if not sock: + return -1 + + request = pack ( '>hhI', SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0 ) # cmd, ver, bodylen + self._Send ( sock, request ) + + response = self._GetResponse ( sock, VER_COMMAND_FLUSHATTRS ) + if not response or len(response)!=4: + self._error = 'unexpected response length' + return -1 + + tag = unpack ( '>L', response[0:4] )[0] + return tag + +def AssertInt32 ( value ): + assert(isinstance(value, (int, long))) + assert(value>=-2**32-1 and value<=2**32-1) + +def AssertUInt32 ( value ): + assert(isinstance(value, (int, long))) + assert(value>=0 and value<=2**32-1) + +def SetBit ( flag, bit, on ): + if on: + flag += ( 1< (3,): + def str_bytes(x): + return bytearray(x, 'utf-8') +else: + def str_bytes(x): + if isinstance(x,unicode): + return bytearray(x, 'utf-8') + else: + return bytearray(x) + +def bytes_str(x): + assert (isinstance(x, bytearray)) + return x.decode('utf-8') + +# +# $Id$ +# diff --git a/plugins/manticoresearch/conf/sphinx.conf b/plugins/manticoresearch/conf/sphinx.conf new file mode 100755 index 000000000..de9e5f270 --- /dev/null +++ b/plugins/manticoresearch/conf/sphinx.conf @@ -0,0 +1,28 @@ +# +# Minimal Sphinx configuration sample (clean, simple, functional) +# + + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$SERVER_APP}/index/searchd.log + query_log = {$SERVER_APP}/index/query.log + pid_file = {$SERVER_APP}/index/searchd.pid + #workers = threads # for RT to work + binlog_path = {$SERVER_APP}/index/binlog + read_timeout = 5 + max_children = 0 + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 +} + +index mydocs +{ + type = rt + path = {$SERVER_APP}/bin/doc + rt_field = title + rt_attr_json = j +} \ No newline at end of file diff --git a/plugins/manticoresearch/ico.png b/plugins/manticoresearch/ico.png new file mode 100644 index 0000000000000000000000000000000000000000..f8f4edeb98a688d081e82864a74e9484b19e89fe GIT binary patch literal 2268 zcmV<22qX82P)24!8f46@SnfZX?3hoG1doWY8{=4j81K*nhaK(*3?AO zRIB4ut4S~!A2V^{Bc`^Biui&m5kV9Xh#>NGVc+}Rdpl=gb-Pf?5-fFwGjrxL-#x!` ze!ug5_kQ197LNWfs*kbuFqApwJJY`=jl3%bZ~gFZ6c=(0QP)a9aq;N`(Va^uT! zQ4+aa;<8w+qA2!@2iG4^(&Bj;)0Ss!RjSn^Jp%+>wKq#neDlXY|DvJ1;z4g%9z>A` zUZ9`|)2FQ3pM6WF;6giv@g6TdfBmpO9>=`*(PeTj^r!o>Z*eAF3iSx+9LE}1YrqAd zo%|?wquXJ7gdL*-4=56{(PH>cFri|r08yMm^A_8NzmLFLC zValV5hYthg)33vvHcT5Gq>;t>K9n6!2Uy({B9h0xy?Wn-a4UF$-=SYE`flJcfl|TCoqIOZD4C%%k__!?MQODTmhZx0$xICnQ3dr zrjA+Vdju#xd-Hd?FkQS%Dh-lz1f+}rQYoktXC^L2A22 zSG+SU&hVNyhf*yM=Xw5d)y=BH=O!RxbVU53cjq5t7?66`eob-piBC3u`m+c5rH4jM zik>n|tBm3p9vC73#wjG6j29{I5=4e$7-)G=fAQe@{r`Xcgkc9edJfZXIv|oP=doi z9~PXzbj6@6Si-PG5WXv}{(NuNz6BjE9cDa;HdHngOr8DeDwYu?o}7#*jY)`{bnUC6 z1A@@E4}?BH#A%C@R)j@sk~zWyI!k!Ce6%3@VR>DRH%6@u3RarP+Yi z-?(|}X8c~GXGV&mjcc+$<_Wk2j!>Q>lyZb}TD>fO!Md5-eKDA$xu)TQR{+PjAw4r` zTS&OZFkk{!Zu!w3m5MRo+QuS`=(Zl*e0&9U^G`P{wO*y1kQ%d8Es=muDgimmvybZ< z3)||O3pv7b9xTJd_!+TF)Hl z3@|R*VYj)R?X4CE?{suX2n9)34&1f<@t6~Hb6s)hSQ|O?Tfnjn)81Doh=G0jKvH^W zdtRp9;&S-MHrAR;iQu+%c2NLhcu>OohcgO4JifGWxJi-dZGeQ~?F}_;CH{EKiMg?c zw;t9;4*e1^>GhbDiPIx8eY;<9sx;>x7psbTif?PS*h{ZfA7OpsjV9%Iqe(G=d)5H> z_HxxRbBoQ^Qx42saI!R~6W_UHCr%H~L@ogd(1s|3R&7i=$PqWw*-;9X){e5=oGY7q z>qc%)!LMv)%RP+u749WtGh6R}w&U_Id(H9DmCaZS-$q@v8RUYTdW!FHBU}I5=^w|B z)ul6dQuMe0ySL=OSy|T7?MutoI?PV%ds zqp{x7<4-PLb2)k1@~ES1`mCr-Z*8TD)tg2H>+-*;KII?Z?O~eqnrUkG&a?vG*|}5L zbZFzszjphjp*J&_RJu_K!xI4nxchBWabvx;B@q0!{j+k%P7Gb*GbC(Xdp@P`QvEgG zxNb{mV#@H@ETN2d`dv2HcRSv>lod#Ew8naC>%}uw=PsV9J{Kr?v>lr+WnpbE59Ea0 z{NuVUq54`&$t&Y^<3bFwc$~srzqyfdyQsO_TjP0~%}VW0nyq)H{a@HJ + +#table xm-select{ + min-height: 30px; + line-height: 30px; +} + +#table xm-select *{ + font-size: 12px; +} + + +
+
+
+

服务

+

自启动

+

配置修改

+

运行日志

+

查询日志

+

运行状态

+

常用功能

+

说明

+
+
+
+
+
+
+ \ No newline at end of file diff --git a/plugins/manticoresearch/index.py b/plugins/manticoresearch/index.py new file mode 100755 index 000000000..cd8ae76cd --- /dev/null +++ b/plugins/manticoresearch/index.py @@ -0,0 +1,483 @@ +# coding:utf-8 + +import sys +import io +import os +import time +import re +import string +import subprocess + +web_dir = os.getcwd() + "/web" +if os.path.exists(web_dir): + sys.path.append(web_dir) + os.chdir(web_dir) + +import core.mw as mw + +app_debug = False +if mw.isAppleSystem(): + app_debug = True + +def getPluginName(): + return 'manticoresearch' + +def getPluginDir(): + return mw.getPluginDir() + '/' + getPluginName() + +sys.path.append(getPluginDir() +"/class") + +def getServerDir(): + return mw.getServerDir() + '/' + getPluginName() + + +def getInitDFile(): + if app_debug: + return '/tmp/' + getPluginName() + return '/etc/init.d/' + getPluginName() + + +def getConfTpl(): + path = getPluginDir() + "/conf/sphinx.conf" + return path + + +def getConf(): + path = getServerDir() + "/sphinx.conf" + return path + + +def getInitDTpl(): + path = getPluginDir() + "/init.d/" + getPluginName() + ".tpl" + return path + + +def getArgs(): + args = sys.argv[2:] + tmp = {} + args_len = len(args) + + if args_len == 1: + t = args[0].strip('{').strip('}') + t = t.split(':') + tmp[t[0]] = t[1] + elif args_len > 1: + for i in range(len(args)): + t = args[i].split(':') + tmp[t[0]] = t[1] + return tmp + + +def checkArgs(data, ck=[]): + for i in range(len(ck)): + if not ck[i] in data: + return (False, mw.returnJson(False, '参数:(' + ck[i] + ')没有!')) + return (True, mw.returnJson(True, 'ok')) + + +def configTpl(): + path = getPluginDir() + '/tpl' + pathFile = os.listdir(path) + tmp = [] + for one in pathFile: + file = path + '/' + one + tmp.append(file) + return mw.getJson(tmp) + + +def readConfigTpl(): + args = getArgs() + data = checkArgs(args, ['file']) + if not data[0]: + return data[1] + + content = mw.readFile(args['file']) + content = contentReplace(content) + return mw.returnJson(True, 'ok', content) + + +def contentReplace(content): + service_path = mw.getServerDir() + content = content.replace('{$ROOT_PATH}', mw.getFatherDir()) + content = content.replace('{$SERVER_PATH}', service_path) + content = content.replace('{$SERVER_APP}', service_path + '/sphinx') + return content + + +def status(): + data = mw.execShell( + "ps -ef|grep sphinx |grep -v grep | grep -v mdserver-web | awk '{print $2}'") + # print(data) + if data[0] == '': + return 'stop' + return 'start' + + +def mkdirAll(): + content = mw.readFile(getConf()) + rep = r'path\s*=\s*(.*)' + p = re.compile(rep) + tmp = p.findall(content) + + for x in tmp: + if x.find('binlog') != -1: + mw.execShell('mkdir -p ' + x) + else: + mw.execShell('mkdir -p ' + os.path.dirname(x)) + + +def initDreplace(): + + file_tpl = getInitDTpl() + service_path = mw.getServerDir() + + initD_path = getServerDir() + '/init.d' + if not os.path.exists(initD_path): + os.mkdir(initD_path) + file_bin = initD_path + '/' + getPluginName() + + # initd replace + if not os.path.exists(file_bin): + content = mw.readFile(file_tpl) + content = contentReplace(content) + mw.writeFile(file_bin, content) + mw.execShell('chmod +x ' + file_bin) + + # config replace + conf_bin = getConf() + if not os.path.exists(conf_bin): + conf_content = mw.readFile(getConfTpl()) + conf_content = contentReplace(conf_content) + mw.writeFile(getServerDir() + '/sphinx.conf', conf_content) + + # systemd + systemDir = mw.systemdCfgDir() + systemService = systemDir + '/sphinx.service' + systemServiceTpl = getPluginDir() + '/init.d/sphinx.service.tpl' + if os.path.exists(systemDir) and not os.path.exists(systemService): + se_content = mw.readFile(systemServiceTpl) + se_content = se_content.replace('{$SERVER_PATH}', service_path) + mw.writeFile(systemService, se_content) + mw.execShell('systemctl daemon-reload') + + mkdirAll() + return file_bin + + +def checkIndexSph(): + content = mw.readFile(getConf()) + rep = r'path\s*=\s*(.*)' + p = re.compile(rep) + tmp = p.findall(content) + for x in tmp: + if x.find('binlog') != -1: + continue + else: + p = x + '.sph' + if os.path.exists(p): + return False + return True + + +def sphOp(method): + file = initDreplace() + + if not mw.isAppleSystem(): + data = mw.execShell('systemctl ' + method + ' sphinx') + if data[1] == '': + return 'ok' + return 'fail' + + data = mw.execShell(file + ' ' + method) + if data[1] == '': + return 'ok' + return data[1] + + +def start(): + # import tool_cron + # tool_cron.createBgTask() + return sphOp('start') + + +def stop(): + # import tool_cron + # tool_cron.removeBgTask() + return sphOp('stop') + + +def restart(): + return sphOp('restart') + + +def reload(): + return sphOp('reload') + + +def rebuild(): + file = initDreplace() + cmd = file + ' rebuild' + data = mw.execShell(cmd) + if data[0].find('successfully')<0: + return data[0].replace("\n","
") + return 'ok' + + +def initdStatus(): + if mw.isAppleSystem(): + return "Apple Computer does not support" + + shell_cmd = 'systemctl status sphinx | grep loaded | grep "enabled;"' + data = mw.execShell(shell_cmd) + if data[0] == '': + return 'fail' + return 'ok' + + +def initdInstall(): + if mw.isAppleSystem(): + return "Apple Computer does not support" + + mw.execShell('systemctl enable sphinx') + return 'ok' + + +def initdUinstall(): + if mw.isAppleSystem(): + return "Apple Computer does not support" + + mw.execShell('systemctl disable sphinx') + return 'ok' + + +def runLog(): + path = getConf() + content = mw.readFile(path) + rep = r'log\s*=\s*(.*)' + tmp = re.search(rep, content) + return tmp.groups()[0] + + +def getPort(): + path = getConf() + content = mw.readFile(path) + rep = r'listen\s*=\s*(.*)' + tmp = re.search(rep, content) + return tmp.groups()[0] + + +def queryLog(): + path = getConf() + content = mw.readFile(path) + rep = r'query_log\s*=\s*(.*)' + tmp = re.search(rep, content) + return tmp.groups()[0] + + +def runStatus(): + s = status() + if s != 'start': + return mw.returnJson(False, '没有启动程序') + + sys.path.append(getPluginDir() + "/class") + import sphinxapi + + sh = sphinxapi.SphinxClient() + port = getPort() + sh.SetServer('127.0.0.1', port) + info_status = sh.Status() + + rData = {} + for x in range(len(info_status)): + rData[info_status[x][0]] = info_status[x][1] + + return mw.returnJson(True, 'ok', rData) + + +def sphinxConfParse(): + file = getConf() + bin_dir = getServerDir() + content = mw.readFile(file) + rep = r'index\s(.*)' + sindex = re.findall(rep, content) + indexlen = len(sindex) + cmd = {} + cmd['cmd'] = bin_dir + '/bin/bin/indexer -c ' + bin_dir + '/sphinx.conf' + + cmd['index'] = [] + cmd_index = [] + cmd_delta = [] + if indexlen > 0: + for x in range(indexlen): + name = sindex[x].strip() + if name == '': + continue + if name.find(':') != -1: + cmd_delta.append(name.strip()) + else: + cmd_index.append(name.strip()) + + # print(cmd_index) + # print(cmd_delta) + + for ci in cmd_index: + val = {} + val['index'] = ci + + for cd in cmd_delta: + cd = cd.replace(" ", '') + if cd.find(":"+ci) > -1: + val['delta'] = cd.split(":")[0].strip() + break + + cmd['index'].append(val) + return cmd + + +def sphinxCmd(): + data = sphinxConfParse() + if 'index' in data: + return mw.returnJson(True, 'ok', data) + else: + return mw.returnJson(False, 'no index') + +def makeDbToSphinxTest(): + conf_file = getConf() + import sphinx_make + sph_make = sphinx_make.sphinxMake() + conf = sph_make.makeSqlToSphinxAll() + + mw.writeFile(conf_file,conf) + print(conf) + # makeSqlToSphinxTable() + return True + +def makeDbToSphinx(): + args = getArgs() + check = checkArgs(args, ['db','tables','is_delta','is_cover']) + if not check[0]: + return check[1] + + db = args['db'] + tables = args['tables'] + is_delta = args['is_delta'] + is_cover = args['is_cover'] + + if is_cover != 'yes': + return mw.returnJson(False,'暂时仅支持覆盖!') + + sph_file = getConf() + + import sphinx_make + sph_make = sphinx_make.sphinxMake() + + version_pl = getServerDir() + "/version.pl" + if os.path.exists(version_pl): + version = mw.readFile(version_pl).strip() + sph_make.setVersion(version) + + if not sph_make.checkDbName(db): + return mw.returnJson(False,'保留数据库名称,不可用!') + is_delta_bool = False + if is_delta == 'yes': + is_delta_bool = True + if is_cover == 'yes': + tables = tables.split(',') + content = sph_make.makeSqlToSphinx(db, tables, is_delta_bool) + mw.writeFile(sph_file,content) + return mw.returnJson(True,'设置成功!') + + return mw.returnJson(True,'测试中') + + +# 全量更新 +def updateAll(): + data = sphinxConfParse() + cmd = data['cmd'] + if not 'index' in data: + return '无更新' + index = data['index'] + + for x in range(len(index)): + cmd_index = cmd + ' ' + index[x]['index'] + ' --rotate' + print(cmd_index) + os.system(cmd_index) + return '' + +#增量更新 +def updateDelta(): + data = sphinxConfParse() + cmd = data['cmd'] + if not 'index' in data: + return '无更新' + index = data['index'] + + for x in range(len(index)): + if 'delta' in index[x]: + cmd_index = cmd + ' ' + index[x]['delta'] + ' --rotate' + print(cmd_index) + os.system(cmd_index) + + cmd_index_merge = cmd + ' --merge ' + index[x]['index'] + ' ' + index[x]['delta'] + ' --rotate' + print(cmd_index_merge) + os.system(cmd_index_merge) + else: + print(index[x]['index'],'no delta') + + return '' + +def installPreInspection(version): + data = mw.execShell('arch') + if data[0].strip().startswith('aarch'): + return '不支持aarch架构' + return 'ok' + +if __name__ == "__main__": + version = "3.1.1" + version_pl = getServerDir() + "/version.pl" + if os.path.exists(version_pl): + version = mw.readFile(version_pl).strip() + + func = sys.argv[1] + if func == 'status': + print(status()) + elif func == 'start': + print(start()) + elif func == 'stop': + print(stop()) + elif func == 'restart': + print(restart()) + elif func == 'reload': + print(reload()) + elif func == 'rebuild': + print(rebuild()) + elif func == 'initd_status': + print(initdStatus()) + elif func == 'initd_install': + print(initdInstall()) + elif func == 'initd_uninstall': + print(initdUinstall()) + elif func == 'install_pre_inspection': + print(installPreInspection(version)) + elif func == 'conf': + print(getConf()) + elif func == 'config_tpl': + print(configTpl()) + elif func == 'read_config_tpl': + print(readConfigTpl()) + elif func == 'run_log': + print(runLog()) + elif func == 'query_log': + print(queryLog()) + elif func == 'run_status': + print(runStatus()) + elif func == 'sphinx_cmd': + print(sphinxCmd()) + elif func == 'db_to_sphinx': + print(makeDbToSphinx()) + elif func == 'update_all': + print(updateAll()) + elif func == 'update_delta': + print(updateDelta()) + else: + print('error') diff --git a/plugins/manticoresearch/info.json b/plugins/manticoresearch/info.json new file mode 100755 index 000000000..8bcad0910 --- /dev/null +++ b/plugins/manticoresearch/info.json @@ -0,0 +1,19 @@ +{ + "sort": 7, + "ps": "是一个高性能开源搜索引擎,基于C++开发,由SphinxSearch项目演变而来,专注于提升搜索效率与扩展性", + "name": "manticoresearch", + "title": "manticoresearch", + "shell": "install.sh", + "versions":["7.4.6"], + "tip": "soft", + "install_pre_inspection":true, + "checks": "server/manticoresearch", + "path": "server/manticoresearch", + "display": 1, + "author": "manticoresoftware", + "date": "2025-07-21", + "home": "https://github.com/manticoresoftware/manticoresearch/", + "doc1": "https://manual.manticoresearch.com/Introduction", + "type": 0, + "pid": "2" +} \ No newline at end of file diff --git a/plugins/manticoresearch/init.d/sphinx.service.tpl b/plugins/manticoresearch/init.d/sphinx.service.tpl new file mode 100644 index 000000000..5524cef02 --- /dev/null +++ b/plugins/manticoresearch/init.d/sphinx.service.tpl @@ -0,0 +1,12 @@ +[Unit] +Description=Open Source Search Server +After=network.target + +[Service] +Type=forking +ExecStart={$SERVER_PATH}/sphinx/bin/bin/searchd -c {$SERVER_PATH}/sphinx/sphinx.conf +ExecReload=/bin/kill -USR2 $MAINPID +Restart=on-failure + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/plugins/manticoresearch/init.d/sphinx.tpl b/plugins/manticoresearch/init.d/sphinx.tpl new file mode 100644 index 000000000..8de34003c --- /dev/null +++ b/plugins/manticoresearch/init.d/sphinx.tpl @@ -0,0 +1,82 @@ +#! /bin/bash +# +# searchd: sphinx Daemon +# +# chkconfig: - 90 25 +# description: sphinx Daemon +# +### BEGIN INIT INFO +# Provides: sphinx +# Required-Start: $syslog +# Required-Stop: $syslog +# Should-Start: $local_fs +# Should-Stop: $local_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: sphinx - Document Index Daemon +# Description: sphinx - Document Index Daemon +### END INIT INFO + +APP_PATH={$SERVER_APP} +APP_CONF={$SERVER_APP}/sphinx.conf +prog="sphinx" + +start () { + echo -n $"Starting $prog: " + ${APP_PATH}/bin/bin/searchd -c ${APP_CONF} + if [ "$?" != 0 ] ; then + echo " failed" + exit 1 + else + echo " done" + fi +} + +rebuild () { + ${APP_PATH}/bin/bin/indexer -c ${APP_CONF} --all --rotate +} + + +stop () { + echo -n $"Stopping $prog: " + if [ ! -e ${APP_PATH}/index/searchd.pid ]; then + echo -n $"$prog is not running." + exit 1 + fi + kill `cat ${APP_PATH}/index/searchd.pid` + if [ "$?" != 0 ] ; then + echo " failed" + exit 1 + else + rm -f ${APP_PATH}/index/searchd.pid + echo " done" + fi +} + +restart () { + $0 stop + sleep 2 + $0 start +} + +# See how we were called. +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart|reload) + restart + ;; + rebuild) + rebuild + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|reload}" + exit 1 + ;; +esac + +exit $? \ No newline at end of file diff --git a/plugins/manticoresearch/install.sh b/plugins/manticoresearch/install.sh new file mode 100755 index 000000000..cdb7402a7 --- /dev/null +++ b/plugins/manticoresearch/install.sh @@ -0,0 +1,28 @@ +#!/bin/bash +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:~/bin:/opt/homebrew/bin +export PATH + +curPath=`pwd` +rootPath=$(dirname "$curPath") +rootPath=$(dirname "$rootPath") +serverPath=$(dirname "$rootPath") +sysName=`uname` +sysArch=`arch` + + +if [ -f ${rootPath}/bin/activate ];then + source ${rootPath}/bin/activate +fi + +ACTION=$1 +VERSION=$2 + +which apt +if [ "$?" == "0" ];then + sh -x $curPath/versions/apt/install.sh $1 $2 +fi + +which yum +if [ "$?" == "0" ];then + sh -x $curPath/versions/yum/install.sh $1 $2 +fi diff --git a/plugins/manticoresearch/js/sphinx.js b/plugins/manticoresearch/js/sphinx.js new file mode 100755 index 000000000..1fd4aa637 --- /dev/null +++ b/plugins/manticoresearch/js/sphinx.js @@ -0,0 +1,311 @@ +function spPostMin(method, args, callback){ + + var req_data = {}; + req_data['name'] = 'sphinx'; + req_data['func'] = method; + + if (typeof(args) != 'undefined' && args!=''){ + req_data['args'] = JSON.stringify(args); + } + + $.post('/plugins/run', req_data, function(data) { + if (!data.status){ + layer.msg(data.msg,{icon:0,time:2000,shade: [0.3, '#000']}); + return; + } + + if(typeof(callback) == 'function'){ + callback(data); + } + },'json'); +} + +function myPost(method, args, callback, title){ + var _args = null; + if (typeof(args) == 'string'){ + _args = JSON.stringify(toArrayObject(args)); + } else { + _args = JSON.stringify(args); + } + + var _title = '正在获取...'; + if (typeof(title) != 'undefined'){ + _title = title; + } + + $.post('/plugins/run', {name:'mysql', func:method, args:_args}, function(data) { + if (!data.status){ + layer.msg(data.msg,{icon:0,time:2000,shade: [0.3, '#000']}); + return; + } + + if(typeof(callback) == 'function'){ + callback(data); + } + },'json'); +} + + +function spPost(method, args, callback){ + var loadT = layer.msg('正在获取...', { icon: 16, time: 0, shade: 0.3 }); + spPostMin(method,args,function(data){ + layer.close(loadT); + if(typeof(callback) == 'function'){ + callback(data); + } + }); +} + +function commonFunc(){ + var con = ''; + con += '   '; + $(".soft-man-con").html(con); +} + +function autoMakeConf(){ + var xm_db_list; + + var con = '
    '; + con += '
  • 如果数据量比较大,第一次启动会失败!(可通过手动建立索引)
    • '; + con += '
  • 以下内容,需手动加入计划任务。
    • '; + layer.open({ + type: 1, + area: ['380px','350px'], + title: '自动创建配置', + closeBtn: 1, + shift: 5, + shadeClose: true, + btn:["提交","关闭"], + content: "
    \ +
    \ + 选择数据库\ +
    \ + \ +
    \ +
    \ +
    \ + 选择表\ +
    \ +
    \ +
    \ +
    \ +
    \ + 是否增量\ +
    \ + \ +
    \ +
    \ +
    \ + 是否覆盖配置\ +
    \ + \ +
    \ +
    \ +
      \ +
    • 具体配置，仍须手动修改!!!
      • \ +
    • 增量索引,需要有更新权限,主从分离时,需要主库配置
      • \ +
    \ +
    \ + ", + + success:function(l,i){ + $(l).find('.layui-layer-content').css('overflow','visible'); + + xm_db_list = xmSelect.render({ + el: '#table', + repeat: false, + toolbar: {show: true}, + data: [], + }); + + myPost('get_db_list', {"page":1,"page_size":20}, function(data){ + var rdata = $.parseJSON(data.data); + var dblist = rdata.data; + + var db_html = ''; + for (var i = 0; i < dblist.length; i++) { + db_html += ""; + } + + if (dblist.length > 0){ + initDbSelect(dblist[0]['name']); + } + $('select[name="dbname"]').html(db_html); + }); + + $('select[name="dbname"]').change(function(){ + var db = $('select[name="dbname"]').val(); + initDbSelect(db); + }); + + }, + yes:function(index){ + var args = {} + args['db'] = $('select[name="dbname"]').val(); + args['is_delta'] = $('select[name="is_delta"]').val(); + args['is_cover'] = $('select[name="is_cover"]').val(); + args['tables'] = xm_db_list.getValue('value').join(','); + // console.log(args); + spPost('db_to_sphinx', args, function(rdata){ + var rdata = $.parseJSON(rdata.data); + // console.log(rdata); + showMsg(rdata.msg,function(){ + if (rdata.status){ + layer.close(index); + confirmRebuildIndex(); + } + },{icon: rdata.status ? 1 : 2}, 2000); + }); + } + + }); + + function initDbSelect(db){ + if (db == ''){ + return; + } + getDbInfo(db, function(rdata){ + var rdata = $.parseJSON(rdata.data); + var tables = rdata.tables; + + var idx_db = []; + for (var i = 0; i < tables.length; i++) { + var t = {}; + t['name'] = tables[i]['table_name']; + t['value'] = tables[i]['table_name']; + idx_db.push(t); + } + xm_db_list = xmSelect.render({el: '#table', filterable: true,repeat: false,toolbar: {show: true},data: idx_db,}); + }); + } + + function getDbInfo(db_name, callback){ + myPost('get_db_info', {name:db_name}, function(data){ + callback(data); + }); + } +} + +function rebuildIndex(){ + spPost('rebuild', '', function(data){ + if (data.data == 'ok'){ + layer.msg('重建成功!',{icon:1,time:2000,shade: [0.3, '#000']}); + } else { + layer.msg(data.data,{icon:2,time:10000,shade: [0.3, '#000']}); + } + }); +} + +function confirmRebuildIndex(){ + layer.confirm("是否重建索引?", {icon:3,closeBtn: 1} , function(){ + rebuildIndex(); + }); +} + + +function tryRebuildIndex(){ + layer.confirm("修改配置后，是否尝试重建索引!", {icon:3,closeBtn: 1} , function(){ + rebuildIndex(); + }); +} + + +function secToTime(s) { + var t; + if(s > -1){ + var hour = Math.floor(s/3600); + var min = Math.floor(s/60) % 60; + var sec = s % 60; + if(hour < 10) { + t = '0'+ hour + ":"; + } else { + t = hour + ":"; + } + + if(min < 10){t += "0";} + t += min + ":"; + if(sec < 10){t += "0";} + t += sec.toFixed(2); + } + return t; +} + + +function runStatus(){ + spPost('run_status', '', function(data){ + var rdata = $.parseJSON(data.data); + if (!rdata['status']){ + layer.msg(rdata['msg'],{icon:2,time:2000,shade: [0.3, '#000']}); + return; + } + var idata = rdata.data; + + var tbody = ''; + for (var i in idata) { + tbody += ''+i+'' + idata[i] + ''+i+''; + } + + var con = '
    \ + \ + \ + \ + \ + \ + \ +
    运行时间' + secToTime(idata.uptime) + '每秒查询' + parseInt(parseInt(idata.queries) / parseInt(idata.uptime)) + '
    总连接次数' + idata.connections + 'work_queue_length' +idata.work_queue_length + '
    agent_connect' + idata.agent_connect+ 'workers_active' + idata.workers_active + '
    agent_retry' + idata.agent_retry + 'workers_total' + idata.workers_total + '
    \ + \ + \ + \ + '+tbody+'\ + \ +
    \ +
    '; + + $(".soft-man-con").html(con); + }); +} + +function readme(){ + spPost('sphinx_cmd', '', function(data){ + + var rdata = $.parseJSON(data.data); + if (!rdata['status']){ + layer.msg(rdata['msg'],{icon:2,time:2000,shade: [0.3, '#000']}); + return; + } + + // console.log(rdata['data']); + var con = '
      '; + + con += '
    • 如果数据量比较大,第一次启动会失败!(可通过手动建立索引)
      • '; + con += '
    • 以下内容,需手动加入计划任务。
      • '; + + con += '
    • 全量:' + rdata['data']['cmd'] + ' --all --rotate
      • '; + + //主索引 + for (var i = 0; i < rdata['data']['index'].length; i++) { + var index_kv = rdata['data']['index'][i]; + var index = index_kv['index']; + // console.log(index); + con += '
    • 主索引 :' + rdata['data']['cmd'] + ' '+ index +' --rotate
      • '; + if (typeof(index_kv['delta']) != 'undefined'){ + var delta = index_kv['delta']; + con += '
    • 增量索引 :' + rdata['data']['cmd'] + ' '+ delta +' --rotate
      • '; + con += '
    • 合并索引 :' + rdata['data']['cmd'] + ' --merge '+ index + ' ' + delta +' --rotate
      • '; + } + } + con += '
    '; + + $(".soft-man-con").html(con); + }); + +} + diff --git a/plugins/manticoresearch/tool_cron.py b/plugins/manticoresearch/tool_cron.py new file mode 100644 index 000000000..65afe5d73 --- /dev/null +++ b/plugins/manticoresearch/tool_cron.py @@ -0,0 +1,211 @@ +# coding:utf-8 + +import sys +import io +import os +import time +import json + +web_dir = os.getcwd() + "/web" +if os.path.exists(web_dir): + sys.path.append(web_dir) + os.chdir(web_dir) + +import core.mw as mw +from utils.crontab import crontab as MwCrontab + + + +app_debug = False +if mw.isAppleSystem(): + app_debug = True + + +def getPluginName(): + return 'manticoresearch' + + +def getPluginDir(): + return mw.getPluginDir() + '/' + getPluginName() + + +def getServerDir(): + return mw.getServerDir() + '/' + getPluginName() + + +def getTaskConf(): + conf = getServerDir() + "/cron_config.json" + return conf + +def getTaskDeltaConf(): + conf = getServerDir() + "/cron_delta_config.json" + return conf + +def getConfigData(): + conf = getTaskConf() + if os.path.exists(conf): + return json.loads(mw.readFile(getTaskConf())) + return { + "task_id": -1, + "period": "day-n", + "where1": "1", + "hour": "0", + "minute": "15", + } + +def getConfigDeltaData(): + conf = getTaskDeltaConf() + if os.path.exists(conf): + return json.loads(mw.readFile(getTaskDeltaConf())) + return { + "task_id": -1, + "period": "minute-n", + "where1": "3", + "hour": "0", + "minute": "0", + } + + +def createBgTask(): + removeBgTask() + removeDeltaBgTask() + + createBgTaskByName(getPluginName()) + createBgTaskDeltaByName(getPluginName()) + return True + + +def createBgTaskByName(name): + args = getConfigData() + _name = "[勿删]Sphinx全量更新[" + name + "]" + res = mw.M("crontab").field("id, name").where("name=?", (_name,)).find() + if res: + return True + + if "task_id" in args and args["task_id"] > 0: + res = mw.M("crontab").field("id, name").where("id=?", (args["task_id"],)).find() + if res and res["id"] == args["task_id"]: + print("计划任务已经存在!") + return True + + mw_dir = mw.getPanelDir() + cmd = ''' +mw_dir=%s +rname=%s +plugin_path=%s +script_path=%s +logs_file=$plugin_path/${rname}.log +''' % (mw_dir, name, getServerDir(), getPluginDir()) + cmd += 'echo "★【`date +"%Y-%m-%d %H:%M:%S"`】 STSRT★" >> $logs_file' + "\n" + cmd += 'echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" >> $logs_file' + "\n" + cmd += 'echo "python3 $script_path/index.py update_all"' + "\n" + cmd += 'cd $mw_dir && python3 $script_path/index.py update_all' + "\n" + cmd += 'echo "【`date +"%Y-%m-%d %H:%M:%S"`】 END★" >> $logs_file' + "\n" + cmd += 'echo "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<" >> $logs_file' + "\n" + + params = { + 'name': _name, + 'type': args['period'], + 'week': "", + 'where1': args['where1'], + 'hour': args['hour'], + 'minute': args['minute'], + 'save': "", + 'backup_to': "", + 'stype': "toShell", + 'sname': '', + 'sbody': cmd, + 'url_address': '', + } + + task_id = MwCrontab.instance().add(params) + if task_id > 0: + args["task_id"] = task_id + args["name"] = name + mw.writeFile(getTaskConf(), json.dumps(args)) + +def createBgTaskDeltaByName(name): + args = getConfigDeltaData() + _name = "[勿删]Sphinx增量更新[" + name + "]" + res = mw.M("crontab").field("id, name").where("name=?", (_name,)).find() + if res: + return True + + if "task_id" in args and args["task_id"] > 0: + res = mw.M("crontab").field("id, name").where("id=?", (args["task_id"],)).find() + if res and res["id"] == args["task_id"]: + print("计划任务已经存在!") + return True + + mw_dir = mw.getPanelDir() + cmd = ''' +mw_dir=%s +rname=%s +plugin_path=%s +script_path=%s +logs_file=$plugin_path/${rname}.log +''' % (mw_dir, name, getServerDir(), getPluginDir()) + cmd += 'echo "★【`date +"%Y-%m-%d %H:%M:%S"`】 STSRT★" >> $logs_file' + "\n" + cmd += 'echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" >> $logs_file' + "\n" + cmd += 'echo "python3 $script_path/index.py update_delta"' + "\n" + cmd += 'cd $mw_dir && python3 $script_path/index.py update_delta' + "\n" + cmd += 'echo "【`date +"%Y-%m-%d %H:%M:%S"`】 END★" >> $logs_file' + "\n" + cmd += 'echo "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<" >> $logs_file' + "\n" + + params = { + 'name': _name, + 'type': args['period'], + 'week': "", + 'where1': args['where1'], + 'hour': args['hour'], + 'minute': args['minute'], + 'save': "", + 'backup_to': "", + 'stype': "toShell", + 'sname': '', + 'sbody': cmd, + 'url_address': '', + } + + task_id = MwCrontab.instance().add(params) + if task_id > 0: + args["task_id"] = task_id + args["name"] = name + mw.writeFile(getTaskConf(), json.dumps(args)) + + +def removeBgTask(): + cfg = getConfigData() + if "task_id" in cfg and cfg["task_id"] > 0: + res = mw.M("crontab").field("id, name").where( + "id=?", (cfg["task_id"],)).find() + if res and res["id"] == cfg["task_id"]: + data = MwCrontab.instance().delete(cfg["task_id"]) + if data[0]: + cfg["task_id"] = -1 + mw.writeFile(getTaskConf(), json.dumps(cfg)) + return True + return False + +def removeDeltaBgTask(): + cfg = getConfigDeltaData() + if "task_id" in cfg and cfg["task_id"] > 0: + res = mw.M("crontab").field("id, name").where( + "id=?", (cfg["task_id"],)).find() + if res and res["id"] == cfg["task_id"]: + data = MwCrontab.instance().delete(cfg["task_id"]) + if data[0]: + cfg["task_id"] = -1 + mw.writeFile(getTaskDeltaConf(), json.dumps(cfg)) + return True + return False + + +if __name__ == "__main__": + if len(sys.argv) > 1: + action = sys.argv[1] + if action == "remove": + removeBgTask() + removeDeltaBgTask() + elif action == "add": + createBgTask() diff --git a/plugins/manticoresearch/tpl/discuz.conf b/plugins/manticoresearch/tpl/discuz.conf new file mode 100644 index 000000000..ab2ba35fe --- /dev/null +++ b/plugins/manticoresearch/tpl/discuz.conf @@ -0,0 +1,127 @@ +# +# Discuz Sphinx Config File +# + +indexer +{ + mem_limit = 32M + write_buffer = 64M +} + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$SERVER_APP}/index/searchd.log + query_log = {$SERVER_APP}/index/query.log + read_timeout = 5 + max_children = 0 + pid_file = {$SERVER_APP}/index/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + #workers = threads # for RT to work + binlog_path = {$SERVER_APP}/index/binlog +} + + +source pre_forum_thread +{ + type = mysql + sql_host = 127.0.0.1 + sql_user = bbs + sql_pass = bbs + sql_db = bbs + sql_port = 3306 + sql_query_pre = SET NAMES UTF8 + sql_query_pre = SET SESSION query_cache_type=OFF + sql_query_pre = REPLACE INTO bbs_common_sphinxcounter SELECT 1, MAX(tid) FROM bbs_forum_thread + sql_query = SELECT t.tid as id,t.tid,t.subject,t.digest,t.displayorder,t.authorid,t.lastpost,t.special \ + FROM bbs_forum_thread AS t WHERE t.tid>=$start AND t.tid<=$end + sql_query_range = SELECT (SELECT MIN(tid) FROM bbs_forum_thread),maxid FROM bbs_common_sphinxcounter WHERE indexid=1 + + sql_range_step = 5000 + sql_attr_uint = tid + + sql_attr_uint = digest + sql_attr_uint = displayorder + sql_attr_uint = authorid + sql_attr_uint = special + sql_attr_timestamp =lastpost +} + +index pre_forum_thread +{ + source = pre_forum_thread + path = {$SERVER_APP}/index/db/pre_forum_thread/index + + min_word_len = 2 + html_strip = 1 + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} + + +# threads_minute +source pre_forum_thread_minute : pre_forum_thread +{ + sql_query_pre = SET NAMES UTF8 + sql_query_pre = SET SESSION query_cache_type=OFF + sql_query_range = SELECT maxid-1,(SELECT MAX(tid) FROM bbs_forum_thread) FROM bbs_common_sphinxcounter WHERE indexid=1 +} + +# threads_minute +index pre_forum_thread_minute : pre_forum_thread +{ + source = pre_forum_thread_minute + path = {$SERVER_APP}/index/db/pre_forum_thread/pre_forum_thread_minute +} + +#posts +source pre_forum_post : pre_forum_thread +{ + type = mysql + sql_query_pre = + sql_query_pre = SET NAMES UTF8 + sql_query_pre = SET SESSION query_cache_type=OFF + sql_query_pre = REPLACE INTO bbs_common_sphinxcounter SELECT 2, MAX(pid) FROM bbs_forum_post + sql_query = SELECT p.pid AS id,p.tid,p.subject,p.message,t.digest,t.displayorder,t.authorid,t.lastpost,t.special \ + FROM bbs_forum_post AS p LEFT JOIN bbs_forum_thread AS t USING(tid) where p.pid >=$start and p.pid <=$end \ + AND p.first=1 + sql_query_range = SELECT (SELECT MIN(pid) FROM bbs_forum_post),maxid FROM bbs_common_sphinxcounter WHERE indexid=2 + sql_range_step = 5000 + sql_attr_uint = tid + sql_attr_uint = digest + sql_attr_uint= displayorder + sql_attr_uint = authorid + sql_attr_uint = special + sql_attr_timestamp = lastpost +} + +#posts +index pre_forum_post +{ + source = pre_forum_post + path = {$SERVER_APP}/index/db/pre_forum_thread/pre_forum_post + + min_word_len = 2 + html_strip = 1 + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} + +#pre_forum_post_minute +source pre_forum_post_minute : pre_forum_post +{ + sql_query_pre = SET NAMES UTF8 + sql_query_pre = SET SESSION query_cache_type=OFF + sql_query_range = SELECT maxid-1,(SELECT MAX(pid) FROM bbs_forum_post) FROM bbs_common_sphinxcounter WHERE indexid=2 +} + +#pre_forum_post_minute +index pre_forum_post_minute : pre_forum_post +{ + source = pre_forum_thread + path = {$SERVER_APP}/index/db/pre_forum_thread/pre_forum_post_minute +} + diff --git a/plugins/manticoresearch/tpl/maccms.conf b/plugins/manticoresearch/tpl/maccms.conf new file mode 100644 index 000000000..ad70ae6f8 --- /dev/null +++ b/plugins/manticoresearch/tpl/maccms.conf @@ -0,0 +1,95 @@ +# +# Maccms Sphinx Config File +# + +# 创建统计表 +# CREATE TABLE `sph_counter` ( `id` bigint(11) unsigned NOT NULL AUTO_INCREMENT, `counter_id` bigint(20) unsigned NOT NULL,`max_doc_id` bigint(20) unsigned NOT NULL,PRIMARY KEY (`id`)) + +indexer +{ + mem_limit = 32M + write_buffer = 64M +} + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$SERVER_APP}/index/searchd.log + query_log = {$SERVER_APP}/index/query.log + read_timeout = 5 + max_children = 0 + pid_file = {$SERVER_APP}/index/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + #workers = threads # for RT to work + binlog_path = {$SERVER_APP}/index/binlog +} + + +# -------- maccms 全量更新 start -------- + +source mac_vod +{ + type = mysql + sql_host = 127.0.0.1 + sql_user = maccms + sql_pass = maccms + sql_db = maccms + sql_port = 3306 + sql_query_range = SELECT min(vod_id), max(vod_id) FROM mac_vod + sql_query_pre = SET NAMES UTF8 + sql_query_pre = SET SESSION query_cache_type=OFF + sql_query = SELECT vod_id as id, vod_name, vod_tag, vod_class, vod_actor, vod_director, vod_time,vod_time_add FROM mac_vod WHERE vod_status=1 and vod_id>=$start AND vod_id<=$end + sql_range_step = 1000 + + + sql_attr_uint = vod_id + sql_attr_timestamp = vod_time + sql_attr_timestamp = vod_time_add + sql_field_string = vod_name + + sql_query_post = UPDATE sph_counter SET max_doc_id=(SELECT MAX(vod_id) FROM mac_vod) where counter_id=1 +} + +index mac_vod +{ + source = mac_vod + path = {$SERVER_APP}/index/db/maccms/index + + min_word_len = 2 + html_strip = 1 + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} + +# -------- maccms 全量更新 end -------- + + +# -------- maccms 增量更新 start -------- + + +source mac_vod_delta : mac_vod +{ + sql_query_pre = SET NAMES utf8 + + sql_query_range = SELECT (SELECT max_doc_id FROM `sph_counter` where counter_id=1) as min, (SELECT max(vod_id) FROM mac_vod) as max + + sql_query = SELECT vod_id as id, vod_name, vod_tag, vod_class, vod_actor, vod_director, vod_time,vod_time_add FROM mac_vod WHERE vod_status=1 and vod_id>=$start AND vod_id<=$end + + sql_query_post = UPDATE sph_counter SET max_doc_id=(SELECT MAX(vod_id) FROM mac_vod) where counter_id=1 +} + +index mac_vod_delta : mac_vod +{ + source = mac_vod_delta + path = {$SERVER_APP}/index/db/maccms_delta/index + + min_word_len = 2 + html_strip = 1 + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} + +# -------- maccms 增量更新 end -------- \ No newline at end of file diff --git a/plugins/manticoresearch/tpl/none.conf b/plugins/manticoresearch/tpl/none.conf new file mode 100755 index 000000000..de9e5f270 --- /dev/null +++ b/plugins/manticoresearch/tpl/none.conf @@ -0,0 +1,28 @@ +# +# Minimal Sphinx configuration sample (clean, simple, functional) +# + + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$SERVER_APP}/index/searchd.log + query_log = {$SERVER_APP}/index/query.log + pid_file = {$SERVER_APP}/index/searchd.pid + #workers = threads # for RT to work + binlog_path = {$SERVER_APP}/index/binlog + read_timeout = 5 + max_children = 0 + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 +} + +index mydocs +{ + type = rt + path = {$SERVER_APP}/bin/doc + rt_field = title + rt_attr_json = j +} \ No newline at end of file diff --git a/plugins/manticoresearch/tpl/qbittorrent.conf b/plugins/manticoresearch/tpl/qbittorrent.conf new file mode 100755 index 000000000..46f61331d --- /dev/null +++ b/plugins/manticoresearch/tpl/qbittorrent.conf @@ -0,0 +1,54 @@ +# +# Minimal Sphinx configuration sample (clean, simple, functional) +# + +indexer +{ + mem_limit = 32M +} + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$SERVER_APP}/index/searchd.log + query_log = {$SERVER_APP}/index/query.log + read_timeout = 5 + max_children = 0 + pid_file = {$SERVER_APP}/index/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + #workers = threads # for RT to work + binlog_path = {$SERVER_APP}/binlog +} + +source qbittorrent +{ + type = mysql + + sql_host = 127.0.0.1 + sql_user = qbittorrent + sql_pass = qbittorrent + sql_db = qbittorrent + sql_port = 3306 # optional, default is 3306 + + sql_query_range = SELECT min(id), max(id) FROM pl_hash_list + sql_range_step = 1000 + + sql_query_pre = SET NAMES utf8 + sql_query = SELECT id, name, UNIX_TIMESTAMP(create_time) AS create_time \ + FROM pl_hash_list where status=0 and id >= $start AND id <= $end + + sql_attr_timestamp = create_time +} + + +index qbittorrent +{ + source = qbittorrent + path = {$SERVER_APP}/index/db/qbittorrent/index + + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} diff --git a/plugins/manticoresearch/tpl/simdht.conf b/plugins/manticoresearch/tpl/simdht.conf new file mode 100755 index 000000000..51d5b26ea --- /dev/null +++ b/plugins/manticoresearch/tpl/simdht.conf @@ -0,0 +1,59 @@ +# +# Minimal Sphinx configuration sample (clean, simple, functional) +# + +indexer +{ + mem_limit = 32M +} + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$SERVER_APP}/index/searchd.log + query_log = {$SERVER_APP}/index/query.log + read_timeout = 5 + max_children = 0 + pid_file = {$SERVER_APP}/index/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + #workers = threads # for RT to work + binlog_path = {$SERVER_APP}/index/binlog +} + +source search_hash +{ + type = mysql + + sql_host = 127.0.0.1 + sql_user = ssbc + sql_pass = ssbc + sql_db = ssbc + sql_port = 3306 # optional, default is 3306 + + sql_query_range = SELECT min(id), max(id) FROM search_hash + sql_range_step = 1000 + + sql_query_pre = SET NAMES utf8 + sql_query = \ + SELECT id, name, CRC32(category) AS category, length, UNIX_TIMESTAMP(create_time) AS create_time, UNIX_TIMESTAMP(last_seen) AS last_seen\ + FROM search_hash where id >= $start AND id <= $end AND is_has=1 + + sql_attr_bigint = length + sql_attr_timestamp = create_time + sql_attr_timestamp = last_seen + sql_attr_uint = category + +} + + +index search_hash +{ + source = search_hash + path = {$SERVER_APP}/index/db/search_hash/index + + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} diff --git a/plugins/manticoresearch/tpl/simdht_delta.conf b/plugins/manticoresearch/tpl/simdht_delta.conf new file mode 100755 index 000000000..e20eed842 --- /dev/null +++ b/plugins/manticoresearch/tpl/simdht_delta.conf @@ -0,0 +1,79 @@ +# +# Minimal Sphinx configuration sample (clean, simple, functional) +# + +indexer +{ + mem_limit = 128M +} + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$SERVER_APP}/index/searchd.log + query_log = {$SERVER_APP}/index/query.log + read_timeout = 5 + max_children = 0 + pid_file = {$SERVER_APP}/index/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + #workers = threads # for RT to work + binlog_path = {$SERVER_APP}/index/binlog +} + +source search_hash +{ + type = mysql + + sql_host = 127.0.0.1 + sql_user = ssbc + sql_pass = ssbc + sql_db = ssbc + sql_port = 3306 # optional, default is 3306 + + sql_query_pre = UPDATE sph_counter SET max_doc_id=(SELECT MAX(id) FROM search_hash) where counter_id=1 + + sql_query_range = SELECT min(id), max(id) FROM search_hash + sql_range_step = 1000 + + sql_query_pre = SET NAMES utf8 + sql_query = SELECT * from ( SELECT s1.id id, s1.name name, s1.category category, s1.length length, UNIX_TIMESTAMP(s1.create_time) as create_time, UNIX_TIMESTAMP(s1.last_seen) as last_seen, s2.file_list file_list, s1.info_hash info_hash FROM search_hash s1 left join search_filelist s2 on s1.info_hash=s2.info_hash where s1.is_has=1 and s1.id >= $start AND s1.id <= $end ) as s WHERE s.file_list is not null + + sql_attr_bigint = length + sql_attr_timestamp = create_time + sql_attr_timestamp = last_seen + sql_attr_uint = category +} + +source search_hash_delta : search_hash +{ + sql_query_range = SELECT (SELECT max_doc_id FROM `sph_counter` where counter_id=1) as min, (SELECT max(id) FROM search_hash) as max + sql_query_pre = SET NAMES utf8 + sql_query = SELECT * from ( SELECT s1.id id, s1.name name, s1.category category, s1.length length, UNIX_TIMESTAMP(s1.create_time) as create_time, UNIX_TIMESTAMP(s1.last_seen) as last_seen, s2.file_list file_list, s1.info_hash info_hash FROM search_hash s1 left join search_filelist s2 on s1.info_hash=s2.info_hash where s1.is_has=1 and s1.id >= $start AND s1.id <= $end ) as s WHERE s.file_list is not null + sql_attr_bigint = length + sql_attr_timestamp = create_time + sql_attr_timestamp = last_seen + sql_attr_uint = category + sql_query_post = UPDATE sph_counter SET max_doc_id=(SELECT MAX(id) FROM search_hash) where counter_id=1 +} + + +index search_hash +{ + source = search_hash + path = {$SERVER_APP}/index/db/search_hash/index + + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} + +index search_hash_delta : search_hash +{ + source = search_hash_delta + path = {$SERVER_APP}/index/db/search_hash_delta/index + + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} diff --git a/plugins/manticoresearch/tpl/video.conf b/plugins/manticoresearch/tpl/video.conf new file mode 100755 index 000000000..2a93c22e5 --- /dev/null +++ b/plugins/manticoresearch/tpl/video.conf @@ -0,0 +1,58 @@ +# +# Minimal Sphinx configuration sample (clean, simple, functional) +# + +indexer +{ + mem_limit = 32M +} + +searchd +{ + listen = 9312 + listen = 9306:mysql41 + log = {$SERVER_APP}/index/searchd.log + query_log = {$SERVER_APP}/index/query.log + read_timeout = 5 + max_children = 0 + pid_file = {$SERVER_APP}/index/searchd.pid + seamless_rotate = 1 + preopen_indexes = 1 + unlink_old = 1 + #workers = threads # for RT to work + binlog_path = {$SERVER_APP}/binlog +} + +source sp_video +{ + type = mysql + + sql_host = 127.0.0.1 + sql_user = video_spider + sql_pass = video_spider + sql_db = video_spider + sql_port = 3306 # optional, default is 3306 + + sql_query_range = SELECT min(id), max(id) FROM sp_video + sql_range_step = 1000 + + sql_query_pre = SET NAMES utf8 + sql_query = SELECT id, name, UNIX_TIMESTAMP(create_time) AS create_time, UNIX_TIMESTAMP(update_time) AS update_time \ + FROM sp_video where id >= $start AND id <= $end + + sql_attr_bigint = length + sql_attr_timestamp = create_time + sql_attr_timestamp = update_time + sql_attr_uint = category + +} + + +index sp_video +{ + source = sp_video + path = {$SERVER_APP}/index/db/sp_video/index + + ngram_len = 1 + ngram_chars = U+3000..U+2FA1F +} diff --git a/plugins/manticoresearch/versions/apt/install.sh b/plugins/manticoresearch/versions/apt/install.sh new file mode 100755 index 000000000..1f69b82c8 --- /dev/null +++ b/plugins/manticoresearch/versions/apt/install.sh @@ -0,0 +1,171 @@ +#!/bin/bash +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:~/bin:/opt/homebrew/bin +export PATH + +curPath=`pwd` +rootPath=$(dirname "$curPath") +rootPath=$(dirname "$rootPath") +serverPath=$(dirname "$rootPath") +sysName=`uname` +sysArch=`arch` + + +if [ -f ${rootPath}/bin/activate ];then + source ${rootPath}/bin/activate +fi + +# cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py rebuild +# cd /www/server/mdserver-web/plugins/sphinx && bash install.sh install 3.6.1 +# cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py db_to_sphinx && /www/server/sphinx/bin/bin/indexer -c /www/server/sphinx/sphinx.conf --all --rotate +# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate + +# cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py sphinx_cmd + +# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate + +# cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py start +bash ${rootPath}/scripts/getos.sh +# echo "bash ${rootPath}/scripts/getos.sh" +OSNAME="macos" +if [ -f ${rootPath}/data/osname.pl ];then + OSNAME=`cat ${rootPath}/data/osname.pl` +fi + +if [ "${OSNAME}" == "centos" ] || + [ "${OSNAME}" == "fedora" ] || + [ "${OSNAME}" == "alma" ]; then + yum install -y postgresql-libs unixODBC +fi + +# http://sphinxsearch.com/files/sphinx-3.7.1-da9f8a4-linux-amd64.tar.gz + +VERSION=$2 + +# echo $VERSION + +if [ "$VERSION" == "3.1.1" ];then + VERSION_NUM=${VERSION}-612d99f +elif [ "$VERSION" == "3.2.1" ]; then + VERSION_NUM=${VERSION}-f152e0b +elif [ "$VERSION" == "3.3.1" ]; then + VERSION_NUM=${VERSION}-b72d67b +elif [ "$VERSION" == "3.4.1" ]; then + VERSION_NUM=${VERSION}-efbcc65 +elif [ "$VERSION" == "3.5.1" ]; then + VERSION_NUM=${VERSION}-82c60cb +elif [ "$VERSION" == "3.6.1" ]; then + VERSION_NUM=${VERSION}-c9dbeda +elif [ "$VERSION" == "3.7.1" ]; then + VERSION_NUM=${VERSION}-da9f8a4 +elif [ "$VERSION" == "3.8.1" ]; then + VERSION_NUM=${VERSION}-d25e0bb +fi + +# echo $VERSION_NUM + +Install_App() +{ + echo '正在安装Sphinx...' + mkdir -p $serverPath/sphinx + + SPHINX_DIR=${serverPath}/source/sphinx + mkdir -p $SPHINX_DIR + + SPH_NAME=amd64 + if [ "$sysArch" == "arm64" ];then + SPH_NAME=amd64 + elif [ "$sysArch" == "x86_64" ]; then + SPH_NAME=amd64 + elif [ "$sysArch" == "aarch64" ]; then + SPH_NAME=aarch64 + fi + + if [ "$sysName" == "Darwin" ] && [ "$VERSION" == "3.7.1" ];then + SPH_NAME=aarch64 + fi + + SPH_SYSNAME=linux + if [ $sysName == 'Darwin' ]; then + SPH_SYSNAME=darwin + elif [ "$sysName" == "aarch64" ]; then + SPH_NAME=aarch64 + elif [ "$sysName" == "freebsd" ]; then + SPH_NAME=freebsd + fi + + if [ "$SPH_SYSNAME" == "linux" ];then + glibc_ver=`ldd --version | grep libc | awk -F ')' '{print $2}'|awk '{gsub(/^\s+|\s+$/, "");print}'` + if [ "$VERSION" == "3.7.1" ] && [ `echo "2.29 > $glibc_ver " | bc` -eq 1 ];then + SPH_NAME=${SPH_NAME}-glibc2.17 + fi + if [ "$VERSION" == "3.6.1" ] && [ `echo "2.29 > $glibc_ver " | bc` -eq 1 ];then + SPH_NAME=${SPH_NAME}-glibc2.17 + fi + fi + + + FILE_NAME=sphinx-${VERSION_NUM}-${SPH_SYSNAME}-${SPH_NAME} + FILE_TGZ=${FILE_NAME}.tar.gz + + echo $FILE_TGZ + # curl -sSLo ${SPHINX_DIR}/${FILE_TGZ} http://sphinxsearch.com/files/${FILE_TGZ} + if [ ! -f ${SPHINX_DIR}/${FILE_TGZ} ];then + wget --no-check-certificate -O ${SPHINX_DIR}/${FILE_TGZ} http://sphinxsearch.com/files/${FILE_TGZ} + fi + + cd ${SPHINX_DIR} && tar -zxvf ${FILE_TGZ} + + if [ "$?" == "0" ];then + mkdir -p $SPHINX_DIR + cp -rf ${SPHINX_DIR}/sphinx-${VERSION}/ $serverPath/sphinx/bin + fi + + if [ -d $serverPath/sphinx ];then + echo "${VERSION}" > $serverPath/sphinx/version.pl + echo '安装Sphinx完成' + cd ${rootPath} && python3 ${rootPath}/plugins/sphinx/index.py start + + if [ $sysName != 'Darwin' ]; then + cd ${rootPath} && python3 ${rootPath}/plugins/sphinx/index.py initd_install + fi + fi + + if [ -d ${SPHINX_DIR}/sphinx-${VERSION} ];then + rm -rf ${SPHINX_DIR}/sphinx-${VERSION} + fi +} + +Uninstall_App() +{ + if [ -f /usr/lib/systemd/system/sphinx.service ] || [ -f /lib/systemd/system/sphinx.service ];then + systemctl stop sphinx + systemctl disable sphinx + + if [ -f /usr/lib/systemd/system/sphinx.service ];then + rm -rf /usr/lib/systemd/system/sphinx.service + fi + + if [ -f /lib/systemd/system/sphinx.service ];then + rm -rf /lib/systemd/system/sphinx.service + fi + systemctl daemon-reload + fi + + if [ -f $serverPath/sphinx/initd/sphinx ];then + $serverPath/sphinx/initd/sphinx stop + fi + + if [ -d $serverPath/sphinx ];then + echo "rm -rf $serverPath/sphinx" + rm -rf $serverPath/sphinx + fi + + echo "卸载sphinx成功" +} + +action=$1 +if [ "${1}" == 'install' ];then + Install_App +else + Uninstall_App +fi diff --git a/plugins/manticoresearch/versions/yum/install.sh b/plugins/manticoresearch/versions/yum/install.sh new file mode 100755 index 000000000..1f69b82c8 --- /dev/null +++ b/plugins/manticoresearch/versions/yum/install.sh @@ -0,0 +1,171 @@ +#!/bin/bash +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:~/bin:/opt/homebrew/bin +export PATH + +curPath=`pwd` +rootPath=$(dirname "$curPath") +rootPath=$(dirname "$rootPath") +serverPath=$(dirname "$rootPath") +sysName=`uname` +sysArch=`arch` + + +if [ -f ${rootPath}/bin/activate ];then + source ${rootPath}/bin/activate +fi + +# cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py rebuild +# cd /www/server/mdserver-web/plugins/sphinx && bash install.sh install 3.6.1 +# cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py db_to_sphinx && /www/server/sphinx/bin/bin/indexer -c /www/server/sphinx/sphinx.conf --all --rotate +# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate + +# cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py sphinx_cmd + +# /Users/midoks/Desktop/mwdev/server/sphinx/bin/bin/indexer /Users/midoks/Desktop/mwdev/server/sphinx/sphinx.conf --all --rotate + +# cd /www/server/mdserver-web && source bin/activate && python3 plugins/sphinx/index.py start +bash ${rootPath}/scripts/getos.sh +# echo "bash ${rootPath}/scripts/getos.sh" +OSNAME="macos" +if [ -f ${rootPath}/data/osname.pl ];then + OSNAME=`cat ${rootPath}/data/osname.pl` +fi + +if [ "${OSNAME}" == "centos" ] || + [ "${OSNAME}" == "fedora" ] || + [ "${OSNAME}" == "alma" ]; then + yum install -y postgresql-libs unixODBC +fi + +# http://sphinxsearch.com/files/sphinx-3.7.1-da9f8a4-linux-amd64.tar.gz + +VERSION=$2 + +# echo $VERSION + +if [ "$VERSION" == "3.1.1" ];then + VERSION_NUM=${VERSION}-612d99f +elif [ "$VERSION" == "3.2.1" ]; then + VERSION_NUM=${VERSION}-f152e0b +elif [ "$VERSION" == "3.3.1" ]; then + VERSION_NUM=${VERSION}-b72d67b +elif [ "$VERSION" == "3.4.1" ]; then + VERSION_NUM=${VERSION}-efbcc65 +elif [ "$VERSION" == "3.5.1" ]; then + VERSION_NUM=${VERSION}-82c60cb +elif [ "$VERSION" == "3.6.1" ]; then + VERSION_NUM=${VERSION}-c9dbeda +elif [ "$VERSION" == "3.7.1" ]; then + VERSION_NUM=${VERSION}-da9f8a4 +elif [ "$VERSION" == "3.8.1" ]; then + VERSION_NUM=${VERSION}-d25e0bb +fi + +# echo $VERSION_NUM + +Install_App() +{ + echo '正在安装Sphinx...' + mkdir -p $serverPath/sphinx + + SPHINX_DIR=${serverPath}/source/sphinx + mkdir -p $SPHINX_DIR + + SPH_NAME=amd64 + if [ "$sysArch" == "arm64" ];then + SPH_NAME=amd64 + elif [ "$sysArch" == "x86_64" ]; then + SPH_NAME=amd64 + elif [ "$sysArch" == "aarch64" ]; then + SPH_NAME=aarch64 + fi + + if [ "$sysName" == "Darwin" ] && [ "$VERSION" == "3.7.1" ];then + SPH_NAME=aarch64 + fi + + SPH_SYSNAME=linux + if [ $sysName == 'Darwin' ]; then + SPH_SYSNAME=darwin + elif [ "$sysName" == "aarch64" ]; then + SPH_NAME=aarch64 + elif [ "$sysName" == "freebsd" ]; then + SPH_NAME=freebsd + fi + + if [ "$SPH_SYSNAME" == "linux" ];then + glibc_ver=`ldd --version | grep libc | awk -F ')' '{print $2}'|awk '{gsub(/^\s+|\s+$/, "");print}'` + if [ "$VERSION" == "3.7.1" ] && [ `echo "2.29 > $glibc_ver " | bc` -eq 1 ];then + SPH_NAME=${SPH_NAME}-glibc2.17 + fi + if [ "$VERSION" == "3.6.1" ] && [ `echo "2.29 > $glibc_ver " | bc` -eq 1 ];then + SPH_NAME=${SPH_NAME}-glibc2.17 + fi + fi + + + FILE_NAME=sphinx-${VERSION_NUM}-${SPH_SYSNAME}-${SPH_NAME} + FILE_TGZ=${FILE_NAME}.tar.gz + + echo $FILE_TGZ + # curl -sSLo ${SPHINX_DIR}/${FILE_TGZ} http://sphinxsearch.com/files/${FILE_TGZ} + if [ ! -f ${SPHINX_DIR}/${FILE_TGZ} ];then + wget --no-check-certificate -O ${SPHINX_DIR}/${FILE_TGZ} http://sphinxsearch.com/files/${FILE_TGZ} + fi + + cd ${SPHINX_DIR} && tar -zxvf ${FILE_TGZ} + + if [ "$?" == "0" ];then + mkdir -p $SPHINX_DIR + cp -rf ${SPHINX_DIR}/sphinx-${VERSION}/ $serverPath/sphinx/bin + fi + + if [ -d $serverPath/sphinx ];then + echo "${VERSION}" > $serverPath/sphinx/version.pl + echo '安装Sphinx完成' + cd ${rootPath} && python3 ${rootPath}/plugins/sphinx/index.py start + + if [ $sysName != 'Darwin' ]; then + cd ${rootPath} && python3 ${rootPath}/plugins/sphinx/index.py initd_install + fi + fi + + if [ -d ${SPHINX_DIR}/sphinx-${VERSION} ];then + rm -rf ${SPHINX_DIR}/sphinx-${VERSION} + fi +} + +Uninstall_App() +{ + if [ -f /usr/lib/systemd/system/sphinx.service ] || [ -f /lib/systemd/system/sphinx.service ];then + systemctl stop sphinx + systemctl disable sphinx + + if [ -f /usr/lib/systemd/system/sphinx.service ];then + rm -rf /usr/lib/systemd/system/sphinx.service + fi + + if [ -f /lib/systemd/system/sphinx.service ];then + rm -rf /lib/systemd/system/sphinx.service + fi + systemctl daemon-reload + fi + + if [ -f $serverPath/sphinx/initd/sphinx ];then + $serverPath/sphinx/initd/sphinx stop + fi + + if [ -d $serverPath/sphinx ];then + echo "rm -rf $serverPath/sphinx" + rm -rf $serverPath/sphinx + fi + + echo "卸载sphinx成功" +} + +action=$1 +if [ "${1}" == 'install' ];then + Install_App +else + Uninstall_App +fi