Verzeichnisstruktur phpBB-3.3.15
- Veröffentlicht
- 28.08.2024
So funktioniert es
|
|
Auf das letzte Element klicken. Dies geht jeweils ein Schritt zurück |
Auf das Icon klicken, dies öffnet das Verzeichnis. Nochmal klicken schließt das Verzeichnis. |
|
|
(Beispiel Datei-Icons)
|
Auf das Icon klicken um den Quellcode anzuzeigen |
fulltext_sphinx.php
0001 <?php
0002 /**
0003 *
0004 * This file is part of the phpBB Forum Software package.
0005 *
0006 * @copyright (c) phpBB Limited <https://www.phpbb.com>
0007 * @license GNU General Public License, version 2 (GPL-2.0)
0008 *
0009 * For full copyright and license information, please see
0010 * the docs/CREDITS.txt file.
0011 *
0012 */
0013
0014 namespace phpbb\search;
0015
0016 define('SPHINX_MAX_MATCHES', 20000);
0017 define('SPHINX_CONNECT_RETRIES', 3);
0018 define('SPHINX_CONNECT_WAIT_TIME', 300);
0019
0020 /**
0021 * Fulltext search based on the sphinx search daemon
0022 */
0023 class fulltext_sphinx
0024 {
0025 /**
0026 * Associative array holding index stats
0027 * @var array
0028 */
0029 protected $stats = array();
0030
0031 /**
0032 * Holds the words entered by user, obtained by splitting the entered query on whitespace
0033 * @var array
0034 */
0035 protected $split_words = array();
0036
0037 /**
0038 * Holds unique sphinx id
0039 * @var string
0040 */
0041 protected $id;
0042
0043 /**
0044 * Stores the names of both main and delta sphinx indexes
0045 * separated by a semicolon
0046 * @var string
0047 */
0048 protected $indexes;
0049
0050 /**
0051 * Sphinx searchd client object
0052 * @var SphinxClient
0053 */
0054 protected $sphinx;
0055
0056 /**
0057 * Relative path to board root
0058 * @var string
0059 */
0060 protected $phpbb_root_path;
0061
0062 /**
0063 * PHP Extension
0064 * @var string
0065 */
0066 protected $php_ext;
0067
0068 /**
0069 * Auth object
0070 * @var \phpbb\auth\auth
0071 */
0072 protected $auth;
0073
0074 /**
0075 * Config object
0076 * @var \phpbb\config\config
0077 */
0078 protected $config;
0079
0080 /**
0081 * Database connection
0082 * @var \phpbb\db\driver\driver_interface
0083 */
0084 protected $db;
0085
0086 /**
0087 * Database Tools object
0088 * @var \phpbb\db\tools\tools_interface
0089 */
0090 protected $db_tools;
0091
0092 /**
0093 * Stores the database type if supported by sphinx
0094 * @var string
0095 */
0096 protected $dbtype;
0097
0098 /**
0099 * phpBB event dispatcher object
0100 * @var \phpbb\event\dispatcher_interface
0101 */
0102 protected $phpbb_dispatcher;
0103
0104 /**
0105 * User object
0106 * @var \phpbb\user
0107 */
0108 protected $user;
0109
0110 /**
0111 * Stores the generated content of the sphinx config file
0112 * @var string
0113 */
0114 protected $config_file_data = '';
0115
0116 /**
0117 * Contains tidied search query.
0118 * Operators are prefixed in search query and common words excluded
0119 * @var string
0120 */
0121 protected $search_query;
0122
0123 /**
0124 * Constructor
0125 * Creates a new \phpbb\search\fulltext_postgres, which is used as a search backend
0126 *
0127 * @param string|bool $error Any error that occurs is passed on through this reference variable otherwise false
0128 * @param string $phpbb_root_path Relative path to phpBB root
0129 * @param string $phpEx PHP file extension
0130 * @param \phpbb\auth\auth $auth Auth object
0131 * @param \phpbb\config\config $config Config object
0132 * @param \phpbb\db\driver\driver_interface $db Database object
0133 * @param \phpbb\user $user User object
0134 * @param \phpbb\event\dispatcher_interface $phpbb_dispatcher Event dispatcher object
0135 */
0136 public function __construct(&$error, $phpbb_root_path, $phpEx, $auth, $config, $db, $user, $phpbb_dispatcher)
0137 {
0138 $this->phpbb_root_path = $phpbb_root_path;
0139 $this->php_ext = $phpEx;
0140 $this->config = $config;
0141 $this->phpbb_dispatcher = $phpbb_dispatcher;
0142 $this->user = $user;
0143 $this->db = $db;
0144 $this->auth = $auth;
0145
0146 // Initialize \phpbb\db\tools\tools object
0147 global $phpbb_container; // TODO inject into object
0148 $this->db_tools = $phpbb_container->get('dbal.tools');
0149
0150 if (!$this->config['fulltext_sphinx_id'])
0151 {
0152 $this->config->set('fulltext_sphinx_id', unique_id());
0153 }
0154 $this->id = $this->config['fulltext_sphinx_id'];
0155 $this->indexes = 'index_phpbb_' . $this->id . '_delta;index_phpbb_' . $this->id . '_main';
0156
0157 if (!class_exists('SphinxClient'))
0158 {
0159 require($this->phpbb_root_path . 'includes/sphinxapi.' . $this->php_ext);
0160 }
0161
0162 // Initialize sphinx client
0163 $this->sphinx = new \SphinxClient();
0164
0165 $this->sphinx->SetServer(($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost'), ($this->config['fulltext_sphinx_port'] ? (int) $this->config['fulltext_sphinx_port'] : 9312));
0166
0167 $error = false;
0168 }
0169
0170 /**
0171 * Returns the name of this search backend to be displayed to administrators
0172 *
0173 * @return string Name
0174 */
0175 public function get_name()
0176 {
0177 return 'Sphinx Fulltext';
0178 }
0179
0180 /**
0181 * Returns the search_query
0182 *
0183 * @return string search query
0184 */
0185 public function get_search_query()
0186 {
0187 return $this->search_query;
0188 }
0189
0190 /**
0191 * Returns false as there is no word_len array
0192 *
0193 * @return false
0194 */
0195 public function get_word_length()
0196 {
0197 return false;
0198 }
0199
0200 /**
0201 * Returns an empty array as there are no common_words
0202 *
0203 * @return array common words that are ignored by search backend
0204 */
0205 public function get_common_words()
0206 {
0207 return array();
0208 }
0209
0210 /**
0211 * Checks permissions and paths, if everything is correct it generates the config file
0212 *
0213 * @return string|bool Language key of the error/incompatibility encountered, or false if successful
0214 */
0215 public function init()
0216 {
0217 if ($this->db->get_sql_layer() != 'mysqli' && $this->db->get_sql_layer() != 'postgres')
0218 {
0219 return $this->user->lang['FULLTEXT_SPHINX_WRONG_DATABASE'];
0220 }
0221
0222 // Move delta to main index each hour
0223 $this->config->set('search_gc', 3600);
0224
0225 return false;
0226 }
0227
0228 /**
0229 * Generates content of sphinx.conf
0230 *
0231 * @return bool True if sphinx.conf content is correctly generated, false otherwise
0232 */
0233 protected function config_generate()
0234 {
0235 // Check if Database is supported by Sphinx
0236 if ($this->db->get_sql_layer() == 'mysqli')
0237 {
0238 $this->dbtype = 'mysql';
0239 }
0240 else if ($this->db->get_sql_layer() == 'postgres')
0241 {
0242 $this->dbtype = 'pgsql';
0243 }
0244 else
0245 {
0246 $this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_WRONG_DATABASE');
0247 return false;
0248 }
0249
0250 // Check if directory paths have been filled
0251 if (!$this->config['fulltext_sphinx_data_path'])
0252 {
0253 $this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_NO_CONFIG_DATA');
0254 return false;
0255 }
0256
0257 include($this->phpbb_root_path . 'config.' . $this->php_ext);
0258
0259 /* Now that we're sure everything was entered correctly,
0260 generate a config for the index. We use a config value
0261 fulltext_sphinx_id for this, as it should be unique. */
0262 $config_object = new \phpbb\search\sphinx\config($this->config_file_data);
0263 $config_data = array(
0264 'source source_phpbb_' . $this->id . '_main' => array(
0265 array('type', $this->dbtype . ' # mysql or pgsql'),
0266 // This config value sql_host needs to be changed incase sphinx and sql are on different servers
0267 array('sql_host', $dbhost . ' # SQL server host sphinx connects to'),
0268 array('sql_user', '[dbuser]'),
0269 array('sql_pass', '[dbpassword]'),
0270 array('sql_db', $dbname),
0271 array('sql_port', $dbport . ' # optional, default is 3306 for mysql and 5432 for pgsql'),
0272 array('sql_query_pre', 'SET NAMES \'utf8\''),
0273 array('sql_query_pre', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = (SELECT MAX(post_id) FROM ' . POSTS_TABLE . ') WHERE counter_id = 1'),
0274 array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''),
0275 array('sql_range_step', '5000'),
0276 array('sql_query', 'SELECT
0277 p.post_id AS id,
0278 p.forum_id,
0279 p.topic_id,
0280 p.poster_id,
0281 p.post_visibility,
0282 CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post,
0283 p.post_time,
0284 p.post_subject,
0285 p.post_subject as title,
0286 p.post_text as data,
0287 t.topic_last_post_time,
0288 0 as deleted
0289 FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
0290 WHERE
0291 p.topic_id = t.topic_id
0292 AND p.post_id >= $start AND p.post_id <= $end'),
0293 array('sql_query_post', ''),
0294 array('sql_query_post_index', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = $maxid WHERE counter_id = 1'),
0295 array('sql_attr_uint', 'forum_id'),
0296 array('sql_attr_uint', 'topic_id'),
0297 array('sql_attr_uint', 'poster_id'),
0298 array('sql_attr_uint', 'post_visibility'),
0299 array('sql_attr_bool', 'topic_first_post'),
0300 array('sql_attr_bool', 'deleted'),
0301 array('sql_attr_timestamp', 'post_time'),
0302 array('sql_attr_timestamp', 'topic_last_post_time'),
0303 array('sql_attr_string', 'post_subject'),
0304 ),
0305 'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array(
0306 array('sql_query_pre', 'SET NAMES \'utf8\''),
0307 array('sql_query_range', ''),
0308 array('sql_range_step', ''),
0309 array('sql_query', 'SELECT
0310 p.post_id AS id,
0311 p.forum_id,
0312 p.topic_id,
0313 p.poster_id,
0314 p.post_visibility,
0315 CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post,
0316 p.post_time,
0317 p.post_subject,
0318 p.post_subject as title,
0319 p.post_text as data,
0320 t.topic_last_post_time,
0321 0 as deleted
0322 FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
0323 WHERE
0324 p.topic_id = t.topic_id
0325 AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'),
0326 array('sql_query_post_index', ''),
0327 ),
0328 'index index_phpbb_' . $this->id . '_main' => array(
0329 array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'),
0330 array('source', 'source_phpbb_' . $this->id . '_main'),
0331 array('docinfo', 'extern'),
0332 array('morphology', 'none'),
0333 array('stopwords', ''),
0334 array('wordforms', ' # optional, specify path to wordforms file. See ./docs/sphinx_wordforms.txt for example'),
0335 array('exceptions', ' # optional, specify path to exceptions file. See ./docs/sphinx_exceptions.txt for example'),
0336 array('min_word_len', '2'),
0337 array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'),
0338 array('ignore_chars', 'U+0027, U+002C'),
0339 array('min_prefix_len', '3 # Minimum number of characters for wildcard searches by prefix (min 1). Default is 3. If specified, set min_infix_len to 0'),
0340 array('min_infix_len', '0 # Minimum number of characters for wildcard searches by infix (min 2). If specified, set min_prefix_len to 0'),
0341 array('html_strip', '1'),
0342 array('index_exact_words', '0 # Set to 1 to enable exact search operator. Requires wordforms or morphology'),
0343 array('blend_chars', 'U+23, U+24, U+25, U+26, U+40'),
0344 ),
0345 'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array(
0346 array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'),
0347 array('source', 'source_phpbb_' . $this->id . '_delta'),
0348 ),
0349 'indexer' => array(
0350 array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'),
0351 ),
0352 'searchd' => array(
0353 array('listen' , ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '9312')),
0354 array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'),
0355 array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'),
0356 array('read_timeout', '5'),
0357 array('max_children', '30'),
0358 array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'),
0359 array('binlog_path', $this->config['fulltext_sphinx_data_path']),
0360 ),
0361 );
0362
0363 $non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true);
0364 $delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true);
0365
0366 /**
0367 * Allow adding/changing the Sphinx configuration data
0368 *
0369 * @event core.search_sphinx_modify_config_data
0370 * @var array config_data Array with the Sphinx configuration data
0371 * @var array non_unique Array with the Sphinx non-unique variables to delete
0372 * @var array delete Array with the Sphinx variables to delete
0373 * @since 3.1.7-RC1
0374 */
0375 $vars = array(
0376 'config_data',
0377 'non_unique',
0378 'delete',
0379 );
0380 extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_modify_config_data', compact($vars)));
0381
0382 foreach ($config_data as $section_name => $section_data)
0383 {
0384 $section = $config_object->get_section_by_name($section_name);
0385 if (!$section)
0386 {
0387 $section = $config_object->add_section($section_name);
0388 }
0389
0390 foreach ($delete as $key => $void)
0391 {
0392 $section->delete_variables_by_name($key);
0393 }
0394
0395 foreach ($non_unique as $key => $void)
0396 {
0397 $section->delete_variables_by_name($key);
0398 }
0399
0400 foreach ($section_data as $entry)
0401 {
0402 $key = $entry[0];
0403 $value = $entry[1];
0404
0405 if (!isset($non_unique[$key]))
0406 {
0407 $variable = $section->get_variable_by_name($key);
0408 if (!$variable)
0409 {
0410 $section->create_variable($key, $value);
0411 }
0412 else
0413 {
0414 $variable->set_value($value);
0415 }
0416 }
0417 else
0418 {
0419 $section->create_variable($key, $value);
0420 }
0421 }
0422 }
0423 $this->config_file_data = $config_object->get_data();
0424
0425 return true;
0426 }
0427
0428 /**
0429 * Splits keywords entered by a user into an array of words stored in $this->split_words
0430 * Stores the tidied search query in $this->search_query
0431 *
0432 * @param string $keywords Contains the keyword as entered by the user
0433 * @param string $terms is either 'all' or 'any'
0434 * @return false if no valid keywords were found and otherwise true
0435 */
0436 public function split_keywords(&$keywords, $terms)
0437 {
0438 // Keep quotes and new lines
0439 $keywords = str_replace(['"', "\n"], ['"', ' '], trim($keywords));
0440
0441 if ($terms == 'all')
0442 {
0443 // Replaces verbal operators OR and NOT with special characters | and -, unless appearing within quotation marks
0444 $match = ['#\sor\s(?=([^"]*"[^"]*")*[^"]*$)#i', '#\snot\s(?=([^"]*"[^"]*")*[^"]*$)#i'];
0445 $replace = [' | ', ' -'];
0446
0447 $keywords = preg_replace($match, $replace, $keywords);
0448 $this->sphinx->SetMatchMode(SPH_MATCH_EXTENDED);
0449 }
0450 else
0451 {
0452 $match = ['\\', '(',')', '|', '!', '@', '~', '/', '^', '$', '=', '&', '<', '>'];
0453
0454 $keywords = str_replace($match, ' ', $keywords);
0455 $this->sphinx->SetMatchMode(SPH_MATCH_ANY);
0456 }
0457
0458 if (strlen($keywords) > 0)
0459 {
0460 $this->search_query = str_replace('"', '"', $keywords);
0461 return true;
0462 }
0463
0464 return false;
0465 }
0466
0467 /**
0468 * Cleans search query passed into Sphinx search engine, as follows:
0469 * 1. Hyphenated words are replaced with keyword search for either the exact phrase with spaces
0470 * or as a single word without spaces eg search for "know-it-all" becomes ("know it all"|"knowitall*")
0471 * 2. Words with apostrophes are contracted eg "it's" becomes "its"
0472 * 3. <, >, " and & are decoded from HTML entities.
0473 * 4. Following special characters used as search operators in Sphinx are preserved when used with correct syntax:
0474 * (a) quorum matching: "the world is a wonderful place"/3
0475 * Finds 3 of the words within the phrase. Number must be between 1 and 9.
0476 * (b) proximity search: "hello world"~10
0477 * Finds hello and world within 10 words of each other. Number can be between 1 and 99.
0478 * (c) strict word order: aaa << bbb << ccc
0479 * Finds "aaa" only where it appears before "bbb" and only where "bbb" appears before "ccc".
0480 * (d) exact match operator: if lemmatizer or stemming enabled,
0481 * search will find exact match only and ignore other grammatical forms of the same word stem.
0482 * eg. raining =cats and =dogs
0483 * will not return "raining cat and dog"
0484 * eg. ="search this exact phrase"
0485 * will not return "searched this exact phrase", "searching these exact phrases".
0486 * 5. Special characters /, ~, << and = not complying with the correct syntax
0487 * and other reserved operators are escaped and searched literally.
0488 * Special characters not explicitly listed in charset_table or blend_chars in sphinx.conf
0489 * will not be indexed and keywords containing them will be ignored by Sphinx.
0490 * By default, only $, %, & and @ characters are indexed and searchable.
0491 * String transformation is in backend only and not visible to the end user
0492 * nor reflected in the results page URL or keyword highlighting.
0493 *
0494 * @param string $search_string
0495 * @return string
0496 */
0497 public function sphinx_clean_search_string($search_string)
0498 {
0499 $from = ['@', '^', '$', '!', '<', '>', '"', '&', '\''];
0500 $to = ['\@', '\^', '\$', '\!', '<', '>', '"', '&', ''];
0501
0502 $search_string = str_replace($from, $to, $search_string);
0503
0504 $search_string = strrev($search_string);
0505 $search_string = preg_replace(['#\/(?!"[^"]+")#', '#~(?!"[^"]+")#'], ['/\\', '~\\'], $search_string);
0506 $search_string = strrev($search_string);
0507
0508 $match = ['#(/|\\\\/)(?)#', '#(~|\\\\~)(?!\d{1,2}(\s|$))#', '#((?:\p{L}|\p{N})+)-((?:\p{L}|\p{N})+)(?:-((?:\p{L}|\p{N})+))?(?:-((?:\p{L}|\p{N})+))?#i', '#<<\s*$#', '#(\S\K=|=(?=\s)|=$)#'];
0509 $replace = ['\/', '\~', '("$1 $2 $3 $4"|$1$2$3$4*)', '\<\<', '\='];
0510
0511 $search_string = preg_replace($match, $replace, $search_string);
0512 $search_string = preg_replace('#\s+"\|#', '"|', $search_string);
0513
0514 /**
0515 * OPTIONAL: Thousands separator stripped from numbers, eg search for '90,000' is queried as '90000'.
0516 * By default commas are stripped from search index so that '90,000' is indexed as '90000'
0517 */
0518 // $search_string = preg_replace('#[0-9]{1,3}\K,(?=[0-9]{3})#', '', $search_string);
0519
0520 return $search_string;
0521 }
0522
0523 /**
0524 * Performs a search on keywords depending on display specific params. You have to run split_keywords() first
0525 *
0526 * @param string $type contains either posts or topics depending on what should be searched for
0527 * @param string $fields contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched)
0528 * @param string $terms is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words)
0529 * @param array $sort_by_sql contains SQL code for the ORDER BY part of a query
0530 * @param string $sort_key is the key of $sort_by_sql for the selected sorting
0531 * @param string $sort_dir is either a or d representing ASC and DESC
0532 * @param string $sort_days specifies the maximum amount of days a post may be old
0533 * @param array $ex_fid_ary specifies an array of forum ids which should not be searched
0534 * @param string $post_visibility specifies which types of posts the user can view in which forums
0535 * @param int $topic_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
0536 * @param array $author_ary an array of author ids if the author should be ignored during the search the array is empty
0537 * @param string $author_name specifies the author match, when ANONYMOUS is also a search-match
0538 * @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
0539 * @param int $start indicates the first index of the page
0540 * @param int $per_page number of ids each page is supposed to contain
0541 * @return boolean|int total number of results
0542 */
0543 public function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page)
0544 {
0545 global $user, $phpbb_log;
0546
0547 // No keywords? No posts.
0548 if (!strlen($this->search_query) && !count($author_ary))
0549 {
0550 return false;
0551 }
0552
0553 $id_ary = array();
0554
0555 // Sorting
0556
0557 if ($type == 'topics')
0558 {
0559 switch ($sort_key)
0560 {
0561 case 'a':
0562 $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'poster_id ' . (($sort_dir == 'a') ? 'ASC' : 'DESC'));
0563 break;
0564
0565 case 'f':
0566 $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'forum_id ' . (($sort_dir == 'a') ? 'ASC' : 'DESC'));
0567 break;
0568
0569 case 'i':
0570
0571 case 's':
0572 $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'post_subject ' . (($sort_dir == 'a') ? 'ASC' : 'DESC'));
0573 break;
0574
0575 case 't':
0576
0577 default:
0578 $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'topic_last_post_time ' . (($sort_dir == 'a') ? 'ASC' : 'DESC'));
0579 break;
0580 }
0581 }
0582 else
0583 {
0584 switch ($sort_key)
0585 {
0586 case 'a':
0587 $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'poster_id');
0588 break;
0589
0590 case 'f':
0591 $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'forum_id');
0592 break;
0593
0594 case 'i':
0595
0596 case 's':
0597 $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_subject');
0598 break;
0599
0600 case 't':
0601
0602 default:
0603 $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_time');
0604 break;
0605 }
0606 }
0607
0608 // Most narrow filters first
0609 if ($topic_id)
0610 {
0611 $this->sphinx->SetFilter('topic_id', array($topic_id));
0612 }
0613
0614 /**
0615 * Allow modifying the Sphinx search options
0616 *
0617 * @event core.search_sphinx_keywords_modify_options
0618 * @var string type Searching type ('posts', 'topics')
0619 * @var string fields Searching fields ('titleonly', 'msgonly', 'firstpost', 'all')
0620 * @var string terms Searching terms ('all', 'any')
0621 * @var int sort_days Time, in days, of the oldest possible post to list
0622 * @var string sort_key The sort type used from the possible sort types
0623 * @var int topic_id Limit the search to this topic_id only
0624 * @var array ex_fid_ary Which forums not to search on
0625 * @var string post_visibility Post visibility data
0626 * @var array author_ary Array of user_id containing the users to filter the results to
0627 * @var string author_name The username to search on
0628 * @var object sphinx The Sphinx searchd client object
0629 * @since 3.1.7-RC1
0630 */
0631 $sphinx = $this->sphinx;
0632 $vars = array(
0633 'type',
0634 'fields',
0635 'terms',
0636 'sort_days',
0637 'sort_key',
0638 'topic_id',
0639 'ex_fid_ary',
0640 'post_visibility',
0641 'author_ary',
0642 'author_name',
0643 'sphinx',
0644 );
0645 extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_keywords_modify_options', compact($vars)));
0646 $this->sphinx = $sphinx;
0647 unset($sphinx);
0648
0649 $search_query_prefix = '';
0650
0651 switch ($fields)
0652 {
0653 case 'titleonly':
0654 // Only search the title
0655 if ($terms == 'all')
0656 {
0657 $search_query_prefix = '@title ';
0658 }
0659 // Weight for the title
0660 $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
0661 // 1 is first_post, 0 is not first post
0662 $this->sphinx->SetFilter('topic_first_post', array(1));
0663 break;
0664
0665 case 'msgonly':
0666 // Only search the body
0667 if ($terms == 'all')
0668 {
0669 $search_query_prefix = '@data ';
0670 }
0671 // Weight for the body
0672 $this->sphinx->SetFieldWeights(array("title" => 1, "data" => 5));
0673 break;
0674
0675 case 'firstpost':
0676 // More relative weight for the title, also search the body
0677 $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
0678 // 1 is first_post, 0 is not first post
0679 $this->sphinx->SetFilter('topic_first_post', array(1));
0680 break;
0681
0682 default:
0683 // More relative weight for the title, also search the body
0684 $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
0685 break;
0686 }
0687
0688 if (count($author_ary))
0689 {
0690 $this->sphinx->SetFilter('poster_id', $author_ary);
0691 }
0692
0693 // As this is not simply possible at the moment, we limit the result to approved posts.
0694 // This will make it impossible for moderators to search unapproved and softdeleted posts,
0695 // but at least it will also cause the same for normal users.
0696 $this->sphinx->SetFilter('post_visibility', array(ITEM_APPROVED));
0697
0698 if (count($ex_fid_ary))
0699 {
0700 // All forums that a user is allowed to access
0701 $fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($this->auth->acl_getf('f_search', true))));
0702 // All forums that the user wants to and can search in
0703 $search_forums = array_diff($fid_ary, $ex_fid_ary);
0704
0705 if (count($search_forums))
0706 {
0707 $this->sphinx->SetFilter('forum_id', $search_forums);
0708 }
0709 }
0710
0711 $this->sphinx->SetFilter('deleted', array(0));
0712
0713 $this->sphinx->SetLimits((int) $start, (int) $per_page, max(SPHINX_MAX_MATCHES, (int) $start + $per_page));
0714 $result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('"', '"', $this->search_query)), $this->indexes);
0715
0716 // Could be connection to localhost:9312 failed (errno=111,
0717 // msg=Connection refused) during rotate, retry if so
0718 $retries = SPHINX_CONNECT_RETRIES;
0719 while (!$result && (strpos($this->sphinx->GetLastError(), "errno=111,") !== false) && $retries--)
0720 {
0721 usleep(SPHINX_CONNECT_WAIT_TIME);
0722 $result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('"', '"', $this->search_query)), $this->indexes);
0723 }
0724
0725 if ($this->sphinx->GetLastError())
0726 {
0727 $phpbb_log->add('critical', $user->data['user_id'], $user->ip, 'LOG_SPHINX_ERROR', false, array($this->sphinx->GetLastError()));
0728 if ($this->auth->acl_get('a_'))
0729 {
0730 trigger_error($this->user->lang('SPHINX_SEARCH_FAILED', $this->sphinx->GetLastError()));
0731 }
0732 else
0733 {
0734 trigger_error($this->user->lang('SPHINX_SEARCH_FAILED_LOG'));
0735 }
0736 }
0737
0738 $result_count = $result['total_found'];
0739
0740 if ($result_count && $start >= $result_count)
0741 {
0742 $start = floor(($result_count - 1) / $per_page) * $per_page;
0743
0744 $this->sphinx->SetLimits((int) $start, (int) $per_page, max(SPHINX_MAX_MATCHES, (int) $start + $per_page));
0745 $result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('"', '"', $this->search_query)), $this->indexes);
0746
0747 // Could be connection to localhost:9312 failed (errno=111,
0748 // msg=Connection refused) during rotate, retry if so
0749 $retries = SPHINX_CONNECT_RETRIES;
0750 while (!$result && (strpos($this->sphinx->GetLastError(), "errno=111,") !== false) && $retries--)
0751 {
0752 usleep(SPHINX_CONNECT_WAIT_TIME);
0753 $result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('"', '"', $this->search_query)), $this->indexes);
0754 }
0755 }
0756
0757 $id_ary = array();
0758 if (isset($result['matches']))
0759 {
0760 if ($type == 'posts')
0761 {
0762 $id_ary = array_keys($result['matches']);
0763 }
0764 else
0765 {
0766 foreach ($result['matches'] as $key => $value)
0767 {
0768 $id_ary[] = $value['attrs']['topic_id'];
0769 }
0770 }
0771 }
0772 else
0773 {
0774 return false;
0775 }
0776
0777 $id_ary = array_slice($id_ary, 0, (int) $per_page);
0778
0779 return $result_count;
0780 }
0781
0782 /**
0783 * Performs a search on an author's posts without caring about message contents. Depends on display specific params
0784 *
0785 * @param string $type contains either posts or topics depending on what should be searched for
0786 * @param boolean $firstpost_only if true, only topic starting posts will be considered
0787 * @param array $sort_by_sql contains SQL code for the ORDER BY part of a query
0788 * @param string $sort_key is the key of $sort_by_sql for the selected sorting
0789 * @param string $sort_dir is either a or d representing ASC and DESC
0790 * @param string $sort_days specifies the maximum amount of days a post may be old
0791 * @param array $ex_fid_ary specifies an array of forum ids which should not be searched
0792 * @param string $post_visibility specifies which types of posts the user can view in which forums
0793 * @param int $topic_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
0794 * @param array $author_ary an array of author ids
0795 * @param string $author_name specifies the author match, when ANONYMOUS is also a search-match
0796 * @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
0797 * @param int $start indicates the first index of the page
0798 * @param int $per_page number of ids each page is supposed to contain
0799 * @return boolean|int total number of results
0800 */
0801 public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, $start, $per_page)
0802 {
0803 $this->search_query = '';
0804
0805 $this->sphinx->SetMatchMode(SPH_MATCH_FULLSCAN);
0806 $fields = ($firstpost_only) ? 'firstpost' : 'all';
0807 $terms = 'all';
0808 return $this->keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, $id_ary, $start, $per_page);
0809 }
0810
0811 /**
0812 * Updates wordlist and wordmatch tables when a message is posted or changed
0813 *
0814 * @param string $mode Contains the post mode: edit, post, reply, quote
0815 * @param int $post_id The id of the post which is modified/created
0816 * @param string &$message New or updated post content
0817 * @param string &$subject New or updated post subject
0818 * @param int $poster_id Post author's user id
0819 * @param int $forum_id The id of the forum in which the post is located
0820 */
0821 public function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id)
0822 {
0823 /**
0824 * Event to modify method arguments before the Sphinx search index is updated
0825 *
0826 * @event core.search_sphinx_index_before
0827 * @var string mode Contains the post mode: edit, post, reply, quote
0828 * @var int post_id The id of the post which is modified/created
0829 * @var string message New or updated post content
0830 * @var string subject New or updated post subject
0831 * @var int poster_id Post author's user id
0832 * @var int forum_id The id of the forum in which the post is located
0833 * @since 3.2.3-RC1
0834 */
0835 $vars = array(
0836 'mode',
0837 'post_id',
0838 'message',
0839 'subject',
0840 'poster_id',
0841 'forum_id',
0842 );
0843 extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_index_before', compact($vars)));
0844
0845 if ($mode == 'edit')
0846 {
0847 $this->sphinx->UpdateAttributes($this->indexes, array('forum_id', 'poster_id'), array((int) $post_id => array((int) $forum_id, (int) $poster_id)));
0848 }
0849 else if ($mode != 'post' && $post_id)
0850 {
0851 // Update topic_last_post_time for full topic
0852 $sql_array = array(
0853 'SELECT' => 'p1.post_id',
0854 'FROM' => array(
0855 POSTS_TABLE => 'p1',
0856 ),
0857 'LEFT_JOIN' => array(array(
0858 'FROM' => array(
0859 POSTS_TABLE => 'p2'
0860 ),
0861 'ON' => 'p1.topic_id = p2.topic_id',
0862 )),
0863 'WHERE' => 'p2.post_id = ' . ((int) $post_id),
0864 );
0865
0866 $sql = $this->db->sql_build_query('SELECT', $sql_array);
0867 $result = $this->db->sql_query($sql);
0868
0869 $post_updates = array();
0870 $post_time = time();
0871 while ($row = $this->db->sql_fetchrow($result))
0872 {
0873 $post_updates[(int) $row['post_id']] = array($post_time);
0874 }
0875 $this->db->sql_freeresult($result);
0876
0877 if (count($post_updates))
0878 {
0879 $this->sphinx->UpdateAttributes($this->indexes, array('topic_last_post_time'), $post_updates);
0880 }
0881 }
0882 }
0883
0884 /**
0885 * Delete a post from the index after it was deleted
0886 */
0887 public function index_remove($post_ids, $author_ids, $forum_ids)
0888 {
0889 $values = array();
0890 foreach ($post_ids as $post_id)
0891 {
0892 $values[$post_id] = array(1);
0893 }
0894
0895 $this->sphinx->UpdateAttributes($this->indexes, array('deleted'), $values);
0896 }
0897
0898 /**
0899 * Nothing needs to be destroyed
0900 */
0901 public function tidy($create = false)
0902 {
0903 $this->config->set('search_last_gc', time(), false);
0904 }
0905
0906 /**
0907 * Create sphinx table
0908 *
0909 * @return string|bool error string is returned incase of errors otherwise false
0910 */
0911 public function create_index($acp_module, $u_action)
0912 {
0913 if (!$this->index_created())
0914 {
0915 $table_data = array(
0916 'COLUMNS' => array(
0917 'counter_id' => array('UINT', 0),
0918 'max_doc_id' => array('UINT', 0),
0919 ),
0920 'PRIMARY_KEY' => 'counter_id',
0921 );
0922 $this->db_tools->sql_create_table(SPHINX_TABLE, $table_data);
0923
0924 $sql = 'TRUNCATE TABLE ' . SPHINX_TABLE;
0925 $this->db->sql_query($sql);
0926
0927 $data = array(
0928 'counter_id' => '1',
0929 'max_doc_id' => '0',
0930 );
0931 $sql = 'INSERT INTO ' . SPHINX_TABLE . ' ' . $this->db->sql_build_array('INSERT', $data);
0932 $this->db->sql_query($sql);
0933 }
0934
0935 return false;
0936 }
0937
0938 /**
0939 * Drop sphinx table
0940 *
0941 * @return string|bool error string is returned incase of errors otherwise false
0942 */
0943 public function delete_index($acp_module, $u_action)
0944 {
0945 if (!$this->index_created())
0946 {
0947 return false;
0948 }
0949
0950 $this->db_tools->sql_table_drop(SPHINX_TABLE);
0951
0952 return false;
0953 }
0954
0955 /**
0956 * Returns true if the sphinx table was created
0957 *
0958 * @return bool true if sphinx table was created
0959 */
0960 public function index_created($allow_new_files = true)
0961 {
0962 $created = false;
0963
0964 if ($this->db_tools->sql_table_exists(SPHINX_TABLE))
0965 {
0966 $created = true;
0967 }
0968
0969 return $created;
0970 }
0971
0972 /**
0973 * Returns an associative array containing information about the indexes
0974 *
0975 * @return string|bool Language string of error false otherwise
0976 */
0977 public function index_stats()
0978 {
0979 if (empty($this->stats))
0980 {
0981 $this->get_stats();
0982 }
0983
0984 return array(
0985 $this->user->lang['FULLTEXT_SPHINX_MAIN_POSTS'] => ($this->index_created()) ? $this->stats['main_posts'] : 0,
0986 $this->user->lang['FULLTEXT_SPHINX_DELTA_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] - $this->stats['main_posts'] : 0,
0987 $this->user->lang['FULLTEXT_MYSQL_TOTAL_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] : 0,
0988 );
0989 }
0990
0991 /**
0992 * Collects stats that can be displayed on the index maintenance page
0993 */
0994 protected function get_stats()
0995 {
0996 if ($this->index_created())
0997 {
0998 $sql = 'SELECT COUNT(post_id) as total_posts
0999 FROM ' . POSTS_TABLE;
1000 $result = $this->db->sql_query($sql);
1001 $this->stats['total_posts'] = (int) $this->db->sql_fetchfield('total_posts');
1002 $this->db->sql_freeresult($result);
1003
1004 $sql = 'SELECT COUNT(p.post_id) as main_posts
1005 FROM ' . POSTS_TABLE . ' p, ' . SPHINX_TABLE . ' m
1006 WHERE p.post_id <= m.max_doc_id
1007 AND m.counter_id = 1';
1008 $result = $this->db->sql_query($sql);
1009 $this->stats['main_posts'] = (int) $this->db->sql_fetchfield('main_posts');
1010 $this->db->sql_freeresult($result);
1011 }
1012 }
1013
1014 /**
1015 * Returns a list of options for the ACP to display
1016 *
1017 * @return associative array containing template and config variables
1018 */
1019 public function acp()
1020 {
1021 $config_vars = array(
1022 'fulltext_sphinx_data_path' => 'string',
1023 'fulltext_sphinx_host' => 'string',
1024 'fulltext_sphinx_port' => 'string',
1025 'fulltext_sphinx_indexer_mem_limit' => 'int',
1026 );
1027
1028 $tpl = '
1029 <span class="error">' . $this->user->lang['FULLTEXT_SPHINX_CONFIGURE']. '</span>
1030 <dl>
1031 <dt><label for="fulltext_sphinx_data_path">' . $this->user->lang['FULLTEXT_SPHINX_DATA_PATH'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_DATA_PATH_EXPLAIN'] . '</span></dt>
1032 <dd><input id="fulltext_sphinx_data_path" type="text" size="40" maxlength="255" name="config[fulltext_sphinx_data_path]" value="' . $this->config['fulltext_sphinx_data_path'] . '" /></dd>
1033 </dl>
1034 <dl>
1035 <dt><label for="fulltext_sphinx_host">' . $this->user->lang['FULLTEXT_SPHINX_HOST'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_HOST_EXPLAIN'] . '</span></dt>
1036 <dd><input id="fulltext_sphinx_host" type="text" size="40" maxlength="255" name="config[fulltext_sphinx_host]" value="' . $this->config['fulltext_sphinx_host'] . '" /></dd>
1037 </dl>
1038 <dl>
1039 <dt><label for="fulltext_sphinx_port">' . $this->user->lang['FULLTEXT_SPHINX_PORT'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '</span></dt>
1040 <dd><input id="fulltext_sphinx_port" type="number" min="0" max="9999999999" name="config[fulltext_sphinx_port]" value="' . $this->config['fulltext_sphinx_port'] . '" /></dd>
1041 </dl>
1042 <dl>
1043 <dt><label for="fulltext_sphinx_indexer_mem_limit">' . $this->user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN'] . '</span></dt>
1044 <dd><input id="fulltext_sphinx_indexer_mem_limit" type="number" min="0" max="9999999999" name="config[fulltext_sphinx_indexer_mem_limit]" value="' . $this->config['fulltext_sphinx_indexer_mem_limit'] . '" /> ' . $this->user->lang['MIB'] . '</dd>
1045 </dl>
1046 <dl>
1047 <dt><label for="fulltext_sphinx_config_file">' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE'] . $this->user->lang['COLON'] . '</label><br /><span>' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN'] . '</span></dt>
1048 <dd>' . (($this->config_generate()) ? '<textarea readonly="readonly" rows="6" id="sphinx_config_data">' . htmlspecialchars($this->config_file_data, ENT_COMPAT) . '</textarea>' : $this->config_file_data) . '</dd>
1049 <dl>
1050 ';
1051
1052 // These are fields required in the config table
1053 return array(
1054 'tpl' => $tpl,
1055 'config' => $config_vars
1056 );
1057 }
1058 }
1059