1
0
mirror of https://github.com/phpbb/phpbb.git synced 2025-08-07 01:06:48 +02:00

- overhauled search system

- updated structure for search backend plugins
  - better result caching using ACM
  - search results no longer session restricted => link to them by copying the URL :)
  - in-topic search
  - indexing posts now uses search backend plugins
  - develop/search_fill.php working again
  - fulltext_mysql not working yet
- tiny bugfixes to ACM


git-svn-id: file:///svn/phpbb/trunk@5441 89ea8834-ac86-4346-8a33-228a782c2dd0
This commit is contained in:
Nils Adermann
2006-01-11 18:56:07 +00:00
parent 9ea5fa1768
commit 0e0b1120fb
22 changed files with 1522 additions and 1036 deletions

View File

@@ -1212,325 +1212,4 @@ class parse_message extends bbcode_firstpass
}
}
/**
* @package phpBB3
* Parses a given message and updates/maintains the fulltext tables
* @todo replace fulltext_search in message_parser with search modules
*/
class fulltext_search
{
function split_words($mode, $text)
{
global $user, $config;
static $drop_char_match, $drop_char_replace, $stopwords, $replace_synonym, $match_synonym;
// Is the fulltext indexer disabled? If yes then we need not
// carry on ... it's okay ... I know when I'm not wanted boo hoo
if (!$config['load_search_upd'])
{
return;
}
if (!is_array($drop_char_match))
{
$drop_char_match = array('-', '^', '$', ';', '#', '&', '(', ')', '<', '>', '`', '\'', '"', '|', ',', '@', '_', '?', '%', '~', '.', '[', ']', '{', '}', ':', '\\', '/', '=', '\'', '!', '*');
$drop_char_replace = array(' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '', '', ' ', ' ', ' ', ' ', '', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '' , ' ', ' ', ' ', ' ', ' ');
if ($fp = @fopen($user->lang_path . '/search_stopwords.txt', 'rb'))
{
$stopwords = explode("\n", str_replace("\r\n", "\n", fread($fp, filesize($user->lang_path . '/search_stopwords.txt'))));
}
fclose($fp);
if ($fp = @fopen($user->lang_path . '/search_synonyms.txt', 'rb'))
{
preg_match_all('#^(.*?) (.*?)$#ms', fread($fp, filesize($user->lang_path . '/search_synonyms.txt')), $match);
$replace_synonym = &$match[1];
$match_synonym = &$match[2];
}
fclose($fp);
}
$match = array();
// Comments for hardcoded bbcode elements (urls, smilies, html)
$match[] = '#<!\-\- .* \-\->(.*?)<!\-\- .* \-\->#is';
// New lines, carriage returns
$match[] = "#[\n\r]+#";
// NCRs like &nbsp; etc.
$match[] = '#(&amp;|&)[\#a-z0-9]+?;#i';
// Do not index code
$match[] = '#\[code=?.*?(\:?[0-9a-z]{5,})\].*?\[\/code(\:?[0-9a-z]{5,})\]#is';
// BBcode
$match[] = '#\[\/?[a-z\*\+\-]+=?.*?(\:?[0-9a-z]{5,})\]#';
// Sequences > min_search_chars & < max_search_chars
// $match[] = '#\s([\b]{1,' . $config['min_search_chars'] . '}|[\b]{' . $config['max_search_chars'] . ',})\s#is';
// $match[] = '#\s((&\#[0-9]+;){1,' . $config['min_search_chars'] . '}|(&\#[0-9]+;){' . $config['max_search_chars'] . ',})\s#is';
// Filter out ; and # but not &#[0-9]+;
// $match[] = '#(&\#[0-9]+;)|;|\#|&#';
$text = preg_replace($match, ' ', ' ' . strtolower(trim($text)) . ' ');
$text = str_replace(array(' + ', ' - '), array(' and ', ' not '), $text);
// Filter out non-alphabetical chars
$text = str_replace($drop_char_match, $drop_char_replace, $text);
// Split words
$text = explode(' ', preg_replace('#\s+#', ' ', trim($text)));
if (sizeof($stopwords))
{
$stopped_words = array_intersect($text, $stopwords);
$text = array_diff($text, $stopwords);
}
if (sizeof($replace_synonym))
{
$text = str_replace($replace_synonym, $match_synonym, $text);
}
foreach ($text as $index => $word)
{
if (strlen($word) < $config['min_search_chars'] || strlen($word) > $config['max_search_chars'])
{
unset($text[$index]);
}
}
return $text;
}
function add($mode, $post_id, &$message, &$subject)
{
global $config, $db;
// Is the fulltext indexer disabled? If yes then we need not
// carry on ... it's okay ... I know when I'm not wanted boo hoo
if (!$config['load_search_upd'])
{
return;
}
// Split old and new post/subject to obtain array of 'words'
$split_text = $this->split_words('post', $message);
$split_title = ($subject) ? $this->split_words('post', $subject) : array();
$words = array();
if ($mode == 'edit')
{
$words['add']['post'] = array();
$words['add']['title'] = array();
$words['del']['post'] = array();
$words['del']['title'] = array();
$sql = 'SELECT w.word_id, w.word_text, m.title_match
FROM ' . SEARCH_WORD_TABLE . ' w, ' . SEARCH_MATCH_TABLE . " m
WHERE m.post_id = $post_id
AND w.word_id = m.word_id";
$result = $db->sql_query($sql);
$cur_words = array();
while ($row = $db->sql_fetchrow($result))
{
$which = ($row['title_match']) ? 'title' : 'post';
$cur_words[$which][$row['word_text']] = $row['word_id'];
}
$db->sql_freeresult($result);
$words['add']['post'] = array_diff($split_text, array_keys($cur_words['post']));
$words['add']['title'] = array_diff($split_title, array_keys($cur_words['title']));
$words['del']['post'] = array_diff(array_keys($cur_words['post']), $split_text);
$words['del']['title'] = array_diff(array_keys($cur_words['title']), $split_title);
}
else
{
$words['add']['post'] = $split_text;
$words['add']['title'] = $split_title;
$words['del']['post'] = array();
$words['del']['title'] = array();
}
unset($split_text);
unset($split_title);
// Get unique words from the above arrays
$unique_add_words = array_unique(array_merge($words['add']['post'], $words['add']['title']));
// We now have unique arrays of all words to be added and removed and
// individual arrays of added and removed words for text and title. What
// we need to do now is add the new words (if they don't already exist)
// and then add (or remove) matches between the words and this post
if (sizeof($unique_add_words))
{
$sql = 'SELECT word_id, word_text
FROM ' . SEARCH_WORD_TABLE . '
WHERE word_text IN (' . implode(', ', preg_replace('#^(.*)$#', '\'$1\'', $unique_add_words)) . ")";
$result = $db->sql_query($sql);
$word_ids = array();
while ($row = $db->sql_fetchrow($result))
{
$word_ids[$row['word_text']] = $row['word_id'];
}
$db->sql_freeresult($result);
$new_words = array_diff($unique_add_words, array_keys($word_ids));
unset($unique_add_words);
if (sizeof($new_words))
{
switch (SQL_LAYER)
{
case 'mysql':
$sql = 'INSERT INTO ' . SEARCH_WORD_TABLE . ' (word_text)
VALUES ' . implode(', ', preg_replace('#^(.*)$#', '(\'$1\')', $new_words));
$db->sql_query($sql);
break;
case 'mysql4':
case 'mysqli':
case 'mssql':
case 'mssql_odbc':
case 'sqlite':
$sql = 'INSERT INTO ' . SEARCH_WORD_TABLE . ' (word_text) ' . implode(' UNION ALL ', preg_replace('#^(.*)$#', "SELECT '\$1'", $new_words));
$db->sql_query($sql);
break;
default:
foreach ($new_words as $word)
{
$sql = 'INSERT INTO ' . SEARCH_WORD_TABLE . " (word_text)
VALUES ('$word')";
$db->sql_query($sql);
}
break;
}
}
unset($new_words);
}
foreach ($words['del'] as $word_in => $word_ary)
{
$title_match = ($word_in == 'title') ? 1 : 0;
if (sizeof($word_ary))
{
$sql_in = array();
foreach ($word_ary as $word)
{
$sql_in[] = $cur_words[$word_in][$word];
}
$sql = 'DELETE FROM ' . SEARCH_MATCH_TABLE . '
WHERE word_id IN (' . implode(', ', $sql_in) . ')
AND post_id = ' . intval($post_id) . "
AND title_match = $title_match";
$db->sql_query($sql);
unset($sql_in);
}
}
foreach ($words['add'] as $word_in => $word_ary)
{
$title_match = ($word_in == 'title') ? 1 : 0;
if (sizeof($word_ary))
{
$sql = 'INSERT INTO ' . SEARCH_MATCH_TABLE . " (post_id, word_id, title_match)
SELECT $post_id, word_id, $title_match
FROM " . SEARCH_WORD_TABLE . '
WHERE word_text IN (' . implode(', ', preg_replace('#^(.*)$#', '\'$1\'', $word_ary)) . ')';
$db->sql_query($sql);
}
}
unset($words);
// Run the cleanup infrequently, once per session cleanup
if ($config['search_last_gc'] < time() - $config['search_gc'])
{
$this->search_tidy();
}
}
// Tidy up indexes, tag 'common words', remove
// words no longer referenced in the match table, etc.
function search_tidy()
{
global $db, $config;
// Is the fulltext indexer disabled? If yes then we need not
// carry on ... it's okay ... I know when I'm not wanted boo hoo
if (!$config['load_search_upd'])
{
return;
}
// Remove common (> 60% of posts ) words
$sql = 'SELECT SUM(forum_posts) AS total_posts
FROM ' . FORUMS_TABLE;
$result = $db->sql_query($sql);
$row = $db->sql_fetchrow($result);
$db->sql_freeresult($result);
if ($row['total_posts'] >= 100)
{
$sql = 'SELECT word_id
FROM ' . SEARCH_MATCH_TABLE . '
GROUP BY word_id
HAVING COUNT(word_id) > ' . floor($row['total_posts'] * 0.6);
$result = $db->sql_query($sql);
if ($row = $db->sql_fetchrow($result))
{
$sql_in = array();
do
{
$sql_in[] = $row['word_id'];
}
while ($row = $db->sql_fetchrow($result));
$sql_in = implode(', ', $sql_in);
$sql = 'UPDATE ' . SEARCH_WORD_TABLE . "
SET word_common = 1
WHERE word_id IN ($sql_in)";
$db->sql_query($sql);
$sql = 'DELETE FROM ' . SEARCH_MATCH_TABLE . "
WHERE word_id IN ($sql_in)";
$db->sql_query($sql);
unset($sql_in);
}
$db->sql_freeresult($result);
}
// Remove words with no matches ... this is a potentially nasty query
$sql = 'SELECT w.word_id
FROM ' . SEARCH_WORD_TABLE . ' w
LEFT JOIN ' . SEARCH_MATCH_TABLE . ' m ON (w.word_id = m.word_id)
WHERE w.word_common = 0 AND m.word_id IS NULL
GROUP BY m.word_id';
$result = $db->sql_query($sql);
if ($row = $db->sql_fetchrow($result))
{
$sql_in = array();
do
{
$sql_in[] = $row['word_id'];
}
while ($row = $db->sql_fetchrow($result));
$sql = 'DELETE FROM ' . SEARCH_WORD_TABLE . '
WHERE word_id IN (' . implode(', ', $sql_in) . ')';
$db->sql_query($sql);
unset($sql_in);
}
$db->sql_freeresult($result);
set_config('search_last_gc', time());
}
}
?>