mirror/php-phpbb
1
0
Fork 0
mirror of https://github.com/phpbb/phpbb.git synced 2025-08-10 18:54:08 +02:00
Code Issues Releases Wiki Activity

- overhauled search system

Browse Source 
- updated structure for search backend plugins
  - better result caching using ACM
  - search results no longer session restricted => link to them by copying the URL :)
  - in-topic search
  - indexing posts now uses search backend plugins
  - develop/search_fill.php working again
  - fulltext_mysql not working yet
- tiny bugfixes to ACM


git-svn-id: file:///svn/phpbb/trunk@5441 89ea8834-ac86-4346-8a33-228a782c2dd0
This commit is contained in:
Nils Adermann
2006-01-11 18:56:07 +00:00
parent 9ea5fa1768
commit 0e0b1120fb
22 changed files with 1522 additions and 1036 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
phpBB/develop/search_fill.php
View File

@@ -13,22 +13,35 @@
//
set_time_limit(0);
$phpbb_root_path = "../";
include($phpbb_root_path . 'extension.inc');
define('IN_PHPBB', true);
$phpbb_root_path = '../';
$phpEx = substr(strrchr(__FILE__, '.'), 1);
include($phpbb_root_path . 'common.'.$phpEx);
include($phpbb_root_path . 'includes/search.'.$phpEx);
$common_percent = 0.4; // Percentage of posts in which a word has to appear to be marked as common
// Start session management
$user->session_begin();
$auth->acl($user->data);
$user->setup();
$search_type = $config['search_type'];
if (!file_exists($phpbb_root_path . 'includes/search/' . $search_type . '.' . $phpEx))
{
trigger_error('NO_SUCH_SEARCH_MODULE');
}
require($phpbb_root_path . 'includes/search/' . $search_type . '.' . $phpEx);
$error = false;
$search = new $search_type($error);
if ($error)
{
trigger_error($error);
}
print "<html>\n<body>\n";
//
// Try and load stopword and synonym files
//
// This needs fixing! Shouldn't be hardcoded to English files!
$stopword_array = file($phpbb_root_path . "language/lang_english/search_stopwords.txt");
$synonym_array = file($phpbb_root_path . "language/lang_english/search_synonyms.txt");
//
// Fetch a batch of posts_text entries
//
@@ -74,9 +87,9 @@ for(;$postcounter <= $max_post_id; $postcounter += $batchsize)
if( $post_rows )
{
// $sql = "LOCK TABLES ".POST_TEXT_TABLE." WRITE";
// $result = $db->sql_query($sql);
print "\n<p>\n<a href='$PHP_SELF?batchstart=$batchstart'>Restart from posting $batchstart</a><br>\n";
// $sql = "LOCK TABLES ".POST_TEXT_TABLE." WRITE";
// $result = $db->sql_query($sql);
print "\n<p>\n<a href='{$_SERVER['PHP_SELF']}?batchstart=$batchstart'>Restart from posting $batchstart</a><br>\n";
// For every post in the batch:
for($post_nr = 0; $post_nr < $post_rows; $post_nr++ )
@@ -86,105 +99,19 @@ for(;$postcounter <= $max_post_id; $postcounter += $batchsize)
$post_id = $rowset[$post_nr]['post_id'];
$matches = array();
$matches['text'] = split_words(clean_words("post", $rowset[$post_nr]['post_text'], $stopword_array, $synonym_array));
$matches['title'] = split_words(clean_words("post", $rowset[$post_nr]['post_subject'], $stopword_array, $synonym_array));
while( list($match_type, $match_ary) = @each($matches) )
{
$title_match = ( $match_type == 'title' ) ? 1 : 0;
$num_matches = count($match_ary);
if ( $num_matches < 1 )
{
// Skip this post if no words where found
continue;
}
// For all words in the posting
$sql_in = "";
$sql_insert = '';
$sql_select = '';
$word = array();
$word_count = array();
for($j = 0; $j < $num_matches; $j++)
{
$this_word = strtolower(trim($match_ary[$j]));
if ( $this_word != '' )
{
$word_count[$this_word] = ( isset($word_count[$this_word]) ) ? $word_count[$this_word] + 1 : 0;
$comma = ($sql_insert != '')? ', ': '';
$sql_insert .= "$comma('" . $this_word . "')";
$sql_select .= "$comma'" . $this_word . "'";
}
}
if ( $sql_insert == '' )
{
die("no words found");
}
$sql = 'INSERT IGNORE INTO ' . SEARCH_WORD_TABLE . "
(word_text)
VALUES $sql_insert";
if ( !$result = $db->sql_query($sql) )
{
$error = $db->sql_error();
die("Couldn't INSERT words :: " . $sql . " :: " . $error['message']);
}
// Get the word_id's out of the DB (to see if they are already there)
$sql = "SELECT word_id, word_text
FROM " . SEARCH_WORD_TABLE . "
WHERE word_text IN ($sql_select)
GROUP BY word_text";
$result = $db->sql_query($sql);
if ( !$result )
{
$error = $db->sql_error();
die("Couldn't select words :: " . $sql . " :: " . $error['message']);
}
$sql_insert = array();
while( $row = $db->sql_fetchrow($result) )
{
$sql_insert[] = "($post_id, " . $row['word_id'] . ", $title_match)";
}
$db->sql_freeresult($result);
$sql = "INSERT INTO " . SEARCH_MATCH_TABLE . "
(post_id, word_id, title_match)
VALUES " . implode(", ", $sql_insert);
$result = $db->sql_query($sql);
if ( !$result )
{
$error = $db->sql_error();
die("Couldn't insert new word match :: " . $sql . " :: " . $error['message']);
}
} // All posts
$search->index('post', $rowset[$post_nr]['post_id'], $rowset[$post_nr]['post_text'], $rowset[$post_nr]['post_subject']);
}
// $sql = "UNLOCK TABLES";
// $result = $db->sql_query($sql);
// $sql = "UNLOCK TABLES";
// $result = $db->sql_query($sql);
}
// Remove common words after the first 2 batches and after every 4th batch after that.
if( $batchcount % 4 == 3 )
{
print "<br>Removing common words (words that appear in more than $common_percent of the posts)<br>\n";
flush();
print "Removed ". remove_common("global", $common_percent) ." words that where too common.<br>";
}
}
print "<br>Removing common words (words that appear in more than 50% of the posts)<br>\n";
flush();
$search->tidy();
print "Removed words that where too common.<br>";
echo "<br>Done";
?>