From c2e970779d5d34cdaeb52e508282a4acb46d3e00 Mon Sep 17 00:00:00 2001 From: David Monllao Date: Tue, 25 Jul 2017 15:56:16 +0200 Subject: [PATCH] MDL-53226 search_simpledb: Refine the patch - Clumsy fallback only when there is no full-text search support - Mimic solr tests - pgsql tokenization using simple configuration - workaround for mysql '*' search issue - total results proper calculation - SQL server FTS support - Standarize dml full-text search checkings - Upgrade note about the new dml method - Set search_simpledb as default engine if no solr config --- admin/settings/plugins.php | 2 +- lib/db/upgrade.php | 9 + lib/dml/mariadb_native_moodle_database.php | 14 ++ lib/dml/moodle_database.php | 10 + lib/dml/mysqli_native_moodle_database.php | 14 ++ lib/dml/pgsql_native_moodle_database.php | 9 + lib/dml/sqlsrv_native_moodle_database.php | 22 ++ lib/upgrade.txt | 2 + search/classes/document.php | 6 +- search/engine/simpledb/classes/engine.php | 178 +++++++++++----- search/engine/simpledb/db/install.php | 47 ++++- search/engine/simpledb/db/uninstall.php | 22 +- search/engine/simpledb/tests/engine_test.php | 207 ++++++++++++++----- search/engine/simpledb/version.php | 4 +- version.php | 2 +- 15 files changed, 431 insertions(+), 117 deletions(-) diff --git a/admin/settings/plugins.php b/admin/settings/plugins.php index c55b1f12dc3..9a2202f158c 100644 --- a/admin/settings/plugins.php +++ b/admin/settings/plugins.php @@ -556,7 +556,7 @@ if ($hassiteconfig) { // Search engine selection. $temp->add(new admin_setting_heading('searchengineheading', new lang_string('searchengine', 'admin'), '')); $temp->add(new admin_setting_configselect('searchengine', - new lang_string('selectsearchengine', 'admin'), '', 'solr', $engines)); + new lang_string('selectsearchengine', 'admin'), '', 'simpledb', $engines)); $temp->add(new admin_setting_heading('searchoptionsheading', new lang_string('searchoptions', 'admin'), '')); $temp->add(new admin_setting_configcheckbox('searchindexwhendisabled', new lang_string('searchindexwhendisabled', 'admin'), new lang_string('searchindexwhendisabled_desc', 'admin'), diff --git a/lib/db/upgrade.php b/lib/db/upgrade.php index 78ebbee0dd0..942fca9c7e1 100644 --- a/lib/db/upgrade.php +++ b/lib/db/upgrade.php @@ -2020,5 +2020,14 @@ function xmldb_main_upgrade($oldversion) { upgrade_main_savepoint(true, 2018022800.03); } + if ($oldversion < 2018031600.01) { + + // Update default search engine to search_simpledb if global search is disabled and there is no solr index defined. + if (empty($CFG->enableglobalsearch) && empty(get_config('search_solr', 'indexname'))) { + set_config('searchengine', 'simpledb'); + } + upgrade_main_savepoint(true, 2018031600.01); + } + return true; } diff --git a/lib/dml/mariadb_native_moodle_database.php b/lib/dml/mariadb_native_moodle_database.php index 48debee57ce..c333222618d 100644 --- a/lib/dml/mariadb_native_moodle_database.php +++ b/lib/dml/mariadb_native_moodle_database.php @@ -107,4 +107,18 @@ class mariadb_native_moodle_database extends mysqli_native_moodle_database { } return true; } + + /** + * Does this mariadb instance support fulltext indexes? + * + * @return bool + */ + public function is_fulltext_search_supported() { + $info = $this->get_server_info(); + + if (version_compare($info['version'], '10.0.5', '>=')) { + return true; + } + return false; + } } diff --git a/lib/dml/moodle_database.php b/lib/dml/moodle_database.php index a16b9336399..77d0b987017 100644 --- a/lib/dml/moodle_database.php +++ b/lib/dml/moodle_database.php @@ -2687,4 +2687,14 @@ abstract class moodle_database { public function perf_get_queries_time() { return $this->queriestime; } + + /** + * Whether the database is able to support full-text search or not. + * + * @return bool + */ + public function is_fulltext_search_supported() { + // No support unless specified. + return false; + } } diff --git a/lib/dml/mysqli_native_moodle_database.php b/lib/dml/mysqli_native_moodle_database.php index 05c6c433ae0..015d3a647d7 100644 --- a/lib/dml/mysqli_native_moodle_database.php +++ b/lib/dml/mysqli_native_moodle_database.php @@ -1989,4 +1989,18 @@ class mysqli_native_moodle_database extends moodle_database { $this->change_database_structure("ALTER TABLE {$prefix}$tablename $rowformat"); } } + + /** + * Does this mysql instance support fulltext indexes? + * + * @return bool + */ + public function is_fulltext_search_supported() { + $info = $this->get_server_info(); + + if (version_compare($info['version'], '5.6.4', '>=')) { + return true; + } + return false; + } } diff --git a/lib/dml/pgsql_native_moodle_database.php b/lib/dml/pgsql_native_moodle_database.php index ae7550175e4..97d94905ff9 100644 --- a/lib/dml/pgsql_native_moodle_database.php +++ b/lib/dml/pgsql_native_moodle_database.php @@ -1497,4 +1497,13 @@ class pgsql_native_moodle_database extends moodle_database { private function trim_quotes($str) { return trim(trim($str), "'\""); } + + /** + * Postgresql supports full-text search indexes. + * + * @return bool + */ + public function is_fulltext_search_supported() { + return true; + } } diff --git a/lib/dml/sqlsrv_native_moodle_database.php b/lib/dml/sqlsrv_native_moodle_database.php index d1d27f128ca..9174f70c600 100644 --- a/lib/dml/sqlsrv_native_moodle_database.php +++ b/lib/dml/sqlsrv_native_moodle_database.php @@ -1586,4 +1586,26 @@ class sqlsrv_native_moodle_database extends moodle_database { $result = sqlsrv_rollback($this->sqlsrv); $this->query_end($result); } + + /** + * Is fulltext search enabled?. + * + * @return bool + */ + public function is_fulltext_search_supported() { + global $CFG; + + $sql = "SELECT FULLTEXTSERVICEPROPERTY('IsFullTextInstalled')"; + $this->query_start($sql, null, SQL_QUERY_AUX); + $result = sqlsrv_query($this->sqlsrv, $sql); + $this->query_end($result); + if ($result) { + if ($row = sqlsrv_fetch_array($result)) { + $property = (bool)reset($row); + } + } + $this->free_result($result); + + return !empty($property); + } } diff --git a/lib/upgrade.txt b/lib/upgrade.txt index 174efd92d20..c569cf603b0 100644 --- a/lib/upgrade.txt +++ b/lib/upgrade.txt @@ -22,6 +22,8 @@ information provided here is intended especially for developers. * Scripts can define a constant NO_SITEPOLICY_CHECK and set it to true before requiring the main config.php file. It will make the require_login() skipping the test for the user's policyagreed status. This is useful for plugins that act as a site policy handler. +* There is a new is_fulltext_search_supported() DML function. The default implementation returns false. This function + is used by 'Simple search' global search engine to determine if the database full-text search capabilities can be used. === 3.4 === diff --git a/search/classes/document.php b/search/classes/document.php index 7625f8173ce..1fea314dd80 100644 --- a/search/classes/document.php +++ b/search/classes/document.php @@ -407,8 +407,7 @@ class document implements \renderable, \templatable { * @return string */ public static function format_string_for_engine($string) { - //FIXME: this shouldn't be required. Where is bad utf8 coming from? - return fix_utf8($string); + return $string; } /** @@ -421,8 +420,7 @@ class document implements \renderable, \templatable { * @return string */ public static function format_text_for_engine($text) { - //FIXME: this shouldn't be required. Where is bad utf8 coming from? - return fix_utf8($text); + return $text; } /** diff --git a/search/engine/simpledb/classes/engine.php b/search/engine/simpledb/classes/engine.php index 2bb43fb0301..f77084b07b9 100644 --- a/search/engine/simpledb/classes/engine.php +++ b/search/engine/simpledb/classes/engine.php @@ -36,14 +36,22 @@ defined('MOODLE_INTERNAL') || die(); class engine extends \core_search\engine { /** - * Prepares a Solr query, applies filters and executes it returning its results. + * Total number of available results. + * + * @var null|int + */ + protected $totalresults = null; + + /** + * Prepares a SQL query, applies filters and executes it returning its results. * * @throws \core_search\engine_exception * @param stdClass $filters Containing query and filters. * @param array $usercontexts Contexts where the user has access. True if the user can access all contexts. + * @param int $limit The maximum number of results to return. * @return \core_search\document[] Results or false if no results */ - public function execute_query($filters, $usercontexts) { + public function execute_query($filters, $usercontexts, $limit = 0) { global $DB, $USER; // Let's keep these changes internal. @@ -54,7 +62,10 @@ class engine extends \core_search\engine { throw new \core_search\engine_exception('engineserverstatus', 'search'); } - $sql = 'SELECT * FROM {search_simpledb_index} WHERE '; + if (empty($limit)) { + $limit = \core_search\manager::MAX_RESULTS; + } + $params = array(); // To store all conditions we will add to where. @@ -71,7 +82,7 @@ class engine extends \core_search\engine { // Join all area contexts into a single array and implode. $allcontexts = array(); foreach ($usercontexts as $areaid => $areacontexts) { - if (!empty($data->areaid) && ($areaid !== $data->areaid)) { + if (!empty($data->areaids) && !in_array($areaid, $data->areaids)) { // Skip unused areas. continue; } @@ -98,16 +109,15 @@ class engine extends \core_search\engine { } // Area id filter. - if (!empty($data->areaid)) { - list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->areaid); + if (!empty($data->areaids)) { + list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->areaids); $ands[] = 'areaid ' . $conditionsql; $params = array_merge($params, $conditionparams); } if (!empty($data->title)) { - list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->title); - $ands[] = 'title ' . $conditionsql; - $params = array_merge($params, $conditionparams); + $ands[] = $DB->sql_like('title', '?', false, false); + $params[] = $data->title; } if (!empty($data->timestart)) { @@ -120,53 +130,77 @@ class engine extends \core_search\engine { } // And finally the main query after applying all AND filters. - switch ($DB->get_dbfamily()) { - case 'postgres': - $ands[] = "(" . - "to_tsvector('simple', title) @@ plainto_tsquery(?) OR ". - "to_tsvector('simple', content) @@ plainto_tsquery(?) OR ". - "to_tsvector('simple', description1) @@ plainto_tsquery(?) OR ". - "to_tsvector('simple', description2) @@ plainto_tsquery(?)". - ")"; - $params[] = $data->q; - $params[] = $data->q; - $params[] = $data->q; - $params[] = $data->q; - break; - case 'mysql': - $ands[] = "MATCH (title, content, description1, description2) AGAINST (?)"; - $params[] = $data->q; - break; - case 'mssql': - $ands[] = "CONTAINS ((title, content, description1, description2), ?)"; - $params[] = $data->q; - break; - default: - $ands[] = '(' . - $DB->sql_like('title', '?', false, false) . ' OR ' . - $DB->sql_like('content', '?', false, false) . ' OR ' . - $DB->sql_like('description1', '?', false, false) . ' OR ' . - $DB->sql_like('description2', '?', false, false) . - ')'; - $params[] = '%' . $data->q . '%'; - $params[] = '%' . $data->q . '%'; - $params[] = '%' . $data->q . '%'; - $params[] = '%' . $data->q . '%'; - break; + if (!empty($data->q)) { + switch ($DB->get_dbfamily()) { + case 'postgres': + $ands[] = "(" . + "to_tsvector('simple', title) @@ plainto_tsquery('simple', ?) OR ". + "to_tsvector('simple', content) @@ plainto_tsquery('simple', ?) OR ". + "to_tsvector('simple', description1) @@ plainto_tsquery('simple', ?) OR ". + "to_tsvector('simple', description2) @@ plainto_tsquery('simple', ?)". + ")"; + $params[] = $data->q; + $params[] = $data->q; + $params[] = $data->q; + $params[] = $data->q; + break; + case 'mysql': + if ($DB->is_fulltext_search_supported()) { + $ands[] = "MATCH (title, content, description1, description2) AGAINST (?)"; + $params[] = $data->q; + + // Sorry for the hack, but it does not seem that we will have a solution for + // this soon (https://bugs.mysql.com/bug.php?id=78485). + if ($data->q === '*') { + return array(); + } + } else { + // Clumsy version for mysql versions with no fulltext support. + list($queryand, $queryparams) = $this->get_simple_query($data->q); + $ands[] = $queryand; + $params = array_merge($params, $queryparams); + } + break; + case 'mssql': + if ($DB->is_fulltext_search_supported()) { + $ands[] = "CONTAINS ((title, content, description1, description2), ?)"; + // Special treatment for double quotes: + // - Puntuation is ignored so we can get rid of them. + // - Phrases should be enclosed in double quotation marks. + $params[] = '"' . str_replace('"', '', $data->q) . '"'; + } else { + // Clumsy version for mysql versions with no fulltext support. + list($queryand, $queryparams) = $this->get_simple_query($data->q); + $ands[] = $queryand; + $params = array_merge($params, $queryparams); + } + break; + default: + list($queryand, $queryparams) = $this->get_simple_query($data->q); + $ands[] = $queryand; + $params = array_merge($params, $queryparams); + break; + } } - $recordset = $DB->get_recordset_sql($sql . implode(' AND ', $ands), $params, 0, \core_search\manager::MAX_RESULTS); + // It is limited to $limit, no need to use recordsets. + $documents = $DB->get_records_select('search_simpledb_index', implode(' AND ', $ands), $params, '', '*', 0, $limit); + + // Hopefully database cached results as this applies the same filters than above. + $this->totalresults = $DB->count_records_select('search_simpledb_index', implode(' AND ', $ands), $params); $numgranted = 0; - if (!$recordset->valid()) { - return array(); - } - // Iterate through the results checking its availability and whether they are available for the user or not. $docs = array(); - foreach ($recordset as $docdata) { + foreach ($documents as $docdata) { + if ($docdata->owneruserid != \core_search\manager::NO_OWNER_ID && $docdata->owneruserid != $USER->id) { + // If owneruserid is set, no other user should be able to access this record. + continue; + } + if (!$searcharea = $this->get_search_area($docdata->areaid)) { + $this->totalresults--; continue; } @@ -178,8 +212,10 @@ class engine extends \core_search\engine { switch ($access) { case \core_search\manager::ACCESS_DELETED: $this->delete_by_id($docdata->id); + $this->totalresults--; break; case \core_search\manager::ACCESS_DENIED: + $this->totalresults--; break; case \core_search\manager::ACCESS_GRANTED: $numgranted++; @@ -188,12 +224,11 @@ class engine extends \core_search\engine { } // This should never happen. - if ($numgranted >= \core_search\manager::MAX_RESULTS) { - $docs = array_slice($docs, 0, \core_search\manager::MAX_RESULTS, true); + if ($numgranted >= $limit) { + $docs = array_slice($docs, 0, $limit, true); break; } } - $recordset->close(); return $docs; } @@ -228,6 +263,7 @@ class engine extends \core_search\engine { } catch (\dml_exception $ex) { debugging('dml error while trying to insert document with id ' . $doc->docid . ': ' . $ex->getMessage(), DEBUG_DEVELOPER); + return false; } return true; @@ -281,4 +317,44 @@ class engine extends \core_search\engine { public function is_installed() { return true; } + + /** + * Returns the total results. + * + * Including skipped results. + * + * @return int + */ + public function get_query_total_count() { + if (!is_null($this->totalresults)) { + // This is a just in case as we count total results in execute_query. + return \core_search\manager::MAX_RESULTS; + } + + return $this->totalresults; + } + + /** + * Returns the default query for db engines. + * + * @param string $q The query string + * @return array SQL string and params list + */ + protected function get_simple_query($q) { + global $DB; + + $sql = '(' . + $DB->sql_like('title', '?', false, false) . ' OR ' . + $DB->sql_like('content', '?', false, false) . ' OR ' . + $DB->sql_like('description1', '?', false, false) . ' OR ' . + $DB->sql_like('description2', '?', false, false) . + ')'; + $params = array( + '%' . $q . '%', + '%' . $q . '%', + '%' . $q . '%', + '%' . $q . '%' + ); + return array($sql, $params); + } } diff --git a/search/engine/simpledb/db/install.php b/search/engine/simpledb/db/install.php index f7f06e37c02..0e1b51b4b36 100644 --- a/search/engine/simpledb/db/install.php +++ b/search/engine/simpledb/db/install.php @@ -24,26 +24,51 @@ defined('MOODLE_INTERNAL') || die; +/** + * Post installation code. + * + * @package search_simpledb + * @copyright 2016 Dan Poltawski + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ function xmldb_search_simpledb_install() { global $DB; switch ($DB->get_dbfamily()) { case 'postgres': - // TODO: There are a few other ways of doing this which avoid the need for individual indicies.. - $DB->execute("CREATE INDEX psql_search_title ON {search_simpledb_index} USING gin(to_tsvector('simple', title))"); - $DB->execute("CREATE INDEX psql_search_content ON {search_simpledb_index} USING gin(to_tsvector('simple', content))"); - $DB->execute("CREATE INDEX psql_search_description1 ON {search_simpledb_index} USING gin(to_tsvector('simple', description1))"); - $DB->execute("CREATE INDEX psql_search_description2 ON {search_simpledb_index} USING gin(to_tsvector('simple', description2))"); + // There are a few other ways of doing this which avoid the need for individual indexes. + $DB->execute("CREATE INDEX {search_simpledb_title} ON {search_simpledb_index} " . + "USING gin(to_tsvector('simple', title))"); + $DB->execute("CREATE INDEX {search_simpledb_content} ON {search_simpledb_index} " . + "USING gin(to_tsvector('simple', content))"); + $DB->execute("CREATE INDEX {search_simpledb_description1} ON {search_simpledb_index} " . + "USING gin(to_tsvector('simple', description1))"); + $DB->execute("CREATE INDEX {search_simpledb_description2} ON {search_simpledb_index} " . + "USING gin(to_tsvector('simple', description2))"); break; case 'mysql': - $DB->execute("CREATE FULLTEXT INDEX mysql_search_index - ON {search_simpledb_index} (title, content, description1, description2)"); + if ($DB->is_fulltext_search_supported()) { + $DB->execute("CREATE FULLTEXT INDEX {search_simpledb_index_index} + ON {search_simpledb_index} (title, content, description1, description2)"); + } break; case 'mssql': - //TODO: workout if fulltext search is installed... select SERVERPROPERTY('IsFullTextInstalled') - $DB->execute("CREATE FULLTEXT CATALOG {search_simpledb_catalog}"); - $DB->execute("CREATE FULLTEXT INDEX ON {search_simpledb_index} (title, content, description1, description2) - KEY INDEX {searsimpinde_id_pk} ON {search_simpledb_catalog}"); + if ($DB->is_fulltext_search_supported()) { + + $catalogname = $DB->get_prefix() . 'search_simpledb_catalog'; + if (!$DB->record_exists_sql('SELECT * FROM sys.fulltext_catalogs WHERE name = ?', array($catalogname))) { + $DB->execute("CREATE FULLTEXT CATALOG {search_simpledb_catalog} WITH ACCENT_SENSITIVITY=OFF"); + } + + if (defined('PHPUNIT_UTIL') and PHPUNIT_UTIL) { + // We want manual tracking for phpunit because the fulltext index does get auto populated fast enough. + $changetracking = 'MANUAL'; + } else { + $changetracking = 'AUTO'; + } + $DB->execute("CREATE FULLTEXT INDEX ON {search_simpledb_index} (title, content, description1, description2) + KEY INDEX {searsimpinde_id_pk} ON {search_simpledb_catalog} WITH CHANGE_TRACKING $changetracking"); + } break; } } diff --git a/search/engine/simpledb/db/uninstall.php b/search/engine/simpledb/db/uninstall.php index 6acbba52cb4..197bcfd0383 100644 --- a/search/engine/simpledb/db/uninstall.php +++ b/search/engine/simpledb/db/uninstall.php @@ -24,12 +24,32 @@ defined('MOODLE_INTERNAL') || die; +/** + * Plugin uninstall code. + * + * @package search_simpledb + * @copyright 2016 Dan Poltawski + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ function xmldb_search_simpledb_uninstall() { global $DB; switch ($DB->get_dbfamily()) { + case 'postgres': + $DB->execute("DROP INDEX {search_simpledb_title}"); + $DB->execute("DROP INDEX {search_simpledb_content}"); + $DB->execute("DROP INDEX {search_simpledb_description1}"); + $DB->execute("DROP INDEX {search_simpledb_description2}"); + break; + case 'mysql': + if ($DB->is_fulltext_search_supported()) { + $DB->execute("ALTER TABLE {search_simpledb_index} DROP INDEX {search_simpledb_index_index}"); + } + break; case 'mssql': - $DB->execute("DROP FULLTEXT CATALOG {search_simpledb_catalog}"); + if ($DB->is_fulltext_search_supported()) { + $DB->execute("DROP FULLTEXT CATALOG {search_simpledb_catalog}"); + } break; } } diff --git a/search/engine/simpledb/tests/engine_test.php b/search/engine/simpledb/tests/engine_test.php index fa37e19eb0b..46dd4a6cb2d 100644 --- a/search/engine/simpledb/tests/engine_test.php +++ b/search/engine/simpledb/tests/engine_test.php @@ -18,7 +18,7 @@ * Simple db search engine tests. * * @package search_simpledb - * @category phpunit + * @category test * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ @@ -33,7 +33,7 @@ require_once($CFG->dirroot . '/search/tests/fixtures/mock_search_area.php'); * Simple search engine base unit tests. * * @package search_simpledb - * @category phpunit + * @category test * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ @@ -44,22 +44,70 @@ class search_simpledb_engine_testcase extends advanced_testcase { */ protected $search = null; + /** + * @var \ + */ + protected $engine = null; + + /** + * @var core_search_generator + */ + protected $generator = null; + + /** + * Initial stuff. + * + * @return void + */ public function setUp() { $this->resetAfterTest(); + + if ($this->requires_manual_index_update()) { + // We need to update fulltext index manually, which requires an alter table statement. + $this->preventResetByRollback(); + } + set_config('enableglobalsearch', true); // Inject search_simpledb engine into the testable core search as we need to add the mock // search component to it. - $searchengine = new \search_simpledb\engine(); - $this->search = testable_core_search::instance($searchengine); - $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities'); - $this->search->add_search_area($areaid, new core_mocksearch\search\role_capabilities()); + + $this->engine = new \search_simpledb\engine(); + $this->search = testable_core_search::instance($this->engine); + $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area'); + $this->search->add_search_area($areaid, new core_mocksearch\search\mock_search_area()); + + $this->generator = self::getDataGenerator()->get_plugin_generator('core_search'); + $this->generator->setup(); + + $this->setAdminUser(); } + /** + * tearDown + * + * @return void + */ + public function tearDown() { + // For unit tests before PHP 7, teardown is called even on skip. So only do our teardown if we did setup. + if ($this->generator) { + // Moodle DML freaks out if we don't teardown the temp table after each run. + $this->generator->teardown(); + $this->generator = null; + } + } + + /** + * Test indexing process. + * + * @return void + */ public function test_index() { global $DB; - $noneditingteacherid = $DB->get_field('role', 'id', array('shortname' => 'teacher')); + $record = new \stdClass(); + $record->timemodified = time() - 1; + $this->generator->create_record($record); // Data gets into the search engine. $this->assertTrue($this->search->index()); @@ -68,8 +116,7 @@ class search_simpledb_engine_testcase extends advanced_testcase { sleep(1); $this->assertFalse($this->search->index()); - assign_capability('moodle/course:renameroles', CAP_ALLOW, $noneditingteacherid, context_system::instance()->id); - accesslib_clear_all_caches_for_unit_testing(); + $this->generator->create_record(); // Indexing again once there is new data. $this->assertTrue($this->search->index()); @@ -83,19 +130,13 @@ class search_simpledb_engine_testcase extends advanced_testcase { public function test_search() { global $USER, $DB; - $this->setAdminUser(); - - $noneditingteacherid = $DB->get_field('role', 'id', array('shortname' => 'teacher')); - - $this->search->index(); - - // Check that docid - id is respected. - $rolecaps = $DB->get_records('role_capabilities', array('capability' => 'moodle/course:renameroles')); - $rolecap = reset($rolecaps); - $rolecap->timemodified = time(); - $DB->update_record('role_capabilities', $rolecap); + $this->generator->create_record(); + $record = new \stdClass(); + $record->title = "Special title"; + $this->generator->create_record($record); $this->search->index(); + $this->update_index(); $querydata = new stdClass(); $querydata->q = 'message'; @@ -104,32 +145,32 @@ class search_simpledb_engine_testcase extends advanced_testcase { // Based on core_mocksearch\search\indexer. $this->assertEquals($USER->id, $results[0]->get('userid')); - $this->assertEquals(\context_system::instance()->id, $results[0]->get('contextid')); + $this->assertEquals(\context_course::instance(SITEID)->id, $results[0]->get('contextid')); // Do a test to make sure we aren't searching non-query fields, like areaid. - $querydata->q = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities'); + $querydata->q = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area'); $this->assertCount(0, $this->search->search($querydata)); $querydata->q = 'message'; sleep(1); $beforeadding = time(); sleep(1); - assign_capability('moodle/course:renameroles', CAP_ALLOW, $noneditingteacherid, context_system::instance()->id); - accesslib_clear_all_caches_for_unit_testing(); + $this->generator->create_record(); $this->search->index(); + $this->update_index(); // Timestart. $querydata->timestart = $beforeadding; - $this->assertCount(2, $this->search->search($querydata)); + $this->assertCount(1, $this->search->search($querydata)); // Timeend. unset($querydata->timestart); $querydata->timeend = $beforeadding; - $this->assertCount(1, $this->search->search($querydata)); + $this->assertCount(2, $this->search->search($querydata)); // Title. unset($querydata->timeend); - $querydata->title = 'moodle/course:renameroles roleid 1'; + $querydata->title = 'Special title'; $this->assertCount(1, $this->search->search($querydata)); // Course IDs. @@ -140,42 +181,69 @@ class search_simpledb_engine_testcase extends advanced_testcase { $querydata->courseids = array(SITEID); $this->assertCount(3, $this->search->search($querydata)); + // Now try some area-id combinations. + unset($querydata->courseids); + $forumpostareaid = \core_search\manager::generate_areaid('mod_forum', 'post'); + $mockareaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area'); + + $querydata->areaids = array($forumpostareaid); + $this->assertCount(0, $this->search->search($querydata)); + + $querydata->areaids = array($forumpostareaid, $mockareaid); + $this->assertCount(3, $this->search->search($querydata)); + + $querydata->areaids = array($mockareaid); + $this->assertCount(3, $this->search->search($querydata)); + + $querydata->areaids = array(); + $this->assertCount(3, $this->search->search($querydata)); + // Check that index contents get updated. - $DB->delete_records('role_capabilities', array('capability' => 'moodle/course:renameroles')); + $this->generator->delete_all(); $this->search->index(true); + $this->update_index(); unset($querydata->title); - $querydata->q = '*renameroles*'; + $querydata->q = ''; $this->assertCount(0, $this->search->search($querydata)); } + /** + * Test delete function + * + * @return void + */ public function test_delete() { + + $this->generator->create_record(); + $this->generator->create_record(); $this->search->index(); + $this->update_index(); $querydata = new stdClass(); $querydata->q = 'message'; $this->assertCount(2, $this->search->search($querydata)); - $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities'); + $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area'); $this->search->delete_index($areaid); + $this->update_index(); $this->assertCount(0, $this->search->search($querydata)); } + /** + * Test user is allowed. + * + * @return void + */ public function test_alloweduserid() { - $engine = $this->search->get_engine(); - $area = new core_mocksearch\search\role_capabilities(); - // Get the first record for the recordset. - $recordset = $area->get_recordset_by_timestamp(); - foreach ($recordset as $r) { - $record = $r; - break; - } - $recordset->close(); + $area = new core_mocksearch\search\mock_search_area(); + + $record = $this->generator->create_record(); // Get the doc and insert the default doc. $doc = $area->get_document($record); - $engine->add_document($doc); + $this->engine->add_document($doc); $users = array(); $users[] = $this->getDataGenerator()->create_user(); @@ -190,10 +258,11 @@ class search_simpledb_engine_testcase extends advanced_testcase { $doc = $area->get_document($record); $doc->set('id', $originalid.'-'.$user->id); $doc->set('owneruserid', $user->id); - $engine->add_document($doc); + $this->engine->add_document($doc); } + $this->update_index(); - $engine->area_index_complete($area->get_area_id()); + $this->engine->area_index_complete($area->get_area_id()); $querydata = new stdClass(); $querydata->q = 'message'; @@ -239,21 +308,24 @@ class search_simpledb_engine_testcase extends advanced_testcase { } public function test_delete_by_id() { - // First get files in the index. + + $this->generator->create_record(); + $this->generator->create_record(); $this->search->index(); - $engine = $this->search->get_engine(); + $this->update_index(); $querydata = new stdClass(); // Then search to make sure they are there. - $querydata->q = 'moodle/course:renameroles'; + $querydata->q = 'message'; $results = $this->search->search($querydata); $this->assertCount(2, $results); $first = reset($results); $deleteid = $first->get('id'); - $engine->delete_by_id($deleteid); + $this->engine->delete_by_id($deleteid); + $this->update_index(); // Check that we don't get a result for it anymore. $results = $this->search->search($querydata); @@ -261,4 +333,47 @@ class search_simpledb_engine_testcase extends advanced_testcase { $result = reset($results); $this->assertNotEquals($deleteid, $result->get('id')); } + + /** + * Updates mssql fulltext index if necessary. + * + * @return bool + */ + private function update_index() { + global $DB; + + if (!$this->requires_manual_index_update()) { + return; + } + + $DB->execute("ALTER FULLTEXT INDEX ON t_search_simpledb_index START UPDATE POPULATION"); + + $catalogname = $DB->get_prefix() . 'search_simpledb_catalog'; + $retries = 0; + do { + // 0.2 seconds. + usleep(200000); + + $record = $DB->get_record_sql("SELECT FULLTEXTCATALOGPROPERTY(cat.name, 'PopulateStatus') AS [PopulateStatus] + FROM sys.fulltext_catalogs AS cat + WHERE cat.name = ?", array($catalogname)); + $retries++; + + } while ($retries < 100 && $record->populatestatus != '0'); + + if ($retries === 100) { + // No update after 20 seconds... + $this->fail('Sorry, your SQL server fulltext search index is too slow.'); + } + } + + /** + * Mssql with fulltext support requires manual updates. + * + * @return bool + */ + private function requires_manual_index_update() { + global $DB; + return ($DB->get_dbfamily() === 'mssql' && $DB->is_fulltext_search_supported()); + } } diff --git a/search/engine/simpledb/version.php b/search/engine/simpledb/version.php index 10e5a8c40b9..f421370feac 100644 --- a/search/engine/simpledb/version.php +++ b/search/engine/simpledb/version.php @@ -24,6 +24,6 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2016030100; -$plugin->requires = 2015111000; +$plugin->version = 2017072700; +$plugin->requires = 2017072700; $plugin->component = 'search_simpledb'; diff --git a/version.php b/version.php index 63eadc474e8..545841dbdfa 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2018031600.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2018031600.01; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes.