MDL-53226 search_simpledb: Refine the patch

- Clumsy fallback only when there is no full-text search support
- Mimic solr tests
- pgsql tokenization using simple configuration
- workaround for mysql '*' search issue
- total results proper calculation
- SQL server FTS support
- Standarize dml full-text search checkings
- Upgrade note about the new dml method
- Set search_simpledb as default engine if no solr config

admin/settings/plugins.php

@@ -556,7 +556,7 @@ if ($hassiteconfig) {
     // Search engine selection.
     $temp->add(new admin_setting_heading('searchengineheading', new lang_string('searchengine', 'admin'), ''));
     $temp->add(new admin_setting_configselect('searchengine',
-                                new lang_string('selectsearchengine', 'admin'), '', 'solr', $engines));
+                                new lang_string('selectsearchengine', 'admin'), '', 'simpledb', $engines));
     $temp->add(new admin_setting_heading('searchoptionsheading', new lang_string('searchoptions', 'admin'), ''));
     $temp->add(new admin_setting_configcheckbox('searchindexwhendisabled',
             new lang_string('searchindexwhendisabled', 'admin'), new lang_string('searchindexwhendisabled_desc', 'admin'),

lib/db/upgrade.php

@@ -2020,5 +2020,14 @@ function xmldb_main_upgrade($oldversion) {
         upgrade_main_savepoint(true, 2018022800.03);
     }
 
+    if ($oldversion < 2018031600.01) {
+
+        // Update default search engine to search_simpledb if global search is disabled and there is no solr index defined.
+        if (empty($CFG->enableglobalsearch) && empty(get_config('search_solr', 'indexname'))) {
+            set_config('searchengine', 'simpledb');
+        }
+        upgrade_main_savepoint(true, 2018031600.01);
+    }
+
     return true;
 }

lib/dml/mariadb_native_moodle_database.php

@@ -107,4 +107,18 @@ class mariadb_native_moodle_database extends mysqli_native_moodle_database {
         }
         return true;
     }
+
+    /**
+     * Does this mariadb instance support fulltext indexes?
+     *
+     * @return bool
+     */
+    public function is_fulltext_search_supported() {
+        $info = $this->get_server_info();
+
+        if (version_compare($info['version'], '10.0.5', '>=')) {
+            return true;
+        }
+        return false;
+    }
 }

lib/dml/moodle_database.php

@@ -2687,4 +2687,14 @@ abstract class moodle_database {
     public function perf_get_queries_time() {
         return $this->queriestime;
     }
+
+    /**
+     * Whether the database is able to support full-text search or not.
+     *
+     * @return bool
+     */
+    public function is_fulltext_search_supported() {
+        // No support unless specified.
+        return false;
+    }
 }

lib/dml/mysqli_native_moodle_database.php

@@ -1989,4 +1989,18 @@ class mysqli_native_moodle_database extends moodle_database {
             $this->change_database_structure("ALTER TABLE {$prefix}$tablename $rowformat");
         }
     }
+
+    /**
+     * Does this mysql instance support fulltext indexes?
+     *
+     * @return bool
+     */
+    public function is_fulltext_search_supported() {
+        $info = $this->get_server_info();
+
+        if (version_compare($info['version'], '5.6.4', '>=')) {
+            return true;
+        }
+        return false;
+    }
 }

lib/dml/pgsql_native_moodle_database.php

@@ -1497,4 +1497,13 @@ class pgsql_native_moodle_database extends moodle_database {
     private function trim_quotes($str) {
         return trim(trim($str), "'\"");
     }
+
+    /**
+     * Postgresql supports full-text search indexes.
+     *
+     * @return bool
+     */
+    public function is_fulltext_search_supported() {
+        return true;
+    }
 }

lib/dml/sqlsrv_native_moodle_database.php

@@ -1586,4 +1586,26 @@ class sqlsrv_native_moodle_database extends moodle_database {
         $result = sqlsrv_rollback($this->sqlsrv);
         $this->query_end($result);
     }
+
+    /**
+     * Is fulltext search enabled?.
+     *
+     * @return bool
+     */
+    public function is_fulltext_search_supported() {
+        global $CFG;
+
+        $sql = "SELECT FULLTEXTSERVICEPROPERTY('IsFullTextInstalled')";
+        $this->query_start($sql, null, SQL_QUERY_AUX);
+        $result = sqlsrv_query($this->sqlsrv, $sql);
+        $this->query_end($result);
+        if ($result) {
+            if ($row = sqlsrv_fetch_array($result)) {
+                $property = (bool)reset($row);
+            }
+        }
+        $this->free_result($result);
+
+        return !empty($property);
+    }
 }

lib/upgrade.txt

@@ -22,6 +22,8 @@ information provided here is intended especially for developers.
 * Scripts can define a constant NO_SITEPOLICY_CHECK and set it to true before requiring the main config.php file. It
   will make the require_login() skipping the test for the user's policyagreed status. This is useful for plugins that
   act as a site policy handler.
+* There is a new is_fulltext_search_supported() DML function. The default implementation returns false. This function
+  is used by 'Simple search' global search engine to determine if the database full-text search capabilities can be used.
 
 === 3.4 ===
 

search/classes/document.php

@@ -407,8 +407,7 @@ class document implements \renderable, \templatable {
      * @return string
      */
     public static function format_string_for_engine($string) {
-        //FIXME: this shouldn't be required. Where is bad utf8 coming from?
-        return fix_utf8($string);
+        return $string;
     }
 
     /**
@@ -421,8 +420,7 @@ class document implements \renderable, \templatable {
      * @return string
      */
     public static function format_text_for_engine($text) {
-        //FIXME: this shouldn't be required. Where is bad utf8 coming from?
-        return fix_utf8($text);
+        return $text;
     }
 
     /**

search/engine/simpledb/classes/engine.php

@@ -36,14 +36,22 @@ defined('MOODLE_INTERNAL') || die();
 class engine extends \core_search\engine {
 
     /**
-     * Prepares a Solr query, applies filters and executes it returning its results.
+     * Total number of available results.
+     *
+     * @var null|int
+     */
+    protected $totalresults = null;
+
+    /**
+     * Prepares a SQL query, applies filters and executes it returning its results.
      *
      * @throws \core_search\engine_exception
      * @param  stdClass     $filters Containing query and filters.
      * @param  array        $usercontexts Contexts where the user has access. True if the user can access all contexts.
+     * @param  int          $limit The maximum number of results to return.
      * @return \core_search\document[] Results or false if no results
      */
-    public function execute_query($filters, $usercontexts) {
+    public function execute_query($filters, $usercontexts, $limit = 0) {
         global $DB, $USER;
 
         // Let's keep these changes internal.
@@ -54,7 +62,10 @@ class engine extends \core_search\engine {
             throw new \core_search\engine_exception('engineserverstatus', 'search');
         }
 
-        $sql = 'SELECT * FROM {search_simpledb_index} WHERE ';
+        if (empty($limit)) {
+            $limit = \core_search\manager::MAX_RESULTS;
+        }
+
         $params = array();
 
         // To store all conditions we will add to where.
@@ -71,7 +82,7 @@ class engine extends \core_search\engine {
             // Join all area contexts into a single array and implode.
             $allcontexts = array();
             foreach ($usercontexts as $areaid => $areacontexts) {
-                if (!empty($data->areaid) && ($areaid !== $data->areaid)) {
+                if (!empty($data->areaids) && !in_array($areaid, $data->areaids)) {
                     // Skip unused areas.
                     continue;
                 }
@@ -98,16 +109,15 @@ class engine extends \core_search\engine {
         }
 
         // Area id filter.
-        if (!empty($data->areaid)) {
-            list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->areaid);
+        if (!empty($data->areaids)) {
+            list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->areaids);
             $ands[] = 'areaid ' . $conditionsql;
             $params = array_merge($params, $conditionparams);
         }
 
         if (!empty($data->title)) {
-            list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->title);
-            $ands[] = 'title ' . $conditionsql;
-            $params = array_merge($params, $conditionparams);
+            $ands[] = $DB->sql_like('title', '?', false, false);
+            $params[] = $data->title;
         }
 
         if (!empty($data->timestart)) {
@@ -120,53 +130,77 @@ class engine extends \core_search\engine {
         }
 
         // And finally the main query after applying all AND filters.
-        switch ($DB->get_dbfamily()) {
-            case 'postgres':
-                $ands[] = "(" .
-                    "to_tsvector('simple', title) @@ plainto_tsquery(?) OR ".
-                    "to_tsvector('simple', content) @@ plainto_tsquery(?) OR ".
-                    "to_tsvector('simple', description1) @@ plainto_tsquery(?) OR ".
-                    "to_tsvector('simple', description2) @@ plainto_tsquery(?)".
-                    ")";
-                $params[] = $data->q;
-                $params[] = $data->q;
-                $params[] = $data->q;
-                $params[] = $data->q;
-                break;
-            case 'mysql':
-                $ands[] = "MATCH (title, content, description1, description2) AGAINST (?)";
-                $params[] = $data->q;
-                break;
-            case 'mssql':
-                $ands[] = "CONTAINS ((title, content, description1, description2), ?)";
-                $params[] = $data->q;
-                break;
-            default:
-                $ands[] = '(' .
-                    $DB->sql_like('title', '?', false, false) . ' OR ' .
-                    $DB->sql_like('content', '?', false, false) . ' OR ' .
-                    $DB->sql_like('description1', '?', false, false) . ' OR ' .
-                    $DB->sql_like('description2', '?', false, false) .
-                    ')';
-                $params[] = '%' . $data->q . '%';
-                $params[] = '%' . $data->q . '%';
-                $params[] = '%' . $data->q . '%';
-                $params[] = '%' . $data->q . '%';
-                break;
+        if (!empty($data->q)) {
+            switch ($DB->get_dbfamily()) {
+                case 'postgres':
+                    $ands[] = "(" .
+                        "to_tsvector('simple', title) @@ plainto_tsquery('simple', ?) OR ".
+                        "to_tsvector('simple', content) @@ plainto_tsquery('simple', ?) OR ".
+                        "to_tsvector('simple', description1) @@ plainto_tsquery('simple', ?) OR ".
+                        "to_tsvector('simple', description2) @@ plainto_tsquery('simple', ?)".
+                        ")";
+                    $params[] = $data->q;
+                    $params[] = $data->q;
+                    $params[] = $data->q;
+                    $params[] = $data->q;
+                    break;
+                case 'mysql':
+                    if ($DB->is_fulltext_search_supported()) {
+                        $ands[] = "MATCH (title, content, description1, description2) AGAINST (?)";
+                        $params[] = $data->q;
+
+                        // Sorry for the hack, but it does not seem that we will have a solution for
+                        // this soon (https://bugs.mysql.com/bug.php?id=78485).
+                        if ($data->q === '*') {
+                            return array();
+                        }
+                    } else {
+                        // Clumsy version for mysql versions with no fulltext support.
+                        list($queryand, $queryparams) = $this->get_simple_query($data->q);
+                        $ands[] = $queryand;
+                        $params = array_merge($params, $queryparams);
+                    }
+                    break;
+                case 'mssql':
+                    if ($DB->is_fulltext_search_supported()) {
+                        $ands[] = "CONTAINS ((title, content, description1, description2), ?)";
+                        // Special treatment for double quotes:
+                        // - Puntuation is ignored so we can get rid of them.
+                        // - Phrases should be enclosed in double quotation marks.
+                        $params[] = '"' . str_replace('"', '', $data->q) . '"';
+                    } else {
+                        // Clumsy version for mysql versions with no fulltext support.
+                        list($queryand, $queryparams) = $this->get_simple_query($data->q);
+                        $ands[] = $queryand;
+                        $params = array_merge($params, $queryparams);
+                    }
+                    break;
+                default:
+                    list($queryand, $queryparams) = $this->get_simple_query($data->q);
+                    $ands[] = $queryand;
+                    $params = array_merge($params, $queryparams);
+                    break;
+            }
         }
 
-        $recordset = $DB->get_recordset_sql($sql . implode(' AND ', $ands), $params, 0, \core_search\manager::MAX_RESULTS);
+        // It is limited to $limit, no need to use recordsets.
+        $documents = $DB->get_records_select('search_simpledb_index', implode(' AND ', $ands), $params, '', '*', 0, $limit);
+
+        // Hopefully database cached results as this applies the same filters than above.
+        $this->totalresults = $DB->count_records_select('search_simpledb_index', implode(' AND ', $ands), $params);
 
         $numgranted = 0;
 
-        if (!$recordset->valid()) {
-            return array();
-        }
-
         // Iterate through the results checking its availability and whether they are available for the user or not.
         $docs = array();
-        foreach ($recordset as $docdata) {
+        foreach ($documents as $docdata) {
+            if ($docdata->owneruserid != \core_search\manager::NO_OWNER_ID && $docdata->owneruserid != $USER->id) {
+                // If owneruserid is set, no other user should be able to access this record.
+                continue;
+            }
+
             if (!$searcharea = $this->get_search_area($docdata->areaid)) {
+                $this->totalresults--;
                 continue;
             }
 
@@ -178,8 +212,10 @@ class engine extends \core_search\engine {
             switch ($access) {
                 case \core_search\manager::ACCESS_DELETED:
                     $this->delete_by_id($docdata->id);
+                    $this->totalresults--;
                     break;
                 case \core_search\manager::ACCESS_DENIED:
+                    $this->totalresults--;
                     break;
                 case \core_search\manager::ACCESS_GRANTED:
                     $numgranted++;
@@ -188,12 +224,11 @@ class engine extends \core_search\engine {
             }
 
             // This should never happen.
-            if ($numgranted >= \core_search\manager::MAX_RESULTS) {
-                $docs = array_slice($docs, 0, \core_search\manager::MAX_RESULTS, true);
+            if ($numgranted >= $limit) {
+                $docs = array_slice($docs, 0, $limit, true);
                 break;
             }
         }
-        $recordset->close();
 
         return $docs;
     }
@@ -228,6 +263,7 @@ class engine extends \core_search\engine {
         } catch (\dml_exception $ex) {
             debugging('dml error while trying to insert document with id ' . $doc->docid . ': ' . $ex->getMessage(),
                 DEBUG_DEVELOPER);
+            return false;
         }
 
         return true;
@@ -281,4 +317,44 @@ class engine extends \core_search\engine {
     public function is_installed() {
         return true;
     }
+
+    /**
+     * Returns the total results.
+     *
+     * Including skipped results.
+     *
+     * @return int
+     */
+    public function get_query_total_count() {
+        if (!is_null($this->totalresults)) {
+            // This is a just in case as we count total results in execute_query.
+            return \core_search\manager::MAX_RESULTS;
+        }
+
+        return $this->totalresults;
+    }
+
+    /**
+     * Returns the default query for db engines.
+     *
+     * @param string $q The query string
+     * @return array SQL string and params list
+     */
+    protected function get_simple_query($q) {
+        global $DB;
+
+        $sql = '(' .
+            $DB->sql_like('title', '?', false, false) . ' OR ' .
+            $DB->sql_like('content', '?', false, false) . ' OR ' .
+            $DB->sql_like('description1', '?', false, false) . ' OR ' .
+            $DB->sql_like('description2', '?', false, false) .
+            ')';
+        $params = array(
+            '%' . $q . '%',
+            '%' . $q . '%',
+            '%' . $q . '%',
+            '%' . $q . '%'
+        );
+        return array($sql, $params);
+    }
 }

search/engine/simpledb/db/install.php

@@ -24,26 +24,51 @@
 
 defined('MOODLE_INTERNAL') || die;
 
+/**
+ * Post installation code.
+ *
+ * @package    search_simpledb
+ * @copyright  2016 Dan Poltawski <dan@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
 function xmldb_search_simpledb_install() {
     global $DB;
 
     switch ($DB->get_dbfamily()) {
         case 'postgres':
-            // TODO: There are a few other ways of doing this which avoid the need for individual indicies..
-            $DB->execute("CREATE INDEX psql_search_title ON {search_simpledb_index} USING gin(to_tsvector('simple', title))");
-            $DB->execute("CREATE INDEX psql_search_content ON {search_simpledb_index} USING gin(to_tsvector('simple', content))");
-            $DB->execute("CREATE INDEX psql_search_description1 ON {search_simpledb_index} USING gin(to_tsvector('simple', description1))");
-            $DB->execute("CREATE INDEX psql_search_description2 ON {search_simpledb_index} USING gin(to_tsvector('simple', description2))");
+            // There are a few other ways of doing this which avoid the need for individual indexes.
+            $DB->execute("CREATE INDEX {search_simpledb_title} ON {search_simpledb_index} " .
+                "USING gin(to_tsvector('simple', title))");
+            $DB->execute("CREATE INDEX {search_simpledb_content} ON {search_simpledb_index} " .
+                "USING gin(to_tsvector('simple', content))");
+            $DB->execute("CREATE INDEX {search_simpledb_description1} ON {search_simpledb_index} " .
+                "USING gin(to_tsvector('simple', description1))");
+            $DB->execute("CREATE INDEX {search_simpledb_description2} ON {search_simpledb_index} " .
+                "USING gin(to_tsvector('simple', description2))");
             break;
         case 'mysql':
-            $DB->execute("CREATE FULLTEXT INDEX mysql_search_index
-                          ON {search_simpledb_index} (title, content, description1, description2)");
+            if ($DB->is_fulltext_search_supported()) {
+                $DB->execute("CREATE FULLTEXT INDEX {search_simpledb_index_index}
+                              ON {search_simpledb_index} (title, content, description1, description2)");
+            }
             break;
         case 'mssql':
-            //TODO: workout if fulltext search is installed... select SERVERPROPERTY('IsFullTextInstalled')
-            $DB->execute("CREATE FULLTEXT CATALOG {search_simpledb_catalog}");
-            $DB->execute("CREATE FULLTEXT INDEX ON {search_simpledb_index} (title, content, description1, description2)
-                          KEY INDEX {searsimpinde_id_pk} ON {search_simpledb_catalog}");
+            if ($DB->is_fulltext_search_supported()) {
+
+                $catalogname = $DB->get_prefix() . 'search_simpledb_catalog';
+                if (!$DB->record_exists_sql('SELECT * FROM sys.fulltext_catalogs WHERE name = ?', array($catalogname))) {
+                    $DB->execute("CREATE FULLTEXT CATALOG {search_simpledb_catalog} WITH ACCENT_SENSITIVITY=OFF");
+                }
+
+                if (defined('PHPUNIT_UTIL') and PHPUNIT_UTIL) {
+                    // We want manual tracking for phpunit because the fulltext index does get auto populated fast enough.
+                    $changetracking = 'MANUAL';
+                } else {
+                    $changetracking = 'AUTO';
+                }
+                $DB->execute("CREATE FULLTEXT INDEX ON {search_simpledb_index} (title, content, description1, description2)
+                              KEY INDEX {searsimpinde_id_pk} ON {search_simpledb_catalog} WITH CHANGE_TRACKING $changetracking");
+            }
             break;
     }
 }

search/engine/simpledb/db/uninstall.php

@@ -24,12 +24,32 @@
 
 defined('MOODLE_INTERNAL') || die;
 
+/**
+ * Plugin uninstall code.
+ *
+ * @package    search_simpledb
+ * @copyright  2016 Dan Poltawski <dan@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
 function xmldb_search_simpledb_uninstall() {
     global $DB;
 
     switch ($DB->get_dbfamily()) {
+        case 'postgres':
+            $DB->execute("DROP INDEX {search_simpledb_title}");
+            $DB->execute("DROP INDEX {search_simpledb_content}");
+            $DB->execute("DROP INDEX {search_simpledb_description1}");
+            $DB->execute("DROP INDEX {search_simpledb_description2}");
+            break;
+        case 'mysql':
+            if ($DB->is_fulltext_search_supported()) {
+                $DB->execute("ALTER TABLE {search_simpledb_index} DROP INDEX {search_simpledb_index_index}");
+            }
+            break;
         case 'mssql':
-            $DB->execute("DROP FULLTEXT CATALOG {search_simpledb_catalog}");
+            if ($DB->is_fulltext_search_supported()) {
+                $DB->execute("DROP FULLTEXT CATALOG {search_simpledb_catalog}");
+            }
             break;
     }
 }

search/engine/simpledb/tests/engine_test.php

@@ -18,7 +18,7 @@
  * Simple db search engine tests.
  *
  * @package     search_simpledb
- * @category    phpunit
+ * @category    test
  * @copyright   2016 David Monllao {@link http://www.davidmonllao.com}
  * @license     http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
@@ -33,7 +33,7 @@ require_once($CFG->dirroot . '/search/tests/fixtures/mock_search_area.php');
  * Simple search engine base unit tests.
  *
  * @package     search_simpledb
- * @category    phpunit
+ * @category    test
  * @copyright   2016 David Monllao {@link http://www.davidmonllao.com}
  * @license     http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
@@ -44,22 +44,70 @@ class search_simpledb_engine_testcase extends advanced_testcase {
      */
     protected $search = null;
 
+    /**
+     * @var \
+     */
+    protected $engine = null;
+
+    /**
+     * @var core_search_generator
+     */
+    protected $generator = null;
+
+    /**
+     * Initial stuff.
+     *
+     * @return void
+     */
     public function setUp() {
         $this->resetAfterTest();
+
+        if ($this->requires_manual_index_update()) {
+            // We need to update fulltext index manually, which requires an alter table statement.
+            $this->preventResetByRollback();
+        }
+
         set_config('enableglobalsearch', true);
 
         // Inject search_simpledb engine into the testable core search as we need to add the mock
         // search component to it.
-        $searchengine = new \search_simpledb\engine();
-        $this->search = testable_core_search::instance($searchengine);
-        $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities');
-        $this->search->add_search_area($areaid, new core_mocksearch\search\role_capabilities());
+
+        $this->engine = new \search_simpledb\engine();
+        $this->search = testable_core_search::instance($this->engine);
+        $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
+        $this->search->add_search_area($areaid, new core_mocksearch\search\mock_search_area());
+
+        $this->generator = self::getDataGenerator()->get_plugin_generator('core_search');
+        $this->generator->setup();
+
+        $this->setAdminUser();
     }
 
+    /**
+     * tearDown
+     *
+     * @return void
+     */
+    public function tearDown() {
+        // For unit tests before PHP 7, teardown is called even on skip. So only do our teardown if we did setup.
+        if ($this->generator) {
+            // Moodle DML freaks out if we don't teardown the temp table after each run.
+            $this->generator->teardown();
+            $this->generator = null;
+        }
+    }
+
+    /**
+     * Test indexing process.
+     *
+     * @return void
+     */
     public function test_index() {
         global $DB;
 
-        $noneditingteacherid = $DB->get_field('role', 'id', array('shortname' => 'teacher'));
+        $record = new \stdClass();
+        $record->timemodified = time() - 1;
+        $this->generator->create_record($record);
 
         // Data gets into the search engine.
         $this->assertTrue($this->search->index());
@@ -68,8 +116,7 @@ class search_simpledb_engine_testcase extends advanced_testcase {
         sleep(1);
         $this->assertFalse($this->search->index());
 
-        assign_capability('moodle/course:renameroles', CAP_ALLOW, $noneditingteacherid, context_system::instance()->id);
-        accesslib_clear_all_caches_for_unit_testing();
+        $this->generator->create_record();
 
         // Indexing again once there is new data.
         $this->assertTrue($this->search->index());
@@ -83,19 +130,13 @@ class search_simpledb_engine_testcase extends advanced_testcase {
     public function test_search() {
         global $USER, $DB;
 
-        $this->setAdminUser();
-
-        $noneditingteacherid = $DB->get_field('role', 'id', array('shortname' => 'teacher'));
-
-        $this->search->index();
-
-        // Check that docid - id is respected.
-        $rolecaps = $DB->get_records('role_capabilities', array('capability' => 'moodle/course:renameroles'));
-        $rolecap = reset($rolecaps);
-        $rolecap->timemodified = time();
-        $DB->update_record('role_capabilities', $rolecap);
+        $this->generator->create_record();
+        $record = new \stdClass();
+        $record->title = "Special title";
+        $this->generator->create_record($record);
 
         $this->search->index();
+        $this->update_index();
 
         $querydata = new stdClass();
         $querydata->q = 'message';
@@ -104,32 +145,32 @@ class search_simpledb_engine_testcase extends advanced_testcase {
 
         // Based on core_mocksearch\search\indexer.
         $this->assertEquals($USER->id, $results[0]->get('userid'));
-        $this->assertEquals(\context_system::instance()->id, $results[0]->get('contextid'));
+        $this->assertEquals(\context_course::instance(SITEID)->id, $results[0]->get('contextid'));
 
         // Do a test to make sure we aren't searching non-query fields, like areaid.
-        $querydata->q = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities');
+        $querydata->q = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
         $this->assertCount(0, $this->search->search($querydata));
         $querydata->q = 'message';
 
         sleep(1);
         $beforeadding = time();
         sleep(1);
-        assign_capability('moodle/course:renameroles', CAP_ALLOW, $noneditingteacherid, context_system::instance()->id);
-        accesslib_clear_all_caches_for_unit_testing();
+        $this->generator->create_record();
         $this->search->index();
+        $this->update_index();
 
         // Timestart.
         $querydata->timestart = $beforeadding;
-        $this->assertCount(2, $this->search->search($querydata));
+        $this->assertCount(1, $this->search->search($querydata));
 
         // Timeend.
         unset($querydata->timestart);
         $querydata->timeend = $beforeadding;
-        $this->assertCount(1, $this->search->search($querydata));
+        $this->assertCount(2, $this->search->search($querydata));
 
         // Title.
         unset($querydata->timeend);
-        $querydata->title = 'moodle/course:renameroles roleid 1';
+        $querydata->title = 'Special title';
         $this->assertCount(1, $this->search->search($querydata));
 
         // Course IDs.
@@ -140,42 +181,69 @@ class search_simpledb_engine_testcase extends advanced_testcase {
         $querydata->courseids = array(SITEID);
         $this->assertCount(3, $this->search->search($querydata));
 
+        // Now try some area-id combinations.
+        unset($querydata->courseids);
+        $forumpostareaid = \core_search\manager::generate_areaid('mod_forum', 'post');
+        $mockareaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
+
+        $querydata->areaids = array($forumpostareaid);
+        $this->assertCount(0, $this->search->search($querydata));
+
+        $querydata->areaids = array($forumpostareaid, $mockareaid);
+        $this->assertCount(3, $this->search->search($querydata));
+
+        $querydata->areaids = array($mockareaid);
+        $this->assertCount(3, $this->search->search($querydata));
+
+        $querydata->areaids = array();
+        $this->assertCount(3, $this->search->search($querydata));
+
         // Check that index contents get updated.
-        $DB->delete_records('role_capabilities', array('capability' => 'moodle/course:renameroles'));
+        $this->generator->delete_all();
         $this->search->index(true);
+        $this->update_index();
         unset($querydata->title);
-        $querydata->q = '*renameroles*';
+        $querydata->q = '';
         $this->assertCount(0, $this->search->search($querydata));
     }
 
+    /**
+     * Test delete function
+     *
+     * @return void
+     */
     public function test_delete() {
+
+        $this->generator->create_record();
+        $this->generator->create_record();
         $this->search->index();
+        $this->update_index();
 
         $querydata = new stdClass();
         $querydata->q = 'message';
 
         $this->assertCount(2, $this->search->search($querydata));
 
-        $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities');
+        $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
         $this->search->delete_index($areaid);
+        $this->update_index();
         $this->assertCount(0, $this->search->search($querydata));
     }
 
+    /**
+     * Test user is allowed.
+     *
+     * @return void
+     */
     public function test_alloweduserid() {
-        $engine = $this->search->get_engine();
-        $area = new core_mocksearch\search\role_capabilities();
 
-        // Get the first record for the recordset.
-        $recordset = $area->get_recordset_by_timestamp();
-        foreach ($recordset as $r) {
-            $record = $r;
-            break;
-        }
-        $recordset->close();
+        $area = new core_mocksearch\search\mock_search_area();
+
+        $record = $this->generator->create_record();
 
         // Get the doc and insert the default doc.
         $doc = $area->get_document($record);
-        $engine->add_document($doc);
+        $this->engine->add_document($doc);
 
         $users = array();
         $users[] = $this->getDataGenerator()->create_user();
@@ -190,10 +258,11 @@ class search_simpledb_engine_testcase extends advanced_testcase {
             $doc = $area->get_document($record);
             $doc->set('id', $originalid.'-'.$user->id);
             $doc->set('owneruserid', $user->id);
-            $engine->add_document($doc);
+            $this->engine->add_document($doc);
         }
+        $this->update_index();
 
-        $engine->area_index_complete($area->get_area_id());
+        $this->engine->area_index_complete($area->get_area_id());
 
         $querydata = new stdClass();
         $querydata->q = 'message';
@@ -239,21 +308,24 @@ class search_simpledb_engine_testcase extends advanced_testcase {
     }
 
     public function test_delete_by_id() {
-        // First get files in the index.
+
+        $this->generator->create_record();
+        $this->generator->create_record();
         $this->search->index();
-        $engine = $this->search->get_engine();
+        $this->update_index();
 
         $querydata = new stdClass();
 
         // Then search to make sure they are there.
-        $querydata->q = 'moodle/course:renameroles';
+        $querydata->q = 'message';
         $results = $this->search->search($querydata);
         $this->assertCount(2, $results);
 
         $first = reset($results);
         $deleteid = $first->get('id');
 
-        $engine->delete_by_id($deleteid);
+        $this->engine->delete_by_id($deleteid);
+        $this->update_index();
 
         // Check that we don't get a result for it anymore.
         $results = $this->search->search($querydata);
@@ -261,4 +333,47 @@ class search_simpledb_engine_testcase extends advanced_testcase {
         $result = reset($results);
         $this->assertNotEquals($deleteid, $result->get('id'));
     }
+
+    /**
+     * Updates mssql fulltext index if necessary.
+     *
+     * @return bool
+     */
+    private function update_index() {
+        global $DB;
+
+        if (!$this->requires_manual_index_update()) {
+            return;
+        }
+
+        $DB->execute("ALTER FULLTEXT INDEX ON t_search_simpledb_index START UPDATE POPULATION");
+
+        $catalogname = $DB->get_prefix() . 'search_simpledb_catalog';
+        $retries = 0;
+        do {
+            // 0.2 seconds.
+            usleep(200000);
+
+            $record = $DB->get_record_sql("SELECT FULLTEXTCATALOGPROPERTY(cat.name, 'PopulateStatus') AS [PopulateStatus]
+                                             FROM sys.fulltext_catalogs AS cat
+                                            WHERE cat.name = ?", array($catalogname));
+            $retries++;
+
+        } while ($retries < 100 && $record->populatestatus != '0');
+
+        if ($retries === 100) {
+            // No update after 20 seconds...
+            $this->fail('Sorry, your SQL server fulltext search index is too slow.');
+        }
+    }
+
+    /**
+     * Mssql with fulltext support requires manual updates.
+     *
+     * @return bool
+     */
+    private function requires_manual_index_update() {
+        global $DB;
+        return ($DB->get_dbfamily() === 'mssql' && $DB->is_fulltext_search_supported());
+    }
 }

search/engine/simpledb/version.php

@@ -24,6 +24,6 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version = 2016030100;
-$plugin->requires = 2015111000;
+$plugin->version = 2017072700;
+$plugin->requires = 2017072700;
 $plugin->component = 'search_simpledb';

version.php

@@ -29,7 +29,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$version  = 2018031600.00;              // YYYYMMDD      = weekly release date of this DEV branch.
+$version  = 2018031600.01;              // YYYYMMDD      = weekly release date of this DEV branch.
                                         //         RR    = release increments - 00 in DEV branches.
                                         //           .XX = incremental changes.