diff --git a/admin/searchareas.php b/admin/searchareas.php index 9ce7af6d032..9cac43969da 100644 --- a/admin/searchareas.php +++ b/admin/searchareas.php @@ -132,7 +132,17 @@ foreach ($searchareas as $area) { $laststatus = ''; } $columns[] = $laststatus; - $columns[] = html_writer::link(admin_searcharea_action_url('delete', $areaid), 'Delete index'); + $accesshide = html_writer::span($area->get_visible_name(), 'accesshide'); + $actions = []; + $actions[] = $OUTPUT->pix_icon('t/delete', '') . + html_writer::link(admin_searcharea_action_url('delete', $areaid), + get_string('deleteindex', 'search', $accesshide)); + if ($area->supports_get_document_recordset()) { + $actions[] = $OUTPUT->pix_icon('i/reload', '') . html_writer::link( + new moodle_url('searchreindex.php', ['areaid' => $areaid]), + get_string('gradualreindex', 'search', $accesshide)); + } + $columns[] = html_writer::alist($actions, ['class' => 'unstyled list-unstyled']); } else { $blankrow = new html_table_cell(get_string('searchnotavailable', 'admin')); @@ -165,6 +175,13 @@ echo $OUTPUT->single_button(admin_searcharea_action_url('deleteall'), get_string echo $OUTPUT->box_end(); echo html_writer::table($table); + +if (empty($searchmanagererror)) { + // Show information about queued index requests for specific contexts. + $searchrenderer = $PAGE->get_renderer('core_search'); + echo $searchrenderer->render_index_requests_info($searchmanager->get_index_requests_info()); +} + echo $OUTPUT->footer(); /** diff --git a/admin/searchreindex.php b/admin/searchreindex.php new file mode 100644 index 00000000000..9fa7a3ccae6 --- /dev/null +++ b/admin/searchreindex.php @@ -0,0 +1,87 @@ +. + +/** + * Adds a search area to the queue for indexing. + * + * @package core_search + * @copyright 2017 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +define('NO_OUTPUT_BUFFERING', true); + +require(__DIR__ . '/../config.php'); + +// Check access. +require_once($CFG->libdir . '/adminlib.php'); + +admin_externalpage_setup('searchareas', '', null, (new moodle_url('/admin/searchreindex.php'))->out(false)); + +// Get area parameter and check it exists. +$areaid = required_param('areaid', PARAM_ALPHAEXT); +$area = \core_search\manager::get_search_area($areaid); +if ($area === false) { + throw new moodle_exception('invalidrequest'); +} +$areaname = $area->get_visible_name(); + +// Start page output. +$heading = get_string('gradualreindex', 'search', ''); +$PAGE->set_title($PAGE->title . ': ' . $heading); +$PAGE->navbar->add($heading); +echo $OUTPUT->header(); +echo $OUTPUT->heading($heading); + +// If sesskey is supplied, actually carry out the action. +if (optional_param('sesskey', '', PARAM_ALPHANUM)) { + require_sesskey(); + + // Get all contexts for search area. This query can take time in large cases. + \core_php_time_limit::raise(0); + $contextiterator = $area->get_contexts_to_reindex(); + + $progress = new \core\progress\display_if_slow(''); + $progress->start_progress($areaname); + + // Request reindexing for each context (with low priority). + $count = 0; + foreach ($contextiterator as $context) { + \core_php_time_limit::raise(30); + \core_search\manager::request_index($context, $area->get_area_id(), + \core_search\manager::INDEX_PRIORITY_REINDEXING); + $progress->progress(); + $count++; + } + + // Unset the iterator which should close the recordset (if there is one). + unset($contextiterator); + + $progress->end_progress(); + + $a = (object)['name' => html_writer::tag('strong', $areaname), 'count' => $count]; + echo $OUTPUT->box(get_string('gradualreindex_queued', 'search', $a)); + + echo $OUTPUT->continue_button(new moodle_url('/admin/searchareas.php')); +} else { + // Display confirmation prompt. + echo $OUTPUT->confirm(get_string('gradualreindex_confirm', 'search', html_writer::tag('strong', $areaname)), + new single_button(new moodle_url('/admin/searchreindex.php', ['areaid' => $areaid, + 'sesskey' => sesskey()]), get_string('continue'), 'post', true), + new single_button(new moodle_url('/admin/searchareas.php'), get_string('cancel'), 'get')); +} + +echo $OUTPUT->footer(); diff --git a/lang/en/search.php b/lang/en/search.php index 552d9b6064f..cba4592d1ce 100644 --- a/lang/en/search.php +++ b/lang/en/search.php @@ -36,6 +36,7 @@ $string['createdon'] = 'Created on'; $string['database'] = 'Database'; $string['databasestate'] = 'Indexing database state'; $string['datadirectory'] = 'Data directory'; +$string['deleteindex'] = 'Delete index {$a}'; $string['deletionsinindex'] = 'Deletions in index'; $string['docmodifiedon'] = 'Last modified on {$a}'; $string['doctype'] = 'Doctype'; @@ -59,6 +60,9 @@ $string['filterheader'] = 'Filter'; $string['fromtime'] = 'Modified after'; $string['globalsearch'] = 'Global search'; $string['globalsearchdisabled'] = 'Global searching is not enabled.'; +$string['gradualreindex'] = 'Gradual reindex {$a}'; +$string['gradualreindex_confirm'] = 'Are you sure you want to reindex {$a}? This may take some time, although existing data will remain available during the reindex.'; +$string['gradualreindex_queued'] = 'Reindexing has been requested for {$a->name} ({$a->count} contexts). This indexing will be carried out by the ‘Global search indexing’ scheduled task.'; $string['checkdb'] = 'Check database'; $string['checkdbadvice'] = 'Check your database for any problems.'; $string['checkdir'] = 'Check dir'; @@ -76,7 +80,12 @@ $string['notitle'] = 'No title'; $string['normalsearch'] = 'Normal search'; $string['openedon'] = 'opened on'; $string['optimize'] = 'Optimize'; +$string['priority'] = 'Priority'; +$string['priority_reindexing'] = 'Reindexing'; +$string['priority_normal'] = 'Normal'; +$string['progress'] = 'Progress'; $string['queryerror'] = 'The query you provided could not be parsed by the search engine: {$a}'; +$string['queueheading'] = 'Additional indexing queue ({$a} items)'; $string['resultsreturnedfor'] = 'results returned for'; $string['runindexer'] = 'Run indexer (real)'; $string['runindexertest'] = 'Run indexer test'; diff --git a/lib/db/install.xml b/lib/db/install.xml index 27c832ae105..593ff37a451 100644 --- a/lib/db/install.xml +++ b/lib/db/install.xml @@ -1,5 +1,5 @@ - @@ -3770,11 +3770,15 @@ + + + + \ No newline at end of file diff --git a/lib/db/upgrade.php b/lib/db/upgrade.php index e2e68c1526b..504fbfdafab 100644 --- a/lib/db/upgrade.php +++ b/lib/db/upgrade.php @@ -1902,5 +1902,38 @@ function xmldb_main_upgrade($oldversion) { upgrade_main_savepoint(true, 2017121900.00); } + if ($oldversion < 2017122200.01) { + + // Define field indexpriority to be added to search_index_requests. Allow null initially. + $table = new xmldb_table('search_index_requests'); + $field = new xmldb_field('indexpriority', XMLDB_TYPE_INTEGER, '10', + null, null, null, null, 'partialtime'); + + // Conditionally add field. + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + + // Set existing values to 'normal' value (100). + $DB->set_field('search_index_requests', 'indexpriority', 100); + + // Now make the field 'NOT NULL'. + $field = new xmldb_field('indexpriority', XMLDB_TYPE_INTEGER, '10', + null, XMLDB_NOTNULL, null, null, 'partialtime'); + $dbman->change_field_notnull($table, $field); + } + + // Define index indexprioritytimerequested (not unique) to be added to search_index_requests. + $index = new xmldb_index('indexprioritytimerequested', XMLDB_INDEX_NOTUNIQUE, + array('indexpriority', 'timerequested')); + + // Conditionally launch add index indexprioritytimerequested. + if (!$dbman->index_exists($table, $index)) { + $dbman->add_index($table, $index); + } + + // Main savepoint reached. + upgrade_main_savepoint(true, 2017122200.01); + } + return true; } diff --git a/mod/forum/classes/search/post.php b/mod/forum/classes/search/post.php index 698d465019e..9f5d68e499a 100644 --- a/mod/forum/classes/search/post.php +++ b/mod/forum/classes/search/post.php @@ -292,4 +292,17 @@ class post extends \core_search\base_mod { } return $this->discussionsdata[$discussionid]; } + + /** + * Changes the context ordering so that the forums with most recent discussions are indexed + * first. + * + * @return string[] SQL join and ORDER BY + */ + protected function get_contexts_to_reindex_extra_sql() { + return [ + 'JOIN {forum_discussions} fd ON fd.course = cm.course AND fd.forum = cm.instance', + 'MAX(fd.timemodified) DESC' + ]; + } } diff --git a/mod/forum/tests/search_test.php b/mod/forum/tests/search_test.php index 351d0a3bfcb..d2cd42368c1 100644 --- a/mod/forum/tests/search_test.php +++ b/mod/forum/tests/search_test.php @@ -390,4 +390,62 @@ class mod_forum_search_testcase extends advanced_testcase { $recordset->close(); $this->assertEquals(3, $nrecords); } + + /** + * Tests that reindexing works in order starting from the forum with most recent discussion. + */ + public function test_posts_get_contexts_to_reindex() { + global $DB; + + $generator = $this->getDataGenerator(); + $adminuser = get_admin(); + + $course1 = $generator->create_course(); + $course2 = $generator->create_course(); + + $time = time() - 1000; + + // Create 3 forums (two in course 1, one in course 2 - doesn't make a difference). + $forum1 = $generator->create_module('forum', ['course' => $course1->id]); + $forum2 = $generator->create_module('forum', ['course' => $course1->id]); + $forum3 = $generator->create_module('forum', ['course' => $course2->id]); + $forum4 = $generator->create_module('forum', ['course' => $course2->id]); + + // Hack added time for the course_modules entries. These should not be used (they would + // be used by the base class implementation). We are setting this so that the order would + // be 4, 3, 2, 1 if this ordering were used (newest first). + $DB->set_field('course_modules', 'added', $time + 100, ['id' => $forum1->cmid]); + $DB->set_field('course_modules', 'added', $time + 110, ['id' => $forum2->cmid]); + $DB->set_field('course_modules', 'added', $time + 120, ['id' => $forum3->cmid]); + $DB->set_field('course_modules', 'added', $time + 130, ['id' => $forum4->cmid]); + + $forumgenerator = $generator->get_plugin_generator('mod_forum'); + + // Create one discussion in forums 1 and 3, three in forum 2, and none in forum 4. + $forumgenerator->create_discussion(['course' => $course1->id, + 'forum' => $forum1->id, 'userid' => $adminuser->id, 'timemodified' => $time + 20]); + + $forumgenerator->create_discussion(['course' => $course1->id, + 'forum' => $forum2->id, 'userid' => $adminuser->id, 'timemodified' => $time + 10]); + $forumgenerator->create_discussion(['course' => $course1->id, + 'forum' => $forum2->id, 'userid' => $adminuser->id, 'timemodified' => $time + 30]); + $forumgenerator->create_discussion(['course' => $course1->id, + 'forum' => $forum2->id, 'userid' => $adminuser->id, 'timemodified' => $time + 11]); + + $forumgenerator->create_discussion(['course' => $course2->id, + 'forum' => $forum3->id, 'userid' => $adminuser->id, 'timemodified' => $time + 25]); + + // Get the contexts in reindex order. + $area = \core_search\manager::get_search_area($this->forumpostareaid); + $contexts = iterator_to_array($area->get_contexts_to_reindex(), false); + + // We expect them in order of newest discussion. Forum 4 is not included at all (which is + // correct because it has no content). + $expected = [ + \context_module::instance($forum2->cmid), + \context_module::instance($forum3->cmid), + \context_module::instance($forum1->cmid) + ]; + $this->assertEquals($expected, $contexts); + } } diff --git a/search/classes/base.php b/search/classes/base.php index 2c10388a32d..97fb8ab8e12 100644 --- a/search/classes/base.php +++ b/search/classes/base.php @@ -285,6 +285,14 @@ abstract class base { * The default implementation returns false, indicating that this facility is not supported and * the older get_recordset_by_timestamp function should be used. * + * This function must accept all possible values for the $context parameter. For example, if + * you are implementing this function for the forum module, it should still operate correctly + * if called with the context for a glossary module, or for the HTML block. (In these cases + * where it will not return any data, it may return null.) + * + * The $context parameter can also be null or the system context; both of these indicate that + * all data, without context restriction, should be returned. + * * @param int $modifiedfrom Return only records modified after this date * @param \context|null $context Context (null means no context restriction) * @return \moodle_recordset|null|false Recordset / null if no results / false if not supported @@ -294,6 +302,19 @@ abstract class base { return false; } + /** + * Checks if get_document_recordset is supported for this search area. + * + * For many uses you can simply call get_document_recordset and see if it returns false, but + * this function is useful when you don't want to actually call the function right away. + */ + public function supports_get_document_recordset() { + // Easiest way to check this is simply to see if the class has overridden the default + // function. + $method = new \ReflectionMethod($this, 'get_document_recordset'); + return $method->getDeclaringClass()->getName() !== self::class; + } + /** * Returns the document related with the provided record. * @@ -474,4 +495,18 @@ abstract class base { return [$sql, $params]; } + + /** + * Gets a list of all contexts to reindex when reindexing this search area. The list should be + * returned in an order that is likely to be suitable when reindexing, for example with newer + * contexts first. + * + * The default implementation simply returns the system context, which will result in + * reindexing everything in normal date order (oldest first). + * + * @return \Iterator Iterator of contexts to reindex + */ + public function get_contexts_to_reindex() { + return new \ArrayIterator([\context_system::instance()]); + } } diff --git a/search/classes/base_block.php b/search/classes/base_block.php index 52d903462d0..42f63d4bb8b 100644 --- a/search/classes/base_block.php +++ b/search/classes/base_block.php @@ -343,4 +343,59 @@ abstract class base_block extends base { return [$sql, $params]; } + + /** + * This can be used in subclasses to change ordering within the get_contexts_to_reindex + * function. + * + * It returns 2 values: + * - Extra SQL joins (tables block_instances 'bi' and context 'x' already exist). + * - An ORDER BY value which must use aggregate functions, by default 'MAX(bi.timemodified) DESC'. + * + * Note the query already includes a GROUP BY on the context fields, so if your joins result + * in multiple rows, you can use aggregate functions in the ORDER BY. See forum for an example. + * + * @return string[] Array with 2 elements; extra joins for the query, and ORDER BY value + */ + protected function get_contexts_to_reindex_extra_sql() { + return ['', 'MAX(bi.timemodified) DESC']; + } + + /** + * Gets a list of all contexts to reindex when reindexing this search area. + * + * For blocks, the default is to return all contexts for blocks of that type, that are on a + * course page, in order of time added (most recent first). + * + * @return \Iterator Iterator of contexts to reindex + * @throws \moodle_exception If any DB error + */ + public function get_contexts_to_reindex() { + global $DB; + + list ($extrajoins, $dborder) = $this->get_contexts_to_reindex_extra_sql(); + $contexts = []; + $selectcolumns = \context_helper::get_preload_record_columns_sql('x'); + $groupbycolumns = ''; + foreach (\context_helper::get_preload_record_columns('x') as $column => $thing) { + if ($groupbycolumns !== '') { + $groupbycolumns .= ','; + } + $groupbycolumns .= $column; + } + $rs = $DB->get_recordset_sql(" + SELECT $selectcolumns + FROM {block_instances} bi + JOIN {context} x ON x.instanceid = bi.id AND x.contextlevel = ? + JOIN {context} parent ON parent.id = bi.parentcontextid + $extrajoins + WHERE bi.blockname = ? AND parent.contextlevel = ? + GROUP BY $groupbycolumns + ORDER BY $dborder", [CONTEXT_BLOCK, $this->get_block_name(), CONTEXT_COURSE]); + return new \core\dml\recordset_walk($rs, function($rec) { + $id = $rec->ctxid; + \context_helper::preload_from_record($rec); + return \context::instance_by_id($id); + }); + } } diff --git a/search/classes/base_mod.php b/search/classes/base_mod.php index ef387508331..1ad7a770e5a 100644 --- a/search/classes/base_mod.php +++ b/search/classes/base_mod.php @@ -192,4 +192,57 @@ abstract class base_mod extends base { return [$sql, $params]; } + /** + * This can be used in subclasses to change ordering within the get_contexts_to_reindex + * function. + * + * It returns 2 values: + * - Extra SQL joins (tables course_modules 'cm' and context 'x' already exist). + * - An ORDER BY value which must use aggregate functions, by default 'MAX(cm.added) DESC'. + * + * Note the query already includes a GROUP BY on the context fields, so if your joins result + * in multiple rows, you can use aggregate functions in the ORDER BY. See forum for an example. + * + * @return string[] Array with 2 elements; extra joins for the query, and ORDER BY value + */ + protected function get_contexts_to_reindex_extra_sql() { + return ['', 'MAX(cm.added) DESC']; + } + + /** + * Gets a list of all contexts to reindex when reindexing this search area. + * + * For modules, the default is to return all contexts for modules of that type, in order of + * time added (most recent first). + * + * @return \Iterator Iterator of contexts to reindex + * @throws \moodle_exception If any DB error + */ + public function get_contexts_to_reindex() { + global $DB; + + list ($extrajoins, $dborder) = $this->get_contexts_to_reindex_extra_sql(); + $contexts = []; + $selectcolumns = \context_helper::get_preload_record_columns_sql('x'); + $groupbycolumns = ''; + foreach (\context_helper::get_preload_record_columns('x') as $column => $thing) { + if ($groupbycolumns !== '') { + $groupbycolumns .= ','; + } + $groupbycolumns .= $column; + } + $rs = $DB->get_recordset_sql(" + SELECT $selectcolumns + FROM {course_modules} cm + JOIN {context} x ON x.instanceid = cm.id AND x.contextlevel = ? + $extrajoins + WHERE cm.module = (SELECT id FROM {modules} WHERE name = ?) + GROUP BY $groupbycolumns + ORDER BY $dborder", [CONTEXT_MODULE, $this->get_module_name()]); + return new \core\dml\recordset_walk($rs, function($rec) { + $id = $rec->ctxid; + \context_helper::preload_from_record($rec); + return \context::instance_by_id($id); + }); + } } diff --git a/search/classes/manager.php b/search/classes/manager.php index dd77270cf12..e04cf9854d7 100644 --- a/search/classes/manager.php +++ b/search/classes/manager.php @@ -87,6 +87,16 @@ class manager { */ const DISPLAY_INDEXING_PROGRESS_EVERY = 30.0; + /** + * @var int Context indexing: normal priority. + */ + const INDEX_PRIORITY_NORMAL = 100; + + /** + * @var int Context indexing: low priority for reindexing. + */ + const INDEX_PRIORITY_REINDEXING = 50; + /** * @var \core_search\base[] Enabled search areas. */ @@ -1143,36 +1153,55 @@ class manager { * added to a queue which is processed by the task. * * This is used after a restore to ensure that restored items are indexed, even though their - * modified time will be older than the latest indexed. + * modified time will be older than the latest indexed. It is also used by the 'Gradual reindex' + * admin feature from the search areas screen. * * @param \context $context Context to index within * @param string $areaid Area to index, '' = all areas + * @param int $priority Priority (INDEX_PRIORITY_xx constant) */ - public static function request_index(\context $context, $areaid = '') { + public static function request_index(\context $context, $areaid = '', + $priority = self::INDEX_PRIORITY_NORMAL) { global $DB; // Check through existing requests for this context or any parent context. list ($contextsql, $contextparams) = $DB->get_in_or_equal( $context->get_parent_context_ids(true)); $existing = $DB->get_records_select('search_index_requests', - 'contextid ' . $contextsql, $contextparams, '', 'id, searcharea, partialarea'); + 'contextid ' . $contextsql, $contextparams, '', + 'id, searcharea, partialarea, indexpriority'); foreach ($existing as $rec) { // If we haven't started processing the existing request yet, and it covers the same // area (or all areas) then that will be sufficient so don't add anything else. if ($rec->partialarea === '' && ($rec->searcharea === $areaid || $rec->searcharea === '')) { - return; + // If the existing request has the same (or higher) priority, no need to add anything. + if ($rec->indexpriority >= $priority) { + return; + } + // The existing request has lower priority. If it is exactly the same, then just + // adjust the priority of the existing request. + if ($rec->searcharea === $areaid) { + $DB->set_field('search_index_requests', 'indexpriority', $priority, + ['id' => $rec->id]); + return; + } + // The existing request would cover this area but is a lower priority. We need to + // add the new request even though that means we will index part of it twice. } } // No suitable existing request, so add a new one. $newrecord = [ 'contextid' => $context->id, 'searcharea' => $areaid, - 'timerequested' => time(), 'partialarea' => '', 'partialtime' => 0 ]; + 'timerequested' => (int)self::get_current_time(), + 'partialarea' => '', 'partialtime' => 0, + 'indexpriority' => $priority ]; $DB->insert_record('search_index_requests', $newrecord); } /** - * Processes outstanding index requests. This will take the first item from the queue and - * process it, continuing until an optional time limit is reached. + * Processes outstanding index requests. This will take the first item from the queue (taking + * account the indexing priority) and process it, continuing until an optional time limit is + * reached. * * If there are no index requests, the function will do nothing. * @@ -1186,7 +1215,6 @@ class manager { $progress = new \null_progress_trace(); } - $complete = false; $before = self::get_current_time(); if ($timelimit) { $stopat = $before + $timelimit; @@ -1194,11 +1222,10 @@ class manager { while (true) { // Retrieve first request, using fully defined ordering. $requests = $DB->get_records('search_index_requests', null, - 'timerequested, contextid, searcharea', + 'indexpriority DESC, timerequested, contextid, searcharea', 'id, contextid, searcharea, partialarea, partialtime', 0, 1); if (!$requests) { // If there are no more requests, stop. - $complete = true; break; } $request = reset($requests); @@ -1208,6 +1235,12 @@ class manager { $beforeindex = self::get_current_time(); if ($timelimit) { $remainingtime = $stopat - $beforeindex; + + // If the time limit expired already, stop now. (Otherwise we might accidentally + // index with no time limit or a negative time limit.) + if ($remainingtime <= 0) { + break; + } } // Show a message before each request, indicating what will be indexed. @@ -1243,6 +1276,52 @@ class manager { } } + /** + * Gets information about the request queue, in the form of a plain object suitable for passing + * to a template for rendering. + * + * @return \stdClass Information about queued index requests + */ + public function get_index_requests_info() { + global $DB; + + $result = new \stdClass(); + + $result->total = $DB->count_records('search_index_requests'); + $result->topten = $DB->get_records('search_index_requests', null, + 'indexpriority DESC, timerequested, contextid, searcharea', + 'id, contextid, timerequested, searcharea, partialarea, partialtime, indexpriority', + 0, 10); + foreach ($result->topten as $item) { + $context = \context::instance_by_id($item->contextid); + $item->contextlink = \html_writer::link($context->get_url(), + s($context->get_context_name())); + if ($item->searcharea) { + $item->areaname = $this->get_search_area($item->searcharea)->get_visible_name(); + } + if ($item->partialarea) { + $item->partialareaname = $this->get_search_area($item->partialarea)->get_visible_name(); + } + switch ($item->indexpriority) { + case self::INDEX_PRIORITY_REINDEXING : + $item->priorityname = get_string('priority_reindexing', 'search'); + break; + case self::INDEX_PRIORITY_NORMAL : + $item->priorityname = get_string('priority_normal', 'search'); + break; + } + } + + // Normalise array indices. + $result->topten = array_values($result->topten); + + if ($result->total > 10) { + $result->ellipsis = true; + } + + return $result; + } + /** * Gets current time for use in search system. * diff --git a/search/classes/output/renderer.php b/search/classes/output/renderer.php index 6f1f51f3d31..97cbe29e110 100644 --- a/search/classes/output/renderer.php +++ b/search/classes/output/renderer.php @@ -103,4 +103,15 @@ class renderer extends \plugin_renderer_base { $content .= $this->output->box_end(); return $content; } + + /** + * Returns information about queued index requests. + * + * @param \stdClass $info Info object from get_index_requests_info + * @return string HTML + * @throws \moodle_exception Any error with template + */ + public function render_index_requests_info(\stdClass $info) { + return $this->output->render_from_template('core_search/index_requests', $info); + } } diff --git a/search/templates/index_requests.mustache b/search/templates/index_requests.mustache new file mode 100644 index 00000000000..784ade70748 --- /dev/null +++ b/search/templates/index_requests.mustache @@ -0,0 +1,110 @@ +{{! + This file is part of Moodle - http://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template core_search/index_requests + + Template to provide admin information about the queue of index requests. + + Classes required for JS: + * none + + Data attributes required for JS: + * none + + Context variables required for this template: + * topten + * count + + Optional context variables for this template: + * ellipsis + + Example context (json): + { + "topten": + [ + { + "id": 42, + "timerequested": 123456789, + "contextid": 123, + "contextlink": "Forum: Tutor group forum", + "searcharea": "mod_forum-activity", + "areaname": "Forum activities", + "partialarea": "mod_forum-activity", + "partialareaname": "Forum activities", + "partialtime": 123400000, + "indexpriority": 100 + } + ], + "total": 1, + "ellipsis": true + } +}} +{{#total}} +
+

+ {{#str}} queueheading, search, {{total}} {{/str}} +

+ + + + + + + + + + + + + + {{#topten}} + + + + + + + + {{/topten}} + + {{#ellipsis}} + + + + {{/ellipsis}} + +
{{#str}} context, role {{/str}}{{#str}} searcharea, search {{/str}}{{#str}} time {{/str}}{{#str}} progress, search {{/str}}{{#str}} priority, search {{/str}}
+ {{{contextlink}}} + + {{#searcharea}} {{areaname}} {{/searcharea}} + {{#userdate}} {{timerequested}}, {{#str}} strftimedatetimeshort {{/str}} {{/userdate}} + {{#partialarea}} + {{partialareaname}}: + {{/partialarea}} + {{#partialtime}} + {{#userdate}} {{partialtime}}, {{#str}} strftimedatetimeshort {{/str}} {{/userdate}} + {{/partialtime}} + + {{#priorityname}} + {{priorityname}} + {{/priorityname}} + {{^priorityname}} + {{indexpriority}} + {{/priorityname}} +
...
+
+{{/total}} diff --git a/search/tests/base_activity_test.php b/search/tests/base_activity_test.php index 7b40c21f042..6c615443212 100644 --- a/search/tests/base_activity_test.php +++ b/search/tests/base_activity_test.php @@ -334,4 +334,43 @@ class search_base_activity_testcase extends advanced_testcase { // that context in the list to search. (This is because the $cm->uservisible access flag // is only valid if the user is known to be able to access the course.) } + + /** + * Tests the module version of get_contexts_to_reindex, which is supposed to return all the + * activity contexts in order of date added. + */ + public function test_get_contexts_to_reindex() { + global $DB; + + $this->resetAfterTest(); + + // Set up a course with two URLs and a Page. + $generator = $this->getDataGenerator(); + $course = $generator->create_course(['fullname' => 'TCourse']); + $url1 = $generator->create_module('url', ['course' => $course->id, 'name' => 'TURL1']); + $url2 = $generator->create_module('url', ['course' => $course->id, 'name' => 'TURL2']); + $page = $generator->create_module('page', ['course' => $course->id, 'name' => 'TPage1']); + + // Hack the items so they have different added times. + $now = time(); + $DB->set_field('course_modules', 'added', $now - 3, ['id' => $url2->cmid]); + $DB->set_field('course_modules', 'added', $now - 2, ['id' => $url1->cmid]); + $DB->set_field('course_modules', 'added', $now - 1, ['id' => $page->cmid]); + + // Check the URL contexts are in date order. + $urlarea = new \mod_url\search\activity(); + $contexts = iterator_to_array($urlarea->get_contexts_to_reindex(), false); + $this->assertEquals([\context_module::instance($url1->cmid), + \context_module::instance($url2->cmid)], $contexts); + + // Check the Page contexts. + $pagearea = new \mod_page\search\activity(); + $contexts = iterator_to_array($pagearea->get_contexts_to_reindex(), false); + $this->assertEquals([\context_module::instance($page->cmid)], $contexts); + + // Check another module area that has no instances. + $glossaryarea = new \mod_glossary\search\activity(); + $contexts = iterator_to_array($glossaryarea->get_contexts_to_reindex(), false); + $this->assertEquals([], $contexts); + } } diff --git a/search/tests/base_block_test.php b/search/tests/base_block_test.php index a7736a7a509..5a6d5c164a0 100644 --- a/search/tests/base_block_test.php +++ b/search/tests/base_block_test.php @@ -384,6 +384,52 @@ class base_block_testcase extends advanced_testcase { $this->assertEquals(\core_search\manager::ACCESS_DELETED, $area->check_access($blockid)); } + /** + * Tests the block version of get_contexts_to_reindex, which is supposed to return all the + * block contexts in order of date added. + */ + public function test_get_contexts_to_reindex() { + global $DB; + + $this->resetAfterTest(); + + // Create course and activity module. + $generator = $this->getDataGenerator(); + $course = $generator->create_course(); + $coursecontext = \context_course::instance($course->id); + $page = $generator->create_module('page', ['course' => $course->id]); + $pagecontext = \context_module::instance($page->cmid); + + // Create blocks on course page, with time modified non-sequential. + $configdata = base64_encode(serialize(new \stdClass())); + $instance = (object)['blockname' => 'mockblock', 'parentcontextid' => $coursecontext->id, + 'showinsubcontexts' => 0, 'pagetypepattern' => 'course-view-*', 'defaultweight' => 0, + 'timecreated' => 1, 'timemodified' => 100, 'configdata' => $configdata]; + $blockid1 = $DB->insert_record('block_instances', $instance); + $context1 = \context_block::instance($blockid1); + $instance->timemodified = 120; + $blockid2 = $DB->insert_record('block_instances', $instance); + $context2 = \context_block::instance($blockid2); + $instance->timemodified = 110; + $blockid3 = $DB->insert_record('block_instances', $instance); + $context3 = \context_block::instance($blockid3); + + // Create another block on the activity page (not included). + $instance->parentcontextid = $pagecontext->id; + $blockid4 = $DB->insert_record('block_instances', $instance); + \context_block::instance($blockid4); + + // Check list of contexts. + $area = new block_mockblock\search\area(); + $contexts = iterator_to_array($area->get_contexts_to_reindex(), false); + $expected = [ + $context2, + $context3, + $context1 + ]; + $this->assertEquals($expected, $contexts); + } + /** * Gets a search document object from the fake search area. * diff --git a/search/tests/base_test.php b/search/tests/base_test.php index 5880b231cf5..08f34426640 100644 --- a/search/tests/base_test.php +++ b/search/tests/base_test.php @@ -130,4 +130,13 @@ class search_base_testcase extends advanced_testcase { $this->assertEquals(1, count($files)); $this->assertEquals($content, $file->get_content()); } + + /** + * Tests the base version (stub) of get_contexts_to_reindex. + */ + public function test_get_contexts_to_reindex() { + $area = new core_mocksearch\search\mock_search_area(); + $this->assertEquals([\context_system::instance()], + iterator_to_array($area->get_contexts_to_reindex(), false)); + } } diff --git a/search/tests/manager_test.php b/search/tests/manager_test.php index 3f1c62147d5..b64cb159929 100644 --- a/search/tests/manager_test.php +++ b/search/tests/manager_test.php @@ -941,6 +941,39 @@ class search_manager_testcase extends advanced_testcase { // if we had already begun processing the previous entry.) \core_search\manager::request_index($forum2ctx); $this->assertEquals(4, $DB->count_records('search_index_requests')); + + // Clear queue and do tests relating to priority. + $DB->delete_records('search_index_requests'); + + // Request forum 1, specific area, priority 100. + \core_search\manager::request_index($forum1ctx, 'forum-post', 100); + $results = array_values($DB->get_records('search_index_requests', null, 'id')); + $this->assertCount(1, $results); + $this->assertEquals(100, $results[0]->indexpriority); + + // Request forum 1, same area, lower priority; no change. + \core_search\manager::request_index($forum1ctx, 'forum-post', 99); + $results = array_values($DB->get_records('search_index_requests', null, 'id')); + $this->assertCount(1, $results); + $this->assertEquals(100, $results[0]->indexpriority); + + // Request forum 1, same area, higher priority; priority stored changes. + \core_search\manager::request_index($forum1ctx, 'forum-post', 101); + $results = array_values($DB->get_records('search_index_requests', null, 'id')); + $this->assertCount(1, $results); + $this->assertEquals(101, $results[0]->indexpriority); + + // Request forum 1, all areas, lower priority; adds second entry. + \core_search\manager::request_index($forum1ctx, '', 100); + $results = array_values($DB->get_records('search_index_requests', null, 'id')); + $this->assertCount(2, $results); + $this->assertEquals(100, $results[1]->indexpriority); + + // Request course 1, all areas, lower priority; adds third entry. + \core_search\manager::request_index($course1ctx, '', 99); + $results = array_values($DB->get_records('search_index_requests', null, 'id')); + $this->assertCount(3, $results); + $this->assertEquals(99, $results[2]->indexpriority); } /** @@ -969,14 +1002,15 @@ class search_manager_testcase extends advanced_testcase { // Hack the forums so they have different creation times. $now = time(); - $DB->set_field('forum', 'timemodified', $now - 3, ['id' => $forum1->id]); - $DB->set_field('forum', 'timemodified', $now - 2, ['id' => $forum2->id]); - $DB->set_field('forum', 'timemodified', $now - 1, ['id' => $forum3->id]); - $forum2time = $now - 2; + $DB->set_field('forum', 'timemodified', $now - 30, ['id' => $forum1->id]); + $DB->set_field('forum', 'timemodified', $now - 20, ['id' => $forum2->id]); + $DB->set_field('forum', 'timemodified', $now - 10, ['id' => $forum3->id]); + $forum2time = $now - 20; // Make 2 index requests. + testable_core_search::fake_current_time($now - 3); $search::request_index(context_course::instance($course->id), 'mod_label-activity'); - $this->waitForSecond(); + testable_core_search::fake_current_time($now - 2); $search::request_index(context_module::instance($forum1->cmid)); // Run with no time limit. @@ -998,12 +1032,13 @@ class search_manager_testcase extends advanced_testcase { $this->assertEquals(0, $DB->count_records('search_index_requests')); // Request indexing the course a couple of times. + testable_core_search::fake_current_time($now - 3); $search::request_index(context_course::instance($course->id), 'mod_forum-activity'); + testable_core_search::fake_current_time($now - 2); $search::request_index(context_course::instance($course->id), 'mod_forum-post'); // Do the processing again with a time limit and indexing delay. The time limit is too // small; because of the way the logic works, this means it will index 2 activities. - testable_core_search::fake_current_time(time()); $search->get_engine()->set_add_delay(0.2); $search->process_index_requests(0.1, $progress); $out = $progress->get_buffer(); @@ -1024,7 +1059,7 @@ class search_manager_testcase extends advanced_testcase { $this->assertEquals($forum2time, $records[0]->partialtime); // Run again and confirm it now finishes. - $search->process_index_requests(0.1, $progress); + $search->process_index_requests(2.0, $progress); $out = $progress->get_buffer(); $progress->reset_buffer(); $this->assertContains( @@ -1036,5 +1071,26 @@ class search_manager_testcase extends advanced_testcase { // Confirm table is now empty. $this->assertEquals(0, $DB->count_records('search_index_requests')); + + // Make 2 requests - first one is low priority. + testable_core_search::fake_current_time($now - 3); + $search::request_index(context_module::instance($forum1->cmid), 'mod_forum-activity', + \core_search\manager::INDEX_PRIORITY_REINDEXING); + testable_core_search::fake_current_time($now - 2); + $search::request_index(context_module::instance($forum2->cmid), 'mod_forum-activity'); + + // Process with short time limit and confirm it does the second one first. + $search->process_index_requests(0.1, $progress); + $out = $progress->get_buffer(); + $progress->reset_buffer(); + $this->assertContains( + 'Completed requested context: Forum: TForum2 (search area: mod_forum-activity)', + $out); + $search->process_index_requests(0.1, $progress); + $out = $progress->get_buffer(); + $progress->reset_buffer(); + $this->assertContains( + 'Completed requested context: Forum: TForum1 (search area: mod_forum-activity)', + $out); } } diff --git a/search/upgrade.txt b/search/upgrade.txt index ba4a4f8d4cd..0371a654b76 100644 --- a/search/upgrade.txt +++ b/search/upgrade.txt @@ -1,6 +1,15 @@ This files describes API changes in /search/*, information provided here is intended especially for developers. +=== 3.5 === + +* Search areas may now optionally implement the get_contexts_to_reindex function (for modules and + blocks, see also get_contexts_to_reindex_extra_sql). This allows a search area to customise the + order in which it is reindexed when doing a gradual reindex, so as to reindex the most important + contexts first. If not implemented, the default behaviour for modules and blocks is to reindex + the newest items first; for other types of search area it will just index the whole system + context, oldest data first. + === 3.4 === * Search indexing now supports time limits to make the scheduled task run more neatly. In order for diff --git a/theme/boost/scss/moodle/search.scss b/theme/boost/scss/moodle/search.scss index 615d18d0330..bc40c9ab90c 100644 --- a/theme/boost/scss/moodle/search.scss +++ b/theme/boost/scss/moodle/search.scss @@ -68,3 +68,11 @@ margin-right: $spacer; display: inline-block; } + +#core-search-areas .lastcol li { + margin-left: 24px; + text-indent: -24px; +} +#core-search-areas .lastcol li > i { + text-indent: 0; +} diff --git a/theme/bootstrapbase/less/moodle/search.less b/theme/bootstrapbase/less/moodle/search.less index 21344348197..649f0c9c498 100644 --- a/theme/bootstrapbase/less/moodle/search.less +++ b/theme/bootstrapbase/less/moodle/search.less @@ -50,3 +50,11 @@ .search-areas-actions > div { display: inline-block; } + +#core-search-areas .lastcol li { + margin-left: 24px; + text-indent: -24px; +} +#core-search-areas .lastcol li > i { + text-indent: 0; +} diff --git a/theme/bootstrapbase/style/moodle.css b/theme/bootstrapbase/style/moodle.css index 7230efa2bdd..600138f46f4 100644 --- a/theme/bootstrapbase/style/moodle.css +++ b/theme/bootstrapbase/style/moodle.css @@ -10046,6 +10046,13 @@ body.path-question-type .mform fieldset.hidden { .search-areas-actions > div { display: inline-block; } +#core-search-areas .lastcol li { + margin-left: 24px; + text-indent: -24px; +} +#core-search-areas .lastcol li > i { + text-indent: 0; +} .popover-region { float: right; position: relative; diff --git a/version.php b/version.php index c2c7a6fc596..f9d19d2b84b 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2017122200.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2017122200.01; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes.