mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-04-24 09:55:33 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'MDL-60981-master' of https://github.com/sammarshallou/moodle

Browse Source
This commit is contained in:
Eloy Lafuente (stronk7) 2017-12-27 00:28:54 +01:00 committed by David Monllao
commit 72614d74d8
22 changed files with 766 additions and 20 deletions

19
admin/searchareas.php

@ -132,7 +132,17 @@ foreach ($searchareas as $area) {
$laststatus = '';
}
$columns[] = $laststatus;
$columns[] = html_writer::link(admin_searcharea_action_url('delete', $areaid), 'Delete index');
$accesshide = html_writer::span($area->get_visible_name(), 'accesshide');
$actions = [];
$actions[] = $OUTPUT->pix_icon('t/delete', '') .
html_writer::link(admin_searcharea_action_url('delete', $areaid),
get_string('deleteindex', 'search', $accesshide));
if ($area->supports_get_document_recordset()) {
$actions[] = $OUTPUT->pix_icon('i/reload', '') . html_writer::link(
new moodle_url('searchreindex.php', ['areaid' => $areaid]),
get_string('gradualreindex', 'search', $accesshide));
}
$columns[] = html_writer::alist($actions, ['class' => 'unstyled list-unstyled']);
} else {
$blankrow = new html_table_cell(get_string('searchnotavailable', 'admin'));
@ -165,6 +175,13 @@ echo $OUTPUT->single_button(admin_searcharea_action_url('deleteall'), get_string
echo $OUTPUT->box_end();
echo html_writer::table($table);
if (empty($searchmanagererror)) {
// Show information about queued index requests for specific contexts.
$searchrenderer = $PAGE->get_renderer('core_search');
echo $searchrenderer->render_index_requests_info($searchmanager->get_index_requests_info());
}
echo $OUTPUT->footer();
/**

87
admin/searchreindex.php Normal file

@ -0,0 +1,87 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Adds a search area to the queue for indexing.
*
* @package core_search
* @copyright 2017 The Open University
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
define('NO_OUTPUT_BUFFERING', true);
require(__DIR__ . '/../config.php');
// Check access.
require_once($CFG->libdir . '/adminlib.php');
admin_externalpage_setup('searchareas', '', null, (new moodle_url('/admin/searchreindex.php'))->out(false));
// Get area parameter and check it exists.
$areaid = required_param('areaid', PARAM_ALPHAEXT);
$area = \core_search\manager::get_search_area($areaid);
if ($area === false) {
throw new moodle_exception('invalidrequest');
}
$areaname = $area->get_visible_name();
// Start page output.
$heading = get_string('gradualreindex', 'search', '');
$PAGE->set_title($PAGE->title . ': ' . $heading);
$PAGE->navbar->add($heading);
echo $OUTPUT->header();
echo $OUTPUT->heading($heading);
// If sesskey is supplied, actually carry out the action.
if (optional_param('sesskey', '', PARAM_ALPHANUM)) {
require_sesskey();
// Get all contexts for search area. This query can take time in large cases.
\core_php_time_limit::raise(0);
$contextiterator = $area->get_contexts_to_reindex();
$progress = new \core\progress\display_if_slow('');
$progress->start_progress($areaname);
// Request reindexing for each context (with low priority).
$count = 0;
foreach ($contextiterator as $context) {
\core_php_time_limit::raise(30);
\core_search\manager::request_index($context, $area->get_area_id(),
\core_search\manager::INDEX_PRIORITY_REINDEXING);
$progress->progress();
$count++;
}
// Unset the iterator which should close the recordset (if there is one).
unset($contextiterator);
$progress->end_progress();
$a = (object)['name' => html_writer::tag('strong', $areaname), 'count' => $count];
echo $OUTPUT->box(get_string('gradualreindex_queued', 'search', $a));
echo $OUTPUT->continue_button(new moodle_url('/admin/searchareas.php'));
} else {
// Display confirmation prompt.
echo $OUTPUT->confirm(get_string('gradualreindex_confirm', 'search', html_writer::tag('strong', $areaname)),
new single_button(new moodle_url('/admin/searchreindex.php', ['areaid' => $areaid,
'sesskey' => sesskey()]), get_string('continue'), 'post', true),
new single_button(new moodle_url('/admin/searchareas.php'), get_string('cancel'), 'get'));
}
echo $OUTPUT->footer();

9
lang/en/search.php

@ -36,6 +36,7 @@ $string['createdon'] = 'Created on';
$string['database'] = 'Database';
$string['databasestate'] = 'Indexing database state';
$string['datadirectory'] = 'Data directory';
$string['deleteindex'] = 'Delete index {$a}';
$string['deletionsinindex'] = 'Deletions in index';
$string['docmodifiedon'] = 'Last modified on {$a}';
$string['doctype'] = 'Doctype';
@ -59,6 +60,9 @@ $string['filterheader'] = 'Filter';
$string['fromtime'] = 'Modified after';
$string['globalsearch'] = 'Global search';
$string['globalsearchdisabled'] = 'Global searching is not enabled.';
$string['gradualreindex'] = 'Gradual reindex {$a}';
$string['gradualreindex_confirm'] = 'Are you sure you want to reindex {$a}? This may take some time, although existing data will remain available during the reindex.';
$string['gradualreindex_queued'] = 'Reindexing has been requested for {$a->name} ({$a->count} contexts). This indexing will be carried out by the &lsquo;Global search indexing&rsquo; scheduled task.';
$string['checkdb'] = 'Check database';
$string['checkdbadvice'] = 'Check your database for any problems.';
$string['checkdir'] = 'Check dir';
@ -76,7 +80,12 @@ $string['notitle'] = 'No title';
$string['normalsearch'] = 'Normal search';
$string['openedon'] = 'opened on';
$string['optimize'] = 'Optimize';
$string['priority'] = 'Priority';
$string['priority_reindexing'] = 'Reindexing';
$string['priority_normal'] = 'Normal';
$string['progress'] = 'Progress';
$string['queryerror'] = 'The query you provided could not be parsed by the search engine: {$a}';
$string['queueheading'] = 'Additional indexing queue ({$a} items)';
$string['resultsreturnedfor'] = 'results returned for';
$string['runindexer'] = 'Run indexer (real)';
$string['runindexertest'] = 'Run indexer test';

6
lib/db/install.xml

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<XMLDB PATH="lib/db" VERSION="20171205" COMMENT="XMLDB file for core Moodle tables"
<XMLDB PATH="lib/db" VERSION="20171222" COMMENT="XMLDB file for core Moodle tables"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="../../lib/xmldb/xmldb.xsd"
>
@ -3770,11 +3770,15 @@
<FIELD NAME="timerequested" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="Time at which this index update was requested."/>
<FIELD NAME="partialarea" TYPE="char" LENGTH="255" NOTNULL="true" SEQUENCE="false" COMMENT="If processing of this context partially completed, set to the area that needs processing next. Blank indicates not processed yet."/>
<FIELD NAME="partialtime" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="If processing partially completed, set to the timestamp within the next area where processing should start. 0 indicates not processed yet."/>
<FIELD NAME="indexpriority" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="Priority value so that important requests can be dealt with first; higher numbers are processed first"/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>
<KEY NAME="contextid" TYPE="foreign" FIELDS="contextid" REFTABLE="context" REFFIELDS="id"/>
</KEYS>
<INDEXES>
<INDEX NAME="indexprioritytimerequested" UNIQUE="false" FIELDS="indexpriority, timerequested"/>
</INDEXES>
</TABLE>
</TABLES>
</XMLDB>

33
lib/db/upgrade.php

@ -1902,5 +1902,38 @@ function xmldb_main_upgrade($oldversion) {
upgrade_main_savepoint(true, 2017121900.00);
}
if ($oldversion < 2017122200.01) {
// Define field indexpriority to be added to search_index_requests. Allow null initially.
$table = new xmldb_table('search_index_requests');
$field = new xmldb_field('indexpriority', XMLDB_TYPE_INTEGER, '10',
null, null, null, null, 'partialtime');
// Conditionally add field.
if (!$dbman->field_exists($table, $field)) {
$dbman->add_field($table, $field);
// Set existing values to 'normal' value (100).
$DB->set_field('search_index_requests', 'indexpriority', 100);
// Now make the field 'NOT NULL'.
$field = new xmldb_field('indexpriority', XMLDB_TYPE_INTEGER, '10',
null, XMLDB_NOTNULL, null, null, 'partialtime');
$dbman->change_field_notnull($table, $field);
}
// Define index indexprioritytimerequested (not unique) to be added to search_index_requests.
$index = new xmldb_index('indexprioritytimerequested', XMLDB_INDEX_NOTUNIQUE,
array('indexpriority', 'timerequested'));
// Conditionally launch add index indexprioritytimerequested.
if (!$dbman->index_exists($table, $index)) {
$dbman->add_index($table, $index);
}
// Main savepoint reached.
upgrade_main_savepoint(true, 2017122200.01);
}
return true;
}

13
mod/forum/classes/search/post.php

@ -292,4 +292,17 @@ class post extends \core_search\base_mod {
}
return $this->discussionsdata[$discussionid];
}
/**
* Changes the context ordering so that the forums with most recent discussions are indexed
* first.
*
* @return string[] SQL join and ORDER BY
*/
protected function get_contexts_to_reindex_extra_sql() {
return [
'JOIN {forum_discussions} fd ON fd.course = cm.course AND fd.forum = cm.instance',
'MAX(fd.timemodified) DESC'
];
}
}

58
mod/forum/tests/search_test.php

@ -390,4 +390,62 @@ class mod_forum_search_testcase extends advanced_testcase {
$recordset->close();
$this->assertEquals(3, $nrecords);
}
/**
* Tests that reindexing works in order starting from the forum with most recent discussion.
*/
public function test_posts_get_contexts_to_reindex() {
global $DB;
$generator = $this->getDataGenerator();
$adminuser = get_admin();
$course1 = $generator->create_course();
$course2 = $generator->create_course();
$time = time() - 1000;
// Create 3 forums (two in course 1, one in course 2 - doesn't make a difference).
$forum1 = $generator->create_module('forum', ['course' => $course1->id]);
$forum2 = $generator->create_module('forum', ['course' => $course1->id]);
$forum3 = $generator->create_module('forum', ['course' => $course2->id]);
$forum4 = $generator->create_module('forum', ['course' => $course2->id]);
// Hack added time for the course_modules entries. These should not be used (they would
// be used by the base class implementation). We are setting this so that the order would
// be 4, 3, 2, 1 if this ordering were used (newest first).
$DB->set_field('course_modules', 'added', $time + 100, ['id' => $forum1->cmid]);
$DB->set_field('course_modules', 'added', $time + 110, ['id' => $forum2->cmid]);
$DB->set_field('course_modules', 'added', $time + 120, ['id' => $forum3->cmid]);
$DB->set_field('course_modules', 'added', $time + 130, ['id' => $forum4->cmid]);
$forumgenerator = $generator->get_plugin_generator('mod_forum');
// Create one discussion in forums 1 and 3, three in forum 2, and none in forum 4.
$forumgenerator->create_discussion(['course' => $course1->id,
'forum' => $forum1->id, 'userid' => $adminuser->id, 'timemodified' => $time + 20]);
$forumgenerator->create_discussion(['course' => $course1->id,
'forum' => $forum2->id, 'userid' => $adminuser->id, 'timemodified' => $time + 10]);
$forumgenerator->create_discussion(['course' => $course1->id,
'forum' => $forum2->id, 'userid' => $adminuser->id, 'timemodified' => $time + 30]);
$forumgenerator->create_discussion(['course' => $course1->id,
'forum' => $forum2->id, 'userid' => $adminuser->id, 'timemodified' => $time + 11]);
$forumgenerator->create_discussion(['course' => $course2->id,
'forum' => $forum3->id, 'userid' => $adminuser->id, 'timemodified' => $time + 25]);
// Get the contexts in reindex order.
$area = \core_search\manager::get_search_area($this->forumpostareaid);
$contexts = iterator_to_array($area->get_contexts_to_reindex(), false);
// We expect them in order of newest discussion. Forum 4 is not included at all (which is
// correct because it has no content).
$expected = [
\context_module::instance($forum2->cmid),
\context_module::instance($forum3->cmid),
\context_module::instance($forum1->cmid)
];
$this->assertEquals($expected, $contexts);
}
}

35
search/classes/base.php

@ -285,6 +285,14 @@ abstract class base {
* The default implementation returns false, indicating that this facility is not supported and
* the older get_recordset_by_timestamp function should be used.
*
* This function must accept all possible values for the $context parameter. For example, if
* you are implementing this function for the forum module, it should still operate correctly
* if called with the context for a glossary module, or for the HTML block. (In these cases
* where it will not return any data, it may return null.)
*
* The $context parameter can also be null or the system context; both of these indicate that
* all data, without context restriction, should be returned.
*
* @param int $modifiedfrom Return only records modified after this date
* @param \context|null $context Context (null means no context restriction)
* @return \moodle_recordset|null|false Recordset / null if no results / false if not supported
@ -294,6 +302,19 @@ abstract class base {
return false;
}
/**
* Checks if get_document_recordset is supported for this search area.
*
* For many uses you can simply call get_document_recordset and see if it returns false, but
* this function is useful when you don't want to actually call the function right away.
*/
public function supports_get_document_recordset() {
// Easiest way to check this is simply to see if the class has overridden the default
// function.
$method = new \ReflectionMethod($this, 'get_document_recordset');
return $method->getDeclaringClass()->getName() !== self::class;
}
/**
* Returns the document related with the provided record.
*
@ -474,4 +495,18 @@ abstract class base {
return [$sql, $params];
}
/**
* Gets a list of all contexts to reindex when reindexing this search area. The list should be
* returned in an order that is likely to be suitable when reindexing, for example with newer
* contexts first.
*
* The default implementation simply returns the system context, which will result in
* reindexing everything in normal date order (oldest first).
*
* @return \Iterator Iterator of contexts to reindex
*/
public function get_contexts_to_reindex() {
return new \ArrayIterator([\context_system::instance()]);
}
}

55
search/classes/base_block.php

@ -343,4 +343,59 @@ abstract class base_block extends base {
return [$sql, $params];
}
/**
* This can be used in subclasses to change ordering within the get_contexts_to_reindex
* function.
*
* It returns 2 values:
* - Extra SQL joins (tables block_instances 'bi' and context 'x' already exist).
* - An ORDER BY value which must use aggregate functions, by default 'MAX(bi.timemodified) DESC'.
*
* Note the query already includes a GROUP BY on the context fields, so if your joins result
* in multiple rows, you can use aggregate functions in the ORDER BY. See forum for an example.
*
* @return string[] Array with 2 elements; extra joins for the query, and ORDER BY value
*/
protected function get_contexts_to_reindex_extra_sql() {
return ['', 'MAX(bi.timemodified) DESC'];
}
/**
* Gets a list of all contexts to reindex when reindexing this search area.
*
* For blocks, the default is to return all contexts for blocks of that type, that are on a
* course page, in order of time added (most recent first).
*
* @return \Iterator Iterator of contexts to reindex
* @throws \moodle_exception If any DB error
*/
public function get_contexts_to_reindex() {
global $DB;
list ($extrajoins, $dborder) = $this->get_contexts_to_reindex_extra_sql();
$contexts = [];
$selectcolumns = \context_helper::get_preload_record_columns_sql('x');
$groupbycolumns = '';
foreach (\context_helper::get_preload_record_columns('x') as $column => $thing) {
if ($groupbycolumns !== '') {
$groupbycolumns .= ',';
}
$groupbycolumns .= $column;
}
$rs = $DB->get_recordset_sql("
SELECT $selectcolumns
FROM {block_instances} bi
JOIN {context} x ON x.instanceid = bi.id AND x.contextlevel = ?
JOIN {context} parent ON parent.id = bi.parentcontextid
$extrajoins
WHERE bi.blockname = ? AND parent.contextlevel = ?
GROUP BY $groupbycolumns
ORDER BY $dborder", [CONTEXT_BLOCK, $this->get_block_name(), CONTEXT_COURSE]);
return new \core\dml\recordset_walk($rs, function($rec) {
$id = $rec->ctxid;
\context_helper::preload_from_record($rec);
return \context::instance_by_id($id);
});
}
}

53
search/classes/base_mod.php

@ -192,4 +192,57 @@ abstract class base_mod extends base {
return [$sql, $params];
}
/**
* This can be used in subclasses to change ordering within the get_contexts_to_reindex
* function.
*
* It returns 2 values:
* - Extra SQL joins (tables course_modules 'cm' and context 'x' already exist).
* - An ORDER BY value which must use aggregate functions, by default 'MAX(cm.added) DESC'.
*
* Note the query already includes a GROUP BY on the context fields, so if your joins result
* in multiple rows, you can use aggregate functions in the ORDER BY. See forum for an example.
*
* @return string[] Array with 2 elements; extra joins for the query, and ORDER BY value
*/
protected function get_contexts_to_reindex_extra_sql() {
return ['', 'MAX(cm.added) DESC'];
}
/**
* Gets a list of all contexts to reindex when reindexing this search area.
*
* For modules, the default is to return all contexts for modules of that type, in order of
* time added (most recent first).
*
* @return \Iterator Iterator of contexts to reindex
* @throws \moodle_exception If any DB error
*/
public function get_contexts_to_reindex() {
global $DB;
list ($extrajoins, $dborder) = $this->get_contexts_to_reindex_extra_sql();
$contexts = [];
$selectcolumns = \context_helper::get_preload_record_columns_sql('x');
$groupbycolumns = '';
foreach (\context_helper::get_preload_record_columns('x') as $column => $thing) {
if ($groupbycolumns !== '') {
$groupbycolumns .= ',';
}
$groupbycolumns .= $column;
}
$rs = $DB->get_recordset_sql("
SELECT $selectcolumns
FROM {course_modules} cm
JOIN {context} x ON x.instanceid = cm.id AND x.contextlevel = ?
$extrajoins
WHERE cm.module = (SELECT id FROM {modules} WHERE name = ?)
GROUP BY $groupbycolumns
ORDER BY $dborder", [CONTEXT_MODULE, $this->get_module_name()]);
return new \core\dml\recordset_walk($rs, function($rec) {
$id = $rec->ctxid;
\context_helper::preload_from_record($rec);
return \context::instance_by_id($id);
});
}
}

99
search/classes/manager.php

@ -87,6 +87,16 @@ class manager {
*/
const DISPLAY_INDEXING_PROGRESS_EVERY = 30.0;
/**
* @var int Context indexing: normal priority.
*/
const INDEX_PRIORITY_NORMAL = 100;
/**
* @var int Context indexing: low priority for reindexing.
*/
const INDEX_PRIORITY_REINDEXING = 50;
/**
* @var \core_search\base[] Enabled search areas.
*/
@ -1143,36 +1153,55 @@ class manager {
* added to a queue which is processed by the task.
*
* This is used after a restore to ensure that restored items are indexed, even though their
* modified time will be older than the latest indexed.
* modified time will be older than the latest indexed. It is also used by the 'Gradual reindex'
* admin feature from the search areas screen.
*
* @param \context $context Context to index within
* @param string $areaid Area to index, '' = all areas
* @param int $priority Priority (INDEX_PRIORITY_xx constant)
*/
public static function request_index(\context $context, $areaid = '') {
public static function request_index(\context $context, $areaid = '',
$priority = self::INDEX_PRIORITY_NORMAL) {
global $DB;
// Check through existing requests for this context or any parent context.
list ($contextsql, $contextparams) = $DB->get_in_or_equal(
$context->get_parent_context_ids(true));
$existing = $DB->get_records_select('search_index_requests',
'contextid ' . $contextsql, $contextparams, '', 'id, searcharea, partialarea');
'contextid ' . $contextsql, $contextparams, '',
'id, searcharea, partialarea, indexpriority');
foreach ($existing as $rec) {
// If we haven't started processing the existing request yet, and it covers the same
// area (or all areas) then that will be sufficient so don't add anything else.
if ($rec->partialarea === '' && ($rec->searcharea === $areaid || $rec->searcharea === '')) {
return;
// If the existing request has the same (or higher) priority, no need to add anything.
if ($rec->indexpriority >= $priority) {
return;
}
// The existing request has lower priority. If it is exactly the same, then just
// adjust the priority of the existing request.
if ($rec->searcharea === $areaid) {
$DB->set_field('search_index_requests', 'indexpriority', $priority,
['id' => $rec->id]);
return;
}
// The existing request would cover this area but is a lower priority. We need to
// add the new request even though that means we will index part of it twice.
}
}
// No suitable existing request, so add a new one.
$newrecord = [ 'contextid' => $context->id, 'searcharea' => $areaid,
'timerequested' => time(), 'partialarea' => '', 'partialtime' => 0 ];
'timerequested' => (int)self::get_current_time(),
'partialarea' => '', 'partialtime' => 0,
'indexpriority' => $priority ];
$DB->insert_record('search_index_requests', $newrecord);
}
/**
* Processes outstanding index requests. This will take the first item from the queue and
* process it, continuing until an optional time limit is reached.
* Processes outstanding index requests. This will take the first item from the queue (taking
* account the indexing priority) and process it, continuing until an optional time limit is
* reached.
*
* If there are no index requests, the function will do nothing.
*
@ -1186,7 +1215,6 @@ class manager {
$progress = new \null_progress_trace();
}
$complete = false;
$before = self::get_current_time();
if ($timelimit) {
$stopat = $before + $timelimit;
@ -1194,11 +1222,10 @@ class manager {
while (true) {
// Retrieve first request, using fully defined ordering.
$requests = $DB->get_records('search_index_requests', null,
'timerequested, contextid, searcharea',
'indexpriority DESC, timerequested, contextid, searcharea',
'id, contextid, searcharea, partialarea, partialtime', 0, 1);
if (!$requests) {
// If there are no more requests, stop.
$complete = true;
break;
}
$request = reset($requests);
@ -1208,6 +1235,12 @@ class manager {
$beforeindex = self::get_current_time();
if ($timelimit) {
$remainingtime = $stopat - $beforeindex;
// If the time limit expired already, stop now. (Otherwise we might accidentally
// index with no time limit or a negative time limit.)
if ($remainingtime <= 0) {
break;
}
}
// Show a message before each request, indicating what will be indexed.
@ -1243,6 +1276,52 @@ class manager {
}
}
/**
* Gets information about the request queue, in the form of a plain object suitable for passing
* to a template for rendering.
*
* @return \stdClass Information about queued index requests
*/
public function get_index_requests_info() {
global $DB;
$result = new \stdClass();
$result->total = $DB->count_records('search_index_requests');
$result->topten = $DB->get_records('search_index_requests', null,
'indexpriority DESC, timerequested, contextid, searcharea',
'id, contextid, timerequested, searcharea, partialarea, partialtime, indexpriority',
0, 10);
foreach ($result->topten as $item) {
$context = \context::instance_by_id($item->contextid);
$item->contextlink = \html_writer::link($context->get_url(),
s($context->get_context_name()));
if ($item->searcharea) {
$item->areaname = $this->get_search_area($item->searcharea)->get_visible_name();
}
if ($item->partialarea) {
$item->partialareaname = $this->get_search_area($item->partialarea)->get_visible_name();
}
switch ($item->indexpriority) {
case self::INDEX_PRIORITY_REINDEXING :
$item->priorityname = get_string('priority_reindexing', 'search');
break;
case self::INDEX_PRIORITY_NORMAL :
$item->priorityname = get_string('priority_normal', 'search');
break;
}
}
// Normalise array indices.
$result->topten = array_values($result->topten);
if ($result->total > 10) {
$result->ellipsis = true;
}
return $result;
}
/**
* Gets current time for use in search system.
*

11
search/classes/output/renderer.php

@ -103,4 +103,15 @@ class renderer extends \plugin_renderer_base {
$content .= $this->output->box_end();
return $content;
}
/**
* Returns information about queued index requests.
*
* @param \stdClass $info Info object from get_index_requests_info
* @return string HTML
* @throws \moodle_exception Any error with template
*/
public function render_index_requests_info(\stdClass $info) {
return $this->output->render_from_template('core_search/index_requests', $info);
}
}

110
search/templates/index_requests.mustache Normal file

@ -0,0 +1,110 @@
{{!
This file is part of Moodle - http://moodle.org/
Moodle is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Moodle is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Moodle. If not, see <http://www.gnu.org/licenses/>.
}}
{{!
@template core_search/index_requests
Template to provide admin information about the queue of index requests.
Classes required for JS:
* none
Data attributes required for JS:
* none
Context variables required for this template:
* topten
* count
Optional context variables for this template:
* ellipsis
Example context (json):
{
"topten":
[
{
"id": 42,
"timerequested": 123456789,
"contextid": 123,
"contextlink": "<a href='...'>Forum: Tutor group forum</a>",
"searcharea": "mod_forum-activity",
"areaname": "Forum activities",
"partialarea": "mod_forum-activity",
"partialareaname": "Forum activities",
"partialtime": 123400000,
"indexpriority": 100
}
],
"total": 1,
"ellipsis": true
}
}}
{{#total}}
<div>
<h3>
{{#str}} queueheading, search, {{total}} {{/str}}
</h3>
<table class="generaltable">
<thead>
<tr>
<th scope="col">{{#str}} context, role {{/str}}</th>
<th scope="col">{{#str}} searcharea, search {{/str}}</th>
<th scope="col">{{#str}} time {{/str}}</th>
<th scope="col">{{#str}} progress, search {{/str}}</th>
<th scope="col">{{#str}} priority, search {{/str}}</th>
</tr>
</thead>
<tbody>
{{#topten}}
<tr>
<td>
{{{contextlink}}}
</td>
<td>
{{#searcharea}} {{areaname}} {{/searcharea}}
</td>
<td>{{#userdate}} {{timerequested}}, {{#str}} strftimedatetimeshort {{/str}} {{/userdate}}</td>
<td>
{{#partialarea}}
{{partialareaname}}:
{{/partialarea}}
{{#partialtime}}
{{#userdate}} {{partialtime}}, {{#str}} strftimedatetimeshort {{/str}} {{/userdate}}
{{/partialtime}}
</td>
<td>
{{#priorityname}}
{{priorityname}}
{{/priorityname}}
{{^priorityname}}
{{indexpriority}}
{{/priorityname}}
</td>
</tr>
{{/topten}}
{{#ellipsis}}
<tr>
<td colspan="5">...</td>
</tr>
{{/ellipsis}}
</tbody>
</table>
</div>
{{/total}}

39
search/tests/base_activity_test.php

@ -334,4 +334,43 @@ class search_base_activity_testcase extends advanced_testcase {
// that context in the list to search. (This is because the $cm->uservisible access flag
// is only valid if the user is known to be able to access the course.)
}
/**
* Tests the module version of get_contexts_to_reindex, which is supposed to return all the
* activity contexts in order of date added.
*/
public function test_get_contexts_to_reindex() {
global $DB;
$this->resetAfterTest();
// Set up a course with two URLs and a Page.
$generator = $this->getDataGenerator();
$course = $generator->create_course(['fullname' => 'TCourse']);
$url1 = $generator->create_module('url', ['course' => $course->id, 'name' => 'TURL1']);
$url2 = $generator->create_module('url', ['course' => $course->id, 'name' => 'TURL2']);
$page = $generator->create_module('page', ['course' => $course->id, 'name' => 'TPage1']);
// Hack the items so they have different added times.
$now = time();
$DB->set_field('course_modules', 'added', $now - 3, ['id' => $url2->cmid]);
$DB->set_field('course_modules', 'added', $now - 2, ['id' => $url1->cmid]);
$DB->set_field('course_modules', 'added', $now - 1, ['id' => $page->cmid]);
// Check the URL contexts are in date order.
$urlarea = new \mod_url\search\activity();
$contexts = iterator_to_array($urlarea->get_contexts_to_reindex(), false);
$this->assertEquals([\context_module::instance($url1->cmid),
\context_module::instance($url2->cmid)], $contexts);
// Check the Page contexts.
$pagearea = new \mod_page\search\activity();
$contexts = iterator_to_array($pagearea->get_contexts_to_reindex(), false);
$this->assertEquals([\context_module::instance($page->cmid)], $contexts);
// Check another module area that has no instances.
$glossaryarea = new \mod_glossary\search\activity();
$contexts = iterator_to_array($glossaryarea->get_contexts_to_reindex(), false);
$this->assertEquals([], $contexts);
}
}

46
search/tests/base_block_test.php

@ -384,6 +384,52 @@ class base_block_testcase extends advanced_testcase {
$this->assertEquals(\core_search\manager::ACCESS_DELETED, $area->check_access($blockid));
}
/**
* Tests the block version of get_contexts_to_reindex, which is supposed to return all the
* block contexts in order of date added.
*/
public function test_get_contexts_to_reindex() {
global $DB;
$this->resetAfterTest();
// Create course and activity module.
$generator = $this->getDataGenerator();
$course = $generator->create_course();
$coursecontext = \context_course::instance($course->id);
$page = $generator->create_module('page', ['course' => $course->id]);
$pagecontext = \context_module::instance($page->cmid);
// Create blocks on course page, with time modified non-sequential.
$configdata = base64_encode(serialize(new \stdClass()));
$instance = (object)['blockname' => 'mockblock', 'parentcontextid' => $coursecontext->id,
'showinsubcontexts' => 0, 'pagetypepattern' => 'course-view-*', 'defaultweight' => 0,
'timecreated' => 1, 'timemodified' => 100, 'configdata' => $configdata];
$blockid1 = $DB->insert_record('block_instances', $instance);
$context1 = \context_block::instance($blockid1);
$instance->timemodified = 120;
$blockid2 = $DB->insert_record('block_instances', $instance);
$context2 = \context_block::instance($blockid2);
$instance->timemodified = 110;
$blockid3 = $DB->insert_record('block_instances', $instance);
$context3 = \context_block::instance($blockid3);
// Create another block on the activity page (not included).
$instance->parentcontextid = $pagecontext->id;
$blockid4 = $DB->insert_record('block_instances', $instance);
\context_block::instance($blockid4);
// Check list of contexts.
$area = new block_mockblock\search\area();
$contexts = iterator_to_array($area->get_contexts_to_reindex(), false);
$expected = [
$context2,
$context3,
$context1
];
$this->assertEquals($expected, $contexts);
}
/**
* Gets a search document object from the fake search area.
*

9
search/tests/base_test.php

@ -130,4 +130,13 @@ class search_base_testcase extends advanced_testcase {
$this->assertEquals(1, count($files));
$this->assertEquals($content, $file->get_content());
}
/**
* Tests the base version (stub) of get_contexts_to_reindex.
*/
public function test_get_contexts_to_reindex() {
$area = new core_mocksearch\search\mock_search_area();
$this->assertEquals([\context_system::instance()],
iterator_to_array($area->get_contexts_to_reindex(), false));
}
}

70
search/tests/manager_test.php

@ -941,6 +941,39 @@ class search_manager_testcase extends advanced_testcase {
// if we had already begun processing the previous entry.)
\core_search\manager::request_index($forum2ctx);
$this->assertEquals(4, $DB->count_records('search_index_requests'));
// Clear queue and do tests relating to priority.
$DB->delete_records('search_index_requests');
// Request forum 1, specific area, priority 100.
\core_search\manager::request_index($forum1ctx, 'forum-post', 100);
$results = array_values($DB->get_records('search_index_requests', null, 'id'));
$this->assertCount(1, $results);
$this->assertEquals(100, $results[0]->indexpriority);
// Request forum 1, same area, lower priority; no change.
\core_search\manager::request_index($forum1ctx, 'forum-post', 99);
$results = array_values($DB->get_records('search_index_requests', null, 'id'));
$this->assertCount(1, $results);
$this->assertEquals(100, $results[0]->indexpriority);
// Request forum 1, same area, higher priority; priority stored changes.
\core_search\manager::request_index($forum1ctx, 'forum-post', 101);
$results = array_values($DB->get_records('search_index_requests', null, 'id'));
$this->assertCount(1, $results);
$this->assertEquals(101, $results[0]->indexpriority);
// Request forum 1, all areas, lower priority; adds second entry.
\core_search\manager::request_index($forum1ctx, '', 100);
$results = array_values($DB->get_records('search_index_requests', null, 'id'));
$this->assertCount(2, $results);
$this->assertEquals(100, $results[1]->indexpriority);
// Request course 1, all areas, lower priority; adds third entry.
\core_search\manager::request_index($course1ctx, '', 99);
$results = array_values($DB->get_records('search_index_requests', null, 'id'));
$this->assertCount(3, $results);
$this->assertEquals(99, $results[2]->indexpriority);
}
/**
@ -969,14 +1002,15 @@ class search_manager_testcase extends advanced_testcase {
// Hack the forums so they have different creation times.
$now = time();
$DB->set_field('forum', 'timemodified', $now - 3, ['id' => $forum1->id]);
$DB->set_field('forum', 'timemodified', $now - 2, ['id' => $forum2->id]);
$DB->set_field('forum', 'timemodified', $now - 1, ['id' => $forum3->id]);
$forum2time = $now - 2;
$DB->set_field('forum', 'timemodified', $now - 30, ['id' => $forum1->id]);
$DB->set_field('forum', 'timemodified', $now - 20, ['id' => $forum2->id]);
$DB->set_field('forum', 'timemodified', $now - 10, ['id' => $forum3->id]);
$forum2time = $now - 20;
// Make 2 index requests.
testable_core_search::fake_current_time($now - 3);
$search::request_index(context_course::instance($course->id), 'mod_label-activity');
$this->waitForSecond();
testable_core_search::fake_current_time($now - 2);
$search::request_index(context_module::instance($forum1->cmid));
// Run with no time limit.
@ -998,12 +1032,13 @@ class search_manager_testcase extends advanced_testcase {
$this->assertEquals(0, $DB->count_records('search_index_requests'));
// Request indexing the course a couple of times.
testable_core_search::fake_current_time($now - 3);
$search::request_index(context_course::instance($course->id), 'mod_forum-activity');
testable_core_search::fake_current_time($now - 2);
$search::request_index(context_course::instance($course->id), 'mod_forum-post');
// Do the processing again with a time limit and indexing delay. The time limit is too
// small; because of the way the logic works, this means it will index 2 activities.
testable_core_search::fake_current_time(time());
$search->get_engine()->set_add_delay(0.2);
$search->process_index_requests(0.1, $progress);
$out = $progress->get_buffer();
@ -1024,7 +1059,7 @@ class search_manager_testcase extends advanced_testcase {
$this->assertEquals($forum2time, $records[0]->partialtime);
// Run again and confirm it now finishes.
$search->process_index_requests(0.1, $progress);
$search->process_index_requests(2.0, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
$this->assertContains(
@ -1036,5 +1071,26 @@ class search_manager_testcase extends advanced_testcase {
// Confirm table is now empty.
$this->assertEquals(0, $DB->count_records('search_index_requests'));
// Make 2 requests - first one is low priority.
testable_core_search::fake_current_time($now - 3);
$search::request_index(context_module::instance($forum1->cmid), 'mod_forum-activity',
\core_search\manager::INDEX_PRIORITY_REINDEXING);
testable_core_search::fake_current_time($now - 2);
$search::request_index(context_module::instance($forum2->cmid), 'mod_forum-activity');
// Process with short time limit and confirm it does the second one first.
$search->process_index_requests(0.1, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
$this->assertContains(
'Completed requested context: Forum: TForum2 (search area: mod_forum-activity)',
$out);
$search->process_index_requests(0.1, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
$this->assertContains(
'Completed requested context: Forum: TForum1 (search area: mod_forum-activity)',
$out);
}
}

9
search/upgrade.txt

@ -1,6 +1,15 @@
This files describes API changes in /search/*,
information provided here is intended especially for developers.
=== 3.5 ===
* Search areas may now optionally implement the get_contexts_to_reindex function (for modules and
blocks, see also get_contexts_to_reindex_extra_sql). This allows a search area to customise the
order in which it is reindexed when doing a gradual reindex, so as to reindex the most important
contexts first. If not implemented, the default behaviour for modules and blocks is to reindex
the newest items first; for other types of search area it will just index the whole system
context, oldest data first.
=== 3.4 ===
* Search indexing now supports time limits to make the scheduled task run more neatly. In order for

8
theme/boost/scss/moodle/search.scss

@ -68,3 +68,11 @@
margin-right: $spacer;
display: inline-block;
}
#core-search-areas .lastcol li {
margin-left: 24px;
text-indent: -24px;
}
#core-search-areas .lastcol li > i {
text-indent: 0;
}

8
theme/bootstrapbase/less/moodle/search.less

@ -50,3 +50,11 @@
.search-areas-actions > div {
display: inline-block;
}
#core-search-areas .lastcol li {
margin-left: 24px;
text-indent: -24px;
}
#core-search-areas .lastcol li > i {
text-indent: 0;
}

7
theme/bootstrapbase/style/moodle.css

@ -10046,6 +10046,13 @@ body.path-question-type .mform fieldset.hidden {
.search-areas-actions > div {
display: inline-block;
}
#core-search-areas .lastcol li {
margin-left: 24px;
text-indent: -24px;
}
#core-search-areas .lastcol li > i {
text-indent: 0;
}
.popover-region {
float: right;
position: relative;

2
version.php

@ -29,7 +29,7 @@
defined('MOODLE_INTERNAL') || die();
$version = 2017122200.00; // YYYYMMDD = weekly release date of this DEV branch.
$version = 2017122200.01; // YYYYMMDD = weekly release date of this DEV branch.
// RR = release increments - 00 in DEV branches.
// .XX = incremental changes.