mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-04-13 12:32:08 +02:00
Code Issues Projects Releases Wiki Activity

MDL-55356 core_search: Make indexing task/CLI do context requests

Browse Source
This commit is contained in:
sam marshall 2017-09-15 14:08:09 +01:00
parent 4ba11aa907
commit 74b7a42d0d
4 changed files with 201 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lib/classes/task/search_index_task.php
View File

@ -51,7 +51,22 @@ class search_index_task extends scheduled_task {
}
$globalsearch = \core_search\manager::instance();
// Indexing database records for modules + rich documents of forum.
$globalsearch->index(false, get_config('core', 'searchindextime'), new \text_progress_trace());
// Get total indexing time limit.
$timelimit = get_config('core', 'searchindextime');
$start = time();
// Do normal indexing.
$globalsearch->index(false, $timelimit, new \text_progress_trace());
// Do requested indexing (if any) for the rest of the time.
if ($timelimit != 0) {
$now = time();
$timelimit -= ($now - $start);
if ($timelimit <= 1) {
// There is hardly any time left, so don't try to do requests.
return;
}
}
$globalsearch->process_index_requests($timelimit, new \text_progress_trace());
}
}

74
search/classes/manager.php
View File

@ -1087,4 +1087,78 @@ class manager {
'timerequested' => time(), 'partialarea' => '', 'partialtime' => 0 ];
$DB->insert_record('search_index_requests', $newrecord);
}
/**
* Processes outstanding index requests. This will take the first item from the queue and
* process it, continuing until an optional time limit is reached.
*
* If there are no index requests, the function will do nothing.
*
* @param float $timelimit Time limit (0 = none)
* @param \progress_trace|null $progress Optional progress indicator
*/
public function process_index_requests($timelimit = 0.0, \progress_trace $progress = null) {
global $DB;
if (!$progress) {
$progress = new \null_progress_trace();
}
$complete = false;
$before = microtime(true);
if ($timelimit) {
$stopat = $before + $timelimit;
}
while (true) {
// Retrieve first request, using fully defined ordering.
$requests = $DB->get_records('search_index_requests', null,
'timerequested, contextid, searcharea',
'id, contextid, searcharea, partialarea, partialtime', 0, 1);
if (!$requests) {
// If there are no more requests, stop.
$complete = true;
break;
}
$request = reset($requests);
// Calculate remaining time.
$remainingtime = 0;
$beforeindex = microtime(true);
if ($timelimit) {
$remainingtime = $stopat - $beforeindex;
}
// Show a message before each request, indicating what will be indexed.
$context = \context::instance_by_id($request->contextid);
$contextname = $context->get_context_name();
if ($request->searcharea) {
$contextname .= ' (search area: ' . $request->searcharea . ')';
}
$progress->output('Indexing requested context: ' . $contextname);
// Actually index the context.
$result = $this->index_context($context, $request->searcharea, $remainingtime,
$progress, $request->partialarea, $request->partialtime);
// Work out shared part of message.
$endmessage = $contextname . ' (' . round(microtime(true) - $beforeindex, 1) . 's)';
// Update database table and continue/stop as appropriate.
if ($result->complete) {
// If we completed the request, remove it from the table.
$DB->delete_records('search_index_requests', ['id' => $request->id]);
$progress->output('Completed requested context: ' . $endmessage);
} else {
// If we didn't complete the request, store the partial details (how far it got).
$DB->update_record('search_index_requests', ['id' => $request->id,
'partialarea' => $result->startfromarea,
'partialtime' => $result->startfromtime]);
$progress->output('Ending requested context: ' . $endmessage);
// The time limit must have expired, so stop looping.
break;
}
}
}
}

17
search/cli/indexer.php
View File

@ -82,11 +82,26 @@ if (empty($options['reindex'])) {
$limitunderline = preg_replace('~.~', '=', $limitinfo);
echo "Running index of site$limitinfo\n";
echo "=====================$limitunderline\n";
$timelimit = (int)$options['timelimit'];
} else {
echo "Running full index of site\n";
echo "==========================\n";
$timelimit = 0;
}
$globalsearch->index(false, $options['timelimit'], new text_progress_trace());
$before = time();
$globalsearch->index(false, $timelimit, new text_progress_trace());
// Do specific index requests with the remaining time.
if ($timelimit) {
$timelimit -= (time() - $before);
// Only do index requests if there is a reasonable amount of time left.
if ($timelimit > 1) {
$globalsearch->process_index_requests($timelimit, new text_progress_trace());
}
} else {
$globalsearch->process_index_requests(0, new text_progress_trace());
}
} else {
echo "Running full reindex of site\n";
echo "============================\n";

94
search/tests/manager_test.php
View File

@ -738,4 +738,98 @@ class search_manager_testcase extends advanced_testcase {
\core_search\manager::request_index($forum2ctx);
$this->assertEquals(4, $DB->count_records('search_index_requests'));
}
/**
* Tests the process_index_requests function.
*/
public function test_process_index_requests() {
global $DB;
$this->resetAfterTest();
$search = testable_core_search::instance();
// When there are no index requests, nothing gets logged.
$progress = new progress_trace_buffer(new text_progress_trace(), false);
$search->process_index_requests(0.0, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
$this->assertEquals('', $out);
// Set up the course with 3 forums.
$generator = $this->getDataGenerator();
$course = $generator->create_course(['fullname' => 'TCourse']);
$forum1 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum1']);
$forum2 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum2']);
$forum3 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum3']);
// Hack the forums so they have different creation times.
$now = time();
$DB->set_field('forum', 'timemodified', $now - 3, ['id' => $forum1->id]);
$DB->set_field('forum', 'timemodified', $now - 2, ['id' => $forum2->id]);
$DB->set_field('forum', 'timemodified', $now - 1, ['id' => $forum3->id]);
$forum2time = $now - 2;
// Make 2 index requests.
$search::request_index(context_course::instance($course->id), 'mod_label-activity');
$this->waitForSecond();
$search::request_index(context_module::instance($forum1->cmid));
// Run with no time limit.
$search->process_index_requests(0.0, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
// Check that it's done both areas.
$this->assertContains(
'Indexing requested context: Course: TCourse (search area: mod_label-activity)',
$out);
$this->assertContains(
'Completed requested context: Course: TCourse (search area: mod_label-activity)',
$out);
$this->assertContains('Indexing requested context: Forum: TForum1', $out);
$this->assertContains('Completed requested context: Forum: TForum1', $out);
// Check the requests database table is now empty.
$this->assertEquals(0, $DB->count_records('search_index_requests'));
// Request indexing the course a couple of times.
$search::request_index(context_course::instance($course->id), 'mod_forum-activity');
$search::request_index(context_course::instance($course->id), 'mod_forum-post');
// Do the processing again with a time limit and indexing delay. The time limit is too
// small; because of the way the logic works, this means it will index 2 activities.
$search->get_engine()->set_add_delay(0.2);
$search->process_index_requests(0.1, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
// Confirm the right wrapper information was logged.
$this->assertContains(
'Indexing requested context: Course: TCourse (search area: mod_forum-activity)',
$out);
$this->assertContains('Stopping indexing due to time limit', $out);
$this->assertContains(
'Ending requested context: Course: TCourse (search area: mod_forum-activity)',
$out);
// Check the database table has been updated with progress.
$records = array_values($DB->get_records('search_index_requests', null, 'searcharea'));
$this->assertEquals('mod_forum-activity', $records[0]->partialarea);
$this->assertEquals($forum2time, $records[0]->partialtime);
// Run again and confirm it now finishes.
$search->process_index_requests(0.1, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
$this->assertContains(
'Completed requested context: Course: TCourse (search area: mod_forum-activity)',
$out);
$this->assertContains(
'Completed requested context: Course: TCourse (search area: mod_forum-post)',
$out);
// Confirm table is now empty.
$this->assertEquals(0, $DB->count_records('search_index_requests'));
}
}