diff --git a/lib/classes/task/search_index_task.php b/lib/classes/task/search_index_task.php index 7b6f8011d6a..31e742c21d8 100644 --- a/lib/classes/task/search_index_task.php +++ b/lib/classes/task/search_index_task.php @@ -51,7 +51,22 @@ class search_index_task extends scheduled_task { } $globalsearch = \core_search\manager::instance(); - // Indexing database records for modules + rich documents of forum. - $globalsearch->index(false, get_config('core', 'searchindextime'), new \text_progress_trace()); + // Get total indexing time limit. + $timelimit = get_config('core', 'searchindextime'); + $start = time(); + + // Do normal indexing. + $globalsearch->index(false, $timelimit, new \text_progress_trace()); + + // Do requested indexing (if any) for the rest of the time. + if ($timelimit != 0) { + $now = time(); + $timelimit -= ($now - $start); + if ($timelimit <= 1) { + // There is hardly any time left, so don't try to do requests. + return; + } + } + $globalsearch->process_index_requests($timelimit, new \text_progress_trace()); } } diff --git a/search/classes/manager.php b/search/classes/manager.php index e8a43a84907..4e85cc7c771 100644 --- a/search/classes/manager.php +++ b/search/classes/manager.php @@ -1087,4 +1087,78 @@ class manager { 'timerequested' => time(), 'partialarea' => '', 'partialtime' => 0 ]; $DB->insert_record('search_index_requests', $newrecord); } + + /** + * Processes outstanding index requests. This will take the first item from the queue and + * process it, continuing until an optional time limit is reached. + * + * If there are no index requests, the function will do nothing. + * + * @param float $timelimit Time limit (0 = none) + * @param \progress_trace|null $progress Optional progress indicator + */ + public function process_index_requests($timelimit = 0.0, \progress_trace $progress = null) { + global $DB; + + if (!$progress) { + $progress = new \null_progress_trace(); + } + + $complete = false; + $before = microtime(true); + if ($timelimit) { + $stopat = $before + $timelimit; + } + while (true) { + // Retrieve first request, using fully defined ordering. + $requests = $DB->get_records('search_index_requests', null, + 'timerequested, contextid, searcharea', + 'id, contextid, searcharea, partialarea, partialtime', 0, 1); + if (!$requests) { + // If there are no more requests, stop. + $complete = true; + break; + } + $request = reset($requests); + + // Calculate remaining time. + $remainingtime = 0; + $beforeindex = microtime(true); + if ($timelimit) { + $remainingtime = $stopat - $beforeindex; + } + + // Show a message before each request, indicating what will be indexed. + $context = \context::instance_by_id($request->contextid); + $contextname = $context->get_context_name(); + if ($request->searcharea) { + $contextname .= ' (search area: ' . $request->searcharea . ')'; + } + $progress->output('Indexing requested context: ' . $contextname); + + // Actually index the context. + $result = $this->index_context($context, $request->searcharea, $remainingtime, + $progress, $request->partialarea, $request->partialtime); + + // Work out shared part of message. + $endmessage = $contextname . ' (' . round(microtime(true) - $beforeindex, 1) . 's)'; + + // Update database table and continue/stop as appropriate. + if ($result->complete) { + // If we completed the request, remove it from the table. + $DB->delete_records('search_index_requests', ['id' => $request->id]); + $progress->output('Completed requested context: ' . $endmessage); + } else { + // If we didn't complete the request, store the partial details (how far it got). + $DB->update_record('search_index_requests', ['id' => $request->id, + 'partialarea' => $result->startfromarea, + 'partialtime' => $result->startfromtime]); + $progress->output('Ending requested context: ' . $endmessage); + + // The time limit must have expired, so stop looping. + break; + } + } + } + } diff --git a/search/cli/indexer.php b/search/cli/indexer.php index 8e204acb7bc..7129cfcc1a5 100644 --- a/search/cli/indexer.php +++ b/search/cli/indexer.php @@ -82,11 +82,26 @@ if (empty($options['reindex'])) { $limitunderline = preg_replace('~.~', '=', $limitinfo); echo "Running index of site$limitinfo\n"; echo "=====================$limitunderline\n"; + $timelimit = (int)$options['timelimit']; } else { echo "Running full index of site\n"; echo "==========================\n"; + $timelimit = 0; } - $globalsearch->index(false, $options['timelimit'], new text_progress_trace()); + $before = time(); + $globalsearch->index(false, $timelimit, new text_progress_trace()); + + // Do specific index requests with the remaining time. + if ($timelimit) { + $timelimit -= (time() - $before); + // Only do index requests if there is a reasonable amount of time left. + if ($timelimit > 1) { + $globalsearch->process_index_requests($timelimit, new text_progress_trace()); + } + } else { + $globalsearch->process_index_requests(0, new text_progress_trace()); + } + } else { echo "Running full reindex of site\n"; echo "============================\n"; diff --git a/search/tests/manager_test.php b/search/tests/manager_test.php index 33fa8c53311..a655b940faa 100644 --- a/search/tests/manager_test.php +++ b/search/tests/manager_test.php @@ -738,4 +738,98 @@ class search_manager_testcase extends advanced_testcase { \core_search\manager::request_index($forum2ctx); $this->assertEquals(4, $DB->count_records('search_index_requests')); } + + /** + * Tests the process_index_requests function. + */ + public function test_process_index_requests() { + global $DB; + + $this->resetAfterTest(); + + $search = testable_core_search::instance(); + + // When there are no index requests, nothing gets logged. + $progress = new progress_trace_buffer(new text_progress_trace(), false); + $search->process_index_requests(0.0, $progress); + $out = $progress->get_buffer(); + $progress->reset_buffer(); + $this->assertEquals('', $out); + + // Set up the course with 3 forums. + $generator = $this->getDataGenerator(); + $course = $generator->create_course(['fullname' => 'TCourse']); + $forum1 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum1']); + $forum2 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum2']); + $forum3 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum3']); + + // Hack the forums so they have different creation times. + $now = time(); + $DB->set_field('forum', 'timemodified', $now - 3, ['id' => $forum1->id]); + $DB->set_field('forum', 'timemodified', $now - 2, ['id' => $forum2->id]); + $DB->set_field('forum', 'timemodified', $now - 1, ['id' => $forum3->id]); + $forum2time = $now - 2; + + // Make 2 index requests. + $search::request_index(context_course::instance($course->id), 'mod_label-activity'); + $this->waitForSecond(); + $search::request_index(context_module::instance($forum1->cmid)); + + // Run with no time limit. + $search->process_index_requests(0.0, $progress); + $out = $progress->get_buffer(); + $progress->reset_buffer(); + + // Check that it's done both areas. + $this->assertContains( + 'Indexing requested context: Course: TCourse (search area: mod_label-activity)', + $out); + $this->assertContains( + 'Completed requested context: Course: TCourse (search area: mod_label-activity)', + $out); + $this->assertContains('Indexing requested context: Forum: TForum1', $out); + $this->assertContains('Completed requested context: Forum: TForum1', $out); + + // Check the requests database table is now empty. + $this->assertEquals(0, $DB->count_records('search_index_requests')); + + // Request indexing the course a couple of times. + $search::request_index(context_course::instance($course->id), 'mod_forum-activity'); + $search::request_index(context_course::instance($course->id), 'mod_forum-post'); + + // Do the processing again with a time limit and indexing delay. The time limit is too + // small; because of the way the logic works, this means it will index 2 activities. + $search->get_engine()->set_add_delay(0.2); + $search->process_index_requests(0.1, $progress); + $out = $progress->get_buffer(); + $progress->reset_buffer(); + + // Confirm the right wrapper information was logged. + $this->assertContains( + 'Indexing requested context: Course: TCourse (search area: mod_forum-activity)', + $out); + $this->assertContains('Stopping indexing due to time limit', $out); + $this->assertContains( + 'Ending requested context: Course: TCourse (search area: mod_forum-activity)', + $out); + + // Check the database table has been updated with progress. + $records = array_values($DB->get_records('search_index_requests', null, 'searcharea')); + $this->assertEquals('mod_forum-activity', $records[0]->partialarea); + $this->assertEquals($forum2time, $records[0]->partialtime); + + // Run again and confirm it now finishes. + $search->process_index_requests(0.1, $progress); + $out = $progress->get_buffer(); + $progress->reset_buffer(); + $this->assertContains( + 'Completed requested context: Course: TCourse (search area: mod_forum-activity)', + $out); + $this->assertContains( + 'Completed requested context: Course: TCourse (search area: mod_forum-post)', + $out); + + // Confirm table is now empty. + $this->assertEquals(0, $DB->count_records('search_index_requests')); + } }