MDL-60981 core_search: UI to gradually reindex a single area

Adds a new 'Gradual reindex' link to the search areas page for each
area. When clicked, this takes you to a confirm prompt, and then
adds each context from that search area to the indexing queue.

The search areas page now displays the 'Additional indexing queue'
(if it is non-empty). The table shows the first 10 items in the
queue, and it also indicates the total number in case there are
more. (I don't think people really need to see the entire
contents of it, so I didn't implement paging.)

admin/searchareas.php

@@ -132,7 +132,17 @@ foreach ($searchareas as $area) {
                 $laststatus = '';
             }
             $columns[] = $laststatus;
-            $columns[] = html_writer::link(admin_searcharea_action_url('delete', $areaid), 'Delete index');
+            $accesshide = html_writer::span($area->get_visible_name(), 'accesshide');
+            $actions = [];
+            $actions[] = $OUTPUT->pix_icon('t/delete', '') .
+                    html_writer::link(admin_searcharea_action_url('delete', $areaid),
+                    get_string('deleteindex', 'search', $accesshide));
+            if ($area->supports_get_document_recordset()) {
+                $actions[] = $OUTPUT->pix_icon('i/reload', '') . html_writer::link(
+                        new moodle_url('searchreindex.php', ['areaid' => $areaid]),
+                        get_string('gradualreindex', 'search', $accesshide));
+            }
+            $columns[] = html_writer::alist($actions, ['class' => 'unstyled list-unstyled']);
 
         } else {
             $blankrow = new html_table_cell(get_string('searchnotavailable', 'admin'));
@@ -165,6 +175,13 @@ echo $OUTPUT->single_button(admin_searcharea_action_url('deleteall'), get_string
 echo $OUTPUT->box_end();
 
 echo html_writer::table($table);
+
+if (empty($searchmanagererror)) {
+    // Show information about queued index requests for specific contexts.
+    $searchrenderer = $PAGE->get_renderer('core_search');
+    echo $searchrenderer->render_index_requests_info($searchmanager->get_index_requests_info());
+}
+
 echo $OUTPUT->footer();
 
 /**

admin/searchreindex.php

@@ -0,0 +1,87 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Adds a search area to the queue for indexing.
+ *
+ * @package core_search
+ * @copyright 2017 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+define('NO_OUTPUT_BUFFERING', true);
+
+require(__DIR__ . '/../config.php');
+
+// Check access.
+require_once($CFG->libdir . '/adminlib.php');
+
+admin_externalpage_setup('searchareas', '', null, (new moodle_url('/admin/searchreindex.php'))->out(false));
+
+// Get area parameter and check it exists.
+$areaid = required_param('areaid', PARAM_ALPHAEXT);
+$area = \core_search\manager::get_search_area($areaid);
+if ($area === false) {
+    throw new moodle_exception('invalidrequest');
+}
+$areaname = $area->get_visible_name();
+
+// Start page output.
+$heading = get_string('gradualreindex', 'search', '');
+$PAGE->set_title($PAGE->title . ': ' . $heading);
+$PAGE->navbar->add($heading);
+echo $OUTPUT->header();
+echo $OUTPUT->heading($heading);
+
+// If sesskey is supplied, actually carry out the action.
+if (optional_param('sesskey', '', PARAM_ALPHANUM)) {
+    require_sesskey();
+
+    // Get all contexts for search area. This query can take time in large cases.
+    \core_php_time_limit::raise(0);
+    $contextiterator = $area->get_contexts_to_reindex();
+
+    $progress = new \core\progress\display_if_slow('');
+    $progress->start_progress($areaname);
+
+    // Request reindexing for each context (with low priority).
+    $count = 0;
+    foreach ($contextiterator as $context) {
+        \core_php_time_limit::raise(30);
+        \core_search\manager::request_index($context, $area->get_area_id(),
+                \core_search\manager::INDEX_PRIORITY_REINDEXING);
+        $progress->progress();
+        $count++;
+    }
+
+    // Unset the iterator which should close the recordset (if there is one).
+    unset($contextiterator);
+
+    $progress->end_progress();
+
+    $a = (object)['name' => html_writer::tag('strong', $areaname), 'count' => $count];
+    echo $OUTPUT->box(get_string('gradualreindex_queued', 'search', $a));
+
+    echo $OUTPUT->continue_button(new moodle_url('/admin/searchareas.php'));
+} else {
+    // Display confirmation prompt.
+    echo $OUTPUT->confirm(get_string('gradualreindex_confirm', 'search', html_writer::tag('strong', $areaname)),
+            new single_button(new moodle_url('/admin/searchreindex.php', ['areaid' => $areaid,
+                'sesskey' => sesskey()]), get_string('continue'), 'post', true),
+            new single_button(new moodle_url('/admin/searchareas.php'), get_string('cancel'), 'get'));
+}
+
+echo $OUTPUT->footer();

lang/en/search.php

@@ -36,6 +36,7 @@ $string['createdon'] = 'Created on';
 $string['database'] = 'Database';
 $string['databasestate'] = 'Indexing database state';
 $string['datadirectory'] = 'Data directory';
+$string['deleteindex'] = 'Delete index {$a}';
 $string['deletionsinindex'] = 'Deletions in index';
 $string['docmodifiedon'] = 'Last modified on {$a}';
 $string['doctype'] = 'Doctype';
@@ -59,6 +60,9 @@ $string['filterheader'] = 'Filter';
 $string['fromtime'] = 'Modified after';
 $string['globalsearch'] = 'Global search';
 $string['globalsearchdisabled'] = 'Global searching is not enabled.';
+$string['gradualreindex'] = 'Gradual reindex {$a}';
+$string['gradualreindex_confirm'] = 'Are you sure you want to reindex {$a}? This may take some time, although existing data will remain available during the reindex.';
+$string['gradualreindex_queued'] = 'Reindexing has been requested for {$a->name} ({$a->count} contexts). This indexing will be carried out by the ‘Global search indexing’ scheduled task.';
 $string['checkdb'] = 'Check database';
 $string['checkdbadvice'] = 'Check your database for any problems.';
 $string['checkdir'] = 'Check dir';
@@ -76,7 +80,12 @@ $string['notitle'] = 'No title';
 $string['normalsearch'] = 'Normal search';
 $string['openedon'] = 'opened on';
 $string['optimize'] = 'Optimize';
+$string['priority'] = 'Priority';
+$string['priority_reindexing'] = 'Reindexing';
+$string['priority_normal'] = 'Normal';
+$string['progress'] = 'Progress';
 $string['queryerror'] = 'The query you provided could not be parsed by the search engine: {$a}';
+$string['queueheading'] = 'Additional indexing queue ({$a} items)';
 $string['resultsreturnedfor'] = 'results returned for';
 $string['runindexer'] = 'Run indexer (real)';
 $string['runindexertest'] = 'Run indexer test';

search/classes/base.php

@@ -302,6 +302,19 @@ abstract class base {
         return false;
     }
 
+    /**
+     * Checks if get_document_recordset is supported for this search area.
+     *
+     * For many uses you can simply call get_document_recordset and see if it returns false, but
+     * this function is useful when you don't want to actually call the function right away.
+     */
+    public function supports_get_document_recordset() {
+        // Easiest way to check this is simply to see if the class has overridden the default
+        // function.
+        $method = new \ReflectionMethod($this, 'get_document_recordset');
+        return $method->getDeclaringClass()->getName() !== self::class;
+    }
+
     /**
      * Returns the document related with the provided record.
      *

search/classes/manager.php

@@ -1153,7 +1153,8 @@ class manager {
      * added to a queue which is processed by the task.
      *
      * This is used after a restore to ensure that restored items are indexed, even though their
-     * modified time will be older than the latest indexed.
+     * modified time will be older than the latest indexed. It is also used by the 'Gradual reindex'
+     * admin feature from the search areas screen.
      *
      * @param \context $context Context to index within
      * @param string $areaid Area to index, '' = all areas
@@ -1275,6 +1276,52 @@ class manager {
         }
     }
 
+    /**
+     * Gets information about the request queue, in the form of a plain object suitable for passing
+     * to a template for rendering.
+     *
+     * @return \stdClass Information about queued index requests
+     */
+    public function get_index_requests_info() {
+        global $DB;
+
+        $result = new \stdClass();
+
+        $result->total = $DB->count_records('search_index_requests');
+        $result->topten = $DB->get_records('search_index_requests', null,
+                'indexpriority DESC, timerequested, contextid, searcharea',
+                'id, contextid, timerequested, searcharea, partialarea, partialtime, indexpriority',
+                0, 10);
+        foreach ($result->topten as $item) {
+            $context = \context::instance_by_id($item->contextid);
+            $item->contextlink = \html_writer::link($context->get_url(),
+                    s($context->get_context_name()));
+            if ($item->searcharea) {
+                $item->areaname = $this->get_search_area($item->searcharea)->get_visible_name();
+            }
+            if ($item->partialarea) {
+                $item->partialareaname = $this->get_search_area($item->partialarea)->get_visible_name();
+            }
+            switch ($item->indexpriority) {
+                case self::INDEX_PRIORITY_REINDEXING :
+                    $item->priorityname = get_string('priority_reindexing', 'search');
+                    break;
+                case self::INDEX_PRIORITY_NORMAL :
+                    $item->priorityname = get_string('priority_normal', 'search');
+                    break;
+            }
+        }
+
+        // Normalise array indices.
+        $result->topten = array_values($result->topten);
+
+        if ($result->total > 10) {
+            $result->ellipsis = true;
+        }
+
+        return $result;
+    }
+
     /**
      * Gets current time for use in search system.
      *

search/classes/output/renderer.php

@@ -103,4 +103,15 @@ class renderer extends \plugin_renderer_base {
         $content .= $this->output->box_end();
         return $content;
     }
+
+    /**
+     * Returns information about queued index requests.
+     *
+     * @param \stdClass $info Info object from get_index_requests_info
+     * @return string HTML
+     * @throws \moodle_exception Any error with template
+     */
+    public function render_index_requests_info(\stdClass $info) {
+        return $this->output->render_from_template('core_search/index_requests', $info);
+    }
 }

search/templates/index_requests.mustache

@@ -0,0 +1,110 @@
+{{!
+    This file is part of Moodle - http://moodle.org/
+
+    Moodle is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Moodle is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+}}
+{{!
+    @template core_search/index_requests
+
+    Template to provide admin information about the queue of index requests.
+
+    Classes required for JS:
+    * none
+
+    Data attributes required for JS:
+    * none
+
+    Context variables required for this template:
+    * topten
+    * count
+
+    Optional context variables for this template:
+    * ellipsis
+
+    Example context (json):
+    {
+        "topten":
+        [
+            {
+                "id": 42,
+                "timerequested": 123456789,
+                "contextid": 123,
+                "contextlink": "<a href='...'>Forum: Tutor group forum</a>",
+                "searcharea": "mod_forum-activity",
+                "areaname": "Forum activities",
+                "partialarea": "mod_forum-activity",
+                "partialareaname": "Forum activities",
+                "partialtime": 123400000,
+                "indexpriority": 100
+            }
+        ],
+        "total": 1,
+        "ellipsis": true
+    }
+}}
+{{#total}}
+    <div>
+        <h3>
+            {{#str}} queueheading, search, {{total}} {{/str}}
+        </h3>
+        <table class="generaltable">
+            <thead>
+            <tr>
+                <th scope="col">{{#str}} context, role {{/str}}</th>
+                <th scope="col">{{#str}} searcharea, search {{/str}}</th>
+                <th scope="col">{{#str}} time {{/str}}</th>
+                <th scope="col">{{#str}} progress, search {{/str}}</th>
+                <th scope="col">{{#str}} priority, search {{/str}}</th>
+            </tr>
+            </thead>
+            <tbody>
+
+
+            {{#topten}}
+                <tr>
+                    <td>
+                        {{{contextlink}}}
+                    </td>
+                    <td>
+                        {{#searcharea}} {{areaname}} {{/searcharea}}
+                    </td>
+                    <td>{{#userdate}} {{timerequested}}, {{#str}} strftimedatetimeshort {{/str}} {{/userdate}}</td>
+                    <td>
+                        {{#partialarea}}
+                            {{partialareaname}}:
+                        {{/partialarea}}
+                        {{#partialtime}}
+                            {{#userdate}} {{partialtime}}, {{#str}} strftimedatetimeshort {{/str}} {{/userdate}}
+                        {{/partialtime}}
+                    </td>
+                    <td>
+                        {{#priorityname}}
+                            {{priorityname}}
+                        {{/priorityname}}
+                        {{^priorityname}}
+                            {{indexpriority}}
+                        {{/priorityname}}
+                    </td>
+                </tr>
+            {{/topten}}
+
+            {{#ellipsis}}
+                <tr>
+                    <td colspan="5">...</td>
+                </tr>
+            {{/ellipsis}}
+            </tbody>
+        </table>
+    </div>
+{{/total}}

search/upgrade.txt

@@ -6,7 +6,9 @@ information provided here is intended especially for developers.
 * Search areas may now optionally implement the get_contexts_to_reindex function (for modules and
   blocks, see also get_contexts_to_reindex_extra_sql). This allows a search area to customise the
   order in which it is reindexed when doing a gradual reindex, so as to reindex the most important
-  contexts first.
+  contexts first. If not implemented, the default behaviour for modules and blocks is to reindex
+  the newest items first; for other types of search area it will just index the whole system
+  context, oldest data first.
 
 === 3.4 ===
 

theme/boost/scss/moodle/search.scss

@@ -68,3 +68,11 @@
     margin-right: $spacer;
     display: inline-block;
 }
+
+#core-search-areas .lastcol li {
+    margin-left: 24px;
+    text-indent: -24px;
+}
+#core-search-areas .lastcol li > i {
+    text-indent: 0;
+}

theme/bootstrapbase/less/moodle/search.less

@@ -50,3 +50,11 @@
 .search-areas-actions > div {
     display: inline-block;
 }
+
+#core-search-areas .lastcol li {
+    margin-left: 24px;
+    text-indent: -24px;
+}
+#core-search-areas .lastcol li > i {
+    text-indent: 0;
+}

theme/bootstrapbase/style/moodle.css

@@ -10046,6 +10046,13 @@ body.path-question-type .mform fieldset.hidden {
 .search-areas-actions > div {
   display: inline-block;
 }
+#core-search-areas .lastcol li {
+  margin-left: 24px;
+  text-indent: -24px;
+}
+#core-search-areas .lastcol li > i {
+  text-indent: 0;
+}
 .popover-region {
   float: right;
   position: relative;