MDL-47271 tool_task: Added Task API status checks

admin/index.php

@@ -843,8 +843,9 @@ $errorsdisplayed = defined('WARN_DISPLAY_ERRORS_ENABLED');
 $lastcron = get_config('tool_task', 'lastcronstart');
 $cronoverdue = ($lastcron < time() - 3600 * 24);
 $lastcroninterval = get_config('tool_task', 'lastcroninterval');
-$expectedfrequency = $CFG->expectedcronfrequency ?? 200;
-$croninfrequent = !$cronoverdue && ($lastcroninterval > $expectedfrequency || $lastcron < time() - $expectedfrequency);
+
+$expectedfrequency = $CFG->expectedcronfrequency ?? MINSECS;
+$croninfrequent = !$cronoverdue && ($lastcroninterval > ($expectedfrequency + MINSECS) || $lastcron < time() - $expectedfrequency);
 $dbproblems = $DB->diagnose();
 $maintenancemode = !empty($CFG->maintenance_enabled);
 

admin/renderer.php

@@ -601,19 +601,9 @@ class core_admin_renderer extends plugin_renderer_base {
             return '';
         }
 
-        if (empty($CFG->cronclionly)) {
-            $url = new moodle_url('/admin/cron.php');
-            if (!empty($CFG->cronremotepassword)) {
-                $url = new moodle_url('/admin/cron.php', array('password' => $CFG->cronremotepassword));
-            }
-
-            return $this->warning(get_string('cronwarning', 'admin', $url->out()) . ' ' .
-                    $this->help_icon('cron', 'admin'));
-        }
-
-        // $CFG->cronclionly is not empty: cron can run only from CLI.
-        return $this->warning(get_string('cronwarningcli', 'admin') . ' ' .
-                $this->help_icon('cron', 'admin'));
+        $check = new \tool_task\check\cronrunning();
+        $result = $check->get_result();
+        return $this->warning($result->get_summary() . ' ' . $this->help_icon('cron', 'admin'));
     }
 
     /**
@@ -629,9 +619,9 @@ class core_admin_renderer extends plugin_renderer_base {
             return '';
         }
 
-        $expectedfrequency = $CFG->expectedcronfrequency ?? 200;
-        return $this->warning(get_string('croninfrequent', 'admin', $expectedfrequency) . ' ' .
-                $this->help_icon('cron', 'admin'));
+        $check = new \tool_task\check\cronrunning();
+        $result = $check->get_result();
+        return $this->warning($result->get_summary() . ' ' . $this->help_icon('cron', 'admin'));
     }
 
     /**

admin/tool/task/classes/check/adhocqueue.php

@@ -0,0 +1,93 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Ad hoc queue checks
+ *
+ * @package    tool_task
+ * @copyright  2020 Brendan Heywood (brendan@catalyst-au.net)
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace tool_task\check;
+
+defined('MOODLE_INTERNAL') || die();
+
+use core\check\check;
+use core\check\result;
+
+/**
+ * Ad hoc queue checks
+ *
+ * @package    tool_task
+ * @copyright  2020 Brendan Heywood (brendan@catalyst-au.net)
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class adhocqueue extends check {
+
+    /**
+     * Constructor
+     */
+    public function __construct() {
+        global $CFG;
+        $this->id = 'adhocqueue';
+        $this->name = get_string('checkadhocqueue', 'tool_task');
+    }
+
+    /**
+     * Return result
+     * @return result
+     */
+    public function get_result() : result {
+        global $DB, $CFG;
+
+        $stats = $DB->get_record_sql('
+            SELECT count(*) cnt,
+                   MAX(? - nextruntime) age
+              FROM {task_adhoc}', [time()]);
+
+        $status = result::OK;
+        $summary = get_string('adhocempty', 'tool_task');
+        $details = '';
+
+        if ($stats->cnt > 0) {
+            // A large queue size by itself is not an issue, only when tasks
+            // are not being processed in a timely fashion is it an issue.
+            $status = result::INFO;
+            $summary = get_string('adhocqueuesize', 'tool_task', $stats->cnt);
+        }
+
+        $max = $CFG->adhoctaskagewarn ?? 10 * MINSECS;
+        if ($stats->age > $max) {
+            $status = result::WARNING;
+            $summary = get_string('adhocqueueold', 'tool_task', [
+                'age' => format_time($stats->age),
+                'max' => format_time($max),
+            ]);
+        }
+
+        $max = $CFG->adhoctaskageerror ?? 4 * HOURSECS;
+        if ($stats->age > $max) {
+            $status = result::ERROR;
+            $summary = get_string('adhocqueueold', 'tool_task', [
+                'age' => format_time($stats->age),
+                'max' => format_time($max),
+            ]);
+        }
+
+        return new result($status, $summary, $details);
+    }
+}

admin/tool/task/classes/check/cronrunning.php

@@ -0,0 +1,118 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Cron running check
+ *
+ * @package    tool_task
+ * @copyright  2020 Brendan Heywood (brendan@catalyst-au.net)
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace tool_task\check;
+
+defined('MOODLE_INTERNAL') || die();
+
+use core\check\check;
+use core\check\result;
+/**
+ * Cron running check
+ *
+ * @package    tool_task
+ * @copyright  2020 Brendan Heywood (brendan@catalyst-au.net)
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class cronrunning extends check {
+
+    /**
+     * Constructor
+     */
+    public function __construct() {
+        global $CFG;
+        $this->id = 'cronrunning';
+        $this->name = get_string('checkcronrunning', 'tool_task');
+        if (empty($CFG->cronclionly)) {
+            $this->actionlink = new \action_link(
+                new \moodle_url('/admin/cron.php'),
+                get_string('cron', 'admin'));
+        }
+    }
+
+    /**
+     * Return result
+     * @return result
+     */
+    public function get_result() : result {
+        global $CFG;
+
+        // Eventually this should replace cron_overdue_warning and
+        // cron_infrequent_warning.
+        $lastcron = get_config('tool_task', 'lastcronstart');
+        $expectedfrequency = $CFG->expectedcronfrequency ?? MINSECS;
+
+        $delta = time() - $lastcron;
+
+        $lastcroninterval = get_config('tool_task', 'lastcroninterval');
+
+        $formatdelta    = format_time($delta);
+        $formatexpected = format_time($expectedfrequency);
+        $formatinterval = format_time($lastcroninterval);
+
+        $details = format_time($delta);
+
+        if ($delta > $expectedfrequency + MINSECS) {
+            $status = result::WARNING;
+
+            if ($delta > DAYSECS) {
+                $status = result::CRITICAL;
+            }
+
+            if (empty($lastcron)) {
+                $summary = get_string('cronwarningnever', 'admin', [
+                    'expected' => $formatexpected,
+                ]);
+            } else if (empty($CFG->cronclionly)) {
+                $url = new \moodle_url('/admin/cron.php');
+                $summary = get_string('cronwarning', 'admin', [
+                    'url' => $url->out(),
+                    'actual'   => $formatdelta,
+                    'expected' => $formatexpected,
+                ]);
+            } else {
+                $summary = get_string('cronwarningcli', 'admin', [
+                    'actual'   => $formatdelta,
+                    'expected' => $formatexpected,
+                ]);
+            }
+            return new result($status, $summary, $details);
+        }
+
+        if ($lastcroninterval > $expectedfrequency) {
+            $status = result::WARNING;
+            $summary = get_string('croninfrequent', 'admin', [
+                'actual'   => $formatinterval,
+                'expected' => $formatexpected,
+            ]);
+            return new result($status, $summary, $details);
+        }
+
+        $status = result::OK;
+        $summary = get_string('cronok', 'tool_task');
+
+        return new result($status, $summary, $details);
+    }
+}
+

admin/tool/task/classes/check/maxfaildelay.php

@@ -0,0 +1,96 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Task fail delay check
+ *
+ * @package    tool_task
+ * @copyright  2020 Brendan Heywood (brendan@catalyst-au.net)
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace tool_task\check;
+
+defined('MOODLE_INTERNAL') || die();
+
+use core\check\check;
+use core\check\result;
+
+/**
+ * Task fail delay check
+ *
+ * @package    tool_task
+ * @copyright  2020 Brendan Heywood (brendan@catalyst-au.net)
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class maxfaildelay extends check {
+
+    /**
+     * Constructor
+     */
+    public function __construct() {
+        global $CFG;
+        $this->id = 'cronfaildelay';
+        $this->name = get_string('checkmaxfaildelay', 'tool_task');
+        $this->actionlink = new \action_link(
+            new \moodle_url('/admin/tool/task/scheduledtasks.php'),
+            get_string('scheduledtasks', 'tool_task'));
+    }
+
+    /**
+     * Return result
+     * @return result
+     */
+    public function get_result() : result {
+        global $CFG;
+
+        $status = result::OK;
+        $summary = get_string('tasknofailures', 'tool_task');
+        $details = '';
+        $failures = 0;
+        $maxdelay = 0;
+
+        $tasks = \core\task\manager::get_all_scheduled_tasks();
+        foreach ($tasks as $task) {
+            if ($task->get_disabled()) {
+                continue;
+            }
+            $faildelay = $task->get_fail_delay();
+            if ($faildelay > $maxdelay) {
+                $maxdelay = $faildelay;
+            }
+            if ($faildelay > 0) {
+                $failures++;
+                $details .= get_string('faildelay', 'tool_task') . ': ' . format_time($faildelay);
+                $details .= ' - ' . $task->get_name() . ' (' .get_class($task) . ")<br>";
+            }
+        }
+
+        if ($failures > 0) {
+            // Intermittent failures are not yet a warning.
+            $status = result::INFO;
+            $summary = get_string('taskfailures', 'tool_task', $failures);
+        }
+        if ($maxdelay > 5 * MINSECS) {
+            $status = result::WARNING;
+        }
+        if ($maxdelay > 4 * HOURSECS) {
+            $status = result::ERROR;
+        }
+
+        return new result($status, $summary, $details);
+    }
+}

admin/tool/task/lang/en/tool_task.php

@@ -23,12 +23,19 @@
  */
 
 $string['asap'] = 'ASAP';
+$string['adhocempty'] = 'Adhoc task queue is empty';
+$string['adhocqueuesize'] = 'Adhoc task queue has {$a} tasks';
+$string['adhocqueueold'] = 'Oldest task is {$a->age} which is more than {$a->max}';
 $string['backtoscheduledtasks'] = 'Back to scheduled tasks';
 $string['blocking'] = 'Blocking';
 $string['cannotfindthepathtothecli'] = 'Cannot find the path to the PHP CLI executable so task execution aborted. Set the \'Path to PHP CLI\' setting in Site administration / Server / System paths.';
+$string['checkadhocqueue'] = 'Adhoc task queue';
+$string['checkcronrunning'] = 'Cron running';
+$string['checkmaxfaildelay'] = 'Tasks max fail delay';
 $string['clearfaildelay_confirm'] = 'Are you sure you want to clear the fail delay for task \'{$a}\'? After clearing the delay, the task will run according to its normal schedule.';
 $string['component'] = 'Component';
 $string['corecomponent'] = 'Core';
+$string['cronok'] = 'Cron is running frequently';
 $string['default'] = 'Default';
 $string['defaultx'] = 'Default: {$a}';
 $string['disabled'] = 'Disabled';
@@ -50,7 +57,9 @@ $string['runpattern'] = 'Run pattern';
 $string['scheduledtasks'] = 'Scheduled tasks';
 $string['scheduledtaskchangesdisabled'] = 'Modifications to the list of scheduled tasks have been prevented in Moodle configuration';
 $string['taskdisabled'] = 'Task disabled';
+$string['taskfailures'] = 'There are {$a} task(s) failing';
 $string['tasklogs'] = 'Task logs';
+$string['tasknofailures'] = 'There are no tasks failing';
 $string['taskscheduleday'] = 'Day';
 $string['taskscheduleday_help'] = 'Day of month field for task schedule. The field uses the same format as unix cron. Some examples are:
 

admin/tool/task/lib.php

@@ -0,0 +1,39 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Task API status checks
+ *
+ * @package    tool_task
+ * @copyright  2020 Brendan Heywood (brendan@catalyst-au.net)
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Add cron related service status checks
+ *
+ * @return array of check objects
+ */
+function tool_task_status_checks() : array {
+    return [
+        new \tool_task\check\cronrunning(),
+        new \tool_task\check\maxfaildelay(),
+        new \tool_task\check\adhocqueue(),
+    ];
+}
+

config-dist.php

@@ -616,6 +616,12 @@ $CFG->admin = 'admin';
 //
 //      $CFG->expectedcronfrequency = 200;
 //
+// Moodle 3.9+ checks how old tasks are in the ad hoc queue and warns at 10 minutes
+// and errors at 4 hours. Set these to override these limits:
+//
+//      $CFG->adhoctaskagewarn = 10 * 60;
+//      $CFG->adhoctaskageerror = 4 * 60 * 60;
+//
 // Session lock warning threshold. Long running pages should release the session using \core\session\manager::write_close().
 // Set this threshold to any value greater than 0 to add developer warnings when a page locks the session for too long.
 // The session should rarely be locked for more than 1 second. The input should be in seconds and may be a float.

lang/en/admin.php

@@ -429,10 +429,11 @@ $string['cron_link'] = 'admin/cron';
 $string['cronclionly'] = 'Cron execution via command line only';
 $string['cronerrorclionly'] = 'Sorry, internet access to this page has been disabled by the administrator.';
 $string['cronerrorpassword'] = 'Sorry, you have not provided a valid password to access this page';
-$string['croninfrequent'] = 'The time between the last two runs of the cron maintenance script was over {$a} seconds. We recommend configuring it to run more frequently.';
+$string['croninfrequent'] = 'There was {$a->actual} between the last two runs of the cron maintenance script and it should run every {$a->expected}. We recommend configuring it to run more frequently.';
 $string['cronremotepassword'] = 'Cron password for remote access';
-$string['cronwarning'] = 'The <a href="{$a}">cron.php maintenance script</a> has not been run for at least 24 hours.';
-$string['cronwarningcli'] = 'The cli/cron.php maintenance script has not been run for at least 24 hours.';
+$string['cronwarning'] = 'The <a href="{$a->url}">admin/cron.php script</a> has not been run for {$a->actual} and should run every {$a->expected}.';
+$string['cronwarningcli'] = 'The <code>admin/cli/cron.php</code> script has not been run for {$a->actual} and should run every {$a->expected}.';
+$string['cronwarningnever'] = 'The <code>admin/cli/cron.php</code> script has never been run and should run every {$a->expected}.';
 $string['ctyperequired'] = 'The ctype PHP extension is now required by Moodle, in order to improve site performance and to offer multilingual compatibility.';
 $string['curlsecurityallowedport'] = 'cURL allowed ports list';
 $string['curlsecurityallowedportsyntax'] = 'List of port numbers that cURL can connect to. Valid entries are integer numbers only. Put each entry on a new line. If left empty, then all ports are allowed. If set, in almost all cases, both 443 and 80 should be specified for cURL to connect to standard HTTPS and HTTP ports.';