Merge branch 'MDL-72775-master' of https://github.com/QihuiChan/moodle

admin/tool/task/classes/check/longrunningtasks.php

@@ -0,0 +1,98 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_task\check;
+
+use core\check\check;
+use core\check\result;
+use core\task\manager;
+
+/**
+ * Long running tasks check
+ *
+ * @package    tool_task
+ * @author     Qihui Chan (qihuichan@catalyst-au.net)
+ * @copyright  2022 Catalyst IT Pty Ltd
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class longrunningtasks extends check {
+
+    /**
+     * Constructor
+     */
+    public function __construct() {
+        global $CFG;
+        $this->id = 'longrunningtasks';
+        $this->name = get_string('checklongrunningtasks', 'tool_task');
+    }
+
+    /**
+     * Links to the running task list
+     *
+     * @return \action_link|null
+     * @throws \coding_exception
+     */
+    public function get_action_link(): ?\action_link {
+        $url = new \moodle_url('/admin/tool/task/runningtasks.php');
+        return new \action_link($url, get_string('runningtasks', 'tool_task'));
+    }
+
+    /**
+     * Return result
+     * @return result
+     */
+    public function get_result(): result {
+        global $CFG;
+        $status = result::OK;
+        $slowtasks = 0;
+        $details = '';
+        $maxruntime = 0;
+        $runtime = 0;
+
+        $tasks = \core\task\manager::get_running_tasks();
+        foreach ($tasks as $record) {
+            $taskmethod = "{$record->type}_task_from_record";
+            $task = manager::$taskmethod($record);
+            $taskname = $task->get_name();
+
+            $result = $task->get_runtime_result();
+            $taskstatus = $result->get_status();
+            $runtime = $task->get_runtime();
+            $runtimedetails = get_string('taskrunningtime', 'tool_task', format_time($runtime));
+            $maxruntime = ($runtime > $maxruntime) ? $runtime : $maxruntime;
+
+            if ($taskstatus == result::OK) {
+                continue;
+            }
+
+            $slowtasks++;
+            $details .= strtoupper($taskstatus) . ": {$taskname}. {$runtimedetails} <br>";
+
+            // The overall check status is the worst tasks status.
+            if ($status !== result::ERROR) {
+                $status = $taskstatus;
+            }
+        }
+
+        $summary = get_string('checklongrunningtaskcount', 'tool_task', $slowtasks);
+        $conclusion = get_string('taskdetails', 'tool_task', ['count' => $slowtasks,
+            'time' => format_time($CFG->taskruntimewarn), 'maxtime' => format_time($maxruntime)]);
+
+        $details = ($slowtasks ? $conclusion : $summary) . "<br>{$details}";
+
+        return new result($status, $summary, $details);
+    }
+}

admin/tool/task/classes/running_tasks_table.php

@@ -28,6 +28,7 @@ namespace tool_task;
 defined('MOODLE_INTERNAL') || die();
 
 require_once($CFG->libdir . '/tablelib.php');
+use core\task\manager;
 
 /**
  * Table to display list of running task.
@@ -47,7 +48,7 @@ class running_tasks_table extends \table_sql {
         $columnheaders = [
             'classname'    => get_string('classname', 'tool_task'),
             'type'         => get_string('tasktype', 'admin'),
-            'time'         => get_string('time'),
+            'time'         => get_string('taskage', 'tool_task'),
             'timestarted'  => get_string('started', 'tool_task'),
             'hostname'     => get_string('hostname', 'tool_task'),
             'pid'          => get_string('pid', 'tool_task'),
@@ -111,7 +112,7 @@ class running_tasks_table extends \table_sql {
         if ($row->type == 'scheduled') {
             $output = \html_writer::span(get_string('scheduled', 'tool_task'), 'badge badge-primary');
         } else if ($row->type == 'adhoc') {
-            $output = \html_writer::span(get_string('adhoc', 'tool_task'), 'badge badge-warning');
+            $output = \html_writer::span(get_string('adhoc', 'tool_task'), 'badge badge-dark');
         } else {
             // This shouldn't ever happen.
             $output = '';
@@ -126,7 +127,21 @@ class running_tasks_table extends \table_sql {
      * @return  string
      */
     public function col_time($row) : string {
-        return format_time($row->time);
+        global $OUTPUT;
+
+        $taskmethod = "{$row->type}_task_from_record";
+        $task = manager::$taskmethod($row);
+
+        $result = $task->get_runtime_result();
+        $extra = '';
+        if ($result->get_status() != $result::OK) {
+            $extra = '<br>';
+            $extra .= $OUTPUT->check_result($result);
+            $extra .= ' ';
+            $extra .= $result->get_details();
+        }
+
+        return format_time($row->time) . $extra;
     }
 
     /**

admin/tool/task/lang/en/tool_task.php

@@ -28,7 +28,7 @@ $string['adhoctasks'] = 'Ad hoc tasks';
 $string['asap'] = 'ASAP';
 $string['adhocempty'] = 'Ad hoc task queue is empty';
 $string['adhocqueuesize'] = 'Ad hoc task queue has {$a} tasks';
-$string['adhocqueueold'] = 'Oldest task is {$a->age} which is more than {$a->max}';
+$string['adhocqueueold'] = 'Oldest unprocessed task is {$a->age} which is more than {$a->max}';
 $string['backtoscheduledtasks'] = 'Back to scheduled tasks';
 $string['blocking'] = 'Blocking';
 $string['cannotfindthepathtothecli'] = 'Cannot find the path to the PHP CLI executable so task execution aborted. Set the \'Path to PHP CLI\' setting in Site administration / Server / System paths.';
@@ -36,6 +36,8 @@ $string['checkadhocqueue'] = 'Ad hoc task queue';
 $string['checkcronrunning'] = 'Cron running';
 $string['checkmaxfaildelay'] = 'Tasks max fail delay';
 $string['classname'] = 'Class name';
+$string['checklongrunningtasks'] = 'Long running tasks';
+$string['checklongrunningtaskcount'] = 'There are {$a} long running tasks';
 $string['clearfaildelay_confirm'] = 'Are you sure you want to clear the fail delay for task \'{$a}\'? After clearing the delay, the task will run according to its normal schedule.';
 $string['component'] = 'Component';
 $string['corecomponent'] = 'Core';
@@ -68,11 +70,15 @@ $string['runpattern'] = 'Run pattern';
 $string['scheduled'] = 'Scheduled';
 $string['scheduledtasks'] = 'Scheduled tasks';
 $string['scheduledtaskchangesdisabled'] = 'Modifications to the list of scheduled tasks have been prevented in Moodle configuration';
+$string['slowtask'] = 'Task has run for longer than {$a}';
 $string['started'] = 'Started';
+$string['taskage'] = 'Run time';
+$string['taskdetails'] = 'There is {$a->count} task(s) running for more than {$a->time} (max {$a->maxtime})';
 $string['taskdisabled'] = 'Task disabled';
 $string['taskfailures'] = '{$a} task(s) failing';
 $string['tasklogs'] = 'Task logs';
 $string['tasknofailures'] = 'There are no tasks failing';
+$string['taskrunningtime'] = 'Task has run for {$a}';
 $string['taskscheduleday'] = 'Day';
 $string['taskscheduleday_help'] = 'Day of month field for task schedule. The field uses the same format as unix cron. Some examples are:
 

admin/tool/task/lib.php

@@ -34,6 +34,7 @@ function tool_task_status_checks() : array {
         new \tool_task\check\cronrunning(),
         new \tool_task\check\maxfaildelay(),
         new \tool_task\check\adhocqueue(),
+        new \tool_task\check\longrunningtasks(),
     ];
 }
 

admin/tool/task/version.php

@@ -24,7 +24,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version   = 2022112800; // The current plugin version (Date: YYYYMMDDXX).
+$plugin->version   = 2022112801; // The current plugin version (Date: YYYYMMDDXX).
 $plugin->requires  = 2022111800; // Requires this Moodle version.
 $plugin->component = 'tool_task'; // Full name of the plugin (used for diagnostics)
 

config-dist.php

@@ -668,6 +668,15 @@ $CFG->admin = 'admin';
 //      $CFG->adhoctaskagewarn = 10 * 60;
 //      $CFG->adhoctaskageerror = 4 * 60 * 60;
 //
+// Moodle 4.2+ checks how long tasks have been running for at warns at 12 hours
+// and errors at 24 hours. Set these to override these limits:
+//
+// $CFG->taskruntimewarn = 12 * 60 * 60;
+// $CFG->taskruntimeerror = 24 * 60 * 60;
+//
+// This is not to be confused with $CFG->task_adhoc_max_runtime which is how long the
+// php process should be allowed to run for, not each specific task.
+//
 // Session lock warning threshold. Long running pages should release the session using \core\session\manager::write_close().
 // Set this threshold to any value greater than 0 to add developer warnings when a page locks the session for too long.
 // The session should rarely be locked for more than 1 second. The input should be in seconds and may be a float.

lib/classes/task/task_base.php

@@ -26,6 +26,7 @@ namespace core\task;
 
 use core_component;
 use core_plugin_manager;
+use core\check\result;
 
 /**
  * Abstract class for common properties of scheduled_task and adhoc_task.
@@ -257,4 +258,40 @@ abstract class task_base {
             return $plugininfo && ($plugininfo->is_enabled() !== false);
         }
     }
+
+    /**
+     * Returns task runtime
+     * @return int
+     */
+    public function get_runtime() {
+        return time() - $this->timestarted;
+    }
+
+    /**
+     * Returns if the task has been running for too long
+     * @return result
+     */
+    public function get_runtime_result() {
+        global $CFG;
+        $runtime = $this->get_runtime();
+        $runtimeerror = $CFG->taskruntimeerror;
+        $runtimewarn = $CFG->taskruntimewarn;
+
+        $status = result::OK;
+        $details = '';
+
+        if ($runtime > $runtimewarn) {
+            $status = result::WARNING;
+            $details = get_string('slowtask', 'tool_task', format_time($runtimewarn));
+        }
+
+        if ($runtime > $runtimeerror) {
+            $status = result::ERROR;
+            $details = get_string('slowtask', 'tool_task', format_time($runtimeerror));
+        }
+
+        // This result is aggregated with other running tasks checks before display.
+        return new result($status, '', $details);
+    }
+
 }

lib/setup.php

@@ -154,6 +154,16 @@ if (defined('BEHAT_SITE_RUNNING')) {
     }
 }
 
+// Set default warn runtime.
+if (!isset($CFG->taskruntimewarn)) {
+    $CFG->taskruntimewarn = 12 * 60 * 60;
+}
+
+// Set default error runtime.
+if (!isset($CFG->taskruntimeerror)) {
+    $CFG->taskruntimeerror = 24 * 60 * 60;
+}
+
 // Normalise dataroot - we do not want any symbolic links, trailing / or any other weirdness there
 if (!isset($CFG->dataroot)) {
     if (isset($_SERVER['REMOTE_ADDR'])) {