MDL-80684 core: Fail running tasks on shutdown

When you kill a process which is executing a task, it now marks it as failed.

lib/classes/shutdown_manager.php

@@ -148,6 +148,9 @@ class core_shutdown_manager {
     public static function shutdown_handler() {
         global $DB;
 
+        // In case we caught an out of memory shutdown we increase memory limit to unlimited, so we can gracefully shut down.
+        raise_memory_limit(MEMORY_UNLIMITED);
+
         // Always ensure we know who the user is in access logs even if they
         // were logged in a weird way midway through the request.
         set_access_log_user();

lib/classes/task/manager.php

@@ -26,6 +26,7 @@ namespace core\task;
 
 use core\lock\lock;
 use core\lock\lock_factory;
+use core_shutdown_manager;
 
 define('CORE_TASK_TASKS_FILENAME', 'db/tasks.php');
 /**
@@ -60,6 +61,16 @@ class manager {
      */
     const MAX_RETRY = 9;
 
+    /**
+     * @var ?task_base $runningtask Used to tell what is the current running task in this process.
+     */
+    public static ?task_base $runningtask = null;
+
+    /**
+     * @var bool Used to tell if the manager's shutdown callback has been registered.
+     */
+    public static bool $registeredshutdownhandler = false;
+
     /**
      * @var array A cached queue of adhoc tasks
      */
@@ -1104,6 +1115,42 @@ class manager {
         return null;
     }
 
+    /**
+     * This function will fail the currently running task, if there is one.
+     */
+    public static function fail_running_task(): void {
+        $runningtask = self::$runningtask;
+
+        if ($runningtask === null) {
+            return;
+        }
+
+        if ($runningtask instanceof scheduled_task) {
+            self::scheduled_task_failed($runningtask);
+            return;
+        }
+
+        if ($runningtask instanceof adhoc_task) {
+            self::adhoc_task_failed($runningtask);
+            return;
+        }
+    }
+
+    /**
+     * This function set's the $runningtask variable and ensures that the shutdown handler is registered.
+     * @param task_base $task
+     */
+    private static function task_starting(task_base $task): void {
+        self::$runningtask = $task;
+
+        // Add \core\task\manager::fail_running_task to shutdown manager, so we can ensure running tasks fail on shutdown.
+        if (!self::$registeredshutdownhandler) {
+            core_shutdown_manager::register_function('\core\task\manager::fail_running_task');
+
+            self::$registeredshutdownhandler = true;
+        }
+    }
+
     /**
      * This function indicates that an adhoc task was not completed successfully and should be retried.
      *
@@ -1145,6 +1192,8 @@ class manager {
             $task->get_cron_lock()->release();
         }
         $task->get_lock()->release();
+
+        self::$runningtask = null;
     }
 
     /**
@@ -1170,6 +1219,8 @@ class manager {
 
         $record = self::record_from_adhoc_task($task);
         $DB->update_record('task_adhoc', $record);
+
+        self::task_starting($task);
     }
 
     /**
@@ -1195,6 +1246,8 @@ class manager {
             $task->get_cron_lock()->release();
         }
         $task->get_lock()->release();
+
+        self::$runningtask = null;
     }
 
     /**
@@ -1239,6 +1292,8 @@ class manager {
             $task->get_cron_lock()->release();
         }
         $task->get_lock()->release();
+
+        self::$runningtask = null;
     }
 
     /**
@@ -1284,6 +1339,8 @@ class manager {
         $record->hostname = $hostname;
         $record->pid = $pid;
         $DB->update_record('task_scheduled', $record);
+
+        self::task_starting($task);
     }
 
     /**
@@ -1318,6 +1375,8 @@ class manager {
             $task->get_cron_lock()->release();
         }
         $task->get_lock()->release();
+
+        self::$runningtask = null;
     }
 
     /**

lib/tests/task/running_test.php

@@ -156,4 +156,64 @@ class running_test extends \advanced_testcase {
         $running = manager::get_running_tasks();
         $this->assertCount(0, $running);
     }
+
+    /**
+     * Test that adhoc tasks are set as failed when shutdown is called during execution.
+     * @covers \core\task\manager::fail_running_task
+     */
+    public function test_adhoc_task_running_will_fail_when_shutdown(): void {
+        $this->resetAfterTest();
+        $this->preventResetByRollback();
+
+        $task1 = new adhoc_test_task();
+        $task1->set_next_run_time(time() - 20);
+        manager::queue_adhoc_task($task1);
+
+        $next1 = manager::get_next_adhoc_task(time());
+        \core\task\manager::adhoc_task_starting($next1);
+
+        self::assertEmpty(manager::get_failed_adhoc_tasks());
+
+        // Trigger shutdown handler.
+        \core_shutdown_manager::shutdown_handler();
+
+        $failedtasks = manager::get_failed_adhoc_tasks();
+
+        self::assertCount(1, $failedtasks);
+        self::assertEquals($next1->get_id(), $failedtasks[0]->get_id());
+    }
+
+    /**
+     * Test that scheduled tasks are set as failed when shutdown is called during execution.
+     * @covers \core\task\manager::fail_running_task
+     */
+    public function test_scheduled_task_running_will_fail_when_shutdown(): void {
+        global $DB;
+
+        $this->resetAfterTest();
+        $this->preventResetByRollback();
+
+        // Disable all the tasks, so we can insert our own and be sure it's the only one being run.
+        $DB->set_field('task_scheduled', 'disabled', 1);
+
+        $task1 = new scheduled_test_task();
+        $task1->set_minute('*');
+        $task1->set_next_run_time(time() - HOURSECS);
+        $DB->insert_record('task_scheduled', manager::record_from_scheduled_task($task1));
+
+        $next1 = \core\task\manager::get_next_scheduled_task(time());
+        \core\task\manager::scheduled_task_starting($next1);
+
+        $running = manager::get_running_tasks();
+        $this->assertCount(1, $running);
+
+        // Trigger shutdown handler.
+        \core_shutdown_manager::shutdown_handler();
+
+        $running = manager::get_running_tasks();
+        $this->assertCount(0, $running);
+
+        $scheduledtask1 = manager::get_scheduled_task(scheduled_test_task::class);
+        self::assertGreaterThan($next1->get_fail_delay(), $scheduledtask1->get_fail_delay());
+    }
 }