MDL-81714 grades: Make large regrades asynchronous

Currently, large courses can take a long time to perform a full regrade.
This is currently handled with a progress bar to prevent frontend
timeouts while the regrade takes place. However, because it can take so
long a teacher may not want to wait with the page open for several
minutes, particularly if they are performing several operations that
trigger a regrade.

This adds a new async flag to grade_regrade_final_grades which is true
by default. Instead of performing the regrade immediately, this queues
an instance of \core\task\regrade_final_grades for the course, which
will be executed in the background.

It is advisable to always leave the async flag set true, except in the
following scenarios:
- Automated tests.
- The regrade_final_grades task which actually wants to do the
calculations immediately.
- When you have performed a check to determine that the regrade process
is unlikely to take a long time, for example there are only a small
number of grade items.

admin/tool/recyclebin/tests/behat/backup_user_data.feature

@@ -54,6 +54,7 @@ Feature: Backup user data
     And I navigate to "Recycle bin" in current page administration
     And I should see "Quiz 1"
     And I click on "Restore" "link" in the "region-main" "region"
+    And I run all adhoc tasks
     When I am on the "Course 1" "grades > User report > View" page logged in as "student1"
     Then "Quiz 1" row "Grade" column of "user-grade" table should contain "50"
     And "Quiz 1" row "Percentage" column of "user-grade" table should contain "50"

backup/moodle2/restore_stepslib.php

@@ -490,7 +490,7 @@ class restore_gradebook_structure_step extends restore_structure_step {
         rebuild_course_cache($this->get_courseid(), true);
 
         // Restore marks items as needing update. Update everything now.
-        grade_regrade_final_grades($this->get_courseid());
+        grade_regrade_final_grades($this->get_courseid(), async: true);
     }
 
     /**

course/classes/task/regrade_final_grades.php

@@ -0,0 +1,77 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace core_course\task;
+
+defined('MOODLE_INTERNAL') || die();
+
+use core\task\adhoc_task;
+
+require_once($CFG->libdir . '/gradelib.php');
+
+/**
+ * Asynchronously regrade a course.
+ *
+ * @copyright 2024 onwards Catalyst IT Europe Ltd.
+ * @author Mark Johnson <mark.johnson@catalyst-eu.net>
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ * @package core_course
+ */
+class regrade_final_grades extends adhoc_task {
+    use \core\task\logging_trait;
+    use \core\task\stored_progress_task_trait;
+
+    /**
+     * Create and return an instance of this task for a given course ID.
+     *
+     * @param int $courseid
+     * @return self
+     */
+    public static function create(int $courseid): self {
+        $task = new regrade_final_grades();
+        $task->set_custom_data((object)['courseid' => $courseid]);
+        $task->set_component('core');
+        return $task;
+    }
+
+    /**
+     * Run regrade_final_grades for the provided course id.
+     *
+     * @return void
+     * @throws \dml_exception
+     */
+    public function execute(): void {
+        $data = $this->get_custom_data();
+        $this->start_stored_progress();
+        $this->log_start("Recalculating grades for course ID {$data->courseid}");
+        // Ensure the course exists.
+        try {
+            $course = get_course($data->courseid);
+        } catch (\dml_missing_record_exception $e) {
+            $this->log("Course with id {$data->courseid} not found. It may have been deleted. Skipping regrade.");
+            return;
+        }
+        $this->log("Found course {$course->shortname}. Starting regrade.");
+        $results = grade_regrade_final_grades($course->id, progress: $this->get_progress());
+        if (is_array($results)) {
+            $this->log('Errors reported during regrade:');
+            foreach ($results as $id => $result) {
+                $this->log("Grade item {$id}: {$result}", 2);
+            }
+        }
+        $this->log_finish('Regrade complete.');
+    }
+}

course/modedit.php

@@ -204,13 +204,6 @@ if ($mform->is_cancelled()) {
         $url = course_get_url($course, $cw->section, $options);
     }
 
-    // If we need to regrade the course with a progress bar as a result of updating this module,
-    // redirect first to the page that will do this.
-    if (isset($fromform->needsfrontendregrade)) {
-        $url = new moodle_url('/course/modregrade.php', ['id' => $fromform->coursemodule,
-                'url' => $url->out_as_local_url(false)]);
-    }
-
     redirect($url);
     exit;
 

course/modlib.php

@@ -404,13 +404,17 @@ function edit_module_post_actions($moduleinfo, $course) {
             // And if it actually needs regrading...
             $courseitem = grade_item::fetch_course_item($course->id);
             if ($courseitem->needsupdate) {
-                // Then don't do it as part of this form save, do it on an extra web request with a
-                // progress bar.
-                $moduleinfo->needsfrontendregrade = true;
+                // Queue an asynchronous regrade.
+                grade_regrade_final_grades($course->id, async: true);
             }
         } else {
             // Regrade now.
-            grade_regrade_final_grades($course->id);
+            $result = grade_regrade_final_grades($course->id);
+            if (is_array($result)) {
+                foreach ($result as $error) {
+                    \core\notification::add($error, \core\output\notification::NOTIFY_ERROR);
+                }
+            }
         }
     }
 

grade/report/overview/lib.php

@@ -137,7 +137,7 @@ class grade_report_overview extends grade_report {
             if ($frontend) {
                 grade_regrade_final_grades_if_required($course);
             } else {
-                grade_regrade_final_grades($course->id);
+                grade_regrade_final_grades($course->id, async: true);
             }
         }
     }

grade/report/overview/tests/lib_test.php

@@ -114,6 +114,10 @@ final class lib_test extends \advanced_testcase {
         $gpr = new \stdClass();
         $report = new \grade_report_overview($user->id, $gpr, '');
         $report->regrade_all_courses_if_needed($frontend);
+        if (!$frontend) {
+            $this->expectOutputRegex("~^Recalculating grades for course ID {$course->id}~");
+            $this->run_all_adhoc_tasks();
+        }
 
         // This should have regraded courses 1 and 3, but not 2 (because the user doesn't belong).
         $this->assertEqualsWithDelta(50.0, $DB->get_field('grade_grades', 'finalgrade',

grade/tests/report_graderlib_test.php

@@ -480,6 +480,9 @@ final class report_graderlib_test extends \advanced_testcase {
         // Set the grade for the second one to 0 (note, you have to do this after creating it,
         // otherwise it doesn't create an ungraded grade item).
         quiz_settings::create($ungradedquiz->id)->get_grade_calculator()->update_quiz_maximum_grade(0);
+        ob_start();
+        $this->run_all_adhoc_tasks();
+        ob_end_clean();
 
         // Set current user.
         $this->setUser($manager);

lang/en/grades.php

@@ -707,6 +707,7 @@ $string['real'] = 'Real';
 $string['realletter'] = 'Real (letter)';
 $string['realpercentage'] = 'Real (percentage)';
 $string['recalculatinggrades'] = 'Recalculating grades';
+$string['recalculatinggradesadhoc'] = 'Grade recalculations are being performed in the background. Displayed grades may be incorrect until the process is complete.';
 $string['recovergradesdefault'] = 'Recover grades default';
 $string['recovergradesdefault_help'] = 'By default recover old grades when re-enrolling a user in a course.';
 $string['refreshpreview'] = 'Refresh preview';

lib/grade/tests/grade_category_test.php

@@ -444,7 +444,7 @@ final class grade_category_test extends \grade_base_testcase {
         $category_grade_item = $grade_category->get_grade_item();
 
         // This creates all the grade_grades we need.
-        grade_regrade_final_grades($this->courseid);
+        grade_regrade_final_grades($this->courseid, async: true);
 
         $grade = $DB->get_record('grade_grades', array('itemid'=>$category_grade_item->id, 'userid'=>$this->userid));
         $this->assertWithinMargin($grade->rawgrade, $grade->rawgrademin, $grade->rawgrademax);

lib/gradelib.php

@@ -383,43 +383,16 @@ function grade_needs_regrade_progress_bar($courseid) {
  * @return moodle_url|false The URL to redirect to if redirecting
  */
 function grade_regrade_final_grades_if_required($course, ?callable $callback = null) {
-    global $PAGE, $OUTPUT;
+    global $PAGE;
 
     if (!grade_needs_regrade_final_grades($course->id)) {
         return false;
     }
 
     if (grade_needs_regrade_progress_bar($course->id)) {
-        if ($PAGE->state !== moodle_page::STATE_IN_BODY) {
-            $PAGE->set_heading($course->fullname);
-            echo $OUTPUT->header();
-        }
-        echo $OUTPUT->heading(get_string('recalculatinggrades', 'grades'));
-        $progress = new \core\progress\display(true);
-        $status = grade_regrade_final_grades($course->id, null, null, $progress);
-
-        // Show regrade errors and set the course to no longer needing regrade (stop endless loop).
-        if (is_array($status)) {
-            foreach ($status as $error) {
-                $errortext = new \core\output\notification($error, \core\output\notification::NOTIFY_ERROR);
-                echo $OUTPUT->render($errortext);
-            }
-            $courseitem = grade_item::fetch_course_item($course->id);
-            $courseitem->regrading_finished();
-        }
-
-        if ($callback) {
-            //
-            $url = call_user_func($callback);
-        }
-
-        if (empty($url)) {
-            $url = $PAGE->url;
-        }
-
-        echo $OUTPUT->continue_button($url);
-        echo $OUTPUT->footer();
-        die();
+        // Queue ad-hoc task and redirect.
+        grade_regrade_final_grades($course->id, async: true);
+        return $callback ? call_user_func($callback) : $PAGE->url;
     } else {
         $result = grade_regrade_final_grades($course->id);
         if ($callback) {
@@ -1153,9 +1126,10 @@ function grade_recover_history_grades($userid, $courseid) {
  * @param int $userid If specified try to do a quick regrading of the grades of this user only
  * @param object $updated_item Optional grade item to be marked for regrading. It is required if $userid is set.
  * @param \core\progress\base $progress If provided, will be used to update progress on this long operation.
+ * @param bool $async If true, and we are recalculating an entire course's grades, defer processing to an ad-hoc task.
  * @return array|true true if ok, array of errors if problems found. Grade item id => error message
  */
-function grade_regrade_final_grades($courseid, $userid=null, $updated_item=null, $progress=null) {
+function grade_regrade_final_grades($courseid, $userid=null, $updated_item=null, $progress=null, bool $async = false) {
     // This may take a very long time and extra memory.
     \core_php_time_limit::raise();
     raise_memory_limit(MEMORY_EXTRA);
@@ -1181,6 +1155,30 @@ function grade_regrade_final_grades($courseid, $userid=null, $updated_item=null,
             // nothing to do :-)
             return true;
         }
+        // Defer recalculation to an ad-hoc task.
+        if ($async) {
+            $regradecache = cache::make_from_params(
+                mode: cache_store::MODE_REQUEST,
+                component: 'core',
+                area: 'grade_regrade_final_grades',
+                options: [
+                    'simplekeys' => true,
+                    'simpledata' => true,
+                ],
+            );
+            // If the courseid already exists in the cache, return so we don't do this multiple times per request.
+            if ($regradecache->get($courseid)) {
+                return true;
+            }
+            $task = \core_course\task\regrade_final_grades::create($courseid);
+            $taskid = \core\task\manager::queue_adhoc_task($task, true);
+            if ($taskid) {
+                $task->set_id($taskid);
+                $task->initialise_stored_progress();
+            }
+            $regradecache->set($courseid, true);
+            return true;
+        }
     }
 
     // Categories might have to run some processing before we fetch the grade items.
@@ -1601,7 +1599,7 @@ function grade_course_reset($courseid) {
     grade_grab_course_grades($courseid);
 
     // recalculate all grades
-    grade_regrade_final_grades($courseid);
+    grade_regrade_final_grades($courseid, async: true);
     return true;
 }
 

lib/moodlelib.php

@@ -5327,7 +5327,7 @@ function reset_course_userdata($data) {
     if (!empty($data->reset_gradebook_items)) {
         remove_course_grades($data->courseid, false);
         grade_grab_course_grades($data->courseid);
-        grade_regrade_final_grades($data->courseid);
+        grade_regrade_final_grades($data->courseid, async: true);
         $status[] = array('component' => $componentstr, 'item' => get_string('removeallcourseitems', 'grades'), 'error' => false);
 
     } else if (!empty($data->reset_gradebook_grades)) {

mod/h5pactivity/tests/behat/h5pactivity_reset_grades.feature

@@ -45,6 +45,7 @@ Feature: Teacher can reset H5P activity grades
     And I click on "Reset course" "button" in the "Reset course?" "dialogue"
     Then I should see "Done" in the "Gradebook" "table_row"
     And I press "Continue"
+    And I run all adhoc tasks
     # Confirm that previously saved grades are gone
     And I navigate to "View > Grader report" in the course gradebook
     And I should not see "7.00" in the "First Student" "table_row"

mod/h5pactivity/tests/local/grader_test.php

@@ -139,6 +139,7 @@ final class grader_test extends \advanced_testcase {
         $grader->grade_item_update($param);
 
         // Check new grade item and grades.
+        grade_regrade_final_grades($course->id);
         $gradeinfo = grade_get_grades($course->id, 'mod', 'h5pactivity', $activity->id, $user->id);
         $item = array_shift($gradeinfo->items);
         $this->assertEquals($scaleid, $item->scaleid);
@@ -245,6 +246,7 @@ final class grader_test extends \advanced_testcase {
         $grader->update_grades($userid);
 
         // Check new grade item and grades.
+        grade_regrade_final_grades($course->id);
         $gradeinfo = grade_get_grades($course->id, 'mod', 'h5pactivity', $activity->id, [$user1->id, $user2->id]);
         $item = array_shift($gradeinfo->items);
         $this->assertArrayHasKey($user1->id, $item->grades);