diff --git a/lang/en/hub.php b/lang/en/hub.php index 99ef82c8bb1..57e9d5f1e42 100644 --- a/lang/en/hub.php +++ b/lang/en/hub.php @@ -24,6 +24,7 @@ */ $string['activeparticipantnumberaverage'] = 'Average number of recently active participants ({$a})'; $string['activeusersnumber'] = 'Number of recently active users ({$a})'; +$string['aiusagestats'] = 'AI usage stats ({$a->timefrom} - {$a->timeto})'; $string['analyticsactions'] = 'Number of actions taken on generated predictions ({$a})'; $string['analyticsactionsnotuseful'] = 'Number of actions marking a prediction as not useful ({$a})'; $string['analyticsenabledmodels'] = 'Number of enabled prediction models ({$a})'; diff --git a/lib/classes/hub/registration.php b/lib/classes/hub/registration.php index ecf24768870..88dd42eb20b 100644 --- a/lib/classes/hub/registration.php +++ b/lib/classes/hub/registration.php @@ -66,6 +66,8 @@ class registration { 2023021700 => ['dbtype', 'coursesnodates', 'sitetheme', 'primaryauthtype'], // Plugin usage added in Moodle 4.5. 2023072300 => ['pluginusage'], + // AI usage added in Moodle 4.5. + 2023081200 => ['aiusage'], ]; /** @var string Site privacy: not displayed */ @@ -191,6 +193,10 @@ class registration { $siteinfo['sitetheme'] = get_config('core', 'theme'); $siteinfo['pluginusage'] = json_encode(self::get_plugin_usage_data()); + // AI usage data. + $aiusagedata = self::get_ai_usage_data(); + $siteinfo['aiusage'] = !empty($aiusagedata) ? json_encode($aiusagedata) : ''; + // Primary auth type. $primaryauthsql = 'SELECT auth, count(auth) as tc FROM {user} GROUP BY auth ORDER BY tc DESC'; $siteinfo['primaryauthtype'] = $DB->get_field_sql($primaryauthsql, null, IGNORE_MULTIPLE); @@ -278,6 +284,7 @@ class registration { 'sitetheme' => get_string('sitetheme', 'hub', $siteinfo['sitetheme']), 'primaryauthtype' => get_string('primaryauthtype', 'hub', $siteinfo['primaryauthtype']), 'pluginusage' => get_string('pluginusagedata', 'hub', $pluginusagelinks), + 'aiusage' => get_string('aiusagestats', 'hub', self::get_ai_usage_time_range(true)), ]; foreach ($senddata as $key => $str) { @@ -685,4 +692,118 @@ class registration { return $data; } + + /** + * Get the time range to use in collected and reporting AI usage data. + * + * @param bool $format Use true to format timestamp. + * @return array + */ + private static function get_ai_usage_time_range(bool $format = false): array { + global $DB; + + // We will try and use the last time this site was last registered for our 'from' time. + // Otherwise, default to using one week's worth of data to roughly match the site rego scheduled task. + $timenow = \core\di::get(\core\clock::class)->time(); + $defaultfrom = $timenow - WEEKSECS; + $timeto = $timenow; + $params = [ + 'huburl' => HUB_MOODLEORGHUBURL, + 'confirmed' => 1, + ]; + $lastregistered = $DB->get_field('registration_hubs', 'timemodified', $params); + $timefrom = $lastregistered ? (int)$lastregistered : $defaultfrom; + + if ($format) { + $timefrom = userdate($timefrom); + $timeto = userdate($timeto); + } + + return [ + 'timefrom' => $timefrom, + 'timeto' => $timeto, + ]; + } + + /** + * Get AI usage data. + * + * @return array + */ + public static function get_ai_usage_data(): array { + global $DB; + + $params = self::get_ai_usage_time_range(); + + $sql = "SELECT aar.* + FROM {ai_action_register} aar + WHERE aar.timecompleted >= :timefrom + AND aar.timecompleted <= :timeto"; + + $actions = $DB->get_records_sql($sql, $params); + + // Build data for site info reporting. + $data = []; + + foreach ($actions as $action) { + $provider = $action->provider; + $actionname = $action->actionname; + + // Initialise data structure. + if (!isset($data[$provider][$actionname])) { + $data[$provider][$actionname] = [ + 'success_count' => 0, + 'fail_count' => 0, + 'times' => [], + 'errors' => [], + ]; + } + + if ($action->success === '1') { + $data[$provider][$actionname]['success_count'] += 1; + // Collect AI processing times for averaging. + $data[$provider][$actionname]['times'][] = (int)$action->timecompleted - (int)$action->timecreated; + + } else { + $data[$provider][$actionname]['fail_count'] += 1; + // Collect errors for determing the predominant one. + $data[$provider][$actionname]['errors'][] = $action->errorcode; + } + } + + // Parse the errors and everage the times, then add them to the data. + foreach ($data as $p => $provider) { + foreach ($provider as $a => $actionname) { + if (isset($data[$p][$a]['errors'])) { + // Create an array with the error codes counted. + $errors = array_count_values($data[$p][$a]['errors']); + if (!empty($errors)) { + // Sort values descending and convert to an array of error codes (most predominant will be at start). + arsort($errors); + $errors = array_keys($errors); + $data[$p][$a]['predominant_error'] = $errors[0]; + } + unset($data[$p][$a]['errors']); + } + + if (isset($data[$p][$a]['times'])) { + $count = count($data[$p][$a]['times']); + if ($count > 0) { + // Average the time to perform the action (seconds). + $totaltime = array_sum($data[$p][$a]['times']); + $data[$p][$a]['average_time'] = round($totaltime / $count); + + } + } + unset($data[$p][$a]['times']); + } + } + + // Include the time range used to help interpret the data. + if (!empty($data)) { + $data['time_range'] = $params; + } + + return $data; + } } diff --git a/lib/tests/hub/registration_test.php b/lib/tests/hub/registration_test.php index e7acddcfdaf..e0dbb519532 100644 --- a/lib/tests/hub/registration_test.php +++ b/lib/tests/hub/registration_test.php @@ -19,7 +19,7 @@ namespace core\hub; /** * Class containing unit tests for the site registration class. * - * @package core + * @package core * @copyright 2023 Matt Porritt * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later * @covers \core\hub\registration @@ -85,4 +85,63 @@ class registration_test extends \advanced_testcase { $this->assertEquals(0, $pluginusage['mod']['feedback']['enabled']); $this->assertEquals(1, $pluginusage['mod']['assign']['enabled']); } + + /** + * Test the AI usage data is calculated correctly. + */ + public function test_get_ai_usage(): void { + global $CFG, $DB; + $this->resetAfterTest(); + $clock = $this->mock_clock_with_frozen(); + + // Record some generated text. + $record = new \stdClass(); + $record->provider = 'openai'; + $record->actionname = 'generate_text'; + $record->actionid = 1; + $record->userid = 1; + $record->contextid = 1; + $record->success = true; + $record->timecreated = $clock->time() - 5; + $record->timecompleted = $clock->time(); + $DB->insert_record('ai_action_register', $record); + + // Record a generated image. + $record->actionname = 'generate_image'; + $record->actionid = 2; + $record->timecreated = $clock->time() - 20; + $DB->insert_record('ai_action_register', $record); + // Record another image. + $record->actionid = 3; + $record->timecreated = $clock->time() - 10; + $DB->insert_record('ai_action_register', $record); + + // Record some errors. + $record->actionname = 'generate_image'; + $record->actionid = 4; + $record->success = false; + $record->errorcode = 403; + $DB->insert_record('ai_action_register', $record); + $record->actionid = 5; + $record->errorcode = 403; + $DB->insert_record('ai_action_register', $record); + $record->actionid = 6; + $record->errorcode = 404; + $DB->insert_record('ai_action_register', $record); + + // Get our site info and check the expected calculations are correct. + $siteinfo = registration::get_site_info(); + $aisuage = json_decode($siteinfo['aiusage']); + // Check generated text. + $this->assertEquals(1, $aisuage->openai->generate_text->success_count); + $this->assertEquals(0, $aisuage->openai->generate_text->fail_count); + // Check generated images. + $this->assertEquals(2, $aisuage->openai->generate_image->success_count); + $this->assertEquals(3, $aisuage->openai->generate_image->fail_count); + $this->assertEquals(15, $aisuage->openai->generate_image->average_time); + $this->assertEquals(403, $aisuage->openai->generate_image->predominant_error); + // Check time range is set correctly. + $this->assertEquals($clock->time() - WEEKSECS, $aisuage->time_range->timefrom); + $this->assertEquals($clock->time(), $aisuage->time_range->timeto); + } }