mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-02-22 02:49:53 +01:00
Code Issues Projects Releases Wiki Activity

MDL-82626 hub: Registration collects AI usage data

Browse Source
This commit is contained in:
David Woloszyn 2024-08-22 11:20:01 +10:00
parent ceb30b01b2
commit 2b1d330783
3 changed files with 182 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lang/en/hub.php
View File

@ -24,6 +24,7 @@
*/
$string['activeparticipantnumberaverage'] = 'Average number of recently active participants ({$a})';
$string['activeusersnumber'] = 'Number of recently active users ({$a})';
$string['aiusagestats'] = 'AI usage stats ({$a->timefrom} - {$a->timeto})';
$string['analyticsactions'] = 'Number of actions taken on generated predictions ({$a})';
$string['analyticsactionsnotuseful'] = 'Number of actions marking a prediction as not useful ({$a})';
$string['analyticsenabledmodels'] = 'Number of enabled prediction models ({$a})';

121
lib/classes/hub/registration.php
View File

@ -66,6 +66,8 @@ class registration {
2023021700 => ['dbtype', 'coursesnodates', 'sitetheme', 'primaryauthtype'],
// Plugin usage added in Moodle 4.5.
2023072300 => ['pluginusage'],
// AI usage added in Moodle 4.5.
2023081200 => ['aiusage'],
];
/** @var string Site privacy: not displayed */
@ -191,6 +193,10 @@ class registration {
$siteinfo['sitetheme'] = get_config('core', 'theme');
$siteinfo['pluginusage'] = json_encode(self::get_plugin_usage_data());
// AI usage data.
$aiusagedata = self::get_ai_usage_data();
$siteinfo['aiusage'] = !empty($aiusagedata) ? json_encode($aiusagedata) : '';
// Primary auth type.
$primaryauthsql = 'SELECT auth, count(auth) as tc FROM {user} GROUP BY auth ORDER BY tc DESC';
$siteinfo['primaryauthtype'] = $DB->get_field_sql($primaryauthsql, null, IGNORE_MULTIPLE);
@ -278,6 +284,7 @@ class registration {
'sitetheme' => get_string('sitetheme', 'hub', $siteinfo['sitetheme']),
'primaryauthtype' => get_string('primaryauthtype', 'hub', $siteinfo['primaryauthtype']),
'pluginusage' => get_string('pluginusagedata', 'hub', $pluginusagelinks),
'aiusage' => get_string('aiusagestats', 'hub', self::get_ai_usage_time_range(true)),
];
foreach ($senddata as $key => $str) {
@ -685,4 +692,118 @@ class registration {
return $data;
}
/**
* Get the time range to use in collected and reporting AI usage data.
*
* @param bool $format Use true to format timestamp.
* @return array
*/
private static function get_ai_usage_time_range(bool $format = false): array {
global $DB;
// We will try and use the last time this site was last registered for our 'from' time.
// Otherwise, default to using one week's worth of data to roughly match the site rego scheduled task.
$timenow = \core\di::get(\core\clock::class)->time();
$defaultfrom = $timenow - WEEKSECS;
$timeto = $timenow;
$params = [
'huburl' => HUB_MOODLEORGHUBURL,
'confirmed' => 1,
];
$lastregistered = $DB->get_field('registration_hubs', 'timemodified', $params);
$timefrom = $lastregistered ? (int)$lastregistered : $defaultfrom;
if ($format) {
$timefrom = userdate($timefrom);
$timeto = userdate($timeto);
}
return [
'timefrom' => $timefrom,
'timeto' => $timeto,
];
}
/**
* Get AI usage data.
*
* @return array
*/
public static function get_ai_usage_data(): array {
global $DB;
$params = self::get_ai_usage_time_range();
$sql = "SELECT aar.*
FROM {ai_action_register} aar
WHERE aar.timecompleted >= :timefrom
AND aar.timecompleted <= :timeto";
$actions = $DB->get_records_sql($sql, $params);
// Build data for site info reporting.
$data = [];
foreach ($actions as $action) {
$provider = $action->provider;
$actionname = $action->actionname;
// Initialise data structure.
if (!isset($data[$provider][$actionname])) {
$data[$provider][$actionname] = [
'success_count' => 0,
'fail_count' => 0,
'times' => [],
'errors' => [],
];
}
if ($action->success === '1') {
$data[$provider][$actionname]['success_count'] += 1;
// Collect AI processing times for averaging.
$data[$provider][$actionname]['times'][] = (int)$action->timecompleted - (int)$action->timecreated;
} else {
$data[$provider][$actionname]['fail_count'] += 1;
// Collect errors for determing the predominant one.
$data[$provider][$actionname]['errors'][] = $action->errorcode;
}
}
// Parse the errors and everage the times, then add them to the data.
foreach ($data as $p => $provider) {
foreach ($provider as $a => $actionname) {
if (isset($data[$p][$a]['errors'])) {
// Create an array with the error codes counted.
$errors = array_count_values($data[$p][$a]['errors']);
if (!empty($errors)) {
// Sort values descending and convert to an array of error codes (most predominant will be at start).
arsort($errors);
$errors = array_keys($errors);
$data[$p][$a]['predominant_error'] = $errors[0];
}
unset($data[$p][$a]['errors']);
}
if (isset($data[$p][$a]['times'])) {
$count = count($data[$p][$a]['times']);
if ($count > 0) {
// Average the time to perform the action (seconds).
$totaltime = array_sum($data[$p][$a]['times']);
$data[$p][$a]['average_time'] = round($totaltime / $count);
}
}
unset($data[$p][$a]['times']);
}
}
// Include the time range used to help interpret the data.
if (!empty($data)) {
$data['time_range'] = $params;
}
return $data;
}
}

61
lib/tests/hub/registration_test.php
View File

@ -19,7 +19,7 @@ namespace core\hub;
/**
* Class containing unit tests for the site registration class.
*
* @package core
* @package core
* @copyright 2023 Matt Porritt <matt.porritt@moodle.com>
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
* @covers \core\hub\registration
@ -85,4 +85,63 @@ class registration_test extends \advanced_testcase {
$this->assertEquals(0, $pluginusage['mod']['feedback']['enabled']);
$this->assertEquals(1, $pluginusage['mod']['assign']['enabled']);
}
/**
* Test the AI usage data is calculated correctly.
*/
public function test_get_ai_usage(): void {
global $CFG, $DB;
$this->resetAfterTest();
$clock = $this->mock_clock_with_frozen();
// Record some generated text.
$record = new \stdClass();
$record->provider = 'openai';
$record->actionname = 'generate_text';
$record->actionid = 1;
$record->userid = 1;
$record->contextid = 1;
$record->success = true;
$record->timecreated = $clock->time() - 5;
$record->timecompleted = $clock->time();
$DB->insert_record('ai_action_register', $record);
// Record a generated image.
$record->actionname = 'generate_image';
$record->actionid = 2;
$record->timecreated = $clock->time() - 20;
$DB->insert_record('ai_action_register', $record);
// Record another image.
$record->actionid = 3;
$record->timecreated = $clock->time() - 10;
$DB->insert_record('ai_action_register', $record);
// Record some errors.
$record->actionname = 'generate_image';
$record->actionid = 4;
$record->success = false;
$record->errorcode = 403;
$DB->insert_record('ai_action_register', $record);
$record->actionid = 5;
$record->errorcode = 403;
$DB->insert_record('ai_action_register', $record);
$record->actionid = 6;
$record->errorcode = 404;
$DB->insert_record('ai_action_register', $record);
// Get our site info and check the expected calculations are correct.
$siteinfo = registration::get_site_info();
$aisuage = json_decode($siteinfo['aiusage']);
// Check generated text.
$this->assertEquals(1, $aisuage->openai->generate_text->success_count);
$this->assertEquals(0, $aisuage->openai->generate_text->fail_count);
// Check generated images.
$this->assertEquals(2, $aisuage->openai->generate_image->success_count);
$this->assertEquals(3, $aisuage->openai->generate_image->fail_count);
$this->assertEquals(15, $aisuage->openai->generate_image->average_time);
$this->assertEquals(403, $aisuage->openai->generate_image->predominant_error);
// Check time range is set correctly.
$this->assertEquals($clock->time() - WEEKSECS, $aisuage->time_range->timefrom);
$this->assertEquals($clock->time(), $aisuage->time_range->timeto);
}
}