mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-01-17 21:49:15 +01:00
Code Issues Projects Releases Wiki Activity

MDL-66498 analytics: SQL IN in chunks

Browse Source
This commit is contained in:
David Monllao 2019-08-27 06:34:56 +08:00 committed by David Monllaó
parent a672f021ea
commit 500111c5d3
2 changed files with 46 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
analytics/classes/manager.php
View File

@ -575,8 +575,22 @@ class manager {
}
// Clean up stuff that depends on analysable ids that do not exist anymore.
$models = self::get_all_models();
foreach ($models as $model) {
// We first dump into memory the list of analysables we have in the database (we could probably do this with 1 single
// query for the 3 tables, but it may be safer to do it separately).
$predictsamplesanalysableids = $DB->get_fieldset_select('analytics_predict_samples', 'DISTINCT analysableid',
'modelid = :modelid', ['modelid' => $model->get_id()]);
$predictsamplesanalysableids = array_flip($predictsamplesanalysableids);
$trainsamplesanalysableids = $DB->get_fieldset_select('analytics_train_samples', 'DISTINCT analysableid',
'modelid = :modelid', ['modelid' => $model->get_id()]);
$trainsamplesanalysableids = array_flip($trainsamplesanalysableids);
$usedanalysablesanalysableids = $DB->get_fieldset_select('analytics_used_analysables', 'DISTINCT analysableid',
'modelid = :modelid', ['modelid' => $model->get_id()]);
$usedanalysablesanalysableids = array_flip($usedanalysablesanalysableids);
$analyser = $model->get_analyser(array('notimesplitting' => true));
$analysables = $analyser->get_analysables_iterator();
@ -585,17 +599,37 @@ class manager {
if (!$analysable) {
continue;
}
$analysableids[] = $analysable->get_id();
}
if (empty($analysableids)) {
continue;
unset($predictsamplesanalysableids[$analysable->get_id()]);
unset($trainsamplesanalysableids[$analysable->get_id()]);
unset($usedanalysablesanalysableids[$analysable->get_id()]);
}
list($notinsql, $params) = $DB->get_in_or_equal($analysableids, SQL_PARAMS_NAMED, 'param', false);
$params['modelid'] = $model->get_id();
$param = ['modelid' => $model->get_id()];
$DB->delete_records_select('analytics_predict_samples', "modelid = :modelid AND analysableid $notinsql", $params);
$DB->delete_records_select('analytics_train_samples', "modelid = :modelid AND analysableid $notinsql", $params);
if ($predictsamplesanalysableids) {
$chunks = array_chunk(array_flip($predictsamplesanalysableids), 1000);
foreach ($chunks as $chunk) {
list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_predict_samples', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
}
if ($trainsamplesanalysableids) {
$chunks = array_chunk(array_flip($trainsamplesanalysableids), 1000);
foreach ($chunks as $chunk) {
list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_train_samples', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
}
if ($usedanalysablesanalysableids) {
$chunks = array_chunk(array_flip($usedanalysablesanalysableids), 1000);
foreach ($chunks as $chunk) {
list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_used_analysables', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
}
}
}

6
analytics/tests/manager_test.php
View File

@ -134,8 +134,9 @@ class analytics_manager_testcase extends advanced_testcase {
$model->train();
$model->predict();
$npredictsamples = $DB->count_records('analytics_predict_samples');
$ntrainsamples = $DB->count_records('analytics_train_samples');
$this->assertNotEmpty($DB->count_records('analytics_predict_samples'));
$this->assertNotEmpty($DB->count_records('analytics_train_samples'));
$this->assertNotEmpty($DB->count_records('analytics_used_analysables'));
// Now we delete an analysable, stored predict and training samples should be deleted.
$deletedcontext = \context_course::instance($coursepredict1->id);
@ -145,6 +146,7 @@ class analytics_manager_testcase extends advanced_testcase {
$this->assertEmpty($DB->count_records('analytics_predict_samples', array('analysableid' => $coursepredict1->id)));
$this->assertEmpty($DB->count_records('analytics_train_samples', array('analysableid' => $coursepredict1->id)));
$this->assertEmpty($DB->count_records('analytics_used_analysables', array('analysableid' => $coursepredict1->id)));
set_config('enabled_stores', '', 'tool_log');
get_log_manager(true);