From 500111c5d38565e6d73f262b7408691336508d74 Mon Sep 17 00:00:00 2001
From: David Monllao <davidm@moodle.com>
Date: Tue, 27 Aug 2019 06:34:56 +0800
Subject: [PATCH] MDL-66498 analytics: SQL IN in chunks

---
 analytics/classes/manager.php    | 50 +++++++++++++++++++++++++++-----
 analytics/tests/manager_test.php |  6 ++--
 2 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/analytics/classes/manager.php b/analytics/classes/manager.php
index 2f476108af1..865518d1f68 100644
--- a/analytics/classes/manager.php
+++ b/analytics/classes/manager.php
@@ -575,8 +575,22 @@ class manager {
         }
 
         // Clean up stuff that depends on analysable ids that do not exist anymore.
+
         $models = self::get_all_models();
         foreach ($models as $model) {
+
+            // We first dump into memory the list of analysables we have in the database (we could probably do this with 1 single
+            // query for the 3 tables, but it may be safer to do it separately).
+            $predictsamplesanalysableids = $DB->get_fieldset_select('analytics_predict_samples', 'DISTINCT analysableid',
+                'modelid = :modelid', ['modelid' => $model->get_id()]);
+            $predictsamplesanalysableids = array_flip($predictsamplesanalysableids);
+            $trainsamplesanalysableids = $DB->get_fieldset_select('analytics_train_samples', 'DISTINCT analysableid',
+                'modelid = :modelid', ['modelid' => $model->get_id()]);
+            $trainsamplesanalysableids = array_flip($trainsamplesanalysableids);
+            $usedanalysablesanalysableids = $DB->get_fieldset_select('analytics_used_analysables', 'DISTINCT analysableid',
+                'modelid = :modelid', ['modelid' => $model->get_id()]);
+            $usedanalysablesanalysableids = array_flip($usedanalysablesanalysableids);
+
             $analyser = $model->get_analyser(array('notimesplitting' => true));
             $analysables = $analyser->get_analysables_iterator();
 
@@ -585,17 +599,37 @@ class manager {
                 if (!$analysable) {
                     continue;
                 }
-                $analysableids[] = $analysable->get_id();
-            }
-            if (empty($analysableids)) {
-                continue;
+                unset($predictsamplesanalysableids[$analysable->get_id()]);
+                unset($trainsamplesanalysableids[$analysable->get_id()]);
+                unset($usedanalysablesanalysableids[$analysable->get_id()]);
             }
 
-            list($notinsql, $params) = $DB->get_in_or_equal($analysableids, SQL_PARAMS_NAMED, 'param', false);
-            $params['modelid'] = $model->get_id();
+            $param = ['modelid' => $model->get_id()];
 
-            $DB->delete_records_select('analytics_predict_samples', "modelid = :modelid AND analysableid $notinsql", $params);
-            $DB->delete_records_select('analytics_train_samples', "modelid = :modelid AND analysableid $notinsql", $params);
+            if ($predictsamplesanalysableids) {
+                $chunks = array_chunk(array_flip($predictsamplesanalysableids), 1000);
+                foreach ($chunks as $chunk) {
+                    list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
+                    $DB->delete_records_select('analytics_predict_samples', "modelid = :modelid AND analysableid $idssql",
+                        $param + $idsparams);
+                }
+            }
+            if ($trainsamplesanalysableids) {
+                $chunks = array_chunk(array_flip($trainsamplesanalysableids), 1000);
+                foreach ($chunks as $chunk) {
+                    list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
+                    $DB->delete_records_select('analytics_train_samples', "modelid = :modelid AND analysableid $idssql",
+                        $param + $idsparams);
+                }
+            }
+            if ($usedanalysablesanalysableids) {
+                $chunks = array_chunk(array_flip($usedanalysablesanalysableids), 1000);
+                foreach ($chunks as $chunk) {
+                    list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
+                    $DB->delete_records_select('analytics_used_analysables', "modelid = :modelid AND analysableid $idssql",
+                        $param + $idsparams);
+                }
+            }
         }
     }
 
diff --git a/analytics/tests/manager_test.php b/analytics/tests/manager_test.php
index 167d41dde0c..74e67216b63 100644
--- a/analytics/tests/manager_test.php
+++ b/analytics/tests/manager_test.php
@@ -134,8 +134,9 @@ class analytics_manager_testcase extends advanced_testcase {
         $model->train();
         $model->predict();
 
-        $npredictsamples = $DB->count_records('analytics_predict_samples');
-        $ntrainsamples = $DB->count_records('analytics_train_samples');
+        $this->assertNotEmpty($DB->count_records('analytics_predict_samples'));
+        $this->assertNotEmpty($DB->count_records('analytics_train_samples'));
+        $this->assertNotEmpty($DB->count_records('analytics_used_analysables'));
 
         // Now we delete an analysable, stored predict and training samples should be deleted.
         $deletedcontext = \context_course::instance($coursepredict1->id);
@@ -145,6 +146,7 @@ class analytics_manager_testcase extends advanced_testcase {
 
         $this->assertEmpty($DB->count_records('analytics_predict_samples', array('analysableid' => $coursepredict1->id)));
         $this->assertEmpty($DB->count_records('analytics_train_samples', array('analysableid' => $coursepredict1->id)));
+        $this->assertEmpty($DB->count_records('analytics_used_analysables', array('analysableid' => $coursepredict1->id)));
 
         set_config('enabled_stores', '', 'tool_log');
         get_log_manager(true);