MDL-64787 tool_analytics: UI for trained models evaluation

admin/tool/analytics/amd/build/model.min.js

@@ -1 +1 @@
-define(["jquery","core/str","core/log","core/notification","core/modal_factory","core/modal_events"],function(a,b,c,d,e,f){var g={clear:{title:{key:"clearpredictions",component:"tool_analytics"},body:{key:"clearmodelpredictions",component:"tool_analytics"}},"delete":{title:{key:"delete",component:"tool_analytics"},body:{key:"deletemodelconfirmation",component:"tool_analytics"}}},h=function(b){return a(b.closest("tr")[0]).find("span.target-name").text()};return{confirmAction:function(i,j){a('[data-action-id="'+i+'"]').on("click",function(i){i.preventDefault();var k=a(i.currentTarget);if("undefined"==typeof g[j])return void c.error('Action "'+j+'" is not allowed.');var l=[g[j].title,g[j].body];l[1].param=h(k);var m=b.get_strings(l),n=e.create({type:e.types.SAVE_CANCEL});a.when(m,n).then(function(a,b){return b.setTitle(a[0]),b.setBody(a[1]),b.setSaveButtonText(a[0]),b.getRoot().on(f.save,function(){window.location.href=k.attr("href")}),b.show(),b}).fail(d.exception)})}}});
+define(["jquery","core/str","core/log","core/notification","core/modal_factory","core/modal_events"],function(a,b,c,d,e,f){var g={clear:{title:{key:"clearpredictions",component:"tool_analytics"},body:{key:"clearmodelpredictions",component:"tool_analytics"}},"delete":{title:{key:"delete",component:"tool_analytics"},body:{key:"deletemodelconfirmation",component:"tool_analytics"}}},h='<div class="box mb-4">{{evaluationmodeinfo}}</div><div class="form-check"><input class="form-check-input" type="radio" name="evaluationmode" id="id-mode-trainedmodel" value="trainedmodel" checked><label class="form-check-label" for="id-mode-trainedmodel">{{trainedmodellabel}}</label></div><div class="form-check"><input class="form-check-input" type="radio" name="evaluationmode" id="id-mode-configuration" value="configuration"><label class="form-check-label" for="id-mode-configuration">{{configurationlabel}}</label></div>',i=function(b){return a(b.closest("tr")[0]).find("span.target-name").text()};return{confirmAction:function(h,j){a('[data-action-id="'+h+'"]').on("click",function(h){h.preventDefault();var k=a(h.currentTarget);if("undefined"==typeof g[j])return void c.error('Action "'+j+'" is not allowed.');var l=[g[j].title,g[j].body];l[1].param=i(k);var m=b.get_strings(l),n=e.create({type:e.types.SAVE_CANCEL});a.when(m,n).then(function(a,b){return b.setTitle(a[0]),b.setBody(a[1]),b.setSaveButtonText(a[0]),b.getRoot().on(f.save,function(){window.location.href=k.attr("href")}),b.show(),b}).fail(d.exception)})},selectEvaluationMode:function(c,g){a('[data-action-id="'+c+'"]').on("click",function(c){c.preventDefault();var i=a(c.currentTarget);if(!g)return void(window.location.href=i.attr("href"));var j=b.get_strings([{key:"evaluatemodel",component:"tool_analytics"},{key:"evaluationmode",component:"tool_analytics"},{key:"evaluationmodeinfo",component:"tool_analytics"},{key:"evaluationmodetrainedmodel",component:"tool_analytics"},{key:"evaluationmodeconfiguration",component:"tool_analytics"}]),k=e.create({type:e.types.SAVE_CANCEL});a.when(j,k).then(function(b,c){c.getRoot().on(f.hidden,c.destroy.bind(c)),c.setTitle(b[1]),c.setSaveButtonText(b[0]);var d=h.replace(/{{evaluationmodeinfo}}/,b[2]).replace(/{{trainedmodellabel}}/,b[3]).replace(/{{configurationlabel}}/,b[4]);return c.setBody(d),c.getRoot().on(f.save,function(){var b=a("input[name='evaluationmode']:checked").val();"trainedmodel"==b&&i.attr("href",i.attr("href")+"&mode=trainedmodel"),window.location.href=i.attr("href")}),c.show(),c}).fail(d.exception)})}}});

admin/tool/analytics/amd/src/model.js

@@ -48,6 +48,20 @@ define(['jquery', 'core/str', 'core/log', 'core/notification', 'core/modal_facto
         }
     };
 
+    /**
+     * Template to display the evaluation mode choices.
+     */
+    var evaluationRadioHTML = '<div class="box mb-4">{{evaluationmodeinfo}}</div>' +
+        '<div class="form-check">' +
+            '<input class="form-check-input" type="radio" name="evaluationmode" id="id-mode-trainedmodel" value="trainedmodel" ' +
+                'checked>' +
+            '<label class="form-check-label" for="id-mode-trainedmodel">{{trainedmodellabel}}</label>' +
+        '</div>' +
+        '<div class="form-check">' +
+            '<input class="form-check-input" type="radio" name="evaluationmode" id="id-mode-configuration" value="configuration">' +
+            '<label class="form-check-label" for="id-mode-configuration">{{configurationlabel}}</label>' +
+        '</div>';
+
     /**
      * Returns the model name.
      *
@@ -94,10 +108,77 @@ define(['jquery', 'core/str', 'core/log', 'core/notification', 'core/modal_facto
                     modal.getRoot().on(ModalEvents.save, function() {
                         window.location.href = a.attr('href');
                     });
+                    modal.show();
+                    return modal;
+                }).fail(Notification.exception);
+            });
+        },
+
+        /**
+         * Displays a select-evaluation-mode choice.
+         *
+         * @param  {String}  actionId
+         * @param  {Boolean} trainedOnlyExternally
+         */
+        selectEvaluationMode: function(actionId, trainedOnlyExternally) {
+            $('[data-action-id="' + actionId + '"]').on('click', function(ev) {
+                ev.preventDefault();
+
+                var a = $(ev.currentTarget);
+
+                if (!trainedOnlyExternally) {
+                    // We can not evaluate trained models if the model was trained using data from this site.
+                    // Default to evaluate the model configuration if that is the case.
+                    window.location.href = a.attr('href');
+                    return;
+                }
+
+                var stringsPromise = Str.get_strings([
+                    {
+                        key: 'evaluatemodel',
+                        component: 'tool_analytics'
+                    }, {
+                        key: 'evaluationmode',
+                        component: 'tool_analytics'
+                    }, {
+                        key: 'evaluationmodeinfo',
+                        component: 'tool_analytics'
+                    }, {
+                        key: 'evaluationmodetrainedmodel',
+                        component: 'tool_analytics'
+                    }, {
+                        key: 'evaluationmodeconfiguration',
+                        component: 'tool_analytics'
+                    }
+                ]);
+                var modalPromise = ModalFactory.create({type: ModalFactory.types.SAVE_CANCEL});
+
+                $.when(stringsPromise, modalPromise).then(function(strings, modal) {
+
+
+                    modal.getRoot().on(ModalEvents.hidden, modal.destroy.bind(modal));
+
+                    modal.setTitle(strings[1]);
+                    modal.setSaveButtonText(strings[0]);
+
+                    var body = evaluationRadioHTML.replace(/{{evaluationmodeinfo}}/, strings[2])
+                        .replace(/{{trainedmodellabel}}/, strings[3])
+                        .replace(/{{configurationlabel}}/, strings[4]);
+                    modal.setBody(body);
+
+                    modal.getRoot().on(ModalEvents.save, function() {
+                        var evaluationMode = $("input[name='evaluationmode']:checked").val();
+                        if (evaluationMode == 'trainedmodel') {
+                            a.attr('href', a.attr('href') + '&mode=trainedmodel');
+                        }
+                        window.location.href = a.attr('href');
+                        return;
+                    });
+
                     modal.show();
                     return modal;
                 }).fail(Notification.exception);
             });
         }
     };
-});
+});

admin/tool/analytics/classes/output/models_list.php

@@ -187,10 +187,16 @@ class models_list implements \renderable, \templatable {
 
             // Evaluate machine-learning-based models.
             if (!$onlycli && $model->get_indicators() && !$model->is_static()) {
+
+                // Extra is_trained call as trained_locally returns false if the model has not been trained yet.
+                $trainedonlyexternally = !$model->trained_locally() && $model->is_trained();
+
+                $actionid = 'evaluate-' . $model->get_id();
+                $PAGE->requires->js_call_amd('tool_analytics/model', 'selectEvaluationMode', [$actionid, $trainedonlyexternally]);
                 $urlparams['action'] = 'evaluate';
                 $url = new \moodle_url('model.php', $urlparams);
                 $icon = new \action_menu_link_secondary($url, new \pix_icon('i/calc', get_string('evaluate', 'tool_analytics')),
-                    get_string('evaluate', 'tool_analytics'));
+                    get_string('evaluate', 'tool_analytics'), ['data-action-id' => $actionid]);
                 $actionsmenu->add($icon);
             }
 

admin/tool/analytics/cli/evaluate_model.php

@@ -35,6 +35,8 @@ Options:
 --non-interactive      Not interactive questions
 --timesplitting        Restrict the evaluation to 1 single time splitting method (Optional)
 --filter               Analyser dependant. e.g. A courseid would evaluate the model using a single course (Optional)
+--mode                 'configuration' or 'trainedmodel'. You can only use mode=trainedmodel when the trained" .
+    " model was imported" . "
 --reuse-prev-analysed  Reuse recently analysed courses instead of analysing the whole site. Set it to false while" .
     " coding indicators. Defaults to true (Optional)" . "
 -h, --help             Print out this help
@@ -50,6 +52,7 @@ list($options, $unrecognized) = cli_get_params(
         'modelid'               => false,
         'list'                  => false,
         'timesplitting'         => false,
+        'mode'                  => 'configuration',
         'reuse-prev-analysed'   => true,
         'non-interactive'       => false,
         'filter'                => false
@@ -74,6 +77,10 @@ if ($options['filter'] !== false) {
     $options['filter'] = explode(',', $options['filter']);
 }
 
+if ($options['mode'] !== 'configuration' && $options['mode'] !== 'trainedmodel') {
+    cli_error('Error: The provided mode is not supported');
+}
+
 // We need admin permissions.
 \core\session\manager::set_user(get_admin());
 
@@ -89,6 +96,7 @@ $analyseroptions = array(
     'filter' => $options['filter'],
     'timesplitting' => $options['timesplitting'],
     'reuseprevanalysed' => $options['reuse-prev-analysed'],
+    'mode' => $options['mode'],
 );
 // Evaluate its suitability to predict accurately.
 $results = $model->evaluate($analyseroptions);

admin/tool/analytics/lang/en/tool_analytics.php

@@ -53,6 +53,10 @@ $string['erroronlycli'] = 'Execution only allowed via command line';
 $string['errortrainingdataexport'] = 'The model training data could not be exported';
 $string['evaluate'] = 'Evaluate';
 $string['evaluatemodel'] = 'Evaluate model';
+$string['evaluationmode'] = 'Evaluation mode';
+$string['evaluationmodeinfo'] = 'This model has been trained using data from another site. You can evaluate the performance of the trained model on your site, or you can evaluate the performance of this model configuration using the data available on this site.';
+$string['evaluationmodetrainedmodel'] = 'Evaluate the trained model';
+$string['evaluationmodeconfiguration'] = 'Evaluate the model configuration';
 $string['evaluationinbatches'] = 'The site contents are calculated and stored in batches. The evaluation process may be stopped at any time. The next time it is run, it will continue from the point when it was stopped.';
 $string['exportmodel'] = 'Export configuration';
 $string['exporttrainingdata'] = 'Export training data';

admin/tool/analytics/model.php

@@ -169,7 +169,13 @@ switch ($action) {
         // Web interface is used by people who can not use CLI nor code stuff, always use
         // cached stuff as they will change the model through the web interface as well
         // which invalidates the previously analysed stuff.
-        $results = $model->evaluate(array('reuseprevanalysed' => true));
+        $options = ['reuseprevanalysed' => true];
+
+        $mode = optional_param('mode', false, PARAM_ALPHANUM);
+        if ($mode == 'trainedmodel') {
+            $options['mode'] = 'trainedmodel';
+        }
+        $results = $model->evaluate($options);
         $renderer = $PAGE->get_renderer('tool_analytics');
         echo $renderer->render_evaluate_results($results, $model->get_analyser()->get_logs());
         break;

analytics/classes/model.php

@@ -1462,6 +1462,29 @@ class model {
         return \core_analytics\dataset_manager::export_training_data($this->get_id(), $timesplittingid);
     }
 
+    /**
+     * Has the model been trained using data from this site?
+     *
+     * This method is useful to determine if a trained model can be evaluated as
+     * we can not use the same data for training and for evaluation.
+     *
+     * @return bool
+     */
+    public function trained_locally() {
+        global $DB;
+
+        if (!$this->is_trained() || $this->is_static()) {
+            // Early exit.
+            return false;
+        }
+
+        if ($DB->record_exists('analytics_train_samples', ['modelid' => $this->model->id])) {
+            return true;
+        }
+
+        return false;
+    }
+
     /**
      * Flag the provided file as used for training or prediction.
      *