mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-04-21 00:12:56 +02:00
Code Issues Projects Releases Wiki Activity

MDL-32188 question CBM: compute summary stats for CBM behaviours

Browse Source 
We now compute the average CBM score, accuracy, CBM bonus and enhanced
accuracy, both for the entire quiz, and for just the questions answered.

Note that these calculations must work correctly in the presence of
descriptions, ungraded questions, and manually graded questions. For
example, imagine a essay added at the end of the quiz "Summarise what
you learned attempting this exercise." This might have max mark zero or
non-zero. The CBM statistics just ignores questions like that.
This commit is contained in:
Tim Hunt 2013-09-30 16:26:21 +01:00
parent 9b2fe16bca
commit e74aa0aa97
9 changed files with 363 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mod/quiz/attemptlib.php
View File

@ -759,8 +759,6 @@ class quiz_attempt {
* The values are arrays with two items, title and content. Each of these
* will be either a string, or a renderable.
*
* @param question_display_options $options display options. Indicates what types
* of information should, or should not, be returned.
* @return array as described above.
*/
public function get_additional_summary_data(question_display_options $options) {

42
question/behaviour/behaviourbase.php
View File

@ -656,6 +656,19 @@ abstract class question_cbm {
);
/**#@-*/
/**#@+ @var array upper and lower limits of the optimal window. */
protected static $lowlimit = array(
self::LOW => 0,
self::MED => 0.666666666666667,
self::HIGH => 0.8,
);
protected static $highlimit = array(
self::LOW => 0.666666666666667,
self::MED => 0.8,
self::HIGH => 1,
);
/**#@-*/
/**
* @return int the default certaintly level that should be assuemd if
* the student does not choose one.
@ -686,10 +699,39 @@ abstract class question_cbm {
return get_string('certainty' . $certainty, 'qbehaviour_deferredcbm');
}
/**
* @param int $certainty one of the LOW/MED/HIGH constants.
* @return string a short textual description of this certainty.
*/
public static function get_short_string($certainty) {
return get_string('certaintyshort' . $certainty, 'qbehaviour_deferredcbm');
}
/**
* Add information about certainty to a response summary.
* @param string $summary the response summary.
* @param int $certainty the level of certainty to add.
*/
public static function summary_with_certainty($summary, $certainty) {
if (is_null($certainty)) {
return $summary;
}
return $summary . ' [' . self::get_string($certainty) . ']';
}
/**
* @param int $certainty one of the LOW/MED/HIGH constants.
* @return float the lower limit of the optimal probability range for this certainty.
*/
public static function optimal_probablility_low($certainty) {
return self::$lowlimit[$certainty];
}
/**
* @param int $certainty one of the LOW/MED/HIGH constants.
* @return float the upper limit of the optimal probability range for this certainty.
*/
public static function optimal_probablility_high($certainty) {
return self::$highlimit[$certainty];
}
}

2
question/behaviour/behaviourtypebase.php
View File

@ -86,8 +86,6 @@ abstract class question_behaviour_type {
* and their unit tests.
*
* @param question_usage_by_activity $quba the usage to provide summary data for.
* @param question_display_options $options display options. Indicates what types
* of information should, or should not, be returned.
* @return array as described above.
*/
public function summarise_usage(question_usage_by_activity $quba,

196
question/behaviour/deferredcbm/behaviourtype.php
View File

@ -38,4 +38,200 @@ class qbehaviour_deferredcbm_type extends qbehaviour_deferredfeedback_type {
public function adjust_random_guess_score($fraction) {
return question_cbm::adjust_fraction($fraction, question_cbm::default_certainty());
}
public function summarise_usage(question_usage_by_activity $quba, question_display_options $options) {
$summarydata = parent::summarise_usage($quba, $options);
if ($options->marks < question_display_options::MARK_AND_MAX) {
return $summarydata;
}
// Prepare accumulators to hold the data we are about to collect.
$notansweredcount = 0;
$notansweredweight = 0;
$attemptcount = array(
question_cbm::HIGH => 0,
question_cbm::MED => 0,
question_cbm::LOW => 0,
);
$totalweight = array(
question_cbm::HIGH => 0,
question_cbm::MED => 0,
question_cbm::LOW => 0,
);
$totalrawscore = array(
question_cbm::HIGH => 0,
question_cbm::MED => 0,
question_cbm::LOW => 0,
);
$totalcbmscore = array(
question_cbm::HIGH => 0,
question_cbm::MED => 0,
question_cbm::LOW => 0,
);
// Loop through the data, and add it to the accumulators.
foreach ($quba->get_attempt_iterator() as $qa) {
if (strpos($qa->get_behaviour_name(), 'cbm') === false || $qa->get_max_mark() < 0.0000005) {
continue;
}
$gradedstep = $qa->get_last_step_with_behaviour_var('_rawfraction');
if (!$gradedstep->has_behaviour_var('_rawfraction')) {
$notansweredcount += 1;
$notansweredweight += $qa->get_max_mark();
continue;
}
$certainty = $qa->get_last_behaviour_var('certainty');
if (is_null($certainty)) {
$certainty = question_cbm::default_certainty();
}
$attemptcount[$certainty] += 1;
$totalweight[$certainty] += $qa->get_max_mark();
$totalrawscore[$certainty] += $qa->get_max_mark() * $gradedstep->get_behaviour_var('_rawfraction');
$totalcbmscore[$certainty] += $qa->get_mark();
}
// Hence compute some statistics.
$totalquestions = $notansweredcount + array_sum($attemptcount);
$grandtotalweight = $notansweredweight + array_sum($totalweight);
$accuracy = array_sum($totalrawscore) / $grandtotalweight;
$averagecbm = array_sum($totalcbmscore) / $grandtotalweight;
$cbmbonus = $this->calculate_bonus($averagecbm, $accuracy);
$accuracyandbonus = $accuracy + $cbmbonus;
// Now we can start generating some of the summary: overall values.
$summarydata['qbehaviour_cbm_entire_quiz_heading'] = array(
'title' => '',
'content' => html_writer::tag('h3',
get_string('forentirequiz', 'qbehaviour_deferredcbm', $totalquestions),
array('class' => 'qbehaviour_deferredcbm_summary_heading')),
);
$summarydata['qbehaviour_cbm_entire_quiz_cbm_average'] = array(
'title' => get_string('averagecbmmark', 'qbehaviour_deferredcbm'),
'content' => format_float($averagecbm, $options->markdp),
);
$summarydata['qbehaviour_cbm_entire_quiz_accuracy'] = array(
'title' => get_string('accuracy', 'qbehaviour_deferredcbm'),
'content' => $this->format_probability($accuracy, 1),
);
$summarydata['qbehaviour_cbm_entire_quiz_cbm_bonus'] = array(
'title' => get_string('cbmbonus', 'qbehaviour_deferredcbm'),
'content' => $this->format_probability($cbmbonus, 1),
);
$summarydata['qbehaviour_cbm_entire_quiz_accuracy_and_bonus'] = array(
'title' => get_string('accuracyandbonus', 'qbehaviour_deferredcbm'),
'content' => $this->format_probability($accuracyandbonus, 1),
);
if ($notansweredcount && array_sum($attemptcount) > 0) {
$totalquestions = array_sum($attemptcount);
$grandtotalweight = array_sum($totalweight);
$accuracy = array_sum($totalrawscore) / $grandtotalweight;
$averagecbm = array_sum($totalcbmscore) / $grandtotalweight;
$cbmbonus = $this->calculate_bonus($averagecbm, $accuracy);
$accuracyandbonus = $accuracy + $cbmbonus;
$summarydata['qbehaviour_cbm_answered_quiz_heading'] = array(
'title' => '',
'content' => html_writer::tag('h3',
get_string('foransweredquestions', 'qbehaviour_deferredcbm', $totalquestions),
array('class' => 'qbehaviour_deferredcbm_summary_heading')),
);
$summarydata['qbehaviour_cbm_answered_quiz_cbm_average'] = array(
'title' => get_string('averagecbmmark', 'qbehaviour_deferredcbm'),
'content' => format_float($averagecbm, $options->markdp),
);
$summarydata['qbehaviour_cbm_answered_quiz_accuracy'] = array(
'title' => get_string('accuracy', 'qbehaviour_deferredcbm'),
'content' => $this->format_probability($accuracy, 1),
);
$summarydata['qbehaviour_cbm_answered_quiz_cbm_bonus'] = array(
'title' => get_string('cbmbonus', 'qbehaviour_deferredcbm'),
'content' => $this->format_probability($cbmbonus, 1),
);
$summarydata['qbehaviour_cbm_answered_quiz_accuracy_and_bonus'] = array(
'title' => get_string('accuracyandbonus', 'qbehaviour_deferredcbm'),
'content' => $this->format_probability($accuracyandbonus, 1),
);
}
// Now per-certainty level values.
$summarydata['qbehaviour_cbm_judgement_heading'] = array(
'title' => '',
'content' => html_writer::tag('h3', get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
array('class' => 'qbehaviour_deferredcbm_summary_heading')),
);
foreach ($attemptcount as $certainty => $count) {
$key = 'qbehaviour_cbm_judgement' . $certainty;
$title = question_cbm::get_short_string($certainty);
if ($count == 0) {
$summarydata[$key] = array(
'title' => $title,
'content' => get_string('noquestions', 'qbehaviour_deferredcbm'),
);
continue;
}
$lowerlimit = question_cbm::optimal_probablility_low($certainty);
$upperlimit = question_cbm::optimal_probablility_high($certainty);
$fraction = $totalrawscore[$certainty] / $totalweight[$certainty];
$a = new stdClass();
$a->responses = $count;
$a->idealrangelow = $this->format_probability($lowerlimit);
$a->idealrangehigh = $this->format_probability($upperlimit);
$a->fraction = html_writer::tag('span', $this->format_probability($fraction),
array('class' => 'qbehaviour_deferredcbm_actual_percentage'));
if ($fraction < $lowerlimit - 0.0000005) {
if ((pow($fraction - $lowerlimit, 2) * $count) > 0.5) { // Rough indicator of significance: t > 1.5 or 1.8.
$judgement = 'overconfident';
} else {
$judgement = 'slightlyoverconfident';
}
} else if ($fraction > $upperlimit + 0.0000005) {
if ((pow($fraction - $upperlimit, 2) * $count) > 0.5) {
$judgement = 'underconfident';
} else {
$judgement = 'slightlyunderconfident';
}
} else {
$judgement = 'judgementok';
}
$a->judgement = html_writer::tag('span', get_string($judgement, 'qbehaviour_deferredcbm'),
array('class' => 'qbehaviour_deferredcbm_' . $judgement));
$summarydata[$key] = array(
'title' => $title,
'content' => get_string('judgementsummary', 'qbehaviour_deferredcbm', $a),
);
}
return $summarydata;
}
protected function format_probability($probability, $dp = 0) {
return format_float($probability * 100, $dp) . '%';
}
public function calculate_bonus($total, $accuracy) {
$expectedforaccuracy = max(
$accuracy * question_cbm::adjust_fraction(1, question_cbm::LOW) +
(1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::LOW),
$accuracy * question_cbm::adjust_fraction(1, question_cbm::MED) +
(1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::MED),
$accuracy * question_cbm::adjust_fraction(1, question_cbm::HIGH) +
(1 - $accuracy) * question_cbm::adjust_fraction(0, question_cbm::HIGH)
);
// The constant 0.1 here is determinted empirically from looking at lots
// for CBM quiz results. See www.ucl.ac.uk/~ucgbarg/tea/IUPS_2013a.pdf.
// It approximately maximises the reliability of accuracy + bonus.
return 0.1 * ($total - $expectedforaccuracy);
}
}

20
question/behaviour/deferredcbm/lang/en/qbehaviour_deferredcbm.php
View File

@ -23,8 +23,13 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
$string['accuracy'] = 'Accuracy';
$string['accuracyandbonus'] = 'Accuracy + Bonus';
$string['assumingcertainty'] = 'You did not select a certainty. Assuming: {$a}.';
$string['averagecbmmark'] = 'Average CBM mark';
$string['basemark'] = 'Base mark {$a}';
$string['breakdownbycertainty'] = 'Break-down by certainty';
$string['cbmbonus'] = 'CBM bonus';
$string['cbmmark'] = 'CBM mark {$a}';
$string['certainty'] = 'Certainty';
$string['certainty_help'] = 'Certainty-based marking requires you to indicate how reliable you think your answer is. The available levels are:
@ -41,5 +46,18 @@ $string['certainty_link'] = 'qbehaviour/deferredcbm/certainty';
$string['certainty1'] = 'C=1 (Unsure: <67%)';
$string['certainty2'] = 'C=2 (Mid: >67%)';
$string['certainty3'] = 'C=3 (Quite sure: >80%)';
$string['howcertainareyou'] = 'How certain are you? {$a}';
$string['certaintyshort1'] = 'C=1';
$string['certaintyshort2'] = 'C=2';
$string['certaintyshort3'] = 'C=3';
$string['dontknow'] = 'No idea';
$string['foransweredquestions'] = 'For just the {$a} answered questions';
$string['forentirequiz'] = 'For the entire quiz ({$a} questions)';
$string['judgementok'] = 'OK';
$string['judgementsummary'] = 'Responses: {$a->responses}. Accuracy: {$a->fraction}. (Optimal range {$a->idealrangelow} to {$a->idealrangehigh}). You were {$a->judgement} using this certainty level.';
$string['howcertainareyou'] = 'Certainty{$a->help}: {$a->choices}';
$string['noquestions'] = 'No responses';
$string['overconfident'] = 'over-confident';
$string['pluginname'] = 'Deferred feedback with CBM';
$string['slightlyoverconfident'] = 'a bit over-confident';
$string['slightlyunderconfident'] = 'a bit under-confident';
$string['underconfident'] = 'under-confident';

23
question/behaviour/deferredcbm/styles.css
View File

@ -1,3 +1,26 @@
.qbehaviour_deferredcbm_slightlyunderconfident,
.qbehaviour_deferredcbm_slightlyoverconfident {
font-weight: bold;
color: #600;
}
.qbehaviour_deferredcbm_underconfident,
.qbehaviour_deferredcbm_overconfident {
font-weight: bold;
color: #c00;
}
.qbehaviour_deferredcbm_judgementok {
font-weight: bold;
color: #080;
}
.qbehaviour_deferredcbm_actual_percentage {
font-weight: bold;
}
.qbehaviour_deferredcbm_summary_heading {
margin: 0;
}
.que.deferredcbm .certaintychoices input[type="radio"] {
margin-left: 0.5em;
}

78
question/behaviour/deferredcbm/tests/behaviourtype_test.php
View File

@ -61,4 +61,82 @@ class qbehaviour_deferredcbm_type_test extends qbehaviour_walkthrough_test_base
$this->assertEquals(0, $this->behaviourtype->adjust_random_guess_score(0));
$this->assertEquals(1, $this->behaviourtype->adjust_random_guess_score(1));
}
public function test_summarise_usage_max_mark_1() {
// Create a usage comprising 3 true-false questions.
$this->quba->set_preferred_behaviour('deferredcbm');
$this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 3);
$this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 3);
$this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 3);
$this->quba->start_all_questions();
// Process responses right, high certainty; right, med certainty; wrong, med certainty.
$this->quba->process_action(1, array('answer' => 1, '-certainty' => 3));
$this->quba->process_action(2, array('answer' => 1, '-certainty' => 2));
$this->quba->process_action(3, array('answer' => 0, '-certainty' => 2));
$this->quba->finish_all_questions();
// Get the summary.
$summarydata = $this->quba->get_summary_information(new question_display_options());
// Verify.
$this->assertContains(get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
$summarydata['qbehaviour_cbm_judgement_heading']['content']);
$this->assertContains('100%',
$summarydata['qbehaviour_cbm_judgement3']['content']);
$this->assertContains(get_string('judgementok', 'qbehaviour_deferredcbm'),
$summarydata['qbehaviour_cbm_judgement3']['content']);
$this->assertContains('50%',
$summarydata['qbehaviour_cbm_judgement2']['content']);
$this->assertContains(get_string('slightlyoverconfident', 'qbehaviour_deferredcbm'),
$summarydata['qbehaviour_cbm_judgement2']['content']);
$this->assertContains(get_string('noquestions', 'qbehaviour_deferredcbm'),
$summarydata['qbehaviour_cbm_judgement1']['content']);
}
public function test_summarise_usage_max_mark_3() {
// Create a usage comprising 3 true-false questions.
$this->quba->set_preferred_behaviour('deferredcbm');
$this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 1);
$this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 1);
$this->quba->add_question(test_question_maker::make_question('truefalse', 'true'), 1);
$this->quba->start_all_questions();
// Process responses right, high certainty; right, med certainty; wrong, med certainty.
$this->quba->process_action(1, array('answer' => 1, '-certainty' => 3));
$this->quba->process_action(2, array('answer' => 1, '-certainty' => 2));
$this->quba->process_action(3, array('answer' => 0, '-certainty' => 2));
$this->quba->finish_all_questions();
// Get the summary.
$summarydata = $this->quba->get_summary_information(new question_display_options());
// Verify.
$this->assertContains(get_string('breakdownbycertainty', 'qbehaviour_deferredcbm'),
$summarydata['qbehaviour_cbm_judgement_heading']['content']);
$this->assertContains('100%',
$summarydata['qbehaviour_cbm_judgement3']['content']);
$this->assertContains(get_string('judgementok', 'qbehaviour_deferredcbm'),
$summarydata['qbehaviour_cbm_judgement3']['content']);
$this->assertContains('50%',
$summarydata['qbehaviour_cbm_judgement2']['content']);
$this->assertContains(get_string('slightlyoverconfident', 'qbehaviour_deferredcbm'),
$summarydata['qbehaviour_cbm_judgement2']['content']);
$this->assertContains(get_string('noquestions', 'qbehaviour_deferredcbm'),
$summarydata['qbehaviour_cbm_judgement1']['content']);
}
public function test_calculate_bonus() {
$this->assertEquals(0.05, $this->behaviourtype->calculate_bonus(1, 1/2));
$this->assertEquals(-0.01, $this->behaviourtype->calculate_bonus(2, 9/10));
$this->assertEquals(0, $this->behaviourtype->calculate_bonus(3, 1));
}
}

9
question/behaviour/immediatecbm/behaviourtype.php
View File

@ -25,7 +25,7 @@
defined('MOODLE_INTERNAL') || die();
require_once(dirname(__FILE__) . '/../immediatefeedback/behaviourtype.php');
require_once(dirname(__FILE__) . '/../deferredcbm/behaviourtype.php');
/**
@ -34,8 +34,9 @@ require_once(dirname(__FILE__) . '/../immediatefeedback/behaviourtype.php');
* @copyright 2012 The Open University
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class qbehaviour_immediatecbm_type extends qbehaviour_immediatefeedback_type {
public function adjust_random_guess_score($fraction) {
return question_cbm::adjust_fraction($fraction, question_cbm::default_certainty());
class qbehaviour_immediatecbm_type extends qbehaviour_deferredcbm_type {
public function get_unused_display_options() {
return array();
}
}

2
question/engine/questionusage.php
View File

@ -340,8 +340,6 @@ class question_usage_by_activity {
* The values are arrays with two items, title and content. Each of these
* will be either a string, or a renderable.
*
* @param question_display_options $options display options. Indicates what types
* of information should, or should not, be returned.
* @return array as described above.
*/
public function get_summary_information(question_display_options $options) {