mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-04-20 07:56:06 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'MDL-66496-master' of git://github.com/tobiasreischmann/moodle

Browse Source
This commit is contained in:
Andrew Nicols 2019-10-17 11:05:34 +08:00
commit a7139f7600
6 changed files with 406 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
mod/data/import.php
View File

@ -91,93 +91,8 @@ if (!$formdata = $form->get_data()) {
echo $OUTPUT->footer();
die;
} else {
// Large files are likely to take their time and memory. Let PHP know
// that we'll take longer, and that the process should be recycled soon
// to free up memory.
core_php_time_limit::raise();
raise_memory_limit(MEMORY_EXTRA);
$iid = csv_import_reader::get_new_iid('moddata');
$cir = new csv_import_reader($iid, 'moddata');
$filecontent = $form->get_file_content('recordsfile');
$readcount = $cir->load_csv_content($filecontent, $formdata->encoding, $formdata->fielddelimiter);
unset($filecontent);
if (empty($readcount)) {
print_error('csvfailed','data',"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}");
} else {
if (!$fieldnames = $cir->get_columns()) {
print_error('cannotreadtmpfile', 'error');
}
$fieldnames = array_flip($fieldnames);
// check the fieldnames are valid
$rawfields = $DB->get_records('data_fields', array('dataid' => $data->id), '', 'name, id, type');
$fields = array();
$errorfield = '';
$safetoskipfields = array(get_string('user'), get_string('username'), get_string('email'),
get_string('timeadded', 'data'), get_string('timemodified', 'data'),
get_string('approved', 'data'), get_string('tags', 'data'));
foreach ($fieldnames as $name => $id) {
if (!isset($rawfields[$name])) {
if (!in_array($name, $safetoskipfields)) {
$errorfield .= "'$name' ";
}
} else {
$field = $rawfields[$name];
require_once("$CFG->dirroot/mod/data/field/$field->type/field.class.php");
$classname = 'data_field_' . $field->type;
$fields[$name] = new $classname($field, $data, $cm);
}
}
if (!empty($errorfield)) {
print_error('fieldnotmatched','data',"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}",$errorfield);
}
$cir->init();
$recordsadded = 0;
while ($record = $cir->next()) {
if ($recordid = data_add_record($data, 0)) { // add instance to data_record
foreach ($fields as $field) {
$fieldid = $fieldnames[$field->field->name];
if (isset($record[$fieldid])) {
$value = $record[$fieldid];
} else {
$value = '';
}
if (method_exists($field, 'update_content_import')) {
$field->update_content_import($recordid, $value, 'field_' . $field->field->id);
} else {
$content = new stdClass();
$content->fieldid = $field->field->id;
$content->content = $value;
$content->recordid = $recordid;
$DB->insert_record('data_content', $content);
}
}
if (core_tag_tag::is_enabled('mod_data', 'data_records') &&
isset($fieldnames[get_string('tags', 'data')])) {
$columnindex = $fieldnames[get_string('tags', 'data')];
$rawtags = $record[$columnindex];
$tags = explode(',', $rawtags);
foreach ($tags as $tag) {
$tag = trim($tag);
if (empty($tag)) {
continue;
}
core_tag_tag::add_item_tag('mod_data', 'data_records', $recordid, $context, $tag);
}
}
$recordsadded++;
print get_string('added', 'moodle', $recordsadded) . ". " . get_string('entry', 'data') . " (ID $recordid)<br />\n";
}
}
$cir->close();
$cir->cleanup(true);
}
$recordsadded = data_import_csv($cm, $data, $filecontent, $formdata->encoding, $formdata->fielddelimiter);
}
if ($recordsadded > 0) {

134
mod/data/lib.php
View File

@ -973,16 +973,17 @@ function data_numentries($data, $userid=null) {
* @global object
* @param object $data
* @param int $groupid
* @param int $userid
* @return bool
*/
function data_add_record($data, $groupid=0){
function data_add_record($data, $groupid=0, $userid=null) {
global $USER, $DB;
$cm = get_coursemodule_from_instance('data', $data->id);
$context = context_module::instance($cm->id);
$record = new stdClass();
$record->userid = $USER->id;
$record->userid = $userid ?? $USER->id;
$record->dataid = $data->id;
$record->groupid = $groupid;
$record->timecreated = $record->timemodified = time();
@ -3044,6 +3045,135 @@ function data_supports($feature) {
default: return null;
}
}
/**
* Import records for a data instance from csv data.
*
* @param object $cm Course module of the data instance.
* @param object $data The data instance.
* @param string $csvdata The csv data to be imported.
* @param string $encoding The encoding of csv data.
* @param string $fielddelimiter The delimiter of the csv data.
* @return int Number of records added.
*/
function data_import_csv($cm, $data, &$csvdata, $encoding, $fielddelimiter) {
global $CFG, $DB;
// Large files are likely to take their time and memory. Let PHP know
// that we'll take longer, and that the process should be recycled soon
// to free up memory.
core_php_time_limit::raise();
raise_memory_limit(MEMORY_EXTRA);
$iid = csv_import_reader::get_new_iid('moddata');
$cir = new csv_import_reader($iid, 'moddata');
$context = context_module::instance($cm->id);
$readcount = $cir->load_csv_content($csvdata, $encoding, $fielddelimiter);
$csvdata = null; // Free memory.
if (empty($readcount)) {
print_error('csvfailed', 'data', "{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}");
} else {
if (!$fieldnames = $cir->get_columns()) {
print_error('cannotreadtmpfile', 'error');
}
// Check the fieldnames are valid.
$rawfields = $DB->get_records('data_fields', array('dataid' => $data->id), '', 'name, id, type');
$fields = array();
$errorfield = '';
$usernamestring = get_string('username');
$safetoskipfields = array(get_string('user'), get_string('email'),
get_string('timeadded', 'data'), get_string('timemodified', 'data'),
get_string('approved', 'data'), get_string('tags', 'data'));
$userfieldid = null;
foreach ($fieldnames as $id => $name) {
if (!isset($rawfields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
} else if (!in_array($name, $safetoskipfields)) {
$errorfield .= "'$name' ";
}
} else {
// If this is the second time, a field with this name comes up, it must be a field not provided by the user...
// like the username.
if (isset($fields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
}
unset($fieldnames[$id]); // To ensure the user provided content fields remain in the array once flipped.
} else {
$field = $rawfields[$name];
require_once("$CFG->dirroot/mod/data/field/$field->type/field.class.php");
$classname = 'data_field_' . $field->type;
$fields[$name] = new $classname($field, $data, $cm);
}
}
}
if (!empty($errorfield)) {
print_error('fieldnotmatched', 'data',
"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}", $errorfield);
}
$fieldnames = array_flip($fieldnames);
$cir->init();
$recordsadded = 0;
while ($record = $cir->next()) {
$authorid = null;
if ($userfieldid) {
if (!($author = core_user::get_user_by_username($record[$userfieldid], 'id'))) {
$authorid = null;
} else {
$authorid = $author->id;
}
}
if ($recordid = data_add_record($data, 0, $authorid)) { // Add instance to data_record.
foreach ($fields as $field) {
$fieldid = $fieldnames[$field->field->name];
if (isset($record[$fieldid])) {
$value = $record[$fieldid];
} else {
$value = '';
}
if (method_exists($field, 'update_content_import')) {
$field->update_content_import($recordid, $value, 'field_' . $field->field->id);
} else {
$content = new stdClass();
$content->fieldid = $field->field->id;
$content->content = $value;
$content->recordid = $recordid;
$DB->insert_record('data_content', $content);
}
}
if (core_tag_tag::is_enabled('mod_data', 'data_records') &&
isset($fieldnames[get_string('tags', 'data')])) {
$columnindex = $fieldnames[get_string('tags', 'data')];
$rawtags = $record[$columnindex];
$tags = explode(',', $rawtags);
foreach ($tags as $tag) {
$tag = trim($tag);
if (empty($tag)) {
continue;
}
core_tag_tag::add_item_tag('mod_data', 'data_records', $recordid, $context, $tag);
}
}
$recordsadded++;
print get_string('added', 'moodle', $recordsadded) . ". " . get_string('entry', 'data') . " (ID $recordid)<br />\n";
}
}
$cir->close();
$cir->cleanup(true);
return $recordsadded;
}
return 0;
}
/**
* @global object
* @param array $export

3
mod/data/tests/fixtures/test_data_import.csv vendored Normal file
View File

@ -0,0 +1,3 @@
ID,Param2
1,"My first entry"
2,"My second entry"
1 ID Param2
2 1 My first entry
3 2 My second entry

4
mod/data/tests/fixtures/test_data_import_with_field_username.csv vendored Normal file
View File

@ -0,0 +1,4 @@
ID,Username,Param2,Username,"Email address"
1,otherusername1,"My first entry",student,student@moodle.org
2,otherusername2,"My second entry",student2,student@moodle.org
3,otherusername3,"My third entry",student,student@moodle.org
1 ID Username Param2 Username Email address
2 1 otherusername1 My first entry student student@moodle.org
3 2 otherusername2 My second entry student2 student@moodle.org
4 3 otherusername3 My third entry student student@moodle.org

3
mod/data/tests/fixtures/test_data_import_with_userdata.csv vendored Normal file
View File

@ -0,0 +1,3 @@
ID,Param2,Username,"Email address"
1,"My first entry",student,student@moodle.org
2,"My second entry",student2,student@moodle.org
1 ID Param2 Username Email address
2 1 My first entry student student@moodle.org
3 2 My second entry student2 student@moodle.org

263
mod/data/tests/import_test.php Normal file
View File

@ -0,0 +1,263 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Unit tests for importing csv files.
*
* @package mod_data
* @category test
* @copyright 2019 Tobias Reischmann
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
defined('MOODLE_INTERNAL') || die();
global $CFG;
require_once($CFG->dirroot . '/mod/data/lib.php');
require_once($CFG->dirroot . '/lib/datalib.php');
require_once($CFG->dirroot . '/lib/csvlib.class.php');
require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php');
require_once($CFG->dirroot . '/mod/data/tests/generator/lib.php');
/**
* Unit tests for import.php.
*
* @package mod_data
* @copyright 2019 Tobias Reischmann
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class mod_data_import_test extends advanced_testcase {
/** @var object $cm Course module of data instance. */
private $cm;
/** @var object $data Data instance. */
private $data;
/** @var mod_data_generator $generator */
private $generator;
/** @var object $student Student object */
private $student;
/** @var object $teacher Teacher object */
private $teacher;
/**
* Set up function. In this instance we are setting up database
* records to be used in the unit tests.
*/
protected function setUp() {
parent::setUp();
$this->resetAfterTest(true);
$this->generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
$course = $this->getDataGenerator()->create_course();
$this->teacher = $this->getDataGenerator()->create_and_enrol($course, 'teacher');
$this->setUser($this->teacher);
$this->student = $this->getDataGenerator()->create_and_enrol($course, 'student', array('username' => 'student'));
$this->data = $this->generator->create_instance(array('course' => $course->id));
$this->cm = get_coursemodule_from_instance('data', $this->data->id);
// Add fields.
$fieldrecord = new StdClass();
$fieldrecord->name = 'ID'; // Identifier of the records for testing.
$fieldrecord->type = 'number';
$this->generator->create_field($fieldrecord, $this->data);
$fieldrecord->name = 'Param2';
$fieldrecord->type = 'text';
$this->generator->create_field($fieldrecord, $this->data);
}
/**
* Test uploading entries for a data instance without userdata.
* @throws dml_exception
*/
public function test_import() {
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import.csv');
ob_start();
data_import_csv($this->cm, $this->data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
// No userdata is present in the file: Fallback is to assign the uploading user as author.
$expecteduserids = array();
$expecteduserids[1] = $this->teacher->id;
$expecteduserids[2] = $this->teacher->id;
$records = $this->get_data_records($this->data->id);
$this->assertCount(2, $records);
foreach ($records as $record) {
$identifier = $record->items['ID']->content;
$this->assertEquals($expecteduserids[$identifier], $record->userid);
}
}
/**
* Test uploading entries for a data instance with userdata.
*
* At least one entry has an identifiable user, which is assigned as author.
* @throws dml_exception
*/
public function test_import_with_userdata() {
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_userdata.csv');
ob_start();
data_import_csv($this->cm, $this->data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$expecteduserids = array();
$expecteduserids[1] = $this->student->id; // User student exists and is assigned as author.
$expecteduserids[2] = $this->teacher->id; // User student2 does not exist. Fallback is the uploading user.
$records = $this->get_data_records($this->data->id);
$this->assertCount(2, $records);
foreach ($records as $record) {
$identifier = $record->items['ID']->content;
$this->assertEquals($expecteduserids[$identifier], $record->userid);
}
}
/**
* Test uploading entries for a data instance with userdata and a defined field 'Username'.
*
* This should test the corner case, in which a user has defined a data fields, which has the same name
* as the current lang string for username. In that case, the first Username entry is used for the field.
* The second one is used to identify the author.
* @throws coding_exception
* @throws dml_exception
*/
public function test_import_with_field_username() {
// Add username field.
$fieldrecord = new StdClass();
$fieldrecord->name = 'Username';
$fieldrecord->type = 'text';
$this->generator->create_field($fieldrecord, $this->data);
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_field_username.csv');
ob_start();
data_import_csv($this->cm, $this->data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$expecteduserids = array();
$expecteduserids[1] = $this->student->id; // User student exists and is assigned as author.
$expecteduserids[2] = $this->teacher->id; // User student2 does not exist. Fallback is the uploading user.
$expecteduserids[3] = $this->student->id; // User student exists and is assigned as author.
$expectedcontent = array();
$expectedcontent[1] = array(
'Username' => 'otherusername1',
'Param2' => 'My first entry',
);
$expectedcontent[2] = array(
'Username' => 'otherusername2',
'Param2' => 'My second entry',
);
$expectedcontent[3] = array(
'Username' => 'otherusername3',
'Param2' => 'My third entry',
);
$records = $this->get_data_records($this->data->id);
$this->assertCount(3, $records);
foreach ($records as $record) {
$identifier = $record->items['ID']->content;
$this->assertEquals($expecteduserids[$identifier], $record->userid);
foreach ($expectedcontent[$identifier] as $field => $value) {
$this->assertEquals($value, $record->items[$field]->content,
"The value of field \"$field\" for the record at position \"$identifier\" ".
"which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
}
}
}
/**
* Test uploading entries for a data instance with a field 'Username' but only one occurrence in the csv file.
*
* This should test the corner case, in which a user has defined a data fields, which has the same name
* as the current lang string for username. In that case, the only Username entry is used for the field.
* The author should not be set.
* @throws coding_exception
* @throws dml_exception
*/
public function test_import_with_field_username_without_userdata() {
// Add username field.
$fieldrecord = new StdClass();
$fieldrecord->name = 'Username';
$fieldrecord->type = 'text';
$this->generator->create_field($fieldrecord, $this->data);
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_userdata.csv');
ob_start();
data_import_csv($this->cm, $this->data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
// No userdata is present in the file: Fallback is to assign the uploading user as author.
$expecteduserids = array();
$expecteduserids[1] = $this->teacher->id;
$expecteduserids[2] = $this->teacher->id;
$expectedcontent = array();
$expectedcontent[1] = array(
'Username' => 'student',
'Param2' => 'My first entry',
);
$expectedcontent[2] = array(
'Username' => 'student2',
'Param2' => 'My second entry',
);
$records = $this->get_data_records($this->data->id);
$this->assertCount(2, $records);
foreach ($records as $record) {
$identifier = $record->items['ID']->content;
$this->assertEquals($expecteduserids[$identifier], $record->userid);
foreach ($expectedcontent[$identifier] as $field => $value) {
$this->assertEquals($value, $record->items[$field]->content,
"The value of field \"$field\" for the record at position \"$identifier\" ".
"which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
}
}
}
/**
* Returns the records of the data instance.
*
* Each records has an item entry, which contains all fields associated with this item.
* Each fields has the parameters name, type and content.
* @param int $dataid Id of the data instance.
* @return array The records of the data instance.
* @throws dml_exception
*
*/
private function get_data_records($dataid) {
global $DB;
$records = $DB->get_records('data_records', ['dataid' => $dataid]);
foreach ($records as $record) {
$sql = 'SELECT f.name, f.type, con.content FROM
{data_content} con JOIN {data_fields} f ON con.fieldid = f.id
WHERE con.recordid = :recordid';
$items = $DB->get_records_sql($sql, array('recordid' => $record->id));
$record->items = $items;
}
return $records;
}
}