Merge branch 'MDL-40084-master' of https://github.com/phmemmel/moodle

This commit is contained in:
Sara Arjona 2023-07-04 13:58:41 +02:00
commit 9c51739570
No known key found for this signature in database
24 changed files with 2243 additions and 378 deletions

View File

@ -0,0 +1,81 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\exporter;
use coding_exception;
use csv_export_writer;
/**
* CSV entries exporter for mod_data.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class csv_entries_exporter extends entries_exporter {
/** @var string[] Possible delimiter names. Only used internally to check if a valid delimiter name
* has been specified.
*/
private const POSSIBLE_DELIMITER_NAMES = ['comma', 'tab', 'semicolon', 'colon', 'cfg'];
/**
* @var string name of the delimiter to use for the csv export. Possible values:
* 'comma', 'tab', 'semicolon', 'colon' or 'cfg'.
*/
private string $delimitername = 'comma';
/**
* Returns the csv data exported by the csv_export_writer for further handling.
*
* @see \mod_data\local\exporter\entries_exporter::get_data_file_content()
*/
public function get_data_file_content(): string {
global $CFG;
require_once($CFG->libdir . '/csvlib.class.php');
return csv_export_writer::print_array($this->exportdata, $this->delimitername, '"', true);
}
/**
* Returns the file extension of this entries exporter.
*
* @see \mod_data\local\exporter\entries_exporter::get_export_data_file_extension()
*/
public function get_export_data_file_extension(): string {
return 'csv';
}
/**
* Setter for the delimiter name which should be used in this csv_entries_exporter object.
*
* Calling this setter is optional, the delimiter name defaults to 'comma'.
*
* @param string $delimitername one of 'comma', 'tab', 'semicolon', 'colon' or 'cfg'
* @return void
* @throws coding_exception if a wrong delimiter name has been specified
*/
public function set_delimiter_name(string $delimitername): void {
if (!in_array($delimitername, self::POSSIBLE_DELIMITER_NAMES)) {
throw new coding_exception('Wrong delimiter type',
'Please choose on of the following delimiters: '
. '\"comma\", \"tab\", \"semicolon\", \"colon\", \"cfg\"');
}
$this->delimitername = $delimitername;
}
}

View File

@ -0,0 +1,277 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\exporter;
use file_serving_exception;
use moodle_exception;
use zip_archive;
/**
* Exporter class for exporting data and - if needed - files as well in a zip archive.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
abstract class entries_exporter {
/** @var int Tracks the currently edited row of the export data file. */
private int $currentrow;
/**
* @var array The data structure containing the data for exporting. It's a 2-dimensional array of
* rows and columns.
*/
protected array $exportdata;
/** @var string Name of the export file name without extension. */
protected string $exportfilename;
/** @var zip_archive The zip archive object we store all the files in, if we need to export files as well. */
private zip_archive $ziparchive;
/** @var bool Tracks the state if the zip archive already has been closed. */
private bool $isziparchiveclosed;
/** @var string full path of the zip archive. */
private string $zipfilepath;
/** @var array Array to store all filenames in the zip archive for export. */
private array $filenamesinzip;
/**
* Creates an entries_exporter object.
*
* This object can be used to export data to different formats including files. If files are added,
* everything will be bundled up in a zip archive.
*/
public function __construct() {
$this->currentrow = 0;
$this->exportdata = [];
$this->exportfilename = 'Exportfile';
$this->filenamesinzip = [];
$this->isziparchiveclosed = true;
}
/**
* Adds a row (array of strings) to the export data.
*
* @param array $row the row to add, $row has to be a plain array of strings
* @return void
*/
public function add_row(array $row): void {
$this->exportdata[] = $row;
$this->currentrow++;
}
/**
* Adds a data string (so the content for a "cell") to the current row.
*
* @param string $cellcontent the content to add to the current row
* @return void
*/
public function add_to_current_row(string $cellcontent): void {
$this->exportdata[$this->currentrow][] = $cellcontent;
}
/**
* Signal the entries_exporter to finish the current row and jump to the next row.
*
* @return void
*/
public function next_row(): void {
$this->currentrow++;
}
/**
* Sets the name of the export file.
*
* Only use the basename without path and without extension here.
*
* @param string $exportfilename name of the file without path and extension
* @return void
*/
public function set_export_file_name(string $exportfilename): void {
$this->exportfilename = $exportfilename;
}
/**
* The entries_exporter will prepare a data file from the rows and columns being added.
* Overwrite this method to generate the data file as string.
*
* @return string the data file as a string
*/
abstract protected function get_data_file_content(): string;
/**
* Overwrite the method to return the file extension your data file will have, for example
* <code>return 'csv';</code> for a csv file entries_exporter.
*
* @return string the file extension of the data file your entries_exporter is using
*/
abstract protected function get_export_data_file_extension(): string;
/**
* Returns the count of currently stored records (rows excluding header row).
*
* @return int the count of records/rows
*/
public function get_records_count(): int {
// The attribute $this->exportdata also contains a header. If only one row is present, this
// usually is the header, so record count should be 0.
if (count($this->exportdata) <= 1) {
return 0;
}
return count($this->exportdata) - 1;
}
/**
* Use this method to add a file which should be exported to the entries_exporter.
*
* @param string $filename the name of the file which should be added
* @param string $filecontent the content of the file as a string
* @param string $zipsubdir the subdirectory in the zip archive. Defaults to 'files/'.
* @return void
* @throws moodle_exception if there is an error adding the file to the zip archive
*/
public function add_file_from_string(string $filename, string $filecontent, string $zipsubdir = 'files/'): void {
if (empty($this->filenamesinzip)) {
// No files added yet, so we need to create a zip archive.
$this->create_zip_archive();
}
if (!str_ends_with($zipsubdir, '/')) {
$zipsubdir .= '/';
}
$zipfilename = $zipsubdir . $filename;
$this->filenamesinzip[] = $zipfilename;
$this->ziparchive->add_file_from_string($zipfilename, $filecontent);
}
/**
* Sends the generated export file.
*
* Care: By default this function finishes the current PHP request and directly serves the file to the user as download.
*
* @param bool $sendtouser true if the file should be sent directly to the user, if false the file content will be returned
* as string
* @return string|null file content as string if $sendtouser is true
* @throws moodle_exception if there is an issue adding the data file
* @throws file_serving_exception if the file could not be served properly
*/
public function send_file(bool $sendtouser = true): null|string {
if (empty($this->filenamesinzip)) {
if ($sendtouser) {
send_file($this->get_data_file_content(),
$this->exportfilename . '.' . $this->get_export_data_file_extension(),
null, 0, true, true);
return null;
} else {
return $this->get_data_file_content();
}
}
$this->add_file_from_string($this->exportfilename . '.' . $this->get_export_data_file_extension(),
$this->get_data_file_content(), '/');
$this->finish_zip_archive();
if ($this->isziparchiveclosed) {
if ($sendtouser) {
send_file($this->zipfilepath, $this->exportfilename . '.zip', null, 0, false, true);
return null;
} else {
return file_get_contents($this->zipfilepath);
}
} else {
throw new file_serving_exception('Could not serve zip file, it could not be closed properly.');
}
}
/**
* Checks if a file with the given name has already been added to the file export bundle.
*
* Care: Filenames are compared to all files in the specified zip subdirectory which
* defaults to 'files/'.
*
* @param string $filename the filename containing the zip path of the file to check
* @param string $zipsubdir The subdirectory in which the filename should be looked for,
* defaults to 'files/'
* @return bool true if file with the given name already exists, false otherwise
*/
public function file_exists(string $filename, string $zipsubdir = 'files/'): bool {
if (!str_ends_with($zipsubdir, '/')) {
$zipsubdir .= '/';
}
if (empty($filename)) {
return false;
}
return in_array($zipsubdir . $filename, $this->filenamesinzip, true);
}
/**
* Creates a unique filename based on the given filename.
*
* This method adds "_1", "_2", ... to the given file name until the newly generated filename
* is not equal to any of the already saved ones in the export file bundle.
*
* @param string $filename the filename based on which a unique filename should be generated
* @return string the unique filename
*/
public function create_unique_filename(string $filename): string {
if (!$this->file_exists($filename)) {
return $filename;
}
$extension = pathinfo($filename, PATHINFO_EXTENSION);
$filenamewithoutextension = empty($extension)
? $filename
: substr($filename, 0,strlen($filename) - strlen($extension) - 1);
$filenamewithoutextension = $filenamewithoutextension . '_1';
$i = 1;
$filename = empty($extension) ? $filenamewithoutextension : $filenamewithoutextension . '.' . $extension;
while ($this->file_exists($filename)) {
// In case we have already a file ending with '_XX' where XX is an ascending number, we have to
// remove '_XX' first before adding '_YY' again where YY is the successor of XX.
$filenamewithoutextension = preg_replace('/_' . $i . '$/', '_' . ($i + 1), $filenamewithoutextension);
$filename = empty($extension) ? $filenamewithoutextension : $filenamewithoutextension . '.' . $extension;
$i++;
}
return $filename;
}
/**
* Prepares the zip archive.
*
* @return void
*/
private function create_zip_archive(): void {
$tmpdir = make_request_directory();
$this->zipfilepath = $tmpdir . '/' . $this->exportfilename . '.zip';
$this->ziparchive = new zip_archive();
$this->isziparchiveclosed = !$this->ziparchive->open($this->zipfilepath);
}
/**
* Closes the zip archive.
*
* @return void
*/
private function finish_zip_archive(): void {
if (!$this->isziparchiveclosed) {
$this->isziparchiveclosed = $this->ziparchive->close();
}
}
}

View File

@ -0,0 +1,65 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\exporter;
use MoodleODSWorkbook;
use MoodleODSWriter;
/**
* ODS entries exporter for mod_data.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class ods_entries_exporter extends entries_exporter {
/**
* Returns the file extension of this entries exporter.
*
* @see \mod_data\local\exporter\entries_exporter::get_export_data_file_extension()
*/
public function get_export_data_file_extension(): string {
return 'ods';
}
/**
* Returns the ods data exported by the ODS library for further handling.
*
* @see \mod_data\local\exporter\entries_exporter::get_data_file_content()
*/
public function get_data_file_content(): string {
global $CFG;
require_once("$CFG->libdir/odslib.class.php");
$filearg = '-';
$workbook = new MoodleODSWorkbook($filearg);
$worksheet = [];
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($this->exportdata as $row) {
$colno = 0;
foreach ($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$writer = new MoodleODSWriter($worksheet);
return $writer->get_file_content();
}
}

View File

@ -0,0 +1,141 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\exporter;
use context;
use context_system;
/**
* Utility class for exporting data from a mod_data instance.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class utils {
/**
* Exports the data of the mod_data instance to an entries_exporter object which then can export it to a file format.
*
* @param int $dataid
* @param array $fields
* @param array $selectedfields
* @param entries_exporter $exporter the entries_exporter object used
* @param int $currentgroup group ID of the current group. This is used for
* exporting data while maintaining group divisions.
* @param context|null $context the context in which the operation is performed (for capability checks)
* @param bool $userdetails whether to include the details of the record author
* @param bool $time whether to include time created/modified
* @param bool $approval whether to include approval status
* @param bool $tags whether to include tags
* @param bool $includefiles whether files should be exported as well
* @return void
*/
public static function data_exportdata(int $dataid, array $fields, array $selectedfields, entries_exporter $exporter,
int $currentgroup = 0, context $context = null, bool $userdetails = false, bool $time = false, bool $approval = false,
bool $tags = false, bool $includefiles = true): void {
global $DB;
if (is_null($context)) {
$context = context_system::instance();
}
// Exporting user data needs special permission.
$userdetails = $userdetails && has_capability('mod/data:exportuserinfo', $context);
// Populate the header in first row of export.
$header = [];
foreach ($fields as $key => $field) {
if (!in_array($field->field->id, $selectedfields)) {
// Ignore values we aren't exporting.
unset($fields[$key]);
} else {
$header[] = $field->field->name;
}
}
if ($tags) {
$header[] = get_string('tags', 'data');
}
if ($userdetails) {
$header[] = get_string('user');
$header[] = get_string('username');
$header[] = get_string('email');
}
if ($time) {
$header[] = get_string('timeadded', 'data');
$header[] = get_string('timemodified', 'data');
}
if ($approval) {
$header[] = get_string('approved', 'data');
}
$exporter->add_row($header);
$datarecords = $DB->get_records('data_records', array('dataid' => $dataid));
ksort($datarecords);
$line = 1;
foreach ($datarecords as $record) {
// Get content indexed by fieldid.
if ($currentgroup) {
$select = 'SELECT c.fieldid, c.content, c.content1, c.content2, c.content3, c.content4 FROM {data_content} c, '
. '{data_records} r WHERE c.recordid = ? AND r.id = c.recordid AND r.groupid = ?';
$where = array($record->id, $currentgroup);
} else {
$select = 'SELECT fieldid, content, content1, content2, content3, content4 FROM {data_content} WHERE recordid = ?';
$where = array($record->id);
}
if ($content = $DB->get_records_sql($select, $where)) {
foreach ($fields as $field) {
$contents = '';
if (isset($content[$field->field->id])) {
$contents = $field->export_text_value($content[$field->field->id]);
if (!empty($contents) && $field->file_export_supported() && $includefiles
&& !is_null($field->export_file_value($record))) {
// For exporting overwrite the content of the column with a unique
// filename, even it is not exactly the name of the file in the
// mod_data instance content. But it's more important to match the name
// of the exported file.
$contents = $exporter->create_unique_filename($contents);
$exporter->add_file_from_string($contents, $field->export_file_value($record));
}
}
// Just be double sure.
$contents = !empty($contents) ? $contents : '';
$exporter->add_to_current_row($contents);
}
if ($tags) {
$itemtags = \core_tag_tag::get_item_tags_array('mod_data', 'data_records', $record->id);
$exporter->add_to_current_row(implode(', ', $itemtags));
}
if ($userdetails) { // Add user details to the export data.
$userdata = get_complete_user_data('id', $record->userid);
$exporter->add_to_current_row(fullname($userdata));
$exporter->add_to_current_row($userdata->username);
$exporter->add_to_current_row($userdata->email);
}
if ($time) { // Add time added / modified.
$exporter->add_to_current_row(userdate($record->timecreated));
$exporter->add_to_current_row(userdate($record->timemodified));
}
if ($approval) { // Add approval status.
$exporter->add_to_current_row((int) $record->approved);
}
}
$exporter->next_row();
}
}
}

View File

@ -0,0 +1,200 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\importer;
use context_module;
use core_php_time_limit;
use core_tag_tag;
use core_user;
use csv_import_reader;
use moodle_exception;
use stdClass;
/**
* CSV entries_importer class for importing data and - if needed - files as well from a zip archive.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class csv_entries_importer extends entries_importer {
/** @var array Log entries for successfully added records. */
private array $addedrecordsmessages = [];
/**
* Declares the entries_importer to use a csv file as data file.
*
* @see entries_importer::get_import_data_file_extension()
*/
public function get_import_data_file_extension(): string {
return 'csv';
}
/**
* Import records for a data instance from csv data.
*
* @param stdClass $cm Course module of the data instance.
* @param stdClass $data The data instance.
* @param string $encoding The encoding of csv data.
* @param string $fielddelimiter The delimiter of the csv data.
*
* @throws moodle_exception
*/
public function import_csv(stdClass $cm, stdClass $data, string $encoding, string $fielddelimiter): void {
global $CFG, $DB;
// Large files are likely to take their time and memory. Let PHP know
// that we'll take longer, and that the process should be recycled soon
// to free up memory.
core_php_time_limit::raise();
raise_memory_limit(MEMORY_HUGE);
$iid = csv_import_reader::get_new_iid('moddata');
$cir = new csv_import_reader($iid, 'moddata');
$context = context_module::instance($cm->id);
$readcount = $cir->load_csv_content($this->get_data_file_content(), $encoding, $fielddelimiter);
if (empty($readcount)) {
throw new \moodle_exception('csvfailed', 'data', "{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}");
} else {
if (!$fieldnames = $cir->get_columns()) {
throw new \moodle_exception('cannotreadtmpfile', 'error');
}
// Check the fieldnames are valid.
$rawfields = $DB->get_records('data_fields', ['dataid' => $data->id], '', 'name, id, type');
$fields = [];
$errorfield = '';
$usernamestring = get_string('username');
$safetoskipfields = [get_string('user'), get_string('email'),
get_string('timeadded', 'data'), get_string('timemodified', 'data'),
get_string('approved', 'data'), get_string('tags', 'data')];
$userfieldid = null;
foreach ($fieldnames as $id => $name) {
if (!isset($rawfields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
} else if (!in_array($name, $safetoskipfields)) {
$errorfield .= "'$name' ";
}
} else {
// If this is the second time, a field with this name comes up, it must be a field not provided by the user...
// like the username.
if (isset($fields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
}
unset($fieldnames[$id]); // To ensure the user provided content fields remain in the array once flipped.
} else {
$field = $rawfields[$name];
$filepath = "$CFG->dirroot/mod/data/field/$field->type/field.class.php";
if (!file_exists($filepath)) {
$errorfield .= "'$name' ";
continue;
}
require_once($filepath);
$classname = 'data_field_' . $field->type;
$fields[$name] = new $classname($field, $data, $cm);
}
}
}
if (!empty($errorfield)) {
throw new \moodle_exception('fieldnotmatched', 'data',
"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}", $errorfield);
}
$fieldnames = array_flip($fieldnames);
$cir->init();
while ($record = $cir->next()) {
$authorid = null;
if ($userfieldid) {
if (!($author = core_user::get_user_by_username($record[$userfieldid], 'id'))) {
$authorid = null;
} else {
$authorid = $author->id;
}
}
if ($recordid = data_add_record($data, 0, $authorid)) { // Add instance to data_record.
foreach ($fields as $field) {
$fieldid = $fieldnames[$field->field->name];
if (isset($record[$fieldid])) {
$value = $record[$fieldid];
} else {
$value = '';
}
if (method_exists($field, 'update_content_import')) {
$field->update_content_import($recordid, $value, 'field_' . $field->field->id);
} else {
$content = new stdClass();
$content->fieldid = $field->field->id;
$content->content = $value;
$content->recordid = $recordid;
if ($field->file_import_supported() && $this->importfiletype === 'zip') {
$filecontent = $this->get_file_content_from_zip($content->content);
if (!$filecontent) {
// No corresponding file in zip archive, so no record for this field being added at all.
continue;
}
$contentid = $DB->insert_record('data_content', $content);
$field->import_file_value($contentid, $filecontent, $content->content);
} else {
$DB->insert_record('data_content', $content);
}
}
}
if (core_tag_tag::is_enabled('mod_data', 'data_records') &&
isset($fieldnames[get_string('tags', 'data')])) {
$columnindex = $fieldnames[get_string('tags', 'data')];
$rawtags = $record[$columnindex];
$tags = explode(',', $rawtags);
foreach ($tags as $tag) {
$tag = trim($tag);
if (empty($tag)) {
continue;
}
core_tag_tag::add_item_tag('mod_data', 'data_records', $recordid, $context, $tag);
}
}
$this->addedrecordsmessages[] = get_string('added', 'moodle',
count($this->addedrecordsmessages) + 1)
. ". " . get_string('entry', 'data')
. " (ID $recordid)\n";
}
}
$cir->close();
$cir->cleanup(true);
}
}
/**
* Getter for the array of messages for added records.
*
* For each successfully added record the array contains a log message.
*
* @return array Array of message strings: For each added record one message string
*/
public function get_added_records_messages(): array {
return $this->addedrecordsmessages;
}
}

View File

@ -0,0 +1,144 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\importer;
use coding_exception;
use core_php_time_limit;
use file_packer;
/**
* Importer class for importing data and - if needed - files as well from a zip archive.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
abstract class entries_importer {
/** @var string The import file path of the file which data should be imported from. */
protected string $importfilepath;
/** @var string The original name of the import file name including extension of the file which data should be imported from. */
protected string $importfilename;
/** @var string $importfiletype The file type of the import file. */
protected string $importfiletype;
/** @var file_packer Zip file packer to extract files from a zip archive. */
private file_packer $packer;
/** @var bool Tracks state if zip archive has been extracted already. */
private bool $zipfileextracted;
/** @var string Temporary directory where zip archive is being extracted to. */
private string $extracteddir;
/**
* Creates an entries_importer object.
*
* This object can be used to import data from data files (like csv) and zip archives both including a data file and files to be
* stored in the course module context.
*
* @param string $importfilepath the complete path of the import file including filename
* @param string $importfilename the import file name as uploaded by the user
* @throws coding_exception if a wrong file type is being used
*/
public function __construct(string $importfilepath, string $importfilename) {
$this->importfilepath = $importfilepath;
$this->importfilename = $importfilename;
$this->importfiletype = pathinfo($importfilename, PATHINFO_EXTENSION);
$this->zipfileextracted = false;
if ($this->importfiletype !== $this->get_import_data_file_extension() && $this->importfiletype !== 'zip') {
throw new coding_exception('Only "zip" or "' . $this->get_import_data_file_extension() . '" files are '
. 'allowed.');
}
}
/**
* Return the file extension of the import data file which is being used, for example 'csv' for a csv entries_importer.
*
* @return string the file extension of the export data file
*/
abstract public function get_import_data_file_extension(): string;
/**
* Returns the file content of the data file.
*
* Returns the content of the file directly if the entries_importer's file is a data file itself.
* If the entries_importer's file is a zip archive, the content of the first found data file in the
* zip archive's root will be returned.
*
* @return false|string the data file content as string; false, if file cannot be found/read
*/
public function get_data_file_content(): false|string {
if ($this->importfiletype !== 'zip') {
// We have no zip archive, so the file itself must be the data file.
return file_get_contents($this->importfilepath);
}
// So we have a zip archive and need to find the right data file in the root of the zip archive.
$this->extract_zip();
$datafilenames = array_filter($this->packer->list_files($this->importfilepath),
fn($file) => pathinfo($file->pathname, PATHINFO_EXTENSION) === $this->get_import_data_file_extension()
&& !str_contains($file->pathname, '/'));
if (empty($datafilenames) || count($datafilenames) > 1) {
return false;
}
return file_get_contents($this->extracteddir . reset($datafilenames)->pathname);
}
/**
* Returns the file content from a file which has been stored in the zip archive.
*
* @param string $filename
* @param string $zipsubdir
* @return false|string the file content as string, false if the file could not be found/read
*/
public function get_file_content_from_zip(string $filename, string $zipsubdir = 'files/'): false|string {
if (empty($filename)) {
// Nothing to return.
return false;
}
// Just to be sure extract if not extracted yet.
$this->extract_zip();
if (!str_ends_with($zipsubdir, '/')) {
$zipsubdir .= '/';
}
$filepathinextractedzip = $this->extracteddir . $zipsubdir . $filename;
return file_exists($filepathinextractedzip) ? file_get_contents($filepathinextractedzip) : false;
}
/**
* Extracts (if not already done and if we have a zip file to deal with) the zip file to a temporary directory.
*
* @return void
*/
private function extract_zip(): void {
if ($this->zipfileextracted || $this->importfiletype !== 'zip') {
return;
}
$this->packer = get_file_packer();
core_php_time_limit::raise(180);
$this->extracteddir = make_request_directory();
if (!str_ends_with($this->extracteddir, '/')) {
$this->extracteddir .= '/';
}
$this->packer->extract_to_pathname($this->importfilepath, $this->extracteddir);
$this->zipfileextracted = true;
}
}

View File

@ -61,3 +61,236 @@ function data_get_completion_state($course, $cm, $userid, $type) {
}
return $result;
}
/**
* @deprecated since Moodle 4.3.
* @global object
* @param array $export
* @param string $dataname
* @param int $count
* @return string
*/
function data_export_xls($export, $dataname, $count) {
global $CFG;
debugging('Function data_export_xls() has been deprecated, because xls export has been dropped.',
DEBUG_DEVELOPER);
require_once("$CFG->libdir/excellib.class.php");
$filename = clean_filename("{$dataname}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$filename .= '.xls';
$filearg = '-';
$workbook = new MoodleExcelWorkbook($filearg);
$workbook->send($filename);
$worksheet = array();
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($export as $row) {
$colno = 0;
foreach($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$workbook->close();
return $filename;
}
/**
* @deprecated since Moodle 4.3, exporting is now being done by \mod_data\local\exporter\csv_entries_exporter
* @global object
* @param array $export
* @param string $delimiter_name
* @param object $database
* @param int $count
* @param bool $return
* @return string|void
*/
function data_export_csv($export, $delimiter_name, $database, $count, $return=false) {
global $CFG;
debugging('Function data_export_csv has been deprecated. Exporting is now being done by '
. '\mod_data\local\csv_exporter.', DEBUG_DEVELOPER);
require_once($CFG->libdir . '/csvlib.class.php');
$filename = $database . '-' . $count . '-record';
if ($count > 1) {
$filename .= 's';
}
if ($return) {
return csv_export_writer::print_array($export, $delimiter_name, '"', true);
} else {
csv_export_writer::download_array($filename, $export, $delimiter_name);
}
}
/**
* @deprecated since Moodle 4.3, exporting is now being done by \mod_data\local\exporter\ods_entries_exporter
* @global object
* @param array $export
* @param string $dataname
* @param int $count
* @param string
*/
function data_export_ods($export, $dataname, $count) {
global $CFG;
debugging('Function data_export_ods has been deprecated. Exporting is now being done by '
. '\mod_data\local\ods_exporter.', DEBUG_DEVELOPER);
require_once("$CFG->libdir/odslib.class.php");
$filename = clean_filename("{$dataname}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$filename .= '.ods';
$filearg = '-';
$workbook = new MoodleODSWorkbook($filearg);
$workbook->send($filename);
$worksheet = array();
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($export as $row) {
$colno = 0;
foreach($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$workbook->close();
return $filename;
}
/**
* @deprecated since Moodle 4.3, use \mod_data\local\exporter\utils::data_exportdata with a \mod_data\local\exporter\entries_exporter object
* @global object
* @param int $dataid
* @param array $fields
* @param array $selectedfields
* @param int $currentgroup group ID of the current group. This is used for
* exporting data while maintaining group divisions.
* @param object $context the context in which the operation is performed (for capability checks)
* @param bool $userdetails whether to include the details of the record author
* @param bool $time whether to include time created/modified
* @param bool $approval whether to include approval status
* @param bool $tags whether to include tags
* @return array
*/
function data_get_exportdata($dataid, $fields, $selectedfields, $currentgroup=0, $context=null,
$userdetails=false, $time=false, $approval=false, $tags = false) {
global $DB;
debugging('Function data_get_exportdata has been deprecated. Use '
. '\mod_data\local\exporter_utils::data_exportdata with a \mod_data\local\exporter object instead',
DEBUG_DEVELOPER);
if (is_null($context)) {
$context = context_system::instance();
}
// exporting user data needs special permission
$userdetails = $userdetails && has_capability('mod/data:exportuserinfo', $context);
$exportdata = array();
// populate the header in first row of export
foreach($fields as $key => $field) {
if (!in_array($field->field->id, $selectedfields)) {
// ignore values we aren't exporting
unset($fields[$key]);
} else {
$exportdata[0][] = $field->field->name;
}
}
if ($tags) {
$exportdata[0][] = get_string('tags', 'data');
}
if ($userdetails) {
$exportdata[0][] = get_string('user');
$exportdata[0][] = get_string('username');
$exportdata[0][] = get_string('email');
}
if ($time) {
$exportdata[0][] = get_string('timeadded', 'data');
$exportdata[0][] = get_string('timemodified', 'data');
}
if ($approval) {
$exportdata[0][] = get_string('approved', 'data');
}
$datarecords = $DB->get_records('data_records', array('dataid'=>$dataid));
ksort($datarecords);
$line = 1;
foreach($datarecords as $record) {
// get content indexed by fieldid
if ($currentgroup) {
$select = 'SELECT c.fieldid, c.content, c.content1, c.content2, c.content3, c.content4 FROM {data_content} c, {data_records} r WHERE c.recordid = ? AND r.id = c.recordid AND r.groupid = ?';
$where = array($record->id, $currentgroup);
} else {
$select = 'SELECT fieldid, content, content1, content2, content3, content4 FROM {data_content} WHERE recordid = ?';
$where = array($record->id);
}
if( $content = $DB->get_records_sql($select, $where) ) {
foreach($fields as $field) {
$contents = '';
if(isset($content[$field->field->id])) {
$contents = $field->export_text_value($content[$field->field->id]);
}
$exportdata[$line][] = $contents;
}
if ($tags) {
$itemtags = \core_tag_tag::get_item_tags_array('mod_data', 'data_records', $record->id);
$exportdata[$line][] = implode(', ', $itemtags);
}
if ($userdetails) { // Add user details to the export data
$userdata = get_complete_user_data('id', $record->userid);
$exportdata[$line][] = fullname($userdata);
$exportdata[$line][] = $userdata->username;
$exportdata[$line][] = $userdata->email;
}
if ($time) { // Add time added / modified
$exportdata[$line][] = userdate($record->timecreated);
$exportdata[$line][] = userdate($record->timemodified);
}
if ($approval) { // Add approval status
$exportdata[$line][] = (int) $record->approved;
}
}
$line++;
}
$line--;
return $exportdata;
}
/**
* @deprecated since Moodle 4.3, importing is now being done by \mod_data\local\importer\csv_importer::import_csv
* Import records for a data instance from csv data.
*
* @param object $cm Course module of the data instance.
* @param object $data The data instance.
* @param string $csvdata The csv data to be imported.
* @param string $encoding The encoding of csv data.
* @param string $fielddelimiter The delimiter of the csv data.
* @return int Number of records added.
*/
function data_import_csv($cm, $data, &$csvdata, $encoding, $fielddelimiter) {
debugging('Function data_import_csv has been deprecated. '
. 'Importing is now being done by \mod_data\local\csv_importer::import_csv.',
DEBUG_DEVELOPER);
// New function needs a file, not the file content, so we have to temporarily put the content into a file.
$tmpdir = make_request_directory();
$tmpfilename = 'tmpfile.csv';
$tmpfilepath = $tmpdir . '/tmpfile.csv';
file_put_contents($tmpfilepath, $csvdata);
$importer = new \mod_data\local\importer\csv_entries_importer($tmpfilepath, $tmpfilename);
$importer->import_csv($cm, $data, $encoding, $fielddelimiter);
return 0;
}

View File

@ -95,20 +95,31 @@ if ($mform->is_cancelled()) {
$currentgroup = groups_get_activity_group($cm);
$exportdata = data_get_exportdata($data->id, $fields, $selectedfields, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags);
$count = count($exportdata);
$exporter = null;
switch ($formdata['exporttype']) {
case 'csv':
data_export_csv($exportdata, $formdata['delimiter_name'], $data->name, $count);
break;
case 'xls':
data_export_xls($exportdata, $data->name, $count);
$exporter = new \mod_data\local\exporter\csv_entries_exporter();
$exporter->set_delimiter_name($formdata['delimiter_name']);
break;
case 'ods':
data_export_ods($exportdata, $data->name, $count);
$exporter = new \mod_data\local\exporter\ods_entries_exporter();
break;
default:
throw new coding_exception('Invalid export format has been specified. '
. 'Only "csv" and "ods" are currently supported.');
}
$includefiles = !empty($formdata['includefiles']);
\mod_data\local\exporter\utils::data_exportdata($data->id, $fields, $selectedfields, $exporter, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags, $includefiles);
$count = $exporter->get_records_count();
$filename = clean_filename("{$data->name}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$exporter->set_export_file_name($filename);
$exporter->send_file();
}
// Build header to match the rest of the UI.

View File

@ -112,6 +112,8 @@ class mod_data_export_form extends moodleform {
$exportoptions[] = $mform->createElement('checkbox', 'exportapproval', get_string('includeapproval', 'data'), '',
$optionattrs);
}
$exportoptions[] = $mform->createElement('checkbox', 'includefiles', get_string('includefiles', 'data'), '', $optionattrs);
$mform->setDefault('includefiles', 1);
$mform->addGroup($exportoptions, 'exportoptions', get_string('selectexportoptions', 'data'), ['<br>'], false);
$this->add_action_buttons(true, get_string('exportentries', 'data'));

View File

@ -218,8 +218,64 @@ class data_field_file extends data_field_base {
$DB->update_record('data_content', $content);
}
function text_export_supported() {
return false;
/**
* Here we export the text value of a file field which is the filename of the exported file.
*
* @param stdClass $record the record which is being exported
* @return string the value which will be stored in the exported file for this field
*/
public function export_text_value(stdClass $record): string {
return !empty($record->content) ? $record->content : '';
}
/**
* Specifies that this field type supports the export of files.
*
* @return bool true which means that file export is being supported by this field type
*/
public function file_export_supported(): bool {
return true;
}
/**
* Specifies that this field type supports the import of files.
*
* @return bool true which means that file import is being supported by this field type
*/
public function file_import_supported(): bool {
return true;
}
/**
* Provides the necessary code for importing a file when importing the content of a mod_data instance.
*
* @param int $contentid the id of the mod_data content record
* @param string $filecontent the content of the file to import as string
* @param string $filename the filename the imported file should get
* @return void
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
$filerecord = [
'contextid' => $this->context->id,
'component' => 'mod_data',
'filearea' => 'content',
'itemid' => $contentid,
'filepath' => '/',
'filename' => $filename,
];
$fs = get_file_storage();
$fs->create_file_from_string($filerecord, $filecontent);
}
/**
* Exports the file content for file export.
*
* @param stdClass $record the data content record the file belongs to
* @return null|string The file content of the stored file or null if no file should be exported for this record
*/
public function export_file_value(stdClass $record): null|string {
$file = $this->get_file($record->id);
return $file ? $file->get_content() : null;
}
function file_ok($path) {

View File

@ -335,12 +335,16 @@ class data_field_picture extends data_field_base {
// If thumbnail width and height are BOTH not specified then no thumbnail is generated, and
// additionally an attempted delete of the existing thumbnail takes place.
$fs = get_file_storage();
$file_record = array('contextid'=>$file->get_contextid(), 'component'=>$file->get_component(), 'filearea'=>$file->get_filearea(),
'itemid'=>$file->get_itemid(), 'filepath'=>$file->get_filepath(),
'filename'=>'thumb_'.$file->get_filename(), 'userid'=>$file->get_userid());
$filerecord = [
'contextid' => $file->get_contextid(), 'component' => $file->get_component(), 'filearea' => $file->get_filearea(),
'itemid' => $file->get_itemid(), 'filepath' => $file->get_filepath(),
'filename' => 'thumb_' . $file->get_filename(), 'userid' => $file->get_userid()
];
try {
// this may fail for various reasons
$fs->convert_image($file_record, $file, (int) $this->field->param4, (int) $this->field->param5, true);
// This may fail for various reasons.
$newwidth = isset($this->field->param4) ? (int) $this->field->param4 : null;
$newheight = isset($this->field->param5) ? (int) $this->field->param5 : null;
$fs->convert_image($filerecord, $file, $newwidth, $newheight, true);
return true;
} catch (Exception $e) {
debugging($e->getMessage());
@ -348,8 +352,65 @@ class data_field_picture extends data_field_base {
}
}
function text_export_supported() {
return false;
/**
* Here we export the text value of a picture field which is the filename of the exported picture.
*
* @param stdClass $record the record which is being exported
* @return string the value which will be stored in the exported file for this field
*/
public function export_text_value(stdClass $record): string {
return !empty($record->content) ? $record->content : '';
}
/**
* Specifies that this field type supports the export of files.
*
* @return bool true which means that file export is being supported by this field type
*/
public function file_export_supported(): bool {
return true;
}
/**
* Exports the file content for file export.
*
* @param stdClass $record the data content record the file belongs to
* @return null|string The file content of the stored file or null if no file should be exported for this record
*/
public function export_file_value(stdClass $record): null|string {
$file = $this->get_file($record->id);
return $file ? $file->get_content() : null;
}
/**
* Specifies that this field type supports the import of files.
*
* @return bool true which means that file import is being supported by this field type
*/
public function file_import_supported(): bool {
return true;
}
/**
* Provides the necessary code for importing a file when importing the content of a mod_data instance.
*
* @param int $contentid the id of the mod_data content record
* @param string $filecontent the content of the file to import as string
* @param string $filename the filename the imported file should get
* @return void
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
$filerecord = [
'contextid' => $this->context->id,
'component' => 'mod_data',
'filearea' => 'content',
'itemid' => $contentid,
'filepath' => '/',
'filename' => $filename,
];
$fs = get_file_storage();
$file = $fs->create_file_from_string($filerecord, $filecontent);
$this->update_thumbnail(null, $file);
}
function file_ok($path) {

View File

@ -86,16 +86,32 @@ echo $OUTPUT->header();
echo $OUTPUT->heading_with_help(get_string('uploadrecords', 'mod_data'), 'uploadrecords', 'mod_data');
if ($formdata = $form->get_data()) {
$filecontent = $form->get_file_content('recordsfile');
$recordsadded = data_import_csv($cm, $data, $filecontent, $formdata->encoding, $formdata->fielddelimiter);
$uploadedfilepath = $form->save_temp_file('recordsfile');
$filestempdir = null;
if ($recordsadded > 0) {
echo $OUTPUT->notification($recordsadded. ' '. get_string('recordssaved', 'data'), '');
} else {
echo $OUTPUT->notification(get_string('recordsnotsaved', 'data'), 'notifysuccess');
if (!$uploadedfilepath) {
throw new coding_exception('No file uploaded.');
}
echo $OUTPUT->continue_button($redirectbackto);
$importer = new \mod_data\local\importer\csv_entries_importer($uploadedfilepath, $form->get_new_filename('recordsfile'));
if (!$importer->get_data_file_content()) {
echo $OUTPUT->notification(get_string('errordatafilenotfound', 'data'),
\core\output\notification::NOTIFY_ERROR);
} else {
$importer->import_csv($cm, $data, $formdata->encoding, $formdata->fielddelimiter);
unlink($uploadedfilepath);
$addedrecordsmessages = $importer->get_added_records_messages();
echo html_writer::div(implode('<br/>', $addedrecordsmessages));
if (count($addedrecordsmessages) > 0) {
echo $OUTPUT->notification(count($addedrecordsmessages) . ' ' . get_string('recordssaved', 'data'),
\core\output\notification::NOTIFY_SUCCESS);
} else {
echo $OUTPUT->notification(get_string('recordsnotsaved', 'data'),
\core\output\notification::NOTIFY_ERROR);
}
}
} else {
/// Upload records section. Only for teachers and the admin.
echo $OUTPUT->box_start('generalbox boxaligncenter boxwidthwide');

View File

@ -14,7 +14,8 @@ class mod_data_import_form extends moodleform {
$dataid = $this->_customdata['dataid'];
$backtourl = $this->_customdata['backtourl'];
$mform->addElement('filepicker', 'recordsfile', get_string('csvfile', 'data'));
$mform->addElement('filepicker', 'recordsfile', get_string('csvfile', 'data'),
null, ['accepted_types' => ['application/zip', 'text/csv']]);
$delimiters = csv_import_reader::get_delimiter_list();
$mform->addElement('select', 'fielddelimiter', get_string('fielddelimiter', 'data'), $delimiters);

View File

@ -83,7 +83,7 @@ $string['createfields'] = 'Create fields to collect different types of data.';
$string['createtemplates'] = 'Templates define the interface of your activity. Once you create fields, templates will be created automatically. Alternatively, you can use a preset, which includes fields and templates.';
$string['csstemplate'] = 'Custom CSS';
$string['csvfailed'] = 'Unable to read the raw data from the CSV file';
$string['csvfile'] = 'CSV file';
$string['csvfile'] = 'CSV or ZIP containing a CSV file';
$string['csvimport'] = 'CSV file import';
$string['csvimport_help'] = 'Entries may be imported via a plain text file with a list of field names as the first line, then the data, with one record per line.';
$string['csvwithselecteddelimiter'] = '<abbr title="Comma Separated Values">CSV</abbr>';
@ -152,6 +152,7 @@ $string['entrieslefttoadd'] = 'You must add {$a->entriesleft} more entry/entries
$string['entrieslefttoaddtoview'] = 'You must add {$a->entrieslefttoview} more entry/entries before you can view other participants\' entries.';
$string['entry'] = 'Entry';
$string['entrysaved'] = 'Your entry has been saved';
$string['errordatafilenotfound'] = 'The file could not be imported. Accepted file types are CSV or a ZIP containing a CSV file in the format used for exporting entries.';
$string['errormustbeteacher'] = 'You need to be a teacher to use this page!';
$string['errorpresetexists'] = 'A preset with this name already exists.';
$string['errorpresetexistsbutnotoverwrite'] = 'A preset with this name already exists. Choose a different name.';
@ -227,6 +228,7 @@ $string['importapreset'] = 'Import a preset';
$string['importsuccess'] = 'Preset applied.';
$string['importpresetmissingcapability'] = 'You don\'t have permission to import a preset.';
$string['includeapproval'] = 'Include approval status';
$string['includefiles'] = 'Include files in export';
$string['includetags'] = 'Include tags';
$string['includetime'] = 'Include time added/modified';
$string['includeuserdetails'] = 'Include user details';
@ -453,14 +455,10 @@ $string['unsupportedfields'] = 'Unsupported fields';
$string['unsupportedfieldslist'] = 'The following fields cannot be exported:';
$string['updatefield'] = 'Update an existing field';
$string['uploadfile'] = 'Upload file';
$string['uploadrecords'] = 'Upload entries from a file';
$string['uploadrecords_help'] = 'Entries may be uploaded via text file. The format of the file should be as follows:
$string['uploadrecords'] = 'Import entries';
$string['uploadrecords_help'] = 'Import entries that you have exported from another database, either via CSV or a ZIP containing a CSV file (if files are included in the export).
* Each line of the file contains one record
* Each record is a series of data separated by the selected separator
* The first record contains a list of fieldnames defining the format of the rest of the file
The field enclosure is a character that surrounds each field in each record. It can normally be left unset.';
Alternatively, to create a CSV file for importing, add one entry to the database and then export it. Edit the CSV file and add more entries.';
$string['uploadrecords_link'] = 'mod/data/import';
$string['url'] = 'URL';
$string['usedate'] = 'Include in search.';

View File

@ -640,16 +640,62 @@ class data_field_base { // Base class for Database Field Types (see field/*/
}
/**
* Per default, return the record's text value only from the "content" field.
* Override this in fields class if necesarry.
* Per default, it is assumed that fields do not support file exporting. Override this (return true)
* on fields supporting file export. You will also have to implement export_file_value().
*
* @param string $record
* @return bool true if field will export a file, false otherwise
*/
public function file_export_supported(): bool {
return false;
}
/**
* Per default, does not return a file (just null).
* Override this in fields class, if you want your field to export a file content.
* In case you are exporting a file value, export_text_value() should return the corresponding file name.
*
* @param stdClass $record
* @return null|string the file content as string or null, if no file content is being provided
*/
public function export_file_value(stdClass $record): null|string {
return null;
}
/**
* Per default, a field does not support the import of files.
*
* A field type can overwrite this function and return true. In this case it also has to implement the function
* import_file_value().
*
* @return false means file imports are not supported
*/
public function file_import_supported(): bool {
return false;
}
/**
* Returns a stored_file object for exporting a file of a given record.
*
* @param int $contentid content id
* @param string $filecontent the content of the file as string
* @param string $filename the filename the file should have
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
return;
}
/**
* Per default, return the record's text value only from the "content" field.
* Override this in fields class if necessary.
*
* @param stdClass $record
* @return string
*/
function export_text_value($record) {
public function export_text_value(stdClass $record) {
if ($this->text_export_supported()) {
return $record->content;
}
return '';
}
/**
@ -2978,328 +3024,6 @@ function data_supports($feature) {
}
}
/**
* Import records for a data instance from csv data.
*
* @param object $cm Course module of the data instance.
* @param object $data The data instance.
* @param string $csvdata The csv data to be imported.
* @param string $encoding The encoding of csv data.
* @param string $fielddelimiter The delimiter of the csv data.
* @return int Number of records added.
*/
function data_import_csv($cm, $data, &$csvdata, $encoding, $fielddelimiter) {
global $CFG, $DB;
// Large files are likely to take their time and memory. Let PHP know
// that we'll take longer, and that the process should be recycled soon
// to free up memory.
core_php_time_limit::raise();
raise_memory_limit(MEMORY_EXTRA);
$iid = csv_import_reader::get_new_iid('moddata');
$cir = new csv_import_reader($iid, 'moddata');
$context = context_module::instance($cm->id);
$readcount = $cir->load_csv_content($csvdata, $encoding, $fielddelimiter);
$csvdata = null; // Free memory.
if (empty($readcount)) {
throw new \moodle_exception('csvfailed', 'data', "{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}");
} else {
if (!$fieldnames = $cir->get_columns()) {
throw new \moodle_exception('cannotreadtmpfile', 'error');
}
// Check the fieldnames are valid.
$rawfields = $DB->get_records('data_fields', array('dataid' => $data->id), '', 'name, id, type');
$fields = array();
$errorfield = '';
$usernamestring = get_string('username');
$safetoskipfields = array(get_string('user'), get_string('email'),
get_string('timeadded', 'data'), get_string('timemodified', 'data'),
get_string('approved', 'data'), get_string('tags', 'data'));
$userfieldid = null;
foreach ($fieldnames as $id => $name) {
if (!isset($rawfields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
} else if (!in_array($name, $safetoskipfields)) {
$errorfield .= "'$name' ";
}
} else {
// If this is the second time, a field with this name comes up, it must be a field not provided by the user...
// like the username.
if (isset($fields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
}
unset($fieldnames[$id]); // To ensure the user provided content fields remain in the array once flipped.
} else {
$field = $rawfields[$name];
$filepath = "$CFG->dirroot/mod/data/field/$field->type/field.class.php";
if (!file_exists($filepath)) {
$errorfield .= "'$name' ";
continue;
}
require_once($filepath);
$classname = 'data_field_' . $field->type;
$fields[$name] = new $classname($field, $data, $cm);
}
}
}
if (!empty($errorfield)) {
throw new \moodle_exception('fieldnotmatched', 'data',
"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}", $errorfield);
}
$fieldnames = array_flip($fieldnames);
$cir->init();
$recordsadded = 0;
while ($record = $cir->next()) {
$authorid = null;
if ($userfieldid) {
if (!($author = core_user::get_user_by_username($record[$userfieldid], 'id'))) {
$authorid = null;
} else {
$authorid = $author->id;
}
}
if ($recordid = data_add_record($data, 0, $authorid)) { // Add instance to data_record.
foreach ($fields as $field) {
$fieldid = $fieldnames[$field->field->name];
if (isset($record[$fieldid])) {
$value = $record[$fieldid];
} else {
$value = '';
}
if (method_exists($field, 'update_content_import')) {
$field->update_content_import($recordid, $value, 'field_' . $field->field->id);
} else {
$content = new stdClass();
$content->fieldid = $field->field->id;
$content->content = $value;
$content->recordid = $recordid;
$DB->insert_record('data_content', $content);
}
}
if (core_tag_tag::is_enabled('mod_data', 'data_records') &&
isset($fieldnames[get_string('tags', 'data')])) {
$columnindex = $fieldnames[get_string('tags', 'data')];
$rawtags = $record[$columnindex];
$tags = explode(',', $rawtags);
foreach ($tags as $tag) {
$tag = trim($tag);
if (empty($tag)) {
continue;
}
core_tag_tag::add_item_tag('mod_data', 'data_records', $recordid, $context, $tag);
}
}
$recordsadded++;
print get_string('added', 'moodle', $recordsadded) . ". " . get_string('entry', 'data') . " (ID $recordid)<br />\n";
}
}
$cir->close();
$cir->cleanup(true);
return $recordsadded;
}
return 0;
}
/**
* @global object
* @param array $export
* @param string $delimiter_name
* @param object $database
* @param int $count
* @param bool $return
* @return string|void
*/
function data_export_csv($export, $delimiter_name, $database, $count, $return=false) {
global $CFG;
require_once($CFG->libdir . '/csvlib.class.php');
$filename = $database . '-' . $count . '-record';
if ($count > 1) {
$filename .= 's';
}
if ($return) {
return csv_export_writer::print_array($export, $delimiter_name, '"', true);
} else {
csv_export_writer::download_array($filename, $export, $delimiter_name);
}
}
/**
* @global object
* @param array $export
* @param string $dataname
* @param int $count
* @return string
*/
function data_export_xls($export, $dataname, $count) {
global $CFG;
require_once("$CFG->libdir/excellib.class.php");
$filename = clean_filename("{$dataname}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$filename .= '.xls';
$filearg = '-';
$workbook = new MoodleExcelWorkbook($filearg);
$workbook->send($filename);
$worksheet = array();
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($export as $row) {
$colno = 0;
foreach($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$workbook->close();
return $filename;
}
/**
* @global object
* @param array $export
* @param string $dataname
* @param int $count
* @param string
*/
function data_export_ods($export, $dataname, $count) {
global $CFG;
require_once("$CFG->libdir/odslib.class.php");
$filename = clean_filename("{$dataname}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$filename .= '.ods';
$filearg = '-';
$workbook = new MoodleODSWorkbook($filearg);
$workbook->send($filename);
$worksheet = array();
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($export as $row) {
$colno = 0;
foreach($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$workbook->close();
return $filename;
}
/**
* @global object
* @param int $dataid
* @param array $fields
* @param array $selectedfields
* @param int $currentgroup group ID of the current group. This is used for
* exporting data while maintaining group divisions.
* @param object $context the context in which the operation is performed (for capability checks)
* @param bool $userdetails whether to include the details of the record author
* @param bool $time whether to include time created/modified
* @param bool $approval whether to include approval status
* @param bool $tags whether to include tags
* @return array
*/
function data_get_exportdata($dataid, $fields, $selectedfields, $currentgroup=0, $context=null,
$userdetails=false, $time=false, $approval=false, $tags = false) {
global $DB;
if (is_null($context)) {
$context = context_system::instance();
}
// exporting user data needs special permission
$userdetails = $userdetails && has_capability('mod/data:exportuserinfo', $context);
$exportdata = array();
// populate the header in first row of export
foreach($fields as $key => $field) {
if (!in_array($field->field->id, $selectedfields)) {
// ignore values we aren't exporting
unset($fields[$key]);
} else {
$exportdata[0][] = $field->field->name;
}
}
if ($tags) {
$exportdata[0][] = get_string('tags', 'data');
}
if ($userdetails) {
$exportdata[0][] = get_string('user');
$exportdata[0][] = get_string('username');
$exportdata[0][] = get_string('email');
}
if ($time) {
$exportdata[0][] = get_string('timeadded', 'data');
$exportdata[0][] = get_string('timemodified', 'data');
}
if ($approval) {
$exportdata[0][] = get_string('approved', 'data');
}
$datarecords = $DB->get_records('data_records', array('dataid'=>$dataid));
ksort($datarecords);
$line = 1;
foreach($datarecords as $record) {
// get content indexed by fieldid
if ($currentgroup) {
$select = 'SELECT c.fieldid, c.content, c.content1, c.content2, c.content3, c.content4 FROM {data_content} c, {data_records} r WHERE c.recordid = ? AND r.id = c.recordid AND r.groupid = ?';
$where = array($record->id, $currentgroup);
} else {
$select = 'SELECT fieldid, content, content1, content2, content3, content4 FROM {data_content} WHERE recordid = ?';
$where = array($record->id);
}
if( $content = $DB->get_records_sql($select, $where) ) {
foreach($fields as $field) {
$contents = '';
if(isset($content[$field->field->id])) {
$contents = $field->export_text_value($content[$field->field->id]);
}
$exportdata[$line][] = $contents;
}
if ($tags) {
$itemtags = \core_tag_tag::get_item_tags_array('mod_data', 'data_records', $record->id);
$exportdata[$line][] = implode(', ', $itemtags);
}
if ($userdetails) { // Add user details to the export data
$userdata = get_complete_user_data('id', $record->userid);
$exportdata[$line][] = fullname($userdata);
$exportdata[$line][] = $userdata->username;
$exportdata[$line][] = $userdata->email;
}
if ($time) { // Add time added / modified
$exportdata[$line][] = userdate($record->timecreated);
$exportdata[$line][] = userdate($record->timemodified);
}
if ($approval) { // Add approval status
$exportdata[$line][] = (int) $record->approved;
}
}
$line++;
}
$line--;
return $exportdata;
}
////////////////////////////////////////////////////////////////////////////////
// File API //
////////////////////////////////////////////////////////////////////////////////

View File

@ -0,0 +1,260 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data;
use context_module;
use mod_data\local\exporter\csv_entries_exporter;
use mod_data\local\exporter\ods_entries_exporter;
use mod_data\local\exporter\utils;
/**
* Unit tests for exporting entries.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class entries_export_test extends \advanced_testcase {
/**
* Get the test data.
*
* In this instance we are setting up database records to be used in the unit tests.
*
* @return array of test instances
*/
protected function get_test_data(): array {
$this->resetAfterTest(true);
/** @var \mod_data_generator $generator */
$generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
$course = $this->getDataGenerator()->create_course();
$teacher = $this->getDataGenerator()->create_and_enrol($course, 'teacher');
$this->setUser($teacher);
$student = $this->getDataGenerator()->create_and_enrol($course, 'student', ['username' => 'student']);
$data = $generator->create_instance(['course' => $course->id]);
$cm = get_coursemodule_from_instance('data', $data->id);
// Add fields.
$fieldrecord = new \stdClass();
$fieldrecord->name = 'numberfield'; // Identifier of the records for testing.
$fieldrecord->type = 'number';
$numberfield = $generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'textfield';
$fieldrecord->type = 'text';
$textfield = $generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield1';
$fieldrecord->type = 'file';
$filefield1 = $generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield2';
$fieldrecord->type = 'file';
$filefield2 = $generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'picturefield';
$fieldrecord->type = 'picture';
$picturefield = $generator->create_field($fieldrecord, $data);
$contents[$numberfield->field->id] = '3';
$contents[$textfield->field->id] = 'a simple text';
$contents[$filefield1->field->id] = 'samplefile.png';
$contents[$filefield2->field->id] = 'samplefile.png';
$contents[$picturefield->field->id] = ['picturefile.png', 'this picture shows something'];
$generator->create_entry($data, $contents);
return [
'teacher' => $teacher,
'student' => $student,
'data' => $data,
'cm' => $cm,
];
}
/**
* Tests the exporting of the content of a mod_data instance by using the csv_entries_exporter.
*
* It also includes more general testing of the functionality of the entries_exporter the csv_entries_exporter
* is inheriting from.
*
* @covers \mod_data\local\exporter\entries_exporter
* @covers \mod_data\local\exporter\entries_exporter::get_records_count()
* @covers \mod_data\local\exporter\entries_exporter::send_file()
* @covers \mod_data\local\exporter\csv_entries_exporter
* @covers \mod_data\local\exporter\utils::data_exportdata
*/
public function test_export_csv(): void {
global $DB;
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$exporter = new csv_entries_exporter();
$exporter->set_export_file_name('testexportfile');
$fieldrecords = $DB->get_records('data_fields', ['dataid' => $data->id], 'id');
$fields = [];
foreach ($fieldrecords as $fieldrecord) {
$fields[] = data_get_field($fieldrecord, $data);
}
// We select all fields.
$selectedfields = array_map(fn($field) => $field->field->id, $fields);
$currentgroup = groups_get_activity_group($cm);
$context = context_module::instance($cm->id);
$exportuser = false;
$exporttime = false;
$exportapproval = false;
$tags = false;
// We first test the export without exporting files.
// This means file and picture fields will be exported, but only as text (which is the filename),
// so we will receive a csv export file.
$includefiles = false;
utils::data_exportdata($data->id, $fields, $selectedfields, $exporter, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags, $includefiles);
$this->assertEquals(file_get_contents(__DIR__ . '/fixtures/test_data_export_without_files.csv'),
$exporter->send_file(false));
$this->assertEquals(1, $exporter->get_records_count());
// We now test the export including files. This will generate a zip archive.
$includefiles = true;
$exporter = new csv_entries_exporter();
$exporter->set_export_file_name('testexportfile');
utils::data_exportdata($data->id, $fields, $selectedfields, $exporter, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags, $includefiles);
// We now write the zip archive temporary to disc to be able to parse it and assert it has the correct structure.
$tmpdir = make_request_directory();
file_put_contents($tmpdir . '/testexportarchive.zip', $exporter->send_file(false));
$ziparchive = new \zip_archive();
$ziparchive->open($tmpdir . '/testexportarchive.zip');
$expectedfilecontents = [
// The test generator for mod_data uses a copy of pix/monologo.png as sample file content for the file stored in a
// file and picture field.
// So we expect that this file has to have the same content as monologo.png.
// Also, the default value for the subdirectory in the zip archive containing the files is 'files/'.
'files/samplefile.png' => 'mod/data/pix/monologo.png',
'files/samplefile_1.png' => 'mod/data/pix/monologo.png',
'files/picturefile.png' => 'mod/data/pix/monologo.png',
// By checking that the content of the exported csv is identical to the fixture file it is verified
// that the filenames in the csv file correspond to the names of the exported file.
// It also verifies that files with identical file names in different fields (or records) will be numbered
// automatically (samplefile.png, samplefile_1.png, ...).
'testexportfile.csv' => __DIR__ . '/fixtures/test_data_export_with_files.csv'
];
for ($i = 0; $i < $ziparchive->count(); $i++) {
// We here iterate over all files in the zip archive and check if their content is identical to the files
// in the $expectedfilecontents array.
$filestream = $ziparchive->get_stream($i);
$fileinfo = $ziparchive->get_info($i);
$filecontent = fread($filestream, $fileinfo->size);
$this->assertEquals(file_get_contents($expectedfilecontents[$fileinfo->pathname]), $filecontent);
fclose($filestream);
}
$ziparchive->close();
unlink($tmpdir . '/testexportarchive.zip');
}
/**
* Tests specific ODS exporting functionality.
*
* @covers \mod_data\local\exporter\ods_entries_exporter
* @covers \mod_data\local\exporter\utils::data_exportdata
*/
public function test_export_ods(): void {
global $DB;
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$exporter = new ods_entries_exporter();
$exporter->set_export_file_name('testexportfile');
$fieldrecords = $DB->get_records('data_fields', ['dataid' => $data->id], 'id');
$fields = [];
foreach ($fieldrecords as $fieldrecord) {
$fields[] = data_get_field($fieldrecord, $data);
}
// We select all fields.
$selectedfields = array_map(fn($field) => $field->field->id, $fields);
$currentgroup = groups_get_activity_group($cm);
$context = context_module::instance($cm->id);
$exportuser = false;
$exporttime = false;
$exportapproval = false;
$tags = false;
// We first test the export without exporting files.
// This means file and picture fields will be exported, but only as text (which is the filename),
// so we will receive an ods export file.
$includefiles = false;
utils::data_exportdata($data->id, $fields, $selectedfields, $exporter, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags, $includefiles);
$odsrows = $this->get_ods_rows_content($exporter->send_file(false));
// Check, if the headings match with the first row of the ods file.
$i = 0;
foreach ($fields as $field) {
$this->assertEquals($field->field->name, $odsrows[0][$i]);
$i++;
}
// Check, if the values match with the field values.
$this->assertEquals('3', $odsrows[1][0]);
$this->assertEquals('a simple text', $odsrows[1][1]);
$this->assertEquals('samplefile.png', $odsrows[1][2]);
$this->assertEquals('samplefile.png', $odsrows[1][3]);
$this->assertEquals('picturefile.png', $odsrows[1][4]);
// As the logic of renaming the files and building a zip archive is implemented in entries_exporter class, we do
// not need to test this for the ods_entries_exporter, because entries_export_test::test_export_csv already does this.
}
/**
* Helper function to extract the text data as row arrays from an ODS document.
*
* @param string $content the file content
* @return array two-dimensional row/column array with the text content of the first spreadsheet
*/
private function get_ods_rows_content(string $content): array {
$file = tempnam(make_request_directory(), 'ods_');
$filestream = fopen($file, "w");
fwrite($filestream, $content);
$reader = new \OpenSpout\Reader\ODS\Reader();
$reader->open($file);
/** @var \OpenSpout\Reader\ODS\Sheet[] $sheets */
$sheets = $reader->getSheetIterator();
$rowscellsvalues = [];
foreach ($sheets as $sheet) {
/** @var \OpenSpout\Common\Entity\Row[] $rows */
$rows = $sheet->getRowIterator();
foreach ($rows as $row) {
$cellvalues = [];
foreach ($row->getCells() as $cell) {
$cellvalues[] = $cell->getValue();
}
$rowscellsvalues[] = $cellvalues;
}
}
return $rowscellsvalues;
}
}

View File

@ -0,0 +1,222 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data;
use context_module;
use mod_data\local\exporter\csv_entries_exporter;
use mod_data\local\exporter\ods_entries_exporter;
use mod_data\local\exporter\utils;
/**
* Unit tests for entries_exporter and csv_entries_exporter classes.
*
* Also {@see entries_export_test} class which provides module tests for exporting entries.
*
* @package mod_data
* @covers \mod_data\local\exporter\entries_exporter
* @covers \mod_data\local\exporter\csv_entries_exporter
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class entries_exporter_test extends \advanced_testcase {
/**
* Tests get_records_count method.
*
* @covers \mod_data\local\exporter\entries_exporter::get_records_count
* @dataProvider get_records_count_provider
* @param array $rows the rows from the data provider to be tested by the exporter
* @param int $expectedcount the expected count of records to be exported
*/
public function test_get_records_count(array $rows, int $expectedcount): void {
$exporter = new csv_entries_exporter();
foreach ($rows as $row) {
$exporter->add_row($row);
}
$this->assertEquals($expectedcount, $exporter->get_records_count());
}
/**
* Data provider method for self::test_get_records_count.
*
* @return array data for testing
*/
public function get_records_count_provider(): array {
return [
'onlyheader' => [
'rows' => [
['numberfield', 'textfield', 'filefield1', 'filefield2', 'picturefield']
],
'expectedcount' => 0 // Only header present, so we expect record count 0.
],
'onerecord' => [
'rows' => [
['numberfield', 'textfield', 'filefield1', 'filefield2', 'picturefield'],
['3', 'a simple text', 'samplefile.png', 'samplefile_1.png', 'picturefile.png']
],
'expectedcount' => 1
],
'tworecords' => [
'rows' => [
['numberfield', 'textfield', 'filefield1', 'filefield2', 'picturefield'],
['3', 'a simple text', 'samplefile.png', 'samplefile_1.png', 'picturefile.png'],
['5', 'a supersimple text', 'anotherfile.png', 'someotherfile.png', 'andapicture.png']
],
'expectedcount' => 2
]
];
}
/**
* Tests adding of files to the exporter to be included in the exported zip archive.
*
* @dataProvider add_file_from_string_provider
* @covers \mod_data\local\exporter\entries_exporter::add_file_from_string
* @covers \mod_data\local\exporter\entries_exporter::file_exists
* @param array $files array of filename and filecontent to be tested for exporting
* @param bool $success if the exporting of files should be successful
*/
public function test_add_file_from_string(array $files, bool $success): void {
$exporter = new csv_entries_exporter();
foreach ($files as $file) {
if (empty($file['subdir'])) {
$exporter->add_file_from_string($file['filename'], $file['filecontent']);
$this->assertEquals($exporter->file_exists($file['filename']), $success);
} else {
$exporter->add_file_from_string($file['filename'], $file['filecontent'], $file['subdir']);
$this->assertEquals($exporter->file_exists($file['filename'], $file['subdir']), $success);
}
}
}
/**
* Data provider method for self::test_add_file_from_string.
*
* @return array data for testing
*/
public function add_file_from_string_provider(): array {
return [
'one file' => [
'files' => [
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent'
],
],
'success' => true
],
'more files, also with subdirs' => [
'files' => [
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent'
],
[
'filename' => 'testfile2.txt',
'filecontent' => 'someothercontent',
'subdir' => 'testsubdir'
],
[
'filename' => 'testfile3.txt',
'filecontent' => 'someverydifferentcontent',
'subdir' => 'files/foo/bar'
],
[
'filename' => 'testfile4.txt',
'filecontent' => 'someverydifferentcontent',
'subdir' => 'files/foo/bar/'
],
[
'filename' => 'testfile5.txt',
'filecontent' => 'someverydifferentcontent',
'subdir' => '/files/foo/bar/'
],
],
'success' => true
],
'nocontent' => [
'files' => [
[
'filename' => '',
'filecontent' => ''
]
],
'success' => false
]
];
}
/**
* Tests if unique filenames are being created correctly.
*
* @covers \mod_data\local\exporter\entries_exporter::create_unique_filename
* @dataProvider create_unique_filename_provider
* @param string $inputfilename the name of the file which should be converted into a unique filename
* @param string $resultfilename the maybe changed $inputfilename, so that it is unique in the exporter
*/
public function test_create_unique_filename(string $inputfilename, string $resultfilename): void {
$exporter = new csv_entries_exporter();
$exporter->add_file_from_string('test.txt', 'somecontent');
$exporter->add_file_from_string('foo.txt', 'somecontent');
$exporter->add_file_from_string('foo_1.txt', 'somecontent');
$exporter->add_file_from_string('foo_2.txt', 'somecontent');
$exporter->add_file_from_string('foo', 'somecontent');
$exporter->add_file_from_string('foo_1', 'somecontent');
$exporter->add_file_from_string('sample_5.txt', 'somecontent');
$exporter->add_file_from_string('bar_1.txt', 'somecontent');
$this->assertEquals($resultfilename, $exporter->create_unique_filename($inputfilename));
}
/**
* Data provider method for self::test_create_unique_filename.
*
* @return array data for testing
*/
public function create_unique_filename_provider(): array {
return [
'does not exist yet' => [
'inputfilename' => 'someuniquename.txt',
'resultfilename' => 'someuniquename.txt'
],
'already exists' => [
'inputfilename' => 'test.txt',
'resultfilename' => 'test_1.txt'
],
'already exists, other numbers as well' => [
'inputfilename' => 'foo.txt',
'resultfilename' => 'foo_3.txt'
],
'file with _5 suffix already exists' => [
'inputfilename' => 'sample_5.txt',
'resultfilename' => 'sample_5_1.txt'
],
'file with _1 suffix already exists' => [
'inputfilename' => 'bar_1.txt',
'resultfilename' => 'bar_1_1.txt'
],
'file without extension unique' => [
'inputfilename' => 'test',
'resultfilename' => 'test'
],
'file without extension not unique' => [
'inputfilename' => 'foo',
'resultfilename' => 'foo_2'
]
];
}
}

View File

@ -16,6 +16,12 @@
namespace mod_data;
use coding_exception;
use dml_exception;
use mod_data\local\importer\csv_entries_importer;
use moodle_exception;
use zip_archive;
/**
* Unit tests for import.php.
*
@ -24,7 +30,7 @@ namespace mod_data;
* @copyright 2019 Tobias Reischmann
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class import_test extends \advanced_testcase {
class entries_import_test extends \advanced_testcase {
/**
* Set up function.
@ -68,6 +74,13 @@ class import_test extends \advanced_testcase {
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield';
$fieldrecord->type = 'file';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'picturefield';
$fieldrecord->type = 'picture';
$generator->create_field($fieldrecord, $data);
return [
'teacher' => $teacher,
@ -79,6 +92,7 @@ class import_test extends \advanced_testcase {
/**
* Test uploading entries for a data instance without userdata.
*
* @throws dml_exception
*/
public function test_import(): void {
@ -88,10 +102,9 @@ class import_test extends \advanced_testcase {
'teacher' => $teacher,
] = $this->get_test_data();
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import.csv');
ob_start();
data_import_csv($cm, $data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import.csv',
'test_data_import.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
// No userdata is present in the file: Fallback is to assign the uploading user as author.
$expecteduserids = array();
@ -110,6 +123,7 @@ class import_test extends \advanced_testcase {
* Test uploading entries for a data instance with userdata.
*
* At least one entry has an identifiable user, which is assigned as author.
*
* @throws dml_exception
*/
public function test_import_with_userdata(): void {
@ -120,10 +134,9 @@ class import_test extends \advanced_testcase {
'student' => $student,
] = $this->get_test_data();
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_userdata.csv');
ob_start();
data_import_csv($cm, $data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_userdata.csv',
'test_data_import_with_userdata.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$expecteduserids = array();
$expecteduserids[1] = $student->id; // User student exists and is assigned as author.
@ -143,6 +156,7 @@ class import_test extends \advanced_testcase {
* This should test the corner case, in which a user has defined a data fields, which has the same name
* as the current lang string for username. In that case, the first Username entry is used for the field.
* The second one is used to identify the author.
*
* @throws coding_exception
* @throws dml_exception
*/
@ -161,10 +175,9 @@ class import_test extends \advanced_testcase {
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_field_username.csv');
ob_start();
data_import_csv($cm, $data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_field_username.csv',
'test_data_import_with_field_username.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$expecteduserids = array();
$expecteduserids[1] = $student->id; // User student exists and is assigned as author.
@ -193,7 +206,7 @@ class import_test extends \advanced_testcase {
foreach ($expectedcontent[$identifier] as $field => $value) {
$this->assertEquals($value, $record->items[$field]->content,
"The value of field \"$field\" for the record at position \"$identifier\" ".
"The value of field \"$field\" for the record at position \"$identifier\" " .
"which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
}
}
@ -205,6 +218,7 @@ class import_test extends \advanced_testcase {
* This should test the corner case, in which a user has defined a data fields, which has the same name
* as the current lang string for username. In that case, the only Username entry is used for the field.
* The author should not be set.
*
* @throws coding_exception
* @throws dml_exception
*/
@ -223,10 +237,9 @@ class import_test extends \advanced_testcase {
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_userdata.csv');
ob_start();
data_import_csv($cm, $data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_userdata.csv',
'test_data_import_with_userdata.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
// No userdata is present in the file: Fallback is to assign the uploading user as author.
$expecteduserids = array();
@ -251,17 +264,98 @@ class import_test extends \advanced_testcase {
foreach ($expectedcontent[$identifier] as $field => $value) {
$this->assertEquals($value, $record->items[$field]->content,
"The value of field \"$field\" for the record at position \"$identifier\" ".
"The value of field \"$field\" for the record at position \"$identifier\" " .
"which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
}
}
}
/**
* Tests the import including files from a zip archive.
*
* @covers \mod_data\local\importer\entries_importer
* @covers \mod_data\local\importer\csv_entries_importer
* @return void
*/
public function test_import_with_files(): void {
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_files.zip',
'test_data_import_with_files.zip');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$records = $this->get_data_records($data->id);
$ziparchive = new zip_archive();
$ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files.zip');
$importedcontent = array_values($records)[0]->items;
$this->assertEquals(17, $importedcontent['ID']->content);
$this->assertEquals('samplefile.png', $importedcontent['filefield']->content);
$this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
// We now check if content of imported file from zip content is identical to the content of the file
// stored in the mod_data record in the field 'filefield'.
$fileindex = array_values(array_map(fn($file) => $file->index,
array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplefile.png')))[0];
$filestream = $ziparchive->get_stream($fileindex);
$filefield = data_get_field_from_name('filefield', $data);
$filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
$this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
$filefieldfilecontent);
fclose($filestream);
// We now check if content of imported picture from zip content is identical to the content of the picture file
// stored in the mod_data record in the field 'picturefield'.
$fileindex = array_values(array_map(fn($file) => $file->index,
array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplepicture.png')))[0];
$filestream = $ziparchive->get_stream($fileindex);
$filefield = data_get_field_from_name('picturefield', $data);
$filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
$this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
$filefieldfilecontent);
fclose($filestream);
$this->assertCount(1, $importer->get_added_records_messages());
$ziparchive->close();
}
/**
* Tests the import including files from a zip archive.
*
* @covers \mod_data\local\importer\entries_importer
* @covers \mod_data\local\importer\csv_entries_importer
* @return void
*/
public function test_import_with_files_missing_file(): void {
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip',
'test_data_import_with_files_missing_file.zip');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$records = $this->get_data_records($data->id);
$ziparchive = new zip_archive();
$ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip');
$importedcontent = array_values($records)[0]->items;
$this->assertEquals(17, $importedcontent['ID']->content);
$this->assertFalse(isset($importedcontent['filefield']));
$this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
$this->assertCount(1, $importer->get_added_records_messages());
$ziparchive->close();
}
/**
* Returns the records of the data instance.
*
* Each records has an item entry, which contains all fields associated with this item.
* Each fields has the parameters name, type and content.
*
* @param int $dataid Id of the data instance.
* @return array The records of the data instance.
* @throws dml_exception
@ -279,4 +373,54 @@ class import_test extends \advanced_testcase {
}
return $records;
}
/**
* Tests if the amount of imported records is counted properly.
*
* @covers \mod_data\local\importer\csv_entries_importer::import_csv
* @covers \mod_data\local\importer\csv_entries_importer::get_added_records_messages
* @dataProvider get_added_record_messages_provider
* @param string $datafilecontent the content of the datafile to test as string
* @param int $expectedcount the expected count of messages depending on the datafile content
*/
public function test_get_added_record_messages(string $datafilecontent, int $expectedcount): void {
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
// First we need to create the zip file from the provided data.
$tmpdir = make_request_directory();
$datafile = $tmpdir . '/entries_import_test_datafile_tmp_' . time() . '.csv';
file_put_contents($datafile, $datafilecontent);
$importer = new csv_entries_importer($datafile, 'testdatafile.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$this->assertEquals($expectedcount, count($importer->get_added_records_messages()));
}
/**
* Data provider method for self::test_get_added_record_messages.
*
* @return array data for testing
*/
public function get_added_record_messages_provider(): array {
return [
'only header' => [
'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL,
'expectedcount' => 0 // One line is being assumed to be the header.
],
'one record' => [
'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL
. '5,"some short text",testfilename.pdf,testpicture.png',
'expectedcount' => 1
],
'two records' => [
'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL
. '5,"some short text",testfilename.pdf,testpicture.png' . PHP_EOL
. '3,"other text",testfilename2.pdf,testpicture2.png',
'expectedcount' => 2
],
];
}
}

View File

@ -0,0 +1,205 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data;
use context_module;
use mod_data\local\exporter\csv_entries_exporter;
use mod_data\local\exporter\ods_entries_exporter;
use mod_data\local\exporter\utils;
use mod_data\local\importer\csv_entries_importer;
use zip_archive;
/**
* Unit tests for entries_importer and csv_entries_importer class.
*
* Also {@see entries_import_test} class which provides module tests for importing entries.
*
* @package mod_data
* @covers \mod_data\local\importer\entries_importer
* @covers \mod_data\local\importer\csv_entries_importer
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class entries_importer_test extends \advanced_testcase {
/**
* Set up function.
*/
protected function setUp(): void {
parent::setUp();
global $CFG;
require_once($CFG->dirroot . '/mod/data/lib.php');
require_once($CFG->dirroot . '/lib/datalib.php');
require_once($CFG->dirroot . '/lib/csvlib.class.php');
require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php');
require_once($CFG->dirroot . '/mod/data/tests/generator/lib.php');
}
/**
* Get the test data.
* In this instance we are setting up database records to be used in the unit tests.
*
* @return array
*/
protected function get_test_data(): array {
$this->resetAfterTest(true);
$generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
$course = $this->getDataGenerator()->create_course();
$teacher = $this->getDataGenerator()->create_and_enrol($course, 'teacher');
$this->setUser($teacher);
$student = $this->getDataGenerator()->create_and_enrol($course, 'student', array('username' => 'student'));
$data = $generator->create_instance(array('course' => $course->id));
$cm = get_coursemodule_from_instance('data', $data->id);
// Add fields.
$fieldrecord = new \stdClass();
$fieldrecord->name = 'ID'; // Identifier of the records for testing.
$fieldrecord->type = 'number';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'Param2';
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield';
$fieldrecord->type = 'file';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'picturefield';
$fieldrecord->type = 'picture';
$generator->create_field($fieldrecord, $data);
return [
'teacher' => $teacher,
'student' => $student,
'data' => $data,
'cm' => $cm,
];
}
/**
* Test importing files from zip archive.
*
* @covers \mod_data\local\importer\entries_importer::get_file_content_from_zip
* @covers \mod_data\local\importer\entries_importer::get_data_file_content
* @dataProvider get_file_content_from_zip_provider
* @param array $files array of filenames and filecontents to test
* @param mixed $datafilecontent the expected result returned by the method which is being tested here
*/
public function test_get_file_content_from_zip(array $files, mixed $datafilecontent): void {
// First we need to create the zip file from the provided data.
$tmpdir = make_request_directory();
$zipfilepath = $tmpdir . '/entries_importer_test_tmp_' . time() . '.zip';
$ziparchive = new zip_archive();
$ziparchive->open($zipfilepath);
foreach ($files as $file) {
$localname = empty($file['subdir']) ? $file['filename'] : $file['subdir'] . '/' . $file['filename'];
$ziparchive->add_file_from_string($localname, $file['filecontent']);
}
$ziparchive->close();
// We now created a zip archive according to the data provider's data. We now can test the importer.
$importer = new csv_entries_importer($zipfilepath, 'testzip.zip');
foreach ($files as $file) {
$subdir = empty($file['subdir']) ? '' : $file['subdir'];
$this->assertEquals($file['filecontent'], $importer->get_file_content_from_zip($file['filename'], $subdir));
}
// Test the method to retrieve the datafile content.
$this->assertEquals($datafilecontent, $importer->get_data_file_content());
unlink($zipfilepath);
}
/**
* Data provider method for self::test_get_file_content_from_zip.
*
* @return array data for testing
*/
public function get_file_content_from_zip_provider(): array {
return [
'some files in the zip archive' => [
'files' => [
[
'filename' => 'datafile.csv',
'filecontent' => 'some,csv,data'
],
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent',
'subdir' => 'files'
],
[
'filename' => 'testfile2.txt',
'filecontent' => 'someothercontent',
'subdir' => 'testsubdir'
]
],
// Should be identical with filecontent of 'datafile.csv' above.
'datafilecontent' => 'some,csv,data'
],
'wrongly placed data file' => [
'files' => [
[
'filename' => 'datafile.csv',
'filecontent' => 'some,csv,data',
'subdir' => 'wrongsubdir'
],
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent',
'subdir' => 'files'
],
[
'filename' => 'testfile2.txt',
'filecontent' => 'someothercontent',
'subdir' => 'testsubdir'
]
],
// Data file is not in the root directory, though no content should be retrieved.
'datafilecontent' => false
],
'two data files where only one is allowed' => [
'files' => [
[
'filename' => 'datafile.csv',
'filecontent' => 'some,csv,data',
],
[
'filename' => 'anothercsvfile.csv',
'filecontent' => 'some,other,csv,data',
],
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent',
'subdir' => 'files'
],
[
'filename' => 'testfile2.txt',
'filecontent' => 'someothercontent',
'subdir' => 'testsubdir'
]
],
// There are two data files in the zip root, so the data cannot be imported.
'datafilecontent' => false
],
];
}
}

View File

@ -0,0 +1,2 @@
numberfield,textfield,filefield1,filefield2,picturefield
3,"a simple text",samplefile.png,samplefile_1.png,picturefile.png
1 numberfield textfield filefield1 filefield2 picturefield
2 3 a simple text samplefile.png samplefile_1.png picturefile.png

View File

@ -0,0 +1,2 @@
numberfield,textfield,filefield1,filefield2,picturefield
3,"a simple text",samplefile.png,samplefile.png,picturefile.png
1 numberfield textfield filefield1 filefield2 picturefield
2 3 a simple text samplefile.png samplefile.png picturefile.png

Binary file not shown.

View File

@ -1,6 +1,26 @@
This files describes API changes in /mod/data - plugins,
information provided here is intended especially for developers.
== 4.3 ==
* Function data_export_xls() has been deprecated and moved to deprecatedlib, because xls support has already been dropped.
* Functions data_export_csv(), data_export_ods() and data_get_exportdata() have been deprecated due to a bigger
refactoring of the way data is being exported. This is now being done by new exporter classes
\mod_data\local\exporter\csv_entries_exporter and \mod_data\local\exporter\ods_entries_exporter (inheriting from
exporter base class \mod_data\local\exporter\entries_exporter) as well as \mod_data\local\exporter\utils::data_exportdata().
* Function data_import_csv() has been deprecated and moved to deprecatedlib due to a bigger rework of the way data is
being imported. This is now being done by new importer class \mod_data\local\importer\csv_importer inheriting from new
class \mod_data\local\importer.
* Field base class now has two new methods file_export_supported() and export_file_value(). The method
file_export_supported() can be overwritten to declare that a field type can/wants to export a file. In this case this
field type will have to implement the method export_file_value() returning this file for exporting. Also: This field
type will have to export the name of the file by overwriting text_export_supported() to return true and make the
method export_text_value() return the name of the file.
* The field types file and picture now are able to export the file/picture.
* Field base class now has two new methods file_import_supported() and import_file_value(). The method
file_import_supported() can be overwritten to declare that a field type is able to import a file. In this case this
field type will have to implement the method import_file_value() doing the actual import of the file being passed.
* The field types file and picture now are able to import the file/picture.
== 4.2 ==
* The field base class now has a method validate(). Overwrite it in the field type to provide validation of field type's
parameters in the field add/modify form.