mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-03-14 20:50:21 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'MDL-40084-master' of https://github.com/phmemmel/moodle

Browse Source
This commit is contained in:
Sara Arjona 2023-07-04 13:58:41 +02:00
commit 9c51739570
No known key found for this signature in database
24 changed files with 2243 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
mod/data/classes/local/exporter/csv_entries_exporter.php Normal file
View File

@ -0,0 +1,81 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\exporter;
use coding_exception;
use csv_export_writer;
/**
* CSV entries exporter for mod_data.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class csv_entries_exporter extends entries_exporter {
/** @var string[] Possible delimiter names. Only used internally to check if a valid delimiter name
* has been specified.
*/
private const POSSIBLE_DELIMITER_NAMES = ['comma', 'tab', 'semicolon', 'colon', 'cfg'];
/**
* @var string name of the delimiter to use for the csv export. Possible values:
* 'comma', 'tab', 'semicolon', 'colon' or 'cfg'.
*/
private string $delimitername = 'comma';
/**
* Returns the csv data exported by the csv_export_writer for further handling.
*
* @see \mod_data\local\exporter\entries_exporter::get_data_file_content()
*/
public function get_data_file_content(): string {
global $CFG;
require_once($CFG->libdir . '/csvlib.class.php');
return csv_export_writer::print_array($this->exportdata, $this->delimitername, '"', true);
}
/**
* Returns the file extension of this entries exporter.
*
* @see \mod_data\local\exporter\entries_exporter::get_export_data_file_extension()
*/
public function get_export_data_file_extension(): string {
return 'csv';
}
/**
* Setter for the delimiter name which should be used in this csv_entries_exporter object.
*
* Calling this setter is optional, the delimiter name defaults to 'comma'.
*
* @param string $delimitername one of 'comma', 'tab', 'semicolon', 'colon' or 'cfg'
* @return void
* @throws coding_exception if a wrong delimiter name has been specified
*/
public function set_delimiter_name(string $delimitername): void {
if (!in_array($delimitername, self::POSSIBLE_DELIMITER_NAMES)) {
throw new coding_exception('Wrong delimiter type',
'Please choose on of the following delimiters: '
. '\"comma\", \"tab\", \"semicolon\", \"colon\", \"cfg\"');
}
$this->delimitername = $delimitername;
}
}

277
mod/data/classes/local/exporter/entries_exporter.php Normal file
View File

@ -0,0 +1,277 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\exporter;
use file_serving_exception;
use moodle_exception;
use zip_archive;
/**
* Exporter class for exporting data and - if needed - files as well in a zip archive.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
abstract class entries_exporter {
/** @var int Tracks the currently edited row of the export data file. */
private int $currentrow;
/**
* @var array The data structure containing the data for exporting. It's a 2-dimensional array of
* rows and columns.
*/
protected array $exportdata;
/** @var string Name of the export file name without extension. */
protected string $exportfilename;
/** @var zip_archive The zip archive object we store all the files in, if we need to export files as well. */
private zip_archive $ziparchive;
/** @var bool Tracks the state if the zip archive already has been closed. */
private bool $isziparchiveclosed;
/** @var string full path of the zip archive. */
private string $zipfilepath;
/** @var array Array to store all filenames in the zip archive for export. */
private array $filenamesinzip;
/**
* Creates an entries_exporter object.
*
* This object can be used to export data to different formats including files. If files are added,
* everything will be bundled up in a zip archive.
*/
public function __construct() {
$this->currentrow = 0;
$this->exportdata = [];
$this->exportfilename = 'Exportfile';
$this->filenamesinzip = [];
$this->isziparchiveclosed = true;
}
/**
* Adds a row (array of strings) to the export data.
*
* @param array $row the row to add, $row has to be a plain array of strings
* @return void
*/
public function add_row(array $row): void {
$this->exportdata[] = $row;
$this->currentrow++;
}
/**
* Adds a data string (so the content for a "cell") to the current row.
*
* @param string $cellcontent the content to add to the current row
* @return void
*/
public function add_to_current_row(string $cellcontent): void {
$this->exportdata[$this->currentrow][] = $cellcontent;
}
/**
* Signal the entries_exporter to finish the current row and jump to the next row.
*
* @return void
*/
public function next_row(): void {
$this->currentrow++;
}
/**
* Sets the name of the export file.
*
* Only use the basename without path and without extension here.
*
* @param string $exportfilename name of the file without path and extension
* @return void
*/
public function set_export_file_name(string $exportfilename): void {
$this->exportfilename = $exportfilename;
}
/**
* The entries_exporter will prepare a data file from the rows and columns being added.
* Overwrite this method to generate the data file as string.
*
* @return string the data file as a string
*/
abstract protected function get_data_file_content(): string;
/**
* Overwrite the method to return the file extension your data file will have, for example
* <code>return 'csv';</code> for a csv file entries_exporter.
*
* @return string the file extension of the data file your entries_exporter is using
*/
abstract protected function get_export_data_file_extension(): string;
/**
* Returns the count of currently stored records (rows excluding header row).
*
* @return int the count of records/rows
*/
public function get_records_count(): int {
// The attribute $this->exportdata also contains a header. If only one row is present, this
// usually is the header, so record count should be 0.
if (count($this->exportdata) <= 1) {
return 0;
}
return count($this->exportdata) - 1;
}
/**
* Use this method to add a file which should be exported to the entries_exporter.
*
* @param string $filename the name of the file which should be added
* @param string $filecontent the content of the file as a string
* @param string $zipsubdir the subdirectory in the zip archive. Defaults to 'files/'.
* @return void
* @throws moodle_exception if there is an error adding the file to the zip archive
*/
public function add_file_from_string(string $filename, string $filecontent, string $zipsubdir = 'files/'): void {
if (empty($this->filenamesinzip)) {
// No files added yet, so we need to create a zip archive.
$this->create_zip_archive();
}
if (!str_ends_with($zipsubdir, '/')) {
$zipsubdir .= '/';
}
$zipfilename = $zipsubdir . $filename;
$this->filenamesinzip[] = $zipfilename;
$this->ziparchive->add_file_from_string($zipfilename, $filecontent);
}
/**
* Sends the generated export file.
*
* Care: By default this function finishes the current PHP request and directly serves the file to the user as download.
*
* @param bool $sendtouser true if the file should be sent directly to the user, if false the file content will be returned
* as string
* @return string|null file content as string if $sendtouser is true
* @throws moodle_exception if there is an issue adding the data file
* @throws file_serving_exception if the file could not be served properly
*/
public function send_file(bool $sendtouser = true): null|string {
if (empty($this->filenamesinzip)) {
if ($sendtouser) {
send_file($this->get_data_file_content(),
$this->exportfilename . '.' . $this->get_export_data_file_extension(),
null, 0, true, true);
return null;
} else {
return $this->get_data_file_content();
}
}
$this->add_file_from_string($this->exportfilename . '.' . $this->get_export_data_file_extension(),
$this->get_data_file_content(), '/');
$this->finish_zip_archive();
if ($this->isziparchiveclosed) {
if ($sendtouser) {
send_file($this->zipfilepath, $this->exportfilename . '.zip', null, 0, false, true);
return null;
} else {
return file_get_contents($this->zipfilepath);
}
} else {
throw new file_serving_exception('Could not serve zip file, it could not be closed properly.');
}
}
/**
* Checks if a file with the given name has already been added to the file export bundle.
*
* Care: Filenames are compared to all files in the specified zip subdirectory which
* defaults to 'files/'.
*
* @param string $filename the filename containing the zip path of the file to check
* @param string $zipsubdir The subdirectory in which the filename should be looked for,
* defaults to 'files/'
* @return bool true if file with the given name already exists, false otherwise
*/
public function file_exists(string $filename, string $zipsubdir = 'files/'): bool {
if (!str_ends_with($zipsubdir, '/')) {
$zipsubdir .= '/';
}
if (empty($filename)) {
return false;
}
return in_array($zipsubdir . $filename, $this->filenamesinzip, true);
}
/**
* Creates a unique filename based on the given filename.
*
* This method adds "_1", "_2", ... to the given file name until the newly generated filename
* is not equal to any of the already saved ones in the export file bundle.
*
* @param string $filename the filename based on which a unique filename should be generated
* @return string the unique filename
*/
public function create_unique_filename(string $filename): string {
if (!$this->file_exists($filename)) {
return $filename;
}
$extension = pathinfo($filename, PATHINFO_EXTENSION);
$filenamewithoutextension = empty($extension)
? $filename
: substr($filename, 0,strlen($filename) - strlen($extension) - 1);
$filenamewithoutextension = $filenamewithoutextension . '_1';
$i = 1;
$filename = empty($extension) ? $filenamewithoutextension : $filenamewithoutextension . '.' . $extension;
while ($this->file_exists($filename)) {
// In case we have already a file ending with '_XX' where XX is an ascending number, we have to
// remove '_XX' first before adding '_YY' again where YY is the successor of XX.
$filenamewithoutextension = preg_replace('/_' . $i . '$/', '_' . ($i + 1), $filenamewithoutextension);
$filename = empty($extension) ? $filenamewithoutextension : $filenamewithoutextension . '.' . $extension;
$i++;
}
return $filename;
}
/**
* Prepares the zip archive.
*
* @return void
*/
private function create_zip_archive(): void {
$tmpdir = make_request_directory();
$this->zipfilepath = $tmpdir . '/' . $this->exportfilename . '.zip';
$this->ziparchive = new zip_archive();
$this->isziparchiveclosed = !$this->ziparchive->open($this->zipfilepath);
}
/**
* Closes the zip archive.
*
* @return void
*/
private function finish_zip_archive(): void {
if (!$this->isziparchiveclosed) {
$this->isziparchiveclosed = $this->ziparchive->close();
}
}
}

65
mod/data/classes/local/exporter/ods_entries_exporter.php Normal file
View File

@ -0,0 +1,65 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\exporter;
use MoodleODSWorkbook;
use MoodleODSWriter;
/**
* ODS entries exporter for mod_data.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class ods_entries_exporter extends entries_exporter {
/**
* Returns the file extension of this entries exporter.
*
* @see \mod_data\local\exporter\entries_exporter::get_export_data_file_extension()
*/
public function get_export_data_file_extension(): string {
return 'ods';
}
/**
* Returns the ods data exported by the ODS library for further handling.
*
* @see \mod_data\local\exporter\entries_exporter::get_data_file_content()
*/
public function get_data_file_content(): string {
global $CFG;
require_once("$CFG->libdir/odslib.class.php");
$filearg = '-';
$workbook = new MoodleODSWorkbook($filearg);
$worksheet = [];
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($this->exportdata as $row) {
$colno = 0;
foreach ($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$writer = new MoodleODSWriter($worksheet);
return $writer->get_file_content();
}
}

141
mod/data/classes/local/exporter/utils.php Normal file
View File

@ -0,0 +1,141 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\exporter;
use context;
use context_system;
/**
* Utility class for exporting data from a mod_data instance.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class utils {
/**
* Exports the data of the mod_data instance to an entries_exporter object which then can export it to a file format.
*
* @param int $dataid
* @param array $fields
* @param array $selectedfields
* @param entries_exporter $exporter the entries_exporter object used
* @param int $currentgroup group ID of the current group. This is used for
* exporting data while maintaining group divisions.
* @param context|null $context the context in which the operation is performed (for capability checks)
* @param bool $userdetails whether to include the details of the record author
* @param bool $time whether to include time created/modified
* @param bool $approval whether to include approval status
* @param bool $tags whether to include tags
* @param bool $includefiles whether files should be exported as well
* @return void
*/
public static function data_exportdata(int $dataid, array $fields, array $selectedfields, entries_exporter $exporter,
int $currentgroup = 0, context $context = null, bool $userdetails = false, bool $time = false, bool $approval = false,
bool $tags = false, bool $includefiles = true): void {
global $DB;
if (is_null($context)) {
$context = context_system::instance();
}
// Exporting user data needs special permission.
$userdetails = $userdetails && has_capability('mod/data:exportuserinfo', $context);
// Populate the header in first row of export.
$header = [];
foreach ($fields as $key => $field) {
if (!in_array($field->field->id, $selectedfields)) {
// Ignore values we aren't exporting.
unset($fields[$key]);
} else {
$header[] = $field->field->name;
}
}
if ($tags) {
$header[] = get_string('tags', 'data');
}
if ($userdetails) {
$header[] = get_string('user');
$header[] = get_string('username');
$header[] = get_string('email');
}
if ($time) {
$header[] = get_string('timeadded', 'data');
$header[] = get_string('timemodified', 'data');
}
if ($approval) {
$header[] = get_string('approved', 'data');
}
$exporter->add_row($header);
$datarecords = $DB->get_records('data_records', array('dataid' => $dataid));
ksort($datarecords);
$line = 1;
foreach ($datarecords as $record) {
// Get content indexed by fieldid.
if ($currentgroup) {
$select = 'SELECT c.fieldid, c.content, c.content1, c.content2, c.content3, c.content4 FROM {data_content} c, '
. '{data_records} r WHERE c.recordid = ? AND r.id = c.recordid AND r.groupid = ?';
$where = array($record->id, $currentgroup);
} else {
$select = 'SELECT fieldid, content, content1, content2, content3, content4 FROM {data_content} WHERE recordid = ?';
$where = array($record->id);
}
if ($content = $DB->get_records_sql($select, $where)) {
foreach ($fields as $field) {
$contents = '';
if (isset($content[$field->field->id])) {
$contents = $field->export_text_value($content[$field->field->id]);
if (!empty($contents) && $field->file_export_supported() && $includefiles
&& !is_null($field->export_file_value($record))) {
// For exporting overwrite the content of the column with a unique
// filename, even it is not exactly the name of the file in the
// mod_data instance content. But it's more important to match the name
// of the exported file.
$contents = $exporter->create_unique_filename($contents);
$exporter->add_file_from_string($contents, $field->export_file_value($record));
}
}
// Just be double sure.
$contents = !empty($contents) ? $contents : '';
$exporter->add_to_current_row($contents);
}
if ($tags) {
$itemtags = \core_tag_tag::get_item_tags_array('mod_data', 'data_records', $record->id);
$exporter->add_to_current_row(implode(', ', $itemtags));
}
if ($userdetails) { // Add user details to the export data.
$userdata = get_complete_user_data('id', $record->userid);
$exporter->add_to_current_row(fullname($userdata));
$exporter->add_to_current_row($userdata->username);
$exporter->add_to_current_row($userdata->email);
}
if ($time) { // Add time added / modified.
$exporter->add_to_current_row(userdate($record->timecreated));
$exporter->add_to_current_row(userdate($record->timemodified));
}
if ($approval) { // Add approval status.
$exporter->add_to_current_row((int) $record->approved);
}
}
$exporter->next_row();
}
}
}

200
mod/data/classes/local/importer/csv_entries_importer.php Normal file
View File

@ -0,0 +1,200 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\importer;
use context_module;
use core_php_time_limit;
use core_tag_tag;
use core_user;
use csv_import_reader;
use moodle_exception;
use stdClass;
/**
* CSV entries_importer class for importing data and - if needed - files as well from a zip archive.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class csv_entries_importer extends entries_importer {
/** @var array Log entries for successfully added records. */
private array $addedrecordsmessages = [];
/**
* Declares the entries_importer to use a csv file as data file.
*
* @see entries_importer::get_import_data_file_extension()
*/
public function get_import_data_file_extension(): string {
return 'csv';
}
/**
* Import records for a data instance from csv data.
*
* @param stdClass $cm Course module of the data instance.
* @param stdClass $data The data instance.
* @param string $encoding The encoding of csv data.
* @param string $fielddelimiter The delimiter of the csv data.
*
* @throws moodle_exception
*/
public function import_csv(stdClass $cm, stdClass $data, string $encoding, string $fielddelimiter): void {
global $CFG, $DB;
// Large files are likely to take their time and memory. Let PHP know
// that we'll take longer, and that the process should be recycled soon
// to free up memory.
core_php_time_limit::raise();
raise_memory_limit(MEMORY_HUGE);
$iid = csv_import_reader::get_new_iid('moddata');
$cir = new csv_import_reader($iid, 'moddata');
$context = context_module::instance($cm->id);
$readcount = $cir->load_csv_content($this->get_data_file_content(), $encoding, $fielddelimiter);
if (empty($readcount)) {
throw new \moodle_exception('csvfailed', 'data', "{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}");
} else {
if (!$fieldnames = $cir->get_columns()) {
throw new \moodle_exception('cannotreadtmpfile', 'error');
}
// Check the fieldnames are valid.
$rawfields = $DB->get_records('data_fields', ['dataid' => $data->id], '', 'name, id, type');
$fields = [];
$errorfield = '';
$usernamestring = get_string('username');
$safetoskipfields = [get_string('user'), get_string('email'),
get_string('timeadded', 'data'), get_string('timemodified', 'data'),
get_string('approved', 'data'), get_string('tags', 'data')];
$userfieldid = null;
foreach ($fieldnames as $id => $name) {
if (!isset($rawfields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
} else if (!in_array($name, $safetoskipfields)) {
$errorfield .= "'$name' ";
}
} else {
// If this is the second time, a field with this name comes up, it must be a field not provided by the user...
// like the username.
if (isset($fields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
}
unset($fieldnames[$id]); // To ensure the user provided content fields remain in the array once flipped.
} else {
$field = $rawfields[$name];
$filepath = "$CFG->dirroot/mod/data/field/$field->type/field.class.php";
if (!file_exists($filepath)) {
$errorfield .= "'$name' ";
continue;
}
require_once($filepath);
$classname = 'data_field_' . $field->type;
$fields[$name] = new $classname($field, $data, $cm);
}
}
}
if (!empty($errorfield)) {
throw new \moodle_exception('fieldnotmatched', 'data',
"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}", $errorfield);
}
$fieldnames = array_flip($fieldnames);
$cir->init();
while ($record = $cir->next()) {
$authorid = null;
if ($userfieldid) {
if (!($author = core_user::get_user_by_username($record[$userfieldid], 'id'))) {
$authorid = null;
} else {
$authorid = $author->id;
}
}
if ($recordid = data_add_record($data, 0, $authorid)) { // Add instance to data_record.
foreach ($fields as $field) {
$fieldid = $fieldnames[$field->field->name];
if (isset($record[$fieldid])) {
$value = $record[$fieldid];
} else {
$value = '';
}
if (method_exists($field, 'update_content_import')) {
$field->update_content_import($recordid, $value, 'field_' . $field->field->id);
} else {
$content = new stdClass();
$content->fieldid = $field->field->id;
$content->content = $value;
$content->recordid = $recordid;
if ($field->file_import_supported() && $this->importfiletype === 'zip') {
$filecontent = $this->get_file_content_from_zip($content->content);
if (!$filecontent) {
// No corresponding file in zip archive, so no record for this field being added at all.
continue;
}
$contentid = $DB->insert_record('data_content', $content);
$field->import_file_value($contentid, $filecontent, $content->content);
} else {
$DB->insert_record('data_content', $content);
}
}
}
if (core_tag_tag::is_enabled('mod_data', 'data_records') &&
isset($fieldnames[get_string('tags', 'data')])) {
$columnindex = $fieldnames[get_string('tags', 'data')];
$rawtags = $record[$columnindex];
$tags = explode(',', $rawtags);
foreach ($tags as $tag) {
$tag = trim($tag);
if (empty($tag)) {
continue;
}
core_tag_tag::add_item_tag('mod_data', 'data_records', $recordid, $context, $tag);
}
}
$this->addedrecordsmessages[] = get_string('added', 'moodle',
count($this->addedrecordsmessages) + 1)
. ". " . get_string('entry', 'data')
. " (ID $recordid)\n";
}
}
$cir->close();
$cir->cleanup(true);
}
}
/**
* Getter for the array of messages for added records.
*
* For each successfully added record the array contains a log message.
*
* @return array Array of message strings: For each added record one message string
*/
public function get_added_records_messages(): array {
return $this->addedrecordsmessages;
}
}

144
mod/data/classes/local/importer/entries_importer.php Normal file
View File

@ -0,0 +1,144 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data\local\importer;
use coding_exception;
use core_php_time_limit;
use file_packer;
/**
* Importer class for importing data and - if needed - files as well from a zip archive.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
abstract class entries_importer {
/** @var string The import file path of the file which data should be imported from. */
protected string $importfilepath;
/** @var string The original name of the import file name including extension of the file which data should be imported from. */
protected string $importfilename;
/** @var string $importfiletype The file type of the import file. */
protected string $importfiletype;
/** @var file_packer Zip file packer to extract files from a zip archive. */
private file_packer $packer;
/** @var bool Tracks state if zip archive has been extracted already. */
private bool $zipfileextracted;
/** @var string Temporary directory where zip archive is being extracted to. */
private string $extracteddir;
/**
* Creates an entries_importer object.
*
* This object can be used to import data from data files (like csv) and zip archives both including a data file and files to be
* stored in the course module context.
*
* @param string $importfilepath the complete path of the import file including filename
* @param string $importfilename the import file name as uploaded by the user
* @throws coding_exception if a wrong file type is being used
*/
public function __construct(string $importfilepath, string $importfilename) {
$this->importfilepath = $importfilepath;
$this->importfilename = $importfilename;
$this->importfiletype = pathinfo($importfilename, PATHINFO_EXTENSION);
$this->zipfileextracted = false;
if ($this->importfiletype !== $this->get_import_data_file_extension() && $this->importfiletype !== 'zip') {
throw new coding_exception('Only "zip" or "' . $this->get_import_data_file_extension() . '" files are '
. 'allowed.');
}
}
/**
* Return the file extension of the import data file which is being used, for example 'csv' for a csv entries_importer.
*
* @return string the file extension of the export data file
*/
abstract public function get_import_data_file_extension(): string;
/**
* Returns the file content of the data file.
*
* Returns the content of the file directly if the entries_importer's file is a data file itself.
* If the entries_importer's file is a zip archive, the content of the first found data file in the
* zip archive's root will be returned.
*
* @return false|string the data file content as string; false, if file cannot be found/read
*/
public function get_data_file_content(): false|string {
if ($this->importfiletype !== 'zip') {
// We have no zip archive, so the file itself must be the data file.
return file_get_contents($this->importfilepath);
}
// So we have a zip archive and need to find the right data file in the root of the zip archive.
$this->extract_zip();
$datafilenames = array_filter($this->packer->list_files($this->importfilepath),
fn($file) => pathinfo($file->pathname, PATHINFO_EXTENSION) === $this->get_import_data_file_extension()
&& !str_contains($file->pathname, '/'));
if (empty($datafilenames) || count($datafilenames) > 1) {
return false;
}
return file_get_contents($this->extracteddir . reset($datafilenames)->pathname);
}
/**
* Returns the file content from a file which has been stored in the zip archive.
*
* @param string $filename
* @param string $zipsubdir
* @return false|string the file content as string, false if the file could not be found/read
*/
public function get_file_content_from_zip(string $filename, string $zipsubdir = 'files/'): false|string {
if (empty($filename)) {
// Nothing to return.
return false;
}
// Just to be sure extract if not extracted yet.
$this->extract_zip();
if (!str_ends_with($zipsubdir, '/')) {
$zipsubdir .= '/';
}
$filepathinextractedzip = $this->extracteddir . $zipsubdir . $filename;
return file_exists($filepathinextractedzip) ? file_get_contents($filepathinextractedzip) : false;
}
/**
* Extracts (if not already done and if we have a zip file to deal with) the zip file to a temporary directory.
*
* @return void
*/
private function extract_zip(): void {
if ($this->zipfileextracted || $this->importfiletype !== 'zip') {
return;
}
$this->packer = get_file_packer();
core_php_time_limit::raise(180);
$this->extracteddir = make_request_directory();
if (!str_ends_with($this->extracteddir, '/')) {
$this->extracteddir .= '/';
}
$this->packer->extract_to_pathname($this->importfilepath, $this->extracteddir);
$this->zipfileextracted = true;
}
}

233
mod/data/deprecatedlib.php
View File

@ -61,3 +61,236 @@ function data_get_completion_state($course, $cm, $userid, $type) {
}
return $result;
}
/**
* @deprecated since Moodle 4.3.
* @global object
* @param array $export
* @param string $dataname
* @param int $count
* @return string
*/
function data_export_xls($export, $dataname, $count) {
global $CFG;
debugging('Function data_export_xls() has been deprecated, because xls export has been dropped.',
DEBUG_DEVELOPER);
require_once("$CFG->libdir/excellib.class.php");
$filename = clean_filename("{$dataname}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$filename .= '.xls';
$filearg = '-';
$workbook = new MoodleExcelWorkbook($filearg);
$workbook->send($filename);
$worksheet = array();
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($export as $row) {
$colno = 0;
foreach($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$workbook->close();
return $filename;
}
/**
* @deprecated since Moodle 4.3, exporting is now being done by \mod_data\local\exporter\csv_entries_exporter
* @global object
* @param array $export
* @param string $delimiter_name
* @param object $database
* @param int $count
* @param bool $return
* @return string|void
*/
function data_export_csv($export, $delimiter_name, $database, $count, $return=false) {
global $CFG;
debugging('Function data_export_csv has been deprecated. Exporting is now being done by '
. '\mod_data\local\csv_exporter.', DEBUG_DEVELOPER);
require_once($CFG->libdir . '/csvlib.class.php');
$filename = $database . '-' . $count . '-record';
if ($count > 1) {
$filename .= 's';
}
if ($return) {
return csv_export_writer::print_array($export, $delimiter_name, '"', true);
} else {
csv_export_writer::download_array($filename, $export, $delimiter_name);
}
}
/**
* @deprecated since Moodle 4.3, exporting is now being done by \mod_data\local\exporter\ods_entries_exporter
* @global object
* @param array $export
* @param string $dataname
* @param int $count
* @param string
*/
function data_export_ods($export, $dataname, $count) {
global $CFG;
debugging('Function data_export_ods has been deprecated. Exporting is now being done by '
. '\mod_data\local\ods_exporter.', DEBUG_DEVELOPER);
require_once("$CFG->libdir/odslib.class.php");
$filename = clean_filename("{$dataname}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$filename .= '.ods';
$filearg = '-';
$workbook = new MoodleODSWorkbook($filearg);
$workbook->send($filename);
$worksheet = array();
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($export as $row) {
$colno = 0;
foreach($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$workbook->close();
return $filename;
}
/**
* @deprecated since Moodle 4.3, use \mod_data\local\exporter\utils::data_exportdata with a \mod_data\local\exporter\entries_exporter object
* @global object
* @param int $dataid
* @param array $fields
* @param array $selectedfields
* @param int $currentgroup group ID of the current group. This is used for
* exporting data while maintaining group divisions.
* @param object $context the context in which the operation is performed (for capability checks)
* @param bool $userdetails whether to include the details of the record author
* @param bool $time whether to include time created/modified
* @param bool $approval whether to include approval status
* @param bool $tags whether to include tags
* @return array
*/
function data_get_exportdata($dataid, $fields, $selectedfields, $currentgroup=0, $context=null,
$userdetails=false, $time=false, $approval=false, $tags = false) {
global $DB;
debugging('Function data_get_exportdata has been deprecated. Use '
. '\mod_data\local\exporter_utils::data_exportdata with a \mod_data\local\exporter object instead',
DEBUG_DEVELOPER);
if (is_null($context)) {
$context = context_system::instance();
}
// exporting user data needs special permission
$userdetails = $userdetails && has_capability('mod/data:exportuserinfo', $context);
$exportdata = array();
// populate the header in first row of export
foreach($fields as $key => $field) {
if (!in_array($field->field->id, $selectedfields)) {
// ignore values we aren't exporting
unset($fields[$key]);
} else {
$exportdata[0][] = $field->field->name;
}
}
if ($tags) {
$exportdata[0][] = get_string('tags', 'data');
}
if ($userdetails) {
$exportdata[0][] = get_string('user');
$exportdata[0][] = get_string('username');
$exportdata[0][] = get_string('email');
}
if ($time) {
$exportdata[0][] = get_string('timeadded', 'data');
$exportdata[0][] = get_string('timemodified', 'data');
}
if ($approval) {
$exportdata[0][] = get_string('approved', 'data');
}
$datarecords = $DB->get_records('data_records', array('dataid'=>$dataid));
ksort($datarecords);
$line = 1;
foreach($datarecords as $record) {
// get content indexed by fieldid
if ($currentgroup) {
$select = 'SELECT c.fieldid, c.content, c.content1, c.content2, c.content3, c.content4 FROM {data_content} c, {data_records} r WHERE c.recordid = ? AND r.id = c.recordid AND r.groupid = ?';
$where = array($record->id, $currentgroup);
} else {
$select = 'SELECT fieldid, content, content1, content2, content3, content4 FROM {data_content} WHERE recordid = ?';
$where = array($record->id);
}
if( $content = $DB->get_records_sql($select, $where) ) {
foreach($fields as $field) {
$contents = '';
if(isset($content[$field->field->id])) {
$contents = $field->export_text_value($content[$field->field->id]);
}
$exportdata[$line][] = $contents;
}
if ($tags) {
$itemtags = \core_tag_tag::get_item_tags_array('mod_data', 'data_records', $record->id);
$exportdata[$line][] = implode(', ', $itemtags);
}
if ($userdetails) { // Add user details to the export data
$userdata = get_complete_user_data('id', $record->userid);
$exportdata[$line][] = fullname($userdata);
$exportdata[$line][] = $userdata->username;
$exportdata[$line][] = $userdata->email;
}
if ($time) { // Add time added / modified
$exportdata[$line][] = userdate($record->timecreated);
$exportdata[$line][] = userdate($record->timemodified);
}
if ($approval) { // Add approval status
$exportdata[$line][] = (int) $record->approved;
}
}
$line++;
}
$line--;
return $exportdata;
}
/**
* @deprecated since Moodle 4.3, importing is now being done by \mod_data\local\importer\csv_importer::import_csv
* Import records for a data instance from csv data.
*
* @param object $cm Course module of the data instance.
* @param object $data The data instance.
* @param string $csvdata The csv data to be imported.
* @param string $encoding The encoding of csv data.
* @param string $fielddelimiter The delimiter of the csv data.
* @return int Number of records added.
*/
function data_import_csv($cm, $data, &$csvdata, $encoding, $fielddelimiter) {
debugging('Function data_import_csv has been deprecated. '
. 'Importing is now being done by \mod_data\local\csv_importer::import_csv.',
DEBUG_DEVELOPER);
// New function needs a file, not the file content, so we have to temporarily put the content into a file.
$tmpdir = make_request_directory();
$tmpfilename = 'tmpfile.csv';
$tmpfilepath = $tmpdir . '/tmpfile.csv';
file_put_contents($tmpfilepath, $csvdata);
$importer = new \mod_data\local\importer\csv_entries_importer($tmpfilepath, $tmpfilename);
$importer->import_csv($cm, $data, $encoding, $fielddelimiter);
return 0;
}

27
mod/data/export.php
View File

@ -95,20 +95,31 @@ if ($mform->is_cancelled()) {
$currentgroup = groups_get_activity_group($cm);
$exportdata = data_get_exportdata($data->id, $fields, $selectedfields, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags);
$count = count($exportdata);
$exporter = null;
switch ($formdata['exporttype']) {
case 'csv':
data_export_csv($exportdata, $formdata['delimiter_name'], $data->name, $count);
break;
case 'xls':
data_export_xls($exportdata, $data->name, $count);
$exporter = new \mod_data\local\exporter\csv_entries_exporter();
$exporter->set_delimiter_name($formdata['delimiter_name']);
break;
case 'ods':
data_export_ods($exportdata, $data->name, $count);
$exporter = new \mod_data\local\exporter\ods_entries_exporter();
break;
default:
throw new coding_exception('Invalid export format has been specified. '
. 'Only "csv" and "ods" are currently supported.');
}
$includefiles = !empty($formdata['includefiles']);
\mod_data\local\exporter\utils::data_exportdata($data->id, $fields, $selectedfields, $exporter, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags, $includefiles);
$count = $exporter->get_records_count();
$filename = clean_filename("{$data->name}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$exporter->set_export_file_name($filename);
$exporter->send_file();
}
// Build header to match the rest of the UI.

2
mod/data/export_form.php
View File

@ -112,6 +112,8 @@ class mod_data_export_form extends moodleform {
$exportoptions[] = $mform->createElement('checkbox', 'exportapproval', get_string('includeapproval', 'data'), '',
$optionattrs);
}
$exportoptions[] = $mform->createElement('checkbox', 'includefiles', get_string('includefiles', 'data'), '', $optionattrs);
$mform->setDefault('includefiles', 1);
$mform->addGroup($exportoptions, 'exportoptions', get_string('selectexportoptions', 'data'), ['<br>'], false);
$this->add_action_buttons(true, get_string('exportentries', 'data'));

60
mod/data/field/file/field.class.php
View File

@ -218,8 +218,64 @@ class data_field_file extends data_field_base {
$DB->update_record('data_content', $content);
}
function text_export_supported() {
return false;
/**
* Here we export the text value of a file field which is the filename of the exported file.
*
* @param stdClass $record the record which is being exported
* @return string the value which will be stored in the exported file for this field
*/
public function export_text_value(stdClass $record): string {
return !empty($record->content) ? $record->content : '';
}
/**
* Specifies that this field type supports the export of files.
*
* @return bool true which means that file export is being supported by this field type
*/
public function file_export_supported(): bool {
return true;
}
/**
* Specifies that this field type supports the import of files.
*
* @return bool true which means that file import is being supported by this field type
*/
public function file_import_supported(): bool {
return true;
}
/**
* Provides the necessary code for importing a file when importing the content of a mod_data instance.
*
* @param int $contentid the id of the mod_data content record
* @param string $filecontent the content of the file to import as string
* @param string $filename the filename the imported file should get
* @return void
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
$filerecord = [
'contextid' => $this->context->id,
'component' => 'mod_data',
'filearea' => 'content',
'itemid' => $contentid,
'filepath' => '/',
'filename' => $filename,
];
$fs = get_file_storage();
$fs->create_file_from_string($filerecord, $filecontent);
}
/**
* Exports the file content for file export.
*
* @param stdClass $record the data content record the file belongs to
* @return null|string The file content of the stored file or null if no file should be exported for this record
*/
public function export_file_value(stdClass $record): null|string {
$file = $this->get_file($record->id);
return $file ? $file->get_content() : null;
}
function file_ok($path) {

75
mod/data/field/picture/field.class.php
View File

@ -335,12 +335,16 @@ class data_field_picture extends data_field_base {
// If thumbnail width and height are BOTH not specified then no thumbnail is generated, and
// additionally an attempted delete of the existing thumbnail takes place.
$fs = get_file_storage();
$file_record = array('contextid'=>$file->get_contextid(), 'component'=>$file->get_component(), 'filearea'=>$file->get_filearea(),
'itemid'=>$file->get_itemid(), 'filepath'=>$file->get_filepath(),
'filename'=>'thumb_'.$file->get_filename(), 'userid'=>$file->get_userid());
$filerecord = [
'contextid' => $file->get_contextid(), 'component' => $file->get_component(), 'filearea' => $file->get_filearea(),
'itemid' => $file->get_itemid(), 'filepath' => $file->get_filepath(),
'filename' => 'thumb_' . $file->get_filename(), 'userid' => $file->get_userid()
];
try {
// this may fail for various reasons
$fs->convert_image($file_record, $file, (int) $this->field->param4, (int) $this->field->param5, true);
// This may fail for various reasons.
$newwidth = isset($this->field->param4) ? (int) $this->field->param4 : null;
$newheight = isset($this->field->param5) ? (int) $this->field->param5 : null;
$fs->convert_image($filerecord, $file, $newwidth, $newheight, true);
return true;
} catch (Exception $e) {
debugging($e->getMessage());
@ -348,8 +352,65 @@ class data_field_picture extends data_field_base {
}
}
function text_export_supported() {
return false;
/**
* Here we export the text value of a picture field which is the filename of the exported picture.
*
* @param stdClass $record the record which is being exported
* @return string the value which will be stored in the exported file for this field
*/
public function export_text_value(stdClass $record): string {
return !empty($record->content) ? $record->content : '';
}
/**
* Specifies that this field type supports the export of files.
*
* @return bool true which means that file export is being supported by this field type
*/
public function file_export_supported(): bool {
return true;
}
/**
* Exports the file content for file export.
*
* @param stdClass $record the data content record the file belongs to
* @return null|string The file content of the stored file or null if no file should be exported for this record
*/
public function export_file_value(stdClass $record): null|string {
$file = $this->get_file($record->id);
return $file ? $file->get_content() : null;
}
/**
* Specifies that this field type supports the import of files.
*
* @return bool true which means that file import is being supported by this field type
*/
public function file_import_supported(): bool {
return true;
}
/**
* Provides the necessary code for importing a file when importing the content of a mod_data instance.
*
* @param int $contentid the id of the mod_data content record
* @param string $filecontent the content of the file to import as string
* @param string $filename the filename the imported file should get
* @return void
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
$filerecord = [
'contextid' => $this->context->id,
'component' => 'mod_data',
'filearea' => 'content',
'itemid' => $contentid,
'filepath' => '/',
'filename' => $filename,
];
$fs = get_file_storage();
$file = $fs->create_file_from_string($filerecord, $filecontent);
$this->update_thumbnail(null, $file);
}
function file_ok($path) {

30
mod/data/import.php
View File

@ -86,16 +86,32 @@ echo $OUTPUT->header();
echo $OUTPUT->heading_with_help(get_string('uploadrecords', 'mod_data'), 'uploadrecords', 'mod_data');
if ($formdata = $form->get_data()) {
$filecontent = $form->get_file_content('recordsfile');
$recordsadded = data_import_csv($cm, $data, $filecontent, $formdata->encoding, $formdata->fielddelimiter);
$uploadedfilepath = $form->save_temp_file('recordsfile');
$filestempdir = null;
if ($recordsadded > 0) {
echo $OUTPUT->notification($recordsadded. ' '. get_string('recordssaved', 'data'), '');
} else {
echo $OUTPUT->notification(get_string('recordsnotsaved', 'data'), 'notifysuccess');
if (!$uploadedfilepath) {
throw new coding_exception('No file uploaded.');
}
echo $OUTPUT->continue_button($redirectbackto);
$importer = new \mod_data\local\importer\csv_entries_importer($uploadedfilepath, $form->get_new_filename('recordsfile'));
if (!$importer->get_data_file_content()) {
echo $OUTPUT->notification(get_string('errordatafilenotfound', 'data'),
\core\output\notification::NOTIFY_ERROR);
} else {
$importer->import_csv($cm, $data, $formdata->encoding, $formdata->fielddelimiter);
unlink($uploadedfilepath);
$addedrecordsmessages = $importer->get_added_records_messages();
echo html_writer::div(implode('<br/>', $addedrecordsmessages));
if (count($addedrecordsmessages) > 0) {
echo $OUTPUT->notification(count($addedrecordsmessages) . ' ' . get_string('recordssaved', 'data'),
\core\output\notification::NOTIFY_SUCCESS);
} else {
echo $OUTPUT->notification(get_string('recordsnotsaved', 'data'),
\core\output\notification::NOTIFY_ERROR);
}
}
} else {
/// Upload records section. Only for teachers and the admin.
echo $OUTPUT->box_start('generalbox boxaligncenter boxwidthwide');

3
mod/data/import_form.php
View File

@ -14,7 +14,8 @@ class mod_data_import_form extends moodleform {
$dataid = $this->_customdata['dataid'];
$backtourl = $this->_customdata['backtourl'];
$mform->addElement('filepicker', 'recordsfile', get_string('csvfile', 'data'));
$mform->addElement('filepicker', 'recordsfile', get_string('csvfile', 'data'),
null, ['accepted_types' => ['application/zip', 'text/csv']]);
$delimiters = csv_import_reader::get_delimiter_list();
$mform->addElement('select', 'fielddelimiter', get_string('fielddelimiter', 'data'), $delimiters);

14
mod/data/lang/en/data.php
View File

@ -83,7 +83,7 @@ $string['createfields'] = 'Create fields to collect different types of data.';
$string['createtemplates'] = 'Templates define the interface of your activity. Once you create fields, templates will be created automatically. Alternatively, you can use a preset, which includes fields and templates.';
$string['csstemplate'] = 'Custom CSS';
$string['csvfailed'] = 'Unable to read the raw data from the CSV file';
$string['csvfile'] = 'CSV file';
$string['csvfile'] = 'CSV or ZIP containing a CSV file';
$string['csvimport'] = 'CSV file import';
$string['csvimport_help'] = 'Entries may be imported via a plain text file with a list of field names as the first line, then the data, with one record per line.';
$string['csvwithselecteddelimiter'] = '<abbr title="Comma Separated Values">CSV</abbr>';
@ -152,6 +152,7 @@ $string['entrieslefttoadd'] = 'You must add {$a->entriesleft} more entry/entries
$string['entrieslefttoaddtoview'] = 'You must add {$a->entrieslefttoview} more entry/entries before you can view other participants\' entries.';
$string['entry'] = 'Entry';
$string['entrysaved'] = 'Your entry has been saved';
$string['errordatafilenotfound'] = 'The file could not be imported. Accepted file types are CSV or a ZIP containing a CSV file in the format used for exporting entries.';
$string['errormustbeteacher'] = 'You need to be a teacher to use this page!';
$string['errorpresetexists'] = 'A preset with this name already exists.';
$string['errorpresetexistsbutnotoverwrite'] = 'A preset with this name already exists. Choose a different name.';
@ -227,6 +228,7 @@ $string['importapreset'] = 'Import a preset';
$string['importsuccess'] = 'Preset applied.';
$string['importpresetmissingcapability'] = 'You don\'t have permission to import a preset.';
$string['includeapproval'] = 'Include approval status';
$string['includefiles'] = 'Include files in export';
$string['includetags'] = 'Include tags';
$string['includetime'] = 'Include time added/modified';
$string['includeuserdetails'] = 'Include user details';
@ -453,14 +455,10 @@ $string['unsupportedfields'] = 'Unsupported fields';
$string['unsupportedfieldslist'] = 'The following fields cannot be exported:';
$string['updatefield'] = 'Update an existing field';
$string['uploadfile'] = 'Upload file';
$string['uploadrecords'] = 'Upload entries from a file';
$string['uploadrecords_help'] = 'Entries may be uploaded via text file. The format of the file should be as follows:
$string['uploadrecords'] = 'Import entries';
$string['uploadrecords_help'] = 'Import entries that you have exported from another database, either via CSV or a ZIP containing a CSV file (if files are included in the export).
* Each line of the file contains one record
* Each record is a series of data separated by the selected separator
* The first record contains a list of fieldnames defining the format of the rest of the file
The field enclosure is a character that surrounds each field in each record. It can normally be left unset.';
Alternatively, to create a CSV file for importing, add one entry to the database and then export it. Edit the CSV file and add more entries.';
$string['uploadrecords_link'] = 'mod/data/import';
$string['url'] = 'URL';
$string['usedate'] = 'Include in search.';

376
mod/data/lib.php
View File

@ -640,16 +640,62 @@ class data_field_base { // Base class for Database Field Types (see field/*/
}
/**
* Per default, return the record's text value only from the "content" field.
* Override this in fields class if necesarry.
* Per default, it is assumed that fields do not support file exporting. Override this (return true)
* on fields supporting file export. You will also have to implement export_file_value().
*
* @param string $record
* @return bool true if field will export a file, false otherwise
*/
public function file_export_supported(): bool {
return false;
}
/**
* Per default, does not return a file (just null).
* Override this in fields class, if you want your field to export a file content.
* In case you are exporting a file value, export_text_value() should return the corresponding file name.
*
* @param stdClass $record
* @return null|string the file content as string or null, if no file content is being provided
*/
public function export_file_value(stdClass $record): null|string {
return null;
}
/**
* Per default, a field does not support the import of files.
*
* A field type can overwrite this function and return true. In this case it also has to implement the function
* import_file_value().
*
* @return false means file imports are not supported
*/
public function file_import_supported(): bool {
return false;
}
/**
* Returns a stored_file object for exporting a file of a given record.
*
* @param int $contentid content id
* @param string $filecontent the content of the file as string
* @param string $filename the filename the file should have
*/
public function import_file_value(int $contentid, string $filecontent, string $filename): void {
return;
}
/**
* Per default, return the record's text value only from the "content" field.
* Override this in fields class if necessary.
*
* @param stdClass $record
* @return string
*/
function export_text_value($record) {
public function export_text_value(stdClass $record) {
if ($this->text_export_supported()) {
return $record->content;
}
return '';
}
/**
@ -2978,328 +3024,6 @@ function data_supports($feature) {
}
}
/**
* Import records for a data instance from csv data.
*
* @param object $cm Course module of the data instance.
* @param object $data The data instance.
* @param string $csvdata The csv data to be imported.
* @param string $encoding The encoding of csv data.
* @param string $fielddelimiter The delimiter of the csv data.
* @return int Number of records added.
*/
function data_import_csv($cm, $data, &$csvdata, $encoding, $fielddelimiter) {
global $CFG, $DB;
// Large files are likely to take their time and memory. Let PHP know
// that we'll take longer, and that the process should be recycled soon
// to free up memory.
core_php_time_limit::raise();
raise_memory_limit(MEMORY_EXTRA);
$iid = csv_import_reader::get_new_iid('moddata');
$cir = new csv_import_reader($iid, 'moddata');
$context = context_module::instance($cm->id);
$readcount = $cir->load_csv_content($csvdata, $encoding, $fielddelimiter);
$csvdata = null; // Free memory.
if (empty($readcount)) {
throw new \moodle_exception('csvfailed', 'data', "{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}");
} else {
if (!$fieldnames = $cir->get_columns()) {
throw new \moodle_exception('cannotreadtmpfile', 'error');
}
// Check the fieldnames are valid.
$rawfields = $DB->get_records('data_fields', array('dataid' => $data->id), '', 'name, id, type');
$fields = array();
$errorfield = '';
$usernamestring = get_string('username');
$safetoskipfields = array(get_string('user'), get_string('email'),
get_string('timeadded', 'data'), get_string('timemodified', 'data'),
get_string('approved', 'data'), get_string('tags', 'data'));
$userfieldid = null;
foreach ($fieldnames as $id => $name) {
if (!isset($rawfields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
} else if (!in_array($name, $safetoskipfields)) {
$errorfield .= "'$name' ";
}
} else {
// If this is the second time, a field with this name comes up, it must be a field not provided by the user...
// like the username.
if (isset($fields[$name])) {
if ($name == $usernamestring) {
$userfieldid = $id;
}
unset($fieldnames[$id]); // To ensure the user provided content fields remain in the array once flipped.
} else {
$field = $rawfields[$name];
$filepath = "$CFG->dirroot/mod/data/field/$field->type/field.class.php";
if (!file_exists($filepath)) {
$errorfield .= "'$name' ";
continue;
}
require_once($filepath);
$classname = 'data_field_' . $field->type;
$fields[$name] = new $classname($field, $data, $cm);
}
}
}
if (!empty($errorfield)) {
throw new \moodle_exception('fieldnotmatched', 'data',
"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}", $errorfield);
}
$fieldnames = array_flip($fieldnames);
$cir->init();
$recordsadded = 0;
while ($record = $cir->next()) {
$authorid = null;
if ($userfieldid) {
if (!($author = core_user::get_user_by_username($record[$userfieldid], 'id'))) {
$authorid = null;
} else {
$authorid = $author->id;
}
}
if ($recordid = data_add_record($data, 0, $authorid)) { // Add instance to data_record.
foreach ($fields as $field) {
$fieldid = $fieldnames[$field->field->name];
if (isset($record[$fieldid])) {
$value = $record[$fieldid];
} else {
$value = '';
}
if (method_exists($field, 'update_content_import')) {
$field->update_content_import($recordid, $value, 'field_' . $field->field->id);
} else {
$content = new stdClass();
$content->fieldid = $field->field->id;
$content->content = $value;
$content->recordid = $recordid;
$DB->insert_record('data_content', $content);
}
}
if (core_tag_tag::is_enabled('mod_data', 'data_records') &&
isset($fieldnames[get_string('tags', 'data')])) {
$columnindex = $fieldnames[get_string('tags', 'data')];
$rawtags = $record[$columnindex];
$tags = explode(',', $rawtags);
foreach ($tags as $tag) {
$tag = trim($tag);
if (empty($tag)) {
continue;
}
core_tag_tag::add_item_tag('mod_data', 'data_records', $recordid, $context, $tag);
}
}
$recordsadded++;
print get_string('added', 'moodle', $recordsadded) . ". " . get_string('entry', 'data') . " (ID $recordid)<br />\n";
}
}
$cir->close();
$cir->cleanup(true);
return $recordsadded;
}
return 0;
}
/**
* @global object
* @param array $export
* @param string $delimiter_name
* @param object $database
* @param int $count
* @param bool $return
* @return string|void
*/
function data_export_csv($export, $delimiter_name, $database, $count, $return=false) {
global $CFG;
require_once($CFG->libdir . '/csvlib.class.php');
$filename = $database . '-' . $count . '-record';
if ($count > 1) {
$filename .= 's';
}
if ($return) {
return csv_export_writer::print_array($export, $delimiter_name, '"', true);
} else {
csv_export_writer::download_array($filename, $export, $delimiter_name);
}
}
/**
* @global object
* @param array $export
* @param string $dataname
* @param int $count
* @return string
*/
function data_export_xls($export, $dataname, $count) {
global $CFG;
require_once("$CFG->libdir/excellib.class.php");
$filename = clean_filename("{$dataname}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$filename .= '.xls';
$filearg = '-';
$workbook = new MoodleExcelWorkbook($filearg);
$workbook->send($filename);
$worksheet = array();
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($export as $row) {
$colno = 0;
foreach($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$workbook->close();
return $filename;
}
/**
* @global object
* @param array $export
* @param string $dataname
* @param int $count
* @param string
*/
function data_export_ods($export, $dataname, $count) {
global $CFG;
require_once("$CFG->libdir/odslib.class.php");
$filename = clean_filename("{$dataname}-{$count}_record");
if ($count > 1) {
$filename .= 's';
}
$filename .= clean_filename('-' . gmdate("Ymd_Hi"));
$filename .= '.ods';
$filearg = '-';
$workbook = new MoodleODSWorkbook($filearg);
$workbook->send($filename);
$worksheet = array();
$worksheet[0] = $workbook->add_worksheet('');
$rowno = 0;
foreach ($export as $row) {
$colno = 0;
foreach($row as $col) {
$worksheet[0]->write($rowno, $colno, $col);
$colno++;
}
$rowno++;
}
$workbook->close();
return $filename;
}
/**
* @global object
* @param int $dataid
* @param array $fields
* @param array $selectedfields
* @param int $currentgroup group ID of the current group. This is used for
* exporting data while maintaining group divisions.
* @param object $context the context in which the operation is performed (for capability checks)
* @param bool $userdetails whether to include the details of the record author
* @param bool $time whether to include time created/modified
* @param bool $approval whether to include approval status
* @param bool $tags whether to include tags
* @return array
*/
function data_get_exportdata($dataid, $fields, $selectedfields, $currentgroup=0, $context=null,
$userdetails=false, $time=false, $approval=false, $tags = false) {
global $DB;
if (is_null($context)) {
$context = context_system::instance();
}
// exporting user data needs special permission
$userdetails = $userdetails && has_capability('mod/data:exportuserinfo', $context);
$exportdata = array();
// populate the header in first row of export
foreach($fields as $key => $field) {
if (!in_array($field->field->id, $selectedfields)) {
// ignore values we aren't exporting
unset($fields[$key]);
} else {
$exportdata[0][] = $field->field->name;
}
}
if ($tags) {
$exportdata[0][] = get_string('tags', 'data');
}
if ($userdetails) {
$exportdata[0][] = get_string('user');
$exportdata[0][] = get_string('username');
$exportdata[0][] = get_string('email');
}
if ($time) {
$exportdata[0][] = get_string('timeadded', 'data');
$exportdata[0][] = get_string('timemodified', 'data');
}
if ($approval) {
$exportdata[0][] = get_string('approved', 'data');
}
$datarecords = $DB->get_records('data_records', array('dataid'=>$dataid));
ksort($datarecords);
$line = 1;
foreach($datarecords as $record) {
// get content indexed by fieldid
if ($currentgroup) {
$select = 'SELECT c.fieldid, c.content, c.content1, c.content2, c.content3, c.content4 FROM {data_content} c, {data_records} r WHERE c.recordid = ? AND r.id = c.recordid AND r.groupid = ?';
$where = array($record->id, $currentgroup);
} else {
$select = 'SELECT fieldid, content, content1, content2, content3, content4 FROM {data_content} WHERE recordid = ?';
$where = array($record->id);
}
if( $content = $DB->get_records_sql($select, $where) ) {
foreach($fields as $field) {
$contents = '';
if(isset($content[$field->field->id])) {
$contents = $field->export_text_value($content[$field->field->id]);
}
$exportdata[$line][] = $contents;
}
if ($tags) {
$itemtags = \core_tag_tag::get_item_tags_array('mod_data', 'data_records', $record->id);
$exportdata[$line][] = implode(', ', $itemtags);
}
if ($userdetails) { // Add user details to the export data
$userdata = get_complete_user_data('id', $record->userid);
$exportdata[$line][] = fullname($userdata);
$exportdata[$line][] = $userdata->username;
$exportdata[$line][] = $userdata->email;
}
if ($time) { // Add time added / modified
$exportdata[$line][] = userdate($record->timecreated);
$exportdata[$line][] = userdate($record->timemodified);
}
if ($approval) { // Add approval status
$exportdata[$line][] = (int) $record->approved;
}
}
$line++;
}
$line--;
return $exportdata;
}
////////////////////////////////////////////////////////////////////////////////
// File API //
////////////////////////////////////////////////////////////////////////////////

260
mod/data/tests/entries_export_test.php Normal file
View File

@ -0,0 +1,260 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data;
use context_module;
use mod_data\local\exporter\csv_entries_exporter;
use mod_data\local\exporter\ods_entries_exporter;
use mod_data\local\exporter\utils;
/**
* Unit tests for exporting entries.
*
* @package mod_data
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class entries_export_test extends \advanced_testcase {
/**
* Get the test data.
*
* In this instance we are setting up database records to be used in the unit tests.
*
* @return array of test instances
*/
protected function get_test_data(): array {
$this->resetAfterTest(true);
/** @var \mod_data_generator $generator */
$generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
$course = $this->getDataGenerator()->create_course();
$teacher = $this->getDataGenerator()->create_and_enrol($course, 'teacher');
$this->setUser($teacher);
$student = $this->getDataGenerator()->create_and_enrol($course, 'student', ['username' => 'student']);
$data = $generator->create_instance(['course' => $course->id]);
$cm = get_coursemodule_from_instance('data', $data->id);
// Add fields.
$fieldrecord = new \stdClass();
$fieldrecord->name = 'numberfield'; // Identifier of the records for testing.
$fieldrecord->type = 'number';
$numberfield = $generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'textfield';
$fieldrecord->type = 'text';
$textfield = $generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield1';
$fieldrecord->type = 'file';
$filefield1 = $generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield2';
$fieldrecord->type = 'file';
$filefield2 = $generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'picturefield';
$fieldrecord->type = 'picture';
$picturefield = $generator->create_field($fieldrecord, $data);
$contents[$numberfield->field->id] = '3';
$contents[$textfield->field->id] = 'a simple text';
$contents[$filefield1->field->id] = 'samplefile.png';
$contents[$filefield2->field->id] = 'samplefile.png';
$contents[$picturefield->field->id] = ['picturefile.png', 'this picture shows something'];
$generator->create_entry($data, $contents);
return [
'teacher' => $teacher,
'student' => $student,
'data' => $data,
'cm' => $cm,
];
}
/**
* Tests the exporting of the content of a mod_data instance by using the csv_entries_exporter.
*
* It also includes more general testing of the functionality of the entries_exporter the csv_entries_exporter
* is inheriting from.
*
* @covers \mod_data\local\exporter\entries_exporter
* @covers \mod_data\local\exporter\entries_exporter::get_records_count()
* @covers \mod_data\local\exporter\entries_exporter::send_file()
* @covers \mod_data\local\exporter\csv_entries_exporter
* @covers \mod_data\local\exporter\utils::data_exportdata
*/
public function test_export_csv(): void {
global $DB;
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$exporter = new csv_entries_exporter();
$exporter->set_export_file_name('testexportfile');
$fieldrecords = $DB->get_records('data_fields', ['dataid' => $data->id], 'id');
$fields = [];
foreach ($fieldrecords as $fieldrecord) {
$fields[] = data_get_field($fieldrecord, $data);
}
// We select all fields.
$selectedfields = array_map(fn($field) => $field->field->id, $fields);
$currentgroup = groups_get_activity_group($cm);
$context = context_module::instance($cm->id);
$exportuser = false;
$exporttime = false;
$exportapproval = false;
$tags = false;
// We first test the export without exporting files.
// This means file and picture fields will be exported, but only as text (which is the filename),
// so we will receive a csv export file.
$includefiles = false;
utils::data_exportdata($data->id, $fields, $selectedfields, $exporter, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags, $includefiles);
$this->assertEquals(file_get_contents(__DIR__ . '/fixtures/test_data_export_without_files.csv'),
$exporter->send_file(false));
$this->assertEquals(1, $exporter->get_records_count());
// We now test the export including files. This will generate a zip archive.
$includefiles = true;
$exporter = new csv_entries_exporter();
$exporter->set_export_file_name('testexportfile');
utils::data_exportdata($data->id, $fields, $selectedfields, $exporter, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags, $includefiles);
// We now write the zip archive temporary to disc to be able to parse it and assert it has the correct structure.
$tmpdir = make_request_directory();
file_put_contents($tmpdir . '/testexportarchive.zip', $exporter->send_file(false));
$ziparchive = new \zip_archive();
$ziparchive->open($tmpdir . '/testexportarchive.zip');
$expectedfilecontents = [
// The test generator for mod_data uses a copy of pix/monologo.png as sample file content for the file stored in a
// file and picture field.
// So we expect that this file has to have the same content as monologo.png.
// Also, the default value for the subdirectory in the zip archive containing the files is 'files/'.
'files/samplefile.png' => 'mod/data/pix/monologo.png',
'files/samplefile_1.png' => 'mod/data/pix/monologo.png',
'files/picturefile.png' => 'mod/data/pix/monologo.png',
// By checking that the content of the exported csv is identical to the fixture file it is verified
// that the filenames in the csv file correspond to the names of the exported file.
// It also verifies that files with identical file names in different fields (or records) will be numbered
// automatically (samplefile.png, samplefile_1.png, ...).
'testexportfile.csv' => __DIR__ . '/fixtures/test_data_export_with_files.csv'
];
for ($i = 0; $i < $ziparchive->count(); $i++) {
// We here iterate over all files in the zip archive and check if their content is identical to the files
// in the $expectedfilecontents array.
$filestream = $ziparchive->get_stream($i);
$fileinfo = $ziparchive->get_info($i);
$filecontent = fread($filestream, $fileinfo->size);
$this->assertEquals(file_get_contents($expectedfilecontents[$fileinfo->pathname]), $filecontent);
fclose($filestream);
}
$ziparchive->close();
unlink($tmpdir . '/testexportarchive.zip');
}
/**
* Tests specific ODS exporting functionality.
*
* @covers \mod_data\local\exporter\ods_entries_exporter
* @covers \mod_data\local\exporter\utils::data_exportdata
*/
public function test_export_ods(): void {
global $DB;
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$exporter = new ods_entries_exporter();
$exporter->set_export_file_name('testexportfile');
$fieldrecords = $DB->get_records('data_fields', ['dataid' => $data->id], 'id');
$fields = [];
foreach ($fieldrecords as $fieldrecord) {
$fields[] = data_get_field($fieldrecord, $data);
}
// We select all fields.
$selectedfields = array_map(fn($field) => $field->field->id, $fields);
$currentgroup = groups_get_activity_group($cm);
$context = context_module::instance($cm->id);
$exportuser = false;
$exporttime = false;
$exportapproval = false;
$tags = false;
// We first test the export without exporting files.
// This means file and picture fields will be exported, but only as text (which is the filename),
// so we will receive an ods export file.
$includefiles = false;
utils::data_exportdata($data->id, $fields, $selectedfields, $exporter, $currentgroup, $context,
$exportuser, $exporttime, $exportapproval, $tags, $includefiles);
$odsrows = $this->get_ods_rows_content($exporter->send_file(false));
// Check, if the headings match with the first row of the ods file.
$i = 0;
foreach ($fields as $field) {
$this->assertEquals($field->field->name, $odsrows[0][$i]);
$i++;
}
// Check, if the values match with the field values.
$this->assertEquals('3', $odsrows[1][0]);
$this->assertEquals('a simple text', $odsrows[1][1]);
$this->assertEquals('samplefile.png', $odsrows[1][2]);
$this->assertEquals('samplefile.png', $odsrows[1][3]);
$this->assertEquals('picturefile.png', $odsrows[1][4]);
// As the logic of renaming the files and building a zip archive is implemented in entries_exporter class, we do
// not need to test this for the ods_entries_exporter, because entries_export_test::test_export_csv already does this.
}
/**
* Helper function to extract the text data as row arrays from an ODS document.
*
* @param string $content the file content
* @return array two-dimensional row/column array with the text content of the first spreadsheet
*/
private function get_ods_rows_content(string $content): array {
$file = tempnam(make_request_directory(), 'ods_');
$filestream = fopen($file, "w");
fwrite($filestream, $content);
$reader = new \OpenSpout\Reader\ODS\Reader();
$reader->open($file);
/** @var \OpenSpout\Reader\ODS\Sheet[] $sheets */
$sheets = $reader->getSheetIterator();
$rowscellsvalues = [];
foreach ($sheets as $sheet) {
/** @var \OpenSpout\Common\Entity\Row[] $rows */
$rows = $sheet->getRowIterator();
foreach ($rows as $row) {
$cellvalues = [];
foreach ($row->getCells() as $cell) {
$cellvalues[] = $cell->getValue();
}
$rowscellsvalues[] = $cellvalues;
}
}
return $rowscellsvalues;
}
}

222
mod/data/tests/entries_exporter_test.php Normal file
View File

@ -0,0 +1,222 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data;
use context_module;
use mod_data\local\exporter\csv_entries_exporter;
use mod_data\local\exporter\ods_entries_exporter;
use mod_data\local\exporter\utils;
/**
* Unit tests for entries_exporter and csv_entries_exporter classes.
*
* Also {@see entries_export_test} class which provides module tests for exporting entries.
*
* @package mod_data
* @covers \mod_data\local\exporter\entries_exporter
* @covers \mod_data\local\exporter\csv_entries_exporter
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class entries_exporter_test extends \advanced_testcase {
/**
* Tests get_records_count method.
*
* @covers \mod_data\local\exporter\entries_exporter::get_records_count
* @dataProvider get_records_count_provider
* @param array $rows the rows from the data provider to be tested by the exporter
* @param int $expectedcount the expected count of records to be exported
*/
public function test_get_records_count(array $rows, int $expectedcount): void {
$exporter = new csv_entries_exporter();
foreach ($rows as $row) {
$exporter->add_row($row);
}
$this->assertEquals($expectedcount, $exporter->get_records_count());
}
/**
* Data provider method for self::test_get_records_count.
*
* @return array data for testing
*/
public function get_records_count_provider(): array {
return [
'onlyheader' => [
'rows' => [
['numberfield', 'textfield', 'filefield1', 'filefield2', 'picturefield']
],
'expectedcount' => 0 // Only header present, so we expect record count 0.
],
'onerecord' => [
'rows' => [
['numberfield', 'textfield', 'filefield1', 'filefield2', 'picturefield'],
['3', 'a simple text', 'samplefile.png', 'samplefile_1.png', 'picturefile.png']
],
'expectedcount' => 1
],
'tworecords' => [
'rows' => [
['numberfield', 'textfield', 'filefield1', 'filefield2', 'picturefield'],
['3', 'a simple text', 'samplefile.png', 'samplefile_1.png', 'picturefile.png'],
['5', 'a supersimple text', 'anotherfile.png', 'someotherfile.png', 'andapicture.png']
],
'expectedcount' => 2
]
];
}
/**
* Tests adding of files to the exporter to be included in the exported zip archive.
*
* @dataProvider add_file_from_string_provider
* @covers \mod_data\local\exporter\entries_exporter::add_file_from_string
* @covers \mod_data\local\exporter\entries_exporter::file_exists
* @param array $files array of filename and filecontent to be tested for exporting
* @param bool $success if the exporting of files should be successful
*/
public function test_add_file_from_string(array $files, bool $success): void {
$exporter = new csv_entries_exporter();
foreach ($files as $file) {
if (empty($file['subdir'])) {
$exporter->add_file_from_string($file['filename'], $file['filecontent']);
$this->assertEquals($exporter->file_exists($file['filename']), $success);
} else {
$exporter->add_file_from_string($file['filename'], $file['filecontent'], $file['subdir']);
$this->assertEquals($exporter->file_exists($file['filename'], $file['subdir']), $success);
}
}
}
/**
* Data provider method for self::test_add_file_from_string.
*
* @return array data for testing
*/
public function add_file_from_string_provider(): array {
return [
'one file' => [
'files' => [
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent'
],
],
'success' => true
],
'more files, also with subdirs' => [
'files' => [
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent'
],
[
'filename' => 'testfile2.txt',
'filecontent' => 'someothercontent',
'subdir' => 'testsubdir'
],
[
'filename' => 'testfile3.txt',
'filecontent' => 'someverydifferentcontent',
'subdir' => 'files/foo/bar'
],
[
'filename' => 'testfile4.txt',
'filecontent' => 'someverydifferentcontent',
'subdir' => 'files/foo/bar/'
],
[
'filename' => 'testfile5.txt',
'filecontent' => 'someverydifferentcontent',
'subdir' => '/files/foo/bar/'
],
],
'success' => true
],
'nocontent' => [
'files' => [
[
'filename' => '',
'filecontent' => ''
]
],
'success' => false
]
];
}
/**
* Tests if unique filenames are being created correctly.
*
* @covers \mod_data\local\exporter\entries_exporter::create_unique_filename
* @dataProvider create_unique_filename_provider
* @param string $inputfilename the name of the file which should be converted into a unique filename
* @param string $resultfilename the maybe changed $inputfilename, so that it is unique in the exporter
*/
public function test_create_unique_filename(string $inputfilename, string $resultfilename): void {
$exporter = new csv_entries_exporter();
$exporter->add_file_from_string('test.txt', 'somecontent');
$exporter->add_file_from_string('foo.txt', 'somecontent');
$exporter->add_file_from_string('foo_1.txt', 'somecontent');
$exporter->add_file_from_string('foo_2.txt', 'somecontent');
$exporter->add_file_from_string('foo', 'somecontent');
$exporter->add_file_from_string('foo_1', 'somecontent');
$exporter->add_file_from_string('sample_5.txt', 'somecontent');
$exporter->add_file_from_string('bar_1.txt', 'somecontent');
$this->assertEquals($resultfilename, $exporter->create_unique_filename($inputfilename));
}
/**
* Data provider method for self::test_create_unique_filename.
*
* @return array data for testing
*/
public function create_unique_filename_provider(): array {
return [
'does not exist yet' => [
'inputfilename' => 'someuniquename.txt',
'resultfilename' => 'someuniquename.txt'
],
'already exists' => [
'inputfilename' => 'test.txt',
'resultfilename' => 'test_1.txt'
],
'already exists, other numbers as well' => [
'inputfilename' => 'foo.txt',
'resultfilename' => 'foo_3.txt'
],
'file with _5 suffix already exists' => [
'inputfilename' => 'sample_5.txt',
'resultfilename' => 'sample_5_1.txt'
],
'file with _1 suffix already exists' => [
'inputfilename' => 'bar_1.txt',
'resultfilename' => 'bar_1_1.txt'
],
'file without extension unique' => [
'inputfilename' => 'test',
'resultfilename' => 'test'
],
'file without extension not unique' => [
'inputfilename' => 'foo',
'resultfilename' => 'foo_2'
]
];
}
}

182
mod/data/tests/import_test.php → mod/data/tests/entries_import_test.php
View File

@ -16,6 +16,12 @@
namespace mod_data;
use coding_exception;
use dml_exception;
use mod_data\local\importer\csv_entries_importer;
use moodle_exception;
use zip_archive;
/**
* Unit tests for import.php.
*
@ -24,7 +30,7 @@ namespace mod_data;
* @copyright 2019 Tobias Reischmann
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class import_test extends \advanced_testcase {
class entries_import_test extends \advanced_testcase {
/**
* Set up function.
@ -68,6 +74,13 @@ class import_test extends \advanced_testcase {
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield';
$fieldrecord->type = 'file';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'picturefield';
$fieldrecord->type = 'picture';
$generator->create_field($fieldrecord, $data);
return [
'teacher' => $teacher,
@ -79,6 +92,7 @@ class import_test extends \advanced_testcase {
/**
* Test uploading entries for a data instance without userdata.
*
* @throws dml_exception
*/
public function test_import(): void {
@ -88,10 +102,9 @@ class import_test extends \advanced_testcase {
'teacher' => $teacher,
] = $this->get_test_data();
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import.csv');
ob_start();
data_import_csv($cm, $data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import.csv',
'test_data_import.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
// No userdata is present in the file: Fallback is to assign the uploading user as author.
$expecteduserids = array();
@ -110,6 +123,7 @@ class import_test extends \advanced_testcase {
* Test uploading entries for a data instance with userdata.
*
* At least one entry has an identifiable user, which is assigned as author.
*
* @throws dml_exception
*/
public function test_import_with_userdata(): void {
@ -120,10 +134,9 @@ class import_test extends \advanced_testcase {
'student' => $student,
] = $this->get_test_data();
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_userdata.csv');
ob_start();
data_import_csv($cm, $data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_userdata.csv',
'test_data_import_with_userdata.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$expecteduserids = array();
$expecteduserids[1] = $student->id; // User student exists and is assigned as author.
@ -143,6 +156,7 @@ class import_test extends \advanced_testcase {
* This should test the corner case, in which a user has defined a data fields, which has the same name
* as the current lang string for username. In that case, the first Username entry is used for the field.
* The second one is used to identify the author.
*
* @throws coding_exception
* @throws dml_exception
*/
@ -161,10 +175,9 @@ class import_test extends \advanced_testcase {
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_field_username.csv');
ob_start();
data_import_csv($cm, $data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_field_username.csv',
'test_data_import_with_field_username.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$expecteduserids = array();
$expecteduserids[1] = $student->id; // User student exists and is assigned as author.
@ -193,7 +206,7 @@ class import_test extends \advanced_testcase {
foreach ($expectedcontent[$identifier] as $field => $value) {
$this->assertEquals($value, $record->items[$field]->content,
"The value of field \"$field\" for the record at position \"$identifier\" ".
"The value of field \"$field\" for the record at position \"$identifier\" " .
"which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
}
}
@ -205,6 +218,7 @@ class import_test extends \advanced_testcase {
* This should test the corner case, in which a user has defined a data fields, which has the same name
* as the current lang string for username. In that case, the only Username entry is used for the field.
* The author should not be set.
*
* @throws coding_exception
* @throws dml_exception
*/
@ -223,10 +237,9 @@ class import_test extends \advanced_testcase {
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_userdata.csv');
ob_start();
data_import_csv($cm, $data, $filecontent, 'UTF-8', 'comma');
ob_end_clean();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_userdata.csv',
'test_data_import_with_userdata.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
// No userdata is present in the file: Fallback is to assign the uploading user as author.
$expecteduserids = array();
@ -251,17 +264,98 @@ class import_test extends \advanced_testcase {
foreach ($expectedcontent[$identifier] as $field => $value) {
$this->assertEquals($value, $record->items[$field]->content,
"The value of field \"$field\" for the record at position \"$identifier\" ".
"The value of field \"$field\" for the record at position \"$identifier\" " .
"which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
}
}
}
/**
* Tests the import including files from a zip archive.
*
* @covers \mod_data\local\importer\entries_importer
* @covers \mod_data\local\importer\csv_entries_importer
* @return void
*/
public function test_import_with_files(): void {
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_files.zip',
'test_data_import_with_files.zip');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$records = $this->get_data_records($data->id);
$ziparchive = new zip_archive();
$ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files.zip');
$importedcontent = array_values($records)[0]->items;
$this->assertEquals(17, $importedcontent['ID']->content);
$this->assertEquals('samplefile.png', $importedcontent['filefield']->content);
$this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
// We now check if content of imported file from zip content is identical to the content of the file
// stored in the mod_data record in the field 'filefield'.
$fileindex = array_values(array_map(fn($file) => $file->index,
array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplefile.png')))[0];
$filestream = $ziparchive->get_stream($fileindex);
$filefield = data_get_field_from_name('filefield', $data);
$filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
$this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
$filefieldfilecontent);
fclose($filestream);
// We now check if content of imported picture from zip content is identical to the content of the picture file
// stored in the mod_data record in the field 'picturefield'.
$fileindex = array_values(array_map(fn($file) => $file->index,
array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplepicture.png')))[0];
$filestream = $ziparchive->get_stream($fileindex);
$filefield = data_get_field_from_name('picturefield', $data);
$filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
$this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
$filefieldfilecontent);
fclose($filestream);
$this->assertCount(1, $importer->get_added_records_messages());
$ziparchive->close();
}
/**
* Tests the import including files from a zip archive.
*
* @covers \mod_data\local\importer\entries_importer
* @covers \mod_data\local\importer\csv_entries_importer
* @return void
*/
public function test_import_with_files_missing_file(): void {
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
$importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip',
'test_data_import_with_files_missing_file.zip');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$records = $this->get_data_records($data->id);
$ziparchive = new zip_archive();
$ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip');
$importedcontent = array_values($records)[0]->items;
$this->assertEquals(17, $importedcontent['ID']->content);
$this->assertFalse(isset($importedcontent['filefield']));
$this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
$this->assertCount(1, $importer->get_added_records_messages());
$ziparchive->close();
}
/**
* Returns the records of the data instance.
*
* Each records has an item entry, which contains all fields associated with this item.
* Each fields has the parameters name, type and content.
*
* @param int $dataid Id of the data instance.
* @return array The records of the data instance.
* @throws dml_exception
@ -279,4 +373,54 @@ class import_test extends \advanced_testcase {
}
return $records;
}
/**
* Tests if the amount of imported records is counted properly.
*
* @covers \mod_data\local\importer\csv_entries_importer::import_csv
* @covers \mod_data\local\importer\csv_entries_importer::get_added_records_messages
* @dataProvider get_added_record_messages_provider
* @param string $datafilecontent the content of the datafile to test as string
* @param int $expectedcount the expected count of messages depending on the datafile content
*/
public function test_get_added_record_messages(string $datafilecontent, int $expectedcount): void {
[
'data' => $data,
'cm' => $cm,
] = $this->get_test_data();
// First we need to create the zip file from the provided data.
$tmpdir = make_request_directory();
$datafile = $tmpdir . '/entries_import_test_datafile_tmp_' . time() . '.csv';
file_put_contents($datafile, $datafilecontent);
$importer = new csv_entries_importer($datafile, 'testdatafile.csv');
$importer->import_csv($cm, $data, 'UTF-8', 'comma');
$this->assertEquals($expectedcount, count($importer->get_added_records_messages()));
}
/**
* Data provider method for self::test_get_added_record_messages.
*
* @return array data for testing
*/
public function get_added_record_messages_provider(): array {
return [
'only header' => [
'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL,
'expectedcount' => 0 // One line is being assumed to be the header.
],
'one record' => [
'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL
. '5,"some short text",testfilename.pdf,testpicture.png',
'expectedcount' => 1
],
'two records' => [
'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL
. '5,"some short text",testfilename.pdf,testpicture.png' . PHP_EOL
. '3,"other text",testfilename2.pdf,testpicture2.png',
'expectedcount' => 2
],
];
}
}

205
mod/data/tests/entries_importer_test.php Normal file
View File

@ -0,0 +1,205 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
namespace mod_data;
use context_module;
use mod_data\local\exporter\csv_entries_exporter;
use mod_data\local\exporter\ods_entries_exporter;
use mod_data\local\exporter\utils;
use mod_data\local\importer\csv_entries_importer;
use zip_archive;
/**
* Unit tests for entries_importer and csv_entries_importer class.
*
* Also {@see entries_import_test} class which provides module tests for importing entries.
*
* @package mod_data
* @covers \mod_data\local\importer\entries_importer
* @covers \mod_data\local\importer\csv_entries_importer
* @copyright 2023 ISB Bayern
* @author Philipp Memmel
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class entries_importer_test extends \advanced_testcase {
/**
* Set up function.
*/
protected function setUp(): void {
parent::setUp();
global $CFG;
require_once($CFG->dirroot . '/mod/data/lib.php');
require_once($CFG->dirroot . '/lib/datalib.php');
require_once($CFG->dirroot . '/lib/csvlib.class.php');
require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php');
require_once($CFG->dirroot . '/mod/data/tests/generator/lib.php');
}
/**
* Get the test data.
* In this instance we are setting up database records to be used in the unit tests.
*
* @return array
*/
protected function get_test_data(): array {
$this->resetAfterTest(true);
$generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
$course = $this->getDataGenerator()->create_course();
$teacher = $this->getDataGenerator()->create_and_enrol($course, 'teacher');
$this->setUser($teacher);
$student = $this->getDataGenerator()->create_and_enrol($course, 'student', array('username' => 'student'));
$data = $generator->create_instance(array('course' => $course->id));
$cm = get_coursemodule_from_instance('data', $data->id);
// Add fields.
$fieldrecord = new \stdClass();
$fieldrecord->name = 'ID'; // Identifier of the records for testing.
$fieldrecord->type = 'number';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'Param2';
$fieldrecord->type = 'text';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'filefield';
$fieldrecord->type = 'file';
$generator->create_field($fieldrecord, $data);
$fieldrecord->name = 'picturefield';
$fieldrecord->type = 'picture';
$generator->create_field($fieldrecord, $data);
return [
'teacher' => $teacher,
'student' => $student,
'data' => $data,
'cm' => $cm,
];
}
/**
* Test importing files from zip archive.
*
* @covers \mod_data\local\importer\entries_importer::get_file_content_from_zip
* @covers \mod_data\local\importer\entries_importer::get_data_file_content
* @dataProvider get_file_content_from_zip_provider
* @param array $files array of filenames and filecontents to test
* @param mixed $datafilecontent the expected result returned by the method which is being tested here
*/
public function test_get_file_content_from_zip(array $files, mixed $datafilecontent): void {
// First we need to create the zip file from the provided data.
$tmpdir = make_request_directory();
$zipfilepath = $tmpdir . '/entries_importer_test_tmp_' . time() . '.zip';
$ziparchive = new zip_archive();
$ziparchive->open($zipfilepath);
foreach ($files as $file) {
$localname = empty($file['subdir']) ? $file['filename'] : $file['subdir'] . '/' . $file['filename'];
$ziparchive->add_file_from_string($localname, $file['filecontent']);
}
$ziparchive->close();
// We now created a zip archive according to the data provider's data. We now can test the importer.
$importer = new csv_entries_importer($zipfilepath, 'testzip.zip');
foreach ($files as $file) {
$subdir = empty($file['subdir']) ? '' : $file['subdir'];
$this->assertEquals($file['filecontent'], $importer->get_file_content_from_zip($file['filename'], $subdir));
}
// Test the method to retrieve the datafile content.
$this->assertEquals($datafilecontent, $importer->get_data_file_content());
unlink($zipfilepath);
}
/**
* Data provider method for self::test_get_file_content_from_zip.
*
* @return array data for testing
*/
public function get_file_content_from_zip_provider(): array {
return [
'some files in the zip archive' => [
'files' => [
[
'filename' => 'datafile.csv',
'filecontent' => 'some,csv,data'
],
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent',
'subdir' => 'files'
],
[
'filename' => 'testfile2.txt',
'filecontent' => 'someothercontent',
'subdir' => 'testsubdir'
]
],
// Should be identical with filecontent of 'datafile.csv' above.
'datafilecontent' => 'some,csv,data'
],
'wrongly placed data file' => [
'files' => [
[
'filename' => 'datafile.csv',
'filecontent' => 'some,csv,data',
'subdir' => 'wrongsubdir'
],
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent',
'subdir' => 'files'
],
[
'filename' => 'testfile2.txt',
'filecontent' => 'someothercontent',
'subdir' => 'testsubdir'
]
],
// Data file is not in the root directory, though no content should be retrieved.
'datafilecontent' => false
],
'two data files where only one is allowed' => [
'files' => [
[
'filename' => 'datafile.csv',
'filecontent' => 'some,csv,data',
],
[
'filename' => 'anothercsvfile.csv',
'filecontent' => 'some,other,csv,data',
],
[
'filename' => 'testfile.txt',
'filecontent' => 'somecontent',
'subdir' => 'files'
],
[
'filename' => 'testfile2.txt',
'filecontent' => 'someothercontent',
'subdir' => 'testsubdir'
]
],
// There are two data files in the zip root, so the data cannot be imported.
'datafilecontent' => false
],
];
}
}

2
mod/data/tests/fixtures/test_data_export_with_files.csv vendored Normal file
View File

@ -0,0 +1,2 @@
numberfield,textfield,filefield1,filefield2,picturefield
3,"a simple text",samplefile.png,samplefile_1.png,picturefile.png
1 numberfield textfield filefield1 filefield2 picturefield
2 3 a simple text samplefile.png samplefile_1.png picturefile.png

2
mod/data/tests/fixtures/test_data_export_without_files.csv vendored Normal file
View File

@ -0,0 +1,2 @@
numberfield,textfield,filefield1,filefield2,picturefield
3,"a simple text",samplefile.png,samplefile.png,picturefile.png
1 numberfield textfield filefield1 filefield2 picturefield
2 3 a simple text samplefile.png samplefile.png picturefile.png

BIN
mod/data/tests/fixtures/test_data_import_with_files.zip vendored Normal file
View File

Binary file not shown.

BIN
mod/data/tests/fixtures/test_data_import_with_files_missing_file.zip vendored Normal file
View File

Binary file not shown.

20
mod/data/upgrade.txt
View File

@ -1,6 +1,26 @@
This files describes API changes in /mod/data - plugins,
information provided here is intended especially for developers.
== 4.3 ==
* Function data_export_xls() has been deprecated and moved to deprecatedlib, because xls support has already been dropped.
* Functions data_export_csv(), data_export_ods() and data_get_exportdata() have been deprecated due to a bigger
refactoring of the way data is being exported. This is now being done by new exporter classes
\mod_data\local\exporter\csv_entries_exporter and \mod_data\local\exporter\ods_entries_exporter (inheriting from
exporter base class \mod_data\local\exporter\entries_exporter) as well as \mod_data\local\exporter\utils::data_exportdata().
* Function data_import_csv() has been deprecated and moved to deprecatedlib due to a bigger rework of the way data is
being imported. This is now being done by new importer class \mod_data\local\importer\csv_importer inheriting from new
class \mod_data\local\importer.
* Field base class now has two new methods file_export_supported() and export_file_value(). The method
file_export_supported() can be overwritten to declare that a field type can/wants to export a file. In this case this
field type will have to implement the method export_file_value() returning this file for exporting. Also: This field
type will have to export the name of the file by overwriting text_export_supported() to return true and make the
method export_text_value() return the name of the file.
* The field types file and picture now are able to export the file/picture.
* Field base class now has two new methods file_import_supported() and import_file_value(). The method
file_import_supported() can be overwritten to declare that a field type is able to import a file. In this case this
field type will have to implement the method import_file_value() doing the actual import of the file being passed.
* The field types file and picture now are able to import the file/picture.
== 4.2 ==
* The field base class now has a method validate(). Overwrite it in the field type to provide validation of field type's
parameters in the field add/modify form.