diff --git a/.upgradenotes/MDL-75850-2024082809421816.yml b/.upgradenotes/MDL-75850-2024082809421816.yml new file mode 100644 index 00000000000..0f0223835c0 --- /dev/null +++ b/.upgradenotes/MDL-75850-2024082809421816.yml @@ -0,0 +1,13 @@ +issueNumber: MDL-75850 +notes: + core_files: + - message: | + The following are the changes made: + - New hook after_file_created + - In the \core_files\file_storage, new additional param $notify (default is true) added to: + - ::create_file_from_storedfile() + - ::create_file_from_pathname() + - ::create_file_from_string() + - ::create_file() + If true, it will trigger the after_file_created hook to re-create the image. + type: improved diff --git a/admin/settings/fileredact.php b/admin/settings/fileredact.php new file mode 100644 index 00000000000..b558e284032 --- /dev/null +++ b/admin/settings/fileredact.php @@ -0,0 +1,43 @@ +. + +/** + * Configure the settings for fileredact. + * + * @package core_admin + * @copyright Meirza + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +if ($hassiteconfig) { + if (!$ADMIN->locate('fileredact')) { + $ADMIN->add('server', new admin_category('fileredact', get_string('fileredact', 'core_files'))); + } + // Get settings from each service. + $servicesdir = "{$CFG->libdir}/classes/fileredact/services/"; + $servicefiles = glob("{$servicesdir}*_service.php"); + foreach ($servicefiles as $servicefile) { + $servicename = basename($servicefile, '_service.php'); + $classname = "\\core\\fileredact\\services\\{$servicename}_service"; + if (class_exists($classname)) { + $fileredactsettings = new admin_settingpage($servicename, new lang_string("fileredact:$servicename", 'core_files')); + call_user_func("{$classname}::add_settings", $fileredactsettings); + $ADMIN->add('fileredact', $fileredactsettings); + } + } +} diff --git a/lang/en/files.php b/lang/en/files.php index 8cc465821f7..57b564773db 100644 --- a/lang/en/files.php +++ b/lang/en/files.php @@ -28,6 +28,29 @@ defined('MOODLE_INTERNAL') || die(); $string['contenthash'] = 'Content hash'; $string['eventfileaddedtodraftarea'] = 'File added to draft area'; $string['eventfiledeletedfromdraftarea'] = 'File deleted from draft area'; +$string['fileredact'] = 'File redact'; +$string['fileredact:exifremover'] = 'EXIF remover'; +$string['fileredact:exifremover:emptyremovetags'] = 'Remove tags can not be empty!'; +$string['fileredact:exifremover:enabled'] = 'Enable EXIF remover'; +$string['fileredact:exifremover:enabled_desc'] = 'By default, EXIF Remover only supports JPG files using PHP GD, or ExifTool if it is configured. +This degrades the quality of the image and removes the orientation tag. + +To enhance the performance of EXIF Remover, please configure the ExifTool settings below. + +More information about installing ExifTool can be found at {$a->link}'; +$string['fileredact:exifremover:failedprocessexiftool'] = 'Redaction failed: failed to process file with ExifTool!'; +$string['fileredact:exifremover:failedprocessgd'] = 'Redaction failed: failed to process file with PHP gd!'; +$string['fileredact:exifremover:heading'] = 'ExifTool'; +$string['fileredact:exifremover:mimetype'] = 'Supported MIME types'; +$string['fileredact:exifremover:mimetype_desc'] = 'To add new MIME types, ensure they\'re included in the File Types.'; +$string['fileredact:exifremover:removetags'] = 'The EXIF tags that will be removed.'; +$string['fileredact:exifremover:removetags_desc'] = 'The EXIF tags that need to be removed.'; +$string['fileredact:exifremover:tag:all'] = 'All'; +$string['fileredact:exifremover:tag:gps'] = 'GPS only'; +$string['fileredact:exifremover:tooldoesnotexist'] = 'Redaction failed: ExifTool does not exist!'; +$string['fileredact:exifremover:toolpath'] = 'Path to ExifTool'; +$string['fileredact:exifremover:toolpath_desc'] = 'To use the ExifTool, please provide the path to the ExifTool executable. +Typically, on Unix/Linux systems, the path is /usr/bin/exiftool.'; $string['privacy:metadata:file_conversions'] = 'A record of the file conversions performed by a user.'; $string['privacy:metadata:file_conversion:usermodified'] = 'The user who started the file conversion.'; $string['privacy:metadata:files'] = 'A record of the files uploaded or shared by users'; diff --git a/lib/classes/fileredact/hook_listener.php b/lib/classes/fileredact/hook_listener.php new file mode 100644 index 00000000000..cdb340d77f3 --- /dev/null +++ b/lib/classes/fileredact/hook_listener.php @@ -0,0 +1,50 @@ +. + +namespace core\fileredact; + +use core\hook\filestorage\after_file_created; + +/** + * Allow the plugin to call as soon as possible before the file is created. + * + * @package core + * @copyright Meirza + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class hook_listener { + /** + * Execute the available services after creating the file. + * + * @param after_file_created $hook + */ + public static function redact_after_file_created(after_file_created $hook): void { + $storedfile = $hook->storedfile; + + // The file mime-type must be present. Otherwise, bypass the process. + if (empty($storedfile->get_mimetype())) { + return; + } + + $manager = new manager($storedfile); + $manager->execute(); + + // Iterates through the errors returned by the manager and outputs each error message. + foreach ($manager->get_errors() as $e) { + debugging($e->getMessage()); + } + } +} diff --git a/lib/classes/fileredact/manager.php b/lib/classes/fileredact/manager.php new file mode 100644 index 00000000000..6b27a9f0b1d --- /dev/null +++ b/lib/classes/fileredact/manager.php @@ -0,0 +1,93 @@ +. + +namespace core\fileredact; + +use stored_file; + +/** + * Fileredact manager. + * + * Manages and executes redaction services. + * + * @package core + * @copyright Meirza + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class manager { + + /** @var array Holds an array of error messages. */ + private $errors = []; + + /** + * Constructor. + * + * @param stored_file $filerecord The file record as a stdClass object, or null if not available. + */ + public function __construct( + /** @var stored_file $filerecord File record. */ + private readonly stored_file $filerecord + ) { + } + + /** + * Executes redaction services. + */ + public function execute(): void { + // Get the file redact services. + $services = $this->get_services(); + foreach ($services as $serviceclass) { + try { + if (class_exists($serviceclass)) { + $service = new $serviceclass($this->filerecord); + // For the given service, execute them if they are enabled, and the given mime type is supported. + if ($service->is_enabled() && $service->is_mimetype_supported($this->filerecord->get_mimetype())) { + $service->execute(); + } + } + } catch (\Throwable $e) { + $this->errors[] = $e; + } + } + } + + /** + * Returns a list of applicable redaction services. + * + * @return string[] return list of services. + */ + protected function get_services(): array { + global $CFG; + $servicesdir = "{$CFG->libdir}/classes/fileredact/services/"; + $servicefiles = glob("{$servicesdir}*_service.php"); + $services = []; + foreach ($servicefiles as $servicefile) { + $servicename = basename($servicefile, '_service.php'); + $serviceclass = "\\core\\fileredact\\services\\{$servicename}_service"; + $services[] = $serviceclass; + } + return $services; + } + + /** + * Retrieves an array of error messages. + * + * @return array An array of error messages. + */ + public function get_errors(): array { + return $this->errors; + } +} diff --git a/lib/classes/fileredact/services/exifremover_service.php b/lib/classes/fileredact/services/exifremover_service.php new file mode 100644 index 00000000000..81c342cb33e --- /dev/null +++ b/lib/classes/fileredact/services/exifremover_service.php @@ -0,0 +1,372 @@ +. + +namespace core\fileredact\services; + +use stored_file; + +/** + * Remove EXIF data from supported image files using PHP GD, or ExifTool if it is configured. + * + * The PHP GD stripping has minimal configuration and removes all EXIF data. + * More stripping is made available when using ExifTool. + * + * @package core + * @copyright Meirza + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class exifremover_service extends service { + + /** @var array REMOVE_TAGS Tags to remove and their corresponding values. */ + const REMOVE_TAGS = [ + "gps" => '"-gps*="', + "all" => "-all=", + ]; + + /** @var string DEFAULT_REMOVE_TAGS Default tags that will be removed. */ + const DEFAULT_REMOVE_TAGS = "gps"; + + /** @var string DEFAULT_MIMETYPE Default MIME type for images. */ + const DEFAULT_MIMETYPE = "image/jpeg"; + + /** + * PRESERVE_TAGS Tag to preserve when stripping EXIF data. + * + * To add a new tag, add the tag with space as a separator. + * For example, if the model tag is preserved, then the value is "-Orientation -Model". + * + * @var string + */ + const PRESERVE_TAGS = "-Orientation"; + + /** @var int DEFAULT_JPEG_COMPRESSION Default JPEG compression quality. */ + const DEFAULT_JPEG_COMPRESSION = 90; + + /** @var bool $useexiftool Flag indicating whether to use ExifTool. */ + private bool $useexiftool = false; + + /** + * Class constructor. + * + * @param stored_file $storedfile The file record. + */ + public function __construct( + /** @var stored_file The file record. */ + private readonly stored_file $storedfile, + ) { + parent::__construct($storedfile); + + // To decide whether to use ExifTool or PHP GD, check the ExifTool path. + if (!empty($this->get_exiftool_path())) { + $this->useexiftool = true; + } + } + + /** + * Performs redaction on the specified file. + */ + public function execute(): void { + if ($this->useexiftool) { + // Use the ExifTool executable to remove the desired EXIF tags. + $this->execute_exiftool(); + } else { + // Use PHP GD lib to remove all EXIF tags. + $this->execute_gd(); + } + } + + /** + * Executes ExifTool to remove metadata from the original file. + * + * @throws \moodle_exception If the ExifTool process fails or the destination file is not created. + */ + private function execute_exiftool(): void { + $tmpfilepath = make_request_directory(); + $filerecordname = $this->clean_filename($this->storedfile->get_filename()); + $neworiginalfile = $tmpfilepath . DIRECTORY_SEPARATOR . 'new_' . $filerecordname; + $destinationfile = $tmpfilepath . DIRECTORY_SEPARATOR . $filerecordname; + + // Copy the original file to a new file. + try { + $this->storedfile->copy_content_to($neworiginalfile); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + + // Prepare the ExifTool command. + $command = $this->get_exiftool_command($neworiginalfile, $destinationfile); + // Run the command. + exec($command, $output, $resultcode); + // If the return code was not zero or the destination file was not successfully created. + if ($resultcode !== 0 || !file_exists($destinationfile)) { + throw new \moodle_exception( + errorcode: 'fileredact:exifremover:failedprocessexiftool', + module: 'core_files', + a: get_class($this), + debuginfo: implode($output), + ); + } + // Replacing the EXIF processed file to the original file. + $this->persist_redacted_file(file_get_contents($destinationfile)); + } + + /** + * Executes GD library to remove metadata from the original file. + */ + private function execute_gd(): void { + $imagedata = $this->recreate_image_gd(); + if (!$imagedata) { + throw new \moodle_exception( + errorcode: 'fileredact:exifremover:failedprocessgd', + module: 'core_files', + a: get_class($this), + ); + } + // Put the image string object data to the original file. + $this->persist_redacted_file($imagedata); + } + /** + * Gets the ExifTool command to strip the file of EXIF data. + * + * @param string $source The source path of the file. + * @param string $destination The destination path of the file. + * @return string The command to use to remove EXIF data from the file. + */ + private function get_exiftool_command(string $source, string $destination): string { + $exiftoolexec = escapeshellarg($this->get_exiftool_path()); + $removetags = $this->get_remove_tags(); + $tempdestination = escapeshellarg($destination); + $tempsource = escapeshellarg($source); + $preservetagsoption = "-tagsfromfile @ " . self::PRESERVE_TAGS; + $command = "$exiftoolexec $removetags $preservetagsoption -o $tempdestination -- $tempsource"; + $command .= " 2> /dev/null"; // Do not output any errors. + return $command; + } + + /** + * Retrieves the remove tag options based on configuration. + * + * @return string The remove tag options. + */ + private function get_remove_tags(): string { + $removetags = get_config('core_fileredact', 'exifremoverremovetags'); + // If the remove tags value is empty or not empty but does not exist in the array, then set the default. + if (!$removetags || ($removetags && !array_key_exists($removetags, self::REMOVE_TAGS))) { + $removetags = self::DEFAULT_REMOVE_TAGS; + } + return self::REMOVE_TAGS[$removetags]; + } + + /** + * Retrieves the path to the ExifTool executable. + * + * @return string The path to the ExifTool executable. + */ + private function get_exiftool_path(): string { + $toolpathconfig = get_config('core_fileredact', 'exifremovertoolpath'); + if (!empty($toolpathconfig) && is_executable($toolpathconfig)) { + return $toolpathconfig; + } + return ''; + } + + /** + * Recreate the image using PHP GD library to strip all EXIF data. + * + * @return string|false The recreated image data as a string if successful, false otherwise. + */ + private function recreate_image_gd(): string|false { + $content = $this->storedfile->get_content(); + // Fetch the image information for this image. + $imageinfo = @getimagesizefromstring($content); + if (empty($imageinfo)) { + return false; + } + // Create a new image from the file. + $image = @imagecreatefromstring($content); + + // Capture the image as a string object, rather than straight to file. + ob_start(); + if (!imagejpeg( + image: $image, + quality: self::DEFAULT_JPEG_COMPRESSION, + ) + ) { + ob_end_clean(); + return false; + } + $data = ob_get_clean(); + imagedestroy($image); + return $data; + } + + /** + * Persists the redacted file to the file storage. + * + * @param string $content File content. + */ + private function persist_redacted_file(string $content): void { + $filerecord = (object) [ + 'id' => $this->storedfile->get_id(), + 'mimetype' => $this->storedfile->get_mimetype(), + 'userid' => $this->storedfile->get_userid(), + 'source' => $this->storedfile->get_source(), + 'contextid' => $this->storedfile->get_contextid(), + 'component' => $this->storedfile->get_component(), + 'filearea' => $this->storedfile->get_filearea(), + 'itemid' => $this->storedfile->get_itemid(), + 'filepath' => $this->storedfile->get_filepath(), + 'filename' => $this->storedfile->get_filename(), + ]; + $fs = get_file_storage(); + $existingfile = $fs->get_file( + $filerecord->contextid, + $filerecord->component, + $filerecord->filearea, + $filerecord->itemid, + $filerecord->filepath, + $filerecord->filename, + ); + if ($existingfile) { + $existingfile->delete(); + } + $redactedfile = $fs->create_file_from_string($filerecord, $content, false); + $this->storedfile->replace_file_with($redactedfile); + } + + /** + * Clean up a file name if it starts with a dash (U+002D) or a Unicode minus sign (U+2212). + * + * According to https://exiftool.org/#security, ensure that input file names do not start with + * a dash (U+002D) or a Unicode minus sign (U+2212). If found, remove the leading dash or Unicode minus sign. + * + * @param string $filename The file name to clean. + * @return string The cleaned file name. + */ + private function clean_filename(string $filename): string { + $pattern = '/^[\x{002D}\x{2212}]/u'; + if (preg_match($pattern, $filename)) { + $filename = preg_replace($pattern, '', $filename); + } + return clean_param($filename, PARAM_PATH); + } + + /** + * Returns true if the service is enabled, and false if it is not. + * + * @return bool + */ + public function is_enabled(): bool { + return (bool) get_config('core_fileredact', 'exifremoverenabled'); + } + + /** + * Determines whether a certain mime-type is supported by the service. + * It will return true if the mime-type is supported, and false if it is not. + * + * @param string $mimetype The mime type of file. + * @return bool + */ + public function is_mimetype_supported(string $mimetype): bool { + if ($mimetype === self::DEFAULT_MIMETYPE) { + return true; + } + + if ($this->useexiftool) { + // Get the supported MIME types from the config if using ExifTool. + $supportedmimetypesconfig = get_config('core_fileredact', 'exifremovermimetype'); + $supportedmimetypes = array_filter(array_map('trim', explode("\n", $supportedmimetypesconfig))); + return in_array($mimetype, $supportedmimetypes) ?? false; + } + + return false; + } + + /** + * Adds settings to the provided admin settings page. + * + * @param \admin_settingpage $settings The admin settings page to which settings are added. + */ + public static function add_settings(\admin_settingpage $settings): void { + global $OUTPUT; + + // Enabled for a fresh install, disabled for an upgrade. + $defaultenabled = 1; + if (!during_initial_install() && empty(get_config('core_fileredact', 'exifremoverenabled'))) { + $defaultenabled = 0; + } + + $icon = $OUTPUT->pix_icon('i/externallink', get_string('opensinnewwindow')); + $a = new \stdClass; + $a->link = \html_writer::link( + url: 'https://exiftool.sourceforge.net/install.html', + text: "https://exiftool.sourceforge.net/install.html $icon", + attributes: ['role' => 'opener', 'rel' => 'noreferrer', 'target' => '_blank'], + ); + + $settings->add( + new \admin_setting_configcheckbox( + name: 'core_fileredact/exifremoverenabled', + visiblename: get_string('fileredact:exifremover:enabled', 'core_files'), + description: get_string('fileredact:exifremover:enabled_desc', 'core_files', $a), + defaultsetting: $defaultenabled, + ), + ); + + $settings->add( + new \admin_setting_heading( + name: 'exifremoverheading', + heading: get_string('fileredact:exifremover:heading', 'core_files'), + information: '', + ) + ); + + $settings->add( + new \admin_setting_configexecutable( + name: 'core_fileredact/exifremovertoolpath', + visiblename: get_string('fileredact:exifremover:toolpath', 'core_files'), + description: get_string('fileredact:exifremover:toolpath_desc', 'core_files'), + defaultdirectory: '', + ) + ); + + foreach (array_keys(self::REMOVE_TAGS) as $key) { + $removedtagchoices[$key] = get_string("fileredact:exifremover:tag:$key", 'core_files'); + } + $settings->add( + new \admin_setting_configselect( + name: 'core_fileredact/exifremoverremovetags', + visiblename: get_string('fileredact:exifremover:removetags', 'core_files'), + description: get_string('fileredact:exifremover:removetags_desc', 'core_files'), + defaultsetting: self::DEFAULT_REMOVE_TAGS, + choices: $removedtagchoices, + ), + ); + + $mimetypedefault = <<add( + new \admin_setting_configtextarea( + name: 'core_fileredact/exifremovermimetype', + visiblename: get_string('fileredact:exifremover:mimetype', 'core_files'), + description: get_string('fileredact:exifremover:mimetype_desc', 'core_files'), + defaultsetting: $mimetypedefault, + ), + ); + } +} diff --git a/lib/classes/fileredact/services/service.php b/lib/classes/fileredact/services/service.php new file mode 100644 index 00000000000..a713af38a2d --- /dev/null +++ b/lib/classes/fileredact/services/service.php @@ -0,0 +1,68 @@ +. + +namespace core\fileredact\services; + +use stored_file; +/** + * The interface of the redaction service outlines the necessary methods for each redaction blueprint. + * + * @package core + * @copyright Meirza + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +abstract class service { + /** + * Class constructor. + * + * @param stored_file $storedfile The file record object. + */ + public function __construct( + /** @var stored_file $storedfile The file record object. */ + private readonly stored_file $storedfile, + ) { + } + + /** + * Performs redaction on the specified file. + */ + abstract public function execute(): void; + + /** + * Returns true if the service is enabled, and false if it is not. + * + * @return bool + */ + abstract public function is_enabled(): bool; + + /** + * Determines whether a certain mime-type is supported by the service. + * It will return true if the mime-type is supported, and false if it is not. + * + * @param string $mimetype + * @return bool + */ + public function is_mimetype_supported(string $mimetype): bool { + return false; + } + + /** + * Adds settings to the provided admin settings page. + * + * @param \admin_settingpage $settings The admin settings page to which settings are added. + */ + abstract public static function add_settings(\admin_settingpage $settings): void; +} diff --git a/lib/classes/hook/filestorage/after_file_created.php b/lib/classes/hook/filestorage/after_file_created.php new file mode 100644 index 00000000000..c3e588b713f --- /dev/null +++ b/lib/classes/hook/filestorage/after_file_created.php @@ -0,0 +1,60 @@ +. + +namespace core\hook\filestorage; + +use core\attribute; +use core\hook\stoppable_trait; + +/** + * Class after_file_created + * + * @package core + * @copyright 2024 Huong Nguyen + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +#[attribute\label('Allows subscribers to modify file after it is created')] +#[attribute\tags('file')] +#[attribute\hook\replaces_callbacks('after_file_created')] +final class after_file_created { + use stoppable_trait; + /** + * Hook to allow subscribers to modify file after it is created. + * + * @param \stored_file $storedfile The stored file. + * @param \stdClass $filerecord The file record. + */ + public function __construct( + /** @var \stored_file The stored file. */ + public readonly \stored_file $storedfile, + /** @var \stdClass The file record. */ + public readonly \stdClass $filerecord, + ) { + } + + /** + * Process legacy callbacks. + */ + public function process_legacy_callbacks(): void { + if ($pluginsfunction = get_plugins_with_function(function: 'after_file_created', migratedtohook: true)) { + foreach ($pluginsfunction as $plugintype => $plugins) { + foreach ($plugins as $pluginfunction) { + $pluginfunction($this->filerecord); + } + } + } + } +} diff --git a/lib/db/hooks.php b/lib/db/hooks.php index b7f8e7bf31b..1f5bfa71898 100644 --- a/lib/db/hooks.php +++ b/lib/db/hooks.php @@ -110,4 +110,8 @@ $callbacks = [ 'hook' => \core\hook\di_configuration::class, 'callback' => [\core\router\hook_callbacks::class, 'provide_di_configuration'], ], + [ + 'hook' => \core\hook\filestorage\after_file_created::class, + 'callback' => \core\fileredact\hook_listener::class . '::redact_after_file_created', + ], ]; diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index 5a3ec192890..732490063b5 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -1069,19 +1069,23 @@ class file_storage { * Add new file record to database and handle callbacks. * * @param stdClass $newrecord + * @param bool $notify Notify the hook about the new file or not */ - protected function create_file($newrecord) { + protected function create_file($newrecord, bool $notify = true) { global $DB; $newrecord->id = $DB->insert_record('files', $newrecord); if ($newrecord->filename !== '.') { - // Callback for file created. - if ($pluginsfunction = get_plugins_with_function('after_file_created')) { - foreach ($pluginsfunction as $plugintype => $plugins) { - foreach ($plugins as $pluginfunction) { - $pluginfunction($newrecord); - } - } + if (defined('PHPUNIT_TEST') && PHPUNIT_TEST) { + return; + } + if ($notify) { + // The $fileinstance is needed for the legacy callback. + $fileinstance = $this->get_file_instance($newrecord); + // Dispatch the new Hook implementation immediately after the legacy callback. + $hook = new \core\hook\filestorage\after_file_created($fileinstance, $newrecord); + \core\di::get(\core\hook\manager::class)->dispatch($hook); + $hook->process_legacy_callbacks(); } } } @@ -1091,9 +1095,10 @@ class file_storage { * * @param stdClass|array $filerecord object or array describing changes * @param stored_file|int $fileorid id or stored_file instance of the existing local file + * @param bool $notify Notify the hook about the new file or not * @return stored_file instance of newly created file */ - public function create_file_from_storedfile($filerecord, $fileorid) { + public function create_file_from_storedfile($filerecord, $fileorid, bool $notify = true) { global $DB; if ($fileorid instanceof stored_file) { @@ -1200,7 +1205,7 @@ class file_storage { } try { - $this->create_file($newrecord); + $this->create_file($newrecord, $notify); } catch (dml_exception $e) { throw new stored_file_creation_exception($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid, $newrecord->filepath, $newrecord->filename, $e->debuginfo); @@ -1272,9 +1277,10 @@ class file_storage { * * @param stdClass|array $filerecord object or array describing file * @param string $pathname path to file or content of file + * @param bool $notify Notify the hook about the new file or not. * @return stored_file */ - public function create_file_from_pathname($filerecord, $pathname) { + public function create_file_from_pathname($filerecord, $pathname, bool $notify = true) { global $DB; $filerecord = (array)$filerecord; // Do not modify the submitted record, this cast unlinks objects. @@ -1368,7 +1374,7 @@ class file_storage { $newrecord->pathnamehash = $this->get_pathname_hash($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid, $newrecord->filepath, $newrecord->filename); try { - $this->create_file($newrecord); + $this->create_file($newrecord, $notify); } catch (dml_exception $e) { if ($newfile) { $this->filesystem->remove_file($newrecord->contenthash); @@ -1387,9 +1393,10 @@ class file_storage { * * @param stdClass|array $filerecord object or array describing file * @param string $content content of file + * @param bool $notify Notify the hook about the new file or not. * @return stored_file */ - public function create_file_from_string($filerecord, $content) { + public function create_file_from_string($filerecord, $content, bool $notify = true) { global $DB; $filerecord = (array)$filerecord; // Do not modify the submitted record, this cast unlinks objects. @@ -1487,7 +1494,7 @@ class file_storage { $newrecord->pathnamehash = $this->get_pathname_hash($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid, $newrecord->filepath, $newrecord->filename); try { - $this->create_file($newrecord); + $this->create_file($newrecord, $notify); } catch (dml_exception $e) { if ($newfile) { $this->filesystem->remove_file($newrecord->contenthash); diff --git a/lib/tests/fileredact/exifremover_service_test.php b/lib/tests/fileredact/exifremover_service_test.php new file mode 100644 index 00000000000..deba6afa478 --- /dev/null +++ b/lib/tests/fileredact/exifremover_service_test.php @@ -0,0 +1,310 @@ +. + +namespace core\fileredact; + +use file_storage; +use stored_file; + +/** + * Tests for the EXIF remover service. + * + * If you wish to use these unit tests all you need to do is add the following definition to + * your config.php file: + * + * define('TEST_PATH_TO_EXIFTOOL', '/usr/bin/exiftool'); + * + * @package core + * @copyright Meirza + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * + * @covers \core\fileredact\services\exifremover_service + */ +final class exifremover_service_test extends \advanced_testcase { + + /** @var file_storage File storage. */ + private file_storage $fs; + + /** + * Set up the test environment. + */ + public function setUp(): void { + parent::setUp(); + $this->resetAfterTest(); + $this->fs = get_file_storage(); + } + + /** + * Creates a temporary file for testing purposes. + * + * @return stored_file The stored file. + */ + private function create_test_file(): stored_file { + $filename = 'dummy.jpg'; + $path = __DIR__ . '/../fixtures/fileredact/' . $filename; + $filerecord = (object) [ + 'contextid' => \context_user::instance(get_admin()->id)->id, + 'component' => 'user', + 'filearea' => 'unittest', + 'itemid' => 0, + 'filepath' => '/', + 'filename' => $filename, + ]; + $file = $this->fs->get_file($filerecord->contextid, $filerecord->component, $filerecord->filearea, $filerecord->itemid, + $filerecord->filepath, $filerecord->filename); + if ($file) { + $file->delete(); + } + + return $this->fs->create_file_from_pathname($filerecord, $path); + } + + /** + * Creates a temporary invalid file for testing purposes. + * + * @return stored_file The stored file. + */ + private function create_invalid_test_file(): stored_file { + $filename = 'dummy_invalid.jpg'; + $filerecord = (object) [ + 'contextid' => \context_user::instance(get_admin()->id)->id, + 'component' => 'user', + 'filearea' => 'unittest', + 'itemid' => 0, + 'filepath' => '/', + 'filename' => $filename, + ]; + $file = $this->fs->get_file($filerecord->contextid, $filerecord->component, $filerecord->filearea, $filerecord->itemid, + $filerecord->filepath, $filerecord->filename); + if ($file) { + $file->delete(); + } + + return $this->fs->create_file_from_string($filerecord, 'Dummy content'); + } + + /** + * Tests the `exifremover_service` functionality using PHP GD. + * + * This test verifies the ability of the `exifremover_service` to remove all EXIF + * tags from an image file when using PHP GD. It ensures that all tags, including + * GPSLatitude, GPSLongitude, and Orientation, are removed from the EXIF data. + * + * @return void + */ + public function test_exifremover_service_with_gd(): void { + $file = $this->create_test_file(); + // Get the EXIF data from the new file. + $currentexif = $this->get_new_exif($file->get_content()); + $this->assertStringContainsString('GPSLatitude', $currentexif); + $this->assertStringContainsString('GPSLongitude', $currentexif); + $this->assertStringContainsString('Orientation', $currentexif); + + $exifremoverservice = new services\exifremover_service($file); + $exifremoverservice->execute(); + // Get the EXIF data from the new file. + $newexif = $this->get_new_exif($file->get_content()); + + // Removing the "all" tags will result in removing all existing tags. + $this->assertStringNotContainsString('GPSLatitude', $newexif); + $this->assertStringNotContainsString('GPSLongitude', $newexif); + $this->assertStringNotContainsString('Orientation', $newexif); + } + + /** + * Tests the `exifremover_service` functionality using ExifTool. + * + * This test verifies the ability of the `exifremover_service` to remove specific + * EXIF tags from an image file when configured to use ExifTool. The test includes + * scenarios for removing all EXIF tags and for removing only GPS tags. + */ + public function test_exifremover_service_with_exiftool(): void { + if ( (defined('TEST_PATH_TO_EXIFTOOL') && TEST_PATH_TO_EXIFTOOL && !is_executable(TEST_PATH_TO_EXIFTOOL)) + || (!defined('TEST_PATH_TO_EXIFTOOL'))) { + $this->markTestSkipped('Could not test the EXIF remover service, missing configuration. ' . + "Example: define('TEST_PATH_TO_EXIFTOOL', '/usr/bin/exiftool');"); + } + + set_config('exifremovertoolpath', TEST_PATH_TO_EXIFTOOL, 'core_fileredact'); + + // Remove All tags. + set_config('exifremoverremovetags', 'all', 'core_fileredact'); + $file1 = $this->create_test_file(); + $exifremoverservice = new services\exifremover_service($file1); + $exifremoverservice->execute(); + // Get the EXIF data from the new file. + $newexif = $this->get_new_exif($file1->get_content()); + // Removing the "all" tags will result in removing all existing tags. + $this->assertStringNotContainsString('GPSLatitude', $newexif); + $this->assertStringNotContainsString('GPSLongitude', $newexif); + $this->assertStringNotContainsString('Aperture', $newexif); + // Orientation is a preserve tag. Ensure it always exists. + $this->assertStringContainsString('Orientation', $newexif); + + // Remove the GPS tag only. + set_config('exifremoverremovetags', 'gps', 'core_fileredact'); + $file2 = $this->create_test_file(); + $exifremoverservice = new services\exifremover_service($file2); + $exifremoverservice->execute(); + // Get the EXIF data from the new file. + $newexif = $this->get_new_exif($file2->get_content()); + // The GPS tag only removal will remove the tag containing "GPS" keyword. + $this->assertStringNotContainsString('GPSLatitude', $newexif); + $this->assertStringNotContainsString('GPSLongitude', $newexif); + // And keep the other tags remaining. + $this->assertStringContainsString('Aperture', $newexif); + // Orientation is a preserve tag. Ensure it always exists. + $this->assertStringContainsString('Orientation', $newexif); + } + + /** + * Tests the `is_mimetype_supported` method. + * + * This test initializes the `exifremover_service` and verifies if the given + * MIME types are supported for EXIF removal using both PHP GD and ExifTool. + */ + public function test_exifremover_service_is_mimetype_supported(): void { + $file = $this->create_test_file(); + // Init uals(false, $resultthe service. + $exifremoverservice = new services\exifremover_service($file); + + // Test using PHP GD. + $rc = new \ReflectionClass(services\exifremover_service::class); + $rcexifremover = $rc->getMethod('is_mimetype_supported'); + // As default, the exif remover only accepts the default mime type. + $result = $rcexifremover->invokeArgs($exifremoverservice, [services\exifremover_service::DEFAULT_MIMETYPE]); + $this->assertEquals(true, $result); + // Other than the default, the function will returns false. + $result = $rcexifremover->invokeArgs($exifremoverservice, ['image/tiff']); + $this->assertEquals(false, $result); + + // Test using ExifTool. + $useexiftool = $rc->getProperty('useexiftool'); + $useexiftool->setValue($exifremoverservice, true); + // Set the supported mime types. + set_config('exifremovermimetype', 'image/tiff', 'core_fileredact'); + // Other than the `image/tiff`, the function will returns false. + $result = $rcexifremover->invokeArgs($exifremoverservice, ['image/png']); + $this->assertEquals(false, $result); + + } + + /** + * Tests the `clean_filename` method. + * + * This test initializes the `exifremover_service` with a mock file record and + * invokes the `clean_filename` method via reflection to ensure it correctly + * processes the given filename. + * + * @dataProvider exifremover_service_clean_filename_provider + * + * @param string $filename The filename to be cleaned by the `clean_filename` method. + * @param string $expected The expected result after cleaning the filename. + */ + public function test_exifremover_service_clean_filename($filename, $expected): void { + $file = $this->create_test_file(); + // Init the service. + $exifremoverservice = new services\exifremover_service($file); + + $rc = new \ReflectionClass(services\exifremover_service::class); + $rccleanfilename = $rc->getMethod('clean_filename'); + + $result = $rccleanfilename->invokeArgs($exifremoverservice, [$filename]); + $this->assertEquals($expected, $result); + } + + /** + * Tests that the EXIF remover service alters the content hash of a file + * when a new file is created from the original. + */ + public function test_exifremover_contenthash(): void { + $file = $this->create_test_file(); + $beforehash = $file->get_contenthash(); + $exifremoverservice = new services\exifremover_service($file); + $exifremoverservice->execute(); + $afterhash = $file->get_contenthash(); + $this->assertNotSame($beforehash, $afterhash); + } + + /** + * Tests the EXIF remover service with an unknown filename and a valid EXIF tool path. + */ + public function test_exiftool_filename_unknown(): void { + if ( (defined('TEST_PATH_TO_EXIFTOOL') && TEST_PATH_TO_EXIFTOOL && !is_executable(TEST_PATH_TO_EXIFTOOL)) + || (!defined('TEST_PATH_TO_EXIFTOOL'))) { + $this->markTestSkipped('Could not test the EXIF remover service, missing configuration. ' . + "Example: define('TEST_PATH_TO_EXIFTOOL', '/usr/bin/exiftool');"); + } + set_config('exifremovertoolpath', TEST_PATH_TO_EXIFTOOL, 'core_fileredact'); + $invalidfile = $this->create_invalid_test_file(); + $exifremoverservice = new services\exifremover_service($invalidfile); + $this->expectException(\Exception::class); + $exifremoverservice->execute(); + } + + /** + * Tests the EXIF remover service with an unknown filename and an invalid EXIF tool path. + */ + public function test_exiftool_notfound_filename_unknown(): void { + set_config('exifremovertoolpath', 'fakeexiftool', 'core_fileredact'); + $invalidfile = $this->create_invalid_test_file(); + $exifremoverservice = new services\exifremover_service($invalidfile); + $this->expectException(\moodle_exception::class); + $this->expectExceptionMessage(get_string('fileredact:exifremover:failedprocessgd', 'core_files')); + $exifremoverservice->execute(); + } + + /** + * Retrieves the EXIF metadata of a file. + * + * @param string $content the content of file. + * @return string The EXIF metadata as a string. + */ + private function get_new_exif(string $content): string { + $logpath = make_request_directory() . '/temp.jpg'; + file_put_contents($logpath, $content); + $exif = exif_read_data($logpath); + $string = ""; + foreach ($exif as $key => $value) { + if (is_array($value)) { + foreach ($value as $subkey => $subvalue) { + $string .= "$subkey: $subvalue\n"; + } + } else { + $string .= "$key: $value\n"; + } + } + return $string; + } + + /** + * Data provider for test_exifremover_service_clean_filename(). + * + * @return array + */ + public static function exifremover_service_clean_filename_provider(): array { + return [ + 'Hyphen minus -' => [ + 'filename' => '-if \'$LensModel eq "18-35mm"\'', + 'expected' => 'if $LensModel eq 18-35mm', + ], + 'Minus −' => [ + 'filename' => '−filename.jpg', + 'expected' => 'filename.jpg', + ], + ]; + } +} diff --git a/lib/tests/fileredact/manager_test.php b/lib/tests/fileredact/manager_test.php new file mode 100644 index 00000000000..7ba5a5b3cf5 --- /dev/null +++ b/lib/tests/fileredact/manager_test.php @@ -0,0 +1,92 @@ +. + +namespace core\fileredact; + +use stored_file; + +/** + * Tests for fileredact manager class. + * + * @package core + * @copyright Meirza + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * + * @covers \core\fileredact\manager + */ +final class manager_test extends \advanced_testcase { + + /** @var stored_file Stored file object. */ + private stored_file $storedfile; + + public function setUp(): void { + parent::setUp(); + $this->resetAfterTest(); + + $file = new \stdClass; + $file->contextid = \context_user::instance(get_admin()->id)->id; + $file->component = 'user'; + $file->filearea = 'private'; + $file->itemid = 0; + $file->filepath = '/'; + $file->filename = 'test.jpg'; + $file->source = 'test'; + + $fs = get_file_storage(); + $this->storedfile = $fs->create_file_from_string($file, 'file1 content'); + } + + /** + * Tests the `get_services` method. + * + * This test initializes the `manager` and verifies that the `get_services` method. + */ + public function test_get_services(): void { + // Init the manager. + $manager = new \core\fileredact\manager($this->storedfile); + + $rc = new \ReflectionClass(\core\fileredact\manager::class); + $rcm = $rc->getMethod('get_services'); + $services = $rcm->invoke($manager); + + $this->assertGreaterThan(0, count($services)); + + } + + /** + * Tests the `execute` method and error handling. + * + * This test mocks the `manager` class to return a dummy service for `get_services` + * and verifies that the `execute` method runs without errors. + */ + public function test_execute(): void { + $managermock = $this->getMockBuilder(\core\fileredact\manager::class) + ->onlyMethods(['get_services']) + ->setConstructorArgs([$this->storedfile]) + ->getMock(); + + $managermock->expects($this->once()) + ->method('get_services') + ->willReturn(['\\core\fileredact\\services\\dummy_service']); + + /** @var \core\fileredact\manager $managermock */ + $managermock->execute(); + $errors = $managermock->get_errors(); + + // If execution is OK, then no errors. + $this->assertEquals([], $errors); + } +} diff --git a/lib/tests/fixtures/fileredact/dummy.jpg b/lib/tests/fixtures/fileredact/dummy.jpg new file mode 100644 index 00000000000..c25d80ba9cc Binary files /dev/null and b/lib/tests/fixtures/fileredact/dummy.jpg differ diff --git a/lib/tests/fixtures/fileredact/dummy_service.php b/lib/tests/fixtures/fileredact/dummy_service.php new file mode 100644 index 00000000000..eed5fabbf30 --- /dev/null +++ b/lib/tests/fixtures/fileredact/dummy_service.php @@ -0,0 +1,63 @@ +. + +namespace core\fileredact\services; + +/** + * Dummy service for testing only. + * + * @package core + * @copyright Meirza + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class dummy_service extends service { + + /** + * Performs redaction on the specified file. + */ + public function execute(): void { + // The function body. + } + + /** + * Returns true if the service is enabled, and "false" if it is not. + * + * @return bool + */ + public function is_enabled(): bool { + return true; + } + + /** + * Determines whether a certain mime-type is supported by the service. + * It will return true if the mime-type is supported, and false if it is not. + * + * @param string $mimetype + * @return bool + */ + public function is_mimetype_supported(string $mimetype): bool { + return true; + } + + /** + * Adds settings to the provided admin settings page. + * + * @param \admin_settingpage $settings The admin settings page to which settings are added. + */ + public static function add_settings(\admin_settingpage $settings): void { + // The function body. + } +} diff --git a/question/format/xml/tests/behat/import_export.feature b/question/format/xml/tests/behat/import_export.feature index c09746929ae..5a836e4c954 100644 --- a/question/format/xml/tests/behat/import_export.feature +++ b/question/format/xml/tests/behat/import_export.feature @@ -34,7 +34,7 @@ Feature: Test importing questions from Moodle XML format. And I set the field "id_format_xml" to "1" And I set the field "Export category" to "TrueFalse" And I press "Export questions to file" - Then following "click here" should download between "57100" and "58150" bytes + Then following "click here" should download between "17042" and "18874" bytes @javascript @_file_upload Scenario: import some multiple choice questions from Moodle XML format diff --git a/question/type/ddimageortext/tests/behat/export.feature b/question/type/ddimageortext/tests/behat/export.feature index 06d83e284ea..eaa227cb774 100644 --- a/question/type/ddimageortext/tests/behat/export.feature +++ b/question/type/ddimageortext/tests/behat/export.feature @@ -26,7 +26,7 @@ Feature: Test exporting drag and drop onto image questions When I am on the "Course 1" "core_question > course question export" page logged in as teacher And I set the field "id_format_xml" to "1" And I press "Export questions to file" - Then following "click here" should download between "18600" and "19150" bytes + Then following "click here" should download between "18500" and "24864" bytes # If the download step is the last in the scenario then we can sometimes run # into the situation where the download page causes a http redirect but behat # has already conducted its reset (generating an error). By putting a logout diff --git a/version.php b/version.php index 9c33ae2a4be..37ce20c4e1e 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2024082900.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2024082900.01; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes. $release = '4.5dev+ (Build: 20240829)'; // Human-friendly version name